diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cd566b8..f84f934 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -83,10 +83,10 @@ jobs: # cargo-zigbuild is invoked directly rather than via `napi build -x` # because the napi CLI can't pass a glibc version suffix; we target # a glibc 2.35 floor. RUSTFLAGS for the same reason as musl below. - build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild --target aarch64-unknown-linux-gnu.2.35 --release && cp target/aarch64-unknown-linux-gnu/release/libfaith.so faith.linux-arm64-gnu.node + build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild -p web-faith-napi --target aarch64-unknown-linux-gnu.2.35 --release && cp target/aarch64-unknown-linux-gnu/release/libfaith.so faith.linux-arm64-gnu.node - host: ubuntu-latest target: armv7-unknown-linux-gnueabihf - build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild --target armv7-unknown-linux-gnueabihf.2.35 --release && cp target/armv7-unknown-linux-gnueabihf/release/libfaith.so faith.linux-arm-gnueabihf.node + build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild -p web-faith-napi --target armv7-unknown-linux-gnueabihf.2.35 --release && cp target/armv7-unknown-linux-gnueabihf/release/libfaith.so faith.linux-arm-gnueabihf.node - host: ubuntu-latest target: aarch64-linux-android build: npm run build -- --target aarch64-linux-android diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html new file mode 100644 index 0000000..46f697b --- /dev/null +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -0,0 +1,372 @@ + + + + + +Rust API shape + + + +
+ +

Rust API shape

+

+ web-faith has one noun and one verb. An Agent owns the pool, jar, cache, and + resolver; fetch is how every request goes out. Defaults match the Node surface: redirects + followed and HTTP/3 upgrade on, cookie jar and HTTP cache opted into. +

+ +
+

The whole surface

+

+ fetch takes a URL or a prebuilt Request and returns a builder. + The builder is a future, so a bare call awaits directly and a configured one awaits after its options. + There is no send() and no second entry point. +

+ +
+
+
Fetch some JSON
+
use web_faith::Agent;
+
+let agent = Agent::new()?;
+
+let things: Vec<Thing> = agent
+    .fetch("https://api.example/things")
+    .await?
+    .json()
+    .await?;
+
+ +
+
Configured POST
+
let agent = Agent::builder()
+    .user_agent("myapp/1.0")
+    .cache(Cache::disk("/var/cache/app"))
+    .cookies(true)
+    .build()?;
+
+let res = agent
+    .fetch("https://api.example/things")
+    .method(Method::POST)
+    .header("authorization", tok)
+    .integrity("sha384-oqVuAfXR")
+    .cache_mode(CacheMode::NoStore)
+    .timeout(Duration::from_secs(5))
+    .json(&thing)
+    .await?;
+
+ +
+
Stream the body
+
let res = agent.fetch(url).await?;
+
+let mut body = res.body_stream();
+while let Some(chunk) = body.next().await {
+    sink.write_all(&chunk?).await?;
+}
+
+let trailers = res.trailers().await;
+let timing = res.timing().await;
+
+
+
+ +
+

Building a request once

+

+ A Request is the built, inert form. Because fetch accepts one and still returns a + builder, a request can be prepared once and tweaked at each call site, or sent unchanged across + two agents. Cloning follows the body: a buffered body clones, a stream body cannot, so it is + try_clone rather than Clone. +

+ +
+
+
Prepare, then vary
+
let probe = Request::new("https://api.example/health")
+    .header("accept", "application/json")
+    .timeout(Duration::from_secs(2))
+    .build()?;
+
+for region in regions {
+    let Some(req) = probe.try_clone() else { break };
+
+    let res = agent
+        .fetch(req)
+        .header("x-region", region)
+        .await?;
+}
+
+ +
+
Same request, two agents
+
let req = Request::new(url)
+    .method(Method::POST)
+    .json(&payload)?
+    .build()?;
+
+let Some(copy) = req.try_clone() else {
+    return Err(Error::NotCloneable);
+};
+
+let (primary, mirror) = tokio::join!(
+    live_agent.fetch(req),
+    audit_agent.fetch(copy),
+);
+
+
+
+ +
+

Agent lifecycle

+

+ An agent is cheap to clone and every clone names the same pool, so close acts on the shared + agent rather than on the handle it was called through. Requests already issued run to completion, even + ones no work has started on; a request issued afterwards fails with the closed-agent error. +

+ +
+
+
Shutting down
+
let agent = Agent::new()?;
+let handle = agent.clone();
+
+tokio::spawn(async move {
+    handle.fetch(url).await
+});
+
+// releases the pool, resolver, and probes
+// for every clone, not just this one
+agent.close();
+
+ +
+
Acting on a live agent
+
agent.network_changed();
+
+let stats = agent.stats();
+let conns = agent.connections();
+let resolvers = agent.resolvers();
+let jar = agent.cookies();
+
+agent.prefetch_dns("api.example").await?;
+agent.preconnect("https://api.example").await?;
+
+
+
+ +

Response

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fetch surfacestatus, status_text, ok, headers, url, + redirected, kind, body_used
Readingtext(), json(), bytes(), body_stream(), + to_file(), discard(). Reading consumes the body and a second read is an + error, following the fetch standard rather than reqwest's owned-response model.
Faith additionspeer, version, trailers(), timing()
ErrorsOne Error type carrying the existing kinds, each reporting the same code the Node + binding does.
CancellationDropping the future cancels the request. timeout stays an option for the deadline case.
RuntimeTokio, as today.
+ +

Ecosystem types

+

+ The http crate's types are canonical wherever one exists, so a Faith request composes with + tower, hyper, and axum code without a shim. Setters stay fetch-flavoured by taking anything that + converts, so a string literal works where JS would pass a string. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConceptCanonical typeWhat setters accept
Methodhttp::Method.method(impl TryInto<Method>), so "POST" and Method::POST both work
Headerhttp::HeaderName, http::HeaderValue.header(impl TryInto<HeaderName>, impl TryInto<HeaderValue>)
Header sethttp::HeaderMap.headers(impl IntoIterator<Item = (K, V)>), which covers a HeaderMap, a + Vec of pairs, and an array of string pairs
Statushttp::StatusCodeRead-only. ok and status_text derive from it
Versionhttp::VersionRead-only, replacing the Node binding's string
URLurl::Urlimpl TryInto<Url>, matching what the fetch standard parses and what reqwest already uses
Request inRequestfetch and Request::new take an impl TryInto<Url>, a + Request, or an http::Request<B>
Response outResponseConverts into http::Response<Body>, and the body implements + http_body::Body so it feeds tower and hyper directly
Conversion errorsError::InvalidHeader and friendsHeld until the builder resolves: at build() for a request, at the await for a fetch. + Either way the error names the offending header
+ +

Considered and dropped

+
+ +
+ +
+ + diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md new file mode 100644 index 0000000..b4f7344 --- /dev/null +++ b/.workhorse/plans/s1/plan.md @@ -0,0 +1,261 @@ +# S1 — Expose a Rust API and publish to crates.io + +Restructure the single `faith` cdylib into a Cargo workspace: a browser-shaped Rust client +`web-faith`, five standalone component crates beneath it, and a thin `web-faith-napi` binding that +ships as `@passcod/faith`. Then publish to crates.io. Target architecture is specified in +[RUST](../../specs/rust/overview.md) and [RSAPI](../../specs/rust/client-api.md). + +## Scope reality + +This is an 8-crate restructure of ~11,500 lines, not a single focused change. It is being built on +this branch as one long-lived PR, at the user's direction, rather than split into a card breakdown. + +Steps 0–8 are done: the workspace stands, the five components are out, and the client owns the agent, +the request path, and the response. Every crate but `web-faith-napi` builds with no napi in its +graph. What remains (steps 9–14) is the caller-facing API, the feature wiring, publishing, and the +spec sweep — step 9 being new design rather than relocation. + +## What the discovery turned up + +Facts that shape the order and difficulty: + +- **The error type is napi-coupled at the base.** `src/error.rs` derives `#[napi(string_enum)]` on + `FaithErrorKind` and holds napi conversions. `integrity` already depends on it. Splitting a + pure-Rust error core from the napi conversion layer is a prerequisite for every component that + reports errors, and for the client itself. This is the load-bearing first move. +- **Component modules are mostly napi-free already.** `alt_svc`, `body`, `cookies`, `dns`, + `encoding`, `integrity`, `retry` have zero `#[napi]`. The coupling concentrates in `agent.rs` + (52), `response.rs` (40), `options.rs` (17), `error.rs` (13), `stream_body.rs` (11), `fetch.rs`. +- **Internal component coupling is small and matches the spec's allowed shape:** `integrity → error`, + `encoding → body::DynStream`, `alt_svc → timing::HeadersStamp` and `alt_svc → dns` (the spec + explicitly allows `web-faith-alt-svc → web-faith-dns`). `cookies` and `dns` have no internal deps. +- **`cookies` is bound to reqwest.** It implements `reqwest::cookie::CookieStore` and takes + `reqwest::Url`. A standalone `web-faith-cookies` should speak `url::Url` and put the reqwest + `CookieStore` impl behind a `reqwest` feature (or move the adapter into the client). +- **The napi build resists a naive relocation.** `build.rs` reads `Cargo.lock` by the relative path + `"Cargo.lock"`; under a workspace the lock is at the root, so this must become + `CARGO_MANIFEST_DIR`/workspace-root aware. `napi build --platform` reads `package.json`'s `napi` + config and builds the crate in cwd; moving the crate means teaching the napi CLI where the crate + is and keeping generated `index.js`/`index.d.ts` at the repo root (the spec requires them there). + `.cargo/config.toml` (`reqwest_unstable`, cross linkers) applies workspace-wide and can stay at root. + +## Target crate family + +- `web-faith` — client: agent, request/response, `fetch`, layering, SRI. Depends on the five components. +- `web-faith-cookies` — the jar ([COOK](../../specs/agent/cookies.md)). +- `web-faith-dns` — resolver, cache, discovery ladder, HTTPS record, Happy Eyeballs ([DNS](../../specs/agent/dns.md)). +- `web-faith-conn-tracker` — live per-connection stats from the OS ([OBS](../../specs/agent/observability.md)). +- `web-faith-alt-svc` — Alt-Svc store + HTTP/3 upgrade/probing ([H3UP](../../specs/http3/upgrade.md), [PROBE](../../specs/http3/probing.md)); may depend on `web-faith-dns`. +- `web-faith-encoding` — request/response content coding ([ENC](../../specs/fetch/content-encoding.md)). +- `web-faith-napi` — the only crate with napi types; ships as `@passcod/faith`. + +QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring alternative), not crates. + +## Build order (each step ends green: `cargo build` + `cargo test` + napi `npm run build`) + +- [x] **0. Workspace scaffold.** Root `[workspace]` with shared `[workspace.package]` + (licence, repository, authors, edition, `rust-version = "1.96"`) and `[workspace.dependencies]`. + Move the current crate to `crates/web-faith-napi`. Fix `build.rs` `Cargo.lock` path. Make + `napi build` target the relocated crate and keep `index.js`/`index.d.ts` at repo root. Verify the + npm build still produces a working `.node`. No behaviour change. +- [x] **1. Error core split.** Pure-Rust `FaithError`/`FaithErrorKind` (no napi) reachable by every + crate; napi conversions live only in `web-faith-napi`. `ERROR_CODES` still generated from the one + source ([ERR](../../specs/errors/errors.md)). Decide where the shared error core lives (likely in + `web-faith`, with components naming their own error types that the client converts — per + [RUST](../../specs/rust/overview.md) "A component crate stands alone"). +- [x] **2. SRI** — extracted as `web-faith-integrity`, then folded back into `web-faith` as a module: + too small for standing alone to buy a caller anything, and not something a build should switch off. +- [x] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). +- [x] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. +- [x] **5. Extract `web-faith-dns`.** +- [x] **6. Extract `web-faith-conn-tracker`** (Linux/macOS/Windows submodules). +- [x] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. +- [x] **8. Stand up `web-faith`** — done. The client holds the agent, the request path, the response + and its reads, the body/timing/retry machinery, and the client recipe. `web-faith-napi` is the + binding: JS option shapes, `AgentOptions` validation, the napi classes wrapping the client's types, + and napi machinery (promises, streams, threadsafe functions). Nothing outside it carries napi. + - [x] `body`, `timing` (measuring; the napi object stays behind), `retry` moved. + - [x] The client-building machinery moved: `ClientRecipe`, `NodeEnvRecipe`, `HttpCacheRecipe`, + `HttpCacheStore`, `H3UpgradeRecipe`, `ResolvedWindows`, `install_https_sink`, and a pure + `RedirectPolicy` the napi `Redirect` converts into. + - [x] **The `Agent` inversion.** Done: `web_faith::agent::Agent` owns the pool, the verbs, and the + per-request settings; `web-faith-napi`'s `Agent` is a napi class holding a handle on it, and + keeps the ~440 lines of `AgentOptions` validation that produce a recipe. `prefetch_dns` and + `preconnect` return futures the binding wraps, so a refusal happens before the future exists. + - [x] `response.rs` inverted: `web_faith::response::Response` holds the state and the reads, and + writing a body out takes a progress closure rather than a threadsafe function. + - [x] `fetch.rs` inverted: `web_faith::request::send` takes an agent, URL, options, body, and an + optional abort future; `fetch.rs` is 59 lines converting a `fetch()` call into those. + - **Left for step 9:** the recipe structs' public fields, which the builder should own the + assembly of. +- [x] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md). + A Rust caller now reaches everything through the client: `Agent::new`/`builder`, `agent.fetch`, + `Request`, and the response's own reads. + - [x] `USER_AGENT` is the client's, composed from its own version and reqwest's, which `web-faith` + now reads in a build script of its own. The binding's constant reads from it. + - [x] Reading a response: the accessors and `bytes`/`text`/`json`/`body_stream`/`discard`/ + `to_file`/`timing`/`trailers` on `web_faith::response::Response`, with the napi methods + delegating. `json` is generic over what it deserialises into. + - [x] `http_body::Body` for the body, and `Response::into_http`. Fallible, since taking the body + can find it already being consumed. + - [x] The option groups and the ~440-line validation moved to `web_faith::options` and + `Agent::from_options`, so both surfaces settle defaults in one place. `Agent::new()` follows. + - [x] **`close()` and `network_changed()` act on the agent, not the handle.** The closeable state + sits in a `Live` behind a shared lock, so every clone sees a close, as + [RSAPI](../../specs/rust/client-api.md) requires. A request takes its handle at the moment it + is issued — `request::send` is given the client rather than reaching for it — which is what + [AGENT](../../specs/agent/overview.md) means by in flight from the moment it is issued. The + JS suite caught the difference: capturing inside the promise instead of at issue stranded a + request that was issued just before a close. + - [x] `Agent::builder()` with nested builders reached through a closure, so a group left alone is + absent from the call. Setters take `Duration` and `IpAddr` rather than the units and strings + the options carry. + - [x] `Request`, `Request::new`, `try_clone`, and `agent.fetch(target)` over `IntoFuture`, with the + layering rules: outermost explicit value wins, untouched settings inherit, headers merge by + name and a removal clears what is underneath, the URL comes from the bottom of the stack. + - [x] Setters take the canonical `http` types or anything converting into them, and a failed + conversion is held until the builder resolves — at `build()` for a request, at the await for + a fetch. The first failure met is the one reported. + - [x] `Agent::cookies()` hands back the jar itself. + - [x] `http::Request` as a target, bringing its method, URL, headers, and body across. + - [ ] Remaining: closing up the recipe and option structs' public fields now that the builder owns + the assembly. Left deliberately: the binding still fills the option structs directly, so + these close up when step 10 settles what the public surface is. +- [x] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling + one drops the code and the API surface it gates (compile error at the call site, not a no-op), and + the dependency too where the capability is a crate. Component, crate, and feature are three axes + and need not line up: a crate can be non-optional, and a feature need not map to a crate. + + `web-faith` carries `cache`, `connection-tracking`, `cookies`, `dns`, `encoding`, `http3`, and the + `tls-aws-lc-rs`/`tls-ring` backend choice; `web-faith-alt-svc` carries `dns` for the HTTPS-record + sink. `web-faith-napi` mirrors the set. Integrity is deliberately not a feature: it is always + built. +- [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; + client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. +- [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, + MSRV 1.96 declared in every published crate and exercised in CI alongside stable, independent + versioning from `1.0.0`. Measure CI cost before adding jobs (see project memory). +- [ ] **13. First publish** to crates.io: the five components, then `web-faith`; `@passcod/faith` + continues from npm via `web-faith-napi`. +- [ ] **14. The both-surfaces spec sweep**, as the closing pass over the tree — deliberately last, + once the Rust API is settled and its names are known. See the section below for the site-by-site + reconnaissance. + +## What the extractions settled + +Decisions taken while doing steps 0–7, worth not relitigating: + +- **The lib is still named `faith`.** `web-faith-napi`'s `[lib] name = "faith"` keeps the artifact + `libfaith.so`, which the release workflow's zigbuild steps copy by name. +- **The benchmark HTTP/3 server is a workspace `exclude`,** not a member: its own manifest says it + keeps a separate lockfile so the quinn/h3 stack stays out of this graph. Adding it as a member + would have pulled that stack in; leaving it unlisted broke it outright. +- **`FaithErrorKind` is gone from the native binding.** It was emitted only because the enum carried + napi's attribute. The package's `exports` map admits nothing but the wrapper, so no consumer could + reach it (verified: a deep import fails with `ERR_PACKAGE_PATH_NOT_EXPORTED`). The documented + surface is `wrapper.js`'s `ERROR_CODES`, unchanged at 22 codes. +- **Do not route a napi enum through `macro_rules!`.** Doc comments arrive as mangled `r"` literals + in the generated `.d.ts`, and the Rust name leaks into `index.js` alongside the `js_name`. This is + why the kinds have a single plain definition in `web-faith` and the codes reach JS through + `errorCodes()` instead. +- **`reqwest` integration sits behind a per-crate feature** on `web-faith-cookies` (`CookieStore`) + and `web-faith-dns` (`Resolve`). Where a trait impl held the only path to real functionality — the + jar's store-and-read — the logic moved to inherent methods and the impl delegates, so the crate is + usable without reqwest rather than merely compilable. +- **`alt_svc` takes the client's timing stamp as a generic,** via an `ArrivalStamp` trait, because + the stamp must exist in non-HTTP/3 builds where the alt-svc crate is not compiled at all. +- **`web-faith` only declares a component dependency once it uses it.** The full + feature-per-component set is step 10. +- **`web-faith` is depended on with `default-features = false`,** set at the workspace root because + Cargo refuses to let a member override a workspace dependency's defaults. Without this the + binding's `http3` feature and the client's drifted: turning the binding's off left the client's on, + and the `#[cfg]`-gated recipe fields stopped lining up. Check both configurations after touching + features — `cargo build` and `cargo build -p web-faith-napi --no-default-features`. +- **The recipe structs carry public fields for now.** The binding assembles them directly; step 9's + builder is what should own that assembly, at which point they can close up again. +- **Spec references go in normal comments, never in doc comments.** A `// spec:DNS#transports` line + sits under the doc block, above the item. Doc comments are published API documentation, and a spec + id means nothing to a reader on docs.rs. +- **A component crate's docs address an external reader,** not a Faith maintainer: what the crate is + for and how to drive it, rather than why Faith needed it factored this way. Keep the reasoning + where it is genuinely about the code's shape, drop the rest. +- **Keep rustdoc clean, and mind that napi doc comments reach TypeScript.** `cargo doc --workspace + --no-deps` is warning-free; making the components public surfaced several links to private items, + which would have shipped as broken docs.rs pages. Rust intra-doc syntax in a `web-faith-napi` doc + comment is emitted verbatim into `index.d.ts`, where `[`X`]` means nothing, so plain backticks + belong on anything a napi item documents. +- **No source file past 1000 lines, tests-only files excepted.** `dns`, `alt-svc`, the client's + agent and request paths, and the binding's agent were each split into modules along their internal + seams. +- **Tests live as a child module of the code they exercise,** `foo/tests.rs` under `foo.rs`, not one + flat `tests.rs` at the crate root. A child module reaches its parent's private items, so the split + costs no visibility: the first attempt widened a dozen internals to `pub(crate)` purely for a + crate-root test module, which is the wrong trade. +- **Splitting a file relocates doc comments as easily as it drops them.** A cut between an item and + its doc block leaves the block dangling at the end of one file and the item bare at the start of + the next, which rustc catches, and a `// spec:` line under a doc block extends how far back the + block starts. Intra-doc links break more quietly: `[`X`]` that resolved within one file needs + `crate::X` once `X` is a sibling module away, and `cargo doc --workspace --no-deps` is what says so. +- **A feature drops the reqwest feature behind it too.** `cookies`, `dns`, `http3`, and the TLS + backend each own their reqwest counterpart, which means those came out of the workspace root's + `reqwest` feature list: a feature that leaves the dependency linked has not dropped anything. + Cargo refuses a member override of a workspace dependency's `default-features`, so + `web-faith-alt-svc` needs `default-features = false` set at the root, as `web-faith` already did. +- **The TLS backend resolves by priority, not by refusal.** Cargo features are additive, so a + build with both `tls-aws-lc-rs` and `tls-ring` — which is what `--all-features` and any `http3` + build are — has to mean something rather than fail. aws-lc-rs wins, and ring is installed as the + process provider only where it is the sole choice. A `compile_error!` remains for *neither*, which + is a real misconfiguration: an HTTPS client that cannot speak TLS is not one. reqwest's `http3` + pins its QUIC stack to aws-lc-rs, which is why `http3` enables that backend rather than tolerating + either. +- **napi's derives ignore `#[cfg]` on a field or an impl method.** `#[napi(object)]` re-emits field + types and `#[napi] impl` enumerates method names, both at macro time, so a gated field or method + leaves generated code referencing an item that is no longer there. Methods can still be removed — + by moving them to their own `#[cfg]`-gated `#[napi] impl` block, which napi accepts — but an + option object keeps its full shape whatever the build. So the binding refuses what it cannot + honour instead: `refuse_absent_capabilities` for an agent option group, and a check in + `faith_fetch` for a per-request one. Silently ignoring the option was the alternative, and it + would make a slim build look like it worked. +- **Gating an option group means gating the tests that set it.** `cargo test` on a slim build was + broken by tests and a doctest reaching for fields that are no longer compiled. The doctest was the + worse of the two, having no per-feature escape: the fix was to illustrate with a group that is + never gated. Check the matrix with `cargo test`, not just `cargo build`. +- **The binding carried two dozen dependencies it no longer used**, left over from before the + extraction — including `web-faith-dns` and `web-faith-alt-svc`, which a feature claimed to drop + while linking them anyway. Worth re-checking after any extraction: `use` roots in the source + against the manifest. + +## Step 14: the both-surfaces spec sweep + +A spec should be one of three things, never a fourth: generic to both surfaces (naming the concept +and linking to where it is defined), specified at the correct site, or explicitly about one surface +so the reader knows which. What it should not be is shared behaviour spelled in one surface's +identifiers, which is what a JavaScript name in a spec covering both amounts to. + +It runs last because the Rust names it will cite are step 9's to settle: sweeping earlier would mean +guessing at them, and a spec that cites a name which then changes is worse than one that has not +been swept yet. + +[FAITH](../../specs/overview.md) and [ENV](../../specs/environment/variables.md) are done. The +remaining sites, from a survey of JS-cased identifiers: + +- `agent/observability.md` (~20: `bodiesStarted`, `responseCount`, `rttUs`, and the rest of the + per-connection fields), `agent/warm-up.md` (~10, `prefetchDns` five times), + `agent/cookies.md` (~8), `agent/dns.md` (~6), `agent/overview.md` (~5), + `agent/flow-control.md` (~4), `agent/connection-pool.md` (~3), and scattered singles. +- `response/response.md` and `response/reading-the-body.md` need judgment rather than a sweep: most + of their identifiers are Resource Timing's own field names (`fetchStart`, `requestStart`), which + are standard vocabulary and should stay. Faith's own (`bodyUsed`, `statusText`) are the ones that + want naming as concepts, [RSAPI](../../specs/rust/client-api.md) spelling them `body_used` and + `status_text`. +- `rust/client-api.md` needs nothing: its `Request` and `Response` are the Rust types. + +So the raw identifier count overstates the work — a blind sweep would churn standard-defined names +and Rust types alike. Roughly 60 genuine sites across about ten files, each needing a decision on +whether the surrounding spec is shared or single-surface. + +## Verification discipline + +Every step must leave `cargo build`, `cargo test`, and the napi `npm run build` green +(tests via `HTTPBIN_URL=http://localhost:8888`, `NODE_ENV=development` for `npm install`). A component +crate's separateness is only proven when `cargo test -p ` passes with no JS runtime present. diff --git a/.workhorse/specs/agent/overview.md b/.workhorse/specs/agent/overview.md index 7228ec9..ee20f4a 100644 --- a/.workhorse/specs/agent/overview.md +++ b/.workhorse/specs/agent/overview.md @@ -33,6 +33,8 @@ Dual-stack hosts are unaffected. `close()` releases the agent's resources on demand: the connection pool, the DNS resolver, in-flight HTTP/3 probes, and the HTTP/3 knowledge cache. This exists because waiting for the garbage collector is not acceptable for code that creates many short-lived agents. Requests already in flight when `close()` is called run to completion; new requests on a closed agent throw a closed-agent error (code `Closed`). +A request counts as in flight from the moment it is issued rather than from when its work begins, so a request issued just before `close()` completes even if nothing had started on it yet. +`close()` acts on the agent itself rather than on the reference it was called through, so every reference to that agent sees it closed. `close()` is idempotent, and the cookie jar remains readable after closing. `networkChanged()` is the other verb that acts on a live agent's own state, discarding what the agent learned from a network that no longer exists while keeping the agent usable (see [NETCHG](network-change.md)). diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index b019178..a922a1a 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -4,39 +4,56 @@ id: ENV # Environment variables -Faith reads a set of environment variables so that `fetch()` behaves like Node's built-in fetch without extra configuration. -The set is deliberately Node's own vocabulary plus the standard proxy and OpenSSL variables, not a Faith-specific namespace. +Faith reads a set of environment variables. +The set is deliberately Node's own vocabulary plus the standard proxy and OpenSSL variables, not a Faith-specific namespace; on the Node surface that is what makes `fetch()` behave like Node's built-in fetch without extra configuration. + +Each section below names the surfaces it applies to. +The `NODE_`-prefixed variables answer to a JavaScript runtime's conventions and belong to the Node surface alone; the rest is platform configuration both surfaces honour (see [RUST](../rust/overview.md)). ## Read-at-construction semantics -Environment variables are read once, when an `Agent` is constructed (including the implicit global default agent). +This applies to both surfaces. + +Environment variables are read once, when an agent is constructed, including the Node surface's implicit global default agent. Changing them afterwards only affects agents created later. ## Trust store -`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any `tls.extraRoots` (see [TLS](../agent/tls.md)); certificates from both sources combine. -It is lenient, matching Node's warn-and-continue behaviour: an empty value, an unreadable file, or an unparseable file is ignored rather than fatal. -(The `tls.extraRoots` option, being an explicit programmatic choice, throws on malformed input instead.) -On Unix platforms other than macOS, `SSL_CERT_FILE` and `SSL_CERT_DIR` override where the system trust store is loaded from, with standard OpenSSL semantics: `SSL_CERT_FILE` replaces the system roots, where `NODE_EXTRA_CA_CERTS` adds to them. +Both surfaces read `SSL_CERT_FILE` and `SSL_CERT_DIR`; `NODE_EXTRA_CA_CERTS` belongs to the Node surface alone. + +On Unix platforms other than macOS, `SSL_CERT_FILE` and `SSL_CERT_DIR` override where the system trust store is loaded from, with standard OpenSSL semantics: `SSL_CERT_FILE` replaces the system roots. On macOS and Windows the OS trust store is used directly and these are ignored, as Node does on those platforms. +`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any extra roots set on the agent (see [TLS](../agent/tls.md)); certificates from both sources combine, and where `SSL_CERT_FILE` replaces the system roots this adds to them. +It is lenient, matching Node's warn-and-continue behaviour: an empty value, an unreadable file, or an unparseable file is ignored rather than fatal. +Extra roots set on the agent are an explicit programmatic choice and fail loudly on malformed input instead. + ## Certificate validation +This applies to the Node surface alone. + `NODE_TLS_REJECT_UNAUTHORIZED` set to exactly `0` disables TLS certificate validation for the agent; any other value or unset keeps validation on. -This matches Node's semantics and exists only for that compatibility; trusting a specific private CA via `NODE_EXTRA_CA_CERTS` or `tls.extraRoots` is the supported path. +This matches Node's semantics and exists only for that compatibility; trusting a specific private CA, through `NODE_EXTRA_CA_CERTS` or the agent's extra roots, is the supported path. ## Proxies +Both surfaces read the proxy variables and the operating system's own proxy settings; `NODE_USE_ENV_PROXY` belongs to the Node surface alone. + `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` (and their lowercase spellings) are honoured automatically: per-scheme proxy selection, a fallback for both schemes, and a comma-separated direct-connection list of hosts, domains, and CIDR ranges. The operating system's proxy settings are also read automatically. + `NODE_USE_ENV_PROXY` set to exactly `0` turns ambient proxy configuration off. Faith proxies by default, so unlike Node (where the same variable opts in), it acts purely as an opt-out. ## Debugging +This applies to both surfaces. + `SSLKEYLOGFILE` names a path to which TLS session keys are written, enabling decryption of captured traffic when debugging. ## Variables with nothing to control +Neither surface reads these. + `NODE_USE_SYSTEM_CA` is ignored because the platform trust store is Faith's only default source of roots; there is no bundled set to toggle away from. `OPENSSL_CONF` is ignored because the TLS stack is not OpenSSL, so OpenSSL's configuration file has nothing to configure. diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index 4fda8f8..3705ea4 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -4,8 +4,13 @@ id: FAITH # Faith -Faith is a fetch API implementation for Node.js backed by a Rust network stack rather than Node's built-in HTTP machinery. -It is published as the native module `@passcod/faith` and aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, while exposing the capabilities that stack unlocks: transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. +Faith is a fetch API implementation for Node.js and Rust. +It aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, with transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. + +It has two public surfaces: + +- the Rust crate `web-faith` (see [RSAPI](rust/client-api.md)) and the component crates at that prefix (see [RUST](rust/overview.md)); +- the Node.js module `@passcod/faith`. The library's contract has two halves: fidelity to the fetch standard, and divergence where the standard assumes a browser. @@ -21,11 +26,9 @@ A divergence is either a browser concept with no server-side meaning or a choice ## Compatibility stance -`fetch(resource, options)` accepts the same shapes as WHATWG fetch: a URL string or stringifiable object (including `URL`), or a Web API `Request` object. -Behaviour follows the fetch standard by default; where browsers and the standards disagree, Faith follows the standards unless a spec here says otherwise (for example, `body` is `null` on responses that cannot have a body). -Browser-only concepts that assume an origin or a browsing context (CORS, `mode`, `referrer`, `referrerPolicy`, `attributionReporting`, `browsingTopics`, `keepalive`) have no server-side meaning; passing them is harmless and they take no effect. -Options in a `RequestInit` that Faith does not recognise are ignored rather than rejected. -Faith-specific extensions (such as `agent`, `timeout`, response `peer`, `version`, `trailers`, `discard()`) are additive: code written against standard fetch runs unmodified. +Behaviour follows the fetch standard by default; where browsers and the standards disagree, Faith follows the standards unless a spec here says otherwise. +Concepts that assume an origin or a browsing context, CORS among them, have no server-side meaning, and a surface that admits the options naming them gives them no effect (see [REQ](fetch/request.md)). +Faith's own extensions add to the standard's surface rather than altering it, so code written against standard fetch runs unmodified. ## Protocol support diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md new file mode 100644 index 0000000..8e205ee --- /dev/null +++ b/.workhorse/specs/rust/client-api.md @@ -0,0 +1,98 @@ +--- +id: RSAPI +--- + +# The Rust client API + +An `Agent` owns the connection pool, resolver, cookie jar, HTTP cache, and HTTP/3 knowledge, and `fetch` is how every request goes out. +The surface is fetch-flavoured rather than a transcription of the JavaScript API: it keeps fetch's vocabulary and the defaults the Node surface has, and it speaks the Rust ecosystem's types wherever one exists for the job. +The agent's own behaviour, its options, and its lifecycle are specified in [AGENT](../agent/overview.md) and the specs beneath it; this spec covers the shape a Rust caller sees. + +## Agent + +`Agent::new()` constructs an agent with default options, and `Agent::builder()` returns a builder whose methods mirror the option groups in [AGENT](../agent/overview.md), ending in `build()`. +Nested option groups are nested builders rather than structs of optional fields, so a group left alone is absent from the call rather than spelled out as absent. +Construction validates options and reports the errors named in [AGENT](../agent/overview.md), and reads the environment variables [ENV](../environment/variables.md) names for this surface. + +An agent is cheap to clone and every clone names the same underlying agent, so cloning one is how a request gets an agent to run on rather than a way to get a second pool. +Because clones share, `close()` acts on the agent itself and every handle to it sees the result. +A request that has already been issued completes, a request issued afterwards fails with the closed-agent error, and `close()` is idempotent, exactly as in [AGENT](../agent/overview.md). +The agent captures what a request needs at the moment the request is issued, which is what lets an in-flight request finish while later ones are refused. + +`network_changed()`, `stats()`, `connections()`, `resolvers()`, `prefetch_dns()`, and `preconnect()` act on a live agent as their counterparts do on the Node surface. + +`cookies()` reaches the agent's jar, returning the `web-faith-cookies` jar itself rather than wrapping it in per-cookie methods, so a caller inserts and reads cookies through the same type that crate documents (see [COOK](../agent/cookies.md)). +An agent with no jar has nothing to hand back and says so in the return type, which is where the Node surface's null-returning reads land in Rust. +The jar outlives `close()` and stays readable from a closed agent, as [AGENT](../agent/overview.md) requires. + +`USER_AGENT` is exported so a caller can prepend its own product token to Faith's default. +The versions it embeds are not exported alongside it, a Rust caller already having its own package metadata to read them from. + +## Making a request + +`agent.fetch(target)` returns a fetch builder. +A target is anything that converts into a `url::Url` — a `Url`, a `&str`, a `String` — or a `Request`, or an `http::Request`. +The builder implements `IntoFuture`, so awaiting it sends the request: a bare call awaits directly, and a configured one awaits after its options. +There is no separate send step, and a builder that is dropped without being awaited sends nothing, so the builder is marked `#[must_use]`. + +Builder methods cover the per-request options in [REQ](../fetch/request.md), [CANCEL](../fetch/cancellation-and-timeouts.md), [SRI](../fetch/integrity.md), [ENC](../fetch/content-encoding.md), and [CACHE](../cache/http-cache.md), and set the method, headers, and body. + +`Request::new(target)` takes the same kinds of target and returns a request builder, which `build()` resolves into a `Request` rather than sending it. +A `Request` is inert, and passing one to `fetch` returns a fetch builder, so a request can be prepared once and adjusted at each call site or sent unchanged on more than one agent. +`try_clone()` copies a request when its body allows it and reports that it cannot when the body is a stream, a stream being consumable once. + +The two builders carry the same option-setting methods, so a call reads the same way whichever one it is written against, and they are distinct types because their terminal steps differ. +A fetch builder is awaited and has no `build()`; a request builder is built and cannot be awaited, so an attempt to send a request that has no agent to send it on is refused by the compiler rather than surfacing when the code runs. +`try_clone()` belongs to `Request` alone, and a fetch builder is not a target, so `fetch` calls do not nest. + +## Layering options + +A request is built in layers: a builder wraps a target, `build()` settles it into a `Request`, and that request can itself be the target of another builder, to any depth. +Each layer wins over the layer it wraps, which is the rule the Node surface follows when options passed to `fetch()` beat the values carried on a `Request` (see [REQ](../fetch/request.md)). +So in `agent.fetch(Request::new(inner).timeout(b)).timeout(c)` the timeout is `c`, whatever `inner` set. +A setting a layer does not touch is inherited from beneath it unchanged, so wrapping a request to adjust one thing leaves the rest as it was. + +Single-valued settings, among them the method, body, timeout, integrity, cache mode, credentials, priority, and request compression, take the outermost value set explicitly. +Setting one twice on a single builder is the same question at a smaller scale, and the later call wins. + +Headers merge by name rather than wholesale, matching how per-request headers beat agent defaults in [REQ](../fetch/request.md). +A name an outer layer sets takes the outer value, a name only an inner layer sets is carried through, and removing a name removes what the layers beneath contributed for it. +Setting a collection of headers applies each entry as a single header would, so it adds to and overrides the set by name rather than replacing it. + +The URL comes from the target at the bottom of the stack, and wrapping a request carries its URL through. + +The agent is not part of this layering: a request runs on whichever agent's `fetch` was called, and a `Request` carries no agent of its own. + +## Reading a response + +A `Response` carries `status`, `status_text`, `ok`, `headers`, `url`, `redirected`, `kind`, and `body_used` with the meanings in [RESP](../response/response.md), and Faith's own `peer` and `version`. +`text()`, `json()`, `bytes()`, `body_stream()`, `to_file()`, and `discard()` read the body, and `trailers()` and `timing()` resolve as in [TRL](../response/trailers.md) and [RESP](../response/response.md). +Reading the body consumes it and a second read fails, following the fetch standard rather than the owned-response model of other Rust clients, as in [BODY](../response/reading-the-body.md). + +The response body implements `http_body::Body`, and a `Response` converts into an `http::Response`, so a Faith response feeds code written against the wider ecosystem without a shim. + +## Ecosystem types + +Types from `http` are canonical wherever one exists: `Method`, `HeaderName`, `HeaderValue`, `HeaderMap`, `StatusCode`, and `Version`. +URLs are `url::Url`, which is what the fetch standard's parsing rules describe and what the stack beneath already uses. +`status_text` and `ok` derive from the status code rather than being carried separately. + +Setters accept anything that converts into the canonical type, so a string literal works where JavaScript would pass a string and a typed value works where a caller already holds one. +A conversion that fails is held until the builder resolves and surfaces there: at `build()` for a request, and at the await for a fetch. +Either way it reports the error naming the offender — `InvalidHeader`, `InvalidMethod`, or `InvalidUrl` for a target that does not parse — as [REQ](../fetch/request.md) and [ERR](../errors/errors.md) require. +Holding the failure is what lets a target be given as a string: an unparseable one is reported where the request is resolved rather than by a call that cannot fail. + +## What a disabled feature removes + +A Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that capability present: no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). +Code written against a capability that is not built fails to compile rather than compiling into a call that does nothing, so a build reports what it does not carry at the point the caller asks for it. + +## Errors and cancellation + +Failures surface as one error type whose variants are the kinds in [ERR](../errors/errors.md), each reporting the same stable code as its JavaScript counterpart. +Errors arriving from a component crate are converted into it at the boundary, so a caller matches on one type whichever layer failed. + +Dropping the future cancels the request, which is how a Rust caller aborts. +The `timeout` option remains for the deadline case, and both surface the errors in [CANCEL](../fetch/cancellation-and-timeouts.md). + +Asynchronous work runs on Tokio. diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md new file mode 100644 index 0000000..1bd0612 --- /dev/null +++ b/.workhorse/specs/rust/overview.md @@ -0,0 +1,71 @@ +--- +id: RUST +--- + +# The Rust distribution + +Faith's network stack is published to crates.io as a family of crates, with `web-faith` as the client a Rust caller reaches for and five component crates beneath it that each stand on their own. +The Node.js native module is built from the same workspace, so the two surfaces are two faces of one implementation rather than two implementations: a behaviour specified anywhere else in these specs holds on both unless that spec says otherwise. +The Rust API surface itself is specified in [RSAPI](client-api.md). + +## The crate family + +`web-faith` is the HTTP client. +It owns the agent, the request and response types, and the `fetch` entry point, and it draws on the component crates for the subsystems beneath it. +The name carries a `web-` prefix because the bare `faith` name on crates.io belongs to an unrelated project, and the prefix reads as the browser-shaped client the crate is. + +Five component crates are published alongside it, each one useful to a caller who wants that piece without the client above it: + +- `web-faith-cookies` is the cookie jar, with the storage, matching, and eviction rules in [COOK](../agent/cookies.md). +- `web-faith-dns` is the resolver, its cache, the discovery ladder, the `HTTPS` record query, and Happy Eyeballs, as in [DNS](../agent/dns.md). +- `web-faith-conn-tracker` reads live per-connection statistics from the operating system, which is the `connections()` view in [OBS](../agent/observability.md); the cumulative counters and the resolver listing that spec also covers belong to `web-faith` and `web-faith-dns` respectively. +- `web-faith-alt-svc` is the Alt-Svc store and the HTTP/3 upgrade machinery, as in [H3UP](../http3/upgrade.md) and [PROBE](../http3/probing.md). +- `web-faith-encoding` is content coding for request and response bodies, as in [ENC](../fetch/content-encoding.md). + +Subresource Integrity parsing and verification is part of `web-faith` itself (see [SRI](../fetch/integrity.md)). + +A component crate depends on another component crate where the subsystems genuinely compose, so `web-faith-alt-svc` draws on `web-faith-dns` for the resolution its probes need. +None of them depends on `web-faith`, which is what makes each one usable on its own. + +`web-faith-napi` is the Node.js binding, and it ships to npm as the prebuilt native module `@passcod/faith`. +It is the one crate in the workspace carrying napi types: every other crate compiles, tests, and documents without a JavaScript runtime present, which is the test that the binding layer is genuinely separate rather than merely renamed. + +## A component crate stands alone + +A component crate names its own error type covering the failures that piece can produce, rather than a type shared across the family. +`web-faith` converts them into its own error as they reach it, and those conversions are what keep the code contract in [ERR](../errors/errors.md) intact across the split. + +A component crate takes and returns types from `http`, `url`, `bytes`, and the other ecosystem crates it already speaks, rather than types belonging to Faith, wherever such a type exists for the job. +Where a component needs a shape from the layer above, it takes it as a generic or through `http::Extensions` rather than depending upward. + +Each component crate carries its own documentation, and its examples run against that crate alone. + +## Subsystems that stay with the client + +QUIC and TLS are not component crates. +Faith reaches both through reqwest, which carries the HTTP/3 implementation Faith uses and offers the choice of TLS backend, so the swapping available there is the one the ecosystem already provides rather than one Faith builds above it. +`web-faith` passes those choices through as features of its own: HTTP/3 has such a feature already, and the TLS backend is selected the same way, with aws-lc-rs the default and ring the alternative. +What the two subsystems do is specified in [QUIC](../http3/transport.md) and [TLS](../agent/tls.md). + +## Choosing what is built + +Cargo features on `web-faith` are how a subsystem is left out of a build that has no use for it. +Features are on by default, so a caller who reaches for the crate without thinking about them gets the whole client. +Turning one off drops the code behind it and the client continues to work without it; the parts of the API that only mean something with that subsystem present go with it, as in [RSAPI](client-api.md). + +A feature decides what is compiled in rather than what is switched on at run time, and the two need not agree: the cookies feature is on by default while the jar itself stays off until an agent asks for it (see [COOK](../agent/cookies.md)). + +Features are the whole of the swapping mechanism: a caller chooses among the implementations Faith builds rather than supplying one. + +## Versioning and support + +Every published crate follows semantic versioning from `1.0.0`, and the crates version independently, so a change confined to one component moves that crate alone. +Releases are prepared by release-plz, and a release runs `cargo-semver-checks` against the previous version of each crate, so a breaking change reaches a major bump rather than a patch. + +The minimum supported Rust version is 1.96, it is declared as `rust-version` in every published crate, and CI builds and tests against it as well as against stable, so the declaration is verified rather than asserted. + +## Workspace layout + +The repository is a Cargo workspace whose members live under `crates/`, one directory per crate named for it. +Shared package metadata (licence, repository, authors, edition, `rust-version`) is declared once at the workspace root and inherited, so the crates cannot drift apart on the fields that describe the same project. +The npm package is built from `crates/web-faith-napi`, and the generated `index.js` and `index.d.ts` continue to sit at the repository root where the package's entry points expect them. diff --git a/Cargo.lock b/Cargo.lock index 46a8716..f39f9db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,46 +577,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "faith" -version = "0.7.0" -dependencies = [ - "async-compression", - "async-stream", - "async-trait", - "bytes", - "cookie", - "cookie_store", - "futures", - "hickory-resolver", - "http", - "http-body-util", - "http-cache-reqwest", - "hyper", - "hyper-util", - "libc", - "moka", - "napi", - "napi-build", - "napi-derive", - "netlink-packet-core", - "netlink-packet-sock-diag", - "netlink-sys", - "reqwest", - "reqwest-middleware", - "serde", - "serde_json", - "ssri", - "stream_shared", - "strum", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "url", - "windows", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -2026,6 +1986,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2834,6 +2795,120 @@ dependencies = [ "web-sys", ] +[[package]] +name = "web-faith" +version = "0.7.0" +dependencies = [ + "async-trait", + "bytes", + "futures", + "http", + "http-body", + "http-body-util", + "http-cache-reqwest", + "hyper", + "hyper-util", + "moka", + "reqwest", + "reqwest-middleware", + "rustls", + "serde", + "serde_json", + "ssri", + "stream_shared", + "strum", + "tokio", + "url", + "web-faith-alt-svc", + "web-faith-conn-tracker", + "web-faith-cookies", + "web-faith-dns", + "web-faith-encoding", +] + +[[package]] +name = "web-faith-alt-svc" +version = "0.7.0" +dependencies = [ + "async-trait", + "http", + "moka", + "reqwest", + "reqwest-middleware", + "tokio", + "url", + "web-faith-dns", +] + +[[package]] +name = "web-faith-conn-tracker" +version = "0.7.0" +dependencies = [ + "libc", + "moka", + "netlink-packet-core", + "netlink-packet-sock-diag", + "netlink-sys", + "tokio", + "windows", +] + +[[package]] +name = "web-faith-cookies" +version = "0.7.0" +dependencies = [ + "cookie", + "cookie_store", + "http", + "reqwest", + "time", + "url", +] + +[[package]] +name = "web-faith-dns" +version = "0.7.0" +dependencies = [ + "futures", + "hickory-resolver", + "moka", + "reqwest", + "tokio", + "url", +] + +[[package]] +name = "web-faith-encoding" +version = "0.7.0" +dependencies = [ + "async-compression", + "bytes", + "futures", + "http", + "tokio", + "tokio-util", +] + +[[package]] +name = "web-faith-napi" +version = "0.7.0" +dependencies = [ + "async-stream", + "bytes", + "futures", + "http-cache-reqwest", + "napi", + "napi-build", + "napi-derive", + "reqwest", + "reqwest-middleware", + "serde_json", + "tokio", + "web-faith", + "web-faith-conn-tracker", + "web-faith-cookies", +] + [[package]] name = "web-sys" version = "0.3.103" diff --git a/Cargo.toml b/Cargo.toml index c15104d..69fe056 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,19 @@ -[package] -name = "faith" +[workspace] +members = ["crates/*"] +# The benchmark HTTP/3 server keeps its own manifest and lockfile on purpose, so +# the quinn/h3 stack stays out of this workspace's dependency graph. +exclude = ["bench/h3-server"] +resolver = "3" + +[workspace.package] version = "0.7.0" edition = "2024" -description = "Faith: a Rust-powered JS fetch" -publish = false - -[lib] -crate-type = ["cdylib"] +rust-version = "1.96" +license = "Apache-2.0 OR MIT" +repository = "https://github.com/passcod/faith" +authors = ["Félix Saparelli "] -[dependencies] +[workspace.dependencies] async-compression = { version = "0.4", default-features = false, features = [ "tokio", "gzip", @@ -31,22 +36,30 @@ hickory-resolver = { version = "0.26", features = [ ] } moka = { version = "0.12", features = ["sync"] } http = "1.4.0" +http-body = "1.0.1" http-body-util = "0.1.3" hyper = "1.8.1" -http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-cacache", "manager-moka"] } +http-cache-reqwest = { version = "1.0.0-alpha.6", features = [ + "manager-cacache", + "manager-moka", +] } hyper-util = { version = "0.1", features = ["client-legacy", "tokio"] } libc = "0.2.179" -napi = { version = "3.7.0", features = ["napi9", "serde-json", "tokio_rt", "web_stream"] } +napi = { version = "3.7.0", features = [ + "napi9", + "serde-json", + "tokio_rt", + "web_stream", +] } +napi-build = "2.3.1" napi-derive = "3.4.0" reqwest = { version = "0.13.4", default-features = false, features = [ - "cookies", - "hickory-dns", "http2", "json", - "rustls", "stream", "system-proxy", ] } +rustls = { version = "0.23.35", default-features = false } reqwest-middleware = { version = "0.5.2", features = ["http2", "rustls"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" @@ -54,25 +67,19 @@ ssri = "9.2.0" stream_shared = { version = "0.8.5", features = ["stats"] } strum = { version = "0.27.2", features = ["derive"] } tokio = { version = "1.48.0", features = ["full"] } -tokio-stream = "0.1.16" time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" - -[target.'cfg(target_os = "linux")'.dependencies] +web-faith = { version = "0.7.0", path = "crates/web-faith", default-features = false } +web-faith-alt-svc = { version = "0.7.0", path = "crates/web-faith-alt-svc", default-features = false } +web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-tracker" } +web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } +web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } +web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } netlink-sys = "0.8.7" - -[target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.62.2", features = [ "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", ] } - -[build-dependencies] -napi-build = "2.3.1" - -[features] -default = ["http3"] -http3 = ["reqwest/http3"] diff --git a/crates/web-faith-alt-svc/Cargo.toml b/crates/web-faith-alt-svc/Cargo.toml new file mode 100644 index 0000000..eea8b42 --- /dev/null +++ b/crates/web-faith-alt-svc/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "web-faith-alt-svc" +description = "An Alt-Svc store and the HTTP/3 upgrade machinery above it" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +async-trait.workspace = true +http.workspace = true +moka.workspace = true +reqwest.workspace = true +reqwest-middleware.workspace = true +tokio.workspace = true +url.workspace = true +web-faith-dns = { workspace = true, optional = true } + +[features] +default = ["dns"] +# Feed `HTTPS` DNS records into the store, so an origin is probe-worthy before anything connects. +dns = ["dep:web-faith-dns"] diff --git a/crates/web-faith-alt-svc/src/cache.rs b/crates/web-faith-alt-svc/src/cache.rs new file mode 100644 index 0000000..3d9f567 --- /dev/null +++ b/crates/web-faith-alt-svc/src/cache.rs @@ -0,0 +1,714 @@ +use std::time::{Duration, Instant}; + +use moka::sync::Cache; + +#[derive(Debug, Clone)] +pub struct AltSvcEntry { + pub port: u16, + pub expires: Instant, +} + +/// An HTTP/3 alternative parsed out of an `Alt-Svc` header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AltSvcAdvertisement { + /// Host the alternative is on. Empty when the header omitted it, which per + /// RFC 7838 means the same host as the origin. + pub host: String, + pub port: u16, + pub max_age: Option, +} + +/// A run of consecutive HTTP/3 failures against one origin. +/// +/// Both instants are carried in the value rather than left to the cache's TTL, +/// because they differ per origin and from each other: the entry deliberately +/// outlives the cooldown it set, so that a count survives the block it caused and +/// can escalate the next one. `advertised` does the same for `ma`. +// spec:H3UP#failure-backoff +#[derive(Debug, Clone, Copy)] +struct FailureEntry { + /// Consecutive failures with no confirmation in between. + count: u32, + /// Until when the origin is blocked from upgrading, probing, and recording + /// advertisements. The only field that gates behaviour. + blocked_until: Instant, + /// Until when `count` still describes a run. Past it the origin is judged + /// from the base cooldown again. + counted_until: Instant, +} + +/// A per-origin exponentially-weighted moving average of time-to-response-headers. +/// +/// Two `f64`s per origin and no sample storage: the average decays stale history +/// by construction, and the count gates decisions until there is enough evidence +/// to mean anything. +#[derive(Debug, Clone, Copy)] +pub struct PathTime { + /// EWMA of time-to-response-headers, in milliseconds. + pub avg_ms: f64, + pub count: u32, +} + +/// Weight of the newest sample in the moving average. +const EWMA_ALPHA: f64 = 0.2; +/// Samples required on *each* side before a slow comparison may act. +const EWMA_MIN_SAMPLES: u32 = 8; +/// Absolute gap the QUIC average must exceed the TCP one by, on top of the +/// factor, so LAN-fast origins don't flap on sub-millisecond noise. +pub const SLOW_FLOOR_MS: f64 = 10.0; + +pub struct AltSvcCacheConfig { + pub advertised_ttl: Duration, + pub confirmed_ttl: Duration, + /// Cooldown a first failure earns; each consecutive one doubles it. + pub failed_ttl: Duration, + /// Ceiling on the doubling. Clamped up to `failed_ttl`, so setting it at or + /// below the base gives a flat cooldown. + pub failed_max_ttl: Duration, + pub capacity: u64, + pub cancel_strikes: u32, + pub strike_window: Duration, + pub follow_advertised_port: bool, + /// Lifetime of a probe's single-flight claim. Doubles as crash recovery: a + /// probe task that dies without reporting frees its origin when this lapses. + pub probe_ttl: Duration, + /// The QUIC path is demoted when its average is worse than TCP's by this + /// factor (and by [`SLOW_FLOOR_MS`] absolutely). `0.0` disables path-time + /// demotion entirely. + pub slow_factor: f64, + /// How long a path-time demotion holds before the origin may be re-probed. + pub slow_ttl: Duration, +} + +#[derive(Clone)] +pub struct AltSvcCache { + advertised: Cache, + confirmed: Cache, + /// Origins that failed over HTTP/3, with their run of consecutive failures. + /// An entry present here is not necessarily blocked: see [`Self::is_failed`]. + failed: Cache, + /// Consecutive cancelled HTTP/3 attempts per origin. Entries expire on a TTL + /// (the strike window), so a run has to be sustained to count. + cancellations: Cache, + /// Single-flight claims for in-flight background probes. + probing: Cache, + /// Origins demoted for being slower over QUIC than over TCP. Distinct from + /// `failed`: the path *works*, so re-advertisements must not be discarded, + /// and expiry re-enters through a probe rather than treating h3 as broken. + slow: Cache, + /// Origins seeded from `http3.hints`, with the port hinted. A hint is the + /// caller's assertion rather than something observed, so it has to be + /// distinguishable from an entry in `confirmed` that a real HTTP/3 response + /// put there: [`Self::network_changed`] demotes the observed ones and + /// re-seeds from here. Unbounded by TTL and outside the capacity bound, + /// because the hints are configuration and there are as many as the caller + /// passed. + // spec:NETCHG#what-the-signal-keeps + hints: Cache, + /// Time-to-headers over TCP (h1 and h2 together), per origin. + tcp_times: Cache, + /// Time-to-headers over QUIC (h3), per origin. + quic_times: Cache, + + advertised_ttl: Duration, + confirmed_ttl: Duration, + failed_ttl: Duration, + failed_max_ttl: Duration, + cancel_strikes: u32, + follow_advertised_port: bool, + slow_factor: f64, +} + +impl std::fmt::Debug for AltSvcCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AltSvcCache") + .field("advertised_count", &self.advertised.entry_count()) + .field("confirmed_count", &self.confirmed.entry_count()) + .field("failed_count", &self.failed.entry_count()) + .field("cancellation_count", &self.cancellations.entry_count()) + .field("probing_count", &self.probing.entry_count()) + .field("slow_count", &self.slow.entry_count()) + .field("hint_count", &self.hints.entry_count()) + .finish() + } +} + +impl AltSvcCache { + pub fn new(config: AltSvcCacheConfig) -> Self { + let AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + capacity, + cancel_strikes, + strike_window, + follow_advertised_port, + probe_ttl, + slow_factor, + slow_ttl, + } = config; + + // A cap below the base would mean the first failure already exceeds it; + // clamping makes that setting a flat cooldown rather than a shorter one. + let failed_max_ttl = failed_max_ttl.max(failed_ttl); + + Self { + advertised: Cache::builder() + .max_capacity(capacity) + .time_to_live(advertised_ttl) + .build(), + confirmed: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + // Twice the longest cooldown: the outer bound on how long an entry + // can be worth keeping, since a count is dropped one cooldown after + // the block it caused lapsed. Per-entry instants do the real work. + failed: Cache::builder() + .max_capacity(capacity) + .time_to_live(failed_max_ttl.saturating_mul(2)) + .build(), + cancellations: Cache::builder() + .max_capacity(capacity) + .time_to_live(strike_window) + .build(), + probing: Cache::builder() + .max_capacity(capacity) + .time_to_live(probe_ttl) + .build(), + slow: Cache::builder() + .max_capacity(capacity) + .time_to_live(slow_ttl) + .build(), + // No TTL and no capacity bound: hints are configuration, held for the + // life of the agent so a network change can re-seed from them. + hints: Cache::builder().build(), + tcp_times: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + quic_times: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + cancel_strikes, + follow_advertised_port, + slow_factor, + } + } + + /// The cooldown the `count`-th consecutive failure earns: the base doubled + /// once per failure before it, capped. + // spec:H3UP#failure-backoff + fn failure_cooldown(&self, count: u32) -> Duration { + let doublings = count.saturating_sub(1).min(u32::BITS - 1); + self.failed_ttl + .saturating_mul(2u32.saturating_pow(doublings)) + .min(self.failed_max_ttl) + } + + /// Whether the origin is inside its failure cooldown. + /// + /// Presence in `failed` is not the question: an entry outlives its cooldown + /// so the failure count survives to escalate the next one. + fn is_failed(&self, origin: &str) -> bool { + self.failed + .get(origin) + .is_some_and(|entry| entry.blocked_until > Instant::now()) + } + + fn origin_key(url: &reqwest::Url) -> Option { + let host = url.host_str()?; + let port = url.port_or_known_default()?; + Some(format!("{}://{}:{}", url.scheme(), host, port)) + } + + pub fn record_alt_svc(&self, url: &reqwest::Url, advertisement: &AltSvcAdvertisement) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // An alternative on a *different host* can never be honoured: reqwest derives + // the HTTP/3 connect target from the request's authority, and rewriting the + // host would also change which certificate is accepted. Unlike a differing + // port — which `follow_advertised_port` can act on — there is nothing to + // gate behind an option, so don't record it at all. RFC 7838 uses an empty + // host to mean "the same host as the origin". + // + // Compared case-insensitively because host names are, and a server naming its + // own host in a different case is still naming its own host. + if !advertisement.host.is_empty() + && !url + .host_str() + .is_some_and(|origin_host| origin_host.eq_ignore_ascii_case(&advertisement.host)) + { + return; + } + + if self.is_failed(&origin) { + return; + } + + if self.confirmed.contains_key(&origin) { + return; + } + + let ttl = advertisement.max_age.unwrap_or(self.advertised_ttl); + let entry = AltSvcEntry { + port: advertisement.port, + expires: Instant::now() + ttl, + }; + + self.advertised.insert(origin, entry); + } + + /// Whether an `HTTPS` DNS record for this origin would tell us anything we do not already + /// know, so the resolver can skip the query rather than send one per lookup. + /// + /// Nothing is learnable while the origin is confirmed (already routing over HTTP/3), failed + /// (blocked whatever a record says), slow (demoted on measurement, which a record cannot + /// overturn), or already carrying a live advertisement (the probe it warrants is already + /// warranted). Each of those states expires, and the query resumes when it does. + // spec:DNS#https-records + pub fn wants_https_record(&self, url: &reqwest::Url) -> bool { + let Some(origin) = Self::origin_key(url) else { + return false; + }; + + !self.is_failed(&origin) + && !self.confirmed.contains_key(&origin) + && !self.slow.contains_key(&origin) + && self + .advertised + .get(&origin) + .is_none_or(|entry| entry.expires <= Instant::now()) + } + + /// Record an HTTP/3 advertisement carried by an `HTTPS` DNS record. + /// + /// An `HTTPS` record and an `Alt-Svc` header are two ways for an origin to say the same thing, + /// so this lands in exactly the state a header advertisement does: the origin becomes + /// probe-worthy, and foreground requests keep to TCP until a probe proves the path. The port + /// and same-host rules are the header's too — [`Self::record_alt_svc`] applies them — because + /// the reasons for them are about what Faith can connect to rather than about where the + /// advertisement was read. + // spec:H3UP#advertisements-from-dns + pub fn record_https_record(&self, url: &reqwest::Url, port: Option, ttl: Duration) { + // A record naming no port describes the origin's own, exactly as an `Alt-Svc` header with + // no alt-authority port would. + let Some(port) = port.or_else(|| url.port_or_known_default()) else { + return; + }; + + self.record_alt_svc( + url, + &AltSvcAdvertisement { + // The same-host case: a record targeting another host is dropped before it gets + // here, since Faith only upgrades to the origin's own host. + host: String::new(), + port, + // The record's own DNS TTL is how long what it says is good for, which is the + // role `ma` plays for a header advertisement. + max_age: Some(ttl), + }, + ); + } + + /// Hints seed `confirmed` directly, not `advertised`: a hint is the *user's* + /// assertion, and routing it through a probe would both second-guess an + /// explicit instruction and break h3-only origins (no TCP listener), which + /// only work if the very first request speaks HTTP/3. Distrust is reserved + /// for what servers advertise. Failure demotes a hinted origin exactly as it + /// does a confirmed one. + pub fn add_hint(&self, host: &str, port: u16) { + let origin = format!("https://{}:{}", host, port); + + // Recorded whether or not it can be acted on right now: the hint is + // configuration, and a failure blocking it is a fact about a path that a + // network change can clear (spec:NETCHG#what-the-signal-keeps). + self.hints.insert(origin.clone(), port); + self.seed_hint(origin, port); + } + + /// Put a hinted origin into `confirmed`, unless a failure currently blocks it. + /// + /// Split out of [`Self::add_hint`] so [`Self::network_changed`] can re-seed the + /// hints it just cleared without re-recording them. + fn seed_hint(&self, origin: String, port: u16) { + if self.is_failed(&origin) { + return; + } + + let entry = AltSvcEntry { + port, + expires: Instant::now() + Duration::from_hours(10_000), // forever + }; + + self.confirmed.insert(origin, entry); + } + + /// Whether an entry advertising `entry_port` can be acted on for this URL. + /// + /// An Alt-Svc advertisement names a network endpoint for the origin; it is not + /// a claim that the origin's *own* port speaks HTTP/3. So when the advertised + /// port differs, upgrading the request on the origin port is an inference the + /// advertisement does not support. + /// + /// Honouring the advertised port properly means connecting to one port while + /// still sending the origin's authority, which reqwest cannot express: it + /// derives the HTTP/3 connect target from the request URI's authority (see + /// ). `follow_advertised_port` + /// opts into doing it anyway by rewriting the request's port, which is not + /// standards-compliant — the request then carries the alternative's authority + /// rather than the origin's. + fn port_actionable(&self, url: &reqwest::Url, entry_port: u16) -> bool { + self.follow_advertised_port || Some(entry_port) == url.port_or_known_default() + } + + /// The port HTTP/3 is *proven* on, or `None` to leave the request on TCP. + /// + /// This is the only lookup foreground routing consults when probing is on: + /// an advertisement is evidence worth probing, not worth routing on. + /// + /// A returned port that differs from the URL's own means the caller opted into + /// `follow_advertised_port` and the request must be rewritten to target it. + pub fn confirmed_port(&self, url: &reqwest::Url) -> Option { + let origin = Self::origin_key(url)?; + + if self.is_failed(&origin) || self.slow.contains_key(&origin) { + return None; + } + + let entry = self.confirmed.get(&origin)?; + if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { + Some(entry.port) + } else { + None + } + } + + /// The advertised port a background probe should verify, or `None` when + /// there is nothing (or no need) to probe: no actionable advertisement, + /// already confirmed, recently failed, or demoted for being slow. + pub fn probe_candidate(&self, url: &reqwest::Url) -> Option { + let origin = Self::origin_key(url)?; + + if self.is_failed(&origin) + || self.slow.contains_key(&origin) + || self.confirmed.contains_key(&origin) + { + return None; + } + + let entry = self.advertised.get(&origin)?; + if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { + Some(entry.port) + } else { + None + } + } + + /// Claim the origin for a probe. Returns `false` when a probe is already in + /// flight; the claim expires on its own (see [`AltSvcCacheConfig::probe_ttl`]) + /// if the prober never reports back. + pub fn claim_probe(&self, url: &reqwest::Url) -> bool { + let Some(origin) = Self::origin_key(url) else { + return false; + }; + self.probing.entry(origin).or_insert(()).is_fresh() + } + + /// Release the origin's probe claim, so a later advertisement can re-probe + /// without waiting out the claim's TTL. + pub fn finish_probe(&self, url: &reqwest::Url) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + self.probing.invalidate(&origin); + } + + /// The port to attempt HTTP/3 on, or `None` to leave the request on TCP. + /// + /// Legacy (probe-less) routing: advertisements are acted on inline, so this + /// consults `advertised` as well as `confirmed`. Only used when + /// probing is off. + pub fn should_use_h3(&self, url: &reqwest::Url) -> Option { + self.confirmed_port(url) + .or_else(|| self.probe_candidate(url)) + } + + /// Record a foreground request's time-to-response-headers for its protocol + /// family, and demote the origin to TCP if QUIC is provenly, sustainedly + /// slower than TCP for it. + /// + /// Time-to-headers includes server think-time, which varies per endpoint far + /// more than per transport; only the averages across many requests are + /// comparable, never individual samples — hence the minimum sample counts. + /// Redirects followed inside the attempt inflate a sample for whichever + /// family carried it, which the averaging absorbs the same way. + /// + /// The comparison is deliberately asymmetric: HTTP/3 is preferred at parity + /// and when moderately slower, because its advantages (no head-of-line + /// blocking, connection migration) pay off beyond the mean. Only a large + /// sustained gap demotes. + pub fn record_path_time(&self, url: &reqwest::Url, version: http::Version, elapsed: Duration) { + if self.slow_factor <= 0.0 { + return; + } + + let Some(origin) = Self::origin_key(url) else { + return; + }; + + let sample_ms = elapsed.as_secs_f64() * 1000.0; + let times = if version == http::Version::HTTP_3 { + &self.quic_times + } else { + &self.tcp_times + }; + + let updated = times + .entry(origin.clone()) + .and_upsert_with(|existing| match existing { + None => PathTime { + avg_ms: sample_ms, + count: 1, + }, + Some(entry) => { + let entry = entry.into_value(); + PathTime { + avg_ms: entry.avg_ms * (1.0 - EWMA_ALPHA) + sample_ms * EWMA_ALPHA, + count: entry.count.saturating_add(1), + } + } + }) + .into_value(); + + if version == http::Version::HTTP_3 + && updated.count >= EWMA_MIN_SAMPLES + && let Some(tcp) = self.tcp_times.get(&origin) + && tcp.count >= EWMA_MIN_SAMPLES + && updated.avg_ms > tcp.avg_ms * self.slow_factor + && updated.avg_ms - tcp.avg_ms > SLOW_FLOOR_MS + { + self.demote_slow(&origin); + } + } + + /// Demote a working-but-slow QUIC origin back to TCP. + /// + /// The confirmed entry moves back to `advertised` rather than being dropped: + /// when the `slow` marker expires, the advertisement is what makes the next + /// request trigger a re-probe — "has this path improved?" asked at zero + /// foreground cost. The QUIC average is cleared so the answer is judged on + /// fresh samples, not held hostage by the history that demoted it. + fn demote_slow(&self, origin: &str) { + let key = origin.to_string(); + let Some(entry) = self.confirmed.get(&key) else { + return; + }; + + self.confirmed.invalidate(&key); + self.advertised.insert( + key.clone(), + AltSvcEntry { + port: entry.port, + expires: Instant::now() + self.advertised_ttl, + }, + ); + self.quic_times.invalidate(&key); + self.slow.insert(key, ()); + } + + /// Record that HTTP/3 worked for this origin, on the port it connected to. + /// + /// `port` must be the port the successful attempt actually used. Recovering it + /// from the caches instead would be unsound: a concurrent failure that cleared + /// them leaves nothing to read, and falling back to the origin's own port would + /// confirm HTTP/3 on a port the server never advertised — for `confirmed_ttl`, + /// and invisibly, since the concurrent failure's `failed` entry masks it until + /// that expires. + pub fn confirm_h3(&self, url: &reqwest::Url, port: u16) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // Promoted out of `advertised`; it has served its purpose. + self.advertised.invalidate(&origin); + // A working h3 response is proof of health; forget any strikes, and end + // whatever run of failures preceded it. + self.cancellations.invalidate(&origin); + self.clear_failure_count(&origin); + + let entry = AltSvcEntry { + port, + expires: Instant::now() + self.confirmed_ttl, + }; + + self.confirmed.insert(origin, entry); + } + + /// Record an HTTP/3 attempt that was cancelled before producing an outcome. + /// + /// This is weaker evidence than an error: the request never got to find out + /// whether HTTP/3 worked, so a single cancellation says nothing about the + /// origin. Only a sustained run of them demotes it, which keeps callers that + /// routinely abort healthy requests from disabling HTTP/3. + /// + /// The window is a TTL measured from the *previous* strike, because moka + /// refreshes an entry's TTL on upsert. Strikes therefore have to arrive + /// within a window of each other, not within a fixed bucket. + pub fn record_h3_cancellation(&self, url: &reqwest::Url) { + if self.cancel_strikes == 0 { + return; + } + + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // This is reachable from a `Drop` impl (see the guard below), which must + // never panic: a panic while already unwinding aborts the process. Use a + // saturating add so an absurd `upgrade_cancel_strikes` can't overflow. + let strikes = self + .cancellations + .entry(origin) + .and_upsert_with(|existing| { + existing.map_or(1, |entry| entry.into_value().saturating_add(1)) + }) + .into_value(); + + if strikes >= self.cancel_strikes { + // Clears the strike count as a side effect. + self.record_h3_failure(url); + } + } + + /// Forget the origin's run of failures, so the next one starts the backoff + /// from the base cooldown again. + /// + /// A cooldown still running is left alone. A confirmation racing a concurrent + /// failure must not unblock the origin that failure just blocked: the failure + /// is the more recent evidence about the path, and [`Self::confirm_h3`] + /// relies on its own entry being masked until the block lapses. + // spec:H3UP#failure-backoff + fn clear_failure_count(&self, origin: &str) { + let Some(entry) = self.failed.get(origin) else { + return; + }; + + if entry.blocked_until > Instant::now() { + self.failed + .insert(origin.to_string(), FailureEntry { count: 0, ..entry }); + } else { + self.failed.invalidate(origin); + } + } + + /// Discard everything this cache learned by observing the network, keeping + /// what it was told. + /// + /// Every state here except `advertised` and `hints` describes the path between + /// this client and an origin, and a network change is exactly the event that + /// invalidates such a description. So the observation-confirmed origins are + /// demoted rather than kept (the path that proved them is gone, and a probe + /// re-proves them without a foreground request paying for it), and the + /// failures, strikes, slow markers and averages go entirely: they are + /// penalties and measurements the old path earned, and carrying them over + /// would judge the new network by the old one's behaviour. + /// + /// What the origin said about itself (`advertised`) and what the caller + /// asserted (`hints`) are not observations, so both survive. + // spec:NETCHG + pub fn network_changed(&self) { + let now = Instant::now(); + + // Demote first, while `confirmed` still holds the entries: an advertisement + // is what makes the next request to the origin trigger a re-probe. + // + // Keys are invalidated one by one rather than with `invalidate_all`, whose + // timestamp-based invalidation would race the hint re-seeding below. + for (origin, entry) in self.confirmed.iter() { + // A hint holds its origin confirmed; it is an assertion, not a finding. + if self.hints.contains_key(origin.as_str()) { + continue; + } + + self.confirmed.invalidate(origin.as_str()); + + // A logically expired entry is not knowledge to carry forward: it would + // come back as a fresh advertisement having just lapsed as a confirmation. + if entry.expires <= now { + continue; + } + + self.advertised.insert( + (*origin).clone(), + AltSvcEntry { + port: entry.port, + expires: now + self.advertised_ttl, + }, + ); + } + + self.failed.invalidate_all(); + self.cancellations.invalidate_all(); + self.slow.invalidate_all(); + // In-flight probes are aborted by the caller of this method, so their + // single-flight claims would otherwise hold their origins until the claim + // TTL lapsed. + self.probing.invalidate_all(); + self.tcp_times.invalidate_all(); + self.quic_times.invalidate_all(); + + // After the failures are cleared, so a hint that a cooldown had been + // blocking takes effect now rather than staying refused. + for (origin, port) in self.hints.iter() { + self.seed_hint((*origin).clone(), port); + } + } + + /// Record a failed HTTP/3 attempt, blocking the origin for a cooldown that + /// lengthens the longer it keeps failing. + // spec:H3UP#failure-backoff + pub fn record_h3_failure(&self, url: &reqwest::Url) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + self.advertised.invalidate(&origin); + self.confirmed.invalidate(&origin); + // Already demoted; further counting is meaningless. + self.cancellations.invalidate(&origin); + + let now = Instant::now(); + // An entry whose run has lapsed is history, not a run in progress: the + // origin went a whole further cooldown without failing again, so it is + // judged from the base. + let count = self + .failed + .get(&origin) + .filter(|entry| entry.counted_until > now) + .map_or(1, |entry| entry.count.saturating_add(1)); + let cooldown = self.failure_cooldown(count); + + self.failed.insert( + origin, + FailureEntry { + count, + blocked_until: now + cooldown, + // The count has to outlive the block it caused, or it could never + // escalate: the next attempt only comes once the block lapses. + counted_until: now + cooldown.saturating_mul(2), + }, + ); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-alt-svc/src/cache/tests.rs b/crates/web-faith-alt-svc/src/cache/tests.rs new file mode 100644 index 0000000..39f7c63 --- /dev/null +++ b/crates/web-faith-alt-svc/src/cache/tests.rs @@ -0,0 +1,1016 @@ +use std::time::{Duration, Instant}; + +use super::*; + +#[test] +fn test_ipv6_origin_accepts_its_own_host_spelled_out() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://[2001:db8::1]/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "[2001:db8::1]".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "an IPv6 origin naming its own address is the same host, brackets and all" + ); +} + +#[test] +fn test_host_comparison_ignores_case() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "ExAmPlE.CoM".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "host names are case-insensitive, so this still names the origin's own host" + ); +} + +#[test] +fn test_alt_svc_on_another_host_is_not_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "cdn.example.net".to_string(), + port: 443, + max_age: None, + }, + ); + + assert!( + cache.should_use_h3(&url).is_none(), + "h3 on another host says nothing about this one, and the port matching is \ + coincidental" + ); +} + +#[test] +fn test_alt_svc_naming_our_own_host_is_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "example.com".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "spelling out the origin's own host is equivalent to omitting it" + ); +} + +#[test] +fn test_confirm_h3_uses_the_port_it_was_given() { + // A concurrent failure can clear both caches between the attempt starting and + // confirming. `confirm_h3` must not fall back to the origin's port then, or it + // would confirm h3 on a port nobody advertised. + let cache = test_cache_with(3, Duration::from_secs(60), true); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + cache.record_h3_failure(&url); + cache.confirm_h3(&url, 8443); + + let entry = cache + .confirmed + .get(&"https://example.com:443".to_string()) + .expect("the successful attempt is confirmed"); + assert_eq!( + entry.port, 8443, + "confirmed on the port actually connected to, not the origin's" + ); +} + +/// A same-host advertisement, the common case. +fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { + AltSvcAdvertisement { + host: String::new(), + port, + max_age, + } +} + +fn test_cache() -> AltSvcCache { + test_cache_with(3, Duration::from_secs(60), false) +} + +fn test_cache_with( + cancel_strikes: u32, + strike_window: Duration, + follow_advertised_port: bool, +) -> AltSvcCache { + test_cache_failing( + cancel_strikes, + strike_window, + follow_advertised_port, + Duration::from_secs(300), + Duration::from_secs(3600), + ) +} + +/// A cache whose knowledge expires soon, for tests about knowledge that has lapsed. +fn test_cache_ttls(advertised_ttl: Duration, confirmed_ttl: Duration) -> AltSvcCache { + AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl: Duration::from_secs(300), + failed_max_ttl: Duration::from_secs(3600), + capacity: 10_000, + cancel_strikes: 3, + strike_window: Duration::from_secs(60), + follow_advertised_port: false, + probe_ttl: Duration::from_secs(10), + slow_factor: 2.5, + slow_ttl: Duration::from_millis(200), + }) +} + +fn test_cache_failing( + cancel_strikes: u32, + strike_window: Duration, + follow_advertised_port: bool, + failed_ttl: Duration, + failed_max_ttl: Duration, +) -> AltSvcCache { + AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl: Duration::from_secs(86400), + confirmed_ttl: Duration::from_secs(86400), + failed_ttl, + failed_max_ttl, + capacity: 10_000, + cancel_strikes, + strike_window, + follow_advertised_port, + probe_ttl: Duration::from_secs(10), + slow_factor: 2.5, + slow_ttl: Duration::from_millis(200), + }) +} + +#[test] +fn test_advertised_port_matching_origin_upgrades() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "an advertisement for the origin's own port is actionable" + ); +} + +#[test] +fn test_advertised_port_mismatch_does_not_upgrade() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + assert!( + cache.should_use_h3(&url).is_none(), + "h3 advertised on :8443 says nothing about :443, so don't upgrade" + ); +} + +#[test] +fn test_advertised_port_mismatch_is_still_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + let entry = cache + .advertised + .get(&"https://example.com:443".to_string()) + .expect("the advertisement is kept even though it isn't actionable"); + assert_eq!( + entry.port, 8443, + "keeping it means the port is available if reqwest ever lets us honour it" + ); +} + +#[test] +fn test_advertised_port_mismatch_upgrades_when_following() { + let cache = test_cache_with(3, Duration::from_secs(60), true); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + assert_eq!( + cache.should_use_h3(&url), + Some(8443), + "opting in returns the advertised port so the request can be rewritten" + ); +} + +#[test] +fn test_cache_flow() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + assert!(cache.should_use_h3(&url).is_none()); + + cache.record_alt_svc(&url, &ad(443, Some(Duration::from_secs(3600)))); + assert_eq!(cache.should_use_h3(&url), Some(443)); + + cache.confirm_h3(&url, 443); + assert_eq!(cache.should_use_h3(&url), Some(443)); + assert!( + !cache + .advertised + .contains_key(&"https://example.com:443".to_string()) + ); + assert!( + cache + .confirmed + .contains_key(&"https://example.com:443".to_string()) + ); +} + +#[test] +fn test_cache_failure() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.should_use_h3(&url).is_some()); + + cache.record_h3_failure(&url); + assert!(cache.should_use_h3(&url).is_none()); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.should_use_h3(&url).is_none()); +} + +#[test] +fn test_hint() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + assert_eq!(cache.should_use_h3(&url), Some(443)); +} + +#[test] +fn test_hint_is_confirmed_not_probed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "a hint is the user's assertion and routes immediately, probe or no probe" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "nothing to verify: the hint already confirmed the origin" + ); +} + +#[test] +fn test_https_record_lands_as_an_advertisement_not_a_confirmation() { + // spec:H3UP#advertisements-from-dns — DNS is a second source of the same advertisement, + // so it makes the origin probe-worthy without routing a foreground request onto an + // unverified QUIC path. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, None, Duration::from_secs(3600)); + + assert!( + cache.confirmed_port(&url).is_none(), + "a record is evidence worth probing, not worth routing on" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "and a record naming no port describes the origin's own" + ); +} + +#[test] +fn test_https_record_port_follows_the_advertised_port_rules() { + // spec:H3UP#advertised-ports — a `port` differing from the origin's is treated exactly as + // an `Alt-Svc` advertised port: recorded, but not acted on by default. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, Some(8443), Duration::from_secs(3600)); + + assert!( + cache.probe_candidate(&url).is_none(), + "h3 on :8443 says nothing about :443, so nothing is probed" + ); + + let following = test_cache_with(3, Duration::from_secs(60), true); + following.record_https_record(&url, Some(8443), Duration::from_secs(3600)); + assert_eq!( + following.probe_candidate(&url), + Some(8443), + "opting into the quirk probes the advertised port" + ); +} + +#[test] +fn test_https_record_is_refused_while_the_origin_is_failed() { + // A failure blocks recording fresh advertisements whatever their source, or a flapping + // origin could re-enter the cycle through DNS (spec:H3UP#advertisements-from-dns). + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.record_https_record(&url, None, Duration::from_secs(3600)); + + assert!(cache.probe_candidate(&url).is_none()); + assert!( + !cache.wants_https_record(&url), + "and there is no point querying again while the cooldown runs" + ); +} + +#[test] +fn test_wants_https_record_only_while_there_is_something_to_learn() { + // spec:DNS#https-records — the gate is what keeps the query off every lookup once the + // origin's HTTP/3 support is settled either way. + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + let unknown = test_cache(); + assert!( + unknown.wants_https_record(&url), + "an origin nothing is known about is worth asking about" + ); + + let advertised = test_cache(); + advertised.record_alt_svc(&url, &ad(443, None)); + assert!( + !advertised.wants_https_record(&url), + "a live advertisement already warrants the probe a record would" + ); + + let confirmed = test_cache(); + confirmed.confirm_h3(&url, 443); + assert!( + !confirmed.wants_https_record(&url), + "a confirmed origin is already routing over HTTP/3" + ); +} + +#[test] +fn test_wants_https_record_again_once_the_advertisement_lapses() { + // The gate must not be permanent: an advertisement that expires without being confirmed + // leaves the origin unknown again, and DNS is how it can be re-learned. + let cache = test_cache_ttls(Duration::from_millis(50), Duration::from_secs(86400)); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(!cache.wants_https_record(&url)); + + std::thread::sleep(Duration::from_millis(120)); + + assert!( + cache.wants_https_record(&url), + "once the advertisement lapses the query is worth making again" + ); +} + +#[test] +fn test_https_record_ttl_bounds_the_advertisement() { + // spec:H3UP#advertisements-from-dns — the record's own DNS TTL is what the advertisement + // lives for, the role `ma` plays for a header. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, None, Duration::from_millis(50)); + assert_eq!(cache.probe_candidate(&url), Some(443)); + + std::thread::sleep(Duration::from_millis(120)); + + assert!( + cache.probe_candidate(&url).is_none(), + "past the record's TTL the advertisement is no longer evidence" + ); +} + +#[test] +fn test_advertised_routes_nothing_but_probes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + assert!( + cache.confirmed_port(&url).is_none(), + "an advertisement is evidence worth probing, not worth routing on" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "and it is exactly what the probe should verify" + ); +} + +#[test] +fn test_probe_candidate_respects_failed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + + assert!( + cache.probe_candidate(&url).is_none(), + "a failed origin is not re-probed until the cooldown lapses" + ); +} + +#[test] +fn test_probe_confirmation_promotes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.claim_probe(&url), "first claim wins"); + cache.confirm_h3(&url, 443); + cache.finish_probe(&url); + + assert_eq!(cache.confirmed_port(&url), Some(443)); + assert!( + cache.probe_candidate(&url).is_none(), + "confirmed origins are not probed again" + ); +} + +#[test] +fn test_claim_probe_is_single_flight() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + assert!(cache.claim_probe(&url)); + assert!( + !cache.claim_probe(&url), + "a second claim while one is in flight loses" + ); + + cache.finish_probe(&url); + assert!( + cache.claim_probe(&url), + "finishing the probe frees the origin for the next one" + ); +} + +#[test] +fn test_slow_demotion_needs_sustained_evidence() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // Plenty of TCP samples at 5ms, but too few QUIC samples to act on. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + } + for _ in 0..(EWMA_MIN_SAMPLES - 1) { + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "below the minimum sample count no comparison may act" + ); +} + +#[test] +fn test_slow_demotion_moves_origin_back_to_probing() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // TCP steady at 5ms, QUIC steady at 50ms: 10x the average and 45ms over, + // clearing both the factor and the absolute floor. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert!( + cache.confirmed_port(&url).is_none(), + "a sustained large gap demotes the origin off HTTP/3" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "while the slow marker lives, the origin is not re-probed either" + ); + assert!( + !cache.is_failed("https://example.com:443"), + "slow is not broken: the failed cache stays out of it" + ); + + // The test cache's slow TTL is short; once it lapses, the advertisement + // preserved by the demotion re-enters through a probe. + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "slow expiry re-enters via the probe, asking whether the path improved" + ); +} + +#[test] +fn test_parity_or_moderately_slower_quic_is_kept() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // QUIC 2x slower and 20ms over: above the floor but below the 2.5x + // factor, so HTTP/3's structural advantages win the tie. + for _ in 0..(EWMA_MIN_SAMPLES * 2) { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(20)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(40)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "moderately slower QUIC is still preferred" + ); +} + +#[test] +fn test_cancellation_below_threshold_keeps_h3() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "two strikes is not enough to demote" + ); +} + +#[test] +fn test_cancellation_at_threshold_demotes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + for _ in 0..3 { + cache.record_h3_cancellation(&url); + } + + assert!( + cache.should_use_h3(&url).is_none(), + "three strikes demotes the origin" + ); + assert!( + cache.is_failed("https://example.com:443"), + "demotion goes through the failed cache, so re-advertisement can't re-arm it" + ); +} + +#[test] +fn test_cancellation_reset_by_h3_success() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + cache.confirm_h3(&url, 443); + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "a working h3 response clears the strikes, so these two start over" + ); +} + +#[test] +fn test_cancellation_disabled_by_zero() { + let cache = test_cache_with(0, Duration::from_secs(60), false); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + for _ in 0..5 { + cache.record_h3_cancellation(&url); + } + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "cancel_strikes: 0 disables cancellation-based demotion" + ); +} + +#[test] +fn test_cancellation_strikes_decay() { + let cache = test_cache_with(3, Duration::from_millis(50), false); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + std::thread::sleep(Duration::from_millis(150)); + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "strikes older than the window don't count towards the run" + ); +} + +fn failure_entry(cache: &AltSvcCache) -> FailureEntry { + cache + .failed + .get("https://example.com:443") + .expect("the origin has a failure on record") +} + +#[test] +fn test_failure_cooldown_doubles_up_to_the_cap() { + let cache = test_cache(); + + let schedule: Vec = (1..=6) + .map(|count| cache.failure_cooldown(count).as_secs()) + .collect(); + + assert_eq!( + schedule, + vec![300, 600, 1200, 2400, 3600, 3600], + "each consecutive failure doubles the base, then holds at the cap" + ); +} + +#[test] +fn test_failure_cooldown_cap_below_base_is_flat() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_secs(300), + Duration::from_secs(60), + ); + + let schedule: Vec = (1..=4) + .map(|count| cache.failure_cooldown(count).as_secs()) + .collect(); + + assert_eq!( + schedule, + vec![300, 300, 300, 300], + "a cap under the base is clamped up to it, giving a cooldown that never backs off" + ); +} + +#[test] +fn test_consecutive_failures_lengthen_the_cooldown() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_millis(200), + Duration::from_secs(60), + ); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + assert!( + cache.is_failed("https://example.com:443"), + "the first failure blocks the origin" + ); + + // Past the first cooldown, but well inside the run's own lifetime: this is + // the retry the cooldown allowed, and it fails too. + std::thread::sleep(Duration::from_millis(250)); + assert!( + !cache.is_failed("https://example.com:443"), + "the first cooldown lapses on its own" + ); + + cache.record_h3_failure(&url); + let entry = failure_entry(&cache); + assert_eq!( + entry.count, 2, + "failing again straight after a lapsed cooldown continues the run" + ); + assert!( + cache.is_failed("https://example.com:443"), + "and blocks the origin again, for twice as long" + ); +} + +#[test] +fn test_run_lapses_when_the_origin_stops_failing() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_millis(100), + Duration::from_secs(60), + ); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + // One cooldown beyond the block it caused: nobody exercised the origin in + // that stretch, so the next failure is judged on its own. + std::thread::sleep(Duration::from_millis(300)); + + cache.record_h3_failure(&url); + assert_eq!( + failure_entry(&cache).count, + 1, + "an origin left alone past its run starts from the base cooldown again" + ); +} + +#[test] +fn test_confirmation_ends_the_run() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.record_h3_failure(&url); + assert_eq!(failure_entry(&cache).count, 2); + + cache.confirm_h3(&url, 443); + cache.record_h3_failure(&url); + + let entry = failure_entry(&cache); + assert_eq!( + entry.count, 1, + "a working h3 response ends the run, so the next failure starts at the base" + ); + assert_eq!( + entry.blocked_until.duration_since(Instant::now()).as_secs(), + 299, + "and is blocked for the base cooldown, not the doubled one" + ); +} + +#[test] +fn test_confirmation_does_not_cut_a_live_cooldown_short() { + // A confirmation can race a concurrent failure. The failure is the more + // recent word on the path, so it keeps the origin blocked; only the run + // is forgotten. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.confirm_h3(&url, 443); + + assert!( + cache.is_failed("https://example.com:443"), + "the cooldown the failure set still runs" + ); + assert_eq!( + failure_entry(&cache).count, + 0, + "but the run behind it is cleared" + ); +} + +#[test] +fn test_network_change_demotes_confirmed_to_advertised() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + cache.network_changed(); + + assert!( + cache.confirmed_port(&url).is_none(), + "the path that proved HTTP/3 is gone, so the origin is no longer confirmed" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "it keeps its advertisement, so a background probe re-verifies it at once" + ); +} + +#[test] +fn test_network_change_clears_failures_and_their_backoff() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.network_changed(); + + assert!( + !cache.is_failed("https://example.com:443"), + "a blocked UDP path is a fact about the old network" + ); + assert!( + cache.failed.get("https://example.com:443").is_none(), + "and so is the run of failures that set the cooldown" + ); + + // The advertisement a failure discards has to come back for the origin to be + // probe-worthy again, so re-record it as a live response would. + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + assert_eq!( + failure_entry(&cache).count, + 1, + "failing on the new network starts the backoff from the base cooldown" + ); +} + +#[test] +fn test_network_change_clears_a_slow_demotion() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + assert!( + cache.probe_candidate(&url).is_none(), + "the slow marker holds the origin off probing before the signal" + ); + + cache.network_changed(); + + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "a slow path was slow on the old network, so the origin re-enters through a probe" + ); +} + +#[test] +fn test_network_change_clears_the_path_time_averages() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + // Enough TCP samples to satisfy the comparison's minimum, all of them fast. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + } + + cache.network_changed(); + cache.confirm_h3(&url, 443); + + // Slow enough to demote several times over, were the old TCP average still there + // to compare against. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "with the TCP average cleared there is nothing to judge QUIC against, so \ + no demotion happens on one side's samples alone" + ); +} + +#[test] +fn test_network_change_keeps_hints_confirmed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + + cache.network_changed(); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "a hint is the caller's assertion, not an observation, so it survives the signal \ + and keeps an HTTP/3-only origin reachable" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "and a hinted origin is still never probed" + ); +} + +#[test] +fn test_network_change_lets_a_blocked_hint_take_effect() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + cache.record_h3_failure(&url); + assert!( + cache.confirmed_port(&url).is_none(), + "a failure demotes a hinted origin like any other" + ); + + cache.network_changed(); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "the failure that was masking the hint belonged to the old network, so the \ + hint holds again once it is cleared" + ); +} + +#[test] +fn test_network_change_keeps_advertisements() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + cache.network_changed(); + + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "an advertisement is the origin's statement about itself, which a change of \ + client network does not revise" + ); +} + +#[test] +fn test_network_change_drops_expired_confirmations() { + let cache = test_cache_ttls(Duration::from_millis(100), Duration::from_millis(100)); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.confirm_h3(&url, 443); + std::thread::sleep(Duration::from_millis(150)); + + cache.network_changed(); + + assert!( + cache.probe_candidate(&url).is_none(), + "a confirmation that had already lapsed is not knowledge to carry forward as \ + a fresh advertisement" + ); +} + +#[test] +fn test_network_change_releases_probe_claims() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.claim_probe(&url), "the first probe claims the origin"); + assert!(!cache.claim_probe(&url), "and holds it single-flight"); + + cache.network_changed(); + + assert!( + cache.claim_probe(&url), + "probes in flight are aborted with the client they ran on, so their claims \ + must not hold the origin for the claim TTL" + ); +} diff --git a/crates/web-faith-alt-svc/src/header.rs b/crates/web-faith-alt-svc/src/header.rs new file mode 100644 index 0000000..f346fa4 --- /dev/null +++ b/crates/web-faith-alt-svc/src/header.rs @@ -0,0 +1,75 @@ +use std::time::Duration; + +use crate::cache::AltSvcAdvertisement; + +pub fn parse_alt_svc_header(value: &str) -> Option { + if value == "clear" { + return None; + } + + for service in value.split(',') { + let service = service.trim(); + if service.is_empty() { + continue; + } + + let mut protocol_id: Option<&str> = None; + let mut host: Option<&str> = None; + let mut port: Option = None; + let mut max_age: Option = None; + + for param in service.split(';') { + let param = param.trim(); + if param.is_empty() { + continue; + } + + let Some((key, value)) = param.split_once('=') else { + continue; + }; + + let key = key.trim(); + let value = value.trim().trim_matches('"'); + + match key { + "ma" => { + if let Ok(secs) = value.parse::() { + max_age = Some(Duration::from_secs(secs)); + } + } + _ if key.starts_with("h3") => { + protocol_id = Some(key); + // The alt-authority is `[host]:port`, where an omitted host means + // the origin's own. Keep the host: acting on an advertisement for + // a different host would be the same unsupported inference as + // acting on one for a different port. + // + // Split on the *last* colon so a bracketed IPv6 literal survives, + // and keep it exactly as written — brackets included. That is the + // form `Url::host_str` also returns for IPv6, so comparing the two + // needs no normalising on either side. + if let Some((alt_host, port_str)) = value.rsplit_once(':') { + host = Some(alt_host); + if let Ok(p) = port_str.parse::() { + port = Some(p); + } + } + } + _ => {} + } + } + + if protocol_id.is_some() && port.is_some() { + return Some(AltSvcAdvertisement { + host: host.unwrap_or_default().to_owned(), + port: port.unwrap(), + max_age, + }); + } + } + + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-alt-svc/src/header/tests.rs b/crates/web-faith-alt-svc/src/header/tests.rs new file mode 100644 index 0000000..5a74872 --- /dev/null +++ b/crates/web-faith-alt-svc/src/header/tests.rs @@ -0,0 +1,86 @@ +use std::time::Duration; + +use super::parse_alt_svc_header; +use crate::cache::AltSvcAdvertisement; + +#[test] +fn test_parse_alt_svc_simple() { + let result = parse_alt_svc_header(r#"h3=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_no_max_age() { + let result = parse_alt_svc_header(r#"h3=":443""#); + assert_eq!(result, Some(ad(443, None))); +} + +#[test] +fn test_parse_alt_svc_different_port() { + let result = parse_alt_svc_header(r#"h3=":8443"; ma=3600"#); + assert_eq!(result, Some(ad(8443, Some(Duration::from_secs(3600))))); +} + +#[test] +fn test_parse_alt_svc_multiple_protocols() { + let result = parse_alt_svc_header(r#"h2=":443", h3=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_h3_variant() { + let result = parse_alt_svc_header(r#"h3-29=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_keeps_the_host() { + let result = parse_alt_svc_header(r#"h3="cdn.example.net:443"; ma=3600"#); + assert_eq!( + result, + Some(AltSvcAdvertisement { + host: "cdn.example.net".to_string(), + port: 443, + max_age: Some(Duration::from_secs(3600)), + }), + "the alt-authority's host must survive parsing, or a different-host \ + advertisement looks same-host once the port matches" + ); +} + +#[test] +fn test_parse_alt_svc_ipv6_host() { + let result = parse_alt_svc_header(r#"h3="[2001:db8::1]:8443""#); + assert_eq!( + result, + Some(AltSvcAdvertisement { + // Brackets kept: this is the form `Url::host_str` returns too, so the + // two compare directly. + host: "[2001:db8::1]".to_string(), + port: 8443, + max_age: None, + }), + "splitting on the last colon keeps a bracketed IPv6 literal intact" + ); +} + +#[test] +fn test_parse_alt_svc_clear() { + let result = parse_alt_svc_header("clear"); + assert_eq!(result, None); +} + +#[test] +fn test_parse_alt_svc_no_h3() { + let result = parse_alt_svc_header(r#"h2=":443"; ma=86400"#); + assert_eq!(result, None); +} + +/// A same-host advertisement, the common case. +fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { + AltSvcAdvertisement { + host: String::new(), + port, + max_age, + } +} diff --git a/crates/web-faith-alt-svc/src/https_sink.rs b/crates/web-faith-alt-svc/src/https_sink.rs new file mode 100644 index 0000000..0fce8a6 --- /dev/null +++ b/crates/web-faith-alt-svc/src/https_sink.rs @@ -0,0 +1,80 @@ +use std::sync::{Arc, Weak}; + +use crate::{cache::AltSvcCache, prober::H3Prober}; + +/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is +/// probe-worthy from its first request rather than from the first TCP response carrying an +/// `Alt-Svc` header. +/// +/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), +/// which is the only place that holds all three: the resolver is built before the cache, and the +/// prober holds a client that holds the resolver, so nothing lower down can own this. +/// +/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the +/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could +/// name. Recording it there is right whichever request triggered the lookup, because what the +/// record describes does not depend on who asked. +// spec:H3UP#advertisements-from-dns +// spec:DNS#https-records +pub struct H3HttpsSink { + cache: Arc, + /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, + /// and the resolver holds this sink. A strong reference here would close that ring and leak + /// the whole graph — connection pool included — past `Agent::close`, which works by dropping + /// the client. The agent owns the only strong reference, so this lives exactly as long as the + /// agent's prober does. + /// + /// `None` rather than a dead handle when probing is off, where an advertisement is acted on + /// inline by the next foreground request instead. + // spec:PROBE + prober: Option>, +} + +impl std::fmt::Debug for H3HttpsSink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("H3HttpsSink") + .field("probing", &self.prober.is_some()) + .finish() + } +} + +impl H3HttpsSink { + pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { + Self { + cache, + prober: prober.map(Arc::downgrade), + } + } + + /// The origin a record at `host` describes: the default HTTPS port, which is the port whose + /// record lives at the bare name. + fn origin_url(host: &str) -> Option { + reqwest::Url::parse(&format!("https://{host}")).ok() + } +} + +impl web_faith_dns::HttpsSink for H3HttpsSink { + fn wants(&self, host: &str) -> bool { + Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) + } + + fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { + let Some(url) = Self::origin_url(host) else { + return; + }; + + self.cache + .record_https_record(&url, advertisement.port, advertisement.ttl); + + // Probe straight away rather than waiting for the request that triggered the lookup to + // finish: the point of reading DNS is that the path can be verified while that request is + // still on TCP, so the one after it upgrades. + // + // A prober that has gone means the agent was closed (or rebuilt) while this query was in + // flight; the advertisement above is still worth keeping, but there is nothing left to + // probe it with, and resurrecting a dropped client to try would be exactly wrong. + if let Some(prober) = self.prober.as_ref().and_then(Weak::upgrade) { + prober.maybe_probe(&url); + } + } +} diff --git a/crates/web-faith-alt-svc/src/lib.rs b/crates/web-faith-alt-svc/src/lib.rs new file mode 100644 index 0000000..7587f5e --- /dev/null +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -0,0 +1,42 @@ +//! An Alt-Svc store, and the machinery that upgrades an origin to HTTP/3 on the strength of it. +//! +//! An origin advertises HTTP/3 in an `Alt-Svc` header, or in an `HTTPS` DNS record. Acting on that +//! is not as simple as believing it: the alternative may be unreachable even though it was +//! advertised, and finding out costs the request that tries. What this does is keep the +//! advertisements ([`AltSvcCache`]) and decide, per origin, whether HTTP/3 is worth attempting. +//! +//! [`AltSvcMiddleware`] is the layer that acts on the decision. Two shapes are available: +//! +//! - With an [`H3Prober`], an advertisement is verified in the background and foreground requests +//! are routed over HTTP/3 only once an origin is confirmed, so no user-visible request pays for +//! discovering a broken alternative. +//! - Without one, the next foreground request is itself the verification, falling back to TCP if the +//! attempt does not produce headers in time. +//! +//! Either way an origin that starts failing, or that turns out to be slower over HTTP/3 than the +//! path it replaced ([`PathTime`]), is demoted, and the cooldown before it is tried again lengthens +//! with each consecutive failure. +//! +//! [`parse_alt_svc_header`] reads a header on its own if all you want is the advertisement, and +//! [`H3HttpsSink`] feeds the store from `HTTPS` record lookups. + +// spec:H3UP spec:PROBE + +mod cache; +mod header; + +#[cfg(feature = "dns")] +mod https_sink; + +mod middleware; +mod prober; + +pub use cache::{ + AltSvcAdvertisement, AltSvcCache, AltSvcCacheConfig, AltSvcEntry, PathTime, SLOW_FLOOR_MS, +}; +pub use header::parse_alt_svc_header; +#[cfg(feature = "dns")] +pub use https_sink::H3HttpsSink; +pub use middleware::{AltSvcMiddleware, ArrivalStamp}; + +pub use prober::H3Prober; diff --git a/crates/web-faith-alt-svc/src/middleware.rs b/crates/web-faith-alt-svc/src/middleware.rs new file mode 100644 index 0000000..365fcf2 --- /dev/null +++ b/crates/web-faith-alt-svc/src/middleware.rs @@ -0,0 +1,278 @@ +use std::{ + marker::PhantomData, + sync::Arc, + time::{Duration, Instant}, +}; + +use http::Extensions; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result}; + +use crate::{cache::AltSvcCache, header::parse_alt_svc_header, prober::H3Prober}; + +/// Recording the moment a response's headers arrived. +/// +/// The stamp itself belongs to the client, which puts it in the request's extensions and reads it +/// back out to surface the timing; this layer is only the one place that observes the arrival, so +/// all it needs is to be able to mark what the client left there. +pub trait ArrivalStamp: Send + Sync + 'static { + fn mark(&self, at: Instant); +} + +/// Records a cancellation if the HTTP/3 attempt it guards is dropped before +/// producing an outcome. +/// +/// [`AltSvcMiddleware`] can only learn that HTTP/3 is broken from the attempt's +/// return value, and a cancelled request never produces one: `faith_fetch` +/// races `send()` against the abort signal in a `select!`, which drops the +/// losing future. Without this guard nothing ever demotes the origin, so a +/// caller whose deadline is shorter than the network's own failure detection +/// re-attempts HTTP/3 over a dead path on every retry, indefinitely. +struct H3AttemptGuard { + cache: Arc, + url: reqwest::Url, + armed: bool, +} + +impl H3AttemptGuard { + fn new(cache: Arc, url: reqwest::Url) -> Self { + Self { + cache, + url, + armed: true, + } + } + + /// The attempt produced an outcome, so it speaks for itself. + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for H3AttemptGuard { + fn drop(&mut self) { + // Must stay infallible: this can run while unwinding, where a panic + // would abort the process. moka's sync cache does not panic on insert. + if self.armed { + self.cache.record_h3_cancellation(&self.url); + } + } +} + +/// The Alt-Svc layer, which upgrades an origin to HTTP/3 once it advertises one. +/// +/// `S` is the client's arrival stamp, which this marks when a response's headers land; see +/// [`ArrivalStamp`]. +#[derive(Clone)] +pub struct AltSvcMiddleware { + cache: Arc, + enabled: bool, + /// Ceiling on how long an HTTP/3 attempt may take to produce response + /// headers before it is treated as failed and retried over TCP. + attempt_timeout: Option, + /// `Some` routes foreground requests on confirmed origins only, verifying + /// advertisements in the background. `None` restores the inline upgrade, + /// where the next foreground request is the verification. + prober: Option>, + stamp: PhantomData, +} + +impl std::fmt::Debug for AltSvcMiddleware { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AltSvcMiddleware") + .field("enabled", &self.enabled) + .field("attempt_timeout", &self.attempt_timeout) + .field("prober", &self.prober) + .field("cache", &self.cache) + .finish() + } +} + +impl AltSvcMiddleware { + pub fn new( + cache: Arc, + enabled: bool, + attempt_timeout: Option, + prober: Option>, + ) -> Self { + Self { + cache, + enabled, + attempt_timeout, + prober, + stamp: PhantomData, + } + } + + #[allow(dead_code)] + pub fn cache(&self) -> &Arc { + &self.cache + } + + /// Kick off a background probe for the URL's origin if one is warranted: + /// probing enabled, an actionable advertisement present, the origin neither + /// confirmed, failed, nor slow, and no probe already in flight. + fn maybe_probe(&self, url: &reqwest::Url) { + if let Some(prober) = &self.prober { + prober.maybe_probe(url); + } + } +} + +/// Run the rest of the stack and stamp the moment the response headers arrive. +/// +/// This is the one place a response's arrival is observed: the returned instant is what the +/// path-time average measures against, and the same instant reaches the caller through the +/// request's extensions to become the surfaced timing, so the two can never disagree. +// spec:RESP#request-timing +async fn run_stamped( + next: Next<'_>, + req: Request, + extensions: &mut Extensions, +) -> (Result, Instant) { + let result = next.run(req, extensions).await; + let at = Instant::now(); + if result.is_ok() + && let Some(stamp) = extensions.get::() + { + stamp.mark(at); + } + (result, at) +} + +#[async_trait::async_trait] +impl Middleware for AltSvcMiddleware { + async fn handle( + &self, + mut req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + if !self.enabled { + return run_stamped::(next, req, extensions).await.0; + } + + let url = req.url().clone(); + + // With a prober, routing consults proven origins only — advertisements + // get verified out-of-band, so no foreground request ever waits on an + // unverified QUIC path. Without one, the legacy inline upgrade applies. + let h3_route = if self.prober.is_some() { + self.cache.confirmed_port(&url) + } else { + self.cache.should_use_h3(&url) + }; + + if let Some(h3_port) = h3_route { + // Clone the request before attempting HTTP/3 so we can retry with TCP if it fails + if let Some(req_clone) = req.try_clone() { + *req.version_mut() = http::Version::HTTP_3; + + // A port differing from the origin's only comes back when + // `follow_advertised_port` is set — `should_use_h3` filters + // mismatches out otherwise. Rewriting the URL is the only way to + // make reqwest connect elsewhere, and it MUST happen after the + // clone above so the TCP fallback still targets the origin. + // + // Every cache operation below keeps using `url`, the origin, so + // confirmations, failures and strikes stay keyed on the origin + // rather than on the alternative endpoint. + if Some(h3_port) != url.port_or_known_default() { + let _ = req.url_mut().set_port(Some(h3_port)); + } + + let mut guard = H3AttemptGuard::new(Arc::clone(&self.cache), url.clone()); + // Measured to response headers: this layer sits inside the cache + // middleware, so `next.run` resolves when headers arrive, before + // any body buffering. + let started = Instant::now(); + // `None` means the attempt ran out of time. Bound in its own + // statement so the mutable borrow of `extensions` ends here, + // leaving the fallback below free to use it. + let outcome = match self.attempt_timeout { + Some(limit) => { + tokio::time::timeout(limit, run_stamped::(next.clone(), req, extensions)) + .await + .ok() + } + None => Some(run_stamped::(next.clone(), req, extensions).await), + }; + // Reached on success, error and expiry alike; only a mid-flight + // drop skips it and leaves the guard armed. + guard.disarm(); + + match outcome { + Some((Ok(response), at)) => { + if response.version() == http::Version::HTTP_3 { + self.cache.confirm_h3(&url, h3_port); + self.cache.record_path_time( + &url, + response.version(), + at.duration_since(started), + ); + } + + if let Some(alt_svc) = response.headers().get("alt-svc") { + if let Ok(value) = alt_svc.to_str() { + if let Some(advertisement) = parse_alt_svc_header(value) { + self.cache.record_alt_svc(&url, &advertisement); + } + } + } + + Ok(response) + } + // An expired deadline is as good as an error: HTTP/3 did not + // deliver. Taking the fallback branch directly avoids having + // to synthesise a reqwest_middleware::Error, which would mean + // adding anyhow as a dependency. + Some((Err(_), _)) | None => { + self.cache.record_h3_failure(&url); + + // Use the cloned request (which still has default HTTP version) + let started = Instant::now(); + let (result, at) = run_stamped::(next, req_clone, extensions).await; + if let Ok(ref response) = result { + self.cache.record_path_time( + &url, + response.version(), + at.duration_since(started), + ); + } + result + } + } + } else { + // Can't clone request (streaming body), just proceed without HTTP/3 + run_stamped::(next, req, extensions).await.0 + } + } else { + // An advertisement from an earlier response may still be waiting on + // verification (or on a fresh single-flight claim after a probe task + // died); this is the belt to the post-response trigger's braces. + self.maybe_probe(&url); + + let started = Instant::now(); + let (result, at) = run_stamped::(next, req, extensions).await; + + // Check for Alt-Svc header in non-HTTP/3 responses + if let Ok(ref response) = result { + self.cache + .record_path_time(&url, response.version(), at.duration_since(started)); + + if let Some(alt_svc) = response.headers().get("alt-svc") { + if let Ok(value) = alt_svc.to_str() { + if let Some(advertisement) = parse_alt_svc_header(value) { + self.cache.record_alt_svc(&url, &advertisement); + // Probe as soon as the advertisement lands, racing + // the gap before the caller's next request. + self.maybe_probe(&url); + } + } + } + } + + result + } + } +} diff --git a/crates/web-faith-alt-svc/src/prober.rs b/crates/web-faith-alt-svc/src/prober.rs new file mode 100644 index 0000000..65f41cf --- /dev/null +++ b/crates/web-faith-alt-svc/src/prober.rs @@ -0,0 +1,128 @@ +use std::{sync::Arc, time::Duration}; + +use crate::cache::AltSvcCache; + +/// Verifies advertised HTTP/3 endpoints in the background, so no foreground +/// request ever waits on an unverified QUIC path. +/// +/// The probe is a real request — `HEAD /` sent with `Version::HTTP_3` — on the +/// **raw** `reqwest::Client`, not the middleware stack. That is load-bearing +/// three times over: it bypasses the HTTP cache, so a replayed cached response +/// (rebuilt with its stored HTTP version) can never fake a confirmation; it +/// bypasses [`AltSvcMiddleware`](crate::AltSvcMiddleware), so probing cannot recurse; and it shares the +/// h3 connection pool with foreground requests, so a successful probe leaves +/// behind a warm QUIC connection the next request rides. Confirmation doubles +/// as prewarming. +/// +/// Any HTTP/3 response confirms, regardless of status: a 401 or 405 to +/// `HEAD /` proves the transport end-to-end just as well as a 200. +pub struct H3Prober { + client: reqwest::Client, + cache: Arc, + /// `None` leaves the attempt bounded only by the QUIC idle timeout. + timeout: Option, + /// Handles for in-flight probes, so `Agent::close` can abort them: a probe + /// holds a clone of the raw client, which would otherwise keep the + /// connection pool alive past close for up to the probe timeout. + tasks: std::sync::Mutex>, +} + +impl std::fmt::Debug for H3Prober { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("H3Prober") + .field("timeout", &self.timeout) + .finish() + } +} + +impl H3Prober { + pub fn new( + client: reqwest::Client, + cache: Arc, + timeout: Option, + ) -> Self { + Self { + client, + cache, + timeout, + tasks: std::sync::Mutex::new(Vec::new()), + } + } + + /// Spawn a probe of `port` for the origin of `url`. The caller must hold the + /// origin's single-flight claim (see [`AltSvcCache::claim_probe`]). + fn spawn(&self, url: reqwest::Url, port: u16) { + let client = self.client.clone(); + let cache = Arc::clone(&self.cache); + let timeout = self.timeout; + + let handle = tokio::spawn(async move { + let mut probe_url = url.clone(); + probe_url.set_path("/"); + probe_url.set_query(None); + probe_url.set_fragment(None); + let _ = probe_url.set_username(""); + let _ = probe_url.set_password(None); + // Same rewrite rule as the foreground path: a port differing from the + // origin's only gets here when `follow_advertised_port` is on. + if Some(port) != url.port_or_known_default() { + let _ = probe_url.set_port(Some(port)); + } + + let attempt = client.head(probe_url).version(http::Version::HTTP_3).send(); + + let outcome = match timeout { + Some(limit) => tokio::time::timeout(limit, attempt).await.ok(), + None => Some(attempt.await), + }; + + // Cache operations stay keyed on `url`, the origin, matching the + // foreground path. + match outcome { + Some(Ok(response)) if response.version() == http::Version::HTTP_3 => { + cache.confirm_h3(&url, port); + } + // A response that is somehow not HTTP/3 is a failure too: the + // h3 route did not deliver, whatever answered. + _ => cache.record_h3_failure(&url), + } + + // An aborted probe never reaches this; its claim expires on the + // probing TTL instead, which is why that TTL exceeds the timeout. + cache.finish_probe(&url); + }); + + // A poisoned lock only means another thread panicked mid-push; the Vec + // is still sound to use, and probing must never take the process down. + let mut tasks = self + .tasks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + tasks.retain(|task| !task.is_finished()); + tasks.push(handle.abort_handle()); + } + + /// Kick off a background probe for the URL's origin if one is warranted: an actionable + /// advertisement present, the origin neither confirmed, failed, nor slow, and no probe + /// already in flight. The same decision the Alt-Svc layer makes on a request, exposed so a + /// `preconnect` TCP warm-up to a probe-worthy origin triggers a probe as a real request would. + pub fn maybe_probe(&self, url: &reqwest::Url) { + let Some(port) = self.cache.probe_candidate(url) else { + return; + }; + if !self.cache.claim_probe(url) { + return; + } + self.spawn(url.clone(), port); + } + + pub fn abort_all(&self) { + let mut tasks = self + .tasks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for task in tasks.drain(..) { + task.abort(); + } + } +} diff --git a/crates/web-faith-conn-tracker/Cargo.toml b/crates/web-faith-conn-tracker/Cargo.toml new file mode 100644 index 0000000..706fac5 --- /dev/null +++ b/crates/web-faith-conn-tracker/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "web-faith-conn-tracker" +description = "Live per-connection TCP statistics, read from the operating system" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +moka.workspace = true +tokio.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +netlink-packet-core.workspace = true +netlink-packet-sock-diag.workspace = true +netlink-sys.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +libc.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +windows.workspace = true diff --git a/src/conn_tracker.rs b/crates/web-faith-conn-tracker/src/lib.rs similarity index 65% rename from src/conn_tracker.rs rename to crates/web-faith-conn-tracker/src/lib.rs index 2e764ed..4a575b3 100644 --- a/src/conn_tracker.rs +++ b/crates/web-faith-conn-tracker/src/lib.rs @@ -1,18 +1,35 @@ +//! Live per-connection TCP statistics, read from the operating system. +//! +//! A connection pool can say which connections it holds, but not how any of them is actually +//! behaving: round-trip time, retransmits, congestion window, delivery rate. The kernel knows, and +//! this reads it. +//! +//! Report traffic on a connection with [`ConnectionTracker::track`], which also answers whether that +//! connection had been seen before, and take the current view with +//! [`ConnectionTracker::snapshot`]. Each entry's statistics are refreshed once a second, and an +//! entry that goes idle for longer than the configured timeout expires out of the tracker. +//! +//! Reading the statistics is per-platform: Linux over netlink, macOS and Windows through their own +//! interfaces. Anywhere else, connections are still tracked but carry no statistics. + +// spec:OBS + #[cfg(target_os = "linux")] +#[path = "platform/linux.rs"] mod linux; #[cfg(target_os = "macos")] +#[path = "platform/macos.rs"] mod macos; #[cfg(target_os = "windows")] +#[path = "platform/windows.rs"] mod windows; use std::net::SocketAddr; use std::sync::Arc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime}; use moka::Expiry; use moka::{ops::compute::Op, sync::Cache}; -use napi::{Env, JsDate}; -use napi_derive::napi; use tokio::{spawn, task::AbortHandle, time::sleep}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -79,25 +96,20 @@ pub struct TcpStats { pub delivery_rate: Option, } -#[napi(object)] -#[derive(Clone)] -pub struct ConnectionInfo<'env> { - pub connection_type: String, - pub local_address: String, - pub local_port: u16, - pub remote_address: String, - pub remote_port: u16, - pub first_seen: Option>, - pub last_seen: Option>, - pub expiry: Option>, - pub response_count: i64, - pub rtt_us: Option, - pub rtt_var_us: Option, - pub lost_packets: Option, - pub retransmits: Option, - pub total_retransmits: Option, - pub congestion_window: Option, - pub delivery_rate_bps: Option, +/// One tracked connection, as a caller reporting on the pool sees it. +#[derive(Debug, Clone)] +pub struct ConnectionSnapshot { + /// The transport the connection runs over. Only TCP is tracked. + pub connection_type: &'static str, + pub local_addr: SocketAddr, + pub remote_addr: SocketAddr, + pub first_seen: SystemTime, + pub last_seen: SystemTime, + /// When the entry falls out of the tracker, unless traffic renews it first. + pub expiry: Option, + pub response_count: u64, + /// What the operating system last reported for this connection, if it has been asked yet. + pub stats: Option, } type Conns = Cache; @@ -169,10 +181,11 @@ impl ConnectionTracker { /// Register a warm-up connection that no request has yet been credited to. /// - /// A `preconnect` connection is listed before any foreground request uses it, at a response - /// count of zero (spec:WARM). An entry already tracked is left untouched: a warm-up to an - /// origin that already holds a pooled connection does no new work, and must not disturb the - /// count or timestamps of the connection it would reuse. + /// A warm-up connection is listed before any foreground request uses it, at a response count of + /// zero. An entry already tracked is left untouched: a warm-up to an origin that already holds a + /// pooled connection does no new work, and must not disturb the count or timestamps of the + /// connection it would reuse. + // spec:WARM pub fn track_warmup(&self, local_addr: SocketAddr, remote_addr: SocketAddr) { let now = SystemTime::now(); let key = ConnectionKey { @@ -193,49 +206,19 @@ impl ConnectionTracker { }); } - pub fn get_for_napi<'env>(&self, env: &'env Env) -> Vec> { + /// Every connection currently tracked. + pub fn snapshot(&self) -> Vec { self.connections .iter() - .map(|(key, conn)| ConnectionInfo { - connection_type: "tcp".to_string(), - local_address: key.local_addr.ip().to_string(), - local_port: key.local_addr.port(), - remote_address: key.remote_addr.ip().to_string(), - remote_port: key.remote_addr.port(), - first_seen: env - .create_date( - conn.first_seen - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok(), - last_seen: env - .create_date( - conn.last_seen - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok(), - expiry: conn.last_seen.checked_add(self.timeout).and_then(|exp| { - env.create_date( - exp.duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok() - }), - response_count: conn.response_count as i64, - rtt_us: conn.latest_stats.map(|s| s.rtt_us as i64), - rtt_var_us: conn.latest_stats.map(|s| s.rtt_var_us as i64), - lost_packets: conn.latest_stats.and_then(|s| s.lost.map(|v| v as i64)), - retransmits: conn.latest_stats.map(|s| s.retrans as i64), - total_retransmits: conn.latest_stats.map(|s| s.total_retrans as i64), - congestion_window: conn.latest_stats.map(|s| s.cwnd as i64), - delivery_rate_bps: conn - .latest_stats - .and_then(|s| s.delivery_rate.map(|v| v as i64)), + .map(|(key, conn)| ConnectionSnapshot { + connection_type: "tcp", + local_addr: key.local_addr, + remote_addr: key.remote_addr, + first_seen: conn.first_seen, + last_seen: conn.last_seen, + expiry: conn.last_seen.checked_add(self.timeout), + response_count: conn.response_count, + stats: conn.latest_stats, }) .collect() } diff --git a/src/conn_tracker/linux.rs b/crates/web-faith-conn-tracker/src/platform/linux.rs similarity index 100% rename from src/conn_tracker/linux.rs rename to crates/web-faith-conn-tracker/src/platform/linux.rs diff --git a/src/conn_tracker/macos.rs b/crates/web-faith-conn-tracker/src/platform/macos.rs similarity index 100% rename from src/conn_tracker/macos.rs rename to crates/web-faith-conn-tracker/src/platform/macos.rs diff --git a/src/conn_tracker/windows.rs b/crates/web-faith-conn-tracker/src/platform/windows.rs similarity index 100% rename from src/conn_tracker/windows.rs rename to crates/web-faith-conn-tracker/src/platform/windows.rs diff --git a/crates/web-faith-cookies/Cargo.toml b/crates/web-faith-cookies/Cargo.toml new file mode 100644 index 0000000..c678d0f --- /dev/null +++ b/crates/web-faith-cookies/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "web-faith-cookies" +description = "A cookie jar with the RFC 6265bis rules that hold outside a browser" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +cookie.workspace = true +cookie_store.workspace = true +http.workspace = true +time.workspace = true +url.workspace = true +reqwest = { workspace = true, optional = true } + +[dev-dependencies] +# Doctests link as their own crate, so the types the API speaks have to be named there too. +url.workspace = true + +[features] +# Implement reqwest's `CookieStore`, so the jar can serve as its cookie provider. +reqwest = ["dep:reqwest", "reqwest/cookies"] diff --git a/src/cookies.rs b/crates/web-faith-cookies/src/lib.rs similarity index 88% rename from src/cookies.rs rename to crates/web-faith-cookies/src/lib.rs index 0ab8007..ad2bcfe 100644 --- a/src/cookies.rs +++ b/crates/web-faith-cookies/src/lib.rs @@ -1,20 +1,52 @@ -//! The agent's cookie jar. (spec:COOK) +//! A cookie jar for HTTP clients, with the storage rules that hold outside a browser. //! -//! `cookie_store` implements the classic RFC 6265 storage model, and reqwest's [`Jar`] wraps it in a -//! private field, so extending it means wrapping the store ourselves rather than the jar. What this -//! adds is the RFC 6265bis rules that mean something without a browsing context: the `__Host-` and -//! `__Secure-` name prefixes, a cap on how far ahead a cookie may expire, and caps on how many -//! cookies and how many bytes one server can accumulate. `SameSite` is left to the `cookie` crate to -//! parse and is never read, governing cross-site behaviour that only a first-party context has. +//! Cookies were specified for browsers, and the parts of RFC 6265 that assume a browsing context do +//! not carry over to a client making requests on its own account. This jar keeps the model that +//! does: the classic storage and matching rules, and the RFC 6265bis additions that still mean +//! something with no browser around them. //! -//! [`Jar`]: reqwest::cookie::Jar +//! - The `__Host-` and `__Secure-` name prefixes, which bind a cookie to the exact host that set it +//! and to a secure transport. +//! - A ceiling on how far ahead a cookie may expire, so a server cannot claim a decade. +//! - Caps on the size of one cookie, and on how many are kept per host and in total. +//! +//! Every rule is applied when a cookie is stored rather than when one is sent. That is what makes +//! the caps bound real memory, and what holds a cookie inserted by hand to the same rules as one +//! that arrived in a `Set-Cookie` header. +//! +//! `SameSite` is parsed but never read: it governs cross-site behaviour that only a first-party +//! context has. +//! +//! # Example +//! +//! ``` +//! use url::Url; +//! use web_faith_cookies::{CookieLimits, FaithJar}; +//! +//! let jar = FaithJar::new(CookieLimits::default()); +//! let url = Url::parse("https://example.com/")?; +//! +//! jar.add_cookie_str("session=abc; Path=/", &url); +//! +//! let header = jar.request_cookie_header(&url).expect("a cookie to send"); +//! assert_eq!(header.to_str()?, "session=abc"); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! # Features +//! +//! `reqwest` implements that client's `CookieStore` for the jar, so it can be handed to a +//! `ClientBuilder` as a cookie provider. + +// spec:COOK use std::{collections::HashMap, sync::RwLock, time::Duration}; use cookie::{Cookie as RawCookie, Expiration}; use cookie_store::{Cookie as StoredCookie, CookieStore as Store, StoreAction}; -use reqwest::{Url, cookie::CookieStore, header::HeaderValue}; +use http::HeaderValue; use time::OffsetDateTime; +use url::Url; /// A cookie may not persist beyond this by default. RFC 6265bis §5.5. pub const DEFAULT_MAX_AGE: Duration = Duration::from_secs(400 * 24 * 60 * 60); @@ -110,7 +142,7 @@ impl FaithJar { /// Gate a cookie on the bis rules, then hand it to the classic storage model. /// /// Gating on the way in rather than filtering on the way out is what makes the caps bound real - /// memory, and what makes the rules apply the same to `addCookie` as to a `Set-Cookie` header. + /// memory, and what makes the rules apply the same to [`FaithJar::add_cookie_str`] as to a `Set-Cookie` header. fn store_one(&self, raw: RawCookie<'static>, url: &Url) { let Some(raw) = self.sanitise(raw, url) else { return; @@ -271,8 +303,16 @@ impl Inner { } } -impl CookieStore for FaithJar { - fn set_cookies(&self, cookie_headers: &mut dyn Iterator, url: &Url) { +impl FaithJar { + /// Store the cookies a response set, ignoring any the jar's rules refuse. + /// + /// A header that is not valid UTF-8, or that does not parse as a cookie, is skipped rather than + /// failing the read: one bad `Set-Cookie` does not spoil the response. + pub fn store_response_cookies<'h>( + &self, + cookie_headers: impl Iterator, + url: &Url, + ) { for header in cookie_headers { let Ok(header) = std::str::from_utf8(header.as_bytes()) else { continue; @@ -286,7 +326,8 @@ impl CookieStore for FaithJar { } } - fn cookies(&self, url: &Url) -> Option { + /// The `Cookie` header to send to `url`, or `None` when the jar has nothing for it. + pub fn request_cookie_header(&self, url: &Url) -> Option { let inner = self.inner.read().unwrap(); let cookies = inner .store @@ -303,6 +344,17 @@ impl CookieStore for FaithJar { } } +#[cfg(feature = "reqwest")] +impl reqwest::cookie::CookieStore for FaithJar { + fn set_cookies(&self, cookie_headers: &mut dyn Iterator, url: &Url) { + self.store_response_cookies(cookie_headers, url); + } + + fn cookies(&self, url: &Url) -> Option { + self.request_cookie_header(url) + } +} + #[cfg(test)] mod tests { use super::*; @@ -321,7 +373,7 @@ mod tests { /// What the jar would send to `url`, as a `Cookie` header value. fn sent(jar: &FaithJar, url: &str) -> Option { - jar.cookies(&self::url(url)) + jar.request_cookie_header(&self::url(url)) .map(|value| value.to_str().unwrap().to_owned()) } @@ -733,7 +785,7 @@ mod tests { HeaderValue::from_static("__Host-bad=1; Path=/"), ]; - jar.set_cookies(&mut headers.iter(), &url("https://example.com/")); + jar.store_response_cookies(headers.iter(), &url("https://example.com/")); let sent = sent(&jar, "https://example.com/").unwrap(); assert!(sent.contains("__Host-good=1"), "{sent}"); diff --git a/crates/web-faith-dns/Cargo.toml b/crates/web-faith-dns/Cargo.toml new file mode 100644 index 0000000..4d3ca4f --- /dev/null +++ b/crates/web-faith-dns/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "web-faith-dns" +description = "A caching DNS resolver: transports, server order, HTTPS records, and Happy Eyeballs" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +futures.workspace = true +hickory-resolver.workspace = true +moka.workspace = true +tokio.workspace = true +url.workspace = true +reqwest = { workspace = true, optional = true } + +[features] +# Implement reqwest's `Resolve`, so the resolver can be installed on its client. +reqwest = ["dep:reqwest", "reqwest/hickory-dns"] diff --git a/crates/web-faith-dns/src/discovery.rs b/crates/web-faith-dns/src/discovery.rs new file mode 100644 index 0000000..21cac62 --- /dev/null +++ b/crates/web-faith-dns/src/discovery.rs @@ -0,0 +1,178 @@ +use hickory_resolver::{ + TokioResolver, + config::{ + GOOGLE, LookupIpStrategy, NameServerConfig, OpportunisticEncryption, ResolveHosts, + ResolverConfig, ServerOrderingStrategy, + }, + net::{NetError, runtime::TokioRuntimeProvider}, + system_conf::read_system_conf, +}; + +use crate::{ + settings::{ResolverReport, ResolverSettings, ResolverSource, report}, + transport::ServerHost, +}; + +/// The resolver and the report of its servers, built together the first time the resolver is used. +pub(crate) struct Built { + pub(crate) resolver: TokioResolver, + pub(crate) reports: Vec, +} + +/// Apply the options common to every resolver Faith builds: race both families for Happy Eyeballs, +/// hold the caller's order fixed rather than reordering by latency, and layer any `dns.*` timeout, +/// ndots, and hosts-file settings on top. +pub(crate) fn apply_options( + builder: &mut hickory_resolver::ResolverBuilder, + settings: &ResolverSettings, +) { + let options = builder.options_mut(); + options.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + // The list expresses the caller's intent, not a performance hint, so a private resolver named + // first must not lose traffic to a closer fallback (spec:DNS#server-order). + options.server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; + if let Some(timeout) = settings.timeout { + options.timeout = timeout; + } + if let Some(ndots) = settings.ndots { + options.ndots = ndots; + } + if let Some(hosts_file) = settings.hosts_file { + options.use_hosts_file = if hosts_file { + ResolveHosts::Always + } else { + ResolveHosts::Never + }; + } +} + +/// Discovery: configure from the system, then let hickory's RFC 9539 opportunistic encryption +/// upgrade those servers to DoT/DoQ where they answer a probe. `dns.searchDomains` overrides the +/// system search list when set. +// spec:DNS#discovery +pub(crate) fn build_discovery(settings: &ResolverSettings) -> Result { + let (mut config, options) = read_system_conf().unwrap_or_else(|_| { + // A host with no readable resolver configuration falls back to Google Public DNS over + // conventional DNS, probed like any other server (spec:DNS#discovery). + ( + ResolverConfig::udp_and_tcp(&GOOGLE), + hickory_resolver::config::ResolverOpts::default(), + ) + }); + + if let Some(search) = &settings.search_domains { + config = ResolverConfig::from_parts(None, search.clone(), config.name_servers().to_vec()); + } + + let reports = report(config.name_servers(), ResolverSource::Conventional); + + let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()) + .with_options(options); + apply_options(&mut builder, settings); + let builder = builder.with_opportunistic_encryption(OpportunisticEncryption::Enabled { + config: Default::default(), + }); + + Ok(Built { + resolver: builder.build()?, + reports, + }) +} + +/// The resolver that bootstraps hostname servers: the listed IP-host servers in order, so an +/// encrypted server placed first resolves its siblings without exposing the hostname in plaintext. +/// Where the list names no IP host, the system's own configuration bootstraps instead. +pub(crate) fn bootstrap_resolver(settings: &ResolverSettings) -> Result { + let ip_servers: Vec = settings + .servers + .iter() + .filter_map(|spec| spec.ip().map(|ip| spec.to_name_server(ip))) + .collect(); + + let mut builder = if ip_servers.is_empty() { + TokioResolver::builder_tokio().unwrap_or_else(|_| { + TokioResolver::builder_with_config( + ResolverConfig::udp_and_tcp(&GOOGLE), + TokioRuntimeProvider::default(), + ) + }) + } else { + TokioResolver::builder_with_config( + ResolverConfig::from_parts(None, vec![], ip_servers), + TokioRuntimeProvider::default(), + ) + }; + builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + builder.options_mut().server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; + builder.build() +} + +/// Build the configured (or discovered) resolver and the report of its servers. +pub(crate) async fn build(settings: &ResolverSettings) -> Result { + if settings.servers.is_empty() { + build_discovery(settings) + } else { + build_listed(settings).await + } +} + +/// The listed-servers path: bootstrap any hostname hosts to addresses, then build the resolver +/// from the parsed specs in order. +// spec:DNS#transports +// spec:DNS#bootstrapping +pub(crate) async fn build_listed(settings: &ResolverSettings) -> Result { + let name_servers = build_name_servers(settings).await?; + + let search = settings.search_domains.clone().unwrap_or_default(); + let config = ResolverConfig::from_parts(None, search, name_servers.clone()); + let reports = report(&name_servers, ResolverSource::Configured); + + let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()); + apply_options(&mut builder, settings); + + Ok(Built { + resolver: builder.build()?, + reports, + }) +} + +/// Resolve the listed servers to hickory name servers, bootstrapping hostname hosts. A hostname +/// that will not resolve drops that server for the life of the agent rather than failing the +/// resolver. +// spec:DNS#bootstrapping +pub(crate) async fn build_name_servers( + settings: &ResolverSettings, +) -> Result, NetError> { + let needs_bootstrap = settings.servers.iter().any(|spec| spec.ip().is_none()); + let bootstrap = if needs_bootstrap { + Some(bootstrap_resolver(settings)?) + } else { + None + }; + + let mut name_servers = Vec::with_capacity(settings.servers.len()); + for spec in &settings.servers { + let ip = match spec.ip() { + Some(ip) => ip, + None => { + let ServerHost::Name(host) = &spec.host else { + unreachable!("ip() is None only for a hostname host"); + }; + let resolver = bootstrap + .as_ref() + .expect("bootstrap resolver built when a hostname host is present"); + match resolver.lookup_ip(host.as_str()).await { + Ok(lookup) => match lookup.iter().next() { + Some(ip) => ip, + None => continue, + }, + // The hostname does not resolve: drop this server for the agent's life. + Err(_) => continue, + } + } + }; + name_servers.push(spec.to_name_server(ip)); + } + + Ok(name_servers) +} diff --git a/crates/web-faith-dns/src/https.rs b/crates/web-faith-dns/src/https.rs new file mode 100644 index 0000000..5acfc5f --- /dev/null +++ b/crates/web-faith-dns/src/https.rs @@ -0,0 +1,110 @@ +use std::time::Duration; + +use hickory_resolver::proto::rr::{ + Name, RData, + rdata::svcb::{SvcParamKey, SvcParamValue}, +}; + +/// What an `HTTPS` record said about an origin's HTTP/3 support. +// spec:DNS#https-records +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HttpsAdvertisement { + /// The record's `port` SvcParam, or `None` when it named none and the origin's own port + /// applies. + pub port: Option, + /// The record's own DNS TTL, which is how long the advertisement it carries lives. + pub ttl: Duration, +} + +/// Where an `HTTPS` record's advertisement goes once the resolver has read one. +/// +/// The resolver cannot own the HTTP/3 upgrade cache directly: that cache is built after the +/// resolver, and the background prober holds a client which holds the resolver in turn. So the +/// caller installs this afterwards (see [`FaithResolver::set_https_sink`](crate::FaithResolver::set_https_sink)), which also keeps this +/// crate free of the upgrade layer's types. +pub trait HttpsSink: Send + Sync { + /// Whether an `HTTPS` record for `host` is worth querying at all right now. + /// + /// Asked before the query so an origin already confirmed, already failed, or already holding a + /// live advertisement costs no DNS traffic to re-learn what is known. + fn wants(&self, host: &str) -> bool; + + /// Fold a record's advertisement into the upgrade layer's knowledge of `host`. + fn record(&self, host: &str, advertisement: HttpsAdvertisement); +} + +/// Whether an ALPN token names a version of HTTP/3. +/// +/// The same family test the `Alt-Svc` reader applies, so a draft token like `h3-29` counts here +/// exactly as it does in a header. +// spec:H3UP#reading-advertisements +pub(crate) fn is_h3_alpn(token: &str) -> bool { + token == "h3" || token.starts_with("h3-") +} + +/// Read the HTTP/3 advertisement out of an `HTTPS` answer for `name`, if it carries one. +/// +/// Only ServiceMode records are considered: an AliasMode record (`svc_priority` 0) redirects to +/// another name rather than describing this one, and following that redirection is a resolution +/// step this does not take. Among the rest the lowest `svc_priority` wins, which is the preference +/// order RFC 9460 defines. +/// +/// A record whose target is neither the root (which per RFC 9460 §2.5.2 means the owner name +/// itself) nor the queried name designates a *different* host, and Faith only upgrades to the +/// origin's own host, so such a record is not acted on. +/// `queried` is the name the answer was actually asked for rather than the host as written, since +/// the search list can requalify a name before it reaches a server; comparison ignores the trailing +/// root so the two are judged on identity rather than on how each was spelled. +// spec:H3UP#advertisements-from-dns +pub(crate) fn read_https_answer( + queried: &Name, + answers: &[hickory_resolver::proto::rr::Record], +) -> Option { + let mut best: Option<(u16, HttpsAdvertisement)> = None; + + for record in answers { + let RData::HTTPS(https) = &record.data else { + continue; + }; + + if https.svc_priority == 0 { + continue; + } + + if !https.target_name.is_root() && !https.target_name.eq_ignore_root(queried) { + continue; + } + + let mut has_h3 = false; + let mut port = None; + for (key, value) in &https.svc_params { + match (key, value) { + (SvcParamKey::Alpn, SvcParamValue::Alpn(alpn)) => { + has_h3 = alpn.0.iter().any(|token| is_h3_alpn(token)); + } + (SvcParamKey::Port, SvcParamValue::Port(value)) => port = Some(*value), + _ => {} + } + } + + if !has_h3 { + continue; + } + + let advertisement = HttpsAdvertisement { + port, + ttl: Duration::from_secs(record.ttl.into()), + }; + if best + .as_ref() + .is_none_or(|(best, _)| https.svc_priority < *best) + { + best = Some((https.svc_priority, advertisement)); + } + } + + best.map(|(_, advertisement)| advertisement) +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/https/tests.rs b/crates/web-faith-dns/src/https/tests.rs new file mode 100644 index 0000000..befece5 --- /dev/null +++ b/crates/web-faith-dns/src/https/tests.rs @@ -0,0 +1,204 @@ +use std::time::Duration; + +use hickory_resolver::proto::rr::{ + Name, RData, Record, + rdata::{ + HTTPS, SVCB, + svcb::{Alpn, SvcParamKey, SvcParamValue}, + }, +}; + +use super::{HttpsAdvertisement, read_https_answer}; + +fn name(input: &str) -> Name { + Name::from_utf8(input).unwrap() +} + +/// One `HTTPS` answer, spelled the way a server would send it. +fn record( + owner: &str, + priority: u16, + target: Name, + params: Vec<(SvcParamKey, SvcParamValue)>, + ttl: u32, +) -> Record { + Record::from_rdata( + name(owner), + ttl, + RData::HTTPS(HTTPS(SVCB::new(priority, target, params))), + ) +} + +fn alpn(tokens: &[&str]) -> (SvcParamKey, SvcParamValue) { + ( + SvcParamKey::Alpn, + SvcParamValue::Alpn(Alpn(tokens.iter().map(|t| (*t).to_owned()).collect())), + ) +} + +#[test] +fn an_h3_alpn_on_the_owner_name_advertises() { + // spec:H3UP#advertisements-from-dns — the ordinary case: `.` as the target means the + // owner name, so the record describes the origin itself. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h2", "h3"])], + 3600, + )]; + + assert_eq!( + read_https_answer(&queried, &answers), + Some(HttpsAdvertisement { + port: None, + ttl: Duration::from_secs(3600), + }), + "an `alpn` listing h3 is an advertisement, and the record's own TTL bounds it" + ); +} + +#[test] +fn a_draft_h3_token_counts_like_the_header_reader_treats_one() { + // spec:H3UP#reading-advertisements — `h3-29` is an h3-family token either way it + // arrives, so DNS must not be stricter than the `Alt-Svc` reader. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h3-29"])], + 60, + )]; + + assert!(read_https_answer(&queried, &answers).is_some()); +} + +#[test] +fn a_record_without_h3_in_its_alpn_advertises_nothing() { + // An origin that speaks only HTTP/2 says so here, and reading that as an h3 + // advertisement would send every such origin down a probe that must fail. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h2"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn the_port_parameter_is_carried_through() { + // spec:H3UP#advertisements-from-dns — a differing port is handled by the same + // machinery an `Alt-Svc` advertised port is. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(8443)), + ], + 3600, + )]; + + assert_eq!( + read_https_answer(&queried, &answers).and_then(|ad| ad.port), + Some(8443) + ); +} + +#[test] +fn a_record_targeting_another_host_is_not_acted_on() { + // Faith only upgrades to the origin's own host, exactly as it refuses an `Alt-Svc` + // advertisement naming a different one (spec:H3UP#advertisements-from-dns). + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + name("cdn.example.net."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn a_record_naming_the_queried_host_itself_is_acted_on() { + // Spelling the owner name out is equivalent to the `.` shorthand, and the comparison + // ignores case and the trailing root the way name equality should. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + name("ExAmPlE.CoM."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert!(read_https_answer(&queried, &answers).is_some()); +} + +#[test] +fn an_alias_mode_record_is_skipped() { + // Priority 0 is AliasMode: it redirects to another name rather than describing this + // one, and following that redirection is a resolution step this does not take. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 0, + name("svc.example.net."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn the_lowest_priority_service_mode_record_wins() { + // RFC 9460 orders ServiceMode records by ascending priority, so the most preferred + // record is the one whose port is acted on. + let queried = name("example.com."); + let answers = [ + record( + "example.com.", + 9, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(9443)), + ], + 3600, + ), + record( + "example.com.", + 2, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(2443)), + ], + 3600, + ), + ]; + + assert_eq!( + read_https_answer(&queried, &answers).and_then(|ad| ad.port), + Some(2443), + "the preferred record is the one acted on" + ); +} + +#[test] +fn an_empty_answer_advertises_nothing() { + // The common case for an origin with no `HTTPS` record at all: nothing learned, and + // nothing that could make the origin probe-worthy. + assert_eq!(read_https_answer(&name("example.com."), &[]), None); +} diff --git a/crates/web-faith-dns/src/lib.rs b/crates/web-faith-dns/src/lib.rs new file mode 100644 index 0000000..f6db349 --- /dev/null +++ b/crates/web-faith-dns/src/lib.rs @@ -0,0 +1,54 @@ +//! A caching DNS resolver for HTTP clients, with a cache you can warm. +//! +//! A client's built-in resolver usually keeps its cache to itself, so the only way to populate it is +//! to make a request. That is no good for prefetching a name ahead of time, which must not touch the +//! origin at all. [`FaithResolver`] is the resolver instead: it can be installed on an HTTP client +//! so every request resolves through it, while [`FaithResolver::prefetch`] is called directly. Both +//! share one resolver and one cache, so a name warmed ahead of time is already there when a request +//! looks it up. +//! +//! # Transports and server order +//! +//! Resolvers are named by URL, and the scheme picks the transport: plaintext `udp` and `tcp`, or +//! encrypted `tls`, `https`, `quic`, and `h3`. The list is queried in the order given rather than +//! reordered by latency. Given no list, the resolver configures itself from the operating system and +//! lets RFC 9539 opportunistic encryption upgrade those servers where it can. +//! +//! [Exempt names] are sent to the system resolver whichever way the rest is configured, so names +//! that only the host knows how to resolve keep resolving. +//! +//! # Beyond addresses +//! +//! Lookups can also read the `HTTPS` record for a name, which is how an origin advertises HTTP/3 +//! before anything has connected to it, and answers can be served stale while a fresh lookup runs. +//! A [network change][FaithResolver::reset] discards what was learned from a network that no longer +//! exists while leaving the resolver usable. +//! +//! [Exempt names]: ResolverSettings::exempt_domains + +// spec:WARM spec:DNS + +mod discovery; +mod https; +mod resolver; +mod settings; +mod transport; + +pub use https::{HttpsAdvertisement, HttpsSink}; +pub use resolver::FaithResolver; +pub use settings::{DEFAULT_MAX_STALE, ResolverReport, ResolverSettings, ResolverSource}; +pub use transport::{ServerSpec, Transport}; + +use hickory_resolver::proto::rr::Name; + +/// Parse a `dns.searchDomains` or `dns.exemptDomains` list into domain names, or return a message +/// for the first entry that is not a valid domain name. +pub fn parse_domains(list: Option>) -> Result>, String> { + list.map(|items| { + items + .iter() + .map(|item| Name::from_utf8(item).map_err(|err| format!("{item:?}: {err}"))) + .collect() + }) + .transpose() +} diff --git a/crates/web-faith-dns/src/resolver.rs b/crates/web-faith-dns/src/resolver.rs new file mode 100644 index 0000000..4f193b6 --- /dev/null +++ b/crates/web-faith-dns/src/resolver.rs @@ -0,0 +1,488 @@ +use std::{ + collections::HashSet, + net::IpAddr, + sync::{Arc, Mutex}, + time::Instant, +}; + +use hickory_resolver::{ + TokioResolver, + config::{GOOGLE, LookupIpStrategy, ResolverConfig}, + net::{DnsError, NetError, runtime::TokioRuntimeProvider}, + proto::rr::{Name, RecordType}, + system_conf::read_system_conf, +}; +use tokio::sync::OnceCell; + +#[cfg(feature = "reqwest")] +use std::net::SocketAddr; + +use crate::{ + discovery::{Built, build}, + https::{HttpsSink, read_https_answer}, + settings::{ResolverReport, ResolverSettings, exempt_suffixes}, +}; + +/// How many names the stale cache holds before evicting the least recently used. +/// +/// Matched to hickory's own default answer-cache size, since the two hold an entry for the same set +/// of names: a stale entry only earns its place while hickory still plausibly holds, or recently +/// held, the answer it came from. Evicting one early costs a blocking lookup rather than a wrong +/// answer, so the bound is about memory rather than correctness. +const STALE_CACHE_SIZE: u64 = 8_192; + +/// A resolved answer kept past its TTL, so an expired lookup is served from it while a refresh runs +/// behind. +// spec:DNS#serving-stale-answers +#[derive(Clone)] +struct StaleEntry { + /// Shared rather than cloned per hit: a hit reads it and hands out a copy of the addresses. + addrs: Arc>, + /// When the answer stopped being fresh, taken from the lookup rather than computed, so it is the + /// TTL the resolver actually gave. + valid_until: Instant, +} + +/// Everything the resolver reads off the network, held together so a network change can drop it in +/// one go. Each field describes the network the agent was on when it was read: which +/// servers discovery found, which suffixes are local to it, and which of its servers answered an +/// encryption probe. The caller's [`ResolverSettings`] deliberately sit outside, being options the +/// agent was constructed with rather than a reading of any network. +// spec:NETCHG +struct Generation { + /// The configured (or discovered) resolver, built lazily inside a tokio runtime. + built: OnceCell>, + /// The system resolver, used for exempt names. Built lazily and independently. + system: OnceCell>, + /// The exempt suffixes, including the system's own, computed once per generation. + exempt: OnceCell>>, + /// Answers held past their TTL, keyed by the host as looked up. Sits in the generation rather + /// than beside the settings so a network change drops it along with the resolvers that produced + /// it: an address learned on the old network is exactly what must not be served on the new one. + stale: moka::sync::Cache, + /// Hosts with a refresh already in flight, so a second stale hit serves the entry rather than + /// starting another lookup. + // spec:DNS#serving-stale-answers + refreshing: Mutex>, +} + +impl Default for Generation { + fn default() -> Self { + Self { + built: OnceCell::new(), + system: OnceCell::new(), + exempt: OnceCell::new(), + stale: moka::sync::Cache::new(STALE_CACHE_SIZE), + refreshing: Mutex::new(HashSet::new()), + } + } +} + +struct Inner { + /// The options the agent was constructed with. A network change does not touch these; they are + /// what the next generation is rebuilt from. + // spec:NETCHG#what-the-signal-keeps + settings: ResolverSettings, + /// Replaced wholesale by [`FaithResolver::reset`]. Read once at the start of a lookup rather + /// than at each step, so a lookup that spans the signal finishes against the one set of + /// resolvers it started on. + // spec:NETCHG#in-flight-requests + generation: Mutex>, + /// Where `HTTPS` records go, installed by the agent once the upgrade cache and prober exist. + /// + /// Sits beside the settings rather than inside the generation deliberately: it is wiring + /// rather than something read off a network, so a network change leaves it in place. Its + /// absence is what turns the `HTTPS` query off, so an agent with HTTP/3 upgrade disabled, or + /// one on the system resolver, never sends one. + https_sink: Mutex>>, +} + +/// A hickory resolver Faith owns, shared between a client's request path and [`FaithResolver::prefetch`]. +#[derive(Clone)] +pub struct FaithResolver { + inner: Arc, +} + +impl Default for FaithResolver { + fn default() -> Self { + Self::new(ResolverSettings::default()) + } +} + +impl std::fmt::Debug for FaithResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FaithResolver").finish_non_exhaustive() + } +} + +impl FaithResolver { + pub fn new(settings: ResolverSettings) -> Self { + Self { + inner: Arc::new(Inner { + settings, + generation: Mutex::new(Arc::new(Generation::default())), + https_sink: Mutex::new(None), + }), + } + } + + /// Install where `HTTPS` records go, enabling the query. + /// + /// Called after the agent's HTTP/3 upgrade cache and prober are built, which cannot happen + /// before the resolver exists. Replaces any previous sink, which is what a network change + /// needs: the prober is rebuilt with the client, so the sink must be too or it would kick + /// probes onto a client that has been dropped. + // spec:DNS#https-records + pub fn set_https_sink(&self, sink: Arc) { + *self + .inner + .https_sink + .lock() + .expect("the HTTPS sink lock is only held to clone or replace an Arc") = Some(sink); + } + + fn https_sink(&self) -> Option> { + self.inner + .https_sink + .lock() + .expect("the HTTPS sink lock is only held to clone or replace an Arc") + .clone() + } + + /// The generation a piece of work resolves against. Taken once per lookup: a reset swaps the + /// generation rather than mutating it, so work already holding one carries on against the + /// resolvers it started with. + // spec:NETCHG#in-flight-requests + fn generation(&self) -> Arc { + self.inner + .generation + .lock() + .expect("the DNS generation lock is only held to clone or replace an Arc") + .clone() + } + + async fn built(&self, generation: &Generation) -> Result, NetError> { + generation + .built + .get_or_try_init(|| async { build(&self.inner.settings).await.map(Arc::new) }) + .await + .cloned() + } + + /// The system resolver, for exempt names. Reads the system configuration and races both + /// address families for Happy Eyeballs like the built-in resolver does. + async fn system(&self, generation: &Generation) -> Result, NetError> { + generation + .system + .get_or_try_init(|| async { + let mut builder = TokioResolver::builder_tokio().unwrap_or_else(|_| { + TokioResolver::builder_with_config( + ResolverConfig::udp_and_tcp(&GOOGLE), + TokioRuntimeProvider::default(), + ) + }); + builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + builder.build().map(Arc::new) + }) + .await + .cloned() + } + + /// The exempt suffixes: `localhost`, `local`, the system's own domain and search suffixes, and + /// the caller's `dns.exemptDomains`. The system's own suffixes are a + /// property of the network, so they are read per generation rather than once per agent. + // spec:DNS#exempt-names + async fn exempt(&self, generation: &Generation) -> Arc> { + generation + .exempt + .get_or_init(|| async { + let system = read_system_conf() + .map(|(config, _)| { + config + .domain() + .into_iter() + .chain(config.search()) + .cloned() + .collect::>() + }) + .unwrap_or_default(); + Arc::new(exempt_suffixes(system, &self.inner.settings.exempt_domains)) + }) + .await + .clone() + } + + /// Whether `host` must go to the system resolver rather than Faith's servers. + async fn is_exempt(&self, generation: &Generation, host: &str) -> bool { + let Ok(name) = Name::from_utf8(host) else { + return false; + }; + self.exempt(generation) + .await + .iter() + .any(|suffix| suffix.zone_of(&name)) + } + + /// Resolve `host` to its addresses, routing exempt names to the system resolver. + async fn lookup(&self, host: &str) -> Result, NetError> { + let generation = self.generation(); + if self.is_exempt(&generation, host).await { + // The system resolver keeps no cache Faith can hold answers in, so an exempt name has + // nothing to go stale and is always resolved for real (spec:DNS#serving-stale-answers). + let resolver = self.system(&generation).await?; + return Ok(resolver.lookup_ip(host).await?.iter().collect()); + } + + // Alongside the addresses rather than after them: the record is a hint for the upgrade + // layer to verify, so nothing about connecting waits on it (spec:DNS#https-records). + self.spawn_https_query(&generation, host); + + if let Some(addrs) = self.stale_addrs(&generation, host) { + self.spawn_refresh(&generation, host); + return Ok(addrs); + } + + let built = self.built(&generation).await?; + let lookup = built.resolver.lookup_ip(host).await?; + let addrs: Vec = lookup.iter().collect(); + self.remember(&generation, host, &addrs, lookup.valid_until()); + Ok(addrs) + } + + /// Ask for `host`'s `HTTPS` record behind the address lookup, so an origin advertising + /// `alpn="h3"` is known before the first connection rather than after the first TCP response. + /// + /// Spawned rather than awaited: an absent, slow, or failed answer must leave address + /// resolution and connecting untouched. Its outcome belongs to the upgrade layer rather than + /// to the request that triggered it, so nothing here reaches a caller, exactly as a stale + /// refresh's outcome does not. + // spec:DNS#https-records + fn spawn_https_query(&self, generation: &Arc, host: &str) { + let Some(sink) = self.https_sink() else { + return; + }; + // Asked before the query, not after: an origin already confirmed, failed, or holding a + // live advertisement has nothing to learn, so it costs no DNS traffic. + if !sink.wants(host) { + return; + } + let Ok(name) = Name::from_utf8(host) else { + return; + }; + + let this = self.clone(); + let generation = Arc::clone(generation); + let host = host.to_owned(); + tokio::spawn(async move { + let Ok(built) = this.built(&generation).await else { + return; + }; + let Ok(lookup) = built.resolver.lookup(name, RecordType::HTTPS).await else { + return; + }; + // The answer's own query name, not the host as written: the search list may have + // requalified it, and the record's target is judged against what was actually asked. + if let Some(advertisement) = read_https_answer(lookup.query().name(), lookup.answers()) + { + sink.record(&host, advertisement); + } + }); + } + + /// The addresses to serve for `host` without waiting, when its answer has expired but is still + /// inside `dns.maxStale`. + /// + /// `None` covers the three cases that must go to the resolver: no entry at all, an entry still + /// fresh (which hickory's own cache answers without a network round trip anyway), and an entry so + /// old it has stopped being evidence about the host. + fn stale_addrs(&self, generation: &Generation, host: &str) -> Option> { + if !self.inner.settings.serve_stale { + return None; + } + let entry = generation.stale.get(host)?; + let now = Instant::now(); + if now <= entry.valid_until { + return None; + } + if now.saturating_duration_since(entry.valid_until) > self.inner.settings.max_stale { + // Dropped rather than left to sit: keeping it would let a refresh that has been failing + // for hours go on being consulted, and the entry can only get older from here. + generation.stale.invalidate(host); + return None; + } + Some(entry.addrs.as_ref().clone()) + } + + /// Keep a successful answer for `host`, so a later lookup past its TTL has something to serve. + fn remember( + &self, + generation: &Generation, + host: &str, + addrs: &[IpAddr], + valid_until: Instant, + ) { + if !self.inner.settings.serve_stale || addrs.is_empty() { + return; + } + generation.stale.insert( + host.to_owned(), + StaleEntry { + addrs: Arc::new(addrs.to_vec()), + valid_until, + }, + ); + } + + /// Refresh `host` behind a stale answer that has already been served. + /// + /// Single-flighted per host: the claim is taken before the task is spawned, so concurrent stale + /// hits serve the entry rather than each starting a lookup. The task outlives the request that + /// triggered it, and its outcome belongs to the cache rather than that request, so nothing here + /// is reported to a caller. + // spec:DNS#serving-stale-answers + fn spawn_refresh(&self, generation: &Arc, host: &str) { + { + let mut refreshing = generation + .refreshing + .lock() + .expect("the DNS refresh lock is only held to insert or remove a host"); + if !refreshing.insert(host.to_owned()) { + return; + } + } + + let this = self.clone(); + let generation = Arc::clone(generation); + let host = host.to_owned(); + tokio::spawn(async move { + match this.refresh(&generation, &host).await { + Ok(()) => {} + Err(err) if is_authoritatively_empty(&err) => { + // The name resolves to nothing now, so the old address is not a stale answer for + // it any more but a wrong one. Dropping the entry makes the next lookup fail + // rather than hand out an address the host no longer answers on. + generation.stale.invalidate(&host); + } + Err(_) => { + // A network error, a server failure, or a timeout says nothing about where the + // host is, so the entry stays and can be served again while this persists. + } + } + generation + .refreshing + .lock() + .expect("the DNS refresh lock is only held to insert or remove a host") + .remove(&host); + }); + } + + /// One refresh lookup, replacing the stale entry when it resolves. + async fn refresh(&self, generation: &Generation, host: &str) -> Result<(), NetError> { + let built = self.built(generation).await?; + let lookup = built.resolver.lookup_ip(host).await?; + let addrs: Vec = lookup.iter().collect(); + self.remember(generation, host, &addrs, lookup.valid_until()); + Ok(()) + } + + /// Drop any stale answer held for `host`, so the next lookup waits for a fresh one. + /// + /// Called when connecting to a served address failed, which is the one piece of evidence that the + /// address was wrong rather than merely old. + // spec:DNS#when-a-stale-address-is-wrong + pub fn invalidate_stale(&self, host: &str) { + self.generation().stale.invalidate(host); + } + + /// Whether a lookup of `host` right now would be served from an expired entry, and so would hand + /// out an address that is assumed rather than confirmed. + /// + /// Deliberately the same window a stale answer is served from, rather than merely "an expired + /// entry exists": an entry past `dns.maxStale` is resolved for real, and treating that as stale + /// would spend a second connection attempt on an address that was already confirmed. + pub fn served_stale(&self, host: &str) -> bool { + if !self.inner.settings.serve_stale { + return false; + } + self.generation().stale.get(host).is_some_and(|entry| { + let now = Instant::now(); + now > entry.valid_until + && now.saturating_duration_since(entry.valid_until) <= self.inner.settings.max_stale + }) + } + + /// Resolve `host` and leave the answer in the shared cache, so a later request skips the + /// lookup. Any failure is swallowed: the warm-up is advisory. + // spec:WARM + pub async fn prefetch(&self, host: &str) { + let _ = self.lookup(host).await; + } + + /// The DNS servers the agent resolves through, in query order. Empty + /// until the resolver has been used, because it reads its configuration on first use. + // spec:OBS#resolvers + pub fn resolvers(&self) -> Vec { + self.generation() + .built + .get() + .map(|built| built.reports.clone()) + .unwrap_or_default() + } + + /// Drop everything read off the network, so the next lookup rebuilds against the network the + /// agent is on now. + /// + /// Flushing cached answers alone would leave the agent resolving them again through the + /// previous network's servers: the discovered server list, the suffixes treated as local, and + /// the results of encryption probes are all readings of a network too, and the whole point of + /// the signal is that the network has changed. Dropping the generation takes the caches with + /// it, since they belong to the resolvers being dropped. + /// + /// The caller's options are untouched, so a listed `dns.servers` set is rebuilt exactly as + /// configured; what it re-reads is what the system supplies and what the network answers. + /// + /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, + /// which keeps it callable from a network-change signal, which is not async. Nothing is rebuilt here + /// either, so an agent that never resolves again pays nothing for the signal. + // spec:NETCHG#what-the-signal-keeps + // spec:NETCHG#reach-across-the-subsystems + pub fn reset(&self) { + *self + .inner + .generation + .lock() + .expect("the DNS generation lock is only held to clone or replace an Arc") = + Arc::new(Generation::default()); + } +} + +/// Installed on a reqwest client with `ClientBuilder::dns_resolver`, so every lookup a request +/// makes goes through the same resolver `prefetch` warms. +#[cfg(feature = "reqwest")] +impl reqwest::dns::Resolve for FaithResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let this = self.clone(); + Box::pin(async move { + let addrs = this.lookup(name.as_str()).await?; + // Port `0` is a placeholder reqwest fills from the URL. The returned `Addrs` has to be + // `'static`, so collect owned rather than borrowing the lookup. + let addrs: Vec = + addrs.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +/// Whether a failed lookup was the resolver answering that the name holds nothing, rather than the +/// resolver failing to answer. +/// +/// The distinction decides what happens to a stale entry: an authoritative "nothing here" retires +/// it, while a failure to reach an answer leaves it in place. Hickory draws the same line, producing +/// `NoRecordsFound` only for `NXDOMAIN` and for `NOERROR` with no answer records, and reporting +/// `SERVFAIL` and the other failure codes as `ResponseCode` instead. +fn is_authoritatively_empty(err: &NetError) -> bool { + matches!(err, NetError::Dns(DnsError::NoRecordsFound(_))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/resolver/tests.rs b/crates/web-faith-dns/src/resolver/tests.rs new file mode 100644 index 0000000..da07582 --- /dev/null +++ b/crates/web-faith-dns/src/resolver/tests.rs @@ -0,0 +1,178 @@ +use std::{ + net::IpAddr, + sync::Arc, + time::{Duration, Instant}, +}; + +use super::{FaithResolver, StaleEntry}; +use crate::{settings::ResolverSettings, transport::ServerSpec}; + +fn spec(input: &str) -> ServerSpec { + ServerSpec::parse(input).expect("valid server URL") +} + +#[tokio::test] +async fn reset_replaces_the_generation_and_what_it_holds() { + // A network change drops what was read off the old network, so the next lookup builds + // against the new one rather than reusing the previous network's servers (spec:NETCHG). + let resolver = FaithResolver::new(ResolverSettings { + servers: vec![spec("udp://127.0.0.1:1")], + timeout: Some(Duration::from_millis(200)), + ..ResolverSettings::default() + }); + + let before = resolver.generation(); + // Build the generation's state, so there is something for the reset to drop. + let _ = resolver.built(&before).await; + assert!( + before.built.get().is_some(), + "the generation built its resolver" + ); + assert_eq!(resolver.resolvers().len(), 1, "which `resolvers()` reports"); + + resolver.reset(); + + let after = resolver.generation(); + assert!( + !Arc::ptr_eq(&before, &after), + "the reset swaps the generation rather than mutating it" + ); + assert!( + after.built.get().is_none(), + "the new generation holds nothing until it is used again" + ); + assert!( + before.built.get().is_some(), + "work already holding the old generation keeps its resolvers" + ); + assert!( + resolver.resolvers().is_empty(), + "`resolvers()` reports nothing until the rebuild (spec:OBS#resolvers)" + ); + + // Configuration survives the signal, so the rebuild uses the servers as configured. + let _ = resolver.built(&after).await; + assert_eq!( + resolver.resolvers().len(), + 1, + "the rebuilt generation resolves through the configured servers again" + ); +} + +/// A resolver with a stale entry for `host` whose freshness ended `ago`. +fn with_stale_entry(settings: ResolverSettings, host: &str, ago: Duration) -> FaithResolver { + let resolver = FaithResolver::new(settings); + resolver.generation().stale.insert( + host.to_owned(), + StaleEntry { + addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), + valid_until: Instant::now() - ago, + }, + ); + resolver +} + +#[test] +fn only_an_expired_entry_inside_the_window_is_served_stale() { + // spec:DNS#serving-stale-answers + let settings = || ResolverSettings { + max_stale: Duration::from_secs(60), + ..ResolverSettings::default() + }; + + // Still fresh: the lookup goes through hickory, which answers from its own cache. + let fresh = FaithResolver::new(settings()); + fresh.generation().stale.insert( + "fresh.test".to_owned(), + StaleEntry { + addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), + valid_until: Instant::now() + Duration::from_secs(60), + }, + ); + let generation = fresh.generation(); + assert!( + fresh.stale_addrs(&generation, "fresh.test").is_none(), + "a fresh entry is not a stale hit" + ); + + // Expired but inside `dns.maxStale`: served immediately. + let stale = with_stale_entry(settings(), "stale.test", Duration::from_secs(5)); + let generation = stale.generation(); + assert!( + stale.stale_addrs(&generation, "stale.test").is_some(), + "an entry expired inside the window is served" + ); + + // Past the window: no longer evidence about the host, so the lookup blocks. + let old = with_stale_entry(settings(), "old.test", Duration::from_secs(120)); + let generation = old.generation(); + assert!( + old.stale_addrs(&generation, "old.test").is_none(), + "an entry past `dns.maxStale` is not served" + ); + assert!( + generation.stale.get("old.test").is_none(), + "and is dropped rather than left to age further" + ); +} + +#[test] +fn serve_stale_off_never_serves_an_expired_entry() { + // spec:DNS#serving-stale-answers — the switch for a caller that must not connect to an + // address it knows to be out of date. + let resolver = with_stale_entry( + ResolverSettings { + serve_stale: false, + ..ResolverSettings::default() + }, + "strict.test", + Duration::from_secs(5), + ); + let generation = resolver.generation(); + assert!(resolver.stale_addrs(&generation, "strict.test").is_none()); + assert!( + !resolver.served_stale("strict.test"), + "and nothing is reported as stale-served, so no retry is armed" + ); +} + +#[test] +fn served_stale_tracks_the_window_it_serves_from() { + // The retry layer arms itself from this, so it must not claim an address was assumed when + // the lookup actually blocked on a fresh one (spec:DNS#when-a-stale-address-is-wrong). + let settings = || ResolverSettings { + max_stale: Duration::from_secs(60), + ..ResolverSettings::default() + }; + + let inside = with_stale_entry(settings(), "inside.test", Duration::from_secs(5)); + assert!(inside.served_stale("inside.test")); + + let outside = with_stale_entry(settings(), "outside.test", Duration::from_secs(120)); + assert!( + !outside.served_stale("outside.test"), + "an entry past the window is resolved for real, so its address is confirmed" + ); + + let absent = FaithResolver::new(settings()); + assert!(!absent.served_stale("absent.test")); +} + +#[test] +fn a_network_change_drops_stale_answers() { + // Addresses read off the old network are exactly what must not be served on the new one + // (spec:NETCHG#reach-across-the-subsystems). + let resolver = with_stale_entry( + ResolverSettings::default(), + "netchg.test", + Duration::from_secs(5), + ); + assert!(resolver.served_stale("netchg.test")); + + resolver.reset(); + + assert!( + !resolver.served_stale("netchg.test"), + "the stale answer goes with the generation that held it" + ); +} diff --git a/crates/web-faith-dns/src/settings.rs b/crates/web-faith-dns/src/settings.rs new file mode 100644 index 0000000..715c9d4 --- /dev/null +++ b/crates/web-faith-dns/src/settings.rs @@ -0,0 +1,134 @@ +use std::{net::SocketAddr, time::Duration}; + +use hickory_resolver::{ + config::{NameServerConfig, ProtocolConfig}, + proto::rr::Name, +}; + +use crate::transport::{ServerSpec, Transport}; + +/// How a server in `resolvers()` came to be reached the way it is. +// spec:OBS#resolvers +#[derive(Clone, Copy, Debug)] +pub enum ResolverSource { + /// Named in `dns.servers` by the caller. + Configured, + /// Discovered from the system's resolver configuration. + Conventional, +} + +impl ResolverSource { + fn label(self) -> &'static str { + match self { + Self::Configured => "configured", + Self::Conventional => "conventional", + } + } +} + +/// One line of `resolvers()`: a server's address, the transport in use, and how it was arrived at. +#[derive(Clone, Debug)] +pub struct ResolverReport { + pub address: String, + pub transport: String, + pub source: String, +} + +/// `dns.maxStale`'s default: how far past expiry an answer may still be served. +/// +/// An hour is long enough that a resolver outage does not stop an agent reaching hosts it already +/// knows, and short enough that a host which really has moved stops being served a dead address for +/// the life of a long-running process. The recovery path bounds the cost of being wrong to one +/// re-resolve, so the window can be generous. +// spec:DNS#serving-stale-answers +pub const DEFAULT_MAX_STALE: Duration = Duration::from_secs(3600); + +/// Everything `dns.*` configures about Faith's resolver, resolved from options at construction. +#[derive(Clone, Debug)] +pub struct ResolverSettings { + /// The `dns.servers` list, in order. Empty means system discovery. + pub servers: Vec, + /// `dns.timeout`, bounding the whole list. `None` leaves hickory's five-second default. + pub timeout: Option, + /// `dns.ndots`. + pub ndots: Option, + /// `dns.searchDomains`, replacing the system's search list when set. + pub search_domains: Option>, + /// `dns.hostsFile`: `Some(true)`/`Some(false)` force it on/off, `None` follows the platform. + pub hosts_file: Option, + /// `dns.exemptDomains`, added to the always-exempt `localhost`, `.local`, and system suffix. + pub exempt_domains: Vec, + /// `dns.serveStale`: whether an expired answer is served while a refresh runs behind it. + pub serve_stale: bool, + /// `dns.maxStale`: how far past expiry an answer may still be served. + pub max_stale: Duration, +} + +impl Default for ResolverSettings { + fn default() -> Self { + Self { + servers: Vec::new(), + timeout: None, + ndots: None, + search_domains: None, + hosts_file: None, + exempt_domains: Vec::new(), + // Defaulted here as well as in the option parsing, so a resolver built directly (in tests, + // and for the global default agent) serves stale like a configured one. + serve_stale: true, + max_stale: DEFAULT_MAX_STALE, + } + } +} + +/// The suffixes handed to the system resolver rather than Faith's servers: `localhost` and `local` +/// always, plus the ones the system supplies and the caller's `dns.exemptDomains`. +/// +/// The root name is never a suffix here, whichever list it arrives in. It is the parent of every +/// name, so admitting it would exempt the lot and route every lookup to the system resolver with +/// `dns.servers` configured and unused. It does arrive in practice: a Windows host with no DNS +/// domain of its own reports the root as its domain, so the check is what keeps the encrypted +/// transports working there rather than being quietly bypassed. +// spec:DNS#exempt-names +pub(crate) fn exempt_suffixes(system: Vec, configured: &[Name]) -> Vec { + let mut names = vec![ + Name::from_ascii("localhost").unwrap(), + Name::from_ascii("local").unwrap(), + ]; + names.extend( + system + .into_iter() + .chain(configured.iter().cloned()) + .filter(|name| !name.is_root()), + ); + names +} + +/// Summarise name servers for `resolvers()`, in the order they are queried. +pub(crate) fn report( + name_servers: &[NameServerConfig], + source: ResolverSource, +) -> Vec { + let mut reports = Vec::new(); + for server in name_servers { + for connection in &server.connections { + let transport = match connection.protocol { + ProtocolConfig::Udp => Transport::Udp, + ProtocolConfig::Tcp => Transport::Tcp, + ProtocolConfig::Tls { .. } => Transport::Tls, + ProtocolConfig::Https { .. } => Transport::Https, + ProtocolConfig::Quic { .. } => Transport::Quic, + ProtocolConfig::H3 { .. } => Transport::H3, + }; + reports.push(ResolverReport { + address: SocketAddr::new(server.ip, connection.port).to_string(), + transport: transport.label().to_owned(), + source: source.label().to_owned(), + }); + } + } + reports +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/settings/tests.rs b/crates/web-faith-dns/src/settings/tests.rs new file mode 100644 index 0000000..b1972bd --- /dev/null +++ b/crates/web-faith-dns/src/settings/tests.rs @@ -0,0 +1,51 @@ +use hickory_resolver::proto::rr::Name; + +use super::exempt_suffixes; + +#[test] +fn a_root_suffix_never_exempts_everything() { + // A Windows host with no DNS domain reports the root as its domain, and the root is the + // parent of every name. Taking it as a suffix exempted every lookup and sent it to the + // system resolver, leaving `dns.servers` configured and unused (spec:DNS#exempt-names). + let suffixes = exempt_suffixes(vec![Name::root()], &[]); + assert!( + !suffixes.iter().any(|suffix| suffix.is_root()), + "the root is not admitted as a suffix" + ); + + let name = Name::from_utf8("nonexistent.example").unwrap(); + assert!( + !suffixes.iter().any(|suffix| suffix.zone_of(&name)), + "so an ordinary name is not exempt and reaches the configured servers" + ); + + // The names that must stay exempt still are, and a real system suffix still counts. + let suffixes = exempt_suffixes( + vec![Name::root(), Name::from_utf8("corp.example").unwrap()], + &[], + ); + for exempt in ["localhost", "printer.local", "host.corp.example"] { + let name = Name::from_utf8(exempt).unwrap(); + assert!( + suffixes.iter().any(|suffix| suffix.zone_of(&name)), + "{exempt} is exempt" + ); + } +} + +#[test] +fn a_root_entry_from_the_caller_is_refused_too() { + // Whichever list it arrives in, the root would disable the caller's own servers. + let suffixes = exempt_suffixes(vec![], &[Name::root()]); + let name = Name::from_utf8("nonexistent.example").unwrap(); + assert!(!suffixes.iter().any(|suffix| suffix.zone_of(&name))); +} + +#[test] +fn exempt_matches_a_suffix_exactly_or_as_a_subdomain() { + // spec:DNS#exempt-names + let local = Name::from_ascii("local").unwrap(); + assert!(local.zone_of(&Name::from_utf8("printer.local").unwrap())); + assert!(local.zone_of(&Name::from_utf8("local").unwrap())); + assert!(!local.zone_of(&Name::from_utf8("mylocal.example").unwrap())); +} diff --git a/crates/web-faith-dns/src/transport.rs b/crates/web-faith-dns/src/transport.rs new file mode 100644 index 0000000..5b34b1c --- /dev/null +++ b/crates/web-faith-dns/src/transport.rs @@ -0,0 +1,166 @@ +use std::{net::IpAddr, sync::Arc}; + +use hickory_resolver::config::{ConnectionConfig, NameServerConfig, ProtocolConfig}; +use url::{Host, Url}; + +/// The default DoH/DoQ query path, used when a `https://`/`h3://` server URL supplies none. +const DEFAULT_DNS_QUERY_PATH: &str = "/dns-query"; + +/// The transport Faith speaks to a resolver, chosen by a server URL's scheme. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Transport { + Udp, + Tcp, + Tls, + Https, + Quic, + H3, +} + +impl Transport { + fn from_scheme(scheme: &str) -> Option { + Some(match scheme { + "udp" => Self::Udp, + "tcp" => Self::Tcp, + "tls" => Self::Tls, + "https" => Self::Https, + "quic" => Self::Quic, + "h3" => Self::H3, + _ => return None, + }) + } + + /// The conventional port for the transport, used when the URL names none. + fn default_port(self) -> u16 { + match self { + Self::Udp | Self::Tcp => 53, + Self::Tls | Self::Quic => 853, + Self::Https | Self::H3 => 443, + } + } + + /// The lowercase label reported by `resolvers()`. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Udp => "udp", + Self::Tcp => "tcp", + Self::Tls => "tls", + Self::Https => "https", + Self::Quic => "quic", + Self::H3 => "h3", + } + } +} + +/// A resolver Faith reaches by IP or by a hostname it bootstraps. +#[derive(Clone, Debug)] +pub(crate) enum ServerHost { + Ip(IpAddr), + Name(String), +} + +/// One entry of `dns.servers`, parsed at agent construction. The IP is not known yet for a +/// hostname host: that is resolved when the resolver is first used (see [`ResolverSettings`](crate::ResolverSettings)). +#[derive(Clone, Debug)] +pub struct ServerSpec { + pub(crate) host: ServerHost, + transport: Transport, + port: u16, + /// DoH/DoQ query path, `None` for the non-HTTP transports. + path: Option>, + /// The name to authenticate the certificate against, from a URL fragment. When absent, a + /// hostname host authenticates against itself and an IP host against the address. + cert_name: Option, +} + +impl ServerSpec { + /// Parse one `dns.servers` URL, or return a message for an unparseable URL or unknown scheme. + pub fn parse(input: &str) -> Result { + let url = Url::parse(input).map_err(|err| format!("{input:?}: {err}"))?; + let transport = Transport::from_scheme(url.scheme()) + .ok_or_else(|| format!("{input:?}: unknown DNS transport scheme {:?}", url.scheme()))?; + + let host = match url.host() { + Some(Host::Ipv4(ip)) => ServerHost::Ip(IpAddr::V4(ip)), + Some(Host::Ipv6(ip)) => ServerHost::Ip(IpAddr::V6(ip)), + Some(Host::Domain(name)) => ServerHost::Name(name.to_owned()), + None => return Err(format!("{input:?}: no host to resolve")), + }; + + let port = url.port().unwrap_or_else(|| transport.default_port()); + let path = match transport { + Transport::Https | Transport::H3 => { + let path = url.path(); + (!path.is_empty() && path != "/").then(|| Arc::from(path)) + } + _ => None, + }; + let cert_name = url.fragment().map(str::to_owned); + + Ok(Self { + host, + transport, + port, + path, + cert_name, + }) + } + + /// The IP host, or `None` for a hostname host that still needs bootstrapping. + pub(crate) fn ip(&self) -> Option { + match self.host { + ServerHost::Ip(ip) => Some(ip), + ServerHost::Name(_) => None, + } + } + + /// The certificate name to authenticate against once the host resolves to `ip`: an explicit + /// fragment, else the hostname, else the address itself. + // spec:DNS#transports + fn server_name(&self, ip: IpAddr) -> Arc { + if let Some(name) = &self.cert_name { + Arc::from(name.as_str()) + } else { + match &self.host { + ServerHost::Name(name) => Arc::from(name.as_str()), + ServerHost::Ip(_) => Arc::from(ip.to_string()), + } + } + } + + /// Build the hickory name server for this spec, reached at `ip`. + pub(crate) fn to_name_server(&self, ip: IpAddr) -> NameServerConfig { + let protocol = match self.transport { + Transport::Udp => ProtocolConfig::Udp, + Transport::Tcp => ProtocolConfig::Tcp, + Transport::Tls => ProtocolConfig::Tls { + server_name: self.server_name(ip), + }, + Transport::Https => ProtocolConfig::Https { + server_name: self.server_name(ip), + path: self + .path + .clone() + .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), + }, + Transport::Quic => ProtocolConfig::Quic { + server_name: self.server_name(ip), + }, + Transport::H3 => ProtocolConfig::H3 { + server_name: self.server_name(ip), + path: self + .path + .clone() + .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), + disable_grease: false, + }, + }; + + let mut connection = ConnectionConfig::new(protocol); + connection.port = self.port; + NameServerConfig::new(ip, true, vec![connection]) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/transport/tests.rs b/crates/web-faith-dns/src/transport/tests.rs new file mode 100644 index 0000000..aa933bb --- /dev/null +++ b/crates/web-faith-dns/src/transport/tests.rs @@ -0,0 +1,82 @@ +use std::{net::IpAddr, sync::Arc}; + +use hickory_resolver::config::ProtocolConfig; + +use super::{DEFAULT_DNS_QUERY_PATH, ServerSpec, Transport}; + +fn spec(input: &str) -> ServerSpec { + ServerSpec::parse(input).expect("valid server URL") +} + +#[test] +fn scheme_selects_transport_and_conventional_port() { + // spec:DNS#transports + assert_eq!(spec("udp://1.1.1.1").transport, Transport::Udp); + assert_eq!(spec("udp://1.1.1.1").port, 53); + assert_eq!(spec("tcp://1.1.1.1").port, 53); + assert_eq!(spec("tls://1.1.1.1").transport, Transport::Tls); + assert_eq!(spec("tls://1.1.1.1").port, 853); + assert_eq!(spec("quic://1.1.1.1").port, 853); + assert_eq!(spec("https://1.1.1.1").transport, Transport::Https); + assert_eq!(spec("https://1.1.1.1").port, 443); + assert_eq!(spec("h3://1.1.1.1").port, 443); +} + +#[test] +fn explicit_port_overrides_the_conventional_one() { + // spec:DNS#transports + assert_eq!(spec("tls://1.1.1.1:8853").port, 8853); +} + +#[test] +fn http_transports_default_the_query_path() { + // spec:DNS#transports — `/dns-query` when the URL supplies none. + assert_eq!(spec("https://dns.google").path, None); + assert_eq!( + spec("https://dns.google") + .to_name_server(IpAddr::from([8, 8, 8, 8])) + .connections[0] + .protocol, + ProtocolConfig::Https { + server_name: Arc::from("dns.google"), + path: Arc::from(DEFAULT_DNS_QUERY_PATH), + } + ); + assert_eq!( + spec("https://dns.google/resolve").path, + Some(Arc::from("/resolve")) + ); +} + +#[test] +fn a_fragment_names_the_certificate() { + // spec:DNS#transports — `tls://1.1.1.1#cloudflare-dns.com`. + let spec = spec("tls://1.1.1.1#cloudflare-dns.com"); + assert_eq!(spec.cert_name.as_deref(), Some("cloudflare-dns.com")); + assert_eq!( + &*spec.server_name(IpAddr::from([1, 1, 1, 1])), + "cloudflare-dns.com" + ); +} + +#[test] +fn a_bare_ip_authenticates_against_the_address() { + // spec:DNS#transports — `tls://1.1.1.1` with no fragment. + let spec = spec("tls://1.1.1.1"); + assert_eq!(spec.cert_name, None); + assert_eq!(&*spec.server_name(IpAddr::from([1, 1, 1, 1])), "1.1.1.1"); +} + +#[test] +fn a_hostname_authenticates_against_itself() { + // spec:DNS#transports + let spec = spec("tls://dns.google"); + assert_eq!(&*spec.server_name(IpAddr::from([8, 8, 8, 8])), "dns.google"); +} + +#[test] +fn an_unknown_scheme_is_rejected() { + // spec:DNS#transports — throws an address-parse error at construction. + assert!(ServerSpec::parse("ftp://1.1.1.1").is_err()); + assert!(ServerSpec::parse("not a url").is_err()); +} diff --git a/crates/web-faith-encoding/Cargo.toml b/crates/web-faith-encoding/Cargo.toml new file mode 100644 index 0000000..8349e99 --- /dev/null +++ b/crates/web-faith-encoding/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "web-faith-encoding" +description = "HTTP content coding for request and response bodies" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +async-compression.workspace = true +bytes.workspace = true +futures.workspace = true +http.workspace = true +tokio.workspace = true +tokio-util.workspace = true diff --git a/src/encoding.rs b/crates/web-faith-encoding/src/lib.rs similarity index 84% rename from src/encoding.rs rename to crates/web-faith-encoding/src/lib.rs index 90231da..957bf4e 100644 --- a/src/encoding.rs +++ b/crates/web-faith-encoding/src/lib.rs @@ -1,8 +1,18 @@ -//! Content coding: Faith owns the decode decision rather than the HTTP stack -//! underneath, so it can rest on the `Accept-Encoding` of the request in hand. -//! The same codings compress a request body under the `compress` option. +//! HTTP content coding for request and response bodies: gzip, deflate, brotli, and zstd. //! -//! spec: ENC +//! An HTTP stack usually decides for itself which codings to advertise and decode. This lets the +//! caller own that decision instead, so decoding can rest on the `Accept-Encoding` of the request +//! actually in hand rather than on whatever the layer beneath negotiated. +//! +//! On the way back, [`AcceptEncoding::parse`] reads what a request advertised and [`decision`] says +//! which coding a response should be decoded under, if any; [`decode_stream`] wraps the body in the +//! decoder for it. On the way out, [`compress_buffer`] and [`compress_stream`] apply a coding to a +//! request body, and [`layer_content_encoding`] names it alongside whatever the caller had already +//! declared. +//! +//! `deflate` is the zlib-wrapped form of RFC 1950, which is what mainstream clients decode it as. + +// spec:ENC use std::{io, pin::Pin}; @@ -12,24 +22,28 @@ use async_compression::tokio::bufread::{ }; use bytes::Bytes; use futures::{Stream, TryStreamExt}; -use reqwest::header::{CONTENT_ENCODING, CONTENT_LENGTH, HeaderMap}; +use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, HeaderMap}; use tokio::io::AsyncReadExt; use tokio_util::io::{ReaderStream, StreamReader}; -use crate::body::DynStream; +/// A body byte-stream, as the decoders take and return one. +/// +/// The client hands its own body streams straight to [`decode_stream`]: the shape is the same one +/// its pipeline already carries, so nothing has to depend on the layer above to name it. +pub type ByteStream = dyn Stream> + Send + Sync; /// The `Accept-Encoding` Faith advertises when the caller advertises none. /// /// Matches the value reqwest's decompression stack sent before Faith took over the /// codings, so the wire is unchanged for the default request. -pub(crate) const DEFAULT_ACCEPT_ENCODING: &str = "zstd,gzip,deflate,br"; +pub const DEFAULT_ACCEPT_ENCODING: &str = "zstd,gzip,deflate,br"; /// A content coding Faith can decode. Wire tokens: `gzip`, `deflate`, `br`, `zstd`. /// /// `deflate` is the zlib-wrapped form (RFC 1950), matching what reqwest and every /// other mainstream client decode it as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Coding { +pub enum Coding { Gzip, Deflate, Brotli, @@ -42,8 +56,9 @@ impl Coding { /// Unlike [`Self::from_token`], which reads a token off the wire and so takes it as /// loosely as HTTP writes it, this matches the four documented tokens exactly: the /// option is an API surface, and an unrecognised value is refused rather than - /// guessed at (spec:ENC#compressing-a-request-body). - pub(crate) fn from_option(value: &str) -> Option { + /// guessed at. + // spec:ENC#compressing-a-request-body + pub fn from_option(value: &str) -> Option { match value { "gzip" => Some(Self::Gzip), "deflate" => Some(Self::Deflate), @@ -54,7 +69,7 @@ impl Coding { } /// The wire token naming this coding in a `Content-Encoding`. - pub(crate) fn token(self) -> &'static str { + pub fn token(self) -> &'static str { match self { Self::Gzip => "gzip", Self::Deflate => "deflate", @@ -64,8 +79,8 @@ impl Coding { } /// Match a single content-coding token, case-insensitively. `None` for - /// `identity`, an unknown coding, or a coding Faith cannot decode. - fn from_token(token: &str) -> Option { + /// `identity`, an unknown coding, or one this cannot decode. + pub fn from_token(token: &str) -> Option { let token = token.trim(); if token.eq_ignore_ascii_case("gzip") || token.eq_ignore_ascii_case("x-gzip") { Some(Self::Gzip) @@ -87,7 +102,7 @@ impl Coding { /// single coding Faith can decode and the request's `Accept-Encoding` accepted it. /// A `Content-Encoding` naming more than one coding, an unknown coding, or a coding the /// request did not accept yields `None`, and the body is delivered as received. -pub(crate) fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option { +pub fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option { // A representation encoded more than once is the caller's to unwind. The codings may // arrive comma-joined on one line or split across several `Content-Encoding` lines -- // the same list either way, so both forms are gathered together before counting. @@ -107,7 +122,7 @@ pub(crate) fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option, deflate: Option, brotli: Option, @@ -126,7 +141,7 @@ pub(crate) struct AcceptEncoding { } impl AcceptEncoding { - pub(crate) fn parse(value: &str) -> Self { + pub fn parse(value: &str) -> Self { let mut accept = Self::default(); for element in value.split(',') { let mut parts = element.split(';'); @@ -210,7 +225,7 @@ fn parse_quality(value: &str) -> Option { /// /// The body stream carries decoded bytes on every read path this way (see [`Coding`]); /// trailers are pulled off the frames before this point, so decoding sees data only. -pub(crate) fn decode_stream(input: Pin>, coding: Coding) -> Pin> { +pub fn decode_stream(input: Pin>, coding: Coding) -> Pin> { let reader = StreamReader::new(input.map_err(io::Error::other)); match coding { Coding::Gzip => reader_stream(GzipDecoder::new(reader)), @@ -225,7 +240,7 @@ pub(crate) fn decode_stream(input: Pin>, coding: Coding) -> Pin(reader: R) -> Pin> +fn reader_stream(reader: R) -> Pin> where R: tokio::io::AsyncRead + Send + Sync + 'static, { @@ -233,13 +248,14 @@ where } /// A request body stream, as reqwest takes one. -pub(crate) type RequestStream = Pin> + Send>>; +pub type RequestStream = Pin> + Send>>; /// Compress a buffered request body, yielding the bytes that go on the wire. /// /// The whole body is known up front, so it compresses in one pass and its length is the -/// `Content-Length` reqwest derives from it (spec:ENC#what-a-compressed-request-sends). -pub(crate) async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result> { +/// `Content-Length` the request can declare. +// spec:ENC#what-a-compressed-request-sends +pub async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result> { let mut output = Vec::new(); match coding { Coding::Gzip => GzipEncoder::new(input).read_to_end(&mut output).await?, @@ -253,9 +269,10 @@ pub(crate) async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result< /// Compress a streaming request body as its chunks arrive. /// /// There is no compressed length to declare before the body ends, so the result goes out -/// chunked (spec:ENC#what-a-compressed-request-sends). The encoder buffers on its own -/// terms, so the bytes for one chunk the caller writes need not leave with it. -pub(crate) fn compress_stream(input: S, coding: Coding) -> RequestStream +/// chunked. The encoder buffers on its own terms, so the bytes for one chunk the caller +/// writes need not leave with it. +// spec:ENC#what-a-compressed-request-sends +pub fn compress_stream(input: S, coding: Coding) -> RequestStream where S: Stream> + Send + 'static, { @@ -275,12 +292,12 @@ where Box::pin(ReaderStream::new(reader)) } -/// Join the codings a request already declares with the one Faith applied. +/// Join the codings a request already declares with the one applied on top. /// -/// The caller's `Content-Encoding` describes the bytes they handed over, so Faith's coding -/// is named after theirs, the order the codings were applied in -/// (spec:ENC#what-a-compressed-request-sends). -pub(crate) fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> String { +/// The caller's `Content-Encoding` describes the bytes they handed over, so the applied coding is +/// named after theirs: the order the codings were applied in. +// spec:ENC#what-a-compressed-request-sends +pub fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> String { match declared.map(str::trim).filter(|value| !value.is_empty()) { Some(declared) => format!("{declared}, {}", applied.token()), None => applied.token().to_owned(), @@ -289,7 +306,7 @@ pub(crate) fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> #[cfg(test)] mod tests { - use reqwest::header::{CONTENT_ENCODING, HeaderMap, HeaderValue}; + use http::header::{CONTENT_ENCODING, HeaderMap, HeaderValue}; use super::*; diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml new file mode 100644 index 0000000..63fab31 --- /dev/null +++ b/crates/web-faith-napi/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "web-faith-napi" +description = "Faith: a Rust-powered JS fetch (Node.js binding)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +publish = false + +[lib] +# Kept as `faith` so the built artifact stays `libfaith.so`/`faith.dll`, which the +# release workflow's cross-compile steps copy by name. +name = "faith" +crate-type = ["cdylib"] + +[dependencies] +async-stream.workspace = true +bytes.workspace = true +futures.workspace = true +http-cache-reqwest = { workspace = true, optional = true } +napi.workspace = true +napi-derive.workspace = true +reqwest.workspace = true +reqwest-middleware.workspace = true +serde_json.workspace = true +tokio.workspace = true +web-faith.workspace = true +web-faith-conn-tracker = { workspace = true, optional = true } +web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } + +[build-dependencies] +napi-build.workspace = true + +[features] +default = [ + "cache", + "connection-tracking", + "cookies", + "dns", + "encoding", + "http3", + "tls-aws-lc-rs", +] +cache = ["dep:http-cache-reqwest", "web-faith/cache"] +connection-tracking = ["dep:web-faith-conn-tracker", "web-faith/connection-tracking"] +cookies = ["dep:web-faith-cookies", "reqwest/cookies", "web-faith/cookies"] +dns = ["web-faith/dns"] +encoding = ["web-faith/encoding"] +http3 = ["reqwest/http3", "tls-aws-lc-rs", "web-faith/http3"] +tls-aws-lc-rs = ["reqwest/rustls", "web-faith/tls-aws-lc-rs"] +tls-ring = ["reqwest/rustls-no-provider", "web-faith/tls-ring"] diff --git a/build.rs b/crates/web-faith-napi/build.rs similarity index 50% rename from build.rs rename to crates/web-faith-napi/build.rs index caebed4..e2f4410 100644 --- a/build.rs +++ b/crates/web-faith-napi/build.rs @@ -1,16 +1,26 @@ -use std::fs; +use std::{env, fs, path::PathBuf}; fn main() { napi_build::setup(); // Parse reqwest version from Cargo.lock - let reqwest_version = extract_reqwest_version().unwrap(); + let lock_path = find_cargo_lock().expect("Cargo.lock not found in any ancestor directory"); + let reqwest_version = extract_reqwest_version(&lock_path).unwrap(); println!("cargo:rustc-env=REQWEST_VERSION={}", reqwest_version); - println!("cargo:rerun-if-changed=Cargo.lock"); + println!("cargo:rerun-if-changed={}", lock_path.display()); } -fn extract_reqwest_version() -> Option { - let cargo_lock = fs::read_to_string("Cargo.lock").ok()?; +/// The lock file lives at the workspace root, which is an ancestor of this crate. +fn find_cargo_lock() -> Option { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR")?); + manifest_dir + .ancestors() + .map(|dir| dir.join("Cargo.lock")) + .find(|lock| lock.is_file()) +} + +fn extract_reqwest_version(lock_path: &PathBuf) -> Option { + let cargo_lock = fs::read_to_string(lock_path).ok()?; // Find the reqwest package entry in Cargo.lock for line in cargo_lock.lines() { diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs new file mode 100644 index 0000000..f8c32bc --- /dev/null +++ b/crates/web-faith-napi/src/agent.rs @@ -0,0 +1,366 @@ +//! The `Agent` class, as JavaScript sees it. + +use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; + +use napi::Env; +use napi_derive::napi; + +use crate::{ + async_task::faith_promise, + error::{FaithError, FaithErrorExt}, +}; + +#[cfg(feature = "connection-tracking")] +use crate::conn_tracker::{ConnectionInfo, connections_for_napi}; + +mod convert; +mod options; + +pub use options::*; + +#[napi] +pub const FAITH_VERSION: &str = env!("CARGO_PKG_VERSION"); +#[napi] +pub const REQWEST_VERSION: &str = env!("REQWEST_VERSION"); +/// Custom user agent string. +/// +/// Default: `Faith/{version} reqwest/{version}`. +/// +/// You may use the `USER_AGENT` constant if you wish to prepend your own agent to the default, e.g. +/// +/// ```javascript +/// import { Agent, USER_AGENT } from '@passcod/faith'; +/// const agent = new Agent({ +/// userAgent: `YourApp/1.2.3 ${USER_AGENT}`, +/// }); +/// ``` +#[napi] +pub const USER_AGENT: &str = web_faith::USER_AGENT; + +#[napi] +#[derive(Debug, Clone, Default)] +pub struct AgentStats { + pub requests_sent: i64, + pub responses_received: i64, + /// Number of response body streams that have been started (converted from raw body to stream). + /// This happens when `.body`, `.text()`, `.json()`, `.bytes()`, or similar methods are called. + pub bodies_started: i64, + /// Number of response body streams that have been fully consumed. + /// When `bodies_started - bodies_finished > 0`, there are bodies holding connections open. + pub bodies_finished: i64, +} + +impl From for AgentStats { + fn from(stats: web_faith::stats::AgentStats) -> Self { + let count = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); + Self { + requests_sent: count(stats.requests_sent), + responses_received: count(stats.responses_received), + bodies_started: count(stats.bodies_started), + bodies_finished: count(stats.bodies_finished), + } + } +} + +/// One entry of `Agent.resolvers()`: a DNS server the agent resolves through (spec:OBS#resolvers). +#[cfg(feature = "dns")] +#[napi(object)] +#[derive(Debug, Clone)] +pub struct ResolverInfo { + /// The server's address, as `ip:port`. + pub address: String, + /// The transport in use: `udp`, `tcp`, `tls`, `https`, `quic`, or `h3`. + pub transport: String, + /// How the transport was arrived at: `configured` by the caller, or `conventional` DNS. + pub source: String, +} + +/// The `Agent` interface of the Faith API represents an instance of an HTTP client. Each `Agent` has +/// its own options, connection pool, caches, etc. There are also conveniences such as `headers` for +/// setting default headers on all requests done with the agent, and statistics collected by the agent. +/// +/// Re-using connections between requests is a significant performance improvement: not only because +/// the TCP and TLS handshake is only performed once across many different requests, but also because +/// the DNS lookup doesn't need to occur for subsequent requests on the same connection. Depending on +/// DNS technology (DoH and DoT add a whole separate handshake to the process) and overall latency, +/// this can not only speed up requests on average, but also reduce system load. +/// +/// For this reason, and also because in browsers this behaviour is standard, **all** requests with +/// Faith use an `Agent`. For `fetch()` calls that don't specify one explicitly, a global agent with +/// default options is created on first use. +/// +/// There are a lot more options that could be exposed here; if you want one, open an issue. +#[napi] +#[derive(Debug, Clone)] +pub struct Agent { + pub(crate) inner: web_faith::agent::Agent, +} + +#[napi] +impl Agent { + pub fn new() -> Result { + Self::with_options(AgentOptions::default()) + } + + pub fn with_options(options: AgentOptions) -> Result { + refuse_absent_capabilities(&options)?; + let options = web_faith::options::AgentOptions::from(options); + // A napi callback can run outside the runtime, and building the HTTP/3 endpoint needs to be + // inside one, so the client is constructed within whichever runtime is to hand. + within_runtime_if_available(|| web_faith::agent::Agent::from_options(options)) + .map(|inner| Self { inner }) + } + + #[napi(constructor)] + pub fn construct(env: Env, options: Option) -> Result { + Ok(if let Some(options) = options { + Self::with_options(options) + } else { + Self::new() + } + .map_err(|err| err.into_js_error(&env))?) + } + + /// Close the agent, releasing its connection pool, DNS resolver, and any + /// background tasks it owns, rather than waiting for the garbage collector + /// to drop it. This is worth doing when you create many short-lived agents; + /// a single long-lived agent can just be left to the GC. + /// + /// Requests already in flight run to completion. Any new request on a closed + /// agent throws a `Closed` error. Calling `close()` more than once is a + /// no-op. The cookie store, if any, remains readable via `getCookie`. + #[napi] + pub fn close(&mut self) { + self.inner.close(); + } + + /// Tell the agent the network underneath it has changed, so it stops deciding from what it + /// learned about a network that is gone. + /// + /// Node has no portable signal for an interface or connectivity change, so Faith cannot + /// detect one; this is the reaction, and wiring it to a trigger (an OS notification, a VPN + /// transition, a captive-portal sign-in) is the caller's own. It drops pooled connections, + /// flushes the DNS cache, demotes the HTTP/3 origins that a real response confirmed back to + /// advertised so a background probe re-verifies them, and clears the HTTP/3 failure and slow + /// states, their cooldown backoff, and the path-time averages. + /// + /// Configuration, `http3.hints`, `Alt-Svc` advertisements, the cookie jar, the HTTP cache and + /// the `stats()` counters are all kept: none of them is a claim about a network path. + /// + /// Requests already in flight are not interrupted and run to completion on the connections + /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a + /// closed agent does nothing, and calling it repeatedly is harmless. + // spec:NETCHG + #[napi] + pub fn network_changed(&mut self) { + self.inner.network_changed(); + } + + /// Returns statistics gathered by this agent: + /// + /// - `requestsSent` + /// - `responsesReceived` + /// - `bodiesStarted` + /// - `bodiesFinished` + #[napi] + pub fn stats(&self) -> AgentStats { + AgentStats::from(self.inner.stats()) + } + + /// Warm the DNS cache for `host`, so a later request to it skips the lookup. + /// + /// Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, + /// port, or path in a fuller string is ignored. The returned promise resolves when the answer + /// lands in the cache and never rejects, whatever happens on the network — a resolution failure + /// resolves quietly, because the work is advisory. Under the system resolver there is no cache + /// to warm, so the call resolves without doing anything. A malformed host throws synchronously, + /// as does a call on a closed agent. + #[napi] + pub fn prefetch_dns<'env>( + &self, + env: &'env Env, + host: String, + ) -> Result, napi::Error> { + let warming = self + .inner + .prefetch_dns(&host) + .map_err(|err| caller_error(env, err))?; + faith_promise(env, async move { + warming.await; + Ok(()) + }) + } + + /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS + /// setup. + /// + /// Mirrors the browser's `preconnect` resource hint. The argument is an origin + /// (`scheme://host[:port]`); a longer URL is reduced to its origin. The warm-up sends a + /// synthetic `HEAD` to the origin's root — the origin sees it — over the transport the next + /// foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every + /// other origin a TCP one. The returned promise resolves when the attempt finishes and never + /// rejects: every network failure resolves quietly. A malformed origin throws synchronously, as + /// does a call on a closed agent. + #[napi] + pub fn preconnect<'env>( + &self, + env: &'env Env, + origin: String, + ) -> Result, napi::Error> { + let warming = self + .inner + .preconnect(&origin) + .map_err(|err| caller_error(env, err))?; + faith_promise(env, async move { + warming.await; + Ok(()) + }) + } +} + +/// Build the JS error a warm-up throws synchronously for a caller mistake, preserving its `.code` +/// and JS error class. Network failures never reach here — they resolve quietly (spec:WARM). +fn caller_error(env: &Env, err: FaithError) -> napi::Error { + napi::Error::from(err.into_js_error(env)) +} + +/// Refuse an option group this build cannot honour. +/// +/// A Cargo feature drops the capability and, on the Rust surface, the API that reaches it. napi's +/// object derive does not honour `#[cfg]` on a field, so an options object here keeps its full shape +/// whatever the build; asking for a capability that is not compiled in is refused rather than +/// quietly ignored, so a slim build says so instead of appearing to work. +fn refuse_absent_capabilities(options: &AgentOptions) -> Result<(), FaithError> { + let absent = |group: &str| -> Result<(), FaithError> { + Err(FaithError::new( + web_faith::FaithErrorKind::Config, + Some(format!("this build has no {group} support")), + )) + }; + + #[cfg(not(feature = "cache"))] + if options.cache.is_some() { + return absent("HTTP cache"); + } + + #[cfg(not(feature = "http3"))] + if options.http3.is_some() { + return absent("HTTP/3"); + } + + #[cfg(not(feature = "cookies"))] + if options.cookies.is_some() { + return absent("cookie"); + } + + // `dns.overrides` reaches reqwest rather than Faith's resolver, so it is honoured either way; + // every other setting in the group configures the resolver this build does not have. + #[cfg(not(feature = "dns"))] + if options.dns.as_ref().is_some_and(|dns| { + dns.system.is_some() + || dns.servers.is_some() + || dns.timeout.is_some() + || dns.search_domains.is_some() + || dns.ndots.is_some() + || dns.hosts_file.is_some() + || dns.exempt_domains.is_some() + || dns.serve_stale.is_some() + || dns.max_stale.is_some() + }) { + return absent("resolver"); + } + + let _ = (options, absent); + Ok(()) +} + +/// Per-connection reporting, which needs the tracker that gathers it. +#[cfg(feature = "connection-tracking")] +#[napi] +impl Agent { + /// Returns information on current connections open by this agent. + /// + /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: + /// this makes it possible to track indicators over time to find the retransmission rate, for + /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other + /// fields might also be missing depending on platform support; and no forward guarantees are made + /// on field availability. If the platform isn't supported at all, this will always return empty. + #[napi] + pub fn connections<'env>(&self, env: &'env Env) -> Vec> { + connections_for_napi(&self.inner.conn_tracker, env) + } +} + +/// The resolver's own observability, which needs a resolver of Faith's own to report on. +#[cfg(feature = "dns")] +#[napi] +impl Agent { + /// Returns the DNS servers this agent resolves through, in the order they are queried, so + /// "are my lookups actually encrypted" is answerable from inside the process. + /// + /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, + /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). + /// The list is empty until the resolver has been used, because it reads its configuration on + /// first use, and empty for an agent using the system resolver. + #[napi] + pub fn resolvers(&self) -> Vec { + self.inner + .resolvers() + .into_iter() + .map(|report| ResolverInfo { + address: report.address, + transport: report.transport, + source: report.source, + }) + .collect() + } +} + +/// The cookie jar's verbs, which exist when the build keeps a jar. +#[cfg(feature = "cookies")] +use std::str::FromStr as _; + +#[cfg(feature = "cookies")] +use reqwest::Url; + +#[cfg(feature = "cookies")] +#[napi] +impl Agent { + /// Add a cookie into the agent. + /// + /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying + /// the scheme and host they read, so this does nothing if: + /// - the cookie store is disabled + /// - the url is malformed + /// - the cookie does not parse + /// - a `__Host-` or `__Secure-` name prefix is not satisfied + /// - the cookie is larger than `cookies.maxSize` + #[napi] + pub fn add_cookie(&self, url: String, cookie: String) { + let Ok(url) = Url::from_str(&url) else { + return; + }; + + let Some(jar) = self.inner.cookies() else { + return; + }; + + jar.add_cookie_str(&cookie, &url); + } + + /// Retrieve a cookie from the store. + /// + /// Returns `null` if: + /// - there's no cookie at this url + /// - the cookie store is disabled + /// - the url is malformed + /// - the cookie cannot be represented as a string + #[napi] + pub fn get_cookie(&self, url: String) -> Option { + let url = Url::from_str(&url).ok()?; + self.inner + .cookies()? + .request_cookie_header(&url) + .and_then(|value| value.to_str().ok().map(ToOwned::to_owned)) + } +} diff --git a/crates/web-faith-napi/src/agent/convert.rs b/crates/web-faith-napi/src/agent/convert.rs new file mode 100644 index 0000000..49eef5a --- /dev/null +++ b/crates/web-faith-napi/src/agent/convert.rs @@ -0,0 +1,160 @@ +//! Reading a JavaScript `AgentOptions` into the options the client validates. + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; +use napi::{Either, bindgen_prelude::Buffer}; + +use web_faith::{client::RedirectPolicy, options}; + +#[cfg(feature = "cookies")] +use web_faith_cookies::CookieLimits; + +use crate::agent::AgentOptions; + +#[cfg(feature = "http3")] +use crate::agent::Http3Congestion; + +#[cfg(feature = "cache")] +use crate::agent::CacheStore; + +/// Read the JavaScript options object into the shape the client validates. +/// +/// A field-for-field mapping wherever the two agree, which is most of them; what differs is where +/// JavaScript expresses a choice as a union or a string that Rust has a type for. +impl From for options::AgentOptions { + fn from(opts: AgentOptions) -> Self { + Self { + #[cfg(feature = "cache")] + cache: opts.cache.map(|cache| options::CacheOptions { + store: cache.store.map(|store| match store { + CacheStore::Disk => options::CacheStore::Disk, + CacheStore::Memory => options::CacheStore::Memory, + }), + capacity: cache.capacity, + mode: cache.mode.map(CacheMode::from), + path: cache.path, + shared: cache.shared, + }), + // `false` and an absent value both mean no jar; `true` means one with default limits. + #[cfg(feature = "cookies")] + cookies: match opts.cookies { + None | Some(Either::A(false)) => None, + Some(Either::A(true)) => Some(CookieLimits::default()), + Some(Either::B(cookies)) => Some((&cookies).into()), + }, + dns: opts.dns.map(|dns| options::DnsOptions { + #[cfg(feature = "dns")] + system: dns.system, + overrides: dns.overrides.map(|overrides| { + overrides + .into_iter() + .map(|o| options::DnsOverride { + domain: o.domain, + addresses: o.addresses, + }) + .collect() + }), + #[cfg(feature = "dns")] + servers: dns.servers, + #[cfg(feature = "dns")] + timeout: dns.timeout, + #[cfg(feature = "dns")] + search_domains: dns.search_domains, + #[cfg(feature = "dns")] + ndots: dns.ndots, + #[cfg(feature = "dns")] + hosts_file: dns.hosts_file, + #[cfg(feature = "dns")] + exempt_domains: dns.exempt_domains, + #[cfg(feature = "dns")] + serve_stale: dns.serve_stale, + #[cfg(feature = "dns")] + max_stale: dns.max_stale, + }), + flow_control: opts.flow_control.map(|flow| options::FlowControlOptions { + stream_window: flow.stream_window, + connection_window: flow.connection_window, + }), + headers: opts.headers.map(|headers| { + headers + .into_iter() + .map(|header| options::Header { + name: header.name, + value: header.value, + sensitive: header.sensitive, + }) + .collect() + }), + http2: opts.http2.map(|http2| options::Http2Options { + stream_window: http2.stream_window, + connection_window: http2.connection_window, + adaptive_window: http2.adaptive_window, + }), + #[cfg(feature = "http3")] + http3: opts.http3.map(|http3| options::Http3Options { + congestion: http3.congestion.map(|c| match c { + Http3Congestion::Cubic => options::Http3Congestion::Cubic, + Http3Congestion::Bbr1 => options::Http3Congestion::Bbr1, + }), + max_idle_timeout: http3.max_idle_timeout, + upgrade_enabled: http3.upgrade_enabled, + upgrade_probe: http3.upgrade_probe, + upgrade_probe_timeout: http3.upgrade_probe_timeout, + upgrade_slow_factor: http3.upgrade_slow_factor, + upgrade_slow_ttl: http3.upgrade_slow_ttl, + upgrade_advertised_ttl: http3.upgrade_advertised_ttl, + upgrade_confirmed_ttl: http3.upgrade_confirmed_ttl, + upgrade_failed_ttl: http3.upgrade_failed_ttl, + upgrade_failed_max_ttl: http3.upgrade_failed_max_ttl, + upgrade_cancel_strikes: http3.upgrade_cancel_strikes, + upgrade_attempt_timeout: http3.upgrade_attempt_timeout, + upgrade_follow_advertised_port: http3.upgrade_follow_advertised_port, + upgrade_cache_capacity: http3.upgrade_cache_capacity, + hints: http3.hints.map(|hints| { + hints + .into_iter() + .map(|hint| options::Http3Hint { + host: hint.host, + port: hint.port, + }) + .collect() + }), + stream_window: http3.stream_window, + connection_window: http3.connection_window, + send_window: http3.send_window, + }), + local_address: opts.local_address, + pool: opts.pool.map(|pool| options::PoolOptions { + idle_timeout: pool.idle_timeout, + max_idle_per_host: pool.max_idle_per_host, + }), + quirks: opts.quirks.map(|quirks| options::QuirksOptions { + h1_request_streaming: quirks.h1_request_streaming, + }), + redirect: opts.redirect.map(RedirectPolicy::from), + timeout: opts.timeout.map(|timeout| options::TimeoutOptions { + connect: timeout.connect, + read: timeout.read, + total: timeout.total, + }), + tls: opts.tls.map(|tls| options::TlsOptions { + early_data: tls.early_data, + // Either spelling of a PEM is the same bytes to the client. + identity: tls.identity.map(|pem| pem_bytes(&pem)), + required: tls.required, + extra_roots: tls + .extra_roots + .map(|roots| roots.iter().map(pem_bytes).collect()), + }), + user_agent: opts.user_agent, + } + } +} + +/// PEM input arrives as a buffer or a string; both are just the bytes. +fn pem_bytes(pem: &Either) -> Vec { + match pem { + Either::A(buf) => buf.to_vec(), + Either::B(string) => string.as_bytes().to_vec(), + } +} diff --git a/crates/web-faith-napi/src/agent/options.rs b/crates/web-faith-napi/src/agent/options.rs new file mode 100644 index 0000000..3237fa1 --- /dev/null +++ b/crates/web-faith-napi/src/agent/options.rs @@ -0,0 +1,756 @@ +//! The `AgentOptions` object as JavaScript spells it, and the option groups under it. + +use std::fmt::Debug; + +use napi::{Either, bindgen_prelude::Buffer}; +use napi_derive::napi; + +#[cfg(feature = "cookies")] +use std::time::Duration; +use web_faith::client::RedirectPolicy; + +#[cfg(feature = "cookies")] +use web_faith_cookies::{ + CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, +}; + +use crate::options::RequestCacheMode; + +#[napi(string_enum)] +#[derive(Debug, Clone, Copy)] +pub enum CacheStore { + #[napi(value = "disk")] + Disk, + + #[napi(value = "memory")] + Memory, +} + +/// Settings related to the HTTP cache. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentCacheOptions { + /// Which cache store to use: either `disk` or `memory`. + /// + /// Default: none (cache disabled). + pub store: Option, + /// If `cache.store: "memory"`, the maximum amount of items stored. + /// + /// Default: 10_000. + pub capacity: Option, + /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if + /// no cache mode is set on a request. + /// + /// Default: `"default"`. + pub mode: Option, + /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. + /// + /// Required if `cache.store: "disk"`. + pub path: Option, + /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is + /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a + /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). + /// `shared: true` is required for proxies and multi-user caches. + /// + /// Default: true. + pub shared: Option, +} + +/// Limits the cookie store enforces, from RFC 6265bis. Each is a cap; a caller who needs more room +/// raises the number. +/// +/// The `__Host-` and `__Secure-` name prefix rules are what those prefixes mean, so they always +/// apply and are not settable here: a cookie that shouldn't carry them is named without one. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentCookieOptions { + /// How far ahead of receipt a cookie may expire, in seconds. A cookie asking for longer, via + /// `Max-Age` or `Expires`, has its expiry reduced to this; a shorter one is left alone and a + /// session cookie stays a session cookie. + /// + /// Default: 34_560_000 (400 days). + pub max_age: Option, + /// The largest cookie stored, as the combined length of its name and value in bytes. A larger + /// cookie is not stored. + /// + /// Default: 4096. + pub max_size: Option, + /// How many cookies are kept for any one domain, which is a cookie's `Domain` attribute when it + /// has one and the host that set it otherwise. + /// + /// Default: 180. + pub max_per_host: Option, + /// How many cookies are kept across the whole store, bounding a server that spreads cookies + /// across subdomains to escape `maxPerHost`. + /// + /// Default: 3000. + pub max_total: Option, +} + +#[cfg(feature = "cookies")] +impl From<&AgentCookieOptions> for CookieLimits { + fn from(options: &AgentCookieOptions) -> Self { + Self { + max_age: options + .max_age + .map_or(DEFAULT_MAX_AGE, |secs| Duration::from_secs(secs.into())), + max_size: options.max_size.map_or(DEFAULT_MAX_SIZE, |n| n as usize), + max_per_host: options + .max_per_host + .map_or(DEFAULT_MAX_PER_HOST, |n| n as usize), + max_total: options.max_total.map_or(DEFAULT_MAX_TOTAL, |n| n as usize), + } + } +} + +#[napi(object)] +#[derive(Debug, Clone)] +pub struct DnsOverride { + pub domain: String, + pub addresses: Vec, +} + +/// Settings related to DNS. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentDnsOptions { + /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on + /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native + /// fetch does, this should be your first port of call. + /// + /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory + /// DNS cache, and may lead to worse performance even discounting the cache. + /// + /// Default: false. + /// + /// [Hickory]: https://hickory-dns.org/ + pub system: Option, + /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. + /// + /// Will throw if addresses are in invalid formats. You may provide a port number as part of the + /// address, it will default to port 0 otherwise, which will select the conventional port for the + /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port + /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively + /// blocks that domain from this agent. + /// + /// Default: no overrides. + pub overrides: Option>, + /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to + /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over + /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on + /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the + /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. + /// + /// The encrypted transports always authenticate the resolver. A URL fragment names the + /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates + /// against the hostname; a bare-IP host authenticates against the address itself. + /// + /// Servers are queried in order, a later one reached only once those before it fail. Setting + /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or + /// its scheme is not one of the above, and combining it with `dns.system` throws. + /// + /// Default: system discovery. + pub servers: Option>, + /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead + /// servers costs a single timeout rather than one per server. + /// + /// Default: 5000. + pub timeout: Option, + /// Replace the system's search list, the domains appended to a name that is not fully + /// qualified. Independent of `dns.servers`. + /// + /// Default: the system's search list. + pub search_domains: Option>, + /// How many dots a name must contain before it is tried as given, ahead of the search list. + /// Independent of `dns.servers`. + /// + /// Default: the system's setting. + pub ndots: Option, + /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. + /// + /// Default: platform convention. + pub hosts_file: Option, + /// Further domains to exempt from the configured or encrypted resolver, for the internal + /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's + /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. + /// + /// Default: no extra exemptions. + pub exempt_domains: Option>, + /// Serve an expired cache entry immediately and refresh it in the background, rather than making + /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is + /// almost always still correct, and a connect failure against one that has moved re-resolves and + /// attempts the request again. + /// + /// Set `false` for an agent that must never connect to an address it knows to be out of date: an + /// expired entry is discarded and the lookup blocks on a fresh answer. + /// + /// Default: true. + pub serve_stale: Option, + /// How far past expiry an answer may still be served, in milliseconds. An entry older than this + /// is discarded rather than served: an answer stale enough stops being evidence about where the + /// host is, and a refresh still failing after that long is the case where the address most likely + /// did change. + /// + /// Default: 3600000 (one hour). + pub max_stale: Option, +} + +/// Sets the default headers for every request. +/// +/// If header names or values are invalid, they are silently omitted. +/// Sensitive headers (e.g. `Authorization`) should be marked. +/// +/// Default: none. +#[napi(object)] +#[derive(Debug, Clone)] +pub struct Header { + pub name: String, + pub value: String, + pub sensitive: Option, +} + +#[napi(string_enum)] +#[derive(Debug, Clone, Copy, Default)] +pub enum Http3Congestion { + #[napi(value = "cubic")] + #[default] + Cubic, + + #[napi(value = "bbr1")] + Bbr1, +} + +/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc +/// cache so the first request to this host will attempt HTTP/3 immediately. +#[napi(object)] +#[derive(Debug, Clone)] +pub struct Http3Hint { + /// The hostname (e.g., "example.com"). + pub host: String, + /// The port number (e.g., 443). + pub port: u16, +} + +/// Settings related to HTTP/3. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentHttp3Options { + /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the + /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with + /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by + /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise + /// bandwidth use and optimises for round-trip time, while ignoring packet loss. + /// + /// In some networks, BBR can lead to pathological degradation of overall network conditions, by + /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, + /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). + /// + /// [2]: https://github.com/quinn-rs/quinn/issues/1254 + /// + /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. + pub congestion: Option, + /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that + /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ + /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith + /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). + /// + /// Default: 30. + pub max_idle_timeout: Option, + /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc + /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. + /// + /// Default: true. + pub upgrade_enabled: Option, + /// Whether advertised HTTP/3 endpoints are verified with a background probe + /// before any foreground request is routed to them. + /// + /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say + /// there is UDP connectivity between you and it. Without probing, the next + /// request after an advertisement attempts HTTP/3 inline, and on a silently + /// broken UDP path it stalls until the QUIC idle timeout or + /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per + /// failure cooldown for as long as the path stays broken. + /// + /// With probing (the default), requests keep using TCP until a background + /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the + /// connection pool, so the first upgraded request rides the probe's warm + /// connection. A broken path costs one failed background request per + /// cooldown and no foreground latency at all. + /// + /// The probe is a synthetic request the server will see in its logs. Set + /// this to `false` to restore the inline upgrade if that is unacceptable + /// (per-request billing, easily-alarmed WAFs). + /// + /// `hints` are exempt either way: a hint is your own assertion, so the first + /// request to a hinted origin speaks HTTP/3 immediately, which is also what + /// makes h3-only origins (no TCP listener) work. + /// + /// Default: true. + pub upgrade_probe: Option, + /// Ceiling on how long a background HTTP/3 probe may take before the origin + /// is treated as failed, in **milliseconds**. + /// + /// This bounds background work only — no foreground request ever waits on a + /// probe — so it can afford to be generous: a healthy handshake plus HEAD + /// completes in one or two round trips. Set to 0 to leave probes bounded + /// only by the QUIC idle timeout. + /// + /// Default: 5000 (5 seconds). + pub upgrade_probe_timeout: Option, + /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than + /// its TCP path by this factor. Set to 0 to disable path-time demotion. + /// + /// Faith keeps a per-origin moving average of time-to-response-headers for + /// each protocol family. HTTP/3 is preferred at parity and when moderately + /// slower — its advantages (no head-of-line blocking, connection migration) + /// pay off beyond the average — so this factor should stay well above 1. + /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC + /// average must also exceed the TCP one by an absolute 10ms so LAN-fast + /// origins don't flap on noise. + /// + /// A demoted origin is not treated as broken: it re-enters through a + /// background probe after `upgradeSlowTtl`, asking whether the path has + /// improved at zero foreground cost. + /// + /// Default: 2.5. + pub upgrade_slow_factor: Option, + /// How long (in seconds) a path-time demotion holds before the origin is + /// re-evaluated. See `upgradeSlowFactor`. + /// + /// Default: 600 (10 minutes). + pub upgrade_slow_ttl: Option, + /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. + /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. + /// + /// Default: 86400 (24 hours). + pub upgrade_advertised_ttl: Option, + /// How long (in seconds) to cache a confirmed working HTTP/3 connection. + /// + /// Default: 86400 (24 hours). + pub upgrade_confirmed_ttl: Option, + /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this + /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends + /// Alt-Svc headers. + /// + /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an + /// origin whose UDP path is blocked for good is retried less and less often instead of + /// forever at this interval. A confirmed HTTP/3 response ends the run. + /// + /// Default: 300 (5 minutes). + pub upgrade_failed_ttl: Option, + /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of + /// `upgradeFailedTtl`. + /// + /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, + /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat + /// cooldown that never backs off. + /// + /// Default: 3600 (1 hour). + pub upgrade_failed_max_ttl: Option, + /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, + /// demote an origin back to TCP. + /// + /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request + /// cancelled via `AbortSignal` never produces that signal, so without this an + /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as + /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a + /// sustained run of them demotes the origin, and any successful HTTP/3 response + /// resets the count. + /// + /// Strikes must land within about a minute of each other to count towards a + /// run. A retry loop whose backoff exceeds that window never accumulates one, + /// so callers with a long backoff should set this to 1 for immediate demotion + /// on the first cancelled attempt. + /// + /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that + /// carries small datagrams but drops full-size ones (an MTU blackhole, say). + /// Response headers still arrive, so the attempt resolves and every mechanism + /// here counts it a success — the transfer then stalls partway through the + /// body, where nothing is watching. `maxIdleTimeout` or the request's own + /// timeout is what ends such a request, and the origin stays on HTTP/3. + /// + /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. + /// + /// Default: 3. + pub upgrade_cancel_strikes: Option, + /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is + /// given up on and the request is retried over TCP, in **milliseconds**. + /// + /// Note the unit: the other `upgrade*` settings are in seconds, but this one + /// is in milliseconds to match the `timeout` settings, because useful values + /// are sub-second. + /// + /// This bounds the wait for response headers, not the response body, so a slow + /// body is unaffected. + /// + /// The default is high, but not unconditionally inert: `maxIdleTimeout` is + /// configurable up to 120 seconds, and above 60 seconds this deadline becomes + /// the effective ceiling. Even below that, "QUIC's own idle timeout fires + /// first" only holds while the connection is idle — a transfer still running + /// past this deadline keeps the connection active, so no idle timeout is + /// coming to end it. + /// + /// On expiry the request is retried over TCP, which means it is re-sent: a + /// timeout often means the server is still processing, so a slow + /// non-idempotent request (a POST, say) can end up delivered twice. Lowering + /// this value trades that double-submission risk for faster recovery when a + /// UDP path breaks. Anyone setting it low should confirm their slowest + /// legitimate time to response headers fits well inside the budget. + /// + /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle + /// timeout and the request's own timeout. + /// + /// Default: 60000 (60 seconds). + pub upgrade_attempt_timeout: Option, + /// Connect to the port a server advertises HTTP/3 on, even when it differs from + /// the origin's own port. **This is not standards-compliant**; it is off by + /// default. + /// + /// An `Alt-Svc` advertisement names a network endpoint for the origin, so + /// honouring one correctly means connecting to that endpoint while still + /// sending the *origin's* authority. reqwest cannot express that — it derives + /// the HTTP/3 connect target from the request URI's authority (tracked + /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). + /// So by default Faith does not upgrade at all when the advertised port + /// differs, rather than guessing that the origin's own port also speaks + /// HTTP/3. + /// + /// Setting this to `true` upgrades anyway, by rewriting the request's port to + /// the advertised one. That gets HTTP/3 working today against servers you + /// control, at the cost of three deviations you should be aware of: + /// + /// - The request's `Host`/`:authority` carries the advertised port instead of + /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) + /// forbids. Servers that route on authority may misroute or reject; servers + /// that ignore it are unaffected. + /// - `response.url` reports the port actually connected to. + /// - `redirected` ignores port differences, since the rewritten port would + /// otherwise look like a redirect on every request. + /// + /// TLS is unaffected: certificates are still validated against the origin's + /// hostname. Only the port changes. + /// + /// Default: `false`. + pub upgrade_follow_advertised_port: Option, + /// Maximum number of origins to track in the Alt-Svc cache. + /// + /// Default: 10000. + pub upgrade_cache_capacity: Option, + /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache + /// on agent initialization, so the first request to these hosts will attempt HTTP/3. + pub hints: Option>, + /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/3 only. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload + /// throughput the way the receive windows bound download. The origin's own flow control + /// applies on top of this, so it is a ceiling rather than a grant. + /// + /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window + /// the peer advertises, with no local cap to set. + /// + /// Default: 10 MB (quinn's own default). + pub send_window: Option, +} + +/// Settings related to HTTP/2. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentHttp2Options { + /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Replace HTTP/2's static windows with windows that start small and grow towards a + /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. + /// + /// This is off by default, and turning it on is usually the wrong move. A fresh connection + /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample + /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and + /// carries *less* throughput than the static window for all but the largest transfers. It + /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. + /// + /// Its one real advantage is memory: it holds a large window open only on connections that + /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling + /// buys the same throughput from the first byte. + /// + /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. + /// + /// Default: `false`. + pub adaptive_window: Option, +} + +/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentFlowControlOptions { + /// Maximum bytes an origin may send on any one stream before it must wait for Faith to + /// acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. + /// The default follows browser practice, and is deliberately at the conservative end of it: + /// a pooled server-side client can hold many connections across many origins, so + /// per-connection memory multiplies harder here than in a browser. + /// + /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. + /// + /// Default: 6 MiB. + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one connection before it must + /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// This is larger than `streamWindow` so concurrent streams on one connection share the + /// connection's headroom, while still bounding the worst-case buffering of a connection + /// carrying many concurrent requests. + /// + /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against + /// the other. + /// + /// Default: 15 MiB. + pub connection_window: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentPoolOptions { + /// How many seconds of inactivity before a connection is closed. + /// + /// Default: 90 seconds. + pub idle_timeout: Option, + /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed + /// to keep the idle connections (per host) under that number. + /// + /// Default: `null` (no limit). + pub max_idle_per_host: Option, +} + +/// Switches that depart from standard behaviour on purpose. This is a nested object. +/// +/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule +/// forbids. All of them are off by default, so an agent constructed with no options is +/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise +/// established that what the rule guards against does not apply to them: turning one on means +/// requests may fail against origins that expect the standard behaviour. +#[napi(object)] +#[derive(Debug, Clone, Copy, Default)] +pub struct AgentQuirksOptions { + /// Allow a streaming request body to be sent over an HTTP/1.x connection. + /// + /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read + /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x + /// origin or an intermediary on the path may refuse it. With this on, such a body sends over + /// whichever protocol the connection negotiates. + /// + /// Default: false. + pub h1_request_streaming: Option, +} + +/// Determines the behavior in case the server replies with a redirect status. +/// One of the following values: +/// +/// - `follow`: automatically follow redirects. Faith limits this to 10 redirects. +/// - `error`: reject the promise with a network error when a redirect status is returned. +/// - ~~`manual`~~: not supported. +/// - `stop`: (Faith custom) don't follow any redirects, return the responses. +/// +/// Defaults to `follow`. +#[napi(string_enum)] +#[derive(Debug, Clone, Copy, Default)] +pub enum Redirect { + #[napi(value = "follow")] + #[default] + Follow, + + #[napi(value = "error")] + Error, + + #[napi(value = "manual")] + Manual, + + #[napi(value = "stop")] + Stop, +} + +/// `manual` is not supported and behaves as `follow`, which is what the client's own policy spells +/// out: it carries no variant for a choice that never differed. +impl From for RedirectPolicy { + fn from(redirect: Redirect) -> Self { + match redirect { + Redirect::Follow | Redirect::Manual => Self::Follow, + Redirect::Error => Self::Error, + Redirect::Stop => Self::Stop, + } + } +} + +/// Timeouts for requests made with this agent. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Copy, Default)] +pub struct AgentTimeoutOptions { + /// Set a timeout for only the connect phase, in milliseconds. + /// + /// Default: none. + pub connect: Option, + /// Set a timeout for read operations, in milliseconds. + /// + /// The timeout applies to each read operation, and resets after a successful read. This is more + /// appropriate for detecting stalled connections when the size isn't known beforehand. + /// + /// Default: none. + pub read: Option, + /// Set a timeout for the entire request-response cycle, in milliseconds. + /// + /// The timeout applies from when the request starts connecting until the response body has finished. + /// Also considered a total deadline. + /// + /// Default: none. + pub total: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[napi(object)] +#[derive(Default)] +pub struct AgentTlsOptions { + /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet + /// of application data alongside the opening packet of the TLS handshake. That can enable the server + /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant + /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should + /// really only be used for static assets or to squeeze out the last drop of performance for endpoints + /// that are replay-safe. + /// + /// Default: false. + pub early_data: Option, + /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also + /// called mutual TLS or mTLS) authentication. + /// + /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The + /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options + /// that will cause the `Agent` constructor to throw if the input is in the wrong format. + pub identity: Option>, + /// Disables plain-text HTTP. + /// + /// Default: false. + pub required: Option, + /// Additional PEM-formatted root certificates to trust, on top of the platform's + /// trust store. Each entry may be a PEM bundle containing multiple certificates. + /// + /// This is mainly useful for connecting to servers with self-signed or private-CA + /// certificates, such as internal services or local test servers. This is one of the + /// few options that will cause the `Agent` constructor to throw if the input is in + /// the wrong format. + pub extra_roots: Option>>, +} + +impl Debug for AgentTlsOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AgentTlsOptions") + .field("early_data", &self.early_data) + .field("identity", &"[sensitive]") + .field("required", &self.required) + .field("extra_roots", &self.extra_roots.as_ref().map(|r| r.len())) + .finish() + } +} + +impl Clone for AgentTlsOptions { + fn clone(&self) -> Self { + Self { + early_data: self.early_data.clone(), + identity: self.identity.as_ref().map(|either| match either { + Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), + Either::B(string) => Either::B(string.clone()), + }), + required: self.required.clone(), + extra_roots: self.extra_roots.as_ref().map(|roots| { + roots + .iter() + .map(|either| match either { + Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), + Either::B(string) => Either::B(string.clone()), + }) + .collect() + }), + } + } +} + +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentOptions { + /// Settings related to the HTTP cache. This is a nested object. + pub cache: Option, + /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and + /// included in additional requests. + /// + /// `true` enables the store with the default limits; an options object enables it and tunes them, + /// so `{}` means the same as `true`. + /// + /// Default: `false`. + /// + /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add + /// and retrieve cookies from the store. + pub cookies: Option>, + /// Settings related to DNS. This is a nested object. + pub dns: Option, + /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. + /// + /// Setting these is the normal way to tune windows: one value applies to whichever protocol + /// a request negotiates, so throughput doesn't change when an origin upgrades from one to + /// the other. The `http2` and `http3` groups override them per protocol. + pub flow_control: Option, + /// Sets the default headers for every request. + /// + /// If header names or values are invalid, they are silently omitted. + /// Sensitive headers (e.g. `Authorization`) should be marked. + /// + /// Default: none. + pub headers: Option>, + /// Settings related to HTTP/2. This is a nested object. + pub http2: Option, + /// Settings related to HTTP/3. This is a nested object. + pub http3: Option, + /// Bind outgoing sockets to this local IP address before connecting. + /// + /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that + /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — + /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and + /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to + /// force a specific source address. Throws if the value does not parse as an IP address. + /// + /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). + pub local_address: Option, + /// Settings related to the connection pool. This is a nested object. + pub pool: Option, + /// Switches that depart from standard behaviour on purpose. This is a nested object. + pub quirks: Option, + /// Determines the behavior in case the server replies with a redirect status. + pub redirect: Option, + /// Timeouts for requests made with this agent. This is a nested object. + pub timeout: Option, + /// Settings related to the connection pool. This is a nested object. + pub tls: Option, + /// Custom user agent string. + /// + /// Default: `Faith/{version} reqwest/{version}`. + pub user_agent: Option, +} diff --git a/src/async_task.rs b/crates/web-faith-napi/src/async_task.rs similarity index 96% rename from src/async_task.rs rename to crates/web-faith-napi/src/async_task.rs index 69706bd..b2d6f0e 100644 --- a/src/async_task.rs +++ b/crates/web-faith-napi/src/async_task.rs @@ -7,7 +7,7 @@ use napi::{ }; use serde_json; -use crate::error::FaithError; +use crate::error::{FaithError, FaithErrorExt}; #[derive(Clone, Debug)] pub struct Value(pub serde_json::Value); diff --git a/crates/web-faith-napi/src/conn_tracker.rs b/crates/web-faith-napi/src/conn_tracker.rs new file mode 100644 index 0000000..9be32a1 --- /dev/null +++ b/crates/web-faith-napi/src/conn_tracker.rs @@ -0,0 +1,78 @@ +//! Handing the connection tracker's view to JavaScript. (spec:OBS) +//! +//! The tracking itself, and reading the operating system's statistics, is +//! [`web_faith_conn_tracker`]'s; what belongs here is turning a snapshot into an object V8 can +//! carry, which means JavaScript `Date`s for the timestamps and `i64` for every count. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use napi::{Env, JsDate}; +use napi_derive::napi; +use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; + +#[napi(object)] +#[derive(Clone)] +pub struct ConnectionInfo<'env> { + pub connection_type: String, + pub local_address: String, + pub local_port: u16, + pub remote_address: String, + pub remote_port: u16, + pub first_seen: Option>, + pub last_seen: Option>, + pub expiry: Option>, + pub response_count: i64, + pub rtt_us: Option, + pub rtt_var_us: Option, + pub lost_packets: Option, + pub retransmits: Option, + pub total_retransmits: Option, + pub congestion_window: Option, + pub delivery_rate_bps: Option, +} + +/// A JavaScript `Date` for a point in time, which is milliseconds since the epoch. +/// +/// A timestamp from before the epoch yields the difference the error carries, matching what this +/// did when the conversion lived on the tracker. +fn js_date<'env>(env: &'env Env, at: SystemTime) -> Option> { + env.create_date( + at.duration_since(UNIX_EPOCH) + .unwrap_or_else(|err| err.duration()) + .as_secs_f64() + * 1000.0, + ) + .ok() +} + +/// The tracker's view, as `agent.connections()` returns it. +pub fn connections_for_napi<'env>( + tracker: &ConnectionTracker, + env: &'env Env, +) -> Vec> { + tracker + .snapshot() + .into_iter() + .map(|conn| { + let ConnectionSnapshot { stats, .. } = &conn; + ConnectionInfo { + connection_type: conn.connection_type.to_string(), + local_address: conn.local_addr.ip().to_string(), + local_port: conn.local_addr.port(), + remote_address: conn.remote_addr.ip().to_string(), + remote_port: conn.remote_addr.port(), + first_seen: js_date(env, conn.first_seen), + last_seen: js_date(env, conn.last_seen), + expiry: conn.expiry.and_then(|at| js_date(env, at)), + response_count: conn.response_count as i64, + rtt_us: stats.map(|s| s.rtt_us as i64), + rtt_var_us: stats.map(|s| s.rtt_var_us as i64), + lost_packets: stats.and_then(|s| s.lost.map(|v| v as i64)), + retransmits: stats.map(|s| s.retrans as i64), + total_retransmits: stats.map(|s| s.total_retrans as i64), + congestion_window: stats.map(|s| s.cwnd as i64), + delivery_rate_bps: stats.and_then(|s| s.delivery_rate.map(|v| v as i64)), + } + }) + .collect() +} diff --git a/crates/web-faith-napi/src/error.rs b/crates/web-faith-napi/src/error.rs new file mode 100644 index 0000000..e43cfb0 --- /dev/null +++ b/crates/web-faith-napi/src/error.rs @@ -0,0 +1,140 @@ +use napi::bindgen_prelude::*; +use napi_derive::napi; +pub use web_faith::{FaithError, FaithErrorKind}; + +#[derive(Debug, Clone, Copy)] +enum JsErrorType { + GenericError, + NamedError(&'static str), + SyntaxError, + TypeError, +} + +/// Which JavaScript error class a kind is thrown as. +/// +/// Faith produces fine-grained errors, but maps them to a few javascript error types for fetch +/// compatibility. The `.code` property on errors thrown from Faith is set to a stable name for each +/// error kind, documented in this comprehensive mapping: +/// +/// - JS `AbortError`: +/// - `Aborted` — request was aborted using `signal` +/// - `Timeout` — request timed out +/// - JS `NetworkError`: +/// - `Network` — network error +/// - `Redirect` — when the agent is configured to error on redirects +/// - `ContentLengthOverrun` — a body written with `response.toFile()` exceeded the advertised `Content-Length` +/// - JS `SyntaxError`: +/// - `AddressParse` — IP parse error for `AgentOptions.dns.overrides` +/// - `InvalidIntegrity` — SRI parse error for `RequestInit.integrity` +/// - `JsonParse` — JSON parse error for `response.json()` +/// - `PemParse` — PEM parse error for `AgentOptions.tls.identity` or `AgentOptions.tls.extraRoots` +/// - JS `TypeError`: +/// - `Closed` — a request was made on an agent that has been closed +/// - `InvalidCompression` — `RequestInit.compress` naming no coding Faith can compress in +/// - `InvalidHeader` — invalid header name or value +/// - `InvalidMethod` — invalid HTTP method +/// - `InvalidPath` — a `response.toFile()` destination that does not name a local path +/// - `InvalidUrl` — invalid URL string +/// - `ResponseAlreadyDisturbed` — body already read (mutually exclusive operations) +/// - `ResponseBodyNull` — `response.toFile()` on a response that cannot carry a body +/// - JS generic `Error`: +/// - `BodyStream` — internal stream handling error +/// - `Config` — invalid agent configuration +/// - `FileExists` — a `response.toFile()` write refusing an occupied destination +/// - `FileWrite` — the filesystem refusing a `response.toFile()` write +/// - `IntegrityMismatch` — SRI checksum mismatch (with `RequestInit.integrity`) +/// +/// The library exports an `ERROR_CODES` object which has every error code the library throws, and +/// every error thrown also has a `code` property that is set to one of those codes. So you can +/// accurately respond to the exact error kind by checking its code and matching against the right +/// constant from `ERROR_CODES`, instead of doing string matching on the error message, or coarse +/// `instance of` matching. +/// +/// Due to technical limitations, when reading a body stream, reads might fail, but that error +/// will not have a `code` property. +fn js_type(kind: FaithErrorKind) -> JsErrorType { + use FaithErrorKind as K; + match kind { + K::BodyStream | K::Config | K::FileExists | K::FileWrite | K::IntegrityMismatch => { + JsErrorType::GenericError + } + K::Aborted | K::Timeout => JsErrorType::NamedError("AbortError"), + K::Network | K::Redirect | K::ContentLengthOverrun => { + JsErrorType::NamedError("NetworkError") + } + K::AddressParse | K::InvalidIntegrity | K::JsonParse | K::PemParse => { + JsErrorType::SyntaxError + } + K::Closed + | K::InvalidCompression + | K::InvalidHeader + | K::InvalidMethod + | K::InvalidPath + | K::InvalidUrl + | K::ResponseAlreadyDisturbed + | K::ResponseBodyNull => JsErrorType::TypeError, + } +} + +/// Throwing a [`FaithError`] into JavaScript. +/// +/// The error itself is the client's, and knows nothing of napi; turning one into a value V8 can +/// carry is this crate's business, which is why it arrives as an extension rather than as methods +/// on the error. +pub trait FaithErrorExt { + /// Convert to a napi error. + /// + /// This is explicit rather than a `From` impl so that it cannot happen by accident, losing the + /// error class and code that [`Self::into_js_error`] preserves. + fn into_napi(self) -> napi::Error; + + fn to_napi(&self) -> napi::Error; + + /// Whenever possible, prefer this so that the error types are correct. + fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env>; +} + +impl FaithErrorExt for FaithError { + fn into_napi(self) -> napi::Error { + self.to_napi() + } + + fn to_napi(&self) -> napi::Error { + napi::Error::new(napi::Status::GenericFailure, format!("{self}")) + } + + fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env> { + let code = self.kind.code(); + let unk = match js_type(self.kind) { + JsErrorType::TypeError => JsTypeError::from(self.into_napi()).into_unknown(*env), + JsErrorType::SyntaxError => JsSyntaxError::from(self.into_napi()).into_unknown(*env), + JsErrorType::GenericError => JsError::from(self.into_napi()).into_unknown(*env), + JsErrorType::NamedError(name) => env + .create_error(self.to_napi()) + .and_then(|mut err| { + err.set_named_property("name", name)?; + Ok(err) + }) + .and_then(|err| err.into_unknown(env)) + .unwrap_or_else(|_| JsError::from(self.into_napi()).into_unknown(*env)), + }; + + // we do this manually instead of using the TryFrom so we can return the untouched Unknown if we fail + let Ok(typ) = unk.get_type() else { return unk }; + if typ != ValueType::Object { + return unk; + } + // SAFETY: we have verified that this value is an Object + let Ok(mut obj) = (unsafe { unk.cast::() }) else { + return unk; + }; + + let _ = obj.set("code", code); + obj.into_unknown(env).unwrap_or(unk) + } +} + +#[napi] +pub fn error_codes() -> Vec { + web_faith::error_codes() +} diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs new file mode 100644 index 0000000..990d791 --- /dev/null +++ b/crates/web-faith-napi/src/fetch.rs @@ -0,0 +1,80 @@ +use napi::{ + Env, + bindgen_prelude::{AbortSignal, PromiseRaw}, +}; +use napi_derive::napi; +use tokio::sync::mpsc; + +use bytes::Bytes; +use web_faith::request::{self, RequestBody}; + +use crate::{ + async_task::faith_promise, + error::FaithErrorKind, + options::{self, FaithOptionsAndBody}, + response::FaithResponse, + stream_body::StreamBody, +}; + +#[napi] +pub fn faith_fetch<'env>( + env: &'env Env, + url: String, + options: FaithOptionsAndBody, + signal: Option, + stream_body: Option<&StreamBody>, +) -> Result, napi::Error> { + // Refused rather than ignored: a build without the coding layer cannot compress a body, and a + // caller who asked for it should hear so. See `refuse_absent_capabilities` for the agent-level + // equivalent. + #[cfg(not(feature = "encoding"))] + if options.compress.is_some() { + return Err(napi::Error::from_reason( + "this build has no content-coding support", + )); + } + + #[cfg(not(feature = "cache"))] + if options.cache.is_some() { + return Err(napi::Error::from_reason("this build has no HTTP cache")); + } + + let (options, agent, body) = options::extract(options); + // Taken here, while `fetch()` is still on the stack, so the request counts as in flight from + // the moment it was issued: closing the agent afterwards does not strand it. + // spec:AGENT + let client = agent.inner.client(); + let (s, abort) = mpsc::channel(8); + let has_signal = signal.is_some(); + if let Some(signal) = signal { + signal.on_abort(move || { + let _ = s.try_send(()); + }); + } + + // Get the stream body receiver if provided + let stream_receiver = stream_body.map(|sb| sb.receiver.clone()); + + faith_promise(env, async move { + let body = match (stream_receiver, body) { + // A streaming body is taken from the sender's channel; the receiver is held behind a + // lock because JavaScript hands the same stream body object to one request only. + (Some(receiver), _) => match receiver.lock().await.take() { + Some(receiver) => RequestBody::Stream(Box::pin(receiver.into_stream())), + None => RequestBody::None, + }, + (None, Some(buffer)) => RequestBody::Bytes(Bytes::copy_from_slice(&buffer)), + (None, None) => RequestBody::None, + }; + + let abort = has_signal.then(|| async move { + let mut abort = abort; + let _ = abort.recv().await; + }); + + let client = client.ok_or(FaithErrorKind::Closed)?; + request::send(&agent.inner, client, &url, options, body, abort) + .await + .map(FaithResponse::from) + }) +} diff --git a/src/lib.rs b/crates/web-faith-napi/src/lib.rs similarity index 77% rename from src/lib.rs rename to crates/web-faith-napi/src/lib.rs index 8ad7ab1..b2a7c28 100644 --- a/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -1,18 +1,11 @@ mod agent; -#[cfg(feature = "http3")] -mod alt_svc; mod async_task; -mod body; +#[cfg(feature = "connection-tracking")] mod conn_tracker; -mod cookies; -mod dns; -mod encoding; mod error; mod fetch; -mod integrity; mod options; mod response; -mod retry; mod stream_body; mod timing; diff --git a/src/options.rs b/crates/web-faith-napi/src/options.rs similarity index 83% rename from src/options.rs rename to crates/web-faith-napi/src/options.rs index e690e3c..c8e18fe 100644 --- a/src/options.rs +++ b/crates/web-faith-napi/src/options.rs @@ -1,9 +1,13 @@ use std::{fmt::Debug, sync::Arc, time::Duration}; +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; + use napi::bindgen_prelude::*; use napi_derive::napi; +use web_faith::request::{Credentials, RequestOptions}; + use crate::agent::Agent; /// The cache mode you want to use for the request. This may be any one of the following values: @@ -68,6 +72,7 @@ pub enum RequestCacheMode { Reload, } +#[cfg(feature = "cache")] impl From for CacheMode { fn from(mode: RequestCacheMode) -> Self { match mode { @@ -134,9 +139,6 @@ pub enum DuplexOption { Half, } -/// The RFC 9218 header the `priority` option maps onto. -pub(crate) const PRIORITY: &str = "priority"; - /// Maps the `priority` option onto an RFC 9218 `Priority` header value. /// /// Urgency runs from 0 (most urgent) to 7 (least urgent), and a request that sends no header @@ -176,49 +178,35 @@ pub struct FaithOptionsAndBody { pub timeout: Option, } -#[derive(Clone, Debug, Default)] -pub(crate) struct FaithOptions { - pub(crate) cache: RequestCacheMode, - /// The `compress` option as given, resolved to a coding where the body is compressed. - pub(crate) compress: Option, - pub(crate) credentials: CredentialsOption, - pub(crate) headers: Option>, - pub(crate) integrity: Option, - pub(crate) method: Option, - /// The `Priority` header value derived from the `priority` option, if it maps to one. - pub(crate) priority: Option<&'static str>, - pub(crate) timeout: Option, -} - -impl FaithOptions { - pub(crate) fn extract(opts: FaithOptionsAndBody) -> (Self, Agent, Option>) { - let credentials = opts.credentials.unwrap_or_default(); - // Transform same-origin to include - let credentials = if credentials == CredentialsOption::SameOrigin { - CredentialsOption::Include - } else { - credentials - }; - - ( - Self { - cache: opts.cache.unwrap_or_default(), - compress: opts.compress, - credentials, - headers: opts.headers, - integrity: opts.integrity, - method: opts.method, - priority: priority_urgency(opts.priority.as_deref()), - timeout: opts.timeout.map(Into::into).map(Duration::from_millis), - }, - Agent::clone(&opts.agent), - opts.body.map(|either| match either { - Either3::A(s) => Arc::new(Buffer::from(s.as_bytes())), - Either3::B(b) => Arc::new(b), - Either3::C(u) => Arc::new(Buffer::from(u.as_ref())), - }), - ) - } +/// Read a `fetch()` call's options into the shape the client takes. +pub(crate) fn extract(opts: FaithOptionsAndBody) -> (RequestOptions, Agent, Option>) { + // `same-origin` means nothing without an origin to be same as, so it lands on `include`, + // which is what a server-side caller means by it. + let credentials = match opts.credentials.unwrap_or_default() { + CredentialsOption::Omit => Credentials::Omit, + CredentialsOption::Include | CredentialsOption::SameOrigin => Credentials::Include, + }; + + ( + RequestOptions { + #[cfg(feature = "cache")] + cache: opts.cache.unwrap_or_default().into(), + #[cfg(feature = "encoding")] + compress: opts.compress, + credentials, + headers: opts.headers, + integrity: opts.integrity, + method: opts.method, + priority: priority_urgency(opts.priority.as_deref()), + timeout: opts.timeout.map(Into::into).map(Duration::from_millis), + }, + Agent::clone(&opts.agent), + opts.body.map(|either| match either { + Either3::A(s) => Arc::new(Buffer::from(s.as_bytes())), + Either3::B(b) => Arc::new(b), + Either3::C(u) => Arc::new(Buffer::from(u.as_ref())), + }), + ) } #[cfg(test)] diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs new file mode 100644 index 0000000..c4df436 --- /dev/null +++ b/crates/web-faith-napi/src/response.rs @@ -0,0 +1,473 @@ +use std::{ + fmt::Debug, + result::Result, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use futures::TryStreamExt; +use napi::{ + bindgen_prelude::*, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, +}; +use napi_derive::napi; + +use web_faith::response::{FileDestination, FileProgress, FileWritten, Response, Trailers}; + +use crate::{ + async_task::{Value, faith_promise}, + error::{FaithError, FaithErrorExt, FaithErrorKind}, + timing::TimingBreakdown, +}; + +/// Options for `toFile()`. +#[napi(object)] +#[derive(Debug, Default)] +pub struct ToFileOptions { + /// Whether to truncate and replace an occupied destination. Defaults to false, which + /// refuses an occupied destination with a `FileExists` error and leaves it untouched. + pub overwrite: Option, + /// The permissions a newly created file is given, defaulting to what Node's own + /// filesystem writes use. Ignored on platforms without Unix file modes. + pub mode: Option, +} + +/// What `toFile()` resolves to. +#[napi(object)] +#[derive(Debug)] +pub struct ToFileResult { + /// The absolute filesystem path written to. + pub path: String, + /// The number of bytes that landed at the destination. + pub bytes_written: i64, +} + +/// A progress report from a `toFile()` write in flight. +#[napi(object)] +#[derive(Debug)] +pub struct ToFileProgress { + /// The number of bytes written to the file so far. + pub bytes_written: i64, + /// What the response advertised in `Content-Length`, when it sent one and Faith is + /// not decoding the body. Absent when the total is not known ahead of time, which is + /// the case for a chunked response and for one Faith decodes. + pub content_length: Option, +} + +/// The `Response` interface of the Fetch API represents the response to a request. +/// +/// Faith does not allow its `Response` object to be constructed. If you need to, you may use the +/// `webResponse()` method to convert one into a Web API `Response` object; note the caveats. +#[napi] +#[derive(Debug, Clone)] +pub struct FaithResponse { + pub(crate) inner: Response, +} + +impl From for FaithResponse { + fn from(inner: Response) -> Self { + Self { inner } + } +} + +impl From<&ToFileOptions> for FileDestination { + fn from(options: &ToFileOptions) -> Self { + Self { + overwrite: options.overwrite.unwrap_or(false), + mode: options.mode, + } + } +} + +impl From for ToFileProgress { + fn from(progress: FileProgress) -> Self { + let count = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); + Self { + bytes_written: count(progress.bytes_written), + content_length: progress.content_length.map(count), + } + } +} + +impl From for ToFileResult { + fn from(written: FileWritten) -> Self { + Self { + path: written.path, + bytes_written: i64::try_from(written.bytes_written).unwrap_or(i64::MAX), + } + } +} + +/// The callback `toFile()` reports progress to. +/// +/// `CalleeHandled = false`: progress is not an error-first callback, so the JavaScript +/// side receives the report on its own rather than as the second argument. +pub type ProgressCallback = + ThreadsafeFunction, ToFileProgress, Status, false>; + +#[napi] +impl FaithResponse { + /// The `headers` read-only property of the `Response` interface contains the `Headers` object + /// associated with the response. + /// + /// Note that Faith does not provide a custom `Headers` class; instead the Web API `Headers` structure + /// is used directly and constructed by Faith when needed. + /// + /// This is a function as an internal implementation detail and the wrapper makes it a property. + #[napi] + pub fn headers(&self) -> Vec<(String, String)> { + self.inner + .headers() + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|v| (name.to_string(), v.to_string())) + }) + .collect() + } + + /// The `ok` read-only property of the `Response` interface contains a boolean stating whether the + /// response was successful (status in the range 200-299) or not. + #[napi(getter)] + pub fn ok(&self) -> bool { + self.inner.ok() + } + + /// Custom to Faith. + /// + /// The `peer` read-only property of the `Response` interface contains an object with information about + /// the remote peer that sent this response: + #[napi(getter, ts_return_type = "{ address?: string; certificate?: Buffer }")] + pub fn peer<'env>(&self, env: &'env Env) -> Result, napi::Error> { + let mut obj = Object::new(env)?; + obj.set( + "address", + self.inner.peer.address.map(|addr| addr.to_string()), + )?; + obj.set( + "certificate", + self.inner + .peer + .certificate + .as_deref() + .map(|cert| Buffer::from(cert)), + )?; + Ok(obj) + } + + /// The `redirected` read-only property of the `Response` interface indicates whether or not the + /// response is the result of a request you made which was redirected. + /// + /// Note that by the time you read this property, the redirect will already have happened, and you + /// cannot prevent it by aborting the fetch at this point. + /// + /// One caveat specific to Faith: with the agent's `http3.upgradeFollowAdvertisedPort` + /// enabled, HTTP/3 responses compare URLs ignoring the port, because the port + /// was rewritten to the advertised one and would otherwise register as a + /// redirect. A genuine redirect differing only in port therefore reads as + /// `false` on those responses. + #[napi(getter)] + pub fn redirected(&self) -> bool { + self.inner.redirected() + } + + /// The `status` read-only property of the `Response` interface contains the HTTP status codes of the + /// response. For example, 200 for success, 404 if the resource could not be found. + /// + /// A value is `0` is returned for a response whose `type` is `opaque`, `opaqueredirect`, or `error`. + #[napi(getter)] + pub fn status(&self) -> u16 { + self.inner.status().as_u16() + } + + /// The `statusText` read-only property of the `Response` interface contains the status message + /// corresponding to the HTTP status code in `Response.status`. For example, this would be `OK` for a + /// status code `200`, `Continue` for `100`, `Not Found` for `404`. + /// + /// Faith always returns the canonical status message for the code. In HTTP/1, servers can send + /// custom status text, but that text is not surfaced here; in HTTP/2 and HTTP/3, custom status + /// text is not supported at all. For status codes with no well-known message, this is an empty + /// string. + #[napi(getter)] + pub fn status_text(&self) -> &'static str { + self.inner.status_text() + } + + /// The `type` read-only property of the `Response` interface contains the type of the response. The + /// type determines whether scripts are able to access the response body and headers. + /// + /// In Faith, this is always set to `basic`. + #[napi(getter, js_name = "type")] + pub fn typ(&self) -> &'static str { + "basic" + } + + /// The `url` read-only property of the `Response` interface contains the URL of the response. The + /// value of the `url` property will be the final URL obtained after any redirects. + #[napi(getter)] + pub fn url(&self) -> String { + self.inner.url().to_string() + } + + /// The `version` read-only property of the `Response` interface contains the HTTP version of the + /// response. The value will be the final HTTP version after any redirects and protocol upgrades. + /// + /// This is custom to Faith. + #[napi(getter)] + pub fn version(&self) -> String { + format!("{:?}", self.inner.version) + } + + /// The `bodyUsed` read-only property of the `Response` interface is a boolean value that indicates + /// whether the body has been read yet. + /// + /// In Faith, this indicates whether the body stream has ever been read from or canceled, as defined + /// [in the standard](https://streams.spec.whatwg.org/#is-readable-stream-disturbed). Note that accessing + /// the `.body` property counts as a read, even if you don't actually consume any bytes of content. + #[napi(getter)] + pub fn body_used(&self) -> bool { + self.inner.disturbed.load(Ordering::SeqCst) + } + + /// The `body` read-only property of the `Response` interface is a `ReadableStream` of the body + /// contents, or `null` for any actual HTTP response that has no body, such as `HEAD` requests and + /// `204 No Content` responses. + /// + /// Note that browsers currently do not return `null` for those responses, but the standard + /// requires it. Faith chooses to respect the standard rather than the browsers in this case. + /// + /// An important consideration exists in conjunction with the connection pool: if you start the + /// body stream, this will hold the connection until the stream is fully consumed. If another + /// request is started during that time, and you don't have an available connection in the pool + /// for the host already, the new request will open one. + /// + /// Note that this is a function as an implementation detail; the wrapper makes it a property. + #[napi] + pub fn body( + &self, + env: Env, + ) -> Result>>, napi::Error> { + // we mark the body as disturbed, but we still allow reading it through here + // as essentially, the body() can be accessed many times as the same stream + let _ = self.inner.check_stream_disturbed(); + + let Some(lock) = &self.inner.body.body else { + return Ok(None); + }; + + // if the lock is taken then we're consuming the body somehow + let mut body = lock + .try_lock() + .map_err(|_| FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed).into_napi())?; + + let stream = self + .inner + .ensure_stream(&mut body, self.inner.body.drained.clone()) + .map_err(|e| e.into_napi())?; + + let stream = napi::bindgen_prelude::ReadableStream::create_with_stream_bytes( + &env, + stream + .map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)).into_napi()), + ) + .map_err(|e| { + napi::Error::from( + FaithError::new(FaithErrorKind::BodyStream, Some(e.to_string())) + .into_js_error(&env), + ) + })?; + Ok(Some(stream)) + } + + /// Discard the response body, releasing the connection back to the pool. + /// + /// This is useful when you don't need the body but want to ensure the connection + /// can be reused for subsequent requests. If you don't call this and don't consume + /// the body, the connection may be held open until the response is garbage collected. + /// + /// For HTTP/1, the remaining body is read and thrown away so the connection can go back + /// to the pool. For HTTP/2 and HTTP/3, the body is dropped instead, which cancels the + /// stream (RST_STREAM / STOP_SENDING) without affecting the multiplexed connection. + /// + /// Returns a promise that resolves when the body has been fully discarded. + #[napi] + pub fn discard<'env>(&self, env: &'env Env) -> Result, napi::Error> { + let this = Clone::clone(self); + faith_promise(env, async move { + this.inner.discard().await; + Ok(()) + }) + } + + /// The `bytes()` method of the `Response` interface takes a `Response` stream and reads it to + /// completion. It returns a promise that resolves with a `Uint8Array`. + /// + /// In Faith, this returns a Node.js `Buffer`, which can be used as (and is a subclass of) a `Uint8Array`. + #[napi] + pub fn bytes<'env>(&self, env: &'env Env) -> Result, napi::Error> { + let this = Clone::clone(self); + faith_promise( + env, + async move { this.inner.bytes().await.map(Buffer::from) }, + ) + } + + /// The `text()` method of the `Response` interface takes a `Response` stream and reads it to + /// completion. It returns a promise that resolves with a `String`. The response is always decoded + /// using UTF-8; as per the standard, invalid UTF-8 sequences are replaced with U+FFFD rather + /// than causing an error. + #[napi] + pub fn text<'env>(&self, env: &'env Env) -> Result, napi::Error> { + let this = Clone::clone(self); + faith_promise(env, async move { this.inner.text().await }) + } + + /// The `json()` method of the `Response` interface takes a `Response` stream and reads it to + /// completion. It returns a promise which resolves with the result of parsing the body text as + /// `JSON`. + /// + /// Note that despite the method being named `json()`, the result is not JSON but is instead the + /// result of taking JSON as input and parsing it to produce a JavaScript object. + /// + /// Further note that, at least in Faith, this method first reads the entire response body as bytes, + /// and then parses that as JSON. This can use up to double the amount of memory. If you need more + /// efficient access, consider handling the response body as a stream. + #[napi] + pub fn json<'env>(&self, env: &'env Env) -> Result, napi::Error> { + let this = Clone::clone(self); + faith_promise(env, async move { this.inner.json().await.map(Value) }) + } + + /// Custom to Faith. + /// + /// `toFile(path, options)` writes the response body to a file on disk, the bytes + /// travelling from the network to the filesystem inside Faith without crossing into + /// JavaScript. It is a whole-body read alongside `bytes()` and its siblings: the first + /// consumer wins, `bodyUsed` becomes true once the read begins, and `integrity` is + /// verified when set. + /// + /// Resolves to `{ path, bytesWritten }`, where `path` is the absolute filesystem path + /// written to and `bytesWritten` counts the bytes that landed there. + /// + /// `onProgress` is reported to as the bytes land, at most every + /// `PROGRESS_INTERVAL`, with a final report once the last byte is written. The + /// wrapper takes it from the options object; it arrives here as its own argument + /// because a threadsafe function cannot be a field of a `#[napi(object)]`. + /// + /// The `file://` URL to path conversion and the `InvalidPath` rejection happen in the + /// wrapper, so this receives a resolved string path. + /// + /// spec:BODY#tofile + #[napi( + ts_args_type = "path: string, options?: ToFileOptions | undefined | null, onProgress?: ((progress: ToFileProgress) => void) | undefined | null" + )] + pub fn to_file<'env>( + &self, + env: &'env Env, + path: String, + options: Option, + on_progress: Option, + ) -> Result, napi::Error> { + let this = Clone::clone(self); + let options = options.unwrap_or_default(); + faith_promise(env, async move { + let destination = FileDestination::from(&options); + let written = this + .inner + .write_to_file(&path, &destination, |progress| { + if let Some(callback) = &on_progress { + callback.call( + ToFileProgress::from(progress), + // Progress is observational: a report the queue cannot take is + // dropped rather than made to hold up the write it describes. + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + }) + .await?; + Ok(ToFileResult::from(written)) + }) + } + + /// Custom to Faith. + /// + /// The measurements behind the `timing` property, which the wrapper turns into a + /// `PerformanceResourceTiming`. + /// + /// A resource timing entry describes a finished request, so this does not resolve until + /// the body has ended: by being read, by `discard()`, or by the collector draining one + /// that was abandoned. A response that cannot carry a body has ended already. + /// + /// Phases are milliseconds from the start of the request rather than absolute times, so + /// the wrapper can place them on the same clock as the platform's other performance + /// entries. + /// + /// This is an async fn as an internal implementation detail and the wrapper makes it a + /// property. + // spec:RESP#request-timing + #[napi] + pub async fn timing(&self) -> TimingBreakdown { + self.inner.timing().await.into() + } + + /// The `trailers()` read-only property of the `Response` interface returns a promise that + /// resolves to either `null` or a `Headers` structure that contains the HTTP/2 or /3 trailing + /// headers. + /// + /// This was once in the standard as a getter, but was removed as no browser implemented it. + /// + /// Trailers only exist once the body has ended, so this does not resolve until the body + /// has been consumed — by `text()`, `bytes()`, `json()`, `blob()`, or reading the `body` + /// stream. Awaiting it first, on its own, waits forever: that is the behaviour the fetch + /// standard's trailers proposal describes (), not + /// a quirk of Faith. Holding the promise while something else reads the body is fine, and + /// costs nothing while it is pending. + /// + /// `discard()` counts as consuming the body but discards its trailers with it, so this + /// then resolves to `null` rather than waiting for trailers that can no longer arrive. + /// + /// This is an async fn as an internal implementation detail and the wrapper makes it a property. + #[napi] + pub async fn trailers(&self) -> Option> { + match self.inner.trailers().await { + // NotYet cannot come back from `settled`, which is what it waits on. + Trailers::NotYet | Trailers::None => None, + Trailers::Some(headers) => Some( + headers + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|v| (name.to_string(), v.to_string())) + }) + .collect(), + ), + } + } + + /// The `clone()` method of the `Response` interface creates a clone of a response object, identical + /// in every way, but stored in a different variable. + /// + /// `clone()` throws an `Error` if the response body has already been used. + /// + /// (Per the standard, this should throw a `TypeError`, but for technical reasons this is not + /// possible with Faith.) + #[napi] + pub fn clone(&self, env: Env) -> Result { + if self.inner.disturbed.load(Ordering::SeqCst) { + return Err(FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed) + .into_js_error(&env) + .into()); + } + + Ok(Self::from(Response { + disturbed: Arc::new(AtomicBool::new(false)), + ..Clone::clone(&self.inner) + })) + } +} diff --git a/src/stream_body.rs b/crates/web-faith-napi/src/stream_body.rs similarity index 100% rename from src/stream_body.rs rename to crates/web-faith-napi/src/stream_body.rs diff --git a/crates/web-faith-napi/src/timing.rs b/crates/web-faith-napi/src/timing.rs new file mode 100644 index 0000000..4ec9e23 --- /dev/null +++ b/crates/web-faith-napi/src/timing.rs @@ -0,0 +1,38 @@ +//! Surfacing a request's timing as a `PerformanceResourceTiming` for the wrapper. +//! +//! The measuring itself is [`web_faith::timing`]'s; what belongs here is the shape JavaScript +//! receives. +//! +//! spec:RESP#request-timing + +use napi_derive::napi; +use web_faith::timing::RequestTiming; + +/// The measurements behind a response's timing breakdown. +/// +/// The wrapper turns these into a `PerformanceResourceTiming`; the phases are milliseconds from +/// the start of the request rather than absolute times, so the wrapper can place them on the +/// same clock as the rest of the platform's performance entries. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct TimingBreakdown { + pub headers_ms: f64, + pub body_ms: Option, + pub reused: bool, + pub next_hop_protocol: String, + pub content_encoding: Option, + pub from_cache: bool, +} + +impl From for TimingBreakdown { + fn from(timing: RequestTiming) -> Self { + Self { + headers_ms: timing.headers_ms, + body_ms: timing.body_ms, + reused: timing.reused, + next_hop_protocol: timing.next_hop_protocol, + content_encoding: timing.content_encoding, + from_cache: timing.from_cache, + } + } +} diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml new file mode 100644 index 0000000..253d0b9 --- /dev/null +++ b/crates/web-faith/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "web-faith" +description = "A browser-shaped HTTP client: fetch semantics over a Rust network stack" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the client API is built out and the release tooling is in place. +publish = false + +[dependencies] +async-trait.workspace = true +bytes.workspace = true +futures.workspace = true +http.workspace = true +http-body.workspace = true +http-body-util.workspace = true +http-cache-reqwest = { workspace = true, optional = true } +hyper.workspace = true +hyper-util.workspace = true +moka.workspace = true +reqwest.workspace = true +rustls = { workspace = true, optional = true } +reqwest-middleware.workspace = true +serde.workspace = true +serde_json.workspace = true +stream_shared.workspace = true +ssri.workspace = true +strum.workspace = true +tokio.workspace = true +url.workspace = true +web-faith-conn-tracker = { workspace = true, optional = true } +web-faith-encoding = { workspace = true, optional = true } +web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } +web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } +web-faith-alt-svc = { workspace = true, optional = true } + +[features] +default = [ + "cache", + "connection-tracking", + "cookies", + "dns", + "encoding", + "http3", + "tls-aws-lc-rs", +] +# The HTTP cache, its store, and the per-request cache mode. +cache = ["dep:http-cache-reqwest"] +# Per-connection kernel counters, and the agent verb that reports them. +connection-tracking = ["dep:web-faith-conn-tracker"] +# The cookie jar, and the agent option and handle that reach it. +cookies = ["dep:web-faith-cookies", "reqwest/cookies"] +# Faith's own caching resolver, which a prefetch warms and an `HTTPS` record lookup reads. +# Without it names resolve through the platform, as reqwest resolves them. +dns = ["dep:web-faith-dns", "reqwest/hickory-dns", "web-faith-alt-svc?/dns"] +# Content codings: negotiating and decoding a response body, and compressing a request one. +encoding = ["dep:web-faith-encoding"] +# Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. +http3 = ["reqwest/http3", "tls-aws-lc-rs", "dep:web-faith-alt-svc"] +# The rustls crypto provider: aws-lc-rs by default, ring as the alternative. Cargo features are +# additive, so enabling both is aws-lc-rs, which is also what an `http3` build resolves to because +# reqwest's QUIC stack is tied to it. +tls-aws-lc-rs = ["reqwest/rustls"] +tls-ring = ["reqwest/rustls-no-provider", "dep:rustls", "rustls/ring"] diff --git a/crates/web-faith/build.rs b/crates/web-faith/build.rs new file mode 100644 index 0000000..884e37b --- /dev/null +++ b/crates/web-faith/build.rs @@ -0,0 +1,42 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + // The default user agent names the version of reqwest the request actually goes out on, which + // only the lock file knows. + let lock_path = find_cargo_lock().expect("Cargo.lock not found in any ancestor directory"); + let reqwest_version = extract_reqwest_version(&lock_path).unwrap(); + println!("cargo:rustc-env=REQWEST_VERSION={}", reqwest_version); + println!("cargo:rerun-if-changed={}", lock_path.display()); +} + +/// The lock file lives at the workspace root, which is an ancestor of this crate. +fn find_cargo_lock() -> Option { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR")?); + manifest_dir + .ancestors() + .map(|dir| dir.join("Cargo.lock")) + .find(|lock| lock.is_file()) +} + +fn extract_reqwest_version(lock_path: &PathBuf) -> Option { + let cargo_lock = fs::read_to_string(lock_path).ok()?; + + // Find the reqwest package entry in Cargo.lock + for line in cargo_lock.lines() { + if line.starts_with("name = \"reqwest\"") { + // Look for the version line in the next few lines + let mut lines_iter = cargo_lock.lines().skip_while(|l| l != &line); + lines_iter.next(); // Skip the name line + + for next_line in lines_iter.take(5) { + if let Some(version) = next_line.trim().strip_prefix("version = \"") { + if let Some(version) = version.strip_suffix("\"") { + return Some(version.to_string()); + } + } + } + } + } + + None +} diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs new file mode 100644 index 0000000..f2d90ec --- /dev/null +++ b/crates/web-faith/src/agent.rs @@ -0,0 +1,384 @@ +//! The agent: what owns a connection pool, and the verbs that act on a live one. + +// spec:AGENT spec:WARM spec:NETCHG spec:OBS + +use std::sync::{ + Arc, RwLock, + atomic::{AtomicU64, Ordering}, +}; + +use moka::sync::Cache as MokaCache; +use reqwest::Client; +use reqwest_middleware::ClientWithMiddleware; +use url::Url; + +#[cfg(feature = "encoding")] +use http::header::HeaderValue; + +#[cfg(feature = "connection-tracking")] +use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; + +#[cfg(feature = "cookies")] +use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] +use web_faith_dns::{FaithResolver, ResolverReport}; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, H3Prober}; + +use crate::{ + client::ClientRecipe, + stats::{AgentStats, InnerAgentStats}, + warm_up::origin_key, +}; + +#[cfg(all(feature = "http3", feature = "dns"))] +use crate::client::install_https_sink; + +mod build; +mod warm; + +#[cfg(test)] +mod tests; + +/// The agent settings a request consults, as opposed to those a client is built from. +#[derive(Debug, Clone, Default)] +pub struct AgentSettings { + /// Whether an HTTP/3 upgrade may follow a port the origin advertised, which a request needs so + /// a rewritten port is not reported as a redirect. + pub h3_follow_advertised_port: bool, + /// Whether the upgrade machinery is on at all. A warm-up needs it to route the way a foreground + /// request would. + #[cfg(feature = "http3")] + pub h3_upgrade_enabled: bool, + /// Whether a streaming request body may go out over HTTP/1.x. + pub quirk_h1_request_streaming: bool, + /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides + /// which codings a response is decoded under when a request adds none of its own. + #[cfg(feature = "encoding")] + pub default_accept_encoding: Option, + /// The agent's default `Content-Encoding`, if one sits among its default headers, which a + /// request layers its own coding on top of rather than displacing. + #[cfg(feature = "encoding")] + pub default_content_encoding: Option, + /// Whether a `Priority` header sits among the agent's default headers, so that default wins + /// over the one a request's priority would derive. + pub has_default_priority: bool, +} + +/// What an agent holds while it is open, and gives up when it is closed. +/// +/// Behind a shared lock because closing acts on the agent rather than on the handle it was called +/// through: every clone names the same one, so every clone sees the result. +#[derive(Debug)] +pub struct Live { + /// The heavy resources (connection pool, DNS resolver, background tasks) live inside this + /// client, so dropping it is what actually releases them. + pub client: ClientWithMiddleware, + /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A warm-up + /// sends its synthetic request here rather than through the middleware stack, which bypasses the + /// HTTP cache and the Alt-Svc layer, and so keeps the warm-up out of request accounting, while + /// still pooling the connection foreground requests reuse. + // spec:WARM + pub raw_client: Client, + /// The DNS resolver, shared with the client so a prefetch warms the cache requests read. `None` + /// under the system resolver, where there is no such cache. + // spec:WARM + #[cfg(feature = "dns")] + pub dns_resolver: Option, + #[cfg(feature = "http3")] + pub alt_svc_cache: Option>, + /// Held so closing can abort in-flight background probes: each one owns a clone of the raw + /// client, which would otherwise keep the connection pool alive past close for up to the probe + /// timeout. + #[cfg(feature = "http3")] + pub h3_prober: Option>, +} + +/// An HTTP client with its own connection pool, caches, and resolver. +/// +/// Cloning one is cheap and every clone names the same underlying agent, so cloning is how a request +/// gets an agent to run on rather than a way to get a second pool. Because clones share, closing +/// acts on the agent itself and every handle to it sees the result. +// spec:AGENT +#[derive(Debug, Clone)] +pub struct Agent { + /// `None` once [`Agent::close`] has been called. + live: Arc>>, + /// Origins with a warm-up connection opened within the pool idle window, so a repeat + /// warm-up does no new work. Keyed by `scheme://host:port`; entries expire with the idle + /// timeout. + // spec:WARM + pub warmed: MokaCache, + /// Single-flight claims for warm-ups in flight, so concurrent calls for the same + /// origin do not open duplicate connections. + // spec:WARM + pub warming: MokaCache, + /// Bumped by [`Self::network_changed`], so a warm-up that was in flight across the signal does + /// not record its origin as warm: its connection went into the pool that was just dropped. + // spec:NETCHG#reach-across-the-subsystems + pub warm_generation: Arc, + /// The jar outlives a close and stays readable from a closed agent. + #[cfg(feature = "cookies")] + pub cookie_jar: Option>, + pub stats: Arc, + #[cfg(feature = "connection-tracking")] + pub conn_tracker: Arc, + /// Whether an upgrade may follow a port the origin advertised. A request needs it to stop a + /// rewritten port from being reported as a redirect. + pub h3_follow_advertised_port: bool, + /// Whether the upgrade machinery is on at all. A warm-up needs it to route the way a foreground + /// request would: with it off, nothing upgrades, whatever the caches hold. + // spec:WARM#preconnect + #[cfg(feature = "http3")] + pub h3_upgrade_enabled: bool, + /// Whether a streaming request body may go out over HTTP/1.x, which the fetch standard otherwise + /// reserves to HTTP/2 and HTTP/3. + // spec:QUIRK#http-1-x-request-body-streaming + pub quirk_h1_request_streaming: bool, + /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides + /// the codings a response is decoded under when a request adds none of its own. + #[cfg(feature = "encoding")] + pub default_accept_encoding: Option, + /// The agent's default `Content-Encoding`, if one sits among its default headers. A request + /// layers its own coding on top of this rather than displacing it. + // spec:ENC + #[cfg(feature = "encoding")] + pub default_content_encoding: Option, + /// Whether a `Priority` header sits among the agent's default headers. That default wins over + /// the header a request's priority would derive. + pub has_default_priority: bool, + /// How to build this agent's clients, so [`Self::network_changed`] can build them again. Shared + /// rather than cloned per handle: every handle builds the same client from the same recipe. + // spec:NETCHG + pub recipe: Arc, +} + +impl Agent { + /// The agent's cookie jar, if it keeps one. + /// + /// The jar itself, rather than per-cookie methods wrapped around it, so cookies go in and out + /// through the type `web-faith-cookies` documents. It outlives a close and stays readable from a + /// closed agent. + // spec:COOK + #[cfg(feature = "cookies")] + pub fn cookies(&self) -> Option<&Arc> { + self.cookie_jar.as_ref() + } + + /// Take a handle on the client, or `None` once the agent is closed. + /// + /// A request takes its own handle at the moment it is issued, which is what lets one already in + /// flight finish while a later one is refused. + // spec:AGENT + pub fn client(&self) -> Option { + self.live().as_ref().map(|live| live.client.clone()) + } + + /// Take a handle on the raw client a warm-up sends through, or `None` once closed. + pub fn raw_client(&self) -> Option { + self.live().as_ref().map(|live| live.raw_client.clone()) + } + + /// The DNS resolver, if the agent has one of its own and is still open. + #[cfg(feature = "dns")] + pub fn dns_resolver(&self) -> Option { + self.live() + .as_ref() + .and_then(|live| live.dns_resolver.clone()) + } + + #[cfg(feature = "http3")] + fn alt_svc_cache(&self) -> Option> { + self.live() + .as_ref() + .and_then(|live| live.alt_svc_cache.clone()) + } + + #[cfg(feature = "http3")] + fn h3_prober(&self) -> Option> { + self.live().as_ref().and_then(|live| live.h3_prober.clone()) + } + + fn live(&self) -> std::sync::RwLockReadGuard<'_, Option> { + self.live + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn live_mut(&self) -> std::sync::RwLockWriteGuard<'_, Option> { + self.live + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Build an agent from options, validating them into the recipe its clients are built from. + /// + + /// Close the agent, releasing its connection pool, DNS resolver, and any + /// background tasks it owns, rather than waiting for the garbage collector + /// to drop it. This is worth doing when you create many short-lived agents; + /// a single long-lived agent can just be left to the GC. + /// + /// Requests already in flight run to completion. Any new request on a closed + /// agent throws a `Closed` error. Calling `close()` more than once is a + /// no-op. The cookie jar, if any, remains readable through [`Self::cookies`]. + pub fn close(&self) { + // Dropping the client releases the reqwest connection pool and the + // Hickory resolver task; the alt-svc cache goes with it. The raw client + // shares that pool and the resolver, so it goes too, and both are what a + // later warm-up checks to refuse with the closed-agent error. + // Taken out of the shared cell, so every handle on this agent sees it closed. + let Some(live) = self.live_mut().take() else { + return; + }; + + #[cfg(feature = "http3")] + // Probes hold a raw client clone; abort them so the pool doesn't outlive close by up to + // the probe timeout. + if let Some(prober) = &live.h3_prober { + prober.abort_all(); + } + + drop(live); + } + + /// Tell the agent the network underneath it has changed, so it stops deciding from what it + /// learned about a network that is gone. + /// + /// Node has no portable signal for an interface or connectivity change, so Faith cannot + /// detect one; this is the reaction, and wiring it to a trigger (an OS notification, a VPN + /// transition, a captive-portal sign-in) is the caller's own. It drops pooled connections, + /// flushes the DNS cache, demotes the HTTP/3 origins that a real response confirmed back to + /// advertised so a background probe re-verifies them, and clears the HTTP/3 failure and slow + /// states, their cooldown backoff, and the path-time averages. + /// + /// Configuration, `http3.hints`, `Alt-Svc` advertisements, the cookie jar, the HTTP cache and + /// the `stats()` counters are all kept: none of them is a claim about a network path. + /// + /// Requests already in flight are not interrupted and run to completion on the connections + /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a + /// closed agent does nothing, and calling it repeatedly is harmless. + // spec:NETCHG + pub fn network_changed(&self) { + { + // Held across the rebuild so a close cannot land halfway through it. + let mut guard = self.live_mut(); + // A closed agent has already released all of this. + let Some(live) = guard.as_mut() else { + return; + }; + + // reqwest cannot drop pooled connections short of dropping the client, so the client is + // rebuilt from the recipe the agent kept for this. Requests in flight hold the handle + // they took when they were issued, so they run to completion and the old pool goes when + // the last of them finishes. + // + // A rebuild that fails leaves the agent on its existing client: the options were already + // validated at construction, so a failure here is not the caller's to answer for, and an + // agent that still works on the old network beats one that works nowhere. + let built = self.recipe.build( + #[cfg(feature = "cookies")] + self.cookie_jar.as_ref(), + #[cfg(feature = "dns")] + live.dns_resolver.as_ref(), + #[cfg(feature = "http3")] + live.alt_svc_cache.as_ref(), + ); + if let Ok(built) = built { + #[cfg(feature = "http3")] + { + // Abort probes running on the old client: each holds a clone of it, and their + // answers would describe the path that has just gone away. + if let Some(prober) = &live.h3_prober { + prober.abort_all(); + } + live.h3_prober = built.prober; + // The sink holds the prober, which has just been replaced along with the client + // it sends on; leaving the old one installed would aim DNS-triggered probes at a + // client that has been dropped. + #[cfg(feature = "dns")] + install_https_sink( + live.dns_resolver.as_ref(), + live.alt_svc_cache.as_ref(), + live.h3_prober.as_ref(), + self.h3_upgrade_enabled, + ); + } + live.client = built.client; + live.raw_client = built.raw_client; + } + + // Names resolve afresh against the new network, through that network's own servers: the + // resolver drops what it read off the old one and reads again when next used. Under the + // system resolver there is no resolver here and so nothing to reset. + // spec:DNS + #[cfg(feature = "dns")] + if let Some(resolver) = &live.dns_resolver { + resolver.reset(); + } + + #[cfg(feature = "http3")] + if let Some(alt_svc_cache) = &live.alt_svc_cache { + alt_svc_cache.network_changed(); + } + } + + // The warm-up records describe pooled connections that have just been dropped, so a + // `preconnect` after the signal opens a connection rather than finding the origin warm + // (spec:NETCHG, spec:WARM). The single-flight claims are left alone: a warm-up still in + // flight is not duplicated by releasing its claim, and the generation bump is what stops + // it recording an origin as warm on the strength of a connection in the dropped pool. + self.warmed.invalidate_all(); + self.warm_generation.fetch_add(1, Ordering::Relaxed); + } + + /// The counters this agent has gathered, as they stand. + pub fn stats(&self) -> AgentStats { + self.stats.snapshot() + } + + /// Returns information on current connections open by this agent. + /// + /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: + /// this makes it possible to track indicators over time to find the retransmission rate, for + /// example. The lost-packet count and delivery rate are only available on Linux. Some other + /// fields might also be missing depending on platform support; and no forward guarantees are made + /// on field availability. If the platform isn't supported at all, this will always return empty. + #[cfg(feature = "connection-tracking")] + pub fn connections(&self) -> Vec { + self.conn_tracker.snapshot() + } + + /// Returns the DNS servers this agent resolves through, in the order they are queried, so + /// "are my lookups actually encrypted" is answerable from inside the process. + /// + /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, + /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). + /// The list is empty until the resolver has been used, because it reads its configuration on + /// first use, and empty for an agent using the system resolver. + // spec:OBS#resolvers + #[cfg(feature = "dns")] + pub fn resolvers(&self) -> Vec { + self.dns_resolver() + .as_ref() + .map(FaithResolver::resolvers) + .unwrap_or_default() + } + + /// Note that a request reached this origin, so it holds a connection the pool keeps idle for + /// the idle window and a `preconnect` for it has no new work to do. + /// + /// Called for foreground requests as well as warm-ups, because the criterion is about the + /// origin holding an idle pooled connection, not about how it came to hold one. + // spec:WARM + pub fn mark_warm(&self, url: &Url) { + self.warmed.insert(origin_key(url), ()); + } + + /// Whether [`Self::close`] has been called. + pub fn is_closed(&self) -> bool { + self.live().is_none() + } +} diff --git a/crates/web-faith/src/agent/build.rs b/crates/web-faith/src/agent/build.rs new file mode 100644 index 0000000..bcd31b2 --- /dev/null +++ b/crates/web-faith/src/agent/build.rs @@ -0,0 +1,607 @@ +//! Turning options into an agent: validating what a caller expressed, and building from it. + +// spec:AGENT + +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + str::FromStr, + sync::{Arc, RwLock}, + time::Duration, +}; + +use http::header::{HeaderMap, HeaderName, HeaderValue}; + +#[cfg(feature = "cache")] +use crate::{ + client::{HttpCacheRecipe, HttpCacheStore}, + options::CacheStore, +}; + +#[cfg(feature = "cache")] +use http_cache_reqwest::{ + CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, +}; +use moka::sync::Cache as MokaCache; +use reqwest::{Identity, tls::Certificate}; + +#[cfg(feature = "connection-tracking")] +use web_faith_conn_tracker::ConnectionTracker; + +#[cfg(feature = "cookies")] +use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] +use web_faith_dns::{ + DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, +}; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig}; + +use crate::{ + USER_AGENT, + agent::{Agent, AgentSettings, Live}, + client::{ClientRecipe, NodeEnvRecipe}, + error::{FaithError, FaithErrorKind}, + options::{AgentOptions, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows}, + request::PRIORITY, +}; + +#[cfg(feature = "http3")] +use crate::{client::H3UpgradeRecipe, options::Http3Congestion}; + +#[cfg(all(feature = "http3", feature = "dns"))] +use crate::client::install_https_sink; + +impl Agent { + /// This is what both surfaces land on, so the defaults a caller gets are settled here rather + /// than once per surface. + // spec:AGENT spec:NETCHG + pub fn from_options(options: AgentOptions) -> Result { + // Destructured rather than read field by field so that a new option cannot be added + // without the compiler pointing here, where every option is turned into the recipe the + // agent's clients are built from (spec:NETCHG). + let AgentOptions { + #[cfg(feature = "cache")] + cache, + #[cfg(feature = "cookies")] + cookies, + dns, + flow_control, + headers, + http2, + #[cfg(feature = "http3")] + http3, + local_address, + pool, + quirks, + redirect, + timeout, + tls, + user_agent, + } = options; + + let quirk_h1_request_streaming = quirks + .and_then(|quirks| quirks.h1_request_streaming) + .unwrap_or(false); + + // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts + // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the + // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and + // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such + // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. + let local_address = match &local_address { + Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { + FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + ) + })?), + None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), + None => None, + }; + + // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) + // The jar is installed on the client by the recipe, so it survives a rebuild + // (spec:NETCHG#what-the-signal-keeps). + #[cfg(feature = "cookies")] + let cookie_jar = cookies.map(|limits| Arc::new(FaithJar::new(limits))); + + let dns = dns.unwrap_or_default(); + // Without Faith's own resolver every name goes to the platform, so that is the only + // answer there is to give. + #[cfg(feature = "dns")] + let dns_system = dns.system.unwrap_or(false); + #[cfg(not(feature = "dns"))] + let dns_system = true; + // Naming servers and asking for the system resolver at once is a contradiction rather than + // a preference, since the system resolver is not Faith's to point at listed servers + // (spec:DNS#system-resolver). + #[cfg(feature = "dns")] + if dns_system + && dns + .servers + .as_ref() + .is_some_and(|servers| !servers.is_empty()) + { + return Err(FaithError::new( + FaithErrorKind::Config, + Some("dns.servers cannot be combined with dns.system".to_string()), + )); + } + // Parsed whichever resolver is in use: overrides take effect under the system resolver + // too (spec:DNS#overrides), and an unparseable address is a construction error either + // way (spec:AGENT#construction). + let dns_overrides = dns + .overrides + .unwrap_or_default() + .into_iter() + .map(|DnsOverride { domain, addresses }| { + let addresses = addresses + .into_iter() + .map(|addr| match SocketAddr::from_str(&addr) { + Ok(addr) => Ok(addr), + Err(err) => match IpAddr::from_str(&addr) { + Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), + Ok(IpAddr::V6(ip)) => { + Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) + } + Err(_) => Err(FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + )), + }, + }) + .collect::, FaithError>>()?; + Ok((domain, addresses)) + }) + .collect::, FaithError>>()?; + + // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so + // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), + // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and + // order each resolver is reached by (spec:DNS#transports). The system resolver + // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there + // and `prefetchDns` resolves as a no-op (spec:WARM). + #[cfg(feature = "dns")] + let dns_resolver = if dns_system { + None + } else { + // These settings configure Faith's own resolver only, so they are read on the path that + // builds one rather than validated under the system resolver that ignores them. + let mut servers = Vec::new(); + for url in dns.servers.unwrap_or_default() { + servers.push(ServerSpec::parse(&url).map_err(|message| { + FaithError::new(FaithErrorKind::AddressParse, Some(message)) + })?); + } + Some(FaithResolver::new(ResolverSettings { + servers, + timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), + ndots: dns.ndots.map(|n| n as usize), + search_domains: parse_domains(dns.search_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, + hosts_file: dns.hosts_file, + exempt_domains: parse_domains(dns.exempt_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? + .unwrap_or_default(), + serve_stale: dns.serve_stale.unwrap_or(true), + max_stale: dns + .max_stale + .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), + })) + }; + + #[cfg(feature = "encoding")] + let mut default_accept_encoding = None; + #[cfg(feature = "encoding")] + let mut default_content_encoding = None; + let mut has_default_priority = false; + let mut default_headers = None; + if let Some(headers) = headers + && !headers.is_empty() + { + let map = HeaderMap::from_iter(headers.into_iter().filter_map( + |Header { + name, + value, + sensitive, + }| { + let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { + return None; + }; + + let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { + return None; + }; + + if sensitive.unwrap_or(false) { + value.set_sensitive(true); + } + + Some((name, value)) + }, + )); + #[cfg(feature = "encoding")] + { + default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); + default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); + } + has_default_priority = map.contains_key(PRIORITY); + default_headers = Some(map); + } + + // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so + // the explicit sizes are not resolved at all when it's on: reqwest would let the later + // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence + // visible here rather than depending on hyper's internal ordering. + let http2 = http2.unwrap_or_default(); + let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); + let http2_windows = (!http2_adaptive_window).then(|| { + resolve_windows( + flow_control.as_ref(), + http2.stream_window, + http2.connection_window, + ) + }); + + #[cfg(feature = "http3")] + let http3_max_idle_timeout = Duration::from_secs( + http3 + .as_ref() + .and_then(|h| h.max_idle_timeout) + .unwrap_or(30) + .clamp(1, 120) + .into(), + ); + + // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window + // inside an unbounded connection window: the stream window is the binding constraint + // on a high-latency link, and the unbounded connection window means a connection with + // many concurrent requests has no ceiling on what it buffers. Both are set here. + #[cfg(feature = "http3")] + let http3_windows = resolve_windows( + flow_control.as_ref(), + http3.as_ref().and_then(|h| h.stream_window), + http3.as_ref().and_then(|h| h.connection_window), + ); + + #[cfg(feature = "http3")] + let http3_congestion_bbr = matches!( + http3.as_ref().and_then(|h| h.congestion), + Some(Http3Congestion::Bbr1) + ); + + #[cfg(feature = "http3")] + let http3_send_window = http3.as_ref().and_then(|h| h.send_window); + + let pool_idle_timeout = pool + .as_ref() + .and_then(|pool| pool.idle_timeout) + .map(|seconds| Duration::from_secs(seconds.into())); + // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). + let pool_max_idle_per_host = pool.as_ref().map(|pool| { + pool.max_idle_per_host + .and_then(|n| n.try_into().ok()) + .unwrap_or(usize::MAX) + }); + + let connect_timeout = timeout + .and_then(|t| t.connect) + .map(|millis| Duration::from_millis(millis.into())); + let read_timeout = timeout + .and_then(|t| t.read) + .map(|millis| Duration::from_millis(millis.into())); + let total_timeout = timeout + .and_then(|t| t.total) + .map(|millis| Duration::from_millis(millis.into())); + + #[cfg(feature = "http3")] + let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); + let tls_required = tls.as_ref().and_then(|tls| tls.required); + // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a + // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). + let (tls_identity, tls_extra_roots) = match tls { + None => (None, Vec::new()), + Some(tls) => { + let identity = match &tls.identity { + None => None, + Some(identity) => Some(Identity::from_pem(identity).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?), + }; + + let mut extra_roots = Vec::new(); + for pem in tls.extra_roots.iter().flatten() { + extra_roots.extend(Certificate::from_pem_bundle(pem).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?); + } + + (identity, extra_roots) + } + }; + + #[cfg(feature = "cache")] + let http_cache = if let Some(cache) = cache + && let Some(store) = cache.store + { + let mode = cache.mode.unwrap_or_default().into(); + let options = HttpCacheOptions { + cache_options: Some(CacheOptions { + shared: cache.shared.unwrap_or(true), + ignore_cargo_cult: true, + ..Default::default() + }), + ..Default::default() + }; + let store = match store { + CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { + path: cache + .path + .ok_or_else(|| { + FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) + })? + .into(), + remove_opts: Default::default(), + }), + CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( + MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), + )), + }; + + Some(HttpCacheRecipe { + mode, + options, + store, + }) + } else { + None + }; + + // Read outside the `alt_svc_cache` block below because `fetch` needs it too, + // to keep a rewritten port from looking like a redirect. + #[cfg(feature = "http3")] + let h3_follow_advertised_port = http3 + .as_ref() + .and_then(|o| o.upgrade_follow_advertised_port) + .unwrap_or(false); + #[cfg(not(feature = "http3"))] + let h3_follow_advertised_port = false; + + // The origin knowledge is built once and outlives every client the agent builds: it is + // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). + #[cfg(feature = "http3")] + let (alt_svc_cache, h3_upgrade) = { + let http3_opts = http3.as_ref(); + let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); + + let advertised_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_advertised_ttl) + .unwrap_or(86400) + .into(), + ); + let confirmed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_confirmed_ttl) + .unwrap_or(86400) + .into(), + ); + let failed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_ttl) + .unwrap_or(300) + .into(), + ); + let failed_max_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_max_ttl) + .unwrap_or(3600) + .into(), + ); + let capacity = http3_opts + .and_then(|o| o.upgrade_cache_capacity) + .unwrap_or(10_000) + .into(); + let cancel_strikes = http3_opts + .and_then(|o| o.upgrade_cancel_strikes) + .unwrap_or(3); + let attempt_timeout = match http3_opts + .and_then(|o| o.upgrade_attempt_timeout) + .unwrap_or(60_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); + let probe_timeout = match http3_opts + .and_then(|o| o.upgrade_probe_timeout) + .unwrap_or(5_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let slow_factor = http3_opts + .and_then(|o| o.upgrade_slow_factor) + .unwrap_or(2.5); + let slow_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_slow_ttl) + .unwrap_or(600) + .into(), + ); + + let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + capacity, + cancel_strikes, + strike_window: Duration::from_secs(60), + follow_advertised_port: h3_follow_advertised_port, + // The single-flight claim must outlive the probe it covers, so + // an aborted probe frees its origin without a report; without a + // probe deadline, the QUIC idle timeout (max 120s) is the bound. + probe_ttl: probe_timeout + .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), + slow_factor, + slow_ttl, + })); + + if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { + for hint in hints { + cache.add_hint(&hint.host, hint.port); + } + } + + ( + Some(cache), + H3UpgradeRecipe { + enabled, + attempt_timeout, + probe, + probe_timeout, + }, + ) + }; + + let recipe = ClientRecipe { + user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), + local_address, + default_headers, + dns_system, + dns_overrides, + http2_adaptive_window, + http2_windows, + #[cfg(feature = "http3")] + http3_max_idle_timeout, + #[cfg(feature = "http3")] + http3_windows, + #[cfg(feature = "http3")] + http3_congestion_bbr, + #[cfg(feature = "http3")] + http3_send_window, + pool_idle_timeout, + pool_max_idle_per_host, + redirect, + connect_timeout, + read_timeout, + total_timeout, + #[cfg(feature = "http3")] + tls_early_data, + tls_identity, + tls_required, + tls_extra_roots, + node_env: NodeEnvRecipe::read(), + #[cfg(feature = "cache")] + http_cache, + #[cfg(feature = "http3")] + h3_upgrade, + }; + + let settings = AgentSettings { + h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming, + #[cfg(feature = "encoding")] + default_accept_encoding, + #[cfg(feature = "encoding")] + default_content_encoding, + has_default_priority, + }; + + Self::build( + recipe, + settings, + #[cfg(feature = "cookies")] + cookie_jar, + #[cfg(feature = "dns")] + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + ) + } + + /// An agent with default options. + pub fn new() -> Result { + Self::from_options(AgentOptions::default()) + } + + /// Build an agent a setting at a time. See [`AgentBuilder`](crate::builder::AgentBuilder). + pub fn builder() -> crate::builder::AgentBuilder { + crate::builder::AgentBuilder::default() + } + + /// Build an agent from a validated recipe. + /// + /// The recipe is what a client is built from, and the settings are what each request consults; + /// validating whatever a caller expressed them as belongs to the surface that took it. + pub fn build( + recipe: ClientRecipe, + settings: AgentSettings, + #[cfg(feature = "cookies")] cookie_jar: Option>, + #[cfg(feature = "dns")] dns_resolver: Option, + #[cfg(feature = "http3")] alt_svc_cache: Option>, + ) -> Result { + let conn_timeout = recipe.conn_timeout(); + let built = recipe.build( + #[cfg(feature = "cookies")] + cookie_jar.as_ref(), + #[cfg(feature = "dns")] + dns_resolver.as_ref(), + #[cfg(feature = "http3")] + alt_svc_cache.as_ref(), + )?; + + // Only now do all three exist: the resolver is built before the cache, and the prober + // holds a client that holds the resolver, so this is the earliest the loop can be closed + // (spec:DNS#https-records). + #[cfg(all(feature = "http3", feature = "dns"))] + install_https_sink( + dns_resolver.as_ref(), + alt_svc_cache.as_ref(), + built.prober.as_ref(), + recipe.h3_upgrade.enabled, + ); + + Ok(Self { + live: Arc::new(RwLock::new(Some(Live { + client: built.client, + raw_client: built.raw_client, + #[cfg(feature = "dns")] + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + #[cfg(feature = "http3")] + h3_prober: built.prober, + }))), + // A warm-up connection is warm only as long as the pool keeps it idle, so the record + // that an origin is warm expires with that same window. + warmed: MokaCache::builder().time_to_live(conn_timeout).build(), + // A safety TTL well past any reasonable warm-up, so a claim that never gets released + // (a warm-up whose task is dropped) frees the origin rather than wedging it. + warming: MokaCache::builder() + .time_to_live(Duration::from_secs(300)) + .build(), + warm_generation: Default::default(), + #[cfg(feature = "cookies")] + cookie_jar, + stats: Default::default(), + #[cfg(feature = "connection-tracking")] + conn_tracker: ConnectionTracker::new(conn_timeout), + h3_follow_advertised_port: settings.h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming: settings.quirk_h1_request_streaming, + #[cfg(feature = "encoding")] + default_accept_encoding: settings.default_accept_encoding, + #[cfg(feature = "encoding")] + default_content_encoding: settings.default_content_encoding, + has_default_priority: settings.has_default_priority, + recipe: Arc::new(recipe), + }) + } +} diff --git a/crates/web-faith/src/agent/tests.rs b/crates/web-faith/src/agent/tests.rs new file mode 100644 index 0000000..834d5bf --- /dev/null +++ b/crates/web-faith/src/agent/tests.rs @@ -0,0 +1,80 @@ +use super::*; + +/// An agent is constructible without a caller assembling a recipe by hand, and comes up with a +/// live client and the default user agent. +#[tokio::test] +async fn a_default_agent_comes_up() { + let agent = Agent::new().expect("default options build an agent"); + + assert!(!agent.is_closed()); + assert_eq!(agent.recipe.user_agent, crate::USER_AGENT); + // No jar until the options ask for one. + #[cfg(feature = "cookies")] + assert!(agent.cookie_jar.is_none()); +} + +/// The builder reaches every group, and a group left alone stays absent rather than being +/// spelled out as absent. +#[cfg(all(feature = "dns", feature = "http3"))] +#[tokio::test] +async fn the_builder_sets_what_it_is_given_and_nothing_else() { + use std::time::Duration; + + let options = Agent::builder() + .user_agent("YourApp/1.2.3") + .dns(|dns| dns.timeout(Duration::from_secs(2)).ndots(3)) + .pool(|pool| pool.max_idle_per_host(8)) + .into_options(); + + assert_eq!(options.user_agent.as_deref(), Some("YourApp/1.2.3")); + let dns = options.dns.expect("the dns group was reached"); + assert_eq!(dns.timeout, Some(2_000)); + assert_eq!(dns.ndots, Some(3)); + // Reached but not set stays unset... + assert_eq!(dns.servers, None); + // ...and a group never reached is absent entirely. + assert!(options.tls.is_none()); + assert!(options.http3.is_none()); +} + +/// What the builder produces is what the agent is built from, so a built agent carries it. +#[tokio::test] +async fn the_builder_builds_an_agent() { + let agent = Agent::builder() + .user_agent("YourApp/1.2.3") + .build() + .expect("the options are valid"); + + assert_eq!(agent.recipe.user_agent, "YourApp/1.2.3"); + assert!(!agent.is_closed()); +} + +/// A handle taken before a close still works afterwards, which is what lets a request issued +/// just before the close run to completion. +#[tokio::test] +async fn a_handle_taken_before_a_close_survives_it() { + let agent = Agent::new().expect("default options build an agent"); + let issued = agent.client().expect("an open agent hands out a client"); + + agent.close(); + + assert!(agent.is_closed()); + assert!( + agent.client().is_none(), + "a closed agent hands out no more clients" + ); + // The handle taken earlier is still usable; dropping it is what releases its share. + drop(issued); +} + +/// Closing acts on the agent itself, so every clone sees it. +#[tokio::test] +async fn closing_a_clone_closes_the_agent() { + let agent = Agent::new().expect("default options build an agent"); + let clone = agent.clone(); + + agent.close(); + + assert!(agent.is_closed()); + assert!(clone.is_closed(), "a clone names the same agent"); +} diff --git a/crates/web-faith/src/agent/warm.rs b/crates/web-faith/src/agent/warm.rs new file mode 100644 index 0000000..09c7b5d --- /dev/null +++ b/crates/web-faith/src/agent/warm.rs @@ -0,0 +1,158 @@ +//! Warming an agent's caches ahead of the request that needs them. + +// spec:WARM + +use std::{future::Future, sync::atomic::Ordering}; + +use moka::sync::Cache as MokaCache; +use reqwest::Version; + +use crate::{ + agent::Agent, + error::{FaithError, FaithErrorKind}, + warm_up::{extract_host, origin_key, reduce_to_origin}, +}; + +impl Agent { + /// Warm the DNS cache for `host`, so a later request to it skips the lookup. + /// + /// The argument is a bare host; a scheme, port, or path in a fuller string is ignored. The + /// returned future completes when the answer lands in the cache and never fails, whatever + /// happens on the network: the work is advisory. Under the system resolver there is no cache to + /// warm, so it completes having done nothing. A host with nothing to resolve, or a closed agent, + /// is refused here rather than by the future. + // spec:WARM + pub fn prefetch_dns(&self, host: &str) -> Result + use<>, FaithError> { + if self.is_closed() { + return Err(FaithErrorKind::Closed.into()); + } + + let Some(host) = extract_host(host) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + + #[cfg(feature = "dns")] + let resolver = self.dns_resolver(); + Ok(async move { + // Nothing to warm without Faith's own resolver: the platform's cache is not ours to fill. + #[cfg(feature = "dns")] + if let Some(resolver) = resolver { + resolver.prefetch(&host).await; + } + #[cfg(not(feature = "dns"))] + let _ = host; + }) + } + + /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS + /// setup. + /// + /// The argument is an origin (`scheme://host[:port]`); a longer URL is reduced to one. The + /// warm-up sends a synthetic `HEAD` to the origin's root -- the origin sees it -- over the + /// transport the next foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC + /// connection, every other origin a TCP one. The returned future completes when the attempt + /// finishes and never fails: every network failure is quiet. Something that cannot be connected + /// to, or a closed agent, is refused here rather than by the future. + // spec:WARM + pub fn preconnect(&self, origin: &str) -> Result + use<>, FaithError> { + let Some(raw_client) = self.raw_client() else { + return Err(FaithErrorKind::Closed.into()); + }; + + let Some(url) = reduce_to_origin(origin) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + let key = origin_key(&url); + + // Already warm within the idle window, or a warm-up for this origin already in flight: + // either way there is no new work to do, so finish without opening a duplicate. + let redundant = self.warmed.contains_key(&key) + || !self.warming.entry(key.clone()).or_insert(()).is_fresh(); + + // The transport the next foreground request would take, decided exactly as the Alt-Svc + // layer decides it: nothing upgrades with the machinery off; with a prober, only a + // confirmed origin routes to QUIC (an advertisement is evidence worth probing, not worth + // routing on); without one, the inline upgrade acts on advertisements too. Diverging here + // would warm the wrong transport. + // spec:WARM#preconnect + #[cfg(feature = "http3")] + let h3_port = self + .alt_svc_cache() + .filter(|_| self.h3_upgrade_enabled) + .and_then(|cache| { + if self.h3_prober().is_some() { + cache.confirmed_port(&url) + } else { + cache.should_use_h3(&url) + } + }); + #[cfg(not(feature = "http3"))] + let h3_port: Option = None; + + #[cfg(feature = "connection-tracking")] + let conn_tracker = self.conn_tracker.clone(); + let warmed = self.warmed.clone(); + let warming = self.warming.clone(); + // Read before the warm-up starts, to compare against once it finishes. + let warm_generation = self.warm_generation.clone(); + let generation = warm_generation.load(Ordering::Relaxed); + + Ok(async move { + if redundant { + return; + } + + // Release the single-flight claim whatever happens, so a later warm-up is not blocked + // by this one having finished. + struct ReleaseClaim { + warming: MokaCache, + key: String, + } + impl Drop for ReleaseClaim { + fn drop(&mut self) { + self.warming.invalidate(&self.key); + } + } + let _release = ReleaseClaim { + warming, + key: key.clone(), + }; + + let request = match h3_port { + Some(port) => { + let mut h3_url = url.clone(); + // A port differing from the origin's only comes back with the + // follow-advertised-port option on; rewriting the URL is how reqwest is told to + // connect there, mirroring the foreground path. + if Some(port) != h3_url.port_or_known_default() { + let _ = h3_url.set_port(Some(port)); + } + raw_client.head(h3_url).version(Version::HTTP_3) + } + None => raw_client.head(url.clone()), + }; + + let outcome = request.send().await; + + // A TCP warm-up leaves a pooled connection to track; a QUIC one does not (QUIC + // connections are not tracked, and a confirmed origin has nothing left to probe). + #[cfg(feature = "connection-tracking")] + if h3_port.is_none() + && let Ok(response) = &outcome + && let Some(info) = response + .extensions() + .get::() + { + conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); + } + + // A network change while this was in flight leaves the origin unmarked: the connection + // landed in the pool that change dropped, so it is not warm however well the request + // went. + // spec:NETCHG#reach-across-the-subsystems + if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { + warmed.insert(key, ()); + } + }) + } +} diff --git a/src/body.rs b/crates/web-faith/src/body.rs similarity index 92% rename from src/body.rs rename to crates/web-faith/src/body.rs index 1c33698..c443b0e 100644 --- a/src/body.rs +++ b/crates/web-faith/src/body.rs @@ -17,23 +17,23 @@ use tokio::sync::Mutex; use crate::timing::TimingSlot; -pub(crate) type DynStream = dyn Stream> + Send + Sync; +pub type DynStream = dyn Stream> + Send + Sync; -pub(crate) enum Body { +pub enum Body { Inner(reqwest::Body), Consumed, Stream(SharedStream>>), } /// Wrapper around the body that auto-drains on drop to release the connection. -pub(crate) struct BodyHolder { +pub struct BodyHolder { pub body: Option>>, /// Flag to prevent drain if body was properly consumed - pub(crate) drained: Arc, + pub drained: Arc, /// HTTP version - HTTP/2+ doesn't need draining for connection reuse - pub(crate) version: Version, + pub version: Version, /// Settled when the body ends, so an abandoned body still finishes its timing - pub(crate) timing: Option>, + pub timing: Option>, } impl BodyHolder { @@ -143,7 +143,7 @@ impl Drop for BodyHolder { /// Drain a body to release the connection back to the pool. /// This reads and discards all remaining bytes. -pub(crate) async fn drain_body_inner(arc: Arc>) { +pub async fn drain_body_inner(arc: Arc>) { let mut guard = arc.lock().await; match replace(&mut *guard, Body::Consumed) { Body::Inner(body) => { diff --git a/crates/web-faith/src/builder.rs b/crates/web-faith/src/builder.rs new file mode 100644 index 0000000..16b0239 --- /dev/null +++ b/crates/web-faith/src/builder.rs @@ -0,0 +1,596 @@ +//! Building an agent a setting at a time. +//! +//! [`Agent::builder`] returns an [`AgentBuilder`], whose methods mirror the option groups. A group +//! is reached through a closure, so one left alone is absent from the call rather than spelled out +//! as absent: +//! +//! ```no_run +//! # use std::time::Duration; +//! # use web_faith::agent::Agent; +//! let agent = Agent::builder() +//! .user_agent("YourApp/1.2.3") +//! .timeout(|timeout| timeout.connect(Duration::from_secs(2))) +//! .pool(|pool| pool.max_idle_per_host(8)) +//! .build()?; +//! # Ok::<(), web_faith::FaithError>(()) +//! ``` +//! +//! Durations are `Duration` here whatever unit the setting is carried in, and a setting left unset +//! takes its default. +//! +//! [`Agent::builder`]: crate::agent::Agent::builder + +// spec:AGENT + +use std::{net::IpAddr, time::Duration}; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; + +#[cfg(feature = "cache")] +use crate::options::{CacheOptions, CacheStore}; + +#[cfg(feature = "http3")] +use crate::options::{Http3Congestion, Http3Hint, Http3Options}; + +#[cfg(feature = "cookies")] +use web_faith_cookies::CookieLimits; + +use crate::{ + agent::Agent, + client::RedirectPolicy, + error::FaithError, + options::{ + AgentOptions, DnsOptions, DnsOverride, FlowControlOptions, Header, Http2Options, + PoolOptions, QuirksOptions, TimeoutOptions, TlsOptions, + }, +}; + +/// Milliseconds, saturating rather than wrapping on a duration no setting could mean. +fn millis(duration: Duration) -> u32 { + duration.as_millis().try_into().unwrap_or(u32::MAX) +} + +/// Whole seconds, rounded down, saturating as above. +fn secs(duration: Duration) -> u32 { + duration.as_secs().try_into().unwrap_or(u32::MAX) +} + +/// Builds an [`Agent`]. See [the module documentation](self). +#[derive(Debug, Default)] +#[must_use = "an agent builder does nothing until built"] +pub struct AgentBuilder { + options: AgentOptions, +} + +impl AgentBuilder { + /// Validate what has been set and build the agent. + pub fn build(self) -> Result { + Agent::from_options(self.options) + } + + /// The options as they stand, for a caller that would rather fill them in directly. + pub fn into_options(self) -> AgentOptions { + self.options + } + + /// The `User-Agent` requests carry. Prepend to [`crate::USER_AGENT`] rather than replacing it, + /// so a server still sees which client is calling. + pub fn user_agent(mut self, user_agent: impl Into) -> Self { + self.options.user_agent = Some(user_agent.into()); + self + } + + /// The local address to bind connections to. + pub fn local_address(mut self, address: IpAddr) -> Self { + self.options.local_address = Some(address.to_string()); + self + } + + /// Add a default header, sent on every request that does not set one of that name itself. + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.options.headers.get_or_insert_default().push(Header { + name: name.into(), + value: value.into(), + sensitive: None, + }); + self + } + + /// Add a default header, marking it sensitive so it is kept out of logs and HPACK indexes. + pub fn sensitive_header(mut self, name: impl Into, value: impl Into) -> Self { + self.options.headers.get_or_insert_default().push(Header { + name: name.into(), + value: value.into(), + sensitive: Some(true), + }); + self + } + + /// What to do with a redirect response. + pub fn redirect(mut self, policy: RedirectPolicy) -> Self { + self.options.redirect = Some(policy); + self + } + + /// Keep a cookie jar, enforcing these limits. Without this call the agent stores no cookies. + #[cfg(feature = "cookies")] + pub fn cookies(mut self, limits: CookieLimits) -> Self { + self.options.cookies = Some(limits); + self + } + + /// Resolver settings. + pub fn dns(mut self, with: impl FnOnce(DnsBuilder) -> DnsBuilder) -> Self { + let group = self.options.dns.take().unwrap_or_default(); + self.options.dns = Some(with(DnsBuilder { group }).group); + self + } + + /// TLS settings. + pub fn tls(mut self, with: impl FnOnce(TlsBuilder) -> TlsBuilder) -> Self { + let group = self.options.tls.take().unwrap_or_default(); + self.options.tls = Some(with(TlsBuilder { group }).group); + self + } + + /// Connection pool settings. + pub fn pool(mut self, with: impl FnOnce(PoolBuilder) -> PoolBuilder) -> Self { + let group = self.options.pool.take().unwrap_or_default(); + self.options.pool = Some(with(PoolBuilder { group }).group); + self + } + + /// Request timeouts. + pub fn timeout(mut self, with: impl FnOnce(TimeoutBuilder) -> TimeoutBuilder) -> Self { + let group = self.options.timeout.take().unwrap_or_default(); + self.options.timeout = Some(with(TimeoutBuilder { group }).group); + self + } + + /// HTTP cache settings. Without this call the agent does not cache. + #[cfg(feature = "cache")] + pub fn cache(mut self, with: impl FnOnce(CacheBuilder) -> CacheBuilder) -> Self { + let group = self.options.cache.take().unwrap_or_default(); + self.options.cache = Some(with(CacheBuilder { group }).group); + self + } + + /// Flow-control windows applied to both protocols, unless one overrides them. + pub fn flow_control( + mut self, + with: impl FnOnce(FlowControlBuilder) -> FlowControlBuilder, + ) -> Self { + let group = self.options.flow_control.take().unwrap_or_default(); + self.options.flow_control = Some(with(FlowControlBuilder { group }).group); + self + } + + /// HTTP/2 settings. + pub fn http2(mut self, with: impl FnOnce(Http2Builder) -> Http2Builder) -> Self { + let group = self.options.http2.take().unwrap_or_default(); + self.options.http2 = Some(with(Http2Builder { group }).group); + self + } + + /// HTTP/3 settings. + #[cfg(feature = "http3")] + pub fn http3(mut self, with: impl FnOnce(Http3Builder) -> Http3Builder) -> Self { + let group = self.options.http3.take().unwrap_or_default(); + self.options.http3 = Some(with(Http3Builder { group }).group); + self + } + + /// Departures from standard behaviour, each opted into deliberately. + pub fn quirks(mut self, with: impl FnOnce(QuirksBuilder) -> QuirksBuilder) -> Self { + let group = self.options.quirks.take().unwrap_or_default(); + self.options.quirks = Some(with(QuirksBuilder { group }).group); + self + } +} + +/// Resolver settings. Reached through [`AgentBuilder::dns`]. +#[derive(Debug, Default)] +#[must_use] +pub struct DnsBuilder { + group: DnsOptions, +} + +impl DnsBuilder { + /// Resolve through the operating system rather than Faith's own resolver. + #[cfg(feature = "dns")] + pub fn system(mut self, system: bool) -> Self { + self.group.system = Some(system); + self + } + + /// Resolve `domain` to these addresses without asking a server. + pub fn r#override( + mut self, + domain: impl Into, + addresses: impl IntoIterator>, + ) -> Self { + self.group + .overrides + .get_or_insert_default() + .push(DnsOverride { + domain: domain.into(), + addresses: addresses.into_iter().map(Into::into).collect(), + }); + self + } + + /// The resolvers to query, in order. Each is a URL whose scheme picks the transport. + #[cfg(feature = "dns")] + pub fn servers(mut self, servers: impl IntoIterator>) -> Self { + self.group.servers = Some(servers.into_iter().map(Into::into).collect()); + self + } + + /// Bound resolution across the whole server list. + #[cfg(feature = "dns")] + pub fn timeout(mut self, timeout: Duration) -> Self { + self.group.timeout = Some(millis(timeout)); + self + } + + /// Suffixes to try for an unqualified name. + #[cfg(feature = "dns")] + pub fn search_domains(mut self, domains: impl IntoIterator>) -> Self { + self.group.search_domains = Some(domains.into_iter().map(Into::into).collect()); + self + } + + /// How many dots a name must contain before it is tried as given, ahead of the search list. + #[cfg(feature = "dns")] + pub fn ndots(mut self, ndots: u32) -> Self { + self.group.ndots = Some(ndots); + self + } + + /// Consult the system hosts file. + #[cfg(feature = "dns")] + pub fn hosts_file(mut self, hosts_file: bool) -> Self { + self.group.hosts_file = Some(hosts_file); + self + } + + /// Names to send to the system resolver whatever the rest of the configuration says. + #[cfg(feature = "dns")] + pub fn exempt_domains(mut self, domains: impl IntoIterator>) -> Self { + self.group.exempt_domains = Some(domains.into_iter().map(Into::into).collect()); + self + } + + /// Serve an expired answer while a fresh lookup runs. + #[cfg(feature = "dns")] + pub fn serve_stale(mut self, serve_stale: bool) -> Self { + self.group.serve_stale = Some(serve_stale); + self + } + + /// How far past expiry an answer may still be served. + #[cfg(feature = "dns")] + pub fn max_stale(mut self, max_stale: Duration) -> Self { + self.group.max_stale = Some(millis(max_stale)); + self + } +} + +/// TLS settings. Reached through [`AgentBuilder::tls`]. +#[derive(Debug, Default)] +#[must_use] +pub struct TlsBuilder { + group: TlsOptions, +} + +impl TlsBuilder { + /// Send early data on resumed connections, which trades a round trip for replayability. + pub fn early_data(mut self, early_data: bool) -> Self { + self.group.early_data = Some(early_data); + self + } + + /// A PEM certificate and private key to present as a client certificate. + pub fn identity(mut self, pem: impl Into>) -> Self { + self.group.identity = Some(pem.into()); + self + } + + /// Trust this PEM root on top of the platform's trust store. + pub fn extra_root(mut self, pem: impl Into>) -> Self { + self.group + .extra_roots + .get_or_insert_default() + .push(pem.into()); + self + } + + /// Refuse a connection that cannot be made over TLS. + pub fn required(mut self, required: bool) -> Self { + self.group.required = Some(required); + self + } +} + +/// Connection pool settings. Reached through [`AgentBuilder::pool`]. +#[derive(Debug, Default)] +#[must_use] +pub struct PoolBuilder { + group: PoolOptions, +} + +impl PoolBuilder { + /// How long a connection may sit idle before it is closed. + pub fn idle_timeout(mut self, idle_timeout: Duration) -> Self { + self.group.idle_timeout = Some(secs(idle_timeout)); + self + } + + /// Most idle connections to keep per host. + pub fn max_idle_per_host(mut self, max: u32) -> Self { + self.group.max_idle_per_host = Some(max); + self + } +} + +/// Request timeouts. Reached through [`AgentBuilder::timeout`]. +#[derive(Debug, Default)] +#[must_use] +pub struct TimeoutBuilder { + group: TimeoutOptions, +} + +impl TimeoutBuilder { + /// Bound the connect phase alone. + pub fn connect(mut self, timeout: Duration) -> Self { + self.group.connect = Some(millis(timeout)); + self + } + + /// Bound each read. + pub fn read(mut self, timeout: Duration) -> Self { + self.group.read = Some(millis(timeout)); + self + } + + /// Bound the whole request and response. + pub fn total(mut self, timeout: Duration) -> Self { + self.group.total = Some(millis(timeout)); + self + } +} + +/// HTTP cache settings. Reached through [`AgentBuilder::cache`]. +#[cfg(feature = "cache")] +#[derive(Debug, Default)] +#[must_use] +pub struct CacheBuilder { + group: CacheOptions, +} + +#[cfg(feature = "cache")] +impl CacheBuilder { + /// Where cached responses are kept. + pub fn store(mut self, store: CacheStore) -> Self { + self.group.store = Some(store); + self + } + + /// For an in-memory store, how many entries to keep. + pub fn capacity(mut self, capacity: u32) -> Self { + self.group.capacity = Some(capacity); + self + } + + /// The default cache mode for requests that name none. + pub fn mode(mut self, mode: CacheMode) -> Self { + self.group.mode = Some(mode); + self + } + + /// For a disk store, the directory to keep it in. + pub fn path(mut self, path: impl Into) -> Self { + self.group.path = Some(path.into()); + self + } + + /// Behave as a shared cache rather than a private one. + pub fn shared(mut self, shared: bool) -> Self { + self.group.shared = Some(shared); + self + } +} + +/// Flow-control windows. Reached through [`AgentBuilder::flow_control`]. +#[derive(Debug, Default)] +#[must_use] +pub struct FlowControlBuilder { + group: FlowControlOptions, +} + +impl FlowControlBuilder { + /// Bytes an origin may send on any one stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } +} + +/// HTTP/2 settings. Reached through [`AgentBuilder::http2`]. +#[derive(Debug, Default)] +#[must_use] +pub struct Http2Builder { + group: Http2Options, +} + +impl Http2Builder { + /// Bytes an origin may send on any one HTTP/2 stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one HTTP/2 connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } + + /// Let the windows size themselves from what the connection is actually doing. + pub fn adaptive_window(mut self, adaptive: bool) -> Self { + self.group.adaptive_window = Some(adaptive); + self + } +} + +/// HTTP/3 settings. Reached through [`AgentBuilder::http3`]. +#[cfg(feature = "http3")] +#[derive(Debug, Default)] +#[must_use] +pub struct Http3Builder { + group: Http3Options, +} + +#[cfg(feature = "http3")] +impl Http3Builder { + /// The congestion controller QUIC runs. + pub fn congestion(mut self, congestion: Http3Congestion) -> Self { + self.group.congestion = Some(congestion); + self + } + + /// Inactivity to accept before timing out a connection. Rounded down to whole seconds, and + /// capped at what the setting can carry. + pub fn max_idle_timeout(mut self, timeout: Duration) -> Self { + self.group.max_idle_timeout = Some(timeout.as_secs().try_into().unwrap_or(u8::MAX)); + self + } + + /// Whether an origin advertising HTTP/3 is upgraded to it at all. + pub fn upgrade_enabled(mut self, enabled: bool) -> Self { + self.group.upgrade_enabled = Some(enabled); + self + } + + /// Verify an advertisement with a background probe rather than on the next request. + pub fn upgrade_probe(mut self, probe: bool) -> Self { + self.group.upgrade_probe = Some(probe); + self + } + + /// Ceiling on how long a background probe may take. + pub fn upgrade_probe_timeout(mut self, timeout: Duration) -> Self { + self.group.upgrade_probe_timeout = Some(millis(timeout)); + self + } + + /// Demote an origin off HTTP/3 when its QUIC path is slower than its TCP one by this factor. + pub fn upgrade_slow_factor(mut self, factor: f64) -> Self { + self.group.upgrade_slow_factor = Some(factor); + self + } + + /// How long a path-time demotion holds before the origin is re-evaluated. + pub fn upgrade_slow_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_slow_ttl = Some(secs(ttl)); + self + } + + /// How long to cache an advertisement before the first HTTP/3 attempt. + pub fn upgrade_advertised_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_advertised_ttl = Some(secs(ttl)); + self + } + + /// How long to cache a confirmed working HTTP/3 connection. + pub fn upgrade_confirmed_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_confirmed_ttl = Some(secs(ttl)); + self + } + + /// How long a first failed attempt blocks an origin. + pub fn upgrade_failed_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_failed_ttl = Some(secs(ttl)); + self + } + + /// Ceiling on the cooldown consecutive failures double out to. + pub fn upgrade_failed_max_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_failed_max_ttl = Some(secs(ttl)); + self + } + + /// How many consecutive cancelled attempts demote an origin. + pub fn upgrade_cancel_strikes(mut self, strikes: u32) -> Self { + self.group.upgrade_cancel_strikes = Some(strikes); + self + } + + /// Ceiling on how long an attempt may take to resolve before it is given up on. + pub fn upgrade_attempt_timeout(mut self, timeout: Duration) -> Self { + self.group.upgrade_attempt_timeout = Some(millis(timeout)); + self + } + + /// Connect to a port an origin advertised, rather than only to its own. + pub fn upgrade_follow_advertised_port(mut self, follow: bool) -> Self { + self.group.upgrade_follow_advertised_port = Some(follow); + self + } + + /// How many origins to track in the Alt-Svc cache. + pub fn upgrade_cache_capacity(mut self, capacity: u32) -> Self { + self.group.upgrade_cache_capacity = Some(capacity); + self + } + + /// Treat HTTP/3 as available at this host and port without waiting for an advertisement. + pub fn hint(mut self, host: impl Into, port: u16) -> Self { + self.group.hints.get_or_insert_default().push(Http3Hint { + host: host.into(), + port, + }); + self + } + + /// Bytes an origin may send on any one HTTP/3 stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one HTTP/3 connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } + + /// Bytes to transmit without acknowledgement, bounding upload memory. + pub fn send_window(mut self, bytes: u32) -> Self { + self.group.send_window = Some(bytes); + self + } +} + +/// Departures from standard behaviour. Reached through [`AgentBuilder::quirks`]. +#[derive(Debug, Default)] +#[must_use] +pub struct QuirksBuilder { + group: QuirksOptions, +} + +impl QuirksBuilder { + /// Send a streaming request body over HTTP/1.x, which the fetch standard otherwise reserves to + /// HTTP/2 and HTTP/3. + pub fn h1_request_streaming(mut self, allow: bool) -> Self { + self.group.h1_request_streaming = Some(allow); + self + } +} diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs new file mode 100644 index 0000000..8af0bff --- /dev/null +++ b/crates/web-faith/src/client.rs @@ -0,0 +1,510 @@ +//! Building the agent's reqwest clients from validated options. +//! +//! The recipe here is what lets a client be rebuilt: `network_changed` has to drop the connection +//! pool, and reqwest offers no way to do that short of dropping the client, so building one is a +//! pure function of settings that were validated once. + +// spec:NETCHG + +use std::{ + net::{IpAddr, SocketAddr}, + time::Duration, +}; + +// Reached only by the pieces the agent shares into a rebuilt client, each behind its own feature. +#[cfg(any(feature = "cookies", feature = "http3"))] +use std::sync::Arc; + +use http::header::HeaderMap; + +#[cfg(feature = "cache")] +mod http_cache; + +#[cfg(feature = "cache")] +pub use http_cache::{HttpCacheRecipe, HttpCacheStore}; + +#[cfg(feature = "cache")] +use http_cache_reqwest::{Cache, HttpCache}; +use reqwest::{Client, Identity, redirect::Policy, tls::Certificate}; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +#[cfg(feature = "cookies")] +use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] +use web_faith_dns::FaithResolver; + +#[cfg(feature = "dns")] +use crate::retry::StaleAddressRetry; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, AltSvcMiddleware, H3Prober}; + +use crate::{ + error::{FaithError, FaithErrorKind}, + retry::DeadConnectionRetry, +}; + +#[cfg(feature = "http3")] +use crate::timing::HeadersStamp; + +/// What to do with a redirect response. +/// +/// The Node surface spells these as fetch's own `redirect` values; this is the same choice in the +/// client's own terms. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum RedirectPolicy { + /// Follow redirects, up to the standard's limit. + #[default] + Follow, + /// Refuse a redirect, reporting it as an error. + Error, + /// Return the redirect response itself rather than following it. + Stop, +} + +/// Per-stream receive window applied to both protocols when nothing overrides it. +/// +/// Chrome's shape: 6 MiB stream inside a 15 MiB connection. Picked over a larger window that +/// measured faster because it is what browsers have proven at scale, and because a pooled +/// server-side client multiplies per-connection memory across far more connections. +// spec:FLOW +pub const DEFAULT_STREAM_WINDOW: u32 = 6 * 1024 * 1024; + +/// Whole-connection receive window applied to both protocols when nothing overrides it. +// spec:FLOW +pub const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; + +// Concurrent streams share the connection's headroom, so the asymmetry is the point of the +// defaults rather than an accident of the numbers (spec:FLOW#common-windows). +const _: () = assert!(DEFAULT_CONNECTION_WINDOW > DEFAULT_STREAM_WINDOW); + +/// The flow-control windows to apply, once the common group, the per-protocol overrides, and the +/// defaults have been reconciled. +// spec:FLOW#per-protocol-windows +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResolvedWindows { + pub stream: u32, + pub connection: u32, +} + +/// What the Node.js networking environment variables asked for, to apply to a reqwest client +/// builder as Node.js honours them for its own clients. This is read for every agent, so +/// `fetch()` behaves like Node's built-in fetch out of the box. +/// +/// - `NODE_EXTRA_CA_CERTS`: a path to a PEM file whose certificates are added to +/// the trust store on top of the platform roots. As in Node.js, a value that +/// is empty, or points at a file that cannot be read or parsed, is ignored +/// rather than fatal, unlike an explicitly configured extra root, which is an +/// error. Certificates load in addition to any configured explicitly. +/// +/// - `NODE_TLS_REJECT_UNAUTHORIZED`: when set to exactly `"0"`, TLS certificate +/// validation is disabled for the agent. This is insecure and exists only to +/// match Node.js semantics; any other value leaves validation enabled. +/// +/// - `NODE_USE_ENV_PROXY`: when set to exactly `"0"`, the agent ignores the +/// ambient proxy configuration (`HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` and the +/// OS proxy settings) that reqwest reads by default. Unlike Node.js — where +/// env-proxy support is opt-*in* and off by default — faith reads it by +/// default and treats this variable purely as an opt-*out* switch, so leaving +/// it unset (or `"1"`) keeps the existing always-on behaviour. +/// +/// `NODE_USE_SYSTEM_CA` is deliberately not honoured: faith bundles no Mozilla +/// root set, so its only default trust source is the platform store the variable +/// would toggle. `=0` could therefore only mean "trust almost nothing", which is +/// never what a caller wants, so the platform store is always used. +/// +/// Read once, at construction: these are layered on top of the explicit options when the agent is +/// built, so a client rebuilt later replays what was read then rather than picking up an environment +/// that has changed since. +// spec:NETCHG +#[derive(Debug, Clone, Default)] +pub struct NodeEnvRecipe { + extra_ca_certs: Vec, + accept_invalid_certs: bool, + no_proxy: bool, +} + +impl NodeEnvRecipe { + pub fn read() -> Self { + let mut recipe = Self::default(); + + if let Ok(path) = std::env::var("NODE_EXTRA_CA_CERTS") + && !path.is_empty() + && let Ok(bytes) = std::fs::read(&path) + && let Ok(certs) = Certificate::from_pem_bundle(&bytes) + { + recipe.extra_ca_certs = certs; + } + + recipe.accept_invalid_certs = + std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").as_deref() == Ok("0"); + recipe.no_proxy = std::env::var("NODE_USE_ENV_PROXY").as_deref() == Ok("0"); + + recipe + } + + fn apply(&self, mut client: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + if !self.extra_ca_certs.is_empty() { + client = client.tls_certs_merge(self.extra_ca_certs.iter().cloned()); + } + + if self.accept_invalid_certs { + client = client.danger_accept_invalid_certs(true); + } + + if self.no_proxy { + client = client.no_proxy(); + } + + client + } +} + +/// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not +/// here: it belongs to the agent and outlives any one client. +// spec:NETCHG +#[cfg(feature = "http3")] +#[derive(Debug, Clone)] +pub struct H3UpgradeRecipe { + pub enabled: bool, + pub attempt_timeout: Option, + pub probe: bool, + pub probe_timeout: Option, +} + +/// Everything needed to build the agent's clients, validated once up front. +/// +/// A client has to be buildable more than once: dropping the connection pool means dropping the +/// client, which is what a network change asks for, so what the client is built from has to outlive +/// any one of them. Options are validated once into these fields, and building a client is then a +/// pure function of them and the agent's shared state. +// spec:NETCHG +#[derive(Debug, Clone)] +pub struct ClientRecipe { + pub user_agent: String, + pub local_address: Option, + pub default_headers: Option, + /// Under the system resolver no hickory resolver is installed at all. + // spec:DNS + pub dns_system: bool, + /// Validated at construction, and applied whichever resolver is in use: reqwest layers + /// overrides on top of the resolver it was given. + // spec:DNS#overrides + pub dns_overrides: Vec<(String, Vec)>, + pub http2_adaptive_window: bool, + /// `None` when adaptive windowing owns the windows itself. + // spec:FLOW#adaptive-windowing + pub http2_windows: Option, + #[cfg(feature = "http3")] + pub http3_max_idle_timeout: Duration, + #[cfg(feature = "http3")] + pub http3_windows: ResolvedWindows, + #[cfg(feature = "http3")] + pub http3_congestion_bbr: bool, + #[cfg(feature = "http3")] + pub http3_send_window: Option, + pub pool_idle_timeout: Option, + pub pool_max_idle_per_host: Option, + pub redirect: Option, + pub connect_timeout: Option, + pub read_timeout: Option, + pub total_timeout: Option, + /// Only reachable over QUIC, so only applied when HTTP/3 is compiled in. + #[cfg(feature = "http3")] + pub tls_early_data: Option, + pub tls_identity: Option, + pub tls_required: Option, + pub tls_extra_roots: Vec, + pub node_env: NodeEnvRecipe, + #[cfg(feature = "cache")] + pub http_cache: Option, + #[cfg(feature = "http3")] + pub h3_upgrade: H3UpgradeRecipe, +} + +/// Point the resolver's `HTTPS` record reading at the upgrade layer, so a record advertising +/// `alpn="h3"` makes an origin probe-worthy before anything has connected to it. +/// +/// A no-op without all the parts: the system resolver is not Faith's to add a query to, and with +/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one. +/// +/// Re-called on a network change, where the prober is rebuilt with the client it sends on. +// spec:DNS#https-records +#[cfg(all(feature = "http3", feature = "dns"))] +pub fn install_https_sink( + dns_resolver: Option<&FaithResolver>, + alt_svc_cache: Option<&Arc>, + prober: Option<&Arc>, + upgrade_enabled: bool, +) { + if !upgrade_enabled { + return; + } + let (Some(resolver), Some(cache)) = (dns_resolver, alt_svc_cache) else { + return; + }; + resolver.set_https_sink(Arc::new(web_faith_alt_svc::H3HttpsSink::new( + Arc::clone(cache), + prober, + ))); +} + +/// The clients [`ClientRecipe::build`] produces, and the prober that sends on them. +pub struct BuiltClients { + pub client: ClientWithMiddleware, + pub raw_client: Client, + #[cfg(feature = "http3")] + pub prober: Option>, +} + +/// Install ring as the process's rustls crypto provider. +/// +/// reqwest reads the process default when it builds a client and panics if there is none, so this +/// runs before the first one is built. Only where ring is the chosen backend: with `tls-aws-lc-rs` +/// also on, reqwest supplies aws-lc-rs itself, which is also what an HTTP/3 build needs. Installing +/// is process-wide and once-only, so a provider the embedding program put in place is left alone. +#[cfg(all(feature = "tls-ring", not(feature = "tls-aws-lc-rs")))] +fn install_crypto_provider() { + use std::sync::Once; + + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + +impl ClientRecipe { + /// The window an idle pooled connection lives in, which is also how long a warm-up counts as + /// warm and how long a connection stays listed. + // spec:POOL + // spec:WARM + // spec:OBS + pub fn conn_timeout(&self) -> Duration { + // reqwest's own default, mirrored because the pool timeout it applies is not readable. + self.pool_idle_timeout.unwrap_or(Duration::from_secs(90)) + } + + /// Build a fresh client and raw client, around state the agent already holds. + /// + /// Everything passed in survives a rebuild by being shared rather than rebuilt: the cookie + /// jar, the resolver (and so its cache), and the HTTP/3 origin knowledge all belong to the + /// agent rather than to any one client. + // spec:NETCHG#what-the-signal-keeps + pub fn build( + &self, + #[cfg(feature = "cookies")] cookie_jar: Option<&Arc>, + #[cfg(feature = "dns")] dns_resolver: Option<&FaithResolver>, + #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, + ) -> Result { + #[cfg(all(feature = "tls-ring", not(feature = "tls-aws-lc-rs")))] + install_crypto_provider(); + + let mut client = Client::builder() + .tls_info(true) + .tls_sslkeylogfile(true) + .user_agent(self.user_agent.clone()); + + if let Some(ip) = self.local_address { + client = client.local_address(ip); + } + + #[cfg(feature = "cookies")] + if let Some(jar) = cookie_jar { + client = client.cookie_provider(jar.clone()); + } + + // Registered whichever resolver is in use: reqwest layers overrides on top of the + // resolver it was given, so they take effect under the system resolver too + // (spec:DNS#overrides). + for (domain, addresses) in &self.dns_overrides { + client = client.resolve_to_addrs(domain, addresses); + } + + #[cfg(feature = "dns")] + if self.dns_system { + client = client.no_hickory_dns(); + } else if let Some(resolver) = dns_resolver { + client = client.dns_resolver(resolver.clone()); + } + + if let Some(headers) = &self.default_headers { + client = client.default_headers(headers.clone()); + } + + if self.http2_adaptive_window { + client = client.http2_adaptive_window(true); + } else if let Some(windows) = self.http2_windows { + client = client + .http2_initial_stream_window_size(windows.stream) + .http2_initial_connection_window_size(windows.connection); + } + + #[cfg(feature = "http3")] + { + client = client + .http3_max_idle_timeout(self.http3_max_idle_timeout) + .http3_stream_receive_window(self.http3_windows.stream.into()) + .http3_conn_receive_window(self.http3_windows.connection.into()); + + if self.http3_congestion_bbr { + client = client.http3_congestion_bbr(); + } + + if let Some(send_window) = self.http3_send_window { + client = client.http3_send_window(send_window.into()); + } + } + + if let Some(timeout) = self.pool_idle_timeout { + client = client.pool_idle_timeout(Some(timeout)); + } + + if let Some(max_idle) = self.pool_max_idle_per_host { + client = client.pool_max_idle_per_host(max_idle); + } + + match self.redirect { + // follow is the default, and we ignore manual + None | Some(RedirectPolicy::Follow) => {} + Some(RedirectPolicy::Error) => { + client = client.redirect(Policy::custom(|attempt| { + // Hand reqwest the error unboxed: it boxes for us, and boxing first would + // put a `Box` in the source chain, which does not downcast + // back to `FaithError` when we come to recover the kind as a `code`. + attempt.error(FaithError::from(FaithErrorKind::Redirect)) + })); + } + Some(RedirectPolicy::Stop) => { + client = client.redirect(Policy::none()); + } + } + + if let Some(timeout) = self.connect_timeout { + client = client.connect_timeout(timeout); + } + + if let Some(timeout) = self.read_timeout { + client = client.read_timeout(timeout); + } + + if let Some(timeout) = self.total_timeout { + client = client.timeout(timeout); + } + + #[cfg(feature = "http3")] + if let Some(early_data) = self.tls_early_data { + client = client.tls_early_data(early_data); + } + + if let Some(identity) = &self.tls_identity { + client = client.identity(identity.clone()); + } + + if let Some(https_only) = self.tls_required { + client = client.https_only(https_only); + } + + if !self.tls_extra_roots.is_empty() { + client = client.tls_certs_merge(self.tls_extra_roots.iter().cloned()); + } + + client = self.node_env.apply(client); + + let raw_client = client + .build() + .map_err(|e| FaithError::new(FaithErrorKind::Config, Some(format!("{e:?}"))))?; + let mut client = ClientBuilder::new(raw_client.clone()); + + #[cfg(feature = "http3")] + let prober = { + // The prober sends on the *raw* client, deliberately: it must skip + // the HTTP cache (a replayed cached response would fake a + // confirmation) and the Alt-Svc middleware (no recursion), while + // sharing the h3 connection pool so a successful probe leaves a warm + // connection for the foreground. Only built when both the upgrade + // machinery and probing are on. + alt_svc_cache + .filter(|_| self.h3_upgrade.enabled && self.h3_upgrade.probe) + .map(|cache| { + Arc::new(H3Prober::new( + raw_client.clone(), + cache.clone(), + self.h3_upgrade.probe_timeout, + )) + }) + }; + + #[cfg(feature = "cache")] + if let Some(cache) = &self.http_cache { + // The two arms differ only in the manager's type, which `HttpCache` is generic over, + // so they cannot share a constructor without boxing the manager. + client = match &cache.store { + HttpCacheStore::Disk(manager) => client.with(Cache(HttpCache { + mode: cache.mode, + manager: manager.clone(), + options: cache.options.clone(), + })), + HttpCacheStore::Memory(manager) => client.with(Cache(HttpCache { + mode: cache.mode, + manager: manager.clone(), + options: cache.options.clone(), + })), + }; + } + + // Registered *after* the HTTP cache, so the Alt-Svc layer sits inside it: + // `reqwest-middleware` runs the first-registered middleware outermost. Being + // inside matters three times over. + // + // A cache hit is served without calling inward, so it never reaches this + // layer. From outside, it would: `http-cache` rebuilds a cached response with + // the *stored* HTTP version, so a response cached from an HTTP/3 exchange + // replays as HTTP/3 and would be taken for a live one — confirming HTTP/3, + // clearing cancellation strikes and refreshing the confirmed TTL on evidence + // that never touched the network. + // + // The cache middleware also buffers the whole response body inside its own + // call inward. From outside, the HTTP/3 attempt guarded here would span that + // buffering, so a cancellation during body download would count as a strike, + // and `upgradeAttemptTimeout` would bound body transfer rather than the wait + // for response headers. + // + // And cache keys are computed before this layer runs, so an advertised-port + // rewrite cannot split HTTP/3 and TCP responses across separate entries. + #[cfg(feature = "http3")] + if let Some(alt_svc_cache) = alt_svc_cache { + client = client.with(AltSvcMiddleware::::new( + alt_svc_cache.clone(), + self.h3_upgrade.enabled, + self.h3_upgrade.attempt_timeout, + prober.clone(), + )); + } + + // Outside the dead-connection layer, so a re-resolved attempt gets the same + // treatment as the original one: the two answer different questions, and a + // fresh address deserves its own chance to draw a dead pooled connection. + // Inside the Alt-Svc and cache layers for the reason given below. + #[cfg(feature = "dns")] + { + client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); + } + + // Registered last, so it sits innermost and wraps nothing but the exchange + // itself. Inside the Alt-Svc layer rather than outside it, because each + // protocol attempt is its own connection and deserves its own retry: a + // failed HTTP/3 attempt is the fallback's business, and re-running the + // upgrade decision from out here would re-attempt HTTP/3 on a path already + // judged dead and record a second failure against the origin for it. Inside + // the HTTP cache for the same reason as the Alt-Svc layer -- a retry should + // re-send the request, not redo the cache lookup that led to it. + client = client.with(DeadConnectionRetry); + + Ok(BuiltClients { + client: client.build(), + raw_client, + #[cfg(feature = "http3")] + prober, + }) + } +} diff --git a/crates/web-faith/src/client/http_cache.rs b/crates/web-faith/src/client/http_cache.rs new file mode 100644 index 0000000..7ba917c --- /dev/null +++ b/crates/web-faith/src/client/http_cache.rs @@ -0,0 +1,25 @@ +//! What an HTTP cache is installed from, held so a rebuilt client keeps the same store. + +use http_cache_reqwest::{CACacheManager, CacheMode, HttpCacheOptions, MokaManager}; + +/// The HTTP cache store to install on a client, held as the built manager rather than as the +/// options that produced it. +/// +/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and +/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while +/// building a fresh one from the same options would empty an in-memory cache — which is why a +/// client rebuilt for a network change clones this. +// spec:NETCHG#what-the-signal-keeps +#[derive(Debug, Clone)] +pub enum HttpCacheStore { + Disk(CACacheManager), + Memory(MokaManager), +} + +/// The HTTP cache middleware to install. +#[derive(Debug, Clone)] +pub struct HttpCacheRecipe { + pub mode: CacheMode, + pub options: HttpCacheOptions, + pub store: HttpCacheStore, +} diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs new file mode 100644 index 0000000..83f6631 --- /dev/null +++ b/crates/web-faith/src/error.rs @@ -0,0 +1,231 @@ +use std::{ + error::Error, + fmt::{Debug, Display}, +}; + +use strum::{EnumIter, IntoEnumIterator}; + +/// The kind of a [`FaithError`], which is also the stable code the error reports. +/// +/// Callers match on the kind rather than on the message: the kind is the API, and the message is +/// for humans. Every kind here is reachable, each one naming a failure some request can produce. +/// +/// This is the one definition of the set, on either surface. The Node surface hands JavaScript the +/// codes through [`error_codes`], which reads them from here, so the exported `ERROR_CODES` map and +/// the errors themselves cannot drift apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)] +pub enum FaithErrorKind { + Aborted, + AddressParse, + BodyStream, + Closed, + Config, + ContentLengthOverrun, + FileExists, + FileWrite, + IntegrityMismatch, + InvalidCompression, + InvalidHeader, + InvalidIntegrity, + InvalidMethod, + InvalidPath, + InvalidUrl, + JsonParse, + Network, + PemParse, + Redirect, + ResponseAlreadyDisturbed, + ResponseBodyNull, + Timeout, +} + +impl FaithErrorKind { + /// The stable name callers match on. + pub fn code(self) -> String { + format!("{self:?}") + } + + pub(crate) fn default_message(self) -> &'static str { + match self { + Self::Aborted => "the request was aborted", + Self::AddressParse => "invalid IP address and/or port", + Self::BodyStream => "internal response body stream copy error", + Self::Closed => "the agent has been closed", + Self::Config => "invalid agent configuration", + Self::ContentLengthOverrun => "response body exceeded the advertised Content-Length", + Self::FileExists => "the destination file already exists", + Self::FileWrite => "could not write the destination file", + Self::IntegrityMismatch => "resource integrity check failed", + Self::InvalidCompression => "invalid request body compression", + Self::InvalidHeader => "invalid header name or value", + Self::InvalidIntegrity => "invalid integrity value", + Self::InvalidMethod => "invalid HTTP method", + Self::InvalidPath => "destination does not name a local path", + Self::InvalidUrl => "invalid URL", + Self::JsonParse => "invalid json in response body", + Self::Network => "network error", + Self::PemParse => "invalid client certificate or key", + Self::Redirect => "got a redirect", + Self::ResponseAlreadyDisturbed => "response body already disturbed", + Self::ResponseBodyNull => "response cannot carry a body to write", + Self::Timeout => "timed out", + } + } +} + +/// Every error code the library reports, in declaration order. +/// +/// The Node surface exports this as `ERROR_CODES`; generating it from the kinds themselves is what +/// keeps the exported map and the errors from drifting. +pub fn error_codes() -> Vec { + FaithErrorKind::iter().map(FaithErrorKind::code).collect() +} + +#[derive(Debug, Clone)] +pub struct FaithError { + pub kind: FaithErrorKind, + pub message: Option, +} + +impl FaithError { + pub fn new(kind: FaithErrorKind, message: Option>) -> Self { + Self { + kind, + message: message.map(|m| m.into()), + } + } +} + +impl From for FaithError { + fn from(kind: FaithErrorKind) -> Self { + Self { + kind, + message: None, + } + } +} + +/// Dig a [`FaithError`] back out of an error chain, if one is in there. +/// +/// The `error` redirect policy refuses a redirect by handing reqwest a [`FaithError`], which comes +/// back to us wrapped in an error of reqwest's own, so the kind we chose has to be recovered from +/// the source chain to survive as a `code`. Redirect failures reqwest raises on its own account +/// (exhausting the hop limit, an https-only downgrade) carry no [`FaithError`] and so fall through +/// to the generic mapping, which is what tells the two apart. +fn faith_kind_in_chain(err: &(dyn Error + 'static)) -> Option { + let mut source = err.source(); + while let Some(e) = source { + if let Some(faith) = e.downcast_ref::() { + return Some(faith.kind); + } + source = e.source(); + } + + None +} + +/// A conversion that cannot fail still has to satisfy the bound on a target, and this is how it +/// does: there is no value to convert. +impl From for FaithError { + fn from(never: std::convert::Infallible) -> Self { + match never {} + } +} + +impl From for FaithError { + fn from(err: reqwest::Error) -> Self { + // Always include full error chain for debugging + let mut msg = format!("{err:?}"); + let mut source = err.source(); + while let Some(e) = source { + msg.push_str(&format!(" -> {e:?}")); + source = e.source(); + } + + if err.is_timeout() { + return FaithError::new(FaithErrorKind::Timeout, Some(msg)); + } + + // A redirect the agent's own policy refused carries the kind we handed reqwest; one reqwest + // raised on its own account stays a plain network error. + let kind = err + .is_redirect() + .then(|| faith_kind_in_chain(&err)) + .flatten() + .unwrap_or(FaithErrorKind::Network); + + FaithError::new(kind, Some(msg)) + } +} + +impl From for FaithError { + fn from(err: reqwest_middleware::Error) -> Self { + match err { + reqwest_middleware::Error::Middleware(err) => { + FaithError::new(FaithErrorKind::Network, Some(err.to_string())) + } + reqwest_middleware::Error::Reqwest(err) => err.into(), + } + } +} + +impl Error for FaithError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } + + fn description(&self) -> &str { + "description() is deprecated; use Display" + } + + fn cause(&self) -> Option<&dyn Error> { + self.source() + } +} + +impl Display for FaithError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:?}: {}", + self.kind, + self.message + .as_deref() + .unwrap_or_else(|| self.kind.default_message()) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_code_is_distinct_and_named() { + let codes = error_codes(); + let unique: std::collections::BTreeSet<_> = codes.iter().collect(); + assert_eq!(unique.len(), codes.len(), "two kinds report the same code"); + assert!(codes.iter().all(|code| !code.is_empty())); + } + + #[test] + fn a_message_is_prefixed_with_the_code_it_reports() { + for kind in FaithErrorKind::iter() { + let code = kind.code(); + let rendered = FaithError::from(kind).to_string(); + assert!( + rendered.starts_with(&format!("{code}: ")), + "{rendered} does not lead with {code}" + ); + } + } + + #[test] + fn a_kind_without_a_message_falls_back_to_its_own() { + let err = FaithError::from(FaithErrorKind::Closed); + assert_eq!(err.to_string(), "Closed: the agent has been closed"); + + let err = FaithError::new(FaithErrorKind::Closed, Some("gone")); + assert_eq!(err.to_string(), "Closed: gone"); + } +} diff --git a/src/integrity.rs b/crates/web-faith/src/integrity.rs similarity index 74% rename from src/integrity.rs rename to crates/web-faith/src/integrity.rs index 7506c4f..59a56e9 100644 --- a/src/integrity.rs +++ b/crates/web-faith/src/integrity.rs @@ -1,7 +1,22 @@ +//! Subresource Integrity parsing and verification. +//! +//! An integrity value names one or more digests a resource is expected to match; the resource is +//! good if it matches any one of them, and digests naming an algorithm too weak to trust are +//! ignored rather than honoured. Verification comes in two shapes: [`verify_integrity`] for a body +//! already in memory, and [`integrity_checker`] with [`finish_integrity`] for one being read as it +//! arrives. +//! +//! This is always built. It is small enough that leaving it out saves nothing worth measuring, and a +//! caller who asks for a digest to be checked is owed the check. + +// spec:SRI + use ssri::{Integrity, IntegrityChecker}; use crate::error::{FaithError, FaithErrorKind}; +/// Algorithm names are compared case-insensitively, the digest itself being base64 and so not ours +/// to touch. fn normalize_integrity(integrity: &str) -> String { integrity .split_whitespace() @@ -26,6 +41,9 @@ fn parse_integrity(integrity: &str) -> Result { }) } +/// Verify data that is already in hand. +/// +/// An absent or blank value has nothing to verify and so passes. pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> { if integrity.trim().is_empty() { return Ok(()); @@ -33,7 +51,7 @@ pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> parse_integrity(integrity)? .check(data) - .map_err(|_| FaithErrorKind::IntegrityMismatch)?; + .map_err(|_| FaithError::from(FaithErrorKind::IntegrityMismatch))?; Ok(()) } @@ -41,7 +59,7 @@ pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> /// Build a streaming integrity checker for a body read that hashes as it goes. /// /// `None` when there is nothing to verify (no value, or an empty one), matching -/// [`verify_integrity`]. A malformed value is rejected up front with `InvalidIntegrity`, +/// [`verify_integrity`]. A malformed value is rejected up front with an invalid-integrity error, /// before any of the body is touched. Feed each chunk to the returned checker with /// [`IntegrityChecker::input`] and finish with [`finish_integrity`]. pub fn integrity_checker(integrity: Option<&str>) -> Result, FaithError> { @@ -55,12 +73,12 @@ pub fn integrity_checker(integrity: Option<&str>) -> Result Result<(), FaithError> { checker .result() .map(|_| ()) - .map_err(|_| FaithErrorKind::IntegrityMismatch.into()) + .map_err(|_| FaithError::from(FaithErrorKind::IntegrityMismatch)) } #[cfg(test)] @@ -94,10 +112,7 @@ mod tests { let integrity = "sha256-wronghashvalue"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err().kind, - FaithErrorKind::IntegrityMismatch - )); + assert_eq!(result.unwrap_err().kind, FaithErrorKind::IntegrityMismatch); } #[test] @@ -113,10 +128,7 @@ mod tests { let integrity = "sha256-wronghash1 sha256-wronghash2"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err().kind, - FaithErrorKind::IntegrityMismatch - )); + assert_eq!(result.unwrap_err().kind, FaithErrorKind::IntegrityMismatch); } #[test] diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs new file mode 100644 index 0000000..5855b7d --- /dev/null +++ b/crates/web-faith/src/lib.rs @@ -0,0 +1,69 @@ +//! A browser-shaped HTTP client: fetch semantics over a Rust network stack. +//! +//! Faith behaves like a browser wherever that translates to a server-side runtime: transparent +//! HTTP/2 and HTTP/3, Happy Eyeballs across IPv4 and IPv6, DNS caching, an optional cookie jar, and +//! HTTP caching. The subsystems beneath it are published on their own, and each can be left out of a +//! build with the feature named for it. +//! +//! Whichever layer a request fails in, the failure arrives as one [`FaithError`] whose +//! [`FaithErrorKind`] is the stable code to match on: a component crate names its own errors, and +//! they are converted at the boundary as they cross into the client. +//! +//! ```no_run +//! # use web_faith::agent::Agent; +//! # async fn example() -> Result<(), web_faith::FaithError> { +//! let agent = Agent::new()?; +//! let body = agent.fetch("https://example.com/").await?.text().await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! An [`Agent`] owns the connection pool, resolver, cookie jar, and caches; [`Agent::builder`] +//! configures one. Cloning an agent is cheap and every clone names the same one. +//! +//! [`Agent::fetch`] returns a builder that sends when awaited, so there is no separate send step. +//! [`Request`] prepares one without sending it, to adjust at each call site or send unchanged on +//! more than one agent. +//! +//! [`Agent`]: agent::Agent +//! [`Agent::builder`]: agent::Agent::builder +//! [`Agent::fetch`]: agent::Agent::fetch +//! [`Request`]: request::Request + +// A build with no crypto provider cannot speak TLS, and an HTTPS client that cannot is not one. +// Selecting a provider is therefore a choice between the two rather than an option to decline. +#[cfg(not(any(feature = "tls-aws-lc-rs", feature = "tls-ring")))] +compile_error!("web-faith needs a TLS backend: enable either tls-aws-lc-rs or tls-ring"); + +pub mod agent; +pub mod body; +pub mod builder; +pub mod client; +pub mod error; +pub mod integrity; +pub mod options; +pub mod request; +pub mod response; +pub mod retry; +pub mod stats; +pub mod timing; +pub mod warm_up; + +/// The `User-Agent` a request carries when nothing overrides it. +/// +/// Prepend your own product token to it rather than replacing it, so a server still sees which +/// client is calling: +/// +/// ``` +/// # use web_faith::USER_AGENT; +/// let ua = format!("YourApp/1.2.3 {USER_AGENT}"); +/// assert!(ua.ends_with(USER_AGENT)); +/// ``` +pub const USER_AGENT: &str = concat!( + "Faith/", + env!("CARGO_PKG_VERSION"), + " reqwest/", + env!("REQWEST_VERSION") +); + +pub use error::{FaithError, FaithErrorKind, error_codes}; diff --git a/crates/web-faith/src/options.rs b/crates/web-faith/src/options.rs new file mode 100644 index 0000000..edb290b --- /dev/null +++ b/crates/web-faith/src/options.rs @@ -0,0 +1,770 @@ +//! The options an agent is built from, one struct per group. +//! +//! [`Agent::from_options`] validates these into the recipe an agent's clients are built from, and +//! settles the defaults for anything left unset. Both surfaces go through it, so a default is +//! decided once rather than once per surface. +//! +//! [`Agent::from_options`]: crate::agent::Agent::from_options + +// spec:AGENT + +use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; + +#[cfg(feature = "cookies")] +use web_faith_cookies::CookieLimits; + +use crate::client::{ + DEFAULT_CONNECTION_WINDOW, DEFAULT_STREAM_WINDOW, RedirectPolicy, ResolvedWindows, +}; + +/// Settings related to the HTTP cache. This is a nested object. +#[cfg(feature = "cache")] +#[derive(Clone, Debug, Default)] +pub struct CacheOptions { + /// Which cache store to use: either `disk` or `memory`. + /// + /// Default: none (cache disabled). + pub store: Option, + /// If `cache.store: "memory"`, the maximum amount of items stored. + /// + /// Default: 10_000. + pub capacity: Option, + /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if + /// no cache mode is set on a request. + /// + /// Default: `"default"`. + pub mode: Option, + /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. + /// + /// Required if `cache.store: "disk"`. + pub path: Option, + /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is + /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a + /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). + /// `shared: true` is required for proxies and multi-user caches. + /// + /// Default: true. + pub shared: Option, +} + +#[cfg(feature = "cache")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CacheStore { + Disk, + + Memory, +} + +#[derive(Clone, Debug, Default)] +pub struct DnsOverride { + pub domain: String, + pub addresses: Vec, +} + +/// Settings related to DNS. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct DnsOptions { + /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on + /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native + /// fetch does, this should be your first port of call. + /// + /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory + /// DNS cache, and may lead to worse performance even discounting the cache. + /// + /// Default: false. + /// + /// [Hickory]: https://hickory-dns.org/ + #[cfg(feature = "dns")] + pub system: Option, + /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. + /// + /// Will throw if addresses are in invalid formats. You may provide a port number as part of the + /// address, it will default to port 0 otherwise, which will select the conventional port for the + /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port + /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively + /// blocks that domain from this agent. + /// + /// Default: no overrides. + pub overrides: Option>, + /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to + /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over + /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on + /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the + /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. + /// + /// The encrypted transports always authenticate the resolver. A URL fragment names the + /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates + /// against the hostname; a bare-IP host authenticates against the address itself. + /// + /// Servers are queried in order, a later one reached only once those before it fail. Setting + /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or + /// its scheme is not one of the above, and combining it with `dns.system` throws. + /// + /// Default: system discovery. + #[cfg(feature = "dns")] + pub servers: Option>, + /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead + /// servers costs a single timeout rather than one per server. + /// + /// Default: 5000. + #[cfg(feature = "dns")] + pub timeout: Option, + /// Replace the system's search list, the domains appended to a name that is not fully + /// qualified. Independent of `dns.servers`. + /// + /// Default: the system's search list. + #[cfg(feature = "dns")] + pub search_domains: Option>, + /// How many dots a name must contain before it is tried as given, ahead of the search list. + /// Independent of `dns.servers`. + /// + /// Default: the system's setting. + #[cfg(feature = "dns")] + pub ndots: Option, + /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. + /// + /// Default: platform convention. + #[cfg(feature = "dns")] + pub hosts_file: Option, + /// Further domains to exempt from the configured or encrypted resolver, for the internal + /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's + /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. + /// + /// Default: no extra exemptions. + #[cfg(feature = "dns")] + pub exempt_domains: Option>, + /// Serve an expired cache entry immediately and refresh it in the background, rather than making + /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is + /// almost always still correct, and a connect failure against one that has moved re-resolves and + /// attempts the request again. + /// + /// Set `false` for an agent that must never connect to an address it knows to be out of date: an + /// expired entry is discarded and the lookup blocks on a fresh answer. + /// + /// Default: true. + #[cfg(feature = "dns")] + pub serve_stale: Option, + /// How far past expiry an answer may still be served, in milliseconds. An entry older than this + /// is discarded rather than served: an answer stale enough stops being evidence about where the + /// host is, and a refresh still failing after that long is the case where the address most likely + /// did change. + /// + /// Default: 3600000 (one hour). + #[cfg(feature = "dns")] + pub max_stale: Option, +} + +/// Sets the default headers for every request. +/// +/// If header names or values are invalid, they are silently omitted. +/// Sensitive headers (e.g. `Authorization`) should be marked. +/// +/// Default: none. +#[derive(Clone, Debug, Default)] +pub struct Header { + pub name: String, + pub value: String, + pub sensitive: Option, +} + +#[cfg(feature = "http3")] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Http3Congestion { + #[default] + Cubic, + + Bbr1, +} + +/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc +/// cache so the first request to this host will attempt HTTP/3 immediately. +#[cfg(feature = "http3")] +#[derive(Clone, Debug, Default)] +pub struct Http3Hint { + /// The hostname (e.g., "example.com"). + pub host: String, + /// The port number (e.g., 443). + pub port: u16, +} + +/// Settings related to HTTP/3. This is a nested object. +#[cfg(feature = "http3")] +#[derive(Clone, Debug, Default)] +pub struct Http3Options { + /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the + /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with + /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by + /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise + /// bandwidth use and optimises for round-trip time, while ignoring packet loss. + /// + /// In some networks, BBR can lead to pathological degradation of overall network conditions, by + /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, + /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). + /// + /// [2]: https://github.com/quinn-rs/quinn/issues/1254 + /// + /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. + pub congestion: Option, + /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that + /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ + /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith + /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). + /// + /// Default: 30. + pub max_idle_timeout: Option, + /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc + /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. + /// + /// Default: true. + pub upgrade_enabled: Option, + /// Whether advertised HTTP/3 endpoints are verified with a background probe + /// before any foreground request is routed to them. + /// + /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say + /// there is UDP connectivity between you and it. Without probing, the next + /// request after an advertisement attempts HTTP/3 inline, and on a silently + /// broken UDP path it stalls until the QUIC idle timeout or + /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per + /// failure cooldown for as long as the path stays broken. + /// + /// With probing (the default), requests keep using TCP until a background + /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the + /// connection pool, so the first upgraded request rides the probe's warm + /// connection. A broken path costs one failed background request per + /// cooldown and no foreground latency at all. + /// + /// The probe is a synthetic request the server will see in its logs. Set + /// this to `false` to restore the inline upgrade if that is unacceptable + /// (per-request billing, easily-alarmed WAFs). + /// + /// `hints` are exempt either way: a hint is your own assertion, so the first + /// request to a hinted origin speaks HTTP/3 immediately, which is also what + /// makes h3-only origins (no TCP listener) work. + /// + /// Default: true. + pub upgrade_probe: Option, + /// Ceiling on how long a background HTTP/3 probe may take before the origin + /// is treated as failed, in **milliseconds**. + /// + /// This bounds background work only — no foreground request ever waits on a + /// probe — so it can afford to be generous: a healthy handshake plus HEAD + /// completes in one or two round trips. Set to 0 to leave probes bounded + /// only by the QUIC idle timeout. + /// + /// Default: 5000 (5 seconds). + pub upgrade_probe_timeout: Option, + /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than + /// its TCP path by this factor. Set to 0 to disable path-time demotion. + /// + /// Faith keeps a per-origin moving average of time-to-response-headers for + /// each protocol family. HTTP/3 is preferred at parity and when moderately + /// slower — its advantages (no head-of-line blocking, connection migration) + /// pay off beyond the average — so this factor should stay well above 1. + /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC + /// average must also exceed the TCP one by an absolute 10ms so LAN-fast + /// origins don't flap on noise. + /// + /// A demoted origin is not treated as broken: it re-enters through a + /// background probe after `upgradeSlowTtl`, asking whether the path has + /// improved at zero foreground cost. + /// + /// Default: 2.5. + pub upgrade_slow_factor: Option, + /// How long (in seconds) a path-time demotion holds before the origin is + /// re-evaluated. See `upgradeSlowFactor`. + /// + /// Default: 600 (10 minutes). + pub upgrade_slow_ttl: Option, + /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. + /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. + /// + /// Default: 86400 (24 hours). + pub upgrade_advertised_ttl: Option, + /// How long (in seconds) to cache a confirmed working HTTP/3 connection. + /// + /// Default: 86400 (24 hours). + pub upgrade_confirmed_ttl: Option, + /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this + /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends + /// Alt-Svc headers. + /// + /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an + /// origin whose UDP path is blocked for good is retried less and less often instead of + /// forever at this interval. A confirmed HTTP/3 response ends the run. + /// + /// Default: 300 (5 minutes). + pub upgrade_failed_ttl: Option, + /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of + /// `upgradeFailedTtl`. + /// + /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, + /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat + /// cooldown that never backs off. + /// + /// Default: 3600 (1 hour). + pub upgrade_failed_max_ttl: Option, + /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, + /// demote an origin back to TCP. + /// + /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request + /// cancelled via `AbortSignal` never produces that signal, so without this an + /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as + /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a + /// sustained run of them demotes the origin, and any successful HTTP/3 response + /// resets the count. + /// + /// Strikes must land within about a minute of each other to count towards a + /// run. A retry loop whose backoff exceeds that window never accumulates one, + /// so callers with a long backoff should set this to 1 for immediate demotion + /// on the first cancelled attempt. + /// + /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that + /// carries small datagrams but drops full-size ones (an MTU blackhole, say). + /// Response headers still arrive, so the attempt resolves and every mechanism + /// here counts it a success — the transfer then stalls partway through the + /// body, where nothing is watching. `maxIdleTimeout` or the request's own + /// timeout is what ends such a request, and the origin stays on HTTP/3. + /// + /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. + /// + /// Default: 3. + pub upgrade_cancel_strikes: Option, + /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is + /// given up on and the request is retried over TCP, in **milliseconds**. + /// + /// Note the unit: the other `upgrade*` settings are in seconds, but this one + /// is in milliseconds to match the `timeout` settings, because useful values + /// are sub-second. + /// + /// This bounds the wait for response headers, not the response body, so a slow + /// body is unaffected. + /// + /// The default is high, but not unconditionally inert: `maxIdleTimeout` is + /// configurable up to 120 seconds, and above 60 seconds this deadline becomes + /// the effective ceiling. Even below that, "QUIC's own idle timeout fires + /// first" only holds while the connection is idle — a transfer still running + /// past this deadline keeps the connection active, so no idle timeout is + /// coming to end it. + /// + /// On expiry the request is retried over TCP, which means it is re-sent: a + /// timeout often means the server is still processing, so a slow + /// non-idempotent request (a POST, say) can end up delivered twice. Lowering + /// this value trades that double-submission risk for faster recovery when a + /// UDP path breaks. Anyone setting it low should confirm their slowest + /// legitimate time to response headers fits well inside the budget. + /// + /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle + /// timeout and the request's own timeout. + /// + /// Default: 60000 (60 seconds). + pub upgrade_attempt_timeout: Option, + /// Connect to the port a server advertises HTTP/3 on, even when it differs from + /// the origin's own port. **This is not standards-compliant**; it is off by + /// default. + /// + /// An `Alt-Svc` advertisement names a network endpoint for the origin, so + /// honouring one correctly means connecting to that endpoint while still + /// sending the *origin's* authority. reqwest cannot express that — it derives + /// the HTTP/3 connect target from the request URI's authority (tracked + /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). + /// So by default Faith does not upgrade at all when the advertised port + /// differs, rather than guessing that the origin's own port also speaks + /// HTTP/3. + /// + /// Setting this to `true` upgrades anyway, by rewriting the request's port to + /// the advertised one. That gets HTTP/3 working today against servers you + /// control, at the cost of three deviations you should be aware of: + /// + /// - The request's `Host`/`:authority` carries the advertised port instead of + /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) + /// forbids. Servers that route on authority may misroute or reject; servers + /// that ignore it are unaffected. + /// - `response.url` reports the port actually connected to. + /// - `redirected` ignores port differences, since the rewritten port would + /// otherwise look like a redirect on every request. + /// + /// TLS is unaffected: certificates are still validated against the origin's + /// hostname. Only the port changes. + /// + /// Default: `false`. + pub upgrade_follow_advertised_port: Option, + /// Maximum number of origins to track in the Alt-Svc cache. + /// + /// Default: 10000. + pub upgrade_cache_capacity: Option, + /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache + /// on agent initialization, so the first request to these hosts will attempt HTTP/3. + pub hints: Option>, + /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/3 only. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload + /// throughput the way the receive windows bound download. The origin's own flow control + /// applies on top of this, so it is a ceiling rather than a grant. + /// + /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window + /// the peer advertises, with no local cap to set. + /// + /// Default: 10 MB (quinn's own default). + pub send_window: Option, +} + +/// Settings related to HTTP/2. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct Http2Options { + /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Replace HTTP/2's static windows with windows that start small and grow towards a + /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. + /// + /// This is off by default, and turning it on is usually the wrong move. A fresh connection + /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample + /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and + /// carries *less* throughput than the static window for all but the largest transfers. It + /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. + /// + /// Its one real advantage is memory: it holds a large window open only on connections that + /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling + /// buys the same throughput from the first byte. + /// + /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. + /// + /// Default: `false`. + pub adaptive_window: Option, +} + +/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct FlowControlOptions { + /// Maximum bytes an origin may send on any one stream before it must wait for Faith to + /// acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. + /// The default follows browser practice, and is deliberately at the conservative end of it: + /// a pooled server-side client can hold many connections across many origins, so + /// per-connection memory multiplies harder here than in a browser. + /// + /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. + /// + /// Default: 6 MiB. + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one connection before it must + /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// This is larger than `streamWindow` so concurrent streams on one connection share the + /// connection's headroom, while still bounding the worst-case buffering of a connection + /// carrying many concurrent requests. + /// + /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against + /// the other. + /// + /// Default: 15 MiB. + pub connection_window: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct PoolOptions { + /// How many seconds of inactivity before a connection is closed. + /// + /// Default: 90 seconds. + pub idle_timeout: Option, + /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed + /// to keep the idle connections (per host) under that number. + /// + /// Default: `null` (no limit). + pub max_idle_per_host: Option, +} + +/// Switches that depart from standard behaviour on purpose. This is a nested object. +/// +/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule +/// forbids. All of them are off by default, so an agent constructed with no options is +/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise +/// established that what the rule guards against does not apply to them: turning one on means +/// requests may fail against origins that expect the standard behaviour. +#[derive(Clone, Copy, Debug, Default)] +pub struct QuirksOptions { + /// Allow a streaming request body to be sent over an HTTP/1.x connection. + /// + /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read + /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x + /// origin or an intermediary on the path may refuse it. With this on, such a body sends over + /// whichever protocol the connection negotiates. + /// + /// Default: false. + pub h1_request_streaming: Option, +} + +/// Timeouts for requests made with this agent. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct TimeoutOptions { + /// Set a timeout for only the connect phase, in milliseconds. + /// + /// Default: none. + pub connect: Option, + /// Set a timeout for read operations, in milliseconds. + /// + /// The timeout applies to each read operation, and resets after a successful read. This is more + /// appropriate for detecting stalled connections when the size isn't known beforehand. + /// + /// Default: none. + pub read: Option, + /// Set a timeout for the entire request-response cycle, in milliseconds. + /// + /// The timeout applies from when the request starts connecting until the response body has finished. + /// Also considered a total deadline. + /// + /// Default: none. + pub total: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct TlsOptions { + /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet + /// of application data alongside the opening packet of the TLS handshake. That can enable the server + /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant + /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should + /// really only be used for static assets or to squeeze out the last drop of performance for endpoints + /// that are replay-safe. + /// + /// Default: false. + pub early_data: Option, + /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also + /// called mutual TLS or mTLS) authentication. + /// + /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The + /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options + /// that will cause the `Agent` constructor to throw if the input is in the wrong format. + pub identity: Option>, + /// Disables plain-text HTTP. + /// + /// Default: false. + pub required: Option, + /// Additional PEM-formatted root certificates to trust, on top of the platform's + /// trust store. Each entry may be a PEM bundle containing multiple certificates. + /// + /// This is mainly useful for connecting to servers with self-signed or private-CA + /// certificates, such as internal services or local test servers. This is one of the + /// few options that will cause the `Agent` constructor to throw if the input is in + /// the wrong format. + pub extra_roots: Option>>, +} + +#[derive(Clone, Debug, Default)] +pub struct AgentOptions { + /// Settings related to the HTTP cache. This is a nested object. + #[cfg(feature = "cache")] + pub cache: Option, + /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and + /// included in additional requests. + /// + /// `true` enables the store with the default limits; an options object enables it and tunes them, + /// so `{}` means the same as `true`. + /// + /// Default: `false`. + /// + /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add + /// and retrieve cookies from the store. + #[cfg(feature = "cookies")] + pub cookies: Option, + /// Settings related to DNS. This is a nested object. + pub dns: Option, + /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. + /// + /// Setting these is the normal way to tune windows: one value applies to whichever protocol + /// a request negotiates, so throughput doesn't change when an origin upgrades from one to + /// the other. The `http2` and `http3` groups override them per protocol. + pub flow_control: Option, + /// Sets the default headers for every request. + /// + /// If header names or values are invalid, they are silently omitted. + /// Sensitive headers (e.g. `Authorization`) should be marked. + /// + /// Default: none. + pub headers: Option>, + /// Settings related to HTTP/2. This is a nested object. + pub http2: Option, + /// Settings related to HTTP/3. This is a nested object. + #[cfg(feature = "http3")] + pub http3: Option, + /// Bind outgoing sockets to this local IP address before connecting. + /// + /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that + /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — + /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and + /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to + /// force a specific source address. Throws if the value does not parse as an IP address. + /// + /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). + pub local_address: Option, + /// Settings related to the connection pool. This is a nested object. + pub pool: Option, + /// Switches that depart from standard behaviour on purpose. This is a nested object. + pub quirks: Option, + /// Determines the behavior in case the server replies with a redirect status. + pub redirect: Option, + /// Timeouts for requests made with this agent. This is a nested object. + pub timeout: Option, + /// Settings related to the connection pool. This is a nested object. + pub tls: Option, + /// Custom user agent string. + /// + /// Default: `Faith/{version} reqwest/{version}`. + pub user_agent: Option, +} + +/// Whether this host can bind the IPv6 wildcard (`[::]`). +/// +/// This is tested using the exact operation reqwest performs when creating the QUIC +/// endpoint with no explicit local address, so it predicts whether the default +/// QUIC bind will succeed. The result is memoised for the life of the process; while +/// IPv6 bindability can in principle change at runtime, this is considered an +/// acceptable tradeoff for performance and simplicity. +pub fn ipv6_wildcard_bindable() -> bool { + use std::sync::OnceLock; + static BINDABLE: OnceLock = OnceLock::new(); + *BINDABLE.get_or_init(|| { + UdpSocket::bind(SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0)).is_ok() + }) +} + +/// Reconcile one protocol's windows: its own setting wins over the common one, which wins over the +/// default (spec:FLOW#per-protocol-windows). +pub fn resolve_windows( + common: Option<&FlowControlOptions>, + protocol_stream: Option, + protocol_connection: Option, +) -> ResolvedWindows { + ResolvedWindows { + stream: protocol_stream + .or_else(|| common.and_then(|c| c.stream_window)) + .unwrap_or(DEFAULT_STREAM_WINDOW), + connection: protocol_connection + .or_else(|| common.and_then(|c| c.connection_window)) + .unwrap_or(DEFAULT_CONNECTION_WINDOW), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::ResolvedWindows; + + fn common(stream: Option, connection: Option) -> FlowControlOptions { + FlowControlOptions { + stream_window: stream, + connection_window: connection, + } + } + + #[test] + fn windows_fall_back_to_the_defaults() { + // An agent configured with nothing at all still gets the large static windows + // (spec:FLOW#common-windows). + assert_eq!( + resolve_windows(None, None, None), + ResolvedWindows { + stream: 6 * 1024 * 1024, + connection: 15 * 1024 * 1024, + } + ); + } + + #[test] + fn the_common_windows_apply_when_a_protocol_says_nothing() { + assert_eq!( + resolve_windows(Some(&common(Some(1024), Some(4096))), None, None), + ResolvedWindows { + stream: 1024, + connection: 4096, + } + ); + } + + #[test] + fn a_protocol_window_beats_the_common_one() { + // The whole point of the per-protocol group: tune one protocol against the other + // (spec:FLOW#per-protocol-windows). + assert_eq!( + resolve_windows( + Some(&common(Some(1024), Some(4096))), + Some(2048), + Some(8192) + ), + ResolvedWindows { + stream: 2048, + connection: 8192, + } + ); + } + + #[test] + fn each_window_falls_back_on_its_own() { + // Overriding the stream window for one protocol leaves that protocol's connection + // window on the common value, rather than dropping it to the default. + assert_eq!( + resolve_windows(Some(&common(Some(1024), Some(4096))), Some(2048), None), + ResolvedWindows { + stream: 2048, + connection: 4096, + } + ); + assert_eq!( + resolve_windows(Some(&common(None, None)), None, Some(8192)), + ResolvedWindows { + stream: DEFAULT_STREAM_WINDOW, + connection: 8192, + } + ); + } + + #[test] + fn a_protocol_window_applies_without_the_common_group() { + assert_eq!( + resolve_windows(None, Some(2048), None), + ResolvedWindows { + stream: 2048, + connection: DEFAULT_CONNECTION_WINDOW, + } + ); + } + + #[test] + fn the_two_protocols_resolve_independently() { + // One `flowControl` value covers both protocols, and overriding it for HTTP/3 leaves + // HTTP/2 where it was (spec:FLOW#per-protocol-windows). + let flow = common(Some(1024), Some(4096)); + let http2 = resolve_windows(Some(&flow), None, None); + let http3 = resolve_windows(Some(&flow), Some(2048), None); + + assert_eq!(http2.stream, 1024); + assert_eq!(http3.stream, 2048); + assert_eq!(http2.connection, http3.connection); + } +} diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs new file mode 100644 index 0000000..09a8c63 --- /dev/null +++ b/crates/web-faith/src/request.rs @@ -0,0 +1,64 @@ +//! Sending a request, and building the response that comes back. + +// spec:REQ spec:ENC spec:CANCEL + +mod builder; +mod send; +mod target; + +pub use builder::{FetchBuilder, Priority, Request, RequestBuilder}; +pub use send::send; +pub use target::Target; + +use std::{pin::Pin, time::Duration}; + +use bytes::Bytes; +use futures::Stream; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; + +/// Whether a request carries its credentials, and how far. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Credentials { + /// Strip credentials from the URL and send no cookies. + Omit, + /// Send them, which is what a server-side caller almost always means. + #[default] + Include, +} + +/// The methods the fetch standard normalises to upper case; any other method is sent as given. +// spec:REQ#method-and-headers +const NORMALISED_METHODS: [&str; 6] = ["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]; + +/// The header a request's priority is expressed in. +// spec:REQ#request-priority +pub const PRIORITY: &str = "priority"; + +/// A request body, as the caller has it. +pub enum RequestBody { + /// No body. + None, + /// A body already in hand, whose length can be declared up front. + Bytes(Bytes), + /// A body arriving in chunks, which goes out chunked because it has no length to declare. + Stream(Pin> + Send>>), +} + +/// What a request carries beyond its method, URL, and body. +#[derive(Clone, Debug, Default)] +pub struct RequestOptions { + #[cfg(feature = "cache")] + pub cache: CacheMode, + /// A coding to compress the body in, named by its wire token. + #[cfg(feature = "encoding")] + pub compress: Option, + pub credentials: Credentials, + pub headers: Option>, + pub integrity: Option, + pub method: Option, + /// The `Priority` header value this request's priority derives, if it derives one. + pub priority: Option<&'static str>, + pub timeout: Option, +} diff --git a/crates/web-faith/src/request/builder.rs b/crates/web-faith/src/request/builder.rs new file mode 100644 index 0000000..87e6f00 --- /dev/null +++ b/crates/web-faith/src/request/builder.rs @@ -0,0 +1,435 @@ +use std::{ + future::{Future, IntoFuture}, + pin::Pin, + time::Duration, +}; + +use bytes::Bytes; +use futures::Stream; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; +use reqwest::{ + Method, + header::{HeaderName, HeaderValue}, +}; +use url::Url; + +use crate::{ + agent::Agent, + error::{FaithError, FaithErrorKind}, + request::{Credentials, RequestBody, RequestOptions, Target, send::send}, + response::Response, +}; + +/// A header the caller set or removed on one layer. +#[derive(Clone, Debug)] +struct HeaderOp { + name: String, + /// `None` removes the name, including whatever the layers beneath contributed for it. + value: Option, +} + +impl std::fmt::Debug for RequestBody { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::None => f.write_str("None"), + Self::Bytes(bytes) => f.debug_tuple("Bytes").field(&bytes.len()).finish(), + Self::Stream(_) => f.write_str("Stream"), + } + } +} + +/// A request prepared but not sent. +/// +/// Inert: it carries no agent, so it can be sent on more than one, and passing it to +/// [`Agent::fetch`] returns a builder that layers over it. +// spec:REQ +#[derive(Debug)] +pub struct Request { + pub(super) url: Url, + pub(super) options: RequestOptions, + pub(super) body: RequestBody, +} + +impl Request { + /// Prepare a request aimed at `target`. + pub fn new(target: T) -> RequestBuilder + where + T: TryInto, + T::Error: Into, + { + RequestBuilder { + layer: Layer::over(target), + } + } + + pub fn url(&self) -> &Url { + &self.url + } + + pub fn options(&self) -> &RequestOptions { + &self.options + } + + /// Copy the request, when its body allows it. + /// + /// `None` when the body is a stream, a stream being consumable once. + // spec:REQ + pub fn try_clone(&self) -> Option { + let body = match &self.body { + RequestBody::None => RequestBody::None, + RequestBody::Bytes(bytes) => RequestBody::Bytes(bytes.clone()), + RequestBody::Stream(_) => return None, + }; + + Some(Self { + url: self.url.clone(), + options: self.options.clone(), + body, + }) + } +} + +/// One layer of settings over a target, which both builders are made of. +struct Layer { + /// The first failure met while setting up, held until the builder resolves. + // spec:REQ + target: Result, + options: RequestOptions, + /// Which single-valued settings this layer set explicitly, so an unset one is inherited rather + /// than overwritten with a default. + set: SetFlags, + headers: Vec, + body: Option, +} + +#[derive(Default)] +struct SetFlags { + #[cfg(feature = "cache")] + cache: bool, + #[cfg(feature = "encoding")] + compress: bool, + credentials: bool, + integrity: bool, + method: bool, + priority: bool, + timeout: bool, +} + +impl Layer { + /// Hold a failure until the builder resolves, keeping the first one met. + // spec:REQ + fn fail(&mut self, err: FaithError) { + if self.target.is_ok() { + self.target = Err(err); + } + } + + fn over(target: T) -> Self + where + T: TryInto, + T::Error: Into, + { + Self { + target: target.try_into().map_err(Into::into), + options: RequestOptions::default(), + set: SetFlags::default(), + headers: Vec::new(), + body: None, + } + } + + /// Settle this layer over whatever it wraps: the outermost explicit value wins, a setting this + /// layer does not touch is inherited unchanged, and headers merge by name. + // spec:REQ + fn settle(self) -> Result { + let (url, mut options, body) = match self.target? { + Target::Url(url) => (url, RequestOptions::default(), RequestBody::None), + // The URL comes from the target at the bottom of the stack. + Target::Request(inner) => (inner.url, inner.options, inner.body), + }; + + #[cfg(feature = "cache")] + if self.set.cache { + options.cache = self.options.cache; + } + #[cfg(feature = "encoding")] + if self.set.compress { + options.compress = self.options.compress; + } + if self.set.credentials { + options.credentials = self.options.credentials; + } + if self.set.integrity { + options.integrity = self.options.integrity; + } + if self.set.method { + options.method = self.options.method; + } + if self.set.priority { + options.priority = self.options.priority; + } + if self.set.timeout { + options.timeout = self.options.timeout; + } + + let mut headers = options.headers.take().unwrap_or_default(); + for op in self.headers { + headers.retain(|(name, _)| !name.eq_ignore_ascii_case(&op.name)); + if let Some(value) = op.value { + headers.push((op.name, value)); + } + } + options.headers = (!headers.is_empty()).then_some(headers); + + Ok(Request { + url, + options, + body: self.body.unwrap_or(body), + }) + } +} + +/// The setters both builders carry, so a call reads the same way whichever it is written against. +macro_rules! layer_setters { + ($builder:ident) => { + impl $builder { + /// The request method. Defaults to `GET`. + /// + /// Takes a `Method` or anything that converts into one; a value that does not is + /// reported where the builder resolves. + pub fn method(mut self, method: M) -> Self + where + M: TryInto, + { + match method.try_into() { + Ok(method) => { + self.layer.options.method = Some(method.to_string()); + self.layer.set.method = true; + } + Err(_) => self.layer.fail(FaithErrorKind::InvalidMethod.into()), + } + self + } + + /// Set a header, replacing whatever any layer beneath contributed for that name. + /// + /// Takes `HeaderName` and `HeaderValue` or anything that converts into them. + pub fn header(mut self, name: N, value: V) -> Self + where + N: TryInto, + V: TryInto, + { + match (name.try_into(), value.try_into()) { + (Ok(name), Ok(value)) => match value.to_str() { + Ok(value) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: Some(value.to_owned()), + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + }, + _ => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// Set several headers, each applied as a single header would be. + pub fn headers(mut self, headers: impl IntoIterator) -> Self + where + N: TryInto, + V: TryInto, + { + for (name, value) in headers { + self = self.header(name, value); + } + self + } + + /// Remove a header, including whatever the layers beneath contributed for it. + pub fn remove_header(mut self, name: N) -> Self + where + N: TryInto, + { + match name.try_into() { + Ok(name) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: None, + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// A body already in hand. + pub fn body(mut self, body: impl Into) -> Self { + self.layer.body = Some(RequestBody::Bytes(body.into())); + self + } + + /// A body arriving in chunks, which goes out chunked having no length to declare. + pub fn body_stream( + mut self, + body: impl Stream> + Send + 'static, + ) -> Self { + self.layer.body = Some(RequestBody::Stream(Box::pin(body))); + self + } + + /// Bound the whole request and response. + // spec:CANCEL + pub fn timeout(mut self, timeout: Duration) -> Self { + self.layer.options.timeout = Some(timeout); + self.layer.set.timeout = true; + self + } + + /// The digests the response body is expected to match. + // spec:SRI + pub fn integrity(mut self, integrity: impl Into) -> Self { + self.layer.options.integrity = Some(integrity.into()); + self.layer.set.integrity = true; + self + } + + /// Compress the request body in this coding, named by its wire token. + // spec:ENC + #[cfg(feature = "encoding")] + pub fn compress(mut self, coding: impl Into) -> Self { + self.layer.options.compress = Some(coding.into()); + self.layer.set.compress = true; + self + } + + /// How the HTTP cache is consulted for this request. + // spec:CACHE + #[cfg(feature = "cache")] + pub fn cache(mut self, mode: CacheMode) -> Self { + self.layer.options.cache = mode; + self.layer.set.cache = true; + self + } + + /// Whether the request carries its credentials. + pub fn credentials(mut self, credentials: Credentials) -> Self { + self.layer.options.credentials = credentials; + self.layer.set.credentials = true; + self + } + + /// How this request ranks against others, as an RFC 9218 urgency. + // spec:REQ#request-priority + pub fn priority(mut self, priority: Priority) -> Self { + self.layer.options.priority = priority.urgency(); + self.layer.set.priority = true; + self + } + } + }; +} + +/// How a request ranks against others. +// spec:REQ#request-priority +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Priority { + /// More urgent than an unmarked request. + High, + /// Less urgent than an unmarked request. + Low, + /// Send no `Priority` header, leaving the request at the scheme's default urgency. + #[default] + Auto, +} + +impl Priority { + fn urgency(self) -> Option<&'static str> { + match self { + Self::High => Some("u=1"), + Self::Low => Some("u=5"), + Self::Auto => None, + } + } +} + +/// Prepares a [`Request`] without sending it. +#[must_use = "a request builder does nothing until built"] +pub struct RequestBuilder { + layer: Layer, +} + +layer_setters!(RequestBuilder); + +impl RequestBuilder { + /// Settle the layers into a request. + /// + /// Where a conversion failed on the way in — an unparseable target, say — it is reported here. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +/// A request bound to an agent, which sends when awaited. +/// +/// There is no separate send step: awaiting is what sends. A builder dropped without being awaited +/// sends nothing, and dropping the future mid-flight cancels the request. +// spec:REQ spec:CANCEL +#[must_use = "a fetch builder sends nothing until awaited"] +pub struct FetchBuilder { + agent: Agent, + layer: Layer, +} + +layer_setters!(FetchBuilder); + +impl FetchBuilder { + /// Settle the layers into the request that would be sent, without sending it. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +impl IntoFuture for FetchBuilder { + type Output = Result; + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(async move { + // Taken before anything is awaited, so the request counts as in flight from here. + // spec:AGENT + let client = self.agent.client().ok_or(FaithErrorKind::Closed)?; + let request = self.layer.settle()?; + + send( + &self.agent, + client, + request.url.as_str(), + request.options, + request.body, + None::>, + ) + .await + }) + } +} + +impl Agent { + /// Aim a request at `target`, to send when awaited. + /// + /// The target is a URL, or a [`Request`] to layer over. Awaiting the builder sends the request; + /// see [`FetchBuilder`]. + // spec:REQ + pub fn fetch(&self, target: T) -> FetchBuilder + where + T: TryInto, + T::Error: Into, + { + FetchBuilder { + agent: self.clone(), + layer: Layer::over(target), + } + } +} + +#[cfg(test)] +#[cfg(test)] +mod tests; diff --git a/crates/web-faith/src/request/builder/tests.rs b/crates/web-faith/src/request/builder/tests.rs new file mode 100644 index 0000000..72a2c67 --- /dev/null +++ b/crates/web-faith/src/request/builder/tests.rs @@ -0,0 +1,197 @@ +use super::*; + +fn headers_of(request: &Request) -> Vec<(String, String)> { + request.options.headers.clone().unwrap_or_default() +} + +/// The outermost layer that set a value wins, whatever the layers beneath said. +#[test] +fn the_outermost_explicit_value_wins() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .timeout(Duration::from_secs(3)) + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(3))); +} + +/// A setting an outer layer does not touch is inherited unchanged. +#[test] +fn an_untouched_setting_is_inherited() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .method("POST") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .integrity("sha256-abc") + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(1))); + assert_eq!(outer.options.method.as_deref(), Some("POST")); + assert_eq!(outer.options.integrity.as_deref(), Some("sha256-abc")); +} + +/// The URL comes from the target at the bottom of the stack. +#[test] +fn wrapping_a_request_carries_its_url_through() { + let inner = Request::new("https://example.com/deep/path?q=1") + .build() + .expect("a valid target"); + let outer = Request::new(inner) + .method("HEAD") + .build() + .expect("layering"); + + assert_eq!(outer.url.as_str(), "https://example.com/deep/path?q=1"); +} + +/// Headers merge by name: an outer value replaces, a name only set inside carries through. +#[test] +fn headers_merge_by_name() { + let inner = Request::new("https://example.com/") + .header("x-keep", "inner") + .header("x-replace", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .header("x-replace", "outer") + .header("x-add", "outer") + .build() + .expect("layering over a request"); + + let mut headers = headers_of(&outer); + headers.sort(); + assert_eq!( + headers, + vec![ + ("x-add".to_owned(), "outer".to_owned()), + ("x-keep".to_owned(), "inner".to_owned()), + ("x-replace".to_owned(), "outer".to_owned()), + ] + ); +} + +/// Removing a name removes what the layers beneath contributed for it. +#[test] +fn removing_a_header_removes_what_is_underneath() { + let inner = Request::new("https://example.com/") + .header("x-gone", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .remove_header("X-Gone") + .build() + .expect("layering over a request"); + + assert!(headers_of(&outer).is_empty(), "the name was removed"); +} + +/// Setting the same thing twice on one builder is the same question at a smaller scale. +#[test] +fn the_later_call_wins_on_one_builder() { + let request = Request::new("https://example.com/") + .method("POST") + .method("PUT") + .build() + .expect("a valid target"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); +} + +/// An `http::Request` brings its method, URL, headers, and body across. +#[test] +fn an_http_request_is_a_target() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/submit") + .header("x-from", "ecosystem") + .body(Bytes::from_static(b"payload")) + .expect("a valid http request"); + + let request = Request::new(http).build().expect("it converts"); + + assert_eq!(request.url.as_str(), "https://example.com/submit"); + assert_eq!(request.options.method.as_deref(), Some("POST")); + assert_eq!( + headers_of(&request), + vec![("x-from".to_owned(), "ecosystem".to_owned())] + ); + assert!(request.try_clone().is_some(), "a buffered body copies"); +} + +/// A layer over an `http::Request` beats what it carried, as over any other target. +#[test] +fn a_layer_beats_what_an_http_request_carried() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/") + .body(Bytes::new()) + .expect("a valid http request"); + + let request = Request::new(http) + .method(http::Method::PUT) + .build() + .expect("layering over it"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); +} + +/// A header name that is not one is reported where the request is resolved too. +#[test] +fn an_invalid_header_surfaces_at_build() { + let err = Request::new("https://example.com/") + .header("not a header name", "value") + .build() + .expect_err("the name is not a header name"); + + assert_eq!(err.kind, FaithErrorKind::InvalidHeader); +} + +/// The first failure met is the one reported, not the last. +#[test] +fn the_first_failure_is_the_one_reported() { + let err = Request::new("not a url") + .header("not a header name", "value") + .build() + .expect_err("both are wrong"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); +} + +/// An unparseable target is reported where the request is resolved, not by the call that took it. +#[test] +fn an_unparseable_target_surfaces_at_build() { + let builder = Request::new("not a url"); + let err = builder.build().expect_err("the target does not parse"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); +} + +/// A request copies when its body allows it, and reports that it cannot when it does not. +#[test] +fn try_clone_copies_what_it_can() { + let buffered = Request::new("https://example.com/") + .body(Bytes::from_static(b"body")) + .build() + .expect("a valid target"); + assert!(buffered.try_clone().is_some()); + + let streamed = Request::new("https://example.com/") + .body_stream(futures::stream::empty()) + .build() + .expect("a valid target"); + assert!( + streamed.try_clone().is_none(), + "a stream is consumable once" + ); +} diff --git a/crates/web-faith/src/request/send.rs b/crates/web-faith/src/request/send.rs new file mode 100644 index 0000000..25b0f89 --- /dev/null +++ b/crates/web-faith/src/request/send.rs @@ -0,0 +1,453 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Instant, +}; + +use reqwest::{ + Method, StatusCode, + header::{CONTENT_ENCODING, HeaderName, HeaderValue}, + tls::TlsInfo, +}; +use reqwest_middleware::ClientWithMiddleware; + +#[cfg(feature = "connection-tracking")] +use hyper_util::client::legacy::connect::HttpInfo; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; + +#[cfg(feature = "encoding")] +use reqwest::header::ACCEPT_ENCODING; + +use tokio::sync::Mutex; +#[cfg(feature = "encoding")] +use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; + +use crate::{ + agent::Agent, + body::{Body, BodyHolder}, + error::{FaithError, FaithErrorKind}, + request::{Credentials, NORMALISED_METHODS, PRIORITY, RequestBody, RequestOptions}, + response::{PeerInformation, Response}, + timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, +}; + +/// Send a request on `agent`, and build the response it produces. +/// +/// `client` is the handle the caller took when the request was issued, rather than one taken here: +/// a request counts as in flight from the moment it is issued, so one issued just before the agent +/// closes runs to completion even though nothing had started on it yet. +/// +/// `abort` is an optional future that, resolving first, cancels the request. +// spec:AGENT +pub async fn send( + agent: &Agent, + client: ClientWithMiddleware, + url: &str, + options: RequestOptions, + body: RequestBody, + abort: Option>, +) -> Result { + let method = options.method.as_deref().unwrap_or("GET"); + // spec:REQ#method-and-headers + let method = NORMALISED_METHODS + .into_iter() + .find(|normalised| normalised.eq_ignore_ascii_case(method)) + .unwrap_or(method); + + let method = + Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; + let is_head = method == Method::HEAD; + + let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; + + // A `compress` naming no coding Faith can compress in is misuse whether or not the + // request turns out to carry a body, so it is refused before anything else looks at + // it (spec:ENC#compressing-a-request-body). + #[cfg(feature = "encoding")] + let compress = options + .compress + .as_deref() + .map(|value| { + Coding::from_option(value).ok_or_else(|| { + FaithError::new( + FaithErrorKind::InvalidCompression, + Some(format!( + "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" + )), + ) + }) + }) + .transpose()?; + + // Handle credentials based on credentials option + if options.credentials == Credentials::Omit { + // Remove credentials from URL if omit is specified + let _ = parsed_url.set_username(""); + let _ = parsed_url.set_password(None); + } + + // The stamp rides along in the request's extensions for the middleware to fill in; + // this side keeps a handle on it so the one measurement taken inside the stack is + // the one surfaced (spec:RESP#request-timing). + let headers_stamp = HeadersStamp::default(); + + let mut request = client + .request(method, parsed_url.clone()) + .with_extension(headers_stamp.clone()); + #[cfg(feature = "cache")] + { + request = request.with_extension(CacheMode::from(options.cache)); + } + + if let Some(headers) = &options.headers { + for (key, value) in headers { + // Skip Cookie header if credentials is omit + if options.credentials == Credentials::Omit && key.eq_ignore_ascii_case("cookie") { + continue; + } + + // Validate header name and value before adding to request + let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header name: {key}")), + ) + })?; + let header_value = HeaderValue::from_str(value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + + // Faith's coding is layered on top of what the caller declares, and reqwest's + // builder appends rather than replaces, so passing the caller's value through + // here would put a second `Content-Encoding` beside the joined one -- the same + // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is + // still validated above, then withheld and re-emitted once below. + #[cfg(feature = "encoding")] + if compress.is_some() && header_name == CONTENT_ENCODING { + continue; + } + + request = request.header(header_name, header_value); + } + } + + // What the caller says they handed over: their own `Content-Encoding`, else the + // agent's, per-request headers winning per name as they do generally (spec: REQ). + // Several lines are the one list, so they are joined as they are read. + #[cfg(feature = "encoding")] + let declared_content_encoding = compress.and_then(|_| { + let from_request = options.headers.as_ref().and_then(|headers| { + let declared = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(); + (!declared.is_empty()).then(|| declared.join(", ")) + }); + from_request.or_else(|| { + agent + .default_content_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + }); + + // The request's `Accept-Encoding` governs which codings Faith decodes on the way + // back (spec: ENC): a value on the request, else one inherited from the agent's + // default headers, else the default Faith sends itself. Neither the request nor the + // agent advertising a value means nothing beneath Faith adds one now that it owns + // the codings, so Faith sends the default explicitly. + #[cfg(feature = "encoding")] + let request_accept_encoding = options.headers.as_ref().and_then(|headers| { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) + .map(|(_, value)| value.clone()) + }); + #[cfg(feature = "encoding")] + let accept_encoding = AcceptEncoding::parse( + &request_accept_encoding + .clone() + .or_else(|| { + agent + .default_accept_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), + ); + #[cfg(feature = "encoding")] + if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { + request = request.header( + ACCEPT_ENCODING, + HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), + ); + } + + // The `priority` option is a hint, so a `Priority` header the caller wrote, or one + // among the agent's default headers, wins over the value derived from it + // (spec: REQ#request-priority). The agent's defaults are consulted here rather than + // left to reqwest: it fills a default header in only where the request carries none + // of that name, so setting the derived value would displace the agent's own. + if let Some(urgency) = options.priority + && !agent.has_default_priority + && !options.headers.as_ref().is_some_and(|headers| { + headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) + }) { + request = request.header( + HeaderName::from_static(PRIORITY), + HeaderValue::from_static(urgency), + ); + } + + // The coding actually applied, which is `compress` only where there was a body to + // apply it to: the option does nothing on a request carrying none, so no + // `Content-Encoding` describes bytes that were never sent + // (spec:ENC#compressing-a-request-body). + #[cfg(feature = "encoding")] + let mut applied_coding = None; + + // Handle body: prefer streaming body over buffered body + match body { + RequestBody::Stream(byte_stream) => { + // A body read from a stream has no length to advertise, which the fetch standard + // allows only over HTTP/2 and HTTP/3. + // spec:REQ#streaming-a-request-body + if !agent.quirk_h1_request_streaming { + // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and + // can be refused without opening a connection to find out. Returning here drops + // the stream, which is what tells whatever is feeding it to stop. + if parsed_url.scheme() != "https" { + return Err(FaithError::new( + FaithErrorKind::Network, + Some(format!( + "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", + parsed_url.as_str() + )), + )); + } + + // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the + // request hands the check to the layer that finds out: the connection is chosen, + // and an HTTP/1.x one is refused there before any of the body is written. + request = request.version(http::Version::HTTP_2); + } + + #[cfg(feature = "encoding")] + let body = match compress { + // Compressed as the chunks arrive, and chunked on the wire either way: + // a stream has no length to declare up front. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) + } + None => reqwest::Body::wrap_stream(byte_stream), + }; + #[cfg(not(feature = "encoding"))] + let body = reqwest::Body::wrap_stream(byte_stream); + request = request.body(body); + } + RequestBody::Bytes(bytes) => { + #[cfg(feature = "encoding")] + let body = match compress { + // The compressed bytes are what reqwest sizes `Content-Length` from, so the + // header counts what goes on the wire. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + encoding::compress_buffer(&bytes, coding) + .await + .map_err(|err| { + FaithError::new( + FaithErrorKind::Network, + Some(format!("could not compress the request body: {err}")), + ) + })? + } + None => bytes.to_vec(), + }; + #[cfg(not(feature = "encoding"))] + let body = bytes.to_vec(); + request = request.body(body); + } + RequestBody::None => {} + } + + // One `Content-Encoding` naming the caller's codings then Faith's, in the order they + // were applied (spec:ENC#what-a-compressed-request-sends). + #[cfg(feature = "encoding")] + if let Some(coding) = applied_coding { + let value = encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); + let value = HeaderValue::from_str(&value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + request = request.header(CONTENT_ENCODING, value); + } + + if let Some(dur) = options.timeout { + request = request.timeout(dur); + } + + agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); + + // The origin every phase is measured from. + let started = Instant::now(); + + // A caller that can abort races the request against that signal; one that cannot just sends. + let response = match abort { + Some(abort) => { + tokio::select! { + result = request.send() => result?, + _ = abort => { + return Err(FaithErrorKind::Aborted.into()); + } + } + } + None => request.send().await?, + }; + + agent + .stats + .responses_received + .fetch_add(1, Ordering::Relaxed); + + let status_code = response.status(); + let empty = status_code == StatusCode::NO_CONTENT || is_head; + + let response_url = response.url().clone(); + let version = response.version(); + + // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the + // request's port to the advertised one, so the response URL's port reflects + // which endpoint answered rather than any redirect. Compare with ports + // normalised away, or every such request would report `redirected`. + // + // Only HTTP/3 responses can have been rewritten — reqwest routes + // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and + // the TCP fallback re-runs the untouched clone. Restricting the normalisation + // to those keeps exact comparison, and so port-only redirect detection, for + // every other response. + let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { + let without_port = |url: &reqwest::Url| { + let mut url = url.clone(); + let _ = url.set_port(None); + url + }; + without_port(&parsed_url) != without_port(&response_url) + } else { + parsed_url != response_url + }; + + // Track connection for TCP stats (if we can get both local and remote addr). + // A connection the tracker has already seen is one the pool handed back, which is + // what `reused` reports (spec:RESP#request-timing). + #[cfg(feature = "connection-tracking")] + let reused = if let Some(http_info) = response.extensions().get::() { + let local_addr = http_info.local_addr(); + let remote_addr = http_info.remote_addr(); + agent.conn_tracker.track(local_addr, remote_addr) + } else { + false + }; + // Whether the pool handed a connection back is what the tracker knows, so without it there is + // no answer to report. + #[cfg(not(feature = "connection-tracking"))] + let reused = false; + + // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has + // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a + // redirect chain marks the origin that actually answered rather than the one asked for. + agent.mark_warm(&response_url); + + let peer = PeerInformation { + address: response.remote_addr(), + certificate: response + .extensions() + .get::() + .and_then(|info| info.peer_certificate()) + .map(|cert| cert.into()), + }; + + let mut headers = response.headers().clone(); + if options.credentials == Credentials::Omit { + headers.remove("set-cookie"); + } + + // A cache hit is served without ever reaching the layer that stamps, so fall back to + // the moment the send resolved, which for a hit is the moment the cache answered. + let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); + let timing = RequestTiming { + headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, + body_ms: None, + reused, + next_hop_protocol: alpn_protocol_id(version, &response_url), + // Captured before a decoded body's `Content-Encoding` is stripped below, so the + // coding the response arrived under is reported either way. + content_encoding: headers + .get(CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + from_cache: headers + .get("x-cache") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), + }; + + // Decode only a body Faith negotiated the coding for; a bodyless response keeps its + // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). + #[cfg(feature = "encoding")] + let decode = if empty { + None + } else { + encoding::decision(&headers, &accept_encoding) + }; + #[cfg(feature = "encoding")] + if decode.is_some() { + encoding::strip_decoded_headers(&mut headers); + } + + let timing = Arc::new(TimingSlot::new(started, timing)); + // A response that cannot carry a body has nothing left to wait for. + if empty { + timing.ended(); + } + + Ok(Response { + body: if empty { + BodyHolder::none() + } else { + let http_response: http::Response<_> = response.into(); + BodyHolder::new( + Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), + version, + timing.clone(), + ) + }, + #[cfg(feature = "encoding")] + decode, + disturbed: Arc::new(AtomicBool::new(false)), + headers, + integrity: options.integrity, + peer: Arc::new(peer), + redirected, + stats: agent.stats.clone(), + status_code, + timing, + trailers: Default::default(), + url: response_url, + version, + }) +} diff --git a/crates/web-faith/src/request/target.rs b/crates/web-faith/src/request/target.rs new file mode 100644 index 0000000..a5754bc --- /dev/null +++ b/crates/web-faith/src/request/target.rs @@ -0,0 +1,90 @@ +use bytes::Bytes; +use url::Url; + +use crate::{ + error::{FaithError, FaithErrorKind}, + request::{Request, RequestBody, RequestOptions}, +}; + +/// What a request is aimed at: a URL, or another request to layer over. +/// +/// Anything that converts into a `url::Url` is a target, as is a [`Request`], which is what lets a +/// prepared request be adjusted at each call site. +// spec:REQ +pub enum Target { + Url(Url), + Request(Box), +} + +impl From for Target { + fn from(url: Url) -> Self { + Self::Url(url) + } +} + +impl From for Target { + fn from(request: Request) -> Self { + Self::Request(Box::new(request)) + } +} + +/// A string target is parsed when the builder resolves, so an unparseable one is reported there +/// rather than by a call that cannot fail. +// spec:REQ +impl TryFrom<&str> for Target { + type Error = FaithError; + + fn try_from(url: &str) -> Result { + Url::parse(url) + .map(Self::Url) + .map_err(|_| FaithErrorKind::InvalidUrl.into()) + } +} + +impl TryFrom for Target { + type Error = FaithError; + + fn try_from(url: String) -> Result { + Self::try_from(url.as_str()) + } +} + +/// An `http::Request` is a target too, so a request built against the wider ecosystem can be sent +/// on an agent: its method, URL, headers, and body come across. +// spec:REQ +impl TryFrom> for Target +where + B: Into, +{ + type Error = FaithError; + + fn try_from(request: http::Request) -> Result { + let (parts, body) = request.into_parts(); + + let url = Url::parse(&parts.uri.to_string()) + .map_err(|_| FaithError::from(FaithErrorKind::InvalidUrl))?; + + let mut headers = Vec::new(); + for (name, value) in parts.headers.iter() { + let Ok(value) = value.to_str() else { + return Err(FaithErrorKind::InvalidHeader.into()); + }; + headers.push((name.to_string(), value.to_owned())); + } + + let body = body.into(); + Ok(Self::Request(Box::new(Request { + url, + options: RequestOptions { + method: Some(parts.method.to_string()), + headers: (!headers.is_empty()).then_some(headers), + ..RequestOptions::default() + }, + body: if body.is_empty() { + RequestBody::None + } else { + RequestBody::Bytes(body) + }, + }))) + } +} diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs new file mode 100644 index 0000000..225fbb7 --- /dev/null +++ b/crates/web-faith/src/response.rs @@ -0,0 +1,819 @@ +//! Reading a response: where trailers land, what is known of the peer, and writing a body out. + +// spec:RESP spec:TRL spec:BODY + +use std::{ + fmt::Debug, + hint::unreachable_unchecked, + mem::replace, + net::SocketAddr, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + task::{Context, Poll}, + time::{Duration, Instant}, +}; + +use bytes::Bytes; +use futures::{Stream, StreamExt, TryStreamExt, stream}; +use http::header::{CONTENT_LENGTH, HeaderMap}; +use http_body_util::BodyStream; +use reqwest::{StatusCode, Url, Version}; +use serde::de::DeserializeOwned; +use stream_shared::SharedStream; +use tokio::{io::AsyncWriteExt, sync::watch}; + +#[cfg(feature = "encoding")] +use web_faith_encoding::{Coding, decode_stream}; + +use crate::{ + body::{Body, BodyHolder, DynStream, drain_body_inner}, + error::{FaithError, FaithErrorKind}, + stats::InnerAgentStats, + timing::TimingSlot, +}; + +use crate::integrity::{finish_integrity, integrity_checker, verify_integrity}; + +/// What is known about the peer that sent a response. +/// +/// - `address`: The IP address and port of the peer, if available. +/// - `certificate`: When connected over HTTPS, this is the DER-encoded leaf certificate of the peer. +#[derive(Debug)] +pub struct PeerInformation { + pub address: Option, + pub certificate: Option>, +} + +/// Where a response body is written, and on what terms. +#[derive(Debug, Clone, Default)] +pub struct FileDestination { + /// Truncate and replace an occupied destination. The default refuses one instead, leaving what + /// is there untouched. + pub overwrite: bool, + /// The permissions a newly created file is given. Ignored on platforms without Unix file modes. + pub mode: Option, +} + +/// The shortest gap between progress reports. +/// +/// Reporting every chunk would cross a surface boundary thousands of times for a large body, +/// which is the cost writing to a file directly exists to avoid. A caller driving a progress bar +/// cannot use updates faster than this anyway, and the final report is always delivered regardless. +pub const PROGRESS_INTERVAL: Duration = Duration::from_millis(50); + +/// Open the destination file for a body write, mapping filesystem refusals to the errors +/// writing a body to a file surfaces. +// spec:BODY#tofile +pub async fn open_destination( + path: &str, + options: &FileDestination, +) -> Result { + let mut open = tokio::fs::OpenOptions::new(); + open.write(true); + if options.overwrite { + // An occupied destination is truncated and replaced. + open.create(true).truncate(true); + } else { + // The safe default refuses an occupied destination outright. + open.create_new(true); + } + #[cfg(unix)] + if let Some(mode) = options.mode { + open.mode(mode); + } + + match open.open(path).await { + Ok(file) => Ok(file), + Err(err) => Err(classify_open_error(path, err).await), + } +} + +/// Classify a failure to open the destination. An occupied destination is `FileExists`, +/// unless what occupies it is a directory: a directory is well-formed but cannot be written +/// to, which is a `FileWrite`. Every other refusal is a `FileWrite` carrying the OS detail. +pub async fn classify_open_error(path: &str, err: std::io::Error) -> FaithError { + let kind = if err.kind() == std::io::ErrorKind::AlreadyExists { + match tokio::fs::symlink_metadata(path).await { + Ok(meta) if meta.is_dir() => FaithErrorKind::FileWrite, + _ => FaithErrorKind::FileExists, + } + } else { + FaithErrorKind::FileWrite + }; + FaithError::new(kind, Some(err.to_string())) +} + +#[derive(Clone, Debug, Default)] +pub enum Trailers { + #[default] + NotYet, + None, + Some(HeaderMap), +} + +/// Where the trailers land: written by whoever finishes the body, awaited by `trailers()`. +/// +/// A watch channel, rather than a lock read in a loop. Per the fetch standard's trailers +/// proposal () this promise is *meant* not to +/// resolve until the body has been consumed, so the wait is unbounded by design -- which is +/// precisely why polling was the wrong shape for it. Awaiting trailers without reading the +/// body now leaves an idle pending promise rather than a pegged core, and the future can be +/// cancelled while it waits. +#[derive(Debug)] +pub struct TrailersSlot(watch::Sender); + +impl Default for TrailersSlot { + fn default() -> Self { + Self(watch::channel(Trailers::NotYet).0) + } +} + +impl TrailersSlot { + /// Record trailers that arrived, waking whoever is waiting. + pub fn arrived(&self, trailers: HeaderMap) { + self.0.send_replace(Trailers::Some(trailers)); + } + + /// Record that the body ended, if no trailers frame got there first. + /// + /// `send_if_modified` so the read and the write are one step, and so waiters are woken + /// only by the call that actually settled it. + pub fn ended(&self) { + self.0.send_if_modified(|state| { + if matches!(state, Trailers::NotYet) { + *state = Trailers::None; + true + } else { + false + } + }); + } + + /// Wait until the body has settled the question. + pub async fn settled(&self) -> Trailers { + let mut rx = self.0.subscribe(); + // `wait_for` tests the current value before waiting, so trailers that already + // arrived return without yielding. Its error case is the sender being gone, which + // means the response was dropped and nothing can ever set this -- no trailers is + // the only answer left. + match rx + .wait_for(|state| !matches!(state, Trailers::NotYet)) + .await + { + Ok(state) => state.clone(), + Err(_) => Trailers::None, + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + future::Future, + pin::pin, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll, Wake, Waker}, + }; + + use super::*; + + /// A waker that counts how many times the task asks to be polled again. + struct CountingWaker(AtomicUsize); + + impl CountingWaker { + fn wakes(&self) -> usize { + self.0.load(Ordering::SeqCst) + } + } + + impl Wake for CountingWaker { + fn wake(self: Arc) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + /// A response that cannot carry a body converts to an `http::Response` with an empty one, + /// carrying its status, version, and headers across. + #[test] + fn a_bodyless_response_converts_to_an_http_response() { + let mut headers = HeaderMap::new(); + headers.insert("x-test", "yes".parse().expect("a valid header value")); + + let response = Response { + body: BodyHolder::none(), + #[cfg(feature = "encoding")] + decode: None, + disturbed: Arc::new(AtomicBool::new(false)), + headers, + integrity: None, + peer: Arc::new(PeerInformation { + address: None, + certificate: None, + }), + redirected: false, + stats: Arc::new(InnerAgentStats::default()), + status_code: StatusCode::NO_CONTENT, + timing: Arc::new(TimingSlot::new( + Instant::now(), + crate::timing::RequestTiming::default(), + )), + trailers: Arc::new(TrailersSlot::default()), + url: Url::parse("https://example.com/").expect("a valid url"), + version: Version::HTTP_2, + }; + + let http = response.into_http().expect("an undisturbed body converts"); + assert_eq!(http.status(), StatusCode::NO_CONTENT); + assert_eq!(http.version(), Version::HTTP_2); + assert_eq!( + http.headers().get("x-test").map(|v| v.as_bytes()), + Some(&b"yes"[..]) + ); + + // Draining it yields nothing, which is what a consumer of the body actually observes. + let collected = + futures::executor::block_on(http_body_util::BodyExt::collect(http.into_body())) + .expect("an empty body collects"); + assert!(collected.to_bytes().is_empty()); + } + + /// Waiting for trailers parks until the body settles the question, rather than polling + /// for it. + /// + /// The bug this guards against was a `yield_now` loop, which is visible here as the shape + /// of the wait rather than as a quantity of CPU: a spin re-arms its own waker on every + /// poll, so it is scheduled again immediately, while a parked wait asks for nothing until + /// something else moves. Asserting the wake count keeps this deterministic -- timing how + /// much CPU the process burns measures the machine as much as the code. + #[test] + fn waiting_for_trailers_parks_rather_than_spinning() { + let slot = TrailersSlot::default(); + let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); + let waker = Waker::from(counter.clone()); + let mut cx = Context::from_waker(&waker); + let mut settled = pin!(slot.settled()); + + // Nothing has settled the question, so the wait parks... + assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); + // ...without scheduling itself to be polled again, which is what a spin does. + assert_eq!(counter.wakes(), 0, "a parked wait asks for no wake-up"); + + // Polling again changes nothing: still parked, still asking for nothing. + assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); + assert_eq!(counter.wakes(), 0, "polling again does not arm a wake-up"); + + // The body ending is what wakes it, and it resolves on the next poll. + slot.ended(); + assert!(counter.wakes() >= 1, "the body ending wakes the waiter"); + assert!(matches!( + settled.as_mut().poll(&mut cx), + Poll::Ready(Trailers::None) + )); + } + + /// Trailers that arrived before anyone asked resolve without parking at all. + #[test] + fn trailers_already_there_resolve_on_the_first_poll() { + let slot = TrailersSlot::default(); + let mut headers = HeaderMap::new(); + headers.insert("x-checksum", "abc123".parse().unwrap()); + slot.arrived(headers); + + let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); + let waker = Waker::from(counter.clone()); + let mut cx = Context::from_waker(&waker); + let mut settled = pin!(slot.settled()); + + assert!(matches!( + settled.as_mut().poll(&mut cx), + Poll::Ready(Trailers::Some(_)) + )); + } +} + +/// A progress report from a body write in flight. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileProgress { + /// Bytes written to the file so far. + pub bytes_written: u64, + /// What the response advertised in `Content-Length`, when it sent one and the body is not + /// being decoded. Absent when the total is not known ahead of time, which is the case for a + /// chunked response and for one being decoded. + pub content_length: Option, +} + +/// What a completed body write reports. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileWritten { + /// The absolute filesystem path written to. + pub path: String, + /// The number of bytes that landed at the destination. + pub bytes_written: u64, +} + +/// A response to a request. +/// +/// A response is not constructed by a caller; it arrives from a request. Reading its body consumes +/// it, following the fetch standard rather than the owned-response model of other Rust clients, so a +/// second read fails. +#[derive(Debug, Clone)] +pub struct Response { + pub body: BodyHolder, + /// The coding to decode the body under, or `None` to deliver it as received. + #[cfg(feature = "encoding")] + /// Set once when the response is built, from the request's `Accept-Encoding` and the + /// response's `Content-Encoding` (see [`web_faith_encoding`]). + pub decode: Option, + pub disturbed: Arc, + pub headers: HeaderMap, + pub integrity: Option, + pub peer: Arc, + pub redirected: bool, + pub stats: Arc, + pub status_code: StatusCode, + pub timing: Arc, + pub trailers: Arc, + pub url: Url, + pub version: Version, +} + +impl Response { + /// The response's status. + pub fn status(&self) -> StatusCode { + self.status_code + } + + /// The canonical reason phrase for the status, or empty for a code with no well-known one. + /// + /// Always the canonical phrase: HTTP/1 lets a server send its own, which is not surfaced here, + /// and HTTP/2 and HTTP/3 carry none at all. + pub fn status_text(&self) -> &'static str { + self.status_code.canonical_reason().unwrap_or_default() + } + + /// Whether the status is in the 2xx range. + pub fn ok(&self) -> bool { + self.status_code.is_success() + } + + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// The URL the response came from, which is the last one after any redirects. + pub fn url(&self) -> &Url { + &self.url + } + + /// Whether a redirect was followed to reach this response. + pub fn redirected(&self) -> bool { + self.redirected + } + + /// The HTTP version the response arrived over. + pub fn version(&self) -> Version { + self.version + } + + /// What is known of the peer that sent the response. + pub fn peer(&self) -> &PeerInformation { + &self.peer + } + + /// Whether the body has been read, or handed out as a stream. + pub fn body_used(&self) -> bool { + self.disturbed.load(Ordering::SeqCst) + } + + /// Read the whole body. + /// + /// Reading consumes the body, so a second read fails with the already-disturbed error, as the + /// fetch standard has it rather than the owned-response model other Rust clients use. An + /// `integrity` value on the request is verified here, once the whole body is in hand. + // spec:BODY + pub async fn bytes(&self) -> Result, FaithError> { + self.check_stream_disturbed()?; + self.gather_contiguous().await + } + + /// Read the whole body as text. + /// + /// Always decoded as UTF-8, with invalid sequences replaced by U+FFFD rather than failing, which + /// is what the fetch standard calls for. + pub async fn text(&self) -> Result { + let bytes = self.bytes().await?; + Ok(String::from_utf8(bytes) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) + } + + /// Read the whole body and deserialise it from JSON. + /// + /// The body is read into memory before it is parsed, which can cost twice its size; read + /// [`Self::body_stream`] instead where that matters. + pub async fn json(&self) -> Result { + let bytes = self.bytes().await?; + serde_json::from_slice(&bytes) + .map_err(|err| FaithError::new(FaithErrorKind::JsonParse, Some(err.to_string()))) + } + + /// Take the body as a stream of chunks, decoded under whichever coding was negotiated. + /// + /// `None` for a response that cannot carry a body. Unlike the collecting reads, this can be + /// called more than once: each call hands back the same shared stream rather than a second one. + /// A body already being consumed elsewhere reports the already-disturbed error rather than + /// waiting for the other reader to finish. + // spec:BODY + pub fn body_stream( + &self, + ) -> Result> + use<>>, FaithError> { + // The body counts as disturbed from here, though the stream itself stays re-readable. + let _ = self.check_stream_disturbed(); + + let Some(lock) = &self.body.body else { + return Ok(None); + }; + + let mut body = lock + .try_lock() + .map_err(|_| FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed))?; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + + Ok(Some(stream.map_err(|err| { + FaithError::new(FaithErrorKind::BodyStream, Some(err)) + }))) + } + + /// Give up on the body, releasing the connection back to the pool. + /// + /// Worth doing when the body is not wanted: left unread, the connection may be held open until + /// the response is dropped. An HTTP/1 body is read and thrown away so the connection can be + /// reused; a multiplexed one is dropped instead, cancelling the stream without touching the + /// connection it shared. + /// + /// This settles the trailers as none rather than leaving them pending: on a multiplexed + /// connection the stream was cancelled before any could arrive, and draining an HTTP/1 body + /// here bypasses the stream that would have collected them. A caller who discards the body and + /// then awaits trailers would otherwise wait for something that can no longer come. + // spec:BODY spec:TRL spec:RESP#request-timing + pub async fn discard(&self) { + if let Some(arc) = self.body.body.clone() { + if self.body.is_multiplexed() { + *arc.lock().await = Body::Consumed; + } else { + drain_body_inner(arc).await; + } + } + self.body.drained.store(true, Ordering::SeqCst); + self.trailers.ended(); + // Discarding is one of the ways a body finishes. + self.timing.ended(); + } + + /// The timing of the request that produced this response, once its body has ended. + // spec:RESP#request-timing + pub async fn timing(&self) -> crate::timing::RequestTiming { + self.timing.settled().await + } + + /// The trailers, once the body has ended. + /// + /// A body that is never read never ends, so this waits indefinitely by design; see [`Trailers`]. + // spec:TRL + pub async fn trailers(&self) -> Trailers { + self.trailers.settled().await + } + + pub fn check_stream_disturbed(&self) -> Result<(), FaithError> { + if self.disturbed.swap(true, Ordering::SeqCst) { + Err(FaithErrorKind::ResponseAlreadyDisturbed.into()) + } else { + Ok(()) + } + } + + /// Ensures the body is converted to a SharedStream, returning a clone of it. + /// + /// This allows multiple consumers (original + clones) to independently read the body. + pub fn ensure_stream( + &self, + body: &mut Body, + drained_flag: Arc, + ) -> Result>>, FaithError> { + match body { + Body::Consumed => Err(FaithErrorKind::ResponseAlreadyDisturbed.into()), + Body::Stream(stream) => Ok(stream.clone()), + lock @ Body::Inner(_) => { + // temporarily replace with Consumed until we can put in the Stream + let Body::Inner(inner) = replace(lock, Body::Consumed) else { + // SAFETY: we're inside the match checking for this exact thing + unsafe { unreachable_unchecked() } + }; + + // Track that we've started consuming a body + self.stats.bodies_started.fetch_add(1, Ordering::Relaxed); + + let trailers_stream = self.trailers.clone(); + let trailers_finish = self.trailers.clone(); + let stats_finish = self.stats.clone(); + let timing_finish = self.timing.clone(); + let drained_finish = drained_flag.clone(); + // The frame stream pulls trailers off to the side (via `arrived`) and yields + // data bytes only, so decoding sees no trailer frames. + let bytes = Box::pin( + BodyStream::new(inner) + .then(move |frame| { + let trailers_lock = trailers_stream.clone(); + async move { + match frame { + Err(err) => Some(Err(err.to_string())), + Ok(frame) => match frame.into_trailers() { + Ok(trailers) => { + trailers_lock.arrived(trailers); + None + } + Err(frame) => Some( + frame + .into_data() + .map_err(|_| "unknown frame kind".to_string()), + ), + }, + } + } + }) + .filter_map(async |item| item), + ) as Pin>; + + #[cfg(feature = "encoding")] + let bytes = match self.decode { + Some(coding) => decode_stream(bytes, coding), + None => bytes, + }; + + // A zero-length chunk carries no bytes, but the body's byte-oriented + // ReadableStream cannot take one: `ReadableByteStreamController.enqueue` + // rejects an empty buffer outright (`ERR_INVALID_STATE`). Some origins end a + // response with an empty DATA frame carrying END_STREAM, so drop empty chunks + // here, before the stream is built, letting it close cleanly. The byte count + // delivered is unchanged, and the collecting paths (`text()`, `bytes()`) never + // noticed the empties anyway. + let bytes = Box::pin(bytes.filter(|item| { + let empty = matches!(item, Ok(chunk) if chunk.is_empty()); + async move { !empty } + })) as Pin>; + + // Chained onto the stream that is actually delivered, above any decoder: a + // decoder reaches the end of its own framing without necessarily polling the + // bytes underneath to completion, so bookkeeping chained below it would never + // run for a decoded body, leaving the trailers promise and the timing pending + // for good. + let bytes = Box::pin( + bytes.chain( + stream::once(async move { + trailers_finish.ended(); + // The last byte of the body: every read path ends here, so + // this is where the timing settles + // (spec:RESP#request-timing). + timing_finish.ended(); + // Track that we've finished consuming a body + stats_finish.bodies_finished.fetch_add(1, Ordering::Relaxed); + // Mark body as drained so Drop doesn't try to drain again + drained_finish.store(true, Ordering::SeqCst); + }) + .filter_map(async |()| None), + ), + ) as Pin>; + + let stream = SharedStream::new(bytes); + + // the _ is the Consumed we put in there earlier + let _ = replace(lock, Body::Stream(stream.clone())); + + Ok(stream) + } + } + } + + /// Underlying efficient response body fetcher. + /// + /// Unlike bytes() and co, this grabs all the chunks of the response but doesn't + /// copy them. Further processing is needed to obtain a `Vec` or whatever is wanted. + pub async fn gather(&self) -> Result, FaithError> { + let Some(lock) = &self.body.body else { + return Ok(Default::default()); + }; + + let mut body = lock.lock().await; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + drop(body); // release lock before consuming stream + + let mut chunks = Vec::new(); + futures::pin_mut!(stream); + while let Some(result) = stream.next().await { + let chunk = + result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; + chunks.push(chunk); + } + + // Mark as drained since we consumed everything + self.body.mark_drained(); + + Ok(Arc::from(chunks.into_boxed_slice())) + } + + /// gather() and then copy into one contiguous buffer + pub async fn gather_contiguous(&self) -> Result, FaithError> { + let body = self.gather().await?; + let length = body.iter().map(|chunk| chunk.len()).sum(); + let mut bytes = Vec::with_capacity(length); + for chunk in body.into_iter() { + bytes.extend_from_slice(chunk); + } + + if let Some(ref integrity) = self.integrity { + verify_integrity(&bytes, integrity)?; + } + + Ok(bytes) + } + + /// Write the body out to a file, reporting progress as the bytes land. + /// + /// `on_progress` is called with the bytes written so far and the advertised length where one is + /// known, no more often than [`PROGRESS_INTERVAL`], and once more when the last byte is written. + // spec:BODY#tofile + pub async fn write_to_file( + &self, + path: &str, + options: &FileDestination, + mut on_progress: impl FnMut(FileProgress), + ) -> Result { + // A response that cannot carry a body has nothing to write, and this is settled + // before any file is created (spec:BODY#tofile). + let Some(lock) = self.body.body.clone() else { + return Err(FaithErrorKind::ResponseBodyNull.into()); + }; + + // A body already read, or whose stream was handed out, has no second read to give. + // Checked without committing so an open failure below still leaves the body + // undisturbed and the caller free to retry to another path. + if self.disturbed.load(Ordering::SeqCst) { + return Err(FaithErrorKind::ResponseAlreadyDisturbed.into()); + } + + // Reject a malformed integrity value up front, before the body is touched, the same + // as the other verified reads reject it when the whole body is in hand. + let mut checker = integrity_checker(self.integrity.as_deref())?; + + // The advertised length, when the server sent one. It is only visible here for a body + // delivered as received: a decoded body has had its Content-Length stripped, so the + // bytes written equal the wire bytes wherever this is Some (spec:BODY#tofile, ENC). + let content_length = self + .headers + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()); + + // The destination is opened before any of the body is read, so a failure to open it + // leaves the body unread and undisturbed. + let mut file = open_destination(path, options).await?; + + // Commit the read now the destination is in hand. A concurrent read that slipped in + // since the load above wins, and this one finds the body already spent. + self.check_stream_disturbed()?; + + let stream = { + let mut body = lock.lock().await; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + drop(body); // release lock before consuming stream + stream + }; + + // Reporting is rate limited rather than per chunk, so a large body does not cross a + // surface boundary thousands of times. + // spec:BODY#tofile + let mut report = |written: u64| { + on_progress(FileProgress { + bytes_written: written, + content_length, + }); + }; + + let mut written: u64 = 0; + let mut reported_at = Instant::now(); + futures::pin_mut!(stream); + while let Some(result) = stream.next().await { + let chunk = + result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; + if let Some(checker) = checker.as_mut() { + checker.input(&chunk); + } + file.write_all(&chunk) + .await + .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; + written += chunk.len() as u64; + // A server cannot send more than it promised: once the bytes off the wire exceed + // the advertised length, the write fails and the bytes so far stay on disk + // (spec:BODY#tofile). + if let Some(limit) = content_length { + if written > limit { + return Err(FaithErrorKind::ContentLengthOverrun.into()); + } + } + if reported_at.elapsed() >= PROGRESS_INTERVAL { + reported_at = Instant::now(); + report(written); + } + } + + file.flush() + .await + .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; + + // The last report always lands, whatever the rate limit allowed along the way, so a + // caller's final view of a completed write is the whole body rather than the last + // interval boundary. An empty body reports once, with nothing written. + report(written); + + // The digest is only known once the last byte has been written, so the file that + // fails verification is on disk when the error arrives (spec:SRI). + if let Some(checker) = checker { + finish_integrity(checker)?; + } + + self.body.mark_drained(); + + Ok(FileWritten { + // A relative path resolves against the process's working directory; the caller + // is handed the absolute path the bytes landed at. + path: std::path::absolute(path) + .map(|abs| abs.to_string_lossy().into_owned()) + .unwrap_or_else(|_| path.to_owned()), + bytes_written: written, + }) + } +} + +/// A [`Response`]'s body, as an [`http_body::Body`]. +/// +/// This is what a response hands to code written against the wider ecosystem: a tower service, a +/// hyper client, anything that takes a body rather than Faith's own reads. +pub struct ResponseBody { + chunks: Pin> + Send>>, +} + +impl Debug for ResponseBody { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ResponseBody").finish_non_exhaustive() + } +} + +impl http_body::Body for ResponseBody { + type Data = Bytes; + type Error = FaithError; + + fn poll_frame( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + self.chunks + .as_mut() + .poll_next(cx) + .map(|chunk| chunk.map(|chunk| chunk.map(http_body::Frame::data))) + } +} + +impl Response { + /// Take the response as an [`http::Response`], so it feeds code written against the ecosystem + /// rather than against Faith. + /// + /// Fails where taking the body would: a body already being consumed elsewhere reports the + /// already-disturbed error. A response that cannot carry a body yields an empty one. + pub fn into_http(self) -> Result, FaithError> { + let chunks: Pin> + Send>> = + match self.body_stream()? { + Some(stream) => Box::pin(stream), + None => Box::pin(stream::empty()), + }; + + let mut response = http::Response::new(ResponseBody { chunks }); + *response.status_mut() = self.status_code; + *response.version_mut() = self.version; + *response.headers_mut() = self.headers.clone(); + Ok(response) + } +} + +impl TryFrom for http::Response { + type Error = FaithError; + + fn try_from(response: Response) -> Result { + response.into_http() + } +} diff --git a/src/retry.rs b/crates/web-faith/src/retry.rs similarity index 73% rename from src/retry.rs rename to crates/web-faith/src/retry.rs index 11ecbe1..6c76a1c 100644 --- a/src/retry.rs +++ b/crates/web-faith/src/retry.rs @@ -15,7 +15,11 @@ use http::{Extensions, Method}; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result}; -use crate::dns::FaithResolver; +#[cfg(feature = "dns")] +mod stale_address; + +#[cfg(feature = "dns")] +pub use stale_address::StaleAddressRetry; /// How many times a request may be replayed before the failure reaches the caller. /// @@ -124,7 +128,8 @@ impl Middleware for DeadConnectionRetry { /// status, a connection that died mid-exchange, a body that stopped early. Those are answers, and an /// address that produced one is confirmed by definition. Only a connect failure leaves open the /// possibility that the address itself was wrong. -fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { +#[cfg(feature = "dns")] +pub(super) fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { match err { reqwest_middleware::Error::Reqwest(err) => err.is_connect(), // A middleware's own error is about this stack rather than about the network. @@ -132,69 +137,6 @@ fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { } } -/// Re-resolves and attempts a request again when connecting to a stale-served address failed. -/// -/// Serving an expired DNS answer trades a round trip against the chance the address has moved, and -/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request -/// (spec:DNS#when-a-stale-address-is-wrong). -#[derive(Debug, Clone)] -pub struct StaleAddressRetry { - /// `None` under `dns.system: true`, where Faith holds no cache and so serves nothing stale. - resolver: Option, -} - -impl StaleAddressRetry { - pub fn new(resolver: Option) -> Self { - Self { resolver } - } -} - -#[async_trait::async_trait] -impl Middleware for StaleAddressRetry { - // spec:DNS#when-a-stale-address-is-wrong - async fn handle( - &self, - req: Request, - extensions: &mut Extensions, - next: Next<'_>, - ) -> Result { - let Some(resolver) = self.resolver.clone() else { - return next.run(req, extensions).await; - }; - - // Asked before the request runs, not after it fails. The lookup this request makes serves the - // stale entry and starts a refresh behind it, so by the time an error is in hand the entry may - // already have been replaced and the question "was this address assumed?" no longer answerable. - // - // Cloned here for the same reason the dead-connection layer clones early: sending consumes the - // body. A `ReadableStream` body does not clone, which is what leaves those requests reporting - // the connect failure rather than being attempted again -- the body has no second copy, whether - // or not it was read. - let host = req.url().host_str().map(str::to_owned); - let replay = match &host { - Some(host) if resolver.served_stale(host) => req.try_clone(), - _ => None, - }; - - let outcome = next.clone().run(req, extensions).await; - - match &outcome { - Err(err) if failed_to_connect(err) => {} - _ => return outcome, - } - - let (Some(host), Some(request)) = (host, replay) else { - return outcome; - }; - - // Drop the entry so the retry's lookup waits for a fresh answer rather than being served the - // address that just failed. One attempt only: the fresh address is confirmed rather than - // assumed, so a second failure is an answer about the origin and belongs to the caller. - resolver.invalidate_stale(&host); - next.run(request, extensions).await - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/web-faith/src/retry/stale_address.rs b/crates/web-faith/src/retry/stale_address.rs new file mode 100644 index 0000000..e57ee46 --- /dev/null +++ b/crates/web-faith/src/retry/stale_address.rs @@ -0,0 +1,69 @@ +use http::Extensions; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result}; +use web_faith_dns::FaithResolver; + +use super::failed_to_connect; + +/// Re-resolves and attempts a request again when connecting to a stale-served address failed. +/// +/// Serving an expired DNS answer trades a round trip against the chance the address has moved, and +/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request. +// spec:DNS#when-a-stale-address-is-wrong +#[derive(Debug, Clone)] +pub struct StaleAddressRetry { + /// `None` under `dns.system: true`, where Faith holds no cache and so serves nothing stale. + resolver: Option, +} + +impl StaleAddressRetry { + pub fn new(resolver: Option) -> Self { + Self { resolver } + } +} + +#[async_trait::async_trait] +impl Middleware for StaleAddressRetry { + // spec:DNS#when-a-stale-address-is-wrong + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + let Some(resolver) = self.resolver.clone() else { + return next.run(req, extensions).await; + }; + + // Asked before the request runs, not after it fails. The lookup this request makes serves the + // stale entry and starts a refresh behind it, so by the time an error is in hand the entry may + // already have been replaced and the question "was this address assumed?" no longer answerable. + // + // Cloned here for the same reason the dead-connection layer clones early: sending consumes the + // body. A `ReadableStream` body does not clone, which is what leaves those requests reporting + // the connect failure rather than being attempted again -- the body has no second copy, whether + // or not it was read. + let host = req.url().host_str().map(str::to_owned); + let replay = match &host { + Some(host) if resolver.served_stale(host) => req.try_clone(), + _ => None, + }; + + let outcome = next.clone().run(req, extensions).await; + + match &outcome { + Err(err) if failed_to_connect(err) => {} + _ => return outcome, + } + + let (Some(host), Some(request)) = (host, replay) else { + return outcome; + }; + + // Drop the entry so the retry's lookup waits for a fresh answer rather than being served the + // address that just failed. One attempt only: the fresh address is confirmed rather than + // assumed, so a second failure is an answer about the origin and belongs to the caller. + resolver.invalidate_stale(&host); + next.run(request, extensions).await + } +} diff --git a/crates/web-faith/src/stats.rs b/crates/web-faith/src/stats.rs new file mode 100644 index 0000000..760dd6f --- /dev/null +++ b/crates/web-faith/src/stats.rs @@ -0,0 +1,42 @@ +//! What an agent counts about the requests it has run. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The agent's running counters, incremented as requests and bodies pass through it. +/// +/// Held behind an `Arc` and shared with every response the agent produces, since a body finishing +/// is what settles two of these and the response is what knows it happened. +#[derive(Debug, Default)] +pub struct InnerAgentStats { + pub requests_sent: AtomicU64, + pub responses_received: AtomicU64, + pub bodies_started: AtomicU64, + pub bodies_finished: AtomicU64, +} + +impl InnerAgentStats { + /// Read the counters as they stand. + pub fn snapshot(&self) -> AgentStats { + AgentStats { + requests_sent: self.requests_sent.load(Ordering::Relaxed), + responses_received: self.responses_received.load(Ordering::Relaxed), + bodies_started: self.bodies_started.load(Ordering::Relaxed), + bodies_finished: self.bodies_finished.load(Ordering::Relaxed), + } + } +} + +/// A reading of an agent's counters, taken at one moment. +/// +/// Non-exhaustive: what an agent counts can grow, and a new counter should not be a breaking change. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct AgentStats { + pub requests_sent: u64, + pub responses_received: u64, + /// Response body streams that have been started, which is what reading a body does. + pub bodies_started: u64, + /// Response body streams that have been read to the end. While more have started than + /// finished, that many bodies are holding connections open. + pub bodies_finished: u64, +} diff --git a/src/timing.rs b/crates/web-faith/src/timing.rs similarity index 81% rename from src/timing.rs rename to crates/web-faith/src/timing.rs index e8c2dc6..f759c90 100644 --- a/src/timing.rs +++ b/crates/web-faith/src/timing.rs @@ -1,13 +1,12 @@ -//! Per-request timing, surfaced as a `PerformanceResourceTiming` by the wrapper. -//! -//! spec:RESP#request-timing +//! Per-request timing. + +// spec:RESP#request-timing use std::{ sync::{Arc, OnceLock}, time::Instant, }; -use napi_derive::napi; use reqwest::{Url, Version}; use tokio::sync::watch; @@ -39,6 +38,15 @@ impl HeadersStamp { } } +/// The Alt-Svc layer is the one place a response's arrival is observed, so it marks the stamp the +/// request carries; reading it back out is this module's business. +#[cfg(feature = "http3")] +impl web_faith_alt_svc::ArrivalStamp for HeadersStamp { + fn mark(&self, at: Instant) { + HeadersStamp::mark(self, at); + } +} + /// What Faith measures of a request, filled in as the request progresses. #[derive(Clone, Debug, Default)] pub struct RequestTiming { @@ -108,35 +116,6 @@ impl TimingSlot { } } -/// The measurements behind a response's timing breakdown. -/// -/// The wrapper turns these into a `PerformanceResourceTiming`; the phases are milliseconds from -/// the start of the request rather than absolute times, so the wrapper can place them on the -/// same clock as the rest of the platform's performance entries. -#[napi(object)] -#[derive(Clone, Debug)] -pub struct TimingBreakdown { - pub headers_ms: f64, - pub body_ms: Option, - pub reused: bool, - pub next_hop_protocol: String, - pub content_encoding: Option, - pub from_cache: bool, -} - -impl From for TimingBreakdown { - fn from(timing: RequestTiming) -> Self { - Self { - headers_ms: timing.headers_ms, - body_ms: timing.body_ms, - reused: timing.reused, - next_hop_protocol: timing.next_hop_protocol, - content_encoding: timing.content_encoding, - from_cache: timing.from_cache, - } - } -} - /// The ALPN Protocol ID (RFC 7301) naming the protocol a response travelled over. /// /// Reported whether or not the connection negotiated over ALPN, which is what a browser does: diff --git a/crates/web-faith/src/warm_up.rs b/crates/web-faith/src/warm_up.rs new file mode 100644 index 0000000..04358ab --- /dev/null +++ b/crates/web-faith/src/warm_up.rs @@ -0,0 +1,164 @@ +//! Reading what a warm-up was asked to warm. +//! +//! Warming a name and warming an origin take their arguments loosely, so the parsing that decides +//! what was meant is worth keeping in one place, away from the verbs that act on it. + +use url::Url; + +// spec:WARM +/// Extract the bare host from a a DNS prefetch argument, ignoring any scheme, port, or path, or +/// `None` if there is no host to resolve. A DNS name carries none of those parts, so a fuller +/// string is reduced to its host. +pub fn extract_host(input: &str) -> Option { + // A string that already spells a scheme is read as the URL it is; anything else is the + // bare-host case, where a name is not a URL on its own and giving it an authority makes it + // parse as one. Telling the two apart on the scheme separator matters both ways: `example.com:8443` + // otherwise parses as a *scheme* of `example.com` with no host, and a schemed string with no host + // (`file:///path`, a bare `https://`) would have its scheme misread as a host by the fallback. + let url = if input.contains("://") { + Url::parse(input).ok()? + } else { + Url::parse(&format!("dns://{input}")).ok()? + }; + let host = url.host_str()?; + // `host_str` brackets an IPv6 literal; the resolver wants it bare. + let host = host + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .unwrap_or(host); + (!host.is_empty()).then(|| host.to_owned()) +} + +/// Reduce a preconnect argument to its origin, or `None` if it is not a connectable origin. Path, +/// query, fragment, and userinfo are stripped — the same reduction the HTTP/3 probe applies — and +/// the scheme must have a known default port so an omitted port resolves. +pub fn reduce_to_origin(input: &str) -> Option { + let mut url = Url::parse(input).ok()?; + if !url.has_host() || url.port_or_known_default().is_none() { + return None; + } + url.set_path("/"); + url.set_query(None); + url.set_fragment(None); + let _ = url.set_username(""); + let _ = url.set_password(None); + Some(url) +} + +/// The `scheme://host:port` key an origin coalesces on, with the port defaulted by scheme so +/// `https://host` and `https://host:443` are the same origin. Matches the Alt-Svc cache's key. +pub fn origin_key(url: &Url) -> String { + format!( + "{}://{}:{}", + url.scheme(), + url.host_str().unwrap_or_default(), + url.port_or_known_default().unwrap_or_default(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefetch_dns_takes_a_bare_host() { + assert_eq!(extract_host("example.com").as_deref(), Some("example.com")); + } + + #[test] + fn prefetch_dns_ignores_the_parts_a_name_does_not_have() { + // A DNS name has no scheme, port, or path, so a fuller string is reduced to its host + // rather than rejected (spec:WARM#prefetchdns). + for input in [ + "https://example.com", + "https://example.com:8443", + "https://example.com/some/path?q=1#frag", + "https://user:pass@example.com/", + "example.com:8443", + ] { + assert_eq!( + extract_host(input).as_deref(), + Some("example.com"), + "{input:?} names example.com whatever else it carries" + ); + } + } + + #[test] + fn prefetch_dns_unwraps_an_ipv6_literal() { + // `host_str` brackets an IPv6 literal, but the resolver wants it bare. + assert_eq!( + extract_host("https://[2001:db8::1]:8443").as_deref(), + Some("2001:db8::1") + ); + } + + #[test] + fn prefetch_dns_rejects_a_string_with_no_host() { + for input in ["", " ", "/just/a/path", "https://"] { + assert!( + extract_host(input).is_none(), + "{input:?} names no host to resolve" + ); + } + } + + #[test] + fn preconnect_reduces_a_longer_url_to_its_origin() { + // The same reduction the HTTP/3 probe applies (spec:WARM#preconnect). + let url = reduce_to_origin("https://user:pass@example.com/some/path?q=1#frag") + .expect("a full URL reduces to its origin"); + + assert_eq!(url.as_str(), "https://example.com/"); + assert_eq!(url.username(), "", "userinfo is stripped"); + assert_eq!(url.password(), None); + assert_eq!(url.query(), None); + assert_eq!(url.fragment(), None); + } + + #[test] + fn preconnect_defaults_the_port_by_scheme() { + // An omitted port defaults by scheme, so an origin spelled either way coalesces on one + // key (spec:WARM#preconnect). + for (bare, spelled) in [ + ("https://example.com", "https://example.com:443"), + ("http://example.com", "http://example.com:80"), + ] { + let bare = origin_key(&reduce_to_origin(bare).expect("parses")); + let spelled = origin_key(&reduce_to_origin(spelled).expect("parses")); + assert_eq!( + bare, spelled, + "the omitted port defaults to the spelled one" + ); + } + } + + #[test] + fn preconnect_keeps_distinct_origins_apart() { + // The pool caps and the warm record are per origin: scheme, host, and port together + // (spec:POOL). + let key = |input: &str| origin_key(&reduce_to_origin(input).expect("parses")); + + assert_ne!(key("https://example.com"), key("https://example.com:8443")); + assert_ne!(key("https://example.com"), key("http://example.com")); + assert_ne!(key("https://example.com"), key("https://other.example")); + } + + #[test] + fn preconnect_rejects_what_cannot_be_connected_to() { + for input in [ + "not an origin", + "", + "/just/a/path", + // No host to connect to. + "file:///etc/hosts", + // No default port for the scheme, and none given. + "unknownscheme://example.com", + ] { + assert!( + reduce_to_origin(input).is_none(), + "{input:?} is not a connectable origin" + ); + } + } +} diff --git a/index.d.ts b/index.d.ts index ee75d6b..042458a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -47,32 +47,8 @@ export declare class Agent { * Requests already in flight are not interrupted and run to completion on the connections * they hold; the reset shapes what requests started afterwards draw on. Calling it on a * closed agent does nothing, and calling it repeatedly is harmless. - * - * spec:NETCHG */ networkChanged(): void - /** - * Add a cookie into the agent. - * - * The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying - * the scheme and host they read, so this does nothing if: - * - the cookie store is disabled - * - the url is malformed - * - the cookie does not parse - * - a `__Host-` or `__Secure-` name prefix is not satisfied - * - the cookie is larger than `cookies.maxSize` - */ - addCookie(url: string, cookie: string): void - /** - * Retrieve a cookie from the store. - * - * Returns `null` if: - * - there's no cookie at this url - * - the cookie store is disabled - * - the url is malformed - * - the cookie cannot be represented as a string - */ - getCookie(url: string): string | null /** * Returns statistics gathered by this agent: * @@ -82,6 +58,30 @@ export declare class Agent { * - `bodiesFinished` */ stats(): AgentStats + /** + * Warm the DNS cache for `host`, so a later request to it skips the lookup. + * + * Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, + * port, or path in a fuller string is ignored. The returned promise resolves when the answer + * lands in the cache and never rejects, whatever happens on the network — a resolution failure + * resolves quietly, because the work is advisory. Under the system resolver there is no cache + * to warm, so the call resolves without doing anything. A malformed host throws synchronously, + * as does a call on a closed agent. + */ + prefetchDns(host: string): Promise + /** + * Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS + * setup. + * + * Mirrors the browser's `preconnect` resource hint. The argument is an origin + * (`scheme://host[:port]`); a longer URL is reduced to its origin. The warm-up sends a + * synthetic `HEAD` to the origin's root — the origin sees it — over the transport the next + * foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every + * other origin a TCP one. The returned promise resolves when the attempt finishes and never + * rejects: every network failure resolves quietly. A malformed origin throws synchronously, as + * does a call on a closed agent. + */ + preconnect(origin: string): Promise /** * Returns information on current connections open by this agent. * @@ -99,33 +99,31 @@ export declare class Agent { * Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, * `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). * The list is empty until the resolver has been used, because it reads its configuration on - * first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) + * first use, and empty for an agent using the system resolver. */ resolvers(): Array /** - * Warm the DNS cache for `host`, so a later request to it skips the lookup. + * Add a cookie into the agent. * - * Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, - * port, or path in a fuller string is ignored. The returned promise resolves when the answer - * lands in the cache and never rejects, whatever happens on the network — a resolution failure - * resolves quietly, because the work is advisory. Under the system resolver there is no cache - * to warm, so the call resolves without doing anything. A malformed host throws synchronously, - * as does a call on a closed agent. (spec:WARM) + * The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying + * the scheme and host they read, so this does nothing if: + * - the cookie store is disabled + * - the url is malformed + * - the cookie does not parse + * - a `__Host-` or `__Secure-` name prefix is not satisfied + * - the cookie is larger than `cookies.maxSize` */ - prefetchDns(host: string): Promise + addCookie(url: string, cookie: string): void /** - * Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS - * setup. + * Retrieve a cookie from the store. * - * Mirrors the browser's `preconnect` resource hint. The argument is an origin - * (`scheme://host[:port]`); a longer URL is reduced to its origin. The warm-up sends a - * synthetic `HEAD` to the origin's root — the origin sees it — over the transport the next - * foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every - * other origin a TCP one. The returned promise resolves when the attempt finishes and never - * rejects: every network failure resolves quietly. A malformed origin throws synchronously, as - * does a call on a closed agent. (spec:WARM) + * Returns `null` if: + * - there's no cookie at this url + * - the cookie store is disabled + * - the url is malformed + * - the cookie cannot be represented as a string */ - preconnect(origin: string): Promise + getCookie(url: string): string | null } export declare class AgentStats { @@ -302,7 +300,7 @@ json(): Promise * written to and `bytesWritten` counts the bytes that landed there. * * `onProgress` is reported to as the bytes land, at most every - * [`PROGRESS_INTERVAL`], with a final report once the last byte is written. The + * `PROGRESS_INTERVAL`, with a final report once the last byte is written. The * wrapper takes it from the options object; it arrives here as its own argument * because a threadsafe function cannot be a field of a `#[napi(object)]`. * @@ -328,8 +326,6 @@ toFile(path: string, options?: ToFileOptions | undefined | null, onProgress?: (( * * This is an async fn as an internal implementation detail and the wrapper makes it a * property. - * - * spec:RESP#request-timing */ timing(): Promise /** @@ -1226,73 +1222,6 @@ export declare function errorCodes(): Array export const FAITH_VERSION: string -/** - * Faith produces fine-grained errors, but maps them to a few javascript error types for fetch - * compatibility. The `.code` property on errors thrown from Faith is set to a stable name for each - * error kind, documented in this comprehensive mapping: - * - * - JS `AbortError`: - * - `Aborted` — request was aborted using `signal` - * - `Timeout` — request timed out - * - JS `NetworkError`: - * - `Network` — network error - * - `Redirect` — when the agent is configured to error on redirects - * - `ContentLengthOverrun` — a body written with `response.toFile()` exceeded the advertised `Content-Length` - * - JS `SyntaxError`: - * - `AddressParse` — IP parse error for `AgentOptions.dns.overrides` - * - `InvalidIntegrity` — SRI parse error for `RequestInit.integrity` - * - `JsonParse` — JSON parse error for `response.json()` - * - `PemParse` — PEM parse error for `AgentOptions.tls.identity` or `AgentOptions.tls.extraRoots` - * - JS `TypeError`: - * - `Closed` — a request was made on an agent that has been closed - * - `InvalidCompression` — `RequestInit.compress` naming no coding Faith can compress in - * - `InvalidHeader` — invalid header name or value - * - `InvalidMethod` — invalid HTTP method - * - `InvalidPath` — a `response.toFile()` destination that does not name a local path - * - `InvalidUrl` — invalid URL string - * - `ResponseAlreadyDisturbed` — body already read (mutually exclusive operations) - * - `ResponseBodyNull` — `response.toFile()` on a response that cannot carry a body - * - JS generic `Error`: - * - `BodyStream` — internal stream handling error - * - `Config` — invalid agent configuration - * - `FileExists` — a `response.toFile()` write refusing an occupied destination - * - `FileWrite` — the filesystem refusing a `response.toFile()` write - * - `IntegrityMismatch` — SRI checksum mismatch (with `RequestInit.integrity`) - * - * The library exports an `ERROR_CODES` object which has every error code the library throws, and - * every error thrown also has a `code` property that is set to one of those codes. So you can - * accurately respond to the exact error kind by checking its code and matching against the right - * constant from `ERROR_CODES`, instead of doing string matching on the error message, or coarse - * `instance of` matching. - * - * Due to technical limitations, when reading a body stream, reads might fail, but that error - * will not have a `code` property. - */ -export declare const enum FaithErrorKind { - Aborted = 'Aborted', - AddressParse = 'AddressParse', - BodyStream = 'BodyStream', - Closed = 'Closed', - Config = 'Config', - ContentLengthOverrun = 'ContentLengthOverrun', - FileExists = 'FileExists', - FileWrite = 'FileWrite', - IntegrityMismatch = 'IntegrityMismatch', - InvalidCompression = 'InvalidCompression', - InvalidHeader = 'InvalidHeader', - InvalidIntegrity = 'InvalidIntegrity', - InvalidMethod = 'InvalidMethod', - InvalidPath = 'InvalidPath', - InvalidUrl = 'InvalidUrl', - JsonParse = 'JsonParse', - Network = 'Network', - PemParse = 'PemParse', - Redirect = 'Redirect', - ResponseAlreadyDisturbed = 'ResponseAlreadyDisturbed', - ResponseBodyNull = 'ResponseBodyNull', - Timeout = 'Timeout' -} - export declare function faithFetch(url: string, options: FaithOptionsAndBody, signal?: AbortSignal | undefined | null, streamBody?: StreamBody | undefined | null): Promise export interface FaithOptionsAndBody { diff --git a/index.js b/index.js index e4b9573..b1423d7 100644 --- a/index.js +++ b/index.js @@ -585,7 +585,6 @@ module.exports.CredentialsOption = nativeBinding.CredentialsOption module.exports.DuplexOption = nativeBinding.DuplexOption module.exports.errorCodes = nativeBinding.errorCodes module.exports.FAITH_VERSION = nativeBinding.FAITH_VERSION -module.exports.FaithErrorKind = nativeBinding.FaithErrorKind module.exports.faithFetch = nativeBinding.faithFetch module.exports.Http3Congestion = nativeBinding.Http3Congestion module.exports.Redirect = nativeBinding.Redirect diff --git a/package.json b/package.json index 0e4dbe8..c32fd22 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ ] }, "scripts": { - "build": "napi build --release --platform", - "build:debug": "napi build --platform", + "build": "napi build --release --platform --manifest-path crates/web-faith-napi/Cargo.toml --output-dir .", + "build:debug": "napi build --platform --manifest-path crates/web-faith-napi/Cargo.toml --output-dir .", "test": "npm run build && cargo test && npm run test:only", "test:only": "tape test/*.test.js", "test:integration": "tape test/integration/*.test.js", diff --git a/src/agent.rs b/src/agent.rs deleted file mode 100644 index ae615c6..0000000 --- a/src/agent.rs +++ /dev/null @@ -1,2549 +0,0 @@ -use std::{ - fmt::Debug, - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, UdpSocket}, - str::FromStr as _, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, - time::Duration, -}; - -use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; - -use http::Version; -use http_cache_reqwest::{ - CACacheManager, Cache, CacheMode, CacheOptions, HttpCache, HttpCacheOptions, MokaCacheBuilder, - MokaManager, -}; -use hyper_util::client::legacy::connect::HttpInfo; -use moka::sync::Cache as MokaCache; -use napi::{Either, Env, bindgen_prelude::Buffer}; -use napi_derive::napi; -use reqwest::{ - Certificate, Client, Identity, Url, - cookie::CookieStore as _, - header::{HeaderMap, HeaderName, HeaderValue}, - redirect::Policy, -}; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; - -#[cfg(feature = "http3")] -use crate::alt_svc::parse_alt_svc_header; -#[cfg(feature = "http3")] -use crate::alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; -use crate::{ - async_task::faith_promise, - conn_tracker::{ConnectionInfo, ConnectionTracker}, - cookies::{ - CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, - FaithJar, - }, - dns::{DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains}, - error::{FaithError, FaithErrorKind}, - options::{PRIORITY, RequestCacheMode}, - retry::{DeadConnectionRetry, StaleAddressRetry}, -}; - -#[napi] -pub const FAITH_VERSION: &str = env!("CARGO_PKG_VERSION"); -#[napi] -pub const REQWEST_VERSION: &str = env!("REQWEST_VERSION"); -/// Custom user agent string. -/// -/// Default: `Faith/{version} reqwest/{version}`. -/// -/// You may use the `USER_AGENT` constant if you wish to prepend your own agent to the default, e.g. -/// -/// ```javascript -/// import { Agent, USER_AGENT } from '@passcod/faith'; -/// const agent = new Agent({ -/// userAgent: `YourApp/1.2.3 ${USER_AGENT}`, -/// }); -/// ``` -#[napi] -pub const USER_AGENT: &str = concat!( - "Faith/", - env!("CARGO_PKG_VERSION"), - " reqwest/", - env!("REQWEST_VERSION") -); - -/// Whether this host can bind the IPv6 wildcard (`[::]`). -/// -/// This is tested using the exact operation reqwest performs when creating the QUIC -/// endpoint with no explicit local address, so it predicts whether the default -/// QUIC bind will succeed. The result is memoised for the life of the process; while -/// IPv6 bindability can in principle change at runtime, this is considered an -/// acceptable tradeoff for performance and simplicity. -fn ipv6_wildcard_bindable() -> bool { - use std::sync::OnceLock; - static BINDABLE: OnceLock = OnceLock::new(); - *BINDABLE.get_or_init(|| { - UdpSocket::bind(SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0)).is_ok() - }) -} - -/// What the Node.js networking environment variables asked for, to apply to a reqwest client -/// builder as Node.js honours them for its own clients. This is read for every agent, so -/// `fetch()` behaves like Node's built-in fetch out of the box. -/// -/// - `NODE_EXTRA_CA_CERTS`: a path to a PEM file whose certificates are added to -/// the trust store on top of the platform roots. As in Node.js, a value that -/// is empty, or points at a file that cannot be read or parsed, is ignored -/// rather than fatal — unlike the explicit [`AgentTlsOptions::extra_roots`] -/// option, which throws. Certificates load in addition to any `extra_roots`. -/// -/// - `NODE_TLS_REJECT_UNAUTHORIZED`: when set to exactly `"0"`, TLS certificate -/// validation is disabled for the agent. This is insecure and exists only to -/// match Node.js semantics; any other value leaves validation enabled. -/// -/// - `NODE_USE_ENV_PROXY`: when set to exactly `"0"`, the agent ignores the -/// ambient proxy configuration (`HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` and the -/// OS proxy settings) that reqwest reads by default. Unlike Node.js — where -/// env-proxy support is opt-*in* and off by default — faith reads it by -/// default and treats this variable purely as an opt-*out* switch, so leaving -/// it unset (or `"1"`) keeps the existing always-on behaviour. -/// -/// `NODE_USE_SYSTEM_CA` is deliberately not honoured: faith bundles no Mozilla -/// root set, so its only default trust source is the platform store the variable -/// would toggle. `=0` could therefore only mean "trust almost nothing", which is -/// never what a caller wants, so the platform store is always used. -/// -/// Read once, at construction: AGENT has these layered on top of the explicit options when the -/// agent is built, so a client rebuilt later (spec:NETCHG) replays what was read then rather than -/// picking up an environment that has changed since. -#[derive(Debug, Clone, Default)] -struct NodeEnvRecipe { - extra_ca_certs: Vec, - accept_invalid_certs: bool, - no_proxy: bool, -} - -impl NodeEnvRecipe { - fn read() -> Self { - let mut recipe = Self::default(); - - if let Ok(path) = std::env::var("NODE_EXTRA_CA_CERTS") - && !path.is_empty() - && let Ok(bytes) = std::fs::read(&path) - && let Ok(certs) = Certificate::from_pem_bundle(&bytes) - { - recipe.extra_ca_certs = certs; - } - - recipe.accept_invalid_certs = - std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").as_deref() == Ok("0"); - recipe.no_proxy = std::env::var("NODE_USE_ENV_PROXY").as_deref() == Ok("0"); - - recipe - } - - fn apply(&self, mut client: reqwest::ClientBuilder) -> reqwest::ClientBuilder { - if !self.extra_ca_certs.is_empty() { - client = client.tls_certs_merge(self.extra_ca_certs.iter().cloned()); - } - - if self.accept_invalid_certs { - client = client.danger_accept_invalid_certs(true); - } - - if self.no_proxy { - client = client.no_proxy(); - } - - client - } -} - -#[napi(string_enum)] -#[derive(Debug, Clone, Copy)] -pub enum CacheStore { - #[napi(value = "disk")] - Disk, - - #[napi(value = "memory")] - Memory, -} - -/// Settings related to the HTTP cache. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentCacheOptions { - /// Which cache store to use: either `disk` or `memory`. - /// - /// Default: none (cache disabled). - pub store: Option, - /// If `cache.store: "memory"`, the maximum amount of items stored. - /// - /// Default: 10_000. - pub capacity: Option, - /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if - /// no cache mode is set on a request. - /// - /// Default: `"default"`. - pub mode: Option, - /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. - /// - /// Required if `cache.store: "disk"`. - pub path: Option, - /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is - /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a - /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). - /// `shared: true` is required for proxies and multi-user caches. - /// - /// Default: true. - pub shared: Option, -} - -/// Limits the cookie store enforces, from RFC 6265bis. Each is a cap; a caller who needs more room -/// raises the number. -/// -/// The `__Host-` and `__Secure-` name prefix rules are what those prefixes mean, so they always -/// apply and are not settable here: a cookie that shouldn't carry them is named without one. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentCookieOptions { - /// How far ahead of receipt a cookie may expire, in seconds. A cookie asking for longer, via - /// `Max-Age` or `Expires`, has its expiry reduced to this; a shorter one is left alone and a - /// session cookie stays a session cookie. - /// - /// Default: 34_560_000 (400 days). - pub max_age: Option, - /// The largest cookie stored, as the combined length of its name and value in bytes. A larger - /// cookie is not stored. - /// - /// Default: 4096. - pub max_size: Option, - /// How many cookies are kept for any one domain, which is a cookie's `Domain` attribute when it - /// has one and the host that set it otherwise. - /// - /// Default: 180. - pub max_per_host: Option, - /// How many cookies are kept across the whole store, bounding a server that spreads cookies - /// across subdomains to escape `maxPerHost`. - /// - /// Default: 3000. - pub max_total: Option, -} - -impl From<&AgentCookieOptions> for CookieLimits { - fn from(options: &AgentCookieOptions) -> Self { - Self { - max_age: options - .max_age - .map_or(DEFAULT_MAX_AGE, |secs| Duration::from_secs(secs.into())), - max_size: options.max_size.map_or(DEFAULT_MAX_SIZE, |n| n as usize), - max_per_host: options - .max_per_host - .map_or(DEFAULT_MAX_PER_HOST, |n| n as usize), - max_total: options.max_total.map_or(DEFAULT_MAX_TOTAL, |n| n as usize), - } - } -} - -#[napi(object)] -#[derive(Debug, Clone)] -pub struct DnsOverride { - pub domain: String, - pub addresses: Vec, -} - -/// Settings related to DNS. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentDnsOptions { - /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on - /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native - /// fetch does, this should be your first port of call. - /// - /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory - /// DNS cache, and may lead to worse performance even discounting the cache. - /// - /// Default: false. - /// - /// [Hickory]: https://hickory-dns.org/ - pub system: Option, - /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. - /// - /// Will throw if addresses are in invalid formats. You may provide a port number as part of the - /// address, it will default to port 0 otherwise, which will select the conventional port for the - /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port - /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively - /// blocks that domain from this agent. - /// - /// Default: no overrides. - pub overrides: Option>, - /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to - /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over - /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on - /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the - /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. - /// - /// The encrypted transports always authenticate the resolver. A URL fragment names the - /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates - /// against the hostname; a bare-IP host authenticates against the address itself. - /// - /// Servers are queried in order, a later one reached only once those before it fail. Setting - /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or - /// its scheme is not one of the above, and combining it with `dns.system` throws. - /// - /// Default: system discovery. - pub servers: Option>, - /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead - /// servers costs a single timeout rather than one per server. - /// - /// Default: 5000. - pub timeout: Option, - /// Replace the system's search list, the domains appended to a name that is not fully - /// qualified. Independent of `dns.servers`. - /// - /// Default: the system's search list. - pub search_domains: Option>, - /// How many dots a name must contain before it is tried as given, ahead of the search list. - /// Independent of `dns.servers`. - /// - /// Default: the system's setting. - pub ndots: Option, - /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. - /// - /// Default: platform convention. - pub hosts_file: Option, - /// Further domains to exempt from the configured or encrypted resolver, for the internal - /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's - /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. - /// - /// Default: no extra exemptions. - pub exempt_domains: Option>, - /// Serve an expired cache entry immediately and refresh it in the background, rather than making - /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is - /// almost always still correct, and a connect failure against one that has moved re-resolves and - /// attempts the request again. - /// - /// Set `false` for an agent that must never connect to an address it knows to be out of date: an - /// expired entry is discarded and the lookup blocks on a fresh answer. - /// - /// Default: true. - pub serve_stale: Option, - /// How far past expiry an answer may still be served, in milliseconds. An entry older than this - /// is discarded rather than served: an answer stale enough stops being evidence about where the - /// host is, and a refresh still failing after that long is the case where the address most likely - /// did change. - /// - /// Default: 3600000 (one hour). - pub max_stale: Option, -} - -/// Sets the default headers for every request. -/// -/// If header names or values are invalid, they are silently omitted. -/// Sensitive headers (e.g. `Authorization`) should be marked. -/// -/// Default: none. -#[napi(object)] -#[derive(Debug, Clone)] -pub struct Header { - pub name: String, - pub value: String, - pub sensitive: Option, -} - -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, Default)] -pub enum Http3Congestion { - #[napi(value = "cubic")] - #[default] - Cubic, - - #[napi(value = "bbr1")] - Bbr1, -} - -/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc -/// cache so the first request to this host will attempt HTTP/3 immediately. -#[napi(object)] -#[derive(Debug, Clone)] -pub struct Http3Hint { - /// The hostname (e.g., "example.com"). - pub host: String, - /// The port number (e.g., 443). - pub port: u16, -} - -/// Settings related to HTTP/3. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentHttp3Options { - /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the - /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with - /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by - /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise - /// bandwidth use and optimises for round-trip time, while ignoring packet loss. - /// - /// In some networks, BBR can lead to pathological degradation of overall network conditions, by - /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, - /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). - /// - /// [2]: https://github.com/quinn-rs/quinn/issues/1254 - /// - /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. - pub congestion: Option, - /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that - /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ - /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith - /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). - /// - /// Default: 30. - pub max_idle_timeout: Option, - /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc - /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. - /// - /// Default: true. - pub upgrade_enabled: Option, - /// Whether advertised HTTP/3 endpoints are verified with a background probe - /// before any foreground request is routed to them. - /// - /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say - /// there is UDP connectivity between you and it. Without probing, the next - /// request after an advertisement attempts HTTP/3 inline, and on a silently - /// broken UDP path it stalls until the QUIC idle timeout or - /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per - /// failure cooldown for as long as the path stays broken. - /// - /// With probing (the default), requests keep using TCP until a background - /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the - /// connection pool, so the first upgraded request rides the probe's warm - /// connection. A broken path costs one failed background request per - /// cooldown and no foreground latency at all. - /// - /// The probe is a synthetic request the server will see in its logs. Set - /// this to `false` to restore the inline upgrade if that is unacceptable - /// (per-request billing, easily-alarmed WAFs). - /// - /// `hints` are exempt either way: a hint is your own assertion, so the first - /// request to a hinted origin speaks HTTP/3 immediately, which is also what - /// makes h3-only origins (no TCP listener) work. - /// - /// Default: true. - pub upgrade_probe: Option, - /// Ceiling on how long a background HTTP/3 probe may take before the origin - /// is treated as failed, in **milliseconds**. - /// - /// This bounds background work only — no foreground request ever waits on a - /// probe — so it can afford to be generous: a healthy handshake plus HEAD - /// completes in one or two round trips. Set to 0 to leave probes bounded - /// only by the QUIC idle timeout. - /// - /// Default: 5000 (5 seconds). - pub upgrade_probe_timeout: Option, - /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than - /// its TCP path by this factor. Set to 0 to disable path-time demotion. - /// - /// Faith keeps a per-origin moving average of time-to-response-headers for - /// each protocol family. HTTP/3 is preferred at parity and when moderately - /// slower — its advantages (no head-of-line blocking, connection migration) - /// pay off beyond the average — so this factor should stay well above 1. - /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC - /// average must also exceed the TCP one by an absolute 10ms so LAN-fast - /// origins don't flap on noise. - /// - /// A demoted origin is not treated as broken: it re-enters through a - /// background probe after `upgradeSlowTtl`, asking whether the path has - /// improved at zero foreground cost. - /// - /// Default: 2.5. - pub upgrade_slow_factor: Option, - /// How long (in seconds) a path-time demotion holds before the origin is - /// re-evaluated. See `upgradeSlowFactor`. - /// - /// Default: 600 (10 minutes). - pub upgrade_slow_ttl: Option, - /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. - /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. - /// - /// Default: 86400 (24 hours). - pub upgrade_advertised_ttl: Option, - /// How long (in seconds) to cache a confirmed working HTTP/3 connection. - /// - /// Default: 86400 (24 hours). - pub upgrade_confirmed_ttl: Option, - /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this - /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends - /// Alt-Svc headers. - /// - /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an - /// origin whose UDP path is blocked for good is retried less and less often instead of - /// forever at this interval. A confirmed HTTP/3 response ends the run. - /// - /// Default: 300 (5 minutes). - pub upgrade_failed_ttl: Option, - /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of - /// `upgradeFailedTtl`. - /// - /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, - /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat - /// cooldown that never backs off. - /// - /// Default: 3600 (1 hour). - pub upgrade_failed_max_ttl: Option, - /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, - /// demote an origin back to TCP. - /// - /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request - /// cancelled via `AbortSignal` never produces that signal, so without this an - /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as - /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a - /// sustained run of them demotes the origin, and any successful HTTP/3 response - /// resets the count. - /// - /// Strikes must land within about a minute of each other to count towards a - /// run. A retry loop whose backoff exceeds that window never accumulates one, - /// so callers with a long backoff should set this to 1 for immediate demotion - /// on the first cancelled attempt. - /// - /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that - /// carries small datagrams but drops full-size ones (an MTU blackhole, say). - /// Response headers still arrive, so the attempt resolves and every mechanism - /// here counts it a success — the transfer then stalls partway through the - /// body, where nothing is watching. `maxIdleTimeout` or the request's own - /// timeout is what ends such a request, and the origin stays on HTTP/3. - /// - /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. - /// - /// Default: 3. - pub upgrade_cancel_strikes: Option, - /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is - /// given up on and the request is retried over TCP, in **milliseconds**. - /// - /// Note the unit: the other `upgrade*` settings are in seconds, but this one - /// is in milliseconds to match the `timeout` settings, because useful values - /// are sub-second. - /// - /// This bounds the wait for response headers, not the response body, so a slow - /// body is unaffected. - /// - /// The default is high, but not unconditionally inert: `maxIdleTimeout` is - /// configurable up to 120 seconds, and above 60 seconds this deadline becomes - /// the effective ceiling. Even below that, "QUIC's own idle timeout fires - /// first" only holds while the connection is idle — a transfer still running - /// past this deadline keeps the connection active, so no idle timeout is - /// coming to end it. - /// - /// On expiry the request is retried over TCP, which means it is re-sent: a - /// timeout often means the server is still processing, so a slow - /// non-idempotent request (a POST, say) can end up delivered twice. Lowering - /// this value trades that double-submission risk for faster recovery when a - /// UDP path breaks. Anyone setting it low should confirm their slowest - /// legitimate time to response headers fits well inside the budget. - /// - /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle - /// timeout and the request's own timeout. - /// - /// Default: 60000 (60 seconds). - pub upgrade_attempt_timeout: Option, - /// Connect to the port a server advertises HTTP/3 on, even when it differs from - /// the origin's own port. **This is not standards-compliant**; it is off by - /// default. - /// - /// An `Alt-Svc` advertisement names a network endpoint for the origin, so - /// honouring one correctly means connecting to that endpoint while still - /// sending the *origin's* authority. reqwest cannot express that — it derives - /// the HTTP/3 connect target from the request URI's authority (tracked - /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). - /// So by default Faith does not upgrade at all when the advertised port - /// differs, rather than guessing that the origin's own port also speaks - /// HTTP/3. - /// - /// Setting this to `true` upgrades anyway, by rewriting the request's port to - /// the advertised one. That gets HTTP/3 working today against servers you - /// control, at the cost of three deviations you should be aware of: - /// - /// - The request's `Host`/`:authority` carries the advertised port instead of - /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) - /// forbids. Servers that route on authority may misroute or reject; servers - /// that ignore it are unaffected. - /// - `response.url` reports the port actually connected to. - /// - `redirected` ignores port differences, since the rewritten port would - /// otherwise look like a redirect on every request. - /// - /// TLS is unaffected: certificates are still validated against the origin's - /// hostname. Only the port changes. - /// - /// Default: `false`. - pub upgrade_follow_advertised_port: Option, - /// Maximum number of origins to track in the Alt-Svc cache. - /// - /// Default: 10000. - pub upgrade_cache_capacity: Option, - /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache - /// on agent initialization, so the first request to these hosts will attempt HTTP/3. - pub hints: Option>, - /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for - /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. - /// - /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it - /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for - /// HTTP/3 only. - /// - /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). - pub connection_window: Option, - /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload - /// throughput the way the receive windows bound download. The origin's own flow control - /// applies on top of this, so it is a ceiling rather than a grant. - /// - /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window - /// the peer advertises, with no local cap to set. - /// - /// Default: 10 MB (quinn's own default). - pub send_window: Option, -} - -/// Settings related to HTTP/2. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentHttp2Options { - /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for - /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. - /// - /// Ignored when `adaptiveWindow` is on. - /// - /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it - /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for - /// HTTP/2 only. - /// - /// Ignored when `adaptiveWindow` is on. - /// - /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). - pub connection_window: Option, - /// Replace HTTP/2's static windows with windows that start small and grow towards a - /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. - /// - /// This is off by default, and turning it on is usually the wrong move. A fresh connection - /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample - /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and - /// carries *less* throughput than the static window for all but the largest transfers. It - /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. - /// - /// Its one real advantage is memory: it holds a large window open only on connections that - /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling - /// buys the same throughput from the first byte. - /// - /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. - /// - /// Default: `false`. - pub adaptive_window: Option, -} - -/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentFlowControlOptions { - /// Maximum bytes an origin may send on any one stream before it must wait for Faith to - /// acknowledge them, for HTTP/2 and HTTP/3 alike. - /// - /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. - /// The default follows browser practice, and is deliberately at the conservative end of it: - /// a pooled server-side client can hold many connections across many origins, so - /// per-connection memory multiplies harder here than in a browser. - /// - /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. - /// - /// Default: 6 MiB. - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one connection before it must - /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. - /// - /// This is larger than `streamWindow` so concurrent streams on one connection share the - /// connection's headroom, while still bounding the worst-case buffering of a connection - /// carrying many concurrent requests. - /// - /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against - /// the other. - /// - /// Default: 15 MiB. - pub connection_window: Option, -} - -/// Per-stream receive window applied to both protocols when nothing overrides it (spec:FLOW). -/// -/// Chrome's shape: 6 MiB stream inside a 15 MiB connection. Picked over a larger window that -/// measured faster because it is what browsers have proven at scale, and because a pooled -/// server-side client multiplies per-connection memory across far more connections. -pub(crate) const DEFAULT_STREAM_WINDOW: u32 = 6 * 1024 * 1024; - -/// Whole-connection receive window applied to both protocols when nothing overrides it (spec:FLOW). -pub(crate) const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; - -// Concurrent streams share the connection's headroom, so the asymmetry is the point of the -// defaults rather than an accident of the numbers (spec:FLOW#common-windows). -const _: () = assert!(DEFAULT_CONNECTION_WINDOW > DEFAULT_STREAM_WINDOW); - -/// The flow-control windows to apply, once the common group, the per-protocol overrides, and the -/// defaults have been reconciled (spec:FLOW#per-protocol-windows). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ResolvedWindows { - pub stream: u32, - pub connection: u32, -} - -/// Reconcile one protocol's windows: its own setting wins over the common one, which wins over the -/// default (spec:FLOW#per-protocol-windows). -pub(crate) fn resolve_windows( - common: Option<&AgentFlowControlOptions>, - protocol_stream: Option, - protocol_connection: Option, -) -> ResolvedWindows { - ResolvedWindows { - stream: protocol_stream - .or_else(|| common.and_then(|c| c.stream_window)) - .unwrap_or(DEFAULT_STREAM_WINDOW), - connection: protocol_connection - .or_else(|| common.and_then(|c| c.connection_window)) - .unwrap_or(DEFAULT_CONNECTION_WINDOW), - } -} - -/// Settings related to the connection pool. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentPoolOptions { - /// How many seconds of inactivity before a connection is closed. - /// - /// Default: 90 seconds. - pub idle_timeout: Option, - /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed - /// to keep the idle connections (per host) under that number. - /// - /// Default: `null` (no limit). - pub max_idle_per_host: Option, -} - -/// Switches that depart from standard behaviour on purpose. This is a nested object. -/// -/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule -/// forbids. All of them are off by default, so an agent constructed with no options is -/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise -/// established that what the rule guards against does not apply to them: turning one on means -/// requests may fail against origins that expect the standard behaviour. -#[napi(object)] -#[derive(Debug, Clone, Copy, Default)] -pub struct AgentQuirksOptions { - /// Allow a streaming request body to be sent over an HTTP/1.x connection. - /// - /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read - /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x - /// origin or an intermediary on the path may refuse it. With this on, such a body sends over - /// whichever protocol the connection negotiates. - /// - /// Default: false. - pub h1_request_streaming: Option, -} - -/// Determines the behavior in case the server replies with a redirect status. -/// One of the following values: -/// -/// - `follow`: automatically follow redirects. Faith limits this to 10 redirects. -/// - `error`: reject the promise with a network error when a redirect status is returned. -/// - ~~`manual`~~: not supported. -/// - `stop`: (Faith custom) don't follow any redirects, return the responses. -/// -/// Defaults to `follow`. -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, Default)] -pub enum Redirect { - #[napi(value = "follow")] - #[default] - Follow, - - #[napi(value = "error")] - Error, - - #[napi(value = "manual")] - Manual, - - #[napi(value = "stop")] - Stop, -} - -/// Timeouts for requests made with this agent. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Copy, Default)] -pub struct AgentTimeoutOptions { - /// Set a timeout for only the connect phase, in milliseconds. - /// - /// Default: none. - pub connect: Option, - /// Set a timeout for read operations, in milliseconds. - /// - /// The timeout applies to each read operation, and resets after a successful read. This is more - /// appropriate for detecting stalled connections when the size isn't known beforehand. - /// - /// Default: none. - pub read: Option, - /// Set a timeout for the entire request-response cycle, in milliseconds. - /// - /// The timeout applies from when the request starts connecting until the response body has finished. - /// Also considered a total deadline. - /// - /// Default: none. - pub total: Option, -} - -/// Settings related to the connection pool. This is a nested object. -#[napi(object)] -#[derive(Default)] -pub struct AgentTlsOptions { - /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet - /// of application data alongside the opening packet of the TLS handshake. That can enable the server - /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant - /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should - /// really only be used for static assets or to squeeze out the last drop of performance for endpoints - /// that are replay-safe. - /// - /// Default: false. - pub early_data: Option, - /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also - /// called mutual TLS or mTLS) authentication. - /// - /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The - /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options - /// that will cause the `Agent` constructor to throw if the input is in the wrong format. - pub identity: Option>, - /// Disables plain-text HTTP. - /// - /// Default: false. - pub required: Option, - /// Additional PEM-formatted root certificates to trust, on top of the platform's - /// trust store. Each entry may be a PEM bundle containing multiple certificates. - /// - /// This is mainly useful for connecting to servers with self-signed or private-CA - /// certificates, such as internal services or local test servers. This is one of the - /// few options that will cause the `Agent` constructor to throw if the input is in - /// the wrong format. - pub extra_roots: Option>>, -} - -impl Debug for AgentTlsOptions { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AgentTlsOptions") - .field("early_data", &self.early_data) - .field("identity", &"[sensitive]") - .field("required", &self.required) - .field("extra_roots", &self.extra_roots.as_ref().map(|r| r.len())) - .finish() - } -} - -impl Clone for AgentTlsOptions { - fn clone(&self) -> Self { - Self { - early_data: self.early_data.clone(), - identity: self.identity.as_ref().map(|either| match either { - Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), - Either::B(string) => Either::B(string.clone()), - }), - required: self.required.clone(), - extra_roots: self.extra_roots.as_ref().map(|roots| { - roots - .iter() - .map(|either| match either { - Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), - Either::B(string) => Either::B(string.clone()), - }) - .collect() - }), - } - } -} - -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentOptions { - /// Settings related to the HTTP cache. This is a nested object. - pub cache: Option, - /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and - /// included in additional requests. - /// - /// `true` enables the store with the default limits; an options object enables it and tunes them, - /// so `{}` means the same as `true`. - /// - /// Default: `false`. - /// - /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add - /// and retrieve cookies from the store. - pub cookies: Option>, - /// Settings related to DNS. This is a nested object. - pub dns: Option, - /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. - /// - /// Setting these is the normal way to tune windows: one value applies to whichever protocol - /// a request negotiates, so throughput doesn't change when an origin upgrades from one to - /// the other. The `http2` and `http3` groups override them per protocol. - pub flow_control: Option, - /// Sets the default headers for every request. - /// - /// If header names or values are invalid, they are silently omitted. - /// Sensitive headers (e.g. `Authorization`) should be marked. - /// - /// Default: none. - pub headers: Option>, - /// Settings related to HTTP/2. This is a nested object. - pub http2: Option, - /// Settings related to HTTP/3. This is a nested object. - pub http3: Option, - /// Bind outgoing sockets to this local IP address before connecting. - /// - /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that - /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — - /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and - /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to - /// force a specific source address. Throws if the value does not parse as an IP address. - /// - /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). - pub local_address: Option, - /// Settings related to the connection pool. This is a nested object. - pub pool: Option, - /// Switches that depart from standard behaviour on purpose. This is a nested object. - pub quirks: Option, - /// Determines the behavior in case the server replies with a redirect status. - pub redirect: Option, - /// Timeouts for requests made with this agent. This is a nested object. - pub timeout: Option, - /// Settings related to the connection pool. This is a nested object. - pub tls: Option, - /// Custom user agent string. - /// - /// Default: `Faith/{version} reqwest/{version}`. - pub user_agent: Option, -} - -#[derive(Debug, Default)] -pub(crate) struct InnerAgentStats { - pub requests_sent: AtomicU64, - pub responses_received: AtomicU64, - pub bodies_started: AtomicU64, - pub bodies_finished: AtomicU64, -} - -#[napi] -#[derive(Debug, Clone, Default)] -pub struct AgentStats { - pub requests_sent: i64, - pub responses_received: i64, - /// Number of response body streams that have been started (converted from raw body to stream). - /// This happens when `.body`, `.text()`, `.json()`, `.bytes()`, or similar methods are called. - pub bodies_started: i64, - /// Number of response body streams that have been fully consumed. - /// When `bodies_started - bodies_finished > 0`, there are bodies holding connections open. - pub bodies_finished: i64, -} - -/// One entry of `Agent.resolvers()`: a DNS server the agent resolves through (spec:OBS#resolvers). -#[napi(object)] -#[derive(Debug, Clone)] -pub struct ResolverInfo { - /// The server's address, as `ip:port`. - pub address: String, - /// The transport in use: `udp`, `tcp`, `tls`, `https`, `quic`, or `h3`. - pub transport: String, - /// How the transport was arrived at: `configured` by the caller, or `conventional` DNS. - pub source: String, -} - -/// The `Agent` interface of the Faith API represents an instance of an HTTP client. Each `Agent` has -/// its own options, connection pool, caches, etc. There are also conveniences such as `headers` for -/// setting default headers on all requests done with the agent, and statistics collected by the agent. -/// -/// Re-using connections between requests is a significant performance improvement: not only because -/// the TCP and TLS handshake is only performed once across many different requests, but also because -/// the DNS lookup doesn't need to occur for subsequent requests on the same connection. Depending on -/// DNS technology (DoH and DoT add a whole separate handshake to the process) and overall latency, -/// this can not only speed up requests on average, but also reduce system load. -/// -/// For this reason, and also because in browsers this behaviour is standard, **all** requests with -/// Faith use an `Agent`. For `fetch()` calls that don't specify one explicitly, a global agent with -/// default options is created on first use. -/// -/// There are a lot more options that could be exposed here; if you want one, open an issue. -#[napi] -#[derive(Debug, Clone)] -pub struct Agent { - /// `None` once [`Agent::close`] has been called. The heavy resources - /// (connection pool, DNS resolver, background tasks) live inside this - /// client, so dropping it is what actually releases them. - pub(crate) client: Option, - /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A - /// `preconnect` warm-up sends its synthetic request here rather than through the middleware - /// stack, which bypasses the HTTP cache and the Alt-Svc layer (and so keeps the warm-up out of - /// request accounting), while still pooling the connection foreground requests reuse. `None` - /// once the agent is closed. (spec:WARM) - pub(crate) raw_client: Option, - /// Faith's DNS resolver, shared with [`Self::client`] so `prefetchDns` warms the cache requests - /// read. `None` under the system resolver, where there is no such cache. (spec:WARM) - pub(crate) dns_resolver: Option, - /// Origins with a warm-up connection opened within the pool idle window, so a repeat - /// `preconnect` does no new work. Keyed by `scheme://host:port`; entries expire with the idle - /// timeout. (spec:WARM) - pub(crate) warmed: MokaCache, - /// Single-flight claims for in-flight `preconnect` warm-ups, so concurrent calls for the same - /// origin do not open duplicate connections. (spec:WARM) - pub(crate) warming: MokaCache, - /// Bumped by `networkChanged`, so a warm-up that was in flight across the signal does not - /// record its origin as warm: its connection went into the pool that was just dropped - /// (spec:NETCHG#reach-across-the-subsystems). - pub(crate) warm_generation: Arc, - pub(crate) cookie_jar: Option>, - pub(crate) stats: Arc, - pub(crate) conn_tracker: Arc, - #[cfg(feature = "http3")] - #[allow(dead_code)] - pub(crate) alt_svc_cache: Option>, - /// Held so `close()` can abort in-flight background probes: each one owns a - /// clone of the raw client, which would otherwise keep the connection pool - /// alive past close for up to the probe timeout. - #[cfg(feature = "http3")] - pub(crate) h3_prober: Option>, - /// Mirrors `http3.upgradeFollowAdvertisedPort`. Lives here because `fetch` needs - /// it to stop a rewritten port from being reported as a redirect. - pub(crate) h3_follow_advertised_port: bool, - /// Mirrors `http3.upgradeEnabled`. A warm-up needs it to route the way a foreground request - /// would: with the upgrade machinery off, nothing upgrades, whatever the caches hold. - /// (spec:WARM#preconnect) - #[cfg(feature = "http3")] - pub(crate) h3_upgrade_enabled: bool, - /// Mirrors `quirks.h1RequestStreaming`. `fetch` consults it to decide whether a streaming - /// request body may go out over HTTP/1.x (spec:QUIRK#http-1-x-request-body-streaming). - pub(crate) quirk_h1_request_streaming: bool, - /// The agent's default `Accept-Encoding`, if one was set among its default headers. - /// `fetch` consults it to decide which codings to decode when a request adds none of - /// its own (see [`crate::encoding`]). - pub(crate) default_accept_encoding: Option, - /// The agent's default `Content-Encoding`, if one was set among its default headers. - /// `fetch` consults it when the `compress` option layers a coding on top of what a - /// request already declares, since setting the joined value on the request would - /// otherwise displace this default rather than build on it (spec:ENC). - pub(crate) default_content_encoding: Option, - /// Whether a `Priority` header sits among the agent's default headers. `fetch` consults - /// it so that default wins over the header the `priority` option would derive. - pub(crate) has_default_priority: bool, - /// How to build this agent's clients, so `networkChanged` can build them again - /// (spec:NETCHG). Shared rather than cloned per agent clone: every clone builds the same - /// client from the same recipe, and `fetch` clones the agent per request. - pub(crate) recipe: Arc, -} - -/// The HTTP cache store to install on a client, held as the built manager rather than as the -/// options that produced it. -/// -/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and -/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while -/// building a fresh one from the same options would empty an in-memory cache — which is why a -/// client rebuilt for a network change clones this (spec:NETCHG#what-the-signal-keeps). -#[derive(Debug, Clone)] -enum HttpCacheStore { - Disk(CACacheManager), - Memory(MokaManager), -} - -/// The HTTP cache middleware to install. -#[derive(Debug, Clone)] -struct HttpCacheRecipe { - mode: CacheMode, - options: HttpCacheOptions, - store: HttpCacheStore, -} - -/// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not -/// here: it belongs to the agent and outlives any one client (spec:NETCHG). -#[cfg(feature = "http3")] -#[derive(Debug, Clone)] -struct H3UpgradeRecipe { - enabled: bool, - attempt_timeout: Option, - probe: bool, - probe_timeout: Option, -} - -/// Everything needed to build the agent's clients, validated once at construction. -/// -/// This exists because `networkChanged` has to drop the connection pool, and reqwest offers no way -/// to do that short of dropping the client, so the client has to be buildable more than once -/// (spec:NETCHG). `AgentOptions` cannot serve: validating it consumes it, and it carries napi -/// values belonging to the JS call that passed them. So validation happens once, into these -/// Rust-native fields, and building a client is a pure function of them and the agent's shared -/// state. -#[derive(Debug, Clone)] -pub(crate) struct ClientRecipe { - user_agent: String, - local_address: Option, - default_headers: Option, - /// Under the system resolver no hickory resolver is installed at all (spec:DNS). - dns_system: bool, - /// Validated at construction, and applied whichever resolver is in use: reqwest layers - /// overrides on top of the resolver it was given (spec:DNS#overrides). - dns_overrides: Vec<(String, Vec)>, - http2_adaptive_window: bool, - /// `None` when adaptive windowing owns the windows itself (spec:FLOW#adaptive-windowing). - http2_windows: Option, - #[cfg(feature = "http3")] - http3_max_idle_timeout: Duration, - #[cfg(feature = "http3")] - http3_windows: ResolvedWindows, - #[cfg(feature = "http3")] - http3_congestion_bbr: bool, - #[cfg(feature = "http3")] - http3_send_window: Option, - pool_idle_timeout: Option, - pool_max_idle_per_host: Option, - redirect: Option, - connect_timeout: Option, - read_timeout: Option, - total_timeout: Option, - /// Only reachable over QUIC, so only applied when HTTP/3 is compiled in. - #[cfg(feature = "http3")] - tls_early_data: Option, - tls_identity: Option, - tls_required: Option, - tls_extra_roots: Vec, - node_env: NodeEnvRecipe, - http_cache: Option, - #[cfg(feature = "http3")] - h3_upgrade: H3UpgradeRecipe, -} - -/// Point the resolver's `HTTPS` record reading at the upgrade layer, so a record advertising -/// `alpn="h3"` makes an origin probe-worthy before anything has connected to it. -/// -/// A no-op without all the parts: the system resolver is not Faith's to add a query to, and with -/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one -/// (spec:DNS#https-records). -/// -/// Re-called on a network change, where the prober is rebuilt with the client it sends on. -#[cfg(feature = "http3")] -fn install_https_sink( - dns_resolver: Option<&FaithResolver>, - alt_svc_cache: Option<&Arc>, - prober: Option<&Arc>, - upgrade_enabled: bool, -) { - if !upgrade_enabled { - return; - } - let (Some(resolver), Some(cache)) = (dns_resolver, alt_svc_cache) else { - return; - }; - resolver.set_https_sink(Arc::new(crate::alt_svc::H3HttpsSink::new( - Arc::clone(cache), - prober, - ))); -} - -/// The clients [`ClientRecipe::build`] produces, and the prober that sends on them. -struct BuiltClients { - client: ClientWithMiddleware, - raw_client: Client, - #[cfg(feature = "http3")] - prober: Option>, -} - -impl ClientRecipe { - /// The window an idle pooled connection lives in, which is also how long a warm-up counts as - /// warm and how long a connection stays listed (spec:POOL, spec:WARM, spec:OBS). - fn conn_timeout(&self) -> Duration { - // reqwest's own default, mirrored because the pool timeout it applies is not readable. - self.pool_idle_timeout.unwrap_or(Duration::from_secs(90)) - } - - /// Build a fresh client and raw client, around state the agent already holds. - /// - /// Everything passed in survives a rebuild by being shared rather than rebuilt: the cookie - /// jar, the resolver (and so its cache), and the HTTP/3 origin knowledge all belong to the - /// agent rather than to any one client (spec:NETCHG#what-the-signal-keeps). - fn build( - &self, - cookie_jar: Option<&Arc>, - dns_resolver: Option<&FaithResolver>, - #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, - ) -> Result { - let mut client = Client::builder() - .tls_info(true) - .tls_sslkeylogfile(true) - .user_agent(self.user_agent.clone()); - - if let Some(ip) = self.local_address { - client = client.local_address(ip); - } - - if let Some(jar) = cookie_jar { - client = client.cookie_provider(jar.clone()); - } - - // Registered whichever resolver is in use: reqwest layers overrides on top of the - // resolver it was given, so they take effect under the system resolver too - // (spec:DNS#overrides). - for (domain, addresses) in &self.dns_overrides { - client = client.resolve_to_addrs(domain, addresses); - } - - if self.dns_system { - client = client.no_hickory_dns(); - } else if let Some(resolver) = dns_resolver { - client = client.dns_resolver(resolver.clone()); - } - - if let Some(headers) = &self.default_headers { - client = client.default_headers(headers.clone()); - } - - if self.http2_adaptive_window { - client = client.http2_adaptive_window(true); - } else if let Some(windows) = self.http2_windows { - client = client - .http2_initial_stream_window_size(windows.stream) - .http2_initial_connection_window_size(windows.connection); - } - - #[cfg(feature = "http3")] - { - client = client - .http3_max_idle_timeout(self.http3_max_idle_timeout) - .http3_stream_receive_window(self.http3_windows.stream.into()) - .http3_conn_receive_window(self.http3_windows.connection.into()); - - if self.http3_congestion_bbr { - client = client.http3_congestion_bbr(); - } - - if let Some(send_window) = self.http3_send_window { - client = client.http3_send_window(send_window.into()); - } - } - - if let Some(timeout) = self.pool_idle_timeout { - client = client.pool_idle_timeout(Some(timeout)); - } - - if let Some(max_idle) = self.pool_max_idle_per_host { - client = client.pool_max_idle_per_host(max_idle); - } - - match self.redirect { - // follow is the default, and we ignore manual - None | Some(Redirect::Follow | Redirect::Manual) => {} - Some(Redirect::Error) => { - client = client.redirect(Policy::custom(|attempt| { - // Hand reqwest the error unboxed: it boxes for us, and boxing first would - // put a `Box` in the source chain, which does not downcast - // back to `FaithError` when we come to recover the kind as a `code`. - attempt.error(FaithError::from(FaithErrorKind::Redirect)) - })); - } - Some(Redirect::Stop) => { - client = client.redirect(Policy::none()); - } - } - - if let Some(timeout) = self.connect_timeout { - client = client.connect_timeout(timeout); - } - - if let Some(timeout) = self.read_timeout { - client = client.read_timeout(timeout); - } - - if let Some(timeout) = self.total_timeout { - client = client.timeout(timeout); - } - - #[cfg(feature = "http3")] - if let Some(early_data) = self.tls_early_data { - client = client.tls_early_data(early_data); - } - - if let Some(identity) = &self.tls_identity { - client = client.identity(identity.clone()); - } - - if let Some(https_only) = self.tls_required { - client = client.https_only(https_only); - } - - if !self.tls_extra_roots.is_empty() { - client = client.tls_certs_merge(self.tls_extra_roots.iter().cloned()); - } - - client = self.node_env.apply(client); - - let raw_client = client - .build() - .map_err(|e| FaithError::new(FaithErrorKind::Config, Some(format!("{e:?}"))))?; - let mut client = ClientBuilder::new(raw_client.clone()); - - #[cfg(feature = "http3")] - let prober = { - // The prober sends on the *raw* client, deliberately: it must skip - // the HTTP cache (a replayed cached response would fake a - // confirmation) and the Alt-Svc middleware (no recursion), while - // sharing the h3 connection pool so a successful probe leaves a warm - // connection for the foreground. Only built when both the upgrade - // machinery and probing are on. - alt_svc_cache - .filter(|_| self.h3_upgrade.enabled && self.h3_upgrade.probe) - .map(|cache| { - Arc::new(H3Prober::new( - raw_client.clone(), - cache.clone(), - self.h3_upgrade.probe_timeout, - )) - }) - }; - - if let Some(cache) = &self.http_cache { - // The two arms differ only in the manager's type, which `HttpCache` is generic over, - // so they cannot share a constructor without boxing the manager. - client = match &cache.store { - HttpCacheStore::Disk(manager) => client.with(Cache(HttpCache { - mode: cache.mode, - manager: manager.clone(), - options: cache.options.clone(), - })), - HttpCacheStore::Memory(manager) => client.with(Cache(HttpCache { - mode: cache.mode, - manager: manager.clone(), - options: cache.options.clone(), - })), - }; - } - - // Registered *after* the HTTP cache, so the Alt-Svc layer sits inside it: - // `reqwest-middleware` runs the first-registered middleware outermost. Being - // inside matters three times over. - // - // A cache hit is served without calling inward, so it never reaches this - // layer. From outside, it would: `http-cache` rebuilds a cached response with - // the *stored* HTTP version, so a response cached from an HTTP/3 exchange - // replays as HTTP/3 and would be taken for a live one — confirming HTTP/3, - // clearing cancellation strikes and refreshing the confirmed TTL on evidence - // that never touched the network. - // - // The cache middleware also buffers the whole response body inside its own - // call inward. From outside, the HTTP/3 attempt guarded here would span that - // buffering, so a cancellation during body download would count as a strike, - // and `upgradeAttemptTimeout` would bound body transfer rather than the wait - // for response headers. - // - // And cache keys are computed before this layer runs, so an advertised-port - // rewrite cannot split HTTP/3 and TCP responses across separate entries. - #[cfg(feature = "http3")] - if let Some(alt_svc_cache) = alt_svc_cache { - client = client.with(AltSvcMiddleware::new( - alt_svc_cache.clone(), - self.h3_upgrade.enabled, - self.h3_upgrade.attempt_timeout, - prober.clone(), - )); - } - - // Outside the dead-connection layer, so a re-resolved attempt gets the same - // treatment as the original one: the two answer different questions, and a - // fresh address deserves its own chance to draw a dead pooled connection. - // Inside the Alt-Svc and cache layers for the reason given below. - client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); - - // Registered last, so it sits innermost and wraps nothing but the exchange - // itself. Inside the Alt-Svc layer rather than outside it, because each - // protocol attempt is its own connection and deserves its own retry: a - // failed HTTP/3 attempt is the fallback's business, and re-running the - // upgrade decision from out here would re-attempt HTTP/3 on a path already - // judged dead and record a second failure against the origin for it. Inside - // the HTTP cache for the same reason as the Alt-Svc layer -- a retry should - // re-send the request, not redo the cache lookup that led to it. - client = client.with(DeadConnectionRetry); - - Ok(BuiltClients { - client: client.build(), - raw_client, - #[cfg(feature = "http3")] - prober, - }) - } -} - -#[napi] -impl Agent { - pub fn new() -> Result { - Self::with_options(AgentOptions::default()) - } - - pub fn with_options(options: AgentOptions) -> Result { - // Wrap in tokio runtime context for HTTP/3 endpoint initialization. - // Quinn's Endpoint::client() requires a tokio runtime to be available. - within_runtime_if_available(|| Self::with_options_inner(options)) - } - - fn with_options_inner(options: AgentOptions) -> Result { - // Destructured rather than read field by field so that a new option cannot be added - // without the compiler pointing here, where every option is turned into the recipe the - // agent's clients are built from (spec:NETCHG). - let AgentOptions { - cache, - cookies, - dns, - flow_control, - headers, - http2, - // Every use of the HTTP/3 options sits behind the feature. - #[cfg_attr(not(feature = "http3"), allow(unused_variables))] - http3, - local_address, - pool, - quirks, - redirect, - timeout, - tls, - user_agent, - } = options; - - let quirk_h1_request_streaming = quirks - .and_then(|quirks| quirks.h1_request_streaming) - .unwrap_or(false); - - // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts - // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the - // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and - // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such - // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. - let local_address = match &local_address { - Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { - FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - ) - })?), - None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), - None => None, - }; - - // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) - // The jar is installed on the client by the recipe, so it survives a rebuild - // (spec:NETCHG#what-the-signal-keeps). - let cookie_jar = match cookies.as_ref() { - None | Some(Either::A(false)) => None, - Some(Either::A(true)) => Some(Arc::new(FaithJar::new(CookieLimits::default()))), - Some(Either::B(cookies)) => Some(Arc::new(FaithJar::new(cookies.into()))), - }; - - let dns = dns.unwrap_or_default(); - let dns_system = dns.system.unwrap_or(false); - // Naming servers and asking for the system resolver at once is a contradiction rather than - // a preference, since the system resolver is not Faith's to point at listed servers - // (spec:DNS#system-resolver). - if dns_system - && dns - .servers - .as_ref() - .is_some_and(|servers| !servers.is_empty()) - { - return Err(FaithError::new( - FaithErrorKind::Config, - Some("dns.servers cannot be combined with dns.system".to_string()), - )); - } - // Parsed whichever resolver is in use: overrides take effect under the system resolver - // too (spec:DNS#overrides), and an unparseable address is a construction error either - // way (spec:AGENT#construction). - let dns_overrides = dns - .overrides - .unwrap_or_default() - .into_iter() - .map(|DnsOverride { domain, addresses }| { - let addresses = addresses - .into_iter() - .map(|addr| match SocketAddr::from_str(&addr) { - Ok(addr) => Ok(addr), - Err(err) => match IpAddr::from_str(&addr) { - Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), - Ok(IpAddr::V6(ip)) => { - Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) - } - Err(_) => Err(FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - )), - }, - }) - .collect::, FaithError>>()?; - Ok((domain, addresses)) - }) - .collect::, FaithError>>()?; - - // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so - // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), - // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and - // order each resolver is reached by (spec:DNS#transports). The system resolver - // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there - // and `prefetchDns` resolves as a no-op (spec:WARM). - let dns_resolver = if dns_system { - None - } else { - // These settings configure Faith's own resolver only, so they are read on the path that - // builds one rather than validated under the system resolver that ignores them. - let mut servers = Vec::new(); - for url in dns.servers.unwrap_or_default() { - servers.push(ServerSpec::parse(&url).map_err(|message| { - FaithError::new(FaithErrorKind::AddressParse, Some(message)) - })?); - } - Some(FaithResolver::new(ResolverSettings { - servers, - timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), - ndots: dns.ndots.map(|n| n as usize), - search_domains: parse_domains(dns.search_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, - hosts_file: dns.hosts_file, - exempt_domains: parse_domains(dns.exempt_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? - .unwrap_or_default(), - serve_stale: dns.serve_stale.unwrap_or(true), - max_stale: dns - .max_stale - .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), - })) - }; - - let mut default_accept_encoding = None; - let mut default_content_encoding = None; - let mut has_default_priority = false; - let mut default_headers = None; - if let Some(headers) = headers - && !headers.is_empty() - { - let map = HeaderMap::from_iter(headers.into_iter().filter_map( - |Header { - name, - value, - sensitive, - }| { - let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { - return None; - }; - - let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { - return None; - }; - - if sensitive.unwrap_or(false) { - value.set_sensitive(true); - } - - Some((name, value)) - }, - )); - default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); - default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); - has_default_priority = map.contains_key(PRIORITY); - default_headers = Some(map); - } - - // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so - // the explicit sizes are not resolved at all when it's on: reqwest would let the later - // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence - // visible here rather than depending on hyper's internal ordering. - let http2 = http2.unwrap_or_default(); - let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); - let http2_windows = (!http2_adaptive_window).then(|| { - resolve_windows( - flow_control.as_ref(), - http2.stream_window, - http2.connection_window, - ) - }); - - #[cfg(feature = "http3")] - let http3_max_idle_timeout = Duration::from_secs( - http3 - .as_ref() - .and_then(|h| h.max_idle_timeout) - .unwrap_or(30) - .clamp(1, 120) - .into(), - ); - - // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window - // inside an unbounded connection window: the stream window is the binding constraint - // on a high-latency link, and the unbounded connection window means a connection with - // many concurrent requests has no ceiling on what it buffers. Both are set here. - #[cfg(feature = "http3")] - let http3_windows = resolve_windows( - flow_control.as_ref(), - http3.as_ref().and_then(|h| h.stream_window), - http3.as_ref().and_then(|h| h.connection_window), - ); - - #[cfg(feature = "http3")] - let http3_congestion_bbr = matches!( - http3.as_ref().and_then(|h| h.congestion), - Some(Http3Congestion::Bbr1) - ); - - #[cfg(feature = "http3")] - let http3_send_window = http3.as_ref().and_then(|h| h.send_window); - - let pool_idle_timeout = pool - .as_ref() - .and_then(|pool| pool.idle_timeout) - .map(|seconds| Duration::from_secs(seconds.into())); - // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). - let pool_max_idle_per_host = pool.as_ref().map(|pool| { - pool.max_idle_per_host - .and_then(|n| n.try_into().ok()) - .unwrap_or(usize::MAX) - }); - - let connect_timeout = timeout - .and_then(|t| t.connect) - .map(|millis| Duration::from_millis(millis.into())); - let read_timeout = timeout - .and_then(|t| t.read) - .map(|millis| Duration::from_millis(millis.into())); - let total_timeout = timeout - .and_then(|t| t.total) - .map(|millis| Duration::from_millis(millis.into())); - - #[cfg(feature = "http3")] - let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); - let tls_required = tls.as_ref().and_then(|tls| tls.required); - // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a - // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). - let (tls_identity, tls_extra_roots) = match tls { - None => (None, Vec::new()), - Some(tls) => { - let identity = match &tls.identity { - None => None, - Some(identity) => Some( - Identity::from_pem(match identity { - Either::A(buf) => buf.as_ref(), - Either::B(string) => string.as_bytes(), - }) - .map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?, - ), - }; - - let mut extra_roots = Vec::new(); - for pem in tls.extra_roots.iter().flatten() { - let bytes = match pem { - Either::A(buf) => buf.as_ref(), - Either::B(string) => string.as_bytes(), - }; - extra_roots.extend(Certificate::from_pem_bundle(bytes).map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?); - } - - (identity, extra_roots) - } - }; - - let http_cache = if let Some(cache) = cache - && let Some(store) = cache.store - { - let mode = cache.mode.unwrap_or_default().into(); - let options = HttpCacheOptions { - cache_options: Some(CacheOptions { - shared: cache.shared.unwrap_or(true), - ignore_cargo_cult: true, - ..Default::default() - }), - ..Default::default() - }; - let store = match store { - CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { - path: cache - .path - .ok_or_else(|| { - FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) - })? - .into(), - remove_opts: Default::default(), - }), - CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( - MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), - )), - }; - - Some(HttpCacheRecipe { - mode, - options, - store, - }) - } else { - None - }; - - // Read outside the `alt_svc_cache` block below because `fetch` needs it too, - // to keep a rewritten port from looking like a redirect. - #[cfg(feature = "http3")] - let h3_follow_advertised_port = http3 - .as_ref() - .and_then(|o| o.upgrade_follow_advertised_port) - .unwrap_or(false); - #[cfg(not(feature = "http3"))] - let h3_follow_advertised_port = false; - - // The origin knowledge is built once and outlives every client the agent builds: it is - // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). - #[cfg(feature = "http3")] - let (alt_svc_cache, h3_upgrade) = { - let http3_opts = http3.as_ref(); - let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); - - let advertised_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_advertised_ttl) - .unwrap_or(86400) - .into(), - ); - let confirmed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_confirmed_ttl) - .unwrap_or(86400) - .into(), - ); - let failed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_ttl) - .unwrap_or(300) - .into(), - ); - let failed_max_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_max_ttl) - .unwrap_or(3600) - .into(), - ); - let capacity = http3_opts - .and_then(|o| o.upgrade_cache_capacity) - .unwrap_or(10_000) - .into(); - let cancel_strikes = http3_opts - .and_then(|o| o.upgrade_cancel_strikes) - .unwrap_or(3); - let attempt_timeout = match http3_opts - .and_then(|o| o.upgrade_attempt_timeout) - .unwrap_or(60_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); - let probe_timeout = match http3_opts - .and_then(|o| o.upgrade_probe_timeout) - .unwrap_or(5_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let slow_factor = http3_opts - .and_then(|o| o.upgrade_slow_factor) - .unwrap_or(2.5); - let slow_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_slow_ttl) - .unwrap_or(600) - .into(), - ); - - let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - capacity, - cancel_strikes, - strike_window: Duration::from_secs(60), - follow_advertised_port: h3_follow_advertised_port, - // The single-flight claim must outlive the probe it covers, so - // an aborted probe frees its origin without a report; without a - // probe deadline, the QUIC idle timeout (max 120s) is the bound. - probe_ttl: probe_timeout - .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), - slow_factor, - slow_ttl, - })); - - if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { - for hint in hints { - cache.add_hint(&hint.host, hint.port); - } - } - - ( - Some(cache), - H3UpgradeRecipe { - enabled, - attempt_timeout, - probe, - probe_timeout, - }, - ) - }; - - let recipe = ClientRecipe { - user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), - local_address, - default_headers, - dns_system, - dns_overrides, - http2_adaptive_window, - http2_windows, - #[cfg(feature = "http3")] - http3_max_idle_timeout, - #[cfg(feature = "http3")] - http3_windows, - #[cfg(feature = "http3")] - http3_congestion_bbr, - #[cfg(feature = "http3")] - http3_send_window, - pool_idle_timeout, - pool_max_idle_per_host, - redirect, - connect_timeout, - read_timeout, - total_timeout, - #[cfg(feature = "http3")] - tls_early_data, - tls_identity, - tls_required, - tls_extra_roots, - node_env: NodeEnvRecipe::read(), - http_cache, - #[cfg(feature = "http3")] - h3_upgrade, - }; - - let conn_timeout = recipe.conn_timeout(); - let built = recipe.build( - cookie_jar.as_ref(), - dns_resolver.as_ref(), - #[cfg(feature = "http3")] - alt_svc_cache.as_ref(), - )?; - - // Only now do all three exist: the resolver is built before the cache, and the prober - // holds a client that holds the resolver, so this is the earliest the loop can be closed - // (spec:DNS#https-records). - #[cfg(feature = "http3")] - install_https_sink( - dns_resolver.as_ref(), - alt_svc_cache.as_ref(), - built.prober.as_ref(), - recipe.h3_upgrade.enabled, - ); - - Ok(Self { - client: Some(built.client), - raw_client: Some(built.raw_client), - dns_resolver, - // A warm-up connection is warm only as long as the pool keeps it idle, so the record - // that an origin is warm expires with that same window. - warmed: MokaCache::builder().time_to_live(conn_timeout).build(), - // A safety TTL well past any reasonable warm-up, so a claim that never gets released - // (a warm-up whose task is dropped) frees the origin rather than wedging it. - warming: MokaCache::builder() - .time_to_live(Duration::from_secs(300)) - .build(), - warm_generation: Default::default(), - cookie_jar, - stats: Default::default(), - conn_tracker: ConnectionTracker::new(conn_timeout), - #[cfg(feature = "http3")] - alt_svc_cache, - #[cfg(feature = "http3")] - h3_prober: built.prober, - h3_follow_advertised_port, - #[cfg(feature = "http3")] - h3_upgrade_enabled: recipe.h3_upgrade.enabled, - quirk_h1_request_streaming, - default_accept_encoding, - default_content_encoding, - has_default_priority, - recipe: Arc::new(recipe), - }) - } - - #[napi(constructor)] - pub fn construct(env: Env, options: Option) -> Result { - Ok(if let Some(options) = options { - Self::with_options(options) - } else { - Self::new() - } - .map_err(|err| err.into_js_error(&env))?) - } - - /// Close the agent, releasing its connection pool, DNS resolver, and any - /// background tasks it owns, rather than waiting for the garbage collector - /// to drop it. This is worth doing when you create many short-lived agents; - /// a single long-lived agent can just be left to the GC. - /// - /// Requests already in flight run to completion. Any new request on a closed - /// agent throws a `Closed` error. Calling `close()` more than once is a - /// no-op. The cookie store, if any, remains readable via `getCookie`. - #[napi] - pub fn close(&mut self) { - // Dropping the client releases the reqwest connection pool and the - // Hickory resolver task; the alt-svc cache goes with it. The raw client - // shares that pool and the resolver, so it goes too, and both are what a - // later `preconnect`/`prefetchDns` checks to throw the closed-agent error. - self.client = None; - self.raw_client = None; - self.dns_resolver = None; - #[cfg(feature = "http3")] - { - // Probes hold a raw client clone; abort them so the pool doesn't - // outlive close by up to the probe timeout. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); - } - self.h3_prober = None; - self.alt_svc_cache = None; - } - } - - /// Tell the agent the network underneath it has changed, so it stops deciding from what it - /// learned about a network that is gone. - /// - /// Node has no portable signal for an interface or connectivity change, so Faith cannot - /// detect one; this is the reaction, and wiring it to a trigger (an OS notification, a VPN - /// transition, a captive-portal sign-in) is the caller's own. It drops pooled connections, - /// flushes the DNS cache, demotes the HTTP/3 origins that a real response confirmed back to - /// advertised so a background probe re-verifies them, and clears the HTTP/3 failure and slow - /// states, their cooldown backoff, and the path-time averages. - /// - /// Configuration, `http3.hints`, `Alt-Svc` advertisements, the cookie jar, the HTTP cache and - /// the `stats()` counters are all kept: none of them is a claim about a network path. - /// - /// Requests already in flight are not interrupted and run to completion on the connections - /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a - /// closed agent does nothing, and calling it repeatedly is harmless. - /// - /// spec:NETCHG - #[napi] - pub fn network_changed(&mut self) { - // A closed agent has already released all of this. - if self.client.is_none() { - return; - } - - // reqwest cannot drop pooled connections short of dropping the client, so the client is - // rebuilt from the recipe the agent kept for this. Requests in flight hold their own - // clone of the old client (`fetch` clones the agent per request), so they run to - // completion and the old pool goes when the last of them finishes. - // - // A rebuild that fails leaves the agent on its existing client: the options were already - // validated at construction, so a failure here is not the caller's to answer for, and an - // agent that still works on the old network beats one that works nowhere. - let built = self.recipe.build( - self.cookie_jar.as_ref(), - self.dns_resolver.as_ref(), - #[cfg(feature = "http3")] - self.alt_svc_cache.as_ref(), - ); - if let Ok(built) = built { - #[cfg(feature = "http3")] - { - // Abort probes running on the old client: each holds a clone of it, and their - // answers would describe the path that has just gone away. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); - } - self.h3_prober = built.prober; - // The sink holds the prober, which has just been replaced along with the client - // it sends on; leaving the old one installed would aim DNS-triggered probes at a - // client that has been dropped. - install_https_sink( - self.dns_resolver.as_ref(), - self.alt_svc_cache.as_ref(), - self.h3_prober.as_ref(), - self.h3_upgrade_enabled, - ); - } - self.client = Some(built.client); - self.raw_client = Some(built.raw_client); - } - - // Names resolve afresh against the new network, through that network's own servers: the - // resolver drops what it read off the old one and reads again when next used. Under the - // system resolver there is no resolver here and so nothing to reset (spec:DNS). - if let Some(resolver) = &self.dns_resolver { - resolver.reset(); - } - - #[cfg(feature = "http3")] - if let Some(alt_svc_cache) = &self.alt_svc_cache { - alt_svc_cache.network_changed(); - } - - // The warm-up records describe pooled connections that have just been dropped, so a - // `preconnect` after the signal opens a connection rather than finding the origin warm - // (spec:NETCHG, spec:WARM). The single-flight claims are left alone: a warm-up still in - // flight is not duplicated by releasing its claim, and the generation bump is what stops - // it recording an origin as warm on the strength of a connection in the dropped pool. - self.warmed.invalidate_all(); - self.warm_generation.fetch_add(1, Ordering::Relaxed); - } - - /// Add a cookie into the agent. - /// - /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying - /// the scheme and host they read, so this does nothing if: - /// - the cookie store is disabled - /// - the url is malformed - /// - the cookie does not parse - /// - a `__Host-` or `__Secure-` name prefix is not satisfied - /// - the cookie is larger than `cookies.maxSize` - #[napi] - pub fn add_cookie(&self, url: String, cookie: String) { - let Some(jar) = &self.cookie_jar else { - return; - }; - - let Ok(url) = Url::from_str(&url) else { - return; - }; - - jar.add_cookie_str(&cookie, &url); - } - - /// Retrieve a cookie from the store. - /// - /// Returns `null` if: - /// - there's no cookie at this url - /// - the cookie store is disabled - /// - the url is malformed - /// - the cookie cannot be represented as a string - #[napi] - pub fn get_cookie(&self, url: String) -> Option { - let Some(jar) = &self.cookie_jar else { - return None; - }; - - let Ok(url) = Url::from_str(&url) else { - return None; - }; - - jar.cookies(&url) - .and_then(|val| val.to_str().ok().map(ToOwned::to_owned)) - } - - /// Returns statistics gathered by this agent: - /// - /// - `requestsSent` - /// - `responsesReceived` - /// - `bodiesStarted` - /// - `bodiesFinished` - #[napi] - pub fn stats(&self) -> AgentStats { - AgentStats { - requests_sent: self - .stats - .requests_sent - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - responses_received: self - .stats - .responses_received - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - bodies_started: self - .stats - .bodies_started - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - bodies_finished: self - .stats - .bodies_finished - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - } - } - - /// Returns information on current connections open by this agent. - /// - /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: - /// this makes it possible to track indicators over time to find the retransmission rate, for - /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other - /// fields might also be missing depending on platform support; and no forward guarantees are made - /// on field availability. If the platform isn't supported at all, this will always return empty. - #[napi] - pub fn connections<'env>(&self, env: &'env Env) -> Vec> { - self.conn_tracker.get_for_napi(env) - } - - /// Returns the DNS servers this agent resolves through, in the order they are queried, so - /// "are my lookups actually encrypted" is answerable from inside the process. - /// - /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, - /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). - /// The list is empty until the resolver has been used, because it reads its configuration on - /// first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) - #[napi] - pub fn resolvers(&self) -> Vec { - self.dns_resolver - .as_ref() - .map(|resolver| { - resolver - .resolvers() - .into_iter() - .map(|report| ResolverInfo { - address: report.address, - transport: report.transport, - source: report.source, - }) - .collect() - }) - .unwrap_or_default() - } - - /// Note that a request reached this origin, so it holds a connection the pool keeps idle for - /// the idle window and a `preconnect` for it has no new work to do (spec:WARM). - /// - /// Called for foreground requests as well as warm-ups, because the criterion is about the - /// origin holding an idle pooled connection, not about how it came to hold one. - pub(crate) fn mark_warm(&self, url: &Url) { - self.warmed.insert(origin_key(url), ()); - } - - /// Warm the DNS cache for `host`, so a later request to it skips the lookup. - /// - /// Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, - /// port, or path in a fuller string is ignored. The returned promise resolves when the answer - /// lands in the cache and never rejects, whatever happens on the network — a resolution failure - /// resolves quietly, because the work is advisory. Under the system resolver there is no cache - /// to warm, so the call resolves without doing anything. A malformed host throws synchronously, - /// as does a call on a closed agent. (spec:WARM) - #[napi] - pub fn prefetch_dns<'env>( - &self, - env: &'env Env, - host: String, - ) -> Result, napi::Error> { - if self.client.is_none() { - return Err(caller_error(env, FaithErrorKind::Closed)); - } - - let Some(host) = extract_host(&host) else { - return Err(caller_error(env, FaithErrorKind::AddressParse)); - }; - - let resolver = self.dns_resolver.clone(); - faith_promise(env, async move { - if let Some(resolver) = resolver { - resolver.prefetch(&host).await; - } - Ok(()) - }) - } - - /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS - /// setup. - /// - /// Mirrors the browser's `preconnect` resource hint. The argument is an origin - /// (`scheme://host[:port]`); a longer URL is reduced to its origin. The warm-up sends a - /// synthetic `HEAD` to the origin's root — the origin sees it — over the transport the next - /// foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every - /// other origin a TCP one. The returned promise resolves when the attempt finishes and never - /// rejects: every network failure resolves quietly. A malformed origin throws synchronously, as - /// does a call on a closed agent. (spec:WARM) - #[napi] - pub fn preconnect<'env>( - &self, - env: &'env Env, - origin: String, - ) -> Result, napi::Error> { - let Some(raw_client) = self.raw_client.clone() else { - return Err(caller_error(env, FaithErrorKind::Closed)); - }; - - let Some(url) = reduce_to_origin(&origin) else { - return Err(caller_error(env, FaithErrorKind::AddressParse)); - }; - let key = origin_key(&url); - - // Already warm within the idle window, or a warm-up for this origin already in flight: - // either way there is no new work to do, so resolve without opening a duplicate. - if self.warmed.contains_key(&key) - || !self.warming.entry(key.clone()).or_insert(()).is_fresh() - { - return faith_promise(env, async move { Ok(()) }); - } - - // The transport the next foreground request would take, decided exactly as - // `AltSvcMiddleware` decides it: nothing upgrades with the machinery off; with a prober, - // only a confirmed origin routes to QUIC (an advertisement is evidence worth probing, not - // worth routing on); without one, the legacy inline upgrade acts on advertisements too. - // Diverging here would warm the wrong transport (spec:WARM#preconnect). - #[cfg(feature = "http3")] - let h3_port = self - .alt_svc_cache - .as_ref() - .filter(|_| self.h3_upgrade_enabled) - .and_then(|cache| { - if self.h3_prober.is_some() { - cache.confirmed_port(&url) - } else { - cache.should_use_h3(&url) - } - }); - #[cfg(not(feature = "http3"))] - let h3_port: Option = None; - - #[cfg(feature = "http3")] - let alt_svc_cache = self.alt_svc_cache.clone(); - #[cfg(feature = "http3")] - let h3_prober = self.h3_prober.clone(); - - let conn_tracker = self.conn_tracker.clone(); - let warmed = self.warmed.clone(); - let warming = self.warming.clone(); - // Read before the warm-up starts, to compare against once it finishes. - let warm_generation = self.warm_generation.clone(); - let generation = warm_generation.load(Ordering::Relaxed); - - faith_promise(env, async move { - // Release the single-flight claim whatever happens, so a later warm-up isn't blocked - // by this one having finished. - struct ReleaseClaim { - warming: MokaCache, - key: String, - } - impl Drop for ReleaseClaim { - fn drop(&mut self) { - self.warming.invalidate(&self.key); - } - } - let _release = ReleaseClaim { - warming, - key: key.clone(), - }; - - let request = match h3_port { - Some(port) => { - let mut h3_url = url.clone(); - // A port differing from the origin's only comes back with - // `upgradeFollowAdvertisedPort` on; rewriting the URL is how reqwest is told to - // connect there (mirrors the foreground path). - if Some(port) != h3_url.port_or_known_default() { - let _ = h3_url.set_port(Some(port)); - } - raw_client.head(h3_url).version(Version::HTTP_3) - } - None => raw_client.head(url.clone()), - }; - - let outcome = request.send().await; - - // A TCP warm-up leaves a pooled connection to track and, in probe mode, may reveal an - // HTTP/3 advertisement to act on; a QUIC warm-up does neither (QUIC connections are not - // tracked, and a confirmed origin has nothing left to probe). - if h3_port.is_none() { - if let Ok(response) = &outcome - && let Some(info) = response.extensions().get::() - { - conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); - } - - // A background probe verifies HTTP/3 for a probe-worthy origin exactly as a real - // TCP-routed request would, folding in any fresh advertisement first; the warm-up - // settles without waiting for it. Both only apply in probe mode (a prober present). - #[cfg(feature = "http3")] - if let Some(prober) = &h3_prober { - if let (Some(cache), Ok(response)) = (&alt_svc_cache, &outcome) - && let Some(value) = response.headers().get("alt-svc") - && let Ok(value) = value.to_str() - && let Some(advertisement) = parse_alt_svc_header(value) - { - cache.record_alt_svc(&url, &advertisement); - } - prober.maybe_probe(&url); - } - } - - // A connection was established, so the origin is warm for the idle window; a failed - // warm-up leaves it unmarked so a later `preconnect` may try again. A network change - // while this was in flight leaves it unmarked too: the connection landed in the pool - // that change dropped, so the origin is not warm however well the request went - // (spec:NETCHG#reach-across-the-subsystems). - if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { - warmed.insert(key, ()); - } - - Ok(()) - }) - } -} - -/// Build the JS error a warm-up throws synchronously for a caller mistake, preserving its `.code` -/// and JS error class. Network failures never reach here — they resolve quietly (spec:WARM). -fn caller_error(env: &Env, kind: FaithErrorKind) -> napi::Error { - napi::Error::from(FaithError::from(kind).into_js_error(env)) -} - -/// Extract the bare host from a `prefetchDns` argument, ignoring any scheme, port, or path, or -/// `None` if there is no host to resolve. A DNS name carries none of those parts, so a fuller -/// string is reduced to its host. (spec:WARM) -fn extract_host(input: &str) -> Option { - // A string that already spells a scheme is read as the URL it is; anything else is the - // bare-host case, where a name is not a URL on its own and giving it an authority makes it - // parse as one. Telling the two apart on the scheme separator matters both ways: `example.com:8443` - // otherwise parses as a *scheme* of `example.com` with no host, and a schemed string with no host - // (`file:///path`, a bare `https://`) would have its scheme misread as a host by the fallback. - let url = if input.contains("://") { - Url::parse(input).ok()? - } else { - Url::parse(&format!("dns://{input}")).ok()? - }; - let host = url.host_str()?; - // `host_str` brackets an IPv6 literal; the resolver wants it bare. - let host = host - .strip_prefix('[') - .and_then(|rest| rest.strip_suffix(']')) - .unwrap_or(host); - (!host.is_empty()).then(|| host.to_owned()) -} - -/// Reduce a `preconnect` argument to its origin, or `None` if it is not a connectable origin. Path, -/// query, fragment, and userinfo are stripped — the same reduction the HTTP/3 probe applies — and -/// the scheme must have a known default port so an omitted port resolves. (spec:WARM) -fn reduce_to_origin(input: &str) -> Option { - let mut url = Url::parse(input).ok()?; - if !url.has_host() || url.port_or_known_default().is_none() { - return None; - } - url.set_path("/"); - url.set_query(None); - url.set_fragment(None); - let _ = url.set_username(""); - let _ = url.set_password(None); - Some(url) -} - -/// The `scheme://host:port` key an origin coalesces on, with the port defaulted by scheme so -/// `https://host` and `https://host:443` are the same origin. Matches the Alt-Svc cache's key. -pub(crate) fn origin_key(url: &Url) -> String { - format!( - "{}://{}:{}", - url.scheme(), - url.host_str().unwrap_or_default(), - url.port_or_known_default().unwrap_or_default(), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn prefetch_dns_takes_a_bare_host() { - assert_eq!(extract_host("example.com").as_deref(), Some("example.com")); - } - - #[test] - fn prefetch_dns_ignores_the_parts_a_name_does_not_have() { - // A DNS name has no scheme, port, or path, so a fuller string is reduced to its host - // rather than rejected (spec:WARM#prefetchdns). - for input in [ - "https://example.com", - "https://example.com:8443", - "https://example.com/some/path?q=1#frag", - "https://user:pass@example.com/", - "example.com:8443", - ] { - assert_eq!( - extract_host(input).as_deref(), - Some("example.com"), - "{input:?} names example.com whatever else it carries" - ); - } - } - - #[test] - fn prefetch_dns_unwraps_an_ipv6_literal() { - // `host_str` brackets an IPv6 literal, but the resolver wants it bare. - assert_eq!( - extract_host("https://[2001:db8::1]:8443").as_deref(), - Some("2001:db8::1") - ); - } - - #[test] - fn prefetch_dns_rejects_a_string_with_no_host() { - for input in ["", " ", "/just/a/path", "https://"] { - assert!( - extract_host(input).is_none(), - "{input:?} names no host to resolve" - ); - } - } - - #[test] - fn preconnect_reduces_a_longer_url_to_its_origin() { - // The same reduction the HTTP/3 probe applies (spec:WARM#preconnect). - let url = reduce_to_origin("https://user:pass@example.com/some/path?q=1#frag") - .expect("a full URL reduces to its origin"); - - assert_eq!(url.as_str(), "https://example.com/"); - assert_eq!(url.username(), "", "userinfo is stripped"); - assert_eq!(url.password(), None); - assert_eq!(url.query(), None); - assert_eq!(url.fragment(), None); - } - - #[test] - fn preconnect_defaults_the_port_by_scheme() { - // An omitted port defaults by scheme, so an origin spelled either way coalesces on one - // key (spec:WARM#preconnect). - for (bare, spelled) in [ - ("https://example.com", "https://example.com:443"), - ("http://example.com", "http://example.com:80"), - ] { - let bare = origin_key(&reduce_to_origin(bare).expect("parses")); - let spelled = origin_key(&reduce_to_origin(spelled).expect("parses")); - assert_eq!( - bare, spelled, - "the omitted port defaults to the spelled one" - ); - } - } - - #[test] - fn preconnect_keeps_distinct_origins_apart() { - // The pool caps and the warm record are per origin: scheme, host, and port together - // (spec:POOL). - let key = |input: &str| origin_key(&reduce_to_origin(input).expect("parses")); - - assert_ne!(key("https://example.com"), key("https://example.com:8443")); - assert_ne!(key("https://example.com"), key("http://example.com")); - assert_ne!(key("https://example.com"), key("https://other.example")); - } - - #[test] - fn preconnect_rejects_what_cannot_be_connected_to() { - for input in [ - "not an origin", - "", - "/just/a/path", - // No host to connect to. - "file:///etc/hosts", - // No default port for the scheme, and none given. - "unknownscheme://example.com", - ] { - assert!( - reduce_to_origin(input).is_none(), - "{input:?} is not a connectable origin" - ); - } - } - - fn common(stream: Option, connection: Option) -> AgentFlowControlOptions { - AgentFlowControlOptions { - stream_window: stream, - connection_window: connection, - } - } - - #[test] - fn windows_fall_back_to_the_defaults() { - // An agent configured with nothing at all still gets the large static windows - // (spec:FLOW#common-windows). - assert_eq!( - resolve_windows(None, None, None), - ResolvedWindows { - stream: 6 * 1024 * 1024, - connection: 15 * 1024 * 1024, - } - ); - } - - #[test] - fn the_common_windows_apply_when_a_protocol_says_nothing() { - assert_eq!( - resolve_windows(Some(&common(Some(1024), Some(4096))), None, None), - ResolvedWindows { - stream: 1024, - connection: 4096, - } - ); - } - - #[test] - fn a_protocol_window_beats_the_common_one() { - // The whole point of the per-protocol group: tune one protocol against the other - // (spec:FLOW#per-protocol-windows). - assert_eq!( - resolve_windows( - Some(&common(Some(1024), Some(4096))), - Some(2048), - Some(8192) - ), - ResolvedWindows { - stream: 2048, - connection: 8192, - } - ); - } - - #[test] - fn each_window_falls_back_on_its_own() { - // Overriding the stream window for one protocol leaves that protocol's connection - // window on the common value, rather than dropping it to the default. - assert_eq!( - resolve_windows(Some(&common(Some(1024), Some(4096))), Some(2048), None), - ResolvedWindows { - stream: 2048, - connection: 4096, - } - ); - assert_eq!( - resolve_windows(Some(&common(None, None)), None, Some(8192)), - ResolvedWindows { - stream: DEFAULT_STREAM_WINDOW, - connection: 8192, - } - ); - } - - #[test] - fn a_protocol_window_applies_without_the_common_group() { - assert_eq!( - resolve_windows(None, Some(2048), None), - ResolvedWindows { - stream: 2048, - connection: DEFAULT_CONNECTION_WINDOW, - } - ); - } - - #[test] - fn the_two_protocols_resolve_independently() { - // One `flowControl` value covers both protocols, and overriding it for HTTP/3 leaves - // HTTP/2 where it was (spec:FLOW#per-protocol-windows). - let flow = common(Some(1024), Some(4096)); - let http2 = resolve_windows(Some(&flow), None, None); - let http3 = resolve_windows(Some(&flow), Some(2048), None); - - assert_eq!(http2.stream, 1024); - assert_eq!(http3.stream, 2048); - assert_eq!(http2.connection, http3.connection); - } -} diff --git a/src/alt_svc.rs b/src/alt_svc.rs deleted file mode 100644 index bd2447d..0000000 --- a/src/alt_svc.rs +++ /dev/null @@ -1,2339 +0,0 @@ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; - -use http::Extensions; -use moka::sync::Cache; -use reqwest::{Request, Response}; -use reqwest_middleware::{Middleware, Next, Result}; - -use crate::timing::HeadersStamp; - -#[derive(Debug, Clone)] -pub struct AltSvcEntry { - pub port: u16, - pub expires: Instant, -} - -/// An HTTP/3 alternative parsed out of an `Alt-Svc` header. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AltSvcAdvertisement { - /// Host the alternative is on. Empty when the header omitted it, which per - /// RFC 7838 means the same host as the origin. - pub host: String, - pub port: u16, - pub max_age: Option, -} - -/// A run of consecutive HTTP/3 failures against one origin. -/// -/// Both instants are carried in the value rather than left to the cache's TTL, -/// because they differ per origin and from each other: the entry deliberately -/// outlives the cooldown it set, so that a count survives the block it caused and -/// can escalate the next one. `advertised` does the same for `ma`. -/// -/// spec:H3UP#failure-backoff -#[derive(Debug, Clone, Copy)] -struct FailureEntry { - /// Consecutive failures with no confirmation in between. - count: u32, - /// Until when the origin is blocked from upgrading, probing, and recording - /// advertisements. The only field that gates behaviour. - blocked_until: Instant, - /// Until when `count` still describes a run. Past it the origin is judged - /// from the base cooldown again. - counted_until: Instant, -} - -/// A per-origin exponentially-weighted moving average of time-to-response-headers. -/// -/// Two `f64`s per origin and no sample storage: the average decays stale history -/// by construction, and the count gates decisions until there is enough evidence -/// to mean anything. -#[derive(Debug, Clone, Copy)] -pub struct PathTime { - /// EWMA of time-to-response-headers, in milliseconds. - pub avg_ms: f64, - pub count: u32, -} - -/// Weight of the newest sample in the moving average. -const EWMA_ALPHA: f64 = 0.2; -/// Samples required on *each* side before a slow comparison may act. -const EWMA_MIN_SAMPLES: u32 = 8; -/// Absolute gap the QUIC average must exceed the TCP one by, on top of the -/// factor, so LAN-fast origins don't flap on sub-millisecond noise. -const SLOW_FLOOR_MS: f64 = 10.0; - -pub struct AltSvcCacheConfig { - pub advertised_ttl: Duration, - pub confirmed_ttl: Duration, - /// Cooldown a first failure earns; each consecutive one doubles it. - pub failed_ttl: Duration, - /// Ceiling on the doubling. Clamped up to `failed_ttl`, so setting it at or - /// below the base gives a flat cooldown. - pub failed_max_ttl: Duration, - pub capacity: u64, - pub cancel_strikes: u32, - pub strike_window: Duration, - pub follow_advertised_port: bool, - /// Lifetime of a probe's single-flight claim. Doubles as crash recovery: a - /// probe task that dies without reporting frees its origin when this lapses. - pub probe_ttl: Duration, - /// The QUIC path is demoted when its average is worse than TCP's by this - /// factor (and by [`SLOW_FLOOR_MS`] absolutely). `0.0` disables path-time - /// demotion entirely. - pub slow_factor: f64, - /// How long a path-time demotion holds before the origin may be re-probed. - pub slow_ttl: Duration, -} - -#[derive(Clone)] -pub struct AltSvcCache { - advertised: Cache, - confirmed: Cache, - /// Origins that failed over HTTP/3, with their run of consecutive failures. - /// An entry present here is not necessarily blocked: see [`Self::is_failed`]. - failed: Cache, - /// Consecutive cancelled HTTP/3 attempts per origin. Entries expire on a TTL - /// (the strike window), so a run has to be sustained to count. - cancellations: Cache, - /// Single-flight claims for in-flight background probes. - probing: Cache, - /// Origins demoted for being slower over QUIC than over TCP. Distinct from - /// `failed`: the path *works*, so re-advertisements must not be discarded, - /// and expiry re-enters through a probe rather than treating h3 as broken. - slow: Cache, - /// Origins seeded from `http3.hints`, with the port hinted. A hint is the - /// caller's assertion rather than something observed, so it has to be - /// distinguishable from an entry in `confirmed` that a real HTTP/3 response - /// put there: [`Self::network_changed`] demotes the observed ones and - /// re-seeds from here. Unbounded by TTL and outside the capacity bound, - /// because the hints are configuration and there are as many as the caller - /// passed. (spec:NETCHG#what-the-signal-keeps) - hints: Cache, - /// Time-to-headers over TCP (h1 and h2 together), per origin. - tcp_times: Cache, - /// Time-to-headers over QUIC (h3), per origin. - quic_times: Cache, - - advertised_ttl: Duration, - confirmed_ttl: Duration, - failed_ttl: Duration, - failed_max_ttl: Duration, - cancel_strikes: u32, - follow_advertised_port: bool, - slow_factor: f64, -} - -impl std::fmt::Debug for AltSvcCache { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AltSvcCache") - .field("advertised_count", &self.advertised.entry_count()) - .field("confirmed_count", &self.confirmed.entry_count()) - .field("failed_count", &self.failed.entry_count()) - .field("cancellation_count", &self.cancellations.entry_count()) - .field("probing_count", &self.probing.entry_count()) - .field("slow_count", &self.slow.entry_count()) - .field("hint_count", &self.hints.entry_count()) - .finish() - } -} - -impl AltSvcCache { - pub fn new(config: AltSvcCacheConfig) -> Self { - let AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - capacity, - cancel_strikes, - strike_window, - follow_advertised_port, - probe_ttl, - slow_factor, - slow_ttl, - } = config; - - // A cap below the base would mean the first failure already exceeds it; - // clamping makes that setting a flat cooldown rather than a shorter one. - let failed_max_ttl = failed_max_ttl.max(failed_ttl); - - Self { - advertised: Cache::builder() - .max_capacity(capacity) - .time_to_live(advertised_ttl) - .build(), - confirmed: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - // Twice the longest cooldown: the outer bound on how long an entry - // can be worth keeping, since a count is dropped one cooldown after - // the block it caused lapsed. Per-entry instants do the real work. - failed: Cache::builder() - .max_capacity(capacity) - .time_to_live(failed_max_ttl.saturating_mul(2)) - .build(), - cancellations: Cache::builder() - .max_capacity(capacity) - .time_to_live(strike_window) - .build(), - probing: Cache::builder() - .max_capacity(capacity) - .time_to_live(probe_ttl) - .build(), - slow: Cache::builder() - .max_capacity(capacity) - .time_to_live(slow_ttl) - .build(), - // No TTL and no capacity bound: hints are configuration, held for the - // life of the agent so a network change can re-seed from them. - hints: Cache::builder().build(), - tcp_times: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - quic_times: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - cancel_strikes, - follow_advertised_port, - slow_factor, - } - } - - /// The cooldown the `count`-th consecutive failure earns: the base doubled - /// once per failure before it, capped. - /// - /// spec:H3UP#failure-backoff - fn failure_cooldown(&self, count: u32) -> Duration { - let doublings = count.saturating_sub(1).min(u32::BITS - 1); - self.failed_ttl - .saturating_mul(2u32.saturating_pow(doublings)) - .min(self.failed_max_ttl) - } - - /// Whether the origin is inside its failure cooldown. - /// - /// Presence in `failed` is not the question: an entry outlives its cooldown - /// so the failure count survives to escalate the next one. - fn is_failed(&self, origin: &str) -> bool { - self.failed - .get(origin) - .is_some_and(|entry| entry.blocked_until > Instant::now()) - } - - fn origin_key(url: &reqwest::Url) -> Option { - let host = url.host_str()?; - let port = url.port_or_known_default()?; - Some(format!("{}://{}:{}", url.scheme(), host, port)) - } - - pub fn record_alt_svc(&self, url: &reqwest::Url, advertisement: &AltSvcAdvertisement) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // An alternative on a *different host* can never be honoured: reqwest derives - // the HTTP/3 connect target from the request's authority, and rewriting the - // host would also change which certificate is accepted. Unlike a differing - // port — which `follow_advertised_port` can act on — there is nothing to - // gate behind an option, so don't record it at all. RFC 7838 uses an empty - // host to mean "the same host as the origin". - // - // Compared case-insensitively because host names are, and a server naming its - // own host in a different case is still naming its own host. - if !advertisement.host.is_empty() - && !url - .host_str() - .is_some_and(|origin_host| origin_host.eq_ignore_ascii_case(&advertisement.host)) - { - return; - } - - if self.is_failed(&origin) { - return; - } - - if self.confirmed.contains_key(&origin) { - return; - } - - let ttl = advertisement.max_age.unwrap_or(self.advertised_ttl); - let entry = AltSvcEntry { - port: advertisement.port, - expires: Instant::now() + ttl, - }; - - self.advertised.insert(origin, entry); - } - - /// Whether an `HTTPS` DNS record for this origin would tell us anything we do not already - /// know, so the resolver can skip the query rather than send one per lookup. - /// - /// Nothing is learnable while the origin is confirmed (already routing over HTTP/3), failed - /// (blocked whatever a record says), slow (demoted on measurement, which a record cannot - /// overturn), or already carrying a live advertisement (the probe it warrants is already - /// warranted). Each of those states expires, and the query resumes when it does. - /// - /// spec:DNS#https-records - pub fn wants_https_record(&self, url: &reqwest::Url) -> bool { - let Some(origin) = Self::origin_key(url) else { - return false; - }; - - !self.is_failed(&origin) - && !self.confirmed.contains_key(&origin) - && !self.slow.contains_key(&origin) - && self - .advertised - .get(&origin) - .is_none_or(|entry| entry.expires <= Instant::now()) - } - - /// Record an HTTP/3 advertisement carried by an `HTTPS` DNS record. - /// - /// An `HTTPS` record and an `Alt-Svc` header are two ways for an origin to say the same thing, - /// so this lands in exactly the state a header advertisement does: the origin becomes - /// probe-worthy, and foreground requests keep to TCP until a probe proves the path. The port - /// and same-host rules are the header's too — [`Self::record_alt_svc`] applies them — because - /// the reasons for them are about what Faith can connect to rather than about where the - /// advertisement was read. - /// - /// spec:H3UP#advertisements-from-dns - pub fn record_https_record(&self, url: &reqwest::Url, port: Option, ttl: Duration) { - // A record naming no port describes the origin's own, exactly as an `Alt-Svc` header with - // no alt-authority port would. - let Some(port) = port.or_else(|| url.port_or_known_default()) else { - return; - }; - - self.record_alt_svc( - url, - &AltSvcAdvertisement { - // The same-host case: a record targeting another host is dropped before it gets - // here, since Faith only upgrades to the origin's own host. - host: String::new(), - port, - // The record's own DNS TTL is how long what it says is good for, which is the - // role `ma` plays for a header advertisement. - max_age: Some(ttl), - }, - ); - } - - /// Hints seed `confirmed` directly, not `advertised`: a hint is the *user's* - /// assertion, and routing it through a probe would both second-guess an - /// explicit instruction and break h3-only origins (no TCP listener), which - /// only work if the very first request speaks HTTP/3. Distrust is reserved - /// for what servers advertise. Failure demotes a hinted origin exactly as it - /// does a confirmed one. - pub fn add_hint(&self, host: &str, port: u16) { - let origin = format!("https://{}:{}", host, port); - - // Recorded whether or not it can be acted on right now: the hint is - // configuration, and a failure blocking it is a fact about a path that a - // network change can clear (spec:NETCHG#what-the-signal-keeps). - self.hints.insert(origin.clone(), port); - self.seed_hint(origin, port); - } - - /// Put a hinted origin into `confirmed`, unless a failure currently blocks it. - /// - /// Split out of [`Self::add_hint`] so [`Self::network_changed`] can re-seed the - /// hints it just cleared without re-recording them. - fn seed_hint(&self, origin: String, port: u16) { - if self.is_failed(&origin) { - return; - } - - let entry = AltSvcEntry { - port, - expires: Instant::now() + Duration::from_hours(10_000), // forever - }; - - self.confirmed.insert(origin, entry); - } - - /// Whether an entry advertising `entry_port` can be acted on for this URL. - /// - /// An Alt-Svc advertisement names a network endpoint for the origin; it is not - /// a claim that the origin's *own* port speaks HTTP/3. So when the advertised - /// port differs, upgrading the request on the origin port is an inference the - /// advertisement does not support. - /// - /// Honouring the advertised port properly means connecting to one port while - /// still sending the origin's authority, which reqwest cannot express: it - /// derives the HTTP/3 connect target from the request URI's authority (see - /// ). `follow_advertised_port` - /// opts into doing it anyway by rewriting the request's port, which is not - /// standards-compliant — the request then carries the alternative's authority - /// rather than the origin's. - fn port_actionable(&self, url: &reqwest::Url, entry_port: u16) -> bool { - self.follow_advertised_port || Some(entry_port) == url.port_or_known_default() - } - - /// The port HTTP/3 is *proven* on, or `None` to leave the request on TCP. - /// - /// This is the only lookup foreground routing consults when probing is on: - /// an advertisement is evidence worth probing, not worth routing on. - /// - /// A returned port that differs from the URL's own means the caller opted into - /// `follow_advertised_port` and the request must be rewritten to target it. - pub fn confirmed_port(&self, url: &reqwest::Url) -> Option { - let origin = Self::origin_key(url)?; - - if self.is_failed(&origin) || self.slow.contains_key(&origin) { - return None; - } - - let entry = self.confirmed.get(&origin)?; - if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { - Some(entry.port) - } else { - None - } - } - - /// The advertised port a background probe should verify, or `None` when - /// there is nothing (or no need) to probe: no actionable advertisement, - /// already confirmed, recently failed, or demoted for being slow. - pub fn probe_candidate(&self, url: &reqwest::Url) -> Option { - let origin = Self::origin_key(url)?; - - if self.is_failed(&origin) - || self.slow.contains_key(&origin) - || self.confirmed.contains_key(&origin) - { - return None; - } - - let entry = self.advertised.get(&origin)?; - if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { - Some(entry.port) - } else { - None - } - } - - /// Claim the origin for a probe. Returns `false` when a probe is already in - /// flight; the claim expires on its own (see [`AltSvcCacheConfig::probe_ttl`]) - /// if the prober never reports back. - pub fn claim_probe(&self, url: &reqwest::Url) -> bool { - let Some(origin) = Self::origin_key(url) else { - return false; - }; - self.probing.entry(origin).or_insert(()).is_fresh() - } - - /// Release the origin's probe claim, so a later advertisement can re-probe - /// without waiting out the claim's TTL. - pub fn finish_probe(&self, url: &reqwest::Url) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - self.probing.invalidate(&origin); - } - - /// The port to attempt HTTP/3 on, or `None` to leave the request on TCP. - /// - /// Legacy (probe-less) routing: advertisements are acted on inline, so this - /// consults `advertised` as well as `confirmed`. Only used when - /// `upgradeProbe` is off. - pub fn should_use_h3(&self, url: &reqwest::Url) -> Option { - self.confirmed_port(url) - .or_else(|| self.probe_candidate(url)) - } - - /// Record a foreground request's time-to-response-headers for its protocol - /// family, and demote the origin to TCP if QUIC is provenly, sustainedly - /// slower than TCP for it. - /// - /// Time-to-headers includes server think-time, which varies per endpoint far - /// more than per transport; only the averages across many requests are - /// comparable, never individual samples — hence the minimum sample counts. - /// Redirects followed inside the attempt inflate a sample for whichever - /// family carried it, which the averaging absorbs the same way. - /// - /// The comparison is deliberately asymmetric: HTTP/3 is preferred at parity - /// and when moderately slower, because its advantages (no head-of-line - /// blocking, connection migration) pay off beyond the mean. Only a large - /// sustained gap demotes. - pub fn record_path_time(&self, url: &reqwest::Url, version: http::Version, elapsed: Duration) { - if self.slow_factor <= 0.0 { - return; - } - - let Some(origin) = Self::origin_key(url) else { - return; - }; - - let sample_ms = elapsed.as_secs_f64() * 1000.0; - let times = if version == http::Version::HTTP_3 { - &self.quic_times - } else { - &self.tcp_times - }; - - let updated = times - .entry(origin.clone()) - .and_upsert_with(|existing| match existing { - None => PathTime { - avg_ms: sample_ms, - count: 1, - }, - Some(entry) => { - let entry = entry.into_value(); - PathTime { - avg_ms: entry.avg_ms * (1.0 - EWMA_ALPHA) + sample_ms * EWMA_ALPHA, - count: entry.count.saturating_add(1), - } - } - }) - .into_value(); - - if version == http::Version::HTTP_3 - && updated.count >= EWMA_MIN_SAMPLES - && let Some(tcp) = self.tcp_times.get(&origin) - && tcp.count >= EWMA_MIN_SAMPLES - && updated.avg_ms > tcp.avg_ms * self.slow_factor - && updated.avg_ms - tcp.avg_ms > SLOW_FLOOR_MS - { - self.demote_slow(&origin); - } - } - - /// Demote a working-but-slow QUIC origin back to TCP. - /// - /// The confirmed entry moves back to `advertised` rather than being dropped: - /// when the `slow` marker expires, the advertisement is what makes the next - /// request trigger a re-probe — "has this path improved?" asked at zero - /// foreground cost. The QUIC average is cleared so the answer is judged on - /// fresh samples, not held hostage by the history that demoted it. - fn demote_slow(&self, origin: &str) { - let key = origin.to_string(); - let Some(entry) = self.confirmed.get(&key) else { - return; - }; - - self.confirmed.invalidate(&key); - self.advertised.insert( - key.clone(), - AltSvcEntry { - port: entry.port, - expires: Instant::now() + self.advertised_ttl, - }, - ); - self.quic_times.invalidate(&key); - self.slow.insert(key, ()); - } - - /// Record that HTTP/3 worked for this origin, on the port it connected to. - /// - /// `port` must be the port the successful attempt actually used. Recovering it - /// from the caches instead would be unsound: a concurrent failure that cleared - /// them leaves nothing to read, and falling back to the origin's own port would - /// confirm HTTP/3 on a port the server never advertised — for `confirmed_ttl`, - /// and invisibly, since the concurrent failure's `failed` entry masks it until - /// that expires. - pub fn confirm_h3(&self, url: &reqwest::Url, port: u16) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // Promoted out of `advertised`; it has served its purpose. - self.advertised.invalidate(&origin); - // A working h3 response is proof of health; forget any strikes, and end - // whatever run of failures preceded it. - self.cancellations.invalidate(&origin); - self.clear_failure_count(&origin); - - let entry = AltSvcEntry { - port, - expires: Instant::now() + self.confirmed_ttl, - }; - - self.confirmed.insert(origin, entry); - } - - /// Record an HTTP/3 attempt that was cancelled before producing an outcome. - /// - /// This is weaker evidence than an error: the request never got to find out - /// whether HTTP/3 worked, so a single cancellation says nothing about the - /// origin. Only a sustained run of them demotes it, which keeps callers that - /// routinely abort healthy requests from disabling HTTP/3. - /// - /// The window is a TTL measured from the *previous* strike, because moka - /// refreshes an entry's TTL on upsert. Strikes therefore have to arrive - /// within a window of each other, not within a fixed bucket. - pub fn record_h3_cancellation(&self, url: &reqwest::Url) { - if self.cancel_strikes == 0 { - return; - } - - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // This is reachable from a `Drop` impl (see the guard below), which must - // never panic: a panic while already unwinding aborts the process. Use a - // saturating add so an absurd `upgrade_cancel_strikes` can't overflow. - let strikes = self - .cancellations - .entry(origin) - .and_upsert_with(|existing| { - existing.map_or(1, |entry| entry.into_value().saturating_add(1)) - }) - .into_value(); - - if strikes >= self.cancel_strikes { - // Clears the strike count as a side effect. - self.record_h3_failure(url); - } - } - - /// Forget the origin's run of failures, so the next one starts the backoff - /// from the base cooldown again. - /// - /// A cooldown still running is left alone. A confirmation racing a concurrent - /// failure must not unblock the origin that failure just blocked: the failure - /// is the more recent evidence about the path, and [`Self::confirm_h3`] - /// relies on its own entry being masked until the block lapses. - /// - /// spec:H3UP#failure-backoff - fn clear_failure_count(&self, origin: &str) { - let Some(entry) = self.failed.get(origin) else { - return; - }; - - if entry.blocked_until > Instant::now() { - self.failed - .insert(origin.to_string(), FailureEntry { count: 0, ..entry }); - } else { - self.failed.invalidate(origin); - } - } - - /// Discard everything this cache learned by observing the network, keeping - /// what it was told. - /// - /// Every state here except `advertised` and `hints` describes the path between - /// this client and an origin, and a network change is exactly the event that - /// invalidates such a description. So the observation-confirmed origins are - /// demoted rather than kept (the path that proved them is gone, and a probe - /// re-proves them without a foreground request paying for it), and the - /// failures, strikes, slow markers and averages go entirely: they are - /// penalties and measurements the old path earned, and carrying them over - /// would judge the new network by the old one's behaviour. - /// - /// What the origin said about itself (`advertised`) and what the caller - /// asserted (`hints`) are not observations, so both survive. - /// - /// spec:NETCHG - pub fn network_changed(&self) { - let now = Instant::now(); - - // Demote first, while `confirmed` still holds the entries: an advertisement - // is what makes the next request to the origin trigger a re-probe. - // - // Keys are invalidated one by one rather than with `invalidate_all`, whose - // timestamp-based invalidation would race the hint re-seeding below. - for (origin, entry) in self.confirmed.iter() { - // A hint holds its origin confirmed; it is an assertion, not a finding. - if self.hints.contains_key(origin.as_str()) { - continue; - } - - self.confirmed.invalidate(origin.as_str()); - - // A logically expired entry is not knowledge to carry forward: it would - // come back as a fresh advertisement having just lapsed as a confirmation. - if entry.expires <= now { - continue; - } - - self.advertised.insert( - (*origin).clone(), - AltSvcEntry { - port: entry.port, - expires: now + self.advertised_ttl, - }, - ); - } - - self.failed.invalidate_all(); - self.cancellations.invalidate_all(); - self.slow.invalidate_all(); - // In-flight probes are aborted by the caller of this method, so their - // single-flight claims would otherwise hold their origins until the claim - // TTL lapsed. - self.probing.invalidate_all(); - self.tcp_times.invalidate_all(); - self.quic_times.invalidate_all(); - - // After the failures are cleared, so a hint that a cooldown had been - // blocking takes effect now rather than staying refused. - for (origin, port) in self.hints.iter() { - self.seed_hint((*origin).clone(), port); - } - } - - /// Record a failed HTTP/3 attempt, blocking the origin for a cooldown that - /// lengthens the longer it keeps failing. - /// - /// spec:H3UP#failure-backoff - pub fn record_h3_failure(&self, url: &reqwest::Url) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - self.advertised.invalidate(&origin); - self.confirmed.invalidate(&origin); - // Already demoted; further counting is meaningless. - self.cancellations.invalidate(&origin); - - let now = Instant::now(); - // An entry whose run has lapsed is history, not a run in progress: the - // origin went a whole further cooldown without failing again, so it is - // judged from the base. - let count = self - .failed - .get(&origin) - .filter(|entry| entry.counted_until > now) - .map_or(1, |entry| entry.count.saturating_add(1)); - let cooldown = self.failure_cooldown(count); - - self.failed.insert( - origin, - FailureEntry { - count, - blocked_until: now + cooldown, - // The count has to outlive the block it caused, or it could never - // escalate: the next attempt only comes once the block lapses. - counted_until: now + cooldown.saturating_mul(2), - }, - ); - } -} - -pub fn parse_alt_svc_header(value: &str) -> Option { - if value == "clear" { - return None; - } - - for service in value.split(',') { - let service = service.trim(); - if service.is_empty() { - continue; - } - - let mut protocol_id: Option<&str> = None; - let mut host: Option<&str> = None; - let mut port: Option = None; - let mut max_age: Option = None; - - for param in service.split(';') { - let param = param.trim(); - if param.is_empty() { - continue; - } - - let Some((key, value)) = param.split_once('=') else { - continue; - }; - - let key = key.trim(); - let value = value.trim().trim_matches('"'); - - match key { - "ma" => { - if let Ok(secs) = value.parse::() { - max_age = Some(Duration::from_secs(secs)); - } - } - _ if key.starts_with("h3") => { - protocol_id = Some(key); - // The alt-authority is `[host]:port`, where an omitted host means - // the origin's own. Keep the host: acting on an advertisement for - // a different host would be the same unsupported inference as - // acting on one for a different port. - // - // Split on the *last* colon so a bracketed IPv6 literal survives, - // and keep it exactly as written — brackets included. That is the - // form `Url::host_str` also returns for IPv6, so comparing the two - // needs no normalising on either side. - if let Some((alt_host, port_str)) = value.rsplit_once(':') { - host = Some(alt_host); - if let Ok(p) = port_str.parse::() { - port = Some(p); - } - } - } - _ => {} - } - } - - if protocol_id.is_some() && port.is_some() { - return Some(AltSvcAdvertisement { - host: host.unwrap_or_default().to_owned(), - port: port.unwrap(), - max_age, - }); - } - } - - None -} - -/// Records a cancellation if the HTTP/3 attempt it guards is dropped before -/// producing an outcome. -/// -/// [`AltSvcMiddleware`] can only learn that HTTP/3 is broken from the attempt's -/// return value, and a cancelled request never produces one: `faith_fetch` -/// races `send()` against the abort signal in a `select!`, which drops the -/// losing future. Without this guard nothing ever demotes the origin, so a -/// caller whose deadline is shorter than the network's own failure detection -/// re-attempts HTTP/3 over a dead path on every retry, indefinitely. -struct H3AttemptGuard { - cache: Arc, - url: reqwest::Url, - armed: bool, -} - -impl H3AttemptGuard { - fn new(cache: Arc, url: reqwest::Url) -> Self { - Self { - cache, - url, - armed: true, - } - } - - /// The attempt produced an outcome, so it speaks for itself. - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for H3AttemptGuard { - fn drop(&mut self) { - // Must stay infallible: this can run while unwinding, where a panic - // would abort the process. moka's sync cache does not panic on insert. - if self.armed { - self.cache.record_h3_cancellation(&self.url); - } - } -} - -/// Verifies advertised HTTP/3 endpoints in the background, so no foreground -/// request ever waits on an unverified QUIC path. -/// -/// The probe is a real request — `HEAD /` sent with `Version::HTTP_3` — on the -/// **raw** `reqwest::Client`, not the middleware stack. That is load-bearing -/// three times over: it bypasses the HTTP cache, so a replayed cached response -/// (rebuilt with its stored HTTP version) can never fake a confirmation; it -/// bypasses [`AltSvcMiddleware`], so probing cannot recurse; and it shares the -/// h3 connection pool with foreground requests, so a successful probe leaves -/// behind a warm QUIC connection the next request rides. Confirmation doubles -/// as prewarming. -/// -/// Any HTTP/3 response confirms, regardless of status: a 401 or 405 to -/// `HEAD /` proves the transport end-to-end just as well as a 200. -pub struct H3Prober { - client: reqwest::Client, - cache: Arc, - /// `None` leaves the attempt bounded only by the QUIC idle timeout. - timeout: Option, - /// Handles for in-flight probes, so `Agent::close` can abort them: a probe - /// holds a clone of the raw client, which would otherwise keep the - /// connection pool alive past close for up to the probe timeout. - tasks: std::sync::Mutex>, -} - -impl std::fmt::Debug for H3Prober { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("H3Prober") - .field("timeout", &self.timeout) - .finish() - } -} - -impl H3Prober { - pub fn new( - client: reqwest::Client, - cache: Arc, - timeout: Option, - ) -> Self { - Self { - client, - cache, - timeout, - tasks: std::sync::Mutex::new(Vec::new()), - } - } - - /// Spawn a probe of `port` for the origin of `url`. The caller must hold the - /// origin's single-flight claim (see [`AltSvcCache::claim_probe`]). - fn spawn(&self, url: reqwest::Url, port: u16) { - let client = self.client.clone(); - let cache = Arc::clone(&self.cache); - let timeout = self.timeout; - - let handle = tokio::spawn(async move { - let mut probe_url = url.clone(); - probe_url.set_path("/"); - probe_url.set_query(None); - probe_url.set_fragment(None); - let _ = probe_url.set_username(""); - let _ = probe_url.set_password(None); - // Same rewrite rule as the foreground path: a port differing from the - // origin's only gets here when `follow_advertised_port` is on. - if Some(port) != url.port_or_known_default() { - let _ = probe_url.set_port(Some(port)); - } - - let attempt = client.head(probe_url).version(http::Version::HTTP_3).send(); - - let outcome = match timeout { - Some(limit) => tokio::time::timeout(limit, attempt).await.ok(), - None => Some(attempt.await), - }; - - // Cache operations stay keyed on `url`, the origin, matching the - // foreground path. - match outcome { - Some(Ok(response)) if response.version() == http::Version::HTTP_3 => { - cache.confirm_h3(&url, port); - } - // A response that is somehow not HTTP/3 is a failure too: the - // h3 route did not deliver, whatever answered. - _ => cache.record_h3_failure(&url), - } - - // An aborted probe never reaches this; its claim expires on the - // probing TTL instead, which is why that TTL exceeds the timeout. - cache.finish_probe(&url); - }); - - // A poisoned lock only means another thread panicked mid-push; the Vec - // is still sound to use, and probing must never take the process down. - let mut tasks = self - .tasks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - tasks.retain(|task| !task.is_finished()); - tasks.push(handle.abort_handle()); - } - - /// Kick off a background probe for the URL's origin if one is warranted: an actionable - /// advertisement present, the origin neither confirmed, failed, nor slow, and no probe - /// already in flight. The same decision [`AltSvcMiddleware::maybe_probe`] makes, exposed so a - /// `preconnect` TCP warm-up to a probe-worthy origin triggers a probe as a real request would. - pub fn maybe_probe(&self, url: &reqwest::Url) { - let Some(port) = self.cache.probe_candidate(url) else { - return; - }; - if !self.cache.claim_probe(url) { - return; - } - self.spawn(url.clone(), port); - } - - pub fn abort_all(&self) { - let mut tasks = self - .tasks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - for task in tasks.drain(..) { - task.abort(); - } - } -} - -/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is -/// probe-worthy from its first request rather than from the first TCP response carrying an -/// `Alt-Svc` header. -/// -/// Installed on the resolver by the agent (see [`crate::dns::FaithResolver::set_https_sink`]), -/// which is the only place that holds all three: the resolver is built before the cache, and the -/// prober holds a client that holds the resolver, so nothing lower down can own this. -/// -/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the -/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could -/// name. Recording it there is right whichever request triggered the lookup, because what the -/// record describes does not depend on who asked. -/// -/// spec:H3UP#advertisements-from-dns spec:DNS#https-records -pub struct H3HttpsSink { - cache: Arc, - /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, - /// and the resolver holds this sink. A strong reference here would close that ring and leak - /// the whole graph — connection pool included — past `Agent::close`, which works by dropping - /// the client. The agent owns the only strong reference, so this lives exactly as long as the - /// agent's prober does. - /// - /// `None` rather than a dead handle when probing is off, where an advertisement is acted on - /// inline by the next foreground request instead (spec:PROBE). - prober: Option>, -} - -impl std::fmt::Debug for H3HttpsSink { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("H3HttpsSink") - .field("probing", &self.prober.is_some()) - .finish() - } -} - -impl H3HttpsSink { - pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { - Self { - cache, - prober: prober.map(Arc::downgrade), - } - } - - /// The origin a record at `host` describes: the default HTTPS port, which is the port whose - /// record lives at the bare name. - fn origin_url(host: &str) -> Option { - reqwest::Url::parse(&format!("https://{host}")).ok() - } -} - -impl crate::dns::HttpsSink for H3HttpsSink { - fn wants(&self, host: &str) -> bool { - Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) - } - - fn record(&self, host: &str, advertisement: crate::dns::HttpsAdvertisement) { - let Some(url) = Self::origin_url(host) else { - return; - }; - - self.cache - .record_https_record(&url, advertisement.port, advertisement.ttl); - - // Probe straight away rather than waiting for the request that triggered the lookup to - // finish: the point of reading DNS is that the path can be verified while that request is - // still on TCP, so the one after it upgrades. - // - // A prober that has gone means the agent was closed (or rebuilt) while this query was in - // flight; the advertisement above is still worth keeping, but there is nothing left to - // probe it with, and resurrecting a dropped client to try would be exactly wrong. - if let Some(prober) = self.prober.as_ref().and_then(std::sync::Weak::upgrade) { - prober.maybe_probe(&url); - } - } -} - -#[derive(Clone)] -pub struct AltSvcMiddleware { - cache: Arc, - enabled: bool, - /// Ceiling on how long an HTTP/3 attempt may take to produce response - /// headers before it is treated as failed and retried over TCP. - attempt_timeout: Option, - /// `Some` routes foreground requests on confirmed origins only, verifying - /// advertisements in the background. `None` restores the inline upgrade, - /// where the next foreground request is the verification. - prober: Option>, -} - -impl std::fmt::Debug for AltSvcMiddleware { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AltSvcMiddleware") - .field("enabled", &self.enabled) - .field("attempt_timeout", &self.attempt_timeout) - .field("prober", &self.prober) - .field("cache", &self.cache) - .finish() - } -} - -impl AltSvcMiddleware { - pub fn new( - cache: Arc, - enabled: bool, - attempt_timeout: Option, - prober: Option>, - ) -> Self { - Self { - cache, - enabled, - attempt_timeout, - prober, - } - } - - #[allow(dead_code)] - pub fn cache(&self) -> &Arc { - &self.cache - } - - /// Kick off a background probe for the URL's origin if one is warranted: - /// probing enabled, an actionable advertisement present, the origin neither - /// confirmed, failed, nor slow, and no probe already in flight. - fn maybe_probe(&self, url: &reqwest::Url) { - if let Some(prober) = &self.prober { - prober.maybe_probe(url); - } - } -} - -/// Run the rest of the stack and stamp the moment the response headers arrive. -/// -/// This is the one place a response's arrival is observed: the returned instant is what the -/// path-time average measures against, and the same instant reaches the caller through the -/// request's extensions to become the surfaced timing, so the two can never disagree. -/// -/// spec:RESP#request-timing -async fn run_stamped( - next: Next<'_>, - req: Request, - extensions: &mut Extensions, -) -> (Result, Instant) { - let result = next.run(req, extensions).await; - let at = Instant::now(); - if result.is_ok() - && let Some(stamp) = extensions.get::() - { - stamp.mark(at); - } - (result, at) -} - -#[async_trait::async_trait] -impl Middleware for AltSvcMiddleware { - async fn handle( - &self, - mut req: Request, - extensions: &mut Extensions, - next: Next<'_>, - ) -> Result { - if !self.enabled { - return run_stamped(next, req, extensions).await.0; - } - - let url = req.url().clone(); - - // With a prober, routing consults proven origins only — advertisements - // get verified out-of-band, so no foreground request ever waits on an - // unverified QUIC path. Without one, the legacy inline upgrade applies. - let h3_route = if self.prober.is_some() { - self.cache.confirmed_port(&url) - } else { - self.cache.should_use_h3(&url) - }; - - if let Some(h3_port) = h3_route { - // Clone the request before attempting HTTP/3 so we can retry with TCP if it fails - if let Some(req_clone) = req.try_clone() { - *req.version_mut() = http::Version::HTTP_3; - - // A port differing from the origin's only comes back when - // `follow_advertised_port` is set — `should_use_h3` filters - // mismatches out otherwise. Rewriting the URL is the only way to - // make reqwest connect elsewhere, and it MUST happen after the - // clone above so the TCP fallback still targets the origin. - // - // Every cache operation below keeps using `url`, the origin, so - // confirmations, failures and strikes stay keyed on the origin - // rather than on the alternative endpoint. - if Some(h3_port) != url.port_or_known_default() { - let _ = req.url_mut().set_port(Some(h3_port)); - } - - let mut guard = H3AttemptGuard::new(Arc::clone(&self.cache), url.clone()); - // Measured to response headers: this layer sits inside the cache - // middleware, so `next.run` resolves when headers arrive, before - // any body buffering. - let started = Instant::now(); - // `None` means the attempt ran out of time. Bound in its own - // statement so the mutable borrow of `extensions` ends here, - // leaving the fallback below free to use it. - let outcome = match self.attempt_timeout { - Some(limit) => { - tokio::time::timeout(limit, run_stamped(next.clone(), req, extensions)) - .await - .ok() - } - None => Some(run_stamped(next.clone(), req, extensions).await), - }; - // Reached on success, error and expiry alike; only a mid-flight - // drop skips it and leaves the guard armed. - guard.disarm(); - - match outcome { - Some((Ok(response), at)) => { - if response.version() == http::Version::HTTP_3 { - self.cache.confirm_h3(&url, h3_port); - self.cache.record_path_time( - &url, - response.version(), - at.duration_since(started), - ); - } - - if let Some(alt_svc) = response.headers().get("alt-svc") { - if let Ok(value) = alt_svc.to_str() { - if let Some(advertisement) = parse_alt_svc_header(value) { - self.cache.record_alt_svc(&url, &advertisement); - } - } - } - - Ok(response) - } - // An expired deadline is as good as an error: HTTP/3 did not - // deliver. Taking the fallback branch directly avoids having - // to synthesise a reqwest_middleware::Error, which would mean - // adding anyhow as a dependency. - Some((Err(_), _)) | None => { - self.cache.record_h3_failure(&url); - - // Use the cloned request (which still has default HTTP version) - let started = Instant::now(); - let (result, at) = run_stamped(next, req_clone, extensions).await; - if let Ok(ref response) = result { - self.cache.record_path_time( - &url, - response.version(), - at.duration_since(started), - ); - } - result - } - } - } else { - // Can't clone request (streaming body), just proceed without HTTP/3 - run_stamped(next, req, extensions).await.0 - } - } else { - // An advertisement from an earlier response may still be waiting on - // verification (or on a fresh single-flight claim after a probe task - // died); this is the belt to the post-response trigger's braces. - self.maybe_probe(&url); - - let started = Instant::now(); - let (result, at) = run_stamped(next, req, extensions).await; - - // Check for Alt-Svc header in non-HTTP/3 responses - if let Ok(ref response) = result { - self.cache - .record_path_time(&url, response.version(), at.duration_since(started)); - - if let Some(alt_svc) = response.headers().get("alt-svc") { - if let Ok(value) = alt_svc.to_str() { - if let Some(advertisement) = parse_alt_svc_header(value) { - self.cache.record_alt_svc(&url, &advertisement); - // Probe as soon as the advertisement lands, racing - // the gap before the caller's next request. - self.maybe_probe(&url); - } - } - } - } - - result - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_alt_svc_simple() { - let result = parse_alt_svc_header(r#"h3=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_no_max_age() { - let result = parse_alt_svc_header(r#"h3=":443""#); - assert_eq!(result, Some(ad(443, None))); - } - - #[test] - fn test_parse_alt_svc_different_port() { - let result = parse_alt_svc_header(r#"h3=":8443"; ma=3600"#); - assert_eq!(result, Some(ad(8443, Some(Duration::from_secs(3600))))); - } - - #[test] - fn test_parse_alt_svc_multiple_protocols() { - let result = parse_alt_svc_header(r#"h2=":443", h3=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_h3_variant() { - let result = parse_alt_svc_header(r#"h3-29=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_keeps_the_host() { - let result = parse_alt_svc_header(r#"h3="cdn.example.net:443"; ma=3600"#); - assert_eq!( - result, - Some(AltSvcAdvertisement { - host: "cdn.example.net".to_string(), - port: 443, - max_age: Some(Duration::from_secs(3600)), - }), - "the alt-authority's host must survive parsing, or a different-host \ - advertisement looks same-host once the port matches" - ); - } - - #[test] - fn test_parse_alt_svc_ipv6_host() { - let result = parse_alt_svc_header(r#"h3="[2001:db8::1]:8443""#); - assert_eq!( - result, - Some(AltSvcAdvertisement { - // Brackets kept: this is the form `Url::host_str` returns too, so the - // two compare directly. - host: "[2001:db8::1]".to_string(), - port: 8443, - max_age: None, - }), - "splitting on the last colon keeps a bracketed IPv6 literal intact" - ); - } - - #[test] - fn test_ipv6_origin_accepts_its_own_host_spelled_out() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://[2001:db8::1]/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "[2001:db8::1]".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "an IPv6 origin naming its own address is the same host, brackets and all" - ); - } - - #[test] - fn test_host_comparison_ignores_case() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "ExAmPlE.CoM".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "host names are case-insensitive, so this still names the origin's own host" - ); - } - - #[test] - fn test_alt_svc_on_another_host_is_not_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "cdn.example.net".to_string(), - port: 443, - max_age: None, - }, - ); - - assert!( - cache.should_use_h3(&url).is_none(), - "h3 on another host says nothing about this one, and the port matching is \ - coincidental" - ); - } - - #[test] - fn test_alt_svc_naming_our_own_host_is_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "example.com".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "spelling out the origin's own host is equivalent to omitting it" - ); - } - - #[test] - fn test_confirm_h3_uses_the_port_it_was_given() { - // A concurrent failure can clear both caches between the attempt starting and - // confirming. `confirm_h3` must not fall back to the origin's port then, or it - // would confirm h3 on a port nobody advertised. - let cache = test_cache_with(3, Duration::from_secs(60), true); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - cache.record_h3_failure(&url); - cache.confirm_h3(&url, 8443); - - let entry = cache - .confirmed - .get(&"https://example.com:443".to_string()) - .expect("the successful attempt is confirmed"); - assert_eq!( - entry.port, 8443, - "confirmed on the port actually connected to, not the origin's" - ); - } - - #[test] - fn test_parse_alt_svc_clear() { - let result = parse_alt_svc_header("clear"); - assert_eq!(result, None); - } - - #[test] - fn test_parse_alt_svc_no_h3() { - let result = parse_alt_svc_header(r#"h2=":443"; ma=86400"#); - assert_eq!(result, None); - } - - /// A same-host advertisement, the common case. - fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { - AltSvcAdvertisement { - host: String::new(), - port, - max_age, - } - } - - fn test_cache() -> AltSvcCache { - test_cache_with(3, Duration::from_secs(60), false) - } - - fn test_cache_with( - cancel_strikes: u32, - strike_window: Duration, - follow_advertised_port: bool, - ) -> AltSvcCache { - test_cache_failing( - cancel_strikes, - strike_window, - follow_advertised_port, - Duration::from_secs(300), - Duration::from_secs(3600), - ) - } - - /// A cache whose knowledge expires soon, for tests about knowledge that has lapsed. - fn test_cache_ttls(advertised_ttl: Duration, confirmed_ttl: Duration) -> AltSvcCache { - AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl: Duration::from_secs(300), - failed_max_ttl: Duration::from_secs(3600), - capacity: 10_000, - cancel_strikes: 3, - strike_window: Duration::from_secs(60), - follow_advertised_port: false, - probe_ttl: Duration::from_secs(10), - slow_factor: 2.5, - slow_ttl: Duration::from_millis(200), - }) - } - - fn test_cache_failing( - cancel_strikes: u32, - strike_window: Duration, - follow_advertised_port: bool, - failed_ttl: Duration, - failed_max_ttl: Duration, - ) -> AltSvcCache { - AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl: Duration::from_secs(86400), - confirmed_ttl: Duration::from_secs(86400), - failed_ttl, - failed_max_ttl, - capacity: 10_000, - cancel_strikes, - strike_window, - follow_advertised_port, - probe_ttl: Duration::from_secs(10), - slow_factor: 2.5, - slow_ttl: Duration::from_millis(200), - }) - } - - #[test] - fn test_advertised_port_matching_origin_upgrades() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "an advertisement for the origin's own port is actionable" - ); - } - - #[test] - fn test_advertised_port_mismatch_does_not_upgrade() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - assert!( - cache.should_use_h3(&url).is_none(), - "h3 advertised on :8443 says nothing about :443, so don't upgrade" - ); - } - - #[test] - fn test_advertised_port_mismatch_is_still_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - let entry = cache - .advertised - .get(&"https://example.com:443".to_string()) - .expect("the advertisement is kept even though it isn't actionable"); - assert_eq!( - entry.port, 8443, - "keeping it means the port is available if reqwest ever lets us honour it" - ); - } - - #[test] - fn test_advertised_port_mismatch_upgrades_when_following() { - let cache = test_cache_with(3, Duration::from_secs(60), true); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - assert_eq!( - cache.should_use_h3(&url), - Some(8443), - "opting in returns the advertised port so the request can be rewritten" - ); - } - - #[test] - fn test_cache_flow() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - assert!(cache.should_use_h3(&url).is_none()); - - cache.record_alt_svc(&url, &ad(443, Some(Duration::from_secs(3600)))); - assert_eq!(cache.should_use_h3(&url), Some(443)); - - cache.confirm_h3(&url, 443); - assert_eq!(cache.should_use_h3(&url), Some(443)); - assert!( - !cache - .advertised - .contains_key(&"https://example.com:443".to_string()) - ); - assert!( - cache - .confirmed - .contains_key(&"https://example.com:443".to_string()) - ); - } - - #[test] - fn test_cache_failure() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.should_use_h3(&url).is_some()); - - cache.record_h3_failure(&url); - assert!(cache.should_use_h3(&url).is_none()); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.should_use_h3(&url).is_none()); - } - - #[test] - fn test_hint() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - assert_eq!(cache.should_use_h3(&url), Some(443)); - } - - #[test] - fn test_hint_is_confirmed_not_probed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "a hint is the user's assertion and routes immediately, probe or no probe" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "nothing to verify: the hint already confirmed the origin" - ); - } - - #[test] - fn test_https_record_lands_as_an_advertisement_not_a_confirmation() { - // spec:H3UP#advertisements-from-dns — DNS is a second source of the same advertisement, - // so it makes the origin probe-worthy without routing a foreground request onto an - // unverified QUIC path. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, None, Duration::from_secs(3600)); - - assert!( - cache.confirmed_port(&url).is_none(), - "a record is evidence worth probing, not worth routing on" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "and a record naming no port describes the origin's own" - ); - } - - #[test] - fn test_https_record_port_follows_the_advertised_port_rules() { - // spec:H3UP#advertised-ports — a `port` differing from the origin's is treated exactly as - // an `Alt-Svc` advertised port: recorded, but not acted on by default. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, Some(8443), Duration::from_secs(3600)); - - assert!( - cache.probe_candidate(&url).is_none(), - "h3 on :8443 says nothing about :443, so nothing is probed" - ); - - let following = test_cache_with(3, Duration::from_secs(60), true); - following.record_https_record(&url, Some(8443), Duration::from_secs(3600)); - assert_eq!( - following.probe_candidate(&url), - Some(8443), - "opting into the quirk probes the advertised port" - ); - } - - #[test] - fn test_https_record_is_refused_while_the_origin_is_failed() { - // A failure blocks recording fresh advertisements whatever their source, or a flapping - // origin could re-enter the cycle through DNS (spec:H3UP#advertisements-from-dns). - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.record_https_record(&url, None, Duration::from_secs(3600)); - - assert!(cache.probe_candidate(&url).is_none()); - assert!( - !cache.wants_https_record(&url), - "and there is no point querying again while the cooldown runs" - ); - } - - #[test] - fn test_wants_https_record_only_while_there_is_something_to_learn() { - // spec:DNS#https-records — the gate is what keeps the query off every lookup once the - // origin's HTTP/3 support is settled either way. - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - let unknown = test_cache(); - assert!( - unknown.wants_https_record(&url), - "an origin nothing is known about is worth asking about" - ); - - let advertised = test_cache(); - advertised.record_alt_svc(&url, &ad(443, None)); - assert!( - !advertised.wants_https_record(&url), - "a live advertisement already warrants the probe a record would" - ); - - let confirmed = test_cache(); - confirmed.confirm_h3(&url, 443); - assert!( - !confirmed.wants_https_record(&url), - "a confirmed origin is already routing over HTTP/3" - ); - } - - #[test] - fn test_wants_https_record_again_once_the_advertisement_lapses() { - // The gate must not be permanent: an advertisement that expires without being confirmed - // leaves the origin unknown again, and DNS is how it can be re-learned. - let cache = test_cache_ttls(Duration::from_millis(50), Duration::from_secs(86400)); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(!cache.wants_https_record(&url)); - - std::thread::sleep(Duration::from_millis(120)); - - assert!( - cache.wants_https_record(&url), - "once the advertisement lapses the query is worth making again" - ); - } - - #[test] - fn test_https_record_ttl_bounds_the_advertisement() { - // spec:H3UP#advertisements-from-dns — the record's own DNS TTL is what the advertisement - // lives for, the role `ma` plays for a header. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, None, Duration::from_millis(50)); - assert_eq!(cache.probe_candidate(&url), Some(443)); - - std::thread::sleep(Duration::from_millis(120)); - - assert!( - cache.probe_candidate(&url).is_none(), - "past the record's TTL the advertisement is no longer evidence" - ); - } - - #[test] - fn test_advertised_routes_nothing_but_probes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - assert!( - cache.confirmed_port(&url).is_none(), - "an advertisement is evidence worth probing, not worth routing on" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "and it is exactly what the probe should verify" - ); - } - - #[test] - fn test_probe_candidate_respects_failed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - - assert!( - cache.probe_candidate(&url).is_none(), - "a failed origin is not re-probed until the cooldown lapses" - ); - } - - #[test] - fn test_probe_confirmation_promotes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.claim_probe(&url), "first claim wins"); - cache.confirm_h3(&url, 443); - cache.finish_probe(&url); - - assert_eq!(cache.confirmed_port(&url), Some(443)); - assert!( - cache.probe_candidate(&url).is_none(), - "confirmed origins are not probed again" - ); - } - - #[test] - fn test_claim_probe_is_single_flight() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - assert!(cache.claim_probe(&url)); - assert!( - !cache.claim_probe(&url), - "a second claim while one is in flight loses" - ); - - cache.finish_probe(&url); - assert!( - cache.claim_probe(&url), - "finishing the probe frees the origin for the next one" - ); - } - - #[test] - fn test_slow_demotion_needs_sustained_evidence() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // Plenty of TCP samples at 5ms, but too few QUIC samples to act on. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - } - for _ in 0..(EWMA_MIN_SAMPLES - 1) { - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "below the minimum sample count no comparison may act" - ); - } - - #[test] - fn test_slow_demotion_moves_origin_back_to_probing() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // TCP steady at 5ms, QUIC steady at 50ms: 10x the average and 45ms over, - // clearing both the factor and the absolute floor. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert!( - cache.confirmed_port(&url).is_none(), - "a sustained large gap demotes the origin off HTTP/3" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "while the slow marker lives, the origin is not re-probed either" - ); - assert!( - !cache.is_failed("https://example.com:443"), - "slow is not broken: the failed cache stays out of it" - ); - - // The test cache's slow TTL is short; once it lapses, the advertisement - // preserved by the demotion re-enters through a probe. - std::thread::sleep(Duration::from_millis(300)); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "slow expiry re-enters via the probe, asking whether the path improved" - ); - } - - #[test] - fn test_parity_or_moderately_slower_quic_is_kept() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // QUIC 2x slower and 20ms over: above the floor but below the 2.5x - // factor, so HTTP/3's structural advantages win the tie. - for _ in 0..(EWMA_MIN_SAMPLES * 2) { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(20)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(40)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "moderately slower QUIC is still preferred" - ); - } - - #[test] - fn test_cancellation_below_threshold_keeps_h3() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "two strikes is not enough to demote" - ); - } - - #[test] - fn test_cancellation_at_threshold_demotes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - for _ in 0..3 { - cache.record_h3_cancellation(&url); - } - - assert!( - cache.should_use_h3(&url).is_none(), - "three strikes demotes the origin" - ); - assert!( - cache.is_failed("https://example.com:443"), - "demotion goes through the failed cache, so re-advertisement can't re-arm it" - ); - } - - #[test] - fn test_cancellation_reset_by_h3_success() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - cache.confirm_h3(&url, 443); - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "a working h3 response clears the strikes, so these two start over" - ); - } - - #[test] - fn test_cancellation_disabled_by_zero() { - let cache = test_cache_with(0, Duration::from_secs(60), false); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - for _ in 0..5 { - cache.record_h3_cancellation(&url); - } - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "cancel_strikes: 0 disables cancellation-based demotion" - ); - } - - #[test] - fn test_cancellation_strikes_decay() { - let cache = test_cache_with(3, Duration::from_millis(50), false); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - std::thread::sleep(Duration::from_millis(150)); - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "strikes older than the window don't count towards the run" - ); - } - - fn failure_entry(cache: &AltSvcCache) -> FailureEntry { - cache - .failed - .get("https://example.com:443") - .expect("the origin has a failure on record") - } - - #[test] - fn test_failure_cooldown_doubles_up_to_the_cap() { - let cache = test_cache(); - - let schedule: Vec = (1..=6) - .map(|count| cache.failure_cooldown(count).as_secs()) - .collect(); - - assert_eq!( - schedule, - vec![300, 600, 1200, 2400, 3600, 3600], - "each consecutive failure doubles the base, then holds at the cap" - ); - } - - #[test] - fn test_failure_cooldown_cap_below_base_is_flat() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_secs(300), - Duration::from_secs(60), - ); - - let schedule: Vec = (1..=4) - .map(|count| cache.failure_cooldown(count).as_secs()) - .collect(); - - assert_eq!( - schedule, - vec![300, 300, 300, 300], - "a cap under the base is clamped up to it, giving a cooldown that never backs off" - ); - } - - #[test] - fn test_consecutive_failures_lengthen_the_cooldown() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_millis(200), - Duration::from_secs(60), - ); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - assert!( - cache.is_failed("https://example.com:443"), - "the first failure blocks the origin" - ); - - // Past the first cooldown, but well inside the run's own lifetime: this is - // the retry the cooldown allowed, and it fails too. - std::thread::sleep(Duration::from_millis(250)); - assert!( - !cache.is_failed("https://example.com:443"), - "the first cooldown lapses on its own" - ); - - cache.record_h3_failure(&url); - let entry = failure_entry(&cache); - assert_eq!( - entry.count, 2, - "failing again straight after a lapsed cooldown continues the run" - ); - assert!( - cache.is_failed("https://example.com:443"), - "and blocks the origin again, for twice as long" - ); - } - - #[test] - fn test_run_lapses_when_the_origin_stops_failing() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_millis(100), - Duration::from_secs(60), - ); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - // One cooldown beyond the block it caused: nobody exercised the origin in - // that stretch, so the next failure is judged on its own. - std::thread::sleep(Duration::from_millis(300)); - - cache.record_h3_failure(&url); - assert_eq!( - failure_entry(&cache).count, - 1, - "an origin left alone past its run starts from the base cooldown again" - ); - } - - #[test] - fn test_confirmation_ends_the_run() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.record_h3_failure(&url); - assert_eq!(failure_entry(&cache).count, 2); - - cache.confirm_h3(&url, 443); - cache.record_h3_failure(&url); - - let entry = failure_entry(&cache); - assert_eq!( - entry.count, 1, - "a working h3 response ends the run, so the next failure starts at the base" - ); - assert_eq!( - entry.blocked_until.duration_since(Instant::now()).as_secs(), - 299, - "and is blocked for the base cooldown, not the doubled one" - ); - } - - #[test] - fn test_confirmation_does_not_cut_a_live_cooldown_short() { - // A confirmation can race a concurrent failure. The failure is the more - // recent word on the path, so it keeps the origin blocked; only the run - // is forgotten. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.confirm_h3(&url, 443); - - assert!( - cache.is_failed("https://example.com:443"), - "the cooldown the failure set still runs" - ); - assert_eq!( - failure_entry(&cache).count, - 0, - "but the run behind it is cleared" - ); - } - - #[test] - fn test_network_change_demotes_confirmed_to_advertised() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - cache.network_changed(); - - assert!( - cache.confirmed_port(&url).is_none(), - "the path that proved HTTP/3 is gone, so the origin is no longer confirmed" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "it keeps its advertisement, so a background probe re-verifies it at once" - ); - } - - #[test] - fn test_network_change_clears_failures_and_their_backoff() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.network_changed(); - - assert!( - !cache.is_failed("https://example.com:443"), - "a blocked UDP path is a fact about the old network" - ); - assert!( - cache.failed.get("https://example.com:443").is_none(), - "and so is the run of failures that set the cooldown" - ); - - // The advertisement a failure discards has to come back for the origin to be - // probe-worthy again, so re-record it as a live response would. - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - assert_eq!( - failure_entry(&cache).count, - 1, - "failing on the new network starts the backoff from the base cooldown" - ); - } - - #[test] - fn test_network_change_clears_a_slow_demotion() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - assert!( - cache.probe_candidate(&url).is_none(), - "the slow marker holds the origin off probing before the signal" - ); - - cache.network_changed(); - - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "a slow path was slow on the old network, so the origin re-enters through a probe" - ); - } - - #[test] - fn test_network_change_clears_the_path_time_averages() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - // Enough TCP samples to satisfy the comparison's minimum, all of them fast. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - } - - cache.network_changed(); - cache.confirm_h3(&url, 443); - - // Slow enough to demote several times over, were the old TCP average still there - // to compare against. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "with the TCP average cleared there is nothing to judge QUIC against, so \ - no demotion happens on one side's samples alone" - ); - } - - #[test] - fn test_network_change_keeps_hints_confirmed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - - cache.network_changed(); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "a hint is the caller's assertion, not an observation, so it survives the signal \ - and keeps an HTTP/3-only origin reachable" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "and a hinted origin is still never probed" - ); - } - - #[test] - fn test_network_change_lets_a_blocked_hint_take_effect() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - cache.record_h3_failure(&url); - assert!( - cache.confirmed_port(&url).is_none(), - "a failure demotes a hinted origin like any other" - ); - - cache.network_changed(); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "the failure that was masking the hint belonged to the old network, so the \ - hint holds again once it is cleared" - ); - } - - #[test] - fn test_network_change_keeps_advertisements() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - cache.network_changed(); - - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "an advertisement is the origin's statement about itself, which a change of \ - client network does not revise" - ); - } - - #[test] - fn test_network_change_drops_expired_confirmations() { - let cache = test_cache_ttls(Duration::from_millis(100), Duration::from_millis(100)); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.confirm_h3(&url, 443); - std::thread::sleep(Duration::from_millis(150)); - - cache.network_changed(); - - assert!( - cache.probe_candidate(&url).is_none(), - "a confirmation that had already lapsed is not knowledge to carry forward as \ - a fresh advertisement" - ); - } - - #[test] - fn test_network_change_releases_probe_claims() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.claim_probe(&url), "the first probe claims the origin"); - assert!(!cache.claim_probe(&url), "and holds it single-flight"); - - cache.network_changed(); - - assert!( - cache.claim_probe(&url), - "probes in flight are aborted with the client they ran on, so their claims \ - must not hold the origin for the claim TTL" - ); - } -} diff --git a/src/dns.rs b/src/dns.rs deleted file mode 100644 index 0077621..0000000 --- a/src/dns.rs +++ /dev/null @@ -1,1535 +0,0 @@ -//! Faith's own DNS resolver. -//! -//! `reqwest`'s built-in hickory resolver and its in-memory cache are `pub(crate)`, so the only -//! way to warm that cache is to make a request through it — which is `preconnect`'s job, not -//! `prefetchDns`'s (that verb must not touch the origin). To let `prefetchDns` populate the cache -//! a later request reads, Faith owns the resolver instead: this type is installed on the `reqwest` -//! client with `ClientBuilder::dns_resolver`, so reqwest routes every lookup through it, and -//! `prefetch` calls the same resolver directly. Both share one `TokioResolver`, so a name warmed -//! by `prefetchDns` is already cached when a request looks it up. -//! -//! Beyond warming, this type is where the DNS transports and server order live. `dns.servers` -//! lists resolver URLs whose scheme picks the transport (`udp`/`tcp`, or the encrypted `tls`, -//! `https`, `quic`, `h3`); the list is queried in order, held fixed with `UserProvidedOrder`. With -//! no list, the resolver configures itself from the system and lets hickory's RFC 9539 -//! opportunistic encryption upgrade those servers where it can. Either way, [exempt names] go to -//! the system resolver instead, so local names keep resolving. -//! -//! [exempt names]: ResolverSettings::exempt_domains -//! -//! spec:WARM spec:DNS - -use std::{ - collections::HashSet, - net::{IpAddr, SocketAddr}, - sync::{Arc, Mutex}, - time::{Duration, Instant}, -}; - -use hickory_resolver::{ - TokioResolver, - config::{ - ConnectionConfig, GOOGLE, LookupIpStrategy, NameServerConfig, OpportunisticEncryption, - ProtocolConfig, ResolveHosts, ResolverConfig, ServerOrderingStrategy, - }, - net::{DnsError, NetError, runtime::TokioRuntimeProvider}, - proto::rr::{ - Name, RData, RecordType, - rdata::svcb::{SvcParamKey, SvcParamValue}, - }, - system_conf::read_system_conf, -}; -use reqwest::dns::{Addrs, Name as ReqName, Resolve, Resolving}; -use tokio::sync::OnceCell; -use url::{Host, Url}; - -/// The default DoH/DoQ query path, used when a `https://`/`h3://` server URL supplies none. -const DEFAULT_DNS_QUERY_PATH: &str = "/dns-query"; - -/// Parse a `dns.searchDomains` or `dns.exemptDomains` list into domain names, or return a message -/// for the first entry that is not a valid domain name. -pub fn parse_domains(list: Option>) -> Result>, String> { - list.map(|items| { - items - .iter() - .map(|item| Name::from_utf8(item).map_err(|err| format!("{item:?}: {err}"))) - .collect() - }) - .transpose() -} - -/// The transport Faith speaks to a resolver, chosen by a server URL's scheme. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Transport { - Udp, - Tcp, - Tls, - Https, - Quic, - H3, -} - -impl Transport { - fn from_scheme(scheme: &str) -> Option { - Some(match scheme { - "udp" => Self::Udp, - "tcp" => Self::Tcp, - "tls" => Self::Tls, - "https" => Self::Https, - "quic" => Self::Quic, - "h3" => Self::H3, - _ => return None, - }) - } - - /// The conventional port for the transport, used when the URL names none. - fn default_port(self) -> u16 { - match self { - Self::Udp | Self::Tcp => 53, - Self::Tls | Self::Quic => 853, - Self::Https | Self::H3 => 443, - } - } - - /// The lowercase label reported by `resolvers()`. - fn label(self) -> &'static str { - match self { - Self::Udp => "udp", - Self::Tcp => "tcp", - Self::Tls => "tls", - Self::Https => "https", - Self::Quic => "quic", - Self::H3 => "h3", - } - } -} - -/// What an `HTTPS` record said about an origin's HTTP/3 support (spec:DNS#https-records). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct HttpsAdvertisement { - /// The record's `port` SvcParam, or `None` when it named none and the origin's own port - /// applies. - pub port: Option, - /// The record's own DNS TTL, which is how long the advertisement it carries lives. - pub ttl: Duration, -} - -/// Where an `HTTPS` record's advertisement goes once the resolver has read one. -/// -/// The resolver cannot own the HTTP/3 upgrade cache directly: that cache is built after the -/// resolver, and the background prober holds a client which holds the resolver in turn. So the -/// agent installs this afterwards (see [`FaithResolver::set_https_sink`]), which also keeps -/// `dns.rs` free of the upgrade layer's types. -pub trait HttpsSink: Send + Sync { - /// Whether an `HTTPS` record for `host` is worth querying at all right now. - /// - /// Asked before the query so an origin already confirmed, already failed, or already holding a - /// live advertisement costs no DNS traffic to re-learn what is known. - fn wants(&self, host: &str) -> bool; - - /// Fold a record's advertisement into the upgrade layer's knowledge of `host`. - fn record(&self, host: &str, advertisement: HttpsAdvertisement); -} - -/// Whether an ALPN token names a version of HTTP/3. -/// -/// The same family test the `Alt-Svc` reader applies, so a draft token like `h3-29` counts here -/// exactly as it does in a header (spec:H3UP#reading-advertisements). -fn is_h3_alpn(token: &str) -> bool { - token == "h3" || token.starts_with("h3-") -} - -/// Read the HTTP/3 advertisement out of an `HTTPS` answer for `name`, if it carries one. -/// -/// Only ServiceMode records are considered: an AliasMode record (`svc_priority` 0) redirects to -/// another name rather than describing this one, and following that redirection is a resolution -/// step this does not take. Among the rest the lowest `svc_priority` wins, which is the preference -/// order RFC 9460 defines. -/// -/// A record whose target is neither the root (which per RFC 9460 §2.5.2 means the owner name -/// itself) nor the queried name designates a *different* host, and Faith only upgrades to the -/// origin's own host, so such a record is not acted on (spec:H3UP#advertisements-from-dns). -/// `queried` is the name the answer was actually asked for rather than the host as written, since -/// the search list can requalify a name before it reaches a server; comparison ignores the trailing -/// root so the two are judged on identity rather than on how each was spelled. -fn read_https_answer( - queried: &Name, - answers: &[hickory_resolver::proto::rr::Record], -) -> Option { - let mut best: Option<(u16, HttpsAdvertisement)> = None; - - for record in answers { - let RData::HTTPS(https) = &record.data else { - continue; - }; - - if https.svc_priority == 0 { - continue; - } - - if !https.target_name.is_root() && !https.target_name.eq_ignore_root(queried) { - continue; - } - - let mut has_h3 = false; - let mut port = None; - for (key, value) in &https.svc_params { - match (key, value) { - (SvcParamKey::Alpn, SvcParamValue::Alpn(alpn)) => { - has_h3 = alpn.0.iter().any(|token| is_h3_alpn(token)); - } - (SvcParamKey::Port, SvcParamValue::Port(value)) => port = Some(*value), - _ => {} - } - } - - if !has_h3 { - continue; - } - - let advertisement = HttpsAdvertisement { - port, - ttl: Duration::from_secs(record.ttl.into()), - }; - if best - .as_ref() - .is_none_or(|(best, _)| https.svc_priority < *best) - { - best = Some((https.svc_priority, advertisement)); - } - } - - best.map(|(_, advertisement)| advertisement) -} - -/// A resolver Faith reaches by IP or by a hostname it bootstraps. -#[derive(Clone, Debug)] -enum ServerHost { - Ip(IpAddr), - Name(String), -} - -/// One entry of `dns.servers`, parsed at agent construction. The IP is not known yet for a -/// hostname host: that is resolved when the resolver is first used (see [`ResolverSettings`]). -#[derive(Clone, Debug)] -pub struct ServerSpec { - host: ServerHost, - transport: Transport, - port: u16, - /// DoH/DoQ query path, `None` for the non-HTTP transports. - path: Option>, - /// The name to authenticate the certificate against, from a URL fragment. When absent, a - /// hostname host authenticates against itself and an IP host against the address. - cert_name: Option, -} - -impl ServerSpec { - /// Parse one `dns.servers` URL, or return a message for an unparseable URL or unknown scheme. - pub fn parse(input: &str) -> Result { - let url = Url::parse(input).map_err(|err| format!("{input:?}: {err}"))?; - let transport = Transport::from_scheme(url.scheme()) - .ok_or_else(|| format!("{input:?}: unknown DNS transport scheme {:?}", url.scheme()))?; - - let host = match url.host() { - Some(Host::Ipv4(ip)) => ServerHost::Ip(IpAddr::V4(ip)), - Some(Host::Ipv6(ip)) => ServerHost::Ip(IpAddr::V6(ip)), - Some(Host::Domain(name)) => ServerHost::Name(name.to_owned()), - None => return Err(format!("{input:?}: no host to resolve")), - }; - - let port = url.port().unwrap_or_else(|| transport.default_port()); - let path = match transport { - Transport::Https | Transport::H3 => { - let path = url.path(); - (!path.is_empty() && path != "/").then(|| Arc::from(path)) - } - _ => None, - }; - let cert_name = url.fragment().map(str::to_owned); - - Ok(Self { - host, - transport, - port, - path, - cert_name, - }) - } - - /// The IP host, or `None` for a hostname host that still needs bootstrapping. - fn ip(&self) -> Option { - match self.host { - ServerHost::Ip(ip) => Some(ip), - ServerHost::Name(_) => None, - } - } - - /// The certificate name to authenticate against once the host resolves to `ip`: an explicit - /// fragment, else the hostname, else the address itself (spec:DNS#transports). - fn server_name(&self, ip: IpAddr) -> Arc { - if let Some(name) = &self.cert_name { - Arc::from(name.as_str()) - } else { - match &self.host { - ServerHost::Name(name) => Arc::from(name.as_str()), - ServerHost::Ip(_) => Arc::from(ip.to_string()), - } - } - } - - /// Build the hickory name server for this spec, reached at `ip`. - fn to_name_server(&self, ip: IpAddr) -> NameServerConfig { - let protocol = match self.transport { - Transport::Udp => ProtocolConfig::Udp, - Transport::Tcp => ProtocolConfig::Tcp, - Transport::Tls => ProtocolConfig::Tls { - server_name: self.server_name(ip), - }, - Transport::Https => ProtocolConfig::Https { - server_name: self.server_name(ip), - path: self - .path - .clone() - .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), - }, - Transport::Quic => ProtocolConfig::Quic { - server_name: self.server_name(ip), - }, - Transport::H3 => ProtocolConfig::H3 { - server_name: self.server_name(ip), - path: self - .path - .clone() - .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), - disable_grease: false, - }, - }; - - let mut connection = ConnectionConfig::new(protocol); - connection.port = self.port; - NameServerConfig::new(ip, true, vec![connection]) - } -} - -/// How a server in `resolvers()` came to be reached the way it is (spec:OBS#resolvers). -#[derive(Clone, Copy, Debug)] -pub enum ResolverSource { - /// Named in `dns.servers` by the caller. - Configured, - /// Discovered from the system's resolver configuration. - Conventional, -} - -impl ResolverSource { - fn label(self) -> &'static str { - match self { - Self::Configured => "configured", - Self::Conventional => "conventional", - } - } -} - -/// One line of `resolvers()`: a server's address, the transport in use, and how it was arrived at. -#[derive(Clone, Debug)] -pub struct ResolverReport { - pub address: String, - pub transport: String, - pub source: String, -} - -/// `dns.maxStale`'s default: how far past expiry an answer may still be served. -/// -/// An hour is long enough that a resolver outage does not stop an agent reaching hosts it already -/// knows, and short enough that a host which really has moved stops being served a dead address for -/// the life of a long-running process. The recovery path bounds the cost of being wrong to one -/// re-resolve, so the window can be generous (spec:DNS#serving-stale-answers). -pub const DEFAULT_MAX_STALE: Duration = Duration::from_secs(3600); - -/// Everything `dns.*` configures about Faith's resolver, resolved from options at construction. -#[derive(Clone, Debug)] -pub struct ResolverSettings { - /// The `dns.servers` list, in order. Empty means system discovery. - pub servers: Vec, - /// `dns.timeout`, bounding the whole list. `None` leaves hickory's five-second default. - pub timeout: Option, - /// `dns.ndots`. - pub ndots: Option, - /// `dns.searchDomains`, replacing the system's search list when set. - pub search_domains: Option>, - /// `dns.hostsFile`: `Some(true)`/`Some(false)` force it on/off, `None` follows the platform. - pub hosts_file: Option, - /// `dns.exemptDomains`, added to the always-exempt `localhost`, `.local`, and system suffix. - pub exempt_domains: Vec, - /// `dns.serveStale`: whether an expired answer is served while a refresh runs behind it. - pub serve_stale: bool, - /// `dns.maxStale`: how far past expiry an answer may still be served. - pub max_stale: Duration, -} - -impl Default for ResolverSettings { - fn default() -> Self { - Self { - servers: Vec::new(), - timeout: None, - ndots: None, - search_domains: None, - hosts_file: None, - exempt_domains: Vec::new(), - // Defaulted here as well as in the option parsing, so a resolver built directly (in tests, - // and for the global default agent) serves stale like a configured one. - serve_stale: true, - max_stale: DEFAULT_MAX_STALE, - } - } -} - -/// How many names the stale cache holds before evicting the least recently used. -/// -/// Matched to hickory's own default answer-cache size, since the two hold an entry for the same set -/// of names: a stale entry only earns its place while hickory still plausibly holds, or recently -/// held, the answer it came from. Evicting one early costs a blocking lookup rather than a wrong -/// answer, so the bound is about memory rather than correctness. -const STALE_CACHE_SIZE: u64 = 8_192; - -/// The resolver and the report of its servers, built together the first time the resolver is used. -struct Built { - resolver: TokioResolver, - reports: Vec, -} - -/// A resolved answer kept past its TTL, so an expired lookup is served from it while a refresh runs -/// behind (spec:DNS#serving-stale-answers). -#[derive(Clone)] -struct StaleEntry { - /// Shared rather than cloned per hit: a hit reads it and hands out a copy of the addresses. - addrs: Arc>, - /// When the answer stopped being fresh, taken from the lookup rather than computed, so it is the - /// TTL the resolver actually gave. - valid_until: Instant, -} - -/// Everything the resolver reads off the network, held together so a network change can drop it in -/// one go (spec:NETCHG). Each field describes the network the agent was on when it was read: which -/// servers discovery found, which suffixes are local to it, and which of its servers answered an -/// encryption probe. The caller's [`ResolverSettings`] deliberately sit outside, being options the -/// agent was constructed with rather than a reading of any network. -struct Generation { - /// The configured (or discovered) resolver, built lazily inside a tokio runtime. - built: OnceCell>, - /// The system resolver, used for exempt names. Built lazily and independently. - system: OnceCell>, - /// The exempt suffixes, including the system's own, computed once per generation. - exempt: OnceCell>>, - /// Answers held past their TTL, keyed by the host as looked up. Sits in the generation rather - /// than beside the settings so a network change drops it along with the resolvers that produced - /// it: an address learned on the old network is exactly what must not be served on the new one. - stale: moka::sync::Cache, - /// Hosts with a refresh already in flight, so a second stale hit serves the entry rather than - /// starting another lookup (spec:DNS#serving-stale-answers). - refreshing: Mutex>, -} - -impl Default for Generation { - fn default() -> Self { - Self { - built: OnceCell::new(), - system: OnceCell::new(), - exempt: OnceCell::new(), - stale: moka::sync::Cache::new(STALE_CACHE_SIZE), - refreshing: Mutex::new(HashSet::new()), - } - } -} - -struct Inner { - /// The options the agent was constructed with. A network change does not touch these - /// (spec:NETCHG#what-the-signal-keeps); they are what the next generation is rebuilt from. - settings: ResolverSettings, - /// Replaced wholesale by [`FaithResolver::reset`]. Read once at the start of a lookup rather - /// than at each step, so a lookup that spans the signal finishes against the one set of - /// resolvers it started on (spec:NETCHG#in-flight-requests). - generation: Mutex>, - /// Where `HTTPS` records go, installed by the agent once the upgrade cache and prober exist. - /// - /// Sits beside the settings rather than inside the generation deliberately: it is wiring - /// rather than something read off a network, so a network change leaves it in place. Its - /// absence is what turns the `HTTPS` query off, so an agent with HTTP/3 upgrade disabled, or - /// one on the system resolver, never sends one. - https_sink: Mutex>>, -} - -/// A hickory resolver Faith owns, shared between reqwest's request path and `prefetchDns`. -#[derive(Clone)] -pub struct FaithResolver { - inner: Arc, -} - -impl Default for FaithResolver { - fn default() -> Self { - Self::new(ResolverSettings::default()) - } -} - -impl std::fmt::Debug for FaithResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FaithResolver").finish_non_exhaustive() - } -} - -impl FaithResolver { - pub fn new(settings: ResolverSettings) -> Self { - Self { - inner: Arc::new(Inner { - settings, - generation: Mutex::new(Arc::new(Generation::default())), - https_sink: Mutex::new(None), - }), - } - } - - /// Install where `HTTPS` records go, enabling the query (spec:DNS#https-records). - /// - /// Called after the agent's HTTP/3 upgrade cache and prober are built, which cannot happen - /// before the resolver exists. Replaces any previous sink, which is what a network change - /// needs: the prober is rebuilt with the client, so the sink must be too or it would kick - /// probes onto a client that has been dropped. - pub fn set_https_sink(&self, sink: Arc) { - *self - .inner - .https_sink - .lock() - .expect("the HTTPS sink lock is only held to clone or replace an Arc") = Some(sink); - } - - fn https_sink(&self) -> Option> { - self.inner - .https_sink - .lock() - .expect("the HTTPS sink lock is only held to clone or replace an Arc") - .clone() - } - - /// The generation a piece of work resolves against. Taken once per lookup: a reset swaps the - /// generation rather than mutating it, so work already holding one carries on against the - /// resolvers it started with (spec:NETCHG#in-flight-requests). - fn generation(&self) -> Arc { - self.inner - .generation - .lock() - .expect("the DNS generation lock is only held to clone or replace an Arc") - .clone() - } - - async fn built(&self, generation: &Generation) -> Result, NetError> { - generation - .built - .get_or_try_init(|| async { build(&self.inner.settings).await.map(Arc::new) }) - .await - .cloned() - } - - /// The system resolver, for exempt names. Reads the system configuration and races both - /// address families for Happy Eyeballs like the built-in resolver does. - async fn system(&self, generation: &Generation) -> Result, NetError> { - generation - .system - .get_or_try_init(|| async { - let mut builder = TokioResolver::builder_tokio().unwrap_or_else(|_| { - TokioResolver::builder_with_config( - ResolverConfig::udp_and_tcp(&GOOGLE), - TokioRuntimeProvider::default(), - ) - }); - builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - builder.build().map(Arc::new) - }) - .await - .cloned() - } - - /// The exempt suffixes: `localhost`, `local`, the system's own domain and search suffixes, and - /// the caller's `dns.exemptDomains` (spec:DNS#exempt-names). The system's own suffixes are a - /// property of the network, so they are read per generation rather than once per agent. - async fn exempt(&self, generation: &Generation) -> Arc> { - generation - .exempt - .get_or_init(|| async { - let system = read_system_conf() - .map(|(config, _)| { - config - .domain() - .into_iter() - .chain(config.search()) - .cloned() - .collect::>() - }) - .unwrap_or_default(); - Arc::new(exempt_suffixes(system, &self.inner.settings.exempt_domains)) - }) - .await - .clone() - } - - /// Whether `host` must go to the system resolver rather than Faith's servers. - async fn is_exempt(&self, generation: &Generation, host: &str) -> bool { - let Ok(name) = Name::from_utf8(host) else { - return false; - }; - self.exempt(generation) - .await - .iter() - .any(|suffix| suffix.zone_of(&name)) - } - - /// Resolve `host` to its addresses, routing exempt names to the system resolver. - async fn lookup(&self, host: &str) -> Result, NetError> { - let generation = self.generation(); - if self.is_exempt(&generation, host).await { - // The system resolver keeps no cache Faith can hold answers in, so an exempt name has - // nothing to go stale and is always resolved for real (spec:DNS#serving-stale-answers). - let resolver = self.system(&generation).await?; - return Ok(resolver.lookup_ip(host).await?.iter().collect()); - } - - // Alongside the addresses rather than after them: the record is a hint for the upgrade - // layer to verify, so nothing about connecting waits on it (spec:DNS#https-records). - self.spawn_https_query(&generation, host); - - if let Some(addrs) = self.stale_addrs(&generation, host) { - self.spawn_refresh(&generation, host); - return Ok(addrs); - } - - let built = self.built(&generation).await?; - let lookup = built.resolver.lookup_ip(host).await?; - let addrs: Vec = lookup.iter().collect(); - self.remember(&generation, host, &addrs, lookup.valid_until()); - Ok(addrs) - } - - /// Ask for `host`'s `HTTPS` record behind the address lookup, so an origin advertising - /// `alpn="h3"` is known before the first connection rather than after the first TCP response - /// (spec:DNS#https-records). - /// - /// Spawned rather than awaited: an absent, slow, or failed answer must leave address - /// resolution and connecting untouched. Its outcome belongs to the upgrade layer rather than - /// to the request that triggered it, so nothing here reaches a caller, exactly as a stale - /// refresh's outcome does not. - fn spawn_https_query(&self, generation: &Arc, host: &str) { - let Some(sink) = self.https_sink() else { - return; - }; - // Asked before the query, not after: an origin already confirmed, failed, or holding a - // live advertisement has nothing to learn, so it costs no DNS traffic. - if !sink.wants(host) { - return; - } - let Ok(name) = Name::from_utf8(host) else { - return; - }; - - let this = self.clone(); - let generation = Arc::clone(generation); - let host = host.to_owned(); - tokio::spawn(async move { - let Ok(built) = this.built(&generation).await else { - return; - }; - let Ok(lookup) = built.resolver.lookup(name, RecordType::HTTPS).await else { - return; - }; - // The answer's own query name, not the host as written: the search list may have - // requalified it, and the record's target is judged against what was actually asked. - if let Some(advertisement) = read_https_answer(lookup.query().name(), lookup.answers()) - { - sink.record(&host, advertisement); - } - }); - } - - /// The addresses to serve for `host` without waiting, when its answer has expired but is still - /// inside `dns.maxStale`. - /// - /// `None` covers the three cases that must go to the resolver: no entry at all, an entry still - /// fresh (which hickory's own cache answers without a network round trip anyway), and an entry so - /// old it has stopped being evidence about the host. - fn stale_addrs(&self, generation: &Generation, host: &str) -> Option> { - if !self.inner.settings.serve_stale { - return None; - } - let entry = generation.stale.get(host)?; - let now = Instant::now(); - if now <= entry.valid_until { - return None; - } - if now.saturating_duration_since(entry.valid_until) > self.inner.settings.max_stale { - // Dropped rather than left to sit: keeping it would let a refresh that has been failing - // for hours go on being consulted, and the entry can only get older from here. - generation.stale.invalidate(host); - return None; - } - Some(entry.addrs.as_ref().clone()) - } - - /// Keep a successful answer for `host`, so a later lookup past its TTL has something to serve. - fn remember( - &self, - generation: &Generation, - host: &str, - addrs: &[IpAddr], - valid_until: Instant, - ) { - if !self.inner.settings.serve_stale || addrs.is_empty() { - return; - } - generation.stale.insert( - host.to_owned(), - StaleEntry { - addrs: Arc::new(addrs.to_vec()), - valid_until, - }, - ); - } - - /// Refresh `host` behind a stale answer that has already been served. - /// - /// Single-flighted per host: the claim is taken before the task is spawned, so concurrent stale - /// hits serve the entry rather than each starting a lookup. The task outlives the request that - /// triggered it, and its outcome belongs to the cache rather than that request, so nothing here - /// is reported to a caller (spec:DNS#serving-stale-answers). - fn spawn_refresh(&self, generation: &Arc, host: &str) { - { - let mut refreshing = generation - .refreshing - .lock() - .expect("the DNS refresh lock is only held to insert or remove a host"); - if !refreshing.insert(host.to_owned()) { - return; - } - } - - let this = self.clone(); - let generation = Arc::clone(generation); - let host = host.to_owned(); - tokio::spawn(async move { - match this.refresh(&generation, &host).await { - Ok(()) => {} - Err(err) if is_authoritatively_empty(&err) => { - // The name resolves to nothing now, so the old address is not a stale answer for - // it any more but a wrong one. Dropping the entry makes the next lookup fail - // rather than hand out an address the host no longer answers on. - generation.stale.invalidate(&host); - } - Err(_) => { - // A network error, a server failure, or a timeout says nothing about where the - // host is, so the entry stays and can be served again while this persists. - } - } - generation - .refreshing - .lock() - .expect("the DNS refresh lock is only held to insert or remove a host") - .remove(&host); - }); - } - - /// One refresh lookup, replacing the stale entry when it resolves. - async fn refresh(&self, generation: &Generation, host: &str) -> Result<(), NetError> { - let built = self.built(generation).await?; - let lookup = built.resolver.lookup_ip(host).await?; - let addrs: Vec = lookup.iter().collect(); - self.remember(generation, host, &addrs, lookup.valid_until()); - Ok(()) - } - - /// Drop any stale answer held for `host`, so the next lookup waits for a fresh one. - /// - /// Called when connecting to a served address failed, which is the one piece of evidence that the - /// address was wrong rather than merely old (spec:DNS#when-a-stale-address-is-wrong). - pub fn invalidate_stale(&self, host: &str) { - self.generation().stale.invalidate(host); - } - - /// Whether a lookup of `host` right now would be served from an expired entry, and so would hand - /// out an address that is assumed rather than confirmed. - /// - /// Deliberately the same window [`Self::stale_addrs`] serves from, rather than merely "an expired - /// entry exists": an entry past `dns.maxStale` is resolved for real, and treating that as stale - /// would spend a second connection attempt on an address that was already confirmed. - pub fn served_stale(&self, host: &str) -> bool { - if !self.inner.settings.serve_stale { - return false; - } - self.generation().stale.get(host).is_some_and(|entry| { - let now = Instant::now(); - now > entry.valid_until - && now.saturating_duration_since(entry.valid_until) <= self.inner.settings.max_stale - }) - } - - /// Resolve `host` and leave the answer in the shared cache, so a later request skips the - /// lookup. Any failure is swallowed: the warm-up is advisory (spec:WARM). - pub async fn prefetch(&self, host: &str) { - let _ = self.lookup(host).await; - } - - /// The DNS servers the agent resolves through, in query order (spec:OBS#resolvers). Empty - /// until the resolver has been used, because it reads its configuration on first use. - pub fn resolvers(&self) -> Vec { - self.generation() - .built - .get() - .map(|built| built.reports.clone()) - .unwrap_or_default() - } - - /// Drop everything read off the network, so the next lookup rebuilds against the network the - /// agent is on now. - /// - /// Flushing cached answers alone would leave the agent resolving them again through the - /// previous network's servers: the discovered server list, the suffixes treated as local, and - /// the results of encryption probes are all readings of a network too, and the whole point of - /// the signal is that the network has changed. Dropping the generation takes the caches with - /// it, since they belong to the resolvers being dropped. - /// - /// The caller's options are untouched, so a listed `dns.servers` set is rebuilt exactly as - /// configured; what it re-reads is what the system supplies and what the network answers - /// (spec:NETCHG#what-the-signal-keeps). - /// - /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, - /// which keeps it callable from `networkChanged`, which is not async. Nothing is rebuilt here - /// either, so an agent that never resolves again pays nothing for the signal. - /// - /// spec:NETCHG#reach-across-the-subsystems - pub fn reset(&self) { - *self - .inner - .generation - .lock() - .expect("the DNS generation lock is only held to clone or replace an Arc") = - Arc::new(Generation::default()); - } -} - -impl Resolve for FaithResolver { - fn resolve(&self, name: ReqName) -> Resolving { - let this = self.clone(); - Box::pin(async move { - let addrs = this.lookup(name.as_str()).await?; - // Port `0` is a placeholder reqwest fills from the URL. The returned `Addrs` has to be - // `'static`, so collect owned rather than borrowing the lookup. - let addrs: Vec = - addrs.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); - Ok(Box::new(addrs.into_iter()) as Addrs) - }) - } -} - -/// Whether a failed lookup was the resolver answering that the name holds nothing, rather than the -/// resolver failing to answer. -/// -/// The distinction decides what happens to a stale entry: an authoritative "nothing here" retires -/// it, while a failure to reach an answer leaves it in place. Hickory draws the same line, producing -/// `NoRecordsFound` only for `NXDOMAIN` and for `NOERROR` with no answer records, and reporting -/// `SERVFAIL` and the other failure codes as `ResponseCode` instead. -fn is_authoritatively_empty(err: &NetError) -> bool { - matches!(err, NetError::Dns(DnsError::NoRecordsFound(_))) -} - -/// Apply the options common to every resolver Faith builds: race both families for Happy Eyeballs, -/// hold the caller's order fixed rather than reordering by latency, and layer any `dns.*` timeout, -/// ndots, and hosts-file settings on top. -fn apply_options( - builder: &mut hickory_resolver::ResolverBuilder, - settings: &ResolverSettings, -) { - let options = builder.options_mut(); - options.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - // The list expresses the caller's intent, not a performance hint, so a private resolver named - // first must not lose traffic to a closer fallback (spec:DNS#server-order). - options.server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; - if let Some(timeout) = settings.timeout { - options.timeout = timeout; - } - if let Some(ndots) = settings.ndots { - options.ndots = ndots; - } - if let Some(hosts_file) = settings.hosts_file { - options.use_hosts_file = if hosts_file { - ResolveHosts::Always - } else { - ResolveHosts::Never - }; - } -} - -/// Build the configured (or discovered) resolver and the report of its servers. -async fn build(settings: &ResolverSettings) -> Result { - if settings.servers.is_empty() { - build_discovery(settings) - } else { - build_listed(settings).await - } -} - -/// Discovery: configure from the system, then let hickory's RFC 9539 opportunistic encryption -/// upgrade those servers to DoT/DoQ where they answer a probe. `dns.searchDomains` overrides the -/// system search list when set (spec:DNS#discovery). -fn build_discovery(settings: &ResolverSettings) -> Result { - let (mut config, options) = read_system_conf().unwrap_or_else(|_| { - // A host with no readable resolver configuration falls back to Google Public DNS over - // conventional DNS, probed like any other server (spec:DNS#discovery). - ( - ResolverConfig::udp_and_tcp(&GOOGLE), - hickory_resolver::config::ResolverOpts::default(), - ) - }); - - if let Some(search) = &settings.search_domains { - config = ResolverConfig::from_parts(None, search.clone(), config.name_servers().to_vec()); - } - - let reports = report(config.name_servers(), ResolverSource::Conventional); - - let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()) - .with_options(options); - apply_options(&mut builder, settings); - let builder = builder.with_opportunistic_encryption(OpportunisticEncryption::Enabled { - config: Default::default(), - }); - - Ok(Built { - resolver: builder.build()?, - reports, - }) -} - -/// The listed-servers path: bootstrap any hostname hosts to addresses, then build the resolver -/// from the parsed specs in order (spec:DNS#transports, spec:DNS#bootstrapping). -async fn build_listed(settings: &ResolverSettings) -> Result { - let name_servers = build_name_servers(settings).await?; - - let search = settings.search_domains.clone().unwrap_or_default(); - let config = ResolverConfig::from_parts(None, search, name_servers.clone()); - let reports = report(&name_servers, ResolverSource::Configured); - - let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()); - apply_options(&mut builder, settings); - - Ok(Built { - resolver: builder.build()?, - reports, - }) -} - -/// Resolve the listed servers to hickory name servers, bootstrapping hostname hosts. A hostname -/// that will not resolve drops that server for the life of the agent rather than failing the -/// resolver (spec:DNS#bootstrapping). -async fn build_name_servers( - settings: &ResolverSettings, -) -> Result, NetError> { - let needs_bootstrap = settings.servers.iter().any(|spec| spec.ip().is_none()); - let bootstrap = if needs_bootstrap { - Some(bootstrap_resolver(settings)?) - } else { - None - }; - - let mut name_servers = Vec::with_capacity(settings.servers.len()); - for spec in &settings.servers { - let ip = match spec.ip() { - Some(ip) => ip, - None => { - let ServerHost::Name(host) = &spec.host else { - unreachable!("ip() is None only for a hostname host"); - }; - let resolver = bootstrap - .as_ref() - .expect("bootstrap resolver built when a hostname host is present"); - match resolver.lookup_ip(host.as_str()).await { - Ok(lookup) => match lookup.iter().next() { - Some(ip) => ip, - None => continue, - }, - // The hostname does not resolve: drop this server for the agent's life. - Err(_) => continue, - } - } - }; - name_servers.push(spec.to_name_server(ip)); - } - - Ok(name_servers) -} - -/// The resolver that bootstraps hostname servers: the listed IP-host servers in order, so an -/// encrypted server placed first resolves its siblings without exposing the hostname in plaintext. -/// Where the list names no IP host, the system's own configuration bootstraps instead. -fn bootstrap_resolver(settings: &ResolverSettings) -> Result { - let ip_servers: Vec = settings - .servers - .iter() - .filter_map(|spec| spec.ip().map(|ip| spec.to_name_server(ip))) - .collect(); - - let mut builder = if ip_servers.is_empty() { - TokioResolver::builder_tokio().unwrap_or_else(|_| { - TokioResolver::builder_with_config( - ResolverConfig::udp_and_tcp(&GOOGLE), - TokioRuntimeProvider::default(), - ) - }) - } else { - TokioResolver::builder_with_config( - ResolverConfig::from_parts(None, vec![], ip_servers), - TokioRuntimeProvider::default(), - ) - }; - builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - builder.options_mut().server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; - builder.build() -} - -/// The suffixes handed to the system resolver rather than Faith's servers: `localhost` and `local` -/// always, plus the ones the system supplies and the caller's `dns.exemptDomains` -/// (spec:DNS#exempt-names). -/// -/// The root name is never a suffix here, whichever list it arrives in. It is the parent of every -/// name, so admitting it would exempt the lot and route every lookup to the system resolver with -/// `dns.servers` configured and unused. It does arrive in practice: a Windows host with no DNS -/// domain of its own reports the root as its domain, so the check is what keeps the encrypted -/// transports working there rather than being quietly bypassed. -fn exempt_suffixes(system: Vec, configured: &[Name]) -> Vec { - let mut names = vec![ - Name::from_ascii("localhost").unwrap(), - Name::from_ascii("local").unwrap(), - ]; - names.extend( - system - .into_iter() - .chain(configured.iter().cloned()) - .filter(|name| !name.is_root()), - ); - names -} - -/// Summarise name servers for `resolvers()`, in the order they are queried. -fn report(name_servers: &[NameServerConfig], source: ResolverSource) -> Vec { - let mut reports = Vec::new(); - for server in name_servers { - for connection in &server.connections { - let transport = match connection.protocol { - ProtocolConfig::Udp => Transport::Udp, - ProtocolConfig::Tcp => Transport::Tcp, - ProtocolConfig::Tls { .. } => Transport::Tls, - ProtocolConfig::Https { .. } => Transport::Https, - ProtocolConfig::Quic { .. } => Transport::Quic, - ProtocolConfig::H3 { .. } => Transport::H3, - }; - reports.push(ResolverReport { - address: SocketAddr::new(server.ip, connection.port).to_string(), - transport: transport.label().to_owned(), - source: source.label().to_owned(), - }); - } - } - reports -} - -#[cfg(test)] -mod tests { - use super::*; - - fn spec(input: &str) -> ServerSpec { - ServerSpec::parse(input).expect("valid server URL") - } - - #[test] - fn scheme_selects_transport_and_conventional_port() { - // spec:DNS#transports - assert_eq!(spec("udp://1.1.1.1").transport, Transport::Udp); - assert_eq!(spec("udp://1.1.1.1").port, 53); - assert_eq!(spec("tcp://1.1.1.1").port, 53); - assert_eq!(spec("tls://1.1.1.1").transport, Transport::Tls); - assert_eq!(spec("tls://1.1.1.1").port, 853); - assert_eq!(spec("quic://1.1.1.1").port, 853); - assert_eq!(spec("https://1.1.1.1").transport, Transport::Https); - assert_eq!(spec("https://1.1.1.1").port, 443); - assert_eq!(spec("h3://1.1.1.1").port, 443); - } - - #[test] - fn explicit_port_overrides_the_conventional_one() { - // spec:DNS#transports - assert_eq!(spec("tls://1.1.1.1:8853").port, 8853); - } - - #[test] - fn http_transports_default_the_query_path() { - // spec:DNS#transports — `/dns-query` when the URL supplies none. - assert_eq!(spec("https://dns.google").path, None); - assert_eq!( - spec("https://dns.google") - .to_name_server(IpAddr::from([8, 8, 8, 8])) - .connections[0] - .protocol, - ProtocolConfig::Https { - server_name: Arc::from("dns.google"), - path: Arc::from(DEFAULT_DNS_QUERY_PATH), - } - ); - assert_eq!( - spec("https://dns.google/resolve").path, - Some(Arc::from("/resolve")) - ); - } - - #[test] - fn a_fragment_names_the_certificate() { - // spec:DNS#transports — `tls://1.1.1.1#cloudflare-dns.com`. - let spec = spec("tls://1.1.1.1#cloudflare-dns.com"); - assert_eq!(spec.cert_name.as_deref(), Some("cloudflare-dns.com")); - assert_eq!( - &*spec.server_name(IpAddr::from([1, 1, 1, 1])), - "cloudflare-dns.com" - ); - } - - #[test] - fn a_bare_ip_authenticates_against_the_address() { - // spec:DNS#transports — `tls://1.1.1.1` with no fragment. - let spec = spec("tls://1.1.1.1"); - assert_eq!(spec.cert_name, None); - assert_eq!(&*spec.server_name(IpAddr::from([1, 1, 1, 1])), "1.1.1.1"); - } - - #[test] - fn a_hostname_authenticates_against_itself() { - // spec:DNS#transports - let spec = spec("tls://dns.google"); - assert_eq!(&*spec.server_name(IpAddr::from([8, 8, 8, 8])), "dns.google"); - } - - #[test] - fn an_unknown_scheme_is_rejected() { - // spec:DNS#transports — throws an address-parse error at construction. - assert!(ServerSpec::parse("ftp://1.1.1.1").is_err()); - assert!(ServerSpec::parse("not a url").is_err()); - } - - #[tokio::test] - async fn reset_replaces_the_generation_and_what_it_holds() { - // A network change drops what was read off the old network, so the next lookup builds - // against the new one rather than reusing the previous network's servers (spec:NETCHG). - let resolver = FaithResolver::new(ResolverSettings { - servers: vec![spec("udp://127.0.0.1:1")], - timeout: Some(Duration::from_millis(200)), - ..ResolverSettings::default() - }); - - let before = resolver.generation(); - // Build the generation's state, so there is something for the reset to drop. - let _ = resolver.built(&before).await; - assert!( - before.built.get().is_some(), - "the generation built its resolver" - ); - assert_eq!(resolver.resolvers().len(), 1, "which `resolvers()` reports"); - - resolver.reset(); - - let after = resolver.generation(); - assert!( - !Arc::ptr_eq(&before, &after), - "the reset swaps the generation rather than mutating it" - ); - assert!( - after.built.get().is_none(), - "the new generation holds nothing until it is used again" - ); - assert!( - before.built.get().is_some(), - "work already holding the old generation keeps its resolvers" - ); - assert!( - resolver.resolvers().is_empty(), - "`resolvers()` reports nothing until the rebuild (spec:OBS#resolvers)" - ); - - // Configuration survives the signal, so the rebuild uses the servers as configured. - let _ = resolver.built(&after).await; - assert_eq!( - resolver.resolvers().len(), - 1, - "the rebuilt generation resolves through the configured servers again" - ); - } - - /// A resolver with a stale entry for `host` whose freshness ended `ago`. - fn with_stale_entry(settings: ResolverSettings, host: &str, ago: Duration) -> FaithResolver { - let resolver = FaithResolver::new(settings); - resolver.generation().stale.insert( - host.to_owned(), - StaleEntry { - addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), - valid_until: Instant::now() - ago, - }, - ); - resolver - } - - #[test] - fn only_an_expired_entry_inside_the_window_is_served_stale() { - // spec:DNS#serving-stale-answers - let settings = || ResolverSettings { - max_stale: Duration::from_secs(60), - ..ResolverSettings::default() - }; - - // Still fresh: the lookup goes through hickory, which answers from its own cache. - let fresh = FaithResolver::new(settings()); - fresh.generation().stale.insert( - "fresh.test".to_owned(), - StaleEntry { - addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), - valid_until: Instant::now() + Duration::from_secs(60), - }, - ); - let generation = fresh.generation(); - assert!( - fresh.stale_addrs(&generation, "fresh.test").is_none(), - "a fresh entry is not a stale hit" - ); - - // Expired but inside `dns.maxStale`: served immediately. - let stale = with_stale_entry(settings(), "stale.test", Duration::from_secs(5)); - let generation = stale.generation(); - assert!( - stale.stale_addrs(&generation, "stale.test").is_some(), - "an entry expired inside the window is served" - ); - - // Past the window: no longer evidence about the host, so the lookup blocks. - let old = with_stale_entry(settings(), "old.test", Duration::from_secs(120)); - let generation = old.generation(); - assert!( - old.stale_addrs(&generation, "old.test").is_none(), - "an entry past `dns.maxStale` is not served" - ); - assert!( - generation.stale.get("old.test").is_none(), - "and is dropped rather than left to age further" - ); - } - - #[test] - fn serve_stale_off_never_serves_an_expired_entry() { - // spec:DNS#serving-stale-answers — the switch for a caller that must not connect to an - // address it knows to be out of date. - let resolver = with_stale_entry( - ResolverSettings { - serve_stale: false, - ..ResolverSettings::default() - }, - "strict.test", - Duration::from_secs(5), - ); - let generation = resolver.generation(); - assert!(resolver.stale_addrs(&generation, "strict.test").is_none()); - assert!( - !resolver.served_stale("strict.test"), - "and nothing is reported as stale-served, so no retry is armed" - ); - } - - #[test] - fn served_stale_tracks_the_window_it_serves_from() { - // The retry layer arms itself from this, so it must not claim an address was assumed when - // the lookup actually blocked on a fresh one (spec:DNS#when-a-stale-address-is-wrong). - let settings = || ResolverSettings { - max_stale: Duration::from_secs(60), - ..ResolverSettings::default() - }; - - let inside = with_stale_entry(settings(), "inside.test", Duration::from_secs(5)); - assert!(inside.served_stale("inside.test")); - - let outside = with_stale_entry(settings(), "outside.test", Duration::from_secs(120)); - assert!( - !outside.served_stale("outside.test"), - "an entry past the window is resolved for real, so its address is confirmed" - ); - - let absent = FaithResolver::new(settings()); - assert!(!absent.served_stale("absent.test")); - } - - #[test] - fn a_network_change_drops_stale_answers() { - // Addresses read off the old network are exactly what must not be served on the new one - // (spec:NETCHG#reach-across-the-subsystems). - let resolver = with_stale_entry( - ResolverSettings::default(), - "netchg.test", - Duration::from_secs(5), - ); - assert!(resolver.served_stale("netchg.test")); - - resolver.reset(); - - assert!( - !resolver.served_stale("netchg.test"), - "the stale answer goes with the generation that held it" - ); - } - - #[test] - fn a_root_suffix_never_exempts_everything() { - // A Windows host with no DNS domain reports the root as its domain, and the root is the - // parent of every name. Taking it as a suffix exempted every lookup and sent it to the - // system resolver, leaving `dns.servers` configured and unused (spec:DNS#exempt-names). - let suffixes = exempt_suffixes(vec![Name::root()], &[]); - assert!( - !suffixes.iter().any(|suffix| suffix.is_root()), - "the root is not admitted as a suffix" - ); - - let name = Name::from_utf8("nonexistent.example").unwrap(); - assert!( - !suffixes.iter().any(|suffix| suffix.zone_of(&name)), - "so an ordinary name is not exempt and reaches the configured servers" - ); - - // The names that must stay exempt still are, and a real system suffix still counts. - let suffixes = exempt_suffixes( - vec![Name::root(), Name::from_utf8("corp.example").unwrap()], - &[], - ); - for exempt in ["localhost", "printer.local", "host.corp.example"] { - let name = Name::from_utf8(exempt).unwrap(); - assert!( - suffixes.iter().any(|suffix| suffix.zone_of(&name)), - "{exempt} is exempt" - ); - } - } - - #[test] - fn a_root_entry_from_the_caller_is_refused_too() { - // Whichever list it arrives in, the root would disable the caller's own servers. - let suffixes = exempt_suffixes(vec![], &[Name::root()]); - let name = Name::from_utf8("nonexistent.example").unwrap(); - assert!(!suffixes.iter().any(|suffix| suffix.zone_of(&name))); - } - - mod https_records { - use hickory_resolver::proto::rr::{ - Record, - rdata::{HTTPS, SVCB, svcb::Alpn}, - }; - - use super::*; - - fn name(input: &str) -> Name { - Name::from_utf8(input).unwrap() - } - - /// One `HTTPS` answer, spelled the way a server would send it. - fn record( - owner: &str, - priority: u16, - target: Name, - params: Vec<(SvcParamKey, SvcParamValue)>, - ttl: u32, - ) -> Record { - Record::from_rdata( - name(owner), - ttl, - RData::HTTPS(HTTPS(SVCB::new(priority, target, params))), - ) - } - - fn alpn(tokens: &[&str]) -> (SvcParamKey, SvcParamValue) { - ( - SvcParamKey::Alpn, - SvcParamValue::Alpn(Alpn(tokens.iter().map(|t| (*t).to_owned()).collect())), - ) - } - - #[test] - fn an_h3_alpn_on_the_owner_name_advertises() { - // spec:H3UP#advertisements-from-dns — the ordinary case: `.` as the target means the - // owner name, so the record describes the origin itself. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h2", "h3"])], - 3600, - )]; - - assert_eq!( - read_https_answer(&queried, &answers), - Some(HttpsAdvertisement { - port: None, - ttl: Duration::from_secs(3600), - }), - "an `alpn` listing h3 is an advertisement, and the record's own TTL bounds it" - ); - } - - #[test] - fn a_draft_h3_token_counts_like_the_header_reader_treats_one() { - // spec:H3UP#reading-advertisements — `h3-29` is an h3-family token either way it - // arrives, so DNS must not be stricter than the `Alt-Svc` reader. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h3-29"])], - 60, - )]; - - assert!(read_https_answer(&queried, &answers).is_some()); - } - - #[test] - fn a_record_without_h3_in_its_alpn_advertises_nothing() { - // An origin that speaks only HTTP/2 says so here, and reading that as an h3 - // advertisement would send every such origin down a probe that must fail. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h2"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn the_port_parameter_is_carried_through() { - // spec:H3UP#advertisements-from-dns — a differing port is handled by the same - // machinery an `Alt-Svc` advertised port is. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(8443)), - ], - 3600, - )]; - - assert_eq!( - read_https_answer(&queried, &answers).and_then(|ad| ad.port), - Some(8443) - ); - } - - #[test] - fn a_record_targeting_another_host_is_not_acted_on() { - // Faith only upgrades to the origin's own host, exactly as it refuses an `Alt-Svc` - // advertisement naming a different one (spec:H3UP#advertisements-from-dns). - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - name("cdn.example.net."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn a_record_naming_the_queried_host_itself_is_acted_on() { - // Spelling the owner name out is equivalent to the `.` shorthand, and the comparison - // ignores case and the trailing root the way name equality should. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - name("ExAmPlE.CoM."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert!(read_https_answer(&queried, &answers).is_some()); - } - - #[test] - fn an_alias_mode_record_is_skipped() { - // Priority 0 is AliasMode: it redirects to another name rather than describing this - // one, and following that redirection is a resolution step this does not take. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 0, - name("svc.example.net."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn the_lowest_priority_service_mode_record_wins() { - // RFC 9460 orders ServiceMode records by ascending priority, so the most preferred - // record is the one whose port is acted on. - let queried = name("example.com."); - let answers = [ - record( - "example.com.", - 9, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(9443)), - ], - 3600, - ), - record( - "example.com.", - 2, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(2443)), - ], - 3600, - ), - ]; - - assert_eq!( - read_https_answer(&queried, &answers).and_then(|ad| ad.port), - Some(2443), - "the preferred record is the one acted on" - ); - } - - #[test] - fn an_empty_answer_advertises_nothing() { - // The common case for an origin with no `HTTPS` record at all: nothing learned, and - // nothing that could make the origin probe-worthy. - assert_eq!(read_https_answer(&name("example.com."), &[]), None); - } - } - - #[test] - fn exempt_matches_a_suffix_exactly_or_as_a_subdomain() { - // spec:DNS#exempt-names - let local = Name::from_ascii("local").unwrap(); - assert!(local.zone_of(&Name::from_utf8("printer.local").unwrap())); - assert!(local.zone_of(&Name::from_utf8("local").unwrap())); - assert!(!local.zone_of(&Name::from_utf8("mylocal.example").unwrap())); - } -} diff --git a/src/error.rs b/src/error.rs deleted file mode 100644 index d385d3b..0000000 --- a/src/error.rs +++ /dev/null @@ -1,290 +0,0 @@ -use std::{ - error::Error, - fmt::{Debug, Display}, -}; - -use napi::bindgen_prelude::*; -use napi_derive::napi; -use strum::{EnumIter, IntoEnumIterator}; - -/// Faith produces fine-grained errors, but maps them to a few javascript error types for fetch -/// compatibility. The `.code` property on errors thrown from Faith is set to a stable name for each -/// error kind, documented in this comprehensive mapping: -/// -/// - JS `AbortError`: -/// - `Aborted` — request was aborted using `signal` -/// - `Timeout` — request timed out -/// - JS `NetworkError`: -/// - `Network` — network error -/// - `Redirect` — when the agent is configured to error on redirects -/// - `ContentLengthOverrun` — a body written with `response.toFile()` exceeded the advertised `Content-Length` -/// - JS `SyntaxError`: -/// - `AddressParse` — IP parse error for `AgentOptions.dns.overrides` -/// - `InvalidIntegrity` — SRI parse error for `RequestInit.integrity` -/// - `JsonParse` — JSON parse error for `response.json()` -/// - `PemParse` — PEM parse error for `AgentOptions.tls.identity` or `AgentOptions.tls.extraRoots` -/// - JS `TypeError`: -/// - `Closed` — a request was made on an agent that has been closed -/// - `InvalidCompression` — `RequestInit.compress` naming no coding Faith can compress in -/// - `InvalidHeader` — invalid header name or value -/// - `InvalidMethod` — invalid HTTP method -/// - `InvalidPath` — a `response.toFile()` destination that does not name a local path -/// - `InvalidUrl` — invalid URL string -/// - `ResponseAlreadyDisturbed` — body already read (mutually exclusive operations) -/// - `ResponseBodyNull` — `response.toFile()` on a response that cannot carry a body -/// - JS generic `Error`: -/// - `BodyStream` — internal stream handling error -/// - `Config` — invalid agent configuration -/// - `FileExists` — a `response.toFile()` write refusing an occupied destination -/// - `FileWrite` — the filesystem refusing a `response.toFile()` write -/// - `IntegrityMismatch` — SRI checksum mismatch (with `RequestInit.integrity`) -/// -/// The library exports an `ERROR_CODES` object which has every error code the library throws, and -/// every error thrown also has a `code` property that is set to one of those codes. So you can -/// accurately respond to the exact error kind by checking its code and matching against the right -/// constant from `ERROR_CODES`, instead of doing string matching on the error message, or coarse -/// `instance of` matching. -/// -/// Due to technical limitations, when reading a body stream, reads might fail, but that error -/// will not have a `code` property. -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, EnumIter)] -pub enum FaithErrorKind { - Aborted, - AddressParse, - BodyStream, - Closed, - Config, - ContentLengthOverrun, - FileExists, - FileWrite, - IntegrityMismatch, - InvalidCompression, - InvalidHeader, - InvalidIntegrity, - InvalidMethod, - InvalidPath, - InvalidUrl, - JsonParse, - Network, - PemParse, - Redirect, - ResponseAlreadyDisturbed, - ResponseBodyNull, - Timeout, -} - -#[derive(Debug, Clone, Copy)] -enum JsErrorType { - GenericError, - NamedError(&'static str), - SyntaxError, - TypeError, -} - -impl FaithErrorKind { - fn default_message(self) -> &'static str { - match self { - Self::Aborted => "the request was aborted", - Self::AddressParse => "invalid IP address and/or port", - Self::BodyStream => "internal response body stream copy error", - Self::Closed => "the agent has been closed", - Self::Config => "invalid agent configuration", - Self::ContentLengthOverrun => "response body exceeded the advertised Content-Length", - Self::FileExists => "the destination file already exists", - Self::FileWrite => "could not write the destination file", - Self::IntegrityMismatch => "resource integrity check failed", - Self::InvalidCompression => "invalid request body compression", - Self::InvalidHeader => "invalid header name or value", - Self::InvalidIntegrity => "invalid integrity value", - Self::InvalidMethod => "invalid HTTP method", - Self::InvalidPath => "destination does not name a local path", - Self::InvalidUrl => "invalid URL", - Self::JsonParse => "invalid json in response body", - Self::Network => "network error", - Self::PemParse => "invalid client certificate or key", - Self::Redirect => "got a redirect", - Self::ResponseAlreadyDisturbed => "response body already disturbed", - Self::ResponseBodyNull => "response cannot carry a body to write", - Self::Timeout => "timed out", - } - } - - fn js_type(self) -> JsErrorType { - match self { - Self::BodyStream - | Self::Config - | Self::FileExists - | Self::FileWrite - | Self::IntegrityMismatch => JsErrorType::GenericError, - Self::Aborted | Self::Timeout => JsErrorType::NamedError("AbortError"), - Self::Network | Self::Redirect | Self::ContentLengthOverrun => { - JsErrorType::NamedError("NetworkError") - } - Self::AddressParse | Self::InvalidIntegrity | Self::JsonParse | Self::PemParse => { - JsErrorType::SyntaxError - } - Self::Closed - | Self::InvalidCompression - | Self::InvalidHeader - | Self::InvalidMethod - | Self::InvalidPath - | Self::InvalidUrl - | Self::ResponseAlreadyDisturbed - | Self::ResponseBodyNull => JsErrorType::TypeError, - } - } -} - -impl From for FaithError { - fn from(kind: FaithErrorKind) -> Self { - Self { - kind, - message: None, - } - } -} - -#[derive(Debug, Clone)] -pub struct FaithError { - pub kind: FaithErrorKind, - pub message: Option, -} - -impl FaithError { - pub fn new(kind: FaithErrorKind, message: Option>) -> Self { - Self { - kind, - message: message.map(|m| m.into()), - } - } - - // we make this explicit instead of adding a From<> so that we can't accidentally do it - pub fn into_napi(self) -> napi::Error { - self.to_napi() - } - fn to_napi(&self) -> napi::Error { - napi::Error::new(napi::Status::GenericFailure, format!("{self}")) - } - - // whenever possible, we should prefer to use this so that the error types are correct - pub fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env> { - let code = format!("{:?}", self.kind); - let unk = match self.kind.js_type() { - JsErrorType::TypeError => JsTypeError::from(self.into_napi()).into_unknown(*env), - JsErrorType::SyntaxError => JsSyntaxError::from(self.into_napi()).into_unknown(*env), - JsErrorType::GenericError => JsError::from(self.into_napi()).into_unknown(*env), - JsErrorType::NamedError(name) => env - .create_error(self.to_napi()) - .and_then(|mut err| { - err.set_named_property("name", name)?; - Ok(err) - }) - .and_then(|err| err.into_unknown(env)) - .unwrap_or_else(|_| JsError::from(self.into_napi()).into_unknown(*env)), - }; - - // we do this manually instead of using the TryFrom so we can return the untouched Unknown if we fail - let Ok(typ) = unk.get_type() else { return unk }; - if typ != ValueType::Object { - return unk; - } - // SAFETY: we have verified that this value is an Object - let Ok(mut obj) = (unsafe { unk.cast::() }) else { - return unk; - }; - - let _ = obj.set("code", code); - obj.into_unknown(env).unwrap_or(unk) - } -} - -/// Dig a [`FaithError`] back out of an error chain, if one is in there. -/// -/// The `error` redirect policy refuses a redirect by handing reqwest a [`FaithError`], which comes -/// back to us wrapped in an error of reqwest's own, so the kind we chose has to be recovered from -/// the source chain to survive as a `code`. Redirect failures reqwest raises on its own account -/// (exhausting the hop limit, an https-only downgrade) carry no [`FaithError`] and so fall through -/// to the generic mapping, which is what tells the two apart. -fn faith_kind_in_chain(err: &(dyn Error + 'static)) -> Option { - let mut source = err.source(); - while let Some(e) = source { - if let Some(faith) = e.downcast_ref::() { - return Some(faith.kind); - } - source = e.source(); - } - - None -} - -impl From for FaithError { - fn from(err: reqwest::Error) -> Self { - // Always include full error chain for debugging - let mut msg = format!("{err:?}"); - let mut source = err.source(); - while let Some(e) = source { - msg.push_str(&format!(" -> {e:?}")); - source = e.source(); - } - - if err.is_timeout() { - return FaithError::new(FaithErrorKind::Timeout, Some(msg)); - } - - // A redirect the agent's own policy refused carries the kind we handed reqwest; one reqwest - // raised on its own account stays a plain network error. - let kind = err - .is_redirect() - .then(|| faith_kind_in_chain(&err)) - .flatten() - .unwrap_or(FaithErrorKind::Network); - - FaithError::new(kind, Some(msg)) - } -} - -impl From for FaithError { - fn from(err: reqwest_middleware::Error) -> Self { - match err { - reqwest_middleware::Error::Middleware(err) => { - FaithError::new(FaithErrorKind::Network, Some(err.to_string())) - } - reqwest_middleware::Error::Reqwest(err) => err.into(), - } - } -} - -impl Error for FaithError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } - - fn description(&self) -> &str { - "description() is deprecated; use Display" - } - - fn cause(&self) -> Option<&dyn Error> { - self.source() - } -} - -impl Display for FaithError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{:?}: {}", - self.kind, - self.message - .as_deref() - .unwrap_or_else(|| self.kind.default_message()) - ) - } -} - -#[napi] -pub fn error_codes() -> Vec { - FaithErrorKind::iter() - .map(|kind| format!("{:?}", kind)) - .collect() -} diff --git a/src/fetch.rs b/src/fetch.rs deleted file mode 100644 index 71791ae..0000000 --- a/src/fetch.rs +++ /dev/null @@ -1,444 +0,0 @@ -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Instant, -}; - -use http_cache_reqwest::CacheMode; -use hyper_util::client::legacy::connect::HttpInfo; -use napi::{ - Env, - bindgen_prelude::{AbortSignal, PromiseRaw}, -}; -use napi_derive::napi; -use reqwest::{Method, StatusCode}; -use reqwest::{ - header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, - tls::TlsInfo, -}; -use tokio::sync::{Mutex, mpsc}; - -use crate::{ - async_task::faith_promise, - body::{Body, BodyHolder}, - encoding::{self, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}, - error::{FaithError, FaithErrorKind}, - options::{CredentialsOption, FaithOptions, FaithOptionsAndBody, PRIORITY}, - response::{FaithResponse, PeerInformation}, - stream_body::StreamBody, - timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, -}; - -/// The methods the fetch standard normalises to upper case; any other method is sent as given. -const NORMALISED_METHODS: [&str; 6] = ["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]; - -#[napi] -pub fn faith_fetch<'env>( - env: &'env Env, - url: String, - options: FaithOptionsAndBody, - signal: Option, - stream_body: Option<&StreamBody>, -) -> Result, napi::Error> { - let (options, agent, body) = FaithOptions::extract(options); - let (s, abort) = mpsc::channel(8); - let has_signal = signal.is_some(); - if let Some(signal) = signal { - signal.on_abort(move || { - let _ = s.try_send(()); - }); - } - - // Get the stream body receiver if provided - let stream_receiver = stream_body.map(|sb| sb.receiver.clone()); - - faith_promise(env, async move { - let mut abort = abort; - let method = options.method.as_deref().unwrap_or("GET"); - // spec:REQ#method-and-headers - let method = NORMALISED_METHODS - .into_iter() - .find(|normalised| normalised.eq_ignore_ascii_case(method)) - .unwrap_or(method); - - let method = - Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; - let is_head = method == Method::HEAD; - - let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; - - // A `compress` naming no coding Faith can compress in is misuse whether or not the - // request turns out to carry a body, so it is refused before anything else looks at - // it (spec:ENC#compressing-a-request-body). - let compress = options - .compress - .as_deref() - .map(|value| { - Coding::from_option(value).ok_or_else(|| { - FaithError::new( - FaithErrorKind::InvalidCompression, - Some(format!( - "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" - )), - ) - }) - }) - .transpose()?; - - // Handle credentials based on credentials option - if options.credentials == CredentialsOption::Omit { - // Remove credentials from URL if omit is specified - let _ = parsed_url.set_username(""); - let _ = parsed_url.set_password(None); - } - - // The stamp rides along in the request's extensions for the middleware to fill in; - // this side keeps a handle on it so the one measurement taken inside the stack is - // the one surfaced (spec:RESP#request-timing). - let headers_stamp = HeadersStamp::default(); - - let mut request = agent - .client - .as_ref() - .ok_or(FaithErrorKind::Closed)? - .request(method, parsed_url.clone()) - .with_extension(CacheMode::from(options.cache)) - .with_extension(headers_stamp.clone()); - - if let Some(headers) = &options.headers { - for (key, value) in headers { - // Skip Cookie header if credentials is omit - if options.credentials == CredentialsOption::Omit - && key.eq_ignore_ascii_case("cookie") - { - continue; - } - - // Validate header name and value before adding to request - let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header name: {key}")), - ) - })?; - let header_value = HeaderValue::from_str(value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - - // Faith's coding is layered on top of what the caller declares, and reqwest's - // builder appends rather than replaces, so passing the caller's value through - // here would put a second `Content-Encoding` beside the joined one -- the same - // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is - // still validated above, then withheld and re-emitted once below. - if compress.is_some() && header_name == CONTENT_ENCODING { - continue; - } - - request = request.header(header_name, header_value); - } - } - - // What the caller says they handed over: their own `Content-Encoding`, else the - // agent's, per-request headers winning per name as they do generally (spec: REQ). - // Several lines are the one list, so they are joined as they are read. - let declared_content_encoding = compress.and_then(|_| { - let from_request = options.headers.as_ref().and_then(|headers| { - let declared = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(); - (!declared.is_empty()).then(|| declared.join(", ")) - }); - from_request.or_else(|| { - agent - .default_content_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - }); - - // The request's `Accept-Encoding` governs which codings Faith decodes on the way - // back (spec: ENC): a value on the request, else one inherited from the agent's - // default headers, else the default Faith sends itself. Neither the request nor the - // agent advertising a value means nothing beneath Faith adds one now that it owns - // the codings, so Faith sends the default explicitly. - let request_accept_encoding = options.headers.as_ref().and_then(|headers| { - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) - .map(|(_, value)| value.clone()) - }); - let accept_encoding = AcceptEncoding::parse( - &request_accept_encoding - .clone() - .or_else(|| { - agent - .default_accept_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), - ); - if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { - request = request.header( - ACCEPT_ENCODING, - HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), - ); - } - - // The `priority` option is a hint, so a `Priority` header the caller wrote, or one - // among the agent's default headers, wins over the value derived from it - // (spec: REQ#request-priority). The agent's defaults are consulted here rather than - // left to reqwest: it fills a default header in only where the request carries none - // of that name, so setting the derived value would displace the agent's own. - if let Some(urgency) = options.priority - && !agent.has_default_priority - && !options.headers.as_ref().is_some_and(|headers| { - headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) - }) { - request = request.header( - HeaderName::from_static(PRIORITY), - HeaderValue::from_static(urgency), - ); - } - - // The coding actually applied, which is `compress` only where there was a body to - // apply it to: the option does nothing on a request carrying none, so no - // `Content-Encoding` describes bytes that were never sent - // (spec:ENC#compressing-a-request-body). - let mut applied_coding = None; - - // Handle body: prefer streaming body over buffered body - if let Some(receiver_arc) = stream_receiver { - // Take the receiver from the Arc>> before anything below can bail - // out. Whatever happens next, this owns the receiving end: dropping it closes the - // channel, which is what tells the JS side pumping chunks in to stop. Leaving it in - // place on a refusal would strand that pump on a channel nobody will ever read, - // blocking once it filled and holding the process open. - let receiver = { - let mut guard = receiver_arc.lock().await; - guard.take() - }; - - // A body read from a `ReadableStream` has no length to advertise, which the fetch - // standard allows only over HTTP/2 and HTTP/3 (spec:REQ#streaming-a-request-body). - if !agent.quirk_h1_request_streaming { - // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and - // can be refused without opening a connection to find out. - if parsed_url.scheme() != "https" { - return Err(FaithError::new( - FaithErrorKind::Network, - Some(format!( - "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", - parsed_url.as_str() - )), - )); - } - - // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the - // request hands the check to the layer that finds out: the connection is chosen, - // and an HTTP/1.x one is refused there before any of the body is written. - request = request.version(http::Version::HTTP_2); - } - - if let Some(receiver) = receiver { - // Convert the receiver into a stream for reqwest - let byte_stream = receiver.into_stream(); - request = request.body(match compress { - // Compressed as the chunks arrive, and chunked on the wire either way: - // a stream has no length to declare up front, compressed or not - // (spec:ENC#what-a-compressed-request-sends). - Some(coding) => { - applied_coding = Some(coding); - reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) - } - None => reqwest::Body::wrap_stream(byte_stream), - }); - } - } else if let Some(body) = &body { - request = request.body(match compress { - // The compressed bytes are what reqwest sizes `Content-Length` from, so the - // header counts what goes on the wire (spec:ENC#what-a-compressed-request-sends). - Some(coding) => { - applied_coding = Some(coding); - encoding::compress_buffer(body, coding) - .await - .map_err(|err| { - FaithError::new( - FaithErrorKind::Network, - Some(format!("could not compress the request body: {err}")), - ) - })? - } - None => body.to_vec(), - }); - } - - // One `Content-Encoding` naming the caller's codings then Faith's, in the order they - // were applied (spec:ENC#what-a-compressed-request-sends). - if let Some(coding) = applied_coding { - let value = - encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); - let value = HeaderValue::from_str(&value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - request = request.header(CONTENT_ENCODING, value); - } - - if let Some(dur) = options.timeout { - request = request.timeout(dur); - } - - agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); - - // The origin every phase is measured from. - let started = Instant::now(); - - // Race the request with the abort signal if signal was provided - let response = if has_signal { - tokio::select! { - result = request.send() => result?, - _ = abort.recv() => { - return Err(FaithErrorKind::Aborted.into()); - } - } - } else { - request.send().await? - }; - - agent - .stats - .responses_received - .fetch_add(1, Ordering::Relaxed); - - let status_code = response.status(); - let empty = status_code == StatusCode::NO_CONTENT || is_head; - - let response_url = response.url().clone(); - let version = response.version(); - - // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the - // request's port to the advertised one, so the response URL's port reflects - // which endpoint answered rather than any redirect. Compare with ports - // normalised away, or every such request would report `redirected`. - // - // Only HTTP/3 responses can have been rewritten — reqwest routes - // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and - // the TCP fallback re-runs the untouched clone. Restricting the normalisation - // to those keeps exact comparison, and so port-only redirect detection, for - // every other response. - let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { - let without_port = |url: &reqwest::Url| { - let mut url = url.clone(); - let _ = url.set_port(None); - url - }; - without_port(&parsed_url) != without_port(&response_url) - } else { - parsed_url != response_url - }; - - // Track connection for TCP stats (if we can get both local and remote addr). - // A connection the tracker has already seen is one the pool handed back, which is - // what `reused` reports (spec:RESP#request-timing). - let reused = if let Some(http_info) = response.extensions().get::() { - let local_addr = http_info.local_addr(); - let remote_addr = http_info.remote_addr(); - agent.conn_tracker.track(local_addr, remote_addr) - } else { - false - }; - - // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has - // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a - // redirect chain marks the origin that actually answered rather than the one asked for. - agent.mark_warm(&response_url); - - let peer = PeerInformation { - address: response.remote_addr(), - certificate: response - .extensions() - .get::() - .and_then(|info| info.peer_certificate()) - .map(|cert| cert.into()), - }; - - let mut headers = response.headers().clone(); - if options.credentials == CredentialsOption::Omit { - headers.remove("set-cookie"); - } - - // A cache hit is served without ever reaching the layer that stamps, so fall back to - // the moment the send resolved, which for a hit is the moment the cache answered. - let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); - let timing = RequestTiming { - headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, - body_ms: None, - reused, - next_hop_protocol: alpn_protocol_id(version, &response_url), - // Captured before a decoded body's `Content-Encoding` is stripped below, so the - // coding the response arrived under is reported either way. - content_encoding: headers - .get(CONTENT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned), - from_cache: headers - .get("x-cache") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), - }; - - // Decode only a body Faith negotiated the coding for; a bodyless response keeps its - // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). - let decode = if empty { - None - } else { - encoding::decision(&headers, &accept_encoding) - }; - if decode.is_some() { - encoding::strip_decoded_headers(&mut headers); - } - - let timing = Arc::new(TimingSlot::new(started, timing)); - // A response that cannot carry a body has nothing left to wait for. - if empty { - timing.ended(); - } - - Ok(FaithResponse { - body: if empty { - BodyHolder::none() - } else { - let http_response: http::Response<_> = response.into(); - BodyHolder::new( - Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), - version, - timing.clone(), - ) - }, - decode, - disturbed: Arc::new(AtomicBool::new(false)), - headers, - integrity: options.integrity, - peer: Arc::new(peer), - redirected, - stats: agent.stats.clone(), - status_code, - timing, - trailers: Default::default(), - url: response_url, - version, - }) - }) -} diff --git a/src/response.rs b/src/response.rs deleted file mode 100644 index b19c898..0000000 --- a/src/response.rs +++ /dev/null @@ -1,962 +0,0 @@ -use std::{ - fmt::Debug, - hint::unreachable_unchecked, - mem::replace, - net::SocketAddr, - pin::Pin, - result::Result, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::{Duration, Instant}, -}; - -use bytes::Bytes; -use futures::{StreamExt, TryStreamExt, stream}; -use http_body_util::BodyStream; -use napi::{ - bindgen_prelude::*, - threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, -}; -use napi_derive::napi; -use reqwest::{ - StatusCode, Url, Version, - header::{CONTENT_LENGTH, HeaderMap}, -}; -use serde_json; -use stream_shared::SharedStream; -use tokio::{io::AsyncWriteExt, sync::watch}; - -use crate::{ - agent::InnerAgentStats, - async_task::{Value, faith_promise}, - body::{Body, BodyHolder, DynStream, drain_body_inner}, - encoding::{Coding, decode_stream}, - error::{FaithError, FaithErrorKind}, - integrity::{finish_integrity, integrity_checker, verify_integrity}, - timing::{TimingBreakdown, TimingSlot}, -}; - -/// The `Response` interface of the Fetch API represents the response to a request. -/// -/// Faith does not allow its `Response` object to be constructed. If you need to, you may use the -/// `webResponse()` method to convert one into a Web API `Response` object; note the caveats. -#[napi] -#[derive(Debug, Clone)] -pub struct FaithResponse { - pub(crate) body: BodyHolder, - /// The coding to decode the body under, or `None` to deliver it as received. - /// Set once when the response is built, from the request's `Accept-Encoding` and the - /// response's `Content-Encoding` (see [`crate::encoding`]). - pub(crate) decode: Option, - pub(crate) disturbed: Arc, - pub(crate) headers: HeaderMap, - pub(crate) integrity: Option, - pub(crate) peer: Arc, - pub(crate) redirected: bool, - pub(crate) stats: Arc, - pub(crate) status_code: StatusCode, - pub(crate) timing: Arc, - pub(crate) trailers: Arc, - pub(crate) url: Url, - pub(crate) version: Version, -} - -/// Custom to Faith. -/// -/// The `peer` read-only property of the `Response` interface contains an object with information about -/// the remote peer that sent this response: -/// -/// - `address`: The IP address and port of the peer, if available. -/// - `certificate`: When connected over HTTPS, this is the DER-encoded leaf certificate of the peer. -#[derive(Debug)] -pub struct PeerInformation { - pub address: Option, - pub certificate: Option>, -} - -/// Options for `toFile()`. -#[napi(object)] -#[derive(Debug, Default)] -pub struct ToFileOptions { - /// Whether to truncate and replace an occupied destination. Defaults to false, which - /// refuses an occupied destination with a `FileExists` error and leaves it untouched. - pub overwrite: Option, - /// The permissions a newly created file is given, defaulting to what Node's own - /// filesystem writes use. Ignored on platforms without Unix file modes. - pub mode: Option, -} - -/// What `toFile()` resolves to. -#[napi(object)] -#[derive(Debug)] -pub struct ToFileResult { - /// The absolute filesystem path written to. - pub path: String, - /// The number of bytes that landed at the destination. - pub bytes_written: i64, -} - -/// A progress report from a `toFile()` write in flight. -#[napi(object)] -#[derive(Debug)] -pub struct ToFileProgress { - /// The number of bytes written to the file so far. - pub bytes_written: i64, - /// What the response advertised in `Content-Length`, when it sent one and Faith is - /// not decoding the body. Absent when the total is not known ahead of time, which is - /// the case for a chunked response and for one Faith decodes. - pub content_length: Option, -} - -/// The callback `toFile()` reports progress to. -/// -/// `CalleeHandled = false`: progress is not an error-first callback, so the JavaScript -/// side receives the report on its own rather than as the second argument. -pub type ProgressCallback = - ThreadsafeFunction, ToFileProgress, Status, false>; - -/// The shortest gap between progress reports. -/// -/// Reporting every chunk would cross into JavaScript thousands of times for a large body, -/// which is the cost `toFile()` exists to avoid. A caller driving a progress bar cannot use -/// updates faster than this anyway, and the final report is always delivered regardless. -const PROGRESS_INTERVAL: Duration = Duration::from_millis(50); - -/// Open the destination file for a body write, mapping filesystem refusals to the errors -/// `toFile()` surfaces (spec:BODY#tofile). -async fn open_destination( - path: &str, - options: &ToFileOptions, -) -> Result { - let mut open = tokio::fs::OpenOptions::new(); - open.write(true); - if options.overwrite.unwrap_or(false) { - // An occupied destination is truncated and replaced. - open.create(true).truncate(true); - } else { - // The safe default refuses an occupied destination outright. - open.create_new(true); - } - #[cfg(unix)] - if let Some(mode) = options.mode { - open.mode(mode); - } - - match open.open(path).await { - Ok(file) => Ok(file), - Err(err) => Err(classify_open_error(path, err).await), - } -} - -/// Classify a failure to open the destination. An occupied destination is `FileExists`, -/// unless what occupies it is a directory: a directory is well-formed but cannot be written -/// to, which is a `FileWrite`. Every other refusal is a `FileWrite` carrying the OS detail. -async fn classify_open_error(path: &str, err: std::io::Error) -> FaithError { - let kind = if err.kind() == std::io::ErrorKind::AlreadyExists { - match tokio::fs::symlink_metadata(path).await { - Ok(meta) if meta.is_dir() => FaithErrorKind::FileWrite, - _ => FaithErrorKind::FileExists, - } - } else { - FaithErrorKind::FileWrite - }; - FaithError::new(kind, Some(err.to_string())) -} - -#[derive(Clone, Debug, Default)] -pub enum Trailers { - #[default] - NotYet, - None, - Some(HeaderMap), -} - -/// Where the trailers land: written by whoever finishes the body, awaited by `trailers()`. -/// -/// A watch channel, rather than a lock read in a loop. Per the fetch standard's trailers -/// proposal () this promise is *meant* not to -/// resolve until the body has been consumed, so the wait is unbounded by design -- which is -/// precisely why polling was the wrong shape for it. Awaiting trailers without reading the -/// body now leaves an idle pending promise rather than a pegged core, and the future can be -/// cancelled while it waits. -#[derive(Debug)] -pub struct TrailersSlot(watch::Sender); - -impl Default for TrailersSlot { - fn default() -> Self { - Self(watch::channel(Trailers::NotYet).0) - } -} - -impl TrailersSlot { - /// Record trailers that arrived, waking whoever is waiting. - fn arrived(&self, trailers: HeaderMap) { - self.0.send_replace(Trailers::Some(trailers)); - } - - /// Record that the body ended, if no trailers frame got there first. - /// - /// `send_if_modified` so the read and the write are one step, and so waiters are woken - /// only by the call that actually settled it. - fn ended(&self) { - self.0.send_if_modified(|state| { - if matches!(state, Trailers::NotYet) { - *state = Trailers::None; - true - } else { - false - } - }); - } - - /// Wait until the body has settled the question. - async fn settled(&self) -> Trailers { - let mut rx = self.0.subscribe(); - // `wait_for` tests the current value before waiting, so trailers that already - // arrived return without yielding. Its error case is the sender being gone, which - // means the response was dropped and nothing can ever set this -- no trailers is - // the only answer left. - match rx - .wait_for(|state| !matches!(state, Trailers::NotYet)) - .await - { - Ok(state) => state.clone(), - Err(_) => Trailers::None, - } - } -} - -#[napi] -impl FaithResponse { - /// The `headers` read-only property of the `Response` interface contains the `Headers` object - /// associated with the response. - /// - /// Note that Faith does not provide a custom `Headers` class; instead the Web API `Headers` structure - /// is used directly and constructed by Faith when needed. - /// - /// This is a function as an internal implementation detail and the wrapper makes it a property. - #[napi] - pub fn headers(&self) -> Vec<(String, String)> { - self.headers - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.to_string(), v.to_string())) - }) - .collect() - } - - /// The `ok` read-only property of the `Response` interface contains a boolean stating whether the - /// response was successful (status in the range 200-299) or not. - #[napi(getter)] - pub fn ok(&self) -> bool { - self.status_code.is_success() - } - - /// Custom to Faith. - /// - /// The `peer` read-only property of the `Response` interface contains an object with information about - /// the remote peer that sent this response: - #[napi(getter, ts_return_type = "{ address?: string; certificate?: Buffer }")] - pub fn peer<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let mut obj = Object::new(env)?; - obj.set("address", self.peer.address.map(|addr| addr.to_string()))?; - obj.set( - "certificate", - self.peer - .certificate - .as_deref() - .map(|cert| Buffer::from(cert)), - )?; - Ok(obj) - } - - /// The `redirected` read-only property of the `Response` interface indicates whether or not the - /// response is the result of a request you made which was redirected. - /// - /// Note that by the time you read this property, the redirect will already have happened, and you - /// cannot prevent it by aborting the fetch at this point. - /// - /// One caveat specific to Faith: with the agent's `http3.upgradeFollowAdvertisedPort` - /// enabled, HTTP/3 responses compare URLs ignoring the port, because the port - /// was rewritten to the advertised one and would otherwise register as a - /// redirect. A genuine redirect differing only in port therefore reads as - /// `false` on those responses. - #[napi(getter)] - pub fn redirected(&self) -> bool { - self.redirected - } - - /// The `status` read-only property of the `Response` interface contains the HTTP status codes of the - /// response. For example, 200 for success, 404 if the resource could not be found. - /// - /// A value is `0` is returned for a response whose `type` is `opaque`, `opaqueredirect`, or `error`. - #[napi(getter)] - pub fn status(&self) -> u16 { - self.status_code.as_u16() - } - - /// The `statusText` read-only property of the `Response` interface contains the status message - /// corresponding to the HTTP status code in `Response.status`. For example, this would be `OK` for a - /// status code `200`, `Continue` for `100`, `Not Found` for `404`. - /// - /// Faith always returns the canonical status message for the code. In HTTP/1, servers can send - /// custom status text, but that text is not surfaced here; in HTTP/2 and HTTP/3, custom status - /// text is not supported at all. For status codes with no well-known message, this is an empty - /// string. - #[napi(getter)] - pub fn status_text(&self) -> &'static str { - self.status_code.canonical_reason().unwrap_or_default() - } - - /// The `type` read-only property of the `Response` interface contains the type of the response. The - /// type determines whether scripts are able to access the response body and headers. - /// - /// In Faith, this is always set to `basic`. - #[napi(getter, js_name = "type")] - pub fn typ(&self) -> &'static str { - "basic" - } - - /// The `url` read-only property of the `Response` interface contains the URL of the response. The - /// value of the `url` property will be the final URL obtained after any redirects. - #[napi(getter)] - pub fn url(&self) -> String { - self.url.to_string() - } - - /// The `version` read-only property of the `Response` interface contains the HTTP version of the - /// response. The value will be the final HTTP version after any redirects and protocol upgrades. - /// - /// This is custom to Faith. - #[napi(getter)] - pub fn version(&self) -> String { - format!("{:?}", self.version) - } - - /// The `bodyUsed` read-only property of the `Response` interface is a boolean value that indicates - /// whether the body has been read yet. - /// - /// In Faith, this indicates whether the body stream has ever been read from or canceled, as defined - /// [in the standard](https://streams.spec.whatwg.org/#is-readable-stream-disturbed). Note that accessing - /// the `.body` property counts as a read, even if you don't actually consume any bytes of content. - #[napi(getter)] - pub fn body_used(&self) -> bool { - self.disturbed.load(Ordering::SeqCst) - } - - /// The `body` read-only property of the `Response` interface is a `ReadableStream` of the body - /// contents, or `null` for any actual HTTP response that has no body, such as `HEAD` requests and - /// `204 No Content` responses. - /// - /// Note that browsers currently do not return `null` for those responses, but the standard - /// requires it. Faith chooses to respect the standard rather than the browsers in this case. - /// - /// An important consideration exists in conjunction with the connection pool: if you start the - /// body stream, this will hold the connection until the stream is fully consumed. If another - /// request is started during that time, and you don't have an available connection in the pool - /// for the host already, the new request will open one. - /// - /// Note that this is a function as an implementation detail; the wrapper makes it a property. - #[napi] - pub fn body( - &self, - env: Env, - ) -> Result>>, napi::Error> { - // we mark the body as disturbed, but we still allow reading it through here - // as essentially, the body() can be accessed many times as the same stream - let _ = self.check_stream_disturbed(); - - let Some(lock) = &self.body.body else { - return Ok(None); - }; - - // if the lock is taken then we're consuming the body somehow - let mut body = lock - .try_lock() - .map_err(|_| FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed).into_napi())?; - - let stream = self - .ensure_stream(&mut body, self.body.drained.clone()) - .map_err(|e| e.into_napi())?; - - let stream = napi::bindgen_prelude::ReadableStream::create_with_stream_bytes( - &env, - stream - .map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)).into_napi()), - ) - .map_err(|e| { - napi::Error::from( - FaithError::new(FaithErrorKind::BodyStream, Some(e.to_string())) - .into_js_error(&env), - ) - })?; - Ok(Some(stream)) - } - - fn check_stream_disturbed(&self) -> Result<(), FaithError> { - if self.disturbed.swap(true, Ordering::SeqCst) { - Err(FaithErrorKind::ResponseAlreadyDisturbed.into()) - } else { - Ok(()) - } - } - - /// Ensures the body is converted to a SharedStream, returning a clone of it. - /// - /// This allows multiple consumers (original + clones) to independently read the body. - fn ensure_stream( - &self, - body: &mut Body, - drained_flag: Arc, - ) -> Result>>, FaithError> { - match body { - Body::Consumed => Err(FaithErrorKind::ResponseAlreadyDisturbed.into()), - Body::Stream(stream) => Ok(stream.clone()), - lock @ Body::Inner(_) => { - // temporarily replace with Consumed until we can put in the Stream - let Body::Inner(inner) = replace(lock, Body::Consumed) else { - // SAFETY: we're inside the match checking for this exact thing - unsafe { unreachable_unchecked() } - }; - - // Track that we've started consuming a body - self.stats.bodies_started.fetch_add(1, Ordering::Relaxed); - - let trailers_stream = self.trailers.clone(); - let trailers_finish = self.trailers.clone(); - let stats_finish = self.stats.clone(); - let timing_finish = self.timing.clone(); - let drained_finish = drained_flag.clone(); - // The frame stream pulls trailers off to the side (via `arrived`) and yields - // data bytes only, so decoding sees no trailer frames. - let bytes = Box::pin( - BodyStream::new(inner) - .then(move |frame| { - let trailers_lock = trailers_stream.clone(); - async move { - match frame { - Err(err) => Some(Err(err.to_string())), - Ok(frame) => match frame.into_trailers() { - Ok(trailers) => { - trailers_lock.arrived(trailers); - None - } - Err(frame) => Some( - frame - .into_data() - .map_err(|_| "unknown frame kind".to_string()), - ), - }, - } - } - }) - .filter_map(async |item| item), - ) as Pin>; - - let bytes = match self.decode { - Some(coding) => decode_stream(bytes, coding), - None => bytes, - }; - - // A zero-length chunk carries no bytes, but the body's byte-oriented - // ReadableStream cannot take one: `ReadableByteStreamController.enqueue` - // rejects an empty buffer outright (`ERR_INVALID_STATE`). Some origins end a - // response with an empty DATA frame carrying END_STREAM, so drop empty chunks - // here, before the stream is built, letting it close cleanly. The byte count - // delivered is unchanged, and the collecting paths (`text()`, `bytes()`) never - // noticed the empties anyway. - let bytes = Box::pin(bytes.filter(|item| { - let empty = matches!(item, Ok(chunk) if chunk.is_empty()); - async move { !empty } - })) as Pin>; - - // Chained onto the stream that is actually delivered, above any decoder: a - // decoder reaches the end of its own framing without necessarily polling the - // bytes underneath to completion, so bookkeeping chained below it would never - // run for a decoded body, leaving the trailers promise and the timing pending - // for good. - let bytes = Box::pin( - bytes.chain( - stream::once(async move { - trailers_finish.ended(); - // The last byte of the body: every read path ends here, so - // this is where the timing settles - // (spec:RESP#request-timing). - timing_finish.ended(); - // Track that we've finished consuming a body - stats_finish.bodies_finished.fetch_add(1, Ordering::Relaxed); - // Mark body as drained so Drop doesn't try to drain again - drained_finish.store(true, Ordering::SeqCst); - }) - .filter_map(async |()| None), - ), - ) as Pin>; - - let stream = SharedStream::new(bytes); - - // the _ is the Consumed we put in there earlier - let _ = replace(lock, Body::Stream(stream.clone())); - - Ok(stream) - } - } - } - - /// Underlying efficient response body fetcher. - /// - /// Unlike bytes() and co, this grabs all the chunks of the response but doesn't - /// copy them. Further processing is needed to obtain a Vec or whatever needed. - async fn gather(&self) -> Result, FaithError> { - let Some(lock) = &self.body.body else { - return Ok(Default::default()); - }; - - let mut body = lock.lock().await; - let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; - drop(body); // release lock before consuming stream - - let mut chunks = Vec::new(); - futures::pin_mut!(stream); - while let Some(result) = stream.next().await { - let chunk = - result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; - chunks.push(chunk); - } - - // Mark as drained since we consumed everything - self.body.mark_drained(); - - Ok(Arc::from(chunks.into_boxed_slice())) - } - - /// Discard the response body, releasing the connection back to the pool. - /// - /// This is useful when you don't need the body but want to ensure the connection - /// can be reused for subsequent requests. If you don't call this and don't consume - /// the body, the connection may be held open until the response is garbage collected. - /// - /// For HTTP/1, the remaining body is read and thrown away so the connection can go back - /// to the pool. For HTTP/2 and HTTP/3, the body is dropped instead, which cancels the - /// stream (RST_STREAM / STOP_SENDING) without affecting the multiplexed connection. - /// - /// Returns a promise that resolves when the body has been fully discarded. - #[napi] - pub fn discard<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let body = self.body.body.clone(); - let drained_flag = self.body.drained.clone(); - let is_multiplexed = self.body.is_multiplexed(); - let trailers = self.trailers.clone(); - let timing = self.timing.clone(); - faith_promise(env, async move { - if let Some(arc) = body { - if is_multiplexed { - // Multiplexed connections don't need draining for reuse; dropping - // the body cancels the stream and frees its resources right away - // instead of waiting for garbage collection. - let mut guard = arc.lock().await; - *guard = Body::Consumed; - } else { - drain_body_inner(arc).await; - } - } - drained_flag.store(true, Ordering::SeqCst); - // Discarding the body discards its trailers: on a multiplexed connection the - // stream was cancelled before any could arrive, and draining an HTTP/1 body - // here bypasses the stream that would have collected them. Settling this as - // "none" rather than leaving it pending is the point -- a caller who discarded - // the body and then awaited trailers used to wait forever. - trailers.ended(); - // Discarding is one of the ways a body finishes (spec:RESP#request-timing). - timing.ended(); - Ok(()) - }) - } - - /// gather() and then copy into one contiguous buffer - async fn gather_contiguous(&self) -> Result, FaithError> { - let body = self.gather().await?; - let length = body.iter().map(|chunk| chunk.len()).sum(); - let mut bytes = Vec::with_capacity(length); - for chunk in body.into_iter() { - bytes.extend_from_slice(chunk); - } - - if let Some(ref integrity) = self.integrity { - verify_integrity(&bytes, integrity)?; - } - - Ok(bytes) - } - - /// The `bytes()` method of the `Response` interface takes a `Response` stream and reads it to - /// completion. It returns a promise that resolves with a `Uint8Array`. - /// - /// In Faith, this returns a Node.js `Buffer`, which can be used as (and is a subclass of) a `Uint8Array`. - #[napi] - pub fn bytes<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let this = Clone::clone(self); - faith_promise(env, async move { - this.check_stream_disturbed()?; - this.gather_contiguous().await.map(Buffer::from) - }) - } - - /// The `text()` method of the `Response` interface takes a `Response` stream and reads it to - /// completion. It returns a promise that resolves with a `String`. The response is always decoded - /// using UTF-8; as per the standard, invalid UTF-8 sequences are replaced with U+FFFD rather - /// than causing an error. - #[napi] - pub fn text<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let this = Clone::clone(self); - faith_promise(env, async move { - this.check_stream_disturbed()?; - let bytes = this.gather_contiguous().await?; - Ok(String::from_utf8(bytes) - .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())) - }) - } - - /// The `json()` method of the `Response` interface takes a `Response` stream and reads it to - /// completion. It returns a promise which resolves with the result of parsing the body text as - /// `JSON`. - /// - /// Note that despite the method being named `json()`, the result is not JSON but is instead the - /// result of taking JSON as input and parsing it to produce a JavaScript object. - /// - /// Further note that, at least in Faith, this method first reads the entire response body as bytes, - /// and then parses that as JSON. This can use up to double the amount of memory. If you need more - /// efficient access, consider handling the response body as a stream. - #[napi] - pub fn json<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let this = Clone::clone(self); - faith_promise(env, async move { - this.check_stream_disturbed()?; - let bytes = this.gather_contiguous().await?; - let value = serde_json::from_slice(&bytes) - .map_err(|e| FaithError::new(FaithErrorKind::JsonParse, Some(e.to_string())))?; - Ok(Value(value)) - }) - } - - /// Custom to Faith. - /// - /// `toFile(path, options)` writes the response body to a file on disk, the bytes - /// travelling from the network to the filesystem inside Faith without crossing into - /// JavaScript. It is a whole-body read alongside `bytes()` and its siblings: the first - /// consumer wins, `bodyUsed` becomes true once the read begins, and `integrity` is - /// verified when set. - /// - /// Resolves to `{ path, bytesWritten }`, where `path` is the absolute filesystem path - /// written to and `bytesWritten` counts the bytes that landed there. - /// - /// `onProgress` is reported to as the bytes land, at most every - /// [`PROGRESS_INTERVAL`], with a final report once the last byte is written. The - /// wrapper takes it from the options object; it arrives here as its own argument - /// because a threadsafe function cannot be a field of a `#[napi(object)]`. - /// - /// The `file://` URL to path conversion and the `InvalidPath` rejection happen in the - /// wrapper, so this receives a resolved string path. - /// - /// spec:BODY#tofile - #[napi( - ts_args_type = "path: string, options?: ToFileOptions | undefined | null, onProgress?: ((progress: ToFileProgress) => void) | undefined | null" - )] - pub fn to_file<'env>( - &self, - env: &'env Env, - path: String, - options: Option, - on_progress: Option, - ) -> Result, napi::Error> { - let this = Clone::clone(self); - let options = options.unwrap_or_default(); - faith_promise(env, async move { - this.write_to_file(path, options, on_progress).await - }) - } - - async fn write_to_file( - &self, - path: String, - options: ToFileOptions, - on_progress: Option, - ) -> Result { - // A response that cannot carry a body has nothing to write, and this is settled - // before any file is created (spec:BODY#tofile). - let Some(lock) = self.body.body.clone() else { - return Err(FaithErrorKind::ResponseBodyNull.into()); - }; - - // A body already read, or whose stream was handed out, has no second read to give. - // Checked without committing so an open failure below still leaves the body - // undisturbed and the caller free to retry to another path. - if self.disturbed.load(Ordering::SeqCst) { - return Err(FaithErrorKind::ResponseAlreadyDisturbed.into()); - } - - // Reject a malformed integrity value up front, before the body is touched, the same - // as the other verified reads reject it when the whole body is in hand. - let mut checker = integrity_checker(self.integrity.as_deref())?; - - // The advertised length, when the server sent one. It is only visible here for a body - // delivered as received: a decoded body has had its Content-Length stripped, so the - // bytes written equal the wire bytes wherever this is Some (spec:BODY#tofile, ENC). - let content_length = self - .headers - .get(CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.trim().parse::().ok()); - - // The destination is opened before any of the body is read, so a failure to open it - // leaves the body unread and undisturbed. - let mut file = open_destination(&path, &options).await?; - - // Commit the read now the destination is in hand. A concurrent read that slipped in - // since the load above wins, and this one finds the body already spent. - self.check_stream_disturbed()?; - - let stream = { - let mut body = lock.lock().await; - let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; - drop(body); // release lock before consuming stream - stream - }; - - // Reporting is rate limited rather than per chunk, so a large body does not cross - // into JavaScript thousands of times (spec:BODY#tofile). - let report = |written: u64| { - if let Some(callback) = &on_progress { - callback.call( - ToFileProgress { - bytes_written: written as i64, - content_length: content_length.map(|len| len as i64), - }, - // Progress is observational: a report the queue cannot take is dropped - // rather than made to hold up the write it is describing. - ThreadsafeFunctionCallMode::NonBlocking, - ); - } - }; - - let mut written: u64 = 0; - let mut reported_at = Instant::now(); - futures::pin_mut!(stream); - while let Some(result) = stream.next().await { - let chunk = - result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; - if let Some(checker) = checker.as_mut() { - checker.input(&chunk); - } - file.write_all(&chunk) - .await - .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; - written += chunk.len() as u64; - // A server cannot send more than it promised: once the bytes off the wire exceed - // the advertised length, the write fails and the bytes so far stay on disk - // (spec:BODY#tofile). - if let Some(limit) = content_length { - if written > limit { - return Err(FaithErrorKind::ContentLengthOverrun.into()); - } - } - if reported_at.elapsed() >= PROGRESS_INTERVAL { - reported_at = Instant::now(); - report(written); - } - } - - file.flush() - .await - .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; - - // The last report always lands, whatever the rate limit allowed along the way, so a - // caller's final view of a completed write is the whole body rather than the last - // interval boundary. An empty body reports once, with nothing written. - report(written); - - // The digest is only known once the last byte has been written, so the file that - // fails verification is on disk when the error arrives (spec:SRI). - if let Some(checker) = checker { - finish_integrity(checker)?; - } - - self.body.mark_drained(); - - Ok(ToFileResult { - // A relative path resolves against the process's working directory; the caller - // is handed the absolute path the bytes landed at. - path: std::path::absolute(&path) - .map(|abs| abs.to_string_lossy().into_owned()) - .unwrap_or(path), - bytes_written: written as i64, - }) - } - - /// Custom to Faith. - /// - /// The measurements behind the `timing` property, which the wrapper turns into a - /// `PerformanceResourceTiming`. - /// - /// A resource timing entry describes a finished request, so this does not resolve until - /// the body has ended: by being read, by `discard()`, or by the collector draining one - /// that was abandoned. A response that cannot carry a body has ended already. - /// - /// Phases are milliseconds from the start of the request rather than absolute times, so - /// the wrapper can place them on the same clock as the platform's other performance - /// entries. - /// - /// This is an async fn as an internal implementation detail and the wrapper makes it a - /// property. - /// - /// spec:RESP#request-timing - #[napi] - pub async fn timing(&self) -> TimingBreakdown { - self.timing.settled().await.into() - } - - /// The `trailers()` read-only property of the `Response` interface returns a promise that - /// resolves to either `null` or a `Headers` structure that contains the HTTP/2 or /3 trailing - /// headers. - /// - /// This was once in the standard as a getter, but was removed as no browser implemented it. - /// - /// Trailers only exist once the body has ended, so this does not resolve until the body - /// has been consumed — by `text()`, `bytes()`, `json()`, `blob()`, or reading the `body` - /// stream. Awaiting it first, on its own, waits forever: that is the behaviour the fetch - /// standard's trailers proposal describes (), not - /// a quirk of Faith. Holding the promise while something else reads the body is fine, and - /// costs nothing while it is pending. - /// - /// `discard()` counts as consuming the body but discards its trailers with it, so this - /// then resolves to `null` rather than waiting for trailers that can no longer arrive. - /// - /// This is an async fn as an internal implementation detail and the wrapper makes it a property. - #[napi] - pub async fn trailers(&self) -> Option> { - match self.trailers.settled().await { - // NotYet cannot come back from `settled`, which is what it waits on. - Trailers::NotYet | Trailers::None => None, - Trailers::Some(headers) => Some( - headers - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.to_string(), v.to_string())) - }) - .collect(), - ), - } - } - - /// The `clone()` method of the `Response` interface creates a clone of a response object, identical - /// in every way, but stored in a different variable. - /// - /// `clone()` throws an `Error` if the response body has already been used. - /// - /// (Per the standard, this should throw a `TypeError`, but for technical reasons this is not - /// possible with Faith.) - #[napi] - pub fn clone(&self, env: Env) -> Result { - if self.disturbed.load(Ordering::SeqCst) { - return Err(FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed) - .into_js_error(&env) - .into()); - } - - Ok(Self { - disturbed: Arc::new(AtomicBool::new(false)), - ..Clone::clone(self) - }) - } -} - -#[cfg(test)] -mod tests { - use std::{ - future::Future, - pin::pin, - sync::atomic::AtomicUsize, - task::{Context, Poll, Wake, Waker}, - }; - - use super::*; - - /// A waker that counts how many times the task asks to be polled again. - struct CountingWaker(AtomicUsize); - - impl CountingWaker { - fn wakes(&self) -> usize { - self.0.load(Ordering::SeqCst) - } - } - - impl Wake for CountingWaker { - fn wake(self: Arc) { - self.wake_by_ref(); - } - - fn wake_by_ref(self: &Arc) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - - /// Waiting for trailers parks until the body settles the question, rather than polling - /// for it. - /// - /// The bug this guards against was a `yield_now` loop, which is visible here as the shape - /// of the wait rather than as a quantity of CPU: a spin re-arms its own waker on every - /// poll, so it is scheduled again immediately, while a parked wait asks for nothing until - /// something else moves. Asserting the wake count keeps this deterministic -- timing how - /// much CPU the process burns measures the machine as much as the code. - #[test] - fn waiting_for_trailers_parks_rather_than_spinning() { - let slot = TrailersSlot::default(); - let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); - let waker = Waker::from(counter.clone()); - let mut cx = Context::from_waker(&waker); - let mut settled = pin!(slot.settled()); - - // Nothing has settled the question, so the wait parks... - assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); - // ...without scheduling itself to be polled again, which is what a spin does. - assert_eq!(counter.wakes(), 0, "a parked wait asks for no wake-up"); - - // Polling again changes nothing: still parked, still asking for nothing. - assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); - assert_eq!(counter.wakes(), 0, "polling again does not arm a wake-up"); - - // The body ending is what wakes it, and it resolves on the next poll. - slot.ended(); - assert!(counter.wakes() >= 1, "the body ending wakes the waiter"); - assert!(matches!( - settled.as_mut().poll(&mut cx), - Poll::Ready(Trailers::None) - )); - } - - /// Trailers that arrived before anyone asked resolve without parking at all. - #[test] - fn trailers_already_there_resolve_on_the_first_poll() { - let slot = TrailersSlot::default(); - let mut headers = HeaderMap::new(); - headers.insert("x-checksum", "abc123".parse().unwrap()); - slot.arrived(headers); - - let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); - let waker = Waker::from(counter.clone()); - let mut cx = Context::from_waker(&waker); - let mut settled = pin!(slot.settled()); - - assert!(matches!( - settled.as_mut().poll(&mut cx), - Poll::Ready(Trailers::Some(_)) - )); - } -} diff --git a/wrapper.d.ts b/wrapper.d.ts index e974ff6..31c15cf 100644 --- a/wrapper.d.ts +++ b/wrapper.d.ts @@ -22,7 +22,7 @@ export { USER_AGENT, } from "./index"; -// NOTE: This must be kept in sync with FaithErrorKind in src/error.rs +// NOTE: This must be kept in sync with the kinds in crates/web-faith/src/error.rs // Run `npm test` to validate sync (test/error-codes.test.js checks this) export const ERROR_CODES: { readonly Aborted: "Aborted";