From 987d5d9cb526a09cd1737033a207232036587777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 10:33:14 +1200 Subject: [PATCH 01/61] S1: update --- .../design/mockups/s1/rust-api-shapes.html | 456 ++++++++++++++++++ 1 file changed, 456 insertions(+) create mode 100644 .workhorse/design/mockups/s1/rust-api-shapes.html diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html new file mode 100644 index 0000000..1a4a174 --- /dev/null +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -0,0 +1,456 @@ + + + + + +Rust API shapes + + + +
+ +

Rust API shapes

+

+ Three candidate surfaces for web-faith, each shown against the same three jobs. + All of them keep browser-like defaults: cookie jar, HTTP cache, redirect following, and HTTP/3 upgrade + are on unless turned off. +

+ + +
+
+ A +

Client-first

+
+

+ A Client owns the pool, jar, and cache; requests are built by method verb off the client. + This is the reqwest arrangement, with Faith's defaults swapped in. +

+ +
+
+
Fetch some JSON
+
use web_faith::Client;
+
+let client = Client::new()?;
+
+let things: Vec<Thing> = client
+    .get("https://api.example/things")
+    .send()
+    .await?
+    .json()
+    .await?;
+
+ +
+
Configured POST
+
let client = Client::builder()
+    .user_agent("myapp/1.0")
+    .cache(Cache::disk("/var/cache/app"))
+    .cookies(false)
+    .build()?;
+
+let res = client
+    .post("https://api.example/things")
+    .header("authorization", tok)
+    .integrity("sha384-oqVuAfXR")
+    .timeout(Duration::from_secs(5))
+    .json(&thing)
+    .send()
+    .await?;
+
+ +
+
Stream the body
+
let res = client.get(url).send().await?;
+
+let mut body = res.body_stream();
+while let Some(chunk) = body.next().await {
+    sink.write_all(&chunk?).await?;
+}
+
+let trailers = res.trailers().await;
+let timing = res.timing().await;
+
+
+ +
+
+

Argues for

+
    +
  • Instantly legible to anyone who has used reqwest
    • +
  • Option structs become nested builders, no Option-soup
    • +
  • The agent is never implicit, so pool ownership is obvious
    • +
+
+
+

Argues against

+
    +
  • No one-line call: the trivial GET still needs a client first
    • +
  • Furthest from the specs' fetch vocabulary
    • +
  • Reads as "another reqwest" rather than a browser
    • +
+
+
+
+ + +
+
+ B +

Free fetch, builder for the rest

+
+

+ fetch(url) is a free function on a process-wide default agent, matching the Node entry point. + Anything beyond a bare GET goes through Request, which carries the options and ends in send(). +

+ +
+
+
Fetch some JSON
+
use web_faith::fetch;
+
+let things: Vec<Thing> =
+    fetch("https://api.example/things")
+        .await?
+        .json()
+        .await?;
+
+ +
+
Configured POST
+
use web_faith::{Agent, Request};
+
+let agent = Agent::builder()
+    .user_agent("myapp/1.0")
+    .cache(Cache::disk("/var/cache/app"))
+    .build()?;
+
+let res = Request::post("https://api.example/things")
+    .header("authorization", tok)
+    .integrity("sha384-oqVuAfXR")
+    .cache_mode(CacheMode::NoStore)
+    .json(&thing)
+    .agent(&agent)
+    .send()
+    .await?;
+
+ +
+
Stream the body
+
let res = fetch(url).await?;
+
+let mut body = res.body_stream();
+while let Some(chunk) = body.next().await {
+    sink.write_all(&chunk?).await?;
+}
+
+let trailers = res.trailers().await;
+let timing = res.timing().await;
+
+
+ +
+
+

Argues for

+
    +
  • The one-liner really is one line
    • +
  • Same default-agent story as Node, so the specs carry over
    • +
  • Request is independent of the agent, so it can be built once and sent on several
    • +
+
+
+

Argues against

+
    +
  • Two entry points to learn, and a cliff between them
    • +
  • .agent() in the middle of a chain is easy to forget, and the default is silent
    • +
  • A process-wide default agent is a hidden global
    • +
+
+
+
+ + +
+
+ C +

One fetch over anything request-shaped

+
+

+ A single fetch() takes anything that converts into a request: a URL string, a Url, + a built Request, or a http::Request. The builder is how you say more, not a second door. + This is the closest Rust gets to fetch(url) and fetch(url, init) being the same call. +

+ +
+
+
Fetch some JSON
+
use web_faith::fetch;
+
+let things: Vec<Thing> =
+    fetch("https://api.example/things")
+        .await?
+        .json()
+        .await?;
+
+ +
+
Configured POST
+
let agent = Agent::builder()
+    .user_agent("myapp/1.0")
+    .cache(Cache::disk("/var/cache/app"))
+    .build()?;
+
+let res = fetch(
+    Request::post("https://api.example/things")
+        .header("authorization", tok)
+        .integrity("sha384-oqVuAfXR")
+        .json(&thing)
+        .on(&agent),
+).await?;
+
+// or, sending from the agent directly
+let res = agent.fetch(url).await?;
+
+ +
+
Stream the body
+
let res = fetch(url).await?;
+
+let mut body = res.body_stream();
+while let Some(chunk) = body.next().await {
+    sink.write_all(&chunk?).await?;
+}
+
+let trailers = res.trailers().await;
+let timing = res.timing().await;
+
+
+ +
+
+

Argues for

+
    +
  • One verb the whole way up, from one-liner to fully configured
    • +
  • agent.fetch(..) and fetch(req.on(&agent)) are the same call, so neither is a special case
    • +
  • Accepting http::Request makes it a drop-in for tower and friends
    • +
+
+
+

Argues against

+
    +
  • The generic argument gives worse compile errors than a plain signature
    • +
  • Nesting the builder inside the call reads awkwardly at four or more options
    • +
  • Still needs the hidden default agent for the bare form
    • +
+
+
+
+ + +

What each shape decides for us

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuestionA · Client-firstB · Free fetchC · One fetch
Bare GETNeeds a clientOne lineOne line
Default agentNone, always explicitProcess-wide, implicitProcess-wide, implicit
Where options liveOn the client's request builderOn a standalone request builderOn a standalone request builder
Can one request go to two agentsNo, it is born on a clientYesYes
Spec vocabularyNeeds a translation per criterionCarries acrossCarries across
Interop with http::RequestExtra methodExtra methodFalls out of the trait
+ +

Settled either way

+ + + + + + + + + + + + + + + + + + + + + + + +
ResponseFetch-shaped: status, headers, ok, url, redirected, + plus text(), json(), bytes(), body_stream(), + and Faith's peer, version, trailers(), timing(), discard().
Body reuseReading a body consumes it, and a second read is an error, matching the fetch standard rather than reqwest's owned-response model.
ErrorsOne Error enum carrying the existing kinds, each with the same code the Node binding reports.
CancellationDropping the future cancels the request; timeout stays an option for the deadline case.
RuntimeTokio, as today.
+ +
+ + From 5ef7a5fdde0c748fb9a131765e8abb73bc5d363d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 14:07:57 +1200 Subject: [PATCH 02/61] S1: update rust-api-shapes.html --- .../design/mockups/s1/rust-api-shapes.html | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html index 1a4a174..8299f06 100644 --- a/.workhorse/design/mockups/s1/rust-api-shapes.html +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -372,6 +372,114 @@

Argues against

+ +

The default agent, if it stays

+

+ B and C both lean on a process-wide agent for the bare fetch(url) form. + Making it reachable turns it from a hidden global into a named one. Two questions decide the shape: + when it may be replaced, and what a closed default does next. +

+ +
+
+ i +

Install once

+
+

+ The default may be set until the moment something uses it, and never after. + This is how rustls takes its crypto provider. +

+
+
+
Setting it up
+
// early in main, before any fetch
+Agent::builder()
+    .user_agent("myapp/1.0")
+    .cache(Cache::disk("/var/cache/app"))
+    .build()?
+    .install_default()?;   // Err if in use
+
+let res = fetch(url).await?;  // uses it
+
+
+
Reaching it
+
let agent = web_faith::default_agent();
+
+let stats = agent.stats();
+agent.network_changed();
+
+// no close: it lives for the process
+
+
+
+
+

Argues for

+
    +
  • No request can ever change agent mid-flight
    • +
  • The failure is loud and at startup, not a silent wrong-pool
    • +
+
+
+

Argues against

+
    +
  • Nothing can release the pool and cache short of process exit
    • +
  • Tests that want a fresh default per case cannot have one
    • +
+
+
+
+ +
+
+ ii +

Swappable

+
+

+ The default sits behind a lock and can be replaced or closed at any point. + Setting hands back whatever was there, so the caller decides what happens to it. +

+
+
+
Swapping
+
let previous = web_faith::set_default_agent(agent);
+
+// the old one is yours to wind down
+if let Some(mut old) = previous {
+    old.close();
+}
+
+
+
Shutting down
+
let agent = web_faith::default_agent();
+let stats = agent.stats();
+
+// closes the shared agent, not a clone
+web_faith::close_default_agent();
+
+// what does this do?
+let res = fetch(url).await;
+
+
+
+
+

Argues for

+
    +
  • The pool, cache, and resolver can be released on demand
    • +
  • Tests can install and tear down a default per case
    • +
  • A network change or a config reload can rebuild it wholesale
    • +
+
+
+

Argues against

+
    +
  • Two concurrent fetch calls can land on different agents
    • +
  • Every bare fetch pays a lock read
    • +
  • Leaves the last line above undecided
    • +
+
+
+
+

What each shape decides for us

From c9b367346844e57996aecadc8b4e8fc35043e99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 14:24:02 +1200 Subject: [PATCH 03/61] S1: update rust-api-shapes.html --- .../design/mockups/s1/rust-api-shapes.html | 504 +++++------------- 1 file changed, 124 insertions(+), 380 deletions(-) diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html index 8299f06..b0ae960 100644 --- a/.workhorse/design/mockups/s1/rust-api-shapes.html +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -1,9 +1,9 @@ - + -Rust API shapes +Rust API shape
-

Rust API shapes

+

Rust API shape

- Three candidate surfaces for web-faith, each shown against the same three jobs. - All of them keep browser-like defaults: cookie jar, HTTP cache, redirect following, and HTTP/3 upgrade - are on unless turned off. + web-faith has one noun and one verb. An Agent owns the pool, jar, cache, and + resolver; fetch is how every request goes out. Browser-like defaults stay on unless turned off: + cookie jar, HTTP cache, redirect following, and HTTP/3 upgrade.

- -
-
- A -

Client-first

-
-

- A Client owns the pool, jar, and cache; requests are built by method verb off the client. - This is the reqwest arrangement, with Faith's defaults swapped in. +

+

The whole surface

+

+ fetch takes a URL or a prebuilt Request and returns a builder. + The builder is a future, so a bare call awaits directly and a configured one awaits after its options. + There is no send() and no second entry point.

-
+
-
Fetch some JSON
-
use web_faith::Client;
+        
Fetch some JSON

+
use web_faith::Agent;
 
-let client = Client::new()?;
+let agent = Agent::new()?;
 
-let things: Vec<Thing> = client
-    .get("https://api.example/things")
-    .send()
+let things: Vec<Thing> = agent
+    .fetch("https://api.example/things")
     .await?
     .json()
     .await?;
-
Configured POST
-
let client = Client::builder()
+        
Configured POST

+
let agent = Agent::builder()
     .user_agent("myapp/1.0")
     .cache(Cache::disk("/var/cache/app"))
     .cookies(false)
     .build()?;
 
-let res = client
-    .post("https://api.example/things")
+let res = agent
+    .fetch("https://api.example/things")
+    .method(Method::POST)
     .header("authorization", tok)
     .integrity("sha384-oqVuAfXR")
+    .cache_mode(CacheMode::NoStore)
     .timeout(Duration::from_secs(5))
     .json(&thing)
-    .send()
     .await?;
-
Stream the body
-
let res = client.get(url).send().await?;
+        
Stream the body

+
let res = agent.fetch(url).await?;
 
 let mut body = res.body_stream();
 while let Some(chunk) = body.next().await {
@@ -198,359 +170,117 @@ 
Client-first

 let timing = res.timing().await;
- -
-
-

Argues for

-
    -
  • Instantly legible to anyone who has used reqwest
    • -
  • Option structs become nested builders, no Option-soup
    • -
  • The agent is never implicit, so pool ownership is obvious
    • -
-
-
-

Argues against

-
    -
  • No one-line call: the trivial GET still needs a client first
    • -
  • Furthest from the specs' fetch vocabulary
    • -
  • Reads as "another reqwest" rather than a browser
    • -
-
-
- -
-
- B -

Free fetch, builder for the rest

-
-

- fetch(url) is a free function on a process-wide default agent, matching the Node entry point. - Anything beyond a bare GET goes through Request, which carries the options and ends in send(). +

+

Building a request once

+

+ A Request is the built, inert form. Because fetch accepts one and still returns a + builder, a request can be prepared once and tweaked at each call site, or sent unchanged across + two agents. Cloning follows the body: a buffered body clones, a stream body cannot, so it is + try_clone rather than Clone.

-
-
-
Fetch some JSON
-
use web_faith::fetch;
-
-let things: Vec<Thing> =
-    fetch("https://api.example/things")
-        .await?
-        .json()
-        .await?;
-
- +
-
Configured POST
-
use web_faith::{Agent, Request};
-
-let agent = Agent::builder()
-    .user_agent("myapp/1.0")
-    .cache(Cache::disk("/var/cache/app"))
+        
Prepare, then vary

+
let probe = Request::new("https://api.example/health")
+    .header("accept", "application/json")
+    .timeout(Duration::from_secs(2))
     .build()?;
 
-let res = Request::post("https://api.example/things")
-    .header("authorization", tok)
-    .integrity("sha384-oqVuAfXR")
-    .cache_mode(CacheMode::NoStore)
-    .json(&thing)
-    .agent(&agent)
-    .send()
-    .await?;

-
- -
-
Stream the body
-
let res = fetch(url).await?;
+for region in regions {
+    let Some(req) = probe.try_clone() else { break };
 
-let mut body = res.body_stream();
-while let Some(chunk) = body.next().await {
-    sink.write_all(&chunk?).await?;
-}
-
-let trailers = res.trailers().await;
-let timing = res.timing().await;
-
-
- -
-
-

Argues for

-
    -
  • The one-liner really is one line
    • -
  • Same default-agent story as Node, so the specs carry over
    • -
  • Request is independent of the agent, so it can be built once and sent on several
    • -
-
-
-

Argues against

-
    -
  • Two entry points to learn, and a cliff between them
    • -
  • .agent() in the middle of a chain is easy to forget, and the default is silent
    • -
  • A process-wide default agent is a hidden global
    • -
-
-
-
- - -
-
- C -

One fetch over anything request-shaped

-
-

- A single fetch() takes anything that converts into a request: a URL string, a Url, - a built Request, or a http::Request. The builder is how you say more, not a second door. - This is the closest Rust gets to fetch(url) and fetch(url, init) being the same call. -

- -
-
-
Fetch some JSON
-
use web_faith::fetch;
-
-let things: Vec<Thing> =
-    fetch("https://api.example/things")
-        .await?
-        .json()
-        .await?;
+ let res = agent + .fetch(req) + .header("x-region", region) + .await?; +}
-
Configured POST
-
let agent = Agent::builder()
-    .user_agent("myapp/1.0")
-    .cache(Cache::disk("/var/cache/app"))
+        
Same request, two agents

+
let req = Request::new(url)
+    .method(Method::POST)
+    .json(&payload)?
     .build()?;
 
-let res = fetch(
-    Request::post("https://api.example/things")
-        .header("authorization", tok)
-        .integrity("sha384-oqVuAfXR")
-        .json(&thing)
-        .on(&agent),
-).await?;
-
-// or, sending from the agent directly
-let res = agent.fetch(url).await?;

-
- -
-
Stream the body
-
let res = fetch(url).await?;
-
-let mut body = res.body_stream();
-while let Some(chunk) = body.next().await {
-    sink.write_all(&chunk?).await?;
-}
-
-let trailers = res.trailers().await;
-let timing = res.timing().await;
-
-
+let Some(copy) = req.try_clone() else { + return Err(Error::NotCloneable); +}; -
-
-

Argues for

-
    -
  • One verb the whole way up, from one-liner to fully configured
    • -
  • agent.fetch(..) and fetch(req.on(&agent)) are the same call, so neither is a special case
    • -
  • Accepting http::Request makes it a drop-in for tower and friends
    • -
-
-
-

Argues against

-
    -
  • The generic argument gives worse compile errors than a plain signature
    • -
  • Nesting the builder inside the call reads awkwardly at four or more options
    • -
  • Still needs the hidden default agent for the bare form
    • -
+let (primary, mirror) = tokio::join!( + live_agent.fetch(req), + audit_agent.fetch(copy), +);
- -

The default agent, if it stays

-

- B and C both lean on a process-wide agent for the bare fetch(url) form. - Making it reachable turns it from a hidden global into a named one. Two questions decide the shape: - when it may be replaced, and what a closed default does next. -

- -
-
- i -

Install once

-
-

- The default may be set until the moment something uses it, and never after. - This is how rustls takes its crypto provider. +

+

Agent lifecycle

+

+ An agent is cheap to clone and every clone names the same pool, so close acts on the shared + agent rather than on the handle it was called through. Requests that have already gone out run to + completion; a request that has not yet started fails with the closed-agent error.

-
-
-
Setting it up
-
// early in main, before any fetch
-Agent::builder()
-    .user_agent("myapp/1.0")
-    .cache(Cache::disk("/var/cache/app"))
-    .build()?
-    .install_default()?;   // Err if in use
 
-let res = fetch(url).await?;  // uses it
-
+
-
Reaching it
-
let agent = web_faith::default_agent();
+        
Shutting down

+
let agent = Agent::new()?;
+let handle = agent.clone();
 
-let stats = agent.stats();
-agent.network_changed();
+tokio::spawn(async move {
+    handle.fetch(url).await
+});
 
-// no close: it lives for the process

-
-
-
-
-

Argues for

-
    -
  • No request can ever change agent mid-flight
    • -
  • The failure is loud and at startup, not a silent wrong-pool
    • -
-
-
-

Argues against

-
    -
  • Nothing can release the pool and cache short of process exit
    • -
  • Tests that want a fresh default per case cannot have one
    • -
+// releases the pool, resolver, and probes +// for every clone, not just this one +agent.close();
-
-
-
-
- ii -

Swappable

-
-

- The default sits behind a lock and can be replaced or closed at any point. - Setting hands back whatever was there, so the caller decides what happens to it. -

-
-
Swapping
-
let previous = web_faith::set_default_agent(agent);
+        
Acting on a live agent

+
agent.network_changed();
 
-// the old one is yours to wind down
-if let Some(mut old) = previous {
-    old.close();
-}

-
-
-
Shutting down
-
let agent = web_faith::default_agent();
 let stats = agent.stats();
+let conns = agent.connections();
+let resolvers = agent.resolvers();
 
-// closes the shared agent, not a clone
-web_faith::close_default_agent();
-
-// what does this do?
-let res = fetch(url).await;
-
-
-
-
-

Argues for

-
    -
  • The pool, cache, and resolver can be released on demand
    • -
  • Tests can install and tear down a default per case
    • -
  • A network change or a config reload can rebuild it wholesale
    • -
-
-
-

Argues against

-
    -
  • Two concurrent fetch calls can land on different agents
    • -
  • Every bare fetch pays a lock read
    • -
  • Leaves the last line above undecided
    • -
+agent.prefetch_dns("api.example").await?; +agent.preconnect("https://api.example").await?;
- -

What each shape decides for us

+

Response

- - - - - - - - - - - - - - - - - - + + - - - - + + - - - - - - - - - - - - - - - - - - -
QuestionA · Client-firstB · Free fetchC · One fetch
Bare GETNeeds a clientOne lineOne line
Default agentNone, always explicitProcess-wide, implicitProcess-wide, implicitFetch surfacestatus, status_text, ok, headers, url, + redirected, kind, body_used
Where options liveOn the client's request builderOn a standalone request builderOn a standalone request builderReadingtext(), json(), bytes(), body_stream(), + to_file(), discard(). Reading consumes the body and a second read is an + error, following the fetch standard rather than reqwest's owned-response model.
Can one request go to two agentsNo, it is born on a clientYesYes
Spec vocabularyNeeds a translation per criterionCarries acrossCarries across
Interop with http::RequestExtra methodExtra methodFalls out of the trait
- -

Settled either way

- - - - - - - - - + + - + - + @@ -559,6 +289,20 @@

Settled either way

ResponseFetch-shaped: status, headers, ok, url, redirected, - plus text(), json(), bytes(), body_stream(), - and Faith's peer, version, trailers(), timing(), discard().
Body reuseReading a body consumes it, and a second read is an error, matching the fetch standard rather than reqwest's owned-response model.Faith additionspeer, version, trailers(), timing()
ErrorsOne Error enum carrying the existing kinds, each with the same code the Node binding reports.One Error type carrying the existing kinds, each reporting the same code the Node + binding does.
CancellationDropping the future cancels the request; timeout stays an option for the deadline case.Dropping the future cancels the request. timeout stays an option for the deadline case.
Runtime
+

Considered and dropped

+
+
    +
  • A process-wide default agent with set and close accessors, + so a bare fetch(url) free function could exist. Dropped because a hidden global owning a + connection pool and a disk cache is worth more scrutiny in Rust than in Node. The cost is that two + libraries in one process cannot share a pool without passing an agent between them.
    • +
  • A reqwest-shaped Client with get, post, and + friends. Dropped in favour of one verb, which keeps the fetch vocabulary the specs are written in.
    • +
  • Crate namespaces (web-faith::cookies). Not available: RFC 3243 is + accepted but unimplemented, with the cargo, crates.io, and docs.rs issues all open.
    • +
+
+ From 3872c2e4dcdbad9de1bf050bd53a631f4293df80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 14:35:14 +1200 Subject: [PATCH 04/61] S1: update rust-api-shapes.html --- .../design/mockups/s1/rust-api-shapes.html | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html index b0ae960..9134fda 100644 --- a/.workhorse/design/mockups/s1/rust-api-shapes.html +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -289,6 +289,69 @@

Response

+

Ecosystem types

+

+ The http crate's types are canonical wherever one exists, so a Faith request composes with + tower, hyper, and axum code without a shim. Setters stay fetch-flavoured by taking anything that + converts, so a string literal works where JS would pass a string. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConceptCanonical typeWhat setters accept
Methodhttp::Method.method(impl TryInto<Method>), so "POST" and Method::POST both work
Headerhttp::HeaderName, http::HeaderValue.header(impl TryInto<HeaderName>, impl TryInto<HeaderValue>)
Header sethttp::HeaderMap.headers(impl IntoIterator<Item = (K, V)>), which covers a HeaderMap, a + Vec of pairs, and an array of string pairs
Statushttp::StatusCodeRead-only. ok and status_text derive from it
Versionhttp::VersionRead-only, replacing the Node binding's string
URLurl::Urlimpl TryInto<Url>, matching what the fetch standard parses and what reqwest already uses
Request inRequestfetch and Request::new take a URL, a Request, or an + http::Request<B>
Response outResponseConverts into http::Response<Body>, and the body implements + http_body::Body so it feeds tower and hyper directly
Conversion errorsError::InvalidHeader and friendsHeld until the builder resolves: at build() for a request, at the await for a fetch. + Either way the error names the offending header
+

Considered and dropped

    From a0283781ac590e4558b853294fa8d042d895d047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 14:48:50 +1200 Subject: [PATCH 05/61] S1: update 3 specs, update overview.md --- .workhorse/specs/agent/overview.md | 2 + .workhorse/specs/errors/errors.md | 1 + .workhorse/specs/overview.md | 8 +++- .workhorse/specs/rust/client-api.md | 64 +++++++++++++++++++++++++++++ .workhorse/specs/rust/overview.md | 61 +++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 .workhorse/specs/rust/client-api.md create mode 100644 .workhorse/specs/rust/overview.md diff --git a/.workhorse/specs/agent/overview.md b/.workhorse/specs/agent/overview.md index 7228ec9..ee20f4a 100644 --- a/.workhorse/specs/agent/overview.md +++ b/.workhorse/specs/agent/overview.md @@ -33,6 +33,8 @@ Dual-stack hosts are unaffected. `close()` releases the agent's resources on demand: the connection pool, the DNS resolver, in-flight HTTP/3 probes, and the HTTP/3 knowledge cache. This exists because waiting for the garbage collector is not acceptable for code that creates many short-lived agents. Requests already in flight when `close()` is called run to completion; new requests on a closed agent throw a closed-agent error (code `Closed`). +A request counts as in flight from the moment it is issued rather than from when its work begins, so a request issued just before `close()` completes even if nothing had started on it yet. +`close()` acts on the agent itself rather than on the reference it was called through, so every reference to that agent sees it closed. `close()` is idempotent, and the cookie jar remains readable after closing. `networkChanged()` is the other verb that acts on a live agent's own state, discarding what the agent learned from a network that no longer exists while keeping the agent usable (see [NETCHG](network-change.md)). diff --git a/.workhorse/specs/errors/errors.md b/.workhorse/specs/errors/errors.md index 1143467..efb73f2 100644 --- a/.workhorse/specs/errors/errors.md +++ b/.workhorse/specs/errors/errors.md @@ -11,6 +11,7 @@ Callers match on `error.code` against the exported `ERROR_CODES` map rather than Every error Faith throws carries a `code` set to a stable name for its kind. `ERROR_CODES` is exported and enumerates the library's error codes; it is generated from the same source as the errors themselves, so the two cannot drift. +That source is the client's own error type: a component crate names its own errors for the failures it can produce, and the client converts them as they cross into it, so the split into crates (see [RUST](../rust/overview.md)) leaves the set of codes and the kind each failure reports unchanged. Every code in `ERROR_CODES` is reachable: each one names a kind that some failure surfaces to the caller, so a branch written for any code in the map can fire. Error messages are prefixed with the kind name and may embed underlying detail; the message is for humans, the code is the API. Errors surfaced through reading the response `body` stream carry no `code`. diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index 4fda8f8..428a431 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -4,8 +4,12 @@ id: FAITH # Faith -Faith is a fetch API implementation for Node.js backed by a Rust network stack rather than Node's built-in HTTP machinery. -It is published as the native module `@passcod/faith` and aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, while exposing the capabilities that stack unlocks: transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. +Faith is a fetch implementation backed by a Rust network stack rather than Node's built-in HTTP machinery. +It aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, while exposing the capabilities that stack unlocks: transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. + +It reaches callers through two surfaces built from one implementation. +The native module `@passcod/faith` is the Node.js surface, and the specs here describe its behaviour in JavaScript terms unless they say otherwise. +The crate `web-faith` is the Rust surface, published to crates.io with the component crates beneath it (see [RUST](rust/overview.md)); it keeps the same behaviour and spells it in Rust (see [RSAPI](rust/client-api.md)). The library's contract has two halves: fidelity to the fetch standard, and divergence where the standard assumes a browser. diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md new file mode 100644 index 0000000..0fd2d0c --- /dev/null +++ b/.workhorse/specs/rust/client-api.md @@ -0,0 +1,64 @@ +--- +id: RSAPI +--- + +# The Rust client API + +`web-faith` has one noun and one verb: an `Agent` owns the connection pool, resolver, cookie jar, HTTP cache, and HTTP/3 knowledge, and `fetch` is how every request goes out. +The surface is fetch-flavoured rather than a transcription of the JavaScript API: it keeps fetch's vocabulary and its browser-like defaults, and it speaks the Rust ecosystem's types wherever one exists for the job. +The agent's own behaviour, its options, and its lifecycle are specified in [AGENT](../agent/overview.md) and the specs beneath it; this spec covers the shape a Rust caller sees. + +## Agent + +`Agent::new()` constructs an agent with default options, and `Agent::builder()` returns a builder whose methods mirror the option groups in [AGENT](../agent/overview.md), ending in `build()`. +Nested option groups are nested builders rather than structs of optional fields, so a group left alone is absent from the call rather than spelled out as absent. +Construction validates options and reports the errors named in [AGENT](../agent/overview.md). + +An agent is cheap to clone and every clone names the same underlying agent, so cloning one is how a request gets an agent to run on rather than a way to get a second pool. +Because clones share, `close()` acts on the agent itself and every handle to it sees the result. +A request that has already been issued completes, a request issued afterwards fails with the closed-agent error, and `close()` is idempotent, exactly as in [AGENT](../agent/overview.md). +The agent captures what a request needs at the moment the request is issued, which is what lets an in-flight request finish while later ones are refused. + +`network_changed()`, `stats()`, `connections()`, `resolvers()`, `prefetch_dns()`, and `preconnect()` act on a live agent as their counterparts do on the Node surface. + +## Making a request + +`agent.fetch(target)` returns a request builder, where `target` is a URL, a `Request`, or an `http::Request`. +The builder implements `IntoFuture`, so awaiting it sends the request: a bare call awaits directly, and a configured one awaits after its options. +There is no separate send step, and a builder that is dropped without being awaited sends nothing, so the builder is marked `#[must_use]`. + +Builder methods cover the per-request options in [REQ](../fetch/request.md), [CANCEL](../fetch/cancellation-and-timeouts.md), [SRI](../fetch/integrity.md), [ENC](../fetch/content-encoding.md), and [CACHE](../cache/http-cache.md), and set the method, headers, and body. + +`Request::new(target)` takes the same three kinds of target and returns the same builder, which `build()` resolves into a `Request` rather than sending it. +A `Request` is inert, and passing one to `fetch` returns a builder again, so a request can be prepared once and adjusted at each call site or sent unchanged on more than one agent. +`try_clone()` copies a request when its body allows it and reports that it cannot when the body is a stream, a stream being consumable once. + +A builder that carries no agent cannot be awaited, and the compiler refuses it rather than the call failing when it runs. + +## Reading a response + +A `Response` carries `status`, `status_text`, `ok`, `headers`, `url`, `redirected`, `kind`, and `body_used` with the meanings in [RESP](../response/response.md), and Faith's own `peer` and `version`. +`text()`, `json()`, `bytes()`, `body_stream()`, `to_file()`, and `discard()` read the body, and `trailers()` and `timing()` resolve as in [TRL](../response/trailers.md) and [RESP](../response/response.md). +Reading the body consumes it and a second read fails, following the fetch standard rather than the owned-response model of other Rust clients, as in [BODY](../response/reading-the-body.md). + +The response body implements `http_body::Body`, and a `Response` converts into an `http::Response`, so a Faith response feeds code written against the wider ecosystem without a shim. + +## Ecosystem types + +Types from `http` are canonical wherever one exists: `Method`, `HeaderName`, `HeaderValue`, `HeaderMap`, `StatusCode`, and `Version`. +URLs are `url::Url`, which is what the fetch standard's parsing rules describe and what the stack beneath already uses. +`status_text` and `ok` derive from the status code rather than being carried separately. + +Setters accept anything that converts into the canonical type, so a string literal works where JavaScript would pass a string and a typed value works where a caller already holds one. +A conversion that fails is held until the builder resolves and surfaces there: at `build()` for a request, and at the await for a fetch. +Either way it reports the invalid-header or invalid-method error naming the offender, as [REQ](../fetch/request.md) requires. + +## Errors and cancellation + +Failures surface as one error type whose variants are the kinds in [ERR](../errors/errors.md), each reporting the same stable code as its JavaScript counterpart. +Errors arriving from a component crate are converted into it at the boundary, so a caller matches on one type whichever layer failed. + +Dropping the future cancels the request, which is how a Rust caller aborts. +The `timeout` option remains for the deadline case, and both surface the errors in [CANCEL](../fetch/cancellation-and-timeouts.md). + +Asynchronous work runs on Tokio. diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md new file mode 100644 index 0000000..1f777bd --- /dev/null +++ b/.workhorse/specs/rust/overview.md @@ -0,0 +1,61 @@ +--- +id: RUST +--- + +# The Rust distribution + +Faith's network stack is published to crates.io as a family of crates, with `web-faith` as the client a Rust caller reaches for and six component crates beneath it that each stand on their own. +The Node.js native module is built from the same workspace, so the two surfaces are two faces of one implementation rather than two implementations: a behaviour specified anywhere else in these specs holds on both unless that spec says otherwise. +The Rust API surface itself is specified in [RSAPI](client-api.md). + +## The crate family + +`web-faith` is the HTTP client. +It owns the agent, the request and response types, and the `fetch` entry point, and it draws on the component crates for the subsystems beneath it. +The name carries a `web-` prefix because the bare `faith` name on crates.io belongs to an unrelated project, and the prefix reads as the browser-shaped client the crate is. + +Six component crates are published alongside it, each one useful to a caller who wants that piece without the client above it: + +- `web-faith-cookies` is the cookie jar, with the storage, matching, and eviction rules in [COOK](../agent/cookies.md). +- `web-faith-dns` is the resolver, its cache, the discovery ladder, the `HTTPS` record query, and Happy Eyeballs, as in [DNS](../agent/dns.md). +- `web-faith-conn-tracker` reads live connection state from the operating system, as in [OBS](../agent/observability.md). +- `web-faith-alt-svc` is the Alt-Svc store and the HTTP/3 upgrade machinery, as in [H3UP](../http3/upgrade.md) and [PROBE](../http3/probing.md). +- `web-faith-encoding` is content coding for request and response bodies, as in [ENC](../fetch/content-encoding.md). +- `web-faith-integrity` is Subresource Integrity parsing and verification, as in [SRI](../fetch/integrity.md). + +A component crate depends on another component crate where the subsystems genuinely compose, so `web-faith-alt-svc` draws on `web-faith-dns` for the resolution its probes need. +None of them depends on `web-faith`, which is what makes each one usable on its own. + +`web-faith-napi` is the Node.js binding, and it ships to npm as the prebuilt native module `@passcod/faith`. +It is the one crate in the workspace carrying napi types: every other crate compiles, tests, and documents without a JavaScript runtime present, which is the test that the binding layer is genuinely separate rather than merely renamed. + +## A component crate stands alone + +A component crate names its own error type covering the failures that piece can produce, rather than a type shared across the family. +`web-faith` converts them into its own error as they reach it, and those conversions are what keep the code contract in [ERR](../errors/errors.md) intact across the split. + +A component crate takes and returns types from `http`, `url`, `bytes`, and the other ecosystem crates it already speaks, rather than types belonging to Faith, wherever such a type exists for the job. +Where a component needs a shape from the layer above, it takes it as a generic or through `http::Extensions` rather than depending upward. + +Each component crate carries its own documentation, and its examples run against that crate alone. + +## Choosing what is built + +Cargo features are how a subsystem is included or left out, so a build that has no use for a piece does not carry it. +Each of the six components has a feature on `web-faith` named for it, and the default set turns on the ones that make the client behave like a browser. +Turning a component's feature off drops the dependency and the behaviour it provides, and the client continues to work without it. + +Features are the whole of the swapping mechanism: a caller chooses among the implementations Faith builds rather than supplying one. + +## Versioning and support + +Every published crate follows semantic versioning from `1.0.0`, and the crates version independently, so a change confined to one component moves that crate alone. +Releases are prepared by release-plz, and a release runs `cargo-semver-checks` against the previous version of each crate, so a breaking change reaches a major bump rather than a patch. + +The minimum supported Rust version is 1.96, it is declared as `rust-version` in every published crate, and CI builds and tests against it as well as against stable, so the declaration is verified rather than asserted. + +## Workspace layout + +The repository is a Cargo workspace whose members live under `crates/`, one directory per crate named for it. +Shared package metadata (licence, repository, authors, edition, `rust-version`) is declared once at the workspace root and inherited, so the crates cannot drift apart on the fields that describe the same project. +The npm package is built from `crates/web-faith-napi`, and the generated `index.js` and `index.d.ts` continue to sit at the repository root where the package's entry points expect them. From f086b5d39bfafead3075bdb13591feb91989b9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 14:55:59 +1200 Subject: [PATCH 06/61] S1: update client-api.md --- .workhorse/specs/rust/client-api.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md index 0fd2d0c..32370b7 100644 --- a/.workhorse/specs/rust/client-api.md +++ b/.workhorse/specs/rust/client-api.md @@ -35,6 +35,24 @@ A `Request` is inert, and passing one to `fetch` returns a builder again, so a r A builder that carries no agent cannot be awaited, and the compiler refuses it rather than the call failing when it runs. +## Layering options + +A request is built in layers: a builder wraps a target, `build()` settles it into a `Request`, and that request can itself be the target of another builder, to any depth. +Each layer wins over the layer it wraps, which is the rule the Node surface follows when options passed to `fetch()` beat the values carried on a `Request` (see [REQ](../fetch/request.md)). +So in `agent.fetch(Request::new(inner).timeout(b)).timeout(c)` the timeout is `c`, whatever `inner` set. +A setting a layer does not touch is inherited from beneath it unchanged, so wrapping a request to adjust one thing leaves the rest as it was. + +Single-valued settings, among them the method, body, timeout, integrity, cache mode, credentials, priority, and request compression, take the outermost value set explicitly. +Setting one twice on a single builder is the same question at a smaller scale, and the later call wins. + +Headers merge by name rather than wholesale, matching how per-request headers beat agent defaults in [REQ](../fetch/request.md). +A name an outer layer sets takes the outer value, a name only an inner layer sets is carried through, and removing a name removes what the layers beneath contributed for it. +Setting a collection of headers applies each entry as a single header would, so it adds to and overrides the set by name rather than replacing it. + +The URL comes from the target at the bottom of the stack, and wrapping a request carries its URL through. + +The agent is not part of this layering: a request runs on whichever agent's `fetch` was called, and a `Request` carries no agent of its own. + ## Reading a response A `Response` carries `status`, `status_text`, `ok`, `headers`, `url`, `redirected`, `kind`, and `body_used` with the meanings in [RESP](../response/response.md), and Faith's own `peer` and `version`. From 52b5011900aa87e63573a8c2279c49f918e9e25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 15:36:42 +1200 Subject: [PATCH 07/61] S1: update 3 specs, update rust-api-shapes.html --- .../design/mockups/s1/rust-api-shapes.html | 15 +++++----- .workhorse/specs/environment/variables.md | 9 ++++++ .workhorse/specs/rust/client-api.md | 30 ++++++++++++++----- .workhorse/specs/rust/overview.md | 15 ++++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.workhorse/design/mockups/s1/rust-api-shapes.html b/.workhorse/design/mockups/s1/rust-api-shapes.html index 9134fda..46f697b 100644 --- a/.workhorse/design/mockups/s1/rust-api-shapes.html +++ b/.workhorse/design/mockups/s1/rust-api-shapes.html @@ -112,8 +112,8 @@

    Rust API shape

    web-faith has one noun and one verb. An Agent owns the pool, jar, cache, and - resolver; fetch is how every request goes out. Browser-like defaults stay on unless turned off: - cookie jar, HTTP cache, redirect following, and HTTP/3 upgrade. + resolver; fetch is how every request goes out. Defaults match the Node surface: redirects + followed and HTTP/3 upgrade on, cookie jar and HTTP cache opted into.

    @@ -143,7 +143,7 @@

    The whole surface

    let agent = Agent::builder()
     .user_agent("myapp/1.0")
     .cache(Cache::disk("/var/cache/app"))
-    .cookies(false)
+    .cookies(true)
     .build()?;
 
 let res = agent
@@ -222,8 +222,8 @@ 
    Building a request once
    
     
    Agent lifecycle
    
     
    
       An agent is cheap to clone and every clone names the same pool, so close acts on the shared
-      agent rather than on the handle it was called through. Requests that have already gone out run to
-      completion; a request that has not yet started fails with the closed-agent error.
+      agent rather than on the handle it was called through. Requests already issued run to completion, even
+      ones no work has started on; a request issued afterwards fails with the closed-agent error.
     
    
 
     
    
@@ -248,6 +248,7 @@ 
    Agent lifecycle
    
 let stats = agent.stats();
 let conns = agent.connections();
 let resolvers = agent.resolvers();
+let jar = agent.cookies();
 
 agent.prefetch_dns("api.example").await?;
 agent.preconnect("https://api.example").await?;
    @@ -334,8 +335,8 @@

    Ecosystem types

    Request in Request - fetch and Request::new take a URL, a Request, or an - http::Request<B> + fetch and Request::new take an impl TryInto<Url>, a + Request, or an http::Request<B> Response out diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index b019178..f143cd9 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -36,6 +36,15 @@ Faith proxies by default, so unlike Node (where the same variable opts in), it a `SSLKEYLOGFILE` names a path to which TLS session keys are written, enabling decryption of captured traffic when debugging. +## What the Rust surface reads + +The vocabulary above is Node's because the Node surface answers to Node's conventions. +`web-faith` reads the part of it that belongs to the platform rather than to a JavaScript runtime (see [RUST](../rust/overview.md)): `SSL_CERT_FILE` and `SSL_CERT_DIR` with the same OpenSSL semantics and the same per-platform reach, `SSLKEYLOGFILE`, and the proxy variables alongside the operating system's own proxy settings. +The `NODE_`-prefixed variables belong to the Node surface alone. +A Rust caller extends the trust store through `tls.extraRoots` (see [TLS](../agent/tls.md)) and keeps certificate validation on, that being what `NODE_TLS_REJECT_UNAUTHORIZED` exists to relax for a compatibility the Rust surface does not owe. + +Both surfaces read their variables once at construction. + ## Variables with nothing to control `NODE_USE_SYSTEM_CA` is ignored because the platform trust store is Faith's only default source of roots; there is no bundled set to toggle away from. diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md index 32370b7..7b05be8 100644 --- a/.workhorse/specs/rust/client-api.md +++ b/.workhorse/specs/rust/client-api.md @@ -5,14 +5,14 @@ id: RSAPI # The Rust client API `web-faith` has one noun and one verb: an `Agent` owns the connection pool, resolver, cookie jar, HTTP cache, and HTTP/3 knowledge, and `fetch` is how every request goes out. -The surface is fetch-flavoured rather than a transcription of the JavaScript API: it keeps fetch's vocabulary and its browser-like defaults, and it speaks the Rust ecosystem's types wherever one exists for the job. +The surface is fetch-flavoured rather than a transcription of the JavaScript API: it keeps fetch's vocabulary and the defaults the Node surface has, and it speaks the Rust ecosystem's types wherever one exists for the job. The agent's own behaviour, its options, and its lifecycle are specified in [AGENT](../agent/overview.md) and the specs beneath it; this spec covers the shape a Rust caller sees. ## Agent `Agent::new()` constructs an agent with default options, and `Agent::builder()` returns a builder whose methods mirror the option groups in [AGENT](../agent/overview.md), ending in `build()`. Nested option groups are nested builders rather than structs of optional fields, so a group left alone is absent from the call rather than spelled out as absent. -Construction validates options and reports the errors named in [AGENT](../agent/overview.md). +Construction validates options and reports the errors named in [AGENT](../agent/overview.md), and reads the environment variables [ENV](../environment/variables.md) names for this surface. An agent is cheap to clone and every clone names the same underlying agent, so cloning one is how a request gets an agent to run on rather than a way to get a second pool. Because clones share, `close()` acts on the agent itself and every handle to it sees the result. @@ -21,19 +21,29 @@ The agent captures what a request needs at the moment the request is issued, whi `network_changed()`, `stats()`, `connections()`, `resolvers()`, `prefetch_dns()`, and `preconnect()` act on a live agent as their counterparts do on the Node surface. +`cookies()` reaches the agent's jar, returning the `web-faith-cookies` jar itself rather than wrapping it in per-cookie methods, so a caller inserts and reads cookies through the same type that crate documents (see [COOK](../agent/cookies.md)). +An agent with no jar has nothing to hand back and says so in the return type, which is where the Node surface's null-returning reads land in Rust. +The jar outlives `close()` and stays readable from a closed agent, as [AGENT](../agent/overview.md) requires. + +`USER_AGENT` is exported so a caller can prepend its own product token to Faith's default. +The versions it embeds are not exported alongside it, a Rust caller already having its own package metadata to read them from. + ## Making a request -`agent.fetch(target)` returns a request builder, where `target` is a URL, a `Request`, or an `http::Request`. +`agent.fetch(target)` returns a fetch builder. +A target is anything that converts into a `url::Url` — a `Url`, a `&str`, a `String` — or a `Request`, or an `http::Request`. The builder implements `IntoFuture`, so awaiting it sends the request: a bare call awaits directly, and a configured one awaits after its options. There is no separate send step, and a builder that is dropped without being awaited sends nothing, so the builder is marked `#[must_use]`. Builder methods cover the per-request options in [REQ](../fetch/request.md), [CANCEL](../fetch/cancellation-and-timeouts.md), [SRI](../fetch/integrity.md), [ENC](../fetch/content-encoding.md), and [CACHE](../cache/http-cache.md), and set the method, headers, and body. -`Request::new(target)` takes the same three kinds of target and returns the same builder, which `build()` resolves into a `Request` rather than sending it. -A `Request` is inert, and passing one to `fetch` returns a builder again, so a request can be prepared once and adjusted at each call site or sent unchanged on more than one agent. +`Request::new(target)` takes the same kinds of target and returns a request builder, which `build()` resolves into a `Request` rather than sending it. +A `Request` is inert, and passing one to `fetch` returns a fetch builder, so a request can be prepared once and adjusted at each call site or sent unchanged on more than one agent. `try_clone()` copies a request when its body allows it and reports that it cannot when the body is a stream, a stream being consumable once. -A builder that carries no agent cannot be awaited, and the compiler refuses it rather than the call failing when it runs. +The two builders carry the same option-setting methods, so a call reads the same way whichever one it is written against, and they are distinct types because their terminal steps differ. +A fetch builder is awaited and has no `build()`; a request builder is built and cannot be awaited, so an attempt to send a request that has no agent to send it on is refused by the compiler rather than surfacing when the code runs. +`try_clone()` belongs to `Request` alone, and a fetch builder is not a target, so `fetch` calls do not nest. ## Layering options @@ -69,7 +79,13 @@ URLs are `url::Url`, which is what the fetch standard's parsing rules describe a Setters accept anything that converts into the canonical type, so a string literal works where JavaScript would pass a string and a typed value works where a caller already holds one. A conversion that fails is held until the builder resolves and surfaces there: at `build()` for a request, and at the await for a fetch. -Either way it reports the invalid-header or invalid-method error naming the offender, as [REQ](../fetch/request.md) requires. +Either way it reports the error naming the offender — `InvalidHeader`, `InvalidMethod`, or `InvalidUrl` for a target that does not parse — as [REQ](../fetch/request.md) and [ERR](../errors/errors.md) require. +Holding the failure is what lets a target be given as a string: an unparseable one is reported where the request is resolved rather than by a call that cannot fail. + +## What a disabled component removes + +A component's Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that component present: no `integrity()` without integrity, no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). +Code written against a component that is not built fails to compile rather than compiling into a call that does nothing, so a build reports what it does not carry at the point the caller asks for it. ## Errors and cancellation diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md index 1f777bd..20d63a5 100644 --- a/.workhorse/specs/rust/overview.md +++ b/.workhorse/specs/rust/overview.md @@ -18,7 +18,7 @@ Six component crates are published alongside it, each one useful to a caller who - `web-faith-cookies` is the cookie jar, with the storage, matching, and eviction rules in [COOK](../agent/cookies.md). - `web-faith-dns` is the resolver, its cache, the discovery ladder, the `HTTPS` record query, and Happy Eyeballs, as in [DNS](../agent/dns.md). -- `web-faith-conn-tracker` reads live connection state from the operating system, as in [OBS](../agent/observability.md). +- `web-faith-conn-tracker` reads live per-connection statistics from the operating system, which is the `connections()` view in [OBS](../agent/observability.md); the cumulative counters and the resolver listing that spec also covers belong to `web-faith` and `web-faith-dns` respectively. - `web-faith-alt-svc` is the Alt-Svc store and the HTTP/3 upgrade machinery, as in [H3UP](../http3/upgrade.md) and [PROBE](../http3/probing.md). - `web-faith-encoding` is content coding for request and response bodies, as in [ENC](../fetch/content-encoding.md). - `web-faith-integrity` is Subresource Integrity parsing and verification, as in [SRI](../fetch/integrity.md). @@ -39,11 +39,20 @@ Where a component needs a shape from the layer above, it takes it as a generic o Each component crate carries its own documentation, and its examples run against that crate alone. +## Subsystems that stay with the client + +QUIC and TLS are not component crates. +Faith reaches both through reqwest, which carries the HTTP/3 implementation Faith uses and offers the choice of TLS backend, so the swapping available there is the one the ecosystem already provides rather than one Faith builds above it. +`web-faith` passes those choices through as features of its own: HTTP/3 has such a feature already, and the TLS backend is selected the same way, with aws-lc-rs the default and ring the alternative. +What the two subsystems do is specified in [QUIC](../http3/transport.md) and [TLS](../agent/tls.md). + ## Choosing what is built Cargo features are how a subsystem is included or left out, so a build that has no use for a piece does not carry it. -Each of the six components has a feature on `web-faith` named for it, and the default set turns on the ones that make the client behave like a browser. -Turning a component's feature off drops the dependency and the behaviour it provides, and the client continues to work without it. +Each of the six components has a feature on `web-faith` named for it, and every one of them is on by default, so a caller who reaches for the crate without thinking about features gets the whole client. +Turning a component's feature off drops the dependency and the code that reaches for it, and the client continues to work without it; it also removes the parts of the API that only mean something with the component present, as in [RSAPI](client-api.md). + +A feature decides what is compiled in rather than what is switched on at run time, and the two need not agree: the cookies feature is on by default while the jar itself stays off until an agent asks for it (see [COOK](../agent/cookies.md)). Features are the whole of the swapping mechanism: a caller chooses among the implementations Faith builds rather than supplying one. From eb4bc01b42596cd62db135dbc6365356841ad7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 18 Aug 2026 15:50:08 +1200 Subject: [PATCH 08/61] S1: update --- .workhorse/plans/s1/plan.md | 95 +++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .workhorse/plans/s1/plan.md diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md new file mode 100644 index 0000000..4fc8220 --- /dev/null +++ b/.workhorse/plans/s1/plan.md @@ -0,0 +1,95 @@ +# S1 — Expose a Rust API and publish to crates.io + +Restructure the single `faith` cdylib into a Cargo workspace: a browser-shaped Rust client +`web-faith`, six standalone component crates beneath it, and a thin `web-faith-napi` binding that +ships as `@passcod/faith`. Then publish to crates.io. Target architecture is specified in +[RUST](../../specs/rust/overview.md) and [RSAPI](../../specs/rust/client-api.md). + +## Scope reality + +This is a multi-week, 8-crate restructure of ~11,500 lines, not a single focused change. The +work almost certainly wants to become a **card breakdown** (one card per crate extraction plus a +publishing card), because a plan of this size does not survive as one green PR on `s1` and, per the +workspace norm, only breakdown entries survive a merge as real cards. The steps below are the build +order whichever way we sequence it — either as the checklist for a single long-lived branch, or as +the spine for the breakdown. + +**Sequencing decision needed from the user before grinding the tree** (see the chat message). + +## What the discovery turned up + +Facts that shape the order and difficulty: + +- **The error type is napi-coupled at the base.** `src/error.rs` derives `#[napi(string_enum)]` on + `FaithErrorKind` and holds napi conversions. `integrity` already depends on it. Splitting a + pure-Rust error core from the napi conversion layer is a prerequisite for every component that + reports errors, and for the client itself. This is the load-bearing first move. +- **Component modules are mostly napi-free already.** `alt_svc`, `body`, `cookies`, `dns`, + `encoding`, `integrity`, `retry` have zero `#[napi]`. The coupling concentrates in `agent.rs` + (52), `response.rs` (40), `options.rs` (17), `error.rs` (13), `stream_body.rs` (11), `fetch.rs`. +- **Internal component coupling is small and matches the spec's allowed shape:** `integrity → error`, + `encoding → body::DynStream`, `alt_svc → timing::HeadersStamp` and `alt_svc → dns` (the spec + explicitly allows `web-faith-alt-svc → web-faith-dns`). `cookies` and `dns` have no internal deps. +- **`cookies` is bound to reqwest.** It implements `reqwest::cookie::CookieStore` and takes + `reqwest::Url`. A standalone `web-faith-cookies` should speak `url::Url` and put the reqwest + `CookieStore` impl behind a `reqwest` feature (or move the adapter into the client). +- **The napi build resists a naive relocation.** `build.rs` reads `Cargo.lock` by the relative path + `"Cargo.lock"`; under a workspace the lock is at the root, so this must become + `CARGO_MANIFEST_DIR`/workspace-root aware. `napi build --platform` reads `package.json`'s `napi` + config and builds the crate in cwd; moving the crate means teaching the napi CLI where the crate + is and keeping generated `index.js`/`index.d.ts` at the repo root (the spec requires them there). + `.cargo/config.toml` (`reqwest_unstable`, cross linkers) applies workspace-wide and can stay at root. + +## Target crate family + +- `web-faith` — client: agent, request/response, `fetch`, layering. Depends on the six components. +- `web-faith-cookies` — the jar ([COOK](../../specs/agent/cookies.md)). +- `web-faith-dns` — resolver, cache, discovery ladder, HTTPS record, Happy Eyeballs ([DNS](../../specs/agent/dns.md)). +- `web-faith-conn-tracker` — live per-connection stats from the OS ([OBS](../../specs/agent/observability.md)). +- `web-faith-alt-svc` — Alt-Svc store + HTTP/3 upgrade/probing ([H3UP](../../specs/http3/upgrade.md), [PROBE](../../specs/http3/probing.md)); may depend on `web-faith-dns`. +- `web-faith-encoding` — request/response content coding ([ENC](../../specs/fetch/content-encoding.md)). +- `web-faith-integrity` — SRI parse + verify ([SRI](../../specs/fetch/integrity.md)). +- `web-faith-napi` — the only crate with napi types; ships as `@passcod/faith`. + +QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring alternative), not crates. + +## Build order (each step ends green: `cargo build` + `cargo test` + napi `npm run build`) + +- [ ] **0. Workspace scaffold.** Root `[workspace]` with shared `[workspace.package]` + (licence, repository, authors, edition, `rust-version = "1.96"`) and `[workspace.dependencies]`. + Move the current crate to `crates/web-faith-napi`. Fix `build.rs` `Cargo.lock` path. Make + `napi build` target the relocated crate and keep `index.js`/`index.d.ts` at repo root. Verify the + npm build still produces a working `.node`. No behaviour change. +- [ ] **1. Error core split.** Pure-Rust `FaithError`/`FaithErrorKind` (no napi) reachable by every + crate; napi conversions live only in `web-faith-napi`. `ERROR_CODES` still generated from the one + source ([ERR](../../specs/errors/errors.md)). Decide where the shared error core lives (likely in + `web-faith`, with components naming their own error types that the client converts — per + [RUST](../../specs/rust/overview.md) "A component crate stands alone"). +- [ ] **2. Extract `web-faith-integrity`** — own error type, own docs, `cargo test -p web-faith-integrity` with no JS runtime. +- [ ] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). +- [ ] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. +- [ ] **5. Extract `web-faith-dns`.** +- [ ] **6. Extract `web-faith-conn-tracker`** (Linux/macOS/Windows submodules). +- [ ] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. +- [ ] **8. Stand up `web-faith`** — move agent/request/response/fetch/options here as a pure-Rust + client; component crates converted into it at the boundary. Reduce `web-faith-napi` to the binding + over `web-faith`. +- [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): + `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder + (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, + `http_body::Body` response + `Into`, feature-gated API surface, Tokio, drop-cancels. +- [ ] **10. Feature wiring** — one default-on feature per component on `web-faith`; disabling one + drops the dep, the code, and the API surface it gates (compile error at the call site, not a no-op). +- [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; + client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. +- [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, + MSRV 1.96 declared in every published crate and exercised in CI alongside stable, independent + versioning from `1.0.0`. Measure CI cost before adding jobs (see project memory). +- [ ] **13. First publish** to crates.io: the six components, then `web-faith`; `@passcod/faith` + continues from npm via `web-faith-napi`. + +## Verification discipline + +Every step must leave `cargo build`, `cargo test`, and the napi `npm run build` green +(tests via `HTTPBIN_URL=http://localhost:8888`, `NODE_ENV=development` for `npm install`). A component +crate's separateness is only proven when `cargo test -p ` passes with no JS runtime present. From ae82b63d4ebef5efbf2d207946e7c3634bfcefc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:17:34 +1200 Subject: [PATCH 09/61] S1: scaffold the Cargo workspace Move the crate to crates/web-faith-napi and turn the root manifest into a workspace, so the component crates and the Rust client have somewhere to land. Shared package metadata (version, edition, MSRV 1.96, licence, repository, authors) and the dependency versions move to the root for the crates to inherit. The lib keeps the name `faith` so the built artifact stays libfaith.so, which the release workflow's cross-compile steps copy by name. build.rs walked to "Cargo.lock" relative to the crate; under a workspace the lock file sits at the root, so it now searches the ancestor directories. The benchmark HTTP/3 server is excluded rather than made a member: it keeps its own lockfile on purpose so the quinn/h3 stack stays out of this graph. No behaviour change; the generated index.js and index.d.ts are unchanged. --- .github/workflows/publish.yml | 4 +- Cargo.lock | 80 +++++++++---------- Cargo.toml | 45 ++++++----- crates/web-faith-napi/Cargo.toml | 62 ++++++++++++++ build.rs => crates/web-faith-napi/build.rs | 20 +++-- {src => crates/web-faith-napi/src}/agent.rs | 0 {src => crates/web-faith-napi/src}/alt_svc.rs | 0 .../web-faith-napi/src}/async_task.rs | 0 {src => crates/web-faith-napi/src}/body.rs | 0 .../web-faith-napi/src}/conn_tracker.rs | 0 .../web-faith-napi/src}/conn_tracker/linux.rs | 0 .../web-faith-napi/src}/conn_tracker/macos.rs | 0 .../src}/conn_tracker/windows.rs | 0 {src => crates/web-faith-napi/src}/cookies.rs | 0 {src => crates/web-faith-napi/src}/dns.rs | 0 .../web-faith-napi/src}/encoding.rs | 0 {src => crates/web-faith-napi/src}/error.rs | 0 {src => crates/web-faith-napi/src}/fetch.rs | 0 .../web-faith-napi/src}/integrity.rs | 0 {src => crates/web-faith-napi/src}/lib.rs | 0 {src => crates/web-faith-napi/src}/options.rs | 0 .../web-faith-napi/src}/response.rs | 0 {src => crates/web-faith-napi/src}/retry.rs | 0 .../web-faith-napi/src}/stream_body.rs | 0 {src => crates/web-faith-napi/src}/timing.rs | 0 package.json | 4 +- 26 files changed, 145 insertions(+), 70 deletions(-) create mode 100644 crates/web-faith-napi/Cargo.toml rename build.rs => crates/web-faith-napi/build.rs (50%) rename {src => crates/web-faith-napi/src}/agent.rs (100%) rename {src => crates/web-faith-napi/src}/alt_svc.rs (100%) rename {src => crates/web-faith-napi/src}/async_task.rs (100%) rename {src => crates/web-faith-napi/src}/body.rs (100%) rename {src => crates/web-faith-napi/src}/conn_tracker.rs (100%) rename {src => crates/web-faith-napi/src}/conn_tracker/linux.rs (100%) rename {src => crates/web-faith-napi/src}/conn_tracker/macos.rs (100%) rename {src => crates/web-faith-napi/src}/conn_tracker/windows.rs (100%) rename {src => crates/web-faith-napi/src}/cookies.rs (100%) rename {src => crates/web-faith-napi/src}/dns.rs (100%) rename {src => crates/web-faith-napi/src}/encoding.rs (100%) rename {src => crates/web-faith-napi/src}/error.rs (100%) rename {src => crates/web-faith-napi/src}/fetch.rs (100%) rename {src => crates/web-faith-napi/src}/integrity.rs (100%) rename {src => crates/web-faith-napi/src}/lib.rs (100%) rename {src => crates/web-faith-napi/src}/options.rs (100%) rename {src => crates/web-faith-napi/src}/response.rs (100%) rename {src => crates/web-faith-napi/src}/retry.rs (100%) rename {src => crates/web-faith-napi/src}/stream_body.rs (100%) rename {src => crates/web-faith-napi/src}/timing.rs (100%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cd566b8..f84f934 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -83,10 +83,10 @@ jobs: # cargo-zigbuild is invoked directly rather than via `napi build -x` # because the napi CLI can't pass a glibc version suffix; we target # a glibc 2.35 floor. RUSTFLAGS for the same reason as musl below. - build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild --target aarch64-unknown-linux-gnu.2.35 --release && cp target/aarch64-unknown-linux-gnu/release/libfaith.so faith.linux-arm64-gnu.node + build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild -p web-faith-napi --target aarch64-unknown-linux-gnu.2.35 --release && cp target/aarch64-unknown-linux-gnu/release/libfaith.so faith.linux-arm64-gnu.node - host: ubuntu-latest target: armv7-unknown-linux-gnueabihf - build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild --target armv7-unknown-linux-gnueabihf.2.35 --release && cp target/armv7-unknown-linux-gnueabihf/release/libfaith.so faith.linux-arm-gnueabihf.node + build: RUSTFLAGS="--cfg reqwest_unstable" cargo zigbuild -p web-faith-napi --target armv7-unknown-linux-gnueabihf.2.35 --release && cp target/armv7-unknown-linux-gnueabihf/release/libfaith.so faith.linux-arm-gnueabihf.node - host: ubuntu-latest target: aarch64-linux-android build: npm run build -- --target aarch64-linux-android diff --git a/Cargo.lock b/Cargo.lock index 46a8716..ad4249e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,46 +577,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "faith" -version = "0.7.0" -dependencies = [ - "async-compression", - "async-stream", - "async-trait", - "bytes", - "cookie", - "cookie_store", - "futures", - "hickory-resolver", - "http", - "http-body-util", - "http-cache-reqwest", - "hyper", - "hyper-util", - "libc", - "moka", - "napi", - "napi-build", - "napi-derive", - "netlink-packet-core", - "netlink-packet-sock-diag", - "netlink-sys", - "reqwest", - "reqwest-middleware", - "serde", - "serde_json", - "ssri", - "stream_shared", - "strum", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "url", - "windows", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -2834,6 +2794,46 @@ dependencies = [ "web-sys", ] +[[package]] +name = "web-faith-napi" +version = "0.7.0" +dependencies = [ + "async-compression", + "async-stream", + "async-trait", + "bytes", + "cookie", + "cookie_store", + "futures", + "hickory-resolver", + "http", + "http-body-util", + "http-cache-reqwest", + "hyper", + "hyper-util", + "libc", + "moka", + "napi", + "napi-build", + "napi-derive", + "netlink-packet-core", + "netlink-packet-sock-diag", + "netlink-sys", + "reqwest", + "reqwest-middleware", + "serde", + "serde_json", + "ssri", + "stream_shared", + "strum", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "url", + "windows", +] + [[package]] name = "web-sys" version = "0.3.103" diff --git a/Cargo.toml b/Cargo.toml index c15104d..05b131b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,19 @@ -[package] -name = "faith" +[workspace] +members = ["crates/*"] +# The benchmark HTTP/3 server keeps its own manifest and lockfile on purpose, so +# the quinn/h3 stack stays out of this workspace's dependency graph. +exclude = ["bench/h3-server"] +resolver = "3" + +[workspace.package] version = "0.7.0" edition = "2024" -description = "Faith: a Rust-powered JS fetch" -publish = false - -[lib] -crate-type = ["cdylib"] +rust-version = "1.96" +license = "Apache-2.0 OR MIT" +repository = "https://github.com/passcod/faith" +authors = ["Félix Saparelli "] -[dependencies] +[workspace.dependencies] async-compression = { version = "0.4", default-features = false, features = [ "tokio", "gzip", @@ -33,10 +38,19 @@ moka = { version = "0.12", features = ["sync"] } http = "1.4.0" http-body-util = "0.1.3" hyper = "1.8.1" -http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-cacache", "manager-moka"] } +http-cache-reqwest = { version = "1.0.0-alpha.6", features = [ + "manager-cacache", + "manager-moka", +] } hyper-util = { version = "0.1", features = ["client-legacy", "tokio"] } libc = "0.2.179" -napi = { version = "3.7.0", features = ["napi9", "serde-json", "tokio_rt", "web_stream"] } +napi = { version = "3.7.0", features = [ + "napi9", + "serde-json", + "tokio_rt", + "web_stream", +] } +napi-build = "2.3.1" napi-derive = "3.4.0" reqwest = { version = "0.13.4", default-features = false, features = [ "cookies", @@ -58,21 +72,10 @@ tokio-stream = "0.1.16" time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" - -[target.'cfg(target_os = "linux")'.dependencies] netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } netlink-sys = "0.8.7" - -[target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.62.2", features = [ "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", ] } - -[build-dependencies] -napi-build = "2.3.1" - -[features] -default = ["http3"] -http3 = ["reqwest/http3"] diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml new file mode 100644 index 0000000..ed548c8 --- /dev/null +++ b/crates/web-faith-napi/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "web-faith-napi" +description = "Faith: a Rust-powered JS fetch (Node.js binding)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +publish = false + +[lib] +# Kept as `faith` so the built artifact stays `libfaith.so`/`faith.dll`, which the +# release workflow's cross-compile steps copy by name. +name = "faith" +crate-type = ["cdylib"] + +[dependencies] +async-compression.workspace = true +async-stream.workspace = true +async-trait.workspace = true +bytes.workspace = true +cookie.workspace = true +cookie_store.workspace = true +futures.workspace = true +hickory-resolver.workspace = true +moka.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper.workspace = true +http-cache-reqwest.workspace = true +hyper-util.workspace = true +libc.workspace = true +napi.workspace = true +napi-derive.workspace = true +reqwest.workspace = true +reqwest-middleware.workspace = true +serde.workspace = true +serde_json.workspace = true +ssri.workspace = true +stream_shared.workspace = true +strum.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +time.workspace = true +tokio-util.workspace = true +url.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +netlink-packet-core.workspace = true +netlink-packet-sock-diag.workspace = true +netlink-sys.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +windows.workspace = true + +[build-dependencies] +napi-build.workspace = true + +[features] +default = ["http3"] +http3 = ["reqwest/http3"] diff --git a/build.rs b/crates/web-faith-napi/build.rs similarity index 50% rename from build.rs rename to crates/web-faith-napi/build.rs index caebed4..e2f4410 100644 --- a/build.rs +++ b/crates/web-faith-napi/build.rs @@ -1,16 +1,26 @@ -use std::fs; +use std::{env, fs, path::PathBuf}; fn main() { napi_build::setup(); // Parse reqwest version from Cargo.lock - let reqwest_version = extract_reqwest_version().unwrap(); + let lock_path = find_cargo_lock().expect("Cargo.lock not found in any ancestor directory"); + let reqwest_version = extract_reqwest_version(&lock_path).unwrap(); println!("cargo:rustc-env=REQWEST_VERSION={}", reqwest_version); - println!("cargo:rerun-if-changed=Cargo.lock"); + println!("cargo:rerun-if-changed={}", lock_path.display()); } -fn extract_reqwest_version() -> Option { - let cargo_lock = fs::read_to_string("Cargo.lock").ok()?; +/// The lock file lives at the workspace root, which is an ancestor of this crate. +fn find_cargo_lock() -> Option { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR")?); + manifest_dir + .ancestors() + .map(|dir| dir.join("Cargo.lock")) + .find(|lock| lock.is_file()) +} + +fn extract_reqwest_version(lock_path: &PathBuf) -> Option { + let cargo_lock = fs::read_to_string(lock_path).ok()?; // Find the reqwest package entry in Cargo.lock for line in cargo_lock.lines() { diff --git a/src/agent.rs b/crates/web-faith-napi/src/agent.rs similarity index 100% rename from src/agent.rs rename to crates/web-faith-napi/src/agent.rs diff --git a/src/alt_svc.rs b/crates/web-faith-napi/src/alt_svc.rs similarity index 100% rename from src/alt_svc.rs rename to crates/web-faith-napi/src/alt_svc.rs diff --git a/src/async_task.rs b/crates/web-faith-napi/src/async_task.rs similarity index 100% rename from src/async_task.rs rename to crates/web-faith-napi/src/async_task.rs diff --git a/src/body.rs b/crates/web-faith-napi/src/body.rs similarity index 100% rename from src/body.rs rename to crates/web-faith-napi/src/body.rs diff --git a/src/conn_tracker.rs b/crates/web-faith-napi/src/conn_tracker.rs similarity index 100% rename from src/conn_tracker.rs rename to crates/web-faith-napi/src/conn_tracker.rs diff --git a/src/conn_tracker/linux.rs b/crates/web-faith-napi/src/conn_tracker/linux.rs similarity index 100% rename from src/conn_tracker/linux.rs rename to crates/web-faith-napi/src/conn_tracker/linux.rs diff --git a/src/conn_tracker/macos.rs b/crates/web-faith-napi/src/conn_tracker/macos.rs similarity index 100% rename from src/conn_tracker/macos.rs rename to crates/web-faith-napi/src/conn_tracker/macos.rs diff --git a/src/conn_tracker/windows.rs b/crates/web-faith-napi/src/conn_tracker/windows.rs similarity index 100% rename from src/conn_tracker/windows.rs rename to crates/web-faith-napi/src/conn_tracker/windows.rs diff --git a/src/cookies.rs b/crates/web-faith-napi/src/cookies.rs similarity index 100% rename from src/cookies.rs rename to crates/web-faith-napi/src/cookies.rs diff --git a/src/dns.rs b/crates/web-faith-napi/src/dns.rs similarity index 100% rename from src/dns.rs rename to crates/web-faith-napi/src/dns.rs diff --git a/src/encoding.rs b/crates/web-faith-napi/src/encoding.rs similarity index 100% rename from src/encoding.rs rename to crates/web-faith-napi/src/encoding.rs diff --git a/src/error.rs b/crates/web-faith-napi/src/error.rs similarity index 100% rename from src/error.rs rename to crates/web-faith-napi/src/error.rs diff --git a/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs similarity index 100% rename from src/fetch.rs rename to crates/web-faith-napi/src/fetch.rs diff --git a/src/integrity.rs b/crates/web-faith-napi/src/integrity.rs similarity index 100% rename from src/integrity.rs rename to crates/web-faith-napi/src/integrity.rs diff --git a/src/lib.rs b/crates/web-faith-napi/src/lib.rs similarity index 100% rename from src/lib.rs rename to crates/web-faith-napi/src/lib.rs diff --git a/src/options.rs b/crates/web-faith-napi/src/options.rs similarity index 100% rename from src/options.rs rename to crates/web-faith-napi/src/options.rs diff --git a/src/response.rs b/crates/web-faith-napi/src/response.rs similarity index 100% rename from src/response.rs rename to crates/web-faith-napi/src/response.rs diff --git a/src/retry.rs b/crates/web-faith-napi/src/retry.rs similarity index 100% rename from src/retry.rs rename to crates/web-faith-napi/src/retry.rs diff --git a/src/stream_body.rs b/crates/web-faith-napi/src/stream_body.rs similarity index 100% rename from src/stream_body.rs rename to crates/web-faith-napi/src/stream_body.rs diff --git a/src/timing.rs b/crates/web-faith-napi/src/timing.rs similarity index 100% rename from src/timing.rs rename to crates/web-faith-napi/src/timing.rs diff --git a/package.json b/package.json index 0e4dbe8..c32fd22 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ ] }, "scripts": { - "build": "napi build --release --platform", - "build:debug": "napi build --platform", + "build": "napi build --release --platform --manifest-path crates/web-faith-napi/Cargo.toml --output-dir .", + "build:debug": "napi build --platform --manifest-path crates/web-faith-napi/Cargo.toml --output-dir .", "test": "npm run build && cargo test && npm run test:only", "test:only": "tape test/*.test.js", "test:integration": "tape test/integration/*.test.js", From 196df43e3ba9c6e60da89abdda304447082badca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:34:59 +1200 Subject: [PATCH 10/61] S1: split the error core out of the napi layer FaithError and FaithErrorKind were defined on a napi string_enum and carried napi conversions, so every module that reported an error was tied to a JavaScript runtime. The kinds and the error now live in web-faith, which builds and documents with no napi in its dependency graph, and the Node surface keeps what is genuinely its own: the mapping from a kind to a JavaScript error class, and turning an error into a value V8 can carry. The napi conversions arrive as an extension trait rather than inherent methods, the error no longer being this crate's type to add methods to. errorCodes() reads the codes from the kinds themselves, so the exported ERROR_CODES map still cannot drift from the errors. This drops FaithErrorKind from the native binding: it was emitted only because the enum carried napi's attribute, and the package's export map admits nothing but the wrapper, so no caller could reach it. The documented surface is wrapper.js's ERROR_CODES, which is unchanged, still holds all 22 codes, and is still checked against errorCodes() by test/error-codes.test.js. --- Cargo.lock | 10 + Cargo.toml | 1 + crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/agent.rs | 2 +- crates/web-faith-napi/src/async_task.rs | 2 +- crates/web-faith-napi/src/error.rs | 254 +++++------------------- crates/web-faith-napi/src/response.rs | 2 +- crates/web-faith/Cargo.toml | 16 ++ crates/web-faith/src/error.rs | 223 +++++++++++++++++++++ crates/web-faith/src/lib.rs | 10 + index.d.ts | 67 ------- index.js | 1 - wrapper.d.ts | 2 +- 13 files changed, 317 insertions(+), 274 deletions(-) create mode 100644 crates/web-faith/Cargo.toml create mode 100644 crates/web-faith/src/error.rs create mode 100644 crates/web-faith/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ad4249e..fdbe404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2794,6 +2794,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "web-faith" +version = "0.7.0" +dependencies = [ + "reqwest", + "reqwest-middleware", + "strum", +] + [[package]] name = "web-faith-napi" version = "0.7.0" @@ -2831,6 +2840,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", + "web-faith", "windows", ] diff --git a/Cargo.toml b/Cargo.toml index 05b131b..2dbf22b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ tokio-stream = "0.1.16" time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" +web-faith = { version = "0.7.0", path = "crates/web-faith" } netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } netlink-sys = "0.8.7" diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index ed548c8..04c2606 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -45,6 +45,7 @@ tokio-stream.workspace = true time.workspace = true tokio-util.workspace = true url.workspace = true +web-faith.workspace = true [target.'cfg(target_os = "linux")'.dependencies] netlink-packet-core.workspace = true diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index ae615c6..23eeb52 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -40,7 +40,7 @@ use crate::{ FaithJar, }, dns::{DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains}, - error::{FaithError, FaithErrorKind}, + error::{FaithError, FaithErrorExt, FaithErrorKind}, options::{PRIORITY, RequestCacheMode}, retry::{DeadConnectionRetry, StaleAddressRetry}, }; diff --git a/crates/web-faith-napi/src/async_task.rs b/crates/web-faith-napi/src/async_task.rs index 69706bd..b2d6f0e 100644 --- a/crates/web-faith-napi/src/async_task.rs +++ b/crates/web-faith-napi/src/async_task.rs @@ -7,7 +7,7 @@ use napi::{ }; use serde_json; -use crate::error::FaithError; +use crate::error::{FaithError, FaithErrorExt}; #[derive(Clone, Debug)] pub struct Value(pub serde_json::Value); diff --git a/crates/web-faith-napi/src/error.rs b/crates/web-faith-napi/src/error.rs index d385d3b..e43cfb0 100644 --- a/crates/web-faith-napi/src/error.rs +++ b/crates/web-faith-napi/src/error.rs @@ -1,12 +1,17 @@ -use std::{ - error::Error, - fmt::{Debug, Display}, -}; - use napi::bindgen_prelude::*; use napi_derive::napi; -use strum::{EnumIter, IntoEnumIterator}; +pub use web_faith::{FaithError, FaithErrorKind}; +#[derive(Debug, Clone, Copy)] +enum JsErrorType { + GenericError, + NamedError(&'static str), + SyntaxError, + TypeError, +} + +/// Which JavaScript error class a kind is thrown as. +/// /// Faith produces fine-grained errors, but maps them to a few javascript error types for fetch /// compatibility. The `.code` property on errors thrown from Faith is set to a stable name for each /// error kind, documented in this comprehensive mapping: @@ -47,130 +52,60 @@ use strum::{EnumIter, IntoEnumIterator}; /// /// Due to technical limitations, when reading a body stream, reads might fail, but that error /// will not have a `code` property. -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, EnumIter)] -pub enum FaithErrorKind { - Aborted, - AddressParse, - BodyStream, - Closed, - Config, - ContentLengthOverrun, - FileExists, - FileWrite, - IntegrityMismatch, - InvalidCompression, - InvalidHeader, - InvalidIntegrity, - InvalidMethod, - InvalidPath, - InvalidUrl, - JsonParse, - Network, - PemParse, - Redirect, - ResponseAlreadyDisturbed, - ResponseBodyNull, - Timeout, -} - -#[derive(Debug, Clone, Copy)] -enum JsErrorType { - GenericError, - NamedError(&'static str), - SyntaxError, - TypeError, -} - -impl FaithErrorKind { - fn default_message(self) -> &'static str { - match self { - Self::Aborted => "the request was aborted", - Self::AddressParse => "invalid IP address and/or port", - Self::BodyStream => "internal response body stream copy error", - Self::Closed => "the agent has been closed", - Self::Config => "invalid agent configuration", - Self::ContentLengthOverrun => "response body exceeded the advertised Content-Length", - Self::FileExists => "the destination file already exists", - Self::FileWrite => "could not write the destination file", - Self::IntegrityMismatch => "resource integrity check failed", - Self::InvalidCompression => "invalid request body compression", - Self::InvalidHeader => "invalid header name or value", - Self::InvalidIntegrity => "invalid integrity value", - Self::InvalidMethod => "invalid HTTP method", - Self::InvalidPath => "destination does not name a local path", - Self::InvalidUrl => "invalid URL", - Self::JsonParse => "invalid json in response body", - Self::Network => "network error", - Self::PemParse => "invalid client certificate or key", - Self::Redirect => "got a redirect", - Self::ResponseAlreadyDisturbed => "response body already disturbed", - Self::ResponseBodyNull => "response cannot carry a body to write", - Self::Timeout => "timed out", +fn js_type(kind: FaithErrorKind) -> JsErrorType { + use FaithErrorKind as K; + match kind { + K::BodyStream | K::Config | K::FileExists | K::FileWrite | K::IntegrityMismatch => { + JsErrorType::GenericError } - } - - fn js_type(self) -> JsErrorType { - match self { - Self::BodyStream - | Self::Config - | Self::FileExists - | Self::FileWrite - | Self::IntegrityMismatch => JsErrorType::GenericError, - Self::Aborted | Self::Timeout => JsErrorType::NamedError("AbortError"), - Self::Network | Self::Redirect | Self::ContentLengthOverrun => { - JsErrorType::NamedError("NetworkError") - } - Self::AddressParse | Self::InvalidIntegrity | Self::JsonParse | Self::PemParse => { - JsErrorType::SyntaxError - } - Self::Closed - | Self::InvalidCompression - | Self::InvalidHeader - | Self::InvalidMethod - | Self::InvalidPath - | Self::InvalidUrl - | Self::ResponseAlreadyDisturbed - | Self::ResponseBodyNull => JsErrorType::TypeError, + K::Aborted | K::Timeout => JsErrorType::NamedError("AbortError"), + K::Network | K::Redirect | K::ContentLengthOverrun => { + JsErrorType::NamedError("NetworkError") } - } -} - -impl From for FaithError { - fn from(kind: FaithErrorKind) -> Self { - Self { - kind, - message: None, + K::AddressParse | K::InvalidIntegrity | K::JsonParse | K::PemParse => { + JsErrorType::SyntaxError } + K::Closed + | K::InvalidCompression + | K::InvalidHeader + | K::InvalidMethod + | K::InvalidPath + | K::InvalidUrl + | K::ResponseAlreadyDisturbed + | K::ResponseBodyNull => JsErrorType::TypeError, } } -#[derive(Debug, Clone)] -pub struct FaithError { - pub kind: FaithErrorKind, - pub message: Option, +/// Throwing a [`FaithError`] into JavaScript. +/// +/// The error itself is the client's, and knows nothing of napi; turning one into a value V8 can +/// carry is this crate's business, which is why it arrives as an extension rather than as methods +/// on the error. +pub trait FaithErrorExt { + /// Convert to a napi error. + /// + /// This is explicit rather than a `From` impl so that it cannot happen by accident, losing the + /// error class and code that [`Self::into_js_error`] preserves. + fn into_napi(self) -> napi::Error; + + fn to_napi(&self) -> napi::Error; + + /// Whenever possible, prefer this so that the error types are correct. + fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env>; } -impl FaithError { - pub fn new(kind: FaithErrorKind, message: Option>) -> Self { - Self { - kind, - message: message.map(|m| m.into()), - } - } - - // we make this explicit instead of adding a From<> so that we can't accidentally do it - pub fn into_napi(self) -> napi::Error { +impl FaithErrorExt for FaithError { + fn into_napi(self) -> napi::Error { self.to_napi() } + fn to_napi(&self) -> napi::Error { napi::Error::new(napi::Status::GenericFailure, format!("{self}")) } - // whenever possible, we should prefer to use this so that the error types are correct - pub fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env> { - let code = format!("{:?}", self.kind); - let unk = match self.kind.js_type() { + fn into_js_error<'env>(self, env: &'env Env) -> Unknown<'env> { + let code = self.kind.code(); + let unk = match js_type(self.kind) { JsErrorType::TypeError => JsTypeError::from(self.into_napi()).into_unknown(*env), JsErrorType::SyntaxError => JsSyntaxError::from(self.into_napi()).into_unknown(*env), JsErrorType::GenericError => JsError::from(self.into_napi()).into_unknown(*env), @@ -199,92 +134,7 @@ impl FaithError { } } -/// Dig a [`FaithError`] back out of an error chain, if one is in there. -/// -/// The `error` redirect policy refuses a redirect by handing reqwest a [`FaithError`], which comes -/// back to us wrapped in an error of reqwest's own, so the kind we chose has to be recovered from -/// the source chain to survive as a `code`. Redirect failures reqwest raises on its own account -/// (exhausting the hop limit, an https-only downgrade) carry no [`FaithError`] and so fall through -/// to the generic mapping, which is what tells the two apart. -fn faith_kind_in_chain(err: &(dyn Error + 'static)) -> Option { - let mut source = err.source(); - while let Some(e) = source { - if let Some(faith) = e.downcast_ref::() { - return Some(faith.kind); - } - source = e.source(); - } - - None -} - -impl From for FaithError { - fn from(err: reqwest::Error) -> Self { - // Always include full error chain for debugging - let mut msg = format!("{err:?}"); - let mut source = err.source(); - while let Some(e) = source { - msg.push_str(&format!(" -> {e:?}")); - source = e.source(); - } - - if err.is_timeout() { - return FaithError::new(FaithErrorKind::Timeout, Some(msg)); - } - - // A redirect the agent's own policy refused carries the kind we handed reqwest; one reqwest - // raised on its own account stays a plain network error. - let kind = err - .is_redirect() - .then(|| faith_kind_in_chain(&err)) - .flatten() - .unwrap_or(FaithErrorKind::Network); - - FaithError::new(kind, Some(msg)) - } -} - -impl From for FaithError { - fn from(err: reqwest_middleware::Error) -> Self { - match err { - reqwest_middleware::Error::Middleware(err) => { - FaithError::new(FaithErrorKind::Network, Some(err.to_string())) - } - reqwest_middleware::Error::Reqwest(err) => err.into(), - } - } -} - -impl Error for FaithError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } - - fn description(&self) -> &str { - "description() is deprecated; use Display" - } - - fn cause(&self) -> Option<&dyn Error> { - self.source() - } -} - -impl Display for FaithError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{:?}: {}", - self.kind, - self.message - .as_deref() - .unwrap_or_else(|| self.kind.default_message()) - ) - } -} - #[napi] pub fn error_codes() -> Vec { - FaithErrorKind::iter() - .map(|kind| format!("{:?}", kind)) - .collect() + web_faith::error_codes() } diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index b19c898..37a7239 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -33,7 +33,7 @@ use crate::{ async_task::{Value, faith_promise}, body::{Body, BodyHolder, DynStream, drain_body_inner}, encoding::{Coding, decode_stream}, - error::{FaithError, FaithErrorKind}, + error::{FaithError, FaithErrorExt, FaithErrorKind}, integrity::{finish_integrity, integrity_checker, verify_integrity}, timing::{TimingBreakdown, TimingSlot}, }; diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml new file mode 100644 index 0000000..88f2e0c --- /dev/null +++ b/crates/web-faith/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "web-faith" +description = "A browser-shaped HTTP client: fetch semantics over a Rust network stack" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the client API is built out and the release tooling is in place. +publish = false + +[dependencies] +reqwest.workspace = true +reqwest-middleware.workspace = true +strum.workspace = true diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs new file mode 100644 index 0000000..f79c910 --- /dev/null +++ b/crates/web-faith/src/error.rs @@ -0,0 +1,223 @@ +use std::{ + error::Error, + fmt::{Debug, Display}, +}; + +use strum::{EnumIter, IntoEnumIterator}; + +/// The kind of a [`FaithError`], which is also the stable code the error reports. +/// +/// Callers match on the kind rather than on the message: the kind is the API, and the message is +/// for humans. Every kind here is reachable, each one naming a failure some request can produce. +/// +/// This is the one definition of the set, on either surface. The Node surface hands JavaScript the +/// codes through [`error_codes`], which reads them from here, so the exported `ERROR_CODES` map and +/// the errors themselves cannot drift apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)] +pub enum FaithErrorKind { + Aborted, + AddressParse, + BodyStream, + Closed, + Config, + ContentLengthOverrun, + FileExists, + FileWrite, + IntegrityMismatch, + InvalidCompression, + InvalidHeader, + InvalidIntegrity, + InvalidMethod, + InvalidPath, + InvalidUrl, + JsonParse, + Network, + PemParse, + Redirect, + ResponseAlreadyDisturbed, + ResponseBodyNull, + Timeout, +} + +impl FaithErrorKind { + /// The stable name callers match on. + pub fn code(self) -> String { + format!("{self:?}") + } + + pub(crate) fn default_message(self) -> &'static str { + match self { + Self::Aborted => "the request was aborted", + Self::AddressParse => "invalid IP address and/or port", + Self::BodyStream => "internal response body stream copy error", + Self::Closed => "the agent has been closed", + Self::Config => "invalid agent configuration", + Self::ContentLengthOverrun => "response body exceeded the advertised Content-Length", + Self::FileExists => "the destination file already exists", + Self::FileWrite => "could not write the destination file", + Self::IntegrityMismatch => "resource integrity check failed", + Self::InvalidCompression => "invalid request body compression", + Self::InvalidHeader => "invalid header name or value", + Self::InvalidIntegrity => "invalid integrity value", + Self::InvalidMethod => "invalid HTTP method", + Self::InvalidPath => "destination does not name a local path", + Self::InvalidUrl => "invalid URL", + Self::JsonParse => "invalid json in response body", + Self::Network => "network error", + Self::PemParse => "invalid client certificate or key", + Self::Redirect => "got a redirect", + Self::ResponseAlreadyDisturbed => "response body already disturbed", + Self::ResponseBodyNull => "response cannot carry a body to write", + Self::Timeout => "timed out", + } + } +} + +/// Every error code the library reports, in declaration order. +/// +/// The Node surface exports this as `ERROR_CODES`; generating it from the kinds themselves is what +/// keeps the exported map and the errors from drifting. +pub fn error_codes() -> Vec { + FaithErrorKind::iter().map(FaithErrorKind::code).collect() +} + +#[derive(Debug, Clone)] +pub struct FaithError { + pub kind: FaithErrorKind, + pub message: Option, +} + +impl FaithError { + pub fn new(kind: FaithErrorKind, message: Option>) -> Self { + Self { + kind, + message: message.map(|m| m.into()), + } + } +} + +impl From for FaithError { + fn from(kind: FaithErrorKind) -> Self { + Self { + kind, + message: None, + } + } +} + +/// Dig a [`FaithError`] back out of an error chain, if one is in there. +/// +/// The `error` redirect policy refuses a redirect by handing reqwest a [`FaithError`], which comes +/// back to us wrapped in an error of reqwest's own, so the kind we chose has to be recovered from +/// the source chain to survive as a `code`. Redirect failures reqwest raises on its own account +/// (exhausting the hop limit, an https-only downgrade) carry no [`FaithError`] and so fall through +/// to the generic mapping, which is what tells the two apart. +fn faith_kind_in_chain(err: &(dyn Error + 'static)) -> Option { + let mut source = err.source(); + while let Some(e) = source { + if let Some(faith) = e.downcast_ref::() { + return Some(faith.kind); + } + source = e.source(); + } + + None +} + +impl From for FaithError { + fn from(err: reqwest::Error) -> Self { + // Always include full error chain for debugging + let mut msg = format!("{err:?}"); + let mut source = err.source(); + while let Some(e) = source { + msg.push_str(&format!(" -> {e:?}")); + source = e.source(); + } + + if err.is_timeout() { + return FaithError::new(FaithErrorKind::Timeout, Some(msg)); + } + + // A redirect the agent's own policy refused carries the kind we handed reqwest; one reqwest + // raised on its own account stays a plain network error. + let kind = err + .is_redirect() + .then(|| faith_kind_in_chain(&err)) + .flatten() + .unwrap_or(FaithErrorKind::Network); + + FaithError::new(kind, Some(msg)) + } +} + +impl From for FaithError { + fn from(err: reqwest_middleware::Error) -> Self { + match err { + reqwest_middleware::Error::Middleware(err) => { + FaithError::new(FaithErrorKind::Network, Some(err.to_string())) + } + reqwest_middleware::Error::Reqwest(err) => err.into(), + } + } +} + +impl Error for FaithError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } + + fn description(&self) -> &str { + "description() is deprecated; use Display" + } + + fn cause(&self) -> Option<&dyn Error> { + self.source() + } +} + +impl Display for FaithError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:?}: {}", + self.kind, + self.message + .as_deref() + .unwrap_or_else(|| self.kind.default_message()) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_code_is_distinct_and_named() { + let codes = error_codes(); + let unique: std::collections::BTreeSet<_> = codes.iter().collect(); + assert_eq!(unique.len(), codes.len(), "two kinds report the same code"); + assert!(codes.iter().all(|code| !code.is_empty())); + } + + #[test] + fn a_message_is_prefixed_with_the_code_it_reports() { + for kind in FaithErrorKind::iter() { + let code = kind.code(); + let rendered = FaithError::from(kind).to_string(); + assert!( + rendered.starts_with(&format!("{code}: ")), + "{rendered} does not lead with {code}" + ); + } + } + + #[test] + fn a_kind_without_a_message_falls_back_to_its_own() { + let err = FaithError::from(FaithErrorKind::Closed); + assert_eq!(err.to_string(), "Closed: the agent has been closed"); + + let err = FaithError::new(FaithErrorKind::Closed, Some("gone")); + assert_eq!(err.to_string(), "Closed: gone"); + } +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs new file mode 100644 index 0000000..f26f792 --- /dev/null +++ b/crates/web-faith/src/lib.rs @@ -0,0 +1,10 @@ +//! A browser-shaped HTTP client: fetch semantics over a Rust network stack. +//! +//! The client is being assembled here; for now this crate carries the error type the whole family +//! reports through. A component crate names its own errors for the failures it can produce, and +//! they are converted into [`FaithError`] as they cross into the client, so a caller matches on one +//! type whichever layer failed. + +pub mod error; + +pub use error::{FaithError, FaithErrorKind, error_codes}; diff --git a/index.d.ts b/index.d.ts index ee75d6b..67aaba1 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1226,73 +1226,6 @@ export declare function errorCodes(): Array export const FAITH_VERSION: string -/** - * Faith produces fine-grained errors, but maps them to a few javascript error types for fetch - * compatibility. The `.code` property on errors thrown from Faith is set to a stable name for each - * error kind, documented in this comprehensive mapping: - * - * - JS `AbortError`: - * - `Aborted` — request was aborted using `signal` - * - `Timeout` — request timed out - * - JS `NetworkError`: - * - `Network` — network error - * - `Redirect` — when the agent is configured to error on redirects - * - `ContentLengthOverrun` — a body written with `response.toFile()` exceeded the advertised `Content-Length` - * - JS `SyntaxError`: - * - `AddressParse` — IP parse error for `AgentOptions.dns.overrides` - * - `InvalidIntegrity` — SRI parse error for `RequestInit.integrity` - * - `JsonParse` — JSON parse error for `response.json()` - * - `PemParse` — PEM parse error for `AgentOptions.tls.identity` or `AgentOptions.tls.extraRoots` - * - JS `TypeError`: - * - `Closed` — a request was made on an agent that has been closed - * - `InvalidCompression` — `RequestInit.compress` naming no coding Faith can compress in - * - `InvalidHeader` — invalid header name or value - * - `InvalidMethod` — invalid HTTP method - * - `InvalidPath` — a `response.toFile()` destination that does not name a local path - * - `InvalidUrl` — invalid URL string - * - `ResponseAlreadyDisturbed` — body already read (mutually exclusive operations) - * - `ResponseBodyNull` — `response.toFile()` on a response that cannot carry a body - * - JS generic `Error`: - * - `BodyStream` — internal stream handling error - * - `Config` — invalid agent configuration - * - `FileExists` — a `response.toFile()` write refusing an occupied destination - * - `FileWrite` — the filesystem refusing a `response.toFile()` write - * - `IntegrityMismatch` — SRI checksum mismatch (with `RequestInit.integrity`) - * - * The library exports an `ERROR_CODES` object which has every error code the library throws, and - * every error thrown also has a `code` property that is set to one of those codes. So you can - * accurately respond to the exact error kind by checking its code and matching against the right - * constant from `ERROR_CODES`, instead of doing string matching on the error message, or coarse - * `instance of` matching. - * - * Due to technical limitations, when reading a body stream, reads might fail, but that error - * will not have a `code` property. - */ -export declare const enum FaithErrorKind { - Aborted = 'Aborted', - AddressParse = 'AddressParse', - BodyStream = 'BodyStream', - Closed = 'Closed', - Config = 'Config', - ContentLengthOverrun = 'ContentLengthOverrun', - FileExists = 'FileExists', - FileWrite = 'FileWrite', - IntegrityMismatch = 'IntegrityMismatch', - InvalidCompression = 'InvalidCompression', - InvalidHeader = 'InvalidHeader', - InvalidIntegrity = 'InvalidIntegrity', - InvalidMethod = 'InvalidMethod', - InvalidPath = 'InvalidPath', - InvalidUrl = 'InvalidUrl', - JsonParse = 'JsonParse', - Network = 'Network', - PemParse = 'PemParse', - Redirect = 'Redirect', - ResponseAlreadyDisturbed = 'ResponseAlreadyDisturbed', - ResponseBodyNull = 'ResponseBodyNull', - Timeout = 'Timeout' -} - export declare function faithFetch(url: string, options: FaithOptionsAndBody, signal?: AbortSignal | undefined | null, streamBody?: StreamBody | undefined | null): Promise export interface FaithOptionsAndBody { diff --git a/index.js b/index.js index e4b9573..b1423d7 100644 --- a/index.js +++ b/index.js @@ -585,7 +585,6 @@ module.exports.CredentialsOption = nativeBinding.CredentialsOption module.exports.DuplexOption = nativeBinding.DuplexOption module.exports.errorCodes = nativeBinding.errorCodes module.exports.FAITH_VERSION = nativeBinding.FAITH_VERSION -module.exports.FaithErrorKind = nativeBinding.FaithErrorKind module.exports.faithFetch = nativeBinding.faithFetch module.exports.Http3Congestion = nativeBinding.Http3Congestion module.exports.Redirect = nativeBinding.Redirect diff --git a/wrapper.d.ts b/wrapper.d.ts index e974ff6..31c15cf 100644 --- a/wrapper.d.ts +++ b/wrapper.d.ts @@ -22,7 +22,7 @@ export { USER_AGENT, } from "./index"; -// NOTE: This must be kept in sync with FaithErrorKind in src/error.rs +// NOTE: This must be kept in sync with the kinds in crates/web-faith/src/error.rs // Run `npm test` to validate sync (test/error-codes.test.js checks this) export const ERROR_CODES: { readonly Aborted: "Aborted"; From 39c943fbbbdaec3a772361716868dd9d9863c84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:40:28 +1200 Subject: [PATCH 11/61] S1: extract web-faith-integrity Subresource Integrity moves to a crate of its own, which builds, tests, and documents with no JavaScript runtime in its graph and speaks only ssri. It names its own error for the two failures it can produce, and web-faith converts them at the boundary, so a caller still matches on one error type and the codes reported for a bad value and a mismatch are unchanged. The client gains an `integrity` feature, on by default, which drops the dependency when turned off. The nine tests move with the code. --- Cargo.lock | 9 +++ Cargo.toml | 1 + crates/web-faith-integrity/Cargo.toml | 14 ++++ .../src/lib.rs} | 75 +++++++++++++------ crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/lib.rs | 1 - crates/web-faith-napi/src/response.rs | 3 +- crates/web-faith/Cargo.toml | 6 ++ crates/web-faith/src/error.rs | 15 ++++ 9 files changed, 100 insertions(+), 25 deletions(-) create mode 100644 crates/web-faith-integrity/Cargo.toml rename crates/{web-faith-napi/src/integrity.rs => web-faith-integrity/src/lib.rs} (62%) diff --git a/Cargo.lock b/Cargo.lock index fdbe404..5531db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2801,6 +2801,14 @@ dependencies = [ "reqwest", "reqwest-middleware", "strum", + "web-faith-integrity", +] + +[[package]] +name = "web-faith-integrity" +version = "0.7.0" +dependencies = [ + "ssri", ] [[package]] @@ -2841,6 +2849,7 @@ dependencies = [ "tokio-util", "url", "web-faith", + "web-faith-integrity", "windows", ] diff --git a/Cargo.toml b/Cargo.toml index 2dbf22b..813f686 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith-integrity = { version = "0.7.0", path = "crates/web-faith-integrity" } netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } netlink-sys = "0.8.7" diff --git a/crates/web-faith-integrity/Cargo.toml b/crates/web-faith-integrity/Cargo.toml new file mode 100644 index 0000000..862b657 --- /dev/null +++ b/crates/web-faith-integrity/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "web-faith-integrity" +description = "Subresource Integrity parsing and verification" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +ssri.workspace = true diff --git a/crates/web-faith-napi/src/integrity.rs b/crates/web-faith-integrity/src/lib.rs similarity index 62% rename from crates/web-faith-napi/src/integrity.rs rename to crates/web-faith-integrity/src/lib.rs index 7506c4f..22f4d14 100644 --- a/crates/web-faith-napi/src/integrity.rs +++ b/crates/web-faith-integrity/src/lib.rs @@ -1,7 +1,40 @@ +//! Subresource Integrity parsing and verification. +//! +//! An integrity value names one or more digests a resource is expected to match; the resource is +//! good if it matches any one of them, and digests naming an algorithm too weak to trust are +//! ignored rather than honoured. Verification comes in two shapes: [`verify_integrity`] for a body +//! already in memory, and [`integrity_checker`] with [`finish_integrity`] for one being read as it +//! arrives. + +use std::{ + error::Error, + fmt::{self, Display}, +}; + use ssri::{Integrity, IntegrityChecker}; -use crate::error::{FaithError, FaithErrorKind}; +/// A resource failing its integrity check, or an integrity value that could not be read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IntegrityError { + /// The integrity value is not one this can parse. + Invalid(String), + /// The resource matched none of the digests it was expected to. + Mismatch, +} +impl Display for IntegrityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Invalid(detail) => write!(f, "failed to parse integrity value: {detail}"), + Self::Mismatch => write!(f, "resource integrity check failed"), + } + } +} + +impl Error for IntegrityError {} + +/// Algorithm names are compared case-insensitively, the digest itself being base64 and so not ours +/// to touch. fn normalize_integrity(integrity: &str) -> String { integrity .split_whitespace() @@ -16,24 +49,24 @@ fn normalize_integrity(integrity: &str) -> String { .join(" ") } -fn parse_integrity(integrity: &str) -> Result { +fn parse_integrity(integrity: &str) -> Result { let normalized = normalize_integrity(integrity); - normalized.parse().map_err(|e| { - FaithError::new( - FaithErrorKind::InvalidIntegrity, - Some(format!("failed to parse integrity value: {e}")), - ) - }) + normalized + .parse() + .map_err(|e| IntegrityError::Invalid(format!("{e}"))) } -pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> { +/// Verify data that is already in hand. +/// +/// An absent or blank value has nothing to verify and so passes. +pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), IntegrityError> { if integrity.trim().is_empty() { return Ok(()); } parse_integrity(integrity)? .check(data) - .map_err(|_| FaithErrorKind::IntegrityMismatch)?; + .map_err(|_| IntegrityError::Mismatch)?; Ok(()) } @@ -41,10 +74,12 @@ pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> /// Build a streaming integrity checker for a body read that hashes as it goes. /// /// `None` when there is nothing to verify (no value, or an empty one), matching -/// [`verify_integrity`]. A malformed value is rejected up front with `InvalidIntegrity`, +/// [`verify_integrity`]. A malformed value is rejected up front with [`IntegrityError::Invalid`], /// before any of the body is touched. Feed each chunk to the returned checker with /// [`IntegrityChecker::input`] and finish with [`finish_integrity`]. -pub fn integrity_checker(integrity: Option<&str>) -> Result, FaithError> { +pub fn integrity_checker( + integrity: Option<&str>, +) -> Result, IntegrityError> { let Some(integrity) = integrity else { return Ok(None); }; @@ -55,12 +90,12 @@ pub fn integrity_checker(integrity: Option<&str>) -> Result Result<(), FaithError> { +/// Finish a streaming integrity check. +pub fn finish_integrity(checker: IntegrityChecker) -> Result<(), IntegrityError> { checker .result() .map(|_| ()) - .map_err(|_| FaithErrorKind::IntegrityMismatch.into()) + .map_err(|_| IntegrityError::Mismatch) } #[cfg(test)] @@ -94,10 +129,7 @@ mod tests { let integrity = "sha256-wronghashvalue"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err().kind, - FaithErrorKind::IntegrityMismatch - )); + assert!(matches!(result.unwrap_err(), IntegrityError::Mismatch)); } #[test] @@ -113,10 +145,7 @@ mod tests { let integrity = "sha256-wronghash1 sha256-wronghash2"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err().kind, - FaithErrorKind::IntegrityMismatch - )); + assert!(matches!(result.unwrap_err(), IntegrityError::Mismatch)); } #[test] diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 04c2606..c08258b 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -46,6 +46,7 @@ time.workspace = true tokio-util.workspace = true url.workspace = true web-faith.workspace = true +web-faith-integrity.workspace = true [target.'cfg(target_os = "linux")'.dependencies] netlink-packet-core.workspace = true diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 8ad7ab1..a97c88c 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -9,7 +9,6 @@ mod dns; mod encoding; mod error; mod fetch; -mod integrity; mod options; mod response; mod retry; diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 37a7239..ba2aaf8 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -28,13 +28,14 @@ use serde_json; use stream_shared::SharedStream; use tokio::{io::AsyncWriteExt, sync::watch}; +use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; + use crate::{ agent::InnerAgentStats, async_task::{Value, faith_promise}, body::{Body, BodyHolder, DynStream, drain_body_inner}, encoding::{Coding, decode_stream}, error::{FaithError, FaithErrorExt, FaithErrorKind}, - integrity::{finish_integrity, integrity_checker, verify_integrity}, timing::{TimingBreakdown, TimingSlot}, }; diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 88f2e0c..d66da65 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -14,3 +14,9 @@ publish = false reqwest.workspace = true reqwest-middleware.workspace = true strum.workspace = true +web-faith-integrity = { workspace = true, optional = true } + +[features] +default = ["integrity"] +# Subresource Integrity: parsing and verifying `integrity` values. +integrity = ["dep:web-faith-integrity"] diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs index f79c910..3736aa1 100644 --- a/crates/web-faith/src/error.rs +++ b/crates/web-faith/src/error.rs @@ -150,6 +150,21 @@ impl From for FaithError { } } +/// A component crate names its own errors; they become the client's as they cross into it, which is +/// what keeps the code a caller sees the same whichever layer failed. +#[cfg(feature = "integrity")] +impl From for FaithError { + fn from(err: web_faith_integrity::IntegrityError) -> Self { + use web_faith_integrity::IntegrityError as E; + match err { + E::Invalid(_) => { + FaithError::new(FaithErrorKind::InvalidIntegrity, Some(err.to_string())) + } + E::Mismatch => FaithErrorKind::IntegrityMismatch.into(), + } + } +} + impl From for FaithError { fn from(err: reqwest_middleware::Error) -> Self { match err { From ee75c48ab4cd234a1efba16b583f11361d4255ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:45:51 +1200 Subject: [PATCH 12/61] S1: extract web-faith-encoding Content coding moves to a crate of its own. It reached into the client for the body-stream type; it now names that shape itself, so the dependency no longer points upward, and it takes header types from http rather than through reqwest, which drops reqwest from its graph entirely. Nothing about the codings, the Accept-Encoding default, or the decode decision changes. The twenty tests move with the code. --- .workhorse/plans/s1/plan.md | 6 +-- Cargo.lock | 13 +++++++ Cargo.toml | 1 + crates/web-faith-encoding/Cargo.toml | 19 ++++++++++ .../src/lib.rs} | 38 ++++++++++--------- crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/fetch.rs | 3 +- crates/web-faith-napi/src/lib.rs | 1 - crates/web-faith-napi/src/response.rs | 4 +- 9 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 crates/web-faith-encoding/Cargo.toml rename crates/{web-faith-napi/src/encoding.rs => web-faith-encoding/src/lib.rs} (92%) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 4fc8220..0e47533 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -55,17 +55,17 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al ## Build order (each step ends green: `cargo build` + `cargo test` + napi `npm run build`) -- [ ] **0. Workspace scaffold.** Root `[workspace]` with shared `[workspace.package]` +- [x] **0. Workspace scaffold.** Root `[workspace]` with shared `[workspace.package]` (licence, repository, authors, edition, `rust-version = "1.96"`) and `[workspace.dependencies]`. Move the current crate to `crates/web-faith-napi`. Fix `build.rs` `Cargo.lock` path. Make `napi build` target the relocated crate and keep `index.js`/`index.d.ts` at repo root. Verify the npm build still produces a working `.node`. No behaviour change. -- [ ] **1. Error core split.** Pure-Rust `FaithError`/`FaithErrorKind` (no napi) reachable by every +- [x] **1. Error core split.** Pure-Rust `FaithError`/`FaithErrorKind` (no napi) reachable by every crate; napi conversions live only in `web-faith-napi`. `ERROR_CODES` still generated from the one source ([ERR](../../specs/errors/errors.md)). Decide where the shared error core lives (likely in `web-faith`, with components naming their own error types that the client converts — per [RUST](../../specs/rust/overview.md) "A component crate stands alone"). -- [ ] **2. Extract `web-faith-integrity`** — own error type, own docs, `cargo test -p web-faith-integrity` with no JS runtime. +- [x] **2. Extract `web-faith-integrity`** — own error type, own docs, `cargo test -p web-faith-integrity` with no JS runtime. - [ ] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). - [ ] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. - [ ] **5. Extract `web-faith-dns`.** diff --git a/Cargo.lock b/Cargo.lock index 5531db4..139af35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2804,6 +2804,18 @@ dependencies = [ "web-faith-integrity", ] +[[package]] +name = "web-faith-encoding" +version = "0.7.0" +dependencies = [ + "async-compression", + "bytes", + "futures", + "http", + "tokio", + "tokio-util", +] + [[package]] name = "web-faith-integrity" version = "0.7.0" @@ -2849,6 +2861,7 @@ dependencies = [ "tokio-util", "url", "web-faith", + "web-faith-encoding", "web-faith-integrity", "windows", ] diff --git a/Cargo.toml b/Cargo.toml index 813f686..5ab43db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } web-faith-integrity = { version = "0.7.0", path = "crates/web-faith-integrity" } netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } diff --git a/crates/web-faith-encoding/Cargo.toml b/crates/web-faith-encoding/Cargo.toml new file mode 100644 index 0000000..8349e99 --- /dev/null +++ b/crates/web-faith-encoding/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "web-faith-encoding" +description = "HTTP content coding for request and response bodies" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +async-compression.workspace = true +bytes.workspace = true +futures.workspace = true +http.workspace = true +tokio.workspace = true +tokio-util.workspace = true diff --git a/crates/web-faith-napi/src/encoding.rs b/crates/web-faith-encoding/src/lib.rs similarity index 92% rename from crates/web-faith-napi/src/encoding.rs rename to crates/web-faith-encoding/src/lib.rs index 90231da..bf8bf34 100644 --- a/crates/web-faith-napi/src/encoding.rs +++ b/crates/web-faith-encoding/src/lib.rs @@ -12,24 +12,28 @@ use async_compression::tokio::bufread::{ }; use bytes::Bytes; use futures::{Stream, TryStreamExt}; -use reqwest::header::{CONTENT_ENCODING, CONTENT_LENGTH, HeaderMap}; +use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, HeaderMap}; use tokio::io::AsyncReadExt; use tokio_util::io::{ReaderStream, StreamReader}; -use crate::body::DynStream; +/// A body byte-stream, as the decoders take and return one. +/// +/// The client hands its own body streams straight to [`decode_stream`]: the shape is the same one +/// its pipeline already carries, so nothing has to depend on the layer above to name it. +pub type ByteStream = dyn Stream> + Send + Sync; /// The `Accept-Encoding` Faith advertises when the caller advertises none. /// /// Matches the value reqwest's decompression stack sent before Faith took over the /// codings, so the wire is unchanged for the default request. -pub(crate) const DEFAULT_ACCEPT_ENCODING: &str = "zstd,gzip,deflate,br"; +pub const DEFAULT_ACCEPT_ENCODING: &str = "zstd,gzip,deflate,br"; /// A content coding Faith can decode. Wire tokens: `gzip`, `deflate`, `br`, `zstd`. /// /// `deflate` is the zlib-wrapped form (RFC 1950), matching what reqwest and every /// other mainstream client decode it as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Coding { +pub enum Coding { Gzip, Deflate, Brotli, @@ -43,7 +47,7 @@ impl Coding { /// loosely as HTTP writes it, this matches the four documented tokens exactly: the /// option is an API surface, and an unrecognised value is refused rather than /// guessed at (spec:ENC#compressing-a-request-body). - pub(crate) fn from_option(value: &str) -> Option { + pub fn from_option(value: &str) -> Option { match value { "gzip" => Some(Self::Gzip), "deflate" => Some(Self::Deflate), @@ -54,7 +58,7 @@ impl Coding { } /// The wire token naming this coding in a `Content-Encoding`. - pub(crate) fn token(self) -> &'static str { + pub fn token(self) -> &'static str { match self { Self::Gzip => "gzip", Self::Deflate => "deflate", @@ -87,7 +91,7 @@ impl Coding { /// single coding Faith can decode and the request's `Accept-Encoding` accepted it. /// A `Content-Encoding` naming more than one coding, an unknown coding, or a coding the /// request did not accept yields `None`, and the body is delivered as received. -pub(crate) fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option { +pub fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option { // A representation encoded more than once is the caller's to unwind. The codings may // arrive comma-joined on one line or split across several `Content-Encoding` lines -- // the same list either way, so both forms are gathered together before counting. @@ -107,7 +111,7 @@ pub(crate) fn decision(headers: &HeaderMap, accept: &AcceptEncoding) -> Option, deflate: Option, brotli: Option, @@ -126,7 +130,7 @@ pub(crate) struct AcceptEncoding { } impl AcceptEncoding { - pub(crate) fn parse(value: &str) -> Self { + pub fn parse(value: &str) -> Self { let mut accept = Self::default(); for element in value.split(',') { let mut parts = element.split(';'); @@ -210,7 +214,7 @@ fn parse_quality(value: &str) -> Option { /// /// The body stream carries decoded bytes on every read path this way (see [`Coding`]); /// trailers are pulled off the frames before this point, so decoding sees data only. -pub(crate) fn decode_stream(input: Pin>, coding: Coding) -> Pin> { +pub fn decode_stream(input: Pin>, coding: Coding) -> Pin> { let reader = StreamReader::new(input.map_err(io::Error::other)); match coding { Coding::Gzip => reader_stream(GzipDecoder::new(reader)), @@ -225,7 +229,7 @@ pub(crate) fn decode_stream(input: Pin>, coding: Coding) -> Pin(reader: R) -> Pin> +fn reader_stream(reader: R) -> Pin> where R: tokio::io::AsyncRead + Send + Sync + 'static, { @@ -233,13 +237,13 @@ where } /// A request body stream, as reqwest takes one. -pub(crate) type RequestStream = Pin> + Send>>; +pub type RequestStream = Pin> + Send>>; /// Compress a buffered request body, yielding the bytes that go on the wire. /// /// The whole body is known up front, so it compresses in one pass and its length is the /// `Content-Length` reqwest derives from it (spec:ENC#what-a-compressed-request-sends). -pub(crate) async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result> { +pub async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result> { let mut output = Vec::new(); match coding { Coding::Gzip => GzipEncoder::new(input).read_to_end(&mut output).await?, @@ -255,7 +259,7 @@ pub(crate) async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result< /// There is no compressed length to declare before the body ends, so the result goes out /// chunked (spec:ENC#what-a-compressed-request-sends). The encoder buffers on its own /// terms, so the bytes for one chunk the caller writes need not leave with it. -pub(crate) fn compress_stream(input: S, coding: Coding) -> RequestStream +pub fn compress_stream(input: S, coding: Coding) -> RequestStream where S: Stream> + Send + 'static, { @@ -280,7 +284,7 @@ where /// The caller's `Content-Encoding` describes the bytes they handed over, so Faith's coding /// is named after theirs, the order the codings were applied in /// (spec:ENC#what-a-compressed-request-sends). -pub(crate) fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> String { +pub fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> String { match declared.map(str::trim).filter(|value| !value.is_empty()) { Some(declared) => format!("{declared}, {}", applied.token()), None => applied.token().to_owned(), @@ -289,7 +293,7 @@ pub(crate) fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> #[cfg(test)] mod tests { - use reqwest::header::{CONTENT_ENCODING, HeaderMap, HeaderValue}; + use http::header::{CONTENT_ENCODING, HeaderMap, HeaderValue}; use super::*; diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index c08258b..74e7cff 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -46,6 +46,7 @@ time.workspace = true tokio-util.workspace = true url.workspace = true web-faith.workspace = true +web-faith-encoding.workspace = true web-faith-integrity.workspace = true [target.'cfg(target_os = "linux")'.dependencies] diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index 71791ae..f71b0d8 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -20,10 +20,11 @@ use reqwest::{ }; use tokio::sync::{Mutex, mpsc}; +use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; + use crate::{ async_task::faith_promise, body::{Body, BodyHolder}, - encoding::{self, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}, error::{FaithError, FaithErrorKind}, options::{CredentialsOption, FaithOptions, FaithOptionsAndBody, PRIORITY}, response::{FaithResponse, PeerInformation}, diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index a97c88c..3d96677 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -6,7 +6,6 @@ mod body; mod conn_tracker; mod cookies; mod dns; -mod encoding; mod error; mod fetch; mod options; diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index ba2aaf8..174d23d 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -28,13 +28,13 @@ use serde_json; use stream_shared::SharedStream; use tokio::{io::AsyncWriteExt, sync::watch}; +use web_faith_encoding::{Coding, decode_stream}; use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; use crate::{ agent::InnerAgentStats, async_task::{Value, faith_promise}, body::{Body, BodyHolder, DynStream, drain_body_inner}, - encoding::{Coding, decode_stream}, error::{FaithError, FaithErrorExt, FaithErrorKind}, timing::{TimingBreakdown, TimingSlot}, }; @@ -49,7 +49,7 @@ pub struct FaithResponse { pub(crate) body: BodyHolder, /// The coding to decode the body under, or `None` to deliver it as received. /// Set once when the response is built, from the request's `Accept-Encoding` and the - /// response's `Content-Encoding` (see [`crate::encoding`]). + /// response's `Content-Encoding` (see [`web_faith_encoding`]). pub(crate) decode: Option, pub(crate) disturbed: Arc, pub(crate) headers: HeaderMap, From 75a85bb13878a410fba34eaa1d4d23504485f5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:51:30 +1200 Subject: [PATCH 13/61] S1: extract web-faith-cookies The jar moves to a crate of its own. It took reqwest's Url and HeaderValue, which are the url and http types under another name, so it now names them directly, and the reqwest CookieStore impl sits behind a `reqwest` feature. That impl held the only way to store a response's cookies or read the header to send, so both are now inherent methods and the trait delegates to them: a caller without reqwest gets a jar that works rather than one it cannot drive. The crate's own tests exercise those methods, so they pass with no reqwest in the graph at all. The storage, matching, and eviction rules are unchanged. --- Cargo.lock | 13 ++++++ Cargo.toml | 1 + crates/web-faith-cookies/Cargo.toml | 23 +++++++++++ .../src/lib.rs} | 40 ++++++++++++++----- crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/agent.rs | 9 +++-- crates/web-faith-napi/src/lib.rs | 1 - 7 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 crates/web-faith-cookies/Cargo.toml rename crates/{web-faith-napi/src/cookies.rs => web-faith-cookies/src/lib.rs} (95%) diff --git a/Cargo.lock b/Cargo.lock index 139af35..bafb7bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2804,6 +2804,18 @@ dependencies = [ "web-faith-integrity", ] +[[package]] +name = "web-faith-cookies" +version = "0.7.0" +dependencies = [ + "cookie", + "cookie_store", + "http", + "reqwest", + "time", + "url", +] + [[package]] name = "web-faith-encoding" version = "0.7.0" @@ -2861,6 +2873,7 @@ dependencies = [ "tokio-util", "url", "web-faith", + "web-faith-cookies", "web-faith-encoding", "web-faith-integrity", "windows", diff --git a/Cargo.toml b/Cargo.toml index 5ab43db..d93b7e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } web-faith-integrity = { version = "0.7.0", path = "crates/web-faith-integrity" } netlink-packet-core = "0.7.0" diff --git a/crates/web-faith-cookies/Cargo.toml b/crates/web-faith-cookies/Cargo.toml new file mode 100644 index 0000000..b9880c3 --- /dev/null +++ b/crates/web-faith-cookies/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "web-faith-cookies" +description = "A cookie jar with the RFC 6265bis rules that hold outside a browser" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +cookie.workspace = true +cookie_store.workspace = true +http.workspace = true +time.workspace = true +url.workspace = true +reqwest = { workspace = true, optional = true } + +[features] +# Implement reqwest's `CookieStore`, so the jar can serve as its cookie provider. +reqwest = ["dep:reqwest"] diff --git a/crates/web-faith-napi/src/cookies.rs b/crates/web-faith-cookies/src/lib.rs similarity index 95% rename from crates/web-faith-napi/src/cookies.rs rename to crates/web-faith-cookies/src/lib.rs index 0ab8007..5ba47a4 100644 --- a/crates/web-faith-napi/src/cookies.rs +++ b/crates/web-faith-cookies/src/lib.rs @@ -1,20 +1,22 @@ -//! The agent's cookie jar. (spec:COOK) +//! A cookie jar for HTTP clients, with the rules that hold outside a browser. (spec:COOK) //! -//! `cookie_store` implements the classic RFC 6265 storage model, and reqwest's [`Jar`] wraps it in a +//! `cookie_store` implements the classic RFC 6265 storage model, and reqwest's `Jar` wraps it in a //! private field, so extending it means wrapping the store ourselves rather than the jar. What this //! adds is the RFC 6265bis rules that mean something without a browsing context: the `__Host-` and //! `__Secure-` name prefixes, a cap on how far ahead a cookie may expire, and caps on how many //! cookies and how many bytes one server can accumulate. `SameSite` is left to the `cookie` crate to //! parse and is never read, governing cross-site behaviour that only a first-party context has. //! -//! [`Jar`]: reqwest::cookie::Jar +//! The `reqwest` feature implements that client's `CookieStore` trait, so the jar can be handed to +//! it as a cookie provider. use std::{collections::HashMap, sync::RwLock, time::Duration}; use cookie::{Cookie as RawCookie, Expiration}; use cookie_store::{Cookie as StoredCookie, CookieStore as Store, StoreAction}; -use reqwest::{Url, cookie::CookieStore, header::HeaderValue}; +use http::HeaderValue; use time::OffsetDateTime; +use url::Url; /// A cookie may not persist beyond this by default. RFC 6265bis §5.5. pub const DEFAULT_MAX_AGE: Duration = Duration::from_secs(400 * 24 * 60 * 60); @@ -271,8 +273,16 @@ impl Inner { } } -impl CookieStore for FaithJar { - fn set_cookies(&self, cookie_headers: &mut dyn Iterator, url: &Url) { +impl FaithJar { + /// Store the cookies a response set, ignoring any the jar's rules refuse. + /// + /// A header that is not valid UTF-8, or that does not parse as a cookie, is skipped rather than + /// failing the read: one bad `Set-Cookie` does not spoil the response. + pub fn store_response_cookies<'h>( + &self, + cookie_headers: impl Iterator, + url: &Url, + ) { for header in cookie_headers { let Ok(header) = std::str::from_utf8(header.as_bytes()) else { continue; @@ -286,7 +296,8 @@ impl CookieStore for FaithJar { } } - fn cookies(&self, url: &Url) -> Option { + /// The `Cookie` header to send to `url`, or `None` when the jar has nothing for it. + pub fn request_cookie_header(&self, url: &Url) -> Option { let inner = self.inner.read().unwrap(); let cookies = inner .store @@ -303,6 +314,17 @@ impl CookieStore for FaithJar { } } +#[cfg(feature = "reqwest")] +impl reqwest::cookie::CookieStore for FaithJar { + fn set_cookies(&self, cookie_headers: &mut dyn Iterator, url: &Url) { + self.store_response_cookies(cookie_headers, url); + } + + fn cookies(&self, url: &Url) -> Option { + self.request_cookie_header(url) + } +} + #[cfg(test)] mod tests { use super::*; @@ -321,7 +343,7 @@ mod tests { /// What the jar would send to `url`, as a `Cookie` header value. fn sent(jar: &FaithJar, url: &str) -> Option { - jar.cookies(&self::url(url)) + jar.request_cookie_header(&self::url(url)) .map(|value| value.to_str().unwrap().to_owned()) } @@ -733,7 +755,7 @@ mod tests { HeaderValue::from_static("__Host-bad=1; Path=/"), ]; - jar.set_cookies(&mut headers.iter(), &url("https://example.com/")); + jar.store_response_cookies(headers.iter(), &url("https://example.com/")); let sent = sent(&jar, "https://example.com/").unwrap(); assert!(sent.contains("__Host-good=1"), "{sent}"); diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 74e7cff..f579c9e 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -46,6 +46,7 @@ time.workspace = true tokio-util.workspace = true url.workspace = true web-faith.workspace = true +web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-encoding.workspace = true web-faith-integrity.workspace = true diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 23eeb52..f283718 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -32,13 +32,14 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use crate::alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] use crate::alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; +use web_faith_cookies::{ + CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, + FaithJar, +}; + use crate::{ async_task::faith_promise, conn_tracker::{ConnectionInfo, ConnectionTracker}, - cookies::{ - CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, - FaithJar, - }, dns::{DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains}, error::{FaithError, FaithErrorExt, FaithErrorKind}, options::{PRIORITY, RequestCacheMode}, diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 3d96677..63d9183 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -4,7 +4,6 @@ mod alt_svc; mod async_task; mod body; mod conn_tracker; -mod cookies; mod dns; mod error; mod fetch; From f64c5f8a7583d19b5ef7719de1179ec11131ef70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:56:27 +1200 Subject: [PATCH 14/61] S1: extract web-faith-dns The resolver moves to a crate of its own: the cache, the discovery ladder, the HTTPS record query, and Happy Eyeballs. It had no ties to the rest of the code at all, so the only coupling to break was reqwest's Resolve impl, which now sits behind a `reqwest` feature the way the jar's does. The resolver itself, its transports, and the server order are unchanged. --- Cargo.lock | 13 +++++++++++ Cargo.toml | 1 + crates/web-faith-dns/Cargo.toml | 23 +++++++++++++++++++ .../src/dns.rs => web-faith-dns/src/lib.rs} | 10 ++++---- crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/agent.rs | 5 +++- crates/web-faith-napi/src/alt_svc.rs | 6 ++--- crates/web-faith-napi/src/lib.rs | 1 - crates/web-faith-napi/src/retry.rs | 2 +- 9 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 crates/web-faith-dns/Cargo.toml rename crates/{web-faith-napi/src/dns.rs => web-faith-dns/src/lib.rs} (99%) diff --git a/Cargo.lock b/Cargo.lock index bafb7bf..abf9aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2816,6 +2816,18 @@ dependencies = [ "url", ] +[[package]] +name = "web-faith-dns" +version = "0.7.0" +dependencies = [ + "futures", + "hickory-resolver", + "moka", + "reqwest", + "tokio", + "url", +] + [[package]] name = "web-faith-encoding" version = "0.7.0" @@ -2874,6 +2886,7 @@ dependencies = [ "url", "web-faith", "web-faith-cookies", + "web-faith-dns", "web-faith-encoding", "web-faith-integrity", "windows", diff --git a/Cargo.toml b/Cargo.toml index d93b7e7..164ea59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,7 @@ tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } +web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } web-faith-integrity = { version = "0.7.0", path = "crates/web-faith-integrity" } netlink-packet-core = "0.7.0" diff --git a/crates/web-faith-dns/Cargo.toml b/crates/web-faith-dns/Cargo.toml new file mode 100644 index 0000000..7be2016 --- /dev/null +++ b/crates/web-faith-dns/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "web-faith-dns" +description = "A caching DNS resolver: transports, server order, HTTPS records, and Happy Eyeballs" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +futures.workspace = true +hickory-resolver.workspace = true +moka.workspace = true +tokio.workspace = true +url.workspace = true +reqwest = { workspace = true, optional = true } + +[features] +# Implement reqwest's `Resolve`, so the resolver can be installed on its client. +reqwest = ["dep:reqwest"] diff --git a/crates/web-faith-napi/src/dns.rs b/crates/web-faith-dns/src/lib.rs similarity index 99% rename from crates/web-faith-napi/src/dns.rs rename to crates/web-faith-dns/src/lib.rs index 0077621..ed262ca 100644 --- a/crates/web-faith-napi/src/dns.rs +++ b/crates/web-faith-dns/src/lib.rs @@ -39,7 +39,6 @@ use hickory_resolver::{ }, system_conf::read_system_conf, }; -use reqwest::dns::{Addrs, Name as ReqName, Resolve, Resolving}; use tokio::sync::OnceCell; use url::{Host, Url}; @@ -811,8 +810,11 @@ impl FaithResolver { } } -impl Resolve for FaithResolver { - fn resolve(&self, name: ReqName) -> Resolving { +/// Installed on a reqwest client with `ClientBuilder::dns_resolver`, so every lookup a request +/// makes goes through the same resolver `prefetch` warms. +#[cfg(feature = "reqwest")] +impl reqwest::dns::Resolve for FaithResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { let this = self.clone(); Box::pin(async move { let addrs = this.lookup(name.as_str()).await?; @@ -820,7 +822,7 @@ impl Resolve for FaithResolver { // `'static`, so collect owned rather than borrowing the lookup. let addrs: Vec = addrs.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); - Ok(Box::new(addrs.into_iter()) as Addrs) + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) }) } } diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index f579c9e..7f556c8 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -47,6 +47,7 @@ tokio-util.workspace = true url.workspace = true web-faith.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } +web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-encoding.workspace = true web-faith-integrity.workspace = true diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index f283718..69e9990 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -32,6 +32,10 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use crate::alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] use crate::alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; +use web_faith_dns::{ + DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, +}; + use web_faith_cookies::{ CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, FaithJar, @@ -40,7 +44,6 @@ use web_faith_cookies::{ use crate::{ async_task::faith_promise, conn_tracker::{ConnectionInfo, ConnectionTracker}, - dns::{DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains}, error::{FaithError, FaithErrorExt, FaithErrorKind}, options::{PRIORITY, RequestCacheMode}, retry::{DeadConnectionRetry, StaleAddressRetry}, diff --git a/crates/web-faith-napi/src/alt_svc.rs b/crates/web-faith-napi/src/alt_svc.rs index bd2447d..2aed17a 100644 --- a/crates/web-faith-napi/src/alt_svc.rs +++ b/crates/web-faith-napi/src/alt_svc.rs @@ -962,7 +962,7 @@ impl H3Prober { /// probe-worthy from its first request rather than from the first TCP response carrying an /// `Alt-Svc` header. /// -/// Installed on the resolver by the agent (see [`crate::dns::FaithResolver::set_https_sink`]), +/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), /// which is the only place that holds all three: the resolver is built before the cache, and the /// prober holds a client that holds the resolver, so nothing lower down can own this. /// @@ -1008,12 +1008,12 @@ impl H3HttpsSink { } } -impl crate::dns::HttpsSink for H3HttpsSink { +impl web_faith_dns::HttpsSink for H3HttpsSink { fn wants(&self, host: &str) -> bool { Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) } - fn record(&self, host: &str, advertisement: crate::dns::HttpsAdvertisement) { + fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { let Some(url) = Self::origin_url(host) else { return; }; diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 63d9183..3e9c9bd 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -4,7 +4,6 @@ mod alt_svc; mod async_task; mod body; mod conn_tracker; -mod dns; mod error; mod fetch; mod options; diff --git a/crates/web-faith-napi/src/retry.rs b/crates/web-faith-napi/src/retry.rs index 11ecbe1..875a1f4 100644 --- a/crates/web-faith-napi/src/retry.rs +++ b/crates/web-faith-napi/src/retry.rs @@ -15,7 +15,7 @@ use http::{Extensions, Method}; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result}; -use crate::dns::FaithResolver; +use web_faith_dns::FaithResolver; /// How many times a request may be replayed before the failure reaches the caller. /// From d32a47cd0cde33cd5cae7a51bc2e8a958bb87648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:02:02 +1200 Subject: [PATCH 15/61] S1: extract web-faith-conn-tracker Reading the operating system's per-connection statistics moves to a crate of its own, along with the Linux, macOS, and Windows implementations behind it. This was the one component holding napi types: the view it returned was a napi object with JavaScript Date fields, so the tracker could not be compiled away from a JS runtime. It now reports a snapshot of plain SystemTime and integer values, and the napi crate turns that into the object `connections()` returns. The JavaScript surface is unchanged, Dates and all. --- Cargo.lock | 14 + Cargo.toml | 1 + crates/web-faith-conn-tracker/Cargo.toml | 26 ++ crates/web-faith-conn-tracker/src/lib.rs | 258 +++++++++++++++ .../src/platform}/linux.rs | 0 .../src/platform}/macos.rs | 0 .../src/platform}/windows.rs | 0 crates/web-faith-napi/Cargo.toml | 1 + crates/web-faith-napi/src/agent.rs | 5 +- crates/web-faith-napi/src/conn_tracker.rs | 298 +++--------------- 10 files changed, 350 insertions(+), 253 deletions(-) create mode 100644 crates/web-faith-conn-tracker/Cargo.toml create mode 100644 crates/web-faith-conn-tracker/src/lib.rs rename crates/{web-faith-napi/src/conn_tracker => web-faith-conn-tracker/src/platform}/linux.rs (100%) rename crates/{web-faith-napi/src/conn_tracker => web-faith-conn-tracker/src/platform}/macos.rs (100%) rename crates/{web-faith-napi/src/conn_tracker => web-faith-conn-tracker/src/platform}/windows.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index abf9aff..c2b5aa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2804,6 +2804,19 @@ dependencies = [ "web-faith-integrity", ] +[[package]] +name = "web-faith-conn-tracker" +version = "0.7.0" +dependencies = [ + "libc", + "moka", + "netlink-packet-core", + "netlink-packet-sock-diag", + "netlink-sys", + "tokio", + "windows", +] + [[package]] name = "web-faith-cookies" version = "0.7.0" @@ -2885,6 +2898,7 @@ dependencies = [ "tokio-util", "url", "web-faith", + "web-faith-conn-tracker", "web-faith-cookies", "web-faith-dns", "web-faith-encoding", diff --git a/Cargo.toml b/Cargo.toml index 164ea59..4423096 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-tracker" } web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } diff --git a/crates/web-faith-conn-tracker/Cargo.toml b/crates/web-faith-conn-tracker/Cargo.toml new file mode 100644 index 0000000..706fac5 --- /dev/null +++ b/crates/web-faith-conn-tracker/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "web-faith-conn-tracker" +description = "Live per-connection TCP statistics, read from the operating system" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +moka.workspace = true +tokio.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +netlink-packet-core.workspace = true +netlink-packet-sock-diag.workspace = true +netlink-sys.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +libc.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +windows.workspace = true diff --git a/crates/web-faith-conn-tracker/src/lib.rs b/crates/web-faith-conn-tracker/src/lib.rs new file mode 100644 index 0000000..acc76e2 --- /dev/null +++ b/crates/web-faith-conn-tracker/src/lib.rs @@ -0,0 +1,258 @@ +//! Live per-connection statistics, read from the operating system. (spec:OBS) +//! +//! The pool's own view says which connections exist; the kernel knows how each one is actually +//! behaving. This tracker keeps an entry per connection the caller reports traffic on, expiring it +//! once it has been idle for the configured timeout, and refreshes each entry's TCP statistics from +//! the OS once a second. +//! +//! Reading those statistics is per-platform: Linux over netlink, macOS and Windows through their own +//! interfaces. On any other platform the entries are tracked without statistics. + +#[cfg(target_os = "linux")] +#[path = "platform/linux.rs"] +mod linux; +#[cfg(target_os = "macos")] +#[path = "platform/macos.rs"] +mod macos; +#[cfg(target_os = "windows")] +#[path = "platform/windows.rs"] +mod windows; + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use moka::Expiry; +use moka::{ops::compute::Op, sync::Cache}; +use tokio::{spawn, task::AbortHandle, time::sleep}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ConnectionKey { + pub local_addr: SocketAddr, + pub remote_addr: SocketAddr, +} + +#[derive(Debug, Clone)] +pub struct TrackedConnection { + pub first_seen: SystemTime, + pub last_seen: SystemTime, + pub response_count: u64, + pub latest_stats: Option, +} + +struct ExpireAfterTimeout(Duration); +impl Expiry for ExpireAfterTimeout { + fn expire_after_create( + &self, + _key: &ConnectionKey, + _value: &TrackedConnection, + _created_at: Instant, + ) -> Option { + Some(self.0) + } + + fn expire_after_read( + &self, + _key: &ConnectionKey, + value: &TrackedConnection, + _read_at: Instant, + _duration_until_expiry: Option, + _last_modified_at: Instant, + ) -> Option { + Some( + self.0 + .saturating_sub(value.last_seen.elapsed().unwrap_or_default()), + ) + } + + fn expire_after_update( + &self, + _key: &ConnectionKey, + value: &TrackedConnection, + _updated_at: Instant, + _duration_until_expiry: Option, + ) -> Option { + Some( + self.0 + .saturating_sub(value.last_seen.elapsed().unwrap_or_default()), + ) + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct TcpStats { + pub rtt_us: u32, + pub rtt_var_us: u32, + pub lost: Option, + pub retrans: u32, + pub total_retrans: u32, + pub cwnd: u32, + pub delivery_rate: Option, +} + +/// One tracked connection, as a caller reporting on the pool sees it. +#[derive(Debug, Clone)] +pub struct ConnectionSnapshot { + /// The transport the connection runs over. Only TCP is tracked. + pub connection_type: &'static str, + pub local_addr: SocketAddr, + pub remote_addr: SocketAddr, + pub first_seen: SystemTime, + pub last_seen: SystemTime, + /// When the entry falls out of the tracker, unless traffic renews it first. + pub expiry: Option, + pub response_count: u64, + /// What the operating system last reported for this connection, if it has been asked yet. + pub stats: Option, +} + +type Conns = Cache; + +#[derive(Debug)] +pub struct ConnectionTracker { + connections: Conns, + timeout: Duration, + task_abort: AbortHandle, +} + +impl Drop for ConnectionTracker { + fn drop(&mut self) { + self.task_abort.abort(); + } +} + +impl ConnectionTracker { + pub fn new(timeout: Duration) -> Arc { + let connections = Cache::builder() + .expire_after(ExpireAfterTimeout(timeout)) + .build(); + + let conns = connections.clone(); + let task_abort = spawn(async move { + loop { + let _ = update_all(conns.clone()); + sleep(Duration::from_secs(1)).await; + } + }) + .abort_handle(); + + Arc::new(Self { + connections, + timeout, + task_abort, + }) + } + + /// Record a response on a connection, returning whether that connection was already known. + /// + /// A connection the tracker has seen before is one the pool handed back rather than one + /// dialled for this request, since a fresh connection takes a local port of its own. + pub fn track(&self, local_addr: SocketAddr, remote_addr: SocketAddr) -> bool { + let now = SystemTime::now(); + let key = ConnectionKey { + local_addr, + remote_addr, + }; + let mut known = false; + self.connections.entry(key).and_compute_with(|entry| { + if let Some(entry) = entry { + known = true; + let mut conn = entry.into_value(); + conn.last_seen = now; + conn.response_count += 1; + Op::Put(conn) + } else { + Op::Put(TrackedConnection { + first_seen: now, + last_seen: now, + response_count: 1, + latest_stats: None, + }) + } + }); + known + } + + /// Register a warm-up connection that no request has yet been credited to. + /// + /// A `preconnect` connection is listed before any foreground request uses it, at a response + /// count of zero (spec:WARM). An entry already tracked is left untouched: a warm-up to an + /// origin that already holds a pooled connection does no new work, and must not disturb the + /// count or timestamps of the connection it would reuse. + pub fn track_warmup(&self, local_addr: SocketAddr, remote_addr: SocketAddr) { + let now = SystemTime::now(); + let key = ConnectionKey { + local_addr, + remote_addr, + }; + self.connections.entry(key).and_compute_with(|entry| { + if entry.is_some() { + Op::Nop + } else { + Op::Put(TrackedConnection { + first_seen: now, + last_seen: now, + response_count: 0, + latest_stats: None, + }) + } + }); + } + + /// Every connection currently tracked. + pub fn snapshot(&self) -> Vec { + self.connections + .iter() + .map(|(key, conn)| ConnectionSnapshot { + connection_type: "tcp", + local_addr: key.local_addr, + remote_addr: key.remote_addr, + first_seen: conn.first_seen, + last_seen: conn.last_seen, + expiry: conn.last_seen.checked_add(self.timeout), + response_count: conn.response_count, + stats: conn.latest_stats, + }) + .collect() + } +} + +fn update_all(conns: Conns) -> std::io::Result<()> { + let keys: Vec = conns.iter().map(|(k, _)| *k).collect(); + if keys.is_empty() { + return Ok(()); + } + + #[allow( + unused_variables, + reason = "when any of the platform-specific impls work, this will be shadowed" + )] + let stats: Vec<(ConnectionKey, TcpStats)> = Vec::new(); + + #[cfg(target_os = "linux")] + let stats = linux::query_tcp_stats(&keys)?; + + #[cfg(target_os = "macos")] + let stats = macos::query_tcp_stats(&keys)?; + + #[cfg(target_os = "windows")] + let stats = windows::query_tcp_stats(&keys)?; + + for (key, tcp_stats) in &stats { + update_stats(&conns, *key, *tcp_stats); + } + + Ok(()) +} + +fn update_stats(conns: &Conns, key: ConnectionKey, stats: TcpStats) { + conns.entry(key).and_compute_with(|entry| { + if let Some(entry) = entry { + let mut entry = entry.into_value(); + entry.latest_stats = Some(stats); + Op::Put(entry) + } else { + Op::Nop + } + }); +} diff --git a/crates/web-faith-napi/src/conn_tracker/linux.rs b/crates/web-faith-conn-tracker/src/platform/linux.rs similarity index 100% rename from crates/web-faith-napi/src/conn_tracker/linux.rs rename to crates/web-faith-conn-tracker/src/platform/linux.rs diff --git a/crates/web-faith-napi/src/conn_tracker/macos.rs b/crates/web-faith-conn-tracker/src/platform/macos.rs similarity index 100% rename from crates/web-faith-napi/src/conn_tracker/macos.rs rename to crates/web-faith-conn-tracker/src/platform/macos.rs diff --git a/crates/web-faith-napi/src/conn_tracker/windows.rs b/crates/web-faith-conn-tracker/src/platform/windows.rs similarity index 100% rename from crates/web-faith-napi/src/conn_tracker/windows.rs rename to crates/web-faith-conn-tracker/src/platform/windows.rs diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 7f556c8..3517f12 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -46,6 +46,7 @@ time.workspace = true tokio-util.workspace = true url.workspace = true web-faith.workspace = true +web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-encoding.workspace = true diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 69e9990..e82321c 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -32,6 +32,7 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use crate::alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] use crate::alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; +use web_faith_conn_tracker::ConnectionTracker; use web_faith_dns::{ DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, }; @@ -43,7 +44,7 @@ use web_faith_cookies::{ use crate::{ async_task::faith_promise, - conn_tracker::{ConnectionInfo, ConnectionTracker}, + conn_tracker::{ConnectionInfo, connections_for_napi}, error::{FaithError, FaithErrorExt, FaithErrorKind}, options::{PRIORITY, RequestCacheMode}, retry::{DeadConnectionRetry, StaleAddressRetry}, @@ -2091,7 +2092,7 @@ impl Agent { /// on field availability. If the platform isn't supported at all, this will always return empty. #[napi] pub fn connections<'env>(&self, env: &'env Env) -> Vec> { - self.conn_tracker.get_for_napi(env) + connections_for_napi(&self.conn_tracker, env) } /// Returns the DNS servers this agent resolves through, in the order they are queried, so diff --git a/crates/web-faith-napi/src/conn_tracker.rs b/crates/web-faith-napi/src/conn_tracker.rs index 2e764ed..9be32a1 100644 --- a/crates/web-faith-napi/src/conn_tracker.rs +++ b/crates/web-faith-napi/src/conn_tracker.rs @@ -1,83 +1,14 @@ -#[cfg(target_os = "linux")] -mod linux; -#[cfg(target_os = "macos")] -mod macos; -#[cfg(target_os = "windows")] -mod windows; +//! Handing the connection tracker's view to JavaScript. (spec:OBS) +//! +//! The tracking itself, and reading the operating system's statistics, is +//! [`web_faith_conn_tracker`]'s; what belongs here is turning a snapshot into an object V8 can +//! carry, which means JavaScript `Date`s for the timestamps and `i64` for every count. -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; -use moka::Expiry; -use moka::{ops::compute::Op, sync::Cache}; use napi::{Env, JsDate}; use napi_derive::napi; -use tokio::{spawn, task::AbortHandle, time::sleep}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct ConnectionKey { - pub local_addr: SocketAddr, - pub remote_addr: SocketAddr, -} - -#[derive(Debug, Clone)] -pub struct TrackedConnection { - pub first_seen: SystemTime, - pub last_seen: SystemTime, - pub response_count: u64, - pub latest_stats: Option, -} - -struct ExpireAfterTimeout(Duration); -impl Expiry for ExpireAfterTimeout { - fn expire_after_create( - &self, - _key: &ConnectionKey, - _value: &TrackedConnection, - _created_at: Instant, - ) -> Option { - Some(self.0) - } - - fn expire_after_read( - &self, - _key: &ConnectionKey, - value: &TrackedConnection, - _read_at: Instant, - _duration_until_expiry: Option, - _last_modified_at: Instant, - ) -> Option { - Some( - self.0 - .saturating_sub(value.last_seen.elapsed().unwrap_or_default()), - ) - } - - fn expire_after_update( - &self, - _key: &ConnectionKey, - value: &TrackedConnection, - _updated_at: Instant, - _duration_until_expiry: Option, - ) -> Option { - Some( - self.0 - .saturating_sub(value.last_seen.elapsed().unwrap_or_default()), - ) - } -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct TcpStats { - pub rtt_us: u32, - pub rtt_var_us: u32, - pub lost: Option, - pub retrans: u32, - pub total_retrans: u32, - pub cwnd: u32, - pub delivery_rate: Option, -} +use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; #[napi(object)] #[derive(Clone)] @@ -100,183 +31,48 @@ pub struct ConnectionInfo<'env> { pub delivery_rate_bps: Option, } -type Conns = Cache; - -#[derive(Debug)] -pub struct ConnectionTracker { - connections: Conns, - timeout: Duration, - task_abort: AbortHandle, -} - -impl Drop for ConnectionTracker { - fn drop(&mut self) { - self.task_abort.abort(); - } +/// A JavaScript `Date` for a point in time, which is milliseconds since the epoch. +/// +/// A timestamp from before the epoch yields the difference the error carries, matching what this +/// did when the conversion lived on the tracker. +fn js_date<'env>(env: &'env Env, at: SystemTime) -> Option> { + env.create_date( + at.duration_since(UNIX_EPOCH) + .unwrap_or_else(|err| err.duration()) + .as_secs_f64() + * 1000.0, + ) + .ok() } -impl ConnectionTracker { - pub fn new(timeout: Duration) -> Arc { - let connections = Cache::builder() - .expire_after(ExpireAfterTimeout(timeout)) - .build(); - - let conns = connections.clone(); - let task_abort = spawn(async move { - loop { - let _ = update_all(conns.clone()); - sleep(Duration::from_secs(1)).await; +/// The tracker's view, as `agent.connections()` returns it. +pub fn connections_for_napi<'env>( + tracker: &ConnectionTracker, + env: &'env Env, +) -> Vec> { + tracker + .snapshot() + .into_iter() + .map(|conn| { + let ConnectionSnapshot { stats, .. } = &conn; + ConnectionInfo { + connection_type: conn.connection_type.to_string(), + local_address: conn.local_addr.ip().to_string(), + local_port: conn.local_addr.port(), + remote_address: conn.remote_addr.ip().to_string(), + remote_port: conn.remote_addr.port(), + first_seen: js_date(env, conn.first_seen), + last_seen: js_date(env, conn.last_seen), + expiry: conn.expiry.and_then(|at| js_date(env, at)), + response_count: conn.response_count as i64, + rtt_us: stats.map(|s| s.rtt_us as i64), + rtt_var_us: stats.map(|s| s.rtt_var_us as i64), + lost_packets: stats.and_then(|s| s.lost.map(|v| v as i64)), + retransmits: stats.map(|s| s.retrans as i64), + total_retransmits: stats.map(|s| s.total_retrans as i64), + congestion_window: stats.map(|s| s.cwnd as i64), + delivery_rate_bps: stats.and_then(|s| s.delivery_rate.map(|v| v as i64)), } }) - .abort_handle(); - - Arc::new(Self { - connections, - timeout, - task_abort, - }) - } - - /// Record a response on a connection, returning whether that connection was already known. - /// - /// A connection the tracker has seen before is one the pool handed back rather than one - /// dialled for this request, since a fresh connection takes a local port of its own. - pub fn track(&self, local_addr: SocketAddr, remote_addr: SocketAddr) -> bool { - let now = SystemTime::now(); - let key = ConnectionKey { - local_addr, - remote_addr, - }; - let mut known = false; - self.connections.entry(key).and_compute_with(|entry| { - if let Some(entry) = entry { - known = true; - let mut conn = entry.into_value(); - conn.last_seen = now; - conn.response_count += 1; - Op::Put(conn) - } else { - Op::Put(TrackedConnection { - first_seen: now, - last_seen: now, - response_count: 1, - latest_stats: None, - }) - } - }); - known - } - - /// Register a warm-up connection that no request has yet been credited to. - /// - /// A `preconnect` connection is listed before any foreground request uses it, at a response - /// count of zero (spec:WARM). An entry already tracked is left untouched: a warm-up to an - /// origin that already holds a pooled connection does no new work, and must not disturb the - /// count or timestamps of the connection it would reuse. - pub fn track_warmup(&self, local_addr: SocketAddr, remote_addr: SocketAddr) { - let now = SystemTime::now(); - let key = ConnectionKey { - local_addr, - remote_addr, - }; - self.connections.entry(key).and_compute_with(|entry| { - if entry.is_some() { - Op::Nop - } else { - Op::Put(TrackedConnection { - first_seen: now, - last_seen: now, - response_count: 0, - latest_stats: None, - }) - } - }); - } - - pub fn get_for_napi<'env>(&self, env: &'env Env) -> Vec> { - self.connections - .iter() - .map(|(key, conn)| ConnectionInfo { - connection_type: "tcp".to_string(), - local_address: key.local_addr.ip().to_string(), - local_port: key.local_addr.port(), - remote_address: key.remote_addr.ip().to_string(), - remote_port: key.remote_addr.port(), - first_seen: env - .create_date( - conn.first_seen - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok(), - last_seen: env - .create_date( - conn.last_seen - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok(), - expiry: conn.last_seen.checked_add(self.timeout).and_then(|exp| { - env.create_date( - exp.duration_since(UNIX_EPOCH) - .unwrap_or_else(|err| err.duration()) - .as_secs_f64() * 1000.0, - ) - .ok() - }), - response_count: conn.response_count as i64, - rtt_us: conn.latest_stats.map(|s| s.rtt_us as i64), - rtt_var_us: conn.latest_stats.map(|s| s.rtt_var_us as i64), - lost_packets: conn.latest_stats.and_then(|s| s.lost.map(|v| v as i64)), - retransmits: conn.latest_stats.map(|s| s.retrans as i64), - total_retransmits: conn.latest_stats.map(|s| s.total_retrans as i64), - congestion_window: conn.latest_stats.map(|s| s.cwnd as i64), - delivery_rate_bps: conn - .latest_stats - .and_then(|s| s.delivery_rate.map(|v| v as i64)), - }) - .collect() - } -} - -fn update_all(conns: Conns) -> std::io::Result<()> { - let keys: Vec = conns.iter().map(|(k, _)| *k).collect(); - if keys.is_empty() { - return Ok(()); - } - - #[allow( - unused_variables, - reason = "when any of the platform-specific impls work, this will be shadowed" - )] - let stats: Vec<(ConnectionKey, TcpStats)> = Vec::new(); - - #[cfg(target_os = "linux")] - let stats = linux::query_tcp_stats(&keys)?; - - #[cfg(target_os = "macos")] - let stats = macos::query_tcp_stats(&keys)?; - - #[cfg(target_os = "windows")] - let stats = windows::query_tcp_stats(&keys)?; - - for (key, tcp_stats) in &stats { - update_stats(&conns, *key, *tcp_stats); - } - - Ok(()) -} - -fn update_stats(conns: &Conns, key: ConnectionKey, stats: TcpStats) { - conns.entry(key).and_compute_with(|entry| { - if let Some(entry) = entry { - let mut entry = entry.into_value(); - entry.latest_stats = Some(stats); - Op::Put(entry) - } else { - Op::Nop - } - }); + .collect() } From cbb4cfc3db507f80adcab56b9cdb25b1d39c70aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:07:49 +1200 Subject: [PATCH 16/61] S1: extract web-faith-alt-svc The Alt-Svc store and the HTTP/3 upgrade machinery move to a crate of its own, which draws on web-faith-dns for the resolution its probes need. It read the client's timing stamp out of the request extensions, so it needed a type from the layer above. It now names what it wants of that type as an ArrivalStamp trait and the middleware is generic over it: the stamp stays the client's, which is what reads it back to surface the timing, and this layer keeps being the one place a response's arrival is observed. The crate is pulled in by the http3 feature, so a build without HTTP/3 does not carry it. Both feature configurations build clean. This is the last of the six components; every one of them now compiles and tests with no JavaScript runtime in its graph. --- Cargo.lock | 15 +++++++ Cargo.toml | 1 + crates/web-faith-alt-svc/Cargo.toml | 21 ++++++++++ .../src/lib.rs} | 40 +++++++++++++------ crates/web-faith-napi/Cargo.toml | 3 +- crates/web-faith-napi/src/agent.rs | 10 +++-- crates/web-faith-napi/src/lib.rs | 2 - crates/web-faith-napi/src/timing.rs | 9 +++++ 8 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 crates/web-faith-alt-svc/Cargo.toml rename crates/{web-faith-napi/src/alt_svc.rs => web-faith-alt-svc/src/lib.rs} (98%) diff --git a/Cargo.lock b/Cargo.lock index c2b5aa5..2a0b394 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2804,6 +2804,20 @@ dependencies = [ "web-faith-integrity", ] +[[package]] +name = "web-faith-alt-svc" +version = "0.7.0" +dependencies = [ + "async-trait", + "http", + "moka", + "reqwest", + "reqwest-middleware", + "tokio", + "url", + "web-faith-dns", +] + [[package]] name = "web-faith-conn-tracker" version = "0.7.0" @@ -2898,6 +2912,7 @@ dependencies = [ "tokio-util", "url", "web-faith", + "web-faith-alt-svc", "web-faith-conn-tracker", "web-faith-cookies", "web-faith-dns", diff --git a/Cargo.toml b/Cargo.toml index 4423096..7ad12ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith-alt-svc = { version = "0.7.0", path = "crates/web-faith-alt-svc" } web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-tracker" } web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } diff --git a/crates/web-faith-alt-svc/Cargo.toml b/crates/web-faith-alt-svc/Cargo.toml new file mode 100644 index 0000000..d065d44 --- /dev/null +++ b/crates/web-faith-alt-svc/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "web-faith-alt-svc" +description = "An Alt-Svc store and the HTTP/3 upgrade machinery above it" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +# Flipped on once the release tooling is in place. +publish = false + +[dependencies] +async-trait.workspace = true +http.workspace = true +moka.workspace = true +reqwest.workspace = true +reqwest-middleware.workspace = true +tokio.workspace = true +url.workspace = true +web-faith-dns.workspace = true diff --git a/crates/web-faith-napi/src/alt_svc.rs b/crates/web-faith-alt-svc/src/lib.rs similarity index 98% rename from crates/web-faith-napi/src/alt_svc.rs rename to crates/web-faith-alt-svc/src/lib.rs index 2aed17a..8fc241f 100644 --- a/crates/web-faith-napi/src/alt_svc.rs +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -1,4 +1,5 @@ use std::{ + marker::PhantomData, sync::Arc, time::{Duration, Instant}, }; @@ -8,7 +9,14 @@ use moka::sync::Cache; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result}; -use crate::timing::HeadersStamp; +/// Recording the moment a response's headers arrived. +/// +/// The stamp itself belongs to the client, which puts it in the request's extensions and reads it +/// back out to surface the timing; this layer is only the one place that observes the arrival, so +/// all it needs is to be able to mark what the client left there. +pub trait ArrivalStamp: Send + Sync + 'static { + fn mark(&self, at: Instant); +} #[derive(Debug, Clone)] pub struct AltSvcEntry { @@ -1034,8 +1042,12 @@ impl web_faith_dns::HttpsSink for H3HttpsSink { } } +/// The Alt-Svc layer, which upgrades an origin to HTTP/3 once it advertises one. +/// +/// `S` is the client's arrival stamp, which this marks when a response's headers land; see +/// [`ArrivalStamp`]. #[derive(Clone)] -pub struct AltSvcMiddleware { +pub struct AltSvcMiddleware { cache: Arc, enabled: bool, /// Ceiling on how long an HTTP/3 attempt may take to produce response @@ -1045,9 +1057,10 @@ pub struct AltSvcMiddleware { /// advertisements in the background. `None` restores the inline upgrade, /// where the next foreground request is the verification. prober: Option>, + stamp: PhantomData, } -impl std::fmt::Debug for AltSvcMiddleware { +impl std::fmt::Debug for AltSvcMiddleware { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AltSvcMiddleware") .field("enabled", &self.enabled) @@ -1058,7 +1071,7 @@ impl std::fmt::Debug for AltSvcMiddleware { } } -impl AltSvcMiddleware { +impl AltSvcMiddleware { pub fn new( cache: Arc, enabled: bool, @@ -1070,6 +1083,7 @@ impl AltSvcMiddleware { enabled, attempt_timeout, prober, + stamp: PhantomData, } } @@ -1095,7 +1109,7 @@ impl AltSvcMiddleware { /// request's extensions to become the surfaced timing, so the two can never disagree. /// /// spec:RESP#request-timing -async fn run_stamped( +async fn run_stamped( next: Next<'_>, req: Request, extensions: &mut Extensions, @@ -1103,7 +1117,7 @@ async fn run_stamped( let result = next.run(req, extensions).await; let at = Instant::now(); if result.is_ok() - && let Some(stamp) = extensions.get::() + && let Some(stamp) = extensions.get::() { stamp.mark(at); } @@ -1111,7 +1125,7 @@ async fn run_stamped( } #[async_trait::async_trait] -impl Middleware for AltSvcMiddleware { +impl Middleware for AltSvcMiddleware { async fn handle( &self, mut req: Request, @@ -1119,7 +1133,7 @@ impl Middleware for AltSvcMiddleware { next: Next<'_>, ) -> Result { if !self.enabled { - return run_stamped(next, req, extensions).await.0; + return run_stamped::(next, req, extensions).await.0; } let url = req.url().clone(); @@ -1161,11 +1175,11 @@ impl Middleware for AltSvcMiddleware { // leaving the fallback below free to use it. let outcome = match self.attempt_timeout { Some(limit) => { - tokio::time::timeout(limit, run_stamped(next.clone(), req, extensions)) + tokio::time::timeout(limit, run_stamped::(next.clone(), req, extensions)) .await .ok() } - None => Some(run_stamped(next.clone(), req, extensions).await), + None => Some(run_stamped::(next.clone(), req, extensions).await), }; // Reached on success, error and expiry alike; only a mid-flight // drop skips it and leaves the guard armed. @@ -1201,7 +1215,7 @@ impl Middleware for AltSvcMiddleware { // Use the cloned request (which still has default HTTP version) let started = Instant::now(); - let (result, at) = run_stamped(next, req_clone, extensions).await; + let (result, at) = run_stamped::(next, req_clone, extensions).await; if let Ok(ref response) = result { self.cache.record_path_time( &url, @@ -1214,7 +1228,7 @@ impl Middleware for AltSvcMiddleware { } } else { // Can't clone request (streaming body), just proceed without HTTP/3 - run_stamped(next, req, extensions).await.0 + run_stamped::(next, req, extensions).await.0 } } else { // An advertisement from an earlier response may still be waiting on @@ -1223,7 +1237,7 @@ impl Middleware for AltSvcMiddleware { self.maybe_probe(&url); let started = Instant::now(); - let (result, at) = run_stamped(next, req, extensions).await; + let (result, at) = run_stamped::(next, req, extensions).await; // Check for Alt-Svc header in non-HTTP/3 responses if let Ok(ref response) = result { diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 3517f12..22c8077 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -46,6 +46,7 @@ time.workspace = true tokio-util.workspace = true url.workspace = true web-faith.workspace = true +web-faith-alt-svc = { workspace = true, optional = true } web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } @@ -65,4 +66,4 @@ napi-build.workspace = true [features] default = ["http3"] -http3 = ["reqwest/http3"] +http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index e82321c..09980aa 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -29,9 +29,11 @@ use reqwest::{ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; #[cfg(feature = "http3")] -use crate::alt_svc::parse_alt_svc_header; +use crate::timing::HeadersStamp; #[cfg(feature = "http3")] -use crate::alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; +use web_faith_alt_svc::parse_alt_svc_header; +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; use web_faith_conn_tracker::ConnectionTracker; use web_faith_dns::{ DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, @@ -1141,7 +1143,7 @@ fn install_https_sink( let (Some(resolver), Some(cache)) = (dns_resolver, alt_svc_cache) else { return; }; - resolver.set_https_sink(Arc::new(crate::alt_svc::H3HttpsSink::new( + resolver.set_https_sink(Arc::new(web_faith_alt_svc::H3HttpsSink::new( Arc::clone(cache), prober, ))); @@ -1345,7 +1347,7 @@ impl ClientRecipe { // rewrite cannot split HTTP/3 and TCP responses across separate entries. #[cfg(feature = "http3")] if let Some(alt_svc_cache) = alt_svc_cache { - client = client.with(AltSvcMiddleware::new( + client = client.with(AltSvcMiddleware::::new( alt_svc_cache.clone(), self.h3_upgrade.enabled, self.h3_upgrade.attempt_timeout, diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 3e9c9bd..8aed28f 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -1,6 +1,4 @@ mod agent; -#[cfg(feature = "http3")] -mod alt_svc; mod async_task; mod body; mod conn_tracker; diff --git a/crates/web-faith-napi/src/timing.rs b/crates/web-faith-napi/src/timing.rs index e8c2dc6..7e2e3b7 100644 --- a/crates/web-faith-napi/src/timing.rs +++ b/crates/web-faith-napi/src/timing.rs @@ -39,6 +39,15 @@ impl HeadersStamp { } } +/// The Alt-Svc layer is the one place a response's arrival is observed, so it marks the stamp the +/// request carries; reading it back out is this module's business. +#[cfg(feature = "http3")] +impl web_faith_alt_svc::ArrivalStamp for HeadersStamp { + fn mark(&self, at: Instant) { + HeadersStamp::mark(self, at); + } +} + /// What Faith measures of a request, filled in as the request progresses. #[derive(Clone, Debug, Default)] pub struct RequestTiming { From 266e1b2037e7e5b74eaa22ce8b396249b2ae3920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:08:33 +1200 Subject: [PATCH 17/61] S1: record the split's progress and decisions in the plan --- .workhorse/plans/s1/plan.md | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 0e47533..b937835 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -7,14 +7,12 @@ ships as `@passcod/faith`. Then publish to crates.io. Target architecture is spe ## Scope reality -This is a multi-week, 8-crate restructure of ~11,500 lines, not a single focused change. The -work almost certainly wants to become a **card breakdown** (one card per crate extraction plus a -publishing card), because a plan of this size does not survive as one green PR on `s1` and, per the -workspace norm, only breakdown entries survive a merge as real cards. The steps below are the build -order whichever way we sequence it — either as the checklist for a single long-lived branch, or as -the spine for the breakdown. +This is an 8-crate restructure of ~11,500 lines, not a single focused change. It is being built on +this branch as one long-lived PR, at the user's direction, rather than split into a card breakdown. -**Sequencing decision needed from the user before grinding the tree** (see the chat message). +The split itself (steps 0–7) is done: the workspace stands, and all six components plus the error +core are out. What remains (steps 8–13) is the larger half, and step 9 in particular is new API +design rather than relocation. ## What the discovery turned up @@ -66,11 +64,11 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al `web-faith`, with components naming their own error types that the client converts — per [RUST](../../specs/rust/overview.md) "A component crate stands alone"). - [x] **2. Extract `web-faith-integrity`** — own error type, own docs, `cargo test -p web-faith-integrity` with no JS runtime. -- [ ] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). -- [ ] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. -- [ ] **5. Extract `web-faith-dns`.** -- [ ] **6. Extract `web-faith-conn-tracker`** (Linux/macOS/Windows submodules). -- [ ] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. +- [x] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). +- [x] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. +- [x] **5. Extract `web-faith-dns`.** +- [x] **6. Extract `web-faith-conn-tracker`** (Linux/macOS/Windows submodules). +- [x] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. - [ ] **8. Stand up `web-faith`** — move agent/request/response/fetch/options here as a pure-Rust client; component crates converted into it at the boundary. Reduce `web-faith-napi` to the binding over `web-faith`. @@ -88,6 +86,33 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [ ] **13. First publish** to crates.io: the six components, then `web-faith`; `@passcod/faith` continues from npm via `web-faith-napi`. +## What the extractions settled + +Decisions taken while doing steps 0–7, worth not relitigating: + +- **The lib is still named `faith`.** `web-faith-napi`'s `[lib] name = "faith"` keeps the artifact + `libfaith.so`, which the release workflow's zigbuild steps copy by name. +- **The benchmark HTTP/3 server is a workspace `exclude`,** not a member: its own manifest says it + keeps a separate lockfile so the quinn/h3 stack stays out of this graph. Adding it as a member + would have pulled that stack in; leaving it unlisted broke it outright. +- **`FaithErrorKind` is gone from the native binding.** It was emitted only because the enum carried + napi's attribute. The package's `exports` map admits nothing but the wrapper, so no consumer could + reach it (verified: a deep import fails with `ERR_PACKAGE_PATH_NOT_EXPORTED`). The documented + surface is `wrapper.js`'s `ERROR_CODES`, unchanged at 22 codes. +- **Do not route a napi enum through `macro_rules!`.** Doc comments arrive as mangled `r"` literals + in the generated `.d.ts`, and the Rust name leaks into `index.js` alongside the `js_name`. This is + why the kinds have a single plain definition in `web-faith` and the codes reach JS through + `errorCodes()` instead. +- **`reqwest` integration sits behind a per-crate feature** on `web-faith-cookies` (`CookieStore`) + and `web-faith-dns` (`Resolve`). Where a trait impl held the only path to real functionality — the + jar's store-and-read — the logic moved to inherent methods and the impl delegates, so the crate is + usable without reqwest rather than merely compilable. +- **`alt_svc` takes the client's timing stamp as a generic,** via an `ArrivalStamp` trait, because + the stamp must exist in non-HTTP/3 builds where the alt-svc crate is not compiled at all. +- **`web-faith` only declares a component dependency once it uses it.** Right now that is + `integrity` alone (it has a real error conversion); the rest arrive with step 8. The full + feature-per-component set is step 10. + ## Verification discipline Every step must leave `cargo build`, `cargo test`, and the napi `npm run build` green From e1daad255ac43fda4d3849a78f00bedb03dfd80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:30:43 +1200 Subject: [PATCH 18/61] S1: move the body, timing, and retry machinery into web-faith The first of the client's own code to leave the binding crate. The body holder and its streams, the timing slot and what it measures, and the two retry layers in the request path were already free of napi, so they move as they are. Timing splits: the measuring is the client's, while the shape JavaScript receives it in stays behind as a napi object built from it. The client gains the http3 feature, since the stamp it owns is marked by the Alt-Svc layer, and the binding's own http3 feature now turns the client's on with it. Both feature configurations build clean. --- Cargo.lock | 10 ++ crates/web-faith-napi/Cargo.toml | 2 +- crates/web-faith-napi/src/agent.rs | 4 +- crates/web-faith-napi/src/fetch.rs | 6 +- crates/web-faith-napi/src/lib.rs | 2 - crates/web-faith-napi/src/response.rs | 7 +- crates/web-faith-napi/src/timing.rs | 143 +----------------- crates/web-faith/Cargo.toml | 14 +- .../{web-faith-napi => web-faith}/src/body.rs | 14 +- crates/web-faith/src/lib.rs | 12 +- .../src/retry.rs | 0 crates/web-faith/src/timing.rs | 141 +++++++++++++++++ 12 files changed, 196 insertions(+), 159 deletions(-) rename crates/{web-faith-napi => web-faith}/src/body.rs (92%) rename crates/{web-faith-napi => web-faith}/src/retry.rs (100%) create mode 100644 crates/web-faith/src/timing.rs diff --git a/Cargo.lock b/Cargo.lock index 2a0b394..54c0083 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2798,9 +2798,19 @@ dependencies = [ name = "web-faith" version = "0.7.0" dependencies = [ + "async-trait", + "bytes", + "futures", + "http", + "http-body-util", + "hyper", "reqwest", "reqwest-middleware", + "stream_shared", "strum", + "tokio", + "web-faith-alt-svc", + "web-faith-dns", "web-faith-integrity", ] diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 22c8077..8a65b5a 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -66,4 +66,4 @@ napi-build.workspace = true [features] default = ["http3"] -http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] +http3 = ["reqwest/http3", "dep:web-faith-alt-svc", "web-faith/http3"] diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 09980aa..254eadd 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -28,8 +28,9 @@ use reqwest::{ }; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use web_faith::retry::{DeadConnectionRetry, StaleAddressRetry}; #[cfg(feature = "http3")] -use crate::timing::HeadersStamp; +use web_faith::timing::HeadersStamp; #[cfg(feature = "http3")] use web_faith_alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] @@ -49,7 +50,6 @@ use crate::{ conn_tracker::{ConnectionInfo, connections_for_napi}, error::{FaithError, FaithErrorExt, FaithErrorKind}, options::{PRIORITY, RequestCacheMode}, - retry::{DeadConnectionRetry, StaleAddressRetry}, }; #[napi] diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index f71b0d8..be5e01b 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -20,16 +20,18 @@ use reqwest::{ }; use tokio::sync::{Mutex, mpsc}; +use web_faith::{ + body::{Body, BodyHolder}, + timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, +}; use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; use crate::{ async_task::faith_promise, - body::{Body, BodyHolder}, error::{FaithError, FaithErrorKind}, options::{CredentialsOption, FaithOptions, FaithOptionsAndBody, PRIORITY}, response::{FaithResponse, PeerInformation}, stream_body::StreamBody, - timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, }; /// The methods the fetch standard normalises to upper case; any other method is sent as given. diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 8aed28f..3f2f632 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -1,12 +1,10 @@ mod agent; mod async_task; -mod body; mod conn_tracker; mod error; mod fetch; mod options; mod response; -mod retry; mod stream_body; mod timing; diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 174d23d..32ba7c8 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -28,15 +28,18 @@ use serde_json; use stream_shared::SharedStream; use tokio::{io::AsyncWriteExt, sync::watch}; +use web_faith::{ + body::{Body, BodyHolder, DynStream, drain_body_inner}, + timing::TimingSlot, +}; use web_faith_encoding::{Coding, decode_stream}; use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; use crate::{ agent::InnerAgentStats, async_task::{Value, faith_promise}, - body::{Body, BodyHolder, DynStream, drain_body_inner}, error::{FaithError, FaithErrorExt, FaithErrorKind}, - timing::{TimingBreakdown, TimingSlot}, + timing::TimingBreakdown, }; /// The `Response` interface of the Fetch API represents the response to a request. diff --git a/crates/web-faith-napi/src/timing.rs b/crates/web-faith-napi/src/timing.rs index 7e2e3b7..4ec9e23 100644 --- a/crates/web-faith-napi/src/timing.rs +++ b/crates/web-faith-napi/src/timing.rs @@ -1,121 +1,12 @@ -//! Per-request timing, surfaced as a `PerformanceResourceTiming` by the wrapper. +//! Surfacing a request's timing as a `PerformanceResourceTiming` for the wrapper. +//! +//! The measuring itself is [`web_faith::timing`]'s; what belongs here is the shape JavaScript +//! receives. //! //! spec:RESP#request-timing -use std::{ - sync::{Arc, OnceLock}, - time::Instant, -}; - use napi_derive::napi; -use reqwest::{Url, Version}; -use tokio::sync::watch; - -/// The moment a response's headers arrived, shared between the middleware that observes it and -/// the request that surfaces it. -/// -/// Carried in the request's extensions so the one stamp taken inside the stack reaches the -/// outside, which is what keeps the surfaced timing and the path-time average reading the same -/// measurement rather than two of their own. -#[derive(Clone, Debug, Default)] -pub struct HeadersStamp(Arc>); - -impl HeadersStamp { - /// Record the arrival, if this is the first response to reach the outside. - /// - /// An HTTP/3 attempt that fails and falls back to TCP runs the stack twice, and only the - /// attempt that produced the response stamps, so the recorded moment always belongs to the - /// response the caller receives. - /// - /// The stamping lives in the Alt-Svc layer, which is only built with HTTP/3 support; without - /// it nothing stamps and the request falls back to timing the send itself. - #[cfg_attr(not(feature = "http3"), allow(dead_code))] - pub fn mark(&self, at: Instant) { - let _ = self.0.set(at); - } - - pub fn get(&self) -> Option { - self.0.get().copied() - } -} - -/// The Alt-Svc layer is the one place a response's arrival is observed, so it marks the stamp the -/// request carries; reading it back out is this module's business. -#[cfg(feature = "http3")] -impl web_faith_alt_svc::ArrivalStamp for HeadersStamp { - fn mark(&self, at: Instant) { - HeadersStamp::mark(self, at); - } -} - -/// What Faith measures of a request, filled in as the request progresses. -#[derive(Clone, Debug, Default)] -pub struct RequestTiming { - /// Milliseconds from the start of the request to the response headers arriving. - pub headers_ms: f64, - /// Milliseconds from the start of the request to the body finishing, once it has. - pub body_ms: Option, - /// Whether the request travelled on a connection that was already in the pool. - pub reused: bool, - /// The ALPN Protocol ID of the protocol the request travelled over. - pub next_hop_protocol: String, - /// The response's `Content-Encoding`, captured before a decoded body's header is stripped. - pub content_encoding: Option, - /// Whether the response was served by the HTTP cache. - pub from_cache: bool, -} - -/// Where the timing lands: written by whoever finishes the body, awaited by `timing()`. -/// -/// A watch channel for the same reason the trailers slot is one: the wait is unbounded by -/// design, since a body that is never read never finishes, and polling would burn a core to -/// find that out. -#[derive(Debug)] -pub struct TimingSlot { - tx: watch::Sender, - started: Instant, -} - -impl TimingSlot { - pub fn new(started: Instant, timing: RequestTiming) -> Self { - Self { - tx: watch::channel(timing).0, - started, - } - } - - /// Record that the body ended, if nothing got there first. - /// - /// `send_if_modified` so the read and the write are one step, and so waiters are woken only - /// by the call that actually settled it. Every route out of a body lands here: the stream - /// ending, `discard()`, and the collector draining one that was abandoned. - pub fn ended(&self) { - let elapsed = self.started.elapsed().as_secs_f64() * 1000.0; - self.tx.send_if_modified(|timing| { - if timing.body_ms.is_none() { - timing.body_ms = Some(elapsed); - true - } else { - false - } - }); - } - - /// Wait until the body has finished. - pub async fn settled(&self) -> RequestTiming { - let mut rx = self.tx.subscribe(); - // `wait_for` tests the current value before waiting, so a body that already finished - // returns without yielding. Its error case is the sender being gone, which means the - // response was dropped with the body unread: the phases reached before that are all - // there is to report, so report them rather than waiting for a moment that can no - // longer come. - let settled = match rx.wait_for(|timing| timing.body_ms.is_some()).await { - Ok(timing) => Some(timing.clone()), - Err(_) => None, - }; - settled.unwrap_or_else(|| rx.borrow().clone()) - } -} +use web_faith::timing::RequestTiming; /// The measurements behind a response's timing breakdown. /// @@ -145,27 +36,3 @@ impl From for TimingBreakdown { } } } - -/// The ALPN Protocol ID (RFC 7301) naming the protocol a response travelled over. -/// -/// Reported whether or not the connection negotiated over ALPN, which is what a browser does: -/// cleartext HTTP/2 is `h2c` and cleartext HTTP/1.1 is still `http/1.1`, neither of which any -/// handshake agreed on. -pub fn alpn_protocol_id(version: Version, url: &Url) -> String { - let secure = url.scheme() == "https"; - match version { - Version::HTTP_3 => "h3", - Version::HTTP_2 => { - if secure { - "h2" - } else { - "h2c" - } - } - Version::HTTP_11 => "http/1.1", - Version::HTTP_10 => "http/1.0", - Version::HTTP_09 => "http/0.9", - _ => "", - } - .to_owned() -} diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index d66da65..9d636b7 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -11,12 +11,24 @@ authors.workspace = true publish = false [dependencies] +async-trait.workspace = true +bytes.workspace = true +futures.workspace = true +http.workspace = true +http-body-util.workspace = true +hyper.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true +stream_shared.workspace = true strum.workspace = true +tokio.workspace = true +web-faith-dns = { workspace = true, features = ["reqwest"] } +web-faith-alt-svc = { workspace = true, optional = true } web-faith-integrity = { workspace = true, optional = true } [features] -default = ["integrity"] +default = ["http3", "integrity"] +# Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. +http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] # Subresource Integrity: parsing and verifying `integrity` values. integrity = ["dep:web-faith-integrity"] diff --git a/crates/web-faith-napi/src/body.rs b/crates/web-faith/src/body.rs similarity index 92% rename from crates/web-faith-napi/src/body.rs rename to crates/web-faith/src/body.rs index 1c33698..c443b0e 100644 --- a/crates/web-faith-napi/src/body.rs +++ b/crates/web-faith/src/body.rs @@ -17,23 +17,23 @@ use tokio::sync::Mutex; use crate::timing::TimingSlot; -pub(crate) type DynStream = dyn Stream> + Send + Sync; +pub type DynStream = dyn Stream> + Send + Sync; -pub(crate) enum Body { +pub enum Body { Inner(reqwest::Body), Consumed, Stream(SharedStream>>), } /// Wrapper around the body that auto-drains on drop to release the connection. -pub(crate) struct BodyHolder { +pub struct BodyHolder { pub body: Option>>, /// Flag to prevent drain if body was properly consumed - pub(crate) drained: Arc, + pub drained: Arc, /// HTTP version - HTTP/2+ doesn't need draining for connection reuse - pub(crate) version: Version, + pub version: Version, /// Settled when the body ends, so an abandoned body still finishes its timing - pub(crate) timing: Option>, + pub timing: Option>, } impl BodyHolder { @@ -143,7 +143,7 @@ impl Drop for BodyHolder { /// Drain a body to release the connection back to the pool. /// This reads and discards all remaining bytes. -pub(crate) async fn drain_body_inner(arc: Arc>) { +pub async fn drain_body_inner(arc: Arc>) { let mut guard = arc.lock().await; match replace(&mut *guard, Body::Consumed) { Body::Inner(body) => { diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index f26f792..f850780 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -1,10 +1,14 @@ //! A browser-shaped HTTP client: fetch semantics over a Rust network stack. //! -//! The client is being assembled here; for now this crate carries the error type the whole family -//! reports through. A component crate names its own errors for the failures it can produce, and -//! they are converted into [`FaithError`] as they cross into the client, so a caller matches on one -//! type whichever layer failed. +//! The client is being assembled here. What it already owns is the error type the whole family +//! reports through, the body and timing machinery a response is built on, and the retry layers that +//! sit in the request path. A component crate names its own errors for the failures it can produce, +//! and they are converted into [`FaithError`] as they cross into the client, so a caller matches on +//! one type whichever layer failed. +pub mod body; pub mod error; +pub mod retry; +pub mod timing; pub use error::{FaithError, FaithErrorKind, error_codes}; diff --git a/crates/web-faith-napi/src/retry.rs b/crates/web-faith/src/retry.rs similarity index 100% rename from crates/web-faith-napi/src/retry.rs rename to crates/web-faith/src/retry.rs diff --git a/crates/web-faith/src/timing.rs b/crates/web-faith/src/timing.rs new file mode 100644 index 0000000..2e2af8f --- /dev/null +++ b/crates/web-faith/src/timing.rs @@ -0,0 +1,141 @@ +//! Per-request timing. +//! +//! spec:RESP#request-timing + +use std::{ + sync::{Arc, OnceLock}, + time::Instant, +}; + +use reqwest::{Url, Version}; +use tokio::sync::watch; + +/// The moment a response's headers arrived, shared between the middleware that observes it and +/// the request that surfaces it. +/// +/// Carried in the request's extensions so the one stamp taken inside the stack reaches the +/// outside, which is what keeps the surfaced timing and the path-time average reading the same +/// measurement rather than two of their own. +#[derive(Clone, Debug, Default)] +pub struct HeadersStamp(Arc>); + +impl HeadersStamp { + /// Record the arrival, if this is the first response to reach the outside. + /// + /// An HTTP/3 attempt that fails and falls back to TCP runs the stack twice, and only the + /// attempt that produced the response stamps, so the recorded moment always belongs to the + /// response the caller receives. + /// + /// The stamping lives in the Alt-Svc layer, which is only built with HTTP/3 support; without + /// it nothing stamps and the request falls back to timing the send itself. + #[cfg_attr(not(feature = "http3"), allow(dead_code))] + pub fn mark(&self, at: Instant) { + let _ = self.0.set(at); + } + + pub fn get(&self) -> Option { + self.0.get().copied() + } +} + +/// The Alt-Svc layer is the one place a response's arrival is observed, so it marks the stamp the +/// request carries; reading it back out is this module's business. +#[cfg(feature = "http3")] +impl web_faith_alt_svc::ArrivalStamp for HeadersStamp { + fn mark(&self, at: Instant) { + HeadersStamp::mark(self, at); + } +} + +/// What Faith measures of a request, filled in as the request progresses. +#[derive(Clone, Debug, Default)] +pub struct RequestTiming { + /// Milliseconds from the start of the request to the response headers arriving. + pub headers_ms: f64, + /// Milliseconds from the start of the request to the body finishing, once it has. + pub body_ms: Option, + /// Whether the request travelled on a connection that was already in the pool. + pub reused: bool, + /// The ALPN Protocol ID of the protocol the request travelled over. + pub next_hop_protocol: String, + /// The response's `Content-Encoding`, captured before a decoded body's header is stripped. + pub content_encoding: Option, + /// Whether the response was served by the HTTP cache. + pub from_cache: bool, +} + +/// Where the timing lands: written by whoever finishes the body, awaited by `timing()`. +/// +/// A watch channel for the same reason the trailers slot is one: the wait is unbounded by +/// design, since a body that is never read never finishes, and polling would burn a core to +/// find that out. +#[derive(Debug)] +pub struct TimingSlot { + tx: watch::Sender, + started: Instant, +} + +impl TimingSlot { + pub fn new(started: Instant, timing: RequestTiming) -> Self { + Self { + tx: watch::channel(timing).0, + started, + } + } + + /// Record that the body ended, if nothing got there first. + /// + /// `send_if_modified` so the read and the write are one step, and so waiters are woken only + /// by the call that actually settled it. Every route out of a body lands here: the stream + /// ending, `discard()`, and the collector draining one that was abandoned. + pub fn ended(&self) { + let elapsed = self.started.elapsed().as_secs_f64() * 1000.0; + self.tx.send_if_modified(|timing| { + if timing.body_ms.is_none() { + timing.body_ms = Some(elapsed); + true + } else { + false + } + }); + } + + /// Wait until the body has finished. + pub async fn settled(&self) -> RequestTiming { + let mut rx = self.tx.subscribe(); + // `wait_for` tests the current value before waiting, so a body that already finished + // returns without yielding. Its error case is the sender being gone, which means the + // response was dropped with the body unread: the phases reached before that are all + // there is to report, so report them rather than waiting for a moment that can no + // longer come. + let settled = match rx.wait_for(|timing| timing.body_ms.is_some()).await { + Ok(timing) => Some(timing.clone()), + Err(_) => None, + }; + settled.unwrap_or_else(|| rx.borrow().clone()) + } +} + +/// The ALPN Protocol ID (RFC 7301) naming the protocol a response travelled over. +/// +/// Reported whether or not the connection negotiated over ALPN, which is what a browser does: +/// cleartext HTTP/2 is `h2c` and cleartext HTTP/1.1 is still `http/1.1`, neither of which any +/// handshake agreed on. +pub fn alpn_protocol_id(version: Version, url: &Url) -> String { + let secure = url.scheme() == "https"; + match version { + Version::HTTP_3 => "h3", + Version::HTTP_2 => { + if secure { + "h2" + } else { + "h2c" + } + } + Version::HTTP_11 => "http/1.1", + Version::HTTP_10 => "http/1.0", + Version::HTTP_09 => "http/0.9", + _ => "", + } + .to_owned() +} From 1ea452d9d50898a4ebca950444b8ac2467d42652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:40:03 +1200 Subject: [PATCH 19/61] S1: move the client-building machinery into web-faith The recipe that builds the agent's reqwest clients, the Node environment variables it layers on, the flow-control windows, and the HTTP cache store move to the client crate. This code was already free of napi by design: the recipe exists because a client has to be buildable more than once for a network change, and AgentOptions could not serve because it carries values belonging to the JS call that passed them. The redirect choice becomes the client's own RedirectPolicy. It carries no `manual` variant, that having never differed from `follow` here, and the Node surface's enum maps onto it, so JavaScript still takes every value it did. web-faith is now depended on with default features off, so the binding's http3 feature turns the client's on rather than the two drifting apart. Both configurations build clean and warning-free. --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/web-faith-napi/Cargo.toml | 2 +- crates/web-faith-napi/src/agent.rs | 460 ++-------------------------- crates/web-faith/Cargo.toml | 2 + crates/web-faith/src/client.rs | 477 +++++++++++++++++++++++++++++ crates/web-faith/src/lib.rs | 1 + 7 files changed, 505 insertions(+), 441 deletions(-) create mode 100644 crates/web-faith/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index 54c0083..c688b87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2803,6 +2803,7 @@ dependencies = [ "futures", "http", "http-body-util", + "http-cache-reqwest", "hyper", "reqwest", "reqwest-middleware", @@ -2810,6 +2811,7 @@ dependencies = [ "strum", "tokio", "web-faith-alt-svc", + "web-faith-cookies", "web-faith-dns", "web-faith-integrity", ] diff --git a/Cargo.toml b/Cargo.toml index 7ad12ec..e29a517 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ tokio-stream = "0.1.16" time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" -web-faith = { version = "0.7.0", path = "crates/web-faith" } +web-faith = { version = "0.7.0", path = "crates/web-faith", default-features = false } web-faith-alt-svc = { version = "0.7.0", path = "crates/web-faith-alt-svc" } web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-tracker" } web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 8a65b5a..de5e9a9 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -45,7 +45,7 @@ tokio-stream.workspace = true time.workspace = true tokio-util.workspace = true url.workspace = true -web-faith.workspace = true +web-faith = { workspace = true, features = ["integrity"] } web-faith-alt-svc = { workspace = true, optional = true } web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 254eadd..fe0b66f 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -13,8 +13,7 @@ use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; use http::Version; use http_cache_reqwest::{ - CACacheManager, Cache, CacheMode, CacheOptions, HttpCache, HttpCacheOptions, MokaCacheBuilder, - MokaManager, + CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, }; use hyper_util::client::legacy::connect::HttpInfo; use moka::sync::Cache as MokaCache; @@ -24,17 +23,19 @@ use reqwest::{ Certificate, Client, Identity, Url, cookie::CookieStore as _, header::{HeaderMap, HeaderName, HeaderValue}, - redirect::Policy, }; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use reqwest_middleware::ClientWithMiddleware; -use web_faith::retry::{DeadConnectionRetry, StaleAddressRetry}; +use web_faith::client::{ + ClientRecipe, DEFAULT_CONNECTION_WINDOW, DEFAULT_STREAM_WINDOW, HttpCacheRecipe, + HttpCacheStore, NodeEnvRecipe, RedirectPolicy, ResolvedWindows, +}; #[cfg(feature = "http3")] -use web_faith::timing::HeadersStamp; +use web_faith::client::{H3UpgradeRecipe, install_https_sink}; #[cfg(feature = "http3")] use web_faith_alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] -use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, AltSvcMiddleware, H3Prober}; +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, H3Prober}; use web_faith_conn_tracker::ConnectionTracker; use web_faith_dns::{ DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, @@ -91,78 +92,6 @@ fn ipv6_wildcard_bindable() -> bool { }) } -/// What the Node.js networking environment variables asked for, to apply to a reqwest client -/// builder as Node.js honours them for its own clients. This is read for every agent, so -/// `fetch()` behaves like Node's built-in fetch out of the box. -/// -/// - `NODE_EXTRA_CA_CERTS`: a path to a PEM file whose certificates are added to -/// the trust store on top of the platform roots. As in Node.js, a value that -/// is empty, or points at a file that cannot be read or parsed, is ignored -/// rather than fatal — unlike the explicit [`AgentTlsOptions::extra_roots`] -/// option, which throws. Certificates load in addition to any `extra_roots`. -/// -/// - `NODE_TLS_REJECT_UNAUTHORIZED`: when set to exactly `"0"`, TLS certificate -/// validation is disabled for the agent. This is insecure and exists only to -/// match Node.js semantics; any other value leaves validation enabled. -/// -/// - `NODE_USE_ENV_PROXY`: when set to exactly `"0"`, the agent ignores the -/// ambient proxy configuration (`HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` and the -/// OS proxy settings) that reqwest reads by default. Unlike Node.js — where -/// env-proxy support is opt-*in* and off by default — faith reads it by -/// default and treats this variable purely as an opt-*out* switch, so leaving -/// it unset (or `"1"`) keeps the existing always-on behaviour. -/// -/// `NODE_USE_SYSTEM_CA` is deliberately not honoured: faith bundles no Mozilla -/// root set, so its only default trust source is the platform store the variable -/// would toggle. `=0` could therefore only mean "trust almost nothing", which is -/// never what a caller wants, so the platform store is always used. -/// -/// Read once, at construction: AGENT has these layered on top of the explicit options when the -/// agent is built, so a client rebuilt later (spec:NETCHG) replays what was read then rather than -/// picking up an environment that has changed since. -#[derive(Debug, Clone, Default)] -struct NodeEnvRecipe { - extra_ca_certs: Vec, - accept_invalid_certs: bool, - no_proxy: bool, -} - -impl NodeEnvRecipe { - fn read() -> Self { - let mut recipe = Self::default(); - - if let Ok(path) = std::env::var("NODE_EXTRA_CA_CERTS") - && !path.is_empty() - && let Ok(bytes) = std::fs::read(&path) - && let Ok(certs) = Certificate::from_pem_bundle(&bytes) - { - recipe.extra_ca_certs = certs; - } - - recipe.accept_invalid_certs = - std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").as_deref() == Ok("0"); - recipe.no_proxy = std::env::var("NODE_USE_ENV_PROXY").as_deref() == Ok("0"); - - recipe - } - - fn apply(&self, mut client: reqwest::ClientBuilder) -> reqwest::ClientBuilder { - if !self.extra_ca_certs.is_empty() { - client = client.tls_certs_merge(self.extra_ca_certs.iter().cloned()); - } - - if self.accept_invalid_certs { - client = client.danger_accept_invalid_certs(true); - } - - if self.no_proxy { - client = client.no_proxy(); - } - - client - } -} - #[napi(string_enum)] #[derive(Debug, Clone, Copy)] pub enum CacheStore { @@ -675,28 +604,6 @@ pub struct AgentFlowControlOptions { pub connection_window: Option, } -/// Per-stream receive window applied to both protocols when nothing overrides it (spec:FLOW). -/// -/// Chrome's shape: 6 MiB stream inside a 15 MiB connection. Picked over a larger window that -/// measured faster because it is what browsers have proven at scale, and because a pooled -/// server-side client multiplies per-connection memory across far more connections. -pub(crate) const DEFAULT_STREAM_WINDOW: u32 = 6 * 1024 * 1024; - -/// Whole-connection receive window applied to both protocols when nothing overrides it (spec:FLOW). -pub(crate) const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; - -// Concurrent streams share the connection's headroom, so the asymmetry is the point of the -// defaults rather than an accident of the numbers (spec:FLOW#common-windows). -const _: () = assert!(DEFAULT_CONNECTION_WINDOW > DEFAULT_STREAM_WINDOW); - -/// The flow-control windows to apply, once the common group, the per-protocol overrides, and the -/// defaults have been reconciled (spec:FLOW#per-protocol-windows). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ResolvedWindows { - pub stream: u32, - pub connection: u32, -} - /// Reconcile one protocol's windows: its own setting wins over the common one, which wins over the /// default (spec:FLOW#per-protocol-windows). pub(crate) fn resolve_windows( @@ -776,6 +683,18 @@ pub enum Redirect { Stop, } +/// `manual` is not supported and behaves as `follow`, which is what the client's own policy spells +/// out: it carries no variant for a choice that never differed. +impl From for RedirectPolicy { + fn from(redirect: Redirect) -> Self { + match redirect { + Redirect::Follow | Redirect::Manual => Self::Follow, + Redirect::Error => Self::Error, + Redirect::Stop => Self::Stop, + } + } +} + /// Timeouts for requests made with this agent. This is a nested object. #[napi(object)] #[derive(Debug, Clone, Copy, Default)] @@ -1043,343 +962,6 @@ pub struct Agent { pub(crate) recipe: Arc, } -/// The HTTP cache store to install on a client, held as the built manager rather than as the -/// options that produced it. -/// -/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and -/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while -/// building a fresh one from the same options would empty an in-memory cache — which is why a -/// client rebuilt for a network change clones this (spec:NETCHG#what-the-signal-keeps). -#[derive(Debug, Clone)] -enum HttpCacheStore { - Disk(CACacheManager), - Memory(MokaManager), -} - -/// The HTTP cache middleware to install. -#[derive(Debug, Clone)] -struct HttpCacheRecipe { - mode: CacheMode, - options: HttpCacheOptions, - store: HttpCacheStore, -} - -/// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not -/// here: it belongs to the agent and outlives any one client (spec:NETCHG). -#[cfg(feature = "http3")] -#[derive(Debug, Clone)] -struct H3UpgradeRecipe { - enabled: bool, - attempt_timeout: Option, - probe: bool, - probe_timeout: Option, -} - -/// Everything needed to build the agent's clients, validated once at construction. -/// -/// This exists because `networkChanged` has to drop the connection pool, and reqwest offers no way -/// to do that short of dropping the client, so the client has to be buildable more than once -/// (spec:NETCHG). `AgentOptions` cannot serve: validating it consumes it, and it carries napi -/// values belonging to the JS call that passed them. So validation happens once, into these -/// Rust-native fields, and building a client is a pure function of them and the agent's shared -/// state. -#[derive(Debug, Clone)] -pub(crate) struct ClientRecipe { - user_agent: String, - local_address: Option, - default_headers: Option, - /// Under the system resolver no hickory resolver is installed at all (spec:DNS). - dns_system: bool, - /// Validated at construction, and applied whichever resolver is in use: reqwest layers - /// overrides on top of the resolver it was given (spec:DNS#overrides). - dns_overrides: Vec<(String, Vec)>, - http2_adaptive_window: bool, - /// `None` when adaptive windowing owns the windows itself (spec:FLOW#adaptive-windowing). - http2_windows: Option, - #[cfg(feature = "http3")] - http3_max_idle_timeout: Duration, - #[cfg(feature = "http3")] - http3_windows: ResolvedWindows, - #[cfg(feature = "http3")] - http3_congestion_bbr: bool, - #[cfg(feature = "http3")] - http3_send_window: Option, - pool_idle_timeout: Option, - pool_max_idle_per_host: Option, - redirect: Option, - connect_timeout: Option, - read_timeout: Option, - total_timeout: Option, - /// Only reachable over QUIC, so only applied when HTTP/3 is compiled in. - #[cfg(feature = "http3")] - tls_early_data: Option, - tls_identity: Option, - tls_required: Option, - tls_extra_roots: Vec, - node_env: NodeEnvRecipe, - http_cache: Option, - #[cfg(feature = "http3")] - h3_upgrade: H3UpgradeRecipe, -} - -/// Point the resolver's `HTTPS` record reading at the upgrade layer, so a record advertising -/// `alpn="h3"` makes an origin probe-worthy before anything has connected to it. -/// -/// A no-op without all the parts: the system resolver is not Faith's to add a query to, and with -/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one -/// (spec:DNS#https-records). -/// -/// Re-called on a network change, where the prober is rebuilt with the client it sends on. -#[cfg(feature = "http3")] -fn install_https_sink( - dns_resolver: Option<&FaithResolver>, - alt_svc_cache: Option<&Arc>, - prober: Option<&Arc>, - upgrade_enabled: bool, -) { - if !upgrade_enabled { - return; - } - let (Some(resolver), Some(cache)) = (dns_resolver, alt_svc_cache) else { - return; - }; - resolver.set_https_sink(Arc::new(web_faith_alt_svc::H3HttpsSink::new( - Arc::clone(cache), - prober, - ))); -} - -/// The clients [`ClientRecipe::build`] produces, and the prober that sends on them. -struct BuiltClients { - client: ClientWithMiddleware, - raw_client: Client, - #[cfg(feature = "http3")] - prober: Option>, -} - -impl ClientRecipe { - /// The window an idle pooled connection lives in, which is also how long a warm-up counts as - /// warm and how long a connection stays listed (spec:POOL, spec:WARM, spec:OBS). - fn conn_timeout(&self) -> Duration { - // reqwest's own default, mirrored because the pool timeout it applies is not readable. - self.pool_idle_timeout.unwrap_or(Duration::from_secs(90)) - } - - /// Build a fresh client and raw client, around state the agent already holds. - /// - /// Everything passed in survives a rebuild by being shared rather than rebuilt: the cookie - /// jar, the resolver (and so its cache), and the HTTP/3 origin knowledge all belong to the - /// agent rather than to any one client (spec:NETCHG#what-the-signal-keeps). - fn build( - &self, - cookie_jar: Option<&Arc>, - dns_resolver: Option<&FaithResolver>, - #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, - ) -> Result { - let mut client = Client::builder() - .tls_info(true) - .tls_sslkeylogfile(true) - .user_agent(self.user_agent.clone()); - - if let Some(ip) = self.local_address { - client = client.local_address(ip); - } - - if let Some(jar) = cookie_jar { - client = client.cookie_provider(jar.clone()); - } - - // Registered whichever resolver is in use: reqwest layers overrides on top of the - // resolver it was given, so they take effect under the system resolver too - // (spec:DNS#overrides). - for (domain, addresses) in &self.dns_overrides { - client = client.resolve_to_addrs(domain, addresses); - } - - if self.dns_system { - client = client.no_hickory_dns(); - } else if let Some(resolver) = dns_resolver { - client = client.dns_resolver(resolver.clone()); - } - - if let Some(headers) = &self.default_headers { - client = client.default_headers(headers.clone()); - } - - if self.http2_adaptive_window { - client = client.http2_adaptive_window(true); - } else if let Some(windows) = self.http2_windows { - client = client - .http2_initial_stream_window_size(windows.stream) - .http2_initial_connection_window_size(windows.connection); - } - - #[cfg(feature = "http3")] - { - client = client - .http3_max_idle_timeout(self.http3_max_idle_timeout) - .http3_stream_receive_window(self.http3_windows.stream.into()) - .http3_conn_receive_window(self.http3_windows.connection.into()); - - if self.http3_congestion_bbr { - client = client.http3_congestion_bbr(); - } - - if let Some(send_window) = self.http3_send_window { - client = client.http3_send_window(send_window.into()); - } - } - - if let Some(timeout) = self.pool_idle_timeout { - client = client.pool_idle_timeout(Some(timeout)); - } - - if let Some(max_idle) = self.pool_max_idle_per_host { - client = client.pool_max_idle_per_host(max_idle); - } - - match self.redirect { - // follow is the default, and we ignore manual - None | Some(Redirect::Follow | Redirect::Manual) => {} - Some(Redirect::Error) => { - client = client.redirect(Policy::custom(|attempt| { - // Hand reqwest the error unboxed: it boxes for us, and boxing first would - // put a `Box` in the source chain, which does not downcast - // back to `FaithError` when we come to recover the kind as a `code`. - attempt.error(FaithError::from(FaithErrorKind::Redirect)) - })); - } - Some(Redirect::Stop) => { - client = client.redirect(Policy::none()); - } - } - - if let Some(timeout) = self.connect_timeout { - client = client.connect_timeout(timeout); - } - - if let Some(timeout) = self.read_timeout { - client = client.read_timeout(timeout); - } - - if let Some(timeout) = self.total_timeout { - client = client.timeout(timeout); - } - - #[cfg(feature = "http3")] - if let Some(early_data) = self.tls_early_data { - client = client.tls_early_data(early_data); - } - - if let Some(identity) = &self.tls_identity { - client = client.identity(identity.clone()); - } - - if let Some(https_only) = self.tls_required { - client = client.https_only(https_only); - } - - if !self.tls_extra_roots.is_empty() { - client = client.tls_certs_merge(self.tls_extra_roots.iter().cloned()); - } - - client = self.node_env.apply(client); - - let raw_client = client - .build() - .map_err(|e| FaithError::new(FaithErrorKind::Config, Some(format!("{e:?}"))))?; - let mut client = ClientBuilder::new(raw_client.clone()); - - #[cfg(feature = "http3")] - let prober = { - // The prober sends on the *raw* client, deliberately: it must skip - // the HTTP cache (a replayed cached response would fake a - // confirmation) and the Alt-Svc middleware (no recursion), while - // sharing the h3 connection pool so a successful probe leaves a warm - // connection for the foreground. Only built when both the upgrade - // machinery and probing are on. - alt_svc_cache - .filter(|_| self.h3_upgrade.enabled && self.h3_upgrade.probe) - .map(|cache| { - Arc::new(H3Prober::new( - raw_client.clone(), - cache.clone(), - self.h3_upgrade.probe_timeout, - )) - }) - }; - - if let Some(cache) = &self.http_cache { - // The two arms differ only in the manager's type, which `HttpCache` is generic over, - // so they cannot share a constructor without boxing the manager. - client = match &cache.store { - HttpCacheStore::Disk(manager) => client.with(Cache(HttpCache { - mode: cache.mode, - manager: manager.clone(), - options: cache.options.clone(), - })), - HttpCacheStore::Memory(manager) => client.with(Cache(HttpCache { - mode: cache.mode, - manager: manager.clone(), - options: cache.options.clone(), - })), - }; - } - - // Registered *after* the HTTP cache, so the Alt-Svc layer sits inside it: - // `reqwest-middleware` runs the first-registered middleware outermost. Being - // inside matters three times over. - // - // A cache hit is served without calling inward, so it never reaches this - // layer. From outside, it would: `http-cache` rebuilds a cached response with - // the *stored* HTTP version, so a response cached from an HTTP/3 exchange - // replays as HTTP/3 and would be taken for a live one — confirming HTTP/3, - // clearing cancellation strikes and refreshing the confirmed TTL on evidence - // that never touched the network. - // - // The cache middleware also buffers the whole response body inside its own - // call inward. From outside, the HTTP/3 attempt guarded here would span that - // buffering, so a cancellation during body download would count as a strike, - // and `upgradeAttemptTimeout` would bound body transfer rather than the wait - // for response headers. - // - // And cache keys are computed before this layer runs, so an advertised-port - // rewrite cannot split HTTP/3 and TCP responses across separate entries. - #[cfg(feature = "http3")] - if let Some(alt_svc_cache) = alt_svc_cache { - client = client.with(AltSvcMiddleware::::new( - alt_svc_cache.clone(), - self.h3_upgrade.enabled, - self.h3_upgrade.attempt_timeout, - prober.clone(), - )); - } - - // Outside the dead-connection layer, so a re-resolved attempt gets the same - // treatment as the original one: the two answer different questions, and a - // fresh address deserves its own chance to draw a dead pooled connection. - // Inside the Alt-Svc and cache layers for the reason given below. - client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); - - // Registered last, so it sits innermost and wraps nothing but the exchange - // itself. Inside the Alt-Svc layer rather than outside it, because each - // protocol attempt is its own connection and deserves its own retry: a - // failed HTTP/3 attempt is the fallback's business, and re-running the - // upgrade decision from out here would re-attempt HTTP/3 on a path already - // judged dead and record a second failure against the origin for it. Inside - // the HTTP cache for the same reason as the Alt-Svc layer -- a retry should - // re-send the request, not redo the cache lookup that led to it. - client = client.with(DeadConnectionRetry); - - Ok(BuiltClients { - client: client.build(), - raw_client, - #[cfg(feature = "http3")] - prober, - }) - } -} - #[napi] impl Agent { pub fn new() -> Result { @@ -1819,7 +1401,7 @@ impl Agent { http3_send_window, pool_idle_timeout, pool_max_idle_per_host, - redirect, + redirect: redirect.map(RedirectPolicy::from), connect_timeout, read_timeout, total_timeout, diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 9d636b7..91cc085 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -16,12 +16,14 @@ bytes.workspace = true futures.workspace = true http.workspace = true http-body-util.workspace = true +http-cache-reqwest.workspace = true hyper.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true stream_shared.workspace = true strum.workspace = true tokio.workspace = true +web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-alt-svc = { workspace = true, optional = true } web-faith-integrity = { workspace = true, optional = true } diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs new file mode 100644 index 0000000..85a2e78 --- /dev/null +++ b/crates/web-faith/src/client.rs @@ -0,0 +1,477 @@ +//! Building the agent's reqwest clients from validated options. +//! +//! The recipe here is what lets a client be rebuilt: `network_changed` has to drop the connection +//! pool, and reqwest offers no way to do that short of dropping the client, so building one is a +//! pure function of settings that were validated once (spec:NETCHG). + +use std::{ + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; + +use http::header::HeaderMap; +use http_cache_reqwest::{ + CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions, MokaManager, +}; +use reqwest::{Client, Identity, redirect::Policy, tls::Certificate}; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use web_faith_cookies::FaithJar; +use web_faith_dns::FaithResolver; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, AltSvcMiddleware, H3Prober}; + +use crate::{ + error::{FaithError, FaithErrorKind}, + retry::{DeadConnectionRetry, StaleAddressRetry}, +}; + +#[cfg(feature = "http3")] +use crate::timing::HeadersStamp; + +/// What to do with a redirect response. +/// +/// The Node surface spells these as fetch's own `redirect` values; this is the same choice in the +/// client's own terms. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum RedirectPolicy { + /// Follow redirects, up to the standard's limit. + #[default] + Follow, + /// Refuse a redirect, reporting it as an error. + Error, + /// Return the redirect response itself rather than following it. + Stop, +} + +/// Per-stream receive window applied to both protocols when nothing overrides it (spec:FLOW). +/// +/// Chrome's shape: 6 MiB stream inside a 15 MiB connection. Picked over a larger window that +/// measured faster because it is what browsers have proven at scale, and because a pooled +/// server-side client multiplies per-connection memory across far more connections. +pub const DEFAULT_STREAM_WINDOW: u32 = 6 * 1024 * 1024; + +/// Whole-connection receive window applied to both protocols when nothing overrides it (spec:FLOW). +pub const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; + +// Concurrent streams share the connection's headroom, so the asymmetry is the point of the +// defaults rather than an accident of the numbers (spec:FLOW#common-windows). +const _: () = assert!(DEFAULT_CONNECTION_WINDOW > DEFAULT_STREAM_WINDOW); + +/// The flow-control windows to apply, once the common group, the per-protocol overrides, and the +/// defaults have been reconciled (spec:FLOW#per-protocol-windows). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResolvedWindows { + pub stream: u32, + pub connection: u32, +} + +/// What the Node.js networking environment variables asked for, to apply to a reqwest client +/// builder as Node.js honours them for its own clients. This is read for every agent, so +/// `fetch()` behaves like Node's built-in fetch out of the box. +/// +/// - `NODE_EXTRA_CA_CERTS`: a path to a PEM file whose certificates are added to +/// the trust store on top of the platform roots. As in Node.js, a value that +/// is empty, or points at a file that cannot be read or parsed, is ignored +/// rather than fatal — unlike the explicit [`AgentTlsOptions::extra_roots`] +/// option, which throws. Certificates load in addition to any `extra_roots`. +/// +/// - `NODE_TLS_REJECT_UNAUTHORIZED`: when set to exactly `"0"`, TLS certificate +/// validation is disabled for the agent. This is insecure and exists only to +/// match Node.js semantics; any other value leaves validation enabled. +/// +/// - `NODE_USE_ENV_PROXY`: when set to exactly `"0"`, the agent ignores the +/// ambient proxy configuration (`HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` and the +/// OS proxy settings) that reqwest reads by default. Unlike Node.js — where +/// env-proxy support is opt-*in* and off by default — faith reads it by +/// default and treats this variable purely as an opt-*out* switch, so leaving +/// it unset (or `"1"`) keeps the existing always-on behaviour. +/// +/// `NODE_USE_SYSTEM_CA` is deliberately not honoured: faith bundles no Mozilla +/// root set, so its only default trust source is the platform store the variable +/// would toggle. `=0` could therefore only mean "trust almost nothing", which is +/// never what a caller wants, so the platform store is always used. +/// +/// Read once, at construction: AGENT has these layered on top of the explicit options when the +/// agent is built, so a client rebuilt later (spec:NETCHG) replays what was read then rather than +/// picking up an environment that has changed since. +#[derive(Debug, Clone, Default)] +pub struct NodeEnvRecipe { + extra_ca_certs: Vec, + accept_invalid_certs: bool, + no_proxy: bool, +} + +impl NodeEnvRecipe { + pub fn read() -> Self { + let mut recipe = Self::default(); + + if let Ok(path) = std::env::var("NODE_EXTRA_CA_CERTS") + && !path.is_empty() + && let Ok(bytes) = std::fs::read(&path) + && let Ok(certs) = Certificate::from_pem_bundle(&bytes) + { + recipe.extra_ca_certs = certs; + } + + recipe.accept_invalid_certs = + std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").as_deref() == Ok("0"); + recipe.no_proxy = std::env::var("NODE_USE_ENV_PROXY").as_deref() == Ok("0"); + + recipe + } + + fn apply(&self, mut client: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + if !self.extra_ca_certs.is_empty() { + client = client.tls_certs_merge(self.extra_ca_certs.iter().cloned()); + } + + if self.accept_invalid_certs { + client = client.danger_accept_invalid_certs(true); + } + + if self.no_proxy { + client = client.no_proxy(); + } + + client + } +} + +/// The HTTP cache store to install on a client, held as the built manager rather than as the +/// options that produced it. +/// +/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and +/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while +/// building a fresh one from the same options would empty an in-memory cache — which is why a +/// client rebuilt for a network change clones this (spec:NETCHG#what-the-signal-keeps). +#[derive(Debug, Clone)] +pub enum HttpCacheStore { + Disk(CACacheManager), + Memory(MokaManager), +} + +/// The HTTP cache middleware to install. +#[derive(Debug, Clone)] +pub struct HttpCacheRecipe { + pub mode: CacheMode, + pub options: HttpCacheOptions, + pub store: HttpCacheStore, +} + +/// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not +/// here: it belongs to the agent and outlives any one client (spec:NETCHG). +#[cfg(feature = "http3")] +#[derive(Debug, Clone)] +pub struct H3UpgradeRecipe { + pub enabled: bool, + pub attempt_timeout: Option, + pub probe: bool, + pub probe_timeout: Option, +} + +/// Everything needed to build the agent's clients, validated once at construction. +/// +/// This exists because `networkChanged` has to drop the connection pool, and reqwest offers no way +/// to do that short of dropping the client, so the client has to be buildable more than once +/// (spec:NETCHG). `AgentOptions` cannot serve: validating it consumes it, and it carries napi +/// values belonging to the JS call that passed them. So validation happens once, into these +/// Rust-native fields, and building a client is a pure function of them and the agent's shared +/// state. +#[derive(Debug, Clone)] +pub struct ClientRecipe { + pub user_agent: String, + pub local_address: Option, + pub default_headers: Option, + /// Under the system resolver no hickory resolver is installed at all (spec:DNS). + pub dns_system: bool, + /// Validated at construction, and applied whichever resolver is in use: reqwest layers + /// overrides on top of the resolver it was given (spec:DNS#overrides). + pub dns_overrides: Vec<(String, Vec)>, + pub http2_adaptive_window: bool, + /// `None` when adaptive windowing owns the windows itself (spec:FLOW#adaptive-windowing). + pub http2_windows: Option, + #[cfg(feature = "http3")] + pub http3_max_idle_timeout: Duration, + #[cfg(feature = "http3")] + pub http3_windows: ResolvedWindows, + #[cfg(feature = "http3")] + pub http3_congestion_bbr: bool, + #[cfg(feature = "http3")] + pub http3_send_window: Option, + pub pool_idle_timeout: Option, + pub pool_max_idle_per_host: Option, + pub redirect: Option, + pub connect_timeout: Option, + pub read_timeout: Option, + pub total_timeout: Option, + /// Only reachable over QUIC, so only applied when HTTP/3 is compiled in. + #[cfg(feature = "http3")] + pub tls_early_data: Option, + pub tls_identity: Option, + pub tls_required: Option, + pub tls_extra_roots: Vec, + pub node_env: NodeEnvRecipe, + pub http_cache: Option, + #[cfg(feature = "http3")] + pub h3_upgrade: H3UpgradeRecipe, +} + +/// Point the resolver's `HTTPS` record reading at the upgrade layer, so a record advertising +/// `alpn="h3"` makes an origin probe-worthy before anything has connected to it. +/// +/// A no-op without all the parts: the system resolver is not Faith's to add a query to, and with +/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one +/// (spec:DNS#https-records). +/// +/// Re-called on a network change, where the prober is rebuilt with the client it sends on. +#[cfg(feature = "http3")] +pub fn install_https_sink( + dns_resolver: Option<&FaithResolver>, + alt_svc_cache: Option<&Arc>, + prober: Option<&Arc>, + upgrade_enabled: bool, +) { + if !upgrade_enabled { + return; + } + let (Some(resolver), Some(cache)) = (dns_resolver, alt_svc_cache) else { + return; + }; + resolver.set_https_sink(Arc::new(web_faith_alt_svc::H3HttpsSink::new( + Arc::clone(cache), + prober, + ))); +} + +/// The clients [`ClientRecipe::build`] produces, and the prober that sends on them. +pub struct BuiltClients { + pub client: ClientWithMiddleware, + pub raw_client: Client, + #[cfg(feature = "http3")] + pub prober: Option>, +} + +impl ClientRecipe { + /// The window an idle pooled connection lives in, which is also how long a warm-up counts as + /// warm and how long a connection stays listed (spec:POOL, spec:WARM, spec:OBS). + pub fn conn_timeout(&self) -> Duration { + // reqwest's own default, mirrored because the pool timeout it applies is not readable. + self.pool_idle_timeout.unwrap_or(Duration::from_secs(90)) + } + + /// Build a fresh client and raw client, around state the agent already holds. + /// + /// Everything passed in survives a rebuild by being shared rather than rebuilt: the cookie + /// jar, the resolver (and so its cache), and the HTTP/3 origin knowledge all belong to the + /// agent rather than to any one client (spec:NETCHG#what-the-signal-keeps). + pub fn build( + &self, + cookie_jar: Option<&Arc>, + dns_resolver: Option<&FaithResolver>, + #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, + ) -> Result { + let mut client = Client::builder() + .tls_info(true) + .tls_sslkeylogfile(true) + .user_agent(self.user_agent.clone()); + + if let Some(ip) = self.local_address { + client = client.local_address(ip); + } + + if let Some(jar) = cookie_jar { + client = client.cookie_provider(jar.clone()); + } + + // Registered whichever resolver is in use: reqwest layers overrides on top of the + // resolver it was given, so they take effect under the system resolver too + // (spec:DNS#overrides). + for (domain, addresses) in &self.dns_overrides { + client = client.resolve_to_addrs(domain, addresses); + } + + if self.dns_system { + client = client.no_hickory_dns(); + } else if let Some(resolver) = dns_resolver { + client = client.dns_resolver(resolver.clone()); + } + + if let Some(headers) = &self.default_headers { + client = client.default_headers(headers.clone()); + } + + if self.http2_adaptive_window { + client = client.http2_adaptive_window(true); + } else if let Some(windows) = self.http2_windows { + client = client + .http2_initial_stream_window_size(windows.stream) + .http2_initial_connection_window_size(windows.connection); + } + + #[cfg(feature = "http3")] + { + client = client + .http3_max_idle_timeout(self.http3_max_idle_timeout) + .http3_stream_receive_window(self.http3_windows.stream.into()) + .http3_conn_receive_window(self.http3_windows.connection.into()); + + if self.http3_congestion_bbr { + client = client.http3_congestion_bbr(); + } + + if let Some(send_window) = self.http3_send_window { + client = client.http3_send_window(send_window.into()); + } + } + + if let Some(timeout) = self.pool_idle_timeout { + client = client.pool_idle_timeout(Some(timeout)); + } + + if let Some(max_idle) = self.pool_max_idle_per_host { + client = client.pool_max_idle_per_host(max_idle); + } + + match self.redirect { + // follow is the default, and we ignore manual + None | Some(RedirectPolicy::Follow) => {} + Some(RedirectPolicy::Error) => { + client = client.redirect(Policy::custom(|attempt| { + // Hand reqwest the error unboxed: it boxes for us, and boxing first would + // put a `Box` in the source chain, which does not downcast + // back to `FaithError` when we come to recover the kind as a `code`. + attempt.error(FaithError::from(FaithErrorKind::Redirect)) + })); + } + Some(RedirectPolicy::Stop) => { + client = client.redirect(Policy::none()); + } + } + + if let Some(timeout) = self.connect_timeout { + client = client.connect_timeout(timeout); + } + + if let Some(timeout) = self.read_timeout { + client = client.read_timeout(timeout); + } + + if let Some(timeout) = self.total_timeout { + client = client.timeout(timeout); + } + + #[cfg(feature = "http3")] + if let Some(early_data) = self.tls_early_data { + client = client.tls_early_data(early_data); + } + + if let Some(identity) = &self.tls_identity { + client = client.identity(identity.clone()); + } + + if let Some(https_only) = self.tls_required { + client = client.https_only(https_only); + } + + if !self.tls_extra_roots.is_empty() { + client = client.tls_certs_merge(self.tls_extra_roots.iter().cloned()); + } + + client = self.node_env.apply(client); + + let raw_client = client + .build() + .map_err(|e| FaithError::new(FaithErrorKind::Config, Some(format!("{e:?}"))))?; + let mut client = ClientBuilder::new(raw_client.clone()); + + #[cfg(feature = "http3")] + let prober = { + // The prober sends on the *raw* client, deliberately: it must skip + // the HTTP cache (a replayed cached response would fake a + // confirmation) and the Alt-Svc middleware (no recursion), while + // sharing the h3 connection pool so a successful probe leaves a warm + // connection for the foreground. Only built when both the upgrade + // machinery and probing are on. + alt_svc_cache + .filter(|_| self.h3_upgrade.enabled && self.h3_upgrade.probe) + .map(|cache| { + Arc::new(H3Prober::new( + raw_client.clone(), + cache.clone(), + self.h3_upgrade.probe_timeout, + )) + }) + }; + + if let Some(cache) = &self.http_cache { + // The two arms differ only in the manager's type, which `HttpCache` is generic over, + // so they cannot share a constructor without boxing the manager. + client = match &cache.store { + HttpCacheStore::Disk(manager) => client.with(Cache(HttpCache { + mode: cache.mode, + manager: manager.clone(), + options: cache.options.clone(), + })), + HttpCacheStore::Memory(manager) => client.with(Cache(HttpCache { + mode: cache.mode, + manager: manager.clone(), + options: cache.options.clone(), + })), + }; + } + + // Registered *after* the HTTP cache, so the Alt-Svc layer sits inside it: + // `reqwest-middleware` runs the first-registered middleware outermost. Being + // inside matters three times over. + // + // A cache hit is served without calling inward, so it never reaches this + // layer. From outside, it would: `http-cache` rebuilds a cached response with + // the *stored* HTTP version, so a response cached from an HTTP/3 exchange + // replays as HTTP/3 and would be taken for a live one — confirming HTTP/3, + // clearing cancellation strikes and refreshing the confirmed TTL on evidence + // that never touched the network. + // + // The cache middleware also buffers the whole response body inside its own + // call inward. From outside, the HTTP/3 attempt guarded here would span that + // buffering, so a cancellation during body download would count as a strike, + // and `upgradeAttemptTimeout` would bound body transfer rather than the wait + // for response headers. + // + // And cache keys are computed before this layer runs, so an advertised-port + // rewrite cannot split HTTP/3 and TCP responses across separate entries. + #[cfg(feature = "http3")] + if let Some(alt_svc_cache) = alt_svc_cache { + client = client.with(AltSvcMiddleware::::new( + alt_svc_cache.clone(), + self.h3_upgrade.enabled, + self.h3_upgrade.attempt_timeout, + prober.clone(), + )); + } + + // Outside the dead-connection layer, so a re-resolved attempt gets the same + // treatment as the original one: the two answer different questions, and a + // fresh address deserves its own chance to draw a dead pooled connection. + // Inside the Alt-Svc and cache layers for the reason given below. + client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); + + // Registered last, so it sits innermost and wraps nothing but the exchange + // itself. Inside the Alt-Svc layer rather than outside it, because each + // protocol attempt is its own connection and deserves its own retry: a + // failed HTTP/3 attempt is the fallback's business, and re-running the + // upgrade decision from out here would re-attempt HTTP/3 on a path already + // judged dead and record a second failure against the origin for it. Inside + // the HTTP cache for the same reason as the Alt-Svc layer -- a retry should + // re-send the request, not redo the cache lookup that led to it. + client = client.with(DeadConnectionRetry); + + Ok(BuiltClients { + client: client.build(), + raw_client, + #[cfg(feature = "http3")] + prober, + }) + } +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index f850780..00559e5 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -7,6 +7,7 @@ //! one type whichever layer failed. pub mod body; +pub mod client; pub mod error; pub mod retry; pub mod timing; From a93fad10e0e0f81f05ba795ca9b17808b5729f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:40:50 +1200 Subject: [PATCH 20/61] S1: record step 8's progress and what the Agent inversion needs --- .workhorse/plans/s1/plan.md | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index b937835..91ddc02 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -71,7 +71,25 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [x] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. - [ ] **8. Stand up `web-faith`** — move agent/request/response/fetch/options here as a pure-Rust client; component crates converted into it at the boundary. Reduce `web-faith-napi` to the binding - over `web-faith`. + over `web-faith`. **In progress:** + - [x] `body`, `timing` (measuring; the napi object stays behind), `retry` moved. + - [x] The client-building machinery moved: `ClientRecipe`, `NodeEnvRecipe`, `HttpCacheRecipe`, + `HttpCacheStore`, `H3UpgradeRecipe`, `ResolvedWindows`, `install_https_sink`, and a pure + `RedirectPolicy` the napi `Redirect` converts into. + - [ ] **The `Agent` inversion — the big remaining piece.** `agent.rs` is down to ~2140 lines: + roughly 860 of `#[napi(object)]` option structs (JS-facing, they stay), ~100 of the `Agent` + struct, and ~1170 of `#[napi] impl Agent` holding 22 methods. The struct's fields are + already pure (reqwest/moka/Arc), so the blocker is that `#[napi] impl` cannot target a + foreign type: moving `Agent` to `web-faith` forces the napi `Agent` to become a distinct + class wrapping it, in the same change. Each of the 22 methods then splits into a pure core + method on `web_faith::Agent` and a thin binding that converts — `close`, `network_changed`, + `stats`, `connections`, `resolvers`, `prefetch_dns`, `preconnect`, `cookies` are the verbs + [RSAPI](../../specs/rust/client-api.md) names, so this is where the real client API starts + rather than a mechanical relocation. + - [ ] `response.rs` (~966 lines, 40 napi refs) and `fetch.rs` (~447) follow the agent, since both + are built around the napi response class. + - [ ] `options.rs` (~246) stays largely JS-facing; the recipe fields it assembles become the + builder's business in step 9. - [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, @@ -109,9 +127,15 @@ Decisions taken while doing steps 0–7, worth not relitigating: usable without reqwest rather than merely compilable. - **`alt_svc` takes the client's timing stamp as a generic,** via an `ArrivalStamp` trait, because the stamp must exist in non-HTTP/3 builds where the alt-svc crate is not compiled at all. -- **`web-faith` only declares a component dependency once it uses it.** Right now that is - `integrity` alone (it has a real error conversion); the rest arrive with step 8. The full +- **`web-faith` only declares a component dependency once it uses it.** The full feature-per-component set is step 10. +- **`web-faith` is depended on with `default-features = false`,** set at the workspace root because + Cargo refuses to let a member override a workspace dependency's defaults. Without this the + binding's `http3` feature and the client's drifted: turning the binding's off left the client's on, + and the `#[cfg]`-gated recipe fields stopped lining up. Check both configurations after touching + features — `cargo build` and `cargo build -p web-faith-napi --no-default-features`. +- **The recipe structs carry public fields for now.** The binding assembles them directly; step 9's + builder is what should own that assembly, at which point they can close up again. ## Verification discipline From 8e2f531e7d909c697fd8cbf47638e3e7fb1723d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:53:14 +1200 Subject: [PATCH 21/61] S1: write the component crates' docs for an external reader Spec references move out of doc comments and into normal comments beside the item they belong to: a spec id is traceability for this repository, not documentation for someone reading the crate on docs.rs. No reference is lost. The crate-level docs said why Faith needed each piece factored out, which is the wrong subject now that each is published on its own. They say what the crate is for and how to drive it instead. The cookie jar leads with the rules it keeps that a browser-shaped model does not, and carries a worked example; the resolver leads with the warmable cache; the Alt-Svc crate had no crate documentation at all, having been a module, and now describes the upgrade decision it exists to make. Making the components public also exposed several doc links to items that stayed private, which would have shipped as broken pages. Two are fixed by publishing what the prose already treated as public: the path-time floor a documented factor is measured against, and reading a content coding off the wire. The rest name the behaviour rather than the private item. `cargo doc --workspace --no-deps` is now warning-free. One line of index.d.ts changes with it: napi emits doc comments into TypeScript, where Rust's intra-doc link syntax was only ever noise. --- .workhorse/plans/s1/plan.md | 11 ++ crates/web-faith-alt-svc/src/lib.rs | 62 +++++++---- crates/web-faith-conn-tracker/src/lib.rs | 29 +++-- crates/web-faith-cookies/Cargo.toml | 4 + crates/web-faith-cookies/src/lib.rs | 48 +++++++-- crates/web-faith-dns/src/lib.rs | 128 ++++++++++++++--------- crates/web-faith-encoding/src/lib.rs | 41 +++++--- crates/web-faith-napi/src/response.rs | 2 +- crates/web-faith/src/client.rs | 64 +++++++----- crates/web-faith/src/lib.rs | 21 +++- crates/web-faith/src/retry.rs | 4 +- crates/web-faith/src/timing.rs | 4 +- index.d.ts | 2 +- 13 files changed, 279 insertions(+), 141 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 91ddc02..598ea51 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -136,6 +136,17 @@ Decisions taken while doing steps 0–7, worth not relitigating: features — `cargo build` and `cargo build -p web-faith-napi --no-default-features`. - **The recipe structs carry public fields for now.** The binding assembles them directly; step 9's builder is what should own that assembly, at which point they can close up again. +- **Spec references go in normal comments, never in doc comments.** A `// spec:DNS#transports` line + sits under the doc block, above the item. Doc comments are published API documentation, and a spec + id means nothing to a reader on docs.rs. +- **A component crate's docs address an external reader,** not a Faith maintainer: what the crate is + for and how to drive it, rather than why Faith needed it factored this way. Keep the reasoning + where it is genuinely about the code's shape, drop the rest. +- **Keep rustdoc clean, and mind that napi doc comments reach TypeScript.** `cargo doc --workspace + --no-deps` is warning-free; making the components public surfaced several links to private items, + which would have shipped as broken docs.rs pages. Rust intra-doc syntax in a `web-faith-napi` doc + comment is emitted verbatim into `index.d.ts`, where `[`X`]` means nothing, so plain backticks + belong on anything a napi item documents. ## Verification discipline diff --git a/crates/web-faith-alt-svc/src/lib.rs b/crates/web-faith-alt-svc/src/lib.rs index 8fc241f..4b607f8 100644 --- a/crates/web-faith-alt-svc/src/lib.rs +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -1,3 +1,27 @@ +//! An Alt-Svc store, and the machinery that upgrades an origin to HTTP/3 on the strength of it. +//! +//! An origin advertises HTTP/3 in an `Alt-Svc` header, or in an `HTTPS` DNS record. Acting on that +//! is not as simple as believing it: the alternative may be unreachable even though it was +//! advertised, and finding out costs the request that tries. What this does is keep the +//! advertisements ([`AltSvcCache`]) and decide, per origin, whether HTTP/3 is worth attempting. +//! +//! [`AltSvcMiddleware`] is the layer that acts on the decision. Two shapes are available: +//! +//! - With an [`H3Prober`], an advertisement is verified in the background and foreground requests +//! are routed over HTTP/3 only once an origin is confirmed, so no user-visible request pays for +//! discovering a broken alternative. +//! - Without one, the next foreground request is itself the verification, falling back to TCP if the +//! attempt does not produce headers in time. +//! +//! Either way an origin that starts failing, or that turns out to be slower over HTTP/3 than the +//! path it replaced ([`PathTime`]), is demoted, and the cooldown before it is tried again lengthens +//! with each consecutive failure. +//! +//! [`parse_alt_svc_header`] reads a header on its own if all you want is the advertisement, and +//! [`H3HttpsSink`] feeds the store from `HTTPS` record lookups. + +// spec:H3UP spec:PROBE + use std::{ marker::PhantomData, sync::Arc, @@ -40,8 +64,7 @@ pub struct AltSvcAdvertisement { /// because they differ per origin and from each other: the entry deliberately /// outlives the cooldown it set, so that a count survives the block it caused and /// can escalate the next one. `advertised` does the same for `ma`. -/// -/// spec:H3UP#failure-backoff +// spec:H3UP#failure-backoff #[derive(Debug, Clone, Copy)] struct FailureEntry { /// Consecutive failures with no confirmation in between. @@ -72,7 +95,7 @@ const EWMA_ALPHA: f64 = 0.2; const EWMA_MIN_SAMPLES: u32 = 8; /// Absolute gap the QUIC average must exceed the TCP one by, on top of the /// factor, so LAN-fast origins don't flap on sub-millisecond noise. -const SLOW_FLOOR_MS: f64 = 10.0; +pub const SLOW_FLOOR_MS: f64 = 10.0; pub struct AltSvcCacheConfig { pub advertised_ttl: Duration, @@ -119,7 +142,8 @@ pub struct AltSvcCache { /// put there: [`Self::network_changed`] demotes the observed ones and /// re-seeds from here. Unbounded by TTL and outside the capacity bound, /// because the hints are configuration and there are as many as the caller - /// passed. (spec:NETCHG#what-the-signal-keeps) + /// passed. + // spec:NETCHG#what-the-signal-keeps hints: Cache, /// Time-to-headers over TCP (h1 and h2 together), per origin. tcp_times: Cache, @@ -220,8 +244,7 @@ impl AltSvcCache { /// The cooldown the `count`-th consecutive failure earns: the base doubled /// once per failure before it, capped. - /// - /// spec:H3UP#failure-backoff + // spec:H3UP#failure-backoff fn failure_cooldown(&self, count: u32) -> Duration { let doublings = count.saturating_sub(1).min(u32::BITS - 1); self.failed_ttl @@ -291,8 +314,7 @@ impl AltSvcCache { /// (blocked whatever a record says), slow (demoted on measurement, which a record cannot /// overturn), or already carrying a live advertisement (the probe it warrants is already /// warranted). Each of those states expires, and the query resumes when it does. - /// - /// spec:DNS#https-records + // spec:DNS#https-records pub fn wants_https_record(&self, url: &reqwest::Url) -> bool { let Some(origin) = Self::origin_key(url) else { return false; @@ -315,8 +337,7 @@ impl AltSvcCache { /// and same-host rules are the header's too — [`Self::record_alt_svc`] applies them — because /// the reasons for them are about what Faith can connect to rather than about where the /// advertisement was read. - /// - /// spec:H3UP#advertisements-from-dns + // spec:H3UP#advertisements-from-dns pub fn record_https_record(&self, url: &reqwest::Url, port: Option, ttl: Duration) { // A record naming no port describes the origin's own, exactly as an `Alt-Svc` header with // no alt-authority port would. @@ -615,8 +636,7 @@ impl AltSvcCache { /// failure must not unblock the origin that failure just blocked: the failure /// is the more recent evidence about the path, and [`Self::confirm_h3`] /// relies on its own entry being masked until the block lapses. - /// - /// spec:H3UP#failure-backoff + // spec:H3UP#failure-backoff fn clear_failure_count(&self, origin: &str) { let Some(entry) = self.failed.get(origin) else { return; @@ -644,8 +664,7 @@ impl AltSvcCache { /// /// What the origin said about itself (`advertised`) and what the caller /// asserted (`hints`) are not observations, so both survive. - /// - /// spec:NETCHG + // spec:NETCHG pub fn network_changed(&self) { let now = Instant::now(); @@ -696,8 +715,7 @@ impl AltSvcCache { /// Record a failed HTTP/3 attempt, blocking the origin for a cooldown that /// lengthens the longer it keeps failing. - /// - /// spec:H3UP#failure-backoff + // spec:H3UP#failure-backoff pub fn record_h3_failure(&self, url: &reqwest::Url) { let Some(origin) = Self::origin_key(url) else { return; @@ -943,7 +961,7 @@ impl H3Prober { /// Kick off a background probe for the URL's origin if one is warranted: an actionable /// advertisement present, the origin neither confirmed, failed, nor slow, and no probe - /// already in flight. The same decision [`AltSvcMiddleware::maybe_probe`] makes, exposed so a + /// already in flight. The same decision the Alt-Svc layer makes on a request, exposed so a /// `preconnect` TCP warm-up to a probe-worthy origin triggers a probe as a real request would. pub fn maybe_probe(&self, url: &reqwest::Url) { let Some(port) = self.cache.probe_candidate(url) else { @@ -978,8 +996,8 @@ impl H3Prober { /// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could /// name. Recording it there is right whichever request triggered the lookup, because what the /// record describes does not depend on who asked. -/// -/// spec:H3UP#advertisements-from-dns spec:DNS#https-records +// spec:H3UP#advertisements-from-dns +// spec:DNS#https-records pub struct H3HttpsSink { cache: Arc, /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, @@ -989,7 +1007,8 @@ pub struct H3HttpsSink { /// agent's prober does. /// /// `None` rather than a dead handle when probing is off, where an advertisement is acted on - /// inline by the next foreground request instead (spec:PROBE). + /// inline by the next foreground request instead. + // spec:PROBE prober: Option>, } @@ -1107,8 +1126,7 @@ impl AltSvcMiddleware { /// This is the one place a response's arrival is observed: the returned instant is what the /// path-time average measures against, and the same instant reaches the caller through the /// request's extensions to become the surfaced timing, so the two can never disagree. -/// -/// spec:RESP#request-timing +// spec:RESP#request-timing async fn run_stamped( next: Next<'_>, req: Request, diff --git a/crates/web-faith-conn-tracker/src/lib.rs b/crates/web-faith-conn-tracker/src/lib.rs index acc76e2..4a575b3 100644 --- a/crates/web-faith-conn-tracker/src/lib.rs +++ b/crates/web-faith-conn-tracker/src/lib.rs @@ -1,12 +1,18 @@ -//! Live per-connection statistics, read from the operating system. (spec:OBS) +//! Live per-connection TCP statistics, read from the operating system. //! -//! The pool's own view says which connections exist; the kernel knows how each one is actually -//! behaving. This tracker keeps an entry per connection the caller reports traffic on, expiring it -//! once it has been idle for the configured timeout, and refreshes each entry's TCP statistics from -//! the OS once a second. +//! A connection pool can say which connections it holds, but not how any of them is actually +//! behaving: round-trip time, retransmits, congestion window, delivery rate. The kernel knows, and +//! this reads it. //! -//! Reading those statistics is per-platform: Linux over netlink, macOS and Windows through their own -//! interfaces. On any other platform the entries are tracked without statistics. +//! Report traffic on a connection with [`ConnectionTracker::track`], which also answers whether that +//! connection had been seen before, and take the current view with +//! [`ConnectionTracker::snapshot`]. Each entry's statistics are refreshed once a second, and an +//! entry that goes idle for longer than the configured timeout expires out of the tracker. +//! +//! Reading the statistics is per-platform: Linux over netlink, macOS and Windows through their own +//! interfaces. Anywhere else, connections are still tracked but carry no statistics. + +// spec:OBS #[cfg(target_os = "linux")] #[path = "platform/linux.rs"] @@ -175,10 +181,11 @@ impl ConnectionTracker { /// Register a warm-up connection that no request has yet been credited to. /// - /// A `preconnect` connection is listed before any foreground request uses it, at a response - /// count of zero (spec:WARM). An entry already tracked is left untouched: a warm-up to an - /// origin that already holds a pooled connection does no new work, and must not disturb the - /// count or timestamps of the connection it would reuse. + /// A warm-up connection is listed before any foreground request uses it, at a response count of + /// zero. An entry already tracked is left untouched: a warm-up to an origin that already holds a + /// pooled connection does no new work, and must not disturb the count or timestamps of the + /// connection it would reuse. + // spec:WARM pub fn track_warmup(&self, local_addr: SocketAddr, remote_addr: SocketAddr) { let now = SystemTime::now(); let key = ConnectionKey { diff --git a/crates/web-faith-cookies/Cargo.toml b/crates/web-faith-cookies/Cargo.toml index b9880c3..a186c23 100644 --- a/crates/web-faith-cookies/Cargo.toml +++ b/crates/web-faith-cookies/Cargo.toml @@ -18,6 +18,10 @@ time.workspace = true url.workspace = true reqwest = { workspace = true, optional = true } +[dev-dependencies] +# Doctests link as their own crate, so the types the API speaks have to be named there too. +url.workspace = true + [features] # Implement reqwest's `CookieStore`, so the jar can serve as its cookie provider. reqwest = ["dep:reqwest"] diff --git a/crates/web-faith-cookies/src/lib.rs b/crates/web-faith-cookies/src/lib.rs index 5ba47a4..5db5afc 100644 --- a/crates/web-faith-cookies/src/lib.rs +++ b/crates/web-faith-cookies/src/lib.rs @@ -1,14 +1,44 @@ -//! A cookie jar for HTTP clients, with the rules that hold outside a browser. (spec:COOK) +//! A cookie jar for HTTP clients, with the storage rules that hold outside a browser. //! -//! `cookie_store` implements the classic RFC 6265 storage model, and reqwest's `Jar` wraps it in a -//! private field, so extending it means wrapping the store ourselves rather than the jar. What this -//! adds is the RFC 6265bis rules that mean something without a browsing context: the `__Host-` and -//! `__Secure-` name prefixes, a cap on how far ahead a cookie may expire, and caps on how many -//! cookies and how many bytes one server can accumulate. `SameSite` is left to the `cookie` crate to -//! parse and is never read, governing cross-site behaviour that only a first-party context has. +//! Cookies were specified for browsers, and the parts of RFC 6265 that assume a browsing context do +//! not carry over to a client making requests on its own account. This jar keeps the model that +//! does: the classic storage and matching rules, and the RFC 6265bis additions that still mean +//! something with no browser around them. //! -//! The `reqwest` feature implements that client's `CookieStore` trait, so the jar can be handed to -//! it as a cookie provider. +//! - The `__Host-` and `__Secure-` name prefixes, which bind a cookie to the exact host that set it +//! and to a secure transport. +//! - A ceiling on how far ahead a cookie may expire, so a server cannot claim a decade. +//! - Caps on the size of one cookie, and on how many are kept per host and in total. +//! +//! Every rule is applied when a cookie is stored rather than when one is sent. That is what makes +//! the caps bound real memory, and what holds a cookie inserted by hand to the same rules as one +//! that arrived in a `Set-Cookie` header. +//! +//! `SameSite` is parsed but never read: it governs cross-site behaviour that only a first-party +//! context has. +//! +//! # Example +//! +//! ``` +//! use url::Url; +//! use web_faith_cookies::{CookieLimits, FaithJar}; +//! +//! let jar = FaithJar::new(CookieLimits::default()); +//! let url = Url::parse("https://example.com/")?; +//! +//! jar.add_cookie_str("session=abc; Path=/", &url); +//! +//! let header = jar.request_cookie_header(&url).expect("a cookie to send"); +//! assert_eq!(header.to_str()?, "session=abc"); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! # Features +//! +//! `reqwest` implements that client's `CookieStore` for the jar, so it can be handed to a +//! `ClientBuilder` as a cookie provider. + +// spec:COOK use std::{collections::HashMap, sync::RwLock, time::Duration}; diff --git a/crates/web-faith-dns/src/lib.rs b/crates/web-faith-dns/src/lib.rs index ed262ca..a703fcb 100644 --- a/crates/web-faith-dns/src/lib.rs +++ b/crates/web-faith-dns/src/lib.rs @@ -1,23 +1,32 @@ -//! Faith's own DNS resolver. +//! A caching DNS resolver for HTTP clients, with a cache you can warm. //! -//! `reqwest`'s built-in hickory resolver and its in-memory cache are `pub(crate)`, so the only -//! way to warm that cache is to make a request through it — which is `preconnect`'s job, not -//! `prefetchDns`'s (that verb must not touch the origin). To let `prefetchDns` populate the cache -//! a later request reads, Faith owns the resolver instead: this type is installed on the `reqwest` -//! client with `ClientBuilder::dns_resolver`, so reqwest routes every lookup through it, and -//! `prefetch` calls the same resolver directly. Both share one `TokioResolver`, so a name warmed -//! by `prefetchDns` is already cached when a request looks it up. +//! A client's built-in resolver usually keeps its cache to itself, so the only way to populate it is +//! to make a request. That is no good for prefetching a name ahead of time, which must not touch the +//! origin at all. [`FaithResolver`] is the resolver instead: it can be installed on an HTTP client +//! so every request resolves through it, while [`FaithResolver::prefetch`] is called directly. Both +//! share one resolver and one cache, so a name warmed ahead of time is already there when a request +//! looks it up. //! -//! Beyond warming, this type is where the DNS transports and server order live. `dns.servers` -//! lists resolver URLs whose scheme picks the transport (`udp`/`tcp`, or the encrypted `tls`, -//! `https`, `quic`, `h3`); the list is queried in order, held fixed with `UserProvidedOrder`. With -//! no list, the resolver configures itself from the system and lets hickory's RFC 9539 -//! opportunistic encryption upgrade those servers where it can. Either way, [exempt names] go to -//! the system resolver instead, so local names keep resolving. +//! # Transports and server order //! -//! [exempt names]: ResolverSettings::exempt_domains +//! Resolvers are named by URL, and the scheme picks the transport: plaintext `udp` and `tcp`, or +//! encrypted `tls`, `https`, `quic`, and `h3`. The list is queried in the order given rather than +//! reordered by latency. Given no list, the resolver configures itself from the operating system and +//! lets RFC 9539 opportunistic encryption upgrade those servers where it can. //! -//! spec:WARM spec:DNS +//! [Exempt names] are sent to the system resolver whichever way the rest is configured, so names +//! that only the host knows how to resolve keep resolving. +//! +//! # Beyond addresses +//! +//! Lookups can also read the `HTTPS` record for a name, which is how an origin advertises HTTP/3 +//! before anything has connected to it, and answers can be served stale while a fresh lookup runs. +//! A [network change][FaithResolver::reset] discards what was learned from a network that no longer +//! exists while leaving the resolver usable. +//! +//! [Exempt names]: ResolverSettings::exempt_domains + +// spec:WARM spec:DNS use std::{ collections::HashSet, @@ -103,7 +112,8 @@ impl Transport { } } -/// What an `HTTPS` record said about an origin's HTTP/3 support (spec:DNS#https-records). +/// What an `HTTPS` record said about an origin's HTTP/3 support. +// spec:DNS#https-records #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct HttpsAdvertisement { /// The record's `port` SvcParam, or `None` when it named none and the origin's own port @@ -117,8 +127,8 @@ pub struct HttpsAdvertisement { /// /// The resolver cannot own the HTTP/3 upgrade cache directly: that cache is built after the /// resolver, and the background prober holds a client which holds the resolver in turn. So the -/// agent installs this afterwards (see [`FaithResolver::set_https_sink`]), which also keeps -/// `dns.rs` free of the upgrade layer's types. +/// caller installs this afterwards (see [`FaithResolver::set_https_sink`]), which also keeps this +/// crate free of the upgrade layer's types. pub trait HttpsSink: Send + Sync { /// Whether an `HTTPS` record for `host` is worth querying at all right now. /// @@ -133,7 +143,8 @@ pub trait HttpsSink: Send + Sync { /// Whether an ALPN token names a version of HTTP/3. /// /// The same family test the `Alt-Svc` reader applies, so a draft token like `h3-29` counts here -/// exactly as it does in a header (spec:H3UP#reading-advertisements). +/// exactly as it does in a header. +// spec:H3UP#reading-advertisements fn is_h3_alpn(token: &str) -> bool { token == "h3" || token.starts_with("h3-") } @@ -147,10 +158,11 @@ fn is_h3_alpn(token: &str) -> bool { /// /// A record whose target is neither the root (which per RFC 9460 §2.5.2 means the owner name /// itself) nor the queried name designates a *different* host, and Faith only upgrades to the -/// origin's own host, so such a record is not acted on (spec:H3UP#advertisements-from-dns). +/// origin's own host, so such a record is not acted on. /// `queried` is the name the answer was actually asked for rather than the host as written, since /// the search list can requalify a name before it reaches a server; comparison ignores the trailing /// root so the two are judged on identity rather than on how each was spelled. +// spec:H3UP#advertisements-from-dns fn read_https_answer( queried: &Name, answers: &[hickory_resolver::proto::rr::Record], @@ -264,7 +276,8 @@ impl ServerSpec { } /// The certificate name to authenticate against once the host resolves to `ip`: an explicit - /// fragment, else the hostname, else the address itself (spec:DNS#transports). + /// fragment, else the hostname, else the address itself. + // spec:DNS#transports fn server_name(&self, ip: IpAddr) -> Arc { if let Some(name) = &self.cert_name { Arc::from(name.as_str()) @@ -310,7 +323,8 @@ impl ServerSpec { } } -/// How a server in `resolvers()` came to be reached the way it is (spec:OBS#resolvers). +/// How a server in `resolvers()` came to be reached the way it is. +// spec:OBS#resolvers #[derive(Clone, Copy, Debug)] pub enum ResolverSource { /// Named in `dns.servers` by the caller. @@ -341,7 +355,8 @@ pub struct ResolverReport { /// An hour is long enough that a resolver outage does not stop an agent reaching hosts it already /// knows, and short enough that a host which really has moved stops being served a dead address for /// the life of a long-running process. The recovery path bounds the cost of being wrong to one -/// re-resolve, so the window can be generous (spec:DNS#serving-stale-answers). +/// re-resolve, so the window can be generous. +// spec:DNS#serving-stale-answers pub const DEFAULT_MAX_STALE: Duration = Duration::from_secs(3600); /// Everything `dns.*` configures about Faith's resolver, resolved from options at construction. @@ -397,7 +412,8 @@ struct Built { } /// A resolved answer kept past its TTL, so an expired lookup is served from it while a refresh runs -/// behind (spec:DNS#serving-stale-answers). +/// behind. +// spec:DNS#serving-stale-answers #[derive(Clone)] struct StaleEntry { /// Shared rather than cloned per hit: a hit reads it and hands out a copy of the addresses. @@ -408,10 +424,11 @@ struct StaleEntry { } /// Everything the resolver reads off the network, held together so a network change can drop it in -/// one go (spec:NETCHG). Each field describes the network the agent was on when it was read: which +/// one go. Each field describes the network the agent was on when it was read: which /// servers discovery found, which suffixes are local to it, and which of its servers answered an /// encryption probe. The caller's [`ResolverSettings`] deliberately sit outside, being options the /// agent was constructed with rather than a reading of any network. +// spec:NETCHG struct Generation { /// The configured (or discovered) resolver, built lazily inside a tokio runtime. built: OnceCell>, @@ -424,7 +441,8 @@ struct Generation { /// it: an address learned on the old network is exactly what must not be served on the new one. stale: moka::sync::Cache, /// Hosts with a refresh already in flight, so a second stale hit serves the entry rather than - /// starting another lookup (spec:DNS#serving-stale-answers). + /// starting another lookup. + // spec:DNS#serving-stale-answers refreshing: Mutex>, } @@ -441,12 +459,14 @@ impl Default for Generation { } struct Inner { - /// The options the agent was constructed with. A network change does not touch these - /// (spec:NETCHG#what-the-signal-keeps); they are what the next generation is rebuilt from. + /// The options the agent was constructed with. A network change does not touch these; they are + /// what the next generation is rebuilt from. + // spec:NETCHG#what-the-signal-keeps settings: ResolverSettings, /// Replaced wholesale by [`FaithResolver::reset`]. Read once at the start of a lookup rather /// than at each step, so a lookup that spans the signal finishes against the one set of - /// resolvers it started on (spec:NETCHG#in-flight-requests). + /// resolvers it started on. + // spec:NETCHG#in-flight-requests generation: Mutex>, /// Where `HTTPS` records go, installed by the agent once the upgrade cache and prober exist. /// @@ -486,12 +506,13 @@ impl FaithResolver { } } - /// Install where `HTTPS` records go, enabling the query (spec:DNS#https-records). + /// Install where `HTTPS` records go, enabling the query. /// /// Called after the agent's HTTP/3 upgrade cache and prober are built, which cannot happen /// before the resolver exists. Replaces any previous sink, which is what a network change /// needs: the prober is rebuilt with the client, so the sink must be too or it would kick /// probes onto a client that has been dropped. + // spec:DNS#https-records pub fn set_https_sink(&self, sink: Arc) { *self .inner @@ -510,7 +531,8 @@ impl FaithResolver { /// The generation a piece of work resolves against. Taken once per lookup: a reset swaps the /// generation rather than mutating it, so work already holding one carries on against the - /// resolvers it started with (spec:NETCHG#in-flight-requests). + /// resolvers it started with. + // spec:NETCHG#in-flight-requests fn generation(&self) -> Arc { self.inner .generation @@ -547,8 +569,9 @@ impl FaithResolver { } /// The exempt suffixes: `localhost`, `local`, the system's own domain and search suffixes, and - /// the caller's `dns.exemptDomains` (spec:DNS#exempt-names). The system's own suffixes are a + /// the caller's `dns.exemptDomains`. The system's own suffixes are a /// property of the network, so they are read per generation rather than once per agent. + // spec:DNS#exempt-names async fn exempt(&self, generation: &Generation) -> Arc> { generation .exempt @@ -607,13 +630,13 @@ impl FaithResolver { } /// Ask for `host`'s `HTTPS` record behind the address lookup, so an origin advertising - /// `alpn="h3"` is known before the first connection rather than after the first TCP response - /// (spec:DNS#https-records). + /// `alpn="h3"` is known before the first connection rather than after the first TCP response. /// /// Spawned rather than awaited: an absent, slow, or failed answer must leave address /// resolution and connecting untouched. Its outcome belongs to the upgrade layer rather than /// to the request that triggered it, so nothing here reaches a caller, exactly as a stale /// refresh's outcome does not. + // spec:DNS#https-records fn spawn_https_query(&self, generation: &Arc, host: &str) { let Some(sink) = self.https_sink() else { return; @@ -695,7 +718,8 @@ impl FaithResolver { /// Single-flighted per host: the claim is taken before the task is spawned, so concurrent stale /// hits serve the entry rather than each starting a lookup. The task outlives the request that /// triggered it, and its outcome belongs to the cache rather than that request, so nothing here - /// is reported to a caller (spec:DNS#serving-stale-answers). + /// is reported to a caller. + // spec:DNS#serving-stale-answers fn spawn_refresh(&self, generation: &Arc, host: &str) { { let mut refreshing = generation @@ -744,7 +768,8 @@ impl FaithResolver { /// Drop any stale answer held for `host`, so the next lookup waits for a fresh one. /// /// Called when connecting to a served address failed, which is the one piece of evidence that the - /// address was wrong rather than merely old (spec:DNS#when-a-stale-address-is-wrong). + /// address was wrong rather than merely old. + // spec:DNS#when-a-stale-address-is-wrong pub fn invalidate_stale(&self, host: &str) { self.generation().stale.invalidate(host); } @@ -752,7 +777,7 @@ impl FaithResolver { /// Whether a lookup of `host` right now would be served from an expired entry, and so would hand /// out an address that is assumed rather than confirmed. /// - /// Deliberately the same window [`Self::stale_addrs`] serves from, rather than merely "an expired + /// Deliberately the same window a stale answer is served from, rather than merely "an expired /// entry exists": an entry past `dns.maxStale` is resolved for real, and treating that as stale /// would spend a second connection attempt on an address that was already confirmed. pub fn served_stale(&self, host: &str) -> bool { @@ -767,13 +792,15 @@ impl FaithResolver { } /// Resolve `host` and leave the answer in the shared cache, so a later request skips the - /// lookup. Any failure is swallowed: the warm-up is advisory (spec:WARM). + /// lookup. Any failure is swallowed: the warm-up is advisory. + // spec:WARM pub async fn prefetch(&self, host: &str) { let _ = self.lookup(host).await; } - /// The DNS servers the agent resolves through, in query order (spec:OBS#resolvers). Empty + /// The DNS servers the agent resolves through, in query order. Empty /// until the resolver has been used, because it reads its configuration on first use. + // spec:OBS#resolvers pub fn resolvers(&self) -> Vec { self.generation() .built @@ -792,14 +819,13 @@ impl FaithResolver { /// it, since they belong to the resolvers being dropped. /// /// The caller's options are untouched, so a listed `dns.servers` set is rebuilt exactly as - /// configured; what it re-reads is what the system supplies and what the network answers - /// (spec:NETCHG#what-the-signal-keeps). + /// configured; what it re-reads is what the system supplies and what the network answers. /// /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, /// which keeps it callable from `networkChanged`, which is not async. Nothing is rebuilt here /// either, so an agent that never resolves again pays nothing for the signal. - /// - /// spec:NETCHG#reach-across-the-subsystems + // spec:NETCHG#what-the-signal-keeps + // spec:NETCHG#reach-across-the-subsystems pub fn reset(&self) { *self .inner @@ -876,7 +902,8 @@ async fn build(settings: &ResolverSettings) -> Result { /// Discovery: configure from the system, then let hickory's RFC 9539 opportunistic encryption /// upgrade those servers to DoT/DoQ where they answer a probe. `dns.searchDomains` overrides the -/// system search list when set (spec:DNS#discovery). +/// system search list when set. +// spec:DNS#discovery fn build_discovery(settings: &ResolverSettings) -> Result { let (mut config, options) = read_system_conf().unwrap_or_else(|_| { // A host with no readable resolver configuration falls back to Google Public DNS over @@ -907,7 +934,9 @@ fn build_discovery(settings: &ResolverSettings) -> Result { } /// The listed-servers path: bootstrap any hostname hosts to addresses, then build the resolver -/// from the parsed specs in order (spec:DNS#transports, spec:DNS#bootstrapping). +/// from the parsed specs in order. +// spec:DNS#transports +// spec:DNS#bootstrapping async fn build_listed(settings: &ResolverSettings) -> Result { let name_servers = build_name_servers(settings).await?; @@ -926,7 +955,8 @@ async fn build_listed(settings: &ResolverSettings) -> Result { /// Resolve the listed servers to hickory name servers, bootstrapping hostname hosts. A hostname /// that will not resolve drops that server for the life of the agent rather than failing the -/// resolver (spec:DNS#bootstrapping). +/// resolver. +// spec:DNS#bootstrapping async fn build_name_servers( settings: &ResolverSettings, ) -> Result, NetError> { @@ -993,14 +1023,14 @@ fn bootstrap_resolver(settings: &ResolverSettings) -> Result, configured: &[Name]) -> Vec { let mut names = vec![ Name::from_ascii("localhost").unwrap(), diff --git a/crates/web-faith-encoding/src/lib.rs b/crates/web-faith-encoding/src/lib.rs index bf8bf34..957bf4e 100644 --- a/crates/web-faith-encoding/src/lib.rs +++ b/crates/web-faith-encoding/src/lib.rs @@ -1,8 +1,18 @@ -//! Content coding: Faith owns the decode decision rather than the HTTP stack -//! underneath, so it can rest on the `Accept-Encoding` of the request in hand. -//! The same codings compress a request body under the `compress` option. +//! HTTP content coding for request and response bodies: gzip, deflate, brotli, and zstd. //! -//! spec: ENC +//! An HTTP stack usually decides for itself which codings to advertise and decode. This lets the +//! caller own that decision instead, so decoding can rest on the `Accept-Encoding` of the request +//! actually in hand rather than on whatever the layer beneath negotiated. +//! +//! On the way back, [`AcceptEncoding::parse`] reads what a request advertised and [`decision`] says +//! which coding a response should be decoded under, if any; [`decode_stream`] wraps the body in the +//! decoder for it. On the way out, [`compress_buffer`] and [`compress_stream`] apply a coding to a +//! request body, and [`layer_content_encoding`] names it alongside whatever the caller had already +//! declared. +//! +//! `deflate` is the zlib-wrapped form of RFC 1950, which is what mainstream clients decode it as. + +// spec:ENC use std::{io, pin::Pin}; @@ -46,7 +56,8 @@ impl Coding { /// Unlike [`Self::from_token`], which reads a token off the wire and so takes it as /// loosely as HTTP writes it, this matches the four documented tokens exactly: the /// option is an API surface, and an unrecognised value is refused rather than - /// guessed at (spec:ENC#compressing-a-request-body). + /// guessed at. + // spec:ENC#compressing-a-request-body pub fn from_option(value: &str) -> Option { match value { "gzip" => Some(Self::Gzip), @@ -68,8 +79,8 @@ impl Coding { } /// Match a single content-coding token, case-insensitively. `None` for - /// `identity`, an unknown coding, or a coding Faith cannot decode. - fn from_token(token: &str) -> Option { + /// `identity`, an unknown coding, or one this cannot decode. + pub fn from_token(token: &str) -> Option { let token = token.trim(); if token.eq_ignore_ascii_case("gzip") || token.eq_ignore_ascii_case("x-gzip") { Some(Self::Gzip) @@ -242,7 +253,8 @@ pub type RequestStream = Pin> + Send>>; /// Compress a buffered request body, yielding the bytes that go on the wire. /// /// The whole body is known up front, so it compresses in one pass and its length is the -/// `Content-Length` reqwest derives from it (spec:ENC#what-a-compressed-request-sends). +/// `Content-Length` the request can declare. +// spec:ENC#what-a-compressed-request-sends pub async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result> { let mut output = Vec::new(); match coding { @@ -257,8 +269,9 @@ pub async fn compress_buffer(input: &[u8], coding: Coding) -> io::Result /// Compress a streaming request body as its chunks arrive. /// /// There is no compressed length to declare before the body ends, so the result goes out -/// chunked (spec:ENC#what-a-compressed-request-sends). The encoder buffers on its own -/// terms, so the bytes for one chunk the caller writes need not leave with it. +/// chunked. The encoder buffers on its own terms, so the bytes for one chunk the caller +/// writes need not leave with it. +// spec:ENC#what-a-compressed-request-sends pub fn compress_stream(input: S, coding: Coding) -> RequestStream where S: Stream> + Send + 'static, @@ -279,11 +292,11 @@ where Box::pin(ReaderStream::new(reader)) } -/// Join the codings a request already declares with the one Faith applied. +/// Join the codings a request already declares with the one applied on top. /// -/// The caller's `Content-Encoding` describes the bytes they handed over, so Faith's coding -/// is named after theirs, the order the codings were applied in -/// (spec:ENC#what-a-compressed-request-sends). +/// The caller's `Content-Encoding` describes the bytes they handed over, so the applied coding is +/// named after theirs: the order the codings were applied in. +// spec:ENC#what-a-compressed-request-sends pub fn layer_content_encoding(declared: Option<&str>, applied: Coding) -> String { match declared.map(str::trim).filter(|value| !value.is_empty()) { Some(declared) => format!("{declared}, {}", applied.token()), diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 32ba7c8..a5d80eb 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -659,7 +659,7 @@ impl FaithResponse { /// written to and `bytesWritten` counts the bytes that landed there. /// /// `onProgress` is reported to as the bytes land, at most every - /// [`PROGRESS_INTERVAL`], with a final report once the last byte is written. The + /// `PROGRESS_INTERVAL`, with a final report once the last byte is written. The /// wrapper takes it from the options object; it arrives here as its own argument /// because a threadsafe function cannot be a field of a `#[napi(object)]`. /// diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs index 85a2e78..1268477 100644 --- a/crates/web-faith/src/client.rs +++ b/crates/web-faith/src/client.rs @@ -2,7 +2,9 @@ //! //! The recipe here is what lets a client be rebuilt: `network_changed` has to drop the connection //! pool, and reqwest offers no way to do that short of dropping the client, so building one is a -//! pure function of settings that were validated once (spec:NETCHG). +//! pure function of settings that were validated once. + +// spec:NETCHG use std::{ net::{IpAddr, SocketAddr}, @@ -45,14 +47,16 @@ pub enum RedirectPolicy { Stop, } -/// Per-stream receive window applied to both protocols when nothing overrides it (spec:FLOW). +/// Per-stream receive window applied to both protocols when nothing overrides it. /// /// Chrome's shape: 6 MiB stream inside a 15 MiB connection. Picked over a larger window that /// measured faster because it is what browsers have proven at scale, and because a pooled /// server-side client multiplies per-connection memory across far more connections. +// spec:FLOW pub const DEFAULT_STREAM_WINDOW: u32 = 6 * 1024 * 1024; -/// Whole-connection receive window applied to both protocols when nothing overrides it (spec:FLOW). +/// Whole-connection receive window applied to both protocols when nothing overrides it. +// spec:FLOW pub const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; // Concurrent streams share the connection's headroom, so the asymmetry is the point of the @@ -60,7 +64,8 @@ pub const DEFAULT_CONNECTION_WINDOW: u32 = 15 * 1024 * 1024; const _: () = assert!(DEFAULT_CONNECTION_WINDOW > DEFAULT_STREAM_WINDOW); /// The flow-control windows to apply, once the common group, the per-protocol overrides, and the -/// defaults have been reconciled (spec:FLOW#per-protocol-windows). +/// defaults have been reconciled. +// spec:FLOW#per-protocol-windows #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ResolvedWindows { pub stream: u32, @@ -74,8 +79,8 @@ pub struct ResolvedWindows { /// - `NODE_EXTRA_CA_CERTS`: a path to a PEM file whose certificates are added to /// the trust store on top of the platform roots. As in Node.js, a value that /// is empty, or points at a file that cannot be read or parsed, is ignored -/// rather than fatal — unlike the explicit [`AgentTlsOptions::extra_roots`] -/// option, which throws. Certificates load in addition to any `extra_roots`. +/// rather than fatal, unlike an explicitly configured extra root, which is an +/// error. Certificates load in addition to any configured explicitly. /// /// - `NODE_TLS_REJECT_UNAUTHORIZED`: when set to exactly `"0"`, TLS certificate /// validation is disabled for the agent. This is insecure and exists only to @@ -93,9 +98,10 @@ pub struct ResolvedWindows { /// would toggle. `=0` could therefore only mean "trust almost nothing", which is /// never what a caller wants, so the platform store is always used. /// -/// Read once, at construction: AGENT has these layered on top of the explicit options when the -/// agent is built, so a client rebuilt later (spec:NETCHG) replays what was read then rather than -/// picking up an environment that has changed since. +/// Read once, at construction: these are layered on top of the explicit options when the agent is +/// built, so a client rebuilt later replays what was read then rather than picking up an environment +/// that has changed since. +// spec:NETCHG #[derive(Debug, Clone, Default)] pub struct NodeEnvRecipe { extra_ca_certs: Vec, @@ -145,7 +151,8 @@ impl NodeEnvRecipe { /// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and /// `CACacheManager` names the directory holding them. So cloning one shares the cache, while /// building a fresh one from the same options would empty an in-memory cache — which is why a -/// client rebuilt for a network change clones this (spec:NETCHG#what-the-signal-keeps). +/// client rebuilt for a network change clones this. +// spec:NETCHG#what-the-signal-keeps #[derive(Debug, Clone)] pub enum HttpCacheStore { Disk(CACacheManager), @@ -161,7 +168,8 @@ pub struct HttpCacheRecipe { } /// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not -/// here: it belongs to the agent and outlives any one client (spec:NETCHG). +/// here: it belongs to the agent and outlives any one client. +// spec:NETCHG #[cfg(feature = "http3")] #[derive(Debug, Clone)] pub struct H3UpgradeRecipe { @@ -171,26 +179,28 @@ pub struct H3UpgradeRecipe { pub probe_timeout: Option, } -/// Everything needed to build the agent's clients, validated once at construction. +/// Everything needed to build the agent's clients, validated once up front. /// -/// This exists because `networkChanged` has to drop the connection pool, and reqwest offers no way -/// to do that short of dropping the client, so the client has to be buildable more than once -/// (spec:NETCHG). `AgentOptions` cannot serve: validating it consumes it, and it carries napi -/// values belonging to the JS call that passed them. So validation happens once, into these -/// Rust-native fields, and building a client is a pure function of them and the agent's shared -/// state. +/// A client has to be buildable more than once: dropping the connection pool means dropping the +/// client, which is what a network change asks for, so what the client is built from has to outlive +/// any one of them. Options are validated once into these fields, and building a client is then a +/// pure function of them and the agent's shared state. +// spec:NETCHG #[derive(Debug, Clone)] pub struct ClientRecipe { pub user_agent: String, pub local_address: Option, pub default_headers: Option, - /// Under the system resolver no hickory resolver is installed at all (spec:DNS). + /// Under the system resolver no hickory resolver is installed at all. + // spec:DNS pub dns_system: bool, /// Validated at construction, and applied whichever resolver is in use: reqwest layers - /// overrides on top of the resolver it was given (spec:DNS#overrides). + /// overrides on top of the resolver it was given. + // spec:DNS#overrides pub dns_overrides: Vec<(String, Vec)>, pub http2_adaptive_window: bool, - /// `None` when adaptive windowing owns the windows itself (spec:FLOW#adaptive-windowing). + /// `None` when adaptive windowing owns the windows itself. + // spec:FLOW#adaptive-windowing pub http2_windows: Option, #[cfg(feature = "http3")] pub http3_max_idle_timeout: Duration, @@ -222,10 +232,10 @@ pub struct ClientRecipe { /// `alpn="h3"` makes an origin probe-worthy before anything has connected to it. /// /// A no-op without all the parts: the system resolver is not Faith's to add a query to, and with -/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one -/// (spec:DNS#https-records). +/// HTTP/3 upgrade off there is nothing an advertisement could feed, so neither sends one. /// /// Re-called on a network change, where the prober is rebuilt with the client it sends on. +// spec:DNS#https-records #[cfg(feature = "http3")] pub fn install_https_sink( dns_resolver: Option<&FaithResolver>, @@ -255,7 +265,10 @@ pub struct BuiltClients { impl ClientRecipe { /// The window an idle pooled connection lives in, which is also how long a warm-up counts as - /// warm and how long a connection stays listed (spec:POOL, spec:WARM, spec:OBS). + /// warm and how long a connection stays listed. + // spec:POOL + // spec:WARM + // spec:OBS pub fn conn_timeout(&self) -> Duration { // reqwest's own default, mirrored because the pool timeout it applies is not readable. self.pool_idle_timeout.unwrap_or(Duration::from_secs(90)) @@ -265,7 +278,8 @@ impl ClientRecipe { /// /// Everything passed in survives a rebuild by being shared rather than rebuilt: the cookie /// jar, the resolver (and so its cache), and the HTTP/3 origin knowledge all belong to the - /// agent rather than to any one client (spec:NETCHG#what-the-signal-keeps). + /// agent rather than to any one client. + // spec:NETCHG#what-the-signal-keeps pub fn build( &self, cookie_jar: Option<&Arc>, diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 00559e5..6c253c6 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -1,10 +1,21 @@ //! A browser-shaped HTTP client: fetch semantics over a Rust network stack. //! -//! The client is being assembled here. What it already owns is the error type the whole family -//! reports through, the body and timing machinery a response is built on, and the retry layers that -//! sit in the request path. A component crate names its own errors for the failures it can produce, -//! and they are converted into [`FaithError`] as they cross into the client, so a caller matches on -//! one type whichever layer failed. +//! Faith behaves like a browser wherever that translates to a server-side runtime: transparent +//! HTTP/2 and HTTP/3, Happy Eyeballs across IPv4 and IPv6, DNS caching, an optional cookie jar, and +//! HTTP caching. The subsystems beneath it are published on their own, and each can be left out of a +//! build with the feature named for it. +//! +//! Whichever layer a request fails in, the failure arrives as one [`FaithError`] whose +//! [`FaithErrorKind`] is the stable code to match on: a component crate names its own errors, and +//! they are converted at the boundary as they cross into the client. +//! +//!
    +//! +//! The client API is still being built out. What is here so far is the error type, the body and +//! timing machinery a response is built on, the retry layers in the request path, and the recipe +//! that builds the HTTP client itself. +//! +//!
    pub mod body; pub mod client; diff --git a/crates/web-faith/src/retry.rs b/crates/web-faith/src/retry.rs index 875a1f4..6177260 100644 --- a/crates/web-faith/src/retry.rs +++ b/crates/web-faith/src/retry.rs @@ -135,8 +135,8 @@ fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { /// Re-resolves and attempts a request again when connecting to a stale-served address failed. /// /// Serving an expired DNS answer trades a round trip against the chance the address has moved, and -/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request -/// (spec:DNS#when-a-stale-address-is-wrong). +/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request. +// spec:DNS#when-a-stale-address-is-wrong #[derive(Debug, Clone)] pub struct StaleAddressRetry { /// `None` under `dns.system: true`, where Faith holds no cache and so serves nothing stale. diff --git a/crates/web-faith/src/timing.rs b/crates/web-faith/src/timing.rs index 2e2af8f..f759c90 100644 --- a/crates/web-faith/src/timing.rs +++ b/crates/web-faith/src/timing.rs @@ -1,6 +1,6 @@ //! Per-request timing. -//! -//! spec:RESP#request-timing + +// spec:RESP#request-timing use std::{ sync::{Arc, OnceLock}, diff --git a/index.d.ts b/index.d.ts index 67aaba1..b3e55b3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -302,7 +302,7 @@ json(): Promise * written to and `bytesWritten` counts the bytes that landed there. * * `onProgress` is reported to as the bytes land, at most every - * [`PROGRESS_INTERVAL`], with a final report once the last byte is written. The + * `PROGRESS_INTERVAL`, with a final report once the last byte is written. The * wrapper takes it from the options object; it arrives here as its own argument * because a threadsafe function cannot be a field of a `#[napi(object)]`. * From 07c862605298d2ef01d60fbbd2814aa2d5a0b67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:56:11 +1200 Subject: [PATCH 22/61] S1: state each environment-variable section's surfaces in place The Rust surface's variables were gathered into a section of their own, which meant reading a variable's semantics in one place and whether it applied to you in another. Each section now opens by naming the surfaces it applies to, and the two that carry a mix say where the line falls: both surfaces read the OpenSSL trust-store and proxy variables, while the NODE_-prefixed ones are the Node surface's alone. Nothing about what any variable does changes. --- .workhorse/specs/environment/variables.md | 40 ++++++++++++++--------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index f143cd9..7968075 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -4,48 +4,58 @@ id: ENV # Environment variables -Faith reads a set of environment variables so that `fetch()` behaves like Node's built-in fetch without extra configuration. -The set is deliberately Node's own vocabulary plus the standard proxy and OpenSSL variables, not a Faith-specific namespace. +Faith reads a set of environment variables rather than asking a caller to configure what the surrounding platform already answers. +The set is deliberately Node's own vocabulary plus the standard proxy and OpenSSL variables, not a Faith-specific namespace; on the Node surface that is what makes `fetch()` behave like Node's built-in fetch without extra configuration. + +Each section below names the surfaces it applies to. +The `NODE_`-prefixed variables answer to a JavaScript runtime's conventions and belong to the Node surface alone; the rest is platform configuration both surfaces honour (see [RUST](../rust/overview.md)). ## Read-at-construction semantics -Environment variables are read once, when an `Agent` is constructed (including the implicit global default agent). +This applies to both surfaces. + +Environment variables are read once, when an agent is constructed, including the Node surface's implicit global default agent. Changing them afterwards only affects agents created later. ## Trust store -`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any `tls.extraRoots` (see [TLS](../agent/tls.md)); certificates from both sources combine. +Both surfaces read `SSL_CERT_FILE` and `SSL_CERT_DIR`; `NODE_EXTRA_CA_CERTS` belongs to the Node surface alone. + +On Unix platforms other than macOS, `SSL_CERT_FILE` and `SSL_CERT_DIR` override where the system trust store is loaded from, with standard OpenSSL semantics: `SSL_CERT_FILE` replaces the system roots. +On macOS and Windows the OS trust store is used directly and these are ignored, as Node does on those platforms. + +`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any `tls.extraRoots` (see [TLS](../agent/tls.md)); certificates from both sources combine, and where `SSL_CERT_FILE` replaces the system roots this adds to them. It is lenient, matching Node's warn-and-continue behaviour: an empty value, an unreadable file, or an unparseable file is ignored rather than fatal. (The `tls.extraRoots` option, being an explicit programmatic choice, throws on malformed input instead.) -On Unix platforms other than macOS, `SSL_CERT_FILE` and `SSL_CERT_DIR` override where the system trust store is loaded from, with standard OpenSSL semantics: `SSL_CERT_FILE` replaces the system roots, where `NODE_EXTRA_CA_CERTS` adds to them. -On macOS and Windows the OS trust store is used directly and these are ignored, as Node does on those platforms. +A Rust caller extends the trust store through `tls.extraRoots`. ## Certificate validation +This applies to the Node surface alone. + `NODE_TLS_REJECT_UNAUTHORIZED` set to exactly `0` disables TLS certificate validation for the agent; any other value or unset keeps validation on. This matches Node's semantics and exists only for that compatibility; trusting a specific private CA via `NODE_EXTRA_CA_CERTS` or `tls.extraRoots` is the supported path. +The Rust surface keeps certificate validation on, that being a compatibility it does not owe. ## Proxies +Both surfaces read the proxy variables and the operating system's own proxy settings; `NODE_USE_ENV_PROXY` belongs to the Node surface alone. + `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` (and their lowercase spellings) are honoured automatically: per-scheme proxy selection, a fallback for both schemes, and a comma-separated direct-connection list of hosts, domains, and CIDR ranges. The operating system's proxy settings are also read automatically. + `NODE_USE_ENV_PROXY` set to exactly `0` turns ambient proxy configuration off. Faith proxies by default, so unlike Node (where the same variable opts in), it acts purely as an opt-out. ## Debugging -`SSLKEYLOGFILE` names a path to which TLS session keys are written, enabling decryption of captured traffic when debugging. - -## What the Rust surface reads +This applies to both surfaces. -The vocabulary above is Node's because the Node surface answers to Node's conventions. -`web-faith` reads the part of it that belongs to the platform rather than to a JavaScript runtime (see [RUST](../rust/overview.md)): `SSL_CERT_FILE` and `SSL_CERT_DIR` with the same OpenSSL semantics and the same per-platform reach, `SSLKEYLOGFILE`, and the proxy variables alongside the operating system's own proxy settings. -The `NODE_`-prefixed variables belong to the Node surface alone. -A Rust caller extends the trust store through `tls.extraRoots` (see [TLS](../agent/tls.md)) and keeps certificate validation on, that being what `NODE_TLS_REJECT_UNAUTHORIZED` exists to relax for a compatibility the Rust surface does not owe. - -Both surfaces read their variables once at construction. +`SSLKEYLOGFILE` names a path to which TLS session keys are written, enabling decryption of captured traffic when debugging. ## Variables with nothing to control +Neither surface reads these. + `NODE_USE_SYSTEM_CA` is ignored because the platform trust store is Faith's only default source of roots; there is no bundled set to toggle away from. `OPENSSL_CONF` is ignored because the TLS stack is not OpenSSL, so OpenSSL's configuration file has nothing to configure. From 34a28fa7ea01fe111cd6f887aa6289221b667ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:59:55 +1200 Subject: [PATCH 23/61] S1: drop the editorialising from the environment spec's opening --- .workhorse/specs/environment/variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index 7968075..6e9fa3f 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -4,7 +4,7 @@ id: ENV # Environment variables -Faith reads a set of environment variables rather than asking a caller to configure what the surrounding platform already answers. +Faith reads a set of environment variables. The set is deliberately Node's own vocabulary plus the standard proxy and OpenSSL variables, not a Faith-specific namespace; on the Node surface that is what makes `fetch()` behave like Node's built-in fetch without extra configuration. Each section below names the surfaces it applies to. From d65fb8482fc7acf7742ffa12b2b10192961bd2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:02:21 +1200 Subject: [PATCH 24/61] S1: name the Rust surface's extra roots by what they are, not by the JS option --- .workhorse/specs/environment/variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index 6e9fa3f..6e51db9 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -27,7 +27,7 @@ On macOS and Windows the OS trust store is used directly and these are ignored, `NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any `tls.extraRoots` (see [TLS](../agent/tls.md)); certificates from both sources combine, and where `SSL_CERT_FILE` replaces the system roots this adds to them. It is lenient, matching Node's warn-and-continue behaviour: an empty value, an unreadable file, or an unparseable file is ignored rather than fatal. (The `tls.extraRoots` option, being an explicit programmatic choice, throws on malformed input instead.) -A Rust caller extends the trust store through `tls.extraRoots`. +On the Rust surface, extra roots come from the agent's TLS options alone. ## Certificate validation From 449301f8c55557df833085a3eddfe64c536456a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:02:41 +1200 Subject: [PATCH 25/61] S1: drop the Rust aside from the certificate-validation section --- .workhorse/specs/environment/variables.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index 6e51db9..109e579 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -35,7 +35,6 @@ This applies to the Node surface alone. `NODE_TLS_REJECT_UNAUTHORIZED` set to exactly `0` disables TLS certificate validation for the agent; any other value or unset keeps validation on. This matches Node's semantics and exists only for that compatibility; trusting a specific private CA via `NODE_EXTRA_CA_CERTS` or `tls.extraRoots` is the supported path. -The Rust surface keeps certificate validation on, that being a compatibility it does not owe. ## Proxies From 3459f7e39b21f0fc88d34cfbb21bccb4130c82d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:04:31 +1200 Subject: [PATCH 26/61] S1: drop the crate-split aside from the error code contract --- .workhorse/specs/errors/errors.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.workhorse/specs/errors/errors.md b/.workhorse/specs/errors/errors.md index efb73f2..1143467 100644 --- a/.workhorse/specs/errors/errors.md +++ b/.workhorse/specs/errors/errors.md @@ -11,7 +11,6 @@ Callers match on `error.code` against the exported `ERROR_CODES` map rather than Every error Faith throws carries a `code` set to a stable name for its kind. `ERROR_CODES` is exported and enumerates the library's error codes; it is generated from the same source as the errors themselves, so the two cannot drift. -That source is the client's own error type: a component crate names its own errors for the failures it can produce, and the client converts them as they cross into it, so the split into crates (see [RUST](../rust/overview.md)) leaves the set of codes and the kind each failure reports unchanged. Every code in `ERROR_CODES` is reachable: each one names a kind that some failure surfaces to the caller, so a branch written for any code in the map can fire. Error messages are prefixed with the kind name and may embed underlying detail; the message is for humans, the code is the API. Errors surfaced through reading the response `body` stream carry no `code`. From 37785ca824189097cbe4222a2779f0ecfce0b938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:06:19 +1200 Subject: [PATCH 27/61] S1: name both surfaces in the overview's opening line --- .workhorse/specs/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index 428a431..8999514 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -4,8 +4,8 @@ id: FAITH # Faith -Faith is a fetch implementation backed by a Rust network stack rather than Node's built-in HTTP machinery. -It aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, while exposing the capabilities that stack unlocks: transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. +Faith is a fetch API implementation for Node.js and Rust. +It aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, with transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. It reaches callers through two surfaces built from one implementation. The native module `@passcod/faith` is the Node.js surface, and the specs here describe its behaviour in JavaScript terms unless they say otherwise. From de982933074ab4aa2756daddbfcb617f4eb37ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:09:34 +1200 Subject: [PATCH 28/61] S1: list the two public surfaces in the overview --- .workhorse/specs/overview.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index 8999514..cf8a98d 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -7,9 +7,13 @@ id: FAITH Faith is a fetch API implementation for Node.js and Rust. It aims to behave like the browser's fetch wherever that concept translates to a server-side runtime, with transparent HTTP/2 and HTTP/3, IPv4/IPv6 Happy Eyeballs, DNS caching, an optional cookie jar, and HTTP caching. -It reaches callers through two surfaces built from one implementation. -The native module `@passcod/faith` is the Node.js surface, and the specs here describe its behaviour in JavaScript terms unless they say otherwise. -The crate `web-faith` is the Rust surface, published to crates.io with the component crates beneath it (see [RUST](rust/overview.md)); it keeps the same behaviour and spells it in Rust (see [RSAPI](rust/client-api.md)). +It has two public surfaces: + +- the Rust crate `web-faith` (see [RSAPI](rust/client-api.md)) and the component crates at that prefix (see [RUST](rust/overview.md)); +- the Node.js module `@passcod/faith`. + +Both are built from one implementation, so a behaviour specified here holds on both unless a spec says otherwise. +The specs describe it in JavaScript terms except where they name the Rust surface. The library's contract has two halves: fidelity to the fetch standard, and divergence where the standard assumes a browser. From 3ec14fa851d2020a7f4eae2a221e14a276394b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:10:54 +1200 Subject: [PATCH 29/61] S1: cut the one-implementation boilerplate from the overview --- .workhorse/specs/overview.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index cf8a98d..25f73e6 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -12,8 +12,7 @@ It has two public surfaces: - the Rust crate `web-faith` (see [RSAPI](rust/client-api.md)) and the component crates at that prefix (see [RUST](rust/overview.md)); - the Node.js module `@passcod/faith`. -Both are built from one implementation, so a behaviour specified here holds on both unless a spec says otherwise. -The specs describe it in JavaScript terms except where they name the Rust surface. +The specs describe behaviour in JavaScript terms except where they name the Rust surface. The library's contract has two halves: fidelity to the fetch standard, and divergence where the standard assumes a browser. From cc4b75019f8fc37a17d6131bfdf7e9706402be3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:17:53 +1200 Subject: [PATCH 30/61] S1: keep the overview to stance, and name shared options rather than spell them The compatibility stance mixed the stance itself with Node API detail: the shapes fetch accepts, the browser-assuming options that are ignored, and the extensions that are additive were each spelled out here as well as at their own site. All three are already specified in REQ and in the response specs, so what is left is the stance a reader comes to this file for. The claim that these specs describe behaviour in JavaScript terms is gone too. It licensed the wrong thing: a spec covering both surfaces should name the concept and link to where it is defined, and one that covers a single surface says so. Following that, the environment spec now names the agent's extra roots rather than spelling the Node option, which is defined in TLS either way. --- .workhorse/specs/environment/variables.md | 7 +++---- .workhorse/specs/overview.md | 10 +++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.workhorse/specs/environment/variables.md b/.workhorse/specs/environment/variables.md index 109e579..a922a1a 100644 --- a/.workhorse/specs/environment/variables.md +++ b/.workhorse/specs/environment/variables.md @@ -24,17 +24,16 @@ Both surfaces read `SSL_CERT_FILE` and `SSL_CERT_DIR`; `NODE_EXTRA_CA_CERTS` bel On Unix platforms other than macOS, `SSL_CERT_FILE` and `SSL_CERT_DIR` override where the system trust store is loaded from, with standard OpenSSL semantics: `SSL_CERT_FILE` replaces the system roots. On macOS and Windows the OS trust store is used directly and these are ignored, as Node does on those platforms. -`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any `tls.extraRoots` (see [TLS](../agent/tls.md)); certificates from both sources combine, and where `SSL_CERT_FILE` replaces the system roots this adds to them. +`NODE_EXTRA_CA_CERTS` names a PEM file whose certificates are added to the trust store on top of the platform roots and any extra roots set on the agent (see [TLS](../agent/tls.md)); certificates from both sources combine, and where `SSL_CERT_FILE` replaces the system roots this adds to them. It is lenient, matching Node's warn-and-continue behaviour: an empty value, an unreadable file, or an unparseable file is ignored rather than fatal. -(The `tls.extraRoots` option, being an explicit programmatic choice, throws on malformed input instead.) -On the Rust surface, extra roots come from the agent's TLS options alone. +Extra roots set on the agent are an explicit programmatic choice and fail loudly on malformed input instead. ## Certificate validation This applies to the Node surface alone. `NODE_TLS_REJECT_UNAUTHORIZED` set to exactly `0` disables TLS certificate validation for the agent; any other value or unset keeps validation on. -This matches Node's semantics and exists only for that compatibility; trusting a specific private CA via `NODE_EXTRA_CA_CERTS` or `tls.extraRoots` is the supported path. +This matches Node's semantics and exists only for that compatibility; trusting a specific private CA, through `NODE_EXTRA_CA_CERTS` or the agent's extra roots, is the supported path. ## Proxies diff --git a/.workhorse/specs/overview.md b/.workhorse/specs/overview.md index 25f73e6..3705ea4 100644 --- a/.workhorse/specs/overview.md +++ b/.workhorse/specs/overview.md @@ -12,8 +12,6 @@ It has two public surfaces: - the Rust crate `web-faith` (see [RSAPI](rust/client-api.md)) and the component crates at that prefix (see [RUST](rust/overview.md)); - the Node.js module `@passcod/faith`. -The specs describe behaviour in JavaScript terms except where they name the Rust surface. - The library's contract has two halves: fidelity to the fetch standard, and divergence where the standard assumes a browser. ## The standards Faith answers to @@ -28,11 +26,9 @@ A divergence is either a browser concept with no server-side meaning or a choice ## Compatibility stance -`fetch(resource, options)` accepts the same shapes as WHATWG fetch: a URL string or stringifiable object (including `URL`), or a Web API `Request` object. -Behaviour follows the fetch standard by default; where browsers and the standards disagree, Faith follows the standards unless a spec here says otherwise (for example, `body` is `null` on responses that cannot have a body). -Browser-only concepts that assume an origin or a browsing context (CORS, `mode`, `referrer`, `referrerPolicy`, `attributionReporting`, `browsingTopics`, `keepalive`) have no server-side meaning; passing them is harmless and they take no effect. -Options in a `RequestInit` that Faith does not recognise are ignored rather than rejected. -Faith-specific extensions (such as `agent`, `timeout`, response `peer`, `version`, `trailers`, `discard()`) are additive: code written against standard fetch runs unmodified. +Behaviour follows the fetch standard by default; where browsers and the standards disagree, Faith follows the standards unless a spec here says otherwise. +Concepts that assume an origin or a browsing context, CORS among them, have no server-side meaning, and a surface that admits the options naming them gives them no effect (see [REQ](fetch/request.md)). +Faith's own extensions add to the standard's surface rather than altering it, so code written against standard fetch runs unmodified. ## Protocol support From cc274999c3ae03f11a5dddac1386e7e6bcafce10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:19:01 +1200 Subject: [PATCH 31/61] S1: record the remaining spec sites that spell shared behaviour in one surface's names --- .workhorse/plans/s1/plan.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 598ea51..62d185c 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -148,6 +148,31 @@ Decisions taken while doing steps 0–7, worth not relitigating: comment is emitted verbatim into `index.d.ts`, where `[`X`]` means nothing, so plain backticks belong on anything a napi item documents. +## Outstanding: make the spec tree serve both surfaces + +A spec should be one of three things, never a fourth: generic to both surfaces (naming the concept +and linking to where it is defined), specified at the correct site, or explicitly about one surface +so the reader knows which. What it should not be is shared behaviour spelled in one surface's +identifiers, which is what a JavaScript name in a spec covering both amounts to. + +[FAITH](../../specs/overview.md) and [ENV](../../specs/environment/variables.md) are done. The +remaining sites, from a survey of JS-cased identifiers: + +- `agent/observability.md` (~20: `bodiesStarted`, `responseCount`, `rttUs`, and the rest of the + per-connection fields), `agent/warm-up.md` (~10, `prefetchDns` five times), + `agent/cookies.md` (~8), `agent/dns.md` (~6), `agent/overview.md` (~5), + `agent/flow-control.md` (~4), `agent/connection-pool.md` (~3), and scattered singles. +- `response/response.md` and `response/reading-the-body.md` need judgment rather than a sweep: most + of their identifiers are Resource Timing's own field names (`fetchStart`, `requestStart`), which + are standard vocabulary and should stay. Faith's own (`bodyUsed`, `statusText`) are the ones that + want naming as concepts, [RSAPI](../../specs/rust/client-api.md) spelling them `body_used` and + `status_text`. +- `rust/client-api.md` needs nothing: its `Request` and `Response` are the Rust types. + +So the raw identifier count overstates the work — a blind sweep would churn standard-defined names +and Rust types alike. Roughly 60 genuine sites across about ten files, each needing a decision on +whether the surrounding spec is shared or single-surface. + ## Verification discipline Every step must leave `cargo build`, `cargo test`, and the napi `npm run build` green From ea11858e8ef06fe576b7d39dfd8e3f50861f3fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:20:31 +1200 Subject: [PATCH 32/61] S1: sequence the spec sweep as the closing step --- .workhorse/plans/s1/plan.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 62d185c..bf73b52 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -103,6 +103,9 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al versioning from `1.0.0`. Measure CI cost before adding jobs (see project memory). - [ ] **13. First publish** to crates.io: the six components, then `web-faith`; `@passcod/faith` continues from npm via `web-faith-napi`. +- [ ] **14. The both-surfaces spec sweep**, as the closing pass over the tree — deliberately last, + once the Rust API is settled and its names are known. See the section below for the site-by-site + reconnaissance. ## What the extractions settled @@ -148,13 +151,17 @@ Decisions taken while doing steps 0–7, worth not relitigating: comment is emitted verbatim into `index.d.ts`, where `[`X`]` means nothing, so plain backticks belong on anything a napi item documents. -## Outstanding: make the spec tree serve both surfaces +## Step 14: the both-surfaces spec sweep A spec should be one of three things, never a fourth: generic to both surfaces (naming the concept and linking to where it is defined), specified at the correct site, or explicitly about one surface so the reader knows which. What it should not be is shared behaviour spelled in one surface's identifiers, which is what a JavaScript name in a spec covering both amounts to. +It runs last because the Rust names it will cite are step 9's to settle: sweeping earlier would mean +guessing at them, and a spec that cites a name which then changes is worse than one that has not +been swept yet. + [FAITH](../../specs/overview.md) and [ENV](../../specs/environment/variables.md) are done. The remaining sites, from a survey of JS-cased identifiers: From 66825a32b36e46e92caa283f87854ab20c7737cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:20:57 +1200 Subject: [PATCH 33/61] S1: open the Rust API spec on the agent rather than a count of its nouns --- .workhorse/specs/rust/client-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md index 7b05be8..c3e16b4 100644 --- a/.workhorse/specs/rust/client-api.md +++ b/.workhorse/specs/rust/client-api.md @@ -4,7 +4,7 @@ id: RSAPI # The Rust client API -`web-faith` has one noun and one verb: an `Agent` owns the connection pool, resolver, cookie jar, HTTP cache, and HTTP/3 knowledge, and `fetch` is how every request goes out. +An `Agent` owns the connection pool, resolver, cookie jar, HTTP cache, and HTTP/3 knowledge, and `fetch` is how every request goes out. The surface is fetch-flavoured rather than a transcription of the JavaScript API: it keeps fetch's vocabulary and the defaults the Node surface has, and it speaks the Rust ecosystem's types wherever one exists for the job. The agent's own behaviour, its options, and its lifecycle are specified in [AGENT](../agent/overview.md) and the specs beneath it; this spec covers the shape a Rust caller sees. From ef83a4a1a044a3e3d58b8043c8294e3fec049dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:27:32 +1200 Subject: [PATCH 34/61] S1: move the warm-up argument parsing into web-faith Warming a name and warming an origin both take their argument loosely: a host with a scheme or a port it does not need, a URL with a path and userinfo that play no part in the origin. The parsing that decides what was meant is pure, so it moves to the client with its eight tests, leaving the verbs to act on the result. The origin key here and the one the Alt-Svc cache derives are the same format arrived at twice; worth merging when the agent itself moves. --- Cargo.lock | 1 + crates/web-faith-napi/src/agent.rs | 153 +-------------------------- crates/web-faith/Cargo.toml | 1 + crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/warm_up.rs | 164 +++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+), 152 deletions(-) create mode 100644 crates/web-faith/src/warm_up.rs diff --git a/Cargo.lock b/Cargo.lock index c688b87..e620dcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2810,6 +2810,7 @@ dependencies = [ "stream_shared", "strum", "tokio", + "url", "web-faith-alt-svc", "web-faith-cookies", "web-faith-dns", diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index fe0b66f..ebb8fdd 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -32,6 +32,7 @@ use web_faith::client::{ }; #[cfg(feature = "http3")] use web_faith::client::{H3UpgradeRecipe, install_https_sink}; +use web_faith::warm_up::{extract_host, origin_key, reduce_to_origin}; #[cfg(feature = "http3")] use web_faith_alt_svc::parse_alt_svc_header; #[cfg(feature = "http3")] @@ -1888,162 +1889,10 @@ fn caller_error(env: &Env, kind: FaithErrorKind) -> napi::Error { napi::Error::from(FaithError::from(kind).into_js_error(env)) } -/// Extract the bare host from a `prefetchDns` argument, ignoring any scheme, port, or path, or -/// `None` if there is no host to resolve. A DNS name carries none of those parts, so a fuller -/// string is reduced to its host. (spec:WARM) -fn extract_host(input: &str) -> Option { - // A string that already spells a scheme is read as the URL it is; anything else is the - // bare-host case, where a name is not a URL on its own and giving it an authority makes it - // parse as one. Telling the two apart on the scheme separator matters both ways: `example.com:8443` - // otherwise parses as a *scheme* of `example.com` with no host, and a schemed string with no host - // (`file:///path`, a bare `https://`) would have its scheme misread as a host by the fallback. - let url = if input.contains("://") { - Url::parse(input).ok()? - } else { - Url::parse(&format!("dns://{input}")).ok()? - }; - let host = url.host_str()?; - // `host_str` brackets an IPv6 literal; the resolver wants it bare. - let host = host - .strip_prefix('[') - .and_then(|rest| rest.strip_suffix(']')) - .unwrap_or(host); - (!host.is_empty()).then(|| host.to_owned()) -} - -/// Reduce a `preconnect` argument to its origin, or `None` if it is not a connectable origin. Path, -/// query, fragment, and userinfo are stripped — the same reduction the HTTP/3 probe applies — and -/// the scheme must have a known default port so an omitted port resolves. (spec:WARM) -fn reduce_to_origin(input: &str) -> Option { - let mut url = Url::parse(input).ok()?; - if !url.has_host() || url.port_or_known_default().is_none() { - return None; - } - url.set_path("/"); - url.set_query(None); - url.set_fragment(None); - let _ = url.set_username(""); - let _ = url.set_password(None); - Some(url) -} - -/// The `scheme://host:port` key an origin coalesces on, with the port defaulted by scheme so -/// `https://host` and `https://host:443` are the same origin. Matches the Alt-Svc cache's key. -pub(crate) fn origin_key(url: &Url) -> String { - format!( - "{}://{}:{}", - url.scheme(), - url.host_str().unwrap_or_default(), - url.port_or_known_default().unwrap_or_default(), - ) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn prefetch_dns_takes_a_bare_host() { - assert_eq!(extract_host("example.com").as_deref(), Some("example.com")); - } - - #[test] - fn prefetch_dns_ignores_the_parts_a_name_does_not_have() { - // A DNS name has no scheme, port, or path, so a fuller string is reduced to its host - // rather than rejected (spec:WARM#prefetchdns). - for input in [ - "https://example.com", - "https://example.com:8443", - "https://example.com/some/path?q=1#frag", - "https://user:pass@example.com/", - "example.com:8443", - ] { - assert_eq!( - extract_host(input).as_deref(), - Some("example.com"), - "{input:?} names example.com whatever else it carries" - ); - } - } - - #[test] - fn prefetch_dns_unwraps_an_ipv6_literal() { - // `host_str` brackets an IPv6 literal, but the resolver wants it bare. - assert_eq!( - extract_host("https://[2001:db8::1]:8443").as_deref(), - Some("2001:db8::1") - ); - } - - #[test] - fn prefetch_dns_rejects_a_string_with_no_host() { - for input in ["", " ", "/just/a/path", "https://"] { - assert!( - extract_host(input).is_none(), - "{input:?} names no host to resolve" - ); - } - } - - #[test] - fn preconnect_reduces_a_longer_url_to_its_origin() { - // The same reduction the HTTP/3 probe applies (spec:WARM#preconnect). - let url = reduce_to_origin("https://user:pass@example.com/some/path?q=1#frag") - .expect("a full URL reduces to its origin"); - - assert_eq!(url.as_str(), "https://example.com/"); - assert_eq!(url.username(), "", "userinfo is stripped"); - assert_eq!(url.password(), None); - assert_eq!(url.query(), None); - assert_eq!(url.fragment(), None); - } - - #[test] - fn preconnect_defaults_the_port_by_scheme() { - // An omitted port defaults by scheme, so an origin spelled either way coalesces on one - // key (spec:WARM#preconnect). - for (bare, spelled) in [ - ("https://example.com", "https://example.com:443"), - ("http://example.com", "http://example.com:80"), - ] { - let bare = origin_key(&reduce_to_origin(bare).expect("parses")); - let spelled = origin_key(&reduce_to_origin(spelled).expect("parses")); - assert_eq!( - bare, spelled, - "the omitted port defaults to the spelled one" - ); - } - } - - #[test] - fn preconnect_keeps_distinct_origins_apart() { - // The pool caps and the warm record are per origin: scheme, host, and port together - // (spec:POOL). - let key = |input: &str| origin_key(&reduce_to_origin(input).expect("parses")); - - assert_ne!(key("https://example.com"), key("https://example.com:8443")); - assert_ne!(key("https://example.com"), key("http://example.com")); - assert_ne!(key("https://example.com"), key("https://other.example")); - } - - #[test] - fn preconnect_rejects_what_cannot_be_connected_to() { - for input in [ - "not an origin", - "", - "/just/a/path", - // No host to connect to. - "file:///etc/hosts", - // No default port for the scheme, and none given. - "unknownscheme://example.com", - ] { - assert!( - reduce_to_origin(input).is_none(), - "{input:?} is not a connectable origin" - ); - } - } - fn common(stream: Option, connection: Option) -> AgentFlowControlOptions { AgentFlowControlOptions { stream_window: stream, diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 91cc085..3f9272a 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -23,6 +23,7 @@ reqwest-middleware.workspace = true stream_shared.workspace = true strum.workspace = true tokio.workspace = true +url.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-alt-svc = { workspace = true, optional = true } diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 6c253c6..d5e036c 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -22,5 +22,6 @@ pub mod client; pub mod error; pub mod retry; pub mod timing; +pub mod warm_up; pub use error::{FaithError, FaithErrorKind, error_codes}; diff --git a/crates/web-faith/src/warm_up.rs b/crates/web-faith/src/warm_up.rs new file mode 100644 index 0000000..04358ab --- /dev/null +++ b/crates/web-faith/src/warm_up.rs @@ -0,0 +1,164 @@ +//! Reading what a warm-up was asked to warm. +//! +//! Warming a name and warming an origin take their arguments loosely, so the parsing that decides +//! what was meant is worth keeping in one place, away from the verbs that act on it. + +use url::Url; + +// spec:WARM +/// Extract the bare host from a a DNS prefetch argument, ignoring any scheme, port, or path, or +/// `None` if there is no host to resolve. A DNS name carries none of those parts, so a fuller +/// string is reduced to its host. +pub fn extract_host(input: &str) -> Option { + // A string that already spells a scheme is read as the URL it is; anything else is the + // bare-host case, where a name is not a URL on its own and giving it an authority makes it + // parse as one. Telling the two apart on the scheme separator matters both ways: `example.com:8443` + // otherwise parses as a *scheme* of `example.com` with no host, and a schemed string with no host + // (`file:///path`, a bare `https://`) would have its scheme misread as a host by the fallback. + let url = if input.contains("://") { + Url::parse(input).ok()? + } else { + Url::parse(&format!("dns://{input}")).ok()? + }; + let host = url.host_str()?; + // `host_str` brackets an IPv6 literal; the resolver wants it bare. + let host = host + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .unwrap_or(host); + (!host.is_empty()).then(|| host.to_owned()) +} + +/// Reduce a preconnect argument to its origin, or `None` if it is not a connectable origin. Path, +/// query, fragment, and userinfo are stripped — the same reduction the HTTP/3 probe applies — and +/// the scheme must have a known default port so an omitted port resolves. +pub fn reduce_to_origin(input: &str) -> Option { + let mut url = Url::parse(input).ok()?; + if !url.has_host() || url.port_or_known_default().is_none() { + return None; + } + url.set_path("/"); + url.set_query(None); + url.set_fragment(None); + let _ = url.set_username(""); + let _ = url.set_password(None); + Some(url) +} + +/// The `scheme://host:port` key an origin coalesces on, with the port defaulted by scheme so +/// `https://host` and `https://host:443` are the same origin. Matches the Alt-Svc cache's key. +pub fn origin_key(url: &Url) -> String { + format!( + "{}://{}:{}", + url.scheme(), + url.host_str().unwrap_or_default(), + url.port_or_known_default().unwrap_or_default(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefetch_dns_takes_a_bare_host() { + assert_eq!(extract_host("example.com").as_deref(), Some("example.com")); + } + + #[test] + fn prefetch_dns_ignores_the_parts_a_name_does_not_have() { + // A DNS name has no scheme, port, or path, so a fuller string is reduced to its host + // rather than rejected (spec:WARM#prefetchdns). + for input in [ + "https://example.com", + "https://example.com:8443", + "https://example.com/some/path?q=1#frag", + "https://user:pass@example.com/", + "example.com:8443", + ] { + assert_eq!( + extract_host(input).as_deref(), + Some("example.com"), + "{input:?} names example.com whatever else it carries" + ); + } + } + + #[test] + fn prefetch_dns_unwraps_an_ipv6_literal() { + // `host_str` brackets an IPv6 literal, but the resolver wants it bare. + assert_eq!( + extract_host("https://[2001:db8::1]:8443").as_deref(), + Some("2001:db8::1") + ); + } + + #[test] + fn prefetch_dns_rejects_a_string_with_no_host() { + for input in ["", " ", "/just/a/path", "https://"] { + assert!( + extract_host(input).is_none(), + "{input:?} names no host to resolve" + ); + } + } + + #[test] + fn preconnect_reduces_a_longer_url_to_its_origin() { + // The same reduction the HTTP/3 probe applies (spec:WARM#preconnect). + let url = reduce_to_origin("https://user:pass@example.com/some/path?q=1#frag") + .expect("a full URL reduces to its origin"); + + assert_eq!(url.as_str(), "https://example.com/"); + assert_eq!(url.username(), "", "userinfo is stripped"); + assert_eq!(url.password(), None); + assert_eq!(url.query(), None); + assert_eq!(url.fragment(), None); + } + + #[test] + fn preconnect_defaults_the_port_by_scheme() { + // An omitted port defaults by scheme, so an origin spelled either way coalesces on one + // key (spec:WARM#preconnect). + for (bare, spelled) in [ + ("https://example.com", "https://example.com:443"), + ("http://example.com", "http://example.com:80"), + ] { + let bare = origin_key(&reduce_to_origin(bare).expect("parses")); + let spelled = origin_key(&reduce_to_origin(spelled).expect("parses")); + assert_eq!( + bare, spelled, + "the omitted port defaults to the spelled one" + ); + } + } + + #[test] + fn preconnect_keeps_distinct_origins_apart() { + // The pool caps and the warm record are per origin: scheme, host, and port together + // (spec:POOL). + let key = |input: &str| origin_key(&reduce_to_origin(input).expect("parses")); + + assert_ne!(key("https://example.com"), key("https://example.com:8443")); + assert_ne!(key("https://example.com"), key("http://example.com")); + assert_ne!(key("https://example.com"), key("https://other.example")); + } + + #[test] + fn preconnect_rejects_what_cannot_be_connected_to() { + for input in [ + "not an origin", + "", + "/just/a/path", + // No host to connect to. + "file:///etc/hosts", + // No default port for the scheme, and none given. + "unknownscheme://example.com", + ] { + assert!( + reduce_to_origin(input).is_none(), + "{input:?} is not a connectable origin" + ); + } + } +} From a261bbd81d3b17eeca24c61407a653f2d1770995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:33:04 +1200 Subject: [PATCH 35/61] S1: move the agent's counters into web-faith The running counters are the agent's, and a response holds a share of them because a body finishing is what settles two of the four. Both surfaces report them, so they move to the client, which reads them as the u64 they are counted in. The Node surface keeps its own shape, converting from that reading: JavaScript takes them as i64, and a count past that saturates rather than wrapping, exactly as before. --- crates/web-faith-napi/src/agent.rs | 48 ++++++++------------------- crates/web-faith-napi/src/response.rs | 2 +- crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/stats.rs | 39 ++++++++++++++++++++++ 4 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 crates/web-faith/src/stats.rs diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index ebb8fdd..a98d8ff 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -32,6 +32,7 @@ use web_faith::client::{ }; #[cfg(feature = "http3")] use web_faith::client::{H3UpgradeRecipe, install_https_sink}; +use web_faith::stats::InnerAgentStats; use web_faith::warm_up::{extract_host, origin_key, reduce_to_origin}; #[cfg(feature = "http3")] use web_faith_alt_svc::parse_alt_svc_header; @@ -848,14 +849,6 @@ pub struct AgentOptions { pub user_agent: Option, } -#[derive(Debug, Default)] -pub(crate) struct InnerAgentStats { - pub requests_sent: AtomicU64, - pub responses_received: AtomicU64, - pub bodies_started: AtomicU64, - pub bodies_finished: AtomicU64, -} - #[napi] #[derive(Debug, Clone, Default)] pub struct AgentStats { @@ -869,6 +862,18 @@ pub struct AgentStats { pub bodies_finished: i64, } +impl From for AgentStats { + fn from(stats: web_faith::stats::AgentStats) -> Self { + let count = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); + Self { + requests_sent: count(stats.requests_sent), + responses_received: count(stats.responses_received), + bodies_started: count(stats.bodies_started), + bodies_finished: count(stats.bodies_finished), + } + } +} + /// One entry of `Agent.resolvers()`: a DNS server the agent resolves through (spec:OBS#resolvers). #[napi(object)] #[derive(Debug, Clone)] @@ -1640,32 +1645,7 @@ impl Agent { /// - `bodiesFinished` #[napi] pub fn stats(&self) -> AgentStats { - AgentStats { - requests_sent: self - .stats - .requests_sent - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - responses_received: self - .stats - .responses_received - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - bodies_started: self - .stats - .bodies_started - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - bodies_finished: self - .stats - .bodies_finished - .load(Ordering::Relaxed) - .try_into() - .unwrap_or(i64::MAX), - } + AgentStats::from(self.stats.snapshot()) } /// Returns information on current connections open by this agent. diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index a5d80eb..34ed075 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -30,13 +30,13 @@ use tokio::{io::AsyncWriteExt, sync::watch}; use web_faith::{ body::{Body, BodyHolder, DynStream, drain_body_inner}, + stats::InnerAgentStats, timing::TimingSlot, }; use web_faith_encoding::{Coding, decode_stream}; use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; use crate::{ - agent::InnerAgentStats, async_task::{Value, faith_promise}, error::{FaithError, FaithErrorExt, FaithErrorKind}, timing::TimingBreakdown, diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index d5e036c..238c4a8 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -21,6 +21,7 @@ pub mod body; pub mod client; pub mod error; pub mod retry; +pub mod stats; pub mod timing; pub mod warm_up; diff --git a/crates/web-faith/src/stats.rs b/crates/web-faith/src/stats.rs new file mode 100644 index 0000000..a055142 --- /dev/null +++ b/crates/web-faith/src/stats.rs @@ -0,0 +1,39 @@ +//! What an agent counts about the requests it has run. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The agent's running counters, incremented as requests and bodies pass through it. +/// +/// Held behind an `Arc` and shared with every response the agent produces, since a body finishing +/// is what settles two of these and the response is what knows it happened. +#[derive(Debug, Default)] +pub struct InnerAgentStats { + pub requests_sent: AtomicU64, + pub responses_received: AtomicU64, + pub bodies_started: AtomicU64, + pub bodies_finished: AtomicU64, +} + +impl InnerAgentStats { + /// Read the counters as they stand. + pub fn snapshot(&self) -> AgentStats { + AgentStats { + requests_sent: self.requests_sent.load(Ordering::Relaxed), + responses_received: self.responses_received.load(Ordering::Relaxed), + bodies_started: self.bodies_started.load(Ordering::Relaxed), + bodies_finished: self.bodies_finished.load(Ordering::Relaxed), + } + } +} + +/// A reading of an agent's counters, taken at one moment. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AgentStats { + pub requests_sent: u64, + pub responses_received: u64, + /// Response body streams that have been started, which is what reading a body does. + pub bodies_started: u64, + /// Response body streams that have been read to the end. While more have started than + /// finished, that many bodies are holding connections open. + pub bodies_finished: u64, +} From 807ae5b5556946462e17adf2b59fd4e7f20330d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:14 +1200 Subject: [PATCH 36/61] S1: invert the agent, so the client owns it and the binding wraps it The agent moves to web-faith. What stays behind is the Node surface's own work: validating whatever a caller passed as AgentOptions into a recipe and settings, and a napi class holding a handle on the client's agent. The verbs are the client's now. close and network_changed were pure already and move as they were. The cookie verbs take a parsed URL rather than a string, since parsing what a caller wrote belongs to the surface that took it. prefetch_dns and preconnect return a future rather than reaching for a JS promise: a warm-up that cannot be attempted is refused before the future exists, which is what lets the binding throw synchronously for a caller's mistake while a network failure stays quiet. The client's agent also carries the settings a request consults, so the split is between what builds a client and what each request reads. The Node surface is unchanged. index.d.ts loses only the spec markers that had been leaking out of doc comments into the published TypeScript; every line of documentation a caller reads is intact. --- Cargo.lock | 3 + crates/web-faith-napi/src/agent.rs | 422 +++-------------------- crates/web-faith-napi/src/fetch.rs | 41 ++- crates/web-faith/Cargo.toml | 3 + crates/web-faith/src/agent.rs | 515 +++++++++++++++++++++++++++++ crates/web-faith/src/lib.rs | 1 + index.d.ts | 8 +- 7 files changed, 600 insertions(+), 393 deletions(-) create mode 100644 crates/web-faith/src/agent.rs diff --git a/Cargo.lock b/Cargo.lock index e620dcd..ee565eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2805,6 +2805,8 @@ dependencies = [ "http-body-util", "http-cache-reqwest", "hyper", + "hyper-util", + "moka", "reqwest", "reqwest-middleware", "stream_shared", @@ -2812,6 +2814,7 @@ dependencies = [ "tokio", "url", "web-faith-alt-svc", + "web-faith-conn-tracker", "web-faith-cookies", "web-faith-dns", "web-faith-integrity", diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index a98d8ff..e75f8bd 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -2,43 +2,31 @@ use std::{ fmt::Debug, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, UdpSocket}, str::FromStr as _, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, + sync::Arc, time::Duration, }; use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; -use http::Version; use http_cache_reqwest::{ CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, }; -use hyper_util::client::legacy::connect::HttpInfo; -use moka::sync::Cache as MokaCache; use napi::{Either, Env, bindgen_prelude::Buffer}; use napi_derive::napi; use reqwest::{ - Certificate, Client, Identity, Url, - cookie::CookieStore as _, + Certificate, Identity, Url, header::{HeaderMap, HeaderName, HeaderValue}, }; -use reqwest_middleware::ClientWithMiddleware; +use web_faith::agent::AgentSettings; +#[cfg(feature = "http3")] +use web_faith::client::H3UpgradeRecipe; use web_faith::client::{ ClientRecipe, DEFAULT_CONNECTION_WINDOW, DEFAULT_STREAM_WINDOW, HttpCacheRecipe, HttpCacheStore, NodeEnvRecipe, RedirectPolicy, ResolvedWindows, }; #[cfg(feature = "http3")] -use web_faith::client::{H3UpgradeRecipe, install_https_sink}; -use web_faith::stats::InnerAgentStats; -use web_faith::warm_up::{extract_host, origin_key, reduce_to_origin}; -#[cfg(feature = "http3")] -use web_faith_alt_svc::parse_alt_svc_header; -#[cfg(feature = "http3")] -use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, H3Prober}; -use web_faith_conn_tracker::ConnectionTracker; +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig}; use web_faith_dns::{ DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, }; @@ -904,68 +892,7 @@ pub struct ResolverInfo { #[napi] #[derive(Debug, Clone)] pub struct Agent { - /// `None` once [`Agent::close`] has been called. The heavy resources - /// (connection pool, DNS resolver, background tasks) live inside this - /// client, so dropping it is what actually releases them. - pub(crate) client: Option, - /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A - /// `preconnect` warm-up sends its synthetic request here rather than through the middleware - /// stack, which bypasses the HTTP cache and the Alt-Svc layer (and so keeps the warm-up out of - /// request accounting), while still pooling the connection foreground requests reuse. `None` - /// once the agent is closed. (spec:WARM) - pub(crate) raw_client: Option, - /// Faith's DNS resolver, shared with [`Self::client`] so `prefetchDns` warms the cache requests - /// read. `None` under the system resolver, where there is no such cache. (spec:WARM) - pub(crate) dns_resolver: Option, - /// Origins with a warm-up connection opened within the pool idle window, so a repeat - /// `preconnect` does no new work. Keyed by `scheme://host:port`; entries expire with the idle - /// timeout. (spec:WARM) - pub(crate) warmed: MokaCache, - /// Single-flight claims for in-flight `preconnect` warm-ups, so concurrent calls for the same - /// origin do not open duplicate connections. (spec:WARM) - pub(crate) warming: MokaCache, - /// Bumped by `networkChanged`, so a warm-up that was in flight across the signal does not - /// record its origin as warm: its connection went into the pool that was just dropped - /// (spec:NETCHG#reach-across-the-subsystems). - pub(crate) warm_generation: Arc, - pub(crate) cookie_jar: Option>, - pub(crate) stats: Arc, - pub(crate) conn_tracker: Arc, - #[cfg(feature = "http3")] - #[allow(dead_code)] - pub(crate) alt_svc_cache: Option>, - /// Held so `close()` can abort in-flight background probes: each one owns a - /// clone of the raw client, which would otherwise keep the connection pool - /// alive past close for up to the probe timeout. - #[cfg(feature = "http3")] - pub(crate) h3_prober: Option>, - /// Mirrors `http3.upgradeFollowAdvertisedPort`. Lives here because `fetch` needs - /// it to stop a rewritten port from being reported as a redirect. - pub(crate) h3_follow_advertised_port: bool, - /// Mirrors `http3.upgradeEnabled`. A warm-up needs it to route the way a foreground request - /// would: with the upgrade machinery off, nothing upgrades, whatever the caches hold. - /// (spec:WARM#preconnect) - #[cfg(feature = "http3")] - pub(crate) h3_upgrade_enabled: bool, - /// Mirrors `quirks.h1RequestStreaming`. `fetch` consults it to decide whether a streaming - /// request body may go out over HTTP/1.x (spec:QUIRK#http-1-x-request-body-streaming). - pub(crate) quirk_h1_request_streaming: bool, - /// The agent's default `Accept-Encoding`, if one was set among its default headers. - /// `fetch` consults it to decide which codings to decode when a request adds none of - /// its own (see [`crate::encoding`]). - pub(crate) default_accept_encoding: Option, - /// The agent's default `Content-Encoding`, if one was set among its default headers. - /// `fetch` consults it when the `compress` option layers a coding on top of what a - /// request already declares, since setting the joined value on the request would - /// otherwise displace this default rather than build on it (spec:ENC). - pub(crate) default_content_encoding: Option, - /// Whether a `Priority` header sits among the agent's default headers. `fetch` consults - /// it so that default wins over the header the `priority` option would derive. - pub(crate) has_default_priority: bool, - /// How to build this agent's clients, so `networkChanged` can build them again - /// (spec:NETCHG). Shared rather than cloned per agent clone: every clone builds the same - /// client from the same recipe, and `fetch` clones the agent per request. - pub(crate) recipe: Arc, + pub(crate) inner: web_faith::agent::Agent, } #[napi] @@ -1422,45 +1349,7 @@ impl Agent { h3_upgrade, }; - let conn_timeout = recipe.conn_timeout(); - let built = recipe.build( - cookie_jar.as_ref(), - dns_resolver.as_ref(), - #[cfg(feature = "http3")] - alt_svc_cache.as_ref(), - )?; - - // Only now do all three exist: the resolver is built before the cache, and the prober - // holds a client that holds the resolver, so this is the earliest the loop can be closed - // (spec:DNS#https-records). - #[cfg(feature = "http3")] - install_https_sink( - dns_resolver.as_ref(), - alt_svc_cache.as_ref(), - built.prober.as_ref(), - recipe.h3_upgrade.enabled, - ); - - Ok(Self { - client: Some(built.client), - raw_client: Some(built.raw_client), - dns_resolver, - // A warm-up connection is warm only as long as the pool keeps it idle, so the record - // that an origin is warm expires with that same window. - warmed: MokaCache::builder().time_to_live(conn_timeout).build(), - // A safety TTL well past any reasonable warm-up, so a claim that never gets released - // (a warm-up whose task is dropped) frees the origin rather than wedging it. - warming: MokaCache::builder() - .time_to_live(Duration::from_secs(300)) - .build(), - warm_generation: Default::default(), - cookie_jar, - stats: Default::default(), - conn_tracker: ConnectionTracker::new(conn_timeout), - #[cfg(feature = "http3")] - alt_svc_cache, - #[cfg(feature = "http3")] - h3_prober: built.prober, + let settings = AgentSettings { h3_follow_advertised_port, #[cfg(feature = "http3")] h3_upgrade_enabled: recipe.h3_upgrade.enabled, @@ -1468,7 +1357,17 @@ impl Agent { default_accept_encoding, default_content_encoding, has_default_priority, - recipe: Arc::new(recipe), + }; + + Ok(Self { + inner: web_faith::agent::Agent::build( + recipe, + settings, + cookie_jar, + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + )?, }) } @@ -1492,23 +1391,7 @@ impl Agent { /// no-op. The cookie store, if any, remains readable via `getCookie`. #[napi] pub fn close(&mut self) { - // Dropping the client releases the reqwest connection pool and the - // Hickory resolver task; the alt-svc cache goes with it. The raw client - // shares that pool and the resolver, so it goes too, and both are what a - // later `preconnect`/`prefetchDns` checks to throw the closed-agent error. - self.client = None; - self.raw_client = None; - self.dns_resolver = None; - #[cfg(feature = "http3")] - { - // Probes hold a raw client clone; abort them so the pool doesn't - // outlive close by up to the probe timeout. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); - } - self.h3_prober = None; - self.alt_svc_cache = None; - } + self.inner.close(); } /// Tell the agent the network underneath it has changed, so it stops deciding from what it @@ -1527,71 +1410,10 @@ impl Agent { /// Requests already in flight are not interrupted and run to completion on the connections /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a /// closed agent does nothing, and calling it repeatedly is harmless. - /// - /// spec:NETCHG + // spec:NETCHG #[napi] pub fn network_changed(&mut self) { - // A closed agent has already released all of this. - if self.client.is_none() { - return; - } - - // reqwest cannot drop pooled connections short of dropping the client, so the client is - // rebuilt from the recipe the agent kept for this. Requests in flight hold their own - // clone of the old client (`fetch` clones the agent per request), so they run to - // completion and the old pool goes when the last of them finishes. - // - // A rebuild that fails leaves the agent on its existing client: the options were already - // validated at construction, so a failure here is not the caller's to answer for, and an - // agent that still works on the old network beats one that works nowhere. - let built = self.recipe.build( - self.cookie_jar.as_ref(), - self.dns_resolver.as_ref(), - #[cfg(feature = "http3")] - self.alt_svc_cache.as_ref(), - ); - if let Ok(built) = built { - #[cfg(feature = "http3")] - { - // Abort probes running on the old client: each holds a clone of it, and their - // answers would describe the path that has just gone away. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); - } - self.h3_prober = built.prober; - // The sink holds the prober, which has just been replaced along with the client - // it sends on; leaving the old one installed would aim DNS-triggered probes at a - // client that has been dropped. - install_https_sink( - self.dns_resolver.as_ref(), - self.alt_svc_cache.as_ref(), - self.h3_prober.as_ref(), - self.h3_upgrade_enabled, - ); - } - self.client = Some(built.client); - self.raw_client = Some(built.raw_client); - } - - // Names resolve afresh against the new network, through that network's own servers: the - // resolver drops what it read off the old one and reads again when next used. Under the - // system resolver there is no resolver here and so nothing to reset (spec:DNS). - if let Some(resolver) = &self.dns_resolver { - resolver.reset(); - } - - #[cfg(feature = "http3")] - if let Some(alt_svc_cache) = &self.alt_svc_cache { - alt_svc_cache.network_changed(); - } - - // The warm-up records describe pooled connections that have just been dropped, so a - // `preconnect` after the signal opens a connection rather than finding the origin warm - // (spec:NETCHG, spec:WARM). The single-flight claims are left alone: a warm-up still in - // flight is not duplicated by releasing its claim, and the generation bump is what stops - // it recording an origin as warm on the strength of a connection in the dropped pool. - self.warmed.invalidate_all(); - self.warm_generation.fetch_add(1, Ordering::Relaxed); + self.inner.network_changed(); } /// Add a cookie into the agent. @@ -1605,15 +1427,11 @@ impl Agent { /// - the cookie is larger than `cookies.maxSize` #[napi] pub fn add_cookie(&self, url: String, cookie: String) { - let Some(jar) = &self.cookie_jar else { - return; - }; - let Ok(url) = Url::from_str(&url) else { return; }; - jar.add_cookie_str(&cookie, &url); + self.inner.add_cookie(&url, &cookie); } /// Retrieve a cookie from the store. @@ -1625,16 +1443,8 @@ impl Agent { /// - the cookie cannot be represented as a string #[napi] pub fn get_cookie(&self, url: String) -> Option { - let Some(jar) = &self.cookie_jar else { - return None; - }; - - let Ok(url) = Url::from_str(&url) else { - return None; - }; - - jar.cookies(&url) - .and_then(|val| val.to_str().ok().map(ToOwned::to_owned)) + let url = Url::from_str(&url).ok()?; + self.inner.cookie_header(&url) } /// Returns statistics gathered by this agent: @@ -1645,7 +1455,7 @@ impl Agent { /// - `bodiesFinished` #[napi] pub fn stats(&self) -> AgentStats { - AgentStats::from(self.stats.snapshot()) + AgentStats::from(self.inner.stats()) } /// Returns information on current connections open by this agent. @@ -1657,7 +1467,7 @@ impl Agent { /// on field availability. If the platform isn't supported at all, this will always return empty. #[napi] pub fn connections<'env>(&self, env: &'env Env) -> Vec> { - connections_for_napi(&self.conn_tracker, env) + connections_for_napi(&self.inner.conn_tracker, env) } /// Returns the DNS servers this agent resolves through, in the order they are queried, so @@ -1666,32 +1476,18 @@ impl Agent { /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). /// The list is empty until the resolver has been used, because it reads its configuration on - /// first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) + /// first use, and empty for an agent using the system resolver. #[napi] pub fn resolvers(&self) -> Vec { - self.dns_resolver - .as_ref() - .map(|resolver| { - resolver - .resolvers() - .into_iter() - .map(|report| ResolverInfo { - address: report.address, - transport: report.transport, - source: report.source, - }) - .collect() + self.inner + .resolvers() + .into_iter() + .map(|report| ResolverInfo { + address: report.address, + transport: report.transport, + source: report.source, }) - .unwrap_or_default() - } - - /// Note that a request reached this origin, so it holds a connection the pool keeps idle for - /// the idle window and a `preconnect` for it has no new work to do (spec:WARM). - /// - /// Called for foreground requests as well as warm-ups, because the criterion is about the - /// origin holding an idle pooled connection, not about how it came to hold one. - pub(crate) fn mark_warm(&self, url: &Url) { - self.warmed.insert(origin_key(url), ()); + .collect() } /// Warm the DNS cache for `host`, so a later request to it skips the lookup. @@ -1701,26 +1497,19 @@ impl Agent { /// lands in the cache and never rejects, whatever happens on the network — a resolution failure /// resolves quietly, because the work is advisory. Under the system resolver there is no cache /// to warm, so the call resolves without doing anything. A malformed host throws synchronously, - /// as does a call on a closed agent. (spec:WARM) + /// as does a call on a closed agent. #[napi] pub fn prefetch_dns<'env>( &self, env: &'env Env, host: String, ) -> Result, napi::Error> { - if self.client.is_none() { - return Err(caller_error(env, FaithErrorKind::Closed)); - } - - let Some(host) = extract_host(&host) else { - return Err(caller_error(env, FaithErrorKind::AddressParse)); - }; - - let resolver = self.dns_resolver.clone(); + let warming = self + .inner + .prefetch_dns(&host) + .map_err(|err| caller_error(env, err))?; faith_promise(env, async move { - if let Some(resolver) = resolver { - resolver.prefetch(&host).await; - } + warming.await; Ok(()) }) } @@ -1734,130 +1523,19 @@ impl Agent { /// foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every /// other origin a TCP one. The returned promise resolves when the attempt finishes and never /// rejects: every network failure resolves quietly. A malformed origin throws synchronously, as - /// does a call on a closed agent. (spec:WARM) + /// does a call on a closed agent. #[napi] pub fn preconnect<'env>( &self, env: &'env Env, origin: String, ) -> Result, napi::Error> { - let Some(raw_client) = self.raw_client.clone() else { - return Err(caller_error(env, FaithErrorKind::Closed)); - }; - - let Some(url) = reduce_to_origin(&origin) else { - return Err(caller_error(env, FaithErrorKind::AddressParse)); - }; - let key = origin_key(&url); - - // Already warm within the idle window, or a warm-up for this origin already in flight: - // either way there is no new work to do, so resolve without opening a duplicate. - if self.warmed.contains_key(&key) - || !self.warming.entry(key.clone()).or_insert(()).is_fresh() - { - return faith_promise(env, async move { Ok(()) }); - } - - // The transport the next foreground request would take, decided exactly as - // `AltSvcMiddleware` decides it: nothing upgrades with the machinery off; with a prober, - // only a confirmed origin routes to QUIC (an advertisement is evidence worth probing, not - // worth routing on); without one, the legacy inline upgrade acts on advertisements too. - // Diverging here would warm the wrong transport (spec:WARM#preconnect). - #[cfg(feature = "http3")] - let h3_port = self - .alt_svc_cache - .as_ref() - .filter(|_| self.h3_upgrade_enabled) - .and_then(|cache| { - if self.h3_prober.is_some() { - cache.confirmed_port(&url) - } else { - cache.should_use_h3(&url) - } - }); - #[cfg(not(feature = "http3"))] - let h3_port: Option = None; - - #[cfg(feature = "http3")] - let alt_svc_cache = self.alt_svc_cache.clone(); - #[cfg(feature = "http3")] - let h3_prober = self.h3_prober.clone(); - - let conn_tracker = self.conn_tracker.clone(); - let warmed = self.warmed.clone(); - let warming = self.warming.clone(); - // Read before the warm-up starts, to compare against once it finishes. - let warm_generation = self.warm_generation.clone(); - let generation = warm_generation.load(Ordering::Relaxed); - + let warming = self + .inner + .preconnect(&origin) + .map_err(|err| caller_error(env, err))?; faith_promise(env, async move { - // Release the single-flight claim whatever happens, so a later warm-up isn't blocked - // by this one having finished. - struct ReleaseClaim { - warming: MokaCache, - key: String, - } - impl Drop for ReleaseClaim { - fn drop(&mut self) { - self.warming.invalidate(&self.key); - } - } - let _release = ReleaseClaim { - warming, - key: key.clone(), - }; - - let request = match h3_port { - Some(port) => { - let mut h3_url = url.clone(); - // A port differing from the origin's only comes back with - // `upgradeFollowAdvertisedPort` on; rewriting the URL is how reqwest is told to - // connect there (mirrors the foreground path). - if Some(port) != h3_url.port_or_known_default() { - let _ = h3_url.set_port(Some(port)); - } - raw_client.head(h3_url).version(Version::HTTP_3) - } - None => raw_client.head(url.clone()), - }; - - let outcome = request.send().await; - - // A TCP warm-up leaves a pooled connection to track and, in probe mode, may reveal an - // HTTP/3 advertisement to act on; a QUIC warm-up does neither (QUIC connections are not - // tracked, and a confirmed origin has nothing left to probe). - if h3_port.is_none() { - if let Ok(response) = &outcome - && let Some(info) = response.extensions().get::() - { - conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); - } - - // A background probe verifies HTTP/3 for a probe-worthy origin exactly as a real - // TCP-routed request would, folding in any fresh advertisement first; the warm-up - // settles without waiting for it. Both only apply in probe mode (a prober present). - #[cfg(feature = "http3")] - if let Some(prober) = &h3_prober { - if let (Some(cache), Ok(response)) = (&alt_svc_cache, &outcome) - && let Some(value) = response.headers().get("alt-svc") - && let Ok(value) = value.to_str() - && let Some(advertisement) = parse_alt_svc_header(value) - { - cache.record_alt_svc(&url, &advertisement); - } - prober.maybe_probe(&url); - } - } - - // A connection was established, so the origin is warm for the idle window; a failed - // warm-up leaves it unmarked so a later `preconnect` may try again. A network change - // while this was in flight leaves it unmarked too: the connection landed in the pool - // that change dropped, so the origin is not warm however well the request went - // (spec:NETCHG#reach-across-the-subsystems). - if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { - warmed.insert(key, ()); - } - + warming.await; Ok(()) }) } @@ -1865,8 +1543,8 @@ impl Agent { /// Build the JS error a warm-up throws synchronously for a caller mistake, preserving its `.code` /// and JS error class. Network failures never reach here — they resolve quietly (spec:WARM). -fn caller_error(env: &Env, kind: FaithErrorKind) -> napi::Error { - napi::Error::from(FaithError::from(kind).into_js_error(env)) +fn caller_error(env: &Env, err: FaithError) -> napi::Error { + napi::Error::from(err.into_js_error(env)) } #[cfg(test)] diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index be5e01b..5e55c7f 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -103,6 +103,7 @@ pub fn faith_fetch<'env>( let headers_stamp = HeadersStamp::default(); let mut request = agent + .inner .client .as_ref() .ok_or(FaithErrorKind::Closed)? @@ -160,6 +161,7 @@ pub fn faith_fetch<'env>( }); from_request.or_else(|| { agent + .inner .default_content_encoding .as_ref() .and_then(|value| value.to_str().ok().map(str::to_owned)) @@ -182,13 +184,14 @@ pub fn faith_fetch<'env>( .clone() .or_else(|| { agent + .inner .default_accept_encoding .as_ref() .and_then(|value| value.to_str().ok().map(str::to_owned)) }) .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), ); - if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { + if request_accept_encoding.is_none() && agent.inner.default_accept_encoding.is_none() { request = request.header( ACCEPT_ENCODING, HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), @@ -201,7 +204,7 @@ pub fn faith_fetch<'env>( // left to reqwest: it fills a default header in only where the request carries none // of that name, so setting the derived value would displace the agent's own. if let Some(urgency) = options.priority - && !agent.has_default_priority + && !agent.inner.has_default_priority && !options.headers.as_ref().is_some_and(|headers| { headers .iter() @@ -233,7 +236,7 @@ pub fn faith_fetch<'env>( // A body read from a `ReadableStream` has no length to advertise, which the fetch // standard allows only over HTTP/2 and HTTP/3 (spec:REQ#streaming-a-request-body). - if !agent.quirk_h1_request_streaming { + if !agent.inner.quirk_h1_request_streaming { // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and // can be refused without opening a connection to find out. if parsed_url.scheme() != "https" { @@ -303,7 +306,11 @@ pub fn faith_fetch<'env>( request = request.timeout(dur); } - agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); + agent + .inner + .stats + .requests_sent + .fetch_add(1, Ordering::Relaxed); // The origin every phase is measured from. let started = Instant::now(); @@ -321,6 +328,7 @@ pub fn faith_fetch<'env>( }; agent + .inner .stats .responses_received .fetch_add(1, Ordering::Relaxed); @@ -341,16 +349,17 @@ pub fn faith_fetch<'env>( // the TCP fallback re-runs the untouched clone. Restricting the normalisation // to those keeps exact comparison, and so port-only redirect detection, for // every other response. - let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { - let without_port = |url: &reqwest::Url| { - let mut url = url.clone(); - let _ = url.set_port(None); - url + let redirected = + if agent.inner.h3_follow_advertised_port && version == http::Version::HTTP_3 { + let without_port = |url: &reqwest::Url| { + let mut url = url.clone(); + let _ = url.set_port(None); + url + }; + without_port(&parsed_url) != without_port(&response_url) + } else { + parsed_url != response_url }; - without_port(&parsed_url) != without_port(&response_url) - } else { - parsed_url != response_url - }; // Track connection for TCP stats (if we can get both local and remote addr). // A connection the tracker has already seen is one the pool handed back, which is @@ -358,7 +367,7 @@ pub fn faith_fetch<'env>( let reused = if let Some(http_info) = response.extensions().get::() { let local_addr = http_info.local_addr(); let remote_addr = http_info.remote_addr(); - agent.conn_tracker.track(local_addr, remote_addr) + agent.inner.conn_tracker.track(local_addr, remote_addr) } else { false }; @@ -366,7 +375,7 @@ pub fn faith_fetch<'env>( // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a // redirect chain marks the origin that actually answered rather than the one asked for. - agent.mark_warm(&response_url); + agent.inner.mark_warm(&response_url); let peer = PeerInformation { address: response.remote_addr(), @@ -436,7 +445,7 @@ pub fn faith_fetch<'env>( integrity: options.integrity, peer: Arc::new(peer), redirected, - stats: agent.stats.clone(), + stats: agent.inner.stats.clone(), status_code, timing, trailers: Default::default(), diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 3f9272a..7e275af 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -18,12 +18,15 @@ http.workspace = true http-body-util.workspace = true http-cache-reqwest.workspace = true hyper.workspace = true +hyper-util.workspace = true +moka.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true stream_shared.workspace = true strum.workspace = true tokio.workspace = true url.workspace = true +web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-alt-svc = { workspace = true, optional = true } diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs new file mode 100644 index 0000000..5c9b96b --- /dev/null +++ b/crates/web-faith/src/agent.rs @@ -0,0 +1,515 @@ +//! The agent: what owns a connection pool, and the verbs that act on a live one. + +// spec:AGENT spec:WARM spec:NETCHG spec:OBS + +use std::{ + future::Future, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use http::header::HeaderValue; +use moka::sync::Cache as MokaCache; +use reqwest::{Client, Version}; +use reqwest_middleware::ClientWithMiddleware; +use url::Url; +use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; +use web_faith_cookies::FaithJar; +use web_faith_dns::{FaithResolver, ResolverReport}; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, H3Prober}; + +use crate::{ + client::ClientRecipe, + error::{FaithError, FaithErrorKind}, + stats::{AgentStats, InnerAgentStats}, + warm_up::{extract_host, origin_key, reduce_to_origin}, +}; + +#[cfg(feature = "http3")] +use crate::client::install_https_sink; + +/// The agent settings a request consults, as opposed to those a client is built from. +#[derive(Debug, Clone, Default)] +pub struct AgentSettings { + /// Whether an HTTP/3 upgrade may follow a port the origin advertised, which a request needs so + /// a rewritten port is not reported as a redirect. + pub h3_follow_advertised_port: bool, + /// Whether the upgrade machinery is on at all. A warm-up needs it to route the way a foreground + /// request would. + #[cfg(feature = "http3")] + pub h3_upgrade_enabled: bool, + /// Whether a streaming request body may go out over HTTP/1.x. + pub quirk_h1_request_streaming: bool, + /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides + /// which codings a response is decoded under when a request adds none of its own. + pub default_accept_encoding: Option, + /// The agent's default `Content-Encoding`, if one sits among its default headers, which a + /// request layers its own coding on top of rather than displacing. + pub default_content_encoding: Option, + /// Whether a `Priority` header sits among the agent's default headers, so that default wins + /// over the one a request's priority would derive. + pub has_default_priority: bool, +} + +/// An HTTP client with its own connection pool, caches, and resolver. +/// +/// Cloning one is cheap and every clone names the same underlying agent, which is what lets a +/// request take a handle of its own without opening a second pool. +#[derive(Debug, Clone)] +pub struct Agent { + /// `None` once [`Agent::close`] has been called. The heavy resources + /// (connection pool, DNS resolver, background tasks) live inside this + /// client, so dropping it is what actually releases them. + pub client: Option, + /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A + /// `preconnect` warm-up sends its synthetic request here rather than through the middleware + /// stack, which bypasses the HTTP cache and the Alt-Svc layer (and so keeps the warm-up out of + /// request accounting), while still pooling the connection foreground requests reuse. `None` + /// once the agent is closed. (spec:WARM) + pub raw_client: Option, + /// Faith's DNS resolver, shared with [`Self::client`] so `prefetchDns` warms the cache requests + /// read. `None` under the system resolver, where there is no such cache. (spec:WARM) + pub dns_resolver: Option, + /// Origins with a warm-up connection opened within the pool idle window, so a repeat + /// `preconnect` does no new work. Keyed by `scheme://host:port`; entries expire with the idle + /// timeout. (spec:WARM) + pub warmed: MokaCache, + /// Single-flight claims for in-flight `preconnect` warm-ups, so concurrent calls for the same + /// origin do not open duplicate connections. (spec:WARM) + pub warming: MokaCache, + /// Bumped by `networkChanged`, so a warm-up that was in flight across the signal does not + /// record its origin as warm: its connection went into the pool that was just dropped + /// (spec:NETCHG#reach-across-the-subsystems). + pub warm_generation: Arc, + pub cookie_jar: Option>, + pub stats: Arc, + pub conn_tracker: Arc, + #[cfg(feature = "http3")] + #[allow(dead_code)] + pub alt_svc_cache: Option>, + /// Held so `close()` can abort in-flight background probes: each one owns a + /// clone of the raw client, which would otherwise keep the connection pool + /// alive past close for up to the probe timeout. + #[cfg(feature = "http3")] + pub h3_prober: Option>, + /// Mirrors `http3.upgradeFollowAdvertisedPort`. Lives here because `fetch` needs + /// it to stop a rewritten port from being reported as a redirect. + pub h3_follow_advertised_port: bool, + /// Mirrors `http3.upgradeEnabled`. A warm-up needs it to route the way a foreground request + /// would: with the upgrade machinery off, nothing upgrades, whatever the caches hold. + /// (spec:WARM#preconnect) + #[cfg(feature = "http3")] + pub h3_upgrade_enabled: bool, + /// Mirrors `quirks.h1RequestStreaming`. `fetch` consults it to decide whether a streaming + /// request body may go out over HTTP/1.x (spec:QUIRK#http-1-x-request-body-streaming). + pub quirk_h1_request_streaming: bool, + /// The agent's default `Accept-Encoding`, if one was set among its default headers. + /// `fetch` consults it to decide which codings to decode when a request adds none of + /// its own (see [`crate::encoding`]). + pub default_accept_encoding: Option, + /// The agent's default `Content-Encoding`, if one was set among its default headers. + /// `fetch` consults it when the `compress` option layers a coding on top of what a + /// request already declares, since setting the joined value on the request would + /// otherwise displace this default rather than build on it (spec:ENC). + pub default_content_encoding: Option, + /// Whether a `Priority` header sits among the agent's default headers. `fetch` consults + /// it so that default wins over the header the `priority` option would derive. + pub has_default_priority: bool, + /// How to build this agent's clients, so `networkChanged` can build them again + /// (spec:NETCHG). Shared rather than cloned per agent clone: every clone builds the same + /// client from the same recipe, and `fetch` clones the agent per request. + pub recipe: Arc, +} + +impl Agent { + /// Build an agent from a validated recipe. + /// + /// The recipe is what a client is built from, and the settings are what each request consults; + /// validating whatever a caller expressed them as belongs to the surface that took it. + pub fn build( + recipe: ClientRecipe, + settings: AgentSettings, + cookie_jar: Option>, + dns_resolver: Option, + #[cfg(feature = "http3")] alt_svc_cache: Option>, + ) -> Result { + let conn_timeout = recipe.conn_timeout(); + let built = recipe.build( + cookie_jar.as_ref(), + dns_resolver.as_ref(), + #[cfg(feature = "http3")] + alt_svc_cache.as_ref(), + )?; + + // Only now do all three exist: the resolver is built before the cache, and the prober + // holds a client that holds the resolver, so this is the earliest the loop can be closed + // (spec:DNS#https-records). + #[cfg(feature = "http3")] + install_https_sink( + dns_resolver.as_ref(), + alt_svc_cache.as_ref(), + built.prober.as_ref(), + recipe.h3_upgrade.enabled, + ); + + Ok(Self { + client: Some(built.client), + raw_client: Some(built.raw_client), + dns_resolver, + // A warm-up connection is warm only as long as the pool keeps it idle, so the record + // that an origin is warm expires with that same window. + warmed: MokaCache::builder().time_to_live(conn_timeout).build(), + // A safety TTL well past any reasonable warm-up, so a claim that never gets released + // (a warm-up whose task is dropped) frees the origin rather than wedging it. + warming: MokaCache::builder() + .time_to_live(Duration::from_secs(300)) + .build(), + warm_generation: Default::default(), + cookie_jar, + stats: Default::default(), + conn_tracker: ConnectionTracker::new(conn_timeout), + #[cfg(feature = "http3")] + alt_svc_cache, + #[cfg(feature = "http3")] + h3_prober: built.prober, + h3_follow_advertised_port: settings.h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming: settings.quirk_h1_request_streaming, + default_accept_encoding: settings.default_accept_encoding, + default_content_encoding: settings.default_content_encoding, + has_default_priority: settings.has_default_priority, + recipe: Arc::new(recipe), + }) + } + + /// Close the agent, releasing its connection pool, DNS resolver, and any + /// background tasks it owns, rather than waiting for the garbage collector + /// to drop it. This is worth doing when you create many short-lived agents; + /// a single long-lived agent can just be left to the GC. + /// + /// Requests already in flight run to completion. Any new request on a closed + /// agent throws a `Closed` error. Calling `close()` more than once is a + /// no-op. The cookie store, if any, remains readable via `getCookie`. + pub fn close(&mut self) { + // Dropping the client releases the reqwest connection pool and the + // Hickory resolver task; the alt-svc cache goes with it. The raw client + // shares that pool and the resolver, so it goes too, and both are what a + // later `preconnect`/`prefetchDns` checks to throw the closed-agent error. + self.client = None; + self.raw_client = None; + self.dns_resolver = None; + #[cfg(feature = "http3")] + { + // Probes hold a raw client clone; abort them so the pool doesn't + // outlive close by up to the probe timeout. + if let Some(prober) = &self.h3_prober { + prober.abort_all(); + } + self.h3_prober = None; + self.alt_svc_cache = None; + } + } + /// Tell the agent the network underneath it has changed, so it stops deciding from what it + /// learned about a network that is gone. + /// + /// Node has no portable signal for an interface or connectivity change, so Faith cannot + /// detect one; this is the reaction, and wiring it to a trigger (an OS notification, a VPN + /// transition, a captive-portal sign-in) is the caller's own. It drops pooled connections, + /// flushes the DNS cache, demotes the HTTP/3 origins that a real response confirmed back to + /// advertised so a background probe re-verifies them, and clears the HTTP/3 failure and slow + /// states, their cooldown backoff, and the path-time averages. + /// + /// Configuration, `http3.hints`, `Alt-Svc` advertisements, the cookie jar, the HTTP cache and + /// the `stats()` counters are all kept: none of them is a claim about a network path. + /// + /// Requests already in flight are not interrupted and run to completion on the connections + /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a + /// closed agent does nothing, and calling it repeatedly is harmless. + /// + /// spec:NETCHG + pub fn network_changed(&mut self) { + // A closed agent has already released all of this. + if self.client.is_none() { + return; + } + + // reqwest cannot drop pooled connections short of dropping the client, so the client is + // rebuilt from the recipe the agent kept for this. Requests in flight hold their own + // clone of the old client (`fetch` clones the agent per request), so they run to + // completion and the old pool goes when the last of them finishes. + // + // A rebuild that fails leaves the agent on its existing client: the options were already + // validated at construction, so a failure here is not the caller's to answer for, and an + // agent that still works on the old network beats one that works nowhere. + let built = self.recipe.build( + self.cookie_jar.as_ref(), + self.dns_resolver.as_ref(), + #[cfg(feature = "http3")] + self.alt_svc_cache.as_ref(), + ); + if let Ok(built) = built { + #[cfg(feature = "http3")] + { + // Abort probes running on the old client: each holds a clone of it, and their + // answers would describe the path that has just gone away. + if let Some(prober) = &self.h3_prober { + prober.abort_all(); + } + self.h3_prober = built.prober; + // The sink holds the prober, which has just been replaced along with the client + // it sends on; leaving the old one installed would aim DNS-triggered probes at a + // client that has been dropped. + install_https_sink( + self.dns_resolver.as_ref(), + self.alt_svc_cache.as_ref(), + self.h3_prober.as_ref(), + self.h3_upgrade_enabled, + ); + } + self.client = Some(built.client); + self.raw_client = Some(built.raw_client); + } + + // Names resolve afresh against the new network, through that network's own servers: the + // resolver drops what it read off the old one and reads again when next used. Under the + // system resolver there is no resolver here and so nothing to reset (spec:DNS). + if let Some(resolver) = &self.dns_resolver { + resolver.reset(); + } + + #[cfg(feature = "http3")] + if let Some(alt_svc_cache) = &self.alt_svc_cache { + alt_svc_cache.network_changed(); + } + + // The warm-up records describe pooled connections that have just been dropped, so a + // `preconnect` after the signal opens a connection rather than finding the origin warm + // (spec:NETCHG, spec:WARM). The single-flight claims are left alone: a warm-up still in + // flight is not duplicated by releasing its claim, and the generation bump is what stops + // it recording an origin as warm on the strength of a connection in the dropped pool. + self.warmed.invalidate_all(); + self.warm_generation.fetch_add(1, Ordering::Relaxed); + } + + /// Add a cookie into the agent. + /// + /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying + /// the scheme and host they read, so this does nothing if: + /// - the cookie store is disabled + /// /// - the cookie does not parse + /// - a `__Host-` or `__Secure-` name prefix is not satisfied + /// - the cookie is larger than `cookies.maxSize` + pub fn add_cookie(&self, url: &Url, cookie: &str) { + let Some(jar) = &self.cookie_jar else { + return; + }; + + jar.add_cookie_str(cookie, url); + } + + /// Retrieve a cookie from the store. + /// + /// `None` if: + /// - there's no cookie at this url + /// - the cookie store is disabled + /// /// - the cookie cannot be represented as a string + pub fn cookie_header(&self, url: &Url) -> Option { + let Some(jar) = &self.cookie_jar else { + return None; + }; + + jar.request_cookie_header(url) + .and_then(|val| val.to_str().ok().map(ToOwned::to_owned)) + } + + /// Returns statistics gathered by this agent: + /// + /// - `requestsSent` + /// - `responsesReceived` + /// - `bodiesStarted` + /// - `bodiesFinished` + pub fn stats(&self) -> AgentStats { + self.stats.snapshot() + } + + /// Returns information on current connections open by this agent. + /// + /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: + /// this makes it possible to track indicators over time to find the retransmission rate, for + /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other + /// fields might also be missing depending on platform support; and no forward guarantees are made + /// on field availability. If the platform isn't supported at all, this will always return empty. + pub fn connections(&self) -> Vec { + self.conn_tracker.snapshot() + } + + /// Returns the DNS servers this agent resolves through, in the order they are queried, so + /// "are my lookups actually encrypted" is answerable from inside the process. + /// + /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, + /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). + /// The list is empty until the resolver has been used, because it reads its configuration on + /// first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) + pub fn resolvers(&self) -> Vec { + self.dns_resolver + .as_ref() + .map(FaithResolver::resolvers) + .unwrap_or_default() + } + + /// Note that a request reached this origin, so it holds a connection the pool keeps idle for + /// the idle window and a `preconnect` for it has no new work to do (spec:WARM). + /// + /// Called for foreground requests as well as warm-ups, because the criterion is about the + /// origin holding an idle pooled connection, not about how it came to hold one. + pub fn mark_warm(&self, url: &Url) { + self.warmed.insert(origin_key(url), ()); + } + + /// Whether [`Self::close`] has been called. + pub fn is_closed(&self) -> bool { + self.client.is_none() + } + + /// Warm the DNS cache for `host`, so a later request to it skips the lookup. + /// + /// The argument is a bare host; a scheme, port, or path in a fuller string is ignored. The + /// returned future completes when the answer lands in the cache and never fails, whatever + /// happens on the network: the work is advisory. Under the system resolver there is no cache to + /// warm, so it completes having done nothing. A host with nothing to resolve, or a closed agent, + /// is refused here rather than by the future. + // spec:WARM + pub fn prefetch_dns(&self, host: &str) -> Result + use<>, FaithError> { + if self.is_closed() { + return Err(FaithErrorKind::Closed.into()); + } + + let Some(host) = extract_host(host) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + + let resolver = self.dns_resolver.clone(); + Ok(async move { + if let Some(resolver) = resolver { + resolver.prefetch(&host).await; + } + }) + } + + /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS + /// setup. + /// + /// The argument is an origin (`scheme://host[:port]`); a longer URL is reduced to one. The + /// warm-up sends a synthetic `HEAD` to the origin's root -- the origin sees it -- over the + /// transport the next foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC + /// connection, every other origin a TCP one. The returned future completes when the attempt + /// finishes and never fails: every network failure is quiet. Something that cannot be connected + /// to, or a closed agent, is refused here rather than by the future. + // spec:WARM + pub fn preconnect(&self, origin: &str) -> Result + use<>, FaithError> { + let Some(raw_client) = self.raw_client.clone() else { + return Err(FaithErrorKind::Closed.into()); + }; + + let Some(url) = reduce_to_origin(origin) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + let key = origin_key(&url); + + // Already warm within the idle window, or a warm-up for this origin already in flight: + // either way there is no new work to do, so finish without opening a duplicate. + let redundant = self.warmed.contains_key(&key) + || !self.warming.entry(key.clone()).or_insert(()).is_fresh(); + + // The transport the next foreground request would take, decided exactly as the Alt-Svc + // layer decides it: nothing upgrades with the machinery off; with a prober, only a + // confirmed origin routes to QUIC (an advertisement is evidence worth probing, not worth + // routing on); without one, the inline upgrade acts on advertisements too. Diverging here + // would warm the wrong transport. + // spec:WARM#preconnect + #[cfg(feature = "http3")] + let h3_port = self + .alt_svc_cache + .as_ref() + .filter(|_| self.h3_upgrade_enabled) + .and_then(|cache| { + if self.h3_prober.is_some() { + cache.confirmed_port(&url) + } else { + cache.should_use_h3(&url) + } + }); + #[cfg(not(feature = "http3"))] + let h3_port: Option = None; + + let conn_tracker = self.conn_tracker.clone(); + let warmed = self.warmed.clone(); + let warming = self.warming.clone(); + // Read before the warm-up starts, to compare against once it finishes. + let warm_generation = self.warm_generation.clone(); + let generation = warm_generation.load(Ordering::Relaxed); + + Ok(async move { + if redundant { + return; + } + + // Release the single-flight claim whatever happens, so a later warm-up is not blocked + // by this one having finished. + struct ReleaseClaim { + warming: MokaCache, + key: String, + } + impl Drop for ReleaseClaim { + fn drop(&mut self) { + self.warming.invalidate(&self.key); + } + } + let _release = ReleaseClaim { + warming, + key: key.clone(), + }; + + let request = match h3_port { + Some(port) => { + let mut h3_url = url.clone(); + // A port differing from the origin's only comes back with the + // follow-advertised-port option on; rewriting the URL is how reqwest is told to + // connect there, mirroring the foreground path. + if Some(port) != h3_url.port_or_known_default() { + let _ = h3_url.set_port(Some(port)); + } + raw_client.head(h3_url).version(Version::HTTP_3) + } + None => raw_client.head(url.clone()), + }; + + let outcome = request.send().await; + + // A TCP warm-up leaves a pooled connection to track; a QUIC one does not (QUIC + // connections are not tracked, and a confirmed origin has nothing left to probe). + if h3_port.is_none() + && let Ok(response) = &outcome + && let Some(info) = response + .extensions() + .get::() + { + conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); + } + + // A network change while this was in flight leaves the origin unmarked: the connection + // landed in the pool that change dropped, so it is not warm however well the request + // went. + // spec:NETCHG#reach-across-the-subsystems + if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { + warmed.insert(key, ()); + } + }) + } +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 238c4a8..eef78b8 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -17,6 +17,7 @@ //! //!
    +pub mod agent; pub mod body; pub mod client; pub mod error; diff --git a/index.d.ts b/index.d.ts index b3e55b3..6ecfcb7 100644 --- a/index.d.ts +++ b/index.d.ts @@ -47,8 +47,6 @@ export declare class Agent { * Requests already in flight are not interrupted and run to completion on the connections * they hold; the reset shapes what requests started afterwards draw on. Calling it on a * closed agent does nothing, and calling it repeatedly is harmless. - * - * spec:NETCHG */ networkChanged(): void /** @@ -99,7 +97,7 @@ export declare class Agent { * Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, * `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). * The list is empty until the resolver has been used, because it reads its configuration on - * first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) + * first use, and empty for an agent using the system resolver. */ resolvers(): Array /** @@ -110,7 +108,7 @@ export declare class Agent { * lands in the cache and never rejects, whatever happens on the network — a resolution failure * resolves quietly, because the work is advisory. Under the system resolver there is no cache * to warm, so the call resolves without doing anything. A malformed host throws synchronously, - * as does a call on a closed agent. (spec:WARM) + * as does a call on a closed agent. */ prefetchDns(host: string): Promise /** @@ -123,7 +121,7 @@ export declare class Agent { * foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC connection, every * other origin a TCP one. The returned promise resolves when the attempt finishes and never * rejects: every network failure resolves quietly. A malformed origin throws synchronously, as - * does a call on a closed agent. (spec:WARM) + * does a call on a closed agent. */ preconnect(origin: string): Promise } From 922c5fba7a74e585116792e8549cbae9baa338c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:35 +1200 Subject: [PATCH 37/61] S1: record the agent inversion as done, and what step 8 has left --- .workhorse/plans/s1/plan.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index bf73b52..d7e2905 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -76,20 +76,17 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [x] The client-building machinery moved: `ClientRecipe`, `NodeEnvRecipe`, `HttpCacheRecipe`, `HttpCacheStore`, `H3UpgradeRecipe`, `ResolvedWindows`, `install_https_sink`, and a pure `RedirectPolicy` the napi `Redirect` converts into. - - [ ] **The `Agent` inversion — the big remaining piece.** `agent.rs` is down to ~2140 lines: - roughly 860 of `#[napi(object)]` option structs (JS-facing, they stay), ~100 of the `Agent` - struct, and ~1170 of `#[napi] impl Agent` holding 22 methods. The struct's fields are - already pure (reqwest/moka/Arc), so the blocker is that `#[napi] impl` cannot target a - foreign type: moving `Agent` to `web-faith` forces the napi `Agent` to become a distinct - class wrapping it, in the same change. Each of the 22 methods then splits into a pure core - method on `web_faith::Agent` and a thin binding that converts — `close`, `network_changed`, - `stats`, `connections`, `resolvers`, `prefetch_dns`, `preconnect`, `cookies` are the verbs - [RSAPI](../../specs/rust/client-api.md) names, so this is where the real client API starts - rather than a mechanical relocation. - - [ ] `response.rs` (~966 lines, 40 napi refs) and `fetch.rs` (~447) follow the agent, since both - are built around the napi response class. - - [ ] `options.rs` (~246) stays largely JS-facing; the recipe fields it assembles become the - builder's business in step 9. + - [x] **The `Agent` inversion.** Done: `web_faith::agent::Agent` owns the pool, the verbs, and the + per-request settings; `web-faith-napi`'s `Agent` is a napi class holding a handle on it, and + keeps the ~440 lines of `AgentOptions` validation that produce a recipe. `prefetch_dns` and + `preconnect` return futures the binding wraps, so a refusal happens before the future exists. + - [ ] `response.rs` (~966 lines, 40 napi refs) and `fetch.rs` (~447) are what is left. Both are + built around the napi response class, so they invert the way the agent did: a pure response + and a pure request path in the client, with the napi classes wrapping them. `fetch.rs` + already reaches the agent through `agent.inner`, which is the seam to pull on. + - [ ] `options.rs` (~246) stays largely JS-facing, holding the `AgentOptions` validation and the + per-request option shapes; the recipe it assembles becomes the builder's business in step 9. + - [ ] `stream_body.rs` and `async_task.rs` are napi machinery and stay where they are. - [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, From d7135f3fa3f8aa05a7fd5591a843978d5354edef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:13:21 +1200 Subject: [PATCH 38/61] S1: move the response's own machinery into web-faith Where trailers land, what is known of the peer, and opening a destination for a body write are all the response's own business rather than JavaScript's, so they move to the client with the two tests that guard the trailers wait against spinning. The file destination becomes the client's own shape, with the defaulting the JS options object leaves implicit made explicit; the napi object converts into it. --- crates/web-faith-napi/src/response.rs | 227 ++------------------------ crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/response.rs | 226 +++++++++++++++++++++++++ 3 files changed, 242 insertions(+), 212 deletions(-) create mode 100644 crates/web-faith/src/response.rs diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 34ed075..1d06213 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -2,14 +2,13 @@ use std::{ fmt::Debug, hint::unreachable_unchecked, mem::replace, - net::SocketAddr, pin::Pin, result::Result, sync::{ Arc, atomic::{AtomicBool, Ordering}, }, - time::{Duration, Instant}, + time::Instant, }; use bytes::Bytes; @@ -26,14 +25,17 @@ use reqwest::{ }; use serde_json; use stream_shared::SharedStream; -use tokio::{io::AsyncWriteExt, sync::watch}; +use tokio::io::AsyncWriteExt; use web_faith::{ body::{Body, BodyHolder, DynStream, drain_body_inner}, + response::{FileDestination, PROGRESS_INTERVAL, Trailers, TrailersSlot, open_destination}, stats::InnerAgentStats, timing::TimingSlot, }; use web_faith_encoding::{Coding, decode_stream}; + +pub use web_faith::response::PeerInformation; use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; use crate::{ @@ -67,19 +69,6 @@ pub struct FaithResponse { pub(crate) version: Version, } -/// Custom to Faith. -/// -/// The `peer` read-only property of the `Response` interface contains an object with information about -/// the remote peer that sent this response: -/// -/// - `address`: The IP address and port of the peer, if available. -/// - `certificate`: When connected over HTTPS, this is the DER-encoded leaf certificate of the peer. -#[derive(Debug)] -pub struct PeerInformation { - pub address: Option, - pub certificate: Option>, -} - /// Options for `toFile()`. #[napi(object)] #[derive(Debug, Default)] @@ -114,6 +103,15 @@ pub struct ToFileProgress { pub content_length: Option, } +impl From<&ToFileOptions> for FileDestination { + fn from(options: &ToFileOptions) -> Self { + Self { + overwrite: options.overwrite.unwrap_or(false), + mode: options.mode, + } + } +} + /// The callback `toFile()` reports progress to. /// /// `CalleeHandled = false`: progress is not an error-first callback, so the JavaScript @@ -121,117 +119,6 @@ pub struct ToFileProgress { pub type ProgressCallback = ThreadsafeFunction, ToFileProgress, Status, false>; -/// The shortest gap between progress reports. -/// -/// Reporting every chunk would cross into JavaScript thousands of times for a large body, -/// which is the cost `toFile()` exists to avoid. A caller driving a progress bar cannot use -/// updates faster than this anyway, and the final report is always delivered regardless. -const PROGRESS_INTERVAL: Duration = Duration::from_millis(50); - -/// Open the destination file for a body write, mapping filesystem refusals to the errors -/// `toFile()` surfaces (spec:BODY#tofile). -async fn open_destination( - path: &str, - options: &ToFileOptions, -) -> Result { - let mut open = tokio::fs::OpenOptions::new(); - open.write(true); - if options.overwrite.unwrap_or(false) { - // An occupied destination is truncated and replaced. - open.create(true).truncate(true); - } else { - // The safe default refuses an occupied destination outright. - open.create_new(true); - } - #[cfg(unix)] - if let Some(mode) = options.mode { - open.mode(mode); - } - - match open.open(path).await { - Ok(file) => Ok(file), - Err(err) => Err(classify_open_error(path, err).await), - } -} - -/// Classify a failure to open the destination. An occupied destination is `FileExists`, -/// unless what occupies it is a directory: a directory is well-formed but cannot be written -/// to, which is a `FileWrite`. Every other refusal is a `FileWrite` carrying the OS detail. -async fn classify_open_error(path: &str, err: std::io::Error) -> FaithError { - let kind = if err.kind() == std::io::ErrorKind::AlreadyExists { - match tokio::fs::symlink_metadata(path).await { - Ok(meta) if meta.is_dir() => FaithErrorKind::FileWrite, - _ => FaithErrorKind::FileExists, - } - } else { - FaithErrorKind::FileWrite - }; - FaithError::new(kind, Some(err.to_string())) -} - -#[derive(Clone, Debug, Default)] -pub enum Trailers { - #[default] - NotYet, - None, - Some(HeaderMap), -} - -/// Where the trailers land: written by whoever finishes the body, awaited by `trailers()`. -/// -/// A watch channel, rather than a lock read in a loop. Per the fetch standard's trailers -/// proposal () this promise is *meant* not to -/// resolve until the body has been consumed, so the wait is unbounded by design -- which is -/// precisely why polling was the wrong shape for it. Awaiting trailers without reading the -/// body now leaves an idle pending promise rather than a pegged core, and the future can be -/// cancelled while it waits. -#[derive(Debug)] -pub struct TrailersSlot(watch::Sender); - -impl Default for TrailersSlot { - fn default() -> Self { - Self(watch::channel(Trailers::NotYet).0) - } -} - -impl TrailersSlot { - /// Record trailers that arrived, waking whoever is waiting. - fn arrived(&self, trailers: HeaderMap) { - self.0.send_replace(Trailers::Some(trailers)); - } - - /// Record that the body ended, if no trailers frame got there first. - /// - /// `send_if_modified` so the read and the write are one step, and so waiters are woken - /// only by the call that actually settled it. - fn ended(&self) { - self.0.send_if_modified(|state| { - if matches!(state, Trailers::NotYet) { - *state = Trailers::None; - true - } else { - false - } - }); - } - - /// Wait until the body has settled the question. - async fn settled(&self) -> Trailers { - let mut rx = self.0.subscribe(); - // `wait_for` tests the current value before waiting, so trailers that already - // arrived return without yielding. Its error case is the sender being gone, which - // means the response was dropped and nothing can ever set this -- no trailers is - // the only answer left. - match rx - .wait_for(|state| !matches!(state, Trailers::NotYet)) - .await - { - Ok(state) => state.clone(), - Err(_) => Trailers::None, - } - } -} - #[napi] impl FaithResponse { /// The `headers` read-only property of the `Response` interface contains the `Headers` object @@ -718,7 +605,7 @@ impl FaithResponse { // The destination is opened before any of the body is read, so a failure to open it // leaves the body unread and undisturbed. - let mut file = open_destination(&path, &options).await?; + let mut file = open_destination(&path, &FileDestination::from(&options)).await?; // Commit the read now the destination is in hand. A concurrent read that slipped in // since the load above wins, and this one finds the body already spent. @@ -880,87 +767,3 @@ impl FaithResponse { }) } } - -#[cfg(test)] -mod tests { - use std::{ - future::Future, - pin::pin, - sync::atomic::AtomicUsize, - task::{Context, Poll, Wake, Waker}, - }; - - use super::*; - - /// A waker that counts how many times the task asks to be polled again. - struct CountingWaker(AtomicUsize); - - impl CountingWaker { - fn wakes(&self) -> usize { - self.0.load(Ordering::SeqCst) - } - } - - impl Wake for CountingWaker { - fn wake(self: Arc) { - self.wake_by_ref(); - } - - fn wake_by_ref(self: &Arc) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - - /// Waiting for trailers parks until the body settles the question, rather than polling - /// for it. - /// - /// The bug this guards against was a `yield_now` loop, which is visible here as the shape - /// of the wait rather than as a quantity of CPU: a spin re-arms its own waker on every - /// poll, so it is scheduled again immediately, while a parked wait asks for nothing until - /// something else moves. Asserting the wake count keeps this deterministic -- timing how - /// much CPU the process burns measures the machine as much as the code. - #[test] - fn waiting_for_trailers_parks_rather_than_spinning() { - let slot = TrailersSlot::default(); - let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); - let waker = Waker::from(counter.clone()); - let mut cx = Context::from_waker(&waker); - let mut settled = pin!(slot.settled()); - - // Nothing has settled the question, so the wait parks... - assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); - // ...without scheduling itself to be polled again, which is what a spin does. - assert_eq!(counter.wakes(), 0, "a parked wait asks for no wake-up"); - - // Polling again changes nothing: still parked, still asking for nothing. - assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); - assert_eq!(counter.wakes(), 0, "polling again does not arm a wake-up"); - - // The body ending is what wakes it, and it resolves on the next poll. - slot.ended(); - assert!(counter.wakes() >= 1, "the body ending wakes the waiter"); - assert!(matches!( - settled.as_mut().poll(&mut cx), - Poll::Ready(Trailers::None) - )); - } - - /// Trailers that arrived before anyone asked resolve without parking at all. - #[test] - fn trailers_already_there_resolve_on_the_first_poll() { - let slot = TrailersSlot::default(); - let mut headers = HeaderMap::new(); - headers.insert("x-checksum", "abc123".parse().unwrap()); - slot.arrived(headers); - - let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); - let waker = Waker::from(counter.clone()); - let mut cx = Context::from_waker(&waker); - let mut settled = pin!(slot.settled()); - - assert!(matches!( - settled.as_mut().poll(&mut cx), - Poll::Ready(Trailers::Some(_)) - )); - } -} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index eef78b8..951ac98 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -21,6 +21,7 @@ pub mod agent; pub mod body; pub mod client; pub mod error; +pub mod response; pub mod retry; pub mod stats; pub mod timing; diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs new file mode 100644 index 0000000..3382dd8 --- /dev/null +++ b/crates/web-faith/src/response.rs @@ -0,0 +1,226 @@ +//! Reading a response: where trailers land, what is known of the peer, and writing a body out. + +// spec:RESP spec:TRL spec:BODY + +use std::{net::SocketAddr, time::Duration}; + +use http::header::HeaderMap; +use tokio::sync::watch; + +use crate::error::{FaithError, FaithErrorKind}; + +/// What is known about the peer that sent a response. +/// +/// - `address`: The IP address and port of the peer, if available. +/// - `certificate`: When connected over HTTPS, this is the DER-encoded leaf certificate of the peer. +#[derive(Debug)] +pub struct PeerInformation { + pub address: Option, + pub certificate: Option>, +} + +/// Where a response body is written, and on what terms. +#[derive(Debug, Clone, Default)] +pub struct FileDestination { + /// Truncate and replace an occupied destination. The default refuses one instead, leaving what + /// is there untouched. + pub overwrite: bool, + /// The permissions a newly created file is given. Ignored on platforms without Unix file modes. + pub mode: Option, +} + +/// The shortest gap between progress reports. +/// +/// Reporting every chunk would cross a surface boundary thousands of times for a large body, +/// which is the cost writing to a file directly exists to avoid. A caller driving a progress bar +/// cannot use updates faster than this anyway, and the final report is always delivered regardless. +pub const PROGRESS_INTERVAL: Duration = Duration::from_millis(50); + +/// Open the destination file for a body write, mapping filesystem refusals to the errors +/// writing a body to a file surfaces. +// spec:BODY#tofile +pub async fn open_destination( + path: &str, + options: &FileDestination, +) -> Result { + let mut open = tokio::fs::OpenOptions::new(); + open.write(true); + if options.overwrite { + // An occupied destination is truncated and replaced. + open.create(true).truncate(true); + } else { + // The safe default refuses an occupied destination outright. + open.create_new(true); + } + #[cfg(unix)] + if let Some(mode) = options.mode { + open.mode(mode); + } + + match open.open(path).await { + Ok(file) => Ok(file), + Err(err) => Err(classify_open_error(path, err).await), + } +} + +/// Classify a failure to open the destination. An occupied destination is `FileExists`, +/// unless what occupies it is a directory: a directory is well-formed but cannot be written +/// to, which is a `FileWrite`. Every other refusal is a `FileWrite` carrying the OS detail. +pub async fn classify_open_error(path: &str, err: std::io::Error) -> FaithError { + let kind = if err.kind() == std::io::ErrorKind::AlreadyExists { + match tokio::fs::symlink_metadata(path).await { + Ok(meta) if meta.is_dir() => FaithErrorKind::FileWrite, + _ => FaithErrorKind::FileExists, + } + } else { + FaithErrorKind::FileWrite + }; + FaithError::new(kind, Some(err.to_string())) +} + +#[derive(Clone, Debug, Default)] +pub enum Trailers { + #[default] + NotYet, + None, + Some(HeaderMap), +} + +/// Where the trailers land: written by whoever finishes the body, awaited by `trailers()`. +/// +/// A watch channel, rather than a lock read in a loop. Per the fetch standard's trailers +/// proposal () this promise is *meant* not to +/// resolve until the body has been consumed, so the wait is unbounded by design -- which is +/// precisely why polling was the wrong shape for it. Awaiting trailers without reading the +/// body now leaves an idle pending promise rather than a pegged core, and the future can be +/// cancelled while it waits. +#[derive(Debug)] +pub struct TrailersSlot(watch::Sender); + +impl Default for TrailersSlot { + fn default() -> Self { + Self(watch::channel(Trailers::NotYet).0) + } +} + +impl TrailersSlot { + /// Record trailers that arrived, waking whoever is waiting. + pub fn arrived(&self, trailers: HeaderMap) { + self.0.send_replace(Trailers::Some(trailers)); + } + + /// Record that the body ended, if no trailers frame got there first. + /// + /// `send_if_modified` so the read and the write are one step, and so waiters are woken + /// only by the call that actually settled it. + pub fn ended(&self) { + self.0.send_if_modified(|state| { + if matches!(state, Trailers::NotYet) { + *state = Trailers::None; + true + } else { + false + } + }); + } + + /// Wait until the body has settled the question. + pub async fn settled(&self) -> Trailers { + let mut rx = self.0.subscribe(); + // `wait_for` tests the current value before waiting, so trailers that already + // arrived return without yielding. Its error case is the sender being gone, which + // means the response was dropped and nothing can ever set this -- no trailers is + // the only answer left. + match rx + .wait_for(|state| !matches!(state, Trailers::NotYet)) + .await + { + Ok(state) => state.clone(), + Err(_) => Trailers::None, + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + future::Future, + pin::pin, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll, Wake, Waker}, + }; + + use super::*; + + /// A waker that counts how many times the task asks to be polled again. + struct CountingWaker(AtomicUsize); + + impl CountingWaker { + fn wakes(&self) -> usize { + self.0.load(Ordering::SeqCst) + } + } + + impl Wake for CountingWaker { + fn wake(self: Arc) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + /// Waiting for trailers parks until the body settles the question, rather than polling + /// for it. + /// + /// The bug this guards against was a `yield_now` loop, which is visible here as the shape + /// of the wait rather than as a quantity of CPU: a spin re-arms its own waker on every + /// poll, so it is scheduled again immediately, while a parked wait asks for nothing until + /// something else moves. Asserting the wake count keeps this deterministic -- timing how + /// much CPU the process burns measures the machine as much as the code. + #[test] + fn waiting_for_trailers_parks_rather_than_spinning() { + let slot = TrailersSlot::default(); + let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); + let waker = Waker::from(counter.clone()); + let mut cx = Context::from_waker(&waker); + let mut settled = pin!(slot.settled()); + + // Nothing has settled the question, so the wait parks... + assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); + // ...without scheduling itself to be polled again, which is what a spin does. + assert_eq!(counter.wakes(), 0, "a parked wait asks for no wake-up"); + + // Polling again changes nothing: still parked, still asking for nothing. + assert!(matches!(settled.as_mut().poll(&mut cx), Poll::Pending)); + assert_eq!(counter.wakes(), 0, "polling again does not arm a wake-up"); + + // The body ending is what wakes it, and it resolves on the next poll. + slot.ended(); + assert!(counter.wakes() >= 1, "the body ending wakes the waiter"); + assert!(matches!( + settled.as_mut().poll(&mut cx), + Poll::Ready(Trailers::None) + )); + } + + /// Trailers that arrived before anyone asked resolve without parking at all. + #[test] + fn trailers_already_there_resolve_on_the_first_poll() { + let slot = TrailersSlot::default(); + let mut headers = HeaderMap::new(); + headers.insert("x-checksum", "abc123".parse().unwrap()); + slot.arrived(headers); + + let counter = Arc::new(CountingWaker(AtomicUsize::new(0))); + let waker = Waker::from(counter.clone()); + let mut cx = Context::from_waker(&waker); + let mut settled = pin!(slot.settled()); + + assert!(matches!( + settled.as_mut().poll(&mut cx), + Poll::Ready(Trailers::Some(_)) + )); + } +} From 1fb5e294d679bc178b6a70ce49d4732dddd57279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:22:39 +1200 Subject: [PATCH 39/61] S1: invert the response, so the client owns it and the binding wraps it web_faith::response::Response holds the response's state and the reads over it: disturbing the body, gathering it, and writing it out to a file. The napi class is a handle on one, and each JS method delegates. Writing to a file takes a progress closure rather than a threadsafe function, and reports in the counts it measures; the binding hands it a closure that crosses into JavaScript, and converts the report and the result into the shapes JS takes. The file destination's defaults, which the options object left implicit, are explicit in the client's own type. The integrity dependency stops being optional. The client's read path verifies unconditionally, and gating the call sites would have meant a build that quietly skips verification -- where the spec asks for a feature to remove the API that offers it instead. That belongs with the request API in step 9, so the feature comes back then rather than standing as a trap now. index.d.ts loses one more spec marker that had been leaking into the published TypeScript. Nothing else about the Node surface changes. --- Cargo.lock | 1 + crates/web-faith-napi/Cargo.toml | 2 +- crates/web-faith-napi/src/fetch.rs | 4 +- crates/web-faith-napi/src/response.rs | 442 ++++++-------------------- crates/web-faith/Cargo.toml | 7 +- crates/web-faith/src/error.rs | 1 - crates/web-faith/src/response.rs | 346 +++++++++++++++++++- index.d.ts | 2 - 8 files changed, 443 insertions(+), 362 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee565eb..57195da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2817,6 +2817,7 @@ dependencies = [ "web-faith-conn-tracker", "web-faith-cookies", "web-faith-dns", + "web-faith-encoding", "web-faith-integrity", ] diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index de5e9a9..8a65b5a 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -45,7 +45,7 @@ tokio-stream.workspace = true time.workspace = true tokio-util.workspace = true url.workspace = true -web-faith = { workspace = true, features = ["integrity"] } +web-faith.workspace = true web-faith-alt-svc = { workspace = true, optional = true } web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index 5e55c7f..91a943c 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -428,7 +428,7 @@ pub fn faith_fetch<'env>( timing.ended(); } - Ok(FaithResponse { + Ok(FaithResponse::from(web_faith::response::Response { body: if empty { BodyHolder::none() } else { @@ -451,6 +451,6 @@ pub fn faith_fetch<'env>( trailers: Default::default(), url: response_url, version, - }) + })) }) } diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 1d06213..3e0acce 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -1,42 +1,25 @@ use std::{ fmt::Debug, - hint::unreachable_unchecked, - mem::replace, - pin::Pin, result::Result, sync::{ Arc, atomic::{AtomicBool, Ordering}, }, - time::Instant, }; -use bytes::Bytes; -use futures::{StreamExt, TryStreamExt, stream}; -use http_body_util::BodyStream; +use futures::TryStreamExt; use napi::{ bindgen_prelude::*, threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, }; use napi_derive::napi; -use reqwest::{ - StatusCode, Url, Version, - header::{CONTENT_LENGTH, HeaderMap}, -}; use serde_json; -use stream_shared::SharedStream; -use tokio::io::AsyncWriteExt; +pub use web_faith::response::PeerInformation; use web_faith::{ - body::{Body, BodyHolder, DynStream, drain_body_inner}, - response::{FileDestination, PROGRESS_INTERVAL, Trailers, TrailersSlot, open_destination}, - stats::InnerAgentStats, - timing::TimingSlot, + body::{Body, drain_body_inner}, + response::{FileDestination, FileProgress, FileWritten, Response, Trailers}, }; -use web_faith_encoding::{Coding, decode_stream}; - -pub use web_faith::response::PeerInformation; -use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; use crate::{ async_task::{Value, faith_promise}, @@ -44,31 +27,6 @@ use crate::{ timing::TimingBreakdown, }; -/// The `Response` interface of the Fetch API represents the response to a request. -/// -/// Faith does not allow its `Response` object to be constructed. If you need to, you may use the -/// `webResponse()` method to convert one into a Web API `Response` object; note the caveats. -#[napi] -#[derive(Debug, Clone)] -pub struct FaithResponse { - pub(crate) body: BodyHolder, - /// The coding to decode the body under, or `None` to deliver it as received. - /// Set once when the response is built, from the request's `Accept-Encoding` and the - /// response's `Content-Encoding` (see [`web_faith_encoding`]). - pub(crate) decode: Option, - pub(crate) disturbed: Arc, - pub(crate) headers: HeaderMap, - pub(crate) integrity: Option, - pub(crate) peer: Arc, - pub(crate) redirected: bool, - pub(crate) stats: Arc, - pub(crate) status_code: StatusCode, - pub(crate) timing: Arc, - pub(crate) trailers: Arc, - pub(crate) url: Url, - pub(crate) version: Version, -} - /// Options for `toFile()`. #[napi(object)] #[derive(Debug, Default)] @@ -103,6 +61,22 @@ pub struct ToFileProgress { pub content_length: Option, } +/// The `Response` interface of the Fetch API represents the response to a request. +/// +/// Faith does not allow its `Response` object to be constructed. If you need to, you may use the +/// `webResponse()` method to convert one into a Web API `Response` object; note the caveats. +#[napi] +#[derive(Debug, Clone)] +pub struct FaithResponse { + pub(crate) inner: Response, +} + +impl From for FaithResponse { + fn from(inner: Response) -> Self { + Self { inner } + } +} + impl From<&ToFileOptions> for FileDestination { fn from(options: &ToFileOptions) -> Self { Self { @@ -112,6 +86,25 @@ impl From<&ToFileOptions> for FileDestination { } } +impl From for ToFileProgress { + fn from(progress: FileProgress) -> Self { + let count = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); + Self { + bytes_written: count(progress.bytes_written), + content_length: progress.content_length.map(count), + } + } +} + +impl From for ToFileResult { + fn from(written: FileWritten) -> Self { + Self { + path: written.path, + bytes_written: i64::try_from(written.bytes_written).unwrap_or(i64::MAX), + } + } +} + /// The callback `toFile()` reports progress to. /// /// `CalleeHandled = false`: progress is not an error-first callback, so the JavaScript @@ -130,7 +123,8 @@ impl FaithResponse { /// This is a function as an internal implementation detail and the wrapper makes it a property. #[napi] pub fn headers(&self) -> Vec<(String, String)> { - self.headers + self.inner + .headers .iter() .filter_map(|(name, value)| { value @@ -145,7 +139,7 @@ impl FaithResponse { /// response was successful (status in the range 200-299) or not. #[napi(getter)] pub fn ok(&self) -> bool { - self.status_code.is_success() + self.inner.status_code.is_success() } /// Custom to Faith. @@ -155,10 +149,14 @@ impl FaithResponse { #[napi(getter, ts_return_type = "{ address?: string; certificate?: Buffer }")] pub fn peer<'env>(&self, env: &'env Env) -> Result, napi::Error> { let mut obj = Object::new(env)?; - obj.set("address", self.peer.address.map(|addr| addr.to_string()))?; + obj.set( + "address", + self.inner.peer.address.map(|addr| addr.to_string()), + )?; obj.set( "certificate", - self.peer + self.inner + .peer .certificate .as_deref() .map(|cert| Buffer::from(cert)), @@ -179,7 +177,7 @@ impl FaithResponse { /// `false` on those responses. #[napi(getter)] pub fn redirected(&self) -> bool { - self.redirected + self.inner.redirected } /// The `status` read-only property of the `Response` interface contains the HTTP status codes of the @@ -188,7 +186,7 @@ impl FaithResponse { /// A value is `0` is returned for a response whose `type` is `opaque`, `opaqueredirect`, or `error`. #[napi(getter)] pub fn status(&self) -> u16 { - self.status_code.as_u16() + self.inner.status_code.as_u16() } /// The `statusText` read-only property of the `Response` interface contains the status message @@ -201,7 +199,10 @@ impl FaithResponse { /// string. #[napi(getter)] pub fn status_text(&self) -> &'static str { - self.status_code.canonical_reason().unwrap_or_default() + self.inner + .status_code + .canonical_reason() + .unwrap_or_default() } /// The `type` read-only property of the `Response` interface contains the type of the response. The @@ -217,7 +218,7 @@ impl FaithResponse { /// value of the `url` property will be the final URL obtained after any redirects. #[napi(getter)] pub fn url(&self) -> String { - self.url.to_string() + self.inner.url.to_string() } /// The `version` read-only property of the `Response` interface contains the HTTP version of the @@ -226,7 +227,7 @@ impl FaithResponse { /// This is custom to Faith. #[napi(getter)] pub fn version(&self) -> String { - format!("{:?}", self.version) + format!("{:?}", self.inner.version) } /// The `bodyUsed` read-only property of the `Response` interface is a boolean value that indicates @@ -237,7 +238,7 @@ impl FaithResponse { /// the `.body` property counts as a read, even if you don't actually consume any bytes of content. #[napi(getter)] pub fn body_used(&self) -> bool { - self.disturbed.load(Ordering::SeqCst) + self.inner.disturbed.load(Ordering::SeqCst) } /// The `body` read-only property of the `Response` interface is a `ReadableStream` of the body @@ -260,9 +261,9 @@ impl FaithResponse { ) -> Result>>, napi::Error> { // we mark the body as disturbed, but we still allow reading it through here // as essentially, the body() can be accessed many times as the same stream - let _ = self.check_stream_disturbed(); + let _ = self.inner.check_stream_disturbed(); - let Some(lock) = &self.body.body else { + let Some(lock) = &self.inner.body.body else { return Ok(None); }; @@ -272,7 +273,8 @@ impl FaithResponse { .map_err(|_| FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed).into_napi())?; let stream = self - .ensure_stream(&mut body, self.body.drained.clone()) + .inner + .ensure_stream(&mut body, self.inner.body.drained.clone()) .map_err(|e| e.into_napi())?; let stream = napi::bindgen_prelude::ReadableStream::create_with_stream_bytes( @@ -289,142 +291,6 @@ impl FaithResponse { Ok(Some(stream)) } - fn check_stream_disturbed(&self) -> Result<(), FaithError> { - if self.disturbed.swap(true, Ordering::SeqCst) { - Err(FaithErrorKind::ResponseAlreadyDisturbed.into()) - } else { - Ok(()) - } - } - - /// Ensures the body is converted to a SharedStream, returning a clone of it. - /// - /// This allows multiple consumers (original + clones) to independently read the body. - fn ensure_stream( - &self, - body: &mut Body, - drained_flag: Arc, - ) -> Result>>, FaithError> { - match body { - Body::Consumed => Err(FaithErrorKind::ResponseAlreadyDisturbed.into()), - Body::Stream(stream) => Ok(stream.clone()), - lock @ Body::Inner(_) => { - // temporarily replace with Consumed until we can put in the Stream - let Body::Inner(inner) = replace(lock, Body::Consumed) else { - // SAFETY: we're inside the match checking for this exact thing - unsafe { unreachable_unchecked() } - }; - - // Track that we've started consuming a body - self.stats.bodies_started.fetch_add(1, Ordering::Relaxed); - - let trailers_stream = self.trailers.clone(); - let trailers_finish = self.trailers.clone(); - let stats_finish = self.stats.clone(); - let timing_finish = self.timing.clone(); - let drained_finish = drained_flag.clone(); - // The frame stream pulls trailers off to the side (via `arrived`) and yields - // data bytes only, so decoding sees no trailer frames. - let bytes = Box::pin( - BodyStream::new(inner) - .then(move |frame| { - let trailers_lock = trailers_stream.clone(); - async move { - match frame { - Err(err) => Some(Err(err.to_string())), - Ok(frame) => match frame.into_trailers() { - Ok(trailers) => { - trailers_lock.arrived(trailers); - None - } - Err(frame) => Some( - frame - .into_data() - .map_err(|_| "unknown frame kind".to_string()), - ), - }, - } - } - }) - .filter_map(async |item| item), - ) as Pin>; - - let bytes = match self.decode { - Some(coding) => decode_stream(bytes, coding), - None => bytes, - }; - - // A zero-length chunk carries no bytes, but the body's byte-oriented - // ReadableStream cannot take one: `ReadableByteStreamController.enqueue` - // rejects an empty buffer outright (`ERR_INVALID_STATE`). Some origins end a - // response with an empty DATA frame carrying END_STREAM, so drop empty chunks - // here, before the stream is built, letting it close cleanly. The byte count - // delivered is unchanged, and the collecting paths (`text()`, `bytes()`) never - // noticed the empties anyway. - let bytes = Box::pin(bytes.filter(|item| { - let empty = matches!(item, Ok(chunk) if chunk.is_empty()); - async move { !empty } - })) as Pin>; - - // Chained onto the stream that is actually delivered, above any decoder: a - // decoder reaches the end of its own framing without necessarily polling the - // bytes underneath to completion, so bookkeeping chained below it would never - // run for a decoded body, leaving the trailers promise and the timing pending - // for good. - let bytes = Box::pin( - bytes.chain( - stream::once(async move { - trailers_finish.ended(); - // The last byte of the body: every read path ends here, so - // this is where the timing settles - // (spec:RESP#request-timing). - timing_finish.ended(); - // Track that we've finished consuming a body - stats_finish.bodies_finished.fetch_add(1, Ordering::Relaxed); - // Mark body as drained so Drop doesn't try to drain again - drained_finish.store(true, Ordering::SeqCst); - }) - .filter_map(async |()| None), - ), - ) as Pin>; - - let stream = SharedStream::new(bytes); - - // the _ is the Consumed we put in there earlier - let _ = replace(lock, Body::Stream(stream.clone())); - - Ok(stream) - } - } - } - - /// Underlying efficient response body fetcher. - /// - /// Unlike bytes() and co, this grabs all the chunks of the response but doesn't - /// copy them. Further processing is needed to obtain a Vec or whatever needed. - async fn gather(&self) -> Result, FaithError> { - let Some(lock) = &self.body.body else { - return Ok(Default::default()); - }; - - let mut body = lock.lock().await; - let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; - drop(body); // release lock before consuming stream - - let mut chunks = Vec::new(); - futures::pin_mut!(stream); - while let Some(result) = stream.next().await { - let chunk = - result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; - chunks.push(chunk); - } - - // Mark as drained since we consumed everything - self.body.mark_drained(); - - Ok(Arc::from(chunks.into_boxed_slice())) - } - /// Discard the response body, releasing the connection back to the pool. /// /// This is useful when you don't need the body but want to ensure the connection @@ -438,11 +304,11 @@ impl FaithResponse { /// Returns a promise that resolves when the body has been fully discarded. #[napi] pub fn discard<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let body = self.body.body.clone(); - let drained_flag = self.body.drained.clone(); - let is_multiplexed = self.body.is_multiplexed(); - let trailers = self.trailers.clone(); - let timing = self.timing.clone(); + let body = self.inner.body.body.clone(); + let drained_flag = self.inner.body.drained.clone(); + let is_multiplexed = self.inner.body.is_multiplexed(); + let trailers = self.inner.trailers.clone(); + let timing = self.inner.timing.clone(); faith_promise(env, async move { if let Some(arc) = body { if is_multiplexed { @@ -468,22 +334,6 @@ impl FaithResponse { }) } - /// gather() and then copy into one contiguous buffer - async fn gather_contiguous(&self) -> Result, FaithError> { - let body = self.gather().await?; - let length = body.iter().map(|chunk| chunk.len()).sum(); - let mut bytes = Vec::with_capacity(length); - for chunk in body.into_iter() { - bytes.extend_from_slice(chunk); - } - - if let Some(ref integrity) = self.integrity { - verify_integrity(&bytes, integrity)?; - } - - Ok(bytes) - } - /// The `bytes()` method of the `Response` interface takes a `Response` stream and reads it to /// completion. It returns a promise that resolves with a `Uint8Array`. /// @@ -492,8 +342,8 @@ impl FaithResponse { pub fn bytes<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); faith_promise(env, async move { - this.check_stream_disturbed()?; - this.gather_contiguous().await.map(Buffer::from) + this.inner.check_stream_disturbed()?; + this.inner.gather_contiguous().await.map(Buffer::from) }) } @@ -505,8 +355,8 @@ impl FaithResponse { pub fn text<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); faith_promise(env, async move { - this.check_stream_disturbed()?; - let bytes = this.gather_contiguous().await?; + this.inner.check_stream_disturbed()?; + let bytes = this.inner.gather_contiguous().await?; Ok(String::from_utf8(bytes) .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())) }) @@ -526,8 +376,8 @@ impl FaithResponse { pub fn json<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); faith_promise(env, async move { - this.check_stream_disturbed()?; - let bytes = this.gather_contiguous().await?; + this.inner.check_stream_disturbed()?; + let bytes = this.inner.gather_contiguous().await?; let value = serde_json::from_slice(&bytes) .map_err(|e| FaithError::new(FaithErrorKind::JsonParse, Some(e.to_string())))?; Ok(Value(value)) @@ -567,124 +417,21 @@ impl FaithResponse { let this = Clone::clone(self); let options = options.unwrap_or_default(); faith_promise(env, async move { - this.write_to_file(path, options, on_progress).await - }) - } - - async fn write_to_file( - &self, - path: String, - options: ToFileOptions, - on_progress: Option, - ) -> Result { - // A response that cannot carry a body has nothing to write, and this is settled - // before any file is created (spec:BODY#tofile). - let Some(lock) = self.body.body.clone() else { - return Err(FaithErrorKind::ResponseBodyNull.into()); - }; - - // A body already read, or whose stream was handed out, has no second read to give. - // Checked without committing so an open failure below still leaves the body - // undisturbed and the caller free to retry to another path. - if self.disturbed.load(Ordering::SeqCst) { - return Err(FaithErrorKind::ResponseAlreadyDisturbed.into()); - } - - // Reject a malformed integrity value up front, before the body is touched, the same - // as the other verified reads reject it when the whole body is in hand. - let mut checker = integrity_checker(self.integrity.as_deref())?; - - // The advertised length, when the server sent one. It is only visible here for a body - // delivered as received: a decoded body has had its Content-Length stripped, so the - // bytes written equal the wire bytes wherever this is Some (spec:BODY#tofile, ENC). - let content_length = self - .headers - .get(CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.trim().parse::().ok()); - - // The destination is opened before any of the body is read, so a failure to open it - // leaves the body unread and undisturbed. - let mut file = open_destination(&path, &FileDestination::from(&options)).await?; - - // Commit the read now the destination is in hand. A concurrent read that slipped in - // since the load above wins, and this one finds the body already spent. - self.check_stream_disturbed()?; - - let stream = { - let mut body = lock.lock().await; - let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; - drop(body); // release lock before consuming stream - stream - }; - - // Reporting is rate limited rather than per chunk, so a large body does not cross - // into JavaScript thousands of times (spec:BODY#tofile). - let report = |written: u64| { - if let Some(callback) = &on_progress { - callback.call( - ToFileProgress { - bytes_written: written as i64, - content_length: content_length.map(|len| len as i64), - }, - // Progress is observational: a report the queue cannot take is dropped - // rather than made to hold up the write it is describing. - ThreadsafeFunctionCallMode::NonBlocking, - ); - } - }; - - let mut written: u64 = 0; - let mut reported_at = Instant::now(); - futures::pin_mut!(stream); - while let Some(result) = stream.next().await { - let chunk = - result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; - if let Some(checker) = checker.as_mut() { - checker.input(&chunk); - } - file.write_all(&chunk) - .await - .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; - written += chunk.len() as u64; - // A server cannot send more than it promised: once the bytes off the wire exceed - // the advertised length, the write fails and the bytes so far stay on disk - // (spec:BODY#tofile). - if let Some(limit) = content_length { - if written > limit { - return Err(FaithErrorKind::ContentLengthOverrun.into()); - } - } - if reported_at.elapsed() >= PROGRESS_INTERVAL { - reported_at = Instant::now(); - report(written); - } - } - - file.flush() - .await - .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; - - // The last report always lands, whatever the rate limit allowed along the way, so a - // caller's final view of a completed write is the whole body rather than the last - // interval boundary. An empty body reports once, with nothing written. - report(written); - - // The digest is only known once the last byte has been written, so the file that - // fails verification is on disk when the error arrives (spec:SRI). - if let Some(checker) = checker { - finish_integrity(checker)?; - } - - self.body.mark_drained(); - - Ok(ToFileResult { - // A relative path resolves against the process's working directory; the caller - // is handed the absolute path the bytes landed at. - path: std::path::absolute(&path) - .map(|abs| abs.to_string_lossy().into_owned()) - .unwrap_or(path), - bytes_written: written as i64, + let destination = FileDestination::from(&options); + let written = this + .inner + .write_to_file(&path, &destination, |progress| { + if let Some(callback) = &on_progress { + callback.call( + ToFileProgress::from(progress), + // Progress is observational: a report the queue cannot take is + // dropped rather than made to hold up the write it describes. + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + }) + .await?; + Ok(ToFileResult::from(written)) }) } @@ -703,11 +450,10 @@ impl FaithResponse { /// /// This is an async fn as an internal implementation detail and the wrapper makes it a /// property. - /// - /// spec:RESP#request-timing + // spec:RESP#request-timing #[napi] pub async fn timing(&self) -> TimingBreakdown { - self.timing.settled().await.into() + self.inner.timing.settled().await.into() } /// The `trailers()` read-only property of the `Response` interface returns a promise that @@ -729,7 +475,7 @@ impl FaithResponse { /// This is an async fn as an internal implementation detail and the wrapper makes it a property. #[napi] pub async fn trailers(&self) -> Option> { - match self.trailers.settled().await { + match self.inner.trailers.settled().await { // NotYet cannot come back from `settled`, which is what it waits on. Trailers::NotYet | Trailers::None => None, Trailers::Some(headers) => Some( @@ -755,15 +501,15 @@ impl FaithResponse { /// possible with Faith.) #[napi] pub fn clone(&self, env: Env) -> Result { - if self.disturbed.load(Ordering::SeqCst) { + if self.inner.disturbed.load(Ordering::SeqCst) { return Err(FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed) .into_js_error(&env) .into()); } - Ok(Self { + Ok(Self::from(Response { disturbed: Arc::new(AtomicBool::new(false)), - ..Clone::clone(self) - }) + ..Clone::clone(&self.inner) + })) } } diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 7e275af..abe45f5 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -27,14 +27,13 @@ strum.workspace = true tokio.workspace = true url.workspace = true web-faith-conn-tracker.workspace = true +web-faith-encoding.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-alt-svc = { workspace = true, optional = true } -web-faith-integrity = { workspace = true, optional = true } +web-faith-integrity.workspace = true [features] -default = ["http3", "integrity"] +default = ["http3"] # Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] -# Subresource Integrity: parsing and verifying `integrity` values. -integrity = ["dep:web-faith-integrity"] diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs index 3736aa1..03af51e 100644 --- a/crates/web-faith/src/error.rs +++ b/crates/web-faith/src/error.rs @@ -152,7 +152,6 @@ impl From for FaithError { /// A component crate names its own errors; they become the client's as they cross into it, which is /// what keeps the code a caller sees the same whichever layer failed. -#[cfg(feature = "integrity")] impl From for FaithError { fn from(err: web_faith_integrity::IntegrityError) -> Self { use web_faith_integrity::IntegrityError as E; diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index 3382dd8..7ea7d64 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -2,12 +2,35 @@ // spec:RESP spec:TRL spec:BODY -use std::{net::SocketAddr, time::Duration}; +use std::{ + hint::unreachable_unchecked, + mem::replace, + net::SocketAddr, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; -use http::header::HeaderMap; -use tokio::sync::watch; +use bytes::Bytes; +use futures::{StreamExt, stream}; +use http::header::{CONTENT_LENGTH, HeaderMap}; +use http_body_util::BodyStream; +use reqwest::{StatusCode, Url, Version}; +use stream_shared::SharedStream; +use tokio::{io::AsyncWriteExt, sync::watch}; +use web_faith_encoding::{Coding, decode_stream}; -use crate::error::{FaithError, FaithErrorKind}; +use crate::{ + body::{Body, BodyHolder, DynStream}, + error::{FaithError, FaithErrorKind}, + stats::InnerAgentStats, + timing::TimingSlot, +}; + +use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; /// What is known about the peer that sent a response. /// @@ -224,3 +247,318 @@ mod tests { )); } } + +/// A progress report from a body write in flight. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileProgress { + /// Bytes written to the file so far. + pub bytes_written: u64, + /// What the response advertised in `Content-Length`, when it sent one and the body is not + /// being decoded. Absent when the total is not known ahead of time, which is the case for a + /// chunked response and for one being decoded. + pub content_length: Option, +} + +/// What a completed body write reports. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileWritten { + /// The absolute filesystem path written to. + pub path: String, + /// The number of bytes that landed at the destination. + pub bytes_written: u64, +} + +/// A response to a request. +/// +/// A response is not constructed by a caller; it arrives from a request. Reading its body consumes +/// it, following the fetch standard rather than the owned-response model of other Rust clients, so a +/// second read fails. +#[derive(Debug, Clone)] +pub struct Response { + pub body: BodyHolder, + /// The coding to decode the body under, or `None` to deliver it as received. + /// Set once when the response is built, from the request's `Accept-Encoding` and the + /// response's `Content-Encoding` (see [`web_faith_encoding`]). + pub decode: Option, + pub disturbed: Arc, + pub headers: HeaderMap, + pub integrity: Option, + pub peer: Arc, + pub redirected: bool, + pub stats: Arc, + pub status_code: StatusCode, + pub timing: Arc, + pub trailers: Arc, + pub url: Url, + pub version: Version, +} + +impl Response { + pub fn check_stream_disturbed(&self) -> Result<(), FaithError> { + if self.disturbed.swap(true, Ordering::SeqCst) { + Err(FaithErrorKind::ResponseAlreadyDisturbed.into()) + } else { + Ok(()) + } + } + + /// Ensures the body is converted to a SharedStream, returning a clone of it. + /// + /// This allows multiple consumers (original + clones) to independently read the body. + pub fn ensure_stream( + &self, + body: &mut Body, + drained_flag: Arc, + ) -> Result>>, FaithError> { + match body { + Body::Consumed => Err(FaithErrorKind::ResponseAlreadyDisturbed.into()), + Body::Stream(stream) => Ok(stream.clone()), + lock @ Body::Inner(_) => { + // temporarily replace with Consumed until we can put in the Stream + let Body::Inner(inner) = replace(lock, Body::Consumed) else { + // SAFETY: we're inside the match checking for this exact thing + unsafe { unreachable_unchecked() } + }; + + // Track that we've started consuming a body + self.stats.bodies_started.fetch_add(1, Ordering::Relaxed); + + let trailers_stream = self.trailers.clone(); + let trailers_finish = self.trailers.clone(); + let stats_finish = self.stats.clone(); + let timing_finish = self.timing.clone(); + let drained_finish = drained_flag.clone(); + // The frame stream pulls trailers off to the side (via `arrived`) and yields + // data bytes only, so decoding sees no trailer frames. + let bytes = Box::pin( + BodyStream::new(inner) + .then(move |frame| { + let trailers_lock = trailers_stream.clone(); + async move { + match frame { + Err(err) => Some(Err(err.to_string())), + Ok(frame) => match frame.into_trailers() { + Ok(trailers) => { + trailers_lock.arrived(trailers); + None + } + Err(frame) => Some( + frame + .into_data() + .map_err(|_| "unknown frame kind".to_string()), + ), + }, + } + } + }) + .filter_map(async |item| item), + ) as Pin>; + + let bytes = match self.decode { + Some(coding) => decode_stream(bytes, coding), + None => bytes, + }; + + // A zero-length chunk carries no bytes, but the body's byte-oriented + // ReadableStream cannot take one: `ReadableByteStreamController.enqueue` + // rejects an empty buffer outright (`ERR_INVALID_STATE`). Some origins end a + // response with an empty DATA frame carrying END_STREAM, so drop empty chunks + // here, before the stream is built, letting it close cleanly. The byte count + // delivered is unchanged, and the collecting paths (`text()`, `bytes()`) never + // noticed the empties anyway. + let bytes = Box::pin(bytes.filter(|item| { + let empty = matches!(item, Ok(chunk) if chunk.is_empty()); + async move { !empty } + })) as Pin>; + + // Chained onto the stream that is actually delivered, above any decoder: a + // decoder reaches the end of its own framing without necessarily polling the + // bytes underneath to completion, so bookkeeping chained below it would never + // run for a decoded body, leaving the trailers promise and the timing pending + // for good. + let bytes = Box::pin( + bytes.chain( + stream::once(async move { + trailers_finish.ended(); + // The last byte of the body: every read path ends here, so + // this is where the timing settles + // (spec:RESP#request-timing). + timing_finish.ended(); + // Track that we've finished consuming a body + stats_finish.bodies_finished.fetch_add(1, Ordering::Relaxed); + // Mark body as drained so Drop doesn't try to drain again + drained_finish.store(true, Ordering::SeqCst); + }) + .filter_map(async |()| None), + ), + ) as Pin>; + + let stream = SharedStream::new(bytes); + + // the _ is the Consumed we put in there earlier + let _ = replace(lock, Body::Stream(stream.clone())); + + Ok(stream) + } + } + } + + /// Underlying efficient response body fetcher. + /// + /// Unlike bytes() and co, this grabs all the chunks of the response but doesn't + /// copy them. Further processing is needed to obtain a Vec or whatever needed. + pub async fn gather(&self) -> Result, FaithError> { + let Some(lock) = &self.body.body else { + return Ok(Default::default()); + }; + + let mut body = lock.lock().await; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + drop(body); // release lock before consuming stream + + let mut chunks = Vec::new(); + futures::pin_mut!(stream); + while let Some(result) = stream.next().await { + let chunk = + result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; + chunks.push(chunk); + } + + // Mark as drained since we consumed everything + self.body.mark_drained(); + + Ok(Arc::from(chunks.into_boxed_slice())) + } + + /// gather() and then copy into one contiguous buffer + pub async fn gather_contiguous(&self) -> Result, FaithError> { + let body = self.gather().await?; + let length = body.iter().map(|chunk| chunk.len()).sum(); + let mut bytes = Vec::with_capacity(length); + for chunk in body.into_iter() { + bytes.extend_from_slice(chunk); + } + + if let Some(ref integrity) = self.integrity { + verify_integrity(&bytes, integrity)?; + } + + Ok(bytes) + } + + /// Write the body out to a file, reporting progress as the bytes land. + /// + /// `on_progress` is called with the bytes written so far and the advertised length where one is + /// known, no more often than [`PROGRESS_INTERVAL`], and once more when the last byte is written. + // spec:BODY#tofile + pub async fn write_to_file( + &self, + path: &str, + options: &FileDestination, + mut on_progress: impl FnMut(FileProgress), + ) -> Result { + // A response that cannot carry a body has nothing to write, and this is settled + // before any file is created (spec:BODY#tofile). + let Some(lock) = self.body.body.clone() else { + return Err(FaithErrorKind::ResponseBodyNull.into()); + }; + + // A body already read, or whose stream was handed out, has no second read to give. + // Checked without committing so an open failure below still leaves the body + // undisturbed and the caller free to retry to another path. + if self.disturbed.load(Ordering::SeqCst) { + return Err(FaithErrorKind::ResponseAlreadyDisturbed.into()); + } + + // Reject a malformed integrity value up front, before the body is touched, the same + // as the other verified reads reject it when the whole body is in hand. + let mut checker = integrity_checker(self.integrity.as_deref())?; + + // The advertised length, when the server sent one. It is only visible here for a body + // delivered as received: a decoded body has had its Content-Length stripped, so the + // bytes written equal the wire bytes wherever this is Some (spec:BODY#tofile, ENC). + let content_length = self + .headers + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()); + + // The destination is opened before any of the body is read, so a failure to open it + // leaves the body unread and undisturbed. + let mut file = open_destination(path, options).await?; + + // Commit the read now the destination is in hand. A concurrent read that slipped in + // since the load above wins, and this one finds the body already spent. + self.check_stream_disturbed()?; + + let stream = { + let mut body = lock.lock().await; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + drop(body); // release lock before consuming stream + stream + }; + + // Reporting is rate limited rather than per chunk, so a large body does not cross a + // surface boundary thousands of times. + // spec:BODY#tofile + let mut report = |written: u64| { + on_progress(FileProgress { + bytes_written: written, + content_length, + }); + }; + + let mut written: u64 = 0; + let mut reported_at = Instant::now(); + futures::pin_mut!(stream); + while let Some(result) = stream.next().await { + let chunk = + result.map_err(|err| FaithError::new(FaithErrorKind::BodyStream, Some(err)))?; + if let Some(checker) = checker.as_mut() { + checker.input(&chunk); + } + file.write_all(&chunk) + .await + .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; + written += chunk.len() as u64; + // A server cannot send more than it promised: once the bytes off the wire exceed + // the advertised length, the write fails and the bytes so far stay on disk + // (spec:BODY#tofile). + if let Some(limit) = content_length { + if written > limit { + return Err(FaithErrorKind::ContentLengthOverrun.into()); + } + } + if reported_at.elapsed() >= PROGRESS_INTERVAL { + reported_at = Instant::now(); + report(written); + } + } + + file.flush() + .await + .map_err(|err| FaithError::new(FaithErrorKind::FileWrite, Some(err.to_string())))?; + + // The last report always lands, whatever the rate limit allowed along the way, so a + // caller's final view of a completed write is the whole body rather than the last + // interval boundary. An empty body reports once, with nothing written. + report(written); + + // The digest is only known once the last byte has been written, so the file that + // fails verification is on disk when the error arrives (spec:SRI). + if let Some(checker) = checker { + finish_integrity(checker)?; + } + + self.body.mark_drained(); + + Ok(FileWritten { + // A relative path resolves against the process's working directory; the caller + // is handed the absolute path the bytes landed at. + path: std::path::absolute(path) + .map(|abs| abs.to_string_lossy().into_owned()) + .unwrap_or_else(|_| path.to_owned()), + bytes_written: written, + }) + } +} diff --git a/index.d.ts b/index.d.ts index 6ecfcb7..c810713 100644 --- a/index.d.ts +++ b/index.d.ts @@ -326,8 +326,6 @@ toFile(path: string, options?: ToFileOptions | undefined | null, onProgress?: (( * * This is an async fn as an internal implementation detail and the wrapper makes it a * property. - * - * spec:RESP#request-timing */ timing(): Promise /** From 3b0231c6676243721042d73d7a6cdc84ba779868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:38:05 +1200 Subject: [PATCH 40/61] S1: move the request path into web-faith Sending a request and building the response that comes back is the client's, so it moves: web_faith::request::send takes an agent, a URL, the options, a body, and an optional abort future. The Node surface reads a fetch() call into those and wraps the result. The body arrives as the client's own shape rather than a napi buffer and a shared receiver: bytes for one already in hand, a stream for one arriving in chunks. The binding still takes the receiver out of its lock before anything can refuse the request, so a refusal drops the stream and whatever is feeding it stops. Cancellation is a future the caller supplies, rather than napi's signal reaching into the send. `same-origin` credentials resolve to `include` at the boundary, that distinction needing an origin the client does not have. --- crates/web-faith-napi/src/fetch.rs | 441 ++---------------------- crates/web-faith-napi/src/options.rs | 72 ++-- crates/web-faith-napi/src/response.rs | 1 - crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/request.rs | 463 ++++++++++++++++++++++++++ 5 files changed, 515 insertions(+), 463 deletions(-) create mode 100644 crates/web-faith/src/request.rs diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index 91a943c..2ba2627 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -1,42 +1,20 @@ -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Instant, -}; - -use http_cache_reqwest::CacheMode; -use hyper_util::client::legacy::connect::HttpInfo; use napi::{ Env, bindgen_prelude::{AbortSignal, PromiseRaw}, }; use napi_derive::napi; -use reqwest::{Method, StatusCode}; -use reqwest::{ - header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, - tls::TlsInfo, -}; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc; -use web_faith::{ - body::{Body, BodyHolder}, - timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, -}; -use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; +use bytes::Bytes; +use web_faith::request::{self, RequestBody}; use crate::{ async_task::faith_promise, - error::{FaithError, FaithErrorKind}, - options::{CredentialsOption, FaithOptions, FaithOptionsAndBody, PRIORITY}, - response::{FaithResponse, PeerInformation}, + options::{self, FaithOptionsAndBody}, + response::FaithResponse, stream_body::StreamBody, }; -/// The methods the fetch standard normalises to upper case; any other method is sent as given. -const NORMALISED_METHODS: [&str; 6] = ["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]; - #[napi] pub fn faith_fetch<'env>( env: &'env Env, @@ -45,7 +23,7 @@ pub fn faith_fetch<'env>( signal: Option, stream_body: Option<&StreamBody>, ) -> Result, napi::Error> { - let (options, agent, body) = FaithOptions::extract(options); + let (options, agent, body) = options::extract(options); let (s, abort) = mpsc::channel(8); let has_signal = signal.is_some(); if let Some(signal) = signal { @@ -58,399 +36,24 @@ pub fn faith_fetch<'env>( let stream_receiver = stream_body.map(|sb| sb.receiver.clone()); faith_promise(env, async move { - let mut abort = abort; - let method = options.method.as_deref().unwrap_or("GET"); - // spec:REQ#method-and-headers - let method = NORMALISED_METHODS - .into_iter() - .find(|normalised| normalised.eq_ignore_ascii_case(method)) - .unwrap_or(method); - - let method = - Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; - let is_head = method == Method::HEAD; - - let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; - - // A `compress` naming no coding Faith can compress in is misuse whether or not the - // request turns out to carry a body, so it is refused before anything else looks at - // it (spec:ENC#compressing-a-request-body). - let compress = options - .compress - .as_deref() - .map(|value| { - Coding::from_option(value).ok_or_else(|| { - FaithError::new( - FaithErrorKind::InvalidCompression, - Some(format!( - "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" - )), - ) - }) - }) - .transpose()?; - - // Handle credentials based on credentials option - if options.credentials == CredentialsOption::Omit { - // Remove credentials from URL if omit is specified - let _ = parsed_url.set_username(""); - let _ = parsed_url.set_password(None); - } - - // The stamp rides along in the request's extensions for the middleware to fill in; - // this side keeps a handle on it so the one measurement taken inside the stack is - // the one surfaced (spec:RESP#request-timing). - let headers_stamp = HeadersStamp::default(); - - let mut request = agent - .inner - .client - .as_ref() - .ok_or(FaithErrorKind::Closed)? - .request(method, parsed_url.clone()) - .with_extension(CacheMode::from(options.cache)) - .with_extension(headers_stamp.clone()); - - if let Some(headers) = &options.headers { - for (key, value) in headers { - // Skip Cookie header if credentials is omit - if options.credentials == CredentialsOption::Omit - && key.eq_ignore_ascii_case("cookie") - { - continue; - } - - // Validate header name and value before adding to request - let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header name: {key}")), - ) - })?; - let header_value = HeaderValue::from_str(value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - - // Faith's coding is layered on top of what the caller declares, and reqwest's - // builder appends rather than replaces, so passing the caller's value through - // here would put a second `Content-Encoding` beside the joined one -- the same - // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is - // still validated above, then withheld and re-emitted once below. - if compress.is_some() && header_name == CONTENT_ENCODING { - continue; - } - - request = request.header(header_name, header_value); - } - } - - // What the caller says they handed over: their own `Content-Encoding`, else the - // agent's, per-request headers winning per name as they do generally (spec: REQ). - // Several lines are the one list, so they are joined as they are read. - let declared_content_encoding = compress.and_then(|_| { - let from_request = options.headers.as_ref().and_then(|headers| { - let declared = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(); - (!declared.is_empty()).then(|| declared.join(", ")) - }); - from_request.or_else(|| { - agent - .inner - .default_content_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - }); - - // The request's `Accept-Encoding` governs which codings Faith decodes on the way - // back (spec: ENC): a value on the request, else one inherited from the agent's - // default headers, else the default Faith sends itself. Neither the request nor the - // agent advertising a value means nothing beneath Faith adds one now that it owns - // the codings, so Faith sends the default explicitly. - let request_accept_encoding = options.headers.as_ref().and_then(|headers| { - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) - .map(|(_, value)| value.clone()) - }); - let accept_encoding = AcceptEncoding::parse( - &request_accept_encoding - .clone() - .or_else(|| { - agent - .inner - .default_accept_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), - ); - if request_accept_encoding.is_none() && agent.inner.default_accept_encoding.is_none() { - request = request.header( - ACCEPT_ENCODING, - HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), - ); - } - - // The `priority` option is a hint, so a `Priority` header the caller wrote, or one - // among the agent's default headers, wins over the value derived from it - // (spec: REQ#request-priority). The agent's defaults are consulted here rather than - // left to reqwest: it fills a default header in only where the request carries none - // of that name, so setting the derived value would displace the agent's own. - if let Some(urgency) = options.priority - && !agent.inner.has_default_priority - && !options.headers.as_ref().is_some_and(|headers| { - headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) - }) { - request = request.header( - HeaderName::from_static(PRIORITY), - HeaderValue::from_static(urgency), - ); - } - - // The coding actually applied, which is `compress` only where there was a body to - // apply it to: the option does nothing on a request carrying none, so no - // `Content-Encoding` describes bytes that were never sent - // (spec:ENC#compressing-a-request-body). - let mut applied_coding = None; - - // Handle body: prefer streaming body over buffered body - if let Some(receiver_arc) = stream_receiver { - // Take the receiver from the Arc>> before anything below can bail - // out. Whatever happens next, this owns the receiving end: dropping it closes the - // channel, which is what tells the JS side pumping chunks in to stop. Leaving it in - // place on a refusal would strand that pump on a channel nobody will ever read, - // blocking once it filled and holding the process open. - let receiver = { - let mut guard = receiver_arc.lock().await; - guard.take() - }; - - // A body read from a `ReadableStream` has no length to advertise, which the fetch - // standard allows only over HTTP/2 and HTTP/3 (spec:REQ#streaming-a-request-body). - if !agent.inner.quirk_h1_request_streaming { - // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and - // can be refused without opening a connection to find out. - if parsed_url.scheme() != "https" { - return Err(FaithError::new( - FaithErrorKind::Network, - Some(format!( - "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", - parsed_url.as_str() - )), - )); - } - - // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the - // request hands the check to the layer that finds out: the connection is chosen, - // and an HTTP/1.x one is refused there before any of the body is written. - request = request.version(http::Version::HTTP_2); - } - - if let Some(receiver) = receiver { - // Convert the receiver into a stream for reqwest - let byte_stream = receiver.into_stream(); - request = request.body(match compress { - // Compressed as the chunks arrive, and chunked on the wire either way: - // a stream has no length to declare up front, compressed or not - // (spec:ENC#what-a-compressed-request-sends). - Some(coding) => { - applied_coding = Some(coding); - reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) - } - None => reqwest::Body::wrap_stream(byte_stream), - }); - } - } else if let Some(body) = &body { - request = request.body(match compress { - // The compressed bytes are what reqwest sizes `Content-Length` from, so the - // header counts what goes on the wire (spec:ENC#what-a-compressed-request-sends). - Some(coding) => { - applied_coding = Some(coding); - encoding::compress_buffer(body, coding) - .await - .map_err(|err| { - FaithError::new( - FaithErrorKind::Network, - Some(format!("could not compress the request body: {err}")), - ) - })? - } - None => body.to_vec(), - }); - } - - // One `Content-Encoding` naming the caller's codings then Faith's, in the order they - // were applied (spec:ENC#what-a-compressed-request-sends). - if let Some(coding) = applied_coding { - let value = - encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); - let value = HeaderValue::from_str(&value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - request = request.header(CONTENT_ENCODING, value); - } - - if let Some(dur) = options.timeout { - request = request.timeout(dur); - } - - agent - .inner - .stats - .requests_sent - .fetch_add(1, Ordering::Relaxed); - - // The origin every phase is measured from. - let started = Instant::now(); - - // Race the request with the abort signal if signal was provided - let response = if has_signal { - tokio::select! { - result = request.send() => result?, - _ = abort.recv() => { - return Err(FaithErrorKind::Aborted.into()); - } - } - } else { - request.send().await? - }; - - agent - .inner - .stats - .responses_received - .fetch_add(1, Ordering::Relaxed); - - let status_code = response.status(); - let empty = status_code == StatusCode::NO_CONTENT || is_head; - - let response_url = response.url().clone(); - let version = response.version(); - - // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the - // request's port to the advertised one, so the response URL's port reflects - // which endpoint answered rather than any redirect. Compare with ports - // normalised away, or every such request would report `redirected`. - // - // Only HTTP/3 responses can have been rewritten — reqwest routes - // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and - // the TCP fallback re-runs the untouched clone. Restricting the normalisation - // to those keeps exact comparison, and so port-only redirect detection, for - // every other response. - let redirected = - if agent.inner.h3_follow_advertised_port && version == http::Version::HTTP_3 { - let without_port = |url: &reqwest::Url| { - let mut url = url.clone(); - let _ = url.set_port(None); - url - }; - without_port(&parsed_url) != without_port(&response_url) - } else { - parsed_url != response_url - }; - - // Track connection for TCP stats (if we can get both local and remote addr). - // A connection the tracker has already seen is one the pool handed back, which is - // what `reused` reports (spec:RESP#request-timing). - let reused = if let Some(http_info) = response.extensions().get::() { - let local_addr = http_info.local_addr(); - let remote_addr = http_info.remote_addr(); - agent.inner.conn_tracker.track(local_addr, remote_addr) - } else { - false - }; - - // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has - // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a - // redirect chain marks the origin that actually answered rather than the one asked for. - agent.inner.mark_warm(&response_url); - - let peer = PeerInformation { - address: response.remote_addr(), - certificate: response - .extensions() - .get::() - .and_then(|info| info.peer_certificate()) - .map(|cert| cert.into()), - }; - - let mut headers = response.headers().clone(); - if options.credentials == CredentialsOption::Omit { - headers.remove("set-cookie"); - } - - // A cache hit is served without ever reaching the layer that stamps, so fall back to - // the moment the send resolved, which for a hit is the moment the cache answered. - let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); - let timing = RequestTiming { - headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, - body_ms: None, - reused, - next_hop_protocol: alpn_protocol_id(version, &response_url), - // Captured before a decoded body's `Content-Encoding` is stripped below, so the - // coding the response arrived under is reported either way. - content_encoding: headers - .get(CONTENT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned), - from_cache: headers - .get("x-cache") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), - }; - - // Decode only a body Faith negotiated the coding for; a bodyless response keeps its - // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). - let decode = if empty { - None - } else { - encoding::decision(&headers, &accept_encoding) + let body = match (stream_receiver, body) { + // A streaming body is taken from the sender's channel; the receiver is held behind a + // lock because JavaScript hands the same stream body object to one request only. + (Some(receiver), _) => match receiver.lock().await.take() { + Some(receiver) => RequestBody::Stream(Box::pin(receiver.into_stream())), + None => RequestBody::None, + }, + (None, Some(buffer)) => RequestBody::Bytes(Bytes::copy_from_slice(&buffer)), + (None, None) => RequestBody::None, }; - if decode.is_some() { - encoding::strip_decoded_headers(&mut headers); - } - let timing = Arc::new(TimingSlot::new(started, timing)); - // A response that cannot carry a body has nothing left to wait for. - if empty { - timing.ended(); - } + let abort = has_signal.then(|| async move { + let mut abort = abort; + let _ = abort.recv().await; + }); - Ok(FaithResponse::from(web_faith::response::Response { - body: if empty { - BodyHolder::none() - } else { - let http_response: http::Response<_> = response.into(); - BodyHolder::new( - Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), - version, - timing.clone(), - ) - }, - decode, - disturbed: Arc::new(AtomicBool::new(false)), - headers, - integrity: options.integrity, - peer: Arc::new(peer), - redirected, - stats: agent.inner.stats.clone(), - status_code, - timing, - trailers: Default::default(), - url: response_url, - version, - })) + request::send(&agent.inner, &url, options, body, abort) + .await + .map(FaithResponse::from) }) } diff --git a/crates/web-faith-napi/src/options.rs b/crates/web-faith-napi/src/options.rs index e690e3c..577e0a0 100644 --- a/crates/web-faith-napi/src/options.rs +++ b/crates/web-faith-napi/src/options.rs @@ -4,6 +4,8 @@ use http_cache_reqwest::CacheMode; use napi::bindgen_prelude::*; use napi_derive::napi; +use web_faith::request::{Credentials, RequestOptions}; + use crate::agent::Agent; /// The cache mode you want to use for the request. This may be any one of the following values: @@ -176,49 +178,33 @@ pub struct FaithOptionsAndBody { pub timeout: Option, } -#[derive(Clone, Debug, Default)] -pub(crate) struct FaithOptions { - pub(crate) cache: RequestCacheMode, - /// The `compress` option as given, resolved to a coding where the body is compressed. - pub(crate) compress: Option, - pub(crate) credentials: CredentialsOption, - pub(crate) headers: Option>, - pub(crate) integrity: Option, - pub(crate) method: Option, - /// The `Priority` header value derived from the `priority` option, if it maps to one. - pub(crate) priority: Option<&'static str>, - pub(crate) timeout: Option, -} - -impl FaithOptions { - pub(crate) fn extract(opts: FaithOptionsAndBody) -> (Self, Agent, Option>) { - let credentials = opts.credentials.unwrap_or_default(); - // Transform same-origin to include - let credentials = if credentials == CredentialsOption::SameOrigin { - CredentialsOption::Include - } else { - credentials - }; - - ( - Self { - cache: opts.cache.unwrap_or_default(), - compress: opts.compress, - credentials, - headers: opts.headers, - integrity: opts.integrity, - method: opts.method, - priority: priority_urgency(opts.priority.as_deref()), - timeout: opts.timeout.map(Into::into).map(Duration::from_millis), - }, - Agent::clone(&opts.agent), - opts.body.map(|either| match either { - Either3::A(s) => Arc::new(Buffer::from(s.as_bytes())), - Either3::B(b) => Arc::new(b), - Either3::C(u) => Arc::new(Buffer::from(u.as_ref())), - }), - ) - } +/// Read a `fetch()` call's options into the shape the client takes. +pub(crate) fn extract(opts: FaithOptionsAndBody) -> (RequestOptions, Agent, Option>) { + // `same-origin` means nothing without an origin to be same as, so it lands on `include`, + // which is what a server-side caller means by it. + let credentials = match opts.credentials.unwrap_or_default() { + CredentialsOption::Omit => Credentials::Omit, + CredentialsOption::Include | CredentialsOption::SameOrigin => Credentials::Include, + }; + + ( + RequestOptions { + cache: opts.cache.unwrap_or_default().into(), + compress: opts.compress, + credentials, + headers: opts.headers, + integrity: opts.integrity, + method: opts.method, + priority: priority_urgency(opts.priority.as_deref()), + timeout: opts.timeout.map(Into::into).map(Duration::from_millis), + }, + Agent::clone(&opts.agent), + opts.body.map(|either| match either { + Either3::A(s) => Arc::new(Buffer::from(s.as_bytes())), + Either3::B(b) => Arc::new(b), + Either3::C(u) => Arc::new(Buffer::from(u.as_ref())), + }), + ) } #[cfg(test)] diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 3e0acce..0a59238 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -15,7 +15,6 @@ use napi::{ use napi_derive::napi; use serde_json; -pub use web_faith::response::PeerInformation; use web_faith::{ body::{Body, drain_body_inner}, response::{FileDestination, FileProgress, FileWritten, Response, Trailers}, diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 951ac98..e0db4fb 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -21,6 +21,7 @@ pub mod agent; pub mod body; pub mod client; pub mod error; +pub mod request; pub mod response; pub mod retry; pub mod stats; diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs new file mode 100644 index 0000000..e697fb2 --- /dev/null +++ b/crates/web-faith/src/request.rs @@ -0,0 +1,463 @@ +//! Sending a request, and building the response that comes back. + +// spec:REQ spec:ENC spec:CANCEL + +use std::{ + future::Future, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use bytes::Bytes; +use futures::Stream; +use http_cache_reqwest::CacheMode; +use hyper_util::client::legacy::connect::HttpInfo; +use reqwest::{ + Method, StatusCode, + header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, + tls::TlsInfo, +}; +use tokio::sync::Mutex; +use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; + +use crate::{ + agent::Agent, + body::{Body, BodyHolder}, + error::{FaithError, FaithErrorKind}, + response::{PeerInformation, Response}, + timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, +}; + +/// Whether a request carries its credentials, and how far. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Credentials { + /// Strip credentials from the URL and send no cookies. + Omit, + /// Send them, which is what a server-side caller almost always means. + #[default] + Include, +} + +/// The methods the fetch standard normalises to upper case; any other method is sent as given. +// spec:REQ#method-and-headers +const NORMALISED_METHODS: [&str; 6] = ["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]; + +/// The header a request's priority is expressed in. +// spec:REQ#request-priority +pub const PRIORITY: &str = "priority"; + +/// A request body, as the caller has it. +pub enum RequestBody { + /// No body. + None, + /// A body already in hand, whose length can be declared up front. + Bytes(Bytes), + /// A body arriving in chunks, which goes out chunked because it has no length to declare. + Stream(Pin> + Send>>), +} + +/// What a request carries beyond its method, URL, and body. +#[derive(Clone, Debug, Default)] +pub struct RequestOptions { + pub cache: CacheMode, + /// A coding to compress the body in, named by its wire token. + pub compress: Option, + pub credentials: Credentials, + pub headers: Option>, + pub integrity: Option, + pub method: Option, + /// The `Priority` header value this request's priority derives, if it derives one. + pub priority: Option<&'static str>, + pub timeout: Option, +} + +/// Send a request on `agent`, and build the response it produces. +/// +/// `abort` is an optional future that, resolving first, cancels the request. +pub async fn send( + agent: &Agent, + url: &str, + options: RequestOptions, + body: RequestBody, + abort: Option>, +) -> Result { + let method = options.method.as_deref().unwrap_or("GET"); + // spec:REQ#method-and-headers + let method = NORMALISED_METHODS + .into_iter() + .find(|normalised| normalised.eq_ignore_ascii_case(method)) + .unwrap_or(method); + + let method = + Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; + let is_head = method == Method::HEAD; + + let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; + + // A `compress` naming no coding Faith can compress in is misuse whether or not the + // request turns out to carry a body, so it is refused before anything else looks at + // it (spec:ENC#compressing-a-request-body). + let compress = options + .compress + .as_deref() + .map(|value| { + Coding::from_option(value).ok_or_else(|| { + FaithError::new( + FaithErrorKind::InvalidCompression, + Some(format!( + "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" + )), + ) + }) + }) + .transpose()?; + + // Handle credentials based on credentials option + if options.credentials == Credentials::Omit { + // Remove credentials from URL if omit is specified + let _ = parsed_url.set_username(""); + let _ = parsed_url.set_password(None); + } + + // The stamp rides along in the request's extensions for the middleware to fill in; + // this side keeps a handle on it so the one measurement taken inside the stack is + // the one surfaced (spec:RESP#request-timing). + let headers_stamp = HeadersStamp::default(); + + let mut request = agent + .client + .as_ref() + .ok_or(FaithErrorKind::Closed)? + .request(method, parsed_url.clone()) + .with_extension(CacheMode::from(options.cache)) + .with_extension(headers_stamp.clone()); + + if let Some(headers) = &options.headers { + for (key, value) in headers { + // Skip Cookie header if credentials is omit + if options.credentials == Credentials::Omit && key.eq_ignore_ascii_case("cookie") { + continue; + } + + // Validate header name and value before adding to request + let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header name: {key}")), + ) + })?; + let header_value = HeaderValue::from_str(value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + + // Faith's coding is layered on top of what the caller declares, and reqwest's + // builder appends rather than replaces, so passing the caller's value through + // here would put a second `Content-Encoding` beside the joined one -- the same + // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is + // still validated above, then withheld and re-emitted once below. + if compress.is_some() && header_name == CONTENT_ENCODING { + continue; + } + + request = request.header(header_name, header_value); + } + } + + // What the caller says they handed over: their own `Content-Encoding`, else the + // agent's, per-request headers winning per name as they do generally (spec: REQ). + // Several lines are the one list, so they are joined as they are read. + let declared_content_encoding = compress.and_then(|_| { + let from_request = options.headers.as_ref().and_then(|headers| { + let declared = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(); + (!declared.is_empty()).then(|| declared.join(", ")) + }); + from_request.or_else(|| { + agent + .default_content_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + }); + + // The request's `Accept-Encoding` governs which codings Faith decodes on the way + // back (spec: ENC): a value on the request, else one inherited from the agent's + // default headers, else the default Faith sends itself. Neither the request nor the + // agent advertising a value means nothing beneath Faith adds one now that it owns + // the codings, so Faith sends the default explicitly. + let request_accept_encoding = options.headers.as_ref().and_then(|headers| { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) + .map(|(_, value)| value.clone()) + }); + let accept_encoding = AcceptEncoding::parse( + &request_accept_encoding + .clone() + .or_else(|| { + agent + .default_accept_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), + ); + if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { + request = request.header( + ACCEPT_ENCODING, + HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), + ); + } + + // The `priority` option is a hint, so a `Priority` header the caller wrote, or one + // among the agent's default headers, wins over the value derived from it + // (spec: REQ#request-priority). The agent's defaults are consulted here rather than + // left to reqwest: it fills a default header in only where the request carries none + // of that name, so setting the derived value would displace the agent's own. + if let Some(urgency) = options.priority + && !agent.has_default_priority + && !options.headers.as_ref().is_some_and(|headers| { + headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) + }) { + request = request.header( + HeaderName::from_static(PRIORITY), + HeaderValue::from_static(urgency), + ); + } + + // The coding actually applied, which is `compress` only where there was a body to + // apply it to: the option does nothing on a request carrying none, so no + // `Content-Encoding` describes bytes that were never sent + // (spec:ENC#compressing-a-request-body). + let mut applied_coding = None; + + // Handle body: prefer streaming body over buffered body + match body { + RequestBody::Stream(byte_stream) => { + // A body read from a stream has no length to advertise, which the fetch standard + // allows only over HTTP/2 and HTTP/3. + // spec:REQ#streaming-a-request-body + if !agent.quirk_h1_request_streaming { + // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and + // can be refused without opening a connection to find out. Returning here drops + // the stream, which is what tells whatever is feeding it to stop. + if parsed_url.scheme() != "https" { + return Err(FaithError::new( + FaithErrorKind::Network, + Some(format!( + "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", + parsed_url.as_str() + )), + )); + } + + // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the + // request hands the check to the layer that finds out: the connection is chosen, + // and an HTTP/1.x one is refused there before any of the body is written. + request = request.version(http::Version::HTTP_2); + } + + request = request.body(match compress { + // Compressed as the chunks arrive, and chunked on the wire either way: + // a stream has no length to declare up front. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) + } + None => reqwest::Body::wrap_stream(byte_stream), + }); + } + RequestBody::Bytes(bytes) => { + request = request.body(match compress { + // The compressed bytes are what reqwest sizes `Content-Length` from, so the + // header counts what goes on the wire. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + encoding::compress_buffer(&bytes, coding) + .await + .map_err(|err| { + FaithError::new( + FaithErrorKind::Network, + Some(format!("could not compress the request body: {err}")), + ) + })? + } + None => bytes.to_vec(), + }); + } + RequestBody::None => {} + } + + // One `Content-Encoding` naming the caller's codings then Faith's, in the order they + // were applied (spec:ENC#what-a-compressed-request-sends). + if let Some(coding) = applied_coding { + let value = encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); + let value = HeaderValue::from_str(&value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + request = request.header(CONTENT_ENCODING, value); + } + + if let Some(dur) = options.timeout { + request = request.timeout(dur); + } + + agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); + + // The origin every phase is measured from. + let started = Instant::now(); + + // A caller that can abort races the request against that signal; one that cannot just sends. + let response = match abort { + Some(abort) => { + tokio::select! { + result = request.send() => result?, + _ = abort => { + return Err(FaithErrorKind::Aborted.into()); + } + } + } + None => request.send().await?, + }; + + agent + .stats + .responses_received + .fetch_add(1, Ordering::Relaxed); + + let status_code = response.status(); + let empty = status_code == StatusCode::NO_CONTENT || is_head; + + let response_url = response.url().clone(); + let version = response.version(); + + // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the + // request's port to the advertised one, so the response URL's port reflects + // which endpoint answered rather than any redirect. Compare with ports + // normalised away, or every such request would report `redirected`. + // + // Only HTTP/3 responses can have been rewritten — reqwest routes + // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and + // the TCP fallback re-runs the untouched clone. Restricting the normalisation + // to those keeps exact comparison, and so port-only redirect detection, for + // every other response. + let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { + let without_port = |url: &reqwest::Url| { + let mut url = url.clone(); + let _ = url.set_port(None); + url + }; + without_port(&parsed_url) != without_port(&response_url) + } else { + parsed_url != response_url + }; + + // Track connection for TCP stats (if we can get both local and remote addr). + // A connection the tracker has already seen is one the pool handed back, which is + // what `reused` reports (spec:RESP#request-timing). + let reused = if let Some(http_info) = response.extensions().get::() { + let local_addr = http_info.local_addr(); + let remote_addr = http_info.remote_addr(); + agent.conn_tracker.track(local_addr, remote_addr) + } else { + false + }; + + // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has + // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a + // redirect chain marks the origin that actually answered rather than the one asked for. + agent.mark_warm(&response_url); + + let peer = PeerInformation { + address: response.remote_addr(), + certificate: response + .extensions() + .get::() + .and_then(|info| info.peer_certificate()) + .map(|cert| cert.into()), + }; + + let mut headers = response.headers().clone(); + if options.credentials == Credentials::Omit { + headers.remove("set-cookie"); + } + + // A cache hit is served without ever reaching the layer that stamps, so fall back to + // the moment the send resolved, which for a hit is the moment the cache answered. + let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); + let timing = RequestTiming { + headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, + body_ms: None, + reused, + next_hop_protocol: alpn_protocol_id(version, &response_url), + // Captured before a decoded body's `Content-Encoding` is stripped below, so the + // coding the response arrived under is reported either way. + content_encoding: headers + .get(CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + from_cache: headers + .get("x-cache") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), + }; + + // Decode only a body Faith negotiated the coding for; a bodyless response keeps its + // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). + let decode = if empty { + None + } else { + encoding::decision(&headers, &accept_encoding) + }; + if decode.is_some() { + encoding::strip_decoded_headers(&mut headers); + } + + let timing = Arc::new(TimingSlot::new(started, timing)); + // A response that cannot carry a body has nothing left to wait for. + if empty { + timing.ended(); + } + + Ok(Response { + body: if empty { + BodyHolder::none() + } else { + let http_response: http::Response<_> = response.into(); + BodyHolder::new( + Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), + version, + timing.clone(), + ) + }, + decode, + disturbed: Arc::new(AtomicBool::new(false)), + headers, + integrity: options.integrity, + peer: Arc::new(peer), + redirected, + stats: agent.stats.clone(), + status_code, + timing, + trailers: Default::default(), + url: response_url, + version, + }) +} From 100720e65b0bfb814d62c8a94a2f9c0cb2e14e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:42:23 +1200 Subject: [PATCH 41/61] S1: finish standing up the client The client's crate documentation now says what it holds, and two doc links that broke when the code moved are fixed, so `cargo doc --workspace` is clean again. Step 8 is done: web-faith holds the agent, the request path, the response and its reads, the body, timing, and retry machinery, and the client recipe. web-faith-napi is the binding it always should have been -- JS option shapes, AgentOptions validation, the napi classes wrapping the client's types, and napi's own promise, stream, and threadsafe-function machinery. --- .workhorse/plans/s1/plan.md | 29 ++++++++++++++++------------- crates/web-faith/src/agent.rs | 2 +- crates/web-faith/src/lib.rs | 6 +++--- crates/web-faith/src/response.rs | 2 +- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index d7e2905..3ddf9cd 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -10,9 +10,10 @@ ships as `@passcod/faith`. Then publish to crates.io. Target architecture is spe This is an 8-crate restructure of ~11,500 lines, not a single focused change. It is being built on this branch as one long-lived PR, at the user's direction, rather than split into a card breakdown. -The split itself (steps 0–7) is done: the workspace stands, and all six components plus the error -core are out. What remains (steps 8–13) is the larger half, and step 9 in particular is new API -design rather than relocation. +Steps 0–8 are done: the workspace stands, the six components are out, and the client owns the agent, +the request path, and the response. Every crate but `web-faith-napi` builds with no napi in its +graph. What remains (steps 9–14) is the caller-facing API, the feature wiring, publishing, and the +spec sweep — step 9 being new design rather than relocation. ## What the discovery turned up @@ -69,9 +70,10 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [x] **5. Extract `web-faith-dns`.** - [x] **6. Extract `web-faith-conn-tracker`** (Linux/macOS/Windows submodules). - [x] **7. Extract `web-faith-alt-svc`** — carry `HeadersStamp` (or take it generically); depend on `web-faith-dns`. -- [ ] **8. Stand up `web-faith`** — move agent/request/response/fetch/options here as a pure-Rust - client; component crates converted into it at the boundary. Reduce `web-faith-napi` to the binding - over `web-faith`. **In progress:** +- [x] **8. Stand up `web-faith`** — done. The client holds the agent, the request path, the response + and its reads, the body/timing/retry machinery, and the client recipe. `web-faith-napi` is the + binding: JS option shapes, `AgentOptions` validation, the napi classes wrapping the client's types, + and napi machinery (promises, streams, threadsafe functions). Nothing outside it carries napi. - [x] `body`, `timing` (measuring; the napi object stays behind), `retry` moved. - [x] The client-building machinery moved: `ClientRecipe`, `NodeEnvRecipe`, `HttpCacheRecipe`, `HttpCacheStore`, `H3UpgradeRecipe`, `ResolvedWindows`, `install_https_sink`, and a pure @@ -80,13 +82,14 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al per-request settings; `web-faith-napi`'s `Agent` is a napi class holding a handle on it, and keeps the ~440 lines of `AgentOptions` validation that produce a recipe. `prefetch_dns` and `preconnect` return futures the binding wraps, so a refusal happens before the future exists. - - [ ] `response.rs` (~966 lines, 40 napi refs) and `fetch.rs` (~447) are what is left. Both are - built around the napi response class, so they invert the way the agent did: a pure response - and a pure request path in the client, with the napi classes wrapping them. `fetch.rs` - already reaches the agent through `agent.inner`, which is the seam to pull on. - - [ ] `options.rs` (~246) stays largely JS-facing, holding the `AgentOptions` validation and the - per-request option shapes; the recipe it assembles becomes the builder's business in step 9. - - [ ] `stream_body.rs` and `async_task.rs` are napi machinery and stay where they are. + - [x] `response.rs` inverted: `web_faith::response::Response` holds the state and the reads, and + writing a body out takes a progress closure rather than a threadsafe function. + - [x] `fetch.rs` inverted: `web_faith::request::send` takes an agent, URL, options, body, and an + optional abort future; `fetch.rs` is 59 lines converting a `fetch()` call into those. + - **Left for step 9:** the `integrity` feature, which had to stop being optional. The client's read + path verifies unconditionally and gating the call sites would mean a build that quietly skips + verification, where the spec wants the feature to remove the API offering it. That API is + step 9's, so the feature returns with it. Same for the recipe structs' public fields. - [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 5c9b96b..29d7f55 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -110,7 +110,7 @@ pub struct Agent { pub quirk_h1_request_streaming: bool, /// The agent's default `Accept-Encoding`, if one was set among its default headers. /// `fetch` consults it to decide which codings to decode when a request adds none of - /// its own (see [`crate::encoding`]). + /// its own (see [`web_faith_encoding`]). pub default_accept_encoding: Option, /// The agent's default `Content-Encoding`, if one was set among its default headers. /// `fetch` consults it when the `compress` option layers a coding on top of what a diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index e0db4fb..d9dbe7e 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -11,9 +11,9 @@ //! //!
    //! -//! The client API is still being built out. What is here so far is the error type, the body and -//! timing machinery a response is built on, the retry layers in the request path, and the recipe -//! that builds the HTTP client itself. +//! The caller-facing API is still being shaped. Everything the client does is here -- building an +//! agent, sending a request, reading a response -- but it is reached through [`request::send`] and +//! the modules below rather than through the fetch-flavoured builder that will front it. //! //!
    diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index 7ea7d64..e996be0 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -406,7 +406,7 @@ impl Response { /// Underlying efficient response body fetcher. /// /// Unlike bytes() and co, this grabs all the chunks of the response but doesn't - /// copy them. Further processing is needed to obtain a Vec or whatever needed. + /// copy them. Further processing is needed to obtain a `Vec` or whatever is wanted. pub async fn gather(&self) -> Result, FaithError> { let Some(lock) = &self.body.body else { return Ok(Default::default()); From 5f443c3fd614357952e748b31343a2e8607f7bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:48:12 +1200 Subject: [PATCH 42/61] S1: write the client's own docs in the client's own terms The counters' listing goes: it spelled the fields in the Node surface's casing, and the type it returns already documents each one with its Rust name and its type, which no prose list can better. AgentStats is non-exhaustive. What an agent counts can grow, and adding a counter should not be a breaking change for anyone reading them. The rest is what the moves carried in and I left: spec references sitting in doc comments rather than beside the item, and JavaScript option paths naming settings the client holds under its own names. The agent's fields now say what they govern rather than which JS option they mirror. --- crates/web-faith-alt-svc/src/lib.rs | 2 +- crates/web-faith-cookies/src/lib.rs | 2 +- crates/web-faith-dns/src/lib.rs | 4 +- crates/web-faith/src/agent.rs | 84 +++++++++++++++-------------- crates/web-faith/src/stats.rs | 3 ++ 5 files changed, 50 insertions(+), 45 deletions(-) diff --git a/crates/web-faith-alt-svc/src/lib.rs b/crates/web-faith-alt-svc/src/lib.rs index 4b607f8..810be61 100644 --- a/crates/web-faith-alt-svc/src/lib.rs +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -476,7 +476,7 @@ impl AltSvcCache { /// /// Legacy (probe-less) routing: advertisements are acted on inline, so this /// consults `advertised` as well as `confirmed`. Only used when - /// `upgradeProbe` is off. + /// probing is off. pub fn should_use_h3(&self, url: &reqwest::Url) -> Option { self.confirmed_port(url) .or_else(|| self.probe_candidate(url)) diff --git a/crates/web-faith-cookies/src/lib.rs b/crates/web-faith-cookies/src/lib.rs index 5db5afc..ad2bcfe 100644 --- a/crates/web-faith-cookies/src/lib.rs +++ b/crates/web-faith-cookies/src/lib.rs @@ -142,7 +142,7 @@ impl FaithJar { /// Gate a cookie on the bis rules, then hand it to the classic storage model. /// /// Gating on the way in rather than filtering on the way out is what makes the caps bound real - /// memory, and what makes the rules apply the same to `addCookie` as to a `Set-Cookie` header. + /// memory, and what makes the rules apply the same to [`FaithJar::add_cookie_str`] as to a `Set-Cookie` header. fn store_one(&self, raw: RawCookie<'static>, url: &Url) { let Some(raw) = self.sanitise(raw, url) else { return; diff --git a/crates/web-faith-dns/src/lib.rs b/crates/web-faith-dns/src/lib.rs index a703fcb..33f10ca 100644 --- a/crates/web-faith-dns/src/lib.rs +++ b/crates/web-faith-dns/src/lib.rs @@ -477,7 +477,7 @@ struct Inner { https_sink: Mutex>>, } -/// A hickory resolver Faith owns, shared between reqwest's request path and `prefetchDns`. +/// A hickory resolver Faith owns, shared between a client's request path and [`FaithResolver::prefetch`]. #[derive(Clone)] pub struct FaithResolver { inner: Arc, @@ -822,7 +822,7 @@ impl FaithResolver { /// configured; what it re-reads is what the system supplies and what the network answers. /// /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, - /// which keeps it callable from `networkChanged`, which is not async. Nothing is rebuilt here + /// which keeps it callable from a network-change signal, which is not async. Nothing is rebuilt here /// either, so an agent that never resolves again pays nothing for the signal. // spec:NETCHG#what-the-signal-keeps // spec:NETCHG#reach-across-the-subsystems diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 29d7f55..524fdf4 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -67,24 +67,28 @@ pub struct Agent { /// client, so dropping it is what actually releases them. pub client: Option, /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A - /// `preconnect` warm-up sends its synthetic request here rather than through the middleware + /// a warm-up sends its synthetic request here rather than through the middleware /// stack, which bypasses the HTTP cache and the Alt-Svc layer (and so keeps the warm-up out of /// request accounting), while still pooling the connection foreground requests reuse. `None` - /// once the agent is closed. (spec:WARM) + /// once the agent is closed. + // spec:WARM pub raw_client: Option, - /// Faith's DNS resolver, shared with [`Self::client`] so `prefetchDns` warms the cache requests - /// read. `None` under the system resolver, where there is no such cache. (spec:WARM) + /// Faith's DNS resolver, shared with [`Self::client`] so [`Self::prefetch_dns`] warms the cache requests + /// read. `None` under the system resolver, where there is no such cache. + // spec:WARM pub dns_resolver: Option, /// Origins with a warm-up connection opened within the pool idle window, so a repeat - /// `preconnect` does no new work. Keyed by `scheme://host:port`; entries expire with the idle - /// timeout. (spec:WARM) + /// warm-up does no new work. Keyed by `scheme://host:port`; entries expire with the idle + /// timeout. + // spec:WARM pub warmed: MokaCache, - /// Single-flight claims for in-flight `preconnect` warm-ups, so concurrent calls for the same - /// origin do not open duplicate connections. (spec:WARM) + /// Single-flight claims for warm-ups in flight, so concurrent calls for the same + /// origin do not open duplicate connections. + // spec:WARM pub warming: MokaCache, - /// Bumped by `networkChanged`, so a warm-up that was in flight across the signal does not - /// record its origin as warm: its connection went into the pool that was just dropped - /// (spec:NETCHG#reach-across-the-subsystems). + /// Bumped by [`Self::network_changed`], so a warm-up that was in flight across the signal does not + /// record its origin as warm: its connection went into the pool that was just dropped. + // spec:NETCHG#reach-across-the-subsystems pub warm_generation: Arc, pub cookie_jar: Option>, pub stats: Arc, @@ -97,32 +101,34 @@ pub struct Agent { /// alive past close for up to the probe timeout. #[cfg(feature = "http3")] pub h3_prober: Option>, - /// Mirrors `http3.upgradeFollowAdvertisedPort`. Lives here because `fetch` needs - /// it to stop a rewritten port from being reported as a redirect. + /// Whether an upgrade may follow a port the origin advertised. A request needs it to stop a + /// rewritten port from being reported as a redirect. pub h3_follow_advertised_port: bool, - /// Mirrors `http3.upgradeEnabled`. A warm-up needs it to route the way a foreground request - /// would: with the upgrade machinery off, nothing upgrades, whatever the caches hold. - /// (spec:WARM#preconnect) + /// Whether the upgrade machinery is on at all. A warm-up needs it to route the way a foreground + /// request would: with it off, nothing upgrades, whatever the caches hold. + // spec:WARM#preconnect #[cfg(feature = "http3")] pub h3_upgrade_enabled: bool, - /// Mirrors `quirks.h1RequestStreaming`. `fetch` consults it to decide whether a streaming - /// request body may go out over HTTP/1.x (spec:QUIRK#http-1-x-request-body-streaming). + /// Whether a streaming request body may go out over HTTP/1.x, which the fetch standard + /// otherwise reserves to HTTP/2 and HTTP/3.. + // spec:QUIRK#http-1-x-request-body-streaming pub quirk_h1_request_streaming: bool, /// The agent's default `Accept-Encoding`, if one was set among its default headers. - /// `fetch` consults it to decide which codings to decode when a request adds none of + /// which decides the codings a response is decoded under when a request adds none of /// its own (see [`web_faith_encoding`]). pub default_accept_encoding: Option, /// The agent's default `Content-Encoding`, if one was set among its default headers. - /// `fetch` consults it when the `compress` option layers a coding on top of what a - /// request already declares, since setting the joined value on the request would - /// otherwise displace this default rather than build on it (spec:ENC). + /// A request layers its own coding on top of this rather than + /// displacing it. + // spec:ENC pub default_content_encoding: Option, - /// Whether a `Priority` header sits among the agent's default headers. `fetch` consults - /// it so that default wins over the header the `priority` option would derive. + /// Whether a `Priority` header sits among the agent's default headers. That default wins over the header a + /// request's priority would derive. pub has_default_priority: bool, - /// How to build this agent's clients, so `networkChanged` can build them again - /// (spec:NETCHG). Shared rather than cloned per agent clone: every clone builds the same - /// client from the same recipe, and `fetch` clones the agent per request. + /// How to build this agent's clients, so [`Self::network_changed`] can build them again + ///. Shared rather than cloned per agent clone: every clone builds the same + /// client from the same recipe, and a request takes a handle per send. + // spec:NETCHG pub recipe: Arc, } @@ -195,12 +201,12 @@ impl Agent { /// /// Requests already in flight run to completion. Any new request on a closed /// agent throws a `Closed` error. Calling `close()` more than once is a - /// no-op. The cookie store, if any, remains readable via `getCookie`. + /// no-op. The cookie store, if any, remains readable through [`Self::cookie_header`]. pub fn close(&mut self) { // Dropping the client releases the reqwest connection pool and the // Hickory resolver task; the alt-svc cache goes with it. The raw client // shares that pool and the resolver, so it goes too, and both are what a - // later `preconnect`/`prefetchDns` checks to throw the closed-agent error. + // later warm-up checks to refuse with the closed-agent error. self.client = None; self.raw_client = None; self.dns_resolver = None; @@ -231,8 +237,7 @@ impl Agent { /// Requests already in flight are not interrupted and run to completion on the connections /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a /// closed agent does nothing, and calling it repeatedly is harmless. - /// - /// spec:NETCHG + // spec:NETCHG pub fn network_changed(&mut self) { // A closed agent has already released all of this. if self.client.is_none() { @@ -304,7 +309,7 @@ impl Agent { /// - the cookie store is disabled /// /// - the cookie does not parse /// - a `__Host-` or `__Secure-` name prefix is not satisfied - /// - the cookie is larger than `cookies.maxSize` + /// - the cookie is larger than the jar's size limit pub fn add_cookie(&self, url: &Url, cookie: &str) { let Some(jar) = &self.cookie_jar else { return; @@ -328,12 +333,7 @@ impl Agent { .and_then(|val| val.to_str().ok().map(ToOwned::to_owned)) } - /// Returns statistics gathered by this agent: - /// - /// - `requestsSent` - /// - `responsesReceived` - /// - `bodiesStarted` - /// - `bodiesFinished` + /// The counters this agent has gathered, as they stand. pub fn stats(&self) -> AgentStats { self.stats.snapshot() } @@ -342,7 +342,7 @@ impl Agent { /// /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: /// this makes it possible to track indicators over time to find the retransmission rate, for - /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other + /// example. The lost-packet count and delivery rate are only available on Linux. Some other /// fields might also be missing depending on platform support; and no forward guarantees are made /// on field availability. If the platform isn't supported at all, this will always return empty. pub fn connections(&self) -> Vec { @@ -355,7 +355,8 @@ impl Agent { /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). /// The list is empty until the resolver has been used, because it reads its configuration on - /// first use, and empty for an agent using the system resolver. (spec:OBS#resolvers) + /// first use, and empty for an agent using the system resolver. + // spec:OBS#resolvers pub fn resolvers(&self) -> Vec { self.dns_resolver .as_ref() @@ -364,10 +365,11 @@ impl Agent { } /// Note that a request reached this origin, so it holds a connection the pool keeps idle for - /// the idle window and a `preconnect` for it has no new work to do (spec:WARM). + /// the idle window and a `preconnect` for it has no new work to do. /// /// Called for foreground requests as well as warm-ups, because the criterion is about the /// origin holding an idle pooled connection, not about how it came to hold one. + // spec:WARM pub fn mark_warm(&self, url: &Url) { self.warmed.insert(origin_key(url), ()); } diff --git a/crates/web-faith/src/stats.rs b/crates/web-faith/src/stats.rs index a055142..760dd6f 100644 --- a/crates/web-faith/src/stats.rs +++ b/crates/web-faith/src/stats.rs @@ -27,7 +27,10 @@ impl InnerAgentStats { } /// A reading of an agent's counters, taken at one moment. +/// +/// Non-exhaustive: what an agent counts can grow, and a new counter should not be a breaking change. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] pub struct AgentStats { pub requests_sent: u64, pub responses_received: u64, From 95464f98f2c1f4bcd60a9540f1d18799f44aad35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:55:19 +1200 Subject: [PATCH 43/61] S1: integrity is always built, by design rather than by expedience There is no good reason for a caller to turn digest verification off, and the component is small enough that leaving it out saves nothing worth measuring. What a feature would buy is a build that quietly skips a check the caller asked for. So integrity has no feature, and the specs say why: RUST no longer claims every component has one, and RSAPI no longer promises that `integrity()` disappears. --- .workhorse/plans/s1/plan.md | 11 +++++------ .workhorse/specs/rust/client-api.md | 2 +- .workhorse/specs/rust/overview.md | 5 ++++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 3ddf9cd..64c846b 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -86,16 +86,15 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al writing a body out takes a progress closure rather than a threadsafe function. - [x] `fetch.rs` inverted: `web_faith::request::send` takes an agent, URL, options, body, and an optional abort future; `fetch.rs` is 59 lines converting a `fetch()` call into those. - - **Left for step 9:** the `integrity` feature, which had to stop being optional. The client's read - path verifies unconditionally and gating the call sites would mean a build that quietly skips - verification, where the spec wants the feature to remove the API offering it. That API is - step 9's, so the feature returns with it. Same for the recipe structs' public fields. + - **Left for step 9:** the recipe structs' public fields, which the builder should own the + assembly of. - [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, `http_body::Body` response + `Into`, feature-gated API surface, Tokio, drop-cancels. -- [ ] **10. Feature wiring** — one default-on feature per component on `web-faith`; disabling one - drops the dep, the code, and the API surface it gates (compile error at the call site, not a no-op). +- [ ] **10. Feature wiring** — a default-on feature per component a build can do without; disabling + one drops the dep, the code, and the API surface it gates (compile error at the call site, not a + no-op). Integrity is deliberately not among them: it is always built, per [RUST](../../specs/rust/overview.md). - [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. - [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md index c3e16b4..ed4e689 100644 --- a/.workhorse/specs/rust/client-api.md +++ b/.workhorse/specs/rust/client-api.md @@ -84,7 +84,7 @@ Holding the failure is what lets a target be given as a string: an unparseable o ## What a disabled component removes -A component's Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that component present: no `integrity()` without integrity, no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). +A component's Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that component present: no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). Code written against a component that is not built fails to compile rather than compiling into a call that does nothing, so a build reports what it does not carry at the point the caller asks for it. ## Errors and cancellation diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md index 20d63a5..92a9b12 100644 --- a/.workhorse/specs/rust/overview.md +++ b/.workhorse/specs/rust/overview.md @@ -49,9 +49,12 @@ What the two subsystems do is specified in [QUIC](../http3/transport.md) and [TL ## Choosing what is built Cargo features are how a subsystem is included or left out, so a build that has no use for a piece does not carry it. -Each of the six components has a feature on `web-faith` named for it, and every one of them is on by default, so a caller who reaches for the crate without thinking about features gets the whole client. +A component a build can do without has a feature on `web-faith` named for it, and every one of them is on by default, so a caller who reaches for the crate without thinking about features gets the whole client. Turning a component's feature off drops the dependency and the code that reaches for it, and the client continues to work without it; it also removes the parts of the API that only mean something with the component present, as in [RSAPI](client-api.md). +Integrity has no such feature and is always built. +It is small enough that leaving it out saves nothing worth measuring, and a caller who asks for a digest to be checked is owed the check rather than a build that quietly skips it. + A feature decides what is compiled in rather than what is switched on at run time, and the two need not agree: the cookies feature is on by default while the jar itself stays off until an agent asks for it (see [COOK](../agent/cookies.md)). Features are the whole of the swapping mechanism: a caller chooses among the implementations Faith builds rather than supplying one. From 5b6963196d26350a2d96dc57372b56a6567aa1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Mon, 24 Aug 2026 16:58:32 +1200 Subject: [PATCH 44/61] S1: update 9 files --- Cargo.lock | 10 +--- Cargo.toml | 1 - crates/web-faith-integrity/Cargo.toml | 14 ----- crates/web-faith-napi/Cargo.toml | 1 - crates/web-faith/Cargo.toml | 2 +- crates/web-faith/src/error.rs | 14 ----- .../src/lib.rs => web-faith/src/integrity.rs} | 57 +++++++------------ crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/response.rs | 2 +- 9 files changed, 24 insertions(+), 78 deletions(-) delete mode 100644 crates/web-faith-integrity/Cargo.toml rename crates/{web-faith-integrity/src/lib.rs => web-faith/src/integrity.rs} (76%) diff --git a/Cargo.lock b/Cargo.lock index 57195da..93c90ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2809,6 +2809,7 @@ dependencies = [ "moka", "reqwest", "reqwest-middleware", + "ssri", "stream_shared", "strum", "tokio", @@ -2818,7 +2819,6 @@ dependencies = [ "web-faith-cookies", "web-faith-dns", "web-faith-encoding", - "web-faith-integrity", ] [[package]] @@ -2884,13 +2884,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "web-faith-integrity" -version = "0.7.0" -dependencies = [ - "ssri", -] - [[package]] name = "web-faith-napi" version = "0.7.0" @@ -2934,7 +2927,6 @@ dependencies = [ "web-faith-cookies", "web-faith-dns", "web-faith-encoding", - "web-faith-integrity", "windows", ] diff --git a/Cargo.toml b/Cargo.toml index e29a517..7d004e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,7 +78,6 @@ web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-trac web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } web-faith-encoding = { version = "0.7.0", path = "crates/web-faith-encoding" } -web-faith-integrity = { version = "0.7.0", path = "crates/web-faith-integrity" } netlink-packet-core = "0.7.0" netlink-packet-sock-diag = { version = "0.4.2", features = ["rich_nlas"] } netlink-sys = "0.8.7" diff --git a/crates/web-faith-integrity/Cargo.toml b/crates/web-faith-integrity/Cargo.toml deleted file mode 100644 index 862b657..0000000 --- a/crates/web-faith-integrity/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "web-faith-integrity" -description = "Subresource Integrity parsing and verification" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true -# Flipped on once the release tooling is in place. -publish = false - -[dependencies] -ssri.workspace = true diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 8a65b5a..952d699 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -51,7 +51,6 @@ web-faith-conn-tracker.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-encoding.workspace = true -web-faith-integrity.workspace = true [target.'cfg(target_os = "linux")'.dependencies] netlink-packet-core.workspace = true diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index abe45f5..fac9e68 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -23,6 +23,7 @@ moka.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true stream_shared.workspace = true +ssri.workspace = true strum.workspace = true tokio.workspace = true url.workspace = true @@ -31,7 +32,6 @@ web-faith-encoding.workspace = true web-faith-cookies = { workspace = true, features = ["reqwest"] } web-faith-dns = { workspace = true, features = ["reqwest"] } web-faith-alt-svc = { workspace = true, optional = true } -web-faith-integrity.workspace = true [features] default = ["http3"] diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs index 03af51e..f79c910 100644 --- a/crates/web-faith/src/error.rs +++ b/crates/web-faith/src/error.rs @@ -150,20 +150,6 @@ impl From for FaithError { } } -/// A component crate names its own errors; they become the client's as they cross into it, which is -/// what keeps the code a caller sees the same whichever layer failed. -impl From for FaithError { - fn from(err: web_faith_integrity::IntegrityError) -> Self { - use web_faith_integrity::IntegrityError as E; - match err { - E::Invalid(_) => { - FaithError::new(FaithErrorKind::InvalidIntegrity, Some(err.to_string())) - } - E::Mismatch => FaithErrorKind::IntegrityMismatch.into(), - } - } -} - impl From for FaithError { fn from(err: reqwest_middleware::Error) -> Self { match err { diff --git a/crates/web-faith-integrity/src/lib.rs b/crates/web-faith/src/integrity.rs similarity index 76% rename from crates/web-faith-integrity/src/lib.rs rename to crates/web-faith/src/integrity.rs index 22f4d14..59a56e9 100644 --- a/crates/web-faith-integrity/src/lib.rs +++ b/crates/web-faith/src/integrity.rs @@ -5,33 +5,15 @@ //! ignored rather than honoured. Verification comes in two shapes: [`verify_integrity`] for a body //! already in memory, and [`integrity_checker`] with [`finish_integrity`] for one being read as it //! arrives. +//! +//! This is always built. It is small enough that leaving it out saves nothing worth measuring, and a +//! caller who asks for a digest to be checked is owed the check. -use std::{ - error::Error, - fmt::{self, Display}, -}; +// spec:SRI use ssri::{Integrity, IntegrityChecker}; -/// A resource failing its integrity check, or an integrity value that could not be read. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum IntegrityError { - /// The integrity value is not one this can parse. - Invalid(String), - /// The resource matched none of the digests it was expected to. - Mismatch, -} - -impl Display for IntegrityError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Invalid(detail) => write!(f, "failed to parse integrity value: {detail}"), - Self::Mismatch => write!(f, "resource integrity check failed"), - } - } -} - -impl Error for IntegrityError {} +use crate::error::{FaithError, FaithErrorKind}; /// Algorithm names are compared case-insensitively, the digest itself being base64 and so not ours /// to touch. @@ -49,24 +31,27 @@ fn normalize_integrity(integrity: &str) -> String { .join(" ") } -fn parse_integrity(integrity: &str) -> Result { +fn parse_integrity(integrity: &str) -> Result { let normalized = normalize_integrity(integrity); - normalized - .parse() - .map_err(|e| IntegrityError::Invalid(format!("{e}"))) + normalized.parse().map_err(|e| { + FaithError::new( + FaithErrorKind::InvalidIntegrity, + Some(format!("failed to parse integrity value: {e}")), + ) + }) } /// Verify data that is already in hand. /// /// An absent or blank value has nothing to verify and so passes. -pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), IntegrityError> { +pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), FaithError> { if integrity.trim().is_empty() { return Ok(()); } parse_integrity(integrity)? .check(data) - .map_err(|_| IntegrityError::Mismatch)?; + .map_err(|_| FaithError::from(FaithErrorKind::IntegrityMismatch))?; Ok(()) } @@ -74,12 +59,10 @@ pub fn verify_integrity(data: &[u8], integrity: &str) -> Result<(), IntegrityErr /// Build a streaming integrity checker for a body read that hashes as it goes. /// /// `None` when there is nothing to verify (no value, or an empty one), matching -/// [`verify_integrity`]. A malformed value is rejected up front with [`IntegrityError::Invalid`], +/// [`verify_integrity`]. A malformed value is rejected up front with an invalid-integrity error, /// before any of the body is touched. Feed each chunk to the returned checker with /// [`IntegrityChecker::input`] and finish with [`finish_integrity`]. -pub fn integrity_checker( - integrity: Option<&str>, -) -> Result, IntegrityError> { +pub fn integrity_checker(integrity: Option<&str>) -> Result, FaithError> { let Some(integrity) = integrity else { return Ok(None); }; @@ -91,11 +74,11 @@ pub fn integrity_checker( } /// Finish a streaming integrity check. -pub fn finish_integrity(checker: IntegrityChecker) -> Result<(), IntegrityError> { +pub fn finish_integrity(checker: IntegrityChecker) -> Result<(), FaithError> { checker .result() .map(|_| ()) - .map_err(|_| IntegrityError::Mismatch) + .map_err(|_| FaithError::from(FaithErrorKind::IntegrityMismatch)) } #[cfg(test)] @@ -129,7 +112,7 @@ mod tests { let integrity = "sha256-wronghashvalue"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), IntegrityError::Mismatch)); + assert_eq!(result.unwrap_err().kind, FaithErrorKind::IntegrityMismatch); } #[test] @@ -145,7 +128,7 @@ mod tests { let integrity = "sha256-wronghash1 sha256-wronghash2"; let result = verify_integrity(data, integrity); assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), IntegrityError::Mismatch)); + assert_eq!(result.unwrap_err().kind, FaithErrorKind::IntegrityMismatch); } #[test] diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index d9dbe7e..8d29dbd 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -21,6 +21,7 @@ pub mod agent; pub mod body; pub mod client; pub mod error; +pub mod integrity; pub mod request; pub mod response; pub mod retry; diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index e996be0..d4e98c2 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -30,7 +30,7 @@ use crate::{ timing::TimingSlot, }; -use web_faith_integrity::{finish_integrity, integrity_checker, verify_integrity}; +use crate::integrity::{finish_integrity, integrity_checker, verify_integrity}; /// What is known about the peer that sent a response. /// From 5f2a8abb9e20950309a5bab7362339aad6441308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:08:21 +1200 Subject: [PATCH 45/61] S1: fold SRI into the client, and untangle component from crate from feature Subresource Integrity is a module of `web-faith` rather than a crate beneath it, and its functions report the client's own error, so the boundary conversion goes with the boundary. The nine tests come along. The specs had been treating component, crate, and feature as one thing: RUST promised a feature per component crate, and RSAPI promised the API a component gated. They are three axes that need not line up. A feature now names a capability, may or may not drop a dependency when turned off, and a subsystem the client is not built without carries none at all. The reasoning about why integrity is always built goes with it. A spec says what the system is, and the system is one where SRI is part of the client. --- .workhorse/plans/s1/plan.md | 19 ++++++++++--------- .workhorse/specs/rust/client-api.md | 6 +++--- .workhorse/specs/rust/overview.md | 17 +++++++++-------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 64c846b..6623321 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -1,7 +1,7 @@ # S1 — Expose a Rust API and publish to crates.io Restructure the single `faith` cdylib into a Cargo workspace: a browser-shaped Rust client -`web-faith`, six standalone component crates beneath it, and a thin `web-faith-napi` binding that +`web-faith`, five standalone component crates beneath it, and a thin `web-faith-napi` binding that ships as `@passcod/faith`. Then publish to crates.io. Target architecture is specified in [RUST](../../specs/rust/overview.md) and [RSAPI](../../specs/rust/client-api.md). @@ -10,7 +10,7 @@ ships as `@passcod/faith`. Then publish to crates.io. Target architecture is spe This is an 8-crate restructure of ~11,500 lines, not a single focused change. It is being built on this branch as one long-lived PR, at the user's direction, rather than split into a card breakdown. -Steps 0–8 are done: the workspace stands, the six components are out, and the client owns the agent, +Steps 0–8 are done: the workspace stands, the five components are out, and the client owns the agent, the request path, and the response. Every crate but `web-faith-napi` builds with no napi in its graph. What remains (steps 9–14) is the caller-facing API, the feature wiring, publishing, and the spec sweep — step 9 being new design rather than relocation. @@ -41,13 +41,12 @@ Facts that shape the order and difficulty: ## Target crate family -- `web-faith` — client: agent, request/response, `fetch`, layering. Depends on the six components. +- `web-faith` — client: agent, request/response, `fetch`, layering, SRI. Depends on the five components. - `web-faith-cookies` — the jar ([COOK](../../specs/agent/cookies.md)). - `web-faith-dns` — resolver, cache, discovery ladder, HTTPS record, Happy Eyeballs ([DNS](../../specs/agent/dns.md)). - `web-faith-conn-tracker` — live per-connection stats from the OS ([OBS](../../specs/agent/observability.md)). - `web-faith-alt-svc` — Alt-Svc store + HTTP/3 upgrade/probing ([H3UP](../../specs/http3/upgrade.md), [PROBE](../../specs/http3/probing.md)); may depend on `web-faith-dns`. - `web-faith-encoding` — request/response content coding ([ENC](../../specs/fetch/content-encoding.md)). -- `web-faith-integrity` — SRI parse + verify ([SRI](../../specs/fetch/integrity.md)). - `web-faith-napi` — the only crate with napi types; ships as `@passcod/faith`. QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring alternative), not crates. @@ -64,7 +63,8 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al source ([ERR](../../specs/errors/errors.md)). Decide where the shared error core lives (likely in `web-faith`, with components naming their own error types that the client converts — per [RUST](../../specs/rust/overview.md) "A component crate stands alone"). -- [x] **2. Extract `web-faith-integrity`** — own error type, own docs, `cargo test -p web-faith-integrity` with no JS runtime. +- [x] **2. SRI** — extracted as `web-faith-integrity`, then folded back into `web-faith` as a module: + too small for standing alone to buy a caller anything, and not something a build should switch off. - [x] **3. Extract `web-faith-encoding`** — decouple from `crate::body::DynStream` (take a generic/`bytes` stream). - [x] **4. Extract `web-faith-cookies`** — `url::Url`; reqwest `CookieStore` behind a feature. - [x] **5. Extract `web-faith-dns`.** @@ -92,15 +92,16 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, `http_body::Body` response + `Into`, feature-gated API surface, Tokio, drop-cancels. -- [ ] **10. Feature wiring** — a default-on feature per component a build can do without; disabling - one drops the dep, the code, and the API surface it gates (compile error at the call site, not a - no-op). Integrity is deliberately not among them: it is always built, per [RUST](../../specs/rust/overview.md). +- [ ] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling + one drops the code and the API surface it gates (compile error at the call site, not a no-op), and + the dependency too where the capability is a crate. Component, crate, and feature are three axes + and need not line up: a crate can be non-optional, and a feature need not map to a crate. - [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. - [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, MSRV 1.96 declared in every published crate and exercised in CI alongside stable, independent versioning from `1.0.0`. Measure CI cost before adding jobs (see project memory). -- [ ] **13. First publish** to crates.io: the six components, then `web-faith`; `@passcod/faith` +- [ ] **13. First publish** to crates.io: the five components, then `web-faith`; `@passcod/faith` continues from npm via `web-faith-napi`. - [ ] **14. The both-surfaces spec sweep**, as the closing pass over the tree — deliberately last, once the Rust API is settled and its names are known. See the section below for the site-by-site diff --git a/.workhorse/specs/rust/client-api.md b/.workhorse/specs/rust/client-api.md index ed4e689..8e205ee 100644 --- a/.workhorse/specs/rust/client-api.md +++ b/.workhorse/specs/rust/client-api.md @@ -82,10 +82,10 @@ A conversion that fails is held until the builder resolves and surfaces there: a Either way it reports the error naming the offender — `InvalidHeader`, `InvalidMethod`, or `InvalidUrl` for a target that does not parse — as [REQ](../fetch/request.md) and [ERR](../errors/errors.md) require. Holding the failure is what lets a target be given as a string: an unparseable one is reported where the request is resolved rather than by a call that cannot fail. -## What a disabled component removes +## What a disabled feature removes -A component's Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that component present: no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). -Code written against a component that is not built fails to compile rather than compiling into a call that does nothing, so a build reports what it does not carry at the point the caller asks for it. +A Cargo feature governs the API as much as the build, so turning one off takes away the methods that only mean something with that capability present: no cookie jar handle without cookies, and the same for request compression and cache mode (see [RUST](overview.md)). +Code written against a capability that is not built fails to compile rather than compiling into a call that does nothing, so a build reports what it does not carry at the point the caller asks for it. ## Errors and cancellation diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md index 92a9b12..335d06d 100644 --- a/.workhorse/specs/rust/overview.md +++ b/.workhorse/specs/rust/overview.md @@ -4,7 +4,7 @@ id: RUST # The Rust distribution -Faith's network stack is published to crates.io as a family of crates, with `web-faith` as the client a Rust caller reaches for and six component crates beneath it that each stand on their own. +Faith's network stack is published to crates.io as a family of crates, with `web-faith` as the client a Rust caller reaches for and five component crates beneath it that each stand on their own. The Node.js native module is built from the same workspace, so the two surfaces are two faces of one implementation rather than two implementations: a behaviour specified anywhere else in these specs holds on both unless that spec says otherwise. The Rust API surface itself is specified in [RSAPI](client-api.md). @@ -14,14 +14,15 @@ The Rust API surface itself is specified in [RSAPI](client-api.md). It owns the agent, the request and response types, and the `fetch` entry point, and it draws on the component crates for the subsystems beneath it. The name carries a `web-` prefix because the bare `faith` name on crates.io belongs to an unrelated project, and the prefix reads as the browser-shaped client the crate is. -Six component crates are published alongside it, each one useful to a caller who wants that piece without the client above it: +Five component crates are published alongside it, each one useful to a caller who wants that piece without the client above it: - `web-faith-cookies` is the cookie jar, with the storage, matching, and eviction rules in [COOK](../agent/cookies.md). - `web-faith-dns` is the resolver, its cache, the discovery ladder, the `HTTPS` record query, and Happy Eyeballs, as in [DNS](../agent/dns.md). - `web-faith-conn-tracker` reads live per-connection statistics from the operating system, which is the `connections()` view in [OBS](../agent/observability.md); the cumulative counters and the resolver listing that spec also covers belong to `web-faith` and `web-faith-dns` respectively. - `web-faith-alt-svc` is the Alt-Svc store and the HTTP/3 upgrade machinery, as in [H3UP](../http3/upgrade.md) and [PROBE](../http3/probing.md). - `web-faith-encoding` is content coding for request and response bodies, as in [ENC](../fetch/content-encoding.md). -- `web-faith-integrity` is Subresource Integrity parsing and verification, as in [SRI](../fetch/integrity.md). + +Subresource Integrity parsing and verification is part of `web-faith` itself (see [SRI](../fetch/integrity.md)). A component crate depends on another component crate where the subsystems genuinely compose, so `web-faith-alt-svc` draws on `web-faith-dns` for the resolution its probes need. None of them depends on `web-faith`, which is what makes each one usable on its own. @@ -48,12 +49,12 @@ What the two subsystems do is specified in [QUIC](../http3/transport.md) and [TL ## Choosing what is built -Cargo features are how a subsystem is included or left out, so a build that has no use for a piece does not carry it. -A component a build can do without has a feature on `web-faith` named for it, and every one of them is on by default, so a caller who reaches for the crate without thinking about features gets the whole client. -Turning a component's feature off drops the dependency and the code that reaches for it, and the client continues to work without it; it also removes the parts of the API that only mean something with the component present, as in [RSAPI](client-api.md). +Cargo features on `web-faith` are how a subsystem is left out of a build that has no use for it. +Features are on by default, so a caller who reaches for the crate without thinking about them gets the whole client. +Turning one off drops the code behind it and the client continues to work without it; the parts of the API that only mean something with that subsystem present go with it, as in [RSAPI](client-api.md). -Integrity has no such feature and is always built. -It is small enough that leaving it out saves nothing worth measuring, and a caller who asks for a digest to be checked is owed the check rather than a build that quietly skips it. +A feature names a capability rather than a crate. +Turning one off may drop a dependency along with the code or may simply compile less of `web-faith` itself, and a subsystem the client is not built without carries no feature at all. A feature decides what is compiled in rather than what is switched on at run time, and the two need not agree: the cookies feature is on by default while the jar itself stays off until an agent asks for it (see [COOK](../agent/cookies.md)). From 267508e74e7f287be7f0339d1c1d37fb3162e731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:12:03 +1200 Subject: [PATCH 46/61] S1: cut the feature explainer from the Rust distribution spec --- .workhorse/specs/rust/overview.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.workhorse/specs/rust/overview.md b/.workhorse/specs/rust/overview.md index 335d06d..1bd0612 100644 --- a/.workhorse/specs/rust/overview.md +++ b/.workhorse/specs/rust/overview.md @@ -53,9 +53,6 @@ Cargo features on `web-faith` are how a subsystem is left out of a build that ha Features are on by default, so a caller who reaches for the crate without thinking about them gets the whole client. Turning one off drops the code behind it and the client continues to work without it; the parts of the API that only mean something with that subsystem present go with it, as in [RSAPI](client-api.md). -A feature names a capability rather than a crate. -Turning one off may drop a dependency along with the code or may simply compile less of `web-faith` itself, and a subsystem the client is not built without carries no feature at all. - A feature decides what is compiled in rather than what is switched on at run time, and the two need not agree: the cookies feature is on by default while the jar itself stays off until an agent asks for it (see [COOK](../agent/cookies.md)). Features are the whole of the swapping mechanism: a caller chooses among the implementations Faith builds rather than supplying one. From eba1c385bcb5fc2b9665b766d6bf484b5ff17d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:36:31 +1200 Subject: [PATCH 47/61] S1: the client owns the user agent it sends The default `User-Agent` names the client and the version of reqwest a request actually goes out on, so the client is what should compose it. web-faith gains a build script to read that version from the lock file, and USER_AGENT with it. The Node surface's constant now reads from the client's, so the value on the wire has one source. It is unchanged: `Faith/0.7.0 reqwest/0.13.4`. RSAPI has the Rust surface export the string but not the versions inside it, a Rust caller having its own package metadata to read those from, so only USER_AGENT is public here. --- crates/web-faith-napi/src/agent.rs | 7 +---- crates/web-faith/build.rs | 42 ++++++++++++++++++++++++++++++ crates/web-faith/src/lib.rs | 17 ++++++++++++ 3 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 crates/web-faith/build.rs diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index e75f8bd..a96fa8e 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -60,12 +60,7 @@ pub const REQWEST_VERSION: &str = env!("REQWEST_VERSION"); /// }); /// ``` #[napi] -pub const USER_AGENT: &str = concat!( - "Faith/", - env!("CARGO_PKG_VERSION"), - " reqwest/", - env!("REQWEST_VERSION") -); +pub const USER_AGENT: &str = web_faith::USER_AGENT; /// Whether this host can bind the IPv6 wildcard (`[::]`). /// diff --git a/crates/web-faith/build.rs b/crates/web-faith/build.rs new file mode 100644 index 0000000..884e37b --- /dev/null +++ b/crates/web-faith/build.rs @@ -0,0 +1,42 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + // The default user agent names the version of reqwest the request actually goes out on, which + // only the lock file knows. + let lock_path = find_cargo_lock().expect("Cargo.lock not found in any ancestor directory"); + let reqwest_version = extract_reqwest_version(&lock_path).unwrap(); + println!("cargo:rustc-env=REQWEST_VERSION={}", reqwest_version); + println!("cargo:rerun-if-changed={}", lock_path.display()); +} + +/// The lock file lives at the workspace root, which is an ancestor of this crate. +fn find_cargo_lock() -> Option { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR")?); + manifest_dir + .ancestors() + .map(|dir| dir.join("Cargo.lock")) + .find(|lock| lock.is_file()) +} + +fn extract_reqwest_version(lock_path: &PathBuf) -> Option { + let cargo_lock = fs::read_to_string(lock_path).ok()?; + + // Find the reqwest package entry in Cargo.lock + for line in cargo_lock.lines() { + if line.starts_with("name = \"reqwest\"") { + // Look for the version line in the next few lines + let mut lines_iter = cargo_lock.lines().skip_while(|l| l != &line); + lines_iter.next(); // Skip the name line + + for next_line in lines_iter.take(5) { + if let Some(version) = next_line.trim().strip_prefix("version = \"") { + if let Some(version) = version.strip_suffix("\"") { + return Some(version.to_string()); + } + } + } + } + } + + None +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 8d29dbd..98b8de6 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -29,4 +29,21 @@ pub mod stats; pub mod timing; pub mod warm_up; +/// The `User-Agent` a request carries when nothing overrides it. +/// +/// Prepend your own product token to it rather than replacing it, so a server still sees which +/// client is calling: +/// +/// ``` +/// # use web_faith::USER_AGENT; +/// let ua = format!("YourApp/1.2.3 {USER_AGENT}"); +/// assert!(ua.ends_with(USER_AGENT)); +/// ``` +pub const USER_AGENT: &str = concat!( + "Faith/", + env!("CARGO_PKG_VERSION"), + " reqwest/", + env!("REQWEST_VERSION") +); + pub use error::{FaithError, FaithErrorKind, error_codes}; From 0bb369ccafaa9502b96336cc87605ba97aae5c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:25:26 +1200 Subject: [PATCH 48/61] S1: give the client's response its own reading API A Rust caller reads a response through the response rather than through the binding: status, status_text, ok, headers, url, redirected, version, peer, and body_used as accessors, and bytes, text, json, body_stream, discard, and the file write as the reads over the body, which is the set RSAPI names. json is generic over what it deserialises into, as a Rust caller expects, rather than handing back a parsed document. body_stream reports the chunk error as the client's own error instead of the string the pipeline carries internally, and can be called more than once: each call hands back the same shared stream. The napi methods now delegate to these, so the reading rules -- a second read fails, integrity is verified once the body is in hand, discarding settles the trailers -- are stated once. --- Cargo.lock | 2 + crates/web-faith-napi/src/response.rs | 75 +++---------- crates/web-faith/Cargo.toml | 2 + crates/web-faith/src/response.rs | 150 +++++++++++++++++++++++++- 4 files changed, 169 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93c90ad..8f114b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2809,6 +2809,8 @@ dependencies = [ "moka", "reqwest", "reqwest-middleware", + "serde", + "serde_json", "ssri", "stream_shared", "strum", diff --git a/crates/web-faith-napi/src/response.rs b/crates/web-faith-napi/src/response.rs index 0a59238..c4df436 100644 --- a/crates/web-faith-napi/src/response.rs +++ b/crates/web-faith-napi/src/response.rs @@ -13,12 +13,8 @@ use napi::{ threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, }; use napi_derive::napi; -use serde_json; -use web_faith::{ - body::{Body, drain_body_inner}, - response::{FileDestination, FileProgress, FileWritten, Response, Trailers}, -}; +use web_faith::response::{FileDestination, FileProgress, FileWritten, Response, Trailers}; use crate::{ async_task::{Value, faith_promise}, @@ -123,7 +119,7 @@ impl FaithResponse { #[napi] pub fn headers(&self) -> Vec<(String, String)> { self.inner - .headers + .headers() .iter() .filter_map(|(name, value)| { value @@ -138,7 +134,7 @@ impl FaithResponse { /// response was successful (status in the range 200-299) or not. #[napi(getter)] pub fn ok(&self) -> bool { - self.inner.status_code.is_success() + self.inner.ok() } /// Custom to Faith. @@ -176,7 +172,7 @@ impl FaithResponse { /// `false` on those responses. #[napi(getter)] pub fn redirected(&self) -> bool { - self.inner.redirected + self.inner.redirected() } /// The `status` read-only property of the `Response` interface contains the HTTP status codes of the @@ -185,7 +181,7 @@ impl FaithResponse { /// A value is `0` is returned for a response whose `type` is `opaque`, `opaqueredirect`, or `error`. #[napi(getter)] pub fn status(&self) -> u16 { - self.inner.status_code.as_u16() + self.inner.status().as_u16() } /// The `statusText` read-only property of the `Response` interface contains the status message @@ -198,10 +194,7 @@ impl FaithResponse { /// string. #[napi(getter)] pub fn status_text(&self) -> &'static str { - self.inner - .status_code - .canonical_reason() - .unwrap_or_default() + self.inner.status_text() } /// The `type` read-only property of the `Response` interface contains the type of the response. The @@ -217,7 +210,7 @@ impl FaithResponse { /// value of the `url` property will be the final URL obtained after any redirects. #[napi(getter)] pub fn url(&self) -> String { - self.inner.url.to_string() + self.inner.url().to_string() } /// The `version` read-only property of the `Response` interface contains the HTTP version of the @@ -303,32 +296,9 @@ impl FaithResponse { /// Returns a promise that resolves when the body has been fully discarded. #[napi] pub fn discard<'env>(&self, env: &'env Env) -> Result, napi::Error> { - let body = self.inner.body.body.clone(); - let drained_flag = self.inner.body.drained.clone(); - let is_multiplexed = self.inner.body.is_multiplexed(); - let trailers = self.inner.trailers.clone(); - let timing = self.inner.timing.clone(); + let this = Clone::clone(self); faith_promise(env, async move { - if let Some(arc) = body { - if is_multiplexed { - // Multiplexed connections don't need draining for reuse; dropping - // the body cancels the stream and frees its resources right away - // instead of waiting for garbage collection. - let mut guard = arc.lock().await; - *guard = Body::Consumed; - } else { - drain_body_inner(arc).await; - } - } - drained_flag.store(true, Ordering::SeqCst); - // Discarding the body discards its trailers: on a multiplexed connection the - // stream was cancelled before any could arrive, and draining an HTTP/1 body - // here bypasses the stream that would have collected them. Settling this as - // "none" rather than leaving it pending is the point -- a caller who discarded - // the body and then awaited trailers used to wait forever. - trailers.ended(); - // Discarding is one of the ways a body finishes (spec:RESP#request-timing). - timing.ended(); + this.inner.discard().await; Ok(()) }) } @@ -340,10 +310,10 @@ impl FaithResponse { #[napi] pub fn bytes<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); - faith_promise(env, async move { - this.inner.check_stream_disturbed()?; - this.inner.gather_contiguous().await.map(Buffer::from) - }) + faith_promise( + env, + async move { this.inner.bytes().await.map(Buffer::from) }, + ) } /// The `text()` method of the `Response` interface takes a `Response` stream and reads it to @@ -353,12 +323,7 @@ impl FaithResponse { #[napi] pub fn text<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); - faith_promise(env, async move { - this.inner.check_stream_disturbed()?; - let bytes = this.inner.gather_contiguous().await?; - Ok(String::from_utf8(bytes) - .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())) - }) + faith_promise(env, async move { this.inner.text().await }) } /// The `json()` method of the `Response` interface takes a `Response` stream and reads it to @@ -374,13 +339,7 @@ impl FaithResponse { #[napi] pub fn json<'env>(&self, env: &'env Env) -> Result, napi::Error> { let this = Clone::clone(self); - faith_promise(env, async move { - this.inner.check_stream_disturbed()?; - let bytes = this.inner.gather_contiguous().await?; - let value = serde_json::from_slice(&bytes) - .map_err(|e| FaithError::new(FaithErrorKind::JsonParse, Some(e.to_string())))?; - Ok(Value(value)) - }) + faith_promise(env, async move { this.inner.json().await.map(Value) }) } /// Custom to Faith. @@ -452,7 +411,7 @@ impl FaithResponse { // spec:RESP#request-timing #[napi] pub async fn timing(&self) -> TimingBreakdown { - self.inner.timing.settled().await.into() + self.inner.timing().await.into() } /// The `trailers()` read-only property of the `Response` interface returns a promise that @@ -474,7 +433,7 @@ impl FaithResponse { /// This is an async fn as an internal implementation detail and the wrapper makes it a property. #[napi] pub async fn trailers(&self) -> Option> { - match self.inner.trailers.settled().await { + match self.inner.trailers().await { // NotYet cannot come back from `settled`, which is what it waits on. Trailers::NotYet | Trailers::None => None, Trailers::Some(headers) => Some( diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index fac9e68..e48ec58 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -22,6 +22,8 @@ hyper-util.workspace = true moka.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true +serde.workspace = true +serde_json.workspace = true stream_shared.workspace = true ssri.workspace = true strum.workspace = true diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index d4e98c2..e5128bd 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -15,16 +15,17 @@ use std::{ }; use bytes::Bytes; -use futures::{StreamExt, stream}; +use futures::{Stream, StreamExt, TryStreamExt, stream}; use http::header::{CONTENT_LENGTH, HeaderMap}; use http_body_util::BodyStream; use reqwest::{StatusCode, Url, Version}; +use serde::de::DeserializeOwned; use stream_shared::SharedStream; use tokio::{io::AsyncWriteExt, sync::watch}; use web_faith_encoding::{Coding, decode_stream}; use crate::{ - body::{Body, BodyHolder, DynStream}, + body::{Body, BodyHolder, DynStream, drain_body_inner}, error::{FaithError, FaithErrorKind}, stats::InnerAgentStats, timing::TimingSlot, @@ -294,6 +295,151 @@ pub struct Response { } impl Response { + /// The response's status. + pub fn status(&self) -> StatusCode { + self.status_code + } + + /// The canonical reason phrase for the status, or empty for a code with no well-known one. + /// + /// Always the canonical phrase: HTTP/1 lets a server send its own, which is not surfaced here, + /// and HTTP/2 and HTTP/3 carry none at all. + pub fn status_text(&self) -> &'static str { + self.status_code.canonical_reason().unwrap_or_default() + } + + /// Whether the status is in the 2xx range. + pub fn ok(&self) -> bool { + self.status_code.is_success() + } + + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// The URL the response came from, which is the last one after any redirects. + pub fn url(&self) -> &Url { + &self.url + } + + /// Whether a redirect was followed to reach this response. + pub fn redirected(&self) -> bool { + self.redirected + } + + /// The HTTP version the response arrived over. + pub fn version(&self) -> Version { + self.version + } + + /// What is known of the peer that sent the response. + pub fn peer(&self) -> &PeerInformation { + &self.peer + } + + /// Whether the body has been read, or handed out as a stream. + pub fn body_used(&self) -> bool { + self.disturbed.load(Ordering::SeqCst) + } + + /// Read the whole body. + /// + /// Reading consumes the body, so a second read fails with the already-disturbed error, as the + /// fetch standard has it rather than the owned-response model other Rust clients use. An + /// `integrity` value on the request is verified here, once the whole body is in hand. + // spec:BODY + pub async fn bytes(&self) -> Result, FaithError> { + self.check_stream_disturbed()?; + self.gather_contiguous().await + } + + /// Read the whole body as text. + /// + /// Always decoded as UTF-8, with invalid sequences replaced by U+FFFD rather than failing, which + /// is what the fetch standard calls for. + pub async fn text(&self) -> Result { + let bytes = self.bytes().await?; + Ok(String::from_utf8(bytes) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) + } + + /// Read the whole body and deserialise it from JSON. + /// + /// The body is read into memory before it is parsed, which can cost twice its size; read + /// [`Self::body_stream`] instead where that matters. + pub async fn json(&self) -> Result { + let bytes = self.bytes().await?; + serde_json::from_slice(&bytes) + .map_err(|err| FaithError::new(FaithErrorKind::JsonParse, Some(err.to_string()))) + } + + /// Take the body as a stream of chunks, decoded under whichever coding was negotiated. + /// + /// `None` for a response that cannot carry a body. Unlike the collecting reads, this can be + /// called more than once: each call hands back the same shared stream rather than a second one. + /// A body already being consumed elsewhere reports the already-disturbed error rather than + /// waiting for the other reader to finish. + // spec:BODY + pub fn body_stream( + &self, + ) -> Result> + use<>>, FaithError> { + // The body counts as disturbed from here, though the stream itself stays re-readable. + let _ = self.check_stream_disturbed(); + + let Some(lock) = &self.body.body else { + return Ok(None); + }; + + let mut body = lock + .try_lock() + .map_err(|_| FaithError::from(FaithErrorKind::ResponseAlreadyDisturbed))?; + let stream = self.ensure_stream(&mut body, self.body.drained.clone())?; + + Ok(Some(stream.map_err(|err| { + FaithError::new(FaithErrorKind::BodyStream, Some(err)) + }))) + } + + /// Give up on the body, releasing the connection back to the pool. + /// + /// Worth doing when the body is not wanted: left unread, the connection may be held open until + /// the response is dropped. An HTTP/1 body is read and thrown away so the connection can be + /// reused; a multiplexed one is dropped instead, cancelling the stream without touching the + /// connection it shared. + /// + /// This settles the trailers as none rather than leaving them pending: on a multiplexed + /// connection the stream was cancelled before any could arrive, and draining an HTTP/1 body + /// here bypasses the stream that would have collected them. A caller who discards the body and + /// then awaits trailers would otherwise wait for something that can no longer come. + // spec:BODY spec:TRL spec:RESP#request-timing + pub async fn discard(&self) { + if let Some(arc) = self.body.body.clone() { + if self.body.is_multiplexed() { + *arc.lock().await = Body::Consumed; + } else { + drain_body_inner(arc).await; + } + } + self.body.drained.store(true, Ordering::SeqCst); + self.trailers.ended(); + // Discarding is one of the ways a body finishes. + self.timing.ended(); + } + + /// The timing of the request that produced this response, once its body has ended. + // spec:RESP#request-timing + pub async fn timing(&self) -> crate::timing::RequestTiming { + self.timing.settled().await + } + + /// The trailers, once the body has ended. + /// + /// A body that is never read never ends, so this waits indefinitely by design; see [`Trailers`]. + // spec:TRL + pub async fn trailers(&self) -> Trailers { + self.trailers.settled().await + } + pub fn check_stream_disturbed(&self) -> Result<(), FaithError> { if self.disturbed.swap(true, Ordering::SeqCst) { Err(FaithErrorKind::ResponseAlreadyDisturbed.into()) From c96538d0c264dd6d4f3888483ac9226d2388cc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:31:06 +1200 Subject: [PATCH 49/61] S1: a response feeds the ecosystem as an http::Response The body implements http_body::Body over the chunks the response already streams, reporting the client's own error, and a response converts into an http::Response carrying its status, version, and headers. That is what lets a Faith response reach code written against the wider ecosystem without a shim. The conversion is fallible rather than infallible: taking the body can find it already being consumed, and a caller should hear that rather than get a body that yields nothing. A response that cannot carry a body converts to an empty one. The test asserts what a consumer observes -- draining yields no bytes -- after a first attempt asserted a size hint this does not implement, which the default hint is entitled not to give. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/web-faith/Cargo.toml | 1 + crates/web-faith/src/response.rs | 104 +++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8f114b9..1e7a2e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2802,6 +2802,7 @@ dependencies = [ "bytes", "futures", "http", + "http-body", "http-body-util", "http-cache-reqwest", "hyper", diff --git a/Cargo.toml b/Cargo.toml index 7d004e1..5cb07ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ hickory-resolver = { version = "0.26", features = [ ] } moka = { version = "0.12", features = ["sync"] } http = "1.4.0" +http-body = "1.0.1" http-body-util = "0.1.3" hyper = "1.8.1" http-cache-reqwest = { version = "1.0.0-alpha.6", features = [ diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index e48ec58..fb2e987 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -15,6 +15,7 @@ async-trait.workspace = true bytes.workspace = true futures.workspace = true http.workspace = true +http-body.workspace = true http-body-util.workspace = true http-cache-reqwest.workspace = true hyper.workspace = true diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index e5128bd..387485b 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -3,6 +3,7 @@ // spec:RESP spec:TRL spec:BODY use std::{ + fmt::Debug, hint::unreachable_unchecked, mem::replace, net::SocketAddr, @@ -11,6 +12,7 @@ use std::{ Arc, atomic::{AtomicBool, Ordering}, }, + task::{Context, Poll}, time::{Duration, Instant}, }; @@ -195,6 +197,50 @@ mod tests { } } + /// A response that cannot carry a body converts to an `http::Response` with an empty one, + /// carrying its status, version, and headers across. + #[test] + fn a_bodyless_response_converts_to_an_http_response() { + let mut headers = HeaderMap::new(); + headers.insert("x-test", "yes".parse().expect("a valid header value")); + + let response = Response { + body: BodyHolder::none(), + decode: None, + disturbed: Arc::new(AtomicBool::new(false)), + headers, + integrity: None, + peer: Arc::new(PeerInformation { + address: None, + certificate: None, + }), + redirected: false, + stats: Arc::new(InnerAgentStats::default()), + status_code: StatusCode::NO_CONTENT, + timing: Arc::new(TimingSlot::new( + Instant::now(), + crate::timing::RequestTiming::default(), + )), + trailers: Arc::new(TrailersSlot::default()), + url: Url::parse("https://example.com/").expect("a valid url"), + version: Version::HTTP_2, + }; + + let http = response.into_http().expect("an undisturbed body converts"); + assert_eq!(http.status(), StatusCode::NO_CONTENT); + assert_eq!(http.version(), Version::HTTP_2); + assert_eq!( + http.headers().get("x-test").map(|v| v.as_bytes()), + Some(&b"yes"[..]) + ); + + // Draining it yields nothing, which is what a consumer of the body actually observes. + let collected = + futures::executor::block_on(http_body_util::BodyExt::collect(http.into_body())) + .expect("an empty body collects"); + assert!(collected.to_bytes().is_empty()); + } + /// Waiting for trailers parks until the body settles the question, rather than polling /// for it. /// @@ -708,3 +754,61 @@ impl Response { }) } } + +/// A [`Response`]'s body, as an [`http_body::Body`]. +/// +/// This is what a response hands to code written against the wider ecosystem: a tower service, a +/// hyper client, anything that takes a body rather than Faith's own reads. +pub struct ResponseBody { + chunks: Pin> + Send>>, +} + +impl Debug for ResponseBody { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ResponseBody").finish_non_exhaustive() + } +} + +impl http_body::Body for ResponseBody { + type Data = Bytes; + type Error = FaithError; + + fn poll_frame( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + self.chunks + .as_mut() + .poll_next(cx) + .map(|chunk| chunk.map(|chunk| chunk.map(http_body::Frame::data))) + } +} + +impl Response { + /// Take the response as an [`http::Response`], so it feeds code written against the ecosystem + /// rather than against Faith. + /// + /// Fails where taking the body would: a body already being consumed elsewhere reports the + /// already-disturbed error. A response that cannot carry a body yields an empty one. + pub fn into_http(self) -> Result, FaithError> { + let chunks: Pin> + Send>> = + match self.body_stream()? { + Some(stream) => Box::pin(stream), + None => Box::pin(stream::empty()), + }; + + let mut response = http::Response::new(ResponseBody { chunks }); + *response.status_mut() = self.status_code; + *response.version_mut() = self.version; + *response.headers_mut() = self.headers.clone(); + Ok(response) + } +} + +impl TryFrom for http::Response { + type Error = FaithError; + + fn try_from(response: Response) -> Result { + response.into_http() + } +} From e8a9733a4ae5e565d4ea95709f6e179566aaf95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:31:22 +1200 Subject: [PATCH 50/61] S1: record step 9's progress and what the agent builder needs --- .workhorse/plans/s1/plan.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 6623321..2fa4c3a 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -88,10 +88,23 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al optional abort future; `fetch.rs` is 59 lines converting a `fetch()` call into those. - **Left for step 9:** the recipe structs' public fields, which the builder should own the assembly of. -- [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md): - `Agent`/`Agent::builder()`, cheap-clone shared agent, `agent.fetch(target) -> IntoFuture` builder - (`#[must_use]`), `Request`/`Request::new`/`try_clone`, layering rules, `http`/`url`/`bytes` types, - `http_body::Body` response + `Into`, feature-gated API surface, Tokio, drop-cancels. +- [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md). + **In progress:** + - [x] `USER_AGENT` is the client's, composed from its own version and reqwest's, which `web-faith` + now reads in a build script of its own. The binding's constant reads from it. + - [x] Reading a response: the accessors and `bytes`/`text`/`json`/`body_stream`/`discard`/ + `to_file`/`timing`/`trailers` on `web_faith::response::Response`, with the napi methods + delegating. `json` is generic over what it deserialises into. + - [x] `http_body::Body` for the body, and `Response::into_http`. Fallible, since taking the body + can find it already being consumed. + - [ ] **`Agent::builder()` — the big remaining piece.** ~13 option groups as nested builders, and + the ~440 lines of validation currently in `web-faith-napi` that turn `AgentOptions` into a + recipe. Move that logic rather than rewrite it: both surfaces must land on the same defaults, + and a second implementation is how they drift. `Agent::new()` then follows, and the recipe + structs' fields close up behind the builder. + - [ ] `Request`, `Request::new`, `try_clone`, and the fetch builder over `IntoFuture`, with the + layering rules (outermost wins; headers merge by name). + - [ ] Setters taking anything convertible, holding a failed conversion until the builder resolves. - [ ] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling one drops the code and the API surface it gates (compile error at the call site, not a no-op), and the dependency too where the capability is a crate. Component, crate, and feature are three axes From 03740ebf10d13e849d39e90809c14d99b023e4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:32:30 +1200 Subject: [PATCH 51/61] S1: the client validates its own options, so Agent::new() exists The ~440 lines that turn options into the recipe an agent's clients are built from move to web-faith, along with the option groups they read, the IPv6 wildcard probe, and the flow-control window reconciliation with its tests. The binding maps its JavaScript options object onto the client's shapes and calls from_options. Moved rather than reimplemented on purpose: both surfaces have to land on the same defaults, and a second implementation is how they drift. What the binding still owns is the JavaScript vocabulary -- a union for cookies, either spelling of a PEM, strings where Rust has enums -- and entering a tokio runtime, which it needs because a napi callback can run outside one. Agent::new() follows, so a Rust caller no longer has to assemble a recipe by hand. --- crates/web-faith-napi/src/agent.rs | 755 +++++---------------------- crates/web-faith-napi/src/options.rs | 3 - crates/web-faith/src/agent.rs | 502 +++++++++++++++++- crates/web-faith/src/lib.rs | 1 + crates/web-faith/src/options.rs | 750 ++++++++++++++++++++++++++ 5 files changed, 1383 insertions(+), 628 deletions(-) create mode 100644 crates/web-faith/src/options.rs diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index a96fa8e..66ad63b 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -1,46 +1,24 @@ -use std::{ - fmt::Debug, - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, UdpSocket}, - str::FromStr as _, - sync::Arc, - time::Duration, -}; +use std::{fmt::Debug, str::FromStr as _, time::Duration}; use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; -use http_cache_reqwest::{ - CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, -}; use napi::{Either, Env, bindgen_prelude::Buffer}; use napi_derive::napi; -use reqwest::{ - Certificate, Identity, Url, - header::{HeaderMap, HeaderName, HeaderValue}, -}; +use reqwest::Url; -use web_faith::agent::AgentSettings; -#[cfg(feature = "http3")] -use web_faith::client::H3UpgradeRecipe; -use web_faith::client::{ - ClientRecipe, DEFAULT_CONNECTION_WINDOW, DEFAULT_STREAM_WINDOW, HttpCacheRecipe, - HttpCacheStore, NodeEnvRecipe, RedirectPolicy, ResolvedWindows, -}; -#[cfg(feature = "http3")] -use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig}; -use web_faith_dns::{ - DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, -}; +use http_cache_reqwest::CacheMode; +use web_faith::client::RedirectPolicy; +use web_faith::options; use web_faith_cookies::{ CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, - FaithJar, }; use crate::{ async_task::faith_promise, conn_tracker::{ConnectionInfo, connections_for_napi}, - error::{FaithError, FaithErrorExt, FaithErrorKind}, - options::{PRIORITY, RequestCacheMode}, + error::{FaithError, FaithErrorExt}, + options::RequestCacheMode, }; #[napi] @@ -62,21 +40,6 @@ pub const REQWEST_VERSION: &str = env!("REQWEST_VERSION"); #[napi] pub const USER_AGENT: &str = web_faith::USER_AGENT; -/// Whether this host can bind the IPv6 wildcard (`[::]`). -/// -/// This is tested using the exact operation reqwest performs when creating the QUIC -/// endpoint with no explicit local address, so it predicts whether the default -/// QUIC bind will succeed. The result is memoised for the life of the process; while -/// IPv6 bindability can in principle change at runtime, this is considered an -/// acceptable tradeoff for performance and simplicity. -fn ipv6_wildcard_bindable() -> bool { - use std::sync::OnceLock; - static BINDABLE: OnceLock = OnceLock::new(); - *BINDABLE.get_or_init(|| { - UdpSocket::bind(SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0)).is_ok() - }) -} - #[napi(string_enum)] #[derive(Debug, Clone, Copy)] pub enum CacheStore { @@ -589,23 +552,6 @@ pub struct AgentFlowControlOptions { pub connection_window: Option, } -/// Reconcile one protocol's windows: its own setting wins over the common one, which wins over the -/// default (spec:FLOW#per-protocol-windows). -pub(crate) fn resolve_windows( - common: Option<&AgentFlowControlOptions>, - protocol_stream: Option, - protocol_connection: Option, -) -> ResolvedWindows { - ResolvedWindows { - stream: protocol_stream - .or_else(|| common.and_then(|c| c.stream_window)) - .unwrap_or(DEFAULT_STREAM_WINDOW), - connection: protocol_connection - .or_else(|| common.and_then(|c| c.connection_window)) - .unwrap_or(DEFAULT_CONNECTION_WINDOW), - } -} - /// Settings related to the connection pool. This is a nested object. #[napi(object)] #[derive(Debug, Clone, Default)] @@ -897,473 +843,11 @@ impl Agent { } pub fn with_options(options: AgentOptions) -> Result { - // Wrap in tokio runtime context for HTTP/3 endpoint initialization. - // Quinn's Endpoint::client() requires a tokio runtime to be available. - within_runtime_if_available(|| Self::with_options_inner(options)) - } - - fn with_options_inner(options: AgentOptions) -> Result { - // Destructured rather than read field by field so that a new option cannot be added - // without the compiler pointing here, where every option is turned into the recipe the - // agent's clients are built from (spec:NETCHG). - let AgentOptions { - cache, - cookies, - dns, - flow_control, - headers, - http2, - // Every use of the HTTP/3 options sits behind the feature. - #[cfg_attr(not(feature = "http3"), allow(unused_variables))] - http3, - local_address, - pool, - quirks, - redirect, - timeout, - tls, - user_agent, - } = options; - - let quirk_h1_request_streaming = quirks - .and_then(|quirks| quirks.h1_request_streaming) - .unwrap_or(false); - - // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts - // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the - // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and - // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such - // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. - let local_address = match &local_address { - Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { - FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - ) - })?), - None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), - None => None, - }; - - // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) - // The jar is installed on the client by the recipe, so it survives a rebuild - // (spec:NETCHG#what-the-signal-keeps). - let cookie_jar = match cookies.as_ref() { - None | Some(Either::A(false)) => None, - Some(Either::A(true)) => Some(Arc::new(FaithJar::new(CookieLimits::default()))), - Some(Either::B(cookies)) => Some(Arc::new(FaithJar::new(cookies.into()))), - }; - - let dns = dns.unwrap_or_default(); - let dns_system = dns.system.unwrap_or(false); - // Naming servers and asking for the system resolver at once is a contradiction rather than - // a preference, since the system resolver is not Faith's to point at listed servers - // (spec:DNS#system-resolver). - if dns_system - && dns - .servers - .as_ref() - .is_some_and(|servers| !servers.is_empty()) - { - return Err(FaithError::new( - FaithErrorKind::Config, - Some("dns.servers cannot be combined with dns.system".to_string()), - )); - } - // Parsed whichever resolver is in use: overrides take effect under the system resolver - // too (spec:DNS#overrides), and an unparseable address is a construction error either - // way (spec:AGENT#construction). - let dns_overrides = dns - .overrides - .unwrap_or_default() - .into_iter() - .map(|DnsOverride { domain, addresses }| { - let addresses = addresses - .into_iter() - .map(|addr| match SocketAddr::from_str(&addr) { - Ok(addr) => Ok(addr), - Err(err) => match IpAddr::from_str(&addr) { - Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), - Ok(IpAddr::V6(ip)) => { - Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) - } - Err(_) => Err(FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - )), - }, - }) - .collect::, FaithError>>()?; - Ok((domain, addresses)) - }) - .collect::, FaithError>>()?; - - // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so - // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), - // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and - // order each resolver is reached by (spec:DNS#transports). The system resolver - // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there - // and `prefetchDns` resolves as a no-op (spec:WARM). - let dns_resolver = if dns_system { - None - } else { - // These settings configure Faith's own resolver only, so they are read on the path that - // builds one rather than validated under the system resolver that ignores them. - let mut servers = Vec::new(); - for url in dns.servers.unwrap_or_default() { - servers.push(ServerSpec::parse(&url).map_err(|message| { - FaithError::new(FaithErrorKind::AddressParse, Some(message)) - })?); - } - Some(FaithResolver::new(ResolverSettings { - servers, - timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), - ndots: dns.ndots.map(|n| n as usize), - search_domains: parse_domains(dns.search_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, - hosts_file: dns.hosts_file, - exempt_domains: parse_domains(dns.exempt_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? - .unwrap_or_default(), - serve_stale: dns.serve_stale.unwrap_or(true), - max_stale: dns - .max_stale - .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), - })) - }; - - let mut default_accept_encoding = None; - let mut default_content_encoding = None; - let mut has_default_priority = false; - let mut default_headers = None; - if let Some(headers) = headers - && !headers.is_empty() - { - let map = HeaderMap::from_iter(headers.into_iter().filter_map( - |Header { - name, - value, - sensitive, - }| { - let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { - return None; - }; - - let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { - return None; - }; - - if sensitive.unwrap_or(false) { - value.set_sensitive(true); - } - - Some((name, value)) - }, - )); - default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); - default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); - has_default_priority = map.contains_key(PRIORITY); - default_headers = Some(map); - } - - // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so - // the explicit sizes are not resolved at all when it's on: reqwest would let the later - // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence - // visible here rather than depending on hyper's internal ordering. - let http2 = http2.unwrap_or_default(); - let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); - let http2_windows = (!http2_adaptive_window).then(|| { - resolve_windows( - flow_control.as_ref(), - http2.stream_window, - http2.connection_window, - ) - }); - - #[cfg(feature = "http3")] - let http3_max_idle_timeout = Duration::from_secs( - http3 - .as_ref() - .and_then(|h| h.max_idle_timeout) - .unwrap_or(30) - .clamp(1, 120) - .into(), - ); - - // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window - // inside an unbounded connection window: the stream window is the binding constraint - // on a high-latency link, and the unbounded connection window means a connection with - // many concurrent requests has no ceiling on what it buffers. Both are set here. - #[cfg(feature = "http3")] - let http3_windows = resolve_windows( - flow_control.as_ref(), - http3.as_ref().and_then(|h| h.stream_window), - http3.as_ref().and_then(|h| h.connection_window), - ); - - #[cfg(feature = "http3")] - let http3_congestion_bbr = matches!( - http3.as_ref().and_then(|h| h.congestion), - Some(Http3Congestion::Bbr1) - ); - - #[cfg(feature = "http3")] - let http3_send_window = http3.as_ref().and_then(|h| h.send_window); - - let pool_idle_timeout = pool - .as_ref() - .and_then(|pool| pool.idle_timeout) - .map(|seconds| Duration::from_secs(seconds.into())); - // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). - let pool_max_idle_per_host = pool.as_ref().map(|pool| { - pool.max_idle_per_host - .and_then(|n| n.try_into().ok()) - .unwrap_or(usize::MAX) - }); - - let connect_timeout = timeout - .and_then(|t| t.connect) - .map(|millis| Duration::from_millis(millis.into())); - let read_timeout = timeout - .and_then(|t| t.read) - .map(|millis| Duration::from_millis(millis.into())); - let total_timeout = timeout - .and_then(|t| t.total) - .map(|millis| Duration::from_millis(millis.into())); - - #[cfg(feature = "http3")] - let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); - let tls_required = tls.as_ref().and_then(|tls| tls.required); - // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a - // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). - let (tls_identity, tls_extra_roots) = match tls { - None => (None, Vec::new()), - Some(tls) => { - let identity = match &tls.identity { - None => None, - Some(identity) => Some( - Identity::from_pem(match identity { - Either::A(buf) => buf.as_ref(), - Either::B(string) => string.as_bytes(), - }) - .map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?, - ), - }; - - let mut extra_roots = Vec::new(); - for pem in tls.extra_roots.iter().flatten() { - let bytes = match pem { - Either::A(buf) => buf.as_ref(), - Either::B(string) => string.as_bytes(), - }; - extra_roots.extend(Certificate::from_pem_bundle(bytes).map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?); - } - - (identity, extra_roots) - } - }; - - let http_cache = if let Some(cache) = cache - && let Some(store) = cache.store - { - let mode = cache.mode.unwrap_or_default().into(); - let options = HttpCacheOptions { - cache_options: Some(CacheOptions { - shared: cache.shared.unwrap_or(true), - ignore_cargo_cult: true, - ..Default::default() - }), - ..Default::default() - }; - let store = match store { - CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { - path: cache - .path - .ok_or_else(|| { - FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) - })? - .into(), - remove_opts: Default::default(), - }), - CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( - MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), - )), - }; - - Some(HttpCacheRecipe { - mode, - options, - store, - }) - } else { - None - }; - - // Read outside the `alt_svc_cache` block below because `fetch` needs it too, - // to keep a rewritten port from looking like a redirect. - #[cfg(feature = "http3")] - let h3_follow_advertised_port = http3 - .as_ref() - .and_then(|o| o.upgrade_follow_advertised_port) - .unwrap_or(false); - #[cfg(not(feature = "http3"))] - let h3_follow_advertised_port = false; - - // The origin knowledge is built once and outlives every client the agent builds: it is - // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). - #[cfg(feature = "http3")] - let (alt_svc_cache, h3_upgrade) = { - let http3_opts = http3.as_ref(); - let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); - - let advertised_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_advertised_ttl) - .unwrap_or(86400) - .into(), - ); - let confirmed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_confirmed_ttl) - .unwrap_or(86400) - .into(), - ); - let failed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_ttl) - .unwrap_or(300) - .into(), - ); - let failed_max_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_max_ttl) - .unwrap_or(3600) - .into(), - ); - let capacity = http3_opts - .and_then(|o| o.upgrade_cache_capacity) - .unwrap_or(10_000) - .into(); - let cancel_strikes = http3_opts - .and_then(|o| o.upgrade_cancel_strikes) - .unwrap_or(3); - let attempt_timeout = match http3_opts - .and_then(|o| o.upgrade_attempt_timeout) - .unwrap_or(60_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); - let probe_timeout = match http3_opts - .and_then(|o| o.upgrade_probe_timeout) - .unwrap_or(5_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let slow_factor = http3_opts - .and_then(|o| o.upgrade_slow_factor) - .unwrap_or(2.5); - let slow_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_slow_ttl) - .unwrap_or(600) - .into(), - ); - - let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - capacity, - cancel_strikes, - strike_window: Duration::from_secs(60), - follow_advertised_port: h3_follow_advertised_port, - // The single-flight claim must outlive the probe it covers, so - // an aborted probe frees its origin without a report; without a - // probe deadline, the QUIC idle timeout (max 120s) is the bound. - probe_ttl: probe_timeout - .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), - slow_factor, - slow_ttl, - })); - - if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { - for hint in hints { - cache.add_hint(&hint.host, hint.port); - } - } - - ( - Some(cache), - H3UpgradeRecipe { - enabled, - attempt_timeout, - probe, - probe_timeout, - }, - ) - }; - - let recipe = ClientRecipe { - user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), - local_address, - default_headers, - dns_system, - dns_overrides, - http2_adaptive_window, - http2_windows, - #[cfg(feature = "http3")] - http3_max_idle_timeout, - #[cfg(feature = "http3")] - http3_windows, - #[cfg(feature = "http3")] - http3_congestion_bbr, - #[cfg(feature = "http3")] - http3_send_window, - pool_idle_timeout, - pool_max_idle_per_host, - redirect: redirect.map(RedirectPolicy::from), - connect_timeout, - read_timeout, - total_timeout, - #[cfg(feature = "http3")] - tls_early_data, - tls_identity, - tls_required, - tls_extra_roots, - node_env: NodeEnvRecipe::read(), - http_cache, - #[cfg(feature = "http3")] - h3_upgrade, - }; - - let settings = AgentSettings { - h3_follow_advertised_port, - #[cfg(feature = "http3")] - h3_upgrade_enabled: recipe.h3_upgrade.enabled, - quirk_h1_request_streaming, - default_accept_encoding, - default_content_encoding, - has_default_priority, - }; - - Ok(Self { - inner: web_faith::agent::Agent::build( - recipe, - settings, - cookie_jar, - dns_resolver, - #[cfg(feature = "http3")] - alt_svc_cache, - )?, - }) + let options = options::AgentOptions::from(options); + // A napi callback can run outside the runtime, and building the HTTP/3 endpoint needs to be + // inside one, so the client is constructed within whichever runtime is to hand. + within_runtime_if_available(|| web_faith::agent::Agent::from_options(options)) + .map(|inner| Self { inner }) } #[napi(constructor)] @@ -1542,99 +1026,132 @@ fn caller_error(env: &Env, err: FaithError) -> napi::Error { napi::Error::from(err.into_js_error(env)) } -#[cfg(test)] -mod tests { - use super::*; - - fn common(stream: Option, connection: Option) -> AgentFlowControlOptions { - AgentFlowControlOptions { - stream_window: stream, - connection_window: connection, +/// Read the JavaScript options object into the shape the client validates. +/// +/// A field-for-field mapping wherever the two agree, which is most of them; what differs is where +/// JavaScript expresses a choice as a union or a string that Rust has a type for. +impl From for options::AgentOptions { + fn from(opts: AgentOptions) -> Self { + Self { + cache: opts.cache.map(|cache| options::CacheOptions { + store: cache.store.map(|store| match store { + CacheStore::Disk => options::CacheStore::Disk, + CacheStore::Memory => options::CacheStore::Memory, + }), + capacity: cache.capacity, + mode: cache.mode.map(CacheMode::from), + path: cache.path, + shared: cache.shared, + }), + // `false` and an absent value both mean no jar; `true` means one with default limits. + cookies: match opts.cookies { + None | Some(Either::A(false)) => None, + Some(Either::A(true)) => Some(CookieLimits::default()), + Some(Either::B(cookies)) => Some((&cookies).into()), + }, + dns: opts.dns.map(|dns| options::DnsOptions { + system: dns.system, + overrides: dns.overrides.map(|overrides| { + overrides + .into_iter() + .map(|o| options::DnsOverride { + domain: o.domain, + addresses: o.addresses, + }) + .collect() + }), + servers: dns.servers, + timeout: dns.timeout, + search_domains: dns.search_domains, + ndots: dns.ndots, + hosts_file: dns.hosts_file, + exempt_domains: dns.exempt_domains, + serve_stale: dns.serve_stale, + max_stale: dns.max_stale, + }), + flow_control: opts.flow_control.map(|flow| options::FlowControlOptions { + stream_window: flow.stream_window, + connection_window: flow.connection_window, + }), + headers: opts.headers.map(|headers| { + headers + .into_iter() + .map(|header| options::Header { + name: header.name, + value: header.value, + sensitive: header.sensitive, + }) + .collect() + }), + http2: opts.http2.map(|http2| options::Http2Options { + stream_window: http2.stream_window, + connection_window: http2.connection_window, + adaptive_window: http2.adaptive_window, + }), + http3: opts.http3.map(|http3| options::Http3Options { + congestion: http3.congestion.map(|c| match c { + Http3Congestion::Cubic => options::Http3Congestion::Cubic, + Http3Congestion::Bbr1 => options::Http3Congestion::Bbr1, + }), + max_idle_timeout: http3.max_idle_timeout, + upgrade_enabled: http3.upgrade_enabled, + upgrade_probe: http3.upgrade_probe, + upgrade_probe_timeout: http3.upgrade_probe_timeout, + upgrade_slow_factor: http3.upgrade_slow_factor, + upgrade_slow_ttl: http3.upgrade_slow_ttl, + upgrade_advertised_ttl: http3.upgrade_advertised_ttl, + upgrade_confirmed_ttl: http3.upgrade_confirmed_ttl, + upgrade_failed_ttl: http3.upgrade_failed_ttl, + upgrade_failed_max_ttl: http3.upgrade_failed_max_ttl, + upgrade_cancel_strikes: http3.upgrade_cancel_strikes, + upgrade_attempt_timeout: http3.upgrade_attempt_timeout, + upgrade_follow_advertised_port: http3.upgrade_follow_advertised_port, + upgrade_cache_capacity: http3.upgrade_cache_capacity, + hints: http3.hints.map(|hints| { + hints + .into_iter() + .map(|hint| options::Http3Hint { + host: hint.host, + port: hint.port, + }) + .collect() + }), + stream_window: http3.stream_window, + connection_window: http3.connection_window, + send_window: http3.send_window, + }), + local_address: opts.local_address, + pool: opts.pool.map(|pool| options::PoolOptions { + idle_timeout: pool.idle_timeout, + max_idle_per_host: pool.max_idle_per_host, + }), + quirks: opts.quirks.map(|quirks| options::QuirksOptions { + h1_request_streaming: quirks.h1_request_streaming, + }), + redirect: opts.redirect.map(RedirectPolicy::from), + timeout: opts.timeout.map(|timeout| options::TimeoutOptions { + connect: timeout.connect, + read: timeout.read, + total: timeout.total, + }), + tls: opts.tls.map(|tls| options::TlsOptions { + early_data: tls.early_data, + // Either spelling of a PEM is the same bytes to the client. + identity: tls.identity.map(|pem| pem_bytes(&pem)), + required: tls.required, + extra_roots: tls + .extra_roots + .map(|roots| roots.iter().map(pem_bytes).collect()), + }), + user_agent: opts.user_agent, } } +} - #[test] - fn windows_fall_back_to_the_defaults() { - // An agent configured with nothing at all still gets the large static windows - // (spec:FLOW#common-windows). - assert_eq!( - resolve_windows(None, None, None), - ResolvedWindows { - stream: 6 * 1024 * 1024, - connection: 15 * 1024 * 1024, - } - ); - } - - #[test] - fn the_common_windows_apply_when_a_protocol_says_nothing() { - assert_eq!( - resolve_windows(Some(&common(Some(1024), Some(4096))), None, None), - ResolvedWindows { - stream: 1024, - connection: 4096, - } - ); - } - - #[test] - fn a_protocol_window_beats_the_common_one() { - // The whole point of the per-protocol group: tune one protocol against the other - // (spec:FLOW#per-protocol-windows). - assert_eq!( - resolve_windows( - Some(&common(Some(1024), Some(4096))), - Some(2048), - Some(8192) - ), - ResolvedWindows { - stream: 2048, - connection: 8192, - } - ); - } - - #[test] - fn each_window_falls_back_on_its_own() { - // Overriding the stream window for one protocol leaves that protocol's connection - // window on the common value, rather than dropping it to the default. - assert_eq!( - resolve_windows(Some(&common(Some(1024), Some(4096))), Some(2048), None), - ResolvedWindows { - stream: 2048, - connection: 4096, - } - ); - assert_eq!( - resolve_windows(Some(&common(None, None)), None, Some(8192)), - ResolvedWindows { - stream: DEFAULT_STREAM_WINDOW, - connection: 8192, - } - ); - } - - #[test] - fn a_protocol_window_applies_without_the_common_group() { - assert_eq!( - resolve_windows(None, Some(2048), None), - ResolvedWindows { - stream: 2048, - connection: DEFAULT_CONNECTION_WINDOW, - } - ); - } - - #[test] - fn the_two_protocols_resolve_independently() { - // One `flowControl` value covers both protocols, and overriding it for HTTP/3 leaves - // HTTP/2 where it was (spec:FLOW#per-protocol-windows). - let flow = common(Some(1024), Some(4096)); - let http2 = resolve_windows(Some(&flow), None, None); - let http3 = resolve_windows(Some(&flow), Some(2048), None); - - assert_eq!(http2.stream, 1024); - assert_eq!(http3.stream, 2048); - assert_eq!(http2.connection, http3.connection); +/// PEM input arrives as a buffer or a string; both are just the bytes. +fn pem_bytes(pem: &Either) -> Vec { + match pem { + Either::A(buf) => buf.to_vec(), + Either::B(string) => string.as_bytes().to_vec(), } } diff --git a/crates/web-faith-napi/src/options.rs b/crates/web-faith-napi/src/options.rs index 577e0a0..7cc028e 100644 --- a/crates/web-faith-napi/src/options.rs +++ b/crates/web-faith-napi/src/options.rs @@ -136,9 +136,6 @@ pub enum DuplexOption { Half, } -/// The RFC 9218 header the `priority` option maps onto. -pub(crate) const PRIORITY: &str = "priority"; - /// Maps the `priority` option onto an RFC 9218 `Priority` header value. /// /// Urgency runs from 0 (most urgent) to 7 (least urgent), and a request that sends no header diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 524fdf4..6b578d8 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -4,6 +4,8 @@ use std::{ future::Future, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + str::FromStr, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -11,27 +13,40 @@ use std::{ time::Duration, }; -use http::header::HeaderValue; +use http::header::{HeaderMap, HeaderName, HeaderValue}; +use http_cache_reqwest::{ + CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, +}; use moka::sync::Cache as MokaCache; -use reqwest::{Client, Version}; +use reqwest::{Client, Identity, Version, tls::Certificate}; use reqwest_middleware::ClientWithMiddleware; use url::Url; use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; use web_faith_cookies::FaithJar; -use web_faith_dns::{FaithResolver, ResolverReport}; +use web_faith_dns::{ + DEFAULT_MAX_STALE, FaithResolver, ResolverReport, ResolverSettings, ServerSpec, parse_domains, +}; #[cfg(feature = "http3")] -use web_faith_alt_svc::{AltSvcCache, H3Prober}; +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, H3Prober}; use crate::{ - client::ClientRecipe, + USER_AGENT, + client::{ClientRecipe, HttpCacheRecipe, HttpCacheStore, NodeEnvRecipe}, error::{FaithError, FaithErrorKind}, + options::{ + AgentOptions, CacheStore, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows, + }, + request::PRIORITY, stats::{AgentStats, InnerAgentStats}, warm_up::{extract_host, origin_key, reduce_to_origin}, }; #[cfg(feature = "http3")] -use crate::client::install_https_sink; +use crate::{ + client::{H3UpgradeRecipe, install_https_sink}, + options::Http3Congestion, +}; /// The agent settings a request consults, as opposed to those a client is built from. #[derive(Debug, Clone, Default)] @@ -133,6 +148,464 @@ pub struct Agent { } impl Agent { + /// Build an agent from options, validating them into the recipe its clients are built from. + /// + /// This is what both surfaces land on, so the defaults a caller gets are settled here rather + /// than once per surface. + // spec:AGENT spec:NETCHG + pub fn from_options(options: AgentOptions) -> Result { + // Destructured rather than read field by field so that a new option cannot be added + // without the compiler pointing here, where every option is turned into the recipe the + // agent's clients are built from (spec:NETCHG). + let AgentOptions { + cache, + cookies, + dns, + flow_control, + headers, + http2, + // Every use of the HTTP/3 options sits behind the feature. + #[cfg_attr(not(feature = "http3"), allow(unused_variables))] + http3, + local_address, + pool, + quirks, + redirect, + timeout, + tls, + user_agent, + } = options; + + let quirk_h1_request_streaming = quirks + .and_then(|quirks| quirks.h1_request_streaming) + .unwrap_or(false); + + // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts + // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the + // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and + // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such + // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. + let local_address = match &local_address { + Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { + FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + ) + })?), + None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), + None => None, + }; + + // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) + // The jar is installed on the client by the recipe, so it survives a rebuild + // (spec:NETCHG#what-the-signal-keeps). + let cookie_jar = cookies.map(|limits| Arc::new(FaithJar::new(limits))); + + let dns = dns.unwrap_or_default(); + let dns_system = dns.system.unwrap_or(false); + // Naming servers and asking for the system resolver at once is a contradiction rather than + // a preference, since the system resolver is not Faith's to point at listed servers + // (spec:DNS#system-resolver). + if dns_system + && dns + .servers + .as_ref() + .is_some_and(|servers| !servers.is_empty()) + { + return Err(FaithError::new( + FaithErrorKind::Config, + Some("dns.servers cannot be combined with dns.system".to_string()), + )); + } + // Parsed whichever resolver is in use: overrides take effect under the system resolver + // too (spec:DNS#overrides), and an unparseable address is a construction error either + // way (spec:AGENT#construction). + let dns_overrides = dns + .overrides + .unwrap_or_default() + .into_iter() + .map(|DnsOverride { domain, addresses }| { + let addresses = addresses + .into_iter() + .map(|addr| match SocketAddr::from_str(&addr) { + Ok(addr) => Ok(addr), + Err(err) => match IpAddr::from_str(&addr) { + Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), + Ok(IpAddr::V6(ip)) => { + Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) + } + Err(_) => Err(FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + )), + }, + }) + .collect::, FaithError>>()?; + Ok((domain, addresses)) + }) + .collect::, FaithError>>()?; + + // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so + // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), + // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and + // order each resolver is reached by (spec:DNS#transports). The system resolver + // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there + // and `prefetchDns` resolves as a no-op (spec:WARM). + let dns_resolver = if dns_system { + None + } else { + // These settings configure Faith's own resolver only, so they are read on the path that + // builds one rather than validated under the system resolver that ignores them. + let mut servers = Vec::new(); + for url in dns.servers.unwrap_or_default() { + servers.push(ServerSpec::parse(&url).map_err(|message| { + FaithError::new(FaithErrorKind::AddressParse, Some(message)) + })?); + } + Some(FaithResolver::new(ResolverSettings { + servers, + timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), + ndots: dns.ndots.map(|n| n as usize), + search_domains: parse_domains(dns.search_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, + hosts_file: dns.hosts_file, + exempt_domains: parse_domains(dns.exempt_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? + .unwrap_or_default(), + serve_stale: dns.serve_stale.unwrap_or(true), + max_stale: dns + .max_stale + .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), + })) + }; + + let mut default_accept_encoding = None; + let mut default_content_encoding = None; + let mut has_default_priority = false; + let mut default_headers = None; + if let Some(headers) = headers + && !headers.is_empty() + { + let map = HeaderMap::from_iter(headers.into_iter().filter_map( + |Header { + name, + value, + sensitive, + }| { + let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { + return None; + }; + + let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { + return None; + }; + + if sensitive.unwrap_or(false) { + value.set_sensitive(true); + } + + Some((name, value)) + }, + )); + default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); + default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); + has_default_priority = map.contains_key(PRIORITY); + default_headers = Some(map); + } + + // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so + // the explicit sizes are not resolved at all when it's on: reqwest would let the later + // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence + // visible here rather than depending on hyper's internal ordering. + let http2 = http2.unwrap_or_default(); + let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); + let http2_windows = (!http2_adaptive_window).then(|| { + resolve_windows( + flow_control.as_ref(), + http2.stream_window, + http2.connection_window, + ) + }); + + #[cfg(feature = "http3")] + let http3_max_idle_timeout = Duration::from_secs( + http3 + .as_ref() + .and_then(|h| h.max_idle_timeout) + .unwrap_or(30) + .clamp(1, 120) + .into(), + ); + + // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window + // inside an unbounded connection window: the stream window is the binding constraint + // on a high-latency link, and the unbounded connection window means a connection with + // many concurrent requests has no ceiling on what it buffers. Both are set here. + #[cfg(feature = "http3")] + let http3_windows = resolve_windows( + flow_control.as_ref(), + http3.as_ref().and_then(|h| h.stream_window), + http3.as_ref().and_then(|h| h.connection_window), + ); + + #[cfg(feature = "http3")] + let http3_congestion_bbr = matches!( + http3.as_ref().and_then(|h| h.congestion), + Some(Http3Congestion::Bbr1) + ); + + #[cfg(feature = "http3")] + let http3_send_window = http3.as_ref().and_then(|h| h.send_window); + + let pool_idle_timeout = pool + .as_ref() + .and_then(|pool| pool.idle_timeout) + .map(|seconds| Duration::from_secs(seconds.into())); + // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). + let pool_max_idle_per_host = pool.as_ref().map(|pool| { + pool.max_idle_per_host + .and_then(|n| n.try_into().ok()) + .unwrap_or(usize::MAX) + }); + + let connect_timeout = timeout + .and_then(|t| t.connect) + .map(|millis| Duration::from_millis(millis.into())); + let read_timeout = timeout + .and_then(|t| t.read) + .map(|millis| Duration::from_millis(millis.into())); + let total_timeout = timeout + .and_then(|t| t.total) + .map(|millis| Duration::from_millis(millis.into())); + + #[cfg(feature = "http3")] + let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); + let tls_required = tls.as_ref().and_then(|tls| tls.required); + // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a + // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). + let (tls_identity, tls_extra_roots) = match tls { + None => (None, Vec::new()), + Some(tls) => { + let identity = match &tls.identity { + None => None, + Some(identity) => Some(Identity::from_pem(identity).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?), + }; + + let mut extra_roots = Vec::new(); + for pem in tls.extra_roots.iter().flatten() { + extra_roots.extend(Certificate::from_pem_bundle(pem).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?); + } + + (identity, extra_roots) + } + }; + + let http_cache = if let Some(cache) = cache + && let Some(store) = cache.store + { + let mode = cache.mode.unwrap_or_default().into(); + let options = HttpCacheOptions { + cache_options: Some(CacheOptions { + shared: cache.shared.unwrap_or(true), + ignore_cargo_cult: true, + ..Default::default() + }), + ..Default::default() + }; + let store = match store { + CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { + path: cache + .path + .ok_or_else(|| { + FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) + })? + .into(), + remove_opts: Default::default(), + }), + CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( + MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), + )), + }; + + Some(HttpCacheRecipe { + mode, + options, + store, + }) + } else { + None + }; + + // Read outside the `alt_svc_cache` block below because `fetch` needs it too, + // to keep a rewritten port from looking like a redirect. + #[cfg(feature = "http3")] + let h3_follow_advertised_port = http3 + .as_ref() + .and_then(|o| o.upgrade_follow_advertised_port) + .unwrap_or(false); + #[cfg(not(feature = "http3"))] + let h3_follow_advertised_port = false; + + // The origin knowledge is built once and outlives every client the agent builds: it is + // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). + #[cfg(feature = "http3")] + let (alt_svc_cache, h3_upgrade) = { + let http3_opts = http3.as_ref(); + let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); + + let advertised_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_advertised_ttl) + .unwrap_or(86400) + .into(), + ); + let confirmed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_confirmed_ttl) + .unwrap_or(86400) + .into(), + ); + let failed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_ttl) + .unwrap_or(300) + .into(), + ); + let failed_max_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_max_ttl) + .unwrap_or(3600) + .into(), + ); + let capacity = http3_opts + .and_then(|o| o.upgrade_cache_capacity) + .unwrap_or(10_000) + .into(); + let cancel_strikes = http3_opts + .and_then(|o| o.upgrade_cancel_strikes) + .unwrap_or(3); + let attempt_timeout = match http3_opts + .and_then(|o| o.upgrade_attempt_timeout) + .unwrap_or(60_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); + let probe_timeout = match http3_opts + .and_then(|o| o.upgrade_probe_timeout) + .unwrap_or(5_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let slow_factor = http3_opts + .and_then(|o| o.upgrade_slow_factor) + .unwrap_or(2.5); + let slow_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_slow_ttl) + .unwrap_or(600) + .into(), + ); + + let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + capacity, + cancel_strikes, + strike_window: Duration::from_secs(60), + follow_advertised_port: h3_follow_advertised_port, + // The single-flight claim must outlive the probe it covers, so + // an aborted probe frees its origin without a report; without a + // probe deadline, the QUIC idle timeout (max 120s) is the bound. + probe_ttl: probe_timeout + .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), + slow_factor, + slow_ttl, + })); + + if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { + for hint in hints { + cache.add_hint(&hint.host, hint.port); + } + } + + ( + Some(cache), + H3UpgradeRecipe { + enabled, + attempt_timeout, + probe, + probe_timeout, + }, + ) + }; + + let recipe = ClientRecipe { + user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), + local_address, + default_headers, + dns_system, + dns_overrides, + http2_adaptive_window, + http2_windows, + #[cfg(feature = "http3")] + http3_max_idle_timeout, + #[cfg(feature = "http3")] + http3_windows, + #[cfg(feature = "http3")] + http3_congestion_bbr, + #[cfg(feature = "http3")] + http3_send_window, + pool_idle_timeout, + pool_max_idle_per_host, + redirect, + connect_timeout, + read_timeout, + total_timeout, + #[cfg(feature = "http3")] + tls_early_data, + tls_identity, + tls_required, + tls_extra_roots, + node_env: NodeEnvRecipe::read(), + http_cache, + #[cfg(feature = "http3")] + h3_upgrade, + }; + + let settings = AgentSettings { + h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming, + default_accept_encoding, + default_content_encoding, + has_default_priority, + }; + + Self::build( + recipe, + settings, + cookie_jar, + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + ) + } + + /// An agent with default options. + pub fn new() -> Result { + Self::from_options(AgentOptions::default()) + } + /// Build an agent from a validated recipe. /// /// The recipe is what a client is built from, and the settings are what each request consults; @@ -515,3 +988,20 @@ impl Agent { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// An agent is constructible without a caller assembling a recipe by hand, and comes up with a + /// live client and the default user agent. + #[tokio::test] + async fn a_default_agent_comes_up() { + let agent = Agent::new().expect("default options build an agent"); + + assert!(!agent.is_closed()); + assert_eq!(agent.recipe.user_agent, crate::USER_AGENT); + // No jar until the options ask for one. + assert!(agent.cookie_jar.is_none()); + } +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 98b8de6..d1b0604 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -22,6 +22,7 @@ pub mod body; pub mod client; pub mod error; pub mod integrity; +pub mod options; pub mod request; pub mod response; pub mod retry; diff --git a/crates/web-faith/src/options.rs b/crates/web-faith/src/options.rs new file mode 100644 index 0000000..5e618e9 --- /dev/null +++ b/crates/web-faith/src/options.rs @@ -0,0 +1,750 @@ +//! The options an agent is built from, one struct per group. +//! +//! [`Agent::from_options`] validates these into the recipe an agent's clients are built from, and +//! settles the defaults for anything left unset. Both surfaces go through it, so a default is +//! decided once rather than once per surface. +//! +//! [`Agent::from_options`]: crate::agent::Agent::from_options + +// spec:AGENT + +use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; + +use http_cache_reqwest::CacheMode; +use web_faith_cookies::CookieLimits; + +use crate::client::{ + DEFAULT_CONNECTION_WINDOW, DEFAULT_STREAM_WINDOW, RedirectPolicy, ResolvedWindows, +}; + +/// Settings related to the HTTP cache. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct CacheOptions { + /// Which cache store to use: either `disk` or `memory`. + /// + /// Default: none (cache disabled). + pub store: Option, + /// If `cache.store: "memory"`, the maximum amount of items stored. + /// + /// Default: 10_000. + pub capacity: Option, + /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if + /// no cache mode is set on a request. + /// + /// Default: `"default"`. + pub mode: Option, + /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. + /// + /// Required if `cache.store: "disk"`. + pub path: Option, + /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is + /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a + /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). + /// `shared: true` is required for proxies and multi-user caches. + /// + /// Default: true. + pub shared: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CacheStore { + Disk, + + Memory, +} + +#[derive(Clone, Debug, Default)] +pub struct DnsOverride { + pub domain: String, + pub addresses: Vec, +} + +/// Settings related to DNS. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct DnsOptions { + /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on + /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native + /// fetch does, this should be your first port of call. + /// + /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory + /// DNS cache, and may lead to worse performance even discounting the cache. + /// + /// Default: false. + /// + /// [Hickory]: https://hickory-dns.org/ + pub system: Option, + /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. + /// + /// Will throw if addresses are in invalid formats. You may provide a port number as part of the + /// address, it will default to port 0 otherwise, which will select the conventional port for the + /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port + /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively + /// blocks that domain from this agent. + /// + /// Default: no overrides. + pub overrides: Option>, + /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to + /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over + /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on + /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the + /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. + /// + /// The encrypted transports always authenticate the resolver. A URL fragment names the + /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates + /// against the hostname; a bare-IP host authenticates against the address itself. + /// + /// Servers are queried in order, a later one reached only once those before it fail. Setting + /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or + /// its scheme is not one of the above, and combining it with `dns.system` throws. + /// + /// Default: system discovery. + pub servers: Option>, + /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead + /// servers costs a single timeout rather than one per server. + /// + /// Default: 5000. + pub timeout: Option, + /// Replace the system's search list, the domains appended to a name that is not fully + /// qualified. Independent of `dns.servers`. + /// + /// Default: the system's search list. + pub search_domains: Option>, + /// How many dots a name must contain before it is tried as given, ahead of the search list. + /// Independent of `dns.servers`. + /// + /// Default: the system's setting. + pub ndots: Option, + /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. + /// + /// Default: platform convention. + pub hosts_file: Option, + /// Further domains to exempt from the configured or encrypted resolver, for the internal + /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's + /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. + /// + /// Default: no extra exemptions. + pub exempt_domains: Option>, + /// Serve an expired cache entry immediately and refresh it in the background, rather than making + /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is + /// almost always still correct, and a connect failure against one that has moved re-resolves and + /// attempts the request again. + /// + /// Set `false` for an agent that must never connect to an address it knows to be out of date: an + /// expired entry is discarded and the lookup blocks on a fresh answer. + /// + /// Default: true. + pub serve_stale: Option, + /// How far past expiry an answer may still be served, in milliseconds. An entry older than this + /// is discarded rather than served: an answer stale enough stops being evidence about where the + /// host is, and a refresh still failing after that long is the case where the address most likely + /// did change. + /// + /// Default: 3600000 (one hour). + pub max_stale: Option, +} + +/// Sets the default headers for every request. +/// +/// If header names or values are invalid, they are silently omitted. +/// Sensitive headers (e.g. `Authorization`) should be marked. +/// +/// Default: none. +#[derive(Clone, Debug, Default)] +pub struct Header { + pub name: String, + pub value: String, + pub sensitive: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Http3Congestion { + #[default] + Cubic, + + Bbr1, +} + +/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc +/// cache so the first request to this host will attempt HTTP/3 immediately. +#[derive(Clone, Debug, Default)] +pub struct Http3Hint { + /// The hostname (e.g., "example.com"). + pub host: String, + /// The port number (e.g., 443). + pub port: u16, +} + +/// Settings related to HTTP/3. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct Http3Options { + /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the + /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with + /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by + /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise + /// bandwidth use and optimises for round-trip time, while ignoring packet loss. + /// + /// In some networks, BBR can lead to pathological degradation of overall network conditions, by + /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, + /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). + /// + /// [2]: https://github.com/quinn-rs/quinn/issues/1254 + /// + /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. + pub congestion: Option, + /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that + /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ + /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith + /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). + /// + /// Default: 30. + pub max_idle_timeout: Option, + /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc + /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. + /// + /// Default: true. + pub upgrade_enabled: Option, + /// Whether advertised HTTP/3 endpoints are verified with a background probe + /// before any foreground request is routed to them. + /// + /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say + /// there is UDP connectivity between you and it. Without probing, the next + /// request after an advertisement attempts HTTP/3 inline, and on a silently + /// broken UDP path it stalls until the QUIC idle timeout or + /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per + /// failure cooldown for as long as the path stays broken. + /// + /// With probing (the default), requests keep using TCP until a background + /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the + /// connection pool, so the first upgraded request rides the probe's warm + /// connection. A broken path costs one failed background request per + /// cooldown and no foreground latency at all. + /// + /// The probe is a synthetic request the server will see in its logs. Set + /// this to `false` to restore the inline upgrade if that is unacceptable + /// (per-request billing, easily-alarmed WAFs). + /// + /// `hints` are exempt either way: a hint is your own assertion, so the first + /// request to a hinted origin speaks HTTP/3 immediately, which is also what + /// makes h3-only origins (no TCP listener) work. + /// + /// Default: true. + pub upgrade_probe: Option, + /// Ceiling on how long a background HTTP/3 probe may take before the origin + /// is treated as failed, in **milliseconds**. + /// + /// This bounds background work only — no foreground request ever waits on a + /// probe — so it can afford to be generous: a healthy handshake plus HEAD + /// completes in one or two round trips. Set to 0 to leave probes bounded + /// only by the QUIC idle timeout. + /// + /// Default: 5000 (5 seconds). + pub upgrade_probe_timeout: Option, + /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than + /// its TCP path by this factor. Set to 0 to disable path-time demotion. + /// + /// Faith keeps a per-origin moving average of time-to-response-headers for + /// each protocol family. HTTP/3 is preferred at parity and when moderately + /// slower — its advantages (no head-of-line blocking, connection migration) + /// pay off beyond the average — so this factor should stay well above 1. + /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC + /// average must also exceed the TCP one by an absolute 10ms so LAN-fast + /// origins don't flap on noise. + /// + /// A demoted origin is not treated as broken: it re-enters through a + /// background probe after `upgradeSlowTtl`, asking whether the path has + /// improved at zero foreground cost. + /// + /// Default: 2.5. + pub upgrade_slow_factor: Option, + /// How long (in seconds) a path-time demotion holds before the origin is + /// re-evaluated. See `upgradeSlowFactor`. + /// + /// Default: 600 (10 minutes). + pub upgrade_slow_ttl: Option, + /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. + /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. + /// + /// Default: 86400 (24 hours). + pub upgrade_advertised_ttl: Option, + /// How long (in seconds) to cache a confirmed working HTTP/3 connection. + /// + /// Default: 86400 (24 hours). + pub upgrade_confirmed_ttl: Option, + /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this + /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends + /// Alt-Svc headers. + /// + /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an + /// origin whose UDP path is blocked for good is retried less and less often instead of + /// forever at this interval. A confirmed HTTP/3 response ends the run. + /// + /// Default: 300 (5 minutes). + pub upgrade_failed_ttl: Option, + /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of + /// `upgradeFailedTtl`. + /// + /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, + /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat + /// cooldown that never backs off. + /// + /// Default: 3600 (1 hour). + pub upgrade_failed_max_ttl: Option, + /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, + /// demote an origin back to TCP. + /// + /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request + /// cancelled via `AbortSignal` never produces that signal, so without this an + /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as + /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a + /// sustained run of them demotes the origin, and any successful HTTP/3 response + /// resets the count. + /// + /// Strikes must land within about a minute of each other to count towards a + /// run. A retry loop whose backoff exceeds that window never accumulates one, + /// so callers with a long backoff should set this to 1 for immediate demotion + /// on the first cancelled attempt. + /// + /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that + /// carries small datagrams but drops full-size ones (an MTU blackhole, say). + /// Response headers still arrive, so the attempt resolves and every mechanism + /// here counts it a success — the transfer then stalls partway through the + /// body, where nothing is watching. `maxIdleTimeout` or the request's own + /// timeout is what ends such a request, and the origin stays on HTTP/3. + /// + /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. + /// + /// Default: 3. + pub upgrade_cancel_strikes: Option, + /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is + /// given up on and the request is retried over TCP, in **milliseconds**. + /// + /// Note the unit: the other `upgrade*` settings are in seconds, but this one + /// is in milliseconds to match the `timeout` settings, because useful values + /// are sub-second. + /// + /// This bounds the wait for response headers, not the response body, so a slow + /// body is unaffected. + /// + /// The default is high, but not unconditionally inert: `maxIdleTimeout` is + /// configurable up to 120 seconds, and above 60 seconds this deadline becomes + /// the effective ceiling. Even below that, "QUIC's own idle timeout fires + /// first" only holds while the connection is idle — a transfer still running + /// past this deadline keeps the connection active, so no idle timeout is + /// coming to end it. + /// + /// On expiry the request is retried over TCP, which means it is re-sent: a + /// timeout often means the server is still processing, so a slow + /// non-idempotent request (a POST, say) can end up delivered twice. Lowering + /// this value trades that double-submission risk for faster recovery when a + /// UDP path breaks. Anyone setting it low should confirm their slowest + /// legitimate time to response headers fits well inside the budget. + /// + /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle + /// timeout and the request's own timeout. + /// + /// Default: 60000 (60 seconds). + pub upgrade_attempt_timeout: Option, + /// Connect to the port a server advertises HTTP/3 on, even when it differs from + /// the origin's own port. **This is not standards-compliant**; it is off by + /// default. + /// + /// An `Alt-Svc` advertisement names a network endpoint for the origin, so + /// honouring one correctly means connecting to that endpoint while still + /// sending the *origin's* authority. reqwest cannot express that — it derives + /// the HTTP/3 connect target from the request URI's authority (tracked + /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). + /// So by default Faith does not upgrade at all when the advertised port + /// differs, rather than guessing that the origin's own port also speaks + /// HTTP/3. + /// + /// Setting this to `true` upgrades anyway, by rewriting the request's port to + /// the advertised one. That gets HTTP/3 working today against servers you + /// control, at the cost of three deviations you should be aware of: + /// + /// - The request's `Host`/`:authority` carries the advertised port instead of + /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) + /// forbids. Servers that route on authority may misroute or reject; servers + /// that ignore it are unaffected. + /// - `response.url` reports the port actually connected to. + /// - `redirected` ignores port differences, since the rewritten port would + /// otherwise look like a redirect on every request. + /// + /// TLS is unaffected: certificates are still validated against the origin's + /// hostname. Only the port changes. + /// + /// Default: `false`. + pub upgrade_follow_advertised_port: Option, + /// Maximum number of origins to track in the Alt-Svc cache. + /// + /// Default: 10000. + pub upgrade_cache_capacity: Option, + /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache + /// on agent initialization, so the first request to these hosts will attempt HTTP/3. + pub hints: Option>, + /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/3 only. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload + /// throughput the way the receive windows bound download. The origin's own flow control + /// applies on top of this, so it is a ceiling rather than a grant. + /// + /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window + /// the peer advertises, with no local cap to set. + /// + /// Default: 10 MB (quinn's own default). + pub send_window: Option, +} + +/// Settings related to HTTP/2. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct Http2Options { + /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Replace HTTP/2's static windows with windows that start small and grow towards a + /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. + /// + /// This is off by default, and turning it on is usually the wrong move. A fresh connection + /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample + /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and + /// carries *less* throughput than the static window for all but the largest transfers. It + /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. + /// + /// Its one real advantage is memory: it holds a large window open only on connections that + /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling + /// buys the same throughput from the first byte. + /// + /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. + /// + /// Default: `false`. + pub adaptive_window: Option, +} + +/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct FlowControlOptions { + /// Maximum bytes an origin may send on any one stream before it must wait for Faith to + /// acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. + /// The default follows browser practice, and is deliberately at the conservative end of it: + /// a pooled server-side client can hold many connections across many origins, so + /// per-connection memory multiplies harder here than in a browser. + /// + /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. + /// + /// Default: 6 MiB. + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one connection before it must + /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// This is larger than `streamWindow` so concurrent streams on one connection share the + /// connection's headroom, while still bounding the worst-case buffering of a connection + /// carrying many concurrent requests. + /// + /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against + /// the other. + /// + /// Default: 15 MiB. + pub connection_window: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct PoolOptions { + /// How many seconds of inactivity before a connection is closed. + /// + /// Default: 90 seconds. + pub idle_timeout: Option, + /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed + /// to keep the idle connections (per host) under that number. + /// + /// Default: `null` (no limit). + pub max_idle_per_host: Option, +} + +/// Switches that depart from standard behaviour on purpose. This is a nested object. +/// +/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule +/// forbids. All of them are off by default, so an agent constructed with no options is +/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise +/// established that what the rule guards against does not apply to them: turning one on means +/// requests may fail against origins that expect the standard behaviour. +#[derive(Clone, Copy, Debug, Default)] +pub struct QuirksOptions { + /// Allow a streaming request body to be sent over an HTTP/1.x connection. + /// + /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read + /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x + /// origin or an intermediary on the path may refuse it. With this on, such a body sends over + /// whichever protocol the connection negotiates. + /// + /// Default: false. + pub h1_request_streaming: Option, +} + +/// Timeouts for requests made with this agent. This is a nested object. +#[derive(Clone, Copy, Debug, Default)] +pub struct TimeoutOptions { + /// Set a timeout for only the connect phase, in milliseconds. + /// + /// Default: none. + pub connect: Option, + /// Set a timeout for read operations, in milliseconds. + /// + /// The timeout applies to each read operation, and resets after a successful read. This is more + /// appropriate for detecting stalled connections when the size isn't known beforehand. + /// + /// Default: none. + pub read: Option, + /// Set a timeout for the entire request-response cycle, in milliseconds. + /// + /// The timeout applies from when the request starts connecting until the response body has finished. + /// Also considered a total deadline. + /// + /// Default: none. + pub total: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[derive(Clone, Debug, Default)] +pub struct TlsOptions { + /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet + /// of application data alongside the opening packet of the TLS handshake. That can enable the server + /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant + /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should + /// really only be used for static assets or to squeeze out the last drop of performance for endpoints + /// that are replay-safe. + /// + /// Default: false. + pub early_data: Option, + /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also + /// called mutual TLS or mTLS) authentication. + /// + /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The + /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options + /// that will cause the `Agent` constructor to throw if the input is in the wrong format. + pub identity: Option>, + /// Disables plain-text HTTP. + /// + /// Default: false. + pub required: Option, + /// Additional PEM-formatted root certificates to trust, on top of the platform's + /// trust store. Each entry may be a PEM bundle containing multiple certificates. + /// + /// This is mainly useful for connecting to servers with self-signed or private-CA + /// certificates, such as internal services or local test servers. This is one of the + /// few options that will cause the `Agent` constructor to throw if the input is in + /// the wrong format. + pub extra_roots: Option>>, +} + +#[derive(Clone, Debug, Default)] +pub struct AgentOptions { + /// Settings related to the HTTP cache. This is a nested object. + pub cache: Option, + /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and + /// included in additional requests. + /// + /// `true` enables the store with the default limits; an options object enables it and tunes them, + /// so `{}` means the same as `true`. + /// + /// Default: `false`. + /// + /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add + /// and retrieve cookies from the store. + pub cookies: Option, + /// Settings related to DNS. This is a nested object. + pub dns: Option, + /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. + /// + /// Setting these is the normal way to tune windows: one value applies to whichever protocol + /// a request negotiates, so throughput doesn't change when an origin upgrades from one to + /// the other. The `http2` and `http3` groups override them per protocol. + pub flow_control: Option, + /// Sets the default headers for every request. + /// + /// If header names or values are invalid, they are silently omitted. + /// Sensitive headers (e.g. `Authorization`) should be marked. + /// + /// Default: none. + pub headers: Option>, + /// Settings related to HTTP/2. This is a nested object. + pub http2: Option, + /// Settings related to HTTP/3. This is a nested object. + pub http3: Option, + /// Bind outgoing sockets to this local IP address before connecting. + /// + /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that + /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — + /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and + /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to + /// force a specific source address. Throws if the value does not parse as an IP address. + /// + /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). + pub local_address: Option, + /// Settings related to the connection pool. This is a nested object. + pub pool: Option, + /// Switches that depart from standard behaviour on purpose. This is a nested object. + pub quirks: Option, + /// Determines the behavior in case the server replies with a redirect status. + pub redirect: Option, + /// Timeouts for requests made with this agent. This is a nested object. + pub timeout: Option, + /// Settings related to the connection pool. This is a nested object. + pub tls: Option, + /// Custom user agent string. + /// + /// Default: `Faith/{version} reqwest/{version}`. + pub user_agent: Option, +} + +/// Whether this host can bind the IPv6 wildcard (`[::]`). +/// +/// This is tested using the exact operation reqwest performs when creating the QUIC +/// endpoint with no explicit local address, so it predicts whether the default +/// QUIC bind will succeed. The result is memoised for the life of the process; while +/// IPv6 bindability can in principle change at runtime, this is considered an +/// acceptable tradeoff for performance and simplicity. +pub fn ipv6_wildcard_bindable() -> bool { + use std::sync::OnceLock; + static BINDABLE: OnceLock = OnceLock::new(); + *BINDABLE.get_or_init(|| { + UdpSocket::bind(SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0)).is_ok() + }) +} + +/// Reconcile one protocol's windows: its own setting wins over the common one, which wins over the +/// default (spec:FLOW#per-protocol-windows). +pub fn resolve_windows( + common: Option<&FlowControlOptions>, + protocol_stream: Option, + protocol_connection: Option, +) -> ResolvedWindows { + ResolvedWindows { + stream: protocol_stream + .or_else(|| common.and_then(|c| c.stream_window)) + .unwrap_or(DEFAULT_STREAM_WINDOW), + connection: protocol_connection + .or_else(|| common.and_then(|c| c.connection_window)) + .unwrap_or(DEFAULT_CONNECTION_WINDOW), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::ResolvedWindows; + + fn common(stream: Option, connection: Option) -> FlowControlOptions { + FlowControlOptions { + stream_window: stream, + connection_window: connection, + } + } + + #[test] + fn windows_fall_back_to_the_defaults() { + // An agent configured with nothing at all still gets the large static windows + // (spec:FLOW#common-windows). + assert_eq!( + resolve_windows(None, None, None), + ResolvedWindows { + stream: 6 * 1024 * 1024, + connection: 15 * 1024 * 1024, + } + ); + } + + #[test] + fn the_common_windows_apply_when_a_protocol_says_nothing() { + assert_eq!( + resolve_windows(Some(&common(Some(1024), Some(4096))), None, None), + ResolvedWindows { + stream: 1024, + connection: 4096, + } + ); + } + + #[test] + fn a_protocol_window_beats_the_common_one() { + // The whole point of the per-protocol group: tune one protocol against the other + // (spec:FLOW#per-protocol-windows). + assert_eq!( + resolve_windows( + Some(&common(Some(1024), Some(4096))), + Some(2048), + Some(8192) + ), + ResolvedWindows { + stream: 2048, + connection: 8192, + } + ); + } + + #[test] + fn each_window_falls_back_on_its_own() { + // Overriding the stream window for one protocol leaves that protocol's connection + // window on the common value, rather than dropping it to the default. + assert_eq!( + resolve_windows(Some(&common(Some(1024), Some(4096))), Some(2048), None), + ResolvedWindows { + stream: 2048, + connection: 4096, + } + ); + assert_eq!( + resolve_windows(Some(&common(None, None)), None, Some(8192)), + ResolvedWindows { + stream: DEFAULT_STREAM_WINDOW, + connection: 8192, + } + ); + } + + #[test] + fn a_protocol_window_applies_without_the_common_group() { + assert_eq!( + resolve_windows(None, Some(2048), None), + ResolvedWindows { + stream: 2048, + connection: DEFAULT_CONNECTION_WINDOW, + } + ); + } + + #[test] + fn the_two_protocols_resolve_independently() { + // One `flowControl` value covers both protocols, and overriding it for HTTP/3 leaves + // HTTP/2 where it was (spec:FLOW#per-protocol-windows). + let flow = common(Some(1024), Some(4096)); + let http2 = resolve_windows(Some(&flow), None, None); + let http3 = resolve_windows(Some(&flow), Some(2048), None); + + assert_eq!(http2.stream, 1024); + assert_eq!(http3.stream, 2048); + assert_eq!(http2.connection, http3.connection); + } +} From 2105ad37da87988d9e656b1f61c426e1659030c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:32:42 +1200 Subject: [PATCH 52/61] S1: record the clone-sharing gap the new agent test found --- .workhorse/plans/s1/plan.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 2fa4c3a..4bb43f4 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -97,11 +97,17 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al delegating. `json` is generic over what it deserialises into. - [x] `http_body::Body` for the body, and `Response::into_http`. Fallible, since taking the body can find it already being consumed. - - [ ] **`Agent::builder()` — the big remaining piece.** ~13 option groups as nested builders, and - the ~440 lines of validation currently in `web-faith-napi` that turn `AgentOptions` into a - recipe. Move that logic rather than rewrite it: both surfaces must land on the same defaults, - and a second implementation is how they drift. `Agent::new()` then follows, and the recipe - structs' fields close up behind the builder. + - [x] The option groups and the ~440-line validation moved to `web_faith::options` and + `Agent::from_options`, so both surfaces settle defaults in one place. `Agent::new()` follows. + - [ ] **`close()` and `network_changed()` must act on the agent, not the handle.** A test written + against [RSAPI](../../specs/rust/client-api.md)'s "every clone names the same underlying + agent" fails: `close()` nulls per-clone fields, so a clone does not see it. The Node surface + never needed this — JavaScript holds one object, and `fetch` cloning the agent per request is + what lets an in-flight request finish. For Rust the closeable state (client, raw client, + resolver, prober, Alt-Svc cache) has to sit behind a shared cell, with a request taking its + own handle at the moment it is issued, which is what RSAPI already describes. ~40 read sites. + - [ ] `Agent::builder()` — the nested builders RSAPI asks for, as sugar over the option groups, + after which the recipe structs' fields can close up. - [ ] `Request`, `Request::new`, `try_clone`, and the fetch builder over `IntoFuture`, with the layering rules (outermost wins; headers merge by name). - [ ] Setters taking anything convertible, holding a failed conversion until the builder resolves. From dd9b7e553600aa602e6a3a6c7490790aac8f049c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 25 Aug 2026 12:59:55 +1200 Subject: [PATCH 53/61] S1: update 4 files --- .workhorse/plans/s1/plan.md | 14 +- crates/web-faith-napi/src/fetch.rs | 8 +- crates/web-faith/src/agent.rs | 317 ++++++++++++++++++----------- crates/web-faith/src/request.rs | 12 +- 4 files changed, 225 insertions(+), 126 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 4bb43f4..fe7e99a 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -99,13 +99,13 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al can find it already being consumed. - [x] The option groups and the ~440-line validation moved to `web_faith::options` and `Agent::from_options`, so both surfaces settle defaults in one place. `Agent::new()` follows. - - [ ] **`close()` and `network_changed()` must act on the agent, not the handle.** A test written - against [RSAPI](../../specs/rust/client-api.md)'s "every clone names the same underlying - agent" fails: `close()` nulls per-clone fields, so a clone does not see it. The Node surface - never needed this — JavaScript holds one object, and `fetch` cloning the agent per request is - what lets an in-flight request finish. For Rust the closeable state (client, raw client, - resolver, prober, Alt-Svc cache) has to sit behind a shared cell, with a request taking its - own handle at the moment it is issued, which is what RSAPI already describes. ~40 read sites. + - [x] **`close()` and `network_changed()` act on the agent, not the handle.** The closeable state + sits in a `Live` behind a shared lock, so every clone sees a close, as + [RSAPI](../../specs/rust/client-api.md) requires. A request takes its handle at the moment it + is issued — `request::send` is given the client rather than reaching for it — which is what + [AGENT](../../specs/agent/overview.md) means by in flight from the moment it is issued. The + JS suite caught the difference: capturing inside the promise instead of at issue stranded a + request that was issued just before a close. - [ ] `Agent::builder()` — the nested builders RSAPI asks for, as sugar over the option groups, after which the recipe structs' fields can close up. - [ ] `Request`, `Request::new`, `try_clone`, and the fetch builder over `IntoFuture`, with the diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index 2ba2627..51b6d5a 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -10,6 +10,7 @@ use web_faith::request::{self, RequestBody}; use crate::{ async_task::faith_promise, + error::FaithErrorKind, options::{self, FaithOptionsAndBody}, response::FaithResponse, stream_body::StreamBody, @@ -24,6 +25,10 @@ pub fn faith_fetch<'env>( stream_body: Option<&StreamBody>, ) -> Result, napi::Error> { let (options, agent, body) = options::extract(options); + // Taken here, while `fetch()` is still on the stack, so the request counts as in flight from + // the moment it was issued: closing the agent afterwards does not strand it. + // spec:AGENT + let client = agent.inner.client(); let (s, abort) = mpsc::channel(8); let has_signal = signal.is_some(); if let Some(signal) = signal { @@ -52,7 +57,8 @@ pub fn faith_fetch<'env>( let _ = abort.recv().await; }); - request::send(&agent.inner, &url, options, body, abort) + let client = client.ok_or(FaithErrorKind::Closed)?; + request::send(&agent.inner, client, &url, options, body, abort) .await .map(FaithResponse::from) }) diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 6b578d8..fe9605a 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -7,7 +7,7 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, str::FromStr, sync::{ - Arc, + Arc, RwLock, atomic::{AtomicU64, Ordering}, }, time::Duration, @@ -71,27 +71,44 @@ pub struct AgentSettings { pub has_default_priority: bool, } -/// An HTTP client with its own connection pool, caches, and resolver. +/// What an agent holds while it is open, and gives up when it is closed. /// -/// Cloning one is cheap and every clone names the same underlying agent, which is what lets a -/// request take a handle of its own without opening a second pool. -#[derive(Debug, Clone)] -pub struct Agent { - /// `None` once [`Agent::close`] has been called. The heavy resources - /// (connection pool, DNS resolver, background tasks) live inside this +/// Behind a shared lock because closing acts on the agent rather than on the handle it was called +/// through: every clone names the same one, so every clone sees the result. +#[derive(Debug)] +pub struct Live { + /// The heavy resources (connection pool, DNS resolver, background tasks) live inside this /// client, so dropping it is what actually releases them. - pub client: Option, - /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A - /// a warm-up sends its synthetic request here rather than through the middleware - /// stack, which bypasses the HTTP cache and the Alt-Svc layer (and so keeps the warm-up out of - /// request accounting), while still pooling the connection foreground requests reuse. `None` - /// once the agent is closed. + pub client: ClientWithMiddleware, + /// The raw `reqwest::Client` underlying [`Self::client`], sharing its connection pool. A warm-up + /// sends its synthetic request here rather than through the middleware stack, which bypasses the + /// HTTP cache and the Alt-Svc layer, and so keeps the warm-up out of request accounting, while + /// still pooling the connection foreground requests reuse. // spec:WARM - pub raw_client: Option, - /// Faith's DNS resolver, shared with [`Self::client`] so [`Self::prefetch_dns`] warms the cache requests - /// read. `None` under the system resolver, where there is no such cache. + pub raw_client: Client, + /// The DNS resolver, shared with the client so a prefetch warms the cache requests read. `None` + /// under the system resolver, where there is no such cache. // spec:WARM pub dns_resolver: Option, + #[cfg(feature = "http3")] + pub alt_svc_cache: Option>, + /// Held so closing can abort in-flight background probes: each one owns a clone of the raw + /// client, which would otherwise keep the connection pool alive past close for up to the probe + /// timeout. + #[cfg(feature = "http3")] + pub h3_prober: Option>, +} + +/// An HTTP client with its own connection pool, caches, and resolver. +/// +/// Cloning one is cheap and every clone names the same underlying agent, so cloning is how a request +/// gets an agent to run on rather than a way to get a second pool. Because clones share, closing +/// acts on the agent itself and every handle to it sees the result. +// spec:AGENT +#[derive(Debug, Clone)] +pub struct Agent { + /// `None` once [`Agent::close`] has been called. + live: Arc>>, /// Origins with a warm-up connection opened within the pool idle window, so a repeat /// warm-up does no new work. Keyed by `scheme://host:port`; entries expire with the idle /// timeout. @@ -101,21 +118,14 @@ pub struct Agent { /// origin do not open duplicate connections. // spec:WARM pub warming: MokaCache, - /// Bumped by [`Self::network_changed`], so a warm-up that was in flight across the signal does not - /// record its origin as warm: its connection went into the pool that was just dropped. + /// Bumped by [`Self::network_changed`], so a warm-up that was in flight across the signal does + /// not record its origin as warm: its connection went into the pool that was just dropped. // spec:NETCHG#reach-across-the-subsystems pub warm_generation: Arc, + /// The jar outlives a close and stays readable from a closed agent. pub cookie_jar: Option>, pub stats: Arc, pub conn_tracker: Arc, - #[cfg(feature = "http3")] - #[allow(dead_code)] - pub alt_svc_cache: Option>, - /// Held so `close()` can abort in-flight background probes: each one owns a - /// clone of the raw client, which would otherwise keep the connection pool - /// alive past close for up to the probe timeout. - #[cfg(feature = "http3")] - pub h3_prober: Option>, /// Whether an upgrade may follow a port the origin advertised. A request needs it to stop a /// rewritten port from being reported as a redirect. pub h3_follow_advertised_port: bool, @@ -124,30 +134,72 @@ pub struct Agent { // spec:WARM#preconnect #[cfg(feature = "http3")] pub h3_upgrade_enabled: bool, - /// Whether a streaming request body may go out over HTTP/1.x, which the fetch standard - /// otherwise reserves to HTTP/2 and HTTP/3.. + /// Whether a streaming request body may go out over HTTP/1.x, which the fetch standard otherwise + /// reserves to HTTP/2 and HTTP/3. // spec:QUIRK#http-1-x-request-body-streaming pub quirk_h1_request_streaming: bool, - /// The agent's default `Accept-Encoding`, if one was set among its default headers. - /// which decides the codings a response is decoded under when a request adds none of - /// its own (see [`web_faith_encoding`]). + /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides + /// the codings a response is decoded under when a request adds none of its own. pub default_accept_encoding: Option, - /// The agent's default `Content-Encoding`, if one was set among its default headers. - /// A request layers its own coding on top of this rather than - /// displacing it. + /// The agent's default `Content-Encoding`, if one sits among its default headers. A request + /// layers its own coding on top of this rather than displacing it. // spec:ENC pub default_content_encoding: Option, - /// Whether a `Priority` header sits among the agent's default headers. That default wins over the header a - /// request's priority would derive. + /// Whether a `Priority` header sits among the agent's default headers. That default wins over + /// the header a request's priority would derive. pub has_default_priority: bool, - /// How to build this agent's clients, so [`Self::network_changed`] can build them again - ///. Shared rather than cloned per agent clone: every clone builds the same - /// client from the same recipe, and a request takes a handle per send. + /// How to build this agent's clients, so [`Self::network_changed`] can build them again. Shared + /// rather than cloned per handle: every handle builds the same client from the same recipe. // spec:NETCHG pub recipe: Arc, } impl Agent { + /// Take a handle on the client, or `None` once the agent is closed. + /// + /// A request takes its own handle at the moment it is issued, which is what lets one already in + /// flight finish while a later one is refused. + // spec:AGENT + pub fn client(&self) -> Option { + self.live().as_ref().map(|live| live.client.clone()) + } + + /// Take a handle on the raw client a warm-up sends through, or `None` once closed. + pub fn raw_client(&self) -> Option { + self.live().as_ref().map(|live| live.raw_client.clone()) + } + + /// The DNS resolver, if the agent has one of its own and is still open. + pub fn dns_resolver(&self) -> Option { + self.live() + .as_ref() + .and_then(|live| live.dns_resolver.clone()) + } + + #[cfg(feature = "http3")] + fn alt_svc_cache(&self) -> Option> { + self.live() + .as_ref() + .and_then(|live| live.alt_svc_cache.clone()) + } + + #[cfg(feature = "http3")] + fn h3_prober(&self) -> Option> { + self.live().as_ref().and_then(|live| live.h3_prober.clone()) + } + + fn live(&self) -> std::sync::RwLockReadGuard<'_, Option> { + self.live + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn live_mut(&self) -> std::sync::RwLockWriteGuard<'_, Option> { + self.live + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + /// Build an agent from options, validating them into the recipe its clients are built from. /// /// This is what both surfaces land on, so the defaults a caller gets are settled here rather @@ -637,9 +689,15 @@ impl Agent { ); Ok(Self { - client: Some(built.client), - raw_client: Some(built.raw_client), - dns_resolver, + live: Arc::new(RwLock::new(Some(Live { + client: built.client, + raw_client: built.raw_client, + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + #[cfg(feature = "http3")] + h3_prober: built.prober, + }))), // A warm-up connection is warm only as long as the pool keeps it idle, so the record // that an origin is warm expires with that same window. warmed: MokaCache::builder().time_to_live(conn_timeout).build(), @@ -652,10 +710,6 @@ impl Agent { cookie_jar, stats: Default::default(), conn_tracker: ConnectionTracker::new(conn_timeout), - #[cfg(feature = "http3")] - alt_svc_cache, - #[cfg(feature = "http3")] - h3_prober: built.prober, h3_follow_advertised_port: settings.h3_follow_advertised_port, #[cfg(feature = "http3")] h3_upgrade_enabled: recipe.h3_upgrade.enabled, @@ -675,25 +729,26 @@ impl Agent { /// Requests already in flight run to completion. Any new request on a closed /// agent throws a `Closed` error. Calling `close()` more than once is a /// no-op. The cookie store, if any, remains readable through [`Self::cookie_header`]. - pub fn close(&mut self) { + pub fn close(&self) { // Dropping the client releases the reqwest connection pool and the // Hickory resolver task; the alt-svc cache goes with it. The raw client // shares that pool and the resolver, so it goes too, and both are what a // later warm-up checks to refuse with the closed-agent error. - self.client = None; - self.raw_client = None; - self.dns_resolver = None; + // Taken out of the shared cell, so every handle on this agent sees it closed. + let Some(live) = self.live_mut().take() else { + return; + }; + #[cfg(feature = "http3")] - { - // Probes hold a raw client clone; abort them so the pool doesn't - // outlive close by up to the probe timeout. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); - } - self.h3_prober = None; - self.alt_svc_cache = None; + // Probes hold a raw client clone; abort them so the pool doesn't outlive close by up to + // the probe timeout. + if let Some(prober) = &live.h3_prober { + prober.abort_all(); } + + drop(live); } + /// Tell the agent the network underneath it has changed, so it stops deciding from what it /// learned about a network that is gone. /// @@ -711,59 +766,64 @@ impl Agent { /// they hold; the reset shapes what requests started afterwards draw on. Calling it on a /// closed agent does nothing, and calling it repeatedly is harmless. // spec:NETCHG - pub fn network_changed(&mut self) { - // A closed agent has already released all of this. - if self.client.is_none() { - return; - } + pub fn network_changed(&self) { + { + // Held across the rebuild so a close cannot land halfway through it. + let mut guard = self.live_mut(); + // A closed agent has already released all of this. + let Some(live) = guard.as_mut() else { + return; + }; - // reqwest cannot drop pooled connections short of dropping the client, so the client is - // rebuilt from the recipe the agent kept for this. Requests in flight hold their own - // clone of the old client (`fetch` clones the agent per request), so they run to - // completion and the old pool goes when the last of them finishes. - // - // A rebuild that fails leaves the agent on its existing client: the options were already - // validated at construction, so a failure here is not the caller's to answer for, and an - // agent that still works on the old network beats one that works nowhere. - let built = self.recipe.build( - self.cookie_jar.as_ref(), - self.dns_resolver.as_ref(), - #[cfg(feature = "http3")] - self.alt_svc_cache.as_ref(), - ); - if let Ok(built) = built { - #[cfg(feature = "http3")] - { - // Abort probes running on the old client: each holds a clone of it, and their - // answers would describe the path that has just gone away. - if let Some(prober) = &self.h3_prober { - prober.abort_all(); + // reqwest cannot drop pooled connections short of dropping the client, so the client is + // rebuilt from the recipe the agent kept for this. Requests in flight hold the handle + // they took when they were issued, so they run to completion and the old pool goes when + // the last of them finishes. + // + // A rebuild that fails leaves the agent on its existing client: the options were already + // validated at construction, so a failure here is not the caller's to answer for, and an + // agent that still works on the old network beats one that works nowhere. + let built = self.recipe.build( + self.cookie_jar.as_ref(), + live.dns_resolver.as_ref(), + #[cfg(feature = "http3")] + live.alt_svc_cache.as_ref(), + ); + if let Ok(built) = built { + #[cfg(feature = "http3")] + { + // Abort probes running on the old client: each holds a clone of it, and their + // answers would describe the path that has just gone away. + if let Some(prober) = &live.h3_prober { + prober.abort_all(); + } + live.h3_prober = built.prober; + // The sink holds the prober, which has just been replaced along with the client + // it sends on; leaving the old one installed would aim DNS-triggered probes at a + // client that has been dropped. + install_https_sink( + live.dns_resolver.as_ref(), + live.alt_svc_cache.as_ref(), + live.h3_prober.as_ref(), + self.h3_upgrade_enabled, + ); } - self.h3_prober = built.prober; - // The sink holds the prober, which has just been replaced along with the client - // it sends on; leaving the old one installed would aim DNS-triggered probes at a - // client that has been dropped. - install_https_sink( - self.dns_resolver.as_ref(), - self.alt_svc_cache.as_ref(), - self.h3_prober.as_ref(), - self.h3_upgrade_enabled, - ); + live.client = built.client; + live.raw_client = built.raw_client; } - self.client = Some(built.client); - self.raw_client = Some(built.raw_client); - } - // Names resolve afresh against the new network, through that network's own servers: the - // resolver drops what it read off the old one and reads again when next used. Under the - // system resolver there is no resolver here and so nothing to reset (spec:DNS). - if let Some(resolver) = &self.dns_resolver { - resolver.reset(); - } + // Names resolve afresh against the new network, through that network's own servers: the + // resolver drops what it read off the old one and reads again when next used. Under the + // system resolver there is no resolver here and so nothing to reset. + // spec:DNS + if let Some(resolver) = &live.dns_resolver { + resolver.reset(); + } - #[cfg(feature = "http3")] - if let Some(alt_svc_cache) = &self.alt_svc_cache { - alt_svc_cache.network_changed(); + #[cfg(feature = "http3")] + if let Some(alt_svc_cache) = &live.alt_svc_cache { + alt_svc_cache.network_changed(); + } } // The warm-up records describe pooled connections that have just been dropped, so a @@ -831,7 +891,7 @@ impl Agent { /// first use, and empty for an agent using the system resolver. // spec:OBS#resolvers pub fn resolvers(&self) -> Vec { - self.dns_resolver + self.dns_resolver() .as_ref() .map(FaithResolver::resolvers) .unwrap_or_default() @@ -849,7 +909,7 @@ impl Agent { /// Whether [`Self::close`] has been called. pub fn is_closed(&self) -> bool { - self.client.is_none() + self.live().is_none() } /// Warm the DNS cache for `host`, so a later request to it skips the lookup. @@ -869,7 +929,7 @@ impl Agent { return Err(FaithErrorKind::AddressParse.into()); }; - let resolver = self.dns_resolver.clone(); + let resolver = self.dns_resolver(); Ok(async move { if let Some(resolver) = resolver { resolver.prefetch(&host).await; @@ -888,7 +948,7 @@ impl Agent { /// to, or a closed agent, is refused here rather than by the future. // spec:WARM pub fn preconnect(&self, origin: &str) -> Result + use<>, FaithError> { - let Some(raw_client) = self.raw_client.clone() else { + let Some(raw_client) = self.raw_client() else { return Err(FaithErrorKind::Closed.into()); }; @@ -910,11 +970,10 @@ impl Agent { // spec:WARM#preconnect #[cfg(feature = "http3")] let h3_port = self - .alt_svc_cache - .as_ref() + .alt_svc_cache() .filter(|_| self.h3_upgrade_enabled) .and_then(|cache| { - if self.h3_prober.is_some() { + if self.h3_prober().is_some() { cache.confirmed_port(&url) } else { cache.should_use_h3(&url) @@ -1004,4 +1063,34 @@ mod tests { // No jar until the options ask for one. assert!(agent.cookie_jar.is_none()); } + + /// A handle taken before a close still works afterwards, which is what lets a request issued + /// just before the close run to completion. + #[tokio::test] + async fn a_handle_taken_before_a_close_survives_it() { + let agent = Agent::new().expect("default options build an agent"); + let issued = agent.client().expect("an open agent hands out a client"); + + agent.close(); + + assert!(agent.is_closed()); + assert!( + agent.client().is_none(), + "a closed agent hands out no more clients" + ); + // The handle taken earlier is still usable; dropping it is what releases its share. + drop(issued); + } + + /// Closing acts on the agent itself, so every clone sees it. + #[tokio::test] + async fn closing_a_clone_closes_the_agent() { + let mut agent = Agent::new().expect("default options build an agent"); + let clone = agent.clone(); + + agent.close(); + + assert!(agent.is_closed()); + assert!(clone.is_closed(), "a clone names the same agent"); + } } diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs index e697fb2..e1e60be 100644 --- a/crates/web-faith/src/request.rs +++ b/crates/web-faith/src/request.rs @@ -21,6 +21,7 @@ use reqwest::{ header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, tls::TlsInfo, }; +use reqwest_middleware::ClientWithMiddleware; use tokio::sync::Mutex; use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; @@ -77,9 +78,15 @@ pub struct RequestOptions { /// Send a request on `agent`, and build the response it produces. /// +/// `client` is the handle the caller took when the request was issued, rather than one taken here: +/// a request counts as in flight from the moment it is issued, so one issued just before the agent +/// closes runs to completion even though nothing had started on it yet. +/// /// `abort` is an optional future that, resolving first, cancels the request. +// spec:AGENT pub async fn send( agent: &Agent, + client: ClientWithMiddleware, url: &str, options: RequestOptions, body: RequestBody, @@ -128,10 +135,7 @@ pub async fn send( // the one surfaced (spec:RESP#request-timing). let headers_stamp = HeadersStamp::default(); - let mut request = agent - .client - .as_ref() - .ok_or(FaithErrorKind::Closed)? + let mut request = client .request(method, parsed_url.clone()) .with_extension(CacheMode::from(options.cache)) .with_extension(headers_stamp.clone()); From 978748454864de7d07184aadd534fde2ffeaaa13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:14:38 +1200 Subject: [PATCH 54/61] S1: Agent::builder(), so an agent is built a setting at a time The builder's methods mirror the option groups, and a group is reached through a closure so one left alone is absent from the call rather than spelled out as absent, which is the shape RSAPI asks for. Setters take what a Rust caller holds rather than what the option carries: Duration whichever unit the setting is stored in, IpAddr for the local address, anything that converts into a String elsewhere. Milliseconds and seconds saturate rather than wrap on a duration no setting could mean. Tests cover both halves of the absent-rather-than-spelled-out rule: a group reached but not set keeps its other settings unset, and a group never reached stays None. --- crates/web-faith/src/agent.rs | 40 +++ crates/web-faith/src/builder.rs | 572 ++++++++++++++++++++++++++++++++ crates/web-faith/src/lib.rs | 1 + 3 files changed, 613 insertions(+) create mode 100644 crates/web-faith/src/builder.rs diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index fe9605a..e9e6e6f 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -658,6 +658,11 @@ impl Agent { Self::from_options(AgentOptions::default()) } + /// Build an agent a setting at a time. See [`AgentBuilder`](crate::builder::AgentBuilder). + pub fn builder() -> crate::builder::AgentBuilder { + crate::builder::AgentBuilder::default() + } + /// Build an agent from a validated recipe. /// /// The recipe is what a client is built from, and the settings are what each request consults; @@ -1064,6 +1069,41 @@ mod tests { assert!(agent.cookie_jar.is_none()); } + /// The builder reaches every group, and a group left alone stays absent rather than being + /// spelled out as absent. + #[tokio::test] + async fn the_builder_sets_what_it_is_given_and_nothing_else() { + use std::time::Duration; + + let options = Agent::builder() + .user_agent("YourApp/1.2.3") + .dns(|dns| dns.timeout(Duration::from_secs(2)).ndots(3)) + .pool(|pool| pool.max_idle_per_host(8)) + .into_options(); + + assert_eq!(options.user_agent.as_deref(), Some("YourApp/1.2.3")); + let dns = options.dns.expect("the dns group was reached"); + assert_eq!(dns.timeout, Some(2_000)); + assert_eq!(dns.ndots, Some(3)); + // Reached but not set stays unset... + assert_eq!(dns.servers, None); + // ...and a group never reached is absent entirely. + assert!(options.tls.is_none()); + assert!(options.http3.is_none()); + } + + /// What the builder produces is what the agent is built from, so a built agent carries it. + #[tokio::test] + async fn the_builder_builds_an_agent() { + let agent = Agent::builder() + .user_agent("YourApp/1.2.3") + .build() + .expect("the options are valid"); + + assert_eq!(agent.recipe.user_agent, "YourApp/1.2.3"); + assert!(!agent.is_closed()); + } + /// A handle taken before a close still works afterwards, which is what lets a request issued /// just before the close run to completion. #[tokio::test] diff --git a/crates/web-faith/src/builder.rs b/crates/web-faith/src/builder.rs new file mode 100644 index 0000000..3422441 --- /dev/null +++ b/crates/web-faith/src/builder.rs @@ -0,0 +1,572 @@ +//! Building an agent a setting at a time. +//! +//! [`Agent::builder`] returns an [`AgentBuilder`], whose methods mirror the option groups. A group +//! is reached through a closure, so one left alone is absent from the call rather than spelled out +//! as absent: +//! +//! ```no_run +//! # use std::time::Duration; +//! # use web_faith::agent::Agent; +//! let agent = Agent::builder() +//! .user_agent("YourApp/1.2.3") +//! .dns(|dns| dns.timeout(Duration::from_secs(2))) +//! .pool(|pool| pool.max_idle_per_host(8)) +//! .build()?; +//! # Ok::<(), web_faith::FaithError>(()) +//! ``` +//! +//! Durations are `Duration` here whatever unit the setting is carried in, and a setting left unset +//! takes its default. +//! +//! [`Agent::builder`]: crate::agent::Agent::builder + +// spec:AGENT + +use std::{net::IpAddr, time::Duration}; + +use http_cache_reqwest::CacheMode; +use web_faith_cookies::CookieLimits; + +use crate::{ + agent::Agent, + client::RedirectPolicy, + error::FaithError, + options::{ + AgentOptions, CacheOptions, CacheStore, DnsOptions, DnsOverride, FlowControlOptions, + Header, Http2Options, Http3Congestion, Http3Hint, Http3Options, PoolOptions, QuirksOptions, + TimeoutOptions, TlsOptions, + }, +}; + +/// Milliseconds, saturating rather than wrapping on a duration no setting could mean. +fn millis(duration: Duration) -> u32 { + duration.as_millis().try_into().unwrap_or(u32::MAX) +} + +/// Whole seconds, rounded down, saturating as above. +fn secs(duration: Duration) -> u32 { + duration.as_secs().try_into().unwrap_or(u32::MAX) +} + +/// Builds an [`Agent`]. See [the module documentation](self). +#[derive(Debug, Default)] +#[must_use = "an agent builder does nothing until built"] +pub struct AgentBuilder { + options: AgentOptions, +} + +impl AgentBuilder { + /// Validate what has been set and build the agent. + pub fn build(self) -> Result { + Agent::from_options(self.options) + } + + /// The options as they stand, for a caller that would rather fill them in directly. + pub fn into_options(self) -> AgentOptions { + self.options + } + + /// The `User-Agent` requests carry. Prepend to [`crate::USER_AGENT`] rather than replacing it, + /// so a server still sees which client is calling. + pub fn user_agent(mut self, user_agent: impl Into) -> Self { + self.options.user_agent = Some(user_agent.into()); + self + } + + /// The local address to bind connections to. + pub fn local_address(mut self, address: IpAddr) -> Self { + self.options.local_address = Some(address.to_string()); + self + } + + /// Add a default header, sent on every request that does not set one of that name itself. + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.options.headers.get_or_insert_default().push(Header { + name: name.into(), + value: value.into(), + sensitive: None, + }); + self + } + + /// Add a default header, marking it sensitive so it is kept out of logs and HPACK indexes. + pub fn sensitive_header(mut self, name: impl Into, value: impl Into) -> Self { + self.options.headers.get_or_insert_default().push(Header { + name: name.into(), + value: value.into(), + sensitive: Some(true), + }); + self + } + + /// What to do with a redirect response. + pub fn redirect(mut self, policy: RedirectPolicy) -> Self { + self.options.redirect = Some(policy); + self + } + + /// Keep a cookie jar, enforcing these limits. Without this call the agent stores no cookies. + pub fn cookies(mut self, limits: CookieLimits) -> Self { + self.options.cookies = Some(limits); + self + } + + /// Resolver settings. + pub fn dns(mut self, with: impl FnOnce(DnsBuilder) -> DnsBuilder) -> Self { + let group = self.options.dns.take().unwrap_or_default(); + self.options.dns = Some(with(DnsBuilder { group }).group); + self + } + + /// TLS settings. + pub fn tls(mut self, with: impl FnOnce(TlsBuilder) -> TlsBuilder) -> Self { + let group = self.options.tls.take().unwrap_or_default(); + self.options.tls = Some(with(TlsBuilder { group }).group); + self + } + + /// Connection pool settings. + pub fn pool(mut self, with: impl FnOnce(PoolBuilder) -> PoolBuilder) -> Self { + let group = self.options.pool.take().unwrap_or_default(); + self.options.pool = Some(with(PoolBuilder { group }).group); + self + } + + /// Request timeouts. + pub fn timeout(mut self, with: impl FnOnce(TimeoutBuilder) -> TimeoutBuilder) -> Self { + let group = self.options.timeout.take().unwrap_or_default(); + self.options.timeout = Some(with(TimeoutBuilder { group }).group); + self + } + + /// HTTP cache settings. Without this call the agent does not cache. + pub fn cache(mut self, with: impl FnOnce(CacheBuilder) -> CacheBuilder) -> Self { + let group = self.options.cache.take().unwrap_or_default(); + self.options.cache = Some(with(CacheBuilder { group }).group); + self + } + + /// Flow-control windows applied to both protocols, unless one overrides them. + pub fn flow_control( + mut self, + with: impl FnOnce(FlowControlBuilder) -> FlowControlBuilder, + ) -> Self { + let group = self.options.flow_control.take().unwrap_or_default(); + self.options.flow_control = Some(with(FlowControlBuilder { group }).group); + self + } + + /// HTTP/2 settings. + pub fn http2(mut self, with: impl FnOnce(Http2Builder) -> Http2Builder) -> Self { + let group = self.options.http2.take().unwrap_or_default(); + self.options.http2 = Some(with(Http2Builder { group }).group); + self + } + + /// HTTP/3 settings. + pub fn http3(mut self, with: impl FnOnce(Http3Builder) -> Http3Builder) -> Self { + let group = self.options.http3.take().unwrap_or_default(); + self.options.http3 = Some(with(Http3Builder { group }).group); + self + } + + /// Departures from standard behaviour, each opted into deliberately. + pub fn quirks(mut self, with: impl FnOnce(QuirksBuilder) -> QuirksBuilder) -> Self { + let group = self.options.quirks.take().unwrap_or_default(); + self.options.quirks = Some(with(QuirksBuilder { group }).group); + self + } +} + +/// Resolver settings. Reached through [`AgentBuilder::dns`]. +#[derive(Debug, Default)] +#[must_use] +pub struct DnsBuilder { + group: DnsOptions, +} + +impl DnsBuilder { + /// Resolve through the operating system rather than Faith's own resolver. + pub fn system(mut self, system: bool) -> Self { + self.group.system = Some(system); + self + } + + /// Resolve `domain` to these addresses without asking a server. + pub fn r#override( + mut self, + domain: impl Into, + addresses: impl IntoIterator>, + ) -> Self { + self.group + .overrides + .get_or_insert_default() + .push(DnsOverride { + domain: domain.into(), + addresses: addresses.into_iter().map(Into::into).collect(), + }); + self + } + + /// The resolvers to query, in order. Each is a URL whose scheme picks the transport. + pub fn servers(mut self, servers: impl IntoIterator>) -> Self { + self.group.servers = Some(servers.into_iter().map(Into::into).collect()); + self + } + + /// Bound resolution across the whole server list. + pub fn timeout(mut self, timeout: Duration) -> Self { + self.group.timeout = Some(millis(timeout)); + self + } + + /// Suffixes to try for an unqualified name. + pub fn search_domains(mut self, domains: impl IntoIterator>) -> Self { + self.group.search_domains = Some(domains.into_iter().map(Into::into).collect()); + self + } + + /// How many dots a name must contain before it is tried as given, ahead of the search list. + pub fn ndots(mut self, ndots: u32) -> Self { + self.group.ndots = Some(ndots); + self + } + + /// Consult the system hosts file. + pub fn hosts_file(mut self, hosts_file: bool) -> Self { + self.group.hosts_file = Some(hosts_file); + self + } + + /// Names to send to the system resolver whatever the rest of the configuration says. + pub fn exempt_domains(mut self, domains: impl IntoIterator>) -> Self { + self.group.exempt_domains = Some(domains.into_iter().map(Into::into).collect()); + self + } + + /// Serve an expired answer while a fresh lookup runs. + pub fn serve_stale(mut self, serve_stale: bool) -> Self { + self.group.serve_stale = Some(serve_stale); + self + } + + /// How far past expiry an answer may still be served. + pub fn max_stale(mut self, max_stale: Duration) -> Self { + self.group.max_stale = Some(millis(max_stale)); + self + } +} + +/// TLS settings. Reached through [`AgentBuilder::tls`]. +#[derive(Debug, Default)] +#[must_use] +pub struct TlsBuilder { + group: TlsOptions, +} + +impl TlsBuilder { + /// Send early data on resumed connections, which trades a round trip for replayability. + pub fn early_data(mut self, early_data: bool) -> Self { + self.group.early_data = Some(early_data); + self + } + + /// A PEM certificate and private key to present as a client certificate. + pub fn identity(mut self, pem: impl Into>) -> Self { + self.group.identity = Some(pem.into()); + self + } + + /// Trust this PEM root on top of the platform's trust store. + pub fn extra_root(mut self, pem: impl Into>) -> Self { + self.group + .extra_roots + .get_or_insert_default() + .push(pem.into()); + self + } + + /// Refuse a connection that cannot be made over TLS. + pub fn required(mut self, required: bool) -> Self { + self.group.required = Some(required); + self + } +} + +/// Connection pool settings. Reached through [`AgentBuilder::pool`]. +#[derive(Debug, Default)] +#[must_use] +pub struct PoolBuilder { + group: PoolOptions, +} + +impl PoolBuilder { + /// How long a connection may sit idle before it is closed. + pub fn idle_timeout(mut self, idle_timeout: Duration) -> Self { + self.group.idle_timeout = Some(secs(idle_timeout)); + self + } + + /// Most idle connections to keep per host. + pub fn max_idle_per_host(mut self, max: u32) -> Self { + self.group.max_idle_per_host = Some(max); + self + } +} + +/// Request timeouts. Reached through [`AgentBuilder::timeout`]. +#[derive(Debug, Default)] +#[must_use] +pub struct TimeoutBuilder { + group: TimeoutOptions, +} + +impl TimeoutBuilder { + /// Bound the connect phase alone. + pub fn connect(mut self, timeout: Duration) -> Self { + self.group.connect = Some(millis(timeout)); + self + } + + /// Bound each read. + pub fn read(mut self, timeout: Duration) -> Self { + self.group.read = Some(millis(timeout)); + self + } + + /// Bound the whole request and response. + pub fn total(mut self, timeout: Duration) -> Self { + self.group.total = Some(millis(timeout)); + self + } +} + +/// HTTP cache settings. Reached through [`AgentBuilder::cache`]. +#[derive(Debug, Default)] +#[must_use] +pub struct CacheBuilder { + group: CacheOptions, +} + +impl CacheBuilder { + /// Where cached responses are kept. + pub fn store(mut self, store: CacheStore) -> Self { + self.group.store = Some(store); + self + } + + /// For an in-memory store, how many entries to keep. + pub fn capacity(mut self, capacity: u32) -> Self { + self.group.capacity = Some(capacity); + self + } + + /// The default cache mode for requests that name none. + pub fn mode(mut self, mode: CacheMode) -> Self { + self.group.mode = Some(mode); + self + } + + /// For a disk store, the directory to keep it in. + pub fn path(mut self, path: impl Into) -> Self { + self.group.path = Some(path.into()); + self + } + + /// Behave as a shared cache rather than a private one. + pub fn shared(mut self, shared: bool) -> Self { + self.group.shared = Some(shared); + self + } +} + +/// Flow-control windows. Reached through [`AgentBuilder::flow_control`]. +#[derive(Debug, Default)] +#[must_use] +pub struct FlowControlBuilder { + group: FlowControlOptions, +} + +impl FlowControlBuilder { + /// Bytes an origin may send on any one stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } +} + +/// HTTP/2 settings. Reached through [`AgentBuilder::http2`]. +#[derive(Debug, Default)] +#[must_use] +pub struct Http2Builder { + group: Http2Options, +} + +impl Http2Builder { + /// Bytes an origin may send on any one HTTP/2 stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one HTTP/2 connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } + + /// Let the windows size themselves from what the connection is actually doing. + pub fn adaptive_window(mut self, adaptive: bool) -> Self { + self.group.adaptive_window = Some(adaptive); + self + } +} + +/// HTTP/3 settings. Reached through [`AgentBuilder::http3`]. +#[derive(Debug, Default)] +#[must_use] +pub struct Http3Builder { + group: Http3Options, +} + +impl Http3Builder { + /// The congestion controller QUIC runs. + pub fn congestion(mut self, congestion: Http3Congestion) -> Self { + self.group.congestion = Some(congestion); + self + } + + /// Inactivity to accept before timing out a connection. Rounded down to whole seconds, and + /// capped at what the setting can carry. + pub fn max_idle_timeout(mut self, timeout: Duration) -> Self { + self.group.max_idle_timeout = Some(timeout.as_secs().try_into().unwrap_or(u8::MAX)); + self + } + + /// Whether an origin advertising HTTP/3 is upgraded to it at all. + pub fn upgrade_enabled(mut self, enabled: bool) -> Self { + self.group.upgrade_enabled = Some(enabled); + self + } + + /// Verify an advertisement with a background probe rather than on the next request. + pub fn upgrade_probe(mut self, probe: bool) -> Self { + self.group.upgrade_probe = Some(probe); + self + } + + /// Ceiling on how long a background probe may take. + pub fn upgrade_probe_timeout(mut self, timeout: Duration) -> Self { + self.group.upgrade_probe_timeout = Some(millis(timeout)); + self + } + + /// Demote an origin off HTTP/3 when its QUIC path is slower than its TCP one by this factor. + pub fn upgrade_slow_factor(mut self, factor: f64) -> Self { + self.group.upgrade_slow_factor = Some(factor); + self + } + + /// How long a path-time demotion holds before the origin is re-evaluated. + pub fn upgrade_slow_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_slow_ttl = Some(secs(ttl)); + self + } + + /// How long to cache an advertisement before the first HTTP/3 attempt. + pub fn upgrade_advertised_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_advertised_ttl = Some(secs(ttl)); + self + } + + /// How long to cache a confirmed working HTTP/3 connection. + pub fn upgrade_confirmed_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_confirmed_ttl = Some(secs(ttl)); + self + } + + /// How long a first failed attempt blocks an origin. + pub fn upgrade_failed_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_failed_ttl = Some(secs(ttl)); + self + } + + /// Ceiling on the cooldown consecutive failures double out to. + pub fn upgrade_failed_max_ttl(mut self, ttl: Duration) -> Self { + self.group.upgrade_failed_max_ttl = Some(secs(ttl)); + self + } + + /// How many consecutive cancelled attempts demote an origin. + pub fn upgrade_cancel_strikes(mut self, strikes: u32) -> Self { + self.group.upgrade_cancel_strikes = Some(strikes); + self + } + + /// Ceiling on how long an attempt may take to resolve before it is given up on. + pub fn upgrade_attempt_timeout(mut self, timeout: Duration) -> Self { + self.group.upgrade_attempt_timeout = Some(millis(timeout)); + self + } + + /// Connect to a port an origin advertised, rather than only to its own. + pub fn upgrade_follow_advertised_port(mut self, follow: bool) -> Self { + self.group.upgrade_follow_advertised_port = Some(follow); + self + } + + /// How many origins to track in the Alt-Svc cache. + pub fn upgrade_cache_capacity(mut self, capacity: u32) -> Self { + self.group.upgrade_cache_capacity = Some(capacity); + self + } + + /// Treat HTTP/3 as available at this host and port without waiting for an advertisement. + pub fn hint(mut self, host: impl Into, port: u16) -> Self { + self.group.hints.get_or_insert_default().push(Http3Hint { + host: host.into(), + port, + }); + self + } + + /// Bytes an origin may send on any one HTTP/3 stream before waiting. + pub fn stream_window(mut self, bytes: u32) -> Self { + self.group.stream_window = Some(bytes); + self + } + + /// Bytes an origin may send across all streams of one HTTP/3 connection before waiting. + pub fn connection_window(mut self, bytes: u32) -> Self { + self.group.connection_window = Some(bytes); + self + } + + /// Bytes to transmit without acknowledgement, bounding upload memory. + pub fn send_window(mut self, bytes: u32) -> Self { + self.group.send_window = Some(bytes); + self + } +} + +/// Departures from standard behaviour. Reached through [`AgentBuilder::quirks`]. +#[derive(Debug, Default)] +#[must_use] +pub struct QuirksBuilder { + group: QuirksOptions, +} + +impl QuirksBuilder { + /// Send a streaming request body over HTTP/1.x, which the fetch standard otherwise reserves to + /// HTTP/2 and HTTP/3. + pub fn h1_request_streaming(mut self, allow: bool) -> Self { + self.group.h1_request_streaming = Some(allow); + self + } +} diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index d1b0604..7a54602 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -19,6 +19,7 @@ pub mod agent; pub mod body; +pub mod builder; pub mod client; pub mod error; pub mod integrity; From 08f35ec40de12ed95ee49abff1b97d38239d6efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 25 Aug 2026 13:24:04 +1200 Subject: [PATCH 55/61] S1: update 6 files --- .workhorse/plans/s1/plan.md | 21 +- crates/web-faith-napi/src/agent.rs | 11 +- crates/web-faith/src/agent.rs | 43 +- crates/web-faith/src/error.rs | 8 + crates/web-faith/src/lib.rs | 23 +- crates/web-faith/src/request.rs | 689 ++++++++++++++++++++++++++++- 6 files changed, 751 insertions(+), 44 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index fe7e99a..462ea44 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -88,8 +88,9 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al optional abort future; `fetch.rs` is 59 lines converting a `fetch()` call into those. - **Left for step 9:** the recipe structs' public fields, which the builder should own the assembly of. -- [ ] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md). - **In progress:** +- [x] **9. Build the fetch-flavoured client API** per [RSAPI](../../specs/rust/client-api.md). + A Rust caller now reaches everything through the client: `Agent::new`/`builder`, `agent.fetch`, + `Request`, and the response's own reads. - [x] `USER_AGENT` is the client's, composed from its own version and reqwest's, which `web-faith` now reads in a build script of its own. The binding's constant reads from it. - [x] Reading a response: the accessors and `bytes`/`text`/`json`/`body_stream`/`discard`/ @@ -106,8 +107,20 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al [AGENT](../../specs/agent/overview.md) means by in flight from the moment it is issued. The JS suite caught the difference: capturing inside the promise instead of at issue stranded a request that was issued just before a close. - - [ ] `Agent::builder()` — the nested builders RSAPI asks for, as sugar over the option groups, - after which the recipe structs' fields can close up. + - [x] `Agent::builder()` with nested builders reached through a closure, so a group left alone is + absent from the call. Setters take `Duration` and `IpAddr` rather than the units and strings + the options carry. + - [x] `Request`, `Request::new`, `try_clone`, and `agent.fetch(target)` over `IntoFuture`, with the + layering rules: outermost explicit value wins, untouched settings inherit, headers merge by + name and a removal clears what is underneath, the URL comes from the bottom of the stack. + - [x] Setters take the canonical `http` types or anything converting into them, and a failed + conversion is held until the builder resolves — at `build()` for a request, at the await for + a fetch. The first failure met is the one reported. + - [x] `Agent::cookies()` hands back the jar itself. + - [x] `http::Request` as a target, bringing its method, URL, headers, and body across. + - [ ] Remaining: closing up the recipe and option structs' public fields now that the builder owns + the assembly. Left deliberately: the binding still fills the option structs directly, so + these close up when step 10 settles what the public surface is. - [ ] `Request`, `Request::new`, `try_clone`, and the fetch builder over `IntoFuture`, with the layering rules (outermost wins; headers merge by name). - [ ] Setters taking anything convertible, holding a failed conversion until the builder resolves. diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 66ad63b..e6738e7 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -910,7 +910,11 @@ impl Agent { return; }; - self.inner.add_cookie(&url, &cookie); + let Some(jar) = self.inner.cookies() else { + return; + }; + + jar.add_cookie_str(&cookie, &url); } /// Retrieve a cookie from the store. @@ -923,7 +927,10 @@ impl Agent { #[napi] pub fn get_cookie(&self, url: String) -> Option { let url = Url::from_str(&url).ok()?; - self.inner.cookie_header(&url) + self.inner + .cookies()? + .request_cookie_header(&url) + .and_then(|value| value.to_str().ok().map(ToOwned::to_owned)) } /// Returns statistics gathered by this agent: diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index e9e6e6f..f4d53ce 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -155,6 +155,16 @@ pub struct Agent { } impl Agent { + /// The agent's cookie jar, if it keeps one. + /// + /// The jar itself, rather than per-cookie methods wrapped around it, so cookies go in and out + /// through the type `web-faith-cookies` documents. It outlives a close and stays readable from a + /// closed agent. + // spec:COOK + pub fn cookies(&self) -> Option<&Arc> { + self.cookie_jar.as_ref() + } + /// Take a handle on the client, or `None` once the agent is closed. /// /// A request takes its own handle at the moment it is issued, which is what lets one already in @@ -733,7 +743,7 @@ impl Agent { /// /// Requests already in flight run to completion. Any new request on a closed /// agent throws a `Closed` error. Calling `close()` more than once is a - /// no-op. The cookie store, if any, remains readable through [`Self::cookie_header`]. + /// no-op. The cookie jar, if any, remains readable through [`Self::cookies`]. pub fn close(&self) { // Dropping the client releases the reqwest connection pool and the // Hickory resolver task; the alt-svc cache goes with it. The raw client @@ -840,37 +850,6 @@ impl Agent { self.warm_generation.fetch_add(1, Ordering::Relaxed); } - /// Add a cookie into the agent. - /// - /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying - /// the scheme and host they read, so this does nothing if: - /// - the cookie store is disabled - /// /// - the cookie does not parse - /// - a `__Host-` or `__Secure-` name prefix is not satisfied - /// - the cookie is larger than the jar's size limit - pub fn add_cookie(&self, url: &Url, cookie: &str) { - let Some(jar) = &self.cookie_jar else { - return; - }; - - jar.add_cookie_str(cookie, url); - } - - /// Retrieve a cookie from the store. - /// - /// `None` if: - /// - there's no cookie at this url - /// - the cookie store is disabled - /// /// - the cookie cannot be represented as a string - pub fn cookie_header(&self, url: &Url) -> Option { - let Some(jar) = &self.cookie_jar else { - return None; - }; - - jar.request_cookie_header(url) - .and_then(|val| val.to_str().ok().map(ToOwned::to_owned)) - } - /// The counters this agent has gathered, as they stand. pub fn stats(&self) -> AgentStats { self.stats.snapshot() diff --git a/crates/web-faith/src/error.rs b/crates/web-faith/src/error.rs index f79c910..83f6631 100644 --- a/crates/web-faith/src/error.rs +++ b/crates/web-faith/src/error.rs @@ -124,6 +124,14 @@ fn faith_kind_in_chain(err: &(dyn Error + 'static)) -> Option { None } +/// A conversion that cannot fail still has to satisfy the bound on a target, and this is how it +/// does: there is no value to convert. +impl From for FaithError { + fn from(never: std::convert::Infallible) -> Self { + match never {} + } +} + impl From for FaithError { fn from(err: reqwest::Error) -> Self { // Always include full error chain for debugging diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index 7a54602..f41b030 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -9,13 +9,26 @@ //! [`FaithErrorKind`] is the stable code to match on: a component crate names its own errors, and //! they are converted at the boundary as they cross into the client. //! -//!
    +//! ```no_run +//! # use web_faith::agent::Agent; +//! # async fn example() -> Result<(), web_faith::FaithError> { +//! let agent = Agent::new()?; +//! let body = agent.fetch("https://example.com/").await?.text().await?; +//! # Ok(()) +//! # } +//! ``` //! -//! The caller-facing API is still being shaped. Everything the client does is here -- building an -//! agent, sending a request, reading a response -- but it is reached through [`request::send`] and -//! the modules below rather than through the fetch-flavoured builder that will front it. +//! An [`Agent`] owns the connection pool, resolver, cookie jar, and caches; [`Agent::builder`] +//! configures one. Cloning an agent is cheap and every clone names the same one. //! -//!
    +//! [`Agent::fetch`] returns a builder that sends when awaited, so there is no separate send step. +//! [`Request`] prepares one without sending it, to adjust at each call site or send unchanged on +//! more than one agent. +//! +//! [`Agent`]: agent::Agent +//! [`Agent::builder`]: agent::Agent::builder +//! [`Agent::fetch`]: agent::Agent::fetch +//! [`Request`]: request::Request pub mod agent; pub mod body; diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs index e1e60be..a7dcf91 100644 --- a/crates/web-faith/src/request.rs +++ b/crates/web-faith/src/request.rs @@ -3,7 +3,7 @@ // spec:REQ spec:ENC spec:CANCEL use std::{ - future::Future, + future::{Future, IntoFuture}, pin::Pin, sync::{ Arc, @@ -23,6 +23,7 @@ use reqwest::{ }; use reqwest_middleware::ClientWithMiddleware; use tokio::sync::Mutex; +use url::Url; use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; use crate::{ @@ -465,3 +466,689 @@ pub async fn send( version, }) } + +/// What a request is aimed at: a URL, or another request to layer over. +/// +/// Anything that converts into a `url::Url` is a target, as is a [`Request`], which is what lets a +/// prepared request be adjusted at each call site. +// spec:REQ +pub enum Target { + Url(Url), + Request(Box), +} + +impl From for Target { + fn from(url: Url) -> Self { + Self::Url(url) + } +} + +impl From for Target { + fn from(request: Request) -> Self { + Self::Request(Box::new(request)) + } +} + +/// A string target is parsed when the builder resolves, so an unparseable one is reported there +/// rather than by a call that cannot fail. +// spec:REQ +impl TryFrom<&str> for Target { + type Error = FaithError; + + fn try_from(url: &str) -> Result { + Url::parse(url) + .map(Self::Url) + .map_err(|_| FaithErrorKind::InvalidUrl.into()) + } +} + +impl TryFrom for Target { + type Error = FaithError; + + fn try_from(url: String) -> Result { + Self::try_from(url.as_str()) + } +} + +/// An `http::Request` is a target too, so a request built against the wider ecosystem can be sent +/// on an agent: its method, URL, headers, and body come across. +// spec:REQ +impl TryFrom> for Target +where + B: Into, +{ + type Error = FaithError; + + fn try_from(request: http::Request) -> Result { + let (parts, body) = request.into_parts(); + + let url = Url::parse(&parts.uri.to_string()) + .map_err(|_| FaithError::from(FaithErrorKind::InvalidUrl))?; + + let mut headers = Vec::new(); + for (name, value) in parts.headers.iter() { + let Ok(value) = value.to_str() else { + return Err(FaithErrorKind::InvalidHeader.into()); + }; + headers.push((name.to_string(), value.to_owned())); + } + + let body = body.into(); + Ok(Self::Request(Box::new(Request { + url, + options: RequestOptions { + method: Some(parts.method.to_string()), + headers: (!headers.is_empty()).then_some(headers), + ..RequestOptions::default() + }, + body: if body.is_empty() { + RequestBody::None + } else { + RequestBody::Bytes(body) + }, + }))) + } +} + +/// A header the caller set or removed on one layer. +#[derive(Clone, Debug)] +struct HeaderOp { + name: String, + /// `None` removes the name, including whatever the layers beneath contributed for it. + value: Option, +} + +impl std::fmt::Debug for RequestBody { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::None => f.write_str("None"), + Self::Bytes(bytes) => f.debug_tuple("Bytes").field(&bytes.len()).finish(), + Self::Stream(_) => f.write_str("Stream"), + } + } +} + +/// A request prepared but not sent. +/// +/// Inert: it carries no agent, so it can be sent on more than one, and passing it to +/// [`Agent::fetch`] returns a builder that layers over it. +// spec:REQ +#[derive(Debug)] +pub struct Request { + url: Url, + options: RequestOptions, + body: RequestBody, +} + +impl Request { + /// Prepare a request aimed at `target`. + pub fn new(target: T) -> RequestBuilder + where + T: TryInto, + T::Error: Into, + { + RequestBuilder { + layer: Layer::over(target), + } + } + + pub fn url(&self) -> &Url { + &self.url + } + + pub fn options(&self) -> &RequestOptions { + &self.options + } + + /// Copy the request, when its body allows it. + /// + /// `None` when the body is a stream, a stream being consumable once. + // spec:REQ + pub fn try_clone(&self) -> Option { + let body = match &self.body { + RequestBody::None => RequestBody::None, + RequestBody::Bytes(bytes) => RequestBody::Bytes(bytes.clone()), + RequestBody::Stream(_) => return None, + }; + + Some(Self { + url: self.url.clone(), + options: self.options.clone(), + body, + }) + } +} + +/// One layer of settings over a target, which both builders are made of. +struct Layer { + /// The first failure met while setting up, held until the builder resolves. + // spec:REQ + target: Result, + options: RequestOptions, + /// Which single-valued settings this layer set explicitly, so an unset one is inherited rather + /// than overwritten with a default. + set: SetFlags, + headers: Vec, + body: Option, +} + +#[derive(Default)] +struct SetFlags { + cache: bool, + compress: bool, + credentials: bool, + integrity: bool, + method: bool, + priority: bool, + timeout: bool, +} + +impl Layer { + /// Hold a failure until the builder resolves, keeping the first one met. + // spec:REQ + fn fail(&mut self, err: FaithError) { + if self.target.is_ok() { + self.target = Err(err); + } + } + + fn over(target: T) -> Self + where + T: TryInto, + T::Error: Into, + { + Self { + target: target.try_into().map_err(Into::into), + options: RequestOptions::default(), + set: SetFlags::default(), + headers: Vec::new(), + body: None, + } + } + + /// Settle this layer over whatever it wraps: the outermost explicit value wins, a setting this + /// layer does not touch is inherited unchanged, and headers merge by name. + // spec:REQ + fn settle(self) -> Result { + let (url, mut options, body) = match self.target? { + Target::Url(url) => (url, RequestOptions::default(), RequestBody::None), + // The URL comes from the target at the bottom of the stack. + Target::Request(inner) => (inner.url, inner.options, inner.body), + }; + + if self.set.cache { + options.cache = self.options.cache; + } + if self.set.compress { + options.compress = self.options.compress; + } + if self.set.credentials { + options.credentials = self.options.credentials; + } + if self.set.integrity { + options.integrity = self.options.integrity; + } + if self.set.method { + options.method = self.options.method; + } + if self.set.priority { + options.priority = self.options.priority; + } + if self.set.timeout { + options.timeout = self.options.timeout; + } + + let mut headers = options.headers.take().unwrap_or_default(); + for op in self.headers { + headers.retain(|(name, _)| !name.eq_ignore_ascii_case(&op.name)); + if let Some(value) = op.value { + headers.push((op.name, value)); + } + } + options.headers = (!headers.is_empty()).then_some(headers); + + Ok(Request { + url, + options, + body: self.body.unwrap_or(body), + }) + } +} + +/// The setters both builders carry, so a call reads the same way whichever it is written against. +macro_rules! layer_setters { + ($builder:ident) => { + impl $builder { + /// The request method. Defaults to `GET`. + /// + /// Takes a `Method` or anything that converts into one; a value that does not is + /// reported where the builder resolves. + pub fn method(mut self, method: M) -> Self + where + M: TryInto, + { + match method.try_into() { + Ok(method) => { + self.layer.options.method = Some(method.to_string()); + self.layer.set.method = true; + } + Err(_) => self.layer.fail(FaithErrorKind::InvalidMethod.into()), + } + self + } + + /// Set a header, replacing whatever any layer beneath contributed for that name. + /// + /// Takes `HeaderName` and `HeaderValue` or anything that converts into them. + pub fn header(mut self, name: N, value: V) -> Self + where + N: TryInto, + V: TryInto, + { + match (name.try_into(), value.try_into()) { + (Ok(name), Ok(value)) => match value.to_str() { + Ok(value) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: Some(value.to_owned()), + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + }, + _ => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// Set several headers, each applied as a single header would be. + pub fn headers(mut self, headers: impl IntoIterator) -> Self + where + N: TryInto, + V: TryInto, + { + for (name, value) in headers { + self = self.header(name, value); + } + self + } + + /// Remove a header, including whatever the layers beneath contributed for it. + pub fn remove_header(mut self, name: N) -> Self + where + N: TryInto, + { + match name.try_into() { + Ok(name) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: None, + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// A body already in hand. + pub fn body(mut self, body: impl Into) -> Self { + self.layer.body = Some(RequestBody::Bytes(body.into())); + self + } + + /// A body arriving in chunks, which goes out chunked having no length to declare. + pub fn body_stream( + mut self, + body: impl Stream> + Send + 'static, + ) -> Self { + self.layer.body = Some(RequestBody::Stream(Box::pin(body))); + self + } + + /// Bound the whole request and response. + // spec:CANCEL + pub fn timeout(mut self, timeout: Duration) -> Self { + self.layer.options.timeout = Some(timeout); + self.layer.set.timeout = true; + self + } + + /// The digests the response body is expected to match. + // spec:SRI + pub fn integrity(mut self, integrity: impl Into) -> Self { + self.layer.options.integrity = Some(integrity.into()); + self.layer.set.integrity = true; + self + } + + /// Compress the request body in this coding, named by its wire token. + // spec:ENC + pub fn compress(mut self, coding: impl Into) -> Self { + self.layer.options.compress = Some(coding.into()); + self.layer.set.compress = true; + self + } + + /// How the HTTP cache is consulted for this request. + // spec:CACHE + pub fn cache(mut self, mode: CacheMode) -> Self { + self.layer.options.cache = mode; + self.layer.set.cache = true; + self + } + + /// Whether the request carries its credentials. + pub fn credentials(mut self, credentials: Credentials) -> Self { + self.layer.options.credentials = credentials; + self.layer.set.credentials = true; + self + } + + /// How this request ranks against others, as an RFC 9218 urgency. + // spec:REQ#request-priority + pub fn priority(mut self, priority: Priority) -> Self { + self.layer.options.priority = priority.urgency(); + self.layer.set.priority = true; + self + } + } + }; +} + +/// How a request ranks against others. +// spec:REQ#request-priority +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Priority { + /// More urgent than an unmarked request. + High, + /// Less urgent than an unmarked request. + Low, + /// Send no `Priority` header, leaving the request at the scheme's default urgency. + #[default] + Auto, +} + +impl Priority { + fn urgency(self) -> Option<&'static str> { + match self { + Self::High => Some("u=1"), + Self::Low => Some("u=5"), + Self::Auto => None, + } + } +} + +/// Prepares a [`Request`] without sending it. +#[must_use = "a request builder does nothing until built"] +pub struct RequestBuilder { + layer: Layer, +} + +layer_setters!(RequestBuilder); + +impl RequestBuilder { + /// Settle the layers into a request. + /// + /// Where a conversion failed on the way in — an unparseable target, say — it is reported here. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +/// A request bound to an agent, which sends when awaited. +/// +/// There is no separate send step: awaiting is what sends. A builder dropped without being awaited +/// sends nothing, and dropping the future mid-flight cancels the request. +// spec:REQ spec:CANCEL +#[must_use = "a fetch builder sends nothing until awaited"] +pub struct FetchBuilder { + agent: Agent, + layer: Layer, +} + +layer_setters!(FetchBuilder); + +impl FetchBuilder { + /// Settle the layers into the request that would be sent, without sending it. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +impl IntoFuture for FetchBuilder { + type Output = Result; + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(async move { + // Taken before anything is awaited, so the request counts as in flight from here. + // spec:AGENT + let client = self.agent.client().ok_or(FaithErrorKind::Closed)?; + let request = self.layer.settle()?; + + send( + &self.agent, + client, + request.url.as_str(), + request.options, + request.body, + None::>, + ) + .await + }) + } +} + +impl Agent { + /// Aim a request at `target`, to send when awaited. + /// + /// The target is a URL, or a [`Request`] to layer over. Awaiting the builder sends the request; + /// see [`FetchBuilder`]. + // spec:REQ + pub fn fetch(&self, target: T) -> FetchBuilder + where + T: TryInto, + T::Error: Into, + { + FetchBuilder { + agent: self.clone(), + layer: Layer::over(target), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers_of(request: &Request) -> Vec<(String, String)> { + request.options.headers.clone().unwrap_or_default() + } + + /// The outermost layer that set a value wins, whatever the layers beneath said. + #[test] + fn the_outermost_explicit_value_wins() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .timeout(Duration::from_secs(3)) + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(3))); + } + + /// A setting an outer layer does not touch is inherited unchanged. + #[test] + fn an_untouched_setting_is_inherited() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .method("POST") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .integrity("sha256-abc") + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(1))); + assert_eq!(outer.options.method.as_deref(), Some("POST")); + assert_eq!(outer.options.integrity.as_deref(), Some("sha256-abc")); + } + + /// The URL comes from the target at the bottom of the stack. + #[test] + fn wrapping_a_request_carries_its_url_through() { + let inner = Request::new("https://example.com/deep/path?q=1") + .build() + .expect("a valid target"); + let outer = Request::new(inner) + .method("HEAD") + .build() + .expect("layering"); + + assert_eq!(outer.url.as_str(), "https://example.com/deep/path?q=1"); + } + + /// Headers merge by name: an outer value replaces, a name only set inside carries through. + #[test] + fn headers_merge_by_name() { + let inner = Request::new("https://example.com/") + .header("x-keep", "inner") + .header("x-replace", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .header("x-replace", "outer") + .header("x-add", "outer") + .build() + .expect("layering over a request"); + + let mut headers = headers_of(&outer); + headers.sort(); + assert_eq!( + headers, + vec![ + ("x-add".to_owned(), "outer".to_owned()), + ("x-keep".to_owned(), "inner".to_owned()), + ("x-replace".to_owned(), "outer".to_owned()), + ] + ); + } + + /// Removing a name removes what the layers beneath contributed for it. + #[test] + fn removing_a_header_removes_what_is_underneath() { + let inner = Request::new("https://example.com/") + .header("x-gone", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .remove_header("X-Gone") + .build() + .expect("layering over a request"); + + assert!(headers_of(&outer).is_empty(), "the name was removed"); + } + + /// Setting the same thing twice on one builder is the same question at a smaller scale. + #[test] + fn the_later_call_wins_on_one_builder() { + let request = Request::new("https://example.com/") + .method("POST") + .method("PUT") + .build() + .expect("a valid target"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); + } + + /// An `http::Request` brings its method, URL, headers, and body across. + #[test] + fn an_http_request_is_a_target() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/submit") + .header("x-from", "ecosystem") + .body(Bytes::from_static(b"payload")) + .expect("a valid http request"); + + let request = Request::new(http).build().expect("it converts"); + + assert_eq!(request.url.as_str(), "https://example.com/submit"); + assert_eq!(request.options.method.as_deref(), Some("POST")); + assert_eq!( + headers_of(&request), + vec![("x-from".to_owned(), "ecosystem".to_owned())] + ); + assert!(request.try_clone().is_some(), "a buffered body copies"); + } + + /// A layer over an `http::Request` beats what it carried, as over any other target. + #[test] + fn a_layer_beats_what_an_http_request_carried() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/") + .body(Bytes::new()) + .expect("a valid http request"); + + let request = Request::new(http) + .method(http::Method::PUT) + .build() + .expect("layering over it"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); + } + + /// A header name that is not one is reported where the request is resolved too. + #[test] + fn an_invalid_header_surfaces_at_build() { + let err = Request::new("https://example.com/") + .header("not a header name", "value") + .build() + .expect_err("the name is not a header name"); + + assert_eq!(err.kind, FaithErrorKind::InvalidHeader); + } + + /// The first failure met is the one reported, not the last. + #[test] + fn the_first_failure_is_the_one_reported() { + let err = Request::new("not a url") + .header("not a header name", "value") + .build() + .expect_err("both are wrong"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); + } + + /// An unparseable target is reported where the request is resolved, not by the call that took it. + #[test] + fn an_unparseable_target_surfaces_at_build() { + let builder = Request::new("not a url"); + let err = builder.build().expect_err("the target does not parse"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); + } + + /// A request copies when its body allows it, and reports that it cannot when it does not. + #[test] + fn try_clone_copies_what_it_can() { + let buffered = Request::new("https://example.com/") + .body(Bytes::from_static(b"body")) + .build() + .expect("a valid target"); + assert!(buffered.try_clone().is_some()); + + let streamed = Request::new("https://example.com/") + .body_stream(futures::stream::empty()) + .build() + .expect("a valid target"); + assert!( + streamed.try_clone().is_none(), + "a stream is consumable once" + ); + } +} From e97451be6ad7597f50fdfcd0a44ca77b06d22c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 25 Aug 2026 17:07:20 +1200 Subject: [PATCH 56/61] Update plan --- .workhorse/plans/s1/plan.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 462ea44..c88c76f 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -121,9 +121,6 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [ ] Remaining: closing up the recipe and option structs' public fields now that the builder owns the assembly. Left deliberately: the binding still fills the option structs directly, so these close up when step 10 settles what the public surface is. - - [ ] `Request`, `Request::new`, `try_clone`, and the fetch builder over `IntoFuture`, with the - layering rules (outermost wins; headers merge by name). - - [ ] Setters taking anything convertible, holding a failed conversion until the builder resolves. - [ ] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling one drops the code and the API surface it gates (compile error at the call site, not a no-op), and the dependency too where the capability is a crate. Component, crate, and feature are three axes From 8defbc4b93d2f8773fd343e840a5a9f8f1a76bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:38:05 +1200 Subject: [PATCH 57/61] S1: split the oversized source files into modules No source file past 1000 lines, tests-only files excepted. Each split follows the internal seams: the dns crate into transport/resolver/ discovery/https/settings, alt-svc into cache/header/prober/middleware, the client's agent into build/warm and its request path into send/target/builder, and the binding's agent into options/convert. Tests move with the code they exercise, as a child module of it, which keeps their access to private items. Co-Authored-By: Claude Opus 5 --- .workhorse/plans/s1/plan.md | 12 + crates/web-faith-alt-svc/src/cache.rs | 714 +++++ crates/web-faith-alt-svc/src/cache/tests.rs | 1016 +++++++ crates/web-faith-alt-svc/src/header.rs | 75 + crates/web-faith-alt-svc/src/header/tests.rs | 86 + crates/web-faith-alt-svc/src/lib.rs | 2356 +---------------- crates/web-faith-alt-svc/src/middleware.rs | 278 ++ crates/web-faith-alt-svc/src/prober.rs | 205 ++ crates/web-faith-dns/src/discovery.rs | 178 ++ crates/web-faith-dns/src/https.rs | 110 + crates/web-faith-dns/src/https/tests.rs | 204 ++ crates/web-faith-dns/src/lib.rs | 1533 +---------- crates/web-faith-dns/src/resolver.rs | 488 ++++ crates/web-faith-dns/src/resolver/tests.rs | 178 ++ crates/web-faith-dns/src/settings.rs | 134 + crates/web-faith-dns/src/settings/tests.rs | 51 + crates/web-faith-dns/src/transport.rs | 166 ++ crates/web-faith-dns/src/transport/tests.rs | 82 + crates/web-faith-napi/src/agent.rs | 890 +------ crates/web-faith-napi/src/agent/convert.rs | 138 + crates/web-faith-napi/src/agent/options.rs | 751 ++++++ crates/web-faith/src/agent.rs | 786 +----- crates/web-faith/src/agent/build.rs | 568 ++++ crates/web-faith/src/agent/tests.rs | 78 + crates/web-faith/src/agent/warm.rs | 151 ++ crates/web-faith/src/request.rs | 1112 +------- crates/web-faith/src/request/builder.rs | 427 +++ crates/web-faith/src/request/builder/tests.rs | 197 ++ crates/web-faith/src/request/send.rs | 417 +++ crates/web-faith/src/request/target.rs | 90 + 30 files changed, 6849 insertions(+), 6622 deletions(-) create mode 100644 crates/web-faith-alt-svc/src/cache.rs create mode 100644 crates/web-faith-alt-svc/src/cache/tests.rs create mode 100644 crates/web-faith-alt-svc/src/header.rs create mode 100644 crates/web-faith-alt-svc/src/header/tests.rs create mode 100644 crates/web-faith-alt-svc/src/middleware.rs create mode 100644 crates/web-faith-alt-svc/src/prober.rs create mode 100644 crates/web-faith-dns/src/discovery.rs create mode 100644 crates/web-faith-dns/src/https.rs create mode 100644 crates/web-faith-dns/src/https/tests.rs create mode 100644 crates/web-faith-dns/src/resolver.rs create mode 100644 crates/web-faith-dns/src/resolver/tests.rs create mode 100644 crates/web-faith-dns/src/settings.rs create mode 100644 crates/web-faith-dns/src/settings/tests.rs create mode 100644 crates/web-faith-dns/src/transport.rs create mode 100644 crates/web-faith-dns/src/transport/tests.rs create mode 100644 crates/web-faith-napi/src/agent/convert.rs create mode 100644 crates/web-faith-napi/src/agent/options.rs create mode 100644 crates/web-faith/src/agent/build.rs create mode 100644 crates/web-faith/src/agent/tests.rs create mode 100644 crates/web-faith/src/agent/warm.rs create mode 100644 crates/web-faith/src/request/builder.rs create mode 100644 crates/web-faith/src/request/builder/tests.rs create mode 100644 crates/web-faith/src/request/send.rs create mode 100644 crates/web-faith/src/request/target.rs diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index c88c76f..7c18aa2 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -179,6 +179,18 @@ Decisions taken while doing steps 0–7, worth not relitigating: which would have shipped as broken docs.rs pages. Rust intra-doc syntax in a `web-faith-napi` doc comment is emitted verbatim into `index.d.ts`, where `[`X`]` means nothing, so plain backticks belong on anything a napi item documents. +- **No source file past 1000 lines, tests-only files excepted.** `dns`, `alt-svc`, the client's + agent and request paths, and the binding's agent were each split into modules along their internal + seams. +- **Tests live as a child module of the code they exercise,** `foo/tests.rs` under `foo.rs`, not one + flat `tests.rs` at the crate root. A child module reaches its parent's private items, so the split + costs no visibility: the first attempt widened a dozen internals to `pub(crate)` purely for a + crate-root test module, which is the wrong trade. +- **Splitting a file relocates doc comments as easily as it drops them.** A cut between an item and + its doc block leaves the block dangling at the end of one file and the item bare at the start of + the next, which rustc catches, and a `// spec:` line under a doc block extends how far back the + block starts. Intra-doc links break more quietly: `[`X`]` that resolved within one file needs + `crate::X` once `X` is a sibling module away, and `cargo doc --workspace --no-deps` is what says so. ## Step 14: the both-surfaces spec sweep diff --git a/crates/web-faith-alt-svc/src/cache.rs b/crates/web-faith-alt-svc/src/cache.rs new file mode 100644 index 0000000..3d9f567 --- /dev/null +++ b/crates/web-faith-alt-svc/src/cache.rs @@ -0,0 +1,714 @@ +use std::time::{Duration, Instant}; + +use moka::sync::Cache; + +#[derive(Debug, Clone)] +pub struct AltSvcEntry { + pub port: u16, + pub expires: Instant, +} + +/// An HTTP/3 alternative parsed out of an `Alt-Svc` header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AltSvcAdvertisement { + /// Host the alternative is on. Empty when the header omitted it, which per + /// RFC 7838 means the same host as the origin. + pub host: String, + pub port: u16, + pub max_age: Option, +} + +/// A run of consecutive HTTP/3 failures against one origin. +/// +/// Both instants are carried in the value rather than left to the cache's TTL, +/// because they differ per origin and from each other: the entry deliberately +/// outlives the cooldown it set, so that a count survives the block it caused and +/// can escalate the next one. `advertised` does the same for `ma`. +// spec:H3UP#failure-backoff +#[derive(Debug, Clone, Copy)] +struct FailureEntry { + /// Consecutive failures with no confirmation in between. + count: u32, + /// Until when the origin is blocked from upgrading, probing, and recording + /// advertisements. The only field that gates behaviour. + blocked_until: Instant, + /// Until when `count` still describes a run. Past it the origin is judged + /// from the base cooldown again. + counted_until: Instant, +} + +/// A per-origin exponentially-weighted moving average of time-to-response-headers. +/// +/// Two `f64`s per origin and no sample storage: the average decays stale history +/// by construction, and the count gates decisions until there is enough evidence +/// to mean anything. +#[derive(Debug, Clone, Copy)] +pub struct PathTime { + /// EWMA of time-to-response-headers, in milliseconds. + pub avg_ms: f64, + pub count: u32, +} + +/// Weight of the newest sample in the moving average. +const EWMA_ALPHA: f64 = 0.2; +/// Samples required on *each* side before a slow comparison may act. +const EWMA_MIN_SAMPLES: u32 = 8; +/// Absolute gap the QUIC average must exceed the TCP one by, on top of the +/// factor, so LAN-fast origins don't flap on sub-millisecond noise. +pub const SLOW_FLOOR_MS: f64 = 10.0; + +pub struct AltSvcCacheConfig { + pub advertised_ttl: Duration, + pub confirmed_ttl: Duration, + /// Cooldown a first failure earns; each consecutive one doubles it. + pub failed_ttl: Duration, + /// Ceiling on the doubling. Clamped up to `failed_ttl`, so setting it at or + /// below the base gives a flat cooldown. + pub failed_max_ttl: Duration, + pub capacity: u64, + pub cancel_strikes: u32, + pub strike_window: Duration, + pub follow_advertised_port: bool, + /// Lifetime of a probe's single-flight claim. Doubles as crash recovery: a + /// probe task that dies without reporting frees its origin when this lapses. + pub probe_ttl: Duration, + /// The QUIC path is demoted when its average is worse than TCP's by this + /// factor (and by [`SLOW_FLOOR_MS`] absolutely). `0.0` disables path-time + /// demotion entirely. + pub slow_factor: f64, + /// How long a path-time demotion holds before the origin may be re-probed. + pub slow_ttl: Duration, +} + +#[derive(Clone)] +pub struct AltSvcCache { + advertised: Cache, + confirmed: Cache, + /// Origins that failed over HTTP/3, with their run of consecutive failures. + /// An entry present here is not necessarily blocked: see [`Self::is_failed`]. + failed: Cache, + /// Consecutive cancelled HTTP/3 attempts per origin. Entries expire on a TTL + /// (the strike window), so a run has to be sustained to count. + cancellations: Cache, + /// Single-flight claims for in-flight background probes. + probing: Cache, + /// Origins demoted for being slower over QUIC than over TCP. Distinct from + /// `failed`: the path *works*, so re-advertisements must not be discarded, + /// and expiry re-enters through a probe rather than treating h3 as broken. + slow: Cache, + /// Origins seeded from `http3.hints`, with the port hinted. A hint is the + /// caller's assertion rather than something observed, so it has to be + /// distinguishable from an entry in `confirmed` that a real HTTP/3 response + /// put there: [`Self::network_changed`] demotes the observed ones and + /// re-seeds from here. Unbounded by TTL and outside the capacity bound, + /// because the hints are configuration and there are as many as the caller + /// passed. + // spec:NETCHG#what-the-signal-keeps + hints: Cache, + /// Time-to-headers over TCP (h1 and h2 together), per origin. + tcp_times: Cache, + /// Time-to-headers over QUIC (h3), per origin. + quic_times: Cache, + + advertised_ttl: Duration, + confirmed_ttl: Duration, + failed_ttl: Duration, + failed_max_ttl: Duration, + cancel_strikes: u32, + follow_advertised_port: bool, + slow_factor: f64, +} + +impl std::fmt::Debug for AltSvcCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AltSvcCache") + .field("advertised_count", &self.advertised.entry_count()) + .field("confirmed_count", &self.confirmed.entry_count()) + .field("failed_count", &self.failed.entry_count()) + .field("cancellation_count", &self.cancellations.entry_count()) + .field("probing_count", &self.probing.entry_count()) + .field("slow_count", &self.slow.entry_count()) + .field("hint_count", &self.hints.entry_count()) + .finish() + } +} + +impl AltSvcCache { + pub fn new(config: AltSvcCacheConfig) -> Self { + let AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + capacity, + cancel_strikes, + strike_window, + follow_advertised_port, + probe_ttl, + slow_factor, + slow_ttl, + } = config; + + // A cap below the base would mean the first failure already exceeds it; + // clamping makes that setting a flat cooldown rather than a shorter one. + let failed_max_ttl = failed_max_ttl.max(failed_ttl); + + Self { + advertised: Cache::builder() + .max_capacity(capacity) + .time_to_live(advertised_ttl) + .build(), + confirmed: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + // Twice the longest cooldown: the outer bound on how long an entry + // can be worth keeping, since a count is dropped one cooldown after + // the block it caused lapsed. Per-entry instants do the real work. + failed: Cache::builder() + .max_capacity(capacity) + .time_to_live(failed_max_ttl.saturating_mul(2)) + .build(), + cancellations: Cache::builder() + .max_capacity(capacity) + .time_to_live(strike_window) + .build(), + probing: Cache::builder() + .max_capacity(capacity) + .time_to_live(probe_ttl) + .build(), + slow: Cache::builder() + .max_capacity(capacity) + .time_to_live(slow_ttl) + .build(), + // No TTL and no capacity bound: hints are configuration, held for the + // life of the agent so a network change can re-seed from them. + hints: Cache::builder().build(), + tcp_times: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + quic_times: Cache::builder() + .max_capacity(capacity) + .time_to_live(confirmed_ttl) + .build(), + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + cancel_strikes, + follow_advertised_port, + slow_factor, + } + } + + /// The cooldown the `count`-th consecutive failure earns: the base doubled + /// once per failure before it, capped. + // spec:H3UP#failure-backoff + fn failure_cooldown(&self, count: u32) -> Duration { + let doublings = count.saturating_sub(1).min(u32::BITS - 1); + self.failed_ttl + .saturating_mul(2u32.saturating_pow(doublings)) + .min(self.failed_max_ttl) + } + + /// Whether the origin is inside its failure cooldown. + /// + /// Presence in `failed` is not the question: an entry outlives its cooldown + /// so the failure count survives to escalate the next one. + fn is_failed(&self, origin: &str) -> bool { + self.failed + .get(origin) + .is_some_and(|entry| entry.blocked_until > Instant::now()) + } + + fn origin_key(url: &reqwest::Url) -> Option { + let host = url.host_str()?; + let port = url.port_or_known_default()?; + Some(format!("{}://{}:{}", url.scheme(), host, port)) + } + + pub fn record_alt_svc(&self, url: &reqwest::Url, advertisement: &AltSvcAdvertisement) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // An alternative on a *different host* can never be honoured: reqwest derives + // the HTTP/3 connect target from the request's authority, and rewriting the + // host would also change which certificate is accepted. Unlike a differing + // port — which `follow_advertised_port` can act on — there is nothing to + // gate behind an option, so don't record it at all. RFC 7838 uses an empty + // host to mean "the same host as the origin". + // + // Compared case-insensitively because host names are, and a server naming its + // own host in a different case is still naming its own host. + if !advertisement.host.is_empty() + && !url + .host_str() + .is_some_and(|origin_host| origin_host.eq_ignore_ascii_case(&advertisement.host)) + { + return; + } + + if self.is_failed(&origin) { + return; + } + + if self.confirmed.contains_key(&origin) { + return; + } + + let ttl = advertisement.max_age.unwrap_or(self.advertised_ttl); + let entry = AltSvcEntry { + port: advertisement.port, + expires: Instant::now() + ttl, + }; + + self.advertised.insert(origin, entry); + } + + /// Whether an `HTTPS` DNS record for this origin would tell us anything we do not already + /// know, so the resolver can skip the query rather than send one per lookup. + /// + /// Nothing is learnable while the origin is confirmed (already routing over HTTP/3), failed + /// (blocked whatever a record says), slow (demoted on measurement, which a record cannot + /// overturn), or already carrying a live advertisement (the probe it warrants is already + /// warranted). Each of those states expires, and the query resumes when it does. + // spec:DNS#https-records + pub fn wants_https_record(&self, url: &reqwest::Url) -> bool { + let Some(origin) = Self::origin_key(url) else { + return false; + }; + + !self.is_failed(&origin) + && !self.confirmed.contains_key(&origin) + && !self.slow.contains_key(&origin) + && self + .advertised + .get(&origin) + .is_none_or(|entry| entry.expires <= Instant::now()) + } + + /// Record an HTTP/3 advertisement carried by an `HTTPS` DNS record. + /// + /// An `HTTPS` record and an `Alt-Svc` header are two ways for an origin to say the same thing, + /// so this lands in exactly the state a header advertisement does: the origin becomes + /// probe-worthy, and foreground requests keep to TCP until a probe proves the path. The port + /// and same-host rules are the header's too — [`Self::record_alt_svc`] applies them — because + /// the reasons for them are about what Faith can connect to rather than about where the + /// advertisement was read. + // spec:H3UP#advertisements-from-dns + pub fn record_https_record(&self, url: &reqwest::Url, port: Option, ttl: Duration) { + // A record naming no port describes the origin's own, exactly as an `Alt-Svc` header with + // no alt-authority port would. + let Some(port) = port.or_else(|| url.port_or_known_default()) else { + return; + }; + + self.record_alt_svc( + url, + &AltSvcAdvertisement { + // The same-host case: a record targeting another host is dropped before it gets + // here, since Faith only upgrades to the origin's own host. + host: String::new(), + port, + // The record's own DNS TTL is how long what it says is good for, which is the + // role `ma` plays for a header advertisement. + max_age: Some(ttl), + }, + ); + } + + /// Hints seed `confirmed` directly, not `advertised`: a hint is the *user's* + /// assertion, and routing it through a probe would both second-guess an + /// explicit instruction and break h3-only origins (no TCP listener), which + /// only work if the very first request speaks HTTP/3. Distrust is reserved + /// for what servers advertise. Failure demotes a hinted origin exactly as it + /// does a confirmed one. + pub fn add_hint(&self, host: &str, port: u16) { + let origin = format!("https://{}:{}", host, port); + + // Recorded whether or not it can be acted on right now: the hint is + // configuration, and a failure blocking it is a fact about a path that a + // network change can clear (spec:NETCHG#what-the-signal-keeps). + self.hints.insert(origin.clone(), port); + self.seed_hint(origin, port); + } + + /// Put a hinted origin into `confirmed`, unless a failure currently blocks it. + /// + /// Split out of [`Self::add_hint`] so [`Self::network_changed`] can re-seed the + /// hints it just cleared without re-recording them. + fn seed_hint(&self, origin: String, port: u16) { + if self.is_failed(&origin) { + return; + } + + let entry = AltSvcEntry { + port, + expires: Instant::now() + Duration::from_hours(10_000), // forever + }; + + self.confirmed.insert(origin, entry); + } + + /// Whether an entry advertising `entry_port` can be acted on for this URL. + /// + /// An Alt-Svc advertisement names a network endpoint for the origin; it is not + /// a claim that the origin's *own* port speaks HTTP/3. So when the advertised + /// port differs, upgrading the request on the origin port is an inference the + /// advertisement does not support. + /// + /// Honouring the advertised port properly means connecting to one port while + /// still sending the origin's authority, which reqwest cannot express: it + /// derives the HTTP/3 connect target from the request URI's authority (see + /// ). `follow_advertised_port` + /// opts into doing it anyway by rewriting the request's port, which is not + /// standards-compliant — the request then carries the alternative's authority + /// rather than the origin's. + fn port_actionable(&self, url: &reqwest::Url, entry_port: u16) -> bool { + self.follow_advertised_port || Some(entry_port) == url.port_or_known_default() + } + + /// The port HTTP/3 is *proven* on, or `None` to leave the request on TCP. + /// + /// This is the only lookup foreground routing consults when probing is on: + /// an advertisement is evidence worth probing, not worth routing on. + /// + /// A returned port that differs from the URL's own means the caller opted into + /// `follow_advertised_port` and the request must be rewritten to target it. + pub fn confirmed_port(&self, url: &reqwest::Url) -> Option { + let origin = Self::origin_key(url)?; + + if self.is_failed(&origin) || self.slow.contains_key(&origin) { + return None; + } + + let entry = self.confirmed.get(&origin)?; + if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { + Some(entry.port) + } else { + None + } + } + + /// The advertised port a background probe should verify, or `None` when + /// there is nothing (or no need) to probe: no actionable advertisement, + /// already confirmed, recently failed, or demoted for being slow. + pub fn probe_candidate(&self, url: &reqwest::Url) -> Option { + let origin = Self::origin_key(url)?; + + if self.is_failed(&origin) + || self.slow.contains_key(&origin) + || self.confirmed.contains_key(&origin) + { + return None; + } + + let entry = self.advertised.get(&origin)?; + if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { + Some(entry.port) + } else { + None + } + } + + /// Claim the origin for a probe. Returns `false` when a probe is already in + /// flight; the claim expires on its own (see [`AltSvcCacheConfig::probe_ttl`]) + /// if the prober never reports back. + pub fn claim_probe(&self, url: &reqwest::Url) -> bool { + let Some(origin) = Self::origin_key(url) else { + return false; + }; + self.probing.entry(origin).or_insert(()).is_fresh() + } + + /// Release the origin's probe claim, so a later advertisement can re-probe + /// without waiting out the claim's TTL. + pub fn finish_probe(&self, url: &reqwest::Url) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + self.probing.invalidate(&origin); + } + + /// The port to attempt HTTP/3 on, or `None` to leave the request on TCP. + /// + /// Legacy (probe-less) routing: advertisements are acted on inline, so this + /// consults `advertised` as well as `confirmed`. Only used when + /// probing is off. + pub fn should_use_h3(&self, url: &reqwest::Url) -> Option { + self.confirmed_port(url) + .or_else(|| self.probe_candidate(url)) + } + + /// Record a foreground request's time-to-response-headers for its protocol + /// family, and demote the origin to TCP if QUIC is provenly, sustainedly + /// slower than TCP for it. + /// + /// Time-to-headers includes server think-time, which varies per endpoint far + /// more than per transport; only the averages across many requests are + /// comparable, never individual samples — hence the minimum sample counts. + /// Redirects followed inside the attempt inflate a sample for whichever + /// family carried it, which the averaging absorbs the same way. + /// + /// The comparison is deliberately asymmetric: HTTP/3 is preferred at parity + /// and when moderately slower, because its advantages (no head-of-line + /// blocking, connection migration) pay off beyond the mean. Only a large + /// sustained gap demotes. + pub fn record_path_time(&self, url: &reqwest::Url, version: http::Version, elapsed: Duration) { + if self.slow_factor <= 0.0 { + return; + } + + let Some(origin) = Self::origin_key(url) else { + return; + }; + + let sample_ms = elapsed.as_secs_f64() * 1000.0; + let times = if version == http::Version::HTTP_3 { + &self.quic_times + } else { + &self.tcp_times + }; + + let updated = times + .entry(origin.clone()) + .and_upsert_with(|existing| match existing { + None => PathTime { + avg_ms: sample_ms, + count: 1, + }, + Some(entry) => { + let entry = entry.into_value(); + PathTime { + avg_ms: entry.avg_ms * (1.0 - EWMA_ALPHA) + sample_ms * EWMA_ALPHA, + count: entry.count.saturating_add(1), + } + } + }) + .into_value(); + + if version == http::Version::HTTP_3 + && updated.count >= EWMA_MIN_SAMPLES + && let Some(tcp) = self.tcp_times.get(&origin) + && tcp.count >= EWMA_MIN_SAMPLES + && updated.avg_ms > tcp.avg_ms * self.slow_factor + && updated.avg_ms - tcp.avg_ms > SLOW_FLOOR_MS + { + self.demote_slow(&origin); + } + } + + /// Demote a working-but-slow QUIC origin back to TCP. + /// + /// The confirmed entry moves back to `advertised` rather than being dropped: + /// when the `slow` marker expires, the advertisement is what makes the next + /// request trigger a re-probe — "has this path improved?" asked at zero + /// foreground cost. The QUIC average is cleared so the answer is judged on + /// fresh samples, not held hostage by the history that demoted it. + fn demote_slow(&self, origin: &str) { + let key = origin.to_string(); + let Some(entry) = self.confirmed.get(&key) else { + return; + }; + + self.confirmed.invalidate(&key); + self.advertised.insert( + key.clone(), + AltSvcEntry { + port: entry.port, + expires: Instant::now() + self.advertised_ttl, + }, + ); + self.quic_times.invalidate(&key); + self.slow.insert(key, ()); + } + + /// Record that HTTP/3 worked for this origin, on the port it connected to. + /// + /// `port` must be the port the successful attempt actually used. Recovering it + /// from the caches instead would be unsound: a concurrent failure that cleared + /// them leaves nothing to read, and falling back to the origin's own port would + /// confirm HTTP/3 on a port the server never advertised — for `confirmed_ttl`, + /// and invisibly, since the concurrent failure's `failed` entry masks it until + /// that expires. + pub fn confirm_h3(&self, url: &reqwest::Url, port: u16) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // Promoted out of `advertised`; it has served its purpose. + self.advertised.invalidate(&origin); + // A working h3 response is proof of health; forget any strikes, and end + // whatever run of failures preceded it. + self.cancellations.invalidate(&origin); + self.clear_failure_count(&origin); + + let entry = AltSvcEntry { + port, + expires: Instant::now() + self.confirmed_ttl, + }; + + self.confirmed.insert(origin, entry); + } + + /// Record an HTTP/3 attempt that was cancelled before producing an outcome. + /// + /// This is weaker evidence than an error: the request never got to find out + /// whether HTTP/3 worked, so a single cancellation says nothing about the + /// origin. Only a sustained run of them demotes it, which keeps callers that + /// routinely abort healthy requests from disabling HTTP/3. + /// + /// The window is a TTL measured from the *previous* strike, because moka + /// refreshes an entry's TTL on upsert. Strikes therefore have to arrive + /// within a window of each other, not within a fixed bucket. + pub fn record_h3_cancellation(&self, url: &reqwest::Url) { + if self.cancel_strikes == 0 { + return; + } + + let Some(origin) = Self::origin_key(url) else { + return; + }; + + // This is reachable from a `Drop` impl (see the guard below), which must + // never panic: a panic while already unwinding aborts the process. Use a + // saturating add so an absurd `upgrade_cancel_strikes` can't overflow. + let strikes = self + .cancellations + .entry(origin) + .and_upsert_with(|existing| { + existing.map_or(1, |entry| entry.into_value().saturating_add(1)) + }) + .into_value(); + + if strikes >= self.cancel_strikes { + // Clears the strike count as a side effect. + self.record_h3_failure(url); + } + } + + /// Forget the origin's run of failures, so the next one starts the backoff + /// from the base cooldown again. + /// + /// A cooldown still running is left alone. A confirmation racing a concurrent + /// failure must not unblock the origin that failure just blocked: the failure + /// is the more recent evidence about the path, and [`Self::confirm_h3`] + /// relies on its own entry being masked until the block lapses. + // spec:H3UP#failure-backoff + fn clear_failure_count(&self, origin: &str) { + let Some(entry) = self.failed.get(origin) else { + return; + }; + + if entry.blocked_until > Instant::now() { + self.failed + .insert(origin.to_string(), FailureEntry { count: 0, ..entry }); + } else { + self.failed.invalidate(origin); + } + } + + /// Discard everything this cache learned by observing the network, keeping + /// what it was told. + /// + /// Every state here except `advertised` and `hints` describes the path between + /// this client and an origin, and a network change is exactly the event that + /// invalidates such a description. So the observation-confirmed origins are + /// demoted rather than kept (the path that proved them is gone, and a probe + /// re-proves them without a foreground request paying for it), and the + /// failures, strikes, slow markers and averages go entirely: they are + /// penalties and measurements the old path earned, and carrying them over + /// would judge the new network by the old one's behaviour. + /// + /// What the origin said about itself (`advertised`) and what the caller + /// asserted (`hints`) are not observations, so both survive. + // spec:NETCHG + pub fn network_changed(&self) { + let now = Instant::now(); + + // Demote first, while `confirmed` still holds the entries: an advertisement + // is what makes the next request to the origin trigger a re-probe. + // + // Keys are invalidated one by one rather than with `invalidate_all`, whose + // timestamp-based invalidation would race the hint re-seeding below. + for (origin, entry) in self.confirmed.iter() { + // A hint holds its origin confirmed; it is an assertion, not a finding. + if self.hints.contains_key(origin.as_str()) { + continue; + } + + self.confirmed.invalidate(origin.as_str()); + + // A logically expired entry is not knowledge to carry forward: it would + // come back as a fresh advertisement having just lapsed as a confirmation. + if entry.expires <= now { + continue; + } + + self.advertised.insert( + (*origin).clone(), + AltSvcEntry { + port: entry.port, + expires: now + self.advertised_ttl, + }, + ); + } + + self.failed.invalidate_all(); + self.cancellations.invalidate_all(); + self.slow.invalidate_all(); + // In-flight probes are aborted by the caller of this method, so their + // single-flight claims would otherwise hold their origins until the claim + // TTL lapsed. + self.probing.invalidate_all(); + self.tcp_times.invalidate_all(); + self.quic_times.invalidate_all(); + + // After the failures are cleared, so a hint that a cooldown had been + // blocking takes effect now rather than staying refused. + for (origin, port) in self.hints.iter() { + self.seed_hint((*origin).clone(), port); + } + } + + /// Record a failed HTTP/3 attempt, blocking the origin for a cooldown that + /// lengthens the longer it keeps failing. + // spec:H3UP#failure-backoff + pub fn record_h3_failure(&self, url: &reqwest::Url) { + let Some(origin) = Self::origin_key(url) else { + return; + }; + + self.advertised.invalidate(&origin); + self.confirmed.invalidate(&origin); + // Already demoted; further counting is meaningless. + self.cancellations.invalidate(&origin); + + let now = Instant::now(); + // An entry whose run has lapsed is history, not a run in progress: the + // origin went a whole further cooldown without failing again, so it is + // judged from the base. + let count = self + .failed + .get(&origin) + .filter(|entry| entry.counted_until > now) + .map_or(1, |entry| entry.count.saturating_add(1)); + let cooldown = self.failure_cooldown(count); + + self.failed.insert( + origin, + FailureEntry { + count, + blocked_until: now + cooldown, + // The count has to outlive the block it caused, or it could never + // escalate: the next attempt only comes once the block lapses. + counted_until: now + cooldown.saturating_mul(2), + }, + ); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-alt-svc/src/cache/tests.rs b/crates/web-faith-alt-svc/src/cache/tests.rs new file mode 100644 index 0000000..39f7c63 --- /dev/null +++ b/crates/web-faith-alt-svc/src/cache/tests.rs @@ -0,0 +1,1016 @@ +use std::time::{Duration, Instant}; + +use super::*; + +#[test] +fn test_ipv6_origin_accepts_its_own_host_spelled_out() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://[2001:db8::1]/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "[2001:db8::1]".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "an IPv6 origin naming its own address is the same host, brackets and all" + ); +} + +#[test] +fn test_host_comparison_ignores_case() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "ExAmPlE.CoM".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "host names are case-insensitive, so this still names the origin's own host" + ); +} + +#[test] +fn test_alt_svc_on_another_host_is_not_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "cdn.example.net".to_string(), + port: 443, + max_age: None, + }, + ); + + assert!( + cache.should_use_h3(&url).is_none(), + "h3 on another host says nothing about this one, and the port matching is \ + coincidental" + ); +} + +#[test] +fn test_alt_svc_naming_our_own_host_is_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc( + &url, + &AltSvcAdvertisement { + host: "example.com".to_string(), + port: 443, + max_age: None, + }, + ); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "spelling out the origin's own host is equivalent to omitting it" + ); +} + +#[test] +fn test_confirm_h3_uses_the_port_it_was_given() { + // A concurrent failure can clear both caches between the attempt starting and + // confirming. `confirm_h3` must not fall back to the origin's port then, or it + // would confirm h3 on a port nobody advertised. + let cache = test_cache_with(3, Duration::from_secs(60), true); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + cache.record_h3_failure(&url); + cache.confirm_h3(&url, 8443); + + let entry = cache + .confirmed + .get(&"https://example.com:443".to_string()) + .expect("the successful attempt is confirmed"); + assert_eq!( + entry.port, 8443, + "confirmed on the port actually connected to, not the origin's" + ); +} + +/// A same-host advertisement, the common case. +fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { + AltSvcAdvertisement { + host: String::new(), + port, + max_age, + } +} + +fn test_cache() -> AltSvcCache { + test_cache_with(3, Duration::from_secs(60), false) +} + +fn test_cache_with( + cancel_strikes: u32, + strike_window: Duration, + follow_advertised_port: bool, +) -> AltSvcCache { + test_cache_failing( + cancel_strikes, + strike_window, + follow_advertised_port, + Duration::from_secs(300), + Duration::from_secs(3600), + ) +} + +/// A cache whose knowledge expires soon, for tests about knowledge that has lapsed. +fn test_cache_ttls(advertised_ttl: Duration, confirmed_ttl: Duration) -> AltSvcCache { + AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl: Duration::from_secs(300), + failed_max_ttl: Duration::from_secs(3600), + capacity: 10_000, + cancel_strikes: 3, + strike_window: Duration::from_secs(60), + follow_advertised_port: false, + probe_ttl: Duration::from_secs(10), + slow_factor: 2.5, + slow_ttl: Duration::from_millis(200), + }) +} + +fn test_cache_failing( + cancel_strikes: u32, + strike_window: Duration, + follow_advertised_port: bool, + failed_ttl: Duration, + failed_max_ttl: Duration, +) -> AltSvcCache { + AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl: Duration::from_secs(86400), + confirmed_ttl: Duration::from_secs(86400), + failed_ttl, + failed_max_ttl, + capacity: 10_000, + cancel_strikes, + strike_window, + follow_advertised_port, + probe_ttl: Duration::from_secs(10), + slow_factor: 2.5, + slow_ttl: Duration::from_millis(200), + }) +} + +#[test] +fn test_advertised_port_matching_origin_upgrades() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "an advertisement for the origin's own port is actionable" + ); +} + +#[test] +fn test_advertised_port_mismatch_does_not_upgrade() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + assert!( + cache.should_use_h3(&url).is_none(), + "h3 advertised on :8443 says nothing about :443, so don't upgrade" + ); +} + +#[test] +fn test_advertised_port_mismatch_is_still_recorded() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + let entry = cache + .advertised + .get(&"https://example.com:443".to_string()) + .expect("the advertisement is kept even though it isn't actionable"); + assert_eq!( + entry.port, 8443, + "keeping it means the port is available if reqwest ever lets us honour it" + ); +} + +#[test] +fn test_advertised_port_mismatch_upgrades_when_following() { + let cache = test_cache_with(3, Duration::from_secs(60), true); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(8443, None)); + + assert_eq!( + cache.should_use_h3(&url), + Some(8443), + "opting in returns the advertised port so the request can be rewritten" + ); +} + +#[test] +fn test_cache_flow() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + assert!(cache.should_use_h3(&url).is_none()); + + cache.record_alt_svc(&url, &ad(443, Some(Duration::from_secs(3600)))); + assert_eq!(cache.should_use_h3(&url), Some(443)); + + cache.confirm_h3(&url, 443); + assert_eq!(cache.should_use_h3(&url), Some(443)); + assert!( + !cache + .advertised + .contains_key(&"https://example.com:443".to_string()) + ); + assert!( + cache + .confirmed + .contains_key(&"https://example.com:443".to_string()) + ); +} + +#[test] +fn test_cache_failure() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.should_use_h3(&url).is_some()); + + cache.record_h3_failure(&url); + assert!(cache.should_use_h3(&url).is_none()); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.should_use_h3(&url).is_none()); +} + +#[test] +fn test_hint() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + assert_eq!(cache.should_use_h3(&url), Some(443)); +} + +#[test] +fn test_hint_is_confirmed_not_probed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "a hint is the user's assertion and routes immediately, probe or no probe" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "nothing to verify: the hint already confirmed the origin" + ); +} + +#[test] +fn test_https_record_lands_as_an_advertisement_not_a_confirmation() { + // spec:H3UP#advertisements-from-dns — DNS is a second source of the same advertisement, + // so it makes the origin probe-worthy without routing a foreground request onto an + // unverified QUIC path. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, None, Duration::from_secs(3600)); + + assert!( + cache.confirmed_port(&url).is_none(), + "a record is evidence worth probing, not worth routing on" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "and a record naming no port describes the origin's own" + ); +} + +#[test] +fn test_https_record_port_follows_the_advertised_port_rules() { + // spec:H3UP#advertised-ports — a `port` differing from the origin's is treated exactly as + // an `Alt-Svc` advertised port: recorded, but not acted on by default. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, Some(8443), Duration::from_secs(3600)); + + assert!( + cache.probe_candidate(&url).is_none(), + "h3 on :8443 says nothing about :443, so nothing is probed" + ); + + let following = test_cache_with(3, Duration::from_secs(60), true); + following.record_https_record(&url, Some(8443), Duration::from_secs(3600)); + assert_eq!( + following.probe_candidate(&url), + Some(8443), + "opting into the quirk probes the advertised port" + ); +} + +#[test] +fn test_https_record_is_refused_while_the_origin_is_failed() { + // A failure blocks recording fresh advertisements whatever their source, or a flapping + // origin could re-enter the cycle through DNS (spec:H3UP#advertisements-from-dns). + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.record_https_record(&url, None, Duration::from_secs(3600)); + + assert!(cache.probe_candidate(&url).is_none()); + assert!( + !cache.wants_https_record(&url), + "and there is no point querying again while the cooldown runs" + ); +} + +#[test] +fn test_wants_https_record_only_while_there_is_something_to_learn() { + // spec:DNS#https-records — the gate is what keeps the query off every lookup once the + // origin's HTTP/3 support is settled either way. + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + let unknown = test_cache(); + assert!( + unknown.wants_https_record(&url), + "an origin nothing is known about is worth asking about" + ); + + let advertised = test_cache(); + advertised.record_alt_svc(&url, &ad(443, None)); + assert!( + !advertised.wants_https_record(&url), + "a live advertisement already warrants the probe a record would" + ); + + let confirmed = test_cache(); + confirmed.confirm_h3(&url, 443); + assert!( + !confirmed.wants_https_record(&url), + "a confirmed origin is already routing over HTTP/3" + ); +} + +#[test] +fn test_wants_https_record_again_once_the_advertisement_lapses() { + // The gate must not be permanent: an advertisement that expires without being confirmed + // leaves the origin unknown again, and DNS is how it can be re-learned. + let cache = test_cache_ttls(Duration::from_millis(50), Duration::from_secs(86400)); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(!cache.wants_https_record(&url)); + + std::thread::sleep(Duration::from_millis(120)); + + assert!( + cache.wants_https_record(&url), + "once the advertisement lapses the query is worth making again" + ); +} + +#[test] +fn test_https_record_ttl_bounds_the_advertisement() { + // spec:H3UP#advertisements-from-dns — the record's own DNS TTL is what the advertisement + // lives for, the role `ma` plays for a header. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_https_record(&url, None, Duration::from_millis(50)); + assert_eq!(cache.probe_candidate(&url), Some(443)); + + std::thread::sleep(Duration::from_millis(120)); + + assert!( + cache.probe_candidate(&url).is_none(), + "past the record's TTL the advertisement is no longer evidence" + ); +} + +#[test] +fn test_advertised_routes_nothing_but_probes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + assert!( + cache.confirmed_port(&url).is_none(), + "an advertisement is evidence worth probing, not worth routing on" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "and it is exactly what the probe should verify" + ); +} + +#[test] +fn test_probe_candidate_respects_failed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + + assert!( + cache.probe_candidate(&url).is_none(), + "a failed origin is not re-probed until the cooldown lapses" + ); +} + +#[test] +fn test_probe_confirmation_promotes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.claim_probe(&url), "first claim wins"); + cache.confirm_h3(&url, 443); + cache.finish_probe(&url); + + assert_eq!(cache.confirmed_port(&url), Some(443)); + assert!( + cache.probe_candidate(&url).is_none(), + "confirmed origins are not probed again" + ); +} + +#[test] +fn test_claim_probe_is_single_flight() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + assert!(cache.claim_probe(&url)); + assert!( + !cache.claim_probe(&url), + "a second claim while one is in flight loses" + ); + + cache.finish_probe(&url); + assert!( + cache.claim_probe(&url), + "finishing the probe frees the origin for the next one" + ); +} + +#[test] +fn test_slow_demotion_needs_sustained_evidence() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // Plenty of TCP samples at 5ms, but too few QUIC samples to act on. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + } + for _ in 0..(EWMA_MIN_SAMPLES - 1) { + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "below the minimum sample count no comparison may act" + ); +} + +#[test] +fn test_slow_demotion_moves_origin_back_to_probing() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // TCP steady at 5ms, QUIC steady at 50ms: 10x the average and 45ms over, + // clearing both the factor and the absolute floor. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert!( + cache.confirmed_port(&url).is_none(), + "a sustained large gap demotes the origin off HTTP/3" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "while the slow marker lives, the origin is not re-probed either" + ); + assert!( + !cache.is_failed("https://example.com:443"), + "slow is not broken: the failed cache stays out of it" + ); + + // The test cache's slow TTL is short; once it lapses, the advertisement + // preserved by the demotion re-enters through a probe. + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "slow expiry re-enters via the probe, asking whether the path improved" + ); +} + +#[test] +fn test_parity_or_moderately_slower_quic_is_kept() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + // QUIC 2x slower and 20ms over: above the floor but below the 2.5x + // factor, so HTTP/3's structural advantages win the tie. + for _ in 0..(EWMA_MIN_SAMPLES * 2) { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(20)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(40)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "moderately slower QUIC is still preferred" + ); +} + +#[test] +fn test_cancellation_below_threshold_keeps_h3() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "two strikes is not enough to demote" + ); +} + +#[test] +fn test_cancellation_at_threshold_demotes() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + for _ in 0..3 { + cache.record_h3_cancellation(&url); + } + + assert!( + cache.should_use_h3(&url).is_none(), + "three strikes demotes the origin" + ); + assert!( + cache.is_failed("https://example.com:443"), + "demotion goes through the failed cache, so re-advertisement can't re-arm it" + ); +} + +#[test] +fn test_cancellation_reset_by_h3_success() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + cache.confirm_h3(&url, 443); + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "a working h3 response clears the strikes, so these two start over" + ); +} + +#[test] +fn test_cancellation_disabled_by_zero() { + let cache = test_cache_with(0, Duration::from_secs(60), false); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + for _ in 0..5 { + cache.record_h3_cancellation(&url); + } + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "cancel_strikes: 0 disables cancellation-based demotion" + ); +} + +#[test] +fn test_cancellation_strikes_decay() { + let cache = test_cache_with(3, Duration::from_millis(50), false); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + std::thread::sleep(Duration::from_millis(150)); + cache.record_h3_cancellation(&url); + cache.record_h3_cancellation(&url); + + assert_eq!( + cache.should_use_h3(&url), + Some(443), + "strikes older than the window don't count towards the run" + ); +} + +fn failure_entry(cache: &AltSvcCache) -> FailureEntry { + cache + .failed + .get("https://example.com:443") + .expect("the origin has a failure on record") +} + +#[test] +fn test_failure_cooldown_doubles_up_to_the_cap() { + let cache = test_cache(); + + let schedule: Vec = (1..=6) + .map(|count| cache.failure_cooldown(count).as_secs()) + .collect(); + + assert_eq!( + schedule, + vec![300, 600, 1200, 2400, 3600, 3600], + "each consecutive failure doubles the base, then holds at the cap" + ); +} + +#[test] +fn test_failure_cooldown_cap_below_base_is_flat() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_secs(300), + Duration::from_secs(60), + ); + + let schedule: Vec = (1..=4) + .map(|count| cache.failure_cooldown(count).as_secs()) + .collect(); + + assert_eq!( + schedule, + vec![300, 300, 300, 300], + "a cap under the base is clamped up to it, giving a cooldown that never backs off" + ); +} + +#[test] +fn test_consecutive_failures_lengthen_the_cooldown() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_millis(200), + Duration::from_secs(60), + ); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + assert!( + cache.is_failed("https://example.com:443"), + "the first failure blocks the origin" + ); + + // Past the first cooldown, but well inside the run's own lifetime: this is + // the retry the cooldown allowed, and it fails too. + std::thread::sleep(Duration::from_millis(250)); + assert!( + !cache.is_failed("https://example.com:443"), + "the first cooldown lapses on its own" + ); + + cache.record_h3_failure(&url); + let entry = failure_entry(&cache); + assert_eq!( + entry.count, 2, + "failing again straight after a lapsed cooldown continues the run" + ); + assert!( + cache.is_failed("https://example.com:443"), + "and blocks the origin again, for twice as long" + ); +} + +#[test] +fn test_run_lapses_when_the_origin_stops_failing() { + let cache = test_cache_failing( + 3, + Duration::from_secs(60), + false, + Duration::from_millis(100), + Duration::from_secs(60), + ); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + // One cooldown beyond the block it caused: nobody exercised the origin in + // that stretch, so the next failure is judged on its own. + std::thread::sleep(Duration::from_millis(300)); + + cache.record_h3_failure(&url); + assert_eq!( + failure_entry(&cache).count, + 1, + "an origin left alone past its run starts from the base cooldown again" + ); +} + +#[test] +fn test_confirmation_ends_the_run() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.record_h3_failure(&url); + assert_eq!(failure_entry(&cache).count, 2); + + cache.confirm_h3(&url, 443); + cache.record_h3_failure(&url); + + let entry = failure_entry(&cache); + assert_eq!( + entry.count, 1, + "a working h3 response ends the run, so the next failure starts at the base" + ); + assert_eq!( + entry.blocked_until.duration_since(Instant::now()).as_secs(), + 299, + "and is blocked for the base cooldown, not the doubled one" + ); +} + +#[test] +fn test_confirmation_does_not_cut_a_live_cooldown_short() { + // A confirmation can race a concurrent failure. The failure is the more + // recent word on the path, so it keeps the origin blocked; only the run + // is forgotten. + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_h3_failure(&url); + cache.confirm_h3(&url, 443); + + assert!( + cache.is_failed("https://example.com:443"), + "the cooldown the failure set still runs" + ); + assert_eq!( + failure_entry(&cache).count, + 0, + "but the run behind it is cleared" + ); +} + +#[test] +fn test_network_change_demotes_confirmed_to_advertised() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + + cache.network_changed(); + + assert!( + cache.confirmed_port(&url).is_none(), + "the path that proved HTTP/3 is gone, so the origin is no longer confirmed" + ); + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "it keeps its advertisement, so a background probe re-verifies it at once" + ); +} + +#[test] +fn test_network_change_clears_failures_and_their_backoff() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + cache.record_alt_svc(&url, &ad(443, None)); + + cache.network_changed(); + + assert!( + !cache.is_failed("https://example.com:443"), + "a blocked UDP path is a fact about the old network" + ); + assert!( + cache.failed.get("https://example.com:443").is_none(), + "and so is the run of failures that set the cooldown" + ); + + // The advertisement a failure discards has to come back for the origin to be + // probe-worthy again, so re-record it as a live response would. + cache.record_alt_svc(&url, &ad(443, None)); + cache.record_h3_failure(&url); + assert_eq!( + failure_entry(&cache).count, + 1, + "failing on the new network starts the backoff from the base cooldown" + ); +} + +#[test] +fn test_network_change_clears_a_slow_demotion() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + assert!( + cache.probe_candidate(&url).is_none(), + "the slow marker holds the origin off probing before the signal" + ); + + cache.network_changed(); + + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "a slow path was slow on the old network, so the origin re-enters through a probe" + ); +} + +#[test] +fn test_network_change_clears_the_path_time_averages() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + cache.confirm_h3(&url, 443); + // Enough TCP samples to satisfy the comparison's minimum, all of them fast. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); + } + + cache.network_changed(); + cache.confirm_h3(&url, 443); + + // Slow enough to demote several times over, were the old TCP average still there + // to compare against. + for _ in 0..EWMA_MIN_SAMPLES { + cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); + } + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "with the TCP average cleared there is nothing to judge QUIC against, so \ + no demotion happens on one side's samples alone" + ); +} + +#[test] +fn test_network_change_keeps_hints_confirmed() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + + cache.network_changed(); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "a hint is the caller's assertion, not an observation, so it survives the signal \ + and keeps an HTTP/3-only origin reachable" + ); + assert!( + cache.probe_candidate(&url).is_none(), + "and a hinted origin is still never probed" + ); +} + +#[test] +fn test_network_change_lets_a_blocked_hint_take_effect() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.add_hint("example.com", 443); + cache.record_h3_failure(&url); + assert!( + cache.confirmed_port(&url).is_none(), + "a failure demotes a hinted origin like any other" + ); + + cache.network_changed(); + + assert_eq!( + cache.confirmed_port(&url), + Some(443), + "the failure that was masking the hint belonged to the old network, so the \ + hint holds again once it is cleared" + ); +} + +#[test] +fn test_network_change_keeps_advertisements() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + + cache.network_changed(); + + assert_eq!( + cache.probe_candidate(&url), + Some(443), + "an advertisement is the origin's statement about itself, which a change of \ + client network does not revise" + ); +} + +#[test] +fn test_network_change_drops_expired_confirmations() { + let cache = test_cache_ttls(Duration::from_millis(100), Duration::from_millis(100)); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.confirm_h3(&url, 443); + std::thread::sleep(Duration::from_millis(150)); + + cache.network_changed(); + + assert!( + cache.probe_candidate(&url).is_none(), + "a confirmation that had already lapsed is not knowledge to carry forward as \ + a fresh advertisement" + ); +} + +#[test] +fn test_network_change_releases_probe_claims() { + let cache = test_cache(); + let url = reqwest::Url::parse("https://example.com/path").unwrap(); + + cache.record_alt_svc(&url, &ad(443, None)); + assert!(cache.claim_probe(&url), "the first probe claims the origin"); + assert!(!cache.claim_probe(&url), "and holds it single-flight"); + + cache.network_changed(); + + assert!( + cache.claim_probe(&url), + "probes in flight are aborted with the client they ran on, so their claims \ + must not hold the origin for the claim TTL" + ); +} diff --git a/crates/web-faith-alt-svc/src/header.rs b/crates/web-faith-alt-svc/src/header.rs new file mode 100644 index 0000000..f346fa4 --- /dev/null +++ b/crates/web-faith-alt-svc/src/header.rs @@ -0,0 +1,75 @@ +use std::time::Duration; + +use crate::cache::AltSvcAdvertisement; + +pub fn parse_alt_svc_header(value: &str) -> Option { + if value == "clear" { + return None; + } + + for service in value.split(',') { + let service = service.trim(); + if service.is_empty() { + continue; + } + + let mut protocol_id: Option<&str> = None; + let mut host: Option<&str> = None; + let mut port: Option = None; + let mut max_age: Option = None; + + for param in service.split(';') { + let param = param.trim(); + if param.is_empty() { + continue; + } + + let Some((key, value)) = param.split_once('=') else { + continue; + }; + + let key = key.trim(); + let value = value.trim().trim_matches('"'); + + match key { + "ma" => { + if let Ok(secs) = value.parse::() { + max_age = Some(Duration::from_secs(secs)); + } + } + _ if key.starts_with("h3") => { + protocol_id = Some(key); + // The alt-authority is `[host]:port`, where an omitted host means + // the origin's own. Keep the host: acting on an advertisement for + // a different host would be the same unsupported inference as + // acting on one for a different port. + // + // Split on the *last* colon so a bracketed IPv6 literal survives, + // and keep it exactly as written — brackets included. That is the + // form `Url::host_str` also returns for IPv6, so comparing the two + // needs no normalising on either side. + if let Some((alt_host, port_str)) = value.rsplit_once(':') { + host = Some(alt_host); + if let Ok(p) = port_str.parse::() { + port = Some(p); + } + } + } + _ => {} + } + } + + if protocol_id.is_some() && port.is_some() { + return Some(AltSvcAdvertisement { + host: host.unwrap_or_default().to_owned(), + port: port.unwrap(), + max_age, + }); + } + } + + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-alt-svc/src/header/tests.rs b/crates/web-faith-alt-svc/src/header/tests.rs new file mode 100644 index 0000000..5a74872 --- /dev/null +++ b/crates/web-faith-alt-svc/src/header/tests.rs @@ -0,0 +1,86 @@ +use std::time::Duration; + +use super::parse_alt_svc_header; +use crate::cache::AltSvcAdvertisement; + +#[test] +fn test_parse_alt_svc_simple() { + let result = parse_alt_svc_header(r#"h3=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_no_max_age() { + let result = parse_alt_svc_header(r#"h3=":443""#); + assert_eq!(result, Some(ad(443, None))); +} + +#[test] +fn test_parse_alt_svc_different_port() { + let result = parse_alt_svc_header(r#"h3=":8443"; ma=3600"#); + assert_eq!(result, Some(ad(8443, Some(Duration::from_secs(3600))))); +} + +#[test] +fn test_parse_alt_svc_multiple_protocols() { + let result = parse_alt_svc_header(r#"h2=":443", h3=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_h3_variant() { + let result = parse_alt_svc_header(r#"h3-29=":443"; ma=86400"#); + assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); +} + +#[test] +fn test_parse_alt_svc_keeps_the_host() { + let result = parse_alt_svc_header(r#"h3="cdn.example.net:443"; ma=3600"#); + assert_eq!( + result, + Some(AltSvcAdvertisement { + host: "cdn.example.net".to_string(), + port: 443, + max_age: Some(Duration::from_secs(3600)), + }), + "the alt-authority's host must survive parsing, or a different-host \ + advertisement looks same-host once the port matches" + ); +} + +#[test] +fn test_parse_alt_svc_ipv6_host() { + let result = parse_alt_svc_header(r#"h3="[2001:db8::1]:8443""#); + assert_eq!( + result, + Some(AltSvcAdvertisement { + // Brackets kept: this is the form `Url::host_str` returns too, so the + // two compare directly. + host: "[2001:db8::1]".to_string(), + port: 8443, + max_age: None, + }), + "splitting on the last colon keeps a bracketed IPv6 literal intact" + ); +} + +#[test] +fn test_parse_alt_svc_clear() { + let result = parse_alt_svc_header("clear"); + assert_eq!(result, None); +} + +#[test] +fn test_parse_alt_svc_no_h3() { + let result = parse_alt_svc_header(r#"h2=":443"; ma=86400"#); + assert_eq!(result, None); +} + +/// A same-host advertisement, the common case. +fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { + AltSvcAdvertisement { + host: String::new(), + port, + max_age, + } +} diff --git a/crates/web-faith-alt-svc/src/lib.rs b/crates/web-faith-alt-svc/src/lib.rs index 810be61..2e23b05 100644 --- a/crates/web-faith-alt-svc/src/lib.rs +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -22,2350 +22,14 @@ // spec:H3UP spec:PROBE -use std::{ - marker::PhantomData, - sync::Arc, - time::{Duration, Instant}, -}; - -use http::Extensions; -use moka::sync::Cache; -use reqwest::{Request, Response}; -use reqwest_middleware::{Middleware, Next, Result}; - -/// Recording the moment a response's headers arrived. -/// -/// The stamp itself belongs to the client, which puts it in the request's extensions and reads it -/// back out to surface the timing; this layer is only the one place that observes the arrival, so -/// all it needs is to be able to mark what the client left there. -pub trait ArrivalStamp: Send + Sync + 'static { - fn mark(&self, at: Instant); -} - -#[derive(Debug, Clone)] -pub struct AltSvcEntry { - pub port: u16, - pub expires: Instant, -} - -/// An HTTP/3 alternative parsed out of an `Alt-Svc` header. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AltSvcAdvertisement { - /// Host the alternative is on. Empty when the header omitted it, which per - /// RFC 7838 means the same host as the origin. - pub host: String, - pub port: u16, - pub max_age: Option, -} - -/// A run of consecutive HTTP/3 failures against one origin. -/// -/// Both instants are carried in the value rather than left to the cache's TTL, -/// because they differ per origin and from each other: the entry deliberately -/// outlives the cooldown it set, so that a count survives the block it caused and -/// can escalate the next one. `advertised` does the same for `ma`. -// spec:H3UP#failure-backoff -#[derive(Debug, Clone, Copy)] -struct FailureEntry { - /// Consecutive failures with no confirmation in between. - count: u32, - /// Until when the origin is blocked from upgrading, probing, and recording - /// advertisements. The only field that gates behaviour. - blocked_until: Instant, - /// Until when `count` still describes a run. Past it the origin is judged - /// from the base cooldown again. - counted_until: Instant, -} - -/// A per-origin exponentially-weighted moving average of time-to-response-headers. -/// -/// Two `f64`s per origin and no sample storage: the average decays stale history -/// by construction, and the count gates decisions until there is enough evidence -/// to mean anything. -#[derive(Debug, Clone, Copy)] -pub struct PathTime { - /// EWMA of time-to-response-headers, in milliseconds. - pub avg_ms: f64, - pub count: u32, -} - -/// Weight of the newest sample in the moving average. -const EWMA_ALPHA: f64 = 0.2; -/// Samples required on *each* side before a slow comparison may act. -const EWMA_MIN_SAMPLES: u32 = 8; -/// Absolute gap the QUIC average must exceed the TCP one by, on top of the -/// factor, so LAN-fast origins don't flap on sub-millisecond noise. -pub const SLOW_FLOOR_MS: f64 = 10.0; - -pub struct AltSvcCacheConfig { - pub advertised_ttl: Duration, - pub confirmed_ttl: Duration, - /// Cooldown a first failure earns; each consecutive one doubles it. - pub failed_ttl: Duration, - /// Ceiling on the doubling. Clamped up to `failed_ttl`, so setting it at or - /// below the base gives a flat cooldown. - pub failed_max_ttl: Duration, - pub capacity: u64, - pub cancel_strikes: u32, - pub strike_window: Duration, - pub follow_advertised_port: bool, - /// Lifetime of a probe's single-flight claim. Doubles as crash recovery: a - /// probe task that dies without reporting frees its origin when this lapses. - pub probe_ttl: Duration, - /// The QUIC path is demoted when its average is worse than TCP's by this - /// factor (and by [`SLOW_FLOOR_MS`] absolutely). `0.0` disables path-time - /// demotion entirely. - pub slow_factor: f64, - /// How long a path-time demotion holds before the origin may be re-probed. - pub slow_ttl: Duration, -} - -#[derive(Clone)] -pub struct AltSvcCache { - advertised: Cache, - confirmed: Cache, - /// Origins that failed over HTTP/3, with their run of consecutive failures. - /// An entry present here is not necessarily blocked: see [`Self::is_failed`]. - failed: Cache, - /// Consecutive cancelled HTTP/3 attempts per origin. Entries expire on a TTL - /// (the strike window), so a run has to be sustained to count. - cancellations: Cache, - /// Single-flight claims for in-flight background probes. - probing: Cache, - /// Origins demoted for being slower over QUIC than over TCP. Distinct from - /// `failed`: the path *works*, so re-advertisements must not be discarded, - /// and expiry re-enters through a probe rather than treating h3 as broken. - slow: Cache, - /// Origins seeded from `http3.hints`, with the port hinted. A hint is the - /// caller's assertion rather than something observed, so it has to be - /// distinguishable from an entry in `confirmed` that a real HTTP/3 response - /// put there: [`Self::network_changed`] demotes the observed ones and - /// re-seeds from here. Unbounded by TTL and outside the capacity bound, - /// because the hints are configuration and there are as many as the caller - /// passed. - // spec:NETCHG#what-the-signal-keeps - hints: Cache, - /// Time-to-headers over TCP (h1 and h2 together), per origin. - tcp_times: Cache, - /// Time-to-headers over QUIC (h3), per origin. - quic_times: Cache, - - advertised_ttl: Duration, - confirmed_ttl: Duration, - failed_ttl: Duration, - failed_max_ttl: Duration, - cancel_strikes: u32, - follow_advertised_port: bool, - slow_factor: f64, -} - -impl std::fmt::Debug for AltSvcCache { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AltSvcCache") - .field("advertised_count", &self.advertised.entry_count()) - .field("confirmed_count", &self.confirmed.entry_count()) - .field("failed_count", &self.failed.entry_count()) - .field("cancellation_count", &self.cancellations.entry_count()) - .field("probing_count", &self.probing.entry_count()) - .field("slow_count", &self.slow.entry_count()) - .field("hint_count", &self.hints.entry_count()) - .finish() - } -} - -impl AltSvcCache { - pub fn new(config: AltSvcCacheConfig) -> Self { - let AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - capacity, - cancel_strikes, - strike_window, - follow_advertised_port, - probe_ttl, - slow_factor, - slow_ttl, - } = config; - - // A cap below the base would mean the first failure already exceeds it; - // clamping makes that setting a flat cooldown rather than a shorter one. - let failed_max_ttl = failed_max_ttl.max(failed_ttl); - - Self { - advertised: Cache::builder() - .max_capacity(capacity) - .time_to_live(advertised_ttl) - .build(), - confirmed: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - // Twice the longest cooldown: the outer bound on how long an entry - // can be worth keeping, since a count is dropped one cooldown after - // the block it caused lapsed. Per-entry instants do the real work. - failed: Cache::builder() - .max_capacity(capacity) - .time_to_live(failed_max_ttl.saturating_mul(2)) - .build(), - cancellations: Cache::builder() - .max_capacity(capacity) - .time_to_live(strike_window) - .build(), - probing: Cache::builder() - .max_capacity(capacity) - .time_to_live(probe_ttl) - .build(), - slow: Cache::builder() - .max_capacity(capacity) - .time_to_live(slow_ttl) - .build(), - // No TTL and no capacity bound: hints are configuration, held for the - // life of the agent so a network change can re-seed from them. - hints: Cache::builder().build(), - tcp_times: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - quic_times: Cache::builder() - .max_capacity(capacity) - .time_to_live(confirmed_ttl) - .build(), - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - cancel_strikes, - follow_advertised_port, - slow_factor, - } - } - - /// The cooldown the `count`-th consecutive failure earns: the base doubled - /// once per failure before it, capped. - // spec:H3UP#failure-backoff - fn failure_cooldown(&self, count: u32) -> Duration { - let doublings = count.saturating_sub(1).min(u32::BITS - 1); - self.failed_ttl - .saturating_mul(2u32.saturating_pow(doublings)) - .min(self.failed_max_ttl) - } - - /// Whether the origin is inside its failure cooldown. - /// - /// Presence in `failed` is not the question: an entry outlives its cooldown - /// so the failure count survives to escalate the next one. - fn is_failed(&self, origin: &str) -> bool { - self.failed - .get(origin) - .is_some_and(|entry| entry.blocked_until > Instant::now()) - } - - fn origin_key(url: &reqwest::Url) -> Option { - let host = url.host_str()?; - let port = url.port_or_known_default()?; - Some(format!("{}://{}:{}", url.scheme(), host, port)) - } - - pub fn record_alt_svc(&self, url: &reqwest::Url, advertisement: &AltSvcAdvertisement) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // An alternative on a *different host* can never be honoured: reqwest derives - // the HTTP/3 connect target from the request's authority, and rewriting the - // host would also change which certificate is accepted. Unlike a differing - // port — which `follow_advertised_port` can act on — there is nothing to - // gate behind an option, so don't record it at all. RFC 7838 uses an empty - // host to mean "the same host as the origin". - // - // Compared case-insensitively because host names are, and a server naming its - // own host in a different case is still naming its own host. - if !advertisement.host.is_empty() - && !url - .host_str() - .is_some_and(|origin_host| origin_host.eq_ignore_ascii_case(&advertisement.host)) - { - return; - } - - if self.is_failed(&origin) { - return; - } - - if self.confirmed.contains_key(&origin) { - return; - } - - let ttl = advertisement.max_age.unwrap_or(self.advertised_ttl); - let entry = AltSvcEntry { - port: advertisement.port, - expires: Instant::now() + ttl, - }; - - self.advertised.insert(origin, entry); - } - - /// Whether an `HTTPS` DNS record for this origin would tell us anything we do not already - /// know, so the resolver can skip the query rather than send one per lookup. - /// - /// Nothing is learnable while the origin is confirmed (already routing over HTTP/3), failed - /// (blocked whatever a record says), slow (demoted on measurement, which a record cannot - /// overturn), or already carrying a live advertisement (the probe it warrants is already - /// warranted). Each of those states expires, and the query resumes when it does. - // spec:DNS#https-records - pub fn wants_https_record(&self, url: &reqwest::Url) -> bool { - let Some(origin) = Self::origin_key(url) else { - return false; - }; - - !self.is_failed(&origin) - && !self.confirmed.contains_key(&origin) - && !self.slow.contains_key(&origin) - && self - .advertised - .get(&origin) - .is_none_or(|entry| entry.expires <= Instant::now()) - } - - /// Record an HTTP/3 advertisement carried by an `HTTPS` DNS record. - /// - /// An `HTTPS` record and an `Alt-Svc` header are two ways for an origin to say the same thing, - /// so this lands in exactly the state a header advertisement does: the origin becomes - /// probe-worthy, and foreground requests keep to TCP until a probe proves the path. The port - /// and same-host rules are the header's too — [`Self::record_alt_svc`] applies them — because - /// the reasons for them are about what Faith can connect to rather than about where the - /// advertisement was read. - // spec:H3UP#advertisements-from-dns - pub fn record_https_record(&self, url: &reqwest::Url, port: Option, ttl: Duration) { - // A record naming no port describes the origin's own, exactly as an `Alt-Svc` header with - // no alt-authority port would. - let Some(port) = port.or_else(|| url.port_or_known_default()) else { - return; - }; - - self.record_alt_svc( - url, - &AltSvcAdvertisement { - // The same-host case: a record targeting another host is dropped before it gets - // here, since Faith only upgrades to the origin's own host. - host: String::new(), - port, - // The record's own DNS TTL is how long what it says is good for, which is the - // role `ma` plays for a header advertisement. - max_age: Some(ttl), - }, - ); - } - - /// Hints seed `confirmed` directly, not `advertised`: a hint is the *user's* - /// assertion, and routing it through a probe would both second-guess an - /// explicit instruction and break h3-only origins (no TCP listener), which - /// only work if the very first request speaks HTTP/3. Distrust is reserved - /// for what servers advertise. Failure demotes a hinted origin exactly as it - /// does a confirmed one. - pub fn add_hint(&self, host: &str, port: u16) { - let origin = format!("https://{}:{}", host, port); - - // Recorded whether or not it can be acted on right now: the hint is - // configuration, and a failure blocking it is a fact about a path that a - // network change can clear (spec:NETCHG#what-the-signal-keeps). - self.hints.insert(origin.clone(), port); - self.seed_hint(origin, port); - } - - /// Put a hinted origin into `confirmed`, unless a failure currently blocks it. - /// - /// Split out of [`Self::add_hint`] so [`Self::network_changed`] can re-seed the - /// hints it just cleared without re-recording them. - fn seed_hint(&self, origin: String, port: u16) { - if self.is_failed(&origin) { - return; - } - - let entry = AltSvcEntry { - port, - expires: Instant::now() + Duration::from_hours(10_000), // forever - }; - - self.confirmed.insert(origin, entry); - } - - /// Whether an entry advertising `entry_port` can be acted on for this URL. - /// - /// An Alt-Svc advertisement names a network endpoint for the origin; it is not - /// a claim that the origin's *own* port speaks HTTP/3. So when the advertised - /// port differs, upgrading the request on the origin port is an inference the - /// advertisement does not support. - /// - /// Honouring the advertised port properly means connecting to one port while - /// still sending the origin's authority, which reqwest cannot express: it - /// derives the HTTP/3 connect target from the request URI's authority (see - /// ). `follow_advertised_port` - /// opts into doing it anyway by rewriting the request's port, which is not - /// standards-compliant — the request then carries the alternative's authority - /// rather than the origin's. - fn port_actionable(&self, url: &reqwest::Url, entry_port: u16) -> bool { - self.follow_advertised_port || Some(entry_port) == url.port_or_known_default() - } - - /// The port HTTP/3 is *proven* on, or `None` to leave the request on TCP. - /// - /// This is the only lookup foreground routing consults when probing is on: - /// an advertisement is evidence worth probing, not worth routing on. - /// - /// A returned port that differs from the URL's own means the caller opted into - /// `follow_advertised_port` and the request must be rewritten to target it. - pub fn confirmed_port(&self, url: &reqwest::Url) -> Option { - let origin = Self::origin_key(url)?; - - if self.is_failed(&origin) || self.slow.contains_key(&origin) { - return None; - } - - let entry = self.confirmed.get(&origin)?; - if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { - Some(entry.port) - } else { - None - } - } - - /// The advertised port a background probe should verify, or `None` when - /// there is nothing (or no need) to probe: no actionable advertisement, - /// already confirmed, recently failed, or demoted for being slow. - pub fn probe_candidate(&self, url: &reqwest::Url) -> Option { - let origin = Self::origin_key(url)?; - - if self.is_failed(&origin) - || self.slow.contains_key(&origin) - || self.confirmed.contains_key(&origin) - { - return None; - } - - let entry = self.advertised.get(&origin)?; - if entry.expires > Instant::now() && self.port_actionable(url, entry.port) { - Some(entry.port) - } else { - None - } - } - - /// Claim the origin for a probe. Returns `false` when a probe is already in - /// flight; the claim expires on its own (see [`AltSvcCacheConfig::probe_ttl`]) - /// if the prober never reports back. - pub fn claim_probe(&self, url: &reqwest::Url) -> bool { - let Some(origin) = Self::origin_key(url) else { - return false; - }; - self.probing.entry(origin).or_insert(()).is_fresh() - } - - /// Release the origin's probe claim, so a later advertisement can re-probe - /// without waiting out the claim's TTL. - pub fn finish_probe(&self, url: &reqwest::Url) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - self.probing.invalidate(&origin); - } - - /// The port to attempt HTTP/3 on, or `None` to leave the request on TCP. - /// - /// Legacy (probe-less) routing: advertisements are acted on inline, so this - /// consults `advertised` as well as `confirmed`. Only used when - /// probing is off. - pub fn should_use_h3(&self, url: &reqwest::Url) -> Option { - self.confirmed_port(url) - .or_else(|| self.probe_candidate(url)) - } - - /// Record a foreground request's time-to-response-headers for its protocol - /// family, and demote the origin to TCP if QUIC is provenly, sustainedly - /// slower than TCP for it. - /// - /// Time-to-headers includes server think-time, which varies per endpoint far - /// more than per transport; only the averages across many requests are - /// comparable, never individual samples — hence the minimum sample counts. - /// Redirects followed inside the attempt inflate a sample for whichever - /// family carried it, which the averaging absorbs the same way. - /// - /// The comparison is deliberately asymmetric: HTTP/3 is preferred at parity - /// and when moderately slower, because its advantages (no head-of-line - /// blocking, connection migration) pay off beyond the mean. Only a large - /// sustained gap demotes. - pub fn record_path_time(&self, url: &reqwest::Url, version: http::Version, elapsed: Duration) { - if self.slow_factor <= 0.0 { - return; - } - - let Some(origin) = Self::origin_key(url) else { - return; - }; - - let sample_ms = elapsed.as_secs_f64() * 1000.0; - let times = if version == http::Version::HTTP_3 { - &self.quic_times - } else { - &self.tcp_times - }; - - let updated = times - .entry(origin.clone()) - .and_upsert_with(|existing| match existing { - None => PathTime { - avg_ms: sample_ms, - count: 1, - }, - Some(entry) => { - let entry = entry.into_value(); - PathTime { - avg_ms: entry.avg_ms * (1.0 - EWMA_ALPHA) + sample_ms * EWMA_ALPHA, - count: entry.count.saturating_add(1), - } - } - }) - .into_value(); - - if version == http::Version::HTTP_3 - && updated.count >= EWMA_MIN_SAMPLES - && let Some(tcp) = self.tcp_times.get(&origin) - && tcp.count >= EWMA_MIN_SAMPLES - && updated.avg_ms > tcp.avg_ms * self.slow_factor - && updated.avg_ms - tcp.avg_ms > SLOW_FLOOR_MS - { - self.demote_slow(&origin); - } - } - - /// Demote a working-but-slow QUIC origin back to TCP. - /// - /// The confirmed entry moves back to `advertised` rather than being dropped: - /// when the `slow` marker expires, the advertisement is what makes the next - /// request trigger a re-probe — "has this path improved?" asked at zero - /// foreground cost. The QUIC average is cleared so the answer is judged on - /// fresh samples, not held hostage by the history that demoted it. - fn demote_slow(&self, origin: &str) { - let key = origin.to_string(); - let Some(entry) = self.confirmed.get(&key) else { - return; - }; - - self.confirmed.invalidate(&key); - self.advertised.insert( - key.clone(), - AltSvcEntry { - port: entry.port, - expires: Instant::now() + self.advertised_ttl, - }, - ); - self.quic_times.invalidate(&key); - self.slow.insert(key, ()); - } - - /// Record that HTTP/3 worked for this origin, on the port it connected to. - /// - /// `port` must be the port the successful attempt actually used. Recovering it - /// from the caches instead would be unsound: a concurrent failure that cleared - /// them leaves nothing to read, and falling back to the origin's own port would - /// confirm HTTP/3 on a port the server never advertised — for `confirmed_ttl`, - /// and invisibly, since the concurrent failure's `failed` entry masks it until - /// that expires. - pub fn confirm_h3(&self, url: &reqwest::Url, port: u16) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // Promoted out of `advertised`; it has served its purpose. - self.advertised.invalidate(&origin); - // A working h3 response is proof of health; forget any strikes, and end - // whatever run of failures preceded it. - self.cancellations.invalidate(&origin); - self.clear_failure_count(&origin); - - let entry = AltSvcEntry { - port, - expires: Instant::now() + self.confirmed_ttl, - }; - - self.confirmed.insert(origin, entry); - } - - /// Record an HTTP/3 attempt that was cancelled before producing an outcome. - /// - /// This is weaker evidence than an error: the request never got to find out - /// whether HTTP/3 worked, so a single cancellation says nothing about the - /// origin. Only a sustained run of them demotes it, which keeps callers that - /// routinely abort healthy requests from disabling HTTP/3. - /// - /// The window is a TTL measured from the *previous* strike, because moka - /// refreshes an entry's TTL on upsert. Strikes therefore have to arrive - /// within a window of each other, not within a fixed bucket. - pub fn record_h3_cancellation(&self, url: &reqwest::Url) { - if self.cancel_strikes == 0 { - return; - } - - let Some(origin) = Self::origin_key(url) else { - return; - }; - - // This is reachable from a `Drop` impl (see the guard below), which must - // never panic: a panic while already unwinding aborts the process. Use a - // saturating add so an absurd `upgrade_cancel_strikes` can't overflow. - let strikes = self - .cancellations - .entry(origin) - .and_upsert_with(|existing| { - existing.map_or(1, |entry| entry.into_value().saturating_add(1)) - }) - .into_value(); - - if strikes >= self.cancel_strikes { - // Clears the strike count as a side effect. - self.record_h3_failure(url); - } - } - - /// Forget the origin's run of failures, so the next one starts the backoff - /// from the base cooldown again. - /// - /// A cooldown still running is left alone. A confirmation racing a concurrent - /// failure must not unblock the origin that failure just blocked: the failure - /// is the more recent evidence about the path, and [`Self::confirm_h3`] - /// relies on its own entry being masked until the block lapses. - // spec:H3UP#failure-backoff - fn clear_failure_count(&self, origin: &str) { - let Some(entry) = self.failed.get(origin) else { - return; - }; - - if entry.blocked_until > Instant::now() { - self.failed - .insert(origin.to_string(), FailureEntry { count: 0, ..entry }); - } else { - self.failed.invalidate(origin); - } - } - - /// Discard everything this cache learned by observing the network, keeping - /// what it was told. - /// - /// Every state here except `advertised` and `hints` describes the path between - /// this client and an origin, and a network change is exactly the event that - /// invalidates such a description. So the observation-confirmed origins are - /// demoted rather than kept (the path that proved them is gone, and a probe - /// re-proves them without a foreground request paying for it), and the - /// failures, strikes, slow markers and averages go entirely: they are - /// penalties and measurements the old path earned, and carrying them over - /// would judge the new network by the old one's behaviour. - /// - /// What the origin said about itself (`advertised`) and what the caller - /// asserted (`hints`) are not observations, so both survive. - // spec:NETCHG - pub fn network_changed(&self) { - let now = Instant::now(); - - // Demote first, while `confirmed` still holds the entries: an advertisement - // is what makes the next request to the origin trigger a re-probe. - // - // Keys are invalidated one by one rather than with `invalidate_all`, whose - // timestamp-based invalidation would race the hint re-seeding below. - for (origin, entry) in self.confirmed.iter() { - // A hint holds its origin confirmed; it is an assertion, not a finding. - if self.hints.contains_key(origin.as_str()) { - continue; - } - - self.confirmed.invalidate(origin.as_str()); - - // A logically expired entry is not knowledge to carry forward: it would - // come back as a fresh advertisement having just lapsed as a confirmation. - if entry.expires <= now { - continue; - } - - self.advertised.insert( - (*origin).clone(), - AltSvcEntry { - port: entry.port, - expires: now + self.advertised_ttl, - }, - ); - } - - self.failed.invalidate_all(); - self.cancellations.invalidate_all(); - self.slow.invalidate_all(); - // In-flight probes are aborted by the caller of this method, so their - // single-flight claims would otherwise hold their origins until the claim - // TTL lapsed. - self.probing.invalidate_all(); - self.tcp_times.invalidate_all(); - self.quic_times.invalidate_all(); - - // After the failures are cleared, so a hint that a cooldown had been - // blocking takes effect now rather than staying refused. - for (origin, port) in self.hints.iter() { - self.seed_hint((*origin).clone(), port); - } - } - - /// Record a failed HTTP/3 attempt, blocking the origin for a cooldown that - /// lengthens the longer it keeps failing. - // spec:H3UP#failure-backoff - pub fn record_h3_failure(&self, url: &reqwest::Url) { - let Some(origin) = Self::origin_key(url) else { - return; - }; - - self.advertised.invalidate(&origin); - self.confirmed.invalidate(&origin); - // Already demoted; further counting is meaningless. - self.cancellations.invalidate(&origin); - - let now = Instant::now(); - // An entry whose run has lapsed is history, not a run in progress: the - // origin went a whole further cooldown without failing again, so it is - // judged from the base. - let count = self - .failed - .get(&origin) - .filter(|entry| entry.counted_until > now) - .map_or(1, |entry| entry.count.saturating_add(1)); - let cooldown = self.failure_cooldown(count); - - self.failed.insert( - origin, - FailureEntry { - count, - blocked_until: now + cooldown, - // The count has to outlive the block it caused, or it could never - // escalate: the next attempt only comes once the block lapses. - counted_until: now + cooldown.saturating_mul(2), - }, - ); - } -} - -pub fn parse_alt_svc_header(value: &str) -> Option { - if value == "clear" { - return None; - } - - for service in value.split(',') { - let service = service.trim(); - if service.is_empty() { - continue; - } - - let mut protocol_id: Option<&str> = None; - let mut host: Option<&str> = None; - let mut port: Option = None; - let mut max_age: Option = None; - - for param in service.split(';') { - let param = param.trim(); - if param.is_empty() { - continue; - } - - let Some((key, value)) = param.split_once('=') else { - continue; - }; - - let key = key.trim(); - let value = value.trim().trim_matches('"'); - - match key { - "ma" => { - if let Ok(secs) = value.parse::() { - max_age = Some(Duration::from_secs(secs)); - } - } - _ if key.starts_with("h3") => { - protocol_id = Some(key); - // The alt-authority is `[host]:port`, where an omitted host means - // the origin's own. Keep the host: acting on an advertisement for - // a different host would be the same unsupported inference as - // acting on one for a different port. - // - // Split on the *last* colon so a bracketed IPv6 literal survives, - // and keep it exactly as written — brackets included. That is the - // form `Url::host_str` also returns for IPv6, so comparing the two - // needs no normalising on either side. - if let Some((alt_host, port_str)) = value.rsplit_once(':') { - host = Some(alt_host); - if let Ok(p) = port_str.parse::() { - port = Some(p); - } - } - } - _ => {} - } - } - - if protocol_id.is_some() && port.is_some() { - return Some(AltSvcAdvertisement { - host: host.unwrap_or_default().to_owned(), - port: port.unwrap(), - max_age, - }); - } - } - - None -} - -/// Records a cancellation if the HTTP/3 attempt it guards is dropped before -/// producing an outcome. -/// -/// [`AltSvcMiddleware`] can only learn that HTTP/3 is broken from the attempt's -/// return value, and a cancelled request never produces one: `faith_fetch` -/// races `send()` against the abort signal in a `select!`, which drops the -/// losing future. Without this guard nothing ever demotes the origin, so a -/// caller whose deadline is shorter than the network's own failure detection -/// re-attempts HTTP/3 over a dead path on every retry, indefinitely. -struct H3AttemptGuard { - cache: Arc, - url: reqwest::Url, - armed: bool, -} - -impl H3AttemptGuard { - fn new(cache: Arc, url: reqwest::Url) -> Self { - Self { - cache, - url, - armed: true, - } - } - - /// The attempt produced an outcome, so it speaks for itself. - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for H3AttemptGuard { - fn drop(&mut self) { - // Must stay infallible: this can run while unwinding, where a panic - // would abort the process. moka's sync cache does not panic on insert. - if self.armed { - self.cache.record_h3_cancellation(&self.url); - } - } -} - -/// Verifies advertised HTTP/3 endpoints in the background, so no foreground -/// request ever waits on an unverified QUIC path. -/// -/// The probe is a real request — `HEAD /` sent with `Version::HTTP_3` — on the -/// **raw** `reqwest::Client`, not the middleware stack. That is load-bearing -/// three times over: it bypasses the HTTP cache, so a replayed cached response -/// (rebuilt with its stored HTTP version) can never fake a confirmation; it -/// bypasses [`AltSvcMiddleware`], so probing cannot recurse; and it shares the -/// h3 connection pool with foreground requests, so a successful probe leaves -/// behind a warm QUIC connection the next request rides. Confirmation doubles -/// as prewarming. -/// -/// Any HTTP/3 response confirms, regardless of status: a 401 or 405 to -/// `HEAD /` proves the transport end-to-end just as well as a 200. -pub struct H3Prober { - client: reqwest::Client, - cache: Arc, - /// `None` leaves the attempt bounded only by the QUIC idle timeout. - timeout: Option, - /// Handles for in-flight probes, so `Agent::close` can abort them: a probe - /// holds a clone of the raw client, which would otherwise keep the - /// connection pool alive past close for up to the probe timeout. - tasks: std::sync::Mutex>, -} - -impl std::fmt::Debug for H3Prober { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("H3Prober") - .field("timeout", &self.timeout) - .finish() - } -} - -impl H3Prober { - pub fn new( - client: reqwest::Client, - cache: Arc, - timeout: Option, - ) -> Self { - Self { - client, - cache, - timeout, - tasks: std::sync::Mutex::new(Vec::new()), - } - } - - /// Spawn a probe of `port` for the origin of `url`. The caller must hold the - /// origin's single-flight claim (see [`AltSvcCache::claim_probe`]). - fn spawn(&self, url: reqwest::Url, port: u16) { - let client = self.client.clone(); - let cache = Arc::clone(&self.cache); - let timeout = self.timeout; - - let handle = tokio::spawn(async move { - let mut probe_url = url.clone(); - probe_url.set_path("/"); - probe_url.set_query(None); - probe_url.set_fragment(None); - let _ = probe_url.set_username(""); - let _ = probe_url.set_password(None); - // Same rewrite rule as the foreground path: a port differing from the - // origin's only gets here when `follow_advertised_port` is on. - if Some(port) != url.port_or_known_default() { - let _ = probe_url.set_port(Some(port)); - } - - let attempt = client.head(probe_url).version(http::Version::HTTP_3).send(); - - let outcome = match timeout { - Some(limit) => tokio::time::timeout(limit, attempt).await.ok(), - None => Some(attempt.await), - }; - - // Cache operations stay keyed on `url`, the origin, matching the - // foreground path. - match outcome { - Some(Ok(response)) if response.version() == http::Version::HTTP_3 => { - cache.confirm_h3(&url, port); - } - // A response that is somehow not HTTP/3 is a failure too: the - // h3 route did not deliver, whatever answered. - _ => cache.record_h3_failure(&url), - } - - // An aborted probe never reaches this; its claim expires on the - // probing TTL instead, which is why that TTL exceeds the timeout. - cache.finish_probe(&url); - }); - - // A poisoned lock only means another thread panicked mid-push; the Vec - // is still sound to use, and probing must never take the process down. - let mut tasks = self - .tasks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - tasks.retain(|task| !task.is_finished()); - tasks.push(handle.abort_handle()); - } - - /// Kick off a background probe for the URL's origin if one is warranted: an actionable - /// advertisement present, the origin neither confirmed, failed, nor slow, and no probe - /// already in flight. The same decision the Alt-Svc layer makes on a request, exposed so a - /// `preconnect` TCP warm-up to a probe-worthy origin triggers a probe as a real request would. - pub fn maybe_probe(&self, url: &reqwest::Url) { - let Some(port) = self.cache.probe_candidate(url) else { - return; - }; - if !self.cache.claim_probe(url) { - return; - } - self.spawn(url.clone(), port); - } - - pub fn abort_all(&self) { - let mut tasks = self - .tasks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - for task in tasks.drain(..) { - task.abort(); - } - } -} - -/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is -/// probe-worthy from its first request rather than from the first TCP response carrying an -/// `Alt-Svc` header. -/// -/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), -/// which is the only place that holds all three: the resolver is built before the cache, and the -/// prober holds a client that holds the resolver, so nothing lower down can own this. -/// -/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the -/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could -/// name. Recording it there is right whichever request triggered the lookup, because what the -/// record describes does not depend on who asked. -// spec:H3UP#advertisements-from-dns -// spec:DNS#https-records -pub struct H3HttpsSink { - cache: Arc, - /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, - /// and the resolver holds this sink. A strong reference here would close that ring and leak - /// the whole graph — connection pool included — past `Agent::close`, which works by dropping - /// the client. The agent owns the only strong reference, so this lives exactly as long as the - /// agent's prober does. - /// - /// `None` rather than a dead handle when probing is off, where an advertisement is acted on - /// inline by the next foreground request instead. - // spec:PROBE - prober: Option>, -} - -impl std::fmt::Debug for H3HttpsSink { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("H3HttpsSink") - .field("probing", &self.prober.is_some()) - .finish() - } -} - -impl H3HttpsSink { - pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { - Self { - cache, - prober: prober.map(Arc::downgrade), - } - } - - /// The origin a record at `host` describes: the default HTTPS port, which is the port whose - /// record lives at the bare name. - fn origin_url(host: &str) -> Option { - reqwest::Url::parse(&format!("https://{host}")).ok() - } -} - -impl web_faith_dns::HttpsSink for H3HttpsSink { - fn wants(&self, host: &str) -> bool { - Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) - } - - fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { - let Some(url) = Self::origin_url(host) else { - return; - }; - - self.cache - .record_https_record(&url, advertisement.port, advertisement.ttl); - - // Probe straight away rather than waiting for the request that triggered the lookup to - // finish: the point of reading DNS is that the path can be verified while that request is - // still on TCP, so the one after it upgrades. - // - // A prober that has gone means the agent was closed (or rebuilt) while this query was in - // flight; the advertisement above is still worth keeping, but there is nothing left to - // probe it with, and resurrecting a dropped client to try would be exactly wrong. - if let Some(prober) = self.prober.as_ref().and_then(std::sync::Weak::upgrade) { - prober.maybe_probe(&url); - } - } -} - -/// The Alt-Svc layer, which upgrades an origin to HTTP/3 once it advertises one. -/// -/// `S` is the client's arrival stamp, which this marks when a response's headers land; see -/// [`ArrivalStamp`]. -#[derive(Clone)] -pub struct AltSvcMiddleware { - cache: Arc, - enabled: bool, - /// Ceiling on how long an HTTP/3 attempt may take to produce response - /// headers before it is treated as failed and retried over TCP. - attempt_timeout: Option, - /// `Some` routes foreground requests on confirmed origins only, verifying - /// advertisements in the background. `None` restores the inline upgrade, - /// where the next foreground request is the verification. - prober: Option>, - stamp: PhantomData, -} - -impl std::fmt::Debug for AltSvcMiddleware { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AltSvcMiddleware") - .field("enabled", &self.enabled) - .field("attempt_timeout", &self.attempt_timeout) - .field("prober", &self.prober) - .field("cache", &self.cache) - .finish() - } -} - -impl AltSvcMiddleware { - pub fn new( - cache: Arc, - enabled: bool, - attempt_timeout: Option, - prober: Option>, - ) -> Self { - Self { - cache, - enabled, - attempt_timeout, - prober, - stamp: PhantomData, - } - } - - #[allow(dead_code)] - pub fn cache(&self) -> &Arc { - &self.cache - } - - /// Kick off a background probe for the URL's origin if one is warranted: - /// probing enabled, an actionable advertisement present, the origin neither - /// confirmed, failed, nor slow, and no probe already in flight. - fn maybe_probe(&self, url: &reqwest::Url) { - if let Some(prober) = &self.prober { - prober.maybe_probe(url); - } - } -} - -/// Run the rest of the stack and stamp the moment the response headers arrive. -/// -/// This is the one place a response's arrival is observed: the returned instant is what the -/// path-time average measures against, and the same instant reaches the caller through the -/// request's extensions to become the surfaced timing, so the two can never disagree. -// spec:RESP#request-timing -async fn run_stamped( - next: Next<'_>, - req: Request, - extensions: &mut Extensions, -) -> (Result, Instant) { - let result = next.run(req, extensions).await; - let at = Instant::now(); - if result.is_ok() - && let Some(stamp) = extensions.get::() - { - stamp.mark(at); - } - (result, at) -} - -#[async_trait::async_trait] -impl Middleware for AltSvcMiddleware { - async fn handle( - &self, - mut req: Request, - extensions: &mut Extensions, - next: Next<'_>, - ) -> Result { - if !self.enabled { - return run_stamped::(next, req, extensions).await.0; - } - - let url = req.url().clone(); - - // With a prober, routing consults proven origins only — advertisements - // get verified out-of-band, so no foreground request ever waits on an - // unverified QUIC path. Without one, the legacy inline upgrade applies. - let h3_route = if self.prober.is_some() { - self.cache.confirmed_port(&url) - } else { - self.cache.should_use_h3(&url) - }; - - if let Some(h3_port) = h3_route { - // Clone the request before attempting HTTP/3 so we can retry with TCP if it fails - if let Some(req_clone) = req.try_clone() { - *req.version_mut() = http::Version::HTTP_3; - - // A port differing from the origin's only comes back when - // `follow_advertised_port` is set — `should_use_h3` filters - // mismatches out otherwise. Rewriting the URL is the only way to - // make reqwest connect elsewhere, and it MUST happen after the - // clone above so the TCP fallback still targets the origin. - // - // Every cache operation below keeps using `url`, the origin, so - // confirmations, failures and strikes stay keyed on the origin - // rather than on the alternative endpoint. - if Some(h3_port) != url.port_or_known_default() { - let _ = req.url_mut().set_port(Some(h3_port)); - } - - let mut guard = H3AttemptGuard::new(Arc::clone(&self.cache), url.clone()); - // Measured to response headers: this layer sits inside the cache - // middleware, so `next.run` resolves when headers arrive, before - // any body buffering. - let started = Instant::now(); - // `None` means the attempt ran out of time. Bound in its own - // statement so the mutable borrow of `extensions` ends here, - // leaving the fallback below free to use it. - let outcome = match self.attempt_timeout { - Some(limit) => { - tokio::time::timeout(limit, run_stamped::(next.clone(), req, extensions)) - .await - .ok() - } - None => Some(run_stamped::(next.clone(), req, extensions).await), - }; - // Reached on success, error and expiry alike; only a mid-flight - // drop skips it and leaves the guard armed. - guard.disarm(); - - match outcome { - Some((Ok(response), at)) => { - if response.version() == http::Version::HTTP_3 { - self.cache.confirm_h3(&url, h3_port); - self.cache.record_path_time( - &url, - response.version(), - at.duration_since(started), - ); - } - - if let Some(alt_svc) = response.headers().get("alt-svc") { - if let Ok(value) = alt_svc.to_str() { - if let Some(advertisement) = parse_alt_svc_header(value) { - self.cache.record_alt_svc(&url, &advertisement); - } - } - } - - Ok(response) - } - // An expired deadline is as good as an error: HTTP/3 did not - // deliver. Taking the fallback branch directly avoids having - // to synthesise a reqwest_middleware::Error, which would mean - // adding anyhow as a dependency. - Some((Err(_), _)) | None => { - self.cache.record_h3_failure(&url); - - // Use the cloned request (which still has default HTTP version) - let started = Instant::now(); - let (result, at) = run_stamped::(next, req_clone, extensions).await; - if let Ok(ref response) = result { - self.cache.record_path_time( - &url, - response.version(), - at.duration_since(started), - ); - } - result - } - } - } else { - // Can't clone request (streaming body), just proceed without HTTP/3 - run_stamped::(next, req, extensions).await.0 - } - } else { - // An advertisement from an earlier response may still be waiting on - // verification (or on a fresh single-flight claim after a probe task - // died); this is the belt to the post-response trigger's braces. - self.maybe_probe(&url); - - let started = Instant::now(); - let (result, at) = run_stamped::(next, req, extensions).await; - - // Check for Alt-Svc header in non-HTTP/3 responses - if let Ok(ref response) = result { - self.cache - .record_path_time(&url, response.version(), at.duration_since(started)); - - if let Some(alt_svc) = response.headers().get("alt-svc") { - if let Ok(value) = alt_svc.to_str() { - if let Some(advertisement) = parse_alt_svc_header(value) { - self.cache.record_alt_svc(&url, &advertisement); - // Probe as soon as the advertisement lands, racing - // the gap before the caller's next request. - self.maybe_probe(&url); - } - } - } - } - - result - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_alt_svc_simple() { - let result = parse_alt_svc_header(r#"h3=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_no_max_age() { - let result = parse_alt_svc_header(r#"h3=":443""#); - assert_eq!(result, Some(ad(443, None))); - } - - #[test] - fn test_parse_alt_svc_different_port() { - let result = parse_alt_svc_header(r#"h3=":8443"; ma=3600"#); - assert_eq!(result, Some(ad(8443, Some(Duration::from_secs(3600))))); - } - - #[test] - fn test_parse_alt_svc_multiple_protocols() { - let result = parse_alt_svc_header(r#"h2=":443", h3=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_h3_variant() { - let result = parse_alt_svc_header(r#"h3-29=":443"; ma=86400"#); - assert_eq!(result, Some(ad(443, Some(Duration::from_secs(86400))))); - } - - #[test] - fn test_parse_alt_svc_keeps_the_host() { - let result = parse_alt_svc_header(r#"h3="cdn.example.net:443"; ma=3600"#); - assert_eq!( - result, - Some(AltSvcAdvertisement { - host: "cdn.example.net".to_string(), - port: 443, - max_age: Some(Duration::from_secs(3600)), - }), - "the alt-authority's host must survive parsing, or a different-host \ - advertisement looks same-host once the port matches" - ); - } - - #[test] - fn test_parse_alt_svc_ipv6_host() { - let result = parse_alt_svc_header(r#"h3="[2001:db8::1]:8443""#); - assert_eq!( - result, - Some(AltSvcAdvertisement { - // Brackets kept: this is the form `Url::host_str` returns too, so the - // two compare directly. - host: "[2001:db8::1]".to_string(), - port: 8443, - max_age: None, - }), - "splitting on the last colon keeps a bracketed IPv6 literal intact" - ); - } - - #[test] - fn test_ipv6_origin_accepts_its_own_host_spelled_out() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://[2001:db8::1]/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "[2001:db8::1]".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "an IPv6 origin naming its own address is the same host, brackets and all" - ); - } - - #[test] - fn test_host_comparison_ignores_case() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "ExAmPlE.CoM".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "host names are case-insensitive, so this still names the origin's own host" - ); - } - - #[test] - fn test_alt_svc_on_another_host_is_not_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "cdn.example.net".to_string(), - port: 443, - max_age: None, - }, - ); - - assert!( - cache.should_use_h3(&url).is_none(), - "h3 on another host says nothing about this one, and the port matching is \ - coincidental" - ); - } - - #[test] - fn test_alt_svc_naming_our_own_host_is_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc( - &url, - &AltSvcAdvertisement { - host: "example.com".to_string(), - port: 443, - max_age: None, - }, - ); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "spelling out the origin's own host is equivalent to omitting it" - ); - } - - #[test] - fn test_confirm_h3_uses_the_port_it_was_given() { - // A concurrent failure can clear both caches between the attempt starting and - // confirming. `confirm_h3` must not fall back to the origin's port then, or it - // would confirm h3 on a port nobody advertised. - let cache = test_cache_with(3, Duration::from_secs(60), true); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - cache.record_h3_failure(&url); - cache.confirm_h3(&url, 8443); - - let entry = cache - .confirmed - .get(&"https://example.com:443".to_string()) - .expect("the successful attempt is confirmed"); - assert_eq!( - entry.port, 8443, - "confirmed on the port actually connected to, not the origin's" - ); - } - - #[test] - fn test_parse_alt_svc_clear() { - let result = parse_alt_svc_header("clear"); - assert_eq!(result, None); - } - - #[test] - fn test_parse_alt_svc_no_h3() { - let result = parse_alt_svc_header(r#"h2=":443"; ma=86400"#); - assert_eq!(result, None); - } - - /// A same-host advertisement, the common case. - fn ad(port: u16, max_age: Option) -> AltSvcAdvertisement { - AltSvcAdvertisement { - host: String::new(), - port, - max_age, - } - } +mod cache; +mod header; +mod middleware; +mod prober; - fn test_cache() -> AltSvcCache { - test_cache_with(3, Duration::from_secs(60), false) - } - - fn test_cache_with( - cancel_strikes: u32, - strike_window: Duration, - follow_advertised_port: bool, - ) -> AltSvcCache { - test_cache_failing( - cancel_strikes, - strike_window, - follow_advertised_port, - Duration::from_secs(300), - Duration::from_secs(3600), - ) - } - - /// A cache whose knowledge expires soon, for tests about knowledge that has lapsed. - fn test_cache_ttls(advertised_ttl: Duration, confirmed_ttl: Duration) -> AltSvcCache { - AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl: Duration::from_secs(300), - failed_max_ttl: Duration::from_secs(3600), - capacity: 10_000, - cancel_strikes: 3, - strike_window: Duration::from_secs(60), - follow_advertised_port: false, - probe_ttl: Duration::from_secs(10), - slow_factor: 2.5, - slow_ttl: Duration::from_millis(200), - }) - } - - fn test_cache_failing( - cancel_strikes: u32, - strike_window: Duration, - follow_advertised_port: bool, - failed_ttl: Duration, - failed_max_ttl: Duration, - ) -> AltSvcCache { - AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl: Duration::from_secs(86400), - confirmed_ttl: Duration::from_secs(86400), - failed_ttl, - failed_max_ttl, - capacity: 10_000, - cancel_strikes, - strike_window, - follow_advertised_port, - probe_ttl: Duration::from_secs(10), - slow_factor: 2.5, - slow_ttl: Duration::from_millis(200), - }) - } - - #[test] - fn test_advertised_port_matching_origin_upgrades() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "an advertisement for the origin's own port is actionable" - ); - } - - #[test] - fn test_advertised_port_mismatch_does_not_upgrade() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - assert!( - cache.should_use_h3(&url).is_none(), - "h3 advertised on :8443 says nothing about :443, so don't upgrade" - ); - } - - #[test] - fn test_advertised_port_mismatch_is_still_recorded() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - let entry = cache - .advertised - .get(&"https://example.com:443".to_string()) - .expect("the advertisement is kept even though it isn't actionable"); - assert_eq!( - entry.port, 8443, - "keeping it means the port is available if reqwest ever lets us honour it" - ); - } - - #[test] - fn test_advertised_port_mismatch_upgrades_when_following() { - let cache = test_cache_with(3, Duration::from_secs(60), true); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(8443, None)); - - assert_eq!( - cache.should_use_h3(&url), - Some(8443), - "opting in returns the advertised port so the request can be rewritten" - ); - } - - #[test] - fn test_cache_flow() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - assert!(cache.should_use_h3(&url).is_none()); - - cache.record_alt_svc(&url, &ad(443, Some(Duration::from_secs(3600)))); - assert_eq!(cache.should_use_h3(&url), Some(443)); - - cache.confirm_h3(&url, 443); - assert_eq!(cache.should_use_h3(&url), Some(443)); - assert!( - !cache - .advertised - .contains_key(&"https://example.com:443".to_string()) - ); - assert!( - cache - .confirmed - .contains_key(&"https://example.com:443".to_string()) - ); - } - - #[test] - fn test_cache_failure() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.should_use_h3(&url).is_some()); - - cache.record_h3_failure(&url); - assert!(cache.should_use_h3(&url).is_none()); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.should_use_h3(&url).is_none()); - } - - #[test] - fn test_hint() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - assert_eq!(cache.should_use_h3(&url), Some(443)); - } - - #[test] - fn test_hint_is_confirmed_not_probed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "a hint is the user's assertion and routes immediately, probe or no probe" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "nothing to verify: the hint already confirmed the origin" - ); - } - - #[test] - fn test_https_record_lands_as_an_advertisement_not_a_confirmation() { - // spec:H3UP#advertisements-from-dns — DNS is a second source of the same advertisement, - // so it makes the origin probe-worthy without routing a foreground request onto an - // unverified QUIC path. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, None, Duration::from_secs(3600)); - - assert!( - cache.confirmed_port(&url).is_none(), - "a record is evidence worth probing, not worth routing on" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "and a record naming no port describes the origin's own" - ); - } - - #[test] - fn test_https_record_port_follows_the_advertised_port_rules() { - // spec:H3UP#advertised-ports — a `port` differing from the origin's is treated exactly as - // an `Alt-Svc` advertised port: recorded, but not acted on by default. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, Some(8443), Duration::from_secs(3600)); - - assert!( - cache.probe_candidate(&url).is_none(), - "h3 on :8443 says nothing about :443, so nothing is probed" - ); - - let following = test_cache_with(3, Duration::from_secs(60), true); - following.record_https_record(&url, Some(8443), Duration::from_secs(3600)); - assert_eq!( - following.probe_candidate(&url), - Some(8443), - "opting into the quirk probes the advertised port" - ); - } - - #[test] - fn test_https_record_is_refused_while_the_origin_is_failed() { - // A failure blocks recording fresh advertisements whatever their source, or a flapping - // origin could re-enter the cycle through DNS (spec:H3UP#advertisements-from-dns). - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.record_https_record(&url, None, Duration::from_secs(3600)); - - assert!(cache.probe_candidate(&url).is_none()); - assert!( - !cache.wants_https_record(&url), - "and there is no point querying again while the cooldown runs" - ); - } - - #[test] - fn test_wants_https_record_only_while_there_is_something_to_learn() { - // spec:DNS#https-records — the gate is what keeps the query off every lookup once the - // origin's HTTP/3 support is settled either way. - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - let unknown = test_cache(); - assert!( - unknown.wants_https_record(&url), - "an origin nothing is known about is worth asking about" - ); - - let advertised = test_cache(); - advertised.record_alt_svc(&url, &ad(443, None)); - assert!( - !advertised.wants_https_record(&url), - "a live advertisement already warrants the probe a record would" - ); - - let confirmed = test_cache(); - confirmed.confirm_h3(&url, 443); - assert!( - !confirmed.wants_https_record(&url), - "a confirmed origin is already routing over HTTP/3" - ); - } - - #[test] - fn test_wants_https_record_again_once_the_advertisement_lapses() { - // The gate must not be permanent: an advertisement that expires without being confirmed - // leaves the origin unknown again, and DNS is how it can be re-learned. - let cache = test_cache_ttls(Duration::from_millis(50), Duration::from_secs(86400)); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(!cache.wants_https_record(&url)); - - std::thread::sleep(Duration::from_millis(120)); - - assert!( - cache.wants_https_record(&url), - "once the advertisement lapses the query is worth making again" - ); - } - - #[test] - fn test_https_record_ttl_bounds_the_advertisement() { - // spec:H3UP#advertisements-from-dns — the record's own DNS TTL is what the advertisement - // lives for, the role `ma` plays for a header. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_https_record(&url, None, Duration::from_millis(50)); - assert_eq!(cache.probe_candidate(&url), Some(443)); - - std::thread::sleep(Duration::from_millis(120)); - - assert!( - cache.probe_candidate(&url).is_none(), - "past the record's TTL the advertisement is no longer evidence" - ); - } - - #[test] - fn test_advertised_routes_nothing_but_probes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - assert!( - cache.confirmed_port(&url).is_none(), - "an advertisement is evidence worth probing, not worth routing on" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "and it is exactly what the probe should verify" - ); - } - - #[test] - fn test_probe_candidate_respects_failed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - - assert!( - cache.probe_candidate(&url).is_none(), - "a failed origin is not re-probed until the cooldown lapses" - ); - } - - #[test] - fn test_probe_confirmation_promotes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.claim_probe(&url), "first claim wins"); - cache.confirm_h3(&url, 443); - cache.finish_probe(&url); - - assert_eq!(cache.confirmed_port(&url), Some(443)); - assert!( - cache.probe_candidate(&url).is_none(), - "confirmed origins are not probed again" - ); - } - - #[test] - fn test_claim_probe_is_single_flight() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - assert!(cache.claim_probe(&url)); - assert!( - !cache.claim_probe(&url), - "a second claim while one is in flight loses" - ); - - cache.finish_probe(&url); - assert!( - cache.claim_probe(&url), - "finishing the probe frees the origin for the next one" - ); - } - - #[test] - fn test_slow_demotion_needs_sustained_evidence() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // Plenty of TCP samples at 5ms, but too few QUIC samples to act on. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - } - for _ in 0..(EWMA_MIN_SAMPLES - 1) { - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "below the minimum sample count no comparison may act" - ); - } - - #[test] - fn test_slow_demotion_moves_origin_back_to_probing() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // TCP steady at 5ms, QUIC steady at 50ms: 10x the average and 45ms over, - // clearing both the factor and the absolute floor. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert!( - cache.confirmed_port(&url).is_none(), - "a sustained large gap demotes the origin off HTTP/3" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "while the slow marker lives, the origin is not re-probed either" - ); - assert!( - !cache.is_failed("https://example.com:443"), - "slow is not broken: the failed cache stays out of it" - ); - - // The test cache's slow TTL is short; once it lapses, the advertisement - // preserved by the demotion re-enters through a probe. - std::thread::sleep(Duration::from_millis(300)); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "slow expiry re-enters via the probe, asking whether the path improved" - ); - } - - #[test] - fn test_parity_or_moderately_slower_quic_is_kept() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - // QUIC 2x slower and 20ms over: above the floor but below the 2.5x - // factor, so HTTP/3's structural advantages win the tie. - for _ in 0..(EWMA_MIN_SAMPLES * 2) { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(20)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(40)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "moderately slower QUIC is still preferred" - ); - } - - #[test] - fn test_cancellation_below_threshold_keeps_h3() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "two strikes is not enough to demote" - ); - } - - #[test] - fn test_cancellation_at_threshold_demotes() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - for _ in 0..3 { - cache.record_h3_cancellation(&url); - } - - assert!( - cache.should_use_h3(&url).is_none(), - "three strikes demotes the origin" - ); - assert!( - cache.is_failed("https://example.com:443"), - "demotion goes through the failed cache, so re-advertisement can't re-arm it" - ); - } - - #[test] - fn test_cancellation_reset_by_h3_success() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - cache.confirm_h3(&url, 443); - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "a working h3 response clears the strikes, so these two start over" - ); - } - - #[test] - fn test_cancellation_disabled_by_zero() { - let cache = test_cache_with(0, Duration::from_secs(60), false); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - for _ in 0..5 { - cache.record_h3_cancellation(&url); - } - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "cancel_strikes: 0 disables cancellation-based demotion" - ); - } - - #[test] - fn test_cancellation_strikes_decay() { - let cache = test_cache_with(3, Duration::from_millis(50), false); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - std::thread::sleep(Duration::from_millis(150)); - cache.record_h3_cancellation(&url); - cache.record_h3_cancellation(&url); - - assert_eq!( - cache.should_use_h3(&url), - Some(443), - "strikes older than the window don't count towards the run" - ); - } - - fn failure_entry(cache: &AltSvcCache) -> FailureEntry { - cache - .failed - .get("https://example.com:443") - .expect("the origin has a failure on record") - } - - #[test] - fn test_failure_cooldown_doubles_up_to_the_cap() { - let cache = test_cache(); - - let schedule: Vec = (1..=6) - .map(|count| cache.failure_cooldown(count).as_secs()) - .collect(); - - assert_eq!( - schedule, - vec![300, 600, 1200, 2400, 3600, 3600], - "each consecutive failure doubles the base, then holds at the cap" - ); - } - - #[test] - fn test_failure_cooldown_cap_below_base_is_flat() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_secs(300), - Duration::from_secs(60), - ); - - let schedule: Vec = (1..=4) - .map(|count| cache.failure_cooldown(count).as_secs()) - .collect(); - - assert_eq!( - schedule, - vec![300, 300, 300, 300], - "a cap under the base is clamped up to it, giving a cooldown that never backs off" - ); - } - - #[test] - fn test_consecutive_failures_lengthen_the_cooldown() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_millis(200), - Duration::from_secs(60), - ); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - assert!( - cache.is_failed("https://example.com:443"), - "the first failure blocks the origin" - ); - - // Past the first cooldown, but well inside the run's own lifetime: this is - // the retry the cooldown allowed, and it fails too. - std::thread::sleep(Duration::from_millis(250)); - assert!( - !cache.is_failed("https://example.com:443"), - "the first cooldown lapses on its own" - ); - - cache.record_h3_failure(&url); - let entry = failure_entry(&cache); - assert_eq!( - entry.count, 2, - "failing again straight after a lapsed cooldown continues the run" - ); - assert!( - cache.is_failed("https://example.com:443"), - "and blocks the origin again, for twice as long" - ); - } - - #[test] - fn test_run_lapses_when_the_origin_stops_failing() { - let cache = test_cache_failing( - 3, - Duration::from_secs(60), - false, - Duration::from_millis(100), - Duration::from_secs(60), - ); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - // One cooldown beyond the block it caused: nobody exercised the origin in - // that stretch, so the next failure is judged on its own. - std::thread::sleep(Duration::from_millis(300)); - - cache.record_h3_failure(&url); - assert_eq!( - failure_entry(&cache).count, - 1, - "an origin left alone past its run starts from the base cooldown again" - ); - } - - #[test] - fn test_confirmation_ends_the_run() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.record_h3_failure(&url); - assert_eq!(failure_entry(&cache).count, 2); - - cache.confirm_h3(&url, 443); - cache.record_h3_failure(&url); - - let entry = failure_entry(&cache); - assert_eq!( - entry.count, 1, - "a working h3 response ends the run, so the next failure starts at the base" - ); - assert_eq!( - entry.blocked_until.duration_since(Instant::now()).as_secs(), - 299, - "and is blocked for the base cooldown, not the doubled one" - ); - } - - #[test] - fn test_confirmation_does_not_cut_a_live_cooldown_short() { - // A confirmation can race a concurrent failure. The failure is the more - // recent word on the path, so it keeps the origin blocked; only the run - // is forgotten. - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_h3_failure(&url); - cache.confirm_h3(&url, 443); - - assert!( - cache.is_failed("https://example.com:443"), - "the cooldown the failure set still runs" - ); - assert_eq!( - failure_entry(&cache).count, - 0, - "but the run behind it is cleared" - ); - } - - #[test] - fn test_network_change_demotes_confirmed_to_advertised() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - - cache.network_changed(); - - assert!( - cache.confirmed_port(&url).is_none(), - "the path that proved HTTP/3 is gone, so the origin is no longer confirmed" - ); - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "it keeps its advertisement, so a background probe re-verifies it at once" - ); - } - - #[test] - fn test_network_change_clears_failures_and_their_backoff() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - cache.record_alt_svc(&url, &ad(443, None)); - - cache.network_changed(); - - assert!( - !cache.is_failed("https://example.com:443"), - "a blocked UDP path is a fact about the old network" - ); - assert!( - cache.failed.get("https://example.com:443").is_none(), - "and so is the run of failures that set the cooldown" - ); - - // The advertisement a failure discards has to come back for the origin to be - // probe-worthy again, so re-record it as a live response would. - cache.record_alt_svc(&url, &ad(443, None)); - cache.record_h3_failure(&url); - assert_eq!( - failure_entry(&cache).count, - 1, - "failing on the new network starts the backoff from the base cooldown" - ); - } - - #[test] - fn test_network_change_clears_a_slow_demotion() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - assert!( - cache.probe_candidate(&url).is_none(), - "the slow marker holds the origin off probing before the signal" - ); - - cache.network_changed(); - - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "a slow path was slow on the old network, so the origin re-enters through a probe" - ); - } - - #[test] - fn test_network_change_clears_the_path_time_averages() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - cache.confirm_h3(&url, 443); - // Enough TCP samples to satisfy the comparison's minimum, all of them fast. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_2, Duration::from_millis(5)); - } - - cache.network_changed(); - cache.confirm_h3(&url, 443); - - // Slow enough to demote several times over, were the old TCP average still there - // to compare against. - for _ in 0..EWMA_MIN_SAMPLES { - cache.record_path_time(&url, http::Version::HTTP_3, Duration::from_millis(50)); - } - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "with the TCP average cleared there is nothing to judge QUIC against, so \ - no demotion happens on one side's samples alone" - ); - } - - #[test] - fn test_network_change_keeps_hints_confirmed() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - - cache.network_changed(); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "a hint is the caller's assertion, not an observation, so it survives the signal \ - and keeps an HTTP/3-only origin reachable" - ); - assert!( - cache.probe_candidate(&url).is_none(), - "and a hinted origin is still never probed" - ); - } - - #[test] - fn test_network_change_lets_a_blocked_hint_take_effect() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.add_hint("example.com", 443); - cache.record_h3_failure(&url); - assert!( - cache.confirmed_port(&url).is_none(), - "a failure demotes a hinted origin like any other" - ); - - cache.network_changed(); - - assert_eq!( - cache.confirmed_port(&url), - Some(443), - "the failure that was masking the hint belonged to the old network, so the \ - hint holds again once it is cleared" - ); - } - - #[test] - fn test_network_change_keeps_advertisements() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - - cache.network_changed(); - - assert_eq!( - cache.probe_candidate(&url), - Some(443), - "an advertisement is the origin's statement about itself, which a change of \ - client network does not revise" - ); - } - - #[test] - fn test_network_change_drops_expired_confirmations() { - let cache = test_cache_ttls(Duration::from_millis(100), Duration::from_millis(100)); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.confirm_h3(&url, 443); - std::thread::sleep(Duration::from_millis(150)); - - cache.network_changed(); - - assert!( - cache.probe_candidate(&url).is_none(), - "a confirmation that had already lapsed is not knowledge to carry forward as \ - a fresh advertisement" - ); - } - - #[test] - fn test_network_change_releases_probe_claims() { - let cache = test_cache(); - let url = reqwest::Url::parse("https://example.com/path").unwrap(); - - cache.record_alt_svc(&url, &ad(443, None)); - assert!(cache.claim_probe(&url), "the first probe claims the origin"); - assert!(!cache.claim_probe(&url), "and holds it single-flight"); - - cache.network_changed(); - - assert!( - cache.claim_probe(&url), - "probes in flight are aborted with the client they ran on, so their claims \ - must not hold the origin for the claim TTL" - ); - } -} +pub use cache::{ + AltSvcAdvertisement, AltSvcCache, AltSvcCacheConfig, AltSvcEntry, PathTime, SLOW_FLOOR_MS, +}; +pub use header::parse_alt_svc_header; +pub use middleware::{AltSvcMiddleware, ArrivalStamp}; +pub use prober::{H3HttpsSink, H3Prober}; diff --git a/crates/web-faith-alt-svc/src/middleware.rs b/crates/web-faith-alt-svc/src/middleware.rs new file mode 100644 index 0000000..365fcf2 --- /dev/null +++ b/crates/web-faith-alt-svc/src/middleware.rs @@ -0,0 +1,278 @@ +use std::{ + marker::PhantomData, + sync::Arc, + time::{Duration, Instant}, +}; + +use http::Extensions; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result}; + +use crate::{cache::AltSvcCache, header::parse_alt_svc_header, prober::H3Prober}; + +/// Recording the moment a response's headers arrived. +/// +/// The stamp itself belongs to the client, which puts it in the request's extensions and reads it +/// back out to surface the timing; this layer is only the one place that observes the arrival, so +/// all it needs is to be able to mark what the client left there. +pub trait ArrivalStamp: Send + Sync + 'static { + fn mark(&self, at: Instant); +} + +/// Records a cancellation if the HTTP/3 attempt it guards is dropped before +/// producing an outcome. +/// +/// [`AltSvcMiddleware`] can only learn that HTTP/3 is broken from the attempt's +/// return value, and a cancelled request never produces one: `faith_fetch` +/// races `send()` against the abort signal in a `select!`, which drops the +/// losing future. Without this guard nothing ever demotes the origin, so a +/// caller whose deadline is shorter than the network's own failure detection +/// re-attempts HTTP/3 over a dead path on every retry, indefinitely. +struct H3AttemptGuard { + cache: Arc, + url: reqwest::Url, + armed: bool, +} + +impl H3AttemptGuard { + fn new(cache: Arc, url: reqwest::Url) -> Self { + Self { + cache, + url, + armed: true, + } + } + + /// The attempt produced an outcome, so it speaks for itself. + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for H3AttemptGuard { + fn drop(&mut self) { + // Must stay infallible: this can run while unwinding, where a panic + // would abort the process. moka's sync cache does not panic on insert. + if self.armed { + self.cache.record_h3_cancellation(&self.url); + } + } +} + +/// The Alt-Svc layer, which upgrades an origin to HTTP/3 once it advertises one. +/// +/// `S` is the client's arrival stamp, which this marks when a response's headers land; see +/// [`ArrivalStamp`]. +#[derive(Clone)] +pub struct AltSvcMiddleware { + cache: Arc, + enabled: bool, + /// Ceiling on how long an HTTP/3 attempt may take to produce response + /// headers before it is treated as failed and retried over TCP. + attempt_timeout: Option, + /// `Some` routes foreground requests on confirmed origins only, verifying + /// advertisements in the background. `None` restores the inline upgrade, + /// where the next foreground request is the verification. + prober: Option>, + stamp: PhantomData, +} + +impl std::fmt::Debug for AltSvcMiddleware { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AltSvcMiddleware") + .field("enabled", &self.enabled) + .field("attempt_timeout", &self.attempt_timeout) + .field("prober", &self.prober) + .field("cache", &self.cache) + .finish() + } +} + +impl AltSvcMiddleware { + pub fn new( + cache: Arc, + enabled: bool, + attempt_timeout: Option, + prober: Option>, + ) -> Self { + Self { + cache, + enabled, + attempt_timeout, + prober, + stamp: PhantomData, + } + } + + #[allow(dead_code)] + pub fn cache(&self) -> &Arc { + &self.cache + } + + /// Kick off a background probe for the URL's origin if one is warranted: + /// probing enabled, an actionable advertisement present, the origin neither + /// confirmed, failed, nor slow, and no probe already in flight. + fn maybe_probe(&self, url: &reqwest::Url) { + if let Some(prober) = &self.prober { + prober.maybe_probe(url); + } + } +} + +/// Run the rest of the stack and stamp the moment the response headers arrive. +/// +/// This is the one place a response's arrival is observed: the returned instant is what the +/// path-time average measures against, and the same instant reaches the caller through the +/// request's extensions to become the surfaced timing, so the two can never disagree. +// spec:RESP#request-timing +async fn run_stamped( + next: Next<'_>, + req: Request, + extensions: &mut Extensions, +) -> (Result, Instant) { + let result = next.run(req, extensions).await; + let at = Instant::now(); + if result.is_ok() + && let Some(stamp) = extensions.get::() + { + stamp.mark(at); + } + (result, at) +} + +#[async_trait::async_trait] +impl Middleware for AltSvcMiddleware { + async fn handle( + &self, + mut req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + if !self.enabled { + return run_stamped::(next, req, extensions).await.0; + } + + let url = req.url().clone(); + + // With a prober, routing consults proven origins only — advertisements + // get verified out-of-band, so no foreground request ever waits on an + // unverified QUIC path. Without one, the legacy inline upgrade applies. + let h3_route = if self.prober.is_some() { + self.cache.confirmed_port(&url) + } else { + self.cache.should_use_h3(&url) + }; + + if let Some(h3_port) = h3_route { + // Clone the request before attempting HTTP/3 so we can retry with TCP if it fails + if let Some(req_clone) = req.try_clone() { + *req.version_mut() = http::Version::HTTP_3; + + // A port differing from the origin's only comes back when + // `follow_advertised_port` is set — `should_use_h3` filters + // mismatches out otherwise. Rewriting the URL is the only way to + // make reqwest connect elsewhere, and it MUST happen after the + // clone above so the TCP fallback still targets the origin. + // + // Every cache operation below keeps using `url`, the origin, so + // confirmations, failures and strikes stay keyed on the origin + // rather than on the alternative endpoint. + if Some(h3_port) != url.port_or_known_default() { + let _ = req.url_mut().set_port(Some(h3_port)); + } + + let mut guard = H3AttemptGuard::new(Arc::clone(&self.cache), url.clone()); + // Measured to response headers: this layer sits inside the cache + // middleware, so `next.run` resolves when headers arrive, before + // any body buffering. + let started = Instant::now(); + // `None` means the attempt ran out of time. Bound in its own + // statement so the mutable borrow of `extensions` ends here, + // leaving the fallback below free to use it. + let outcome = match self.attempt_timeout { + Some(limit) => { + tokio::time::timeout(limit, run_stamped::(next.clone(), req, extensions)) + .await + .ok() + } + None => Some(run_stamped::(next.clone(), req, extensions).await), + }; + // Reached on success, error and expiry alike; only a mid-flight + // drop skips it and leaves the guard armed. + guard.disarm(); + + match outcome { + Some((Ok(response), at)) => { + if response.version() == http::Version::HTTP_3 { + self.cache.confirm_h3(&url, h3_port); + self.cache.record_path_time( + &url, + response.version(), + at.duration_since(started), + ); + } + + if let Some(alt_svc) = response.headers().get("alt-svc") { + if let Ok(value) = alt_svc.to_str() { + if let Some(advertisement) = parse_alt_svc_header(value) { + self.cache.record_alt_svc(&url, &advertisement); + } + } + } + + Ok(response) + } + // An expired deadline is as good as an error: HTTP/3 did not + // deliver. Taking the fallback branch directly avoids having + // to synthesise a reqwest_middleware::Error, which would mean + // adding anyhow as a dependency. + Some((Err(_), _)) | None => { + self.cache.record_h3_failure(&url); + + // Use the cloned request (which still has default HTTP version) + let started = Instant::now(); + let (result, at) = run_stamped::(next, req_clone, extensions).await; + if let Ok(ref response) = result { + self.cache.record_path_time( + &url, + response.version(), + at.duration_since(started), + ); + } + result + } + } + } else { + // Can't clone request (streaming body), just proceed without HTTP/3 + run_stamped::(next, req, extensions).await.0 + } + } else { + // An advertisement from an earlier response may still be waiting on + // verification (or on a fresh single-flight claim after a probe task + // died); this is the belt to the post-response trigger's braces. + self.maybe_probe(&url); + + let started = Instant::now(); + let (result, at) = run_stamped::(next, req, extensions).await; + + // Check for Alt-Svc header in non-HTTP/3 responses + if let Ok(ref response) = result { + self.cache + .record_path_time(&url, response.version(), at.duration_since(started)); + + if let Some(alt_svc) = response.headers().get("alt-svc") { + if let Ok(value) = alt_svc.to_str() { + if let Some(advertisement) = parse_alt_svc_header(value) { + self.cache.record_alt_svc(&url, &advertisement); + // Probe as soon as the advertisement lands, racing + // the gap before the caller's next request. + self.maybe_probe(&url); + } + } + } + } + + result + } + } +} diff --git a/crates/web-faith-alt-svc/src/prober.rs b/crates/web-faith-alt-svc/src/prober.rs new file mode 100644 index 0000000..c66630a --- /dev/null +++ b/crates/web-faith-alt-svc/src/prober.rs @@ -0,0 +1,205 @@ +use std::{sync::Arc, time::Duration}; + +use crate::cache::AltSvcCache; + +/// Verifies advertised HTTP/3 endpoints in the background, so no foreground +/// request ever waits on an unverified QUIC path. +/// +/// The probe is a real request — `HEAD /` sent with `Version::HTTP_3` — on the +/// **raw** `reqwest::Client`, not the middleware stack. That is load-bearing +/// three times over: it bypasses the HTTP cache, so a replayed cached response +/// (rebuilt with its stored HTTP version) can never fake a confirmation; it +/// bypasses [`AltSvcMiddleware`](crate::AltSvcMiddleware), so probing cannot recurse; and it shares the +/// h3 connection pool with foreground requests, so a successful probe leaves +/// behind a warm QUIC connection the next request rides. Confirmation doubles +/// as prewarming. +/// +/// Any HTTP/3 response confirms, regardless of status: a 401 or 405 to +/// `HEAD /` proves the transport end-to-end just as well as a 200. +pub struct H3Prober { + client: reqwest::Client, + cache: Arc, + /// `None` leaves the attempt bounded only by the QUIC idle timeout. + timeout: Option, + /// Handles for in-flight probes, so `Agent::close` can abort them: a probe + /// holds a clone of the raw client, which would otherwise keep the + /// connection pool alive past close for up to the probe timeout. + tasks: std::sync::Mutex>, +} + +impl std::fmt::Debug for H3Prober { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("H3Prober") + .field("timeout", &self.timeout) + .finish() + } +} + +impl H3Prober { + pub fn new( + client: reqwest::Client, + cache: Arc, + timeout: Option, + ) -> Self { + Self { + client, + cache, + timeout, + tasks: std::sync::Mutex::new(Vec::new()), + } + } + + /// Spawn a probe of `port` for the origin of `url`. The caller must hold the + /// origin's single-flight claim (see [`AltSvcCache::claim_probe`]). + fn spawn(&self, url: reqwest::Url, port: u16) { + let client = self.client.clone(); + let cache = Arc::clone(&self.cache); + let timeout = self.timeout; + + let handle = tokio::spawn(async move { + let mut probe_url = url.clone(); + probe_url.set_path("/"); + probe_url.set_query(None); + probe_url.set_fragment(None); + let _ = probe_url.set_username(""); + let _ = probe_url.set_password(None); + // Same rewrite rule as the foreground path: a port differing from the + // origin's only gets here when `follow_advertised_port` is on. + if Some(port) != url.port_or_known_default() { + let _ = probe_url.set_port(Some(port)); + } + + let attempt = client.head(probe_url).version(http::Version::HTTP_3).send(); + + let outcome = match timeout { + Some(limit) => tokio::time::timeout(limit, attempt).await.ok(), + None => Some(attempt.await), + }; + + // Cache operations stay keyed on `url`, the origin, matching the + // foreground path. + match outcome { + Some(Ok(response)) if response.version() == http::Version::HTTP_3 => { + cache.confirm_h3(&url, port); + } + // A response that is somehow not HTTP/3 is a failure too: the + // h3 route did not deliver, whatever answered. + _ => cache.record_h3_failure(&url), + } + + // An aborted probe never reaches this; its claim expires on the + // probing TTL instead, which is why that TTL exceeds the timeout. + cache.finish_probe(&url); + }); + + // A poisoned lock only means another thread panicked mid-push; the Vec + // is still sound to use, and probing must never take the process down. + let mut tasks = self + .tasks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + tasks.retain(|task| !task.is_finished()); + tasks.push(handle.abort_handle()); + } + + /// Kick off a background probe for the URL's origin if one is warranted: an actionable + /// advertisement present, the origin neither confirmed, failed, nor slow, and no probe + /// already in flight. The same decision the Alt-Svc layer makes on a request, exposed so a + /// `preconnect` TCP warm-up to a probe-worthy origin triggers a probe as a real request would. + pub fn maybe_probe(&self, url: &reqwest::Url) { + let Some(port) = self.cache.probe_candidate(url) else { + return; + }; + if !self.cache.claim_probe(url) { + return; + } + self.spawn(url.clone(), port); + } + + pub fn abort_all(&self) { + let mut tasks = self + .tasks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for task in tasks.drain(..) { + task.abort(); + } + } +} + +/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is +/// probe-worthy from its first request rather than from the first TCP response carrying an +/// `Alt-Svc` header. +/// +/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), +/// which is the only place that holds all three: the resolver is built before the cache, and the +/// prober holds a client that holds the resolver, so nothing lower down can own this. +/// +/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the +/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could +/// name. Recording it there is right whichever request triggered the lookup, because what the +/// record describes does not depend on who asked. +// spec:H3UP#advertisements-from-dns +// spec:DNS#https-records +pub struct H3HttpsSink { + cache: Arc, + /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, + /// and the resolver holds this sink. A strong reference here would close that ring and leak + /// the whole graph — connection pool included — past `Agent::close`, which works by dropping + /// the client. The agent owns the only strong reference, so this lives exactly as long as the + /// agent's prober does. + /// + /// `None` rather than a dead handle when probing is off, where an advertisement is acted on + /// inline by the next foreground request instead. + // spec:PROBE + prober: Option>, +} + +impl std::fmt::Debug for H3HttpsSink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("H3HttpsSink") + .field("probing", &self.prober.is_some()) + .finish() + } +} + +impl H3HttpsSink { + pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { + Self { + cache, + prober: prober.map(Arc::downgrade), + } + } + + /// The origin a record at `host` describes: the default HTTPS port, which is the port whose + /// record lives at the bare name. + fn origin_url(host: &str) -> Option { + reqwest::Url::parse(&format!("https://{host}")).ok() + } +} + +impl web_faith_dns::HttpsSink for H3HttpsSink { + fn wants(&self, host: &str) -> bool { + Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) + } + + fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { + let Some(url) = Self::origin_url(host) else { + return; + }; + + self.cache + .record_https_record(&url, advertisement.port, advertisement.ttl); + + // Probe straight away rather than waiting for the request that triggered the lookup to + // finish: the point of reading DNS is that the path can be verified while that request is + // still on TCP, so the one after it upgrades. + // + // A prober that has gone means the agent was closed (or rebuilt) while this query was in + // flight; the advertisement above is still worth keeping, but there is nothing left to + // probe it with, and resurrecting a dropped client to try would be exactly wrong. + if let Some(prober) = self.prober.as_ref().and_then(std::sync::Weak::upgrade) { + prober.maybe_probe(&url); + } + } +} diff --git a/crates/web-faith-dns/src/discovery.rs b/crates/web-faith-dns/src/discovery.rs new file mode 100644 index 0000000..21cac62 --- /dev/null +++ b/crates/web-faith-dns/src/discovery.rs @@ -0,0 +1,178 @@ +use hickory_resolver::{ + TokioResolver, + config::{ + GOOGLE, LookupIpStrategy, NameServerConfig, OpportunisticEncryption, ResolveHosts, + ResolverConfig, ServerOrderingStrategy, + }, + net::{NetError, runtime::TokioRuntimeProvider}, + system_conf::read_system_conf, +}; + +use crate::{ + settings::{ResolverReport, ResolverSettings, ResolverSource, report}, + transport::ServerHost, +}; + +/// The resolver and the report of its servers, built together the first time the resolver is used. +pub(crate) struct Built { + pub(crate) resolver: TokioResolver, + pub(crate) reports: Vec, +} + +/// Apply the options common to every resolver Faith builds: race both families for Happy Eyeballs, +/// hold the caller's order fixed rather than reordering by latency, and layer any `dns.*` timeout, +/// ndots, and hosts-file settings on top. +pub(crate) fn apply_options( + builder: &mut hickory_resolver::ResolverBuilder, + settings: &ResolverSettings, +) { + let options = builder.options_mut(); + options.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + // The list expresses the caller's intent, not a performance hint, so a private resolver named + // first must not lose traffic to a closer fallback (spec:DNS#server-order). + options.server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; + if let Some(timeout) = settings.timeout { + options.timeout = timeout; + } + if let Some(ndots) = settings.ndots { + options.ndots = ndots; + } + if let Some(hosts_file) = settings.hosts_file { + options.use_hosts_file = if hosts_file { + ResolveHosts::Always + } else { + ResolveHosts::Never + }; + } +} + +/// Discovery: configure from the system, then let hickory's RFC 9539 opportunistic encryption +/// upgrade those servers to DoT/DoQ where they answer a probe. `dns.searchDomains` overrides the +/// system search list when set. +// spec:DNS#discovery +pub(crate) fn build_discovery(settings: &ResolverSettings) -> Result { + let (mut config, options) = read_system_conf().unwrap_or_else(|_| { + // A host with no readable resolver configuration falls back to Google Public DNS over + // conventional DNS, probed like any other server (spec:DNS#discovery). + ( + ResolverConfig::udp_and_tcp(&GOOGLE), + hickory_resolver::config::ResolverOpts::default(), + ) + }); + + if let Some(search) = &settings.search_domains { + config = ResolverConfig::from_parts(None, search.clone(), config.name_servers().to_vec()); + } + + let reports = report(config.name_servers(), ResolverSource::Conventional); + + let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()) + .with_options(options); + apply_options(&mut builder, settings); + let builder = builder.with_opportunistic_encryption(OpportunisticEncryption::Enabled { + config: Default::default(), + }); + + Ok(Built { + resolver: builder.build()?, + reports, + }) +} + +/// The resolver that bootstraps hostname servers: the listed IP-host servers in order, so an +/// encrypted server placed first resolves its siblings without exposing the hostname in plaintext. +/// Where the list names no IP host, the system's own configuration bootstraps instead. +pub(crate) fn bootstrap_resolver(settings: &ResolverSettings) -> Result { + let ip_servers: Vec = settings + .servers + .iter() + .filter_map(|spec| spec.ip().map(|ip| spec.to_name_server(ip))) + .collect(); + + let mut builder = if ip_servers.is_empty() { + TokioResolver::builder_tokio().unwrap_or_else(|_| { + TokioResolver::builder_with_config( + ResolverConfig::udp_and_tcp(&GOOGLE), + TokioRuntimeProvider::default(), + ) + }) + } else { + TokioResolver::builder_with_config( + ResolverConfig::from_parts(None, vec![], ip_servers), + TokioRuntimeProvider::default(), + ) + }; + builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + builder.options_mut().server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; + builder.build() +} + +/// Build the configured (or discovered) resolver and the report of its servers. +pub(crate) async fn build(settings: &ResolverSettings) -> Result { + if settings.servers.is_empty() { + build_discovery(settings) + } else { + build_listed(settings).await + } +} + +/// The listed-servers path: bootstrap any hostname hosts to addresses, then build the resolver +/// from the parsed specs in order. +// spec:DNS#transports +// spec:DNS#bootstrapping +pub(crate) async fn build_listed(settings: &ResolverSettings) -> Result { + let name_servers = build_name_servers(settings).await?; + + let search = settings.search_domains.clone().unwrap_or_default(); + let config = ResolverConfig::from_parts(None, search, name_servers.clone()); + let reports = report(&name_servers, ResolverSource::Configured); + + let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()); + apply_options(&mut builder, settings); + + Ok(Built { + resolver: builder.build()?, + reports, + }) +} + +/// Resolve the listed servers to hickory name servers, bootstrapping hostname hosts. A hostname +/// that will not resolve drops that server for the life of the agent rather than failing the +/// resolver. +// spec:DNS#bootstrapping +pub(crate) async fn build_name_servers( + settings: &ResolverSettings, +) -> Result, NetError> { + let needs_bootstrap = settings.servers.iter().any(|spec| spec.ip().is_none()); + let bootstrap = if needs_bootstrap { + Some(bootstrap_resolver(settings)?) + } else { + None + }; + + let mut name_servers = Vec::with_capacity(settings.servers.len()); + for spec in &settings.servers { + let ip = match spec.ip() { + Some(ip) => ip, + None => { + let ServerHost::Name(host) = &spec.host else { + unreachable!("ip() is None only for a hostname host"); + }; + let resolver = bootstrap + .as_ref() + .expect("bootstrap resolver built when a hostname host is present"); + match resolver.lookup_ip(host.as_str()).await { + Ok(lookup) => match lookup.iter().next() { + Some(ip) => ip, + None => continue, + }, + // The hostname does not resolve: drop this server for the agent's life. + Err(_) => continue, + } + } + }; + name_servers.push(spec.to_name_server(ip)); + } + + Ok(name_servers) +} diff --git a/crates/web-faith-dns/src/https.rs b/crates/web-faith-dns/src/https.rs new file mode 100644 index 0000000..5acfc5f --- /dev/null +++ b/crates/web-faith-dns/src/https.rs @@ -0,0 +1,110 @@ +use std::time::Duration; + +use hickory_resolver::proto::rr::{ + Name, RData, + rdata::svcb::{SvcParamKey, SvcParamValue}, +}; + +/// What an `HTTPS` record said about an origin's HTTP/3 support. +// spec:DNS#https-records +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HttpsAdvertisement { + /// The record's `port` SvcParam, or `None` when it named none and the origin's own port + /// applies. + pub port: Option, + /// The record's own DNS TTL, which is how long the advertisement it carries lives. + pub ttl: Duration, +} + +/// Where an `HTTPS` record's advertisement goes once the resolver has read one. +/// +/// The resolver cannot own the HTTP/3 upgrade cache directly: that cache is built after the +/// resolver, and the background prober holds a client which holds the resolver in turn. So the +/// caller installs this afterwards (see [`FaithResolver::set_https_sink`](crate::FaithResolver::set_https_sink)), which also keeps this +/// crate free of the upgrade layer's types. +pub trait HttpsSink: Send + Sync { + /// Whether an `HTTPS` record for `host` is worth querying at all right now. + /// + /// Asked before the query so an origin already confirmed, already failed, or already holding a + /// live advertisement costs no DNS traffic to re-learn what is known. + fn wants(&self, host: &str) -> bool; + + /// Fold a record's advertisement into the upgrade layer's knowledge of `host`. + fn record(&self, host: &str, advertisement: HttpsAdvertisement); +} + +/// Whether an ALPN token names a version of HTTP/3. +/// +/// The same family test the `Alt-Svc` reader applies, so a draft token like `h3-29` counts here +/// exactly as it does in a header. +// spec:H3UP#reading-advertisements +pub(crate) fn is_h3_alpn(token: &str) -> bool { + token == "h3" || token.starts_with("h3-") +} + +/// Read the HTTP/3 advertisement out of an `HTTPS` answer for `name`, if it carries one. +/// +/// Only ServiceMode records are considered: an AliasMode record (`svc_priority` 0) redirects to +/// another name rather than describing this one, and following that redirection is a resolution +/// step this does not take. Among the rest the lowest `svc_priority` wins, which is the preference +/// order RFC 9460 defines. +/// +/// A record whose target is neither the root (which per RFC 9460 §2.5.2 means the owner name +/// itself) nor the queried name designates a *different* host, and Faith only upgrades to the +/// origin's own host, so such a record is not acted on. +/// `queried` is the name the answer was actually asked for rather than the host as written, since +/// the search list can requalify a name before it reaches a server; comparison ignores the trailing +/// root so the two are judged on identity rather than on how each was spelled. +// spec:H3UP#advertisements-from-dns +pub(crate) fn read_https_answer( + queried: &Name, + answers: &[hickory_resolver::proto::rr::Record], +) -> Option { + let mut best: Option<(u16, HttpsAdvertisement)> = None; + + for record in answers { + let RData::HTTPS(https) = &record.data else { + continue; + }; + + if https.svc_priority == 0 { + continue; + } + + if !https.target_name.is_root() && !https.target_name.eq_ignore_root(queried) { + continue; + } + + let mut has_h3 = false; + let mut port = None; + for (key, value) in &https.svc_params { + match (key, value) { + (SvcParamKey::Alpn, SvcParamValue::Alpn(alpn)) => { + has_h3 = alpn.0.iter().any(|token| is_h3_alpn(token)); + } + (SvcParamKey::Port, SvcParamValue::Port(value)) => port = Some(*value), + _ => {} + } + } + + if !has_h3 { + continue; + } + + let advertisement = HttpsAdvertisement { + port, + ttl: Duration::from_secs(record.ttl.into()), + }; + if best + .as_ref() + .is_none_or(|(best, _)| https.svc_priority < *best) + { + best = Some((https.svc_priority, advertisement)); + } + } + + best.map(|(_, advertisement)| advertisement) +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/https/tests.rs b/crates/web-faith-dns/src/https/tests.rs new file mode 100644 index 0000000..befece5 --- /dev/null +++ b/crates/web-faith-dns/src/https/tests.rs @@ -0,0 +1,204 @@ +use std::time::Duration; + +use hickory_resolver::proto::rr::{ + Name, RData, Record, + rdata::{ + HTTPS, SVCB, + svcb::{Alpn, SvcParamKey, SvcParamValue}, + }, +}; + +use super::{HttpsAdvertisement, read_https_answer}; + +fn name(input: &str) -> Name { + Name::from_utf8(input).unwrap() +} + +/// One `HTTPS` answer, spelled the way a server would send it. +fn record( + owner: &str, + priority: u16, + target: Name, + params: Vec<(SvcParamKey, SvcParamValue)>, + ttl: u32, +) -> Record { + Record::from_rdata( + name(owner), + ttl, + RData::HTTPS(HTTPS(SVCB::new(priority, target, params))), + ) +} + +fn alpn(tokens: &[&str]) -> (SvcParamKey, SvcParamValue) { + ( + SvcParamKey::Alpn, + SvcParamValue::Alpn(Alpn(tokens.iter().map(|t| (*t).to_owned()).collect())), + ) +} + +#[test] +fn an_h3_alpn_on_the_owner_name_advertises() { + // spec:H3UP#advertisements-from-dns — the ordinary case: `.` as the target means the + // owner name, so the record describes the origin itself. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h2", "h3"])], + 3600, + )]; + + assert_eq!( + read_https_answer(&queried, &answers), + Some(HttpsAdvertisement { + port: None, + ttl: Duration::from_secs(3600), + }), + "an `alpn` listing h3 is an advertisement, and the record's own TTL bounds it" + ); +} + +#[test] +fn a_draft_h3_token_counts_like_the_header_reader_treats_one() { + // spec:H3UP#reading-advertisements — `h3-29` is an h3-family token either way it + // arrives, so DNS must not be stricter than the `Alt-Svc` reader. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h3-29"])], + 60, + )]; + + assert!(read_https_answer(&queried, &answers).is_some()); +} + +#[test] +fn a_record_without_h3_in_its_alpn_advertises_nothing() { + // An origin that speaks only HTTP/2 says so here, and reading that as an h3 + // advertisement would send every such origin down a probe that must fail. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![alpn(&["h2"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn the_port_parameter_is_carried_through() { + // spec:H3UP#advertisements-from-dns — a differing port is handled by the same + // machinery an `Alt-Svc` advertised port is. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(8443)), + ], + 3600, + )]; + + assert_eq!( + read_https_answer(&queried, &answers).and_then(|ad| ad.port), + Some(8443) + ); +} + +#[test] +fn a_record_targeting_another_host_is_not_acted_on() { + // Faith only upgrades to the origin's own host, exactly as it refuses an `Alt-Svc` + // advertisement naming a different one (spec:H3UP#advertisements-from-dns). + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + name("cdn.example.net."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn a_record_naming_the_queried_host_itself_is_acted_on() { + // Spelling the owner name out is equivalent to the `.` shorthand, and the comparison + // ignores case and the trailing root the way name equality should. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 1, + name("ExAmPlE.CoM."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert!(read_https_answer(&queried, &answers).is_some()); +} + +#[test] +fn an_alias_mode_record_is_skipped() { + // Priority 0 is AliasMode: it redirects to another name rather than describing this + // one, and following that redirection is a resolution step this does not take. + let queried = name("example.com."); + let answers = [record( + "example.com.", + 0, + name("svc.example.net."), + vec![alpn(&["h3"])], + 3600, + )]; + + assert_eq!(read_https_answer(&queried, &answers), None); +} + +#[test] +fn the_lowest_priority_service_mode_record_wins() { + // RFC 9460 orders ServiceMode records by ascending priority, so the most preferred + // record is the one whose port is acted on. + let queried = name("example.com."); + let answers = [ + record( + "example.com.", + 9, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(9443)), + ], + 3600, + ), + record( + "example.com.", + 2, + Name::root(), + vec![ + alpn(&["h3"]), + (SvcParamKey::Port, SvcParamValue::Port(2443)), + ], + 3600, + ), + ]; + + assert_eq!( + read_https_answer(&queried, &answers).and_then(|ad| ad.port), + Some(2443), + "the preferred record is the one acted on" + ); +} + +#[test] +fn an_empty_answer_advertises_nothing() { + // The common case for an origin with no `HTTPS` record at all: nothing learned, and + // nothing that could make the origin probe-worthy. + assert_eq!(read_https_answer(&name("example.com."), &[]), None); +} diff --git a/crates/web-faith-dns/src/lib.rs b/crates/web-faith-dns/src/lib.rs index 33f10ca..f6db349 100644 --- a/crates/web-faith-dns/src/lib.rs +++ b/crates/web-faith-dns/src/lib.rs @@ -28,31 +28,18 @@ // spec:WARM spec:DNS -use std::{ - collections::HashSet, - net::{IpAddr, SocketAddr}, - sync::{Arc, Mutex}, - time::{Duration, Instant}, -}; +mod discovery; +mod https; +mod resolver; +mod settings; +mod transport; -use hickory_resolver::{ - TokioResolver, - config::{ - ConnectionConfig, GOOGLE, LookupIpStrategy, NameServerConfig, OpportunisticEncryption, - ProtocolConfig, ResolveHosts, ResolverConfig, ServerOrderingStrategy, - }, - net::{DnsError, NetError, runtime::TokioRuntimeProvider}, - proto::rr::{ - Name, RData, RecordType, - rdata::svcb::{SvcParamKey, SvcParamValue}, - }, - system_conf::read_system_conf, -}; -use tokio::sync::OnceCell; -use url::{Host, Url}; +pub use https::{HttpsAdvertisement, HttpsSink}; +pub use resolver::FaithResolver; +pub use settings::{DEFAULT_MAX_STALE, ResolverReport, ResolverSettings, ResolverSource}; +pub use transport::{ServerSpec, Transport}; -/// The default DoH/DoQ query path, used when a `https://`/`h3://` server URL supplies none. -const DEFAULT_DNS_QUERY_PATH: &str = "/dns-query"; +use hickory_resolver::proto::rr::Name; /// Parse a `dns.searchDomains` or `dns.exemptDomains` list into domain names, or return a message /// for the first entry that is not a valid domain name. @@ -65,1503 +52,3 @@ pub fn parse_domains(list: Option>) -> Result>, Str }) .transpose() } - -/// The transport Faith speaks to a resolver, chosen by a server URL's scheme. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Transport { - Udp, - Tcp, - Tls, - Https, - Quic, - H3, -} - -impl Transport { - fn from_scheme(scheme: &str) -> Option { - Some(match scheme { - "udp" => Self::Udp, - "tcp" => Self::Tcp, - "tls" => Self::Tls, - "https" => Self::Https, - "quic" => Self::Quic, - "h3" => Self::H3, - _ => return None, - }) - } - - /// The conventional port for the transport, used when the URL names none. - fn default_port(self) -> u16 { - match self { - Self::Udp | Self::Tcp => 53, - Self::Tls | Self::Quic => 853, - Self::Https | Self::H3 => 443, - } - } - - /// The lowercase label reported by `resolvers()`. - fn label(self) -> &'static str { - match self { - Self::Udp => "udp", - Self::Tcp => "tcp", - Self::Tls => "tls", - Self::Https => "https", - Self::Quic => "quic", - Self::H3 => "h3", - } - } -} - -/// What an `HTTPS` record said about an origin's HTTP/3 support. -// spec:DNS#https-records -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct HttpsAdvertisement { - /// The record's `port` SvcParam, or `None` when it named none and the origin's own port - /// applies. - pub port: Option, - /// The record's own DNS TTL, which is how long the advertisement it carries lives. - pub ttl: Duration, -} - -/// Where an `HTTPS` record's advertisement goes once the resolver has read one. -/// -/// The resolver cannot own the HTTP/3 upgrade cache directly: that cache is built after the -/// resolver, and the background prober holds a client which holds the resolver in turn. So the -/// caller installs this afterwards (see [`FaithResolver::set_https_sink`]), which also keeps this -/// crate free of the upgrade layer's types. -pub trait HttpsSink: Send + Sync { - /// Whether an `HTTPS` record for `host` is worth querying at all right now. - /// - /// Asked before the query so an origin already confirmed, already failed, or already holding a - /// live advertisement costs no DNS traffic to re-learn what is known. - fn wants(&self, host: &str) -> bool; - - /// Fold a record's advertisement into the upgrade layer's knowledge of `host`. - fn record(&self, host: &str, advertisement: HttpsAdvertisement); -} - -/// Whether an ALPN token names a version of HTTP/3. -/// -/// The same family test the `Alt-Svc` reader applies, so a draft token like `h3-29` counts here -/// exactly as it does in a header. -// spec:H3UP#reading-advertisements -fn is_h3_alpn(token: &str) -> bool { - token == "h3" || token.starts_with("h3-") -} - -/// Read the HTTP/3 advertisement out of an `HTTPS` answer for `name`, if it carries one. -/// -/// Only ServiceMode records are considered: an AliasMode record (`svc_priority` 0) redirects to -/// another name rather than describing this one, and following that redirection is a resolution -/// step this does not take. Among the rest the lowest `svc_priority` wins, which is the preference -/// order RFC 9460 defines. -/// -/// A record whose target is neither the root (which per RFC 9460 §2.5.2 means the owner name -/// itself) nor the queried name designates a *different* host, and Faith only upgrades to the -/// origin's own host, so such a record is not acted on. -/// `queried` is the name the answer was actually asked for rather than the host as written, since -/// the search list can requalify a name before it reaches a server; comparison ignores the trailing -/// root so the two are judged on identity rather than on how each was spelled. -// spec:H3UP#advertisements-from-dns -fn read_https_answer( - queried: &Name, - answers: &[hickory_resolver::proto::rr::Record], -) -> Option { - let mut best: Option<(u16, HttpsAdvertisement)> = None; - - for record in answers { - let RData::HTTPS(https) = &record.data else { - continue; - }; - - if https.svc_priority == 0 { - continue; - } - - if !https.target_name.is_root() && !https.target_name.eq_ignore_root(queried) { - continue; - } - - let mut has_h3 = false; - let mut port = None; - for (key, value) in &https.svc_params { - match (key, value) { - (SvcParamKey::Alpn, SvcParamValue::Alpn(alpn)) => { - has_h3 = alpn.0.iter().any(|token| is_h3_alpn(token)); - } - (SvcParamKey::Port, SvcParamValue::Port(value)) => port = Some(*value), - _ => {} - } - } - - if !has_h3 { - continue; - } - - let advertisement = HttpsAdvertisement { - port, - ttl: Duration::from_secs(record.ttl.into()), - }; - if best - .as_ref() - .is_none_or(|(best, _)| https.svc_priority < *best) - { - best = Some((https.svc_priority, advertisement)); - } - } - - best.map(|(_, advertisement)| advertisement) -} - -/// A resolver Faith reaches by IP or by a hostname it bootstraps. -#[derive(Clone, Debug)] -enum ServerHost { - Ip(IpAddr), - Name(String), -} - -/// One entry of `dns.servers`, parsed at agent construction. The IP is not known yet for a -/// hostname host: that is resolved when the resolver is first used (see [`ResolverSettings`]). -#[derive(Clone, Debug)] -pub struct ServerSpec { - host: ServerHost, - transport: Transport, - port: u16, - /// DoH/DoQ query path, `None` for the non-HTTP transports. - path: Option>, - /// The name to authenticate the certificate against, from a URL fragment. When absent, a - /// hostname host authenticates against itself and an IP host against the address. - cert_name: Option, -} - -impl ServerSpec { - /// Parse one `dns.servers` URL, or return a message for an unparseable URL or unknown scheme. - pub fn parse(input: &str) -> Result { - let url = Url::parse(input).map_err(|err| format!("{input:?}: {err}"))?; - let transport = Transport::from_scheme(url.scheme()) - .ok_or_else(|| format!("{input:?}: unknown DNS transport scheme {:?}", url.scheme()))?; - - let host = match url.host() { - Some(Host::Ipv4(ip)) => ServerHost::Ip(IpAddr::V4(ip)), - Some(Host::Ipv6(ip)) => ServerHost::Ip(IpAddr::V6(ip)), - Some(Host::Domain(name)) => ServerHost::Name(name.to_owned()), - None => return Err(format!("{input:?}: no host to resolve")), - }; - - let port = url.port().unwrap_or_else(|| transport.default_port()); - let path = match transport { - Transport::Https | Transport::H3 => { - let path = url.path(); - (!path.is_empty() && path != "/").then(|| Arc::from(path)) - } - _ => None, - }; - let cert_name = url.fragment().map(str::to_owned); - - Ok(Self { - host, - transport, - port, - path, - cert_name, - }) - } - - /// The IP host, or `None` for a hostname host that still needs bootstrapping. - fn ip(&self) -> Option { - match self.host { - ServerHost::Ip(ip) => Some(ip), - ServerHost::Name(_) => None, - } - } - - /// The certificate name to authenticate against once the host resolves to `ip`: an explicit - /// fragment, else the hostname, else the address itself. - // spec:DNS#transports - fn server_name(&self, ip: IpAddr) -> Arc { - if let Some(name) = &self.cert_name { - Arc::from(name.as_str()) - } else { - match &self.host { - ServerHost::Name(name) => Arc::from(name.as_str()), - ServerHost::Ip(_) => Arc::from(ip.to_string()), - } - } - } - - /// Build the hickory name server for this spec, reached at `ip`. - fn to_name_server(&self, ip: IpAddr) -> NameServerConfig { - let protocol = match self.transport { - Transport::Udp => ProtocolConfig::Udp, - Transport::Tcp => ProtocolConfig::Tcp, - Transport::Tls => ProtocolConfig::Tls { - server_name: self.server_name(ip), - }, - Transport::Https => ProtocolConfig::Https { - server_name: self.server_name(ip), - path: self - .path - .clone() - .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), - }, - Transport::Quic => ProtocolConfig::Quic { - server_name: self.server_name(ip), - }, - Transport::H3 => ProtocolConfig::H3 { - server_name: self.server_name(ip), - path: self - .path - .clone() - .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), - disable_grease: false, - }, - }; - - let mut connection = ConnectionConfig::new(protocol); - connection.port = self.port; - NameServerConfig::new(ip, true, vec![connection]) - } -} - -/// How a server in `resolvers()` came to be reached the way it is. -// spec:OBS#resolvers -#[derive(Clone, Copy, Debug)] -pub enum ResolverSource { - /// Named in `dns.servers` by the caller. - Configured, - /// Discovered from the system's resolver configuration. - Conventional, -} - -impl ResolverSource { - fn label(self) -> &'static str { - match self { - Self::Configured => "configured", - Self::Conventional => "conventional", - } - } -} - -/// One line of `resolvers()`: a server's address, the transport in use, and how it was arrived at. -#[derive(Clone, Debug)] -pub struct ResolverReport { - pub address: String, - pub transport: String, - pub source: String, -} - -/// `dns.maxStale`'s default: how far past expiry an answer may still be served. -/// -/// An hour is long enough that a resolver outage does not stop an agent reaching hosts it already -/// knows, and short enough that a host which really has moved stops being served a dead address for -/// the life of a long-running process. The recovery path bounds the cost of being wrong to one -/// re-resolve, so the window can be generous. -// spec:DNS#serving-stale-answers -pub const DEFAULT_MAX_STALE: Duration = Duration::from_secs(3600); - -/// Everything `dns.*` configures about Faith's resolver, resolved from options at construction. -#[derive(Clone, Debug)] -pub struct ResolverSettings { - /// The `dns.servers` list, in order. Empty means system discovery. - pub servers: Vec, - /// `dns.timeout`, bounding the whole list. `None` leaves hickory's five-second default. - pub timeout: Option, - /// `dns.ndots`. - pub ndots: Option, - /// `dns.searchDomains`, replacing the system's search list when set. - pub search_domains: Option>, - /// `dns.hostsFile`: `Some(true)`/`Some(false)` force it on/off, `None` follows the platform. - pub hosts_file: Option, - /// `dns.exemptDomains`, added to the always-exempt `localhost`, `.local`, and system suffix. - pub exempt_domains: Vec, - /// `dns.serveStale`: whether an expired answer is served while a refresh runs behind it. - pub serve_stale: bool, - /// `dns.maxStale`: how far past expiry an answer may still be served. - pub max_stale: Duration, -} - -impl Default for ResolverSettings { - fn default() -> Self { - Self { - servers: Vec::new(), - timeout: None, - ndots: None, - search_domains: None, - hosts_file: None, - exempt_domains: Vec::new(), - // Defaulted here as well as in the option parsing, so a resolver built directly (in tests, - // and for the global default agent) serves stale like a configured one. - serve_stale: true, - max_stale: DEFAULT_MAX_STALE, - } - } -} - -/// How many names the stale cache holds before evicting the least recently used. -/// -/// Matched to hickory's own default answer-cache size, since the two hold an entry for the same set -/// of names: a stale entry only earns its place while hickory still plausibly holds, or recently -/// held, the answer it came from. Evicting one early costs a blocking lookup rather than a wrong -/// answer, so the bound is about memory rather than correctness. -const STALE_CACHE_SIZE: u64 = 8_192; - -/// The resolver and the report of its servers, built together the first time the resolver is used. -struct Built { - resolver: TokioResolver, - reports: Vec, -} - -/// A resolved answer kept past its TTL, so an expired lookup is served from it while a refresh runs -/// behind. -// spec:DNS#serving-stale-answers -#[derive(Clone)] -struct StaleEntry { - /// Shared rather than cloned per hit: a hit reads it and hands out a copy of the addresses. - addrs: Arc>, - /// When the answer stopped being fresh, taken from the lookup rather than computed, so it is the - /// TTL the resolver actually gave. - valid_until: Instant, -} - -/// Everything the resolver reads off the network, held together so a network change can drop it in -/// one go. Each field describes the network the agent was on when it was read: which -/// servers discovery found, which suffixes are local to it, and which of its servers answered an -/// encryption probe. The caller's [`ResolverSettings`] deliberately sit outside, being options the -/// agent was constructed with rather than a reading of any network. -// spec:NETCHG -struct Generation { - /// The configured (or discovered) resolver, built lazily inside a tokio runtime. - built: OnceCell>, - /// The system resolver, used for exempt names. Built lazily and independently. - system: OnceCell>, - /// The exempt suffixes, including the system's own, computed once per generation. - exempt: OnceCell>>, - /// Answers held past their TTL, keyed by the host as looked up. Sits in the generation rather - /// than beside the settings so a network change drops it along with the resolvers that produced - /// it: an address learned on the old network is exactly what must not be served on the new one. - stale: moka::sync::Cache, - /// Hosts with a refresh already in flight, so a second stale hit serves the entry rather than - /// starting another lookup. - // spec:DNS#serving-stale-answers - refreshing: Mutex>, -} - -impl Default for Generation { - fn default() -> Self { - Self { - built: OnceCell::new(), - system: OnceCell::new(), - exempt: OnceCell::new(), - stale: moka::sync::Cache::new(STALE_CACHE_SIZE), - refreshing: Mutex::new(HashSet::new()), - } - } -} - -struct Inner { - /// The options the agent was constructed with. A network change does not touch these; they are - /// what the next generation is rebuilt from. - // spec:NETCHG#what-the-signal-keeps - settings: ResolverSettings, - /// Replaced wholesale by [`FaithResolver::reset`]. Read once at the start of a lookup rather - /// than at each step, so a lookup that spans the signal finishes against the one set of - /// resolvers it started on. - // spec:NETCHG#in-flight-requests - generation: Mutex>, - /// Where `HTTPS` records go, installed by the agent once the upgrade cache and prober exist. - /// - /// Sits beside the settings rather than inside the generation deliberately: it is wiring - /// rather than something read off a network, so a network change leaves it in place. Its - /// absence is what turns the `HTTPS` query off, so an agent with HTTP/3 upgrade disabled, or - /// one on the system resolver, never sends one. - https_sink: Mutex>>, -} - -/// A hickory resolver Faith owns, shared between a client's request path and [`FaithResolver::prefetch`]. -#[derive(Clone)] -pub struct FaithResolver { - inner: Arc, -} - -impl Default for FaithResolver { - fn default() -> Self { - Self::new(ResolverSettings::default()) - } -} - -impl std::fmt::Debug for FaithResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FaithResolver").finish_non_exhaustive() - } -} - -impl FaithResolver { - pub fn new(settings: ResolverSettings) -> Self { - Self { - inner: Arc::new(Inner { - settings, - generation: Mutex::new(Arc::new(Generation::default())), - https_sink: Mutex::new(None), - }), - } - } - - /// Install where `HTTPS` records go, enabling the query. - /// - /// Called after the agent's HTTP/3 upgrade cache and prober are built, which cannot happen - /// before the resolver exists. Replaces any previous sink, which is what a network change - /// needs: the prober is rebuilt with the client, so the sink must be too or it would kick - /// probes onto a client that has been dropped. - // spec:DNS#https-records - pub fn set_https_sink(&self, sink: Arc) { - *self - .inner - .https_sink - .lock() - .expect("the HTTPS sink lock is only held to clone or replace an Arc") = Some(sink); - } - - fn https_sink(&self) -> Option> { - self.inner - .https_sink - .lock() - .expect("the HTTPS sink lock is only held to clone or replace an Arc") - .clone() - } - - /// The generation a piece of work resolves against. Taken once per lookup: a reset swaps the - /// generation rather than mutating it, so work already holding one carries on against the - /// resolvers it started with. - // spec:NETCHG#in-flight-requests - fn generation(&self) -> Arc { - self.inner - .generation - .lock() - .expect("the DNS generation lock is only held to clone or replace an Arc") - .clone() - } - - async fn built(&self, generation: &Generation) -> Result, NetError> { - generation - .built - .get_or_try_init(|| async { build(&self.inner.settings).await.map(Arc::new) }) - .await - .cloned() - } - - /// The system resolver, for exempt names. Reads the system configuration and races both - /// address families for Happy Eyeballs like the built-in resolver does. - async fn system(&self, generation: &Generation) -> Result, NetError> { - generation - .system - .get_or_try_init(|| async { - let mut builder = TokioResolver::builder_tokio().unwrap_or_else(|_| { - TokioResolver::builder_with_config( - ResolverConfig::udp_and_tcp(&GOOGLE), - TokioRuntimeProvider::default(), - ) - }); - builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - builder.build().map(Arc::new) - }) - .await - .cloned() - } - - /// The exempt suffixes: `localhost`, `local`, the system's own domain and search suffixes, and - /// the caller's `dns.exemptDomains`. The system's own suffixes are a - /// property of the network, so they are read per generation rather than once per agent. - // spec:DNS#exempt-names - async fn exempt(&self, generation: &Generation) -> Arc> { - generation - .exempt - .get_or_init(|| async { - let system = read_system_conf() - .map(|(config, _)| { - config - .domain() - .into_iter() - .chain(config.search()) - .cloned() - .collect::>() - }) - .unwrap_or_default(); - Arc::new(exempt_suffixes(system, &self.inner.settings.exempt_domains)) - }) - .await - .clone() - } - - /// Whether `host` must go to the system resolver rather than Faith's servers. - async fn is_exempt(&self, generation: &Generation, host: &str) -> bool { - let Ok(name) = Name::from_utf8(host) else { - return false; - }; - self.exempt(generation) - .await - .iter() - .any(|suffix| suffix.zone_of(&name)) - } - - /// Resolve `host` to its addresses, routing exempt names to the system resolver. - async fn lookup(&self, host: &str) -> Result, NetError> { - let generation = self.generation(); - if self.is_exempt(&generation, host).await { - // The system resolver keeps no cache Faith can hold answers in, so an exempt name has - // nothing to go stale and is always resolved for real (spec:DNS#serving-stale-answers). - let resolver = self.system(&generation).await?; - return Ok(resolver.lookup_ip(host).await?.iter().collect()); - } - - // Alongside the addresses rather than after them: the record is a hint for the upgrade - // layer to verify, so nothing about connecting waits on it (spec:DNS#https-records). - self.spawn_https_query(&generation, host); - - if let Some(addrs) = self.stale_addrs(&generation, host) { - self.spawn_refresh(&generation, host); - return Ok(addrs); - } - - let built = self.built(&generation).await?; - let lookup = built.resolver.lookup_ip(host).await?; - let addrs: Vec = lookup.iter().collect(); - self.remember(&generation, host, &addrs, lookup.valid_until()); - Ok(addrs) - } - - /// Ask for `host`'s `HTTPS` record behind the address lookup, so an origin advertising - /// `alpn="h3"` is known before the first connection rather than after the first TCP response. - /// - /// Spawned rather than awaited: an absent, slow, or failed answer must leave address - /// resolution and connecting untouched. Its outcome belongs to the upgrade layer rather than - /// to the request that triggered it, so nothing here reaches a caller, exactly as a stale - /// refresh's outcome does not. - // spec:DNS#https-records - fn spawn_https_query(&self, generation: &Arc, host: &str) { - let Some(sink) = self.https_sink() else { - return; - }; - // Asked before the query, not after: an origin already confirmed, failed, or holding a - // live advertisement has nothing to learn, so it costs no DNS traffic. - if !sink.wants(host) { - return; - } - let Ok(name) = Name::from_utf8(host) else { - return; - }; - - let this = self.clone(); - let generation = Arc::clone(generation); - let host = host.to_owned(); - tokio::spawn(async move { - let Ok(built) = this.built(&generation).await else { - return; - }; - let Ok(lookup) = built.resolver.lookup(name, RecordType::HTTPS).await else { - return; - }; - // The answer's own query name, not the host as written: the search list may have - // requalified it, and the record's target is judged against what was actually asked. - if let Some(advertisement) = read_https_answer(lookup.query().name(), lookup.answers()) - { - sink.record(&host, advertisement); - } - }); - } - - /// The addresses to serve for `host` without waiting, when its answer has expired but is still - /// inside `dns.maxStale`. - /// - /// `None` covers the three cases that must go to the resolver: no entry at all, an entry still - /// fresh (which hickory's own cache answers without a network round trip anyway), and an entry so - /// old it has stopped being evidence about the host. - fn stale_addrs(&self, generation: &Generation, host: &str) -> Option> { - if !self.inner.settings.serve_stale { - return None; - } - let entry = generation.stale.get(host)?; - let now = Instant::now(); - if now <= entry.valid_until { - return None; - } - if now.saturating_duration_since(entry.valid_until) > self.inner.settings.max_stale { - // Dropped rather than left to sit: keeping it would let a refresh that has been failing - // for hours go on being consulted, and the entry can only get older from here. - generation.stale.invalidate(host); - return None; - } - Some(entry.addrs.as_ref().clone()) - } - - /// Keep a successful answer for `host`, so a later lookup past its TTL has something to serve. - fn remember( - &self, - generation: &Generation, - host: &str, - addrs: &[IpAddr], - valid_until: Instant, - ) { - if !self.inner.settings.serve_stale || addrs.is_empty() { - return; - } - generation.stale.insert( - host.to_owned(), - StaleEntry { - addrs: Arc::new(addrs.to_vec()), - valid_until, - }, - ); - } - - /// Refresh `host` behind a stale answer that has already been served. - /// - /// Single-flighted per host: the claim is taken before the task is spawned, so concurrent stale - /// hits serve the entry rather than each starting a lookup. The task outlives the request that - /// triggered it, and its outcome belongs to the cache rather than that request, so nothing here - /// is reported to a caller. - // spec:DNS#serving-stale-answers - fn spawn_refresh(&self, generation: &Arc, host: &str) { - { - let mut refreshing = generation - .refreshing - .lock() - .expect("the DNS refresh lock is only held to insert or remove a host"); - if !refreshing.insert(host.to_owned()) { - return; - } - } - - let this = self.clone(); - let generation = Arc::clone(generation); - let host = host.to_owned(); - tokio::spawn(async move { - match this.refresh(&generation, &host).await { - Ok(()) => {} - Err(err) if is_authoritatively_empty(&err) => { - // The name resolves to nothing now, so the old address is not a stale answer for - // it any more but a wrong one. Dropping the entry makes the next lookup fail - // rather than hand out an address the host no longer answers on. - generation.stale.invalidate(&host); - } - Err(_) => { - // A network error, a server failure, or a timeout says nothing about where the - // host is, so the entry stays and can be served again while this persists. - } - } - generation - .refreshing - .lock() - .expect("the DNS refresh lock is only held to insert or remove a host") - .remove(&host); - }); - } - - /// One refresh lookup, replacing the stale entry when it resolves. - async fn refresh(&self, generation: &Generation, host: &str) -> Result<(), NetError> { - let built = self.built(generation).await?; - let lookup = built.resolver.lookup_ip(host).await?; - let addrs: Vec = lookup.iter().collect(); - self.remember(generation, host, &addrs, lookup.valid_until()); - Ok(()) - } - - /// Drop any stale answer held for `host`, so the next lookup waits for a fresh one. - /// - /// Called when connecting to a served address failed, which is the one piece of evidence that the - /// address was wrong rather than merely old. - // spec:DNS#when-a-stale-address-is-wrong - pub fn invalidate_stale(&self, host: &str) { - self.generation().stale.invalidate(host); - } - - /// Whether a lookup of `host` right now would be served from an expired entry, and so would hand - /// out an address that is assumed rather than confirmed. - /// - /// Deliberately the same window a stale answer is served from, rather than merely "an expired - /// entry exists": an entry past `dns.maxStale` is resolved for real, and treating that as stale - /// would spend a second connection attempt on an address that was already confirmed. - pub fn served_stale(&self, host: &str) -> bool { - if !self.inner.settings.serve_stale { - return false; - } - self.generation().stale.get(host).is_some_and(|entry| { - let now = Instant::now(); - now > entry.valid_until - && now.saturating_duration_since(entry.valid_until) <= self.inner.settings.max_stale - }) - } - - /// Resolve `host` and leave the answer in the shared cache, so a later request skips the - /// lookup. Any failure is swallowed: the warm-up is advisory. - // spec:WARM - pub async fn prefetch(&self, host: &str) { - let _ = self.lookup(host).await; - } - - /// The DNS servers the agent resolves through, in query order. Empty - /// until the resolver has been used, because it reads its configuration on first use. - // spec:OBS#resolvers - pub fn resolvers(&self) -> Vec { - self.generation() - .built - .get() - .map(|built| built.reports.clone()) - .unwrap_or_default() - } - - /// Drop everything read off the network, so the next lookup rebuilds against the network the - /// agent is on now. - /// - /// Flushing cached answers alone would leave the agent resolving them again through the - /// previous network's servers: the discovered server list, the suffixes treated as local, and - /// the results of encryption probes are all readings of a network too, and the whole point of - /// the signal is that the network has changed. Dropping the generation takes the caches with - /// it, since they belong to the resolvers being dropped. - /// - /// The caller's options are untouched, so a listed `dns.servers` set is rebuilt exactly as - /// configured; what it re-reads is what the system supplies and what the network answers. - /// - /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, - /// which keeps it callable from a network-change signal, which is not async. Nothing is rebuilt here - /// either, so an agent that never resolves again pays nothing for the signal. - // spec:NETCHG#what-the-signal-keeps - // spec:NETCHG#reach-across-the-subsystems - pub fn reset(&self) { - *self - .inner - .generation - .lock() - .expect("the DNS generation lock is only held to clone or replace an Arc") = - Arc::new(Generation::default()); - } -} - -/// Installed on a reqwest client with `ClientBuilder::dns_resolver`, so every lookup a request -/// makes goes through the same resolver `prefetch` warms. -#[cfg(feature = "reqwest")] -impl reqwest::dns::Resolve for FaithResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - let this = self.clone(); - Box::pin(async move { - let addrs = this.lookup(name.as_str()).await?; - // Port `0` is a placeholder reqwest fills from the URL. The returned `Addrs` has to be - // `'static`, so collect owned rather than borrowing the lookup. - let addrs: Vec = - addrs.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); - Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) - }) - } -} - -/// Whether a failed lookup was the resolver answering that the name holds nothing, rather than the -/// resolver failing to answer. -/// -/// The distinction decides what happens to a stale entry: an authoritative "nothing here" retires -/// it, while a failure to reach an answer leaves it in place. Hickory draws the same line, producing -/// `NoRecordsFound` only for `NXDOMAIN` and for `NOERROR` with no answer records, and reporting -/// `SERVFAIL` and the other failure codes as `ResponseCode` instead. -fn is_authoritatively_empty(err: &NetError) -> bool { - matches!(err, NetError::Dns(DnsError::NoRecordsFound(_))) -} - -/// Apply the options common to every resolver Faith builds: race both families for Happy Eyeballs, -/// hold the caller's order fixed rather than reordering by latency, and layer any `dns.*` timeout, -/// ndots, and hosts-file settings on top. -fn apply_options( - builder: &mut hickory_resolver::ResolverBuilder, - settings: &ResolverSettings, -) { - let options = builder.options_mut(); - options.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - // The list expresses the caller's intent, not a performance hint, so a private resolver named - // first must not lose traffic to a closer fallback (spec:DNS#server-order). - options.server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; - if let Some(timeout) = settings.timeout { - options.timeout = timeout; - } - if let Some(ndots) = settings.ndots { - options.ndots = ndots; - } - if let Some(hosts_file) = settings.hosts_file { - options.use_hosts_file = if hosts_file { - ResolveHosts::Always - } else { - ResolveHosts::Never - }; - } -} - -/// Build the configured (or discovered) resolver and the report of its servers. -async fn build(settings: &ResolverSettings) -> Result { - if settings.servers.is_empty() { - build_discovery(settings) - } else { - build_listed(settings).await - } -} - -/// Discovery: configure from the system, then let hickory's RFC 9539 opportunistic encryption -/// upgrade those servers to DoT/DoQ where they answer a probe. `dns.searchDomains` overrides the -/// system search list when set. -// spec:DNS#discovery -fn build_discovery(settings: &ResolverSettings) -> Result { - let (mut config, options) = read_system_conf().unwrap_or_else(|_| { - // A host with no readable resolver configuration falls back to Google Public DNS over - // conventional DNS, probed like any other server (spec:DNS#discovery). - ( - ResolverConfig::udp_and_tcp(&GOOGLE), - hickory_resolver::config::ResolverOpts::default(), - ) - }); - - if let Some(search) = &settings.search_domains { - config = ResolverConfig::from_parts(None, search.clone(), config.name_servers().to_vec()); - } - - let reports = report(config.name_servers(), ResolverSource::Conventional); - - let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()) - .with_options(options); - apply_options(&mut builder, settings); - let builder = builder.with_opportunistic_encryption(OpportunisticEncryption::Enabled { - config: Default::default(), - }); - - Ok(Built { - resolver: builder.build()?, - reports, - }) -} - -/// The listed-servers path: bootstrap any hostname hosts to addresses, then build the resolver -/// from the parsed specs in order. -// spec:DNS#transports -// spec:DNS#bootstrapping -async fn build_listed(settings: &ResolverSettings) -> Result { - let name_servers = build_name_servers(settings).await?; - - let search = settings.search_domains.clone().unwrap_or_default(); - let config = ResolverConfig::from_parts(None, search, name_servers.clone()); - let reports = report(&name_servers, ResolverSource::Configured); - - let mut builder = TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()); - apply_options(&mut builder, settings); - - Ok(Built { - resolver: builder.build()?, - reports, - }) -} - -/// Resolve the listed servers to hickory name servers, bootstrapping hostname hosts. A hostname -/// that will not resolve drops that server for the life of the agent rather than failing the -/// resolver. -// spec:DNS#bootstrapping -async fn build_name_servers( - settings: &ResolverSettings, -) -> Result, NetError> { - let needs_bootstrap = settings.servers.iter().any(|spec| spec.ip().is_none()); - let bootstrap = if needs_bootstrap { - Some(bootstrap_resolver(settings)?) - } else { - None - }; - - let mut name_servers = Vec::with_capacity(settings.servers.len()); - for spec in &settings.servers { - let ip = match spec.ip() { - Some(ip) => ip, - None => { - let ServerHost::Name(host) = &spec.host else { - unreachable!("ip() is None only for a hostname host"); - }; - let resolver = bootstrap - .as_ref() - .expect("bootstrap resolver built when a hostname host is present"); - match resolver.lookup_ip(host.as_str()).await { - Ok(lookup) => match lookup.iter().next() { - Some(ip) => ip, - None => continue, - }, - // The hostname does not resolve: drop this server for the agent's life. - Err(_) => continue, - } - } - }; - name_servers.push(spec.to_name_server(ip)); - } - - Ok(name_servers) -} - -/// The resolver that bootstraps hostname servers: the listed IP-host servers in order, so an -/// encrypted server placed first resolves its siblings without exposing the hostname in plaintext. -/// Where the list names no IP host, the system's own configuration bootstraps instead. -fn bootstrap_resolver(settings: &ResolverSettings) -> Result { - let ip_servers: Vec = settings - .servers - .iter() - .filter_map(|spec| spec.ip().map(|ip| spec.to_name_server(ip))) - .collect(); - - let mut builder = if ip_servers.is_empty() { - TokioResolver::builder_tokio().unwrap_or_else(|_| { - TokioResolver::builder_with_config( - ResolverConfig::udp_and_tcp(&GOOGLE), - TokioRuntimeProvider::default(), - ) - }) - } else { - TokioResolver::builder_with_config( - ResolverConfig::from_parts(None, vec![], ip_servers), - TokioRuntimeProvider::default(), - ) - }; - builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; - builder.options_mut().server_ordering_strategy = ServerOrderingStrategy::UserProvidedOrder; - builder.build() -} - -/// The suffixes handed to the system resolver rather than Faith's servers: `localhost` and `local` -/// always, plus the ones the system supplies and the caller's `dns.exemptDomains`. -/// -/// The root name is never a suffix here, whichever list it arrives in. It is the parent of every -/// name, so admitting it would exempt the lot and route every lookup to the system resolver with -/// `dns.servers` configured and unused. It does arrive in practice: a Windows host with no DNS -/// domain of its own reports the root as its domain, so the check is what keeps the encrypted -/// transports working there rather than being quietly bypassed. -// spec:DNS#exempt-names -fn exempt_suffixes(system: Vec, configured: &[Name]) -> Vec { - let mut names = vec![ - Name::from_ascii("localhost").unwrap(), - Name::from_ascii("local").unwrap(), - ]; - names.extend( - system - .into_iter() - .chain(configured.iter().cloned()) - .filter(|name| !name.is_root()), - ); - names -} - -/// Summarise name servers for `resolvers()`, in the order they are queried. -fn report(name_servers: &[NameServerConfig], source: ResolverSource) -> Vec { - let mut reports = Vec::new(); - for server in name_servers { - for connection in &server.connections { - let transport = match connection.protocol { - ProtocolConfig::Udp => Transport::Udp, - ProtocolConfig::Tcp => Transport::Tcp, - ProtocolConfig::Tls { .. } => Transport::Tls, - ProtocolConfig::Https { .. } => Transport::Https, - ProtocolConfig::Quic { .. } => Transport::Quic, - ProtocolConfig::H3 { .. } => Transport::H3, - }; - reports.push(ResolverReport { - address: SocketAddr::new(server.ip, connection.port).to_string(), - transport: transport.label().to_owned(), - source: source.label().to_owned(), - }); - } - } - reports -} - -#[cfg(test)] -mod tests { - use super::*; - - fn spec(input: &str) -> ServerSpec { - ServerSpec::parse(input).expect("valid server URL") - } - - #[test] - fn scheme_selects_transport_and_conventional_port() { - // spec:DNS#transports - assert_eq!(spec("udp://1.1.1.1").transport, Transport::Udp); - assert_eq!(spec("udp://1.1.1.1").port, 53); - assert_eq!(spec("tcp://1.1.1.1").port, 53); - assert_eq!(spec("tls://1.1.1.1").transport, Transport::Tls); - assert_eq!(spec("tls://1.1.1.1").port, 853); - assert_eq!(spec("quic://1.1.1.1").port, 853); - assert_eq!(spec("https://1.1.1.1").transport, Transport::Https); - assert_eq!(spec("https://1.1.1.1").port, 443); - assert_eq!(spec("h3://1.1.1.1").port, 443); - } - - #[test] - fn explicit_port_overrides_the_conventional_one() { - // spec:DNS#transports - assert_eq!(spec("tls://1.1.1.1:8853").port, 8853); - } - - #[test] - fn http_transports_default_the_query_path() { - // spec:DNS#transports — `/dns-query` when the URL supplies none. - assert_eq!(spec("https://dns.google").path, None); - assert_eq!( - spec("https://dns.google") - .to_name_server(IpAddr::from([8, 8, 8, 8])) - .connections[0] - .protocol, - ProtocolConfig::Https { - server_name: Arc::from("dns.google"), - path: Arc::from(DEFAULT_DNS_QUERY_PATH), - } - ); - assert_eq!( - spec("https://dns.google/resolve").path, - Some(Arc::from("/resolve")) - ); - } - - #[test] - fn a_fragment_names_the_certificate() { - // spec:DNS#transports — `tls://1.1.1.1#cloudflare-dns.com`. - let spec = spec("tls://1.1.1.1#cloudflare-dns.com"); - assert_eq!(spec.cert_name.as_deref(), Some("cloudflare-dns.com")); - assert_eq!( - &*spec.server_name(IpAddr::from([1, 1, 1, 1])), - "cloudflare-dns.com" - ); - } - - #[test] - fn a_bare_ip_authenticates_against_the_address() { - // spec:DNS#transports — `tls://1.1.1.1` with no fragment. - let spec = spec("tls://1.1.1.1"); - assert_eq!(spec.cert_name, None); - assert_eq!(&*spec.server_name(IpAddr::from([1, 1, 1, 1])), "1.1.1.1"); - } - - #[test] - fn a_hostname_authenticates_against_itself() { - // spec:DNS#transports - let spec = spec("tls://dns.google"); - assert_eq!(&*spec.server_name(IpAddr::from([8, 8, 8, 8])), "dns.google"); - } - - #[test] - fn an_unknown_scheme_is_rejected() { - // spec:DNS#transports — throws an address-parse error at construction. - assert!(ServerSpec::parse("ftp://1.1.1.1").is_err()); - assert!(ServerSpec::parse("not a url").is_err()); - } - - #[tokio::test] - async fn reset_replaces_the_generation_and_what_it_holds() { - // A network change drops what was read off the old network, so the next lookup builds - // against the new one rather than reusing the previous network's servers (spec:NETCHG). - let resolver = FaithResolver::new(ResolverSettings { - servers: vec![spec("udp://127.0.0.1:1")], - timeout: Some(Duration::from_millis(200)), - ..ResolverSettings::default() - }); - - let before = resolver.generation(); - // Build the generation's state, so there is something for the reset to drop. - let _ = resolver.built(&before).await; - assert!( - before.built.get().is_some(), - "the generation built its resolver" - ); - assert_eq!(resolver.resolvers().len(), 1, "which `resolvers()` reports"); - - resolver.reset(); - - let after = resolver.generation(); - assert!( - !Arc::ptr_eq(&before, &after), - "the reset swaps the generation rather than mutating it" - ); - assert!( - after.built.get().is_none(), - "the new generation holds nothing until it is used again" - ); - assert!( - before.built.get().is_some(), - "work already holding the old generation keeps its resolvers" - ); - assert!( - resolver.resolvers().is_empty(), - "`resolvers()` reports nothing until the rebuild (spec:OBS#resolvers)" - ); - - // Configuration survives the signal, so the rebuild uses the servers as configured. - let _ = resolver.built(&after).await; - assert_eq!( - resolver.resolvers().len(), - 1, - "the rebuilt generation resolves through the configured servers again" - ); - } - - /// A resolver with a stale entry for `host` whose freshness ended `ago`. - fn with_stale_entry(settings: ResolverSettings, host: &str, ago: Duration) -> FaithResolver { - let resolver = FaithResolver::new(settings); - resolver.generation().stale.insert( - host.to_owned(), - StaleEntry { - addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), - valid_until: Instant::now() - ago, - }, - ); - resolver - } - - #[test] - fn only_an_expired_entry_inside_the_window_is_served_stale() { - // spec:DNS#serving-stale-answers - let settings = || ResolverSettings { - max_stale: Duration::from_secs(60), - ..ResolverSettings::default() - }; - - // Still fresh: the lookup goes through hickory, which answers from its own cache. - let fresh = FaithResolver::new(settings()); - fresh.generation().stale.insert( - "fresh.test".to_owned(), - StaleEntry { - addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), - valid_until: Instant::now() + Duration::from_secs(60), - }, - ); - let generation = fresh.generation(); - assert!( - fresh.stale_addrs(&generation, "fresh.test").is_none(), - "a fresh entry is not a stale hit" - ); - - // Expired but inside `dns.maxStale`: served immediately. - let stale = with_stale_entry(settings(), "stale.test", Duration::from_secs(5)); - let generation = stale.generation(); - assert!( - stale.stale_addrs(&generation, "stale.test").is_some(), - "an entry expired inside the window is served" - ); - - // Past the window: no longer evidence about the host, so the lookup blocks. - let old = with_stale_entry(settings(), "old.test", Duration::from_secs(120)); - let generation = old.generation(); - assert!( - old.stale_addrs(&generation, "old.test").is_none(), - "an entry past `dns.maxStale` is not served" - ); - assert!( - generation.stale.get("old.test").is_none(), - "and is dropped rather than left to age further" - ); - } - - #[test] - fn serve_stale_off_never_serves_an_expired_entry() { - // spec:DNS#serving-stale-answers — the switch for a caller that must not connect to an - // address it knows to be out of date. - let resolver = with_stale_entry( - ResolverSettings { - serve_stale: false, - ..ResolverSettings::default() - }, - "strict.test", - Duration::from_secs(5), - ); - let generation = resolver.generation(); - assert!(resolver.stale_addrs(&generation, "strict.test").is_none()); - assert!( - !resolver.served_stale("strict.test"), - "and nothing is reported as stale-served, so no retry is armed" - ); - } - - #[test] - fn served_stale_tracks_the_window_it_serves_from() { - // The retry layer arms itself from this, so it must not claim an address was assumed when - // the lookup actually blocked on a fresh one (spec:DNS#when-a-stale-address-is-wrong). - let settings = || ResolverSettings { - max_stale: Duration::from_secs(60), - ..ResolverSettings::default() - }; - - let inside = with_stale_entry(settings(), "inside.test", Duration::from_secs(5)); - assert!(inside.served_stale("inside.test")); - - let outside = with_stale_entry(settings(), "outside.test", Duration::from_secs(120)); - assert!( - !outside.served_stale("outside.test"), - "an entry past the window is resolved for real, so its address is confirmed" - ); - - let absent = FaithResolver::new(settings()); - assert!(!absent.served_stale("absent.test")); - } - - #[test] - fn a_network_change_drops_stale_answers() { - // Addresses read off the old network are exactly what must not be served on the new one - // (spec:NETCHG#reach-across-the-subsystems). - let resolver = with_stale_entry( - ResolverSettings::default(), - "netchg.test", - Duration::from_secs(5), - ); - assert!(resolver.served_stale("netchg.test")); - - resolver.reset(); - - assert!( - !resolver.served_stale("netchg.test"), - "the stale answer goes with the generation that held it" - ); - } - - #[test] - fn a_root_suffix_never_exempts_everything() { - // A Windows host with no DNS domain reports the root as its domain, and the root is the - // parent of every name. Taking it as a suffix exempted every lookup and sent it to the - // system resolver, leaving `dns.servers` configured and unused (spec:DNS#exempt-names). - let suffixes = exempt_suffixes(vec![Name::root()], &[]); - assert!( - !suffixes.iter().any(|suffix| suffix.is_root()), - "the root is not admitted as a suffix" - ); - - let name = Name::from_utf8("nonexistent.example").unwrap(); - assert!( - !suffixes.iter().any(|suffix| suffix.zone_of(&name)), - "so an ordinary name is not exempt and reaches the configured servers" - ); - - // The names that must stay exempt still are, and a real system suffix still counts. - let suffixes = exempt_suffixes( - vec![Name::root(), Name::from_utf8("corp.example").unwrap()], - &[], - ); - for exempt in ["localhost", "printer.local", "host.corp.example"] { - let name = Name::from_utf8(exempt).unwrap(); - assert!( - suffixes.iter().any(|suffix| suffix.zone_of(&name)), - "{exempt} is exempt" - ); - } - } - - #[test] - fn a_root_entry_from_the_caller_is_refused_too() { - // Whichever list it arrives in, the root would disable the caller's own servers. - let suffixes = exempt_suffixes(vec![], &[Name::root()]); - let name = Name::from_utf8("nonexistent.example").unwrap(); - assert!(!suffixes.iter().any(|suffix| suffix.zone_of(&name))); - } - - mod https_records { - use hickory_resolver::proto::rr::{ - Record, - rdata::{HTTPS, SVCB, svcb::Alpn}, - }; - - use super::*; - - fn name(input: &str) -> Name { - Name::from_utf8(input).unwrap() - } - - /// One `HTTPS` answer, spelled the way a server would send it. - fn record( - owner: &str, - priority: u16, - target: Name, - params: Vec<(SvcParamKey, SvcParamValue)>, - ttl: u32, - ) -> Record { - Record::from_rdata( - name(owner), - ttl, - RData::HTTPS(HTTPS(SVCB::new(priority, target, params))), - ) - } - - fn alpn(tokens: &[&str]) -> (SvcParamKey, SvcParamValue) { - ( - SvcParamKey::Alpn, - SvcParamValue::Alpn(Alpn(tokens.iter().map(|t| (*t).to_owned()).collect())), - ) - } - - #[test] - fn an_h3_alpn_on_the_owner_name_advertises() { - // spec:H3UP#advertisements-from-dns — the ordinary case: `.` as the target means the - // owner name, so the record describes the origin itself. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h2", "h3"])], - 3600, - )]; - - assert_eq!( - read_https_answer(&queried, &answers), - Some(HttpsAdvertisement { - port: None, - ttl: Duration::from_secs(3600), - }), - "an `alpn` listing h3 is an advertisement, and the record's own TTL bounds it" - ); - } - - #[test] - fn a_draft_h3_token_counts_like_the_header_reader_treats_one() { - // spec:H3UP#reading-advertisements — `h3-29` is an h3-family token either way it - // arrives, so DNS must not be stricter than the `Alt-Svc` reader. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h3-29"])], - 60, - )]; - - assert!(read_https_answer(&queried, &answers).is_some()); - } - - #[test] - fn a_record_without_h3_in_its_alpn_advertises_nothing() { - // An origin that speaks only HTTP/2 says so here, and reading that as an h3 - // advertisement would send every such origin down a probe that must fail. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![alpn(&["h2"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn the_port_parameter_is_carried_through() { - // spec:H3UP#advertisements-from-dns — a differing port is handled by the same - // machinery an `Alt-Svc` advertised port is. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(8443)), - ], - 3600, - )]; - - assert_eq!( - read_https_answer(&queried, &answers).and_then(|ad| ad.port), - Some(8443) - ); - } - - #[test] - fn a_record_targeting_another_host_is_not_acted_on() { - // Faith only upgrades to the origin's own host, exactly as it refuses an `Alt-Svc` - // advertisement naming a different one (spec:H3UP#advertisements-from-dns). - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - name("cdn.example.net."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn a_record_naming_the_queried_host_itself_is_acted_on() { - // Spelling the owner name out is equivalent to the `.` shorthand, and the comparison - // ignores case and the trailing root the way name equality should. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 1, - name("ExAmPlE.CoM."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert!(read_https_answer(&queried, &answers).is_some()); - } - - #[test] - fn an_alias_mode_record_is_skipped() { - // Priority 0 is AliasMode: it redirects to another name rather than describing this - // one, and following that redirection is a resolution step this does not take. - let queried = name("example.com."); - let answers = [record( - "example.com.", - 0, - name("svc.example.net."), - vec![alpn(&["h3"])], - 3600, - )]; - - assert_eq!(read_https_answer(&queried, &answers), None); - } - - #[test] - fn the_lowest_priority_service_mode_record_wins() { - // RFC 9460 orders ServiceMode records by ascending priority, so the most preferred - // record is the one whose port is acted on. - let queried = name("example.com."); - let answers = [ - record( - "example.com.", - 9, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(9443)), - ], - 3600, - ), - record( - "example.com.", - 2, - Name::root(), - vec![ - alpn(&["h3"]), - (SvcParamKey::Port, SvcParamValue::Port(2443)), - ], - 3600, - ), - ]; - - assert_eq!( - read_https_answer(&queried, &answers).and_then(|ad| ad.port), - Some(2443), - "the preferred record is the one acted on" - ); - } - - #[test] - fn an_empty_answer_advertises_nothing() { - // The common case for an origin with no `HTTPS` record at all: nothing learned, and - // nothing that could make the origin probe-worthy. - assert_eq!(read_https_answer(&name("example.com."), &[]), None); - } - } - - #[test] - fn exempt_matches_a_suffix_exactly_or_as_a_subdomain() { - // spec:DNS#exempt-names - let local = Name::from_ascii("local").unwrap(); - assert!(local.zone_of(&Name::from_utf8("printer.local").unwrap())); - assert!(local.zone_of(&Name::from_utf8("local").unwrap())); - assert!(!local.zone_of(&Name::from_utf8("mylocal.example").unwrap())); - } -} diff --git a/crates/web-faith-dns/src/resolver.rs b/crates/web-faith-dns/src/resolver.rs new file mode 100644 index 0000000..4f193b6 --- /dev/null +++ b/crates/web-faith-dns/src/resolver.rs @@ -0,0 +1,488 @@ +use std::{ + collections::HashSet, + net::IpAddr, + sync::{Arc, Mutex}, + time::Instant, +}; + +use hickory_resolver::{ + TokioResolver, + config::{GOOGLE, LookupIpStrategy, ResolverConfig}, + net::{DnsError, NetError, runtime::TokioRuntimeProvider}, + proto::rr::{Name, RecordType}, + system_conf::read_system_conf, +}; +use tokio::sync::OnceCell; + +#[cfg(feature = "reqwest")] +use std::net::SocketAddr; + +use crate::{ + discovery::{Built, build}, + https::{HttpsSink, read_https_answer}, + settings::{ResolverReport, ResolverSettings, exempt_suffixes}, +}; + +/// How many names the stale cache holds before evicting the least recently used. +/// +/// Matched to hickory's own default answer-cache size, since the two hold an entry for the same set +/// of names: a stale entry only earns its place while hickory still plausibly holds, or recently +/// held, the answer it came from. Evicting one early costs a blocking lookup rather than a wrong +/// answer, so the bound is about memory rather than correctness. +const STALE_CACHE_SIZE: u64 = 8_192; + +/// A resolved answer kept past its TTL, so an expired lookup is served from it while a refresh runs +/// behind. +// spec:DNS#serving-stale-answers +#[derive(Clone)] +struct StaleEntry { + /// Shared rather than cloned per hit: a hit reads it and hands out a copy of the addresses. + addrs: Arc>, + /// When the answer stopped being fresh, taken from the lookup rather than computed, so it is the + /// TTL the resolver actually gave. + valid_until: Instant, +} + +/// Everything the resolver reads off the network, held together so a network change can drop it in +/// one go. Each field describes the network the agent was on when it was read: which +/// servers discovery found, which suffixes are local to it, and which of its servers answered an +/// encryption probe. The caller's [`ResolverSettings`] deliberately sit outside, being options the +/// agent was constructed with rather than a reading of any network. +// spec:NETCHG +struct Generation { + /// The configured (or discovered) resolver, built lazily inside a tokio runtime. + built: OnceCell>, + /// The system resolver, used for exempt names. Built lazily and independently. + system: OnceCell>, + /// The exempt suffixes, including the system's own, computed once per generation. + exempt: OnceCell>>, + /// Answers held past their TTL, keyed by the host as looked up. Sits in the generation rather + /// than beside the settings so a network change drops it along with the resolvers that produced + /// it: an address learned on the old network is exactly what must not be served on the new one. + stale: moka::sync::Cache, + /// Hosts with a refresh already in flight, so a second stale hit serves the entry rather than + /// starting another lookup. + // spec:DNS#serving-stale-answers + refreshing: Mutex>, +} + +impl Default for Generation { + fn default() -> Self { + Self { + built: OnceCell::new(), + system: OnceCell::new(), + exempt: OnceCell::new(), + stale: moka::sync::Cache::new(STALE_CACHE_SIZE), + refreshing: Mutex::new(HashSet::new()), + } + } +} + +struct Inner { + /// The options the agent was constructed with. A network change does not touch these; they are + /// what the next generation is rebuilt from. + // spec:NETCHG#what-the-signal-keeps + settings: ResolverSettings, + /// Replaced wholesale by [`FaithResolver::reset`]. Read once at the start of a lookup rather + /// than at each step, so a lookup that spans the signal finishes against the one set of + /// resolvers it started on. + // spec:NETCHG#in-flight-requests + generation: Mutex>, + /// Where `HTTPS` records go, installed by the agent once the upgrade cache and prober exist. + /// + /// Sits beside the settings rather than inside the generation deliberately: it is wiring + /// rather than something read off a network, so a network change leaves it in place. Its + /// absence is what turns the `HTTPS` query off, so an agent with HTTP/3 upgrade disabled, or + /// one on the system resolver, never sends one. + https_sink: Mutex>>, +} + +/// A hickory resolver Faith owns, shared between a client's request path and [`FaithResolver::prefetch`]. +#[derive(Clone)] +pub struct FaithResolver { + inner: Arc, +} + +impl Default for FaithResolver { + fn default() -> Self { + Self::new(ResolverSettings::default()) + } +} + +impl std::fmt::Debug for FaithResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FaithResolver").finish_non_exhaustive() + } +} + +impl FaithResolver { + pub fn new(settings: ResolverSettings) -> Self { + Self { + inner: Arc::new(Inner { + settings, + generation: Mutex::new(Arc::new(Generation::default())), + https_sink: Mutex::new(None), + }), + } + } + + /// Install where `HTTPS` records go, enabling the query. + /// + /// Called after the agent's HTTP/3 upgrade cache and prober are built, which cannot happen + /// before the resolver exists. Replaces any previous sink, which is what a network change + /// needs: the prober is rebuilt with the client, so the sink must be too or it would kick + /// probes onto a client that has been dropped. + // spec:DNS#https-records + pub fn set_https_sink(&self, sink: Arc) { + *self + .inner + .https_sink + .lock() + .expect("the HTTPS sink lock is only held to clone or replace an Arc") = Some(sink); + } + + fn https_sink(&self) -> Option> { + self.inner + .https_sink + .lock() + .expect("the HTTPS sink lock is only held to clone or replace an Arc") + .clone() + } + + /// The generation a piece of work resolves against. Taken once per lookup: a reset swaps the + /// generation rather than mutating it, so work already holding one carries on against the + /// resolvers it started with. + // spec:NETCHG#in-flight-requests + fn generation(&self) -> Arc { + self.inner + .generation + .lock() + .expect("the DNS generation lock is only held to clone or replace an Arc") + .clone() + } + + async fn built(&self, generation: &Generation) -> Result, NetError> { + generation + .built + .get_or_try_init(|| async { build(&self.inner.settings).await.map(Arc::new) }) + .await + .cloned() + } + + /// The system resolver, for exempt names. Reads the system configuration and races both + /// address families for Happy Eyeballs like the built-in resolver does. + async fn system(&self, generation: &Generation) -> Result, NetError> { + generation + .system + .get_or_try_init(|| async { + let mut builder = TokioResolver::builder_tokio().unwrap_or_else(|_| { + TokioResolver::builder_with_config( + ResolverConfig::udp_and_tcp(&GOOGLE), + TokioRuntimeProvider::default(), + ) + }); + builder.options_mut().ip_strategy = LookupIpStrategy::Ipv4AndIpv6; + builder.build().map(Arc::new) + }) + .await + .cloned() + } + + /// The exempt suffixes: `localhost`, `local`, the system's own domain and search suffixes, and + /// the caller's `dns.exemptDomains`. The system's own suffixes are a + /// property of the network, so they are read per generation rather than once per agent. + // spec:DNS#exempt-names + async fn exempt(&self, generation: &Generation) -> Arc> { + generation + .exempt + .get_or_init(|| async { + let system = read_system_conf() + .map(|(config, _)| { + config + .domain() + .into_iter() + .chain(config.search()) + .cloned() + .collect::>() + }) + .unwrap_or_default(); + Arc::new(exempt_suffixes(system, &self.inner.settings.exempt_domains)) + }) + .await + .clone() + } + + /// Whether `host` must go to the system resolver rather than Faith's servers. + async fn is_exempt(&self, generation: &Generation, host: &str) -> bool { + let Ok(name) = Name::from_utf8(host) else { + return false; + }; + self.exempt(generation) + .await + .iter() + .any(|suffix| suffix.zone_of(&name)) + } + + /// Resolve `host` to its addresses, routing exempt names to the system resolver. + async fn lookup(&self, host: &str) -> Result, NetError> { + let generation = self.generation(); + if self.is_exempt(&generation, host).await { + // The system resolver keeps no cache Faith can hold answers in, so an exempt name has + // nothing to go stale and is always resolved for real (spec:DNS#serving-stale-answers). + let resolver = self.system(&generation).await?; + return Ok(resolver.lookup_ip(host).await?.iter().collect()); + } + + // Alongside the addresses rather than after them: the record is a hint for the upgrade + // layer to verify, so nothing about connecting waits on it (spec:DNS#https-records). + self.spawn_https_query(&generation, host); + + if let Some(addrs) = self.stale_addrs(&generation, host) { + self.spawn_refresh(&generation, host); + return Ok(addrs); + } + + let built = self.built(&generation).await?; + let lookup = built.resolver.lookup_ip(host).await?; + let addrs: Vec = lookup.iter().collect(); + self.remember(&generation, host, &addrs, lookup.valid_until()); + Ok(addrs) + } + + /// Ask for `host`'s `HTTPS` record behind the address lookup, so an origin advertising + /// `alpn="h3"` is known before the first connection rather than after the first TCP response. + /// + /// Spawned rather than awaited: an absent, slow, or failed answer must leave address + /// resolution and connecting untouched. Its outcome belongs to the upgrade layer rather than + /// to the request that triggered it, so nothing here reaches a caller, exactly as a stale + /// refresh's outcome does not. + // spec:DNS#https-records + fn spawn_https_query(&self, generation: &Arc, host: &str) { + let Some(sink) = self.https_sink() else { + return; + }; + // Asked before the query, not after: an origin already confirmed, failed, or holding a + // live advertisement has nothing to learn, so it costs no DNS traffic. + if !sink.wants(host) { + return; + } + let Ok(name) = Name::from_utf8(host) else { + return; + }; + + let this = self.clone(); + let generation = Arc::clone(generation); + let host = host.to_owned(); + tokio::spawn(async move { + let Ok(built) = this.built(&generation).await else { + return; + }; + let Ok(lookup) = built.resolver.lookup(name, RecordType::HTTPS).await else { + return; + }; + // The answer's own query name, not the host as written: the search list may have + // requalified it, and the record's target is judged against what was actually asked. + if let Some(advertisement) = read_https_answer(lookup.query().name(), lookup.answers()) + { + sink.record(&host, advertisement); + } + }); + } + + /// The addresses to serve for `host` without waiting, when its answer has expired but is still + /// inside `dns.maxStale`. + /// + /// `None` covers the three cases that must go to the resolver: no entry at all, an entry still + /// fresh (which hickory's own cache answers without a network round trip anyway), and an entry so + /// old it has stopped being evidence about the host. + fn stale_addrs(&self, generation: &Generation, host: &str) -> Option> { + if !self.inner.settings.serve_stale { + return None; + } + let entry = generation.stale.get(host)?; + let now = Instant::now(); + if now <= entry.valid_until { + return None; + } + if now.saturating_duration_since(entry.valid_until) > self.inner.settings.max_stale { + // Dropped rather than left to sit: keeping it would let a refresh that has been failing + // for hours go on being consulted, and the entry can only get older from here. + generation.stale.invalidate(host); + return None; + } + Some(entry.addrs.as_ref().clone()) + } + + /// Keep a successful answer for `host`, so a later lookup past its TTL has something to serve. + fn remember( + &self, + generation: &Generation, + host: &str, + addrs: &[IpAddr], + valid_until: Instant, + ) { + if !self.inner.settings.serve_stale || addrs.is_empty() { + return; + } + generation.stale.insert( + host.to_owned(), + StaleEntry { + addrs: Arc::new(addrs.to_vec()), + valid_until, + }, + ); + } + + /// Refresh `host` behind a stale answer that has already been served. + /// + /// Single-flighted per host: the claim is taken before the task is spawned, so concurrent stale + /// hits serve the entry rather than each starting a lookup. The task outlives the request that + /// triggered it, and its outcome belongs to the cache rather than that request, so nothing here + /// is reported to a caller. + // spec:DNS#serving-stale-answers + fn spawn_refresh(&self, generation: &Arc, host: &str) { + { + let mut refreshing = generation + .refreshing + .lock() + .expect("the DNS refresh lock is only held to insert or remove a host"); + if !refreshing.insert(host.to_owned()) { + return; + } + } + + let this = self.clone(); + let generation = Arc::clone(generation); + let host = host.to_owned(); + tokio::spawn(async move { + match this.refresh(&generation, &host).await { + Ok(()) => {} + Err(err) if is_authoritatively_empty(&err) => { + // The name resolves to nothing now, so the old address is not a stale answer for + // it any more but a wrong one. Dropping the entry makes the next lookup fail + // rather than hand out an address the host no longer answers on. + generation.stale.invalidate(&host); + } + Err(_) => { + // A network error, a server failure, or a timeout says nothing about where the + // host is, so the entry stays and can be served again while this persists. + } + } + generation + .refreshing + .lock() + .expect("the DNS refresh lock is only held to insert or remove a host") + .remove(&host); + }); + } + + /// One refresh lookup, replacing the stale entry when it resolves. + async fn refresh(&self, generation: &Generation, host: &str) -> Result<(), NetError> { + let built = self.built(generation).await?; + let lookup = built.resolver.lookup_ip(host).await?; + let addrs: Vec = lookup.iter().collect(); + self.remember(generation, host, &addrs, lookup.valid_until()); + Ok(()) + } + + /// Drop any stale answer held for `host`, so the next lookup waits for a fresh one. + /// + /// Called when connecting to a served address failed, which is the one piece of evidence that the + /// address was wrong rather than merely old. + // spec:DNS#when-a-stale-address-is-wrong + pub fn invalidate_stale(&self, host: &str) { + self.generation().stale.invalidate(host); + } + + /// Whether a lookup of `host` right now would be served from an expired entry, and so would hand + /// out an address that is assumed rather than confirmed. + /// + /// Deliberately the same window a stale answer is served from, rather than merely "an expired + /// entry exists": an entry past `dns.maxStale` is resolved for real, and treating that as stale + /// would spend a second connection attempt on an address that was already confirmed. + pub fn served_stale(&self, host: &str) -> bool { + if !self.inner.settings.serve_stale { + return false; + } + self.generation().stale.get(host).is_some_and(|entry| { + let now = Instant::now(); + now > entry.valid_until + && now.saturating_duration_since(entry.valid_until) <= self.inner.settings.max_stale + }) + } + + /// Resolve `host` and leave the answer in the shared cache, so a later request skips the + /// lookup. Any failure is swallowed: the warm-up is advisory. + // spec:WARM + pub async fn prefetch(&self, host: &str) { + let _ = self.lookup(host).await; + } + + /// The DNS servers the agent resolves through, in query order. Empty + /// until the resolver has been used, because it reads its configuration on first use. + // spec:OBS#resolvers + pub fn resolvers(&self) -> Vec { + self.generation() + .built + .get() + .map(|built| built.reports.clone()) + .unwrap_or_default() + } + + /// Drop everything read off the network, so the next lookup rebuilds against the network the + /// agent is on now. + /// + /// Flushing cached answers alone would leave the agent resolving them again through the + /// previous network's servers: the discovered server list, the suffixes treated as local, and + /// the results of encryption probes are all readings of a network too, and the whole point of + /// the signal is that the network has changed. Dropping the generation takes the caches with + /// it, since they belong to the resolvers being dropped. + /// + /// The caller's options are untouched, so a listed `dns.servers` set is rebuilt exactly as + /// configured; what it re-reads is what the system supplies and what the network answers. + /// + /// Synchronous, unlike the rest of this type: it swaps an `Arc` rather than building anything, + /// which keeps it callable from a network-change signal, which is not async. Nothing is rebuilt here + /// either, so an agent that never resolves again pays nothing for the signal. + // spec:NETCHG#what-the-signal-keeps + // spec:NETCHG#reach-across-the-subsystems + pub fn reset(&self) { + *self + .inner + .generation + .lock() + .expect("the DNS generation lock is only held to clone or replace an Arc") = + Arc::new(Generation::default()); + } +} + +/// Installed on a reqwest client with `ClientBuilder::dns_resolver`, so every lookup a request +/// makes goes through the same resolver `prefetch` warms. +#[cfg(feature = "reqwest")] +impl reqwest::dns::Resolve for FaithResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let this = self.clone(); + Box::pin(async move { + let addrs = this.lookup(name.as_str()).await?; + // Port `0` is a placeholder reqwest fills from the URL. The returned `Addrs` has to be + // `'static`, so collect owned rather than borrowing the lookup. + let addrs: Vec = + addrs.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +/// Whether a failed lookup was the resolver answering that the name holds nothing, rather than the +/// resolver failing to answer. +/// +/// The distinction decides what happens to a stale entry: an authoritative "nothing here" retires +/// it, while a failure to reach an answer leaves it in place. Hickory draws the same line, producing +/// `NoRecordsFound` only for `NXDOMAIN` and for `NOERROR` with no answer records, and reporting +/// `SERVFAIL` and the other failure codes as `ResponseCode` instead. +fn is_authoritatively_empty(err: &NetError) -> bool { + matches!(err, NetError::Dns(DnsError::NoRecordsFound(_))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/resolver/tests.rs b/crates/web-faith-dns/src/resolver/tests.rs new file mode 100644 index 0000000..da07582 --- /dev/null +++ b/crates/web-faith-dns/src/resolver/tests.rs @@ -0,0 +1,178 @@ +use std::{ + net::IpAddr, + sync::Arc, + time::{Duration, Instant}, +}; + +use super::{FaithResolver, StaleEntry}; +use crate::{settings::ResolverSettings, transport::ServerSpec}; + +fn spec(input: &str) -> ServerSpec { + ServerSpec::parse(input).expect("valid server URL") +} + +#[tokio::test] +async fn reset_replaces_the_generation_and_what_it_holds() { + // A network change drops what was read off the old network, so the next lookup builds + // against the new one rather than reusing the previous network's servers (spec:NETCHG). + let resolver = FaithResolver::new(ResolverSettings { + servers: vec![spec("udp://127.0.0.1:1")], + timeout: Some(Duration::from_millis(200)), + ..ResolverSettings::default() + }); + + let before = resolver.generation(); + // Build the generation's state, so there is something for the reset to drop. + let _ = resolver.built(&before).await; + assert!( + before.built.get().is_some(), + "the generation built its resolver" + ); + assert_eq!(resolver.resolvers().len(), 1, "which `resolvers()` reports"); + + resolver.reset(); + + let after = resolver.generation(); + assert!( + !Arc::ptr_eq(&before, &after), + "the reset swaps the generation rather than mutating it" + ); + assert!( + after.built.get().is_none(), + "the new generation holds nothing until it is used again" + ); + assert!( + before.built.get().is_some(), + "work already holding the old generation keeps its resolvers" + ); + assert!( + resolver.resolvers().is_empty(), + "`resolvers()` reports nothing until the rebuild (spec:OBS#resolvers)" + ); + + // Configuration survives the signal, so the rebuild uses the servers as configured. + let _ = resolver.built(&after).await; + assert_eq!( + resolver.resolvers().len(), + 1, + "the rebuilt generation resolves through the configured servers again" + ); +} + +/// A resolver with a stale entry for `host` whose freshness ended `ago`. +fn with_stale_entry(settings: ResolverSettings, host: &str, ago: Duration) -> FaithResolver { + let resolver = FaithResolver::new(settings); + resolver.generation().stale.insert( + host.to_owned(), + StaleEntry { + addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), + valid_until: Instant::now() - ago, + }, + ); + resolver +} + +#[test] +fn only_an_expired_entry_inside_the_window_is_served_stale() { + // spec:DNS#serving-stale-answers + let settings = || ResolverSettings { + max_stale: Duration::from_secs(60), + ..ResolverSettings::default() + }; + + // Still fresh: the lookup goes through hickory, which answers from its own cache. + let fresh = FaithResolver::new(settings()); + fresh.generation().stale.insert( + "fresh.test".to_owned(), + StaleEntry { + addrs: Arc::new(vec![IpAddr::from([127, 0, 0, 1])]), + valid_until: Instant::now() + Duration::from_secs(60), + }, + ); + let generation = fresh.generation(); + assert!( + fresh.stale_addrs(&generation, "fresh.test").is_none(), + "a fresh entry is not a stale hit" + ); + + // Expired but inside `dns.maxStale`: served immediately. + let stale = with_stale_entry(settings(), "stale.test", Duration::from_secs(5)); + let generation = stale.generation(); + assert!( + stale.stale_addrs(&generation, "stale.test").is_some(), + "an entry expired inside the window is served" + ); + + // Past the window: no longer evidence about the host, so the lookup blocks. + let old = with_stale_entry(settings(), "old.test", Duration::from_secs(120)); + let generation = old.generation(); + assert!( + old.stale_addrs(&generation, "old.test").is_none(), + "an entry past `dns.maxStale` is not served" + ); + assert!( + generation.stale.get("old.test").is_none(), + "and is dropped rather than left to age further" + ); +} + +#[test] +fn serve_stale_off_never_serves_an_expired_entry() { + // spec:DNS#serving-stale-answers — the switch for a caller that must not connect to an + // address it knows to be out of date. + let resolver = with_stale_entry( + ResolverSettings { + serve_stale: false, + ..ResolverSettings::default() + }, + "strict.test", + Duration::from_secs(5), + ); + let generation = resolver.generation(); + assert!(resolver.stale_addrs(&generation, "strict.test").is_none()); + assert!( + !resolver.served_stale("strict.test"), + "and nothing is reported as stale-served, so no retry is armed" + ); +} + +#[test] +fn served_stale_tracks_the_window_it_serves_from() { + // The retry layer arms itself from this, so it must not claim an address was assumed when + // the lookup actually blocked on a fresh one (spec:DNS#when-a-stale-address-is-wrong). + let settings = || ResolverSettings { + max_stale: Duration::from_secs(60), + ..ResolverSettings::default() + }; + + let inside = with_stale_entry(settings(), "inside.test", Duration::from_secs(5)); + assert!(inside.served_stale("inside.test")); + + let outside = with_stale_entry(settings(), "outside.test", Duration::from_secs(120)); + assert!( + !outside.served_stale("outside.test"), + "an entry past the window is resolved for real, so its address is confirmed" + ); + + let absent = FaithResolver::new(settings()); + assert!(!absent.served_stale("absent.test")); +} + +#[test] +fn a_network_change_drops_stale_answers() { + // Addresses read off the old network are exactly what must not be served on the new one + // (spec:NETCHG#reach-across-the-subsystems). + let resolver = with_stale_entry( + ResolverSettings::default(), + "netchg.test", + Duration::from_secs(5), + ); + assert!(resolver.served_stale("netchg.test")); + + resolver.reset(); + + assert!( + !resolver.served_stale("netchg.test"), + "the stale answer goes with the generation that held it" + ); +} diff --git a/crates/web-faith-dns/src/settings.rs b/crates/web-faith-dns/src/settings.rs new file mode 100644 index 0000000..715c9d4 --- /dev/null +++ b/crates/web-faith-dns/src/settings.rs @@ -0,0 +1,134 @@ +use std::{net::SocketAddr, time::Duration}; + +use hickory_resolver::{ + config::{NameServerConfig, ProtocolConfig}, + proto::rr::Name, +}; + +use crate::transport::{ServerSpec, Transport}; + +/// How a server in `resolvers()` came to be reached the way it is. +// spec:OBS#resolvers +#[derive(Clone, Copy, Debug)] +pub enum ResolverSource { + /// Named in `dns.servers` by the caller. + Configured, + /// Discovered from the system's resolver configuration. + Conventional, +} + +impl ResolverSource { + fn label(self) -> &'static str { + match self { + Self::Configured => "configured", + Self::Conventional => "conventional", + } + } +} + +/// One line of `resolvers()`: a server's address, the transport in use, and how it was arrived at. +#[derive(Clone, Debug)] +pub struct ResolverReport { + pub address: String, + pub transport: String, + pub source: String, +} + +/// `dns.maxStale`'s default: how far past expiry an answer may still be served. +/// +/// An hour is long enough that a resolver outage does not stop an agent reaching hosts it already +/// knows, and short enough that a host which really has moved stops being served a dead address for +/// the life of a long-running process. The recovery path bounds the cost of being wrong to one +/// re-resolve, so the window can be generous. +// spec:DNS#serving-stale-answers +pub const DEFAULT_MAX_STALE: Duration = Duration::from_secs(3600); + +/// Everything `dns.*` configures about Faith's resolver, resolved from options at construction. +#[derive(Clone, Debug)] +pub struct ResolverSettings { + /// The `dns.servers` list, in order. Empty means system discovery. + pub servers: Vec, + /// `dns.timeout`, bounding the whole list. `None` leaves hickory's five-second default. + pub timeout: Option, + /// `dns.ndots`. + pub ndots: Option, + /// `dns.searchDomains`, replacing the system's search list when set. + pub search_domains: Option>, + /// `dns.hostsFile`: `Some(true)`/`Some(false)` force it on/off, `None` follows the platform. + pub hosts_file: Option, + /// `dns.exemptDomains`, added to the always-exempt `localhost`, `.local`, and system suffix. + pub exempt_domains: Vec, + /// `dns.serveStale`: whether an expired answer is served while a refresh runs behind it. + pub serve_stale: bool, + /// `dns.maxStale`: how far past expiry an answer may still be served. + pub max_stale: Duration, +} + +impl Default for ResolverSettings { + fn default() -> Self { + Self { + servers: Vec::new(), + timeout: None, + ndots: None, + search_domains: None, + hosts_file: None, + exempt_domains: Vec::new(), + // Defaulted here as well as in the option parsing, so a resolver built directly (in tests, + // and for the global default agent) serves stale like a configured one. + serve_stale: true, + max_stale: DEFAULT_MAX_STALE, + } + } +} + +/// The suffixes handed to the system resolver rather than Faith's servers: `localhost` and `local` +/// always, plus the ones the system supplies and the caller's `dns.exemptDomains`. +/// +/// The root name is never a suffix here, whichever list it arrives in. It is the parent of every +/// name, so admitting it would exempt the lot and route every lookup to the system resolver with +/// `dns.servers` configured and unused. It does arrive in practice: a Windows host with no DNS +/// domain of its own reports the root as its domain, so the check is what keeps the encrypted +/// transports working there rather than being quietly bypassed. +// spec:DNS#exempt-names +pub(crate) fn exempt_suffixes(system: Vec, configured: &[Name]) -> Vec { + let mut names = vec![ + Name::from_ascii("localhost").unwrap(), + Name::from_ascii("local").unwrap(), + ]; + names.extend( + system + .into_iter() + .chain(configured.iter().cloned()) + .filter(|name| !name.is_root()), + ); + names +} + +/// Summarise name servers for `resolvers()`, in the order they are queried. +pub(crate) fn report( + name_servers: &[NameServerConfig], + source: ResolverSource, +) -> Vec { + let mut reports = Vec::new(); + for server in name_servers { + for connection in &server.connections { + let transport = match connection.protocol { + ProtocolConfig::Udp => Transport::Udp, + ProtocolConfig::Tcp => Transport::Tcp, + ProtocolConfig::Tls { .. } => Transport::Tls, + ProtocolConfig::Https { .. } => Transport::Https, + ProtocolConfig::Quic { .. } => Transport::Quic, + ProtocolConfig::H3 { .. } => Transport::H3, + }; + reports.push(ResolverReport { + address: SocketAddr::new(server.ip, connection.port).to_string(), + transport: transport.label().to_owned(), + source: source.label().to_owned(), + }); + } + } + reports +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/settings/tests.rs b/crates/web-faith-dns/src/settings/tests.rs new file mode 100644 index 0000000..b1972bd --- /dev/null +++ b/crates/web-faith-dns/src/settings/tests.rs @@ -0,0 +1,51 @@ +use hickory_resolver::proto::rr::Name; + +use super::exempt_suffixes; + +#[test] +fn a_root_suffix_never_exempts_everything() { + // A Windows host with no DNS domain reports the root as its domain, and the root is the + // parent of every name. Taking it as a suffix exempted every lookup and sent it to the + // system resolver, leaving `dns.servers` configured and unused (spec:DNS#exempt-names). + let suffixes = exempt_suffixes(vec![Name::root()], &[]); + assert!( + !suffixes.iter().any(|suffix| suffix.is_root()), + "the root is not admitted as a suffix" + ); + + let name = Name::from_utf8("nonexistent.example").unwrap(); + assert!( + !suffixes.iter().any(|suffix| suffix.zone_of(&name)), + "so an ordinary name is not exempt and reaches the configured servers" + ); + + // The names that must stay exempt still are, and a real system suffix still counts. + let suffixes = exempt_suffixes( + vec![Name::root(), Name::from_utf8("corp.example").unwrap()], + &[], + ); + for exempt in ["localhost", "printer.local", "host.corp.example"] { + let name = Name::from_utf8(exempt).unwrap(); + assert!( + suffixes.iter().any(|suffix| suffix.zone_of(&name)), + "{exempt} is exempt" + ); + } +} + +#[test] +fn a_root_entry_from_the_caller_is_refused_too() { + // Whichever list it arrives in, the root would disable the caller's own servers. + let suffixes = exempt_suffixes(vec![], &[Name::root()]); + let name = Name::from_utf8("nonexistent.example").unwrap(); + assert!(!suffixes.iter().any(|suffix| suffix.zone_of(&name))); +} + +#[test] +fn exempt_matches_a_suffix_exactly_or_as_a_subdomain() { + // spec:DNS#exempt-names + let local = Name::from_ascii("local").unwrap(); + assert!(local.zone_of(&Name::from_utf8("printer.local").unwrap())); + assert!(local.zone_of(&Name::from_utf8("local").unwrap())); + assert!(!local.zone_of(&Name::from_utf8("mylocal.example").unwrap())); +} diff --git a/crates/web-faith-dns/src/transport.rs b/crates/web-faith-dns/src/transport.rs new file mode 100644 index 0000000..5b34b1c --- /dev/null +++ b/crates/web-faith-dns/src/transport.rs @@ -0,0 +1,166 @@ +use std::{net::IpAddr, sync::Arc}; + +use hickory_resolver::config::{ConnectionConfig, NameServerConfig, ProtocolConfig}; +use url::{Host, Url}; + +/// The default DoH/DoQ query path, used when a `https://`/`h3://` server URL supplies none. +const DEFAULT_DNS_QUERY_PATH: &str = "/dns-query"; + +/// The transport Faith speaks to a resolver, chosen by a server URL's scheme. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Transport { + Udp, + Tcp, + Tls, + Https, + Quic, + H3, +} + +impl Transport { + fn from_scheme(scheme: &str) -> Option { + Some(match scheme { + "udp" => Self::Udp, + "tcp" => Self::Tcp, + "tls" => Self::Tls, + "https" => Self::Https, + "quic" => Self::Quic, + "h3" => Self::H3, + _ => return None, + }) + } + + /// The conventional port for the transport, used when the URL names none. + fn default_port(self) -> u16 { + match self { + Self::Udp | Self::Tcp => 53, + Self::Tls | Self::Quic => 853, + Self::Https | Self::H3 => 443, + } + } + + /// The lowercase label reported by `resolvers()`. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Udp => "udp", + Self::Tcp => "tcp", + Self::Tls => "tls", + Self::Https => "https", + Self::Quic => "quic", + Self::H3 => "h3", + } + } +} + +/// A resolver Faith reaches by IP or by a hostname it bootstraps. +#[derive(Clone, Debug)] +pub(crate) enum ServerHost { + Ip(IpAddr), + Name(String), +} + +/// One entry of `dns.servers`, parsed at agent construction. The IP is not known yet for a +/// hostname host: that is resolved when the resolver is first used (see [`ResolverSettings`](crate::ResolverSettings)). +#[derive(Clone, Debug)] +pub struct ServerSpec { + pub(crate) host: ServerHost, + transport: Transport, + port: u16, + /// DoH/DoQ query path, `None` for the non-HTTP transports. + path: Option>, + /// The name to authenticate the certificate against, from a URL fragment. When absent, a + /// hostname host authenticates against itself and an IP host against the address. + cert_name: Option, +} + +impl ServerSpec { + /// Parse one `dns.servers` URL, or return a message for an unparseable URL or unknown scheme. + pub fn parse(input: &str) -> Result { + let url = Url::parse(input).map_err(|err| format!("{input:?}: {err}"))?; + let transport = Transport::from_scheme(url.scheme()) + .ok_or_else(|| format!("{input:?}: unknown DNS transport scheme {:?}", url.scheme()))?; + + let host = match url.host() { + Some(Host::Ipv4(ip)) => ServerHost::Ip(IpAddr::V4(ip)), + Some(Host::Ipv6(ip)) => ServerHost::Ip(IpAddr::V6(ip)), + Some(Host::Domain(name)) => ServerHost::Name(name.to_owned()), + None => return Err(format!("{input:?}: no host to resolve")), + }; + + let port = url.port().unwrap_or_else(|| transport.default_port()); + let path = match transport { + Transport::Https | Transport::H3 => { + let path = url.path(); + (!path.is_empty() && path != "/").then(|| Arc::from(path)) + } + _ => None, + }; + let cert_name = url.fragment().map(str::to_owned); + + Ok(Self { + host, + transport, + port, + path, + cert_name, + }) + } + + /// The IP host, or `None` for a hostname host that still needs bootstrapping. + pub(crate) fn ip(&self) -> Option { + match self.host { + ServerHost::Ip(ip) => Some(ip), + ServerHost::Name(_) => None, + } + } + + /// The certificate name to authenticate against once the host resolves to `ip`: an explicit + /// fragment, else the hostname, else the address itself. + // spec:DNS#transports + fn server_name(&self, ip: IpAddr) -> Arc { + if let Some(name) = &self.cert_name { + Arc::from(name.as_str()) + } else { + match &self.host { + ServerHost::Name(name) => Arc::from(name.as_str()), + ServerHost::Ip(_) => Arc::from(ip.to_string()), + } + } + } + + /// Build the hickory name server for this spec, reached at `ip`. + pub(crate) fn to_name_server(&self, ip: IpAddr) -> NameServerConfig { + let protocol = match self.transport { + Transport::Udp => ProtocolConfig::Udp, + Transport::Tcp => ProtocolConfig::Tcp, + Transport::Tls => ProtocolConfig::Tls { + server_name: self.server_name(ip), + }, + Transport::Https => ProtocolConfig::Https { + server_name: self.server_name(ip), + path: self + .path + .clone() + .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), + }, + Transport::Quic => ProtocolConfig::Quic { + server_name: self.server_name(ip), + }, + Transport::H3 => ProtocolConfig::H3 { + server_name: self.server_name(ip), + path: self + .path + .clone() + .unwrap_or_else(|| Arc::from(DEFAULT_DNS_QUERY_PATH)), + disable_grease: false, + }, + }; + + let mut connection = ConnectionConfig::new(protocol); + connection.port = self.port; + NameServerConfig::new(ip, true, vec![connection]) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/web-faith-dns/src/transport/tests.rs b/crates/web-faith-dns/src/transport/tests.rs new file mode 100644 index 0000000..aa933bb --- /dev/null +++ b/crates/web-faith-dns/src/transport/tests.rs @@ -0,0 +1,82 @@ +use std::{net::IpAddr, sync::Arc}; + +use hickory_resolver::config::ProtocolConfig; + +use super::{DEFAULT_DNS_QUERY_PATH, ServerSpec, Transport}; + +fn spec(input: &str) -> ServerSpec { + ServerSpec::parse(input).expect("valid server URL") +} + +#[test] +fn scheme_selects_transport_and_conventional_port() { + // spec:DNS#transports + assert_eq!(spec("udp://1.1.1.1").transport, Transport::Udp); + assert_eq!(spec("udp://1.1.1.1").port, 53); + assert_eq!(spec("tcp://1.1.1.1").port, 53); + assert_eq!(spec("tls://1.1.1.1").transport, Transport::Tls); + assert_eq!(spec("tls://1.1.1.1").port, 853); + assert_eq!(spec("quic://1.1.1.1").port, 853); + assert_eq!(spec("https://1.1.1.1").transport, Transport::Https); + assert_eq!(spec("https://1.1.1.1").port, 443); + assert_eq!(spec("h3://1.1.1.1").port, 443); +} + +#[test] +fn explicit_port_overrides_the_conventional_one() { + // spec:DNS#transports + assert_eq!(spec("tls://1.1.1.1:8853").port, 8853); +} + +#[test] +fn http_transports_default_the_query_path() { + // spec:DNS#transports — `/dns-query` when the URL supplies none. + assert_eq!(spec("https://dns.google").path, None); + assert_eq!( + spec("https://dns.google") + .to_name_server(IpAddr::from([8, 8, 8, 8])) + .connections[0] + .protocol, + ProtocolConfig::Https { + server_name: Arc::from("dns.google"), + path: Arc::from(DEFAULT_DNS_QUERY_PATH), + } + ); + assert_eq!( + spec("https://dns.google/resolve").path, + Some(Arc::from("/resolve")) + ); +} + +#[test] +fn a_fragment_names_the_certificate() { + // spec:DNS#transports — `tls://1.1.1.1#cloudflare-dns.com`. + let spec = spec("tls://1.1.1.1#cloudflare-dns.com"); + assert_eq!(spec.cert_name.as_deref(), Some("cloudflare-dns.com")); + assert_eq!( + &*spec.server_name(IpAddr::from([1, 1, 1, 1])), + "cloudflare-dns.com" + ); +} + +#[test] +fn a_bare_ip_authenticates_against_the_address() { + // spec:DNS#transports — `tls://1.1.1.1` with no fragment. + let spec = spec("tls://1.1.1.1"); + assert_eq!(spec.cert_name, None); + assert_eq!(&*spec.server_name(IpAddr::from([1, 1, 1, 1])), "1.1.1.1"); +} + +#[test] +fn a_hostname_authenticates_against_itself() { + // spec:DNS#transports + let spec = spec("tls://dns.google"); + assert_eq!(&*spec.server_name(IpAddr::from([8, 8, 8, 8])), "dns.google"); +} + +#[test] +fn an_unknown_scheme_is_rejected() { + // spec:DNS#transports — throws an address-parse error at construction. + assert!(ServerSpec::parse("ftp://1.1.1.1").is_err()); + assert!(ServerSpec::parse("not a url").is_err()); +} diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index e6738e7..e2bcdbf 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -1,26 +1,24 @@ -use std::{fmt::Debug, str::FromStr as _, time::Duration}; +//! The `Agent` class, as JavaScript sees it. + +use std::str::FromStr as _; use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; -use napi::{Either, Env, bindgen_prelude::Buffer}; +use napi::Env; use napi_derive::napi; use reqwest::Url; -use http_cache_reqwest::CacheMode; -use web_faith::client::RedirectPolicy; -use web_faith::options; - -use web_faith_cookies::{ - CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, -}; - use crate::{ async_task::faith_promise, conn_tracker::{ConnectionInfo, connections_for_napi}, error::{FaithError, FaithErrorExt}, - options::RequestCacheMode, }; +mod convert; +mod options; + +pub use options::*; + #[napi] pub const FAITH_VERSION: &str = env!("CARGO_PKG_VERSION"); #[napi] @@ -40,744 +38,6 @@ pub const REQWEST_VERSION: &str = env!("REQWEST_VERSION"); #[napi] pub const USER_AGENT: &str = web_faith::USER_AGENT; -#[napi(string_enum)] -#[derive(Debug, Clone, Copy)] -pub enum CacheStore { - #[napi(value = "disk")] - Disk, - - #[napi(value = "memory")] - Memory, -} - -/// Settings related to the HTTP cache. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentCacheOptions { - /// Which cache store to use: either `disk` or `memory`. - /// - /// Default: none (cache disabled). - pub store: Option, - /// If `cache.store: "memory"`, the maximum amount of items stored. - /// - /// Default: 10_000. - pub capacity: Option, - /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if - /// no cache mode is set on a request. - /// - /// Default: `"default"`. - pub mode: Option, - /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. - /// - /// Required if `cache.store: "disk"`. - pub path: Option, - /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is - /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a - /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). - /// `shared: true` is required for proxies and multi-user caches. - /// - /// Default: true. - pub shared: Option, -} - -/// Limits the cookie store enforces, from RFC 6265bis. Each is a cap; a caller who needs more room -/// raises the number. -/// -/// The `__Host-` and `__Secure-` name prefix rules are what those prefixes mean, so they always -/// apply and are not settable here: a cookie that shouldn't carry them is named without one. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentCookieOptions { - /// How far ahead of receipt a cookie may expire, in seconds. A cookie asking for longer, via - /// `Max-Age` or `Expires`, has its expiry reduced to this; a shorter one is left alone and a - /// session cookie stays a session cookie. - /// - /// Default: 34_560_000 (400 days). - pub max_age: Option, - /// The largest cookie stored, as the combined length of its name and value in bytes. A larger - /// cookie is not stored. - /// - /// Default: 4096. - pub max_size: Option, - /// How many cookies are kept for any one domain, which is a cookie's `Domain` attribute when it - /// has one and the host that set it otherwise. - /// - /// Default: 180. - pub max_per_host: Option, - /// How many cookies are kept across the whole store, bounding a server that spreads cookies - /// across subdomains to escape `maxPerHost`. - /// - /// Default: 3000. - pub max_total: Option, -} - -impl From<&AgentCookieOptions> for CookieLimits { - fn from(options: &AgentCookieOptions) -> Self { - Self { - max_age: options - .max_age - .map_or(DEFAULT_MAX_AGE, |secs| Duration::from_secs(secs.into())), - max_size: options.max_size.map_or(DEFAULT_MAX_SIZE, |n| n as usize), - max_per_host: options - .max_per_host - .map_or(DEFAULT_MAX_PER_HOST, |n| n as usize), - max_total: options.max_total.map_or(DEFAULT_MAX_TOTAL, |n| n as usize), - } - } -} - -#[napi(object)] -#[derive(Debug, Clone)] -pub struct DnsOverride { - pub domain: String, - pub addresses: Vec, -} - -/// Settings related to DNS. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentDnsOptions { - /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on - /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native - /// fetch does, this should be your first port of call. - /// - /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory - /// DNS cache, and may lead to worse performance even discounting the cache. - /// - /// Default: false. - /// - /// [Hickory]: https://hickory-dns.org/ - pub system: Option, - /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. - /// - /// Will throw if addresses are in invalid formats. You may provide a port number as part of the - /// address, it will default to port 0 otherwise, which will select the conventional port for the - /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port - /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively - /// blocks that domain from this agent. - /// - /// Default: no overrides. - pub overrides: Option>, - /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to - /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over - /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on - /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the - /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. - /// - /// The encrypted transports always authenticate the resolver. A URL fragment names the - /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates - /// against the hostname; a bare-IP host authenticates against the address itself. - /// - /// Servers are queried in order, a later one reached only once those before it fail. Setting - /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or - /// its scheme is not one of the above, and combining it with `dns.system` throws. - /// - /// Default: system discovery. - pub servers: Option>, - /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead - /// servers costs a single timeout rather than one per server. - /// - /// Default: 5000. - pub timeout: Option, - /// Replace the system's search list, the domains appended to a name that is not fully - /// qualified. Independent of `dns.servers`. - /// - /// Default: the system's search list. - pub search_domains: Option>, - /// How many dots a name must contain before it is tried as given, ahead of the search list. - /// Independent of `dns.servers`. - /// - /// Default: the system's setting. - pub ndots: Option, - /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. - /// - /// Default: platform convention. - pub hosts_file: Option, - /// Further domains to exempt from the configured or encrypted resolver, for the internal - /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's - /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. - /// - /// Default: no extra exemptions. - pub exempt_domains: Option>, - /// Serve an expired cache entry immediately and refresh it in the background, rather than making - /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is - /// almost always still correct, and a connect failure against one that has moved re-resolves and - /// attempts the request again. - /// - /// Set `false` for an agent that must never connect to an address it knows to be out of date: an - /// expired entry is discarded and the lookup blocks on a fresh answer. - /// - /// Default: true. - pub serve_stale: Option, - /// How far past expiry an answer may still be served, in milliseconds. An entry older than this - /// is discarded rather than served: an answer stale enough stops being evidence about where the - /// host is, and a refresh still failing after that long is the case where the address most likely - /// did change. - /// - /// Default: 3600000 (one hour). - pub max_stale: Option, -} - -/// Sets the default headers for every request. -/// -/// If header names or values are invalid, they are silently omitted. -/// Sensitive headers (e.g. `Authorization`) should be marked. -/// -/// Default: none. -#[napi(object)] -#[derive(Debug, Clone)] -pub struct Header { - pub name: String, - pub value: String, - pub sensitive: Option, -} - -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, Default)] -pub enum Http3Congestion { - #[napi(value = "cubic")] - #[default] - Cubic, - - #[napi(value = "bbr1")] - Bbr1, -} - -/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc -/// cache so the first request to this host will attempt HTTP/3 immediately. -#[napi(object)] -#[derive(Debug, Clone)] -pub struct Http3Hint { - /// The hostname (e.g., "example.com"). - pub host: String, - /// The port number (e.g., 443). - pub port: u16, -} - -/// Settings related to HTTP/3. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentHttp3Options { - /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the - /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with - /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by - /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise - /// bandwidth use and optimises for round-trip time, while ignoring packet loss. - /// - /// In some networks, BBR can lead to pathological degradation of overall network conditions, by - /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, - /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). - /// - /// [2]: https://github.com/quinn-rs/quinn/issues/1254 - /// - /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. - pub congestion: Option, - /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that - /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ - /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith - /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). - /// - /// Default: 30. - pub max_idle_timeout: Option, - /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc - /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. - /// - /// Default: true. - pub upgrade_enabled: Option, - /// Whether advertised HTTP/3 endpoints are verified with a background probe - /// before any foreground request is routed to them. - /// - /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say - /// there is UDP connectivity between you and it. Without probing, the next - /// request after an advertisement attempts HTTP/3 inline, and on a silently - /// broken UDP path it stalls until the QUIC idle timeout or - /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per - /// failure cooldown for as long as the path stays broken. - /// - /// With probing (the default), requests keep using TCP until a background - /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the - /// connection pool, so the first upgraded request rides the probe's warm - /// connection. A broken path costs one failed background request per - /// cooldown and no foreground latency at all. - /// - /// The probe is a synthetic request the server will see in its logs. Set - /// this to `false` to restore the inline upgrade if that is unacceptable - /// (per-request billing, easily-alarmed WAFs). - /// - /// `hints` are exempt either way: a hint is your own assertion, so the first - /// request to a hinted origin speaks HTTP/3 immediately, which is also what - /// makes h3-only origins (no TCP listener) work. - /// - /// Default: true. - pub upgrade_probe: Option, - /// Ceiling on how long a background HTTP/3 probe may take before the origin - /// is treated as failed, in **milliseconds**. - /// - /// This bounds background work only — no foreground request ever waits on a - /// probe — so it can afford to be generous: a healthy handshake plus HEAD - /// completes in one or two round trips. Set to 0 to leave probes bounded - /// only by the QUIC idle timeout. - /// - /// Default: 5000 (5 seconds). - pub upgrade_probe_timeout: Option, - /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than - /// its TCP path by this factor. Set to 0 to disable path-time demotion. - /// - /// Faith keeps a per-origin moving average of time-to-response-headers for - /// each protocol family. HTTP/3 is preferred at parity and when moderately - /// slower — its advantages (no head-of-line blocking, connection migration) - /// pay off beyond the average — so this factor should stay well above 1. - /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC - /// average must also exceed the TCP one by an absolute 10ms so LAN-fast - /// origins don't flap on noise. - /// - /// A demoted origin is not treated as broken: it re-enters through a - /// background probe after `upgradeSlowTtl`, asking whether the path has - /// improved at zero foreground cost. - /// - /// Default: 2.5. - pub upgrade_slow_factor: Option, - /// How long (in seconds) a path-time demotion holds before the origin is - /// re-evaluated. See `upgradeSlowFactor`. - /// - /// Default: 600 (10 minutes). - pub upgrade_slow_ttl: Option, - /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. - /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. - /// - /// Default: 86400 (24 hours). - pub upgrade_advertised_ttl: Option, - /// How long (in seconds) to cache a confirmed working HTTP/3 connection. - /// - /// Default: 86400 (24 hours). - pub upgrade_confirmed_ttl: Option, - /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this - /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends - /// Alt-Svc headers. - /// - /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an - /// origin whose UDP path is blocked for good is retried less and less often instead of - /// forever at this interval. A confirmed HTTP/3 response ends the run. - /// - /// Default: 300 (5 minutes). - pub upgrade_failed_ttl: Option, - /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of - /// `upgradeFailedTtl`. - /// - /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, - /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat - /// cooldown that never backs off. - /// - /// Default: 3600 (1 hour). - pub upgrade_failed_max_ttl: Option, - /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, - /// demote an origin back to TCP. - /// - /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request - /// cancelled via `AbortSignal` never produces that signal, so without this an - /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as - /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a - /// sustained run of them demotes the origin, and any successful HTTP/3 response - /// resets the count. - /// - /// Strikes must land within about a minute of each other to count towards a - /// run. A retry loop whose backoff exceeds that window never accumulates one, - /// so callers with a long backoff should set this to 1 for immediate demotion - /// on the first cancelled attempt. - /// - /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that - /// carries small datagrams but drops full-size ones (an MTU blackhole, say). - /// Response headers still arrive, so the attempt resolves and every mechanism - /// here counts it a success — the transfer then stalls partway through the - /// body, where nothing is watching. `maxIdleTimeout` or the request's own - /// timeout is what ends such a request, and the origin stays on HTTP/3. - /// - /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. - /// - /// Default: 3. - pub upgrade_cancel_strikes: Option, - /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is - /// given up on and the request is retried over TCP, in **milliseconds**. - /// - /// Note the unit: the other `upgrade*` settings are in seconds, but this one - /// is in milliseconds to match the `timeout` settings, because useful values - /// are sub-second. - /// - /// This bounds the wait for response headers, not the response body, so a slow - /// body is unaffected. - /// - /// The default is high, but not unconditionally inert: `maxIdleTimeout` is - /// configurable up to 120 seconds, and above 60 seconds this deadline becomes - /// the effective ceiling. Even below that, "QUIC's own idle timeout fires - /// first" only holds while the connection is idle — a transfer still running - /// past this deadline keeps the connection active, so no idle timeout is - /// coming to end it. - /// - /// On expiry the request is retried over TCP, which means it is re-sent: a - /// timeout often means the server is still processing, so a slow - /// non-idempotent request (a POST, say) can end up delivered twice. Lowering - /// this value trades that double-submission risk for faster recovery when a - /// UDP path breaks. Anyone setting it low should confirm their slowest - /// legitimate time to response headers fits well inside the budget. - /// - /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle - /// timeout and the request's own timeout. - /// - /// Default: 60000 (60 seconds). - pub upgrade_attempt_timeout: Option, - /// Connect to the port a server advertises HTTP/3 on, even when it differs from - /// the origin's own port. **This is not standards-compliant**; it is off by - /// default. - /// - /// An `Alt-Svc` advertisement names a network endpoint for the origin, so - /// honouring one correctly means connecting to that endpoint while still - /// sending the *origin's* authority. reqwest cannot express that — it derives - /// the HTTP/3 connect target from the request URI's authority (tracked - /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). - /// So by default Faith does not upgrade at all when the advertised port - /// differs, rather than guessing that the origin's own port also speaks - /// HTTP/3. - /// - /// Setting this to `true` upgrades anyway, by rewriting the request's port to - /// the advertised one. That gets HTTP/3 working today against servers you - /// control, at the cost of three deviations you should be aware of: - /// - /// - The request's `Host`/`:authority` carries the advertised port instead of - /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) - /// forbids. Servers that route on authority may misroute or reject; servers - /// that ignore it are unaffected. - /// - `response.url` reports the port actually connected to. - /// - `redirected` ignores port differences, since the rewritten port would - /// otherwise look like a redirect on every request. - /// - /// TLS is unaffected: certificates are still validated against the origin's - /// hostname. Only the port changes. - /// - /// Default: `false`. - pub upgrade_follow_advertised_port: Option, - /// Maximum number of origins to track in the Alt-Svc cache. - /// - /// Default: 10000. - pub upgrade_cache_capacity: Option, - /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache - /// on agent initialization, so the first request to these hosts will attempt HTTP/3. - pub hints: Option>, - /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for - /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. - /// - /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it - /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for - /// HTTP/3 only. - /// - /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). - pub connection_window: Option, - /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload - /// throughput the way the receive windows bound download. The origin's own flow control - /// applies on top of this, so it is a ceiling rather than a grant. - /// - /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window - /// the peer advertises, with no local cap to set. - /// - /// Default: 10 MB (quinn's own default). - pub send_window: Option, -} - -/// Settings related to HTTP/2. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentHttp2Options { - /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for - /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. - /// - /// Ignored when `adaptiveWindow` is on. - /// - /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it - /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for - /// HTTP/2 only. - /// - /// Ignored when `adaptiveWindow` is on. - /// - /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). - pub connection_window: Option, - /// Replace HTTP/2's static windows with windows that start small and grow towards a - /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. - /// - /// This is off by default, and turning it on is usually the wrong move. A fresh connection - /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample - /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and - /// carries *less* throughput than the static window for all but the largest transfers. It - /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. - /// - /// Its one real advantage is memory: it holds a large window open only on connections that - /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling - /// buys the same throughput from the first byte. - /// - /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. - /// - /// Default: `false`. - pub adaptive_window: Option, -} - -/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentFlowControlOptions { - /// Maximum bytes an origin may send on any one stream before it must wait for Faith to - /// acknowledge them, for HTTP/2 and HTTP/3 alike. - /// - /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. - /// The default follows browser practice, and is deliberately at the conservative end of it: - /// a pooled server-side client can hold many connections across many origins, so - /// per-connection memory multiplies harder here than in a browser. - /// - /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. - /// - /// Default: 6 MiB. - pub stream_window: Option, - /// Maximum bytes an origin may send across all streams of one connection before it must - /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. - /// - /// This is larger than `streamWindow` so concurrent streams on one connection share the - /// connection's headroom, while still bounding the worst-case buffering of a connection - /// carrying many concurrent requests. - /// - /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against - /// the other. - /// - /// Default: 15 MiB. - pub connection_window: Option, -} - -/// Settings related to the connection pool. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentPoolOptions { - /// How many seconds of inactivity before a connection is closed. - /// - /// Default: 90 seconds. - pub idle_timeout: Option, - /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed - /// to keep the idle connections (per host) under that number. - /// - /// Default: `null` (no limit). - pub max_idle_per_host: Option, -} - -/// Switches that depart from standard behaviour on purpose. This is a nested object. -/// -/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule -/// forbids. All of them are off by default, so an agent constructed with no options is -/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise -/// established that what the rule guards against does not apply to them: turning one on means -/// requests may fail against origins that expect the standard behaviour. -#[napi(object)] -#[derive(Debug, Clone, Copy, Default)] -pub struct AgentQuirksOptions { - /// Allow a streaming request body to be sent over an HTTP/1.x connection. - /// - /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read - /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x - /// origin or an intermediary on the path may refuse it. With this on, such a body sends over - /// whichever protocol the connection negotiates. - /// - /// Default: false. - pub h1_request_streaming: Option, -} - -/// Determines the behavior in case the server replies with a redirect status. -/// One of the following values: -/// -/// - `follow`: automatically follow redirects. Faith limits this to 10 redirects. -/// - `error`: reject the promise with a network error when a redirect status is returned. -/// - ~~`manual`~~: not supported. -/// - `stop`: (Faith custom) don't follow any redirects, return the responses. -/// -/// Defaults to `follow`. -#[napi(string_enum)] -#[derive(Debug, Clone, Copy, Default)] -pub enum Redirect { - #[napi(value = "follow")] - #[default] - Follow, - - #[napi(value = "error")] - Error, - - #[napi(value = "manual")] - Manual, - - #[napi(value = "stop")] - Stop, -} - -/// `manual` is not supported and behaves as `follow`, which is what the client's own policy spells -/// out: it carries no variant for a choice that never differed. -impl From for RedirectPolicy { - fn from(redirect: Redirect) -> Self { - match redirect { - Redirect::Follow | Redirect::Manual => Self::Follow, - Redirect::Error => Self::Error, - Redirect::Stop => Self::Stop, - } - } -} - -/// Timeouts for requests made with this agent. This is a nested object. -#[napi(object)] -#[derive(Debug, Clone, Copy, Default)] -pub struct AgentTimeoutOptions { - /// Set a timeout for only the connect phase, in milliseconds. - /// - /// Default: none. - pub connect: Option, - /// Set a timeout for read operations, in milliseconds. - /// - /// The timeout applies to each read operation, and resets after a successful read. This is more - /// appropriate for detecting stalled connections when the size isn't known beforehand. - /// - /// Default: none. - pub read: Option, - /// Set a timeout for the entire request-response cycle, in milliseconds. - /// - /// The timeout applies from when the request starts connecting until the response body has finished. - /// Also considered a total deadline. - /// - /// Default: none. - pub total: Option, -} - -/// Settings related to the connection pool. This is a nested object. -#[napi(object)] -#[derive(Default)] -pub struct AgentTlsOptions { - /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet - /// of application data alongside the opening packet of the TLS handshake. That can enable the server - /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant - /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should - /// really only be used for static assets or to squeeze out the last drop of performance for endpoints - /// that are replay-safe. - /// - /// Default: false. - pub early_data: Option, - /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also - /// called mutual TLS or mTLS) authentication. - /// - /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The - /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options - /// that will cause the `Agent` constructor to throw if the input is in the wrong format. - pub identity: Option>, - /// Disables plain-text HTTP. - /// - /// Default: false. - pub required: Option, - /// Additional PEM-formatted root certificates to trust, on top of the platform's - /// trust store. Each entry may be a PEM bundle containing multiple certificates. - /// - /// This is mainly useful for connecting to servers with self-signed or private-CA - /// certificates, such as internal services or local test servers. This is one of the - /// few options that will cause the `Agent` constructor to throw if the input is in - /// the wrong format. - pub extra_roots: Option>>, -} - -impl Debug for AgentTlsOptions { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AgentTlsOptions") - .field("early_data", &self.early_data) - .field("identity", &"[sensitive]") - .field("required", &self.required) - .field("extra_roots", &self.extra_roots.as_ref().map(|r| r.len())) - .finish() - } -} - -impl Clone for AgentTlsOptions { - fn clone(&self) -> Self { - Self { - early_data: self.early_data.clone(), - identity: self.identity.as_ref().map(|either| match either { - Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), - Either::B(string) => Either::B(string.clone()), - }), - required: self.required.clone(), - extra_roots: self.extra_roots.as_ref().map(|roots| { - roots - .iter() - .map(|either| match either { - Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), - Either::B(string) => Either::B(string.clone()), - }) - .collect() - }), - } - } -} - -#[napi(object)] -#[derive(Debug, Clone, Default)] -pub struct AgentOptions { - /// Settings related to the HTTP cache. This is a nested object. - pub cache: Option, - /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and - /// included in additional requests. - /// - /// `true` enables the store with the default limits; an options object enables it and tunes them, - /// so `{}` means the same as `true`. - /// - /// Default: `false`. - /// - /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add - /// and retrieve cookies from the store. - pub cookies: Option>, - /// Settings related to DNS. This is a nested object. - pub dns: Option, - /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. - /// - /// Setting these is the normal way to tune windows: one value applies to whichever protocol - /// a request negotiates, so throughput doesn't change when an origin upgrades from one to - /// the other. The `http2` and `http3` groups override them per protocol. - pub flow_control: Option, - /// Sets the default headers for every request. - /// - /// If header names or values are invalid, they are silently omitted. - /// Sensitive headers (e.g. `Authorization`) should be marked. - /// - /// Default: none. - pub headers: Option>, - /// Settings related to HTTP/2. This is a nested object. - pub http2: Option, - /// Settings related to HTTP/3. This is a nested object. - pub http3: Option, - /// Bind outgoing sockets to this local IP address before connecting. - /// - /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that - /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — - /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and - /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to - /// force a specific source address. Throws if the value does not parse as an IP address. - /// - /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). - pub local_address: Option, - /// Settings related to the connection pool. This is a nested object. - pub pool: Option, - /// Switches that depart from standard behaviour on purpose. This is a nested object. - pub quirks: Option, - /// Determines the behavior in case the server replies with a redirect status. - pub redirect: Option, - /// Timeouts for requests made with this agent. This is a nested object. - pub timeout: Option, - /// Settings related to the connection pool. This is a nested object. - pub tls: Option, - /// Custom user agent string. - /// - /// Default: `Faith/{version} reqwest/{version}`. - pub user_agent: Option, -} - #[napi] #[derive(Debug, Clone, Default)] pub struct AgentStats { @@ -843,7 +103,7 @@ impl Agent { } pub fn with_options(options: AgentOptions) -> Result { - let options = options::AgentOptions::from(options); + let options = web_faith::options::AgentOptions::from(options); // A napi callback can run outside the runtime, and building the HTTP/3 endpoint needs to be // inside one, so the client is constructed within whichever runtime is to hand. within_runtime_if_available(|| web_faith::agent::Agent::from_options(options)) @@ -1032,133 +292,3 @@ impl Agent { fn caller_error(env: &Env, err: FaithError) -> napi::Error { napi::Error::from(err.into_js_error(env)) } - -/// Read the JavaScript options object into the shape the client validates. -/// -/// A field-for-field mapping wherever the two agree, which is most of them; what differs is where -/// JavaScript expresses a choice as a union or a string that Rust has a type for. -impl From for options::AgentOptions { - fn from(opts: AgentOptions) -> Self { - Self { - cache: opts.cache.map(|cache| options::CacheOptions { - store: cache.store.map(|store| match store { - CacheStore::Disk => options::CacheStore::Disk, - CacheStore::Memory => options::CacheStore::Memory, - }), - capacity: cache.capacity, - mode: cache.mode.map(CacheMode::from), - path: cache.path, - shared: cache.shared, - }), - // `false` and an absent value both mean no jar; `true` means one with default limits. - cookies: match opts.cookies { - None | Some(Either::A(false)) => None, - Some(Either::A(true)) => Some(CookieLimits::default()), - Some(Either::B(cookies)) => Some((&cookies).into()), - }, - dns: opts.dns.map(|dns| options::DnsOptions { - system: dns.system, - overrides: dns.overrides.map(|overrides| { - overrides - .into_iter() - .map(|o| options::DnsOverride { - domain: o.domain, - addresses: o.addresses, - }) - .collect() - }), - servers: dns.servers, - timeout: dns.timeout, - search_domains: dns.search_domains, - ndots: dns.ndots, - hosts_file: dns.hosts_file, - exempt_domains: dns.exempt_domains, - serve_stale: dns.serve_stale, - max_stale: dns.max_stale, - }), - flow_control: opts.flow_control.map(|flow| options::FlowControlOptions { - stream_window: flow.stream_window, - connection_window: flow.connection_window, - }), - headers: opts.headers.map(|headers| { - headers - .into_iter() - .map(|header| options::Header { - name: header.name, - value: header.value, - sensitive: header.sensitive, - }) - .collect() - }), - http2: opts.http2.map(|http2| options::Http2Options { - stream_window: http2.stream_window, - connection_window: http2.connection_window, - adaptive_window: http2.adaptive_window, - }), - http3: opts.http3.map(|http3| options::Http3Options { - congestion: http3.congestion.map(|c| match c { - Http3Congestion::Cubic => options::Http3Congestion::Cubic, - Http3Congestion::Bbr1 => options::Http3Congestion::Bbr1, - }), - max_idle_timeout: http3.max_idle_timeout, - upgrade_enabled: http3.upgrade_enabled, - upgrade_probe: http3.upgrade_probe, - upgrade_probe_timeout: http3.upgrade_probe_timeout, - upgrade_slow_factor: http3.upgrade_slow_factor, - upgrade_slow_ttl: http3.upgrade_slow_ttl, - upgrade_advertised_ttl: http3.upgrade_advertised_ttl, - upgrade_confirmed_ttl: http3.upgrade_confirmed_ttl, - upgrade_failed_ttl: http3.upgrade_failed_ttl, - upgrade_failed_max_ttl: http3.upgrade_failed_max_ttl, - upgrade_cancel_strikes: http3.upgrade_cancel_strikes, - upgrade_attempt_timeout: http3.upgrade_attempt_timeout, - upgrade_follow_advertised_port: http3.upgrade_follow_advertised_port, - upgrade_cache_capacity: http3.upgrade_cache_capacity, - hints: http3.hints.map(|hints| { - hints - .into_iter() - .map(|hint| options::Http3Hint { - host: hint.host, - port: hint.port, - }) - .collect() - }), - stream_window: http3.stream_window, - connection_window: http3.connection_window, - send_window: http3.send_window, - }), - local_address: opts.local_address, - pool: opts.pool.map(|pool| options::PoolOptions { - idle_timeout: pool.idle_timeout, - max_idle_per_host: pool.max_idle_per_host, - }), - quirks: opts.quirks.map(|quirks| options::QuirksOptions { - h1_request_streaming: quirks.h1_request_streaming, - }), - redirect: opts.redirect.map(RedirectPolicy::from), - timeout: opts.timeout.map(|timeout| options::TimeoutOptions { - connect: timeout.connect, - read: timeout.read, - total: timeout.total, - }), - tls: opts.tls.map(|tls| options::TlsOptions { - early_data: tls.early_data, - // Either spelling of a PEM is the same bytes to the client. - identity: tls.identity.map(|pem| pem_bytes(&pem)), - required: tls.required, - extra_roots: tls - .extra_roots - .map(|roots| roots.iter().map(pem_bytes).collect()), - }), - user_agent: opts.user_agent, - } - } -} - -/// PEM input arrives as a buffer or a string; both are just the bytes. -fn pem_bytes(pem: &Either) -> Vec { - match pem { - Either::A(buf) => buf.to_vec(), - Either::B(string) => string.as_bytes().to_vec(), - } -} diff --git a/crates/web-faith-napi/src/agent/convert.rs b/crates/web-faith-napi/src/agent/convert.rs new file mode 100644 index 0000000..59f2979 --- /dev/null +++ b/crates/web-faith-napi/src/agent/convert.rs @@ -0,0 +1,138 @@ +//! Reading a JavaScript `AgentOptions` into the options the client validates. + +use http_cache_reqwest::CacheMode; +use napi::{Either, bindgen_prelude::Buffer}; +use web_faith::{client::RedirectPolicy, options}; +use web_faith_cookies::CookieLimits; + +use crate::agent::{AgentOptions, CacheStore, Http3Congestion}; + +/// Read the JavaScript options object into the shape the client validates. +/// +/// A field-for-field mapping wherever the two agree, which is most of them; what differs is where +/// JavaScript expresses a choice as a union or a string that Rust has a type for. +impl From for options::AgentOptions { + fn from(opts: AgentOptions) -> Self { + Self { + cache: opts.cache.map(|cache| options::CacheOptions { + store: cache.store.map(|store| match store { + CacheStore::Disk => options::CacheStore::Disk, + CacheStore::Memory => options::CacheStore::Memory, + }), + capacity: cache.capacity, + mode: cache.mode.map(CacheMode::from), + path: cache.path, + shared: cache.shared, + }), + // `false` and an absent value both mean no jar; `true` means one with default limits. + cookies: match opts.cookies { + None | Some(Either::A(false)) => None, + Some(Either::A(true)) => Some(CookieLimits::default()), + Some(Either::B(cookies)) => Some((&cookies).into()), + }, + dns: opts.dns.map(|dns| options::DnsOptions { + system: dns.system, + overrides: dns.overrides.map(|overrides| { + overrides + .into_iter() + .map(|o| options::DnsOverride { + domain: o.domain, + addresses: o.addresses, + }) + .collect() + }), + servers: dns.servers, + timeout: dns.timeout, + search_domains: dns.search_domains, + ndots: dns.ndots, + hosts_file: dns.hosts_file, + exempt_domains: dns.exempt_domains, + serve_stale: dns.serve_stale, + max_stale: dns.max_stale, + }), + flow_control: opts.flow_control.map(|flow| options::FlowControlOptions { + stream_window: flow.stream_window, + connection_window: flow.connection_window, + }), + headers: opts.headers.map(|headers| { + headers + .into_iter() + .map(|header| options::Header { + name: header.name, + value: header.value, + sensitive: header.sensitive, + }) + .collect() + }), + http2: opts.http2.map(|http2| options::Http2Options { + stream_window: http2.stream_window, + connection_window: http2.connection_window, + adaptive_window: http2.adaptive_window, + }), + http3: opts.http3.map(|http3| options::Http3Options { + congestion: http3.congestion.map(|c| match c { + Http3Congestion::Cubic => options::Http3Congestion::Cubic, + Http3Congestion::Bbr1 => options::Http3Congestion::Bbr1, + }), + max_idle_timeout: http3.max_idle_timeout, + upgrade_enabled: http3.upgrade_enabled, + upgrade_probe: http3.upgrade_probe, + upgrade_probe_timeout: http3.upgrade_probe_timeout, + upgrade_slow_factor: http3.upgrade_slow_factor, + upgrade_slow_ttl: http3.upgrade_slow_ttl, + upgrade_advertised_ttl: http3.upgrade_advertised_ttl, + upgrade_confirmed_ttl: http3.upgrade_confirmed_ttl, + upgrade_failed_ttl: http3.upgrade_failed_ttl, + upgrade_failed_max_ttl: http3.upgrade_failed_max_ttl, + upgrade_cancel_strikes: http3.upgrade_cancel_strikes, + upgrade_attempt_timeout: http3.upgrade_attempt_timeout, + upgrade_follow_advertised_port: http3.upgrade_follow_advertised_port, + upgrade_cache_capacity: http3.upgrade_cache_capacity, + hints: http3.hints.map(|hints| { + hints + .into_iter() + .map(|hint| options::Http3Hint { + host: hint.host, + port: hint.port, + }) + .collect() + }), + stream_window: http3.stream_window, + connection_window: http3.connection_window, + send_window: http3.send_window, + }), + local_address: opts.local_address, + pool: opts.pool.map(|pool| options::PoolOptions { + idle_timeout: pool.idle_timeout, + max_idle_per_host: pool.max_idle_per_host, + }), + quirks: opts.quirks.map(|quirks| options::QuirksOptions { + h1_request_streaming: quirks.h1_request_streaming, + }), + redirect: opts.redirect.map(RedirectPolicy::from), + timeout: opts.timeout.map(|timeout| options::TimeoutOptions { + connect: timeout.connect, + read: timeout.read, + total: timeout.total, + }), + tls: opts.tls.map(|tls| options::TlsOptions { + early_data: tls.early_data, + // Either spelling of a PEM is the same bytes to the client. + identity: tls.identity.map(|pem| pem_bytes(&pem)), + required: tls.required, + extra_roots: tls + .extra_roots + .map(|roots| roots.iter().map(pem_bytes).collect()), + }), + user_agent: opts.user_agent, + } + } +} + +/// PEM input arrives as a buffer or a string; both are just the bytes. +fn pem_bytes(pem: &Either) -> Vec { + match pem { + Either::A(buf) => buf.to_vec(), + Either::B(string) => string.as_bytes().to_vec(), + } +} diff --git a/crates/web-faith-napi/src/agent/options.rs b/crates/web-faith-napi/src/agent/options.rs new file mode 100644 index 0000000..051cc22 --- /dev/null +++ b/crates/web-faith-napi/src/agent/options.rs @@ -0,0 +1,751 @@ +//! The `AgentOptions` object as JavaScript spells it, and the option groups under it. + +use std::{fmt::Debug, time::Duration}; + +use napi::{Either, bindgen_prelude::Buffer}; +use napi_derive::napi; + +use web_faith::client::RedirectPolicy; +use web_faith_cookies::{ + CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, +}; + +use crate::options::RequestCacheMode; + +#[napi(string_enum)] +#[derive(Debug, Clone, Copy)] +pub enum CacheStore { + #[napi(value = "disk")] + Disk, + + #[napi(value = "memory")] + Memory, +} + +/// Settings related to the HTTP cache. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentCacheOptions { + /// Which cache store to use: either `disk` or `memory`. + /// + /// Default: none (cache disabled). + pub store: Option, + /// If `cache.store: "memory"`, the maximum amount of items stored. + /// + /// Default: 10_000. + pub capacity: Option, + /// Default cache mode. This is the same as [`FetchOptions.cache`](#fetchoptionscache), and is used if + /// no cache mode is set on a request. + /// + /// Default: `"default"`. + pub mode: Option, + /// If `cache.store: "disk"`, then this is the path at which the cache data is. Must be writeable. + /// + /// Required if `cache.store: "disk"`. + pub path: Option, + /// If `true`, then the response is evaluated from a perspective of a shared cache (i.e. `private` is + /// not cacheable and `s-maxage` is respected). If `false`, then the response is evaluated from a + /// perspective of a single-user cache (i.e. `private` is cacheable and `s-maxage` is ignored). + /// `shared: true` is required for proxies and multi-user caches. + /// + /// Default: true. + pub shared: Option, +} + +/// Limits the cookie store enforces, from RFC 6265bis. Each is a cap; a caller who needs more room +/// raises the number. +/// +/// The `__Host-` and `__Secure-` name prefix rules are what those prefixes mean, so they always +/// apply and are not settable here: a cookie that shouldn't carry them is named without one. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentCookieOptions { + /// How far ahead of receipt a cookie may expire, in seconds. A cookie asking for longer, via + /// `Max-Age` or `Expires`, has its expiry reduced to this; a shorter one is left alone and a + /// session cookie stays a session cookie. + /// + /// Default: 34_560_000 (400 days). + pub max_age: Option, + /// The largest cookie stored, as the combined length of its name and value in bytes. A larger + /// cookie is not stored. + /// + /// Default: 4096. + pub max_size: Option, + /// How many cookies are kept for any one domain, which is a cookie's `Domain` attribute when it + /// has one and the host that set it otherwise. + /// + /// Default: 180. + pub max_per_host: Option, + /// How many cookies are kept across the whole store, bounding a server that spreads cookies + /// across subdomains to escape `maxPerHost`. + /// + /// Default: 3000. + pub max_total: Option, +} + +impl From<&AgentCookieOptions> for CookieLimits { + fn from(options: &AgentCookieOptions) -> Self { + Self { + max_age: options + .max_age + .map_or(DEFAULT_MAX_AGE, |secs| Duration::from_secs(secs.into())), + max_size: options.max_size.map_or(DEFAULT_MAX_SIZE, |n| n as usize), + max_per_host: options + .max_per_host + .map_or(DEFAULT_MAX_PER_HOST, |n| n as usize), + max_total: options.max_total.map_or(DEFAULT_MAX_TOTAL, |n| n as usize), + } + } +} + +#[napi(object)] +#[derive(Debug, Clone)] +pub struct DnsOverride { + pub domain: String, + pub addresses: Vec, +} + +/// Settings related to DNS. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentDnsOptions { + /// Use the system's DNS (via `getaddrinfo` or equivalent) rather than Faith's own DNS client (based on + /// [Hickory]). If you experience issues with DNS where Faith does not work but e.g. curl or native + /// fetch does, this should be your first port of call. + /// + /// Enabling this also disables Happy Eyeballs (for IPv6 / IPv4 best-effort resolution), the in-memory + /// DNS cache, and may lead to worse performance even discounting the cache. + /// + /// Default: false. + /// + /// [Hickory]: https://hickory-dns.org/ + pub system: Option, + /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. + /// + /// Will throw if addresses are in invalid formats. You may provide a port number as part of the + /// address, it will default to port 0 otherwise, which will select the conventional port for the + /// protocol in use (e.g. 80 for plaintext HTTP). If the URL passed to `fetch()` has an explicit port + /// number, that one will be used instead. Resolving a domain to an empty `addresses` array effectively + /// blocks that domain from this agent. + /// + /// Default: no overrides. + pub overrides: Option>, + /// An ordered list of resolver URLs, each URL's scheme selecting the transport Faith speaks to + /// that resolver: `udp://` and `tcp://` for conventional DNS on port 53, `tls://` for DNS over + /// TLS on port 853, `https://` for DNS over HTTPS on port 443, `quic://` for DNS over QUIC on + /// port 853, and `h3://` for DNS over HTTP/3 on port 443. A port in the URL overrides the + /// conventional one, and the HTTP transports use `/dns-query` when the URL supplies no path. + /// + /// The encrypted transports always authenticate the resolver. A URL fragment names the + /// certificate to expect (`tls://1.1.1.1#cloudflare-dns.com`); a hostname host authenticates + /// against the hostname; a bare-IP host authenticates against the address itself. + /// + /// Servers are queried in order, a later one reached only once those before it fail. Setting + /// this replaces the system's servers, so no discovery runs. Throws if a URL is unparseable or + /// its scheme is not one of the above, and combining it with `dns.system` throws. + /// + /// Default: system discovery. + pub servers: Option>, + /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead + /// servers costs a single timeout rather than one per server. + /// + /// Default: 5000. + pub timeout: Option, + /// Replace the system's search list, the domains appended to a name that is not fully + /// qualified. Independent of `dns.servers`. + /// + /// Default: the system's search list. + pub search_domains: Option>, + /// How many dots a name must contain before it is tried as given, ahead of the search list. + /// Independent of `dns.servers`. + /// + /// Default: the system's setting. + pub ndots: Option, + /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. + /// + /// Default: platform convention. + pub hosts_file: Option, + /// Further domains to exempt from the configured or encrypted resolver, for the internal + /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's + /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. + /// + /// Default: no extra exemptions. + pub exempt_domains: Option>, + /// Serve an expired cache entry immediately and refresh it in the background, rather than making + /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is + /// almost always still correct, and a connect failure against one that has moved re-resolves and + /// attempts the request again. + /// + /// Set `false` for an agent that must never connect to an address it knows to be out of date: an + /// expired entry is discarded and the lookup blocks on a fresh answer. + /// + /// Default: true. + pub serve_stale: Option, + /// How far past expiry an answer may still be served, in milliseconds. An entry older than this + /// is discarded rather than served: an answer stale enough stops being evidence about where the + /// host is, and a refresh still failing after that long is the case where the address most likely + /// did change. + /// + /// Default: 3600000 (one hour). + pub max_stale: Option, +} + +/// Sets the default headers for every request. +/// +/// If header names or values are invalid, they are silently omitted. +/// Sensitive headers (e.g. `Authorization`) should be marked. +/// +/// Default: none. +#[napi(object)] +#[derive(Debug, Clone)] +pub struct Header { + pub name: String, + pub value: String, + pub sensitive: Option, +} + +#[napi(string_enum)] +#[derive(Debug, Clone, Copy, Default)] +pub enum Http3Congestion { + #[napi(value = "cubic")] + #[default] + Cubic, + + #[napi(value = "bbr1")] + Bbr1, +} + +/// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc +/// cache so the first request to this host will attempt HTTP/3 immediately. +#[napi(object)] +#[derive(Debug, Clone)] +pub struct Http3Hint { + /// The hostname (e.g., "example.com"). + pub host: String, + /// The port number (e.g., 443). + pub port: u16, +} + +/// Settings related to HTTP/3. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentHttp3Options { + /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the + /// Linux stack. It's fair for all traffic, but not the most optimal, especially for networks with + /// a lot of available bandwidth, high latency, or a lot of packet loss. Cubic reacts to packet loss by + /// dropping the speed by 30%, and takes a long time to recover. BBR instead tries to maximise + /// bandwidth use and optimises for round-trip time, while ignoring packet loss. + /// + /// In some networks, BBR can lead to pathological degradation of overall network conditions, by + /// flooding the network by up to **100 times** more retransmissions. This is fixed in BBRv2 and BBRv3, + /// but Faith (or rather its underlying QUIC library quinn, [does not implement those yet][2]). + /// + /// [2]: https://github.com/quinn-rs/quinn/issues/1254 + /// + /// Default: `cubic`. Accepted values: `cubic`, `bbr1`. + pub congestion: Option, + /// Maximum duration of inactivity to accept before timing out the connection, in seconds. Note that + /// this only sets the timeout on this side of the connection: the true idle timeout is the _minimum_ + /// of this and the peer's own max idle timeout. While the underlying library has no limits, Faith + /// defines bounds for safety: minimum 1 second, maximum 2 minutes (120 seconds). + /// + /// Default: 30. + pub max_idle_timeout: Option, + /// Whether HTTP/3 upgrade via Alt-Svc is enabled. When enabled, the agent will track Alt-Svc + /// headers from responses and automatically upgrade subsequent requests to HTTP/3 when available. + /// + /// Default: true. + pub upgrade_enabled: Option, + /// Whether advertised HTTP/3 endpoints are verified with a background probe + /// before any foreground request is routed to them. + /// + /// An `Alt-Svc` advertisement says the server listens on UDP; it cannot say + /// there is UDP connectivity between you and it. Without probing, the next + /// request after an advertisement attempts HTTP/3 inline, and on a silently + /// broken UDP path it stalls until the QUIC idle timeout or + /// `upgradeAttemptTimeout` before falling back to TCP — recurring once per + /// failure cooldown for as long as the path stays broken. + /// + /// With probing (the default), requests keep using TCP until a background + /// `HEAD /` over HTTP/3 has confirmed the path. The probe shares the + /// connection pool, so the first upgraded request rides the probe's warm + /// connection. A broken path costs one failed background request per + /// cooldown and no foreground latency at all. + /// + /// The probe is a synthetic request the server will see in its logs. Set + /// this to `false` to restore the inline upgrade if that is unacceptable + /// (per-request billing, easily-alarmed WAFs). + /// + /// `hints` are exempt either way: a hint is your own assertion, so the first + /// request to a hinted origin speaks HTTP/3 immediately, which is also what + /// makes h3-only origins (no TCP listener) work. + /// + /// Default: true. + pub upgrade_probe: Option, + /// Ceiling on how long a background HTTP/3 probe may take before the origin + /// is treated as failed, in **milliseconds**. + /// + /// This bounds background work only — no foreground request ever waits on a + /// probe — so it can afford to be generous: a healthy handshake plus HEAD + /// completes in one or two round trips. Set to 0 to leave probes bounded + /// only by the QUIC idle timeout. + /// + /// Default: 5000 (5 seconds). + pub upgrade_probe_timeout: Option, + /// Demote an origin off HTTP/3 when its QUIC path is provenly slower than + /// its TCP path by this factor. Set to 0 to disable path-time demotion. + /// + /// Faith keeps a per-origin moving average of time-to-response-headers for + /// each protocol family. HTTP/3 is preferred at parity and when moderately + /// slower — its advantages (no head-of-line blocking, connection migration) + /// pay off beyond the average — so this factor should stay well above 1. + /// Only a sustained gap acts: at least 8 samples on each side, and the QUIC + /// average must also exceed the TCP one by an absolute 10ms so LAN-fast + /// origins don't flap on noise. + /// + /// A demoted origin is not treated as broken: it re-enters through a + /// background probe after `upgradeSlowTtl`, asking whether the path has + /// improved at zero foreground cost. + /// + /// Default: 2.5. + pub upgrade_slow_factor: Option, + /// How long (in seconds) a path-time demotion holds before the origin is + /// re-evaluated. See `upgradeSlowFactor`. + /// + /// Default: 600 (10 minutes). + pub upgrade_slow_ttl: Option, + /// How long (in seconds) to cache an Alt-Svc advertisement before the first HTTP/3 attempt. + /// This is overridden by the `ma` (max-age) parameter in the Alt-Svc header if present. + /// + /// Default: 86400 (24 hours). + pub upgrade_advertised_ttl: Option, + /// How long (in seconds) to cache a confirmed working HTTP/3 connection. + /// + /// Default: 86400 (24 hours). + pub upgrade_confirmed_ttl: Option, + /// How long (in seconds) a *first* failed HTTP/3 attempt blocks an origin. During this + /// time, no HTTP/3 upgrades will be attempted for the origin, even if the server sends + /// Alt-Svc headers. + /// + /// Each consecutive failure doubles the cooldown, up to `upgradeFailedMaxTtl`, so an + /// origin whose UDP path is blocked for good is retried less and less often instead of + /// forever at this interval. A confirmed HTTP/3 response ends the run. + /// + /// Default: 300 (5 minutes). + pub upgrade_failed_ttl: Option, + /// Ceiling (in seconds) on the cooldown that consecutive HTTP/3 failures double out of + /// `upgradeFailedTtl`. + /// + /// On the defaults an origin that keeps failing is blocked for 5 minutes, then 10, 20, + /// 40, and an hour thereafter. Set this at or below `upgradeFailedTtl` for a flat + /// cooldown that never backs off. + /// + /// Default: 3600 (1 hour). + pub upgrade_failed_max_ttl: Option, + /// How many consecutive cancelled HTTP/3 attempts, within a 60-second window, + /// demote an origin back to TCP. + /// + /// Faith normally learns that HTTP/3 is broken from a failed attempt. A request + /// cancelled via `AbortSignal` never produces that signal, so without this an + /// origin whose UDP path breaks keeps being retried over HTTP/3 for as long as + /// the Alt-Svc entry lives. Cancellations are treated as weak evidence: only a + /// sustained run of them demotes the origin, and any successful HTTP/3 response + /// resets the count. + /// + /// Strikes must land within about a minute of each other to count towards a + /// run. A retry loop whose backoff exceeds that window never accumulates one, + /// so callers with a long backoff should set this to 1 for immediate demotion + /// on the first cancelled attempt. + /// + /// One fault neither this nor `upgradeAttemptTimeout` catches: a path that + /// carries small datagrams but drops full-size ones (an MTU blackhole, say). + /// Response headers still arrive, so the attempt resolves and every mechanism + /// here counts it a success — the transfer then stalls partway through the + /// body, where nothing is watching. `maxIdleTimeout` or the request's own + /// timeout is what ends such a request, and the origin stays on HTTP/3. + /// + /// Set to 0 to disable, so only real HTTP/3 errors demote an origin. + /// + /// Default: 3. + pub upgrade_cancel_strikes: Option, + /// Ceiling on how long an HTTP/3 attempt may take to resolve before it is + /// given up on and the request is retried over TCP, in **milliseconds**. + /// + /// Note the unit: the other `upgrade*` settings are in seconds, but this one + /// is in milliseconds to match the `timeout` settings, because useful values + /// are sub-second. + /// + /// This bounds the wait for response headers, not the response body, so a slow + /// body is unaffected. + /// + /// The default is high, but not unconditionally inert: `maxIdleTimeout` is + /// configurable up to 120 seconds, and above 60 seconds this deadline becomes + /// the effective ceiling. Even below that, "QUIC's own idle timeout fires + /// first" only holds while the connection is idle — a transfer still running + /// past this deadline keeps the connection active, so no idle timeout is + /// coming to end it. + /// + /// On expiry the request is retried over TCP, which means it is re-sent: a + /// timeout often means the server is still processing, so a slow + /// non-idempotent request (a POST, say) can end up delivered twice. Lowering + /// this value trades that double-submission risk for faster recovery when a + /// UDP path breaks. Anyone setting it low should confirm their slowest + /// legitimate time to response headers fits well inside the budget. + /// + /// Set to 0 to disable, so an HTTP/3 attempt is bounded only by the QUIC idle + /// timeout and the request's own timeout. + /// + /// Default: 60000 (60 seconds). + pub upgrade_attempt_timeout: Option, + /// Connect to the port a server advertises HTTP/3 on, even when it differs from + /// the origin's own port. **This is not standards-compliant**; it is off by + /// default. + /// + /// An `Alt-Svc` advertisement names a network endpoint for the origin, so + /// honouring one correctly means connecting to that endpoint while still + /// sending the *origin's* authority. reqwest cannot express that — it derives + /// the HTTP/3 connect target from the request URI's authority (tracked + /// upstream as [reqwest#1138](https://github.com/seanmonstar/reqwest/issues/1138)). + /// So by default Faith does not upgrade at all when the advertised port + /// differs, rather than guessing that the origin's own port also speaks + /// HTTP/3. + /// + /// Setting this to `true` upgrades anyway, by rewriting the request's port to + /// the advertised one. That gets HTTP/3 working today against servers you + /// control, at the cost of three deviations you should be aware of: + /// + /// - The request's `Host`/`:authority` carries the advertised port instead of + /// the origin's, which [RFC 7838](https://www.rfc-editor.org/rfc/rfc7838) + /// forbids. Servers that route on authority may misroute or reject; servers + /// that ignore it are unaffected. + /// - `response.url` reports the port actually connected to. + /// - `redirected` ignores port differences, since the rewritten port would + /// otherwise look like a redirect on every request. + /// + /// TLS is unaffected: certificates are still validated against the origin's + /// hostname. Only the port changes. + /// + /// Default: `false`. + pub upgrade_follow_advertised_port: Option, + /// Maximum number of origins to track in the Alt-Svc cache. + /// + /// Default: 10000. + pub upgrade_cache_capacity: Option, + /// Hints for hosts that are known to support HTTP/3. These are added to the Alt-Svc cache + /// on agent initialization, so the first request to these hosts will attempt HTTP/3. + pub hints: Option>, + /// Maximum bytes an origin may send on any one HTTP/3 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/3 only. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/3 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/3 only. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Maximum bytes Faith transmits to an origin without acknowledgement, bounding upload + /// throughput the way the receive windows bound download. The origin's own flow control + /// applies on top of this, so it is a ceiling rather than a grant. + /// + /// This has no HTTP/2 counterpart: HTTP/2's send side is governed entirely by the window + /// the peer advertises, with no local cap to set. + /// + /// Default: 10 MB (quinn's own default). + pub send_window: Option, +} + +/// Settings related to HTTP/2. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentHttp2Options { + /// Maximum bytes an origin may send on any one HTTP/2 stream before it must wait for + /// Faith to acknowledge them. Overrides `flowControl.streamWindow` for HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.streamWindow`, itself 6 MiB by default). + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one HTTP/2 connection before it + /// must wait for Faith to acknowledge them. Overrides `flowControl.connectionWindow` for + /// HTTP/2 only. + /// + /// Ignored when `adaptiveWindow` is on. + /// + /// Default: unset (`flowControl.connectionWindow`, itself 15 MiB by default). + pub connection_window: Option, + /// Replace HTTP/2's static windows with windows that start small and grow towards a + /// bandwidth-delay estimate sampled from connection pings, capped at 16 MiB. + /// + /// This is off by default, and turning it on is usually the wrong move. A fresh connection + /// opens at 64 KiB, 96 times below the static default, and doubles only when a ping sample + /// reaches two thirds of the current estimate — so it takes many round trips to ramp up and + /// carries *less* throughput than the static window for all but the largest transfers. It + /// also takes over both windows, so `streamWindow` and `connectionWindow` stop applying. + /// + /// Its one real advantage is memory: it holds a large window open only on connections that + /// demonstrably need one. Since it caps at 16 MiB anyway, a static window near that ceiling + /// buys the same throughput from the first byte. + /// + /// HTTP/3 is unaffected either way, and keeps whichever windows apply to it. + /// + /// Default: `false`. + pub adaptive_window: Option, +} + +/// Settings related to HTTP flow control, shared by HTTP/2 and HTTP/3. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentFlowControlOptions { + /// Maximum bytes an origin may send on any one stream before it must wait for Faith to + /// acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// Larger windows keep a high-latency link full, at the cost of buffering more per stream. + /// The default follows browser practice, and is deliberately at the conservative end of it: + /// a pooled server-side client can hold many connections across many origins, so + /// per-connection memory multiplies harder here than in a browser. + /// + /// Set `http2.streamWindow` or `http3.streamWindow` to tune one protocol against the other. + /// + /// Default: 6 MiB. + pub stream_window: Option, + /// Maximum bytes an origin may send across all streams of one connection before it must + /// wait for Faith to acknowledge them, for HTTP/2 and HTTP/3 alike. + /// + /// This is larger than `streamWindow` so concurrent streams on one connection share the + /// connection's headroom, while still bounding the worst-case buffering of a connection + /// carrying many concurrent requests. + /// + /// Set `http2.connectionWindow` or `http3.connectionWindow` to tune one protocol against + /// the other. + /// + /// Default: 15 MiB. + pub connection_window: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentPoolOptions { + /// How many seconds of inactivity before a connection is closed. + /// + /// Default: 90 seconds. + pub idle_timeout: Option, + /// The maximum amount of idle connections per host to allow in the pool. Connections will be closed + /// to keep the idle connections (per host) under that number. + /// + /// Default: `null` (no limit). + pub max_idle_per_host: Option, +} + +/// Switches that depart from standard behaviour on purpose. This is a nested object. +/// +/// Each quirk turns off a rule Faith otherwise upholds, in exchange for a capability the rule +/// forbids. All of them are off by default, so an agent constructed with no options is +/// standards-compliant. A quirk is for a caller who controls the origin, or has otherwise +/// established that what the rule guards against does not apply to them: turning one on means +/// requests may fail against origins that expect the standard behaviour. +#[napi(object)] +#[derive(Debug, Clone, Copy, Default)] +pub struct AgentQuirksOptions { + /// Allow a streaming request body to be sent over an HTTP/1.x connection. + /// + /// The fetch standard reserves streaming request bodies for HTTP/2 and HTTP/3: a body read + /// from a `ReadableStream` has no known length when the headers go out, and an HTTP/1.x + /// origin or an intermediary on the path may refuse it. With this on, such a body sends over + /// whichever protocol the connection negotiates. + /// + /// Default: false. + pub h1_request_streaming: Option, +} + +/// Determines the behavior in case the server replies with a redirect status. +/// One of the following values: +/// +/// - `follow`: automatically follow redirects. Faith limits this to 10 redirects. +/// - `error`: reject the promise with a network error when a redirect status is returned. +/// - ~~`manual`~~: not supported. +/// - `stop`: (Faith custom) don't follow any redirects, return the responses. +/// +/// Defaults to `follow`. +#[napi(string_enum)] +#[derive(Debug, Clone, Copy, Default)] +pub enum Redirect { + #[napi(value = "follow")] + #[default] + Follow, + + #[napi(value = "error")] + Error, + + #[napi(value = "manual")] + Manual, + + #[napi(value = "stop")] + Stop, +} + +/// `manual` is not supported and behaves as `follow`, which is what the client's own policy spells +/// out: it carries no variant for a choice that never differed. +impl From for RedirectPolicy { + fn from(redirect: Redirect) -> Self { + match redirect { + Redirect::Follow | Redirect::Manual => Self::Follow, + Redirect::Error => Self::Error, + Redirect::Stop => Self::Stop, + } + } +} + +/// Timeouts for requests made with this agent. This is a nested object. +#[napi(object)] +#[derive(Debug, Clone, Copy, Default)] +pub struct AgentTimeoutOptions { + /// Set a timeout for only the connect phase, in milliseconds. + /// + /// Default: none. + pub connect: Option, + /// Set a timeout for read operations, in milliseconds. + /// + /// The timeout applies to each read operation, and resets after a successful read. This is more + /// appropriate for detecting stalled connections when the size isn't known beforehand. + /// + /// Default: none. + pub read: Option, + /// Set a timeout for the entire request-response cycle, in milliseconds. + /// + /// The timeout applies from when the request starts connecting until the response body has finished. + /// Also considered a total deadline. + /// + /// Default: none. + pub total: Option, +} + +/// Settings related to the connection pool. This is a nested object. +#[napi(object)] +#[derive(Default)] +pub struct AgentTlsOptions { + /// Enable TLS 1.3 Early Data. Early data is an optimisation where the client sends the first packet + /// of application data alongside the opening packet of the TLS handshake. That can enable the server + /// to answer faster, improving latency by up to one round-trip. However, Early Data has significant + /// security implications: it's vulnerable to replay attacks and has weaker forward secrecy. It should + /// really only be used for static assets or to squeeze out the last drop of performance for endpoints + /// that are replay-safe. + /// + /// Default: false. + pub early_data: Option, + /// Provide a PEM-formatted certificate and private key to present as a TLS client certificate (also + /// called mutual TLS or mTLS) authentication. + /// + /// The input should contain a PEM encoded private key and at least one PEM encoded certificate. The + /// private key must be in RSA, SEC1 Elliptic Curve or PKCS#8 format. This is one of the few options + /// that will cause the `Agent` constructor to throw if the input is in the wrong format. + pub identity: Option>, + /// Disables plain-text HTTP. + /// + /// Default: false. + pub required: Option, + /// Additional PEM-formatted root certificates to trust, on top of the platform's + /// trust store. Each entry may be a PEM bundle containing multiple certificates. + /// + /// This is mainly useful for connecting to servers with self-signed or private-CA + /// certificates, such as internal services or local test servers. This is one of the + /// few options that will cause the `Agent` constructor to throw if the input is in + /// the wrong format. + pub extra_roots: Option>>, +} + +impl Debug for AgentTlsOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AgentTlsOptions") + .field("early_data", &self.early_data) + .field("identity", &"[sensitive]") + .field("required", &self.required) + .field("extra_roots", &self.extra_roots.as_ref().map(|r| r.len())) + .finish() + } +} + +impl Clone for AgentTlsOptions { + fn clone(&self) -> Self { + Self { + early_data: self.early_data.clone(), + identity: self.identity.as_ref().map(|either| match either { + Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), + Either::B(string) => Either::B(string.clone()), + }), + required: self.required.clone(), + extra_roots: self.extra_roots.as_ref().map(|roots| { + roots + .iter() + .map(|either| match either { + Either::A(buf) => Either::A(Buffer::from(buf.as_ref())), + Either::B(string) => Either::B(string.clone()), + }) + .collect() + }), + } + } +} + +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct AgentOptions { + /// Settings related to the HTTP cache. This is a nested object. + pub cache: Option, + /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and + /// included in additional requests. + /// + /// `true` enables the store with the default limits; an options object enables it and tunes them, + /// so `{}` means the same as `true`. + /// + /// Default: `false`. + /// + /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add + /// and retrieve cookies from the store. + pub cookies: Option>, + /// Settings related to DNS. This is a nested object. + pub dns: Option, + /// Flow-control windows shared by HTTP/2 and HTTP/3. This is a nested object. + /// + /// Setting these is the normal way to tune windows: one value applies to whichever protocol + /// a request negotiates, so throughput doesn't change when an origin upgrades from one to + /// the other. The `http2` and `http3` groups override them per protocol. + pub flow_control: Option, + /// Sets the default headers for every request. + /// + /// If header names or values are invalid, they are silently omitted. + /// Sensitive headers (e.g. `Authorization`) should be marked. + /// + /// Default: none. + pub headers: Option>, + /// Settings related to HTTP/2. This is a nested object. + pub http2: Option, + /// Settings related to HTTP/3. This is a nested object. + pub http3: Option, + /// Bind outgoing sockets to this local IP address before connecting. + /// + /// This also selects the address family of the HTTP/3 (QUIC) socket. By default that + /// socket binds the IPv6 wildcard (`[::]`), which fails on hosts without usable IPv6 — + /// there, HTTP/3 silently falls back to TCP. Faith detects that case automatically and + /// binds `0.0.0.0` instead, so you normally don't need to set this; provide it only to + /// force a specific source address. Throws if the value does not parse as an IP address. + /// + /// Default: unset (IPv6 wildcard for QUIC where available, else `0.0.0.0`). + pub local_address: Option, + /// Settings related to the connection pool. This is a nested object. + pub pool: Option, + /// Switches that depart from standard behaviour on purpose. This is a nested object. + pub quirks: Option, + /// Determines the behavior in case the server replies with a redirect status. + pub redirect: Option, + /// Timeouts for requests made with this agent. This is a nested object. + pub timeout: Option, + /// Settings related to the connection pool. This is a nested object. + pub tls: Option, + /// Custom user agent string. + /// + /// Default: `Faith/{version} reqwest/{version}`. + pub user_agent: Option, +} diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index f4d53ce..96224dd 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -2,51 +2,37 @@ // spec:AGENT spec:WARM spec:NETCHG spec:OBS -use std::{ - future::Future, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, - str::FromStr, - sync::{ - Arc, RwLock, - atomic::{AtomicU64, Ordering}, - }, - time::Duration, +use std::sync::{ + Arc, RwLock, + atomic::{AtomicU64, Ordering}, }; -use http::header::{HeaderMap, HeaderName, HeaderValue}; -use http_cache_reqwest::{ - CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, -}; +use http::header::HeaderValue; use moka::sync::Cache as MokaCache; -use reqwest::{Client, Identity, Version, tls::Certificate}; +use reqwest::Client; use reqwest_middleware::ClientWithMiddleware; use url::Url; use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; use web_faith_cookies::FaithJar; -use web_faith_dns::{ - DEFAULT_MAX_STALE, FaithResolver, ResolverReport, ResolverSettings, ServerSpec, parse_domains, -}; +use web_faith_dns::{FaithResolver, ResolverReport}; #[cfg(feature = "http3")] -use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, H3Prober}; +use web_faith_alt_svc::{AltSvcCache, H3Prober}; use crate::{ - USER_AGENT, - client::{ClientRecipe, HttpCacheRecipe, HttpCacheStore, NodeEnvRecipe}, - error::{FaithError, FaithErrorKind}, - options::{ - AgentOptions, CacheStore, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows, - }, - request::PRIORITY, + client::ClientRecipe, stats::{AgentStats, InnerAgentStats}, - warm_up::{extract_host, origin_key, reduce_to_origin}, + warm_up::origin_key, }; #[cfg(feature = "http3")] -use crate::{ - client::{H3UpgradeRecipe, install_https_sink}, - options::Http3Congestion, -}; +use crate::client::install_https_sink; + +mod build; +mod warm; + +#[cfg(test)] +mod tests; /// The agent settings a request consults, as opposed to those a client is built from. #[derive(Debug, Clone, Default)] @@ -212,529 +198,6 @@ impl Agent { /// Build an agent from options, validating them into the recipe its clients are built from. /// - /// This is what both surfaces land on, so the defaults a caller gets are settled here rather - /// than once per surface. - // spec:AGENT spec:NETCHG - pub fn from_options(options: AgentOptions) -> Result { - // Destructured rather than read field by field so that a new option cannot be added - // without the compiler pointing here, where every option is turned into the recipe the - // agent's clients are built from (spec:NETCHG). - let AgentOptions { - cache, - cookies, - dns, - flow_control, - headers, - http2, - // Every use of the HTTP/3 options sits behind the feature. - #[cfg_attr(not(feature = "http3"), allow(unused_variables))] - http3, - local_address, - pool, - quirks, - redirect, - timeout, - tls, - user_agent, - } = options; - - let quirk_h1_request_streaming = quirks - .and_then(|quirks| quirks.h1_request_streaming) - .unwrap_or(false); - - // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts - // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the - // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and - // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such - // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. - let local_address = match &local_address { - Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { - FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - ) - })?), - None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), - None => None, - }; - - // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) - // The jar is installed on the client by the recipe, so it survives a rebuild - // (spec:NETCHG#what-the-signal-keeps). - let cookie_jar = cookies.map(|limits| Arc::new(FaithJar::new(limits))); - - let dns = dns.unwrap_or_default(); - let dns_system = dns.system.unwrap_or(false); - // Naming servers and asking for the system resolver at once is a contradiction rather than - // a preference, since the system resolver is not Faith's to point at listed servers - // (spec:DNS#system-resolver). - if dns_system - && dns - .servers - .as_ref() - .is_some_and(|servers| !servers.is_empty()) - { - return Err(FaithError::new( - FaithErrorKind::Config, - Some("dns.servers cannot be combined with dns.system".to_string()), - )); - } - // Parsed whichever resolver is in use: overrides take effect under the system resolver - // too (spec:DNS#overrides), and an unparseable address is a construction error either - // way (spec:AGENT#construction). - let dns_overrides = dns - .overrides - .unwrap_or_default() - .into_iter() - .map(|DnsOverride { domain, addresses }| { - let addresses = addresses - .into_iter() - .map(|addr| match SocketAddr::from_str(&addr) { - Ok(addr) => Ok(addr), - Err(err) => match IpAddr::from_str(&addr) { - Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), - Ok(IpAddr::V6(ip)) => { - Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) - } - Err(_) => Err(FaithError::new( - FaithErrorKind::AddressParse, - Some(format!("{addr:?}: {err}")), - )), - }, - }) - .collect::, FaithError>>()?; - Ok((domain, addresses)) - }) - .collect::, FaithError>>()?; - - // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so - // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), - // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and - // order each resolver is reached by (spec:DNS#transports). The system resolver - // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there - // and `prefetchDns` resolves as a no-op (spec:WARM). - let dns_resolver = if dns_system { - None - } else { - // These settings configure Faith's own resolver only, so they are read on the path that - // builds one rather than validated under the system resolver that ignores them. - let mut servers = Vec::new(); - for url in dns.servers.unwrap_or_default() { - servers.push(ServerSpec::parse(&url).map_err(|message| { - FaithError::new(FaithErrorKind::AddressParse, Some(message)) - })?); - } - Some(FaithResolver::new(ResolverSettings { - servers, - timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), - ndots: dns.ndots.map(|n| n as usize), - search_domains: parse_domains(dns.search_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, - hosts_file: dns.hosts_file, - exempt_domains: parse_domains(dns.exempt_domains) - .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? - .unwrap_or_default(), - serve_stale: dns.serve_stale.unwrap_or(true), - max_stale: dns - .max_stale - .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), - })) - }; - - let mut default_accept_encoding = None; - let mut default_content_encoding = None; - let mut has_default_priority = false; - let mut default_headers = None; - if let Some(headers) = headers - && !headers.is_empty() - { - let map = HeaderMap::from_iter(headers.into_iter().filter_map( - |Header { - name, - value, - sensitive, - }| { - let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { - return None; - }; - - let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { - return None; - }; - - if sensitive.unwrap_or(false) { - value.set_sensitive(true); - } - - Some((name, value)) - }, - )); - default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); - default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); - has_default_priority = map.contains_key(PRIORITY); - default_headers = Some(map); - } - - // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so - // the explicit sizes are not resolved at all when it's on: reqwest would let the later - // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence - // visible here rather than depending on hyper's internal ordering. - let http2 = http2.unwrap_or_default(); - let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); - let http2_windows = (!http2_adaptive_window).then(|| { - resolve_windows( - flow_control.as_ref(), - http2.stream_window, - http2.connection_window, - ) - }); - - #[cfg(feature = "http3")] - let http3_max_idle_timeout = Duration::from_secs( - http3 - .as_ref() - .and_then(|h| h.max_idle_timeout) - .unwrap_or(30) - .clamp(1, 120) - .into(), - ); - - // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window - // inside an unbounded connection window: the stream window is the binding constraint - // on a high-latency link, and the unbounded connection window means a connection with - // many concurrent requests has no ceiling on what it buffers. Both are set here. - #[cfg(feature = "http3")] - let http3_windows = resolve_windows( - flow_control.as_ref(), - http3.as_ref().and_then(|h| h.stream_window), - http3.as_ref().and_then(|h| h.connection_window), - ); - - #[cfg(feature = "http3")] - let http3_congestion_bbr = matches!( - http3.as_ref().and_then(|h| h.congestion), - Some(Http3Congestion::Bbr1) - ); - - #[cfg(feature = "http3")] - let http3_send_window = http3.as_ref().and_then(|h| h.send_window); - - let pool_idle_timeout = pool - .as_ref() - .and_then(|pool| pool.idle_timeout) - .map(|seconds| Duration::from_secs(seconds.into())); - // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). - let pool_max_idle_per_host = pool.as_ref().map(|pool| { - pool.max_idle_per_host - .and_then(|n| n.try_into().ok()) - .unwrap_or(usize::MAX) - }); - - let connect_timeout = timeout - .and_then(|t| t.connect) - .map(|millis| Duration::from_millis(millis.into())); - let read_timeout = timeout - .and_then(|t| t.read) - .map(|millis| Duration::from_millis(millis.into())); - let total_timeout = timeout - .and_then(|t| t.total) - .map(|millis| Duration::from_millis(millis.into())); - - #[cfg(feature = "http3")] - let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); - let tls_required = tls.as_ref().and_then(|tls| tls.required); - // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a - // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). - let (tls_identity, tls_extra_roots) = match tls { - None => (None, Vec::new()), - Some(tls) => { - let identity = match &tls.identity { - None => None, - Some(identity) => Some(Identity::from_pem(identity).map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?), - }; - - let mut extra_roots = Vec::new(); - for pem in tls.extra_roots.iter().flatten() { - extra_roots.extend(Certificate::from_pem_bundle(pem).map_err(|err| { - FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) - })?); - } - - (identity, extra_roots) - } - }; - - let http_cache = if let Some(cache) = cache - && let Some(store) = cache.store - { - let mode = cache.mode.unwrap_or_default().into(); - let options = HttpCacheOptions { - cache_options: Some(CacheOptions { - shared: cache.shared.unwrap_or(true), - ignore_cargo_cult: true, - ..Default::default() - }), - ..Default::default() - }; - let store = match store { - CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { - path: cache - .path - .ok_or_else(|| { - FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) - })? - .into(), - remove_opts: Default::default(), - }), - CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( - MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), - )), - }; - - Some(HttpCacheRecipe { - mode, - options, - store, - }) - } else { - None - }; - - // Read outside the `alt_svc_cache` block below because `fetch` needs it too, - // to keep a rewritten port from looking like a redirect. - #[cfg(feature = "http3")] - let h3_follow_advertised_port = http3 - .as_ref() - .and_then(|o| o.upgrade_follow_advertised_port) - .unwrap_or(false); - #[cfg(not(feature = "http3"))] - let h3_follow_advertised_port = false; - - // The origin knowledge is built once and outlives every client the agent builds: it is - // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). - #[cfg(feature = "http3")] - let (alt_svc_cache, h3_upgrade) = { - let http3_opts = http3.as_ref(); - let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); - - let advertised_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_advertised_ttl) - .unwrap_or(86400) - .into(), - ); - let confirmed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_confirmed_ttl) - .unwrap_or(86400) - .into(), - ); - let failed_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_ttl) - .unwrap_or(300) - .into(), - ); - let failed_max_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_failed_max_ttl) - .unwrap_or(3600) - .into(), - ); - let capacity = http3_opts - .and_then(|o| o.upgrade_cache_capacity) - .unwrap_or(10_000) - .into(); - let cancel_strikes = http3_opts - .and_then(|o| o.upgrade_cancel_strikes) - .unwrap_or(3); - let attempt_timeout = match http3_opts - .and_then(|o| o.upgrade_attempt_timeout) - .unwrap_or(60_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); - let probe_timeout = match http3_opts - .and_then(|o| o.upgrade_probe_timeout) - .unwrap_or(5_000) - { - 0 => None, - millis => Some(Duration::from_millis(millis.into())), - }; - let slow_factor = http3_opts - .and_then(|o| o.upgrade_slow_factor) - .unwrap_or(2.5); - let slow_ttl = Duration::from_secs( - http3_opts - .and_then(|o| o.upgrade_slow_ttl) - .unwrap_or(600) - .into(), - ); - - let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { - advertised_ttl, - confirmed_ttl, - failed_ttl, - failed_max_ttl, - capacity, - cancel_strikes, - strike_window: Duration::from_secs(60), - follow_advertised_port: h3_follow_advertised_port, - // The single-flight claim must outlive the probe it covers, so - // an aborted probe frees its origin without a report; without a - // probe deadline, the QUIC idle timeout (max 120s) is the bound. - probe_ttl: probe_timeout - .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), - slow_factor, - slow_ttl, - })); - - if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { - for hint in hints { - cache.add_hint(&hint.host, hint.port); - } - } - - ( - Some(cache), - H3UpgradeRecipe { - enabled, - attempt_timeout, - probe, - probe_timeout, - }, - ) - }; - - let recipe = ClientRecipe { - user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), - local_address, - default_headers, - dns_system, - dns_overrides, - http2_adaptive_window, - http2_windows, - #[cfg(feature = "http3")] - http3_max_idle_timeout, - #[cfg(feature = "http3")] - http3_windows, - #[cfg(feature = "http3")] - http3_congestion_bbr, - #[cfg(feature = "http3")] - http3_send_window, - pool_idle_timeout, - pool_max_idle_per_host, - redirect, - connect_timeout, - read_timeout, - total_timeout, - #[cfg(feature = "http3")] - tls_early_data, - tls_identity, - tls_required, - tls_extra_roots, - node_env: NodeEnvRecipe::read(), - http_cache, - #[cfg(feature = "http3")] - h3_upgrade, - }; - - let settings = AgentSettings { - h3_follow_advertised_port, - #[cfg(feature = "http3")] - h3_upgrade_enabled: recipe.h3_upgrade.enabled, - quirk_h1_request_streaming, - default_accept_encoding, - default_content_encoding, - has_default_priority, - }; - - Self::build( - recipe, - settings, - cookie_jar, - dns_resolver, - #[cfg(feature = "http3")] - alt_svc_cache, - ) - } - - /// An agent with default options. - pub fn new() -> Result { - Self::from_options(AgentOptions::default()) - } - - /// Build an agent a setting at a time. See [`AgentBuilder`](crate::builder::AgentBuilder). - pub fn builder() -> crate::builder::AgentBuilder { - crate::builder::AgentBuilder::default() - } - - /// Build an agent from a validated recipe. - /// - /// The recipe is what a client is built from, and the settings are what each request consults; - /// validating whatever a caller expressed them as belongs to the surface that took it. - pub fn build( - recipe: ClientRecipe, - settings: AgentSettings, - cookie_jar: Option>, - dns_resolver: Option, - #[cfg(feature = "http3")] alt_svc_cache: Option>, - ) -> Result { - let conn_timeout = recipe.conn_timeout(); - let built = recipe.build( - cookie_jar.as_ref(), - dns_resolver.as_ref(), - #[cfg(feature = "http3")] - alt_svc_cache.as_ref(), - )?; - - // Only now do all three exist: the resolver is built before the cache, and the prober - // holds a client that holds the resolver, so this is the earliest the loop can be closed - // (spec:DNS#https-records). - #[cfg(feature = "http3")] - install_https_sink( - dns_resolver.as_ref(), - alt_svc_cache.as_ref(), - built.prober.as_ref(), - recipe.h3_upgrade.enabled, - ); - - Ok(Self { - live: Arc::new(RwLock::new(Some(Live { - client: built.client, - raw_client: built.raw_client, - dns_resolver, - #[cfg(feature = "http3")] - alt_svc_cache, - #[cfg(feature = "http3")] - h3_prober: built.prober, - }))), - // A warm-up connection is warm only as long as the pool keeps it idle, so the record - // that an origin is warm expires with that same window. - warmed: MokaCache::builder().time_to_live(conn_timeout).build(), - // A safety TTL well past any reasonable warm-up, so a claim that never gets released - // (a warm-up whose task is dropped) frees the origin rather than wedging it. - warming: MokaCache::builder() - .time_to_live(Duration::from_secs(300)) - .build(), - warm_generation: Default::default(), - cookie_jar, - stats: Default::default(), - conn_tracker: ConnectionTracker::new(conn_timeout), - h3_follow_advertised_port: settings.h3_follow_advertised_port, - #[cfg(feature = "http3")] - h3_upgrade_enabled: recipe.h3_upgrade.enabled, - quirk_h1_request_streaming: settings.quirk_h1_request_streaming, - default_accept_encoding: settings.default_accept_encoding, - default_content_encoding: settings.default_content_encoding, - has_default_priority: settings.has_default_priority, - recipe: Arc::new(recipe), - }) - } /// Close the agent, releasing its connection pool, DNS resolver, and any /// background tasks it owns, rather than waiting for the garbage collector @@ -895,221 +358,4 @@ impl Agent { pub fn is_closed(&self) -> bool { self.live().is_none() } - - /// Warm the DNS cache for `host`, so a later request to it skips the lookup. - /// - /// The argument is a bare host; a scheme, port, or path in a fuller string is ignored. The - /// returned future completes when the answer lands in the cache and never fails, whatever - /// happens on the network: the work is advisory. Under the system resolver there is no cache to - /// warm, so it completes having done nothing. A host with nothing to resolve, or a closed agent, - /// is refused here rather than by the future. - // spec:WARM - pub fn prefetch_dns(&self, host: &str) -> Result + use<>, FaithError> { - if self.is_closed() { - return Err(FaithErrorKind::Closed.into()); - } - - let Some(host) = extract_host(host) else { - return Err(FaithErrorKind::AddressParse.into()); - }; - - let resolver = self.dns_resolver(); - Ok(async move { - if let Some(resolver) = resolver { - resolver.prefetch(&host).await; - } - }) - } - - /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS - /// setup. - /// - /// The argument is an origin (`scheme://host[:port]`); a longer URL is reduced to one. The - /// warm-up sends a synthetic `HEAD` to the origin's root -- the origin sees it -- over the - /// transport the next foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC - /// connection, every other origin a TCP one. The returned future completes when the attempt - /// finishes and never fails: every network failure is quiet. Something that cannot be connected - /// to, or a closed agent, is refused here rather than by the future. - // spec:WARM - pub fn preconnect(&self, origin: &str) -> Result + use<>, FaithError> { - let Some(raw_client) = self.raw_client() else { - return Err(FaithErrorKind::Closed.into()); - }; - - let Some(url) = reduce_to_origin(origin) else { - return Err(FaithErrorKind::AddressParse.into()); - }; - let key = origin_key(&url); - - // Already warm within the idle window, or a warm-up for this origin already in flight: - // either way there is no new work to do, so finish without opening a duplicate. - let redundant = self.warmed.contains_key(&key) - || !self.warming.entry(key.clone()).or_insert(()).is_fresh(); - - // The transport the next foreground request would take, decided exactly as the Alt-Svc - // layer decides it: nothing upgrades with the machinery off; with a prober, only a - // confirmed origin routes to QUIC (an advertisement is evidence worth probing, not worth - // routing on); without one, the inline upgrade acts on advertisements too. Diverging here - // would warm the wrong transport. - // spec:WARM#preconnect - #[cfg(feature = "http3")] - let h3_port = self - .alt_svc_cache() - .filter(|_| self.h3_upgrade_enabled) - .and_then(|cache| { - if self.h3_prober().is_some() { - cache.confirmed_port(&url) - } else { - cache.should_use_h3(&url) - } - }); - #[cfg(not(feature = "http3"))] - let h3_port: Option = None; - - let conn_tracker = self.conn_tracker.clone(); - let warmed = self.warmed.clone(); - let warming = self.warming.clone(); - // Read before the warm-up starts, to compare against once it finishes. - let warm_generation = self.warm_generation.clone(); - let generation = warm_generation.load(Ordering::Relaxed); - - Ok(async move { - if redundant { - return; - } - - // Release the single-flight claim whatever happens, so a later warm-up is not blocked - // by this one having finished. - struct ReleaseClaim { - warming: MokaCache, - key: String, - } - impl Drop for ReleaseClaim { - fn drop(&mut self) { - self.warming.invalidate(&self.key); - } - } - let _release = ReleaseClaim { - warming, - key: key.clone(), - }; - - let request = match h3_port { - Some(port) => { - let mut h3_url = url.clone(); - // A port differing from the origin's only comes back with the - // follow-advertised-port option on; rewriting the URL is how reqwest is told to - // connect there, mirroring the foreground path. - if Some(port) != h3_url.port_or_known_default() { - let _ = h3_url.set_port(Some(port)); - } - raw_client.head(h3_url).version(Version::HTTP_3) - } - None => raw_client.head(url.clone()), - }; - - let outcome = request.send().await; - - // A TCP warm-up leaves a pooled connection to track; a QUIC one does not (QUIC - // connections are not tracked, and a confirmed origin has nothing left to probe). - if h3_port.is_none() - && let Ok(response) = &outcome - && let Some(info) = response - .extensions() - .get::() - { - conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); - } - - // A network change while this was in flight leaves the origin unmarked: the connection - // landed in the pool that change dropped, so it is not warm however well the request - // went. - // spec:NETCHG#reach-across-the-subsystems - if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { - warmed.insert(key, ()); - } - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// An agent is constructible without a caller assembling a recipe by hand, and comes up with a - /// live client and the default user agent. - #[tokio::test] - async fn a_default_agent_comes_up() { - let agent = Agent::new().expect("default options build an agent"); - - assert!(!agent.is_closed()); - assert_eq!(agent.recipe.user_agent, crate::USER_AGENT); - // No jar until the options ask for one. - assert!(agent.cookie_jar.is_none()); - } - - /// The builder reaches every group, and a group left alone stays absent rather than being - /// spelled out as absent. - #[tokio::test] - async fn the_builder_sets_what_it_is_given_and_nothing_else() { - use std::time::Duration; - - let options = Agent::builder() - .user_agent("YourApp/1.2.3") - .dns(|dns| dns.timeout(Duration::from_secs(2)).ndots(3)) - .pool(|pool| pool.max_idle_per_host(8)) - .into_options(); - - assert_eq!(options.user_agent.as_deref(), Some("YourApp/1.2.3")); - let dns = options.dns.expect("the dns group was reached"); - assert_eq!(dns.timeout, Some(2_000)); - assert_eq!(dns.ndots, Some(3)); - // Reached but not set stays unset... - assert_eq!(dns.servers, None); - // ...and a group never reached is absent entirely. - assert!(options.tls.is_none()); - assert!(options.http3.is_none()); - } - - /// What the builder produces is what the agent is built from, so a built agent carries it. - #[tokio::test] - async fn the_builder_builds_an_agent() { - let agent = Agent::builder() - .user_agent("YourApp/1.2.3") - .build() - .expect("the options are valid"); - - assert_eq!(agent.recipe.user_agent, "YourApp/1.2.3"); - assert!(!agent.is_closed()); - } - - /// A handle taken before a close still works afterwards, which is what lets a request issued - /// just before the close run to completion. - #[tokio::test] - async fn a_handle_taken_before_a_close_survives_it() { - let agent = Agent::new().expect("default options build an agent"); - let issued = agent.client().expect("an open agent hands out a client"); - - agent.close(); - - assert!(agent.is_closed()); - assert!( - agent.client().is_none(), - "a closed agent hands out no more clients" - ); - // The handle taken earlier is still usable; dropping it is what releases its share. - drop(issued); - } - - /// Closing acts on the agent itself, so every clone sees it. - #[tokio::test] - async fn closing_a_clone_closes_the_agent() { - let mut agent = Agent::new().expect("default options build an agent"); - let clone = agent.clone(); - - agent.close(); - - assert!(agent.is_closed()); - assert!(clone.is_closed(), "a clone names the same agent"); - } } diff --git a/crates/web-faith/src/agent/build.rs b/crates/web-faith/src/agent/build.rs new file mode 100644 index 0000000..a3e9de1 --- /dev/null +++ b/crates/web-faith/src/agent/build.rs @@ -0,0 +1,568 @@ +//! Turning options into an agent: validating what a caller expressed, and building from it. + +// spec:AGENT + +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + str::FromStr, + sync::{Arc, RwLock}, + time::Duration, +}; + +use http::header::{HeaderMap, HeaderName, HeaderValue}; +use http_cache_reqwest::{ + CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, +}; +use moka::sync::Cache as MokaCache; +use reqwest::{Identity, tls::Certificate}; +use web_faith_conn_tracker::ConnectionTracker; +use web_faith_cookies::FaithJar; +use web_faith_dns::{ + DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, +}; + +#[cfg(feature = "http3")] +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig}; + +use crate::{ + USER_AGENT, + agent::{Agent, AgentSettings, Live}, + client::{ClientRecipe, HttpCacheRecipe, HttpCacheStore, NodeEnvRecipe}, + error::{FaithError, FaithErrorKind}, + options::{ + AgentOptions, CacheStore, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows, + }, + request::PRIORITY, +}; + +#[cfg(feature = "http3")] +use crate::{ + client::{H3UpgradeRecipe, install_https_sink}, + options::Http3Congestion, +}; + +impl Agent { + /// This is what both surfaces land on, so the defaults a caller gets are settled here rather + /// than once per surface. + // spec:AGENT spec:NETCHG + pub fn from_options(options: AgentOptions) -> Result { + // Destructured rather than read field by field so that a new option cannot be added + // without the compiler pointing here, where every option is turned into the recipe the + // agent's clients are built from (spec:NETCHG). + let AgentOptions { + cache, + cookies, + dns, + flow_control, + headers, + http2, + // Every use of the HTTP/3 options sits behind the feature. + #[cfg_attr(not(feature = "http3"), allow(unused_variables))] + http3, + local_address, + pool, + quirks, + redirect, + timeout, + tls, + user_agent, + } = options; + + let quirk_h1_request_streaming = quirks + .and_then(|quirks| quirks.h1_request_streaming) + .unwrap_or(false); + + // Local bind address. An explicit value is honoured as-is. Otherwise, on hosts + // without usable IPv6, bind 0.0.0.0: reqwest binds the QUIC (HTTP/3) socket to the + // IPv6 wildcard `[::]` by default, which fails to construct on IPv4-only hosts and + // makes HTTP/3 silently fall back to TCP. Binding 0.0.0.0 there costs nothing (such + // a host can't use IPv6 for TCP either) and keeps HTTP/3 working. + let local_address = match &local_address { + Some(addr) => Some(IpAddr::from_str(addr).map_err(|err| { + FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + ) + })?), + None if !ipv6_wildcard_bindable() => Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), + None => None, + }; + + // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) + // The jar is installed on the client by the recipe, so it survives a rebuild + // (spec:NETCHG#what-the-signal-keeps). + let cookie_jar = cookies.map(|limits| Arc::new(FaithJar::new(limits))); + + let dns = dns.unwrap_or_default(); + let dns_system = dns.system.unwrap_or(false); + // Naming servers and asking for the system resolver at once is a contradiction rather than + // a preference, since the system resolver is not Faith's to point at listed servers + // (spec:DNS#system-resolver). + if dns_system + && dns + .servers + .as_ref() + .is_some_and(|servers| !servers.is_empty()) + { + return Err(FaithError::new( + FaithErrorKind::Config, + Some("dns.servers cannot be combined with dns.system".to_string()), + )); + } + // Parsed whichever resolver is in use: overrides take effect under the system resolver + // too (spec:DNS#overrides), and an unparseable address is a construction error either + // way (spec:AGENT#construction). + let dns_overrides = dns + .overrides + .unwrap_or_default() + .into_iter() + .map(|DnsOverride { domain, addresses }| { + let addresses = addresses + .into_iter() + .map(|addr| match SocketAddr::from_str(&addr) { + Ok(addr) => Ok(addr), + Err(err) => match IpAddr::from_str(&addr) { + Ok(IpAddr::V4(ip)) => Ok(SocketAddr::V4(SocketAddrV4::new(ip, 0))), + Ok(IpAddr::V6(ip)) => { + Ok(SocketAddr::V6(SocketAddrV6::new(ip, 0, 0, 0))) + } + Err(_) => Err(FaithError::new( + FaithErrorKind::AddressParse, + Some(format!("{addr:?}: {err}")), + )), + }, + }) + .collect::, FaithError>>()?; + Ok((domain, addresses)) + }) + .collect::, FaithError>>()?; + + // Faith owns the hickory resolver rather than leaving it to reqwest's built-in one, so + // `prefetchDns` can warm the very cache reqwest's requests read (spec:WARM), + // `networkChanged` can flush it (spec:NETCHG), and `dns.servers` can pick the transport and + // order each resolver is reached by (spec:DNS#transports). The system resolver + // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there + // and `prefetchDns` resolves as a no-op (spec:WARM). + let dns_resolver = if dns_system { + None + } else { + // These settings configure Faith's own resolver only, so they are read on the path that + // builds one rather than validated under the system resolver that ignores them. + let mut servers = Vec::new(); + for url in dns.servers.unwrap_or_default() { + servers.push(ServerSpec::parse(&url).map_err(|message| { + FaithError::new(FaithErrorKind::AddressParse, Some(message)) + })?); + } + Some(FaithResolver::new(ResolverSettings { + servers, + timeout: dns.timeout.map(|ms| Duration::from_millis(ms.into())), + ndots: dns.ndots.map(|n| n as usize), + search_domains: parse_domains(dns.search_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))?, + hosts_file: dns.hosts_file, + exempt_domains: parse_domains(dns.exempt_domains) + .map_err(|message| FaithError::new(FaithErrorKind::Config, Some(message)))? + .unwrap_or_default(), + serve_stale: dns.serve_stale.unwrap_or(true), + max_stale: dns + .max_stale + .map_or(DEFAULT_MAX_STALE, |ms| Duration::from_millis(ms.into())), + })) + }; + + let mut default_accept_encoding = None; + let mut default_content_encoding = None; + let mut has_default_priority = false; + let mut default_headers = None; + if let Some(headers) = headers + && !headers.is_empty() + { + let map = HeaderMap::from_iter(headers.into_iter().filter_map( + |Header { + name, + value, + sensitive, + }| { + let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { + return None; + }; + + let Ok(mut value) = HeaderValue::from_bytes(value.as_bytes()) else { + return None; + }; + + if sensitive.unwrap_or(false) { + value.set_sensitive(true); + } + + Some((name, value)) + }, + )); + default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); + default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); + has_default_priority = map.contains_key(PRIORITY); + default_headers = Some(map); + } + + // HTTP/2 flow control (spec:FLOW). Adaptive windowing takes over both windows itself, so + // the explicit sizes are not resolved at all when it's on: reqwest would let the later + // `http2_adaptive_window` call win regardless, but leaving them out makes the precedence + // visible here rather than depending on hyper's internal ordering. + let http2 = http2.unwrap_or_default(); + let http2_adaptive_window = http2.adaptive_window.unwrap_or(false); + let http2_windows = (!http2_adaptive_window).then(|| { + resolve_windows( + flow_control.as_ref(), + http2.stream_window, + http2.connection_window, + ) + }); + + #[cfg(feature = "http3")] + let http3_max_idle_timeout = Duration::from_secs( + http3 + .as_ref() + .and_then(|h| h.max_idle_timeout) + .unwrap_or(30) + .clamp(1, 120) + .into(), + ); + + // QUIC flow control (spec:FLOW). quinn's own defaults are a ~1.25MB stream window + // inside an unbounded connection window: the stream window is the binding constraint + // on a high-latency link, and the unbounded connection window means a connection with + // many concurrent requests has no ceiling on what it buffers. Both are set here. + #[cfg(feature = "http3")] + let http3_windows = resolve_windows( + flow_control.as_ref(), + http3.as_ref().and_then(|h| h.stream_window), + http3.as_ref().and_then(|h| h.connection_window), + ); + + #[cfg(feature = "http3")] + let http3_congestion_bbr = matches!( + http3.as_ref().and_then(|h| h.congestion), + Some(Http3Congestion::Bbr1) + ); + + #[cfg(feature = "http3")] + let http3_send_window = http3.as_ref().and_then(|h| h.send_window); + + let pool_idle_timeout = pool + .as_ref() + .and_then(|pool| pool.idle_timeout) + .map(|seconds| Duration::from_secs(seconds.into())); + // A pool group with no cap set means no limit, which is reqwest's own default (spec:POOL). + let pool_max_idle_per_host = pool.as_ref().map(|pool| { + pool.max_idle_per_host + .and_then(|n| n.try_into().ok()) + .unwrap_or(usize::MAX) + }); + + let connect_timeout = timeout + .and_then(|t| t.connect) + .map(|millis| Duration::from_millis(millis.into())); + let read_timeout = timeout + .and_then(|t| t.read) + .map(|millis| Duration::from_millis(millis.into())); + let total_timeout = timeout + .and_then(|t| t.total) + .map(|millis| Duration::from_millis(millis.into())); + + #[cfg(feature = "http3")] + let tls_early_data = tls.as_ref().and_then(|tls| tls.early_data); + let tls_required = tls.as_ref().and_then(|tls| tls.required); + // PEM inputs are parsed here rather than kept as bytes: the parsed forms are what a + // rebuilt client needs, and a syntax error belongs to construction (spec:AGENT#construction). + let (tls_identity, tls_extra_roots) = match tls { + None => (None, Vec::new()), + Some(tls) => { + let identity = match &tls.identity { + None => None, + Some(identity) => Some(Identity::from_pem(identity).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?), + }; + + let mut extra_roots = Vec::new(); + for pem in tls.extra_roots.iter().flatten() { + extra_roots.extend(Certificate::from_pem_bundle(pem).map_err(|err| { + FaithError::new(FaithErrorKind::PemParse, Some(err.to_string())) + })?); + } + + (identity, extra_roots) + } + }; + + let http_cache = if let Some(cache) = cache + && let Some(store) = cache.store + { + let mode = cache.mode.unwrap_or_default().into(); + let options = HttpCacheOptions { + cache_options: Some(CacheOptions { + shared: cache.shared.unwrap_or(true), + ignore_cargo_cult: true, + ..Default::default() + }), + ..Default::default() + }; + let store = match store { + CacheStore::Disk => HttpCacheStore::Disk(CACacheManager { + path: cache + .path + .ok_or_else(|| { + FaithError::new(FaithErrorKind::Config, Some("missing cache.path")) + })? + .into(), + remove_opts: Default::default(), + }), + CacheStore::Memory => HttpCacheStore::Memory(MokaManager::new( + MokaCacheBuilder::new(cache.capacity.map_or(10_000, |n| n.into())).build(), + )), + }; + + Some(HttpCacheRecipe { + mode, + options, + store, + }) + } else { + None + }; + + // Read outside the `alt_svc_cache` block below because `fetch` needs it too, + // to keep a rewritten port from looking like a redirect. + #[cfg(feature = "http3")] + let h3_follow_advertised_port = http3 + .as_ref() + .and_then(|o| o.upgrade_follow_advertised_port) + .unwrap_or(false); + #[cfg(not(feature = "http3"))] + let h3_follow_advertised_port = false; + + // The origin knowledge is built once and outlives every client the agent builds: it is + // the agent's own, and a network change edits it rather than replacing it (spec:NETCHG). + #[cfg(feature = "http3")] + let (alt_svc_cache, h3_upgrade) = { + let http3_opts = http3.as_ref(); + let enabled = http3_opts.and_then(|o| o.upgrade_enabled).unwrap_or(true); + + let advertised_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_advertised_ttl) + .unwrap_or(86400) + .into(), + ); + let confirmed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_confirmed_ttl) + .unwrap_or(86400) + .into(), + ); + let failed_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_ttl) + .unwrap_or(300) + .into(), + ); + let failed_max_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_failed_max_ttl) + .unwrap_or(3600) + .into(), + ); + let capacity = http3_opts + .and_then(|o| o.upgrade_cache_capacity) + .unwrap_or(10_000) + .into(); + let cancel_strikes = http3_opts + .and_then(|o| o.upgrade_cancel_strikes) + .unwrap_or(3); + let attempt_timeout = match http3_opts + .and_then(|o| o.upgrade_attempt_timeout) + .unwrap_or(60_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let probe = http3_opts.and_then(|o| o.upgrade_probe).unwrap_or(true); + let probe_timeout = match http3_opts + .and_then(|o| o.upgrade_probe_timeout) + .unwrap_or(5_000) + { + 0 => None, + millis => Some(Duration::from_millis(millis.into())), + }; + let slow_factor = http3_opts + .and_then(|o| o.upgrade_slow_factor) + .unwrap_or(2.5); + let slow_ttl = Duration::from_secs( + http3_opts + .and_then(|o| o.upgrade_slow_ttl) + .unwrap_or(600) + .into(), + ); + + let cache = Arc::new(AltSvcCache::new(AltSvcCacheConfig { + advertised_ttl, + confirmed_ttl, + failed_ttl, + failed_max_ttl, + capacity, + cancel_strikes, + strike_window: Duration::from_secs(60), + follow_advertised_port: h3_follow_advertised_port, + // The single-flight claim must outlive the probe it covers, so + // an aborted probe frees its origin without a report; without a + // probe deadline, the QUIC idle timeout (max 120s) is the bound. + probe_ttl: probe_timeout + .map_or(Duration::from_secs(125), |t| t + Duration::from_secs(5)), + slow_factor, + slow_ttl, + })); + + if let Some(hints) = http3_opts.and_then(|o| o.hints.as_ref()) { + for hint in hints { + cache.add_hint(&hint.host, hint.port); + } + } + + ( + Some(cache), + H3UpgradeRecipe { + enabled, + attempt_timeout, + probe, + probe_timeout, + }, + ) + }; + + let recipe = ClientRecipe { + user_agent: user_agent.unwrap_or_else(|| USER_AGENT.to_owned()), + local_address, + default_headers, + dns_system, + dns_overrides, + http2_adaptive_window, + http2_windows, + #[cfg(feature = "http3")] + http3_max_idle_timeout, + #[cfg(feature = "http3")] + http3_windows, + #[cfg(feature = "http3")] + http3_congestion_bbr, + #[cfg(feature = "http3")] + http3_send_window, + pool_idle_timeout, + pool_max_idle_per_host, + redirect, + connect_timeout, + read_timeout, + total_timeout, + #[cfg(feature = "http3")] + tls_early_data, + tls_identity, + tls_required, + tls_extra_roots, + node_env: NodeEnvRecipe::read(), + http_cache, + #[cfg(feature = "http3")] + h3_upgrade, + }; + + let settings = AgentSettings { + h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming, + default_accept_encoding, + default_content_encoding, + has_default_priority, + }; + + Self::build( + recipe, + settings, + cookie_jar, + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + ) + } + + /// An agent with default options. + pub fn new() -> Result { + Self::from_options(AgentOptions::default()) + } + + /// Build an agent a setting at a time. See [`AgentBuilder`](crate::builder::AgentBuilder). + pub fn builder() -> crate::builder::AgentBuilder { + crate::builder::AgentBuilder::default() + } + + /// Build an agent from a validated recipe. + /// + /// The recipe is what a client is built from, and the settings are what each request consults; + /// validating whatever a caller expressed them as belongs to the surface that took it. + pub fn build( + recipe: ClientRecipe, + settings: AgentSettings, + cookie_jar: Option>, + dns_resolver: Option, + #[cfg(feature = "http3")] alt_svc_cache: Option>, + ) -> Result { + let conn_timeout = recipe.conn_timeout(); + let built = recipe.build( + cookie_jar.as_ref(), + dns_resolver.as_ref(), + #[cfg(feature = "http3")] + alt_svc_cache.as_ref(), + )?; + + // Only now do all three exist: the resolver is built before the cache, and the prober + // holds a client that holds the resolver, so this is the earliest the loop can be closed + // (spec:DNS#https-records). + #[cfg(feature = "http3")] + install_https_sink( + dns_resolver.as_ref(), + alt_svc_cache.as_ref(), + built.prober.as_ref(), + recipe.h3_upgrade.enabled, + ); + + Ok(Self { + live: Arc::new(RwLock::new(Some(Live { + client: built.client, + raw_client: built.raw_client, + dns_resolver, + #[cfg(feature = "http3")] + alt_svc_cache, + #[cfg(feature = "http3")] + h3_prober: built.prober, + }))), + // A warm-up connection is warm only as long as the pool keeps it idle, so the record + // that an origin is warm expires with that same window. + warmed: MokaCache::builder().time_to_live(conn_timeout).build(), + // A safety TTL well past any reasonable warm-up, so a claim that never gets released + // (a warm-up whose task is dropped) frees the origin rather than wedging it. + warming: MokaCache::builder() + .time_to_live(Duration::from_secs(300)) + .build(), + warm_generation: Default::default(), + cookie_jar, + stats: Default::default(), + conn_tracker: ConnectionTracker::new(conn_timeout), + h3_follow_advertised_port: settings.h3_follow_advertised_port, + #[cfg(feature = "http3")] + h3_upgrade_enabled: recipe.h3_upgrade.enabled, + quirk_h1_request_streaming: settings.quirk_h1_request_streaming, + default_accept_encoding: settings.default_accept_encoding, + default_content_encoding: settings.default_content_encoding, + has_default_priority: settings.has_default_priority, + recipe: Arc::new(recipe), + }) + } +} diff --git a/crates/web-faith/src/agent/tests.rs b/crates/web-faith/src/agent/tests.rs new file mode 100644 index 0000000..c099352 --- /dev/null +++ b/crates/web-faith/src/agent/tests.rs @@ -0,0 +1,78 @@ +use super::*; + +/// An agent is constructible without a caller assembling a recipe by hand, and comes up with a +/// live client and the default user agent. +#[tokio::test] +async fn a_default_agent_comes_up() { + let agent = Agent::new().expect("default options build an agent"); + + assert!(!agent.is_closed()); + assert_eq!(agent.recipe.user_agent, crate::USER_AGENT); + // No jar until the options ask for one. + assert!(agent.cookie_jar.is_none()); +} + +/// The builder reaches every group, and a group left alone stays absent rather than being +/// spelled out as absent. +#[tokio::test] +async fn the_builder_sets_what_it_is_given_and_nothing_else() { + use std::time::Duration; + + let options = Agent::builder() + .user_agent("YourApp/1.2.3") + .dns(|dns| dns.timeout(Duration::from_secs(2)).ndots(3)) + .pool(|pool| pool.max_idle_per_host(8)) + .into_options(); + + assert_eq!(options.user_agent.as_deref(), Some("YourApp/1.2.3")); + let dns = options.dns.expect("the dns group was reached"); + assert_eq!(dns.timeout, Some(2_000)); + assert_eq!(dns.ndots, Some(3)); + // Reached but not set stays unset... + assert_eq!(dns.servers, None); + // ...and a group never reached is absent entirely. + assert!(options.tls.is_none()); + assert!(options.http3.is_none()); +} + +/// What the builder produces is what the agent is built from, so a built agent carries it. +#[tokio::test] +async fn the_builder_builds_an_agent() { + let agent = Agent::builder() + .user_agent("YourApp/1.2.3") + .build() + .expect("the options are valid"); + + assert_eq!(agent.recipe.user_agent, "YourApp/1.2.3"); + assert!(!agent.is_closed()); +} + +/// A handle taken before a close still works afterwards, which is what lets a request issued +/// just before the close run to completion. +#[tokio::test] +async fn a_handle_taken_before_a_close_survives_it() { + let agent = Agent::new().expect("default options build an agent"); + let issued = agent.client().expect("an open agent hands out a client"); + + agent.close(); + + assert!(agent.is_closed()); + assert!( + agent.client().is_none(), + "a closed agent hands out no more clients" + ); + // The handle taken earlier is still usable; dropping it is what releases its share. + drop(issued); +} + +/// Closing acts on the agent itself, so every clone sees it. +#[tokio::test] +async fn closing_a_clone_closes_the_agent() { + let agent = Agent::new().expect("default options build an agent"); + let clone = agent.clone(); + + agent.close(); + + assert!(agent.is_closed()); + assert!(clone.is_closed(), "a clone names the same agent"); +} diff --git a/crates/web-faith/src/agent/warm.rs b/crates/web-faith/src/agent/warm.rs new file mode 100644 index 0000000..c13f767 --- /dev/null +++ b/crates/web-faith/src/agent/warm.rs @@ -0,0 +1,151 @@ +//! Warming an agent's caches ahead of the request that needs them. + +// spec:WARM + +use std::{future::Future, sync::atomic::Ordering}; + +use moka::sync::Cache as MokaCache; +use reqwest::Version; + +use crate::{ + agent::Agent, + error::{FaithError, FaithErrorKind}, + warm_up::{extract_host, origin_key, reduce_to_origin}, +}; + +impl Agent { + /// Warm the DNS cache for `host`, so a later request to it skips the lookup. + /// + /// The argument is a bare host; a scheme, port, or path in a fuller string is ignored. The + /// returned future completes when the answer lands in the cache and never fails, whatever + /// happens on the network: the work is advisory. Under the system resolver there is no cache to + /// warm, so it completes having done nothing. A host with nothing to resolve, or a closed agent, + /// is refused here rather than by the future. + // spec:WARM + pub fn prefetch_dns(&self, host: &str) -> Result + use<>, FaithError> { + if self.is_closed() { + return Err(FaithErrorKind::Closed.into()); + } + + let Some(host) = extract_host(host) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + + let resolver = self.dns_resolver(); + Ok(async move { + if let Some(resolver) = resolver { + resolver.prefetch(&host).await; + } + }) + } + + /// Open a pooled connection to `origin`, so the first request to it skips DNS, TCP, and TLS + /// setup. + /// + /// The argument is an origin (`scheme://host[:port]`); a longer URL is reduced to one. The + /// warm-up sends a synthetic `HEAD` to the origin's root -- the origin sees it -- over the + /// transport the next foreground request would use: a confirmed HTTP/3 origin gets a warm QUIC + /// connection, every other origin a TCP one. The returned future completes when the attempt + /// finishes and never fails: every network failure is quiet. Something that cannot be connected + /// to, or a closed agent, is refused here rather than by the future. + // spec:WARM + pub fn preconnect(&self, origin: &str) -> Result + use<>, FaithError> { + let Some(raw_client) = self.raw_client() else { + return Err(FaithErrorKind::Closed.into()); + }; + + let Some(url) = reduce_to_origin(origin) else { + return Err(FaithErrorKind::AddressParse.into()); + }; + let key = origin_key(&url); + + // Already warm within the idle window, or a warm-up for this origin already in flight: + // either way there is no new work to do, so finish without opening a duplicate. + let redundant = self.warmed.contains_key(&key) + || !self.warming.entry(key.clone()).or_insert(()).is_fresh(); + + // The transport the next foreground request would take, decided exactly as the Alt-Svc + // layer decides it: nothing upgrades with the machinery off; with a prober, only a + // confirmed origin routes to QUIC (an advertisement is evidence worth probing, not worth + // routing on); without one, the inline upgrade acts on advertisements too. Diverging here + // would warm the wrong transport. + // spec:WARM#preconnect + #[cfg(feature = "http3")] + let h3_port = self + .alt_svc_cache() + .filter(|_| self.h3_upgrade_enabled) + .and_then(|cache| { + if self.h3_prober().is_some() { + cache.confirmed_port(&url) + } else { + cache.should_use_h3(&url) + } + }); + #[cfg(not(feature = "http3"))] + let h3_port: Option = None; + + let conn_tracker = self.conn_tracker.clone(); + let warmed = self.warmed.clone(); + let warming = self.warming.clone(); + // Read before the warm-up starts, to compare against once it finishes. + let warm_generation = self.warm_generation.clone(); + let generation = warm_generation.load(Ordering::Relaxed); + + Ok(async move { + if redundant { + return; + } + + // Release the single-flight claim whatever happens, so a later warm-up is not blocked + // by this one having finished. + struct ReleaseClaim { + warming: MokaCache, + key: String, + } + impl Drop for ReleaseClaim { + fn drop(&mut self) { + self.warming.invalidate(&self.key); + } + } + let _release = ReleaseClaim { + warming, + key: key.clone(), + }; + + let request = match h3_port { + Some(port) => { + let mut h3_url = url.clone(); + // A port differing from the origin's only comes back with the + // follow-advertised-port option on; rewriting the URL is how reqwest is told to + // connect there, mirroring the foreground path. + if Some(port) != h3_url.port_or_known_default() { + let _ = h3_url.set_port(Some(port)); + } + raw_client.head(h3_url).version(Version::HTTP_3) + } + None => raw_client.head(url.clone()), + }; + + let outcome = request.send().await; + + // A TCP warm-up leaves a pooled connection to track; a QUIC one does not (QUIC + // connections are not tracked, and a confirmed origin has nothing left to probe). + if h3_port.is_none() + && let Ok(response) = &outcome + && let Some(info) = response + .extensions() + .get::() + { + conn_tracker.track_warmup(info.local_addr(), info.remote_addr()); + } + + // A network change while this was in flight leaves the origin unmarked: the connection + // landed in the pool that change dropped, so it is not warm however well the request + // went. + // spec:NETCHG#reach-across-the-subsystems + if outcome.is_ok() && warm_generation.load(Ordering::Relaxed) == generation { + warmed.insert(key, ()); + } + }) + } +} diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs index a7dcf91..8293a28 100644 --- a/crates/web-faith/src/request.rs +++ b/crates/web-faith/src/request.rs @@ -2,37 +2,19 @@ // spec:REQ spec:ENC spec:CANCEL -use std::{ - future::{Future, IntoFuture}, - pin::Pin, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::{Duration, Instant}, -}; +mod builder; +mod send; +mod target; + +pub use builder::{FetchBuilder, Priority, Request, RequestBuilder}; +pub use send::send; +pub use target::Target; + +use std::{pin::Pin, time::Duration}; use bytes::Bytes; use futures::Stream; use http_cache_reqwest::CacheMode; -use hyper_util::client::legacy::connect::HttpInfo; -use reqwest::{ - Method, StatusCode, - header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, - tls::TlsInfo, -}; -use reqwest_middleware::ClientWithMiddleware; -use tokio::sync::Mutex; -use url::Url; -use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; - -use crate::{ - agent::Agent, - body::{Body, BodyHolder}, - error::{FaithError, FaithErrorKind}, - response::{PeerInformation, Response}, - timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, -}; /// Whether a request carries its credentials, and how far. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -76,1079 +58,3 @@ pub struct RequestOptions { pub priority: Option<&'static str>, pub timeout: Option, } - -/// Send a request on `agent`, and build the response it produces. -/// -/// `client` is the handle the caller took when the request was issued, rather than one taken here: -/// a request counts as in flight from the moment it is issued, so one issued just before the agent -/// closes runs to completion even though nothing had started on it yet. -/// -/// `abort` is an optional future that, resolving first, cancels the request. -// spec:AGENT -pub async fn send( - agent: &Agent, - client: ClientWithMiddleware, - url: &str, - options: RequestOptions, - body: RequestBody, - abort: Option>, -) -> Result { - let method = options.method.as_deref().unwrap_or("GET"); - // spec:REQ#method-and-headers - let method = NORMALISED_METHODS - .into_iter() - .find(|normalised| normalised.eq_ignore_ascii_case(method)) - .unwrap_or(method); - - let method = - Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; - let is_head = method == Method::HEAD; - - let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; - - // A `compress` naming no coding Faith can compress in is misuse whether or not the - // request turns out to carry a body, so it is refused before anything else looks at - // it (spec:ENC#compressing-a-request-body). - let compress = options - .compress - .as_deref() - .map(|value| { - Coding::from_option(value).ok_or_else(|| { - FaithError::new( - FaithErrorKind::InvalidCompression, - Some(format!( - "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" - )), - ) - }) - }) - .transpose()?; - - // Handle credentials based on credentials option - if options.credentials == Credentials::Omit { - // Remove credentials from URL if omit is specified - let _ = parsed_url.set_username(""); - let _ = parsed_url.set_password(None); - } - - // The stamp rides along in the request's extensions for the middleware to fill in; - // this side keeps a handle on it so the one measurement taken inside the stack is - // the one surfaced (spec:RESP#request-timing). - let headers_stamp = HeadersStamp::default(); - - let mut request = client - .request(method, parsed_url.clone()) - .with_extension(CacheMode::from(options.cache)) - .with_extension(headers_stamp.clone()); - - if let Some(headers) = &options.headers { - for (key, value) in headers { - // Skip Cookie header if credentials is omit - if options.credentials == Credentials::Omit && key.eq_ignore_ascii_case("cookie") { - continue; - } - - // Validate header name and value before adding to request - let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header name: {key}")), - ) - })?; - let header_value = HeaderValue::from_str(value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - - // Faith's coding is layered on top of what the caller declares, and reqwest's - // builder appends rather than replaces, so passing the caller's value through - // here would put a second `Content-Encoding` beside the joined one -- the same - // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is - // still validated above, then withheld and re-emitted once below. - if compress.is_some() && header_name == CONTENT_ENCODING { - continue; - } - - request = request.header(header_name, header_value); - } - } - - // What the caller says they handed over: their own `Content-Encoding`, else the - // agent's, per-request headers winning per name as they do generally (spec: REQ). - // Several lines are the one list, so they are joined as they are read. - let declared_content_encoding = compress.and_then(|_| { - let from_request = options.headers.as_ref().and_then(|headers| { - let declared = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(); - (!declared.is_empty()).then(|| declared.join(", ")) - }); - from_request.or_else(|| { - agent - .default_content_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - }); - - // The request's `Accept-Encoding` governs which codings Faith decodes on the way - // back (spec: ENC): a value on the request, else one inherited from the agent's - // default headers, else the default Faith sends itself. Neither the request nor the - // agent advertising a value means nothing beneath Faith adds one now that it owns - // the codings, so Faith sends the default explicitly. - let request_accept_encoding = options.headers.as_ref().and_then(|headers| { - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) - .map(|(_, value)| value.clone()) - }); - let accept_encoding = AcceptEncoding::parse( - &request_accept_encoding - .clone() - .or_else(|| { - agent - .default_accept_encoding - .as_ref() - .and_then(|value| value.to_str().ok().map(str::to_owned)) - }) - .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), - ); - if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { - request = request.header( - ACCEPT_ENCODING, - HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), - ); - } - - // The `priority` option is a hint, so a `Priority` header the caller wrote, or one - // among the agent's default headers, wins over the value derived from it - // (spec: REQ#request-priority). The agent's defaults are consulted here rather than - // left to reqwest: it fills a default header in only where the request carries none - // of that name, so setting the derived value would displace the agent's own. - if let Some(urgency) = options.priority - && !agent.has_default_priority - && !options.headers.as_ref().is_some_and(|headers| { - headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) - }) { - request = request.header( - HeaderName::from_static(PRIORITY), - HeaderValue::from_static(urgency), - ); - } - - // The coding actually applied, which is `compress` only where there was a body to - // apply it to: the option does nothing on a request carrying none, so no - // `Content-Encoding` describes bytes that were never sent - // (spec:ENC#compressing-a-request-body). - let mut applied_coding = None; - - // Handle body: prefer streaming body over buffered body - match body { - RequestBody::Stream(byte_stream) => { - // A body read from a stream has no length to advertise, which the fetch standard - // allows only over HTTP/2 and HTTP/3. - // spec:REQ#streaming-a-request-body - if !agent.quirk_h1_request_streaming { - // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and - // can be refused without opening a connection to find out. Returning here drops - // the stream, which is what tells whatever is feeding it to stop. - if parsed_url.scheme() != "https" { - return Err(FaithError::new( - FaithErrorKind::Network, - Some(format!( - "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", - parsed_url.as_str() - )), - )); - } - - // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the - // request hands the check to the layer that finds out: the connection is chosen, - // and an HTTP/1.x one is refused there before any of the body is written. - request = request.version(http::Version::HTTP_2); - } - - request = request.body(match compress { - // Compressed as the chunks arrive, and chunked on the wire either way: - // a stream has no length to declare up front. - // spec:ENC#what-a-compressed-request-sends - Some(coding) => { - applied_coding = Some(coding); - reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) - } - None => reqwest::Body::wrap_stream(byte_stream), - }); - } - RequestBody::Bytes(bytes) => { - request = request.body(match compress { - // The compressed bytes are what reqwest sizes `Content-Length` from, so the - // header counts what goes on the wire. - // spec:ENC#what-a-compressed-request-sends - Some(coding) => { - applied_coding = Some(coding); - encoding::compress_buffer(&bytes, coding) - .await - .map_err(|err| { - FaithError::new( - FaithErrorKind::Network, - Some(format!("could not compress the request body: {err}")), - ) - })? - } - None => bytes.to_vec(), - }); - } - RequestBody::None => {} - } - - // One `Content-Encoding` naming the caller's codings then Faith's, in the order they - // were applied (spec:ENC#what-a-compressed-request-sends). - if let Some(coding) = applied_coding { - let value = encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); - let value = HeaderValue::from_str(&value).map_err(|_| { - FaithError::new( - FaithErrorKind::InvalidHeader, - Some(format!("invalid header value: {value}")), - ) - })?; - request = request.header(CONTENT_ENCODING, value); - } - - if let Some(dur) = options.timeout { - request = request.timeout(dur); - } - - agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); - - // The origin every phase is measured from. - let started = Instant::now(); - - // A caller that can abort races the request against that signal; one that cannot just sends. - let response = match abort { - Some(abort) => { - tokio::select! { - result = request.send() => result?, - _ = abort => { - return Err(FaithErrorKind::Aborted.into()); - } - } - } - None => request.send().await?, - }; - - agent - .stats - .responses_received - .fetch_add(1, Ordering::Relaxed); - - let status_code = response.status(); - let empty = status_code == StatusCode::NO_CONTENT || is_head; - - let response_url = response.url().clone(); - let version = response.version(); - - // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the - // request's port to the advertised one, so the response URL's port reflects - // which endpoint answered rather than any redirect. Compare with ports - // normalised away, or every such request would report `redirected`. - // - // Only HTTP/3 responses can have been rewritten — reqwest routes - // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and - // the TCP fallback re-runs the untouched clone. Restricting the normalisation - // to those keeps exact comparison, and so port-only redirect detection, for - // every other response. - let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { - let without_port = |url: &reqwest::Url| { - let mut url = url.clone(); - let _ = url.set_port(None); - url - }; - without_port(&parsed_url) != without_port(&response_url) - } else { - parsed_url != response_url - }; - - // Track connection for TCP stats (if we can get both local and remote addr). - // A connection the tracker has already seen is one the pool handed back, which is - // what `reused` reports (spec:RESP#request-timing). - let reused = if let Some(http_info) = response.extensions().get::() { - let local_addr = http_info.local_addr(); - let remote_addr = http_info.remote_addr(); - agent.conn_tracker.track(local_addr, remote_addr) - } else { - false - }; - - // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has - // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a - // redirect chain marks the origin that actually answered rather than the one asked for. - agent.mark_warm(&response_url); - - let peer = PeerInformation { - address: response.remote_addr(), - certificate: response - .extensions() - .get::() - .and_then(|info| info.peer_certificate()) - .map(|cert| cert.into()), - }; - - let mut headers = response.headers().clone(); - if options.credentials == Credentials::Omit { - headers.remove("set-cookie"); - } - - // A cache hit is served without ever reaching the layer that stamps, so fall back to - // the moment the send resolved, which for a hit is the moment the cache answered. - let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); - let timing = RequestTiming { - headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, - body_ms: None, - reused, - next_hop_protocol: alpn_protocol_id(version, &response_url), - // Captured before a decoded body's `Content-Encoding` is stripped below, so the - // coding the response arrived under is reported either way. - content_encoding: headers - .get(CONTENT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned), - from_cache: headers - .get("x-cache") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), - }; - - // Decode only a body Faith negotiated the coding for; a bodyless response keeps its - // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). - let decode = if empty { - None - } else { - encoding::decision(&headers, &accept_encoding) - }; - if decode.is_some() { - encoding::strip_decoded_headers(&mut headers); - } - - let timing = Arc::new(TimingSlot::new(started, timing)); - // A response that cannot carry a body has nothing left to wait for. - if empty { - timing.ended(); - } - - Ok(Response { - body: if empty { - BodyHolder::none() - } else { - let http_response: http::Response<_> = response.into(); - BodyHolder::new( - Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), - version, - timing.clone(), - ) - }, - decode, - disturbed: Arc::new(AtomicBool::new(false)), - headers, - integrity: options.integrity, - peer: Arc::new(peer), - redirected, - stats: agent.stats.clone(), - status_code, - timing, - trailers: Default::default(), - url: response_url, - version, - }) -} - -/// What a request is aimed at: a URL, or another request to layer over. -/// -/// Anything that converts into a `url::Url` is a target, as is a [`Request`], which is what lets a -/// prepared request be adjusted at each call site. -// spec:REQ -pub enum Target { - Url(Url), - Request(Box), -} - -impl From for Target { - fn from(url: Url) -> Self { - Self::Url(url) - } -} - -impl From for Target { - fn from(request: Request) -> Self { - Self::Request(Box::new(request)) - } -} - -/// A string target is parsed when the builder resolves, so an unparseable one is reported there -/// rather than by a call that cannot fail. -// spec:REQ -impl TryFrom<&str> for Target { - type Error = FaithError; - - fn try_from(url: &str) -> Result { - Url::parse(url) - .map(Self::Url) - .map_err(|_| FaithErrorKind::InvalidUrl.into()) - } -} - -impl TryFrom for Target { - type Error = FaithError; - - fn try_from(url: String) -> Result { - Self::try_from(url.as_str()) - } -} - -/// An `http::Request` is a target too, so a request built against the wider ecosystem can be sent -/// on an agent: its method, URL, headers, and body come across. -// spec:REQ -impl TryFrom> for Target -where - B: Into, -{ - type Error = FaithError; - - fn try_from(request: http::Request) -> Result { - let (parts, body) = request.into_parts(); - - let url = Url::parse(&parts.uri.to_string()) - .map_err(|_| FaithError::from(FaithErrorKind::InvalidUrl))?; - - let mut headers = Vec::new(); - for (name, value) in parts.headers.iter() { - let Ok(value) = value.to_str() else { - return Err(FaithErrorKind::InvalidHeader.into()); - }; - headers.push((name.to_string(), value.to_owned())); - } - - let body = body.into(); - Ok(Self::Request(Box::new(Request { - url, - options: RequestOptions { - method: Some(parts.method.to_string()), - headers: (!headers.is_empty()).then_some(headers), - ..RequestOptions::default() - }, - body: if body.is_empty() { - RequestBody::None - } else { - RequestBody::Bytes(body) - }, - }))) - } -} - -/// A header the caller set or removed on one layer. -#[derive(Clone, Debug)] -struct HeaderOp { - name: String, - /// `None` removes the name, including whatever the layers beneath contributed for it. - value: Option, -} - -impl std::fmt::Debug for RequestBody { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::None => f.write_str("None"), - Self::Bytes(bytes) => f.debug_tuple("Bytes").field(&bytes.len()).finish(), - Self::Stream(_) => f.write_str("Stream"), - } - } -} - -/// A request prepared but not sent. -/// -/// Inert: it carries no agent, so it can be sent on more than one, and passing it to -/// [`Agent::fetch`] returns a builder that layers over it. -// spec:REQ -#[derive(Debug)] -pub struct Request { - url: Url, - options: RequestOptions, - body: RequestBody, -} - -impl Request { - /// Prepare a request aimed at `target`. - pub fn new(target: T) -> RequestBuilder - where - T: TryInto, - T::Error: Into, - { - RequestBuilder { - layer: Layer::over(target), - } - } - - pub fn url(&self) -> &Url { - &self.url - } - - pub fn options(&self) -> &RequestOptions { - &self.options - } - - /// Copy the request, when its body allows it. - /// - /// `None` when the body is a stream, a stream being consumable once. - // spec:REQ - pub fn try_clone(&self) -> Option { - let body = match &self.body { - RequestBody::None => RequestBody::None, - RequestBody::Bytes(bytes) => RequestBody::Bytes(bytes.clone()), - RequestBody::Stream(_) => return None, - }; - - Some(Self { - url: self.url.clone(), - options: self.options.clone(), - body, - }) - } -} - -/// One layer of settings over a target, which both builders are made of. -struct Layer { - /// The first failure met while setting up, held until the builder resolves. - // spec:REQ - target: Result, - options: RequestOptions, - /// Which single-valued settings this layer set explicitly, so an unset one is inherited rather - /// than overwritten with a default. - set: SetFlags, - headers: Vec, - body: Option, -} - -#[derive(Default)] -struct SetFlags { - cache: bool, - compress: bool, - credentials: bool, - integrity: bool, - method: bool, - priority: bool, - timeout: bool, -} - -impl Layer { - /// Hold a failure until the builder resolves, keeping the first one met. - // spec:REQ - fn fail(&mut self, err: FaithError) { - if self.target.is_ok() { - self.target = Err(err); - } - } - - fn over(target: T) -> Self - where - T: TryInto, - T::Error: Into, - { - Self { - target: target.try_into().map_err(Into::into), - options: RequestOptions::default(), - set: SetFlags::default(), - headers: Vec::new(), - body: None, - } - } - - /// Settle this layer over whatever it wraps: the outermost explicit value wins, a setting this - /// layer does not touch is inherited unchanged, and headers merge by name. - // spec:REQ - fn settle(self) -> Result { - let (url, mut options, body) = match self.target? { - Target::Url(url) => (url, RequestOptions::default(), RequestBody::None), - // The URL comes from the target at the bottom of the stack. - Target::Request(inner) => (inner.url, inner.options, inner.body), - }; - - if self.set.cache { - options.cache = self.options.cache; - } - if self.set.compress { - options.compress = self.options.compress; - } - if self.set.credentials { - options.credentials = self.options.credentials; - } - if self.set.integrity { - options.integrity = self.options.integrity; - } - if self.set.method { - options.method = self.options.method; - } - if self.set.priority { - options.priority = self.options.priority; - } - if self.set.timeout { - options.timeout = self.options.timeout; - } - - let mut headers = options.headers.take().unwrap_or_default(); - for op in self.headers { - headers.retain(|(name, _)| !name.eq_ignore_ascii_case(&op.name)); - if let Some(value) = op.value { - headers.push((op.name, value)); - } - } - options.headers = (!headers.is_empty()).then_some(headers); - - Ok(Request { - url, - options, - body: self.body.unwrap_or(body), - }) - } -} - -/// The setters both builders carry, so a call reads the same way whichever it is written against. -macro_rules! layer_setters { - ($builder:ident) => { - impl $builder { - /// The request method. Defaults to `GET`. - /// - /// Takes a `Method` or anything that converts into one; a value that does not is - /// reported where the builder resolves. - pub fn method(mut self, method: M) -> Self - where - M: TryInto, - { - match method.try_into() { - Ok(method) => { - self.layer.options.method = Some(method.to_string()); - self.layer.set.method = true; - } - Err(_) => self.layer.fail(FaithErrorKind::InvalidMethod.into()), - } - self - } - - /// Set a header, replacing whatever any layer beneath contributed for that name. - /// - /// Takes `HeaderName` and `HeaderValue` or anything that converts into them. - pub fn header(mut self, name: N, value: V) -> Self - where - N: TryInto, - V: TryInto, - { - match (name.try_into(), value.try_into()) { - (Ok(name), Ok(value)) => match value.to_str() { - Ok(value) => self.layer.headers.push(HeaderOp { - name: name.to_string(), - value: Some(value.to_owned()), - }), - Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), - }, - _ => self.layer.fail(FaithErrorKind::InvalidHeader.into()), - } - self - } - - /// Set several headers, each applied as a single header would be. - pub fn headers(mut self, headers: impl IntoIterator) -> Self - where - N: TryInto, - V: TryInto, - { - for (name, value) in headers { - self = self.header(name, value); - } - self - } - - /// Remove a header, including whatever the layers beneath contributed for it. - pub fn remove_header(mut self, name: N) -> Self - where - N: TryInto, - { - match name.try_into() { - Ok(name) => self.layer.headers.push(HeaderOp { - name: name.to_string(), - value: None, - }), - Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), - } - self - } - - /// A body already in hand. - pub fn body(mut self, body: impl Into) -> Self { - self.layer.body = Some(RequestBody::Bytes(body.into())); - self - } - - /// A body arriving in chunks, which goes out chunked having no length to declare. - pub fn body_stream( - mut self, - body: impl Stream> + Send + 'static, - ) -> Self { - self.layer.body = Some(RequestBody::Stream(Box::pin(body))); - self - } - - /// Bound the whole request and response. - // spec:CANCEL - pub fn timeout(mut self, timeout: Duration) -> Self { - self.layer.options.timeout = Some(timeout); - self.layer.set.timeout = true; - self - } - - /// The digests the response body is expected to match. - // spec:SRI - pub fn integrity(mut self, integrity: impl Into) -> Self { - self.layer.options.integrity = Some(integrity.into()); - self.layer.set.integrity = true; - self - } - - /// Compress the request body in this coding, named by its wire token. - // spec:ENC - pub fn compress(mut self, coding: impl Into) -> Self { - self.layer.options.compress = Some(coding.into()); - self.layer.set.compress = true; - self - } - - /// How the HTTP cache is consulted for this request. - // spec:CACHE - pub fn cache(mut self, mode: CacheMode) -> Self { - self.layer.options.cache = mode; - self.layer.set.cache = true; - self - } - - /// Whether the request carries its credentials. - pub fn credentials(mut self, credentials: Credentials) -> Self { - self.layer.options.credentials = credentials; - self.layer.set.credentials = true; - self - } - - /// How this request ranks against others, as an RFC 9218 urgency. - // spec:REQ#request-priority - pub fn priority(mut self, priority: Priority) -> Self { - self.layer.options.priority = priority.urgency(); - self.layer.set.priority = true; - self - } - } - }; -} - -/// How a request ranks against others. -// spec:REQ#request-priority -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Priority { - /// More urgent than an unmarked request. - High, - /// Less urgent than an unmarked request. - Low, - /// Send no `Priority` header, leaving the request at the scheme's default urgency. - #[default] - Auto, -} - -impl Priority { - fn urgency(self) -> Option<&'static str> { - match self { - Self::High => Some("u=1"), - Self::Low => Some("u=5"), - Self::Auto => None, - } - } -} - -/// Prepares a [`Request`] without sending it. -#[must_use = "a request builder does nothing until built"] -pub struct RequestBuilder { - layer: Layer, -} - -layer_setters!(RequestBuilder); - -impl RequestBuilder { - /// Settle the layers into a request. - /// - /// Where a conversion failed on the way in — an unparseable target, say — it is reported here. - pub fn build(self) -> Result { - self.layer.settle() - } -} - -/// A request bound to an agent, which sends when awaited. -/// -/// There is no separate send step: awaiting is what sends. A builder dropped without being awaited -/// sends nothing, and dropping the future mid-flight cancels the request. -// spec:REQ spec:CANCEL -#[must_use = "a fetch builder sends nothing until awaited"] -pub struct FetchBuilder { - agent: Agent, - layer: Layer, -} - -layer_setters!(FetchBuilder); - -impl FetchBuilder { - /// Settle the layers into the request that would be sent, without sending it. - pub fn build(self) -> Result { - self.layer.settle() - } -} - -impl IntoFuture for FetchBuilder { - type Output = Result; - type IntoFuture = Pin + Send>>; - - fn into_future(self) -> Self::IntoFuture { - Box::pin(async move { - // Taken before anything is awaited, so the request counts as in flight from here. - // spec:AGENT - let client = self.agent.client().ok_or(FaithErrorKind::Closed)?; - let request = self.layer.settle()?; - - send( - &self.agent, - client, - request.url.as_str(), - request.options, - request.body, - None::>, - ) - .await - }) - } -} - -impl Agent { - /// Aim a request at `target`, to send when awaited. - /// - /// The target is a URL, or a [`Request`] to layer over. Awaiting the builder sends the request; - /// see [`FetchBuilder`]. - // spec:REQ - pub fn fetch(&self, target: T) -> FetchBuilder - where - T: TryInto, - T::Error: Into, - { - FetchBuilder { - agent: self.clone(), - layer: Layer::over(target), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn headers_of(request: &Request) -> Vec<(String, String)> { - request.options.headers.clone().unwrap_or_default() - } - - /// The outermost layer that set a value wins, whatever the layers beneath said. - #[test] - fn the_outermost_explicit_value_wins() { - let inner = Request::new("https://example.com/") - .timeout(Duration::from_secs(1)) - .build() - .expect("a valid target"); - - let outer = Request::new(inner) - .timeout(Duration::from_secs(3)) - .build() - .expect("layering over a request"); - - assert_eq!(outer.options.timeout, Some(Duration::from_secs(3))); - } - - /// A setting an outer layer does not touch is inherited unchanged. - #[test] - fn an_untouched_setting_is_inherited() { - let inner = Request::new("https://example.com/") - .timeout(Duration::from_secs(1)) - .method("POST") - .build() - .expect("a valid target"); - - let outer = Request::new(inner) - .integrity("sha256-abc") - .build() - .expect("layering over a request"); - - assert_eq!(outer.options.timeout, Some(Duration::from_secs(1))); - assert_eq!(outer.options.method.as_deref(), Some("POST")); - assert_eq!(outer.options.integrity.as_deref(), Some("sha256-abc")); - } - - /// The URL comes from the target at the bottom of the stack. - #[test] - fn wrapping_a_request_carries_its_url_through() { - let inner = Request::new("https://example.com/deep/path?q=1") - .build() - .expect("a valid target"); - let outer = Request::new(inner) - .method("HEAD") - .build() - .expect("layering"); - - assert_eq!(outer.url.as_str(), "https://example.com/deep/path?q=1"); - } - - /// Headers merge by name: an outer value replaces, a name only set inside carries through. - #[test] - fn headers_merge_by_name() { - let inner = Request::new("https://example.com/") - .header("x-keep", "inner") - .header("x-replace", "inner") - .build() - .expect("a valid target"); - - let outer = Request::new(inner) - .header("x-replace", "outer") - .header("x-add", "outer") - .build() - .expect("layering over a request"); - - let mut headers = headers_of(&outer); - headers.sort(); - assert_eq!( - headers, - vec![ - ("x-add".to_owned(), "outer".to_owned()), - ("x-keep".to_owned(), "inner".to_owned()), - ("x-replace".to_owned(), "outer".to_owned()), - ] - ); - } - - /// Removing a name removes what the layers beneath contributed for it. - #[test] - fn removing_a_header_removes_what_is_underneath() { - let inner = Request::new("https://example.com/") - .header("x-gone", "inner") - .build() - .expect("a valid target"); - - let outer = Request::new(inner) - .remove_header("X-Gone") - .build() - .expect("layering over a request"); - - assert!(headers_of(&outer).is_empty(), "the name was removed"); - } - - /// Setting the same thing twice on one builder is the same question at a smaller scale. - #[test] - fn the_later_call_wins_on_one_builder() { - let request = Request::new("https://example.com/") - .method("POST") - .method("PUT") - .build() - .expect("a valid target"); - - assert_eq!(request.options.method.as_deref(), Some("PUT")); - } - - /// An `http::Request` brings its method, URL, headers, and body across. - #[test] - fn an_http_request_is_a_target() { - let http = http::Request::builder() - .method(http::Method::POST) - .uri("https://example.com/submit") - .header("x-from", "ecosystem") - .body(Bytes::from_static(b"payload")) - .expect("a valid http request"); - - let request = Request::new(http).build().expect("it converts"); - - assert_eq!(request.url.as_str(), "https://example.com/submit"); - assert_eq!(request.options.method.as_deref(), Some("POST")); - assert_eq!( - headers_of(&request), - vec![("x-from".to_owned(), "ecosystem".to_owned())] - ); - assert!(request.try_clone().is_some(), "a buffered body copies"); - } - - /// A layer over an `http::Request` beats what it carried, as over any other target. - #[test] - fn a_layer_beats_what_an_http_request_carried() { - let http = http::Request::builder() - .method(http::Method::POST) - .uri("https://example.com/") - .body(Bytes::new()) - .expect("a valid http request"); - - let request = Request::new(http) - .method(http::Method::PUT) - .build() - .expect("layering over it"); - - assert_eq!(request.options.method.as_deref(), Some("PUT")); - } - - /// A header name that is not one is reported where the request is resolved too. - #[test] - fn an_invalid_header_surfaces_at_build() { - let err = Request::new("https://example.com/") - .header("not a header name", "value") - .build() - .expect_err("the name is not a header name"); - - assert_eq!(err.kind, FaithErrorKind::InvalidHeader); - } - - /// The first failure met is the one reported, not the last. - #[test] - fn the_first_failure_is_the_one_reported() { - let err = Request::new("not a url") - .header("not a header name", "value") - .build() - .expect_err("both are wrong"); - - assert_eq!(err.kind, FaithErrorKind::InvalidUrl); - } - - /// An unparseable target is reported where the request is resolved, not by the call that took it. - #[test] - fn an_unparseable_target_surfaces_at_build() { - let builder = Request::new("not a url"); - let err = builder.build().expect_err("the target does not parse"); - - assert_eq!(err.kind, FaithErrorKind::InvalidUrl); - } - - /// A request copies when its body allows it, and reports that it cannot when it does not. - #[test] - fn try_clone_copies_what_it_can() { - let buffered = Request::new("https://example.com/") - .body(Bytes::from_static(b"body")) - .build() - .expect("a valid target"); - assert!(buffered.try_clone().is_some()); - - let streamed = Request::new("https://example.com/") - .body_stream(futures::stream::empty()) - .build() - .expect("a valid target"); - assert!( - streamed.try_clone().is_none(), - "a stream is consumable once" - ); - } -} diff --git a/crates/web-faith/src/request/builder.rs b/crates/web-faith/src/request/builder.rs new file mode 100644 index 0000000..f3bd6c4 --- /dev/null +++ b/crates/web-faith/src/request/builder.rs @@ -0,0 +1,427 @@ +use std::{ + future::{Future, IntoFuture}, + pin::Pin, + time::Duration, +}; + +use bytes::Bytes; +use futures::Stream; +use http_cache_reqwest::CacheMode; +use reqwest::{ + Method, + header::{HeaderName, HeaderValue}, +}; +use url::Url; + +use crate::{ + agent::Agent, + error::{FaithError, FaithErrorKind}, + request::{Credentials, RequestBody, RequestOptions, Target, send::send}, + response::Response, +}; + +/// A header the caller set or removed on one layer. +#[derive(Clone, Debug)] +struct HeaderOp { + name: String, + /// `None` removes the name, including whatever the layers beneath contributed for it. + value: Option, +} + +impl std::fmt::Debug for RequestBody { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::None => f.write_str("None"), + Self::Bytes(bytes) => f.debug_tuple("Bytes").field(&bytes.len()).finish(), + Self::Stream(_) => f.write_str("Stream"), + } + } +} + +/// A request prepared but not sent. +/// +/// Inert: it carries no agent, so it can be sent on more than one, and passing it to +/// [`Agent::fetch`] returns a builder that layers over it. +// spec:REQ +#[derive(Debug)] +pub struct Request { + pub(super) url: Url, + pub(super) options: RequestOptions, + pub(super) body: RequestBody, +} + +impl Request { + /// Prepare a request aimed at `target`. + pub fn new(target: T) -> RequestBuilder + where + T: TryInto, + T::Error: Into, + { + RequestBuilder { + layer: Layer::over(target), + } + } + + pub fn url(&self) -> &Url { + &self.url + } + + pub fn options(&self) -> &RequestOptions { + &self.options + } + + /// Copy the request, when its body allows it. + /// + /// `None` when the body is a stream, a stream being consumable once. + // spec:REQ + pub fn try_clone(&self) -> Option { + let body = match &self.body { + RequestBody::None => RequestBody::None, + RequestBody::Bytes(bytes) => RequestBody::Bytes(bytes.clone()), + RequestBody::Stream(_) => return None, + }; + + Some(Self { + url: self.url.clone(), + options: self.options.clone(), + body, + }) + } +} + +/// One layer of settings over a target, which both builders are made of. +struct Layer { + /// The first failure met while setting up, held until the builder resolves. + // spec:REQ + target: Result, + options: RequestOptions, + /// Which single-valued settings this layer set explicitly, so an unset one is inherited rather + /// than overwritten with a default. + set: SetFlags, + headers: Vec, + body: Option, +} + +#[derive(Default)] +struct SetFlags { + cache: bool, + compress: bool, + credentials: bool, + integrity: bool, + method: bool, + priority: bool, + timeout: bool, +} + +impl Layer { + /// Hold a failure until the builder resolves, keeping the first one met. + // spec:REQ + fn fail(&mut self, err: FaithError) { + if self.target.is_ok() { + self.target = Err(err); + } + } + + fn over(target: T) -> Self + where + T: TryInto, + T::Error: Into, + { + Self { + target: target.try_into().map_err(Into::into), + options: RequestOptions::default(), + set: SetFlags::default(), + headers: Vec::new(), + body: None, + } + } + + /// Settle this layer over whatever it wraps: the outermost explicit value wins, a setting this + /// layer does not touch is inherited unchanged, and headers merge by name. + // spec:REQ + fn settle(self) -> Result { + let (url, mut options, body) = match self.target? { + Target::Url(url) => (url, RequestOptions::default(), RequestBody::None), + // The URL comes from the target at the bottom of the stack. + Target::Request(inner) => (inner.url, inner.options, inner.body), + }; + + if self.set.cache { + options.cache = self.options.cache; + } + if self.set.compress { + options.compress = self.options.compress; + } + if self.set.credentials { + options.credentials = self.options.credentials; + } + if self.set.integrity { + options.integrity = self.options.integrity; + } + if self.set.method { + options.method = self.options.method; + } + if self.set.priority { + options.priority = self.options.priority; + } + if self.set.timeout { + options.timeout = self.options.timeout; + } + + let mut headers = options.headers.take().unwrap_or_default(); + for op in self.headers { + headers.retain(|(name, _)| !name.eq_ignore_ascii_case(&op.name)); + if let Some(value) = op.value { + headers.push((op.name, value)); + } + } + options.headers = (!headers.is_empty()).then_some(headers); + + Ok(Request { + url, + options, + body: self.body.unwrap_or(body), + }) + } +} + +/// The setters both builders carry, so a call reads the same way whichever it is written against. +macro_rules! layer_setters { + ($builder:ident) => { + impl $builder { + /// The request method. Defaults to `GET`. + /// + /// Takes a `Method` or anything that converts into one; a value that does not is + /// reported where the builder resolves. + pub fn method(mut self, method: M) -> Self + where + M: TryInto, + { + match method.try_into() { + Ok(method) => { + self.layer.options.method = Some(method.to_string()); + self.layer.set.method = true; + } + Err(_) => self.layer.fail(FaithErrorKind::InvalidMethod.into()), + } + self + } + + /// Set a header, replacing whatever any layer beneath contributed for that name. + /// + /// Takes `HeaderName` and `HeaderValue` or anything that converts into them. + pub fn header(mut self, name: N, value: V) -> Self + where + N: TryInto, + V: TryInto, + { + match (name.try_into(), value.try_into()) { + (Ok(name), Ok(value)) => match value.to_str() { + Ok(value) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: Some(value.to_owned()), + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + }, + _ => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// Set several headers, each applied as a single header would be. + pub fn headers(mut self, headers: impl IntoIterator) -> Self + where + N: TryInto, + V: TryInto, + { + for (name, value) in headers { + self = self.header(name, value); + } + self + } + + /// Remove a header, including whatever the layers beneath contributed for it. + pub fn remove_header(mut self, name: N) -> Self + where + N: TryInto, + { + match name.try_into() { + Ok(name) => self.layer.headers.push(HeaderOp { + name: name.to_string(), + value: None, + }), + Err(_) => self.layer.fail(FaithErrorKind::InvalidHeader.into()), + } + self + } + + /// A body already in hand. + pub fn body(mut self, body: impl Into) -> Self { + self.layer.body = Some(RequestBody::Bytes(body.into())); + self + } + + /// A body arriving in chunks, which goes out chunked having no length to declare. + pub fn body_stream( + mut self, + body: impl Stream> + Send + 'static, + ) -> Self { + self.layer.body = Some(RequestBody::Stream(Box::pin(body))); + self + } + + /// Bound the whole request and response. + // spec:CANCEL + pub fn timeout(mut self, timeout: Duration) -> Self { + self.layer.options.timeout = Some(timeout); + self.layer.set.timeout = true; + self + } + + /// The digests the response body is expected to match. + // spec:SRI + pub fn integrity(mut self, integrity: impl Into) -> Self { + self.layer.options.integrity = Some(integrity.into()); + self.layer.set.integrity = true; + self + } + + /// Compress the request body in this coding, named by its wire token. + // spec:ENC + pub fn compress(mut self, coding: impl Into) -> Self { + self.layer.options.compress = Some(coding.into()); + self.layer.set.compress = true; + self + } + + /// How the HTTP cache is consulted for this request. + // spec:CACHE + pub fn cache(mut self, mode: CacheMode) -> Self { + self.layer.options.cache = mode; + self.layer.set.cache = true; + self + } + + /// Whether the request carries its credentials. + pub fn credentials(mut self, credentials: Credentials) -> Self { + self.layer.options.credentials = credentials; + self.layer.set.credentials = true; + self + } + + /// How this request ranks against others, as an RFC 9218 urgency. + // spec:REQ#request-priority + pub fn priority(mut self, priority: Priority) -> Self { + self.layer.options.priority = priority.urgency(); + self.layer.set.priority = true; + self + } + } + }; +} + +/// How a request ranks against others. +// spec:REQ#request-priority +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Priority { + /// More urgent than an unmarked request. + High, + /// Less urgent than an unmarked request. + Low, + /// Send no `Priority` header, leaving the request at the scheme's default urgency. + #[default] + Auto, +} + +impl Priority { + fn urgency(self) -> Option<&'static str> { + match self { + Self::High => Some("u=1"), + Self::Low => Some("u=5"), + Self::Auto => None, + } + } +} + +/// Prepares a [`Request`] without sending it. +#[must_use = "a request builder does nothing until built"] +pub struct RequestBuilder { + layer: Layer, +} + +layer_setters!(RequestBuilder); + +impl RequestBuilder { + /// Settle the layers into a request. + /// + /// Where a conversion failed on the way in — an unparseable target, say — it is reported here. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +/// A request bound to an agent, which sends when awaited. +/// +/// There is no separate send step: awaiting is what sends. A builder dropped without being awaited +/// sends nothing, and dropping the future mid-flight cancels the request. +// spec:REQ spec:CANCEL +#[must_use = "a fetch builder sends nothing until awaited"] +pub struct FetchBuilder { + agent: Agent, + layer: Layer, +} + +layer_setters!(FetchBuilder); + +impl FetchBuilder { + /// Settle the layers into the request that would be sent, without sending it. + pub fn build(self) -> Result { + self.layer.settle() + } +} + +impl IntoFuture for FetchBuilder { + type Output = Result; + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(async move { + // Taken before anything is awaited, so the request counts as in flight from here. + // spec:AGENT + let client = self.agent.client().ok_or(FaithErrorKind::Closed)?; + let request = self.layer.settle()?; + + send( + &self.agent, + client, + request.url.as_str(), + request.options, + request.body, + None::>, + ) + .await + }) + } +} + +impl Agent { + /// Aim a request at `target`, to send when awaited. + /// + /// The target is a URL, or a [`Request`] to layer over. Awaiting the builder sends the request; + /// see [`FetchBuilder`]. + // spec:REQ + pub fn fetch(&self, target: T) -> FetchBuilder + where + T: TryInto, + T::Error: Into, + { + FetchBuilder { + agent: self.clone(), + layer: Layer::over(target), + } + } +} + +#[cfg(test)] +#[cfg(test)] +mod tests; diff --git a/crates/web-faith/src/request/builder/tests.rs b/crates/web-faith/src/request/builder/tests.rs new file mode 100644 index 0000000..72a2c67 --- /dev/null +++ b/crates/web-faith/src/request/builder/tests.rs @@ -0,0 +1,197 @@ +use super::*; + +fn headers_of(request: &Request) -> Vec<(String, String)> { + request.options.headers.clone().unwrap_or_default() +} + +/// The outermost layer that set a value wins, whatever the layers beneath said. +#[test] +fn the_outermost_explicit_value_wins() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .timeout(Duration::from_secs(3)) + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(3))); +} + +/// A setting an outer layer does not touch is inherited unchanged. +#[test] +fn an_untouched_setting_is_inherited() { + let inner = Request::new("https://example.com/") + .timeout(Duration::from_secs(1)) + .method("POST") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .integrity("sha256-abc") + .build() + .expect("layering over a request"); + + assert_eq!(outer.options.timeout, Some(Duration::from_secs(1))); + assert_eq!(outer.options.method.as_deref(), Some("POST")); + assert_eq!(outer.options.integrity.as_deref(), Some("sha256-abc")); +} + +/// The URL comes from the target at the bottom of the stack. +#[test] +fn wrapping_a_request_carries_its_url_through() { + let inner = Request::new("https://example.com/deep/path?q=1") + .build() + .expect("a valid target"); + let outer = Request::new(inner) + .method("HEAD") + .build() + .expect("layering"); + + assert_eq!(outer.url.as_str(), "https://example.com/deep/path?q=1"); +} + +/// Headers merge by name: an outer value replaces, a name only set inside carries through. +#[test] +fn headers_merge_by_name() { + let inner = Request::new("https://example.com/") + .header("x-keep", "inner") + .header("x-replace", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .header("x-replace", "outer") + .header("x-add", "outer") + .build() + .expect("layering over a request"); + + let mut headers = headers_of(&outer); + headers.sort(); + assert_eq!( + headers, + vec![ + ("x-add".to_owned(), "outer".to_owned()), + ("x-keep".to_owned(), "inner".to_owned()), + ("x-replace".to_owned(), "outer".to_owned()), + ] + ); +} + +/// Removing a name removes what the layers beneath contributed for it. +#[test] +fn removing_a_header_removes_what_is_underneath() { + let inner = Request::new("https://example.com/") + .header("x-gone", "inner") + .build() + .expect("a valid target"); + + let outer = Request::new(inner) + .remove_header("X-Gone") + .build() + .expect("layering over a request"); + + assert!(headers_of(&outer).is_empty(), "the name was removed"); +} + +/// Setting the same thing twice on one builder is the same question at a smaller scale. +#[test] +fn the_later_call_wins_on_one_builder() { + let request = Request::new("https://example.com/") + .method("POST") + .method("PUT") + .build() + .expect("a valid target"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); +} + +/// An `http::Request` brings its method, URL, headers, and body across. +#[test] +fn an_http_request_is_a_target() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/submit") + .header("x-from", "ecosystem") + .body(Bytes::from_static(b"payload")) + .expect("a valid http request"); + + let request = Request::new(http).build().expect("it converts"); + + assert_eq!(request.url.as_str(), "https://example.com/submit"); + assert_eq!(request.options.method.as_deref(), Some("POST")); + assert_eq!( + headers_of(&request), + vec![("x-from".to_owned(), "ecosystem".to_owned())] + ); + assert!(request.try_clone().is_some(), "a buffered body copies"); +} + +/// A layer over an `http::Request` beats what it carried, as over any other target. +#[test] +fn a_layer_beats_what_an_http_request_carried() { + let http = http::Request::builder() + .method(http::Method::POST) + .uri("https://example.com/") + .body(Bytes::new()) + .expect("a valid http request"); + + let request = Request::new(http) + .method(http::Method::PUT) + .build() + .expect("layering over it"); + + assert_eq!(request.options.method.as_deref(), Some("PUT")); +} + +/// A header name that is not one is reported where the request is resolved too. +#[test] +fn an_invalid_header_surfaces_at_build() { + let err = Request::new("https://example.com/") + .header("not a header name", "value") + .build() + .expect_err("the name is not a header name"); + + assert_eq!(err.kind, FaithErrorKind::InvalidHeader); +} + +/// The first failure met is the one reported, not the last. +#[test] +fn the_first_failure_is_the_one_reported() { + let err = Request::new("not a url") + .header("not a header name", "value") + .build() + .expect_err("both are wrong"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); +} + +/// An unparseable target is reported where the request is resolved, not by the call that took it. +#[test] +fn an_unparseable_target_surfaces_at_build() { + let builder = Request::new("not a url"); + let err = builder.build().expect_err("the target does not parse"); + + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); +} + +/// A request copies when its body allows it, and reports that it cannot when it does not. +#[test] +fn try_clone_copies_what_it_can() { + let buffered = Request::new("https://example.com/") + .body(Bytes::from_static(b"body")) + .build() + .expect("a valid target"); + assert!(buffered.try_clone().is_some()); + + let streamed = Request::new("https://example.com/") + .body_stream(futures::stream::empty()) + .build() + .expect("a valid target"); + assert!( + streamed.try_clone().is_none(), + "a stream is consumable once" + ); +} diff --git a/crates/web-faith/src/request/send.rs b/crates/web-faith/src/request/send.rs new file mode 100644 index 0000000..3cba9d4 --- /dev/null +++ b/crates/web-faith/src/request/send.rs @@ -0,0 +1,417 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Instant, +}; + +use http_cache_reqwest::CacheMode; +use hyper_util::client::legacy::connect::HttpInfo; +use reqwest::{ + Method, StatusCode, + header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, + tls::TlsInfo, +}; +use reqwest_middleware::ClientWithMiddleware; +use tokio::sync::Mutex; +use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; + +use crate::{ + agent::Agent, + body::{Body, BodyHolder}, + error::{FaithError, FaithErrorKind}, + request::{Credentials, NORMALISED_METHODS, PRIORITY, RequestBody, RequestOptions}, + response::{PeerInformation, Response}, + timing::{HeadersStamp, RequestTiming, TimingSlot, alpn_protocol_id}, +}; + +/// Send a request on `agent`, and build the response it produces. +/// +/// `client` is the handle the caller took when the request was issued, rather than one taken here: +/// a request counts as in flight from the moment it is issued, so one issued just before the agent +/// closes runs to completion even though nothing had started on it yet. +/// +/// `abort` is an optional future that, resolving first, cancels the request. +// spec:AGENT +pub async fn send( + agent: &Agent, + client: ClientWithMiddleware, + url: &str, + options: RequestOptions, + body: RequestBody, + abort: Option>, +) -> Result { + let method = options.method.as_deref().unwrap_or("GET"); + // spec:REQ#method-and-headers + let method = NORMALISED_METHODS + .into_iter() + .find(|normalised| normalised.eq_ignore_ascii_case(method)) + .unwrap_or(method); + + let method = + Method::from_bytes(method.as_bytes()).map_err(|_| FaithErrorKind::InvalidMethod)?; + let is_head = method == Method::HEAD; + + let mut parsed_url = reqwest::Url::parse(&url).map_err(|_| FaithErrorKind::InvalidUrl)?; + + // A `compress` naming no coding Faith can compress in is misuse whether or not the + // request turns out to carry a body, so it is refused before anything else looks at + // it (spec:ENC#compressing-a-request-body). + let compress = options + .compress + .as_deref() + .map(|value| { + Coding::from_option(value).ok_or_else(|| { + FaithError::new( + FaithErrorKind::InvalidCompression, + Some(format!( + "compress: {value:?} names no coding; expected gzip, deflate, br, or zstd" + )), + ) + }) + }) + .transpose()?; + + // Handle credentials based on credentials option + if options.credentials == Credentials::Omit { + // Remove credentials from URL if omit is specified + let _ = parsed_url.set_username(""); + let _ = parsed_url.set_password(None); + } + + // The stamp rides along in the request's extensions for the middleware to fill in; + // this side keeps a handle on it so the one measurement taken inside the stack is + // the one surfaced (spec:RESP#request-timing). + let headers_stamp = HeadersStamp::default(); + + let mut request = client + .request(method, parsed_url.clone()) + .with_extension(CacheMode::from(options.cache)) + .with_extension(headers_stamp.clone()); + + if let Some(headers) = &options.headers { + for (key, value) in headers { + // Skip Cookie header if credentials is omit + if options.credentials == Credentials::Omit && key.eq_ignore_ascii_case("cookie") { + continue; + } + + // Validate header name and value before adding to request + let header_name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header name: {key}")), + ) + })?; + let header_value = HeaderValue::from_str(value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + + // Faith's coding is layered on top of what the caller declares, and reqwest's + // builder appends rather than replaces, so passing the caller's value through + // here would put a second `Content-Encoding` beside the joined one -- the same + // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is + // still validated above, then withheld and re-emitted once below. + if compress.is_some() && header_name == CONTENT_ENCODING { + continue; + } + + request = request.header(header_name, header_value); + } + } + + // What the caller says they handed over: their own `Content-Encoding`, else the + // agent's, per-request headers winning per name as they do generally (spec: REQ). + // Several lines are the one list, so they are joined as they are read. + let declared_content_encoding = compress.and_then(|_| { + let from_request = options.headers.as_ref().and_then(|headers| { + let declared = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(CONTENT_ENCODING.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(); + (!declared.is_empty()).then(|| declared.join(", ")) + }); + from_request.or_else(|| { + agent + .default_content_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + }); + + // The request's `Accept-Encoding` governs which codings Faith decodes on the way + // back (spec: ENC): a value on the request, else one inherited from the agent's + // default headers, else the default Faith sends itself. Neither the request nor the + // agent advertising a value means nothing beneath Faith adds one now that it owns + // the codings, so Faith sends the default explicitly. + let request_accept_encoding = options.headers.as_ref().and_then(|headers| { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) + .map(|(_, value)| value.clone()) + }); + let accept_encoding = AcceptEncoding::parse( + &request_accept_encoding + .clone() + .or_else(|| { + agent + .default_accept_encoding + .as_ref() + .and_then(|value| value.to_str().ok().map(str::to_owned)) + }) + .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), + ); + if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { + request = request.header( + ACCEPT_ENCODING, + HeaderValue::from_static(DEFAULT_ACCEPT_ENCODING), + ); + } + + // The `priority` option is a hint, so a `Priority` header the caller wrote, or one + // among the agent's default headers, wins over the value derived from it + // (spec: REQ#request-priority). The agent's defaults are consulted here rather than + // left to reqwest: it fills a default header in only where the request carries none + // of that name, so setting the derived value would displace the agent's own. + if let Some(urgency) = options.priority + && !agent.has_default_priority + && !options.headers.as_ref().is_some_and(|headers| { + headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case(PRIORITY)) + }) { + request = request.header( + HeaderName::from_static(PRIORITY), + HeaderValue::from_static(urgency), + ); + } + + // The coding actually applied, which is `compress` only where there was a body to + // apply it to: the option does nothing on a request carrying none, so no + // `Content-Encoding` describes bytes that were never sent + // (spec:ENC#compressing-a-request-body). + let mut applied_coding = None; + + // Handle body: prefer streaming body over buffered body + match body { + RequestBody::Stream(byte_stream) => { + // A body read from a stream has no length to advertise, which the fetch standard + // allows only over HTTP/2 and HTTP/3. + // spec:REQ#streaming-a-request-body + if !agent.quirk_h1_request_streaming { + // Faith never negotiates h2c, so a plaintext origin is HTTP/1.x for certain and + // can be refused without opening a connection to find out. Returning here drops + // the stream, which is what tells whatever is feeding it to stop. + if parsed_url.scheme() != "https" { + return Err(FaithError::new( + FaithErrorKind::Network, + Some(format!( + "a streaming request body requires HTTP/2 or HTTP/3, and {} is served over HTTP/1.1; set the agent's quirks.h1RequestStreaming to send it anyway", + parsed_url.as_str() + )), + )); + } + + // Over TLS the protocol is only known once ALPN has run. Asserting HTTP/2 on the + // request hands the check to the layer that finds out: the connection is chosen, + // and an HTTP/1.x one is refused there before any of the body is written. + request = request.version(http::Version::HTTP_2); + } + + request = request.body(match compress { + // Compressed as the chunks arrive, and chunked on the wire either way: + // a stream has no length to declare up front. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) + } + None => reqwest::Body::wrap_stream(byte_stream), + }); + } + RequestBody::Bytes(bytes) => { + request = request.body(match compress { + // The compressed bytes are what reqwest sizes `Content-Length` from, so the + // header counts what goes on the wire. + // spec:ENC#what-a-compressed-request-sends + Some(coding) => { + applied_coding = Some(coding); + encoding::compress_buffer(&bytes, coding) + .await + .map_err(|err| { + FaithError::new( + FaithErrorKind::Network, + Some(format!("could not compress the request body: {err}")), + ) + })? + } + None => bytes.to_vec(), + }); + } + RequestBody::None => {} + } + + // One `Content-Encoding` naming the caller's codings then Faith's, in the order they + // were applied (spec:ENC#what-a-compressed-request-sends). + if let Some(coding) = applied_coding { + let value = encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); + let value = HeaderValue::from_str(&value).map_err(|_| { + FaithError::new( + FaithErrorKind::InvalidHeader, + Some(format!("invalid header value: {value}")), + ) + })?; + request = request.header(CONTENT_ENCODING, value); + } + + if let Some(dur) = options.timeout { + request = request.timeout(dur); + } + + agent.stats.requests_sent.fetch_add(1, Ordering::Relaxed); + + // The origin every phase is measured from. + let started = Instant::now(); + + // A caller that can abort races the request against that signal; one that cannot just sends. + let response = match abort { + Some(abort) => { + tokio::select! { + result = request.send() => result?, + _ = abort => { + return Err(FaithErrorKind::Aborted.into()); + } + } + } + None => request.send().await?, + }; + + agent + .stats + .responses_received + .fetch_add(1, Ordering::Relaxed); + + let status_code = response.status(); + let empty = status_code == StatusCode::NO_CONTENT || is_head; + + let response_url = response.url().clone(); + let version = response.version(); + + // With `http3.upgradeFollowAdvertisedPort` on, an HTTP/3 attempt rewrites the + // request's port to the advertised one, so the response URL's port reflects + // which endpoint answered rather than any redirect. Compare with ports + // normalised away, or every such request would report `redirected`. + // + // Only HTTP/3 responses can have been rewritten — reqwest routes + // `Version::HTTP_3` exclusively to the h3 client with no silent downgrade, and + // the TCP fallback re-runs the untouched clone. Restricting the normalisation + // to those keeps exact comparison, and so port-only redirect detection, for + // every other response. + let redirected = if agent.h3_follow_advertised_port && version == http::Version::HTTP_3 { + let without_port = |url: &reqwest::Url| { + let mut url = url.clone(); + let _ = url.set_port(None); + url + }; + without_port(&parsed_url) != without_port(&response_url) + } else { + parsed_url != response_url + }; + + // Track connection for TCP stats (if we can get both local and remote addr). + // A connection the tracker has already seen is one the pool handed back, which is + // what `reused` reports (spec:RESP#request-timing). + let reused = if let Some(http_info) = response.extensions().get::() { + let local_addr = http_info.local_addr(); + let remote_addr = http_info.remote_addr(); + agent.conn_tracker.track(local_addr, remote_addr) + } else { + false + }; + + // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has + // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a + // redirect chain marks the origin that actually answered rather than the one asked for. + agent.mark_warm(&response_url); + + let peer = PeerInformation { + address: response.remote_addr(), + certificate: response + .extensions() + .get::() + .and_then(|info| info.peer_certificate()) + .map(|cert| cert.into()), + }; + + let mut headers = response.headers().clone(); + if options.credentials == Credentials::Omit { + headers.remove("set-cookie"); + } + + // A cache hit is served without ever reaching the layer that stamps, so fall back to + // the moment the send resolved, which for a hit is the moment the cache answered. + let headers_at = headers_stamp.get().unwrap_or_else(Instant::now); + let timing = RequestTiming { + headers_ms: headers_at.duration_since(started).as_secs_f64() * 1000.0, + body_ms: None, + reused, + next_hop_protocol: alpn_protocol_id(version, &response_url), + // Captured before a decoded body's `Content-Encoding` is stripped below, so the + // coding the response arrived under is reported either way. + content_encoding: headers + .get(CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + from_cache: headers + .get("x-cache") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("HIT")), + }; + + // Decode only a body Faith negotiated the coding for; a bodyless response keeps its + // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). + let decode = if empty { + None + } else { + encoding::decision(&headers, &accept_encoding) + }; + if decode.is_some() { + encoding::strip_decoded_headers(&mut headers); + } + + let timing = Arc::new(TimingSlot::new(started, timing)); + // A response that cannot carry a body has nothing left to wait for. + if empty { + timing.ended(); + } + + Ok(Response { + body: if empty { + BodyHolder::none() + } else { + let http_response: http::Response<_> = response.into(); + BodyHolder::new( + Some(Arc::new(Mutex::new(Body::Inner(http_response.into_body())))), + version, + timing.clone(), + ) + }, + decode, + disturbed: Arc::new(AtomicBool::new(false)), + headers, + integrity: options.integrity, + peer: Arc::new(peer), + redirected, + stats: agent.stats.clone(), + status_code, + timing, + trailers: Default::default(), + url: response_url, + version, + }) +} diff --git a/crates/web-faith/src/request/target.rs b/crates/web-faith/src/request/target.rs new file mode 100644 index 0000000..a5754bc --- /dev/null +++ b/crates/web-faith/src/request/target.rs @@ -0,0 +1,90 @@ +use bytes::Bytes; +use url::Url; + +use crate::{ + error::{FaithError, FaithErrorKind}, + request::{Request, RequestBody, RequestOptions}, +}; + +/// What a request is aimed at: a URL, or another request to layer over. +/// +/// Anything that converts into a `url::Url` is a target, as is a [`Request`], which is what lets a +/// prepared request be adjusted at each call site. +// spec:REQ +pub enum Target { + Url(Url), + Request(Box), +} + +impl From for Target { + fn from(url: Url) -> Self { + Self::Url(url) + } +} + +impl From for Target { + fn from(request: Request) -> Self { + Self::Request(Box::new(request)) + } +} + +/// A string target is parsed when the builder resolves, so an unparseable one is reported there +/// rather than by a call that cannot fail. +// spec:REQ +impl TryFrom<&str> for Target { + type Error = FaithError; + + fn try_from(url: &str) -> Result { + Url::parse(url) + .map(Self::Url) + .map_err(|_| FaithErrorKind::InvalidUrl.into()) + } +} + +impl TryFrom for Target { + type Error = FaithError; + + fn try_from(url: String) -> Result { + Self::try_from(url.as_str()) + } +} + +/// An `http::Request` is a target too, so a request built against the wider ecosystem can be sent +/// on an agent: its method, URL, headers, and body come across. +// spec:REQ +impl TryFrom> for Target +where + B: Into, +{ + type Error = FaithError; + + fn try_from(request: http::Request) -> Result { + let (parts, body) = request.into_parts(); + + let url = Url::parse(&parts.uri.to_string()) + .map_err(|_| FaithError::from(FaithErrorKind::InvalidUrl))?; + + let mut headers = Vec::new(); + for (name, value) in parts.headers.iter() { + let Ok(value) = value.to_str() else { + return Err(FaithErrorKind::InvalidHeader.into()); + }; + headers.push((name.to_string(), value.to_owned())); + } + + let body = body.into(); + Ok(Self::Request(Box::new(Request { + url, + options: RequestOptions { + method: Some(parts.method.to_string()), + headers: (!headers.is_empty()).then_some(headers), + ..RequestOptions::default() + }, + body: if body.is_empty() { + RequestBody::None + } else { + RequestBody::Bytes(body) + }, + }))) + } +} From 1bdf705abae39baec8d8b2365a918f1381e82658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:43:59 +1200 Subject: [PATCH 58/61] S1: a feature per capability, starting with cookies and DNS Each is default-on; turning one off drops the component crate, the reqwest feature behind it, and the API that only means something with it present. alt-svc gains a dns feature of its own for the HTTPS-record sink, so the Alt-Svc store is usable without the resolver crate. napi's derives do not honour cfg on an object field or an impl method, so the binding removes methods through separate gated impl blocks and refuses an option group the build cannot honour. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 4 +- crates/web-faith-alt-svc/Cargo.toml | 7 +- crates/web-faith-alt-svc/src/https_sink.rs | 80 +++++++++ crates/web-faith-alt-svc/src/lib.rs | 9 +- crates/web-faith-alt-svc/src/prober.rs | 77 --------- crates/web-faith-cookies/Cargo.toml | 2 +- crates/web-faith-dns/Cargo.toml | 2 +- crates/web-faith-napi/Cargo.toml | 13 +- crates/web-faith-napi/src/agent.rs | 177 +++++++++++++------- crates/web-faith-napi/src/agent/convert.rs | 12 ++ crates/web-faith-napi/src/agent/options.rs | 7 +- crates/web-faith/Cargo.toml | 11 +- crates/web-faith/src/agent.rs | 15 +- crates/web-faith/src/agent/build.rs | 33 +++- crates/web-faith/src/agent/warm.rs | 5 + crates/web-faith/src/builder.rs | 12 ++ crates/web-faith/src/client.rs | 26 ++- crates/web-faith/src/options.rs | 12 ++ crates/web-faith/src/retry.rs | 72 +------- crates/web-faith/src/retry/stale_address.rs | 69 ++++++++ index.d.ts | 64 +++---- 21 files changed, 446 insertions(+), 263 deletions(-) create mode 100644 crates/web-faith-alt-svc/src/https_sink.rs create mode 100644 crates/web-faith/src/retry/stale_address.rs diff --git a/Cargo.toml b/Cargo.toml index 5cb07ed..2a9593d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,8 +54,6 @@ napi = { version = "3.7.0", features = [ napi-build = "2.3.1" napi-derive = "3.4.0" reqwest = { version = "0.13.4", default-features = false, features = [ - "cookies", - "hickory-dns", "http2", "json", "rustls", @@ -74,7 +72,7 @@ time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" web-faith = { version = "0.7.0", path = "crates/web-faith", default-features = false } -web-faith-alt-svc = { version = "0.7.0", path = "crates/web-faith-alt-svc" } +web-faith-alt-svc = { version = "0.7.0", path = "crates/web-faith-alt-svc", default-features = false } web-faith-conn-tracker = { version = "0.7.0", path = "crates/web-faith-conn-tracker" } web-faith-cookies = { version = "0.7.0", path = "crates/web-faith-cookies" } web-faith-dns = { version = "0.7.0", path = "crates/web-faith-dns" } diff --git a/crates/web-faith-alt-svc/Cargo.toml b/crates/web-faith-alt-svc/Cargo.toml index d065d44..eea8b42 100644 --- a/crates/web-faith-alt-svc/Cargo.toml +++ b/crates/web-faith-alt-svc/Cargo.toml @@ -18,4 +18,9 @@ reqwest.workspace = true reqwest-middleware.workspace = true tokio.workspace = true url.workspace = true -web-faith-dns.workspace = true +web-faith-dns = { workspace = true, optional = true } + +[features] +default = ["dns"] +# Feed `HTTPS` DNS records into the store, so an origin is probe-worthy before anything connects. +dns = ["dep:web-faith-dns"] diff --git a/crates/web-faith-alt-svc/src/https_sink.rs b/crates/web-faith-alt-svc/src/https_sink.rs new file mode 100644 index 0000000..0fce8a6 --- /dev/null +++ b/crates/web-faith-alt-svc/src/https_sink.rs @@ -0,0 +1,80 @@ +use std::sync::{Arc, Weak}; + +use crate::{cache::AltSvcCache, prober::H3Prober}; + +/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is +/// probe-worthy from its first request rather than from the first TCP response carrying an +/// `Alt-Svc` header. +/// +/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), +/// which is the only place that holds all three: the resolver is built before the cache, and the +/// prober holds a client that holds the resolver, so nothing lower down can own this. +/// +/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the +/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could +/// name. Recording it there is right whichever request triggered the lookup, because what the +/// record describes does not depend on who asked. +// spec:H3UP#advertisements-from-dns +// spec:DNS#https-records +pub struct H3HttpsSink { + cache: Arc, + /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, + /// and the resolver holds this sink. A strong reference here would close that ring and leak + /// the whole graph — connection pool included — past `Agent::close`, which works by dropping + /// the client. The agent owns the only strong reference, so this lives exactly as long as the + /// agent's prober does. + /// + /// `None` rather than a dead handle when probing is off, where an advertisement is acted on + /// inline by the next foreground request instead. + // spec:PROBE + prober: Option>, +} + +impl std::fmt::Debug for H3HttpsSink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("H3HttpsSink") + .field("probing", &self.prober.is_some()) + .finish() + } +} + +impl H3HttpsSink { + pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { + Self { + cache, + prober: prober.map(Arc::downgrade), + } + } + + /// The origin a record at `host` describes: the default HTTPS port, which is the port whose + /// record lives at the bare name. + fn origin_url(host: &str) -> Option { + reqwest::Url::parse(&format!("https://{host}")).ok() + } +} + +impl web_faith_dns::HttpsSink for H3HttpsSink { + fn wants(&self, host: &str) -> bool { + Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) + } + + fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { + let Some(url) = Self::origin_url(host) else { + return; + }; + + self.cache + .record_https_record(&url, advertisement.port, advertisement.ttl); + + // Probe straight away rather than waiting for the request that triggered the lookup to + // finish: the point of reading DNS is that the path can be verified while that request is + // still on TCP, so the one after it upgrades. + // + // A prober that has gone means the agent was closed (or rebuilt) while this query was in + // flight; the advertisement above is still worth keeping, but there is nothing left to + // probe it with, and resurrecting a dropped client to try would be exactly wrong. + if let Some(prober) = self.prober.as_ref().and_then(Weak::upgrade) { + prober.maybe_probe(&url); + } + } +} diff --git a/crates/web-faith-alt-svc/src/lib.rs b/crates/web-faith-alt-svc/src/lib.rs index 2e23b05..7587f5e 100644 --- a/crates/web-faith-alt-svc/src/lib.rs +++ b/crates/web-faith-alt-svc/src/lib.rs @@ -24,6 +24,10 @@ mod cache; mod header; + +#[cfg(feature = "dns")] +mod https_sink; + mod middleware; mod prober; @@ -31,5 +35,8 @@ pub use cache::{ AltSvcAdvertisement, AltSvcCache, AltSvcCacheConfig, AltSvcEntry, PathTime, SLOW_FLOOR_MS, }; pub use header::parse_alt_svc_header; +#[cfg(feature = "dns")] +pub use https_sink::H3HttpsSink; pub use middleware::{AltSvcMiddleware, ArrivalStamp}; -pub use prober::{H3HttpsSink, H3Prober}; + +pub use prober::H3Prober; diff --git a/crates/web-faith-alt-svc/src/prober.rs b/crates/web-faith-alt-svc/src/prober.rs index c66630a..65f41cf 100644 --- a/crates/web-faith-alt-svc/src/prober.rs +++ b/crates/web-faith-alt-svc/src/prober.rs @@ -126,80 +126,3 @@ impl H3Prober { } } } - -/// Feeds `HTTPS` DNS records into the upgrade layer, so an origin advertising `alpn="h3"` is -/// probe-worthy from its first request rather than from the first TCP response carrying an -/// `Alt-Svc` header. -/// -/// Installed on the resolver by the agent (see [`web_faith_dns::FaithResolver::set_https_sink`]), -/// which is the only place that holds all three: the resolver is built before the cache, and the -/// prober holds a client that holds the resolver, so nothing lower down can own this. -/// -/// The record is read at the bare name, which per RFC 9460 is the record for the origin at the -/// default HTTPS port; the resolver sees only a hostname, so that is also the only origin it could -/// name. Recording it there is right whichever request triggered the lookup, because what the -/// record describes does not depend on who asked. -// spec:H3UP#advertisements-from-dns -// spec:DNS#https-records -pub struct H3HttpsSink { - cache: Arc, - /// Weak, and load-bearingly so: the prober holds the client, the client holds the resolver, - /// and the resolver holds this sink. A strong reference here would close that ring and leak - /// the whole graph — connection pool included — past `Agent::close`, which works by dropping - /// the client. The agent owns the only strong reference, so this lives exactly as long as the - /// agent's prober does. - /// - /// `None` rather than a dead handle when probing is off, where an advertisement is acted on - /// inline by the next foreground request instead. - // spec:PROBE - prober: Option>, -} - -impl std::fmt::Debug for H3HttpsSink { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("H3HttpsSink") - .field("probing", &self.prober.is_some()) - .finish() - } -} - -impl H3HttpsSink { - pub fn new(cache: Arc, prober: Option<&Arc>) -> Self { - Self { - cache, - prober: prober.map(Arc::downgrade), - } - } - - /// The origin a record at `host` describes: the default HTTPS port, which is the port whose - /// record lives at the bare name. - fn origin_url(host: &str) -> Option { - reqwest::Url::parse(&format!("https://{host}")).ok() - } -} - -impl web_faith_dns::HttpsSink for H3HttpsSink { - fn wants(&self, host: &str) -> bool { - Self::origin_url(host).is_some_and(|url| self.cache.wants_https_record(&url)) - } - - fn record(&self, host: &str, advertisement: web_faith_dns::HttpsAdvertisement) { - let Some(url) = Self::origin_url(host) else { - return; - }; - - self.cache - .record_https_record(&url, advertisement.port, advertisement.ttl); - - // Probe straight away rather than waiting for the request that triggered the lookup to - // finish: the point of reading DNS is that the path can be verified while that request is - // still on TCP, so the one after it upgrades. - // - // A prober that has gone means the agent was closed (or rebuilt) while this query was in - // flight; the advertisement above is still worth keeping, but there is nothing left to - // probe it with, and resurrecting a dropped client to try would be exactly wrong. - if let Some(prober) = self.prober.as_ref().and_then(std::sync::Weak::upgrade) { - prober.maybe_probe(&url); - } - } -} diff --git a/crates/web-faith-cookies/Cargo.toml b/crates/web-faith-cookies/Cargo.toml index a186c23..c678d0f 100644 --- a/crates/web-faith-cookies/Cargo.toml +++ b/crates/web-faith-cookies/Cargo.toml @@ -24,4 +24,4 @@ url.workspace = true [features] # Implement reqwest's `CookieStore`, so the jar can serve as its cookie provider. -reqwest = ["dep:reqwest"] +reqwest = ["dep:reqwest", "reqwest/cookies"] diff --git a/crates/web-faith-dns/Cargo.toml b/crates/web-faith-dns/Cargo.toml index 7be2016..4d3ca4f 100644 --- a/crates/web-faith-dns/Cargo.toml +++ b/crates/web-faith-dns/Cargo.toml @@ -20,4 +20,4 @@ reqwest = { workspace = true, optional = true } [features] # Implement reqwest's `Resolve`, so the resolver can be installed on its client. -reqwest = ["dep:reqwest"] +reqwest = ["dep:reqwest", "reqwest/hickory-dns"] diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 952d699..7d4eed3 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -48,8 +48,8 @@ url.workspace = true web-faith.workspace = true web-faith-alt-svc = { workspace = true, optional = true } web-faith-conn-tracker.workspace = true -web-faith-cookies = { workspace = true, features = ["reqwest"] } -web-faith-dns = { workspace = true, features = ["reqwest"] } +web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } +web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } web-faith-encoding.workspace = true [target.'cfg(target_os = "linux")'.dependencies] @@ -64,5 +64,12 @@ windows.workspace = true napi-build.workspace = true [features] -default = ["http3"] +default = ["cookies", "dns", "http3"] +cookies = ["dep:web-faith-cookies", "reqwest/cookies", "web-faith/cookies"] +dns = [ + "dep:web-faith-dns", + "reqwest/hickory-dns", + "web-faith/dns", + "web-faith-alt-svc?/dns", +] http3 = ["reqwest/http3", "dep:web-faith-alt-svc", "web-faith/http3"] diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index e2bcdbf..466e572 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -1,12 +1,9 @@ //! The `Agent` class, as JavaScript sees it. -use std::str::FromStr as _; - use napi::bindgen_prelude::{PromiseRaw, within_runtime_if_available}; use napi::Env; use napi_derive::napi; -use reqwest::Url; use crate::{ async_task::faith_promise, @@ -64,6 +61,7 @@ impl From for AgentStats { } /// One entry of `Agent.resolvers()`: a DNS server the agent resolves through (spec:OBS#resolvers). +#[cfg(feature = "dns")] #[napi(object)] #[derive(Debug, Clone)] pub struct ResolverInfo { @@ -103,6 +101,7 @@ impl Agent { } pub fn with_options(options: AgentOptions) -> Result { + refuse_absent_capabilities(&options)?; let options = web_faith::options::AgentOptions::from(options); // A napi callback can run outside the runtime, and building the HTTP/3 endpoint needs to be // inside one, so the client is constructed within whichever runtime is to hand. @@ -155,44 +154,6 @@ impl Agent { self.inner.network_changed(); } - /// Add a cookie into the agent. - /// - /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying - /// the scheme and host they read, so this does nothing if: - /// - the cookie store is disabled - /// - the url is malformed - /// - the cookie does not parse - /// - a `__Host-` or `__Secure-` name prefix is not satisfied - /// - the cookie is larger than `cookies.maxSize` - #[napi] - pub fn add_cookie(&self, url: String, cookie: String) { - let Ok(url) = Url::from_str(&url) else { - return; - }; - - let Some(jar) = self.inner.cookies() else { - return; - }; - - jar.add_cookie_str(&cookie, &url); - } - - /// Retrieve a cookie from the store. - /// - /// Returns `null` if: - /// - there's no cookie at this url - /// - the cookie store is disabled - /// - the url is malformed - /// - the cookie cannot be represented as a string - #[napi] - pub fn get_cookie(&self, url: String) -> Option { - let url = Url::from_str(&url).ok()?; - self.inner - .cookies()? - .request_cookie_header(&url) - .and_then(|value| value.to_str().ok().map(ToOwned::to_owned)) - } - /// Returns statistics gathered by this agent: /// /// - `requestsSent` @@ -216,26 +177,6 @@ impl Agent { connections_for_napi(&self.inner.conn_tracker, env) } - /// Returns the DNS servers this agent resolves through, in the order they are queried, so - /// "are my lookups actually encrypted" is answerable from inside the process. - /// - /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, - /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). - /// The list is empty until the resolver has been used, because it reads its configuration on - /// first use, and empty for an agent using the system resolver. - #[napi] - pub fn resolvers(&self) -> Vec { - self.inner - .resolvers() - .into_iter() - .map(|report| ResolverInfo { - address: report.address, - transport: report.transport, - source: report.source, - }) - .collect() - } - /// Warm the DNS cache for `host`, so a later request to it skips the lookup. /// /// Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, @@ -292,3 +233,117 @@ impl Agent { fn caller_error(env: &Env, err: FaithError) -> napi::Error { napi::Error::from(err.into_js_error(env)) } + +/// Refuse an option group this build cannot honour. +/// +/// A Cargo feature drops the capability and, on the Rust surface, the API that reaches it. napi's +/// object derive does not honour `#[cfg]` on a field, so an options object here keeps its full shape +/// whatever the build; asking for a capability that is not compiled in is refused rather than +/// quietly ignored, so a slim build says so instead of appearing to work. +fn refuse_absent_capabilities(options: &AgentOptions) -> Result<(), FaithError> { + let absent = |group: &str| -> Result<(), FaithError> { + Err(FaithError::new( + web_faith::FaithErrorKind::Config, + Some(format!("this build has no {group} support")), + )) + }; + + #[cfg(not(feature = "cookies"))] + if options.cookies.is_some() { + return absent("cookie"); + } + + // `dns.overrides` reaches reqwest rather than Faith's resolver, so it is honoured either way; + // every other setting in the group configures the resolver this build does not have. + #[cfg(not(feature = "dns"))] + if options.dns.as_ref().is_some_and(|dns| { + dns.system.is_some() + || dns.servers.is_some() + || dns.timeout.is_some() + || dns.search_domains.is_some() + || dns.ndots.is_some() + || dns.hosts_file.is_some() + || dns.exempt_domains.is_some() + || dns.serve_stale.is_some() + || dns.max_stale.is_some() + }) { + return absent("resolver"); + } + + let _ = (options, absent); + Ok(()) +} + +/// The resolver's own observability, which needs a resolver of Faith's own to report on. +#[cfg(feature = "dns")] +#[napi] +impl Agent { + /// Returns the DNS servers this agent resolves through, in the order they are queried, so + /// "are my lookups actually encrypted" is answerable from inside the process. + /// + /// Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, + /// `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). + /// The list is empty until the resolver has been used, because it reads its configuration on + /// first use, and empty for an agent using the system resolver. + #[napi] + pub fn resolvers(&self) -> Vec { + self.inner + .resolvers() + .into_iter() + .map(|report| ResolverInfo { + address: report.address, + transport: report.transport, + source: report.source, + }) + .collect() + } +} + +/// The cookie jar's verbs, which exist when the build keeps a jar. +#[cfg(feature = "cookies")] +use std::str::FromStr as _; + +#[cfg(feature = "cookies")] +use reqwest::Url; + +#[cfg(feature = "cookies")] +#[napi] +impl Agent { + /// Add a cookie into the agent. + /// + /// The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying + /// the scheme and host they read, so this does nothing if: + /// - the cookie store is disabled + /// - the url is malformed + /// - the cookie does not parse + /// - a `__Host-` or `__Secure-` name prefix is not satisfied + /// - the cookie is larger than `cookies.maxSize` + #[napi] + pub fn add_cookie(&self, url: String, cookie: String) { + let Ok(url) = Url::from_str(&url) else { + return; + }; + + let Some(jar) = self.inner.cookies() else { + return; + }; + + jar.add_cookie_str(&cookie, &url); + } + + /// Retrieve a cookie from the store. + /// + /// Returns `null` if: + /// - there's no cookie at this url + /// - the cookie store is disabled + /// - the url is malformed + /// - the cookie cannot be represented as a string + #[napi] + pub fn get_cookie(&self, url: String) -> Option { + let url = Url::from_str(&url).ok()?; + self.inner + .cookies()? + .request_cookie_header(&url) + .and_then(|value| value.to_str().ok().map(ToOwned::to_owned)) + } +} diff --git a/crates/web-faith-napi/src/agent/convert.rs b/crates/web-faith-napi/src/agent/convert.rs index 59f2979..69c8e06 100644 --- a/crates/web-faith-napi/src/agent/convert.rs +++ b/crates/web-faith-napi/src/agent/convert.rs @@ -3,6 +3,8 @@ use http_cache_reqwest::CacheMode; use napi::{Either, bindgen_prelude::Buffer}; use web_faith::{client::RedirectPolicy, options}; + +#[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; use crate::agent::{AgentOptions, CacheStore, Http3Congestion}; @@ -25,12 +27,14 @@ impl From for options::AgentOptions { shared: cache.shared, }), // `false` and an absent value both mean no jar; `true` means one with default limits. + #[cfg(feature = "cookies")] cookies: match opts.cookies { None | Some(Either::A(false)) => None, Some(Either::A(true)) => Some(CookieLimits::default()), Some(Either::B(cookies)) => Some((&cookies).into()), }, dns: opts.dns.map(|dns| options::DnsOptions { + #[cfg(feature = "dns")] system: dns.system, overrides: dns.overrides.map(|overrides| { overrides @@ -41,13 +45,21 @@ impl From for options::AgentOptions { }) .collect() }), + #[cfg(feature = "dns")] servers: dns.servers, + #[cfg(feature = "dns")] timeout: dns.timeout, + #[cfg(feature = "dns")] search_domains: dns.search_domains, + #[cfg(feature = "dns")] ndots: dns.ndots, + #[cfg(feature = "dns")] hosts_file: dns.hosts_file, + #[cfg(feature = "dns")] exempt_domains: dns.exempt_domains, + #[cfg(feature = "dns")] serve_stale: dns.serve_stale, + #[cfg(feature = "dns")] max_stale: dns.max_stale, }), flow_control: opts.flow_control.map(|flow| options::FlowControlOptions { diff --git a/crates/web-faith-napi/src/agent/options.rs b/crates/web-faith-napi/src/agent/options.rs index 051cc22..3237fa1 100644 --- a/crates/web-faith-napi/src/agent/options.rs +++ b/crates/web-faith-napi/src/agent/options.rs @@ -1,11 +1,15 @@ //! The `AgentOptions` object as JavaScript spells it, and the option groups under it. -use std::{fmt::Debug, time::Duration}; +use std::fmt::Debug; use napi::{Either, bindgen_prelude::Buffer}; use napi_derive::napi; +#[cfg(feature = "cookies")] +use std::time::Duration; use web_faith::client::RedirectPolicy; + +#[cfg(feature = "cookies")] use web_faith_cookies::{ CookieLimits, DEFAULT_MAX_AGE, DEFAULT_MAX_PER_HOST, DEFAULT_MAX_SIZE, DEFAULT_MAX_TOTAL, }; @@ -83,6 +87,7 @@ pub struct AgentCookieOptions { pub max_total: Option, } +#[cfg(feature = "cookies")] impl From<&AgentCookieOptions> for CookieLimits { fn from(options: &AgentCookieOptions) -> Self { Self { diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index fb2e987..709a80d 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -32,11 +32,16 @@ tokio.workspace = true url.workspace = true web-faith-conn-tracker.workspace = true web-faith-encoding.workspace = true -web-faith-cookies = { workspace = true, features = ["reqwest"] } -web-faith-dns = { workspace = true, features = ["reqwest"] } +web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } +web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } web-faith-alt-svc = { workspace = true, optional = true } [features] -default = ["http3"] +default = ["cookies", "dns", "http3"] +# The cookie jar, and the agent option and handle that reach it. +cookies = ["dep:web-faith-cookies", "reqwest/cookies"] +# Faith's own caching resolver, which a prefetch warms and an `HTTPS` record lookup reads. +# Without it names resolve through the platform, as reqwest resolves them. +dns = ["dep:web-faith-dns", "reqwest/hickory-dns", "web-faith-alt-svc?/dns"] # Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 96224dd..1d68b7e 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -13,7 +13,11 @@ use reqwest::Client; use reqwest_middleware::ClientWithMiddleware; use url::Url; use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; + +#[cfg(feature = "cookies")] use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] use web_faith_dns::{FaithResolver, ResolverReport}; #[cfg(feature = "http3")] @@ -25,7 +29,7 @@ use crate::{ warm_up::origin_key, }; -#[cfg(feature = "http3")] +#[cfg(all(feature = "http3", feature = "dns"))] use crate::client::install_https_sink; mod build; @@ -75,6 +79,7 @@ pub struct Live { /// The DNS resolver, shared with the client so a prefetch warms the cache requests read. `None` /// under the system resolver, where there is no such cache. // spec:WARM + #[cfg(feature = "dns")] pub dns_resolver: Option, #[cfg(feature = "http3")] pub alt_svc_cache: Option>, @@ -109,6 +114,7 @@ pub struct Agent { // spec:NETCHG#reach-across-the-subsystems pub warm_generation: Arc, /// The jar outlives a close and stays readable from a closed agent. + #[cfg(feature = "cookies")] pub cookie_jar: Option>, pub stats: Arc, pub conn_tracker: Arc, @@ -147,6 +153,7 @@ impl Agent { /// through the type `web-faith-cookies` documents. It outlives a close and stays readable from a /// closed agent. // spec:COOK + #[cfg(feature = "cookies")] pub fn cookies(&self) -> Option<&Arc> { self.cookie_jar.as_ref() } @@ -166,6 +173,7 @@ impl Agent { } /// The DNS resolver, if the agent has one of its own and is still open. + #[cfg(feature = "dns")] pub fn dns_resolver(&self) -> Option { self.live() .as_ref() @@ -262,7 +270,9 @@ impl Agent { // validated at construction, so a failure here is not the caller's to answer for, and an // agent that still works on the old network beats one that works nowhere. let built = self.recipe.build( + #[cfg(feature = "cookies")] self.cookie_jar.as_ref(), + #[cfg(feature = "dns")] live.dns_resolver.as_ref(), #[cfg(feature = "http3")] live.alt_svc_cache.as_ref(), @@ -279,6 +289,7 @@ impl Agent { // The sink holds the prober, which has just been replaced along with the client // it sends on; leaving the old one installed would aim DNS-triggered probes at a // client that has been dropped. + #[cfg(feature = "dns")] install_https_sink( live.dns_resolver.as_ref(), live.alt_svc_cache.as_ref(), @@ -294,6 +305,7 @@ impl Agent { // resolver drops what it read off the old one and reads again when next used. Under the // system resolver there is no resolver here and so nothing to reset. // spec:DNS + #[cfg(feature = "dns")] if let Some(resolver) = &live.dns_resolver { resolver.reset(); } @@ -337,6 +349,7 @@ impl Agent { /// The list is empty until the resolver has been used, because it reads its configuration on /// first use, and empty for an agent using the system resolver. // spec:OBS#resolvers + #[cfg(feature = "dns")] pub fn resolvers(&self) -> Vec { self.dns_resolver() .as_ref() diff --git a/crates/web-faith/src/agent/build.rs b/crates/web-faith/src/agent/build.rs index a3e9de1..ec05f05 100644 --- a/crates/web-faith/src/agent/build.rs +++ b/crates/web-faith/src/agent/build.rs @@ -16,7 +16,11 @@ use http_cache_reqwest::{ use moka::sync::Cache as MokaCache; use reqwest::{Identity, tls::Certificate}; use web_faith_conn_tracker::ConnectionTracker; + +#[cfg(feature = "cookies")] use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] use web_faith_dns::{ DEFAULT_MAX_STALE, FaithResolver, ResolverSettings, ServerSpec, parse_domains, }; @@ -36,10 +40,10 @@ use crate::{ }; #[cfg(feature = "http3")] -use crate::{ - client::{H3UpgradeRecipe, install_https_sink}, - options::Http3Congestion, -}; +use crate::{client::H3UpgradeRecipe, options::Http3Congestion}; + +#[cfg(all(feature = "http3", feature = "dns"))] +use crate::client::install_https_sink; impl Agent { /// This is what both surfaces land on, so the defaults a caller gets are settled here rather @@ -51,6 +55,7 @@ impl Agent { // agent's clients are built from (spec:NETCHG). let AgentOptions { cache, + #[cfg(feature = "cookies")] cookies, dns, flow_control, @@ -91,13 +96,20 @@ impl Agent { // `cookies: true` takes the default limits; an options object tunes them. (spec:COOK) // The jar is installed on the client by the recipe, so it survives a rebuild // (spec:NETCHG#what-the-signal-keeps). + #[cfg(feature = "cookies")] let cookie_jar = cookies.map(|limits| Arc::new(FaithJar::new(limits))); let dns = dns.unwrap_or_default(); + // Without Faith's own resolver every name goes to the platform, so that is the only + // answer there is to give. + #[cfg(feature = "dns")] let dns_system = dns.system.unwrap_or(false); + #[cfg(not(feature = "dns"))] + let dns_system = true; // Naming servers and asking for the system resolver at once is a contradiction rather than // a preference, since the system resolver is not Faith's to point at listed servers // (spec:DNS#system-resolver). + #[cfg(feature = "dns")] if dns_system && dns .servers @@ -143,6 +155,7 @@ impl Agent { // order each resolver is reached by (spec:DNS#transports). The system resolver // (getaddrinfo) has no in-process cache Faith can warm, so no resolver is installed there // and `prefetchDns` resolves as a no-op (spec:WARM). + #[cfg(feature = "dns")] let dns_resolver = if dns_system { None } else { @@ -486,7 +499,9 @@ impl Agent { Self::build( recipe, settings, + #[cfg(feature = "cookies")] cookie_jar, + #[cfg(feature = "dns")] dns_resolver, #[cfg(feature = "http3")] alt_svc_cache, @@ -510,13 +525,15 @@ impl Agent { pub fn build( recipe: ClientRecipe, settings: AgentSettings, - cookie_jar: Option>, - dns_resolver: Option, + #[cfg(feature = "cookies")] cookie_jar: Option>, + #[cfg(feature = "dns")] dns_resolver: Option, #[cfg(feature = "http3")] alt_svc_cache: Option>, ) -> Result { let conn_timeout = recipe.conn_timeout(); let built = recipe.build( + #[cfg(feature = "cookies")] cookie_jar.as_ref(), + #[cfg(feature = "dns")] dns_resolver.as_ref(), #[cfg(feature = "http3")] alt_svc_cache.as_ref(), @@ -525,7 +542,7 @@ impl Agent { // Only now do all three exist: the resolver is built before the cache, and the prober // holds a client that holds the resolver, so this is the earliest the loop can be closed // (spec:DNS#https-records). - #[cfg(feature = "http3")] + #[cfg(all(feature = "http3", feature = "dns"))] install_https_sink( dns_resolver.as_ref(), alt_svc_cache.as_ref(), @@ -537,6 +554,7 @@ impl Agent { live: Arc::new(RwLock::new(Some(Live { client: built.client, raw_client: built.raw_client, + #[cfg(feature = "dns")] dns_resolver, #[cfg(feature = "http3")] alt_svc_cache, @@ -552,6 +570,7 @@ impl Agent { .time_to_live(Duration::from_secs(300)) .build(), warm_generation: Default::default(), + #[cfg(feature = "cookies")] cookie_jar, stats: Default::default(), conn_tracker: ConnectionTracker::new(conn_timeout), diff --git a/crates/web-faith/src/agent/warm.rs b/crates/web-faith/src/agent/warm.rs index c13f767..b8272ba 100644 --- a/crates/web-faith/src/agent/warm.rs +++ b/crates/web-faith/src/agent/warm.rs @@ -31,11 +31,16 @@ impl Agent { return Err(FaithErrorKind::AddressParse.into()); }; + #[cfg(feature = "dns")] let resolver = self.dns_resolver(); Ok(async move { + // Nothing to warm without Faith's own resolver: the platform's cache is not ours to fill. + #[cfg(feature = "dns")] if let Some(resolver) = resolver { resolver.prefetch(&host).await; } + #[cfg(not(feature = "dns"))] + let _ = host; }) } diff --git a/crates/web-faith/src/builder.rs b/crates/web-faith/src/builder.rs index 3422441..973a3c2 100644 --- a/crates/web-faith/src/builder.rs +++ b/crates/web-faith/src/builder.rs @@ -25,6 +25,8 @@ use std::{net::IpAddr, time::Duration}; use http_cache_reqwest::CacheMode; + +#[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; use crate::{ @@ -106,6 +108,7 @@ impl AgentBuilder { } /// Keep a cookie jar, enforcing these limits. Without this call the agent stores no cookies. + #[cfg(feature = "cookies")] pub fn cookies(mut self, limits: CookieLimits) -> Self { self.options.cookies = Some(limits); self @@ -187,6 +190,7 @@ pub struct DnsBuilder { impl DnsBuilder { /// Resolve through the operating system rather than Faith's own resolver. + #[cfg(feature = "dns")] pub fn system(mut self, system: bool) -> Self { self.group.system = Some(system); self @@ -209,48 +213,56 @@ impl DnsBuilder { } /// The resolvers to query, in order. Each is a URL whose scheme picks the transport. + #[cfg(feature = "dns")] pub fn servers(mut self, servers: impl IntoIterator>) -> Self { self.group.servers = Some(servers.into_iter().map(Into::into).collect()); self } /// Bound resolution across the whole server list. + #[cfg(feature = "dns")] pub fn timeout(mut self, timeout: Duration) -> Self { self.group.timeout = Some(millis(timeout)); self } /// Suffixes to try for an unqualified name. + #[cfg(feature = "dns")] pub fn search_domains(mut self, domains: impl IntoIterator>) -> Self { self.group.search_domains = Some(domains.into_iter().map(Into::into).collect()); self } /// How many dots a name must contain before it is tried as given, ahead of the search list. + #[cfg(feature = "dns")] pub fn ndots(mut self, ndots: u32) -> Self { self.group.ndots = Some(ndots); self } /// Consult the system hosts file. + #[cfg(feature = "dns")] pub fn hosts_file(mut self, hosts_file: bool) -> Self { self.group.hosts_file = Some(hosts_file); self } /// Names to send to the system resolver whatever the rest of the configuration says. + #[cfg(feature = "dns")] pub fn exempt_domains(mut self, domains: impl IntoIterator>) -> Self { self.group.exempt_domains = Some(domains.into_iter().map(Into::into).collect()); self } /// Serve an expired answer while a fresh lookup runs. + #[cfg(feature = "dns")] pub fn serve_stale(mut self, serve_stale: bool) -> Self { self.group.serve_stale = Some(serve_stale); self } /// How far past expiry an answer may still be served. + #[cfg(feature = "dns")] pub fn max_stale(mut self, max_stale: Duration) -> Self { self.group.max_stale = Some(millis(max_stale)); self diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs index 1268477..f166c38 100644 --- a/crates/web-faith/src/client.rs +++ b/crates/web-faith/src/client.rs @@ -8,25 +8,34 @@ use std::{ net::{IpAddr, SocketAddr}, - sync::Arc, time::Duration, }; +// Reached only by the pieces the agent shares into a rebuilt client, each behind its own feature. +#[cfg(any(feature = "cookies", feature = "http3"))] +use std::sync::Arc; + use http::header::HeaderMap; use http_cache_reqwest::{ CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions, MokaManager, }; use reqwest::{Client, Identity, redirect::Policy, tls::Certificate}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +#[cfg(feature = "cookies")] use web_faith_cookies::FaithJar; + +#[cfg(feature = "dns")] use web_faith_dns::FaithResolver; +#[cfg(feature = "dns")] +use crate::retry::StaleAddressRetry; + #[cfg(feature = "http3")] use web_faith_alt_svc::{AltSvcCache, AltSvcMiddleware, H3Prober}; use crate::{ error::{FaithError, FaithErrorKind}, - retry::{DeadConnectionRetry, StaleAddressRetry}, + retry::DeadConnectionRetry, }; #[cfg(feature = "http3")] @@ -236,7 +245,7 @@ pub struct ClientRecipe { /// /// Re-called on a network change, where the prober is rebuilt with the client it sends on. // spec:DNS#https-records -#[cfg(feature = "http3")] +#[cfg(all(feature = "http3", feature = "dns"))] pub fn install_https_sink( dns_resolver: Option<&FaithResolver>, alt_svc_cache: Option<&Arc>, @@ -282,8 +291,8 @@ impl ClientRecipe { // spec:NETCHG#what-the-signal-keeps pub fn build( &self, - cookie_jar: Option<&Arc>, - dns_resolver: Option<&FaithResolver>, + #[cfg(feature = "cookies")] cookie_jar: Option<&Arc>, + #[cfg(feature = "dns")] dns_resolver: Option<&FaithResolver>, #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, ) -> Result { let mut client = Client::builder() @@ -295,6 +304,7 @@ impl ClientRecipe { client = client.local_address(ip); } + #[cfg(feature = "cookies")] if let Some(jar) = cookie_jar { client = client.cookie_provider(jar.clone()); } @@ -306,6 +316,7 @@ impl ClientRecipe { client = client.resolve_to_addrs(domain, addresses); } + #[cfg(feature = "dns")] if self.dns_system { client = client.no_hickory_dns(); } else if let Some(resolver) = dns_resolver { @@ -469,7 +480,10 @@ impl ClientRecipe { // treatment as the original one: the two answer different questions, and a // fresh address deserves its own chance to draw a dead pooled connection. // Inside the Alt-Svc and cache layers for the reason given below. - client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); + #[cfg(feature = "dns")] + { + client = client.with(StaleAddressRetry::new(dns_resolver.cloned())); + } // Registered last, so it sits innermost and wraps nothing but the exchange // itself. Inside the Alt-Svc layer rather than outside it, because each diff --git a/crates/web-faith/src/options.rs b/crates/web-faith/src/options.rs index 5e618e9..26a6812 100644 --- a/crates/web-faith/src/options.rs +++ b/crates/web-faith/src/options.rs @@ -11,6 +11,8 @@ use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; use http_cache_reqwest::CacheMode; + +#[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; use crate::client::{ @@ -72,6 +74,7 @@ pub struct DnsOptions { /// Default: false. /// /// [Hickory]: https://hickory-dns.org/ + #[cfg(feature = "dns")] pub system: Option, /// Override DNS resolution for specific domains. This takes effect even with `dns.system: true`. /// @@ -98,31 +101,37 @@ pub struct DnsOptions { /// its scheme is not one of the above, and combining it with `dns.system` throws. /// /// Default: system discovery. + #[cfg(feature = "dns")] pub servers: Option>, /// Bound name resolution across the whole server list, in milliseconds. Exhausting several dead /// servers costs a single timeout rather than one per server. /// /// Default: 5000. + #[cfg(feature = "dns")] pub timeout: Option, /// Replace the system's search list, the domains appended to a name that is not fully /// qualified. Independent of `dns.servers`. /// /// Default: the system's search list. + #[cfg(feature = "dns")] pub search_domains: Option>, /// How many dots a name must contain before it is tried as given, ahead of the search list. /// Independent of `dns.servers`. /// /// Default: the system's setting. + #[cfg(feature = "dns")] pub ndots: Option, /// Turn hosts-file lookup on or off. When unset, follows the platform's own convention. /// /// Default: platform convention. + #[cfg(feature = "dns")] pub hosts_file: Option, /// Further domains to exempt from the configured or encrypted resolver, for the internal /// suffixes a network uses. Added to the always-exempt `localhost`, `.local`, and the network's /// own DNS suffix; a domain is exempt when it matches an entry exactly or is a subdomain of one. /// /// Default: no extra exemptions. + #[cfg(feature = "dns")] pub exempt_domains: Option>, /// Serve an expired cache entry immediately and refresh it in the background, rather than making /// the lookup wait for a fresh answer. A host's address changes rarely, so an expired answer is @@ -133,6 +142,7 @@ pub struct DnsOptions { /// expired entry is discarded and the lookup blocks on a fresh answer. /// /// Default: true. + #[cfg(feature = "dns")] pub serve_stale: Option, /// How far past expiry an answer may still be served, in milliseconds. An entry older than this /// is discarded rather than served: an answer stale enough stops being evidence about where the @@ -140,6 +150,7 @@ pub struct DnsOptions { /// did change. /// /// Default: 3600000 (one hour). + #[cfg(feature = "dns")] pub max_stale: Option, } @@ -573,6 +584,7 @@ pub struct AgentOptions { /// /// You may use `agent.getCookie(url: string)` and `agent.addCookie(url: string, value: string)` to add /// and retrieve cookies from the store. + #[cfg(feature = "cookies")] pub cookies: Option, /// Settings related to DNS. This is a nested object. pub dns: Option, diff --git a/crates/web-faith/src/retry.rs b/crates/web-faith/src/retry.rs index 6177260..6c76a1c 100644 --- a/crates/web-faith/src/retry.rs +++ b/crates/web-faith/src/retry.rs @@ -15,7 +15,11 @@ use http::{Extensions, Method}; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result}; -use web_faith_dns::FaithResolver; +#[cfg(feature = "dns")] +mod stale_address; + +#[cfg(feature = "dns")] +pub use stale_address::StaleAddressRetry; /// How many times a request may be replayed before the failure reaches the caller. /// @@ -124,7 +128,8 @@ impl Middleware for DeadConnectionRetry { /// status, a connection that died mid-exchange, a body that stopped early. Those are answers, and an /// address that produced one is confirmed by definition. Only a connect failure leaves open the /// possibility that the address itself was wrong. -fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { +#[cfg(feature = "dns")] +pub(super) fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { match err { reqwest_middleware::Error::Reqwest(err) => err.is_connect(), // A middleware's own error is about this stack rather than about the network. @@ -132,69 +137,6 @@ fn failed_to_connect(err: &reqwest_middleware::Error) -> bool { } } -/// Re-resolves and attempts a request again when connecting to a stale-served address failed. -/// -/// Serving an expired DNS answer trades a round trip against the chance the address has moved, and -/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request. -// spec:DNS#when-a-stale-address-is-wrong -#[derive(Debug, Clone)] -pub struct StaleAddressRetry { - /// `None` under `dns.system: true`, where Faith holds no cache and so serves nothing stale. - resolver: Option, -} - -impl StaleAddressRetry { - pub fn new(resolver: Option) -> Self { - Self { resolver } - } -} - -#[async_trait::async_trait] -impl Middleware for StaleAddressRetry { - // spec:DNS#when-a-stale-address-is-wrong - async fn handle( - &self, - req: Request, - extensions: &mut Extensions, - next: Next<'_>, - ) -> Result { - let Some(resolver) = self.resolver.clone() else { - return next.run(req, extensions).await; - }; - - // Asked before the request runs, not after it fails. The lookup this request makes serves the - // stale entry and starts a refresh behind it, so by the time an error is in hand the entry may - // already have been replaced and the question "was this address assumed?" no longer answerable. - // - // Cloned here for the same reason the dead-connection layer clones early: sending consumes the - // body. A `ReadableStream` body does not clone, which is what leaves those requests reporting - // the connect failure rather than being attempted again -- the body has no second copy, whether - // or not it was read. - let host = req.url().host_str().map(str::to_owned); - let replay = match &host { - Some(host) if resolver.served_stale(host) => req.try_clone(), - _ => None, - }; - - let outcome = next.clone().run(req, extensions).await; - - match &outcome { - Err(err) if failed_to_connect(err) => {} - _ => return outcome, - } - - let (Some(host), Some(request)) = (host, replay) else { - return outcome; - }; - - // Drop the entry so the retry's lookup waits for a fresh answer rather than being served the - // address that just failed. One attempt only: the fresh address is confirmed rather than - // assumed, so a second failure is an answer about the origin and belongs to the caller. - resolver.invalidate_stale(&host); - next.run(request, extensions).await - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/web-faith/src/retry/stale_address.rs b/crates/web-faith/src/retry/stale_address.rs new file mode 100644 index 0000000..e57ee46 --- /dev/null +++ b/crates/web-faith/src/retry/stale_address.rs @@ -0,0 +1,69 @@ +use http::Extensions; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result}; +use web_faith_dns::FaithResolver; + +use super::failed_to_connect; + +/// Re-resolves and attempts a request again when connecting to a stale-served address failed. +/// +/// Serving an expired DNS answer trades a round trip against the chance the address has moved, and +/// this layer is what bounds the cost of being wrong to one re-resolve rather than a failed request. +// spec:DNS#when-a-stale-address-is-wrong +#[derive(Debug, Clone)] +pub struct StaleAddressRetry { + /// `None` under `dns.system: true`, where Faith holds no cache and so serves nothing stale. + resolver: Option, +} + +impl StaleAddressRetry { + pub fn new(resolver: Option) -> Self { + Self { resolver } + } +} + +#[async_trait::async_trait] +impl Middleware for StaleAddressRetry { + // spec:DNS#when-a-stale-address-is-wrong + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + let Some(resolver) = self.resolver.clone() else { + return next.run(req, extensions).await; + }; + + // Asked before the request runs, not after it fails. The lookup this request makes serves the + // stale entry and starts a refresh behind it, so by the time an error is in hand the entry may + // already have been replaced and the question "was this address assumed?" no longer answerable. + // + // Cloned here for the same reason the dead-connection layer clones early: sending consumes the + // body. A `ReadableStream` body does not clone, which is what leaves those requests reporting + // the connect failure rather than being attempted again -- the body has no second copy, whether + // or not it was read. + let host = req.url().host_str().map(str::to_owned); + let replay = match &host { + Some(host) if resolver.served_stale(host) => req.try_clone(), + _ => None, + }; + + let outcome = next.clone().run(req, extensions).await; + + match &outcome { + Err(err) if failed_to_connect(err) => {} + _ => return outcome, + } + + let (Some(host), Some(request)) = (host, replay) else { + return outcome; + }; + + // Drop the entry so the retry's lookup waits for a fresh answer rather than being served the + // address that just failed. One attempt only: the fresh address is confirmed rather than + // assumed, so a second failure is an answer about the origin and belongs to the caller. + resolver.invalidate_stale(&host); + next.run(request, extensions).await + } +} diff --git a/index.d.ts b/index.d.ts index c810713..51935af 100644 --- a/index.d.ts +++ b/index.d.ts @@ -49,28 +49,6 @@ export declare class Agent { * closed agent does nothing, and calling it repeatedly is harmless. */ networkChanged(): void - /** - * Add a cookie into the agent. - * - * The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying - * the scheme and host they read, so this does nothing if: - * - the cookie store is disabled - * - the url is malformed - * - the cookie does not parse - * - a `__Host-` or `__Secure-` name prefix is not satisfied - * - the cookie is larger than `cookies.maxSize` - */ - addCookie(url: string, cookie: string): void - /** - * Retrieve a cookie from the store. - * - * Returns `null` if: - * - there's no cookie at this url - * - the cookie store is disabled - * - the url is malformed - * - the cookie cannot be represented as a string - */ - getCookie(url: string): string | null /** * Returns statistics gathered by this agent: * @@ -90,16 +68,6 @@ export declare class Agent { * on field availability. If the platform isn't supported at all, this will always return empty. */ connections(): Array - /** - * Returns the DNS servers this agent resolves through, in the order they are queried, so - * "are my lookups actually encrypted" is answerable from inside the process. - * - * Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, - * `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). - * The list is empty until the resolver has been used, because it reads its configuration on - * first use, and empty for an agent using the system resolver. - */ - resolvers(): Array /** * Warm the DNS cache for `host`, so a later request to it skips the lookup. * @@ -124,6 +92,38 @@ export declare class Agent { * does a call on a closed agent. */ preconnect(origin: string): Promise + /** + * Returns the DNS servers this agent resolves through, in the order they are queried, so + * "are my lookups actually encrypted" is answerable from inside the process. + * + * Each entry gives the server's address, the transport in use (`udp`, `tcp`, `tls`, `https`, + * `quic`, or `h3`), and how that transport was arrived at (`configured` or `conventional`). + * The list is empty until the resolver has been used, because it reads its configuration on + * first use, and empty for an agent using the system resolver. + */ + resolvers(): Array + /** + * Add a cookie into the agent. + * + * The cookie goes through the same rules a `Set-Cookie` header would, with the url supplying + * the scheme and host they read, so this does nothing if: + * - the cookie store is disabled + * - the url is malformed + * - the cookie does not parse + * - a `__Host-` or `__Secure-` name prefix is not satisfied + * - the cookie is larger than `cookies.maxSize` + */ + addCookie(url: string, cookie: string): void + /** + * Retrieve a cookie from the store. + * + * Returns `null` if: + * - there's no cookie at this url + * - the cookie store is disabled + * - the url is malformed + * - the cookie cannot be represented as a string + */ + getCookie(url: string): string | null } export declare class AgentStats { From d3a9b80db19189899e47c7620ac6933ad0a69cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:59:36 +1200 Subject: [PATCH 59/61] S1: features for the cache, codings, and connection tracking Also clears the binding's dead dependencies: it carried two dozen crates from before the extraction, several of which a feature claimed to drop while linking them anyway. A slim binding refuses an option it cannot honour, at agent construction for an option group and at fetch for a per-request one. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 26 ----------- Cargo.toml | 1 - crates/web-faith-napi/Cargo.toml | 48 ++++---------------- crates/web-faith-napi/src/agent.rs | 38 ++++++++++------ crates/web-faith-napi/src/agent/convert.rs | 8 +++- crates/web-faith-napi/src/fetch.rs | 15 +++++++ crates/web-faith-napi/src/lib.rs | 1 + crates/web-faith-napi/src/options.rs | 5 +++ crates/web-faith/Cargo.toml | 14 ++++-- crates/web-faith/src/agent.rs | 12 ++++- crates/web-faith/src/agent/build.rs | 33 +++++++++++--- crates/web-faith/src/agent/warm.rs | 2 + crates/web-faith/src/builder.rs | 13 ++++-- crates/web-faith/src/client.rs | 36 +++++---------- crates/web-faith/src/client/http_cache.rs | 25 +++++++++++ crates/web-faith/src/options.rs | 4 ++ crates/web-faith/src/request.rs | 4 ++ crates/web-faith/src/request/builder.rs | 8 ++++ crates/web-faith/src/request/send.rs | 52 ++++++++++++++++++---- crates/web-faith/src/response.rs | 5 +++ index.d.ts | 20 ++++----- 21 files changed, 232 insertions(+), 138 deletions(-) create mode 100644 crates/web-faith/src/client/http_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 1e7a2e1..0653e31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2891,46 +2891,20 @@ dependencies = [ name = "web-faith-napi" version = "0.7.0" dependencies = [ - "async-compression", "async-stream", - "async-trait", "bytes", - "cookie", - "cookie_store", "futures", - "hickory-resolver", - "http", - "http-body-util", "http-cache-reqwest", - "hyper", - "hyper-util", - "libc", - "moka", "napi", "napi-build", "napi-derive", - "netlink-packet-core", - "netlink-packet-sock-diag", - "netlink-sys", "reqwest", "reqwest-middleware", - "serde", "serde_json", - "ssri", - "stream_shared", - "strum", - "time", "tokio", - "tokio-stream", - "tokio-util", - "url", "web-faith", - "web-faith-alt-svc", "web-faith-conn-tracker", "web-faith-cookies", - "web-faith-dns", - "web-faith-encoding", - "windows", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2a9593d..2867b71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,6 @@ ssri = "9.2.0" stream_shared = { version = "0.8.5", features = ["stats"] } strum = { version = "0.27.2", features = ["derive"] } tokio = { version = "1.48.0", features = ["full"] } -tokio-stream = "0.1.16" time = "0.3.53" tokio-util = { version = "0.7.10", features = ["io"] } url = "2.5.7" diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index 7d4eed3..c6f395a 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -16,60 +16,28 @@ name = "faith" crate-type = ["cdylib"] [dependencies] -async-compression.workspace = true async-stream.workspace = true -async-trait.workspace = true bytes.workspace = true -cookie.workspace = true -cookie_store.workspace = true futures.workspace = true -hickory-resolver.workspace = true -moka.workspace = true -http.workspace = true -http-body-util.workspace = true -hyper.workspace = true -http-cache-reqwest.workspace = true -hyper-util.workspace = true -libc.workspace = true +http-cache-reqwest = { workspace = true, optional = true } napi.workspace = true napi-derive.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true -serde.workspace = true serde_json.workspace = true -ssri.workspace = true -stream_shared.workspace = true -strum.workspace = true tokio.workspace = true -tokio-stream.workspace = true -time.workspace = true -tokio-util.workspace = true -url.workspace = true web-faith.workspace = true -web-faith-alt-svc = { workspace = true, optional = true } -web-faith-conn-tracker.workspace = true +web-faith-conn-tracker = { workspace = true, optional = true } web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } -web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } -web-faith-encoding.workspace = true - -[target.'cfg(target_os = "linux")'.dependencies] -netlink-packet-core.workspace = true -netlink-packet-sock-diag.workspace = true -netlink-sys.workspace = true - -[target.'cfg(target_os = "windows")'.dependencies] -windows.workspace = true [build-dependencies] napi-build.workspace = true [features] -default = ["cookies", "dns", "http3"] +default = ["cache", "connection-tracking", "cookies", "dns", "encoding", "http3"] +cache = ["dep:http-cache-reqwest", "web-faith/cache"] +connection-tracking = ["dep:web-faith-conn-tracker", "web-faith/connection-tracking"] cookies = ["dep:web-faith-cookies", "reqwest/cookies", "web-faith/cookies"] -dns = [ - "dep:web-faith-dns", - "reqwest/hickory-dns", - "web-faith/dns", - "web-faith-alt-svc?/dns", -] -http3 = ["reqwest/http3", "dep:web-faith-alt-svc", "web-faith/http3"] +dns = ["web-faith/dns"] +encoding = ["web-faith/encoding"] +http3 = ["reqwest/http3", "web-faith/http3"] diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index 466e572..a068ab0 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -7,10 +7,12 @@ use napi_derive::napi; use crate::{ async_task::faith_promise, - conn_tracker::{ConnectionInfo, connections_for_napi}, error::{FaithError, FaithErrorExt}, }; +#[cfg(feature = "connection-tracking")] +use crate::conn_tracker::{ConnectionInfo, connections_for_napi}; + mod convert; mod options; @@ -165,18 +167,6 @@ impl Agent { AgentStats::from(self.inner.stats()) } - /// Returns information on current connections open by this agent. - /// - /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: - /// this makes it possible to track indicators over time to find the retransmission rate, for - /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other - /// fields might also be missing depending on platform support; and no forward guarantees are made - /// on field availability. If the platform isn't supported at all, this will always return empty. - #[napi] - pub fn connections<'env>(&self, env: &'env Env) -> Vec> { - connections_for_napi(&self.inner.conn_tracker, env) - } - /// Warm the DNS cache for `host`, so a later request to it skips the lookup. /// /// Mirrors the browser's `dns-prefetch` resource hint. The argument is a bare host; a scheme, @@ -248,6 +238,11 @@ fn refuse_absent_capabilities(options: &AgentOptions) -> Result<(), FaithError> )) }; + #[cfg(not(feature = "cache"))] + if options.cache.is_some() { + return absent("HTTP cache"); + } + #[cfg(not(feature = "cookies"))] if options.cookies.is_some() { return absent("cookie"); @@ -274,6 +269,23 @@ fn refuse_absent_capabilities(options: &AgentOptions) -> Result<(), FaithError> Ok(()) } +/// Per-connection reporting, which needs the tracker that gathers it. +#[cfg(feature = "connection-tracking")] +#[napi] +impl Agent { + /// Returns information on current connections open by this agent. + /// + /// Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: + /// this makes it possible to track indicators over time to find the retransmission rate, for + /// example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other + /// fields might also be missing depending on platform support; and no forward guarantees are made + /// on field availability. If the platform isn't supported at all, this will always return empty. + #[napi] + pub fn connections<'env>(&self, env: &'env Env) -> Vec> { + connections_for_napi(&self.inner.conn_tracker, env) + } +} + /// The resolver's own observability, which needs a resolver of Faith's own to report on. #[cfg(feature = "dns")] #[napi] diff --git a/crates/web-faith-napi/src/agent/convert.rs b/crates/web-faith-napi/src/agent/convert.rs index 69c8e06..ebf545d 100644 --- a/crates/web-faith-napi/src/agent/convert.rs +++ b/crates/web-faith-napi/src/agent/convert.rs @@ -1,13 +1,18 @@ //! Reading a JavaScript `AgentOptions` into the options the client validates. +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; use napi::{Either, bindgen_prelude::Buffer}; + use web_faith::{client::RedirectPolicy, options}; #[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; -use crate::agent::{AgentOptions, CacheStore, Http3Congestion}; +use crate::agent::{AgentOptions, Http3Congestion}; + +#[cfg(feature = "cache")] +use crate::agent::CacheStore; /// Read the JavaScript options object into the shape the client validates. /// @@ -16,6 +21,7 @@ use crate::agent::{AgentOptions, CacheStore, Http3Congestion}; impl From for options::AgentOptions { fn from(opts: AgentOptions) -> Self { Self { + #[cfg(feature = "cache")] cache: opts.cache.map(|cache| options::CacheOptions { store: cache.store.map(|store| match store { CacheStore::Disk => options::CacheStore::Disk, diff --git a/crates/web-faith-napi/src/fetch.rs b/crates/web-faith-napi/src/fetch.rs index 51b6d5a..990d791 100644 --- a/crates/web-faith-napi/src/fetch.rs +++ b/crates/web-faith-napi/src/fetch.rs @@ -24,6 +24,21 @@ pub fn faith_fetch<'env>( signal: Option, stream_body: Option<&StreamBody>, ) -> Result, napi::Error> { + // Refused rather than ignored: a build without the coding layer cannot compress a body, and a + // caller who asked for it should hear so. See `refuse_absent_capabilities` for the agent-level + // equivalent. + #[cfg(not(feature = "encoding"))] + if options.compress.is_some() { + return Err(napi::Error::from_reason( + "this build has no content-coding support", + )); + } + + #[cfg(not(feature = "cache"))] + if options.cache.is_some() { + return Err(napi::Error::from_reason("this build has no HTTP cache")); + } + let (options, agent, body) = options::extract(options); // Taken here, while `fetch()` is still on the stack, so the request counts as in flight from // the moment it was issued: closing the agent afterwards does not strand it. diff --git a/crates/web-faith-napi/src/lib.rs b/crates/web-faith-napi/src/lib.rs index 3f2f632..b2a7c28 100644 --- a/crates/web-faith-napi/src/lib.rs +++ b/crates/web-faith-napi/src/lib.rs @@ -1,5 +1,6 @@ mod agent; mod async_task; +#[cfg(feature = "connection-tracking")] mod conn_tracker; mod error; mod fetch; diff --git a/crates/web-faith-napi/src/options.rs b/crates/web-faith-napi/src/options.rs index 7cc028e..c8e18fe 100644 --- a/crates/web-faith-napi/src/options.rs +++ b/crates/web-faith-napi/src/options.rs @@ -1,6 +1,8 @@ use std::{fmt::Debug, sync::Arc, time::Duration}; +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; + use napi::bindgen_prelude::*; use napi_derive::napi; @@ -70,6 +72,7 @@ pub enum RequestCacheMode { Reload, } +#[cfg(feature = "cache")] impl From for CacheMode { fn from(mode: RequestCacheMode) -> Self { match mode { @@ -186,7 +189,9 @@ pub(crate) fn extract(opts: FaithOptionsAndBody) -> (RequestOptions, Agent, Opti ( RequestOptions { + #[cfg(feature = "cache")] cache: opts.cache.unwrap_or_default().into(), + #[cfg(feature = "encoding")] compress: opts.compress, credentials, headers: opts.headers, diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 709a80d..a36f582 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -17,7 +17,7 @@ futures.workspace = true http.workspace = true http-body.workspace = true http-body-util.workspace = true -http-cache-reqwest.workspace = true +http-cache-reqwest = { workspace = true, optional = true } hyper.workspace = true hyper-util.workspace = true moka.workspace = true @@ -30,18 +30,24 @@ ssri.workspace = true strum.workspace = true tokio.workspace = true url.workspace = true -web-faith-conn-tracker.workspace = true -web-faith-encoding.workspace = true +web-faith-conn-tracker = { workspace = true, optional = true } +web-faith-encoding = { workspace = true, optional = true } web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true } web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } web-faith-alt-svc = { workspace = true, optional = true } [features] -default = ["cookies", "dns", "http3"] +default = ["cache", "connection-tracking", "cookies", "dns", "encoding", "http3"] +# The HTTP cache, its store, and the per-request cache mode. +cache = ["dep:http-cache-reqwest"] +# Per-connection kernel counters, and the agent verb that reports them. +connection-tracking = ["dep:web-faith-conn-tracker"] # The cookie jar, and the agent option and handle that reach it. cookies = ["dep:web-faith-cookies", "reqwest/cookies"] # Faith's own caching resolver, which a prefetch warms and an `HTTPS` record lookup reads. # Without it names resolve through the platform, as reqwest resolves them. dns = ["dep:web-faith-dns", "reqwest/hickory-dns", "web-faith-alt-svc?/dns"] +# Content codings: negotiating and decoding a response body, and compressing a request one. +encoding = ["dep:web-faith-encoding"] # Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] diff --git a/crates/web-faith/src/agent.rs b/crates/web-faith/src/agent.rs index 1d68b7e..f2d90ec 100644 --- a/crates/web-faith/src/agent.rs +++ b/crates/web-faith/src/agent.rs @@ -7,11 +7,15 @@ use std::sync::{ atomic::{AtomicU64, Ordering}, }; -use http::header::HeaderValue; use moka::sync::Cache as MokaCache; use reqwest::Client; use reqwest_middleware::ClientWithMiddleware; use url::Url; + +#[cfg(feature = "encoding")] +use http::header::HeaderValue; + +#[cfg(feature = "connection-tracking")] use web_faith_conn_tracker::{ConnectionSnapshot, ConnectionTracker}; #[cfg(feature = "cookies")] @@ -52,9 +56,11 @@ pub struct AgentSettings { pub quirk_h1_request_streaming: bool, /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides /// which codings a response is decoded under when a request adds none of its own. + #[cfg(feature = "encoding")] pub default_accept_encoding: Option, /// The agent's default `Content-Encoding`, if one sits among its default headers, which a /// request layers its own coding on top of rather than displacing. + #[cfg(feature = "encoding")] pub default_content_encoding: Option, /// Whether a `Priority` header sits among the agent's default headers, so that default wins /// over the one a request's priority would derive. @@ -117,6 +123,7 @@ pub struct Agent { #[cfg(feature = "cookies")] pub cookie_jar: Option>, pub stats: Arc, + #[cfg(feature = "connection-tracking")] pub conn_tracker: Arc, /// Whether an upgrade may follow a port the origin advertised. A request needs it to stop a /// rewritten port from being reported as a redirect. @@ -132,10 +139,12 @@ pub struct Agent { pub quirk_h1_request_streaming: bool, /// The agent's default `Accept-Encoding`, if one sits among its default headers, which decides /// the codings a response is decoded under when a request adds none of its own. + #[cfg(feature = "encoding")] pub default_accept_encoding: Option, /// The agent's default `Content-Encoding`, if one sits among its default headers. A request /// layers its own coding on top of this rather than displacing it. // spec:ENC + #[cfg(feature = "encoding")] pub default_content_encoding: Option, /// Whether a `Priority` header sits among the agent's default headers. That default wins over /// the header a request's priority would derive. @@ -337,6 +346,7 @@ impl Agent { /// example. The lost-packet count and delivery rate are only available on Linux. Some other /// fields might also be missing depending on platform support; and no forward guarantees are made /// on field availability. If the platform isn't supported at all, this will always return empty. + #[cfg(feature = "connection-tracking")] pub fn connections(&self) -> Vec { self.conn_tracker.snapshot() } diff --git a/crates/web-faith/src/agent/build.rs b/crates/web-faith/src/agent/build.rs index ec05f05..99cf236 100644 --- a/crates/web-faith/src/agent/build.rs +++ b/crates/web-faith/src/agent/build.rs @@ -10,11 +10,21 @@ use std::{ }; use http::header::{HeaderMap, HeaderName, HeaderValue}; + +#[cfg(feature = "cache")] +use crate::{ + client::{HttpCacheRecipe, HttpCacheStore}, + options::CacheStore, +}; + +#[cfg(feature = "cache")] use http_cache_reqwest::{ CACacheManager, CacheOptions, HttpCacheOptions, MokaCacheBuilder, MokaManager, }; use moka::sync::Cache as MokaCache; use reqwest::{Identity, tls::Certificate}; + +#[cfg(feature = "connection-tracking")] use web_faith_conn_tracker::ConnectionTracker; #[cfg(feature = "cookies")] @@ -31,11 +41,9 @@ use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig}; use crate::{ USER_AGENT, agent::{Agent, AgentSettings, Live}, - client::{ClientRecipe, HttpCacheRecipe, HttpCacheStore, NodeEnvRecipe}, + client::{ClientRecipe, NodeEnvRecipe}, error::{FaithError, FaithErrorKind}, - options::{ - AgentOptions, CacheStore, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows, - }, + options::{AgentOptions, DnsOverride, Header, ipv6_wildcard_bindable, resolve_windows}, request::PRIORITY, }; @@ -54,6 +62,7 @@ impl Agent { // without the compiler pointing here, where every option is turned into the recipe the // agent's clients are built from (spec:NETCHG). let AgentOptions { + #[cfg(feature = "cache")] cache, #[cfg(feature = "cookies")] cookies, @@ -184,7 +193,9 @@ impl Agent { })) }; + #[cfg(feature = "encoding")] let mut default_accept_encoding = None; + #[cfg(feature = "encoding")] let mut default_content_encoding = None; let mut has_default_priority = false; let mut default_headers = None; @@ -212,8 +223,11 @@ impl Agent { Some((name, value)) }, )); - default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); - default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); + #[cfg(feature = "encoding")] + { + default_accept_encoding = map.get(reqwest::header::ACCEPT_ENCODING).cloned(); + default_content_encoding = map.get(reqwest::header::CONTENT_ENCODING).cloned(); + } has_default_priority = map.contains_key(PRIORITY); default_headers = Some(map); } @@ -309,6 +323,7 @@ impl Agent { } }; + #[cfg(feature = "cache")] let http_cache = if let Some(cache) = cache && let Some(store) = cache.store { @@ -481,6 +496,7 @@ impl Agent { tls_required, tls_extra_roots, node_env: NodeEnvRecipe::read(), + #[cfg(feature = "cache")] http_cache, #[cfg(feature = "http3")] h3_upgrade, @@ -491,7 +507,9 @@ impl Agent { #[cfg(feature = "http3")] h3_upgrade_enabled: recipe.h3_upgrade.enabled, quirk_h1_request_streaming, + #[cfg(feature = "encoding")] default_accept_encoding, + #[cfg(feature = "encoding")] default_content_encoding, has_default_priority, }; @@ -573,12 +591,15 @@ impl Agent { #[cfg(feature = "cookies")] cookie_jar, stats: Default::default(), + #[cfg(feature = "connection-tracking")] conn_tracker: ConnectionTracker::new(conn_timeout), h3_follow_advertised_port: settings.h3_follow_advertised_port, #[cfg(feature = "http3")] h3_upgrade_enabled: recipe.h3_upgrade.enabled, quirk_h1_request_streaming: settings.quirk_h1_request_streaming, + #[cfg(feature = "encoding")] default_accept_encoding: settings.default_accept_encoding, + #[cfg(feature = "encoding")] default_content_encoding: settings.default_content_encoding, has_default_priority: settings.has_default_priority, recipe: Arc::new(recipe), diff --git a/crates/web-faith/src/agent/warm.rs b/crates/web-faith/src/agent/warm.rs index b8272ba..09c7b5d 100644 --- a/crates/web-faith/src/agent/warm.rs +++ b/crates/web-faith/src/agent/warm.rs @@ -89,6 +89,7 @@ impl Agent { #[cfg(not(feature = "http3"))] let h3_port: Option = None; + #[cfg(feature = "connection-tracking")] let conn_tracker = self.conn_tracker.clone(); let warmed = self.warmed.clone(); let warming = self.warming.clone(); @@ -135,6 +136,7 @@ impl Agent { // A TCP warm-up leaves a pooled connection to track; a QUIC one does not (QUIC // connections are not tracked, and a confirmed origin has nothing left to probe). + #[cfg(feature = "connection-tracking")] if h3_port.is_none() && let Ok(response) = &outcome && let Some(info) = response diff --git a/crates/web-faith/src/builder.rs b/crates/web-faith/src/builder.rs index 973a3c2..8b0e84e 100644 --- a/crates/web-faith/src/builder.rs +++ b/crates/web-faith/src/builder.rs @@ -24,8 +24,12 @@ use std::{net::IpAddr, time::Duration}; +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; +#[cfg(feature = "cache")] +use crate::options::{CacheOptions, CacheStore}; + #[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; @@ -34,9 +38,9 @@ use crate::{ client::RedirectPolicy, error::FaithError, options::{ - AgentOptions, CacheOptions, CacheStore, DnsOptions, DnsOverride, FlowControlOptions, - Header, Http2Options, Http3Congestion, Http3Hint, Http3Options, PoolOptions, QuirksOptions, - TimeoutOptions, TlsOptions, + AgentOptions, DnsOptions, DnsOverride, FlowControlOptions, Header, Http2Options, + Http3Congestion, Http3Hint, Http3Options, PoolOptions, QuirksOptions, TimeoutOptions, + TlsOptions, }, }; @@ -143,6 +147,7 @@ impl AgentBuilder { } /// HTTP cache settings. Without this call the agent does not cache. + #[cfg(feature = "cache")] pub fn cache(mut self, with: impl FnOnce(CacheBuilder) -> CacheBuilder) -> Self { let group = self.options.cache.take().unwrap_or_default(); self.options.cache = Some(with(CacheBuilder { group }).group); @@ -354,12 +359,14 @@ impl TimeoutBuilder { } /// HTTP cache settings. Reached through [`AgentBuilder::cache`]. +#[cfg(feature = "cache")] #[derive(Debug, Default)] #[must_use] pub struct CacheBuilder { group: CacheOptions, } +#[cfg(feature = "cache")] impl CacheBuilder { /// Where cached responses are kept. pub fn store(mut self, store: CacheStore) -> Self { diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs index f166c38..cbe94fc 100644 --- a/crates/web-faith/src/client.rs +++ b/crates/web-faith/src/client.rs @@ -16,9 +16,15 @@ use std::{ use std::sync::Arc; use http::header::HeaderMap; -use http_cache_reqwest::{ - CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions, MokaManager, -}; + +#[cfg(feature = "cache")] +mod http_cache; + +#[cfg(feature = "cache")] +pub use http_cache::{HttpCacheRecipe, HttpCacheStore}; + +#[cfg(feature = "cache")] +use http_cache_reqwest::{Cache, HttpCache}; use reqwest::{Client, Identity, redirect::Policy, tls::Certificate}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; #[cfg(feature = "cookies")] @@ -154,28 +160,6 @@ impl NodeEnvRecipe { } } -/// The HTTP cache store to install on a client, held as the built manager rather than as the -/// options that produced it. -/// -/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and -/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while -/// building a fresh one from the same options would empty an in-memory cache — which is why a -/// client rebuilt for a network change clones this. -// spec:NETCHG#what-the-signal-keeps -#[derive(Debug, Clone)] -pub enum HttpCacheStore { - Disk(CACacheManager), - Memory(MokaManager), -} - -/// The HTTP cache middleware to install. -#[derive(Debug, Clone)] -pub struct HttpCacheRecipe { - pub mode: CacheMode, - pub options: HttpCacheOptions, - pub store: HttpCacheStore, -} - /// The HTTP/3 upgrade settings a client's middleware needs. The origin knowledge itself is not /// here: it belongs to the agent and outlives any one client. // spec:NETCHG @@ -232,6 +216,7 @@ pub struct ClientRecipe { pub tls_required: Option, pub tls_extra_roots: Vec, pub node_env: NodeEnvRecipe, + #[cfg(feature = "cache")] pub http_cache: Option, #[cfg(feature = "http3")] pub h3_upgrade: H3UpgradeRecipe, @@ -430,6 +415,7 @@ impl ClientRecipe { }) }; + #[cfg(feature = "cache")] if let Some(cache) = &self.http_cache { // The two arms differ only in the manager's type, which `HttpCache` is generic over, // so they cannot share a constructor without boxing the manager. diff --git a/crates/web-faith/src/client/http_cache.rs b/crates/web-faith/src/client/http_cache.rs new file mode 100644 index 0000000..7ba917c --- /dev/null +++ b/crates/web-faith/src/client/http_cache.rs @@ -0,0 +1,25 @@ +//! What an HTTP cache is installed from, held so a rebuilt client keeps the same store. + +use http_cache_reqwest::{CACacheManager, CacheMode, HttpCacheOptions, MokaManager}; + +/// The HTTP cache store to install on a client, held as the built manager rather than as the +/// options that produced it. +/// +/// The manager *is* the store: `MokaManager` holds the cached entries behind an `Arc`, and +/// `CACacheManager` names the directory holding them. So cloning one shares the cache, while +/// building a fresh one from the same options would empty an in-memory cache — which is why a +/// client rebuilt for a network change clones this. +// spec:NETCHG#what-the-signal-keeps +#[derive(Debug, Clone)] +pub enum HttpCacheStore { + Disk(CACacheManager), + Memory(MokaManager), +} + +/// The HTTP cache middleware to install. +#[derive(Debug, Clone)] +pub struct HttpCacheRecipe { + pub mode: CacheMode, + pub options: HttpCacheOptions, + pub store: HttpCacheStore, +} diff --git a/crates/web-faith/src/options.rs b/crates/web-faith/src/options.rs index 26a6812..d95753e 100644 --- a/crates/web-faith/src/options.rs +++ b/crates/web-faith/src/options.rs @@ -10,6 +10,7 @@ use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; #[cfg(feature = "cookies")] @@ -20,6 +21,7 @@ use crate::client::{ }; /// Settings related to the HTTP cache. This is a nested object. +#[cfg(feature = "cache")] #[derive(Clone, Debug, Default)] pub struct CacheOptions { /// Which cache store to use: either `disk` or `memory`. @@ -48,6 +50,7 @@ pub struct CacheOptions { pub shared: Option, } +#[cfg(feature = "cache")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CacheStore { Disk, @@ -573,6 +576,7 @@ pub struct TlsOptions { #[derive(Clone, Debug, Default)] pub struct AgentOptions { /// Settings related to the HTTP cache. This is a nested object. + #[cfg(feature = "cache")] pub cache: Option, /// Enable a persistent cookie store for the agent. Cookies received in responses will be preserved and /// included in additional requests. diff --git a/crates/web-faith/src/request.rs b/crates/web-faith/src/request.rs index 8293a28..09a8c63 100644 --- a/crates/web-faith/src/request.rs +++ b/crates/web-faith/src/request.rs @@ -14,6 +14,8 @@ use std::{pin::Pin, time::Duration}; use bytes::Bytes; use futures::Stream; + +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; /// Whether a request carries its credentials, and how far. @@ -47,8 +49,10 @@ pub enum RequestBody { /// What a request carries beyond its method, URL, and body. #[derive(Clone, Debug, Default)] pub struct RequestOptions { + #[cfg(feature = "cache")] pub cache: CacheMode, /// A coding to compress the body in, named by its wire token. + #[cfg(feature = "encoding")] pub compress: Option, pub credentials: Credentials, pub headers: Option>, diff --git a/crates/web-faith/src/request/builder.rs b/crates/web-faith/src/request/builder.rs index f3bd6c4..87e6f00 100644 --- a/crates/web-faith/src/request/builder.rs +++ b/crates/web-faith/src/request/builder.rs @@ -6,6 +6,8 @@ use std::{ use bytes::Bytes; use futures::Stream; + +#[cfg(feature = "cache")] use http_cache_reqwest::CacheMode; use reqwest::{ Method, @@ -104,7 +106,9 @@ struct Layer { #[derive(Default)] struct SetFlags { + #[cfg(feature = "cache")] cache: bool, + #[cfg(feature = "encoding")] compress: bool, credentials: bool, integrity: bool, @@ -146,9 +150,11 @@ impl Layer { Target::Request(inner) => (inner.url, inner.options, inner.body), }; + #[cfg(feature = "cache")] if self.set.cache { options.cache = self.options.cache; } + #[cfg(feature = "encoding")] if self.set.compress { options.compress = self.options.compress; } @@ -288,6 +294,7 @@ macro_rules! layer_setters { /// Compress the request body in this coding, named by its wire token. // spec:ENC + #[cfg(feature = "encoding")] pub fn compress(mut self, coding: impl Into) -> Self { self.layer.options.compress = Some(coding.into()); self.layer.set.compress = true; @@ -296,6 +303,7 @@ macro_rules! layer_setters { /// How the HTTP cache is consulted for this request. // spec:CACHE + #[cfg(feature = "cache")] pub fn cache(mut self, mode: CacheMode) -> Self { self.layer.options.cache = mode; self.layer.set.cache = true; diff --git a/crates/web-faith/src/request/send.rs b/crates/web-faith/src/request/send.rs index 3cba9d4..25b0f89 100644 --- a/crates/web-faith/src/request/send.rs +++ b/crates/web-faith/src/request/send.rs @@ -6,15 +6,24 @@ use std::{ time::Instant, }; -use http_cache_reqwest::CacheMode; -use hyper_util::client::legacy::connect::HttpInfo; use reqwest::{ Method, StatusCode, - header::{ACCEPT_ENCODING, CONTENT_ENCODING, HeaderName, HeaderValue}, + header::{CONTENT_ENCODING, HeaderName, HeaderValue}, tls::TlsInfo, }; use reqwest_middleware::ClientWithMiddleware; + +#[cfg(feature = "connection-tracking")] +use hyper_util::client::legacy::connect::HttpInfo; + +#[cfg(feature = "cache")] +use http_cache_reqwest::CacheMode; + +#[cfg(feature = "encoding")] +use reqwest::header::ACCEPT_ENCODING; + use tokio::sync::Mutex; +#[cfg(feature = "encoding")] use web_faith_encoding::{self as encoding, AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING}; use crate::{ @@ -58,6 +67,7 @@ pub async fn send( // A `compress` naming no coding Faith can compress in is misuse whether or not the // request turns out to carry a body, so it is refused before anything else looks at // it (spec:ENC#compressing-a-request-body). + #[cfg(feature = "encoding")] let compress = options .compress .as_deref() @@ -87,8 +97,11 @@ pub async fn send( let mut request = client .request(method, parsed_url.clone()) - .with_extension(CacheMode::from(options.cache)) .with_extension(headers_stamp.clone()); + #[cfg(feature = "cache")] + { + request = request.with_extension(CacheMode::from(options.cache)); + } if let Some(headers) = &options.headers { for (key, value) in headers { @@ -116,6 +129,7 @@ pub async fn send( // here would put a second `Content-Encoding` beside the joined one -- the same // list read twice over (spec:ENC#what-a-compressed-request-sends). The value is // still validated above, then withheld and re-emitted once below. + #[cfg(feature = "encoding")] if compress.is_some() && header_name == CONTENT_ENCODING { continue; } @@ -127,6 +141,7 @@ pub async fn send( // What the caller says they handed over: their own `Content-Encoding`, else the // agent's, per-request headers winning per name as they do generally (spec: REQ). // Several lines are the one list, so they are joined as they are read. + #[cfg(feature = "encoding")] let declared_content_encoding = compress.and_then(|_| { let from_request = options.headers.as_ref().and_then(|headers| { let declared = headers @@ -149,12 +164,14 @@ pub async fn send( // default headers, else the default Faith sends itself. Neither the request nor the // agent advertising a value means nothing beneath Faith adds one now that it owns // the codings, so Faith sends the default explicitly. + #[cfg(feature = "encoding")] let request_accept_encoding = options.headers.as_ref().and_then(|headers| { headers .iter() .find(|(name, _)| name.eq_ignore_ascii_case("accept-encoding")) .map(|(_, value)| value.clone()) }); + #[cfg(feature = "encoding")] let accept_encoding = AcceptEncoding::parse( &request_accept_encoding .clone() @@ -166,6 +183,7 @@ pub async fn send( }) .unwrap_or_else(|| DEFAULT_ACCEPT_ENCODING.to_owned()), ); + #[cfg(feature = "encoding")] if request_accept_encoding.is_none() && agent.default_accept_encoding.is_none() { request = request.header( ACCEPT_ENCODING, @@ -195,6 +213,7 @@ pub async fn send( // apply it to: the option does nothing on a request carrying none, so no // `Content-Encoding` describes bytes that were never sent // (spec:ENC#compressing-a-request-body). + #[cfg(feature = "encoding")] let mut applied_coding = None; // Handle body: prefer streaming body over buffered body @@ -223,7 +242,8 @@ pub async fn send( request = request.version(http::Version::HTTP_2); } - request = request.body(match compress { + #[cfg(feature = "encoding")] + let body = match compress { // Compressed as the chunks arrive, and chunked on the wire either way: // a stream has no length to declare up front. // spec:ENC#what-a-compressed-request-sends @@ -232,10 +252,14 @@ pub async fn send( reqwest::Body::wrap_stream(encoding::compress_stream(byte_stream, coding)) } None => reqwest::Body::wrap_stream(byte_stream), - }); + }; + #[cfg(not(feature = "encoding"))] + let body = reqwest::Body::wrap_stream(byte_stream); + request = request.body(body); } RequestBody::Bytes(bytes) => { - request = request.body(match compress { + #[cfg(feature = "encoding")] + let body = match compress { // The compressed bytes are what reqwest sizes `Content-Length` from, so the // header counts what goes on the wire. // spec:ENC#what-a-compressed-request-sends @@ -251,13 +275,17 @@ pub async fn send( })? } None => bytes.to_vec(), - }); + }; + #[cfg(not(feature = "encoding"))] + let body = bytes.to_vec(); + request = request.body(body); } RequestBody::None => {} } // One `Content-Encoding` naming the caller's codings then Faith's, in the order they // were applied (spec:ENC#what-a-compressed-request-sends). + #[cfg(feature = "encoding")] if let Some(coding) = applied_coding { let value = encoding::layer_content_encoding(declared_content_encoding.as_deref(), coding); let value = HeaderValue::from_str(&value).map_err(|_| { @@ -326,6 +354,7 @@ pub async fn send( // Track connection for TCP stats (if we can get both local and remote addr). // A connection the tracker has already seen is one the pool handed back, which is // what `reused` reports (spec:RESP#request-timing). + #[cfg(feature = "connection-tracking")] let reused = if let Some(http_info) = response.extensions().get::() { let local_addr = http_info.local_addr(); let remote_addr = http_info.remote_addr(); @@ -333,6 +362,10 @@ pub async fn send( } else { false }; + // Whether the pool handed a connection back is what the tracker knows, so without it there is + // no answer to report. + #[cfg(not(feature = "connection-tracking"))] + let reused = false; // The origin now holds a connection the pool keeps idle, so a `preconnect` for it has // nothing left to do (spec:WARM). Keyed on the URL the request was sent to, so a @@ -375,11 +408,13 @@ pub async fn send( // Decode only a body Faith negotiated the coding for; a bodyless response keeps its // `Content-Encoding` and `Content-Length` describing the representation (spec: ENC). + #[cfg(feature = "encoding")] let decode = if empty { None } else { encoding::decision(&headers, &accept_encoding) }; + #[cfg(feature = "encoding")] if decode.is_some() { encoding::strip_decoded_headers(&mut headers); } @@ -401,6 +436,7 @@ pub async fn send( timing.clone(), ) }, + #[cfg(feature = "encoding")] decode, disturbed: Arc::new(AtomicBool::new(false)), headers, diff --git a/crates/web-faith/src/response.rs b/crates/web-faith/src/response.rs index 387485b..225fbb7 100644 --- a/crates/web-faith/src/response.rs +++ b/crates/web-faith/src/response.rs @@ -24,6 +24,8 @@ use reqwest::{StatusCode, Url, Version}; use serde::de::DeserializeOwned; use stream_shared::SharedStream; use tokio::{io::AsyncWriteExt, sync::watch}; + +#[cfg(feature = "encoding")] use web_faith_encoding::{Coding, decode_stream}; use crate::{ @@ -206,6 +208,7 @@ mod tests { let response = Response { body: BodyHolder::none(), + #[cfg(feature = "encoding")] decode: None, disturbed: Arc::new(AtomicBool::new(false)), headers, @@ -324,6 +327,7 @@ pub struct FileWritten { pub struct Response { pub body: BodyHolder, /// The coding to decode the body under, or `None` to deliver it as received. + #[cfg(feature = "encoding")] /// Set once when the response is built, from the request's `Accept-Encoding` and the /// response's `Content-Encoding` (see [`web_faith_encoding`]). pub decode: Option, @@ -546,6 +550,7 @@ impl Response { .filter_map(async |item| item), ) as Pin>; + #[cfg(feature = "encoding")] let bytes = match self.decode { Some(coding) => decode_stream(bytes, coding), None => bytes, diff --git a/index.d.ts b/index.d.ts index 51935af..042458a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -58,16 +58,6 @@ export declare class Agent { * - `bodiesFinished` */ stats(): AgentStats - /** - * Returns information on current connections open by this agent. - * - * Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: - * this makes it possible to track indicators over time to find the retransmission rate, for - * example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other - * fields might also be missing depending on platform support; and no forward guarantees are made - * on field availability. If the platform isn't supported at all, this will always return empty. - */ - connections(): Array /** * Warm the DNS cache for `host`, so a later request to it skips the lookup. * @@ -92,6 +82,16 @@ export declare class Agent { * does a call on a closed agent. */ preconnect(origin: string): Promise + /** + * Returns information on current connections open by this agent. + * + * Only tracks TCP connections currently (upstream limitation). Stats are updated once a second: + * this makes it possible to track indicators over time to find the retransmission rate, for + * example. The `lostPackets` and `deliveryRateBps` stats are only available on Linux. Some other + * fields might also be missing depending on platform support; and no forward guarantees are made + * on field availability. If the platform isn't supported at all, this will always return empty. + */ + connections(): Array /** * Returns the DNS servers this agent resolves through, in the order they are queried, so * "are my lookups actually encrypted" is answerable from inside the process. From 142f56c0334e022e0617b00d3e2bff532b6a5cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:10:38 +1200 Subject: [PATCH 60/61] S1: gate the HTTP/3 options, and make the TLS backend a choice The http3 option group was accepted and ignored without the feature; it now goes with it. reqwest's rustls feature moves out of the workspace root into tls-aws-lc-rs, with tls-ring the alternative, installing ring as the process crypto provider where it is the only choice. Neither backend is a compile error. Co-Authored-By: Claude Opus 5 --- .workhorse/plans/s1/plan.md | 35 +++++++++++++++++++++- Cargo.lock | 2 ++ Cargo.toml | 2 +- crates/web-faith-napi/Cargo.toml | 14 +++++++-- crates/web-faith-napi/src/agent.rs | 5 ++++ crates/web-faith-napi/src/agent/convert.rs | 6 +++- crates/web-faith/Cargo.toml | 18 +++++++++-- crates/web-faith/src/agent/build.rs | 3 +- crates/web-faith/src/agent/tests.rs | 2 ++ crates/web-faith/src/builder.rs | 11 +++++-- crates/web-faith/src/client.rs | 19 ++++++++++++ crates/web-faith/src/lib.rs | 5 ++++ crates/web-faith/src/options.rs | 4 +++ 13 files changed, 114 insertions(+), 12 deletions(-) diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index 7c18aa2..b4f7344 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -121,10 +121,15 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al - [ ] Remaining: closing up the recipe and option structs' public fields now that the builder owns the assembly. Left deliberately: the binding still fills the option structs directly, so these close up when step 10 settles what the public surface is. -- [ ] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling +- [x] **10. Feature wiring** — a default-on feature per capability a build can do without; disabling one drops the code and the API surface it gates (compile error at the call site, not a no-op), and the dependency too where the capability is a crate. Component, crate, and feature are three axes and need not line up: a crate can be non-optional, and a feature need not map to a crate. + + `web-faith` carries `cache`, `connection-tracking`, `cookies`, `dns`, `encoding`, `http3`, and the + `tls-aws-lc-rs`/`tls-ring` backend choice; `web-faith-alt-svc` carries `dns` for the HTTPS-record + sink. `web-faith-napi` mirrors the set. Integrity is deliberately not a feature: it is always + built. - [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. - [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, @@ -191,6 +196,34 @@ Decisions taken while doing steps 0–7, worth not relitigating: the next, which rustc catches, and a `// spec:` line under a doc block extends how far back the block starts. Intra-doc links break more quietly: `[`X`]` that resolved within one file needs `crate::X` once `X` is a sibling module away, and `cargo doc --workspace --no-deps` is what says so. +- **A feature drops the reqwest feature behind it too.** `cookies`, `dns`, `http3`, and the TLS + backend each own their reqwest counterpart, which means those came out of the workspace root's + `reqwest` feature list: a feature that leaves the dependency linked has not dropped anything. + Cargo refuses a member override of a workspace dependency's `default-features`, so + `web-faith-alt-svc` needs `default-features = false` set at the root, as `web-faith` already did. +- **The TLS backend resolves by priority, not by refusal.** Cargo features are additive, so a + build with both `tls-aws-lc-rs` and `tls-ring` — which is what `--all-features` and any `http3` + build are — has to mean something rather than fail. aws-lc-rs wins, and ring is installed as the + process provider only where it is the sole choice. A `compile_error!` remains for *neither*, which + is a real misconfiguration: an HTTPS client that cannot speak TLS is not one. reqwest's `http3` + pins its QUIC stack to aws-lc-rs, which is why `http3` enables that backend rather than tolerating + either. +- **napi's derives ignore `#[cfg]` on a field or an impl method.** `#[napi(object)]` re-emits field + types and `#[napi] impl` enumerates method names, both at macro time, so a gated field or method + leaves generated code referencing an item that is no longer there. Methods can still be removed — + by moving them to their own `#[cfg]`-gated `#[napi] impl` block, which napi accepts — but an + option object keeps its full shape whatever the build. So the binding refuses what it cannot + honour instead: `refuse_absent_capabilities` for an agent option group, and a check in + `faith_fetch` for a per-request one. Silently ignoring the option was the alternative, and it + would make a slim build look like it worked. +- **Gating an option group means gating the tests that set it.** `cargo test` on a slim build was + broken by tests and a doctest reaching for fields that are no longer compiled. The doctest was the + worse of the two, having no per-feature escape: the fix was to illustrate with a group that is + never gated. Check the matrix with `cargo test`, not just `cargo build`. +- **The binding carried two dozen dependencies it no longer used**, left over from before the + extraction — including `web-faith-dns` and `web-faith-alt-svc`, which a feature claimed to drop + while linking them anyway. Worth re-checking after any extraction: `use` roots in the source + against the manifest. ## Step 14: the both-surfaces spec sweep diff --git a/Cargo.lock b/Cargo.lock index 0653e31..f39f9db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1986,6 +1986,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2810,6 +2811,7 @@ dependencies = [ "moka", "reqwest", "reqwest-middleware", + "rustls", "serde", "serde_json", "ssri", diff --git a/Cargo.toml b/Cargo.toml index 2867b71..69fe056 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,10 +56,10 @@ napi-derive = "3.4.0" reqwest = { version = "0.13.4", default-features = false, features = [ "http2", "json", - "rustls", "stream", "system-proxy", ] } +rustls = { version = "0.23.35", default-features = false } reqwest-middleware = { version = "0.5.2", features = ["http2", "rustls"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" diff --git a/crates/web-faith-napi/Cargo.toml b/crates/web-faith-napi/Cargo.toml index c6f395a..63fab31 100644 --- a/crates/web-faith-napi/Cargo.toml +++ b/crates/web-faith-napi/Cargo.toml @@ -34,10 +34,20 @@ web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true napi-build.workspace = true [features] -default = ["cache", "connection-tracking", "cookies", "dns", "encoding", "http3"] +default = [ + "cache", + "connection-tracking", + "cookies", + "dns", + "encoding", + "http3", + "tls-aws-lc-rs", +] cache = ["dep:http-cache-reqwest", "web-faith/cache"] connection-tracking = ["dep:web-faith-conn-tracker", "web-faith/connection-tracking"] cookies = ["dep:web-faith-cookies", "reqwest/cookies", "web-faith/cookies"] dns = ["web-faith/dns"] encoding = ["web-faith/encoding"] -http3 = ["reqwest/http3", "web-faith/http3"] +http3 = ["reqwest/http3", "tls-aws-lc-rs", "web-faith/http3"] +tls-aws-lc-rs = ["reqwest/rustls", "web-faith/tls-aws-lc-rs"] +tls-ring = ["reqwest/rustls-no-provider", "web-faith/tls-ring"] diff --git a/crates/web-faith-napi/src/agent.rs b/crates/web-faith-napi/src/agent.rs index a068ab0..f8c32bc 100644 --- a/crates/web-faith-napi/src/agent.rs +++ b/crates/web-faith-napi/src/agent.rs @@ -243,6 +243,11 @@ fn refuse_absent_capabilities(options: &AgentOptions) -> Result<(), FaithError> return absent("HTTP cache"); } + #[cfg(not(feature = "http3"))] + if options.http3.is_some() { + return absent("HTTP/3"); + } + #[cfg(not(feature = "cookies"))] if options.cookies.is_some() { return absent("cookie"); diff --git a/crates/web-faith-napi/src/agent/convert.rs b/crates/web-faith-napi/src/agent/convert.rs index ebf545d..49eef5a 100644 --- a/crates/web-faith-napi/src/agent/convert.rs +++ b/crates/web-faith-napi/src/agent/convert.rs @@ -9,7 +9,10 @@ use web_faith::{client::RedirectPolicy, options}; #[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; -use crate::agent::{AgentOptions, Http3Congestion}; +use crate::agent::AgentOptions; + +#[cfg(feature = "http3")] +use crate::agent::Http3Congestion; #[cfg(feature = "cache")] use crate::agent::CacheStore; @@ -87,6 +90,7 @@ impl From for options::AgentOptions { connection_window: http2.connection_window, adaptive_window: http2.adaptive_window, }), + #[cfg(feature = "http3")] http3: opts.http3.map(|http3| options::Http3Options { congestion: http3.congestion.map(|c| match c { Http3Congestion::Cubic => options::Http3Congestion::Cubic, diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index a36f582..253d0b9 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -22,6 +22,7 @@ hyper.workspace = true hyper-util.workspace = true moka.workspace = true reqwest.workspace = true +rustls = { workspace = true, optional = true } reqwest-middleware.workspace = true serde.workspace = true serde_json.workspace = true @@ -37,7 +38,15 @@ web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } web-faith-alt-svc = { workspace = true, optional = true } [features] -default = ["cache", "connection-tracking", "cookies", "dns", "encoding", "http3"] +default = [ + "cache", + "connection-tracking", + "cookies", + "dns", + "encoding", + "http3", + "tls-aws-lc-rs", +] # The HTTP cache, its store, and the per-request cache mode. cache = ["dep:http-cache-reqwest"] # Per-connection kernel counters, and the agent verb that reports them. @@ -50,4 +59,9 @@ dns = ["dep:web-faith-dns", "reqwest/hickory-dns", "web-faith-alt-svc?/dns"] # Content codings: negotiating and decoding a response body, and compressing a request one. encoding = ["dep:web-faith-encoding"] # Transparent HTTP/3, and the Alt-Svc machinery that upgrades an origin to it. -http3 = ["reqwest/http3", "dep:web-faith-alt-svc"] +http3 = ["reqwest/http3", "tls-aws-lc-rs", "dep:web-faith-alt-svc"] +# The rustls crypto provider: aws-lc-rs by default, ring as the alternative. Cargo features are +# additive, so enabling both is aws-lc-rs, which is also what an `http3` build resolves to because +# reqwest's QUIC stack is tied to it. +tls-aws-lc-rs = ["reqwest/rustls"] +tls-ring = ["reqwest/rustls-no-provider", "dep:rustls", "rustls/ring"] diff --git a/crates/web-faith/src/agent/build.rs b/crates/web-faith/src/agent/build.rs index 99cf236..bcd31b2 100644 --- a/crates/web-faith/src/agent/build.rs +++ b/crates/web-faith/src/agent/build.rs @@ -70,8 +70,7 @@ impl Agent { flow_control, headers, http2, - // Every use of the HTTP/3 options sits behind the feature. - #[cfg_attr(not(feature = "http3"), allow(unused_variables))] + #[cfg(feature = "http3")] http3, local_address, pool, diff --git a/crates/web-faith/src/agent/tests.rs b/crates/web-faith/src/agent/tests.rs index c099352..834d5bf 100644 --- a/crates/web-faith/src/agent/tests.rs +++ b/crates/web-faith/src/agent/tests.rs @@ -9,11 +9,13 @@ async fn a_default_agent_comes_up() { assert!(!agent.is_closed()); assert_eq!(agent.recipe.user_agent, crate::USER_AGENT); // No jar until the options ask for one. + #[cfg(feature = "cookies")] assert!(agent.cookie_jar.is_none()); } /// The builder reaches every group, and a group left alone stays absent rather than being /// spelled out as absent. +#[cfg(all(feature = "dns", feature = "http3"))] #[tokio::test] async fn the_builder_sets_what_it_is_given_and_nothing_else() { use std::time::Duration; diff --git a/crates/web-faith/src/builder.rs b/crates/web-faith/src/builder.rs index 8b0e84e..16b0239 100644 --- a/crates/web-faith/src/builder.rs +++ b/crates/web-faith/src/builder.rs @@ -9,7 +9,7 @@ //! # use web_faith::agent::Agent; //! let agent = Agent::builder() //! .user_agent("YourApp/1.2.3") -//! .dns(|dns| dns.timeout(Duration::from_secs(2))) +//! .timeout(|timeout| timeout.connect(Duration::from_secs(2))) //! .pool(|pool| pool.max_idle_per_host(8)) //! .build()?; //! # Ok::<(), web_faith::FaithError>(()) @@ -30,6 +30,9 @@ use http_cache_reqwest::CacheMode; #[cfg(feature = "cache")] use crate::options::{CacheOptions, CacheStore}; +#[cfg(feature = "http3")] +use crate::options::{Http3Congestion, Http3Hint, Http3Options}; + #[cfg(feature = "cookies")] use web_faith_cookies::CookieLimits; @@ -39,8 +42,7 @@ use crate::{ error::FaithError, options::{ AgentOptions, DnsOptions, DnsOverride, FlowControlOptions, Header, Http2Options, - Http3Congestion, Http3Hint, Http3Options, PoolOptions, QuirksOptions, TimeoutOptions, - TlsOptions, + PoolOptions, QuirksOptions, TimeoutOptions, TlsOptions, }, }; @@ -172,6 +174,7 @@ impl AgentBuilder { } /// HTTP/3 settings. + #[cfg(feature = "http3")] pub fn http3(mut self, with: impl FnOnce(Http3Builder) -> Http3Builder) -> Self { let group = self.options.http3.take().unwrap_or_default(); self.options.http3 = Some(with(Http3Builder { group }).group); @@ -448,12 +451,14 @@ impl Http2Builder { } /// HTTP/3 settings. Reached through [`AgentBuilder::http3`]. +#[cfg(feature = "http3")] #[derive(Debug, Default)] #[must_use] pub struct Http3Builder { group: Http3Options, } +#[cfg(feature = "http3")] impl Http3Builder { /// The congestion controller QUIC runs. pub fn congestion(mut self, congestion: Http3Congestion) -> Self { diff --git a/crates/web-faith/src/client.rs b/crates/web-faith/src/client.rs index cbe94fc..8af0bff 100644 --- a/crates/web-faith/src/client.rs +++ b/crates/web-faith/src/client.rs @@ -257,6 +257,22 @@ pub struct BuiltClients { pub prober: Option>, } +/// Install ring as the process's rustls crypto provider. +/// +/// reqwest reads the process default when it builds a client and panics if there is none, so this +/// runs before the first one is built. Only where ring is the chosen backend: with `tls-aws-lc-rs` +/// also on, reqwest supplies aws-lc-rs itself, which is also what an HTTP/3 build needs. Installing +/// is process-wide and once-only, so a provider the embedding program put in place is left alone. +#[cfg(all(feature = "tls-ring", not(feature = "tls-aws-lc-rs")))] +fn install_crypto_provider() { + use std::sync::Once; + + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + impl ClientRecipe { /// The window an idle pooled connection lives in, which is also how long a warm-up counts as /// warm and how long a connection stays listed. @@ -280,6 +296,9 @@ impl ClientRecipe { #[cfg(feature = "dns")] dns_resolver: Option<&FaithResolver>, #[cfg(feature = "http3")] alt_svc_cache: Option<&Arc>, ) -> Result { + #[cfg(all(feature = "tls-ring", not(feature = "tls-aws-lc-rs")))] + install_crypto_provider(); + let mut client = Client::builder() .tls_info(true) .tls_sslkeylogfile(true) diff --git a/crates/web-faith/src/lib.rs b/crates/web-faith/src/lib.rs index f41b030..5855b7d 100644 --- a/crates/web-faith/src/lib.rs +++ b/crates/web-faith/src/lib.rs @@ -30,6 +30,11 @@ //! [`Agent::fetch`]: agent::Agent::fetch //! [`Request`]: request::Request +// A build with no crypto provider cannot speak TLS, and an HTTPS client that cannot is not one. +// Selecting a provider is therefore a choice between the two rather than an option to decline. +#[cfg(not(any(feature = "tls-aws-lc-rs", feature = "tls-ring")))] +compile_error!("web-faith needs a TLS backend: enable either tls-aws-lc-rs or tls-ring"); + pub mod agent; pub mod body; pub mod builder; diff --git a/crates/web-faith/src/options.rs b/crates/web-faith/src/options.rs index d95753e..edb290b 100644 --- a/crates/web-faith/src/options.rs +++ b/crates/web-faith/src/options.rs @@ -170,6 +170,7 @@ pub struct Header { pub sensitive: Option, } +#[cfg(feature = "http3")] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum Http3Congestion { #[default] @@ -180,6 +181,7 @@ pub enum Http3Congestion { /// A hint that HTTP/3 is available at a specific host and port. This pre-populates the Alt-Svc /// cache so the first request to this host will attempt HTTP/3 immediately. +#[cfg(feature = "http3")] #[derive(Clone, Debug, Default)] pub struct Http3Hint { /// The hostname (e.g., "example.com"). @@ -189,6 +191,7 @@ pub struct Http3Hint { } /// Settings related to HTTP/3. This is a nested object. +#[cfg(feature = "http3")] #[derive(Clone, Debug, Default)] pub struct Http3Options { /// The congestion control algorithm. The default is `cubic`, which is the same used in TCP in the @@ -608,6 +611,7 @@ pub struct AgentOptions { /// Settings related to HTTP/2. This is a nested object. pub http2: Option, /// Settings related to HTTP/3. This is a nested object. + #[cfg(feature = "http3")] pub http3: Option, /// Bind outgoing sockets to this local IP address before connecting. /// From 6969237f7ccdec79edab9a8d931addc6d268821d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= <155787+passcod@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:09:17 +1200 Subject: [PATCH 61/61] S1: per-crate examples, client integration tests, and test cases Each component crate gains an example naming only itself, which is what proves it stands alone; crates/web-faith/tests/fetch.rs covers the fetch-flavoured surface against a live origin, skipping when HTTPBIN_URL names none. Writing the examples surfaced two gaps: AltSvcCacheConfig had no Default, so the store could not be used without copying eleven fields out of web-faith. Co-Authored-By: Claude Opus 5 --- .workhorse/plans/s1/plan.md | 20 +- .workhorse/test-cases/s1/overview.md | 116 ++++++ crates/web-faith-alt-svc/Cargo.toml | 6 + crates/web-faith-alt-svc/examples/upgrade.rs | 51 +++ crates/web-faith-alt-svc/src/cache.rs | 20 ++ .../examples/snapshot.rs | 43 +++ crates/web-faith-cookies/Cargo.toml | 3 +- crates/web-faith-cookies/examples/jar.rs | 35 ++ crates/web-faith-dns/Cargo.toml | 4 + crates/web-faith-dns/examples/resolve.rs | 43 +++ crates/web-faith-encoding/examples/codings.rs | 53 +++ crates/web-faith/Cargo.toml | 5 + crates/web-faith/examples/fetch.rs | 50 +++ crates/web-faith/tests/fetch.rs | 340 ++++++++++++++++++ 14 files changed, 787 insertions(+), 2 deletions(-) create mode 100644 .workhorse/test-cases/s1/overview.md create mode 100644 crates/web-faith-alt-svc/examples/upgrade.rs create mode 100644 crates/web-faith-conn-tracker/examples/snapshot.rs create mode 100644 crates/web-faith-cookies/examples/jar.rs create mode 100644 crates/web-faith-dns/examples/resolve.rs create mode 100644 crates/web-faith-encoding/examples/codings.rs create mode 100644 crates/web-faith/examples/fetch.rs create mode 100644 crates/web-faith/tests/fetch.rs diff --git a/.workhorse/plans/s1/plan.md b/.workhorse/plans/s1/plan.md index b4f7344..3cc0fb9 100644 --- a/.workhorse/plans/s1/plan.md +++ b/.workhorse/plans/s1/plan.md @@ -130,8 +130,12 @@ QUIC/TLS stay inside `web-faith` as reqwest features (aws-lc-rs default, ring al `tls-aws-lc-rs`/`tls-ring` backend choice; `web-faith-alt-svc` carries `dns` for the HTTPS-record sink. `web-faith-napi` mirrors the set. Integrity is deliberately not a feature: it is always built. -- [ ] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; +- [x] **11. Rust-facing tests + examples** — per-crate examples that run against that crate alone; client integration tests mirroring the JS suite where it translates. Add `.workhorse/test-cases/s1/`. + + Five component examples plus the client's, and `crates/web-faith/tests/fetch.rs` covering the + fetch-flavoured surface against a live origin. Test cases are in + [`.workhorse/test-cases/s1/overview.md`](../../test-cases/s1/overview.md). - [ ] **12. Publishing infra** — release-plz, `cargo-semver-checks` against previous version per crate, MSRV 1.96 declared in every published crate and exercised in CI alongside stable, independent versioning from `1.0.0`. Measure CI cost before adding jobs (see project memory). @@ -220,6 +224,20 @@ Decisions taken while doing steps 0–7, worth not relitigating: broken by tests and a doctest reaching for fields that are no longer compiled. The doctest was the worse of the two, having no per-feature escape: the fix was to illustrate with a group that is never gated. Check the matrix with `cargo test`, not just `cargo build`. +- **An integration test that needs an origin skips rather than fails.** `crates/web-faith/tests/fetch.rs` + reads `HTTPBIN_URL` with no default and reports the skip when it is unset, so `cargo test` works + on a machine with no server to hand while CI gets the full run. The JS suite defaults to + `localhost:8888` instead, which is why it cannot be run without one. +- **Examples are the proof a component crate stands alone.** Each names only its own crate, so a + dependency that had leaked upward would not compile. Two API gaps surfaced from writing them: + `AltSvcCacheConfig` had no `Default`, which made the store unusable without copying eleven fields + out of `web-faith`, and `ConnectionTracker::new` needs a tokio runtime (it spawns the counter + refresh), which the example now says out loud. +- **go-httpbin runs as a native binary, not only a container.** `podman run ghcr.io/mccutchen/go-httpbin` + fails on a host with no `/etc/subuid` range, because the image wants uid 65532 and unpacking it + chowns to that. `~/go/bin/go-httpbin -port 8888` needs no root and serves the same endpoints. Note + its header values arrive as arrays (`{"Name": ["value"]}`), and it echoes a body with no + `Content-Type` back as a base64 data URL. - **The binding carried two dozen dependencies it no longer used**, left over from before the extraction — including `web-faith-dns` and `web-faith-alt-svc`, which a feature claimed to drop while linking them anyway. Worth re-checking after any extraction: `use` roots in the source diff --git a/.workhorse/test-cases/s1/overview.md b/.workhorse/test-cases/s1/overview.md new file mode 100644 index 0000000..c41fd1b --- /dev/null +++ b/.workhorse/test-cases/s1/overview.md @@ -0,0 +1,116 @@ +# Rust API and crates.io publication + +Coverage for splitting the single napi cdylib into a Cargo workspace, exposing `web-faith` as a +Rust client with component crates beneath it, and publishing the family. + +An unticked box is coverage this card still owes, not a scenario that was considered and dropped. + +## No regression on the Node surface + +The binding's behaviour is the control for the whole restructure: the JS suite passed before the +split and must pass after it, unchanged. + +- [x] The full JS suite passes (`HTTPBIN_URL=… npm run test:only`, 2111 assertions). +- [x] `index.js` and `index.d.ts` are byte-identical after a rebuild, or differ only by declaration + order where a method moved to a gated `impl` block. Verified by diffing after + `npm run build:debug`. +- [x] The generated TypeScript keeps the documentation napi emits from Rust doc comments: no class + or method loses its docs to the move. Verifies spec: RUST. +- [ ] `@passcod/faith` installs and loads from a packed tarball, so the published module is not + missing a file the workspace layout moved. + +## The Rust client's shape + +Verifies spec: RSAPI. + +- [x] `Agent::new` builds an agent with defaults, and `Agent::builder` reaches every option group; + a group left alone is absent rather than spelled out as absent. +- [x] `agent.fetch(url).await` sends without a separate send step (`IntoFuture`). +- [x] `Request::new(...).build()` prepares a request that carries no agent, can be sent more than + once, and can be layered over per call site. +- [x] Layering puts the outermost explicit value in charge; headers merge by name, and a removal + reaches through to whatever the layers beneath contributed. +- [x] `try_clone` copies a request with a buffered body and returns `None` for a stream body. +- [x] A setter taking anything convertible holds a failed conversion until the builder resolves, + and the first failure met is the one reported. +- [x] `text()`, `json()`, and `bytes()` each read the one body; a second read reports + `ResponseAlreadyDisturbed`. +- [x] Cloning an agent names the same agent: closing through one clone closes it for all, and a + request issued afterwards reports `Closed`. +- [x] A request issued before a close runs to completion. +- [ ] `body_stream()` delivers chunks, and the trailers and timing promises settle after the last + one. +- [ ] `write_to_file` writes the body and refuses an existing destination. +- [ ] `into_http` hands over an `http::Response` whose body is the undisturbed stream. + +## Errors keep their codes across the split + +Verifies spec: ERR. + +- [x] `FaithErrorKind` is one definition on both surfaces, and `error_codes()` is generated from it + rather than listed separately. +- [x] Every kind renders a message led by its own code, and one with no message falls back to its + default. +- [x] A timeout reports `Timeout`, an integrity mismatch `IntegrityMismatch`, a malformed integrity + value `InvalidIntegrity`, an unparseable URL `InvalidUrl`, and a bad method `InvalidMethod` — + each rather than a generic `Network`. +- [x] A redirect the agent's own policy refused keeps the kind Faith chose, rather than being + flattened into reqwest's own redirect error. + +## Each component crate stands alone + +Verifies spec: RUST. + +- [x] Every component crate carries an example that runs against that crate alone, with no + `web-faith` dependency: cookies, codings, connection tracking, the Alt-Svc store, and the + resolver. +- [x] Each example compiles and runs (`cargo run -p --example `). +- [x] `web-faith-alt-svc` builds and its example compiles without `web-faith-dns`, the HTTPS-record + sink being the only thing that needed it. +- [x] No component crate has napi anywhere in its dependency graph; only `web-faith-napi` does. +- [ ] `cargo package` succeeds for each crate, so nothing depends on a path that only exists in the + workspace. +- [ ] `cargo publish --dry-run` succeeds for the family in dependency order. + +## Features drop what they name + +Verifies spec: RUST. + +- [x] `cargo build` and `cargo test` pass for: default, `--all-features`, TLS alone, and TLS plus + each of `cache`, `connection-tracking`, `cookies`, `dns`, `encoding`, and `http3` + individually, on both `web-faith` and `web-faith-napi`. +- [x] Turning a feature off removes the API that only means something with it: no cookie jar handle + without `cookies`, no cache mode without `cache`, no request compression without `encoding`, + no `resolvers()` without `dns`. +- [x] Turning a feature off drops the dependency too, including the reqwest feature behind it. +- [x] A slim binding refuses an option group it cannot honour at agent construction, and a + per-request option at `fetch`, rather than ignoring it. +- [x] `tls-ring` builds and a client comes up under it, reqwest finding the installed provider + rather than panicking. +- [x] Enabling both TLS backends resolves to aws-lc-rs rather than failing, so `--all-features` and + any `http3` build work. +- [x] Selecting neither TLS backend is a compile error naming both options. +- [ ] Each feature combination passes the JS suite where the binding is what changed, not just + `cargo build`. + +## Publication and versioning + +Verifies spec: RUST. + +- [ ] `cargo-semver-checks` runs against the previous version of each crate and passes on a release + that claims to be non-breaking. +- [ ] `rust-version` is declared in every published crate and inherited from the workspace root. +- [ ] CI builds and tests against MSRV 1.96 as well as stable, so the declaration is verified + rather than asserted. +- [ ] release-plz prepares a release, and a change confined to one component moves that crate's + version alone. +- [ ] The published crates resolve from crates.io in a fresh project, `web-faith` pulling its + components at the versions it declares. + +## Housekeeping the restructure owes + +- [x] `cargo doc --workspace --no-deps` is warning-free, so no published crate ships a broken + intra-doc link. +- [x] `cargo fmt --all --check` is clean. +- [x] No source file exceeds 1000 lines, files that are entirely tests excepted. +- [x] The binding's manifest names only crates it uses. diff --git a/crates/web-faith-alt-svc/Cargo.toml b/crates/web-faith-alt-svc/Cargo.toml index eea8b42..f8d04b7 100644 --- a/crates/web-faith-alt-svc/Cargo.toml +++ b/crates/web-faith-alt-svc/Cargo.toml @@ -20,6 +20,12 @@ tokio.workspace = true url.workspace = true web-faith-dns = { workspace = true, optional = true } +[dev-dependencies] +# Examples link as their own crate, so the types the API speaks have to be named there too. +http.workspace = true +reqwest.workspace = true +tokio = { workspace = true } + [features] default = ["dns"] # Feed `HTTPS` DNS records into the store, so an origin is probe-worthy before anything connects. diff --git a/crates/web-faith-alt-svc/examples/upgrade.rs b/crates/web-faith-alt-svc/examples/upgrade.rs new file mode 100644 index 0000000..3f31d36 --- /dev/null +++ b/crates/web-faith-alt-svc/examples/upgrade.rs @@ -0,0 +1,51 @@ +//! Read an origin's `Alt-Svc` header into the store, then follow the upgrade decision it drives. +//! +//! Run with `cargo run -p web-faith-alt-svc --example upgrade`. + +use std::time::Duration; + +use reqwest::Url; +use web_faith_alt_svc::{AltSvcCache, AltSvcCacheConfig, parse_alt_svc_header}; + +fn report(cache: &AltSvcCache, url: &Url, stage: &str) { + println!( + "{stage}: route on {:?}, probe {:?}", + cache.confirmed_port(url), + cache.probe_candidate(url) + ); +} + +fn main() { + let cache = AltSvcCache::new(AltSvcCacheConfig::default()); + let origin = Url::parse("https://example.com/").expect("a valid URL"); + + // An advertisement is evidence worth probing, not evidence worth routing on: it says the + // alternative exists, not that it works. + let advertisement = + parse_alt_svc_header(r#"h3=":443"; ma=86400"#).expect("the header advertises h3"); + cache.record_alt_svc(&origin, &advertisement); + report(&cache, &origin, "advertised"); + + // A probe that reaches the origin over HTTP/3 is what promotes it to routable. + let port = cache + .probe_candidate(&origin) + .expect("the advertisement is probe-worthy"); + cache.claim_probe(&origin); + cache.confirm_h3(&origin, port); + cache.finish_probe(&origin); + report(&cache, &origin, "confirmed"); + + // A confirmed origin that turns out sustainedly slower over HTTP/3 than the path it replaced + // is demoted, and is not re-probed while the slow marker lives: once it lapses the preserved + // advertisement re-enters through a probe, asking whether the path has improved. + for _ in 0..16 { + cache.record_path_time(&origin, http::Version::HTTP_11, Duration::from_millis(20)); + cache.record_path_time(&origin, http::Version::HTTP_3, Duration::from_millis(400)); + } + report(&cache, &origin, "slow"); + + // A network change drops what was learned about a network that no longer exists, keeping the + // advertisement, which came from the origin rather than from the path. + cache.network_changed(); + report(&cache, &origin, "after a network change"); +} diff --git a/crates/web-faith-alt-svc/src/cache.rs b/crates/web-faith-alt-svc/src/cache.rs index 3d9f567..aedc4b2 100644 --- a/crates/web-faith-alt-svc/src/cache.rs +++ b/crates/web-faith-alt-svc/src/cache.rs @@ -79,6 +79,26 @@ pub struct AltSvcCacheConfig { /// How long a path-time demotion holds before the origin may be re-probed. pub slow_ttl: Duration, } +impl Default for AltSvcCacheConfig { + /// The same values `web-faith` settles on when a caller names none. + fn default() -> Self { + Self { + advertised_ttl: Duration::from_secs(86_400), + confirmed_ttl: Duration::from_secs(86_400), + failed_ttl: Duration::from_secs(300), + failed_max_ttl: Duration::from_secs(3_600), + capacity: 10_000, + cancel_strikes: 3, + strike_window: Duration::from_secs(60), + follow_advertised_port: false, + // Long enough to outlive a probe that is never reported, so an aborted one frees its + // origin: a probe deadline plus a margin, or the QUIC idle timeout without one. + probe_ttl: Duration::from_secs(125), + slow_factor: 2.5, + slow_ttl: Duration::from_secs(600), + } + } +} #[derive(Clone)] pub struct AltSvcCache { diff --git a/crates/web-faith-conn-tracker/examples/snapshot.rs b/crates/web-faith-conn-tracker/examples/snapshot.rs new file mode 100644 index 0000000..33434b0 --- /dev/null +++ b/crates/web-faith-conn-tracker/examples/snapshot.rs @@ -0,0 +1,43 @@ +//! Track a connection and read back the kernel's own view of it. +//! +//! The addresses here are made up, so the snapshot carries the tracker's own accounting without +//! the per-socket TCP counters a live connection would have. Point it at a real socket pair to see +//! those filled in. +//! +//! Run with `cargo run -p web-faith-conn-tracker --example snapshot`. + +use std::{net::SocketAddr, time::Duration}; + +use web_faith_conn_tracker::ConnectionTracker; + +// The tracker spawns a task to refresh the kernel's counters, so it is built inside a runtime. +#[tokio::main] +async fn main() { + let tracker = ConnectionTracker::new(Duration::from_secs(90)); + + let local: SocketAddr = "127.0.0.1:54321".parse().expect("a valid address"); + let remote: SocketAddr = "93.184.216.34:443".parse().expect("a valid address"); + + // The first sighting is a new connection; a later one is the pool handing it back, which is + // what the return value reports. + println!( + "first request reused a connection: {}", + tracker.track(local, remote) + ); + println!("second request reused it: {}", tracker.track(local, remote)); + + // A warm-up's connection is tracked too, so a later request on it counts as a reuse. + let warmed: SocketAddr = "127.0.0.1:54322".parse().expect("a valid address"); + tracker.track_warmup(warmed, remote); + + for connection in tracker.snapshot() { + println!( + "{} {} -> {}: {} responses, os stats: {}", + connection.connection_type, + connection.local_addr, + connection.remote_addr, + connection.response_count, + connection.stats.is_some() + ); + } +} diff --git a/crates/web-faith-cookies/Cargo.toml b/crates/web-faith-cookies/Cargo.toml index c678d0f..3e94e4c 100644 --- a/crates/web-faith-cookies/Cargo.toml +++ b/crates/web-faith-cookies/Cargo.toml @@ -19,7 +19,8 @@ url.workspace = true reqwest = { workspace = true, optional = true } [dev-dependencies] -# Doctests link as their own crate, so the types the API speaks have to be named there too. +# Doctests and examples link as their own crate, so the types the API speaks have to be named there too. +http.workspace = true url.workspace = true [features] diff --git a/crates/web-faith-cookies/examples/jar.rs b/crates/web-faith-cookies/examples/jar.rs new file mode 100644 index 0000000..261adca --- /dev/null +++ b/crates/web-faith-cookies/examples/jar.rs @@ -0,0 +1,35 @@ +//! Store the cookies a response set, then read back the header the next request should carry. +//! +//! Run with `cargo run -p web-faith-cookies --example jar`. + +use http::HeaderValue; +use url::Url; +use web_faith_cookies::{CookieLimits, FaithJar}; + +fn main() { + let jar = FaithJar::new(CookieLimits::default()); + let origin = Url::parse("https://example.com/login").expect("a valid URL"); + + // As a response would set them: one plain, one host-locked, one that expires long past the cap. + let set_cookie = [ + HeaderValue::from_static("session=abc123; Path=/; Secure; HttpOnly"), + HeaderValue::from_static("__Host-csrf=xyz; Path=/; Secure"), + HeaderValue::from_static("stale=1; Max-Age=999999999"), + ]; + jar.store_response_cookies(set_cookie.iter(), &origin); + + // A cookie can also be added by hand, the way a caller seeds a jar. + jar.add_cookie_str("theme=dark; Path=/", &origin); + + match jar.request_cookie_header(&origin) { + Some(header) => println!("Cookie: {}", header.to_str().expect("ASCII cookie values")), + None => println!("the jar has nothing for {origin}"), + } + + // A different origin sees none of them: the jar matches on host and path. + let elsewhere = Url::parse("https://other.example/").expect("a valid URL"); + println!( + "other origin: {:?}", + jar.request_cookie_header(&elsewhere).is_none() + ); +} diff --git a/crates/web-faith-dns/Cargo.toml b/crates/web-faith-dns/Cargo.toml index 4d3ca4f..c4f89a5 100644 --- a/crates/web-faith-dns/Cargo.toml +++ b/crates/web-faith-dns/Cargo.toml @@ -18,6 +18,10 @@ tokio.workspace = true url.workspace = true reqwest = { workspace = true, optional = true } +[dev-dependencies] +# Examples link as their own crate, so the runtime they need has to be named there too. +tokio = { workspace = true } + [features] # Implement reqwest's `Resolve`, so the resolver can be installed on its client. reqwest = ["dep:reqwest", "reqwest/hickory-dns"] diff --git a/crates/web-faith-dns/examples/resolve.rs b/crates/web-faith-dns/examples/resolve.rs new file mode 100644 index 0000000..511523b --- /dev/null +++ b/crates/web-faith-dns/examples/resolve.rs @@ -0,0 +1,43 @@ +//! Warm the cache for a name, then read back which servers the lookup went through. +//! +//! Resolves against the operating system's own servers, since those are what a machine running +//! this is configured for. Pass a name to look up something other than `localhost`. +//! +//! Run with `cargo run -p web-faith-dns --example resolve -- example.com`. + +use web_faith_dns::{FaithResolver, ResolverSettings}; + +#[tokio::main] +async fn main() { + let host = std::env::args() + .nth(1) + .unwrap_or_else(|| "localhost".into()); + + // No servers named, so the resolver configures itself from the operating system and lets + // opportunistic encryption upgrade those servers where it can. + let resolver = FaithResolver::new(ResolverSettings::default()); + + // Prefetching populates the very cache a request's lookup reads, and never fails: warming is + // advisory, so a name that does not resolve simply leaves the cache as it was. + resolver.prefetch(&host).await; + + // The configuration is read on first use, so this is empty until something has resolved. + let reports = resolver.resolvers(); + if reports.is_empty() { + println!("nothing resolved through Faith's own servers for {host}"); + } + for report in reports { + println!( + "{} over {} ({})", + report.address, report.transport, report.source + ); + } + + // A network change discards what was learned from a network that no longer exists, leaving the + // resolver usable rather than needing to be rebuilt. + resolver.reset(); + println!( + "after a network change: {} servers", + resolver.resolvers().len() + ); +} diff --git a/crates/web-faith-encoding/examples/codings.rs b/crates/web-faith-encoding/examples/codings.rs new file mode 100644 index 0000000..58a5dab --- /dev/null +++ b/crates/web-faith-encoding/examples/codings.rs @@ -0,0 +1,53 @@ +//! Negotiate a coding from a response's headers, decode a body under it, and compress one on the +//! way out. +//! +//! Run with `cargo run -p web-faith-encoding --example codings`. + +use bytes::Bytes; +use futures::StreamExt as _; +use http::{HeaderMap, HeaderValue}; +use web_faith_encoding::{ + AcceptEncoding, Coding, DEFAULT_ACCEPT_ENCODING, compress_buffer, decision, decode_stream, + layer_content_encoding, strip_decoded_headers, +}; + +#[tokio::main] +async fn main() { + let accept = AcceptEncoding::parse(DEFAULT_ACCEPT_ENCODING); + + // What a response's headers negotiate against what the request asked for. + let mut headers = HeaderMap::new(); + headers.insert("content-encoding", HeaderValue::from_static("gzip")); + headers.insert("content-length", HeaderValue::from_static("42")); + let coding = decision(&headers, &accept).expect("gzip is in the default Accept-Encoding"); + println!("negotiated: {coding:?}"); + + // A decoded body's length and coding no longer describe what the caller receives. + strip_decoded_headers(&mut headers); + println!("headers after decoding: {:?}", headers.keys().count()); + + // Round-trip a body through the coding that was negotiated. + let original = b"the quick brown fox jumps over the lazy dog".repeat(8); + let compressed = compress_buffer(&original, coding) + .await + .expect("gzip compresses"); + println!( + "{} bytes in, {} bytes out", + original.len(), + compressed.len() + ); + + let stream = futures::stream::once(async move { Ok(Bytes::from(compressed)) }); + let mut decoded = decode_stream(Box::pin(stream), coding); + let mut round_tripped = Vec::new(); + while let Some(chunk) = decoded.next().await { + round_tripped.extend_from_slice(&chunk.expect("the body decodes")); + } + println!("round-tripped intact: {}", round_tripped == original); + + // A request that compresses on top of a coding the caller already applied names both, in order. + println!( + "Content-Encoding: {}", + layer_content_encoding(Some("br"), Coding::Gzip) + ); +} diff --git a/crates/web-faith/Cargo.toml b/crates/web-faith/Cargo.toml index 253d0b9..a756757 100644 --- a/crates/web-faith/Cargo.toml +++ b/crates/web-faith/Cargo.toml @@ -37,6 +37,11 @@ web-faith-cookies = { workspace = true, features = ["reqwest"], optional = true web-faith-dns = { workspace = true, features = ["reqwest"], optional = true } web-faith-alt-svc = { workspace = true, optional = true } +[dev-dependencies] +# Examples and integration tests link as their own crate, so what they name has to be here too. +serde.workspace = true +tokio = { workspace = true } + [features] default = [ "cache", diff --git a/crates/web-faith/examples/fetch.rs b/crates/web-faith/examples/fetch.rs new file mode 100644 index 0000000..fddd582 --- /dev/null +++ b/crates/web-faith/examples/fetch.rs @@ -0,0 +1,50 @@ +//! Fetch a URL, then prepare a request once and send it more than once. +//! +//! Run with `cargo run -p web-faith --example fetch -- https://example.com/`. + +use std::time::Duration; + +use web_faith::{FaithError, agent::Agent, request::Request}; + +#[tokio::main] +async fn main() -> Result<(), FaithError> { + let url = std::env::args() + .nth(1) + .unwrap_or_else(|| "https://example.com/".into()); + + // An agent owns the connection pool, resolver, and caches. Cloning one is cheap and every + // clone names the same agent, so it is what a request runs on rather than a second pool. + let agent = Agent::builder() + .user_agent(format!("fetch-example/1.0 {}", web_faith::USER_AGENT)) + .timeout(|timeout| timeout.total(Duration::from_secs(30))) + .build()?; + + // `fetch` returns a builder that sends when awaited, so there is no separate send step. + let response = agent.fetch(url.as_str()).await?; + println!("{} {}", response.status(), response.url()); + println!("{} over {:?}", response.status_text(), response.version()); + + let body = response.text().await?; + println!("{} bytes of body", body.len()); + + // A prepared request carries no agent, so it can be adjusted per call site or sent unchanged + // on more than one agent. Layering it puts the outermost value in charge, and headers merge. + let probe = Request::new(url.as_str()) + .method("HEAD") + .header("x-example", "prepared") + .build()?; + + for _ in 0..2 { + let response = agent + .fetch(probe.try_clone().expect("no stream body")) + .await?; + println!("HEAD -> {}", response.status()); + } + + // Closing releases the pool and the resolver rather than waiting for the agent to be dropped. + // Requests already in flight run to completion; a new one is refused. + agent.close(); + println!("closed: {}", agent.is_closed()); + + Ok(()) +} diff --git a/crates/web-faith/tests/fetch.rs b/crates/web-faith/tests/fetch.rs new file mode 100644 index 0000000..c9c893a --- /dev/null +++ b/crates/web-faith/tests/fetch.rs @@ -0,0 +1,340 @@ +//! What the fetch-flavoured surface owes a caller, exercised against a real origin. +//! +//! Set `HTTPBIN_URL` to a [go-httpbin] and these run; leave it unset and they report that they +//! were skipped rather than failing, so `cargo test` works without a server to hand. +//! +//! ```console +//! $ HTTPBIN_URL=http://127.0.0.1:8888 cargo test -p web-faith +//! ``` +//! +//! [go-httpbin]: https://github.com/mccutchen/go-httpbin + +// spec:RSAPI spec:REQ spec:RESP + +use std::time::Duration; + +use serde::Deserialize; +use web_faith::{ + FaithErrorKind, + agent::Agent, + request::{Priority, Request}, +}; + +/// The origin under test, or `None` when there is none configured. +fn origin() -> Option { + match std::env::var("HTTPBIN_URL") { + Ok(url) if !url.is_empty() => Some(url.trim_end_matches('/').to_owned()), + _ => None, + } +} + +/// Run `body` against the configured origin, or report the skip and return. +/// +/// A macro rather than a function taking a closure: an async closure returning a future that +/// borrows its argument is more ceremony than the tests are worth. +macro_rules! against_origin { + ($origin:ident => $body:block) => { + let Some($origin) = origin() else { + eprintln!("skipped: set HTTPBIN_URL to run this against an origin"); + return; + }; + $body + }; +} + +fn agent() -> Agent { + Agent::builder() + .timeout(|timeout| timeout.total(Duration::from_secs(30))) + .build() + .expect("the options are valid") +} + +/// What go-httpbin echoes back about the request it received. +/// +/// Header values arrive as arrays, a name being repeatable, so `header` reads the first. +#[derive(Deserialize)] +struct Echo { + url: String, + #[serde(default)] + headers: std::collections::HashMap>, + #[serde(default)] + data: String, +} + +impl Echo { + fn header(&self, name: &str) -> Option<&str> { + self.headers.get(name)?.first().map(String::as_str) + } +} + +/// A fetch builder sends when awaited, so a request is one expression. +#[tokio::test] +async fn awaiting_the_builder_sends_the_request() { + against_origin!(origin => { + let response = agent() + .fetch(format!("{origin}/get")) + .await + .expect("the request reaches the origin"); + + assert!(response.ok()); + assert_eq!(response.status(), 200); + assert_eq!(response.status_text(), "OK"); + assert!(!response.redirected()); + assert!(!response.body_used()); + }); +} + +/// The body readers each consume the one body, and say so afterwards. +#[tokio::test] +async fn a_body_is_read_once_by_whichever_reader_asks() { + against_origin!(origin => { + let agent = agent(); + + let response = agent.fetch(format!("{origin}/get")).await.expect("sent"); + let echo: Echo = response.json().await.expect("httpbin answers with json"); + assert!(echo.url.ends_with("/get")); + assert!(response.body_used(), "reading the body marks it used"); + + let response = agent.fetch(format!("{origin}/get")).await.expect("sent"); + let text = response.text().await.expect("the body is text"); + assert!(text.contains("\"url\"")); + + let response = agent.fetch(format!("{origin}/get")).await.expect("sent"); + let bytes = response.bytes().await.expect("the body is bytes"); + assert_eq!(bytes.len(), text.len(), "the same body either way"); + + // A second read is refused rather than returning an empty body. + let err = response.text().await.expect_err("the body is spent"); + assert_eq!(err.kind, FaithErrorKind::ResponseAlreadyDisturbed); + }); +} + +/// A method, headers, and a body all reach the origin. +#[tokio::test] +async fn what_the_builder_sets_is_what_the_origin_sees() { + against_origin!(origin => { + let echo: Echo = agent() + .fetch(format!("{origin}/post")) + .method("POST") + // Named, because go-httpbin echoes an unlabelled body back as a data URL. + .header("content-type", "text/plain") + .header("x-faith-test", "present") + .body("the body") + .await + .expect("sent") + .json() + .await + .expect("httpbin echoes the request"); + + assert_eq!(echo.data, "the body"); + assert_eq!(echo.header("X-Faith-Test"), Some("present")); + }); +} + +/// A prepared request carries no agent, so it can be sent more than once and layered over. +#[tokio::test] +async fn a_prepared_request_is_reusable_and_layerable() { + against_origin!(origin => { + let agent = agent(); + let prepared = Request::new(format!("{origin}/get")) + .header("x-base", "from-the-request") + .header("x-overridden", "from-the-request") + .build() + .expect("the target parses"); + + // Sent unchanged, twice: the request is inert. + for _ in 0..2 { + let response = agent + .fetch(prepared.try_clone().expect("no stream body")) + .await + .expect("sent"); + assert!(response.ok()); + } + + // Layered over: the outermost value wins, and headers merge by name. + let echo: Echo = agent + .fetch(prepared.try_clone().expect("no stream body")) + .header("x-overridden", "from-the-layer") + .header("x-added", "from-the-layer") + .await + .expect("sent") + .json() + .await + .expect("httpbin echoes the request"); + + assert_eq!(echo.header("X-Base"), Some("from-the-request")); + assert_eq!(echo.header("X-Overridden"), Some("from-the-layer")); + assert_eq!(echo.header("X-Added"), Some("from-the-layer")); + }); +} + +/// Removing a header takes away whatever the layers beneath contributed for it. +#[tokio::test] +async fn removing_a_header_reaches_through_the_layers() { + against_origin!(origin => { + let prepared = Request::new(format!("{origin}/get")) + .header("x-removed", "from-the-request") + .build() + .expect("the target parses"); + + let echo: Echo = agent() + .fetch(prepared) + .remove_header("x-removed") + .await + .expect("sent") + .json() + .await + .expect("httpbin echoes the request"); + + assert!(!echo.headers.contains_key("X-Removed")); + }); +} + +/// A conversion that failed on the way in is reported where the builder resolves, not at the +/// setter that took it. +#[tokio::test] +async fn a_failed_conversion_surfaces_when_the_builder_resolves() { + let err = agent() + .fetch("https://example.com/") + .method("a method with spaces") + .await + .expect_err("the method does not convert"); + assert_eq!(err.kind, FaithErrorKind::InvalidMethod); + + // The first failure met is the one reported, whatever follows it. + let err = Request::new("not a url") + .header("x-fine", "value") + .build() + .expect_err("the target does not parse"); + assert_eq!(err.kind, FaithErrorKind::InvalidUrl); +} + +/// A timeout shorter than the origin's delay is a `Timeout`, not a generic network failure. +#[tokio::test] +async fn a_timeout_reports_itself_as_one() { + against_origin!(origin => { + let err = agent() + .fetch(format!("{origin}/delay/5")) + .timeout(Duration::from_millis(250)) + .await + .expect_err("the origin is slower than the deadline"); + assert_eq!(err.kind, FaithErrorKind::Timeout); + }); +} + +/// A body whose digest does not match the integrity the caller named is refused. +/// +/// The check needs the whole body, so it fires where the body is read rather than where the +/// response arrives. +#[tokio::test] +async fn integrity_is_checked_against_the_body() { + against_origin!(origin => { + let response = agent() + .fetch(format!("{origin}/get")) + // A well-formed digest of the wrong bytes: 32 zero bytes, which no body hashes to. + .integrity("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + .await + .expect("the response itself arrives"); + + let err = response.bytes().await.expect_err("the digest cannot match"); + assert_eq!(err.kind, FaithErrorKind::IntegrityMismatch); + + // A value naming no algorithm is refused before the body is touched at all. + let response = agent() + .fetch(format!("{origin}/get")) + .integrity("not-an-integrity-value") + .await + .expect("the response itself arrives"); + let err = response.bytes().await.expect_err("the value does not parse"); + assert_eq!(err.kind, FaithErrorKind::InvalidIntegrity); + }); +} + +/// A closed agent refuses a new request, and every clone sees the close. +#[tokio::test] +async fn a_closed_agent_refuses_new_requests() { + against_origin!(origin => { + let agent = agent(); + let clone = agent.clone(); + + agent + .fetch(format!("{origin}/get")) + .await + .expect("the agent is open"); + + agent.close(); + assert!(clone.is_closed(), "a clone names the same agent"); + + let err = clone + .fetch(format!("{origin}/get")) + .await + .expect_err("the agent is closed"); + assert_eq!(err.kind, FaithErrorKind::Closed); + }); +} + +/// Redirects are followed by default, and the response says which URL answered. +#[tokio::test] +async fn a_followed_redirect_reports_the_url_that_answered() { + against_origin!(origin => { + let response = agent() + .fetch(format!("{origin}/redirect/2")) + .await + .expect("sent"); + + assert!(response.ok()); + assert!(response.redirected()); + assert!(response.url().as_str().ends_with("/get")); + }); +} + +/// A priority is a hint carried as a header, and one the caller wrote wins over it. +#[tokio::test] +async fn a_priority_derives_a_header_a_caller_can_override() { + against_origin!(origin => { + let echo: Echo = agent() + .fetch(format!("{origin}/get")) + .priority(Priority::High) + .await + .expect("sent") + .json() + .await + .expect("httpbin echoes the request"); + assert_eq!(echo.header("Priority"), Some("u=1")); + + let echo: Echo = agent() + .fetch(format!("{origin}/get")) + .priority(Priority::High) + .header("priority", "u=5") + .await + .expect("sent") + .json() + .await + .expect("httpbin echoes the request"); + assert_eq!(echo.header("Priority"), Some("u=5")); + }); +} + +/// Warming is advisory: it never fails, and a closed agent refuses it up front. +#[tokio::test] +async fn warming_is_advisory_but_a_closed_agent_refuses_it() { + against_origin!(origin => { + let agent = agent(); + + agent.prefetch_dns("localhost").expect("a host to warm").await; + agent.preconnect(&origin).expect("an origin to warm").await; + + // A string with no host is refused where the caller can see it, not by the future. The + // future itself is not `Debug`, so the refusal is read off the `Err` rather than unwrapped. + let Err(err) = agent.prefetch_dns("") else { + panic!("there is no host in an empty string"); + }; + assert_eq!(err.kind, FaithErrorKind::AddressParse); + + agent.close(); + let Err(err) = agent.prefetch_dns("localhost") else { + panic!("a closed agent has nothing to warm"); + }; + assert_eq!(err.kind, FaithErrorKind::Closed); + }); +}