From 6b25fc606dd5e19013ebe300851db020dee7cf50 Mon Sep 17 00:00:00 2001 From: Timothy Miller Date: Fri, 21 Aug 2026 13:18:55 +0900 Subject: [PATCH] Enable Jetstream v2 --- CarpaNet.Samples.slnx | 1 + docs/docs/jetstream.md | 70 ++- .../JetstreamV2Test/JetstreamV2Test.csproj | 20 + samples/JetstreamV2Test/Program.cs | 176 ++++++ samples/JetstreamV2Test/README.md | 51 ++ skills/carpanet/SKILL.md | 53 +- .../CarpaNet.Jetstream.csproj | 5 + src/CarpaNet.Jetstream/README.md | 37 +- .../V2/JetstreamSegmentFormat.cs | 347 ++++++++++++ .../V2/JetstreamSegmentHeader.cs | 35 ++ .../V2/JetstreamSegmentRow.cs | 40 ++ .../V2/JetstreamSegmentRowKind.cs | 29 + .../V2/JetstreamSnapshotPlan.cs | 106 ++++ .../V2/JetstreamV2Account.cs | 34 ++ .../V2/JetstreamV2Client.cs | 215 ++++++++ .../V2/JetstreamV2ClientOptions.cs | 98 ++++ .../V2/JetstreamV2Commit.cs | 92 ++++ .../V2/JetstreamV2CommitOperation.cs | 16 + .../V2/JetstreamV2Engine.cs | 439 +++++++++++++++ src/CarpaNet.Jetstream/V2/JetstreamV2Event.cs | 56 ++ .../V2/JetstreamV2EventKind.cs | 20 + .../V2/JetstreamV2Exception.cs | 70 +++ .../V2/JetstreamV2FrameDecoder.cs | 399 ++++++++++++++ .../V2/JetstreamV2Identity.cs | 28 + .../V2/JetstreamV2LiveSession.cs | 513 ++++++++++++++++++ .../V2/JetstreamV2Matcher.cs | 187 +++++++ .../V2/JetstreamV2PlanConverter.cs | 95 ++++ .../V2/JetstreamV2RecordDecoder.cs | 319 +++++++++++ .../V2/JetstreamV2SubscribeOptions.cs | 160 ++++++ src/CarpaNet.Jetstream/V2/JetstreamV2Sync.cs | 28 + .../V2/JetstreamV2Transport.cs | 326 +++++++++++ .../V2/JetstreamV2WireModels.cs | 262 +++++++++ .../V2/JetstreamZstdDictionary.cs | 35 ++ src/CarpaNet/BlueskyServices.cs | 5 + .../CarpaNet.UnitTests.csproj | 1 + .../Jetstream/JetstreamSegmentFormatTests.cs | 372 +++++++++++++ .../Jetstream/JetstreamV2FrameDecoderTests.cs | 235 ++++++++ .../Jetstream/JetstreamV2MatcherTests.cs | 131 +++++ .../Jetstream/JetstreamV2OptionsTests.cs | 221 ++++++++ .../JetstreamV2RecordDecoderTests.cs | 268 +++++++++ 40 files changed, 5589 insertions(+), 6 deletions(-) create mode 100644 samples/JetstreamV2Test/JetstreamV2Test.csproj create mode 100644 samples/JetstreamV2Test/Program.cs create mode 100644 samples/JetstreamV2Test/README.md create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamSegmentFormat.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamSegmentHeader.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamSegmentRow.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamSegmentRowKind.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamSnapshotPlan.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Account.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Client.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2ClientOptions.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Commit.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2CommitOperation.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Engine.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Event.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2EventKind.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Exception.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2FrameDecoder.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Identity.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2LiveSession.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Matcher.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2PlanConverter.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2RecordDecoder.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2SubscribeOptions.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Sync.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2Transport.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamV2WireModels.cs create mode 100644 src/CarpaNet.Jetstream/V2/JetstreamZstdDictionary.cs create mode 100644 tests/CarpaNet.UnitTests/Jetstream/JetstreamSegmentFormatTests.cs create mode 100644 tests/CarpaNet.UnitTests/Jetstream/JetstreamV2FrameDecoderTests.cs create mode 100644 tests/CarpaNet.UnitTests/Jetstream/JetstreamV2MatcherTests.cs create mode 100644 tests/CarpaNet.UnitTests/Jetstream/JetstreamV2OptionsTests.cs create mode 100644 tests/CarpaNet.UnitTests/Jetstream/JetstreamV2RecordDecoderTests.cs diff --git a/CarpaNet.Samples.slnx b/CarpaNet.Samples.slnx index 8b28771..220fe10 100644 --- a/CarpaNet.Samples.slnx +++ b/CarpaNet.Samples.slnx @@ -10,6 +10,7 @@ + diff --git a/docs/docs/jetstream.md b/docs/docs/jetstream.md index b9d4023..bfc3a7a 100644 --- a/docs/docs/jetstream.md +++ b/docs/docs/jetstream.md @@ -3,6 +3,69 @@ Jetstream provides a lightweight, JSON-based WebSocket event stream. Requires the `CarpaNet.Jetstream` package. This is useful when you only care for a type of collection that you want to respond to. It also uses far less data than the full Firehose. +The package contains two clients: + +- **`JetstreamV2Client`** — the current Jetstream service ([bluesky-social/jetstream](https://github.com/bluesky-social/jetstream)): the `network.bsky.jetstream.subscribeEvents` websocket plus a full-network sealed archive you can replay over HTTP. Use this for new code. +- **`JetstreamClient`** — the legacy v1 `/subscribe` wire, kept for servers that only expose it. + +## Jetstream v2 + +`JetstreamV2Client.SubscribeAsync` is a managed stream: it reconnects with exponential backoff, resumes at the last delivered seq, deduplicates the at-least-once overlap, fetches and rotates the zstd compression dictionary, and, when you ask for history, downloads the sealed archive over HTTP and cuts over to the live tail with no gap. + +```csharp +using CarpaNet; +using CarpaNet.Jetstream; + +using var client = new JetstreamV2Client( + new Uri(BlueskyServices.JetstreamUsEast), + new JetstreamV2ClientOptions + { + EnableCompression = true, // dict-zstd; the dictionary is fetched automatically + }); + +var options = new JetstreamV2SubscribeOptions +{ + Collections = new[] { "app.bsky.feed.post" }, // exact NSIDs or wildcards like app.bsky.feed.* + // LiveCursor = lastSeq, // resume a live tail from a saved seq + // AfterSeq = 0, // or: replay the whole archive first, then go live + // SnapshotOnly = true, // or: archive dump only, no websocket +}; + +await foreach (var evt in client.SubscribeAsync(options)) +{ + switch (evt.Kind) + { + case JetstreamV2EventKind.Commit when evt.Commit is { } commit: + Console.WriteLine($"[{commit.Operation}] seq={evt.Seq} {commit.Collection}/{commit.Rkey}"); + // Typed records via your generated JSON context: + // var post = commit.GetRecord(ATProtoJsonContext.Default.AppBskyFeedPost); + break; + + case JetstreamV2EventKind.Identity when evt.Identity is { } identity: + Console.WriteLine($"[Identity] {evt.Did} → {identity.Handle}"); + break; + + case JetstreamV2EventKind.Account when evt.Account is { } account: + Console.WriteLine($"[Account] {evt.Did} active={account.Active} status={account.Status}"); + break; + + case JetstreamV2EventKind.Sync when evt.Sync is { } sync: + Console.WriteLine($"[Sync] {evt.Did} rev={sync.Rev}"); + break; + } +} +``` + +Things worth knowing: + +- `evt.Seq` is the cursor: Persist the last seen value and pass it back as `LiveCursor` (pure live) or `AfterSeq` (archive replay) to resume. Delivery is at-least-once across restarts — fold idempotently or dedup by seq. +- Collection filters never suppress `#identity`/`#account`/`#sync`: those DID-level events are your only signal to purge a deleted account's records. Ask for `Kinds = [JetstreamV2EventKind.Commit]` explicitly if you want commits only. +- Filters are immutable per connection on the v2 wire; there is no `options_update`. Start a new subscription to change them. +- `CursorTooOld`: on a backfill-enabled stream the client automatically re-enters archive replay; on a pure live tail it throws `JetstreamV2Exception` — resume with `AfterSeq` instead. +- The archive endpoints (`PlanSnapshotAsync`, `GetSegmentAsync`, `GetBlockAsync`, decoded via `JetstreamSegmentFormat`) are also public for direct use, and `JetstreamV2ClientOptions.ApiKey` sends a bearer key on them (never on the public websocket or dictionary fetch). + +## Jetstream v1 (legacy) + ```csharp using CarpaNet.Jetstream; @@ -42,11 +105,14 @@ await foreach (var evt in client.SubscribeAsync(options)) } ``` -## Dynamic Filter Updates +### Dynamic Filter Updates (v1 only) ```csharp await client.SendOptionsUpdateAsync(new JetstreamOptionsUpdate { - WantedCollections = new[] { "app.bsky.graph.follow" }, + Payload = new JetstreamOptionsPayload + { + WantedCollections = new List { "app.bsky.graph.follow" }, + }, }); ``` diff --git a/samples/JetstreamV2Test/JetstreamV2Test.csproj b/samples/JetstreamV2Test/JetstreamV2Test.csproj new file mode 100644 index 0000000..8203755 --- /dev/null +++ b/samples/JetstreamV2Test/JetstreamV2Test.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/samples/JetstreamV2Test/Program.cs b/samples/JetstreamV2Test/Program.cs new file mode 100644 index 0000000..3fa48b4 --- /dev/null +++ b/samples/JetstreamV2Test/Program.cs @@ -0,0 +1,176 @@ +using CarpaNet; +using CarpaNet.Jetstream; +using Microsoft.Extensions.Logging; + +using var cts = new CancellationTokenSource(); +Console.CancelKeyPress += (_, e) => +{ + e.Cancel = true; + cts.Cancel(); + Console.WriteLine(); + Console.WriteLine("Shutting down..."); +}; + +// Parse arguments +long? liveCursor = null; +long? afterSeq = null; +long? beforeSeq = null; +bool snapshotOnly = false; +bool compress = false; +bool verbose = false; +var collections = new List(); +var dids = new List(); +var kinds = new List(); +string endpoint = BlueskyServices.JetstreamUsEast; +string? apiKey = Environment.GetEnvironmentVariable("JETSTREAM_CLIENT_API_KEY"); + +for (int i = 0; i < args.Length; i++) +{ + switch (args[i]) + { + case "--cursor" when i + 1 < args.Length && long.TryParse(args[i + 1], out var c): + liveCursor = c; + i++; + break; + case "--after-seq" when i + 1 < args.Length && long.TryParse(args[i + 1], out var a): + afterSeq = a; + i++; + break; + case "--before-seq" when i + 1 < args.Length && long.TryParse(args[i + 1], out var b): + beforeSeq = b; + i++; + break; + case "--snapshot-only": + snapshotOnly = true; + break; + case "--collection" when i + 1 < args.Length: + collections.Add(args[i + 1]); + i++; + break; + case "--did" when i + 1 < args.Length: + dids.Add(args[i + 1]); + i++; + break; + case "--kind" when i + 1 < args.Length && Enum.TryParse(args[i + 1], ignoreCase: true, out var k): + kinds.Add(k); + i++; + break; + case "--endpoint" when i + 1 < args.Length: + endpoint = args[i + 1]; + i++; + break; + case "--api-key" when i + 1 < args.Length: + apiKey = args[i + 1]; + i++; + break; + case "--compress": + compress = true; + break; + case "--verbose": + verbose = true; + break; + case "--help": + Console.WriteLine(""" + CarpaNet Jetstream v2 sample. + + Live tail: JetstreamV2Test [--cursor ] + Full replay then live: JetstreamV2Test --after-seq 0 + Point-in-time snapshot: JetstreamV2Test --after-seq 0 [--before-seq ] --snapshot-only + + Options: + --collection Filter to a collection + --did Filter to a repo DID + --kind Filter to commit|identity|account|sync + --endpoint Jetstream v2 instance (default: jetstream.us-east.bsky.network) + --api-key Archive API key (or JETSTREAM_CLIENT_API_KEY env var) + --compress Enable dict-zstd live-tail compression + --verbose Enable client diagnostics logging + """); + return; + } +} + +Console.WriteLine("=== CarpaNet Jetstream v2 Test ==="); +Console.WriteLine($"Endpoint: {endpoint}"); +if (afterSeq.HasValue) +{ + Console.WriteLine(snapshotOnly + ? $"Mode: archive snapshot ({afterSeq}, {(beforeSeq.HasValue ? beforeSeq.ToString() : "sealed tip")}]" + : $"Mode: backfill from seq {afterSeq}, then live"); +} +else +{ + Console.WriteLine(liveCursor.HasValue ? $"Mode: live from cursor {liveCursor}" : "Mode: live from the current tip"); +} + +Console.WriteLine("Press Ctrl+C to stop."); +Console.WriteLine(); + +using var loggerFactory = verbose + ? LoggerFactory.Create(builder => builder.AddSimpleConsole(o => o.SingleLine = true).SetMinimumLevel(LogLevel.Information)) + : null; + +using var client = new JetstreamV2Client(new Uri(endpoint), new JetstreamV2ClientOptions +{ + EnableCompression = compress, + ApiKey = apiKey, + LoggerFactory = loggerFactory, +}); + +var options = new JetstreamV2SubscribeOptions +{ + LiveCursor = liveCursor, + AfterSeq = afterSeq, + BeforeSeq = beforeSeq, + SnapshotOnly = snapshotOnly, + Collections = collections.Count > 0 ? collections : null, + Dids = dids.Count > 0 ? dids : null, + Kinds = kinds.Count > 0 ? kinds : null, +}; + +long count = 0; +long lastSeq = 0; +try +{ + await foreach (var evt in client.SubscribeAsync(options, cts.Token)) + { + count++; + lastSeq = evt.Seq; + switch (evt.Kind) + { + case JetstreamV2EventKind.Commit when evt.Commit != null: + var commit = evt.Commit; + Console.WriteLine($"[Commit seq={evt.Seq}] {commit.Operation} {commit.Collection}/{commit.Rkey} from {evt.Did}"); + if (commit.Record.HasValue && + commit.Record.Value.TryGetProperty("$type", out var typeEl)) + { + Console.WriteLine($" record $type={typeEl.GetString()} cid={commit.Cid}"); + } + + break; + + case JetstreamV2EventKind.Identity when evt.Identity != null: + Console.WriteLine($"[Identity seq={evt.Seq}] did={evt.Did} handle={evt.Identity.Handle}"); + break; + + case JetstreamV2EventKind.Account when evt.Account != null: + Console.WriteLine($"[Account seq={evt.Seq}] did={evt.Did} active={evt.Account.Active} status={evt.Account.Status}"); + break; + + case JetstreamV2EventKind.Sync when evt.Sync != null: + Console.WriteLine($"[Sync seq={evt.Seq}] did={evt.Did} rev={evt.Sync.Rev}"); + break; + } + } +} +catch (OperationCanceledException) +{ + // Expected on Ctrl+C +} +catch (JetstreamV2Exception ex) +{ + Console.WriteLine($"Stream failed{(ex.ErrorName != null ? $" ({ex.ErrorName})" : string.Empty)}: {ex.Message}"); +} + +Console.WriteLine(); +Console.WriteLine($"Stream ended after {count} events; last seq {lastSeq} (pass it back with --cursor or --after-seq to resume)."); diff --git a/samples/JetstreamV2Test/README.md b/samples/JetstreamV2Test/README.md new file mode 100644 index 0000000..5168ccf --- /dev/null +++ b/samples/JetstreamV2Test/README.md @@ -0,0 +1,51 @@ +# JetstreamV2Test + +Demonstrates `JetstreamV2Client`: the managed Jetstream v2 stream that welds sealed-archive +replay (planSnapshot → getSegment/getBlock over HTTP) and the live +`network.bsky.jetstream.subscribeEvents` websocket into one seq-ordered event stream with +automatic reconnect, resume, and compression handling. + +## Usage + +```bash +# Live tail from the current tip +dotnet run + +# Live tail, filtered, with dict-zstd compression (dictionary fetched automatically) +dotnet run -- --collection app.bsky.feed.post --compress + +# Resume a live tail from a saved seq +dotnet run -- --cursor 24959600000 + +# Replay sealed history from a seq, then cut over to live with no gap +dotnet run -- --after-seq 24959600000 --collection app.bsky.feed.post + +# Point-in-time archive snapshot (no websocket) +dotnet run -- --after-seq 1000000 --before-seq 1000400 --snapshot-only +``` + +Options: + +| Flag | Meaning | +| --- | --- | +| `--collection ` | Filter commits to a collection (repeatable; wildcards like `app.bsky.feed.*`) | +| `--did ` | Filter to a repo DID (repeatable) | +| `--kind ` | Filter to `commit`, `identity`, `account`, or `sync` (repeatable) | +| `--cursor ` | Resume a pure live tail from a saved seq | +| `--after-seq ` | Replay the sealed archive after this seq, then go live (0 = everything) | +| `--before-seq ` | Snapshot upper bound (requires `--snapshot-only`) | +| `--snapshot-only` | Archive dump only; the stream ends at the sealed tip | +| `--compress` | Enable dict-zstd live-tail compression | +| `--endpoint ` | Jetstream v2 instance (default: `jetstream.us-east.bsky.network`) | +| `--api-key ` | Archive API key (also read from `JETSTREAM_CLIENT_API_KEY`) | +| `--verbose` | Show client diagnostics (reconnects, sweeps, degraded compression) | + +The final line prints the last delivered seq — pass it back with `--cursor` (live) or +`--after-seq` (replay) to resume. Delivery is at-least-once across restarts; consumers fold +idempotently or dedup by seq. + +## Jetstream v2 vs v1 + +The v2 wire (`JetstreamV2Client`) adds a durable per-event `seq` cursor, an archive you can +replay from seq 0, a `sync` event kind, and self-describing frames. The v1 `/subscribe` wire +(`JetstreamClient`, see the `JetstreamTest` sample) remains for legacy servers. diff --git a/skills/carpanet/SKILL.md b/skills/carpanet/SKILL.md index 81df525..c696566 100644 --- a/skills/carpanet/SKILL.md +++ b/skills/carpanet/SKILL.md @@ -391,7 +391,51 @@ public sealed class FileOAuthSessionStore : IOAuthSessionStore ## Jetstream (Real-Time Events) -Jetstream provides a lightweight, JSON-based WebSocket event stream. Requires the `CarpaNet.Jetstream` package. +Jetstream provides a lightweight, JSON-based WebSocket event stream. Requires the `CarpaNet.Jetstream` package. The package has two clients: `JetstreamV2Client` (current service: managed live tail + sealed-archive replay; prefer for new code) and `JetstreamClient` (legacy v1 `/subscribe` wire). + +### Jetstream v2 (preferred) + +```csharp +using CarpaNet.Jetstream; + +using var client = new JetstreamV2Client( + new Uri(BlueskyServices.JetstreamUsEast), + new JetstreamV2ClientOptions { EnableCompression = true }); // dict-zstd, dictionary auto-fetched + +var options = new JetstreamV2SubscribeOptions +{ + Collections = new[] { "app.bsky.feed.post", "app.bsky.graph.*" }, // exact NSIDs or wildcards + Dids = new[] { "did:plc:z72i7hdynmk6r22z27h6tvur" }, // optional, max 10,000 + // LiveCursor = lastSeq, // resume a live tail from a saved seq (evt.Seq) + // AfterSeq = 0, // or: replay the whole sealed archive, then cut over to live + // SnapshotOnly = true, // or: archive dump only, no websocket +}; + +await foreach (var evt in client.SubscribeAsync(options)) +{ + switch (evt.Kind) + { + case JetstreamV2EventKind.Commit when evt.Commit is { } commit: + Console.WriteLine($"[{commit.Operation}] seq={evt.Seq} {commit.Collection}/{commit.Rkey}"); + // Typed record via your generated context: + // var post = commit.GetRecord(ATProtoJsonContext.Default.AppBskyFeedPost); + break; + case JetstreamV2EventKind.Identity when evt.Identity is { } identity: + Console.WriteLine($"[Identity] {evt.Did} → {identity.Handle}"); + break; + case JetstreamV2EventKind.Account when evt.Account is { } account: + Console.WriteLine($"[Account] {evt.Did} active={account.Active} status={account.Status}"); + break; + case JetstreamV2EventKind.Sync when evt.Sync is { } sync: + Console.WriteLine($"[Sync] {evt.Did} rev={sync.Rev}"); + break; + } +} +``` + +The managed stream reconnects with backoff, resumes at the last delivered seq, dedups the at-least-once overlap, and recovers from `CursorTooOld` by re-entering archive replay (backfill mode) — persist `evt.Seq` to resume across restarts. Collection filters never suppress `#identity`/`#account`/`#sync` (the account-deletion purge signal); v2 filters are immutable per connection (no `options_update`). Archive endpoints (`PlanSnapshotAsync`/`GetSegmentAsync`/`GetBlockAsync` + `JetstreamSegmentFormat`) are public; `JetstreamV2ClientOptions.ApiKey` authenticates them. + +### Jetstream v1 (legacy) ```csharp using CarpaNet.Jetstream; @@ -432,12 +476,15 @@ await foreach (var evt in client.SubscribeAsync(options)) } ``` -### Dynamic Filter Updates +### Dynamic Filter Updates (v1 only) ```csharp await client.SendOptionsUpdateAsync(new JetstreamOptionsUpdate { - WantedCollections = new[] { "app.bsky.graph.follow" }, + Payload = new JetstreamOptionsPayload + { + WantedCollections = new List { "app.bsky.graph.follow" }, + }, }); ``` diff --git a/src/CarpaNet.Jetstream/CarpaNet.Jetstream.csproj b/src/CarpaNet.Jetstream/CarpaNet.Jetstream.csproj index 1d6eafb..3e860f6 100644 --- a/src/CarpaNet.Jetstream/CarpaNet.Jetstream.csproj +++ b/src/CarpaNet.Jetstream/CarpaNet.Jetstream.csproj @@ -17,6 +17,11 @@ + + + + + diff --git a/src/CarpaNet.Jetstream/README.md b/src/CarpaNet.Jetstream/README.md index 7bcb9c4..afc8457 100644 --- a/src/CarpaNet.Jetstream/README.md +++ b/src/CarpaNet.Jetstream/README.md @@ -10,7 +10,42 @@ CarpaNet.Jetstream lets you connect to a Bluesky Jetstream instance. This library is experimental and not stable. Expect issues and bugs! -# How to use +The package contains two clients: + +- **`JetstreamV2Client`** — the current Jetstream service ([bluesky-social/jetstream](https://github.com/bluesky-social/jetstream)): the `network.bsky.jetstream.subscribeEvents` live websocket plus the sealed-archive replay API. Use this for new code. +- **`JetstreamClient`** — the legacy v1 `/subscribe` wire. + +# Jetstream v2 + +`SubscribeAsync` is a managed stream: automatic reconnect with backoff, seq-based resume and dedup, optional dict-zstd compression with automatic dictionary fetch/rotation, and seamless archive-backfill-to-live cutover. + +```csharp +using var client = new JetstreamV2Client( + new Uri(BlueskyServices.JetstreamUsEast), + new JetstreamV2ClientOptions { EnableCompression = true }); + +var options = new JetstreamV2SubscribeOptions +{ + Collections = new[] { "app.bsky.feed.post" }, // exact NSIDs or wildcards like app.bsky.feed.* + // LiveCursor = lastSeq, // resume a live tail from a saved seq + // AfterSeq = 0, // or: replay the whole sealed archive first, then go live + // SnapshotOnly = true, // or: archive dump only, no websocket +}; + +await foreach (var evt in client.SubscribeAsync(options, cts.Token)) +{ + if (evt.Kind == JetstreamV2EventKind.Commit && evt.Commit is { } commit) + { + Console.WriteLine($"[{commit.Operation}] seq={evt.Seq} {commit.Collection}/{commit.Rkey}"); + // Typed records via your generated context: + // var post = commit.GetRecord(ATProtoJsonContext.Default.AppBskyFeedPost); + } +} +``` + +Persist the last `evt.Seq` and pass it back (`LiveCursor` for live, `AfterSeq` for replay) to resume. Delivery is at-least-once — fold idempotently or dedup by seq. The archive endpoints (`PlanSnapshotAsync`, `GetSegmentAsync`, `GetBlockAsync`, plus the `JetstreamSegmentFormat` decoder) are also available directly. + +# Jetstream v1 (legacy) ```csharp diff --git a/src/CarpaNet.Jetstream/V2/JetstreamSegmentFormat.cs b/src/CarpaNet.Jetstream/V2/JetstreamSegmentFormat.cs new file mode 100644 index 0000000..371a8a2 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamSegmentFormat.cs @@ -0,0 +1,347 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Text; +using ZstdSharp; + +namespace CarpaNet.Jetstream; + +/// +/// Reads the sealed Jetstream v2 segment file format: the fixed header, the stored +/// per-block zstd frames, and the columnar block body. +/// +public static class JetstreamSegmentFormat +{ + /// Size in bytes of the fixed segment header. + public const int HeaderSize = 256; + + private const ushort CurrentHeaderVersion = 1; + private const int BlockIndexEntrySize = 52; + + private const int MaxBlockEvents = 1 << 18; + + private const long MaxDecodedBlockBytes = 1L << 30; + + private static readonly byte[] SegmentMagic = { (byte)'j', (byte)'s', (byte)'s', (byte)'0' }; + + /// + /// Parses and validates the fixed header of a sealed segment file. The input may be the + /// whole file or just its first bytes. + /// + /// The sealed segment file bytes. + /// The parsed header. + /// The header is truncated, unsealed, or corrupt. + public static JetstreamSegmentHeader ReadHeader(ReadOnlySpan segmentBytes) + { + if (segmentBytes.Length < HeaderSize) + { + throw new JetstreamV2Exception($"segment header is {segmentBytes.Length} bytes, want at least {HeaderSize}"); + } + + if (!segmentBytes.Slice(0, 4).SequenceEqual(SegmentMagic)) + { + throw new JetstreamV2Exception("segment has bad magic; not a sealed Jetstream segment file"); + } + + var checksum = BinaryPrimitives.ReadUInt64LittleEndian(segmentBytes.Slice(4, 8)); + if (checksum == 0) + { + throw new JetstreamV2Exception("segment checksum field is zero; this is an active (unsealed) segment"); + } + + var version = BinaryPrimitives.ReadUInt16LittleEndian(segmentBytes.Slice(12, 2)); + if (version != CurrentHeaderVersion) + { + throw new JetstreamV2Exception($"segment header version {version}, want {CurrentHeaderVersion}"); + } + + var blockCount = BinaryPrimitives.ReadUInt32LittleEndian(segmentBytes.Slice(14, 4)); + if (blockCount > int.MaxValue) + { + throw new JetstreamV2Exception($"segment block count {blockCount} is out of range"); + } + + var footerOffset = BinaryPrimitives.ReadUInt64LittleEndian(segmentBytes.Slice(58, 8)); + var blockIndexOffset = BinaryPrimitives.ReadUInt64LittleEndian(segmentBytes.Slice(90, 8)); + if (footerOffset > long.MaxValue || blockIndexOffset > long.MaxValue) + { + throw new JetstreamV2Exception("segment header offsets are out of range"); + } + + return new JetstreamSegmentHeader + { + Version = version, + BlockCount = (int)blockCount, + EventCount = BinaryPrimitives.ReadUInt32LittleEndian(segmentBytes.Slice(18, 4)), + MinSeq = ReadSeq(segmentBytes.Slice(26, 8), "minSeq"), + MaxSeq = ReadSeq(segmentBytes.Slice(34, 8), "maxSeq"), + MinWitnessedAt = (long)BinaryPrimitives.ReadUInt64LittleEndian(segmentBytes.Slice(42, 8)), + MaxWitnessedAt = (long)BinaryPrimitives.ReadUInt64LittleEndian(segmentBytes.Slice(50, 8)), + FooterOffset = (long)footerOffset, + BlockIndexOffset = (long)blockIndexOffset, + }; + } + + /// + /// Extracts the raw, stored zstd frame for one block from a whole downloaded segment file, + /// using the block index the header points at. + /// + /// The whole sealed segment file. + /// The header parsed from the same bytes. + /// The zero-based block index. + /// The block's compressed frame bytes. + /// The index is out of range or the block index is corrupt. + public static byte[] GetBlockFrame(byte[] segmentBytes, JetstreamSegmentHeader header, int index) + { + if (segmentBytes == null) + { + throw new ArgumentNullException(nameof(segmentBytes)); + } + + if (header == null) + { + throw new ArgumentNullException(nameof(header)); + } + + if (index < 0 || index >= header.BlockCount) + { + throw new JetstreamV2Exception($"block index {index} out of range; segment has {header.BlockCount} blocks"); + } + + if (header.FooterOffset < HeaderSize || header.FooterOffset > segmentBytes.Length) + { + throw new JetstreamV2Exception($"segment footer offset {header.FooterOffset} is out of range"); + } + + if (header.BlockIndexOffset != header.FooterOffset) + { + throw new JetstreamV2Exception( + $"segment block index offset {header.BlockIndexOffset} does not match footer offset {header.FooterOffset}"); + } + + var entryOffset = header.BlockIndexOffset + (long)index * BlockIndexEntrySize; + if (entryOffset + BlockIndexEntrySize > segmentBytes.Length) + { + throw new JetstreamV2Exception($"segment block index entry {index} lies past the end of the file"); + } + + var entry = segmentBytes.AsSpan((int)entryOffset, BlockIndexEntrySize); + var offset = BinaryPrimitives.ReadUInt64LittleEndian(entry.Slice(0, 8)); + var compressedSize = BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8, 4)); + + // Validate the frame range lies within [HeaderSize, FooterOffset). The stored frame is + // preceded by an 8-byte length prefix that is not part of the frame itself. + var footer = (ulong)header.FooterOffset; + if (compressedSize > footer || + offset > footer - 8 || + compressedSize > footer - offset - 8 || + offset < HeaderSize) + { + throw new JetstreamV2Exception($"segment block {index} range is outside the data region"); + } + + var frame = new byte[compressedSize]; + Array.Copy(segmentBytes, (long)offset + 8, frame, 0, compressedSize); + return frame; + } + + /// + /// Decompresses and decodes a single raw block frame into its rows. + /// + /// The compressed block frame. + /// The decoded rows, in stored (per-DID) order. + /// The frame is corrupt or would decompress past the safety cap. + public static IReadOnlyList DecodeBlockFrame(byte[] frame) + { + if (frame == null) + { + throw new ArgumentNullException(nameof(frame)); + } + + byte[] body; + try + { + using var decompressor = new Decompressor(); + var contentSize = Decompressor.GetDecompressedSize(frame); + if (contentSize > MaxDecodedBlockBytes) + { + throw new JetstreamV2Exception( + $"segment block would decompress to {contentSize} bytes, over the {MaxDecodedBlockBytes} byte cap"); + } + + body = decompressor.Unwrap(frame).ToArray(); + } + catch (JetstreamV2Exception) + { + throw; + } + catch (Exception ex) + { + throw new JetstreamV2Exception($"segment block zstd decompress failed: {ex.Message}", ex); + } + + return DecodeBlockBody(body); + } + + /// + /// Decodes an uncompressed columnar block body. Exposed for tooling; most callers use + /// . + /// + /// The uncompressed block body. + /// The decoded rows. + /// The body is truncated or malformed. + public static IReadOnlyList DecodeBlockBody(byte[] body) + { + if (body == null) + { + throw new ArgumentNullException(nameof(body)); + } + + const int FixedPerEvent = 8 + 8 + 8 + 1 + 1 + 2 + 1 + 1 + 4; + + if (body.Length < 4) + { + throw Truncated(); + } + + var span = body.AsSpan(); + long eventCount = BinaryPrimitives.ReadUInt32LittleEndian(span.Slice(0, 4)); + var offset = 4; + + if (eventCount > MaxBlockEvents) + { + throw Truncated(); + } + + if (body.Length - offset < eventCount * FixedPerEvent) + { + throw Truncated(); + } + + var count = (int)eventCount; + if (count == 0) + { + if (offset != body.Length) + { + throw Truncated(); + } + + return Array.Empty(); + } + + var rows = new JetstreamSegmentRow[count]; + for (var i = 0; i < count; i++) + { + rows[i] = new JetstreamSegmentRow(); + } + + // Fixed-size columns, in spec order. + for (var i = 0; i < count; i++) + { + rows[i].Seq = ReadSeq(span.Slice(offset + i * 8, 8), "row seq"); + } + + offset += count * 8; + for (var i = 0; i < count; i++) + { + rows[i].WitnessedAt = (long)BinaryPrimitives.ReadUInt64LittleEndian(span.Slice(offset + i * 8, 8)); + } + + offset += count * 8; + for (var i = 0; i < count; i++) + { + rows[i].IndexedAt = (long)BinaryPrimitives.ReadUInt64LittleEndian(span.Slice(offset + i * 8, 8)); + } + + offset += count * 8; + for (var i = 0; i < count; i++) + { + var kind = body[offset + i]; + if (kind < (byte)JetstreamSegmentRowKind.Create || kind > (byte)JetstreamSegmentRowKind.CreateResync) + { + throw Truncated(); + } + + rows[i].Kind = (JetstreamSegmentRowKind)kind; + } + + offset += count; + var collLenOffset = offset; + offset += count; + var didLenOffset = offset; + offset += count * 2; + var rkeyLenOffset = offset; + offset += count; + var revLenOffset = offset; + offset += count; + var payloadLenOffset = offset; + offset += count * 4; + + // Sum the variable-length blobs, trapping overflow before any allocation. + long totalColl = 0, totalDid = 0, totalRkey = 0, totalRev = 0, totalPayload = 0; + for (var i = 0; i < count; i++) + { + totalColl += body[collLenOffset + i]; + totalDid += BinaryPrimitives.ReadUInt16LittleEndian(span.Slice(didLenOffset + i * 2, 2)); + totalRkey += body[rkeyLenOffset + i]; + totalRev += body[revLenOffset + i]; + totalPayload += BinaryPrimitives.ReadUInt32LittleEndian(span.Slice(payloadLenOffset + i * 4, 4)); + } + + var remaining = (long)body.Length - offset; + if (totalColl + totalDid + totalRkey + totalRev + totalPayload != remaining) + { + // encodeBlock produces an exact-length buffer; anything else is corruption. + throw Truncated(); + } + + var collOffset = offset; + var didOffset = collOffset + (int)totalColl; + var rkeyOffset = didOffset + (int)totalDid; + var revOffset = rkeyOffset + (int)totalRkey; + var payloadOffset = revOffset + (int)totalRev; + + for (var i = 0; i < count; i++) + { + var collLen = body[collLenOffset + i]; + var didLen = BinaryPrimitives.ReadUInt16LittleEndian(span.Slice(didLenOffset + i * 2, 2)); + var rkeyLen = body[rkeyLenOffset + i]; + var revLen = body[revLenOffset + i]; + var payloadLen = (int)BinaryPrimitives.ReadUInt32LittleEndian(span.Slice(payloadLenOffset + i * 4, 4)); + + var row = rows[i]; + row.Collection = collLen == 0 ? string.Empty : Encoding.UTF8.GetString(body, collOffset, collLen); + row.Did = didLen == 0 ? string.Empty : Encoding.UTF8.GetString(body, didOffset, didLen); + row.Rkey = rkeyLen == 0 ? string.Empty : Encoding.UTF8.GetString(body, rkeyOffset, rkeyLen); + row.Rev = revLen == 0 ? string.Empty : Encoding.UTF8.GetString(body, revOffset, revLen); + if (payloadLen > 0) + { + var payload = new byte[payloadLen]; + Array.Copy(body, payloadOffset, payload, 0, payloadLen); + row.Payload = payload; + } + + collOffset += collLen; + didOffset += didLen; + rkeyOffset += rkeyLen; + revOffset += revLen; + payloadOffset += payloadLen; + } + + return rows; + } + + private static long ReadSeq(ReadOnlySpan span, string field) + { + var value = BinaryPrimitives.ReadUInt64LittleEndian(span); + if (value > long.MaxValue) + { + throw new JetstreamV2Exception($"segment {field} {value} is out of range"); + } + + return (long)value; + } + + private static JetstreamV2Exception Truncated() => + new("truncated or malformed segment block"); +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamSegmentHeader.cs b/src/CarpaNet.Jetstream/V2/JetstreamSegmentHeader.cs new file mode 100644 index 0000000..af9467a --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamSegmentHeader.cs @@ -0,0 +1,35 @@ +namespace CarpaNet.Jetstream; + +/// +/// The parsed fixed header at offset 0 of every sealed Jetstream segment file. +/// All offsets are absolute file offsets. +/// +public sealed class JetstreamSegmentHeader +{ + /// The segment format version (currently 1). + public int Version { get; set; } + + /// Number of blocks in the segment. + public int BlockCount { get; set; } + + /// Number of events across all blocks. + public long EventCount { get; set; } + + /// Lowest sequence number stored in the segment. + public long MinSeq { get; set; } + + /// Highest sequence number stored in the segment. + public long MaxSeq { get; set; } + + /// Earliest witnessed-at time in the segment, unix microseconds. + public long MinWitnessedAt { get; set; } + + /// Latest witnessed-at time in the segment, unix microseconds. + public long MaxWitnessedAt { get; set; } + + /// Absolute file offset of the footer (which begins with the block index). + public long FooterOffset { get; set; } + + /// Absolute file offset of the block index within the footer. + public long BlockIndexOffset { get; set; } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamSegmentRow.cs b/src/CarpaNet.Jetstream/V2/JetstreamSegmentRow.cs new file mode 100644 index 0000000..3587330 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamSegmentRow.cs @@ -0,0 +1,40 @@ +namespace CarpaNet.Jetstream; + +/// +/// One row inside a sealed-segment block. +/// +public sealed class JetstreamSegmentRow +{ + /// Jetstream's sequence number for this event. + public long Seq { get; set; } + + /// When Jetstream first saw the event, unix microseconds. + public long WitnessedAt { get; set; } + + /// Operator-imported display timestamp, unix microseconds; 0 when none was imported. + public long IndexedAt { get; set; } + + /// The row's event kind. + public JetstreamSegmentRowKind Kind { get; set; } + + /// The repository DID. + public string Did { get; set; } = string.Empty; + + /// The record's collection NSID; empty for non-commit kinds. + public string Collection { get; set; } = string.Empty; + + /// The record key; empty for non-commit kinds. + public string Rkey { get; set; } = string.Empty; + + /// The repo revision; empty for non-commit kinds. + public string Rev { get; set; } = string.Empty; + + /// The raw DAG-CBOR payload bytes, or null when the row carries none (deletes). + public byte[]? Payload { get; set; } + + /// + /// The timestamp shown to subscribers on the wire: the operator-imported + /// value when one was set, otherwise . + /// + public long DisplayTimeUs => IndexedAt != 0 ? IndexedAt : WitnessedAt; +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamSegmentRowKind.cs b/src/CarpaNet.Jetstream/V2/JetstreamSegmentRowKind.cs new file mode 100644 index 0000000..6bac2d5 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamSegmentRowKind.cs @@ -0,0 +1,29 @@ +namespace CarpaNet.Jetstream; + +/// +/// Discriminates which firehose event type a sealed-segment row represents. +/// Values are the on-disk wire format. +/// +public enum JetstreamSegmentRowKind : byte +{ + /// A record create. + Create = 1, + + /// A record update. + Update = 2, + + /// A record delete. + Delete = 3, + + /// A #identity event; the payload is the upstream event's DAG-CBOR. + Identity = 4, + + /// A #account event; the payload is the upstream event's DAG-CBOR. + Account = 5, + + /// A #sync event; the payload is the upstream event's DAG-CBOR. + Sync = 6, + + /// A resync replacement record (rendered as a create commit). + CreateResync = 7, +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamSnapshotPlan.cs b/src/CarpaNet.Jetstream/V2/JetstreamSnapshotPlan.cs new file mode 100644 index 0000000..58bb6f4 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamSnapshotPlan.cs @@ -0,0 +1,106 @@ +using System.Collections.Generic; + +namespace CarpaNet.Jetstream; + +/// +/// A request for . Empty kind, DID, and +/// collection lists mean match-all. is an exclusive lower bound; +/// (when set) is an inclusive upper bound. +/// +public sealed class JetstreamSnapshotPlanRequest +{ + /// Event kinds to include; null or empty includes all kinds. + public IReadOnlyList? Kinds { get; set; } + + /// Only include data for these DIDs; null or empty includes all DIDs. + public IReadOnlyList? Dids { get; set; } + + /// Collection NSIDs or namespace wildcards such as "app.bsky.feed.*"; constrains commit events only. + public IReadOnlyList? Collections { get; set; } + + /// Start after this sequence number (exclusive); 0 plans from the start of the archive. + public long AfterSeq { get; set; } + + /// Stop at this sequence number (inclusive); null plans through the sealed tip. + public long? BeforeSeq { get; set; } +} + +/// +/// How a planned segment's rows should be fetched. +/// +public enum JetstreamSegmentPlanMode +{ + /// Download the whole segment file with getSegment. + WholeSegment, + + /// Download only the listed block ranges with getBlock. + Blocks, +} + +/// +/// An inclusive range of block indices within a segment. +/// +public sealed class JetstreamBlockRange +{ + /// Index of the first block in the range. + public int First { get; set; } + + /// Index of the last block in the range (inclusive). + public int Last { get; set; } +} + +/// +/// One unit of sealed-archive transport work: either a whole segment or a set of inclusive +/// block ranges within a segment. Seq bounds are transport hints with a one-sided contract +/// (no false negatives, possible false positives) — the client must re-apply exact filtering +/// after decode. +/// +public sealed class JetstreamPlannedSegment +{ + /// The segment filename accepted by getSegment and getBlock. + public string Name { get; set; } = string.Empty; + + /// The zero-based segment index (ascending = creation order). + public int Index { get; set; } + + /// The segment's xxh3 metadata checksum (16 hex chars). Equals the getSegment ETag + /// and uniquely identifies a segment generation. + public string Checksum { get; set; } = string.Empty; + + /// Lowest seq this entry may contain. + public long MinSeq { get; set; } + + /// Highest seq this entry may contain. + public long MaxSeq { get; set; } + + /// Whole-segment vs block-range download. + public JetstreamSegmentPlanMode Mode { get; set; } + + /// The inclusive block ranges to fetch when is + /// ; empty otherwise. + public IReadOnlyList Blocks { get; set; } = System.Array.Empty(); +} + +/// +/// The ordered transport plan returned by the server for a historical backfill query, plus the +/// sealed-archive coverage horizon. +/// +public sealed class JetstreamSnapshotPlan +{ + /// + /// The continuation cursor: the highest sealed seq this page accounts for. Fetch the next + /// page with afterSeq = PlannedThroughSeq; planning is complete once it reaches + /// . + /// + public long PlannedThroughSeq { get; set; } + + /// + /// The pagination goal: the sealed-archive tip, capped by beforeSeq when provided. Pin it + /// from the first page (pass it as beforeSeq on later pages) so the snapshot does not move + /// while it is being downloaded. + /// + public long SealedTipSeq { get; set; } + + /// The segments/block-ranges to download, in ascending order. + public IReadOnlyList Segments { get; set; } = System.Array.Empty(); +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Account.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Account.cs new file mode 100644 index 0000000..48a2332 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Account.cs @@ -0,0 +1,34 @@ +namespace CarpaNet.Jetstream; + +/// +/// A #account event: a change to an account's hosting status, wrapping the upstream +/// com.atproto.sync.subscribeRepos event verbatim. +/// +public sealed class JetstreamV2Account +{ + /// + /// The DID whose hosting status changed. + /// + public string Did { get; set; } = string.Empty; + + /// + /// Whether the account is active on its host. + /// + public bool Active { get; set; } + + /// + /// The inactive reason (e.g. "deleted", "suspended", "takendown") when + /// is false; null when active. + /// + public string? Status { get; set; } + + /// + /// The upstream relay sequence number carried by the event (not Jetstream's seq). + /// + public long Seq { get; set; } + + /// + /// The RFC 3339 timestamp from the upstream event. + /// + public string Time { get; set; } = string.Empty; +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Client.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Client.cs new file mode 100644 index 0000000..4456e64 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Client.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace CarpaNet.Jetstream; + +/// +/// Client for Jetstream v2: the archive-backed ATProtocol event stream service +/// (github.com/bluesky-social/jetstream). One facade covers both transports — sealed history +/// over HTTP/XRPC (planSnapshot → getSegment/getBlock) and the live +/// network.bsky.jetstream.subscribeEvents websocket — and welds +/// them into a single managed stream: replay in seq order, cut over to live with no gap, +/// reconnect with backoff, resume at the last delivered seq, and recover from cursor and +/// compression-dictionary rejections automatically. +/// +/// +/// For the legacy v1 /subscribe wire, use . Servers running +/// Jetstream v2 serve both endpoints; this client speaks only the v2 wire. +/// +public sealed class JetstreamV2Client : IDisposable +{ + private readonly JetstreamV2ClientOptions _options; + private readonly HttpClient? _ownedHttpClient; + private readonly JetstreamV2Transport _transport; + private readonly JetstreamV2Engine _engine; + private bool _disposed; + + /// + /// Creates a new Jetstream v2 client. + /// + /// The Jetstream instance URI (e.g. https://jetstream.us-east.bsky.network). + /// Optional client configuration. + public JetstreamV2Client(Uri baseUri, JetstreamV2ClientOptions? options = null) + { + if (baseUri == null) + { + throw new ArgumentNullException(nameof(baseUri)); + } + + _options = options ?? new JetstreamV2ClientOptions(); + _options.Validate(); + + var httpClient = _options.HttpClient; + if (httpClient == null) + { + // Bulk segment downloads are large transfers; a short wall-clock timeout would + // prematurely kill them, so the owned client relies on cancellation tokens. + _ownedHttpClient = new HttpClient { Timeout = Timeout.InfiniteTimeSpan }; + httpClient = _ownedHttpClient; + } + + var logger = (ILogger?)_options.LoggerFactory?.CreateLogger("CarpaNet.Jetstream.JetstreamV2Client") + ?? NullLogger.Instance; + + _transport = new JetstreamV2Transport(baseUri, httpClient, _options.ApiKey, _options.MaxDownloadAttempts, logger); + _engine = new JetstreamV2Engine(baseUri, _options, _transport, logger); + } + + /// + /// Subscribes to the managed Jetstream v2 stream and yields events in seq order until + /// cancellation. Depending on this is a pure live tail, a + /// backfill of sealed history that cuts over to live, or a point-in-time archive snapshot + /// (see ). + /// + /// + /// Delivery is at-least-once across every boundary; the client dedups by seq internally, + /// but consumers that persist a cursor should treat re-delivery of a persisted seq as + /// possible and fold idempotently. Recoverable conditions (reconnects, dictionary + /// rotation, malformed frames, malformed rows) are logged and handled internally; + /// unrecoverable ones throw . + /// + /// Filters, cursors, and stream mode; null tails live from the current tip. + /// Ends the stream when cancelled. + /// The managed event stream. + public IAsyncEnumerable SubscribeAsync( + JetstreamV2SubscribeOptions? options = null, + CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + var resolved = options ?? new JetstreamV2SubscribeOptions(); + resolved.Validate(); + return _engine.RunAsync(resolved, cancellationToken); + } + + /// + /// Calls network.bsky.jetstream.planSnapshot: builds a transport plan naming the sealed + /// segments (or block ranges) that may contain events for the requested filters. The plan + /// over-approximates; exact filtering happens after decode. Page by re-issuing with + /// afterSeq = the previous page's . + /// + /// The filters and seq window to plan for. + /// Cancellation token. + /// The validated plan page. + public async Task PlanSnapshotAsync( + JetstreamSnapshotPlanRequest request, + CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + var input = new JetstreamV2PlanSnapshotInput + { + Kinds = ToListOrNull(JetstreamV2Engine.KindStrings(request.Kinds)), + Dids = ToListOrNull(request.Dids), + Collections = ToListOrNull(request.Collections), + AfterSeq = request.AfterSeq > 0 ? request.AfterSeq : null, + BeforeSeq = request.BeforeSeq, + }; + + var output = await _transport.PlanSnapshotAsync(input, cancellationToken).ConfigureAwait(false); + return JetstreamV2PlanConverter.Convert(output); + } + + /// + /// Downloads a whole sealed segment file via network.bsky.jetstream.getSegment. Decode it + /// with . + /// + /// The segment filename from a plan (e.g. "seg_000000002a.jss"). + /// Cancellation token. + /// The raw segment file bytes. + public Task GetSegmentAsync(string name, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentException("Segment name is required.", nameof(name)); + } + + return _transport.GetSegmentAsync(name, null, cancellationToken); + } + + /// + /// Downloads a single stored block frame via network.bsky.jetstream.getBlock. The result + /// is exactly the compressed frame + /// accepts. + /// + /// The sealed segment filename. + /// The zero-based block index within the segment. + /// Cancellation token. + /// The raw zstd block frame. + public Task GetBlockAsync(string segment, int blockIndex, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + if (string.IsNullOrEmpty(segment)) + { + throw new ArgumentException("Segment name is required.", nameof(segment)); + } + + if (blockIndex < 0) + { + throw new ArgumentOutOfRangeException(nameof(blockIndex)); + } + + return _transport.GetBlockAsync(segment, blockIndex, cancellationToken); + } + + /// + /// Downloads a zstd compression dictionary via network.bsky.jetstream.getZstdDictionary. + /// The blob is a structured dictionary (RFC 8878 §5), immutable for a given ID. The managed + /// stream fetches and rotates dictionaries automatically when + /// is set; this method exists for + /// direct wire access. + /// + /// A specific dictionary ID, or null for the server's current one. + /// Cancellation token. + /// The dictionary bytes. + public Task GetZstdDictionaryAsync(long? id = null, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + return _transport.GetZstdDictionaryAsync(id, cancellationToken); + } + + /// + /// Disposes the client and any HTTP client it owns. + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _ownedHttpClient?.Dispose(); + } + + private static List? ToListOrNull(IReadOnlyList? values) + { + if (values == null || values.Count == 0) + { + return null; + } + + return new List(values); + } + + private void ThrowIfDisposed() + { +#if NET8_0_OR_GREATER + ObjectDisposedException.ThrowIf(_disposed, this); +#else + if (_disposed) + { + throw new ObjectDisposedException(GetType().FullName); + } +#endif + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2ClientOptions.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2ClientOptions.cs new file mode 100644 index 0000000..036bd52 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2ClientOptions.cs @@ -0,0 +1,98 @@ +using System; +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace CarpaNet.Jetstream; + +/// +/// Configuration for a . All properties have working defaults. +/// +public sealed class JetstreamV2ClientOptions +{ + private static readonly int DefaultConcurrency = + Math.Max(4, Math.Min(32, Environment.ProcessorCount)); + + /// + /// HTTP client used for the archive XRPC calls (planSnapshot, getSegment, getBlock), + /// the public dictionary fetch, and handshake-error classification. When null the client + /// creates and owns its own. + /// + public HttpClient? HttpClient { get; set; } + + /// + /// Logger factory for diagnostics (reconnects, degraded compression, skipped frames). + /// Null discards all output. + /// + public ILoggerFactory? LoggerFactory { get; set; } + + /// + /// Bearer API key for the archive endpoints. Sent as "Authorization: Bearer <key>" on + /// planSnapshot, getSegment, and getBlock only — never on getZstdDictionary or the live + /// websocket, which remain public. The value is a bearer secret: use TLS and keep it out + /// of logs and process arguments. + /// + public string? ApiKey { get; set; } + + /// + /// Opts the live tail into the dictionary-zstd compression scheme: the client fetches the + /// server's current dictionary via getZstdDictionary before the first dial, negotiates it + /// with ?zstdDictionary=<id>, and transparently decompresses binary frames. Fetch or + /// rotation-recovery failure degrades to an uncompressed tail (logged), never a stream + /// failure. Default false. + /// + public bool EnableCompression { get; set; } + + /// + /// Archive-replay parallelism: how many block frames are fetched and decoded concurrently. + /// Defaults to the processor count clamped to [4, 32]. + /// + public int DownloadConcurrency { get; set; } = DefaultConcurrency; + + /// + /// Upper bound on a single live websocket message (and on its decompressed size when + /// compression is on), guarding against unbounded allocations. Default 32 MiB — v2 frames + /// embed whole records. + /// + public int ReadLimitBytes { get; set; } = 32 * 1024 * 1024; + + /// + /// Total attempts (initial request plus retries) for each archive HTTP request before the + /// failure is surfaced. Default 3; 1 disables retries. + /// + public int MaxDownloadAttempts { get; set; } = 3; + + /// + /// Live-tail reconnect backoff floor. Default 250 ms; the delay doubles per failed attempt + /// up to and resets when a session delivers events. + /// + public TimeSpan ReconnectBackoffMin { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Live-tail reconnect backoff ceiling. Default 30 seconds. + /// + public TimeSpan ReconnectBackoffMax { get; set; } = TimeSpan.FromSeconds(30); + + internal void Validate() + { + if (DownloadConcurrency < 1) + { + throw new ArgumentException($"{nameof(DownloadConcurrency)} must be at least 1."); + } + + if (ReadLimitBytes < 1024) + { + throw new ArgumentException($"{nameof(ReadLimitBytes)} must be at least 1024."); + } + + if (MaxDownloadAttempts < 1) + { + throw new ArgumentException($"{nameof(MaxDownloadAttempts)} must be at least 1."); + } + + if (ReconnectBackoffMin <= TimeSpan.Zero || ReconnectBackoffMax < ReconnectBackoffMin) + { + throw new ArgumentException( + $"{nameof(ReconnectBackoffMin)} must be positive and no greater than {nameof(ReconnectBackoffMax)}."); + } + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Commit.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Commit.cs new file mode 100644 index 0000000..2639139 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Commit.cs @@ -0,0 +1,92 @@ +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace CarpaNet.Jetstream; + +/// +/// A single record mutation (create, update, or delete). +/// +public sealed class JetstreamV2Commit +{ + /// + /// The operation: create, update, or delete. + /// + public JetstreamV2CommitOperation Operation { get; set; } + + /// + /// The record's collection NSID, e.g. "app.bsky.feed.post". + /// + public string Collection { get; set; } = string.Empty; + + /// + /// The record key within the collection. + /// + public string Rkey { get; set; } = string.Empty; + + /// + /// The repo revision that produced this commit. + /// + public string Rev { get; set; } = string.Empty; + + /// + /// The content identifier of the record. Null for deletes. + /// + public string? Cid { get; set; } + + /// + /// The record in the atproto JSON data model (byte strings appear as + /// {"$bytes": ...}, CID links as {"$link": ...}). Null for deletes. + /// Use to deserialize into a typed record. + /// + public JsonElement? Record { get; set; } + + /// + /// Deserializes into a typed record using a source-generated + /// (for example from a generated ATProtoJsonContext). + /// Returns default when the commit carries no record (deletes). + /// + /// The record type. + /// The source-generated type info for . + /// The deserialized record, or default for deletes. + public T? GetRecord(JsonTypeInfo typeInfo) + { + if (typeInfo == null) + { + throw new ArgumentNullException(nameof(typeInfo)); + } + + if (Record == null) + { + return default; + } + + return Record.Value.Deserialize(typeInfo); + } + + /// + /// Attempts to deserialize into a typed record, returning false + /// instead of throwing when the commit has no record or the record does not match the shape. + /// + /// The record type. + /// The source-generated type info for . + /// The deserialized record on success. + /// True when a record was deserialized. + public bool TryGetRecord(JsonTypeInfo typeInfo, out T? record) + { + record = default; + if (typeInfo == null || Record == null) + { + return false; + } + + try + { + record = Record.Value.Deserialize(typeInfo); + return record != null; + } + catch (JsonException) + { + return false; + } + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2CommitOperation.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2CommitOperation.cs new file mode 100644 index 0000000..e7874c7 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2CommitOperation.cs @@ -0,0 +1,16 @@ +namespace CarpaNet.Jetstream; + +/// +/// The kind of mutation a carries. +/// +public enum JetstreamV2CommitOperation +{ + /// A record was created (includes resync replacement records). + Create, + + /// A record was updated. + Update, + + /// A record was deleted. and are null. + Delete, +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Engine.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Engine.cs new file mode 100644 index 0000000..61f72af --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Engine.cs @@ -0,0 +1,439 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace CarpaNet.Jetstream; + +/// +/// Orchestrates the managed Jetstream v2 stream. +/// +internal sealed class JetstreamV2Engine +{ + private const int MaxRebackfillStalls = 5; + + private readonly Uri _baseUri; + private readonly JetstreamV2ClientOptions _clientOptions; + private readonly JetstreamV2Transport _transport; + private readonly ILogger _logger; + + public JetstreamV2Engine(Uri baseUri, JetstreamV2ClientOptions clientOptions, JetstreamV2Transport transport, ILogger logger) + { + _baseUri = baseUri; + _clientOptions = clientOptions; + _transport = transport; + _logger = logger; + } + + public async IAsyncEnumerable RunAsync( + JetstreamV2SubscribeOptions options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var matcher = new JetstreamV2Matcher(options); + + if (!options.BackfillRequested) + { + await foreach (var evt in TailLiveAsync(options, matcher, options.LiveCursor, options.LiveCursor ?? 0, cancellationToken).ConfigureAwait(false)) + { + yield return evt; + } + + yield break; + } + + var cursor = options.AfterSeq ?? 0; + + if (options.SnapshotOnly) + { + var snapshotState = new SweepState(); + await foreach (var evt in SweepSealedArchiveAsync(options, matcher, cursor, snapshotState, cancellationToken).ConfigureAwait(false)) + { + yield return evt; + } + + yield break; + } + + var stalls = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var sweep = new SweepState(); + await foreach (var evt in SweepSealedArchiveAsync(options, matcher, cursor, sweep, cancellationToken).ConfigureAwait(false)) + { + yield return evt; + } + + var cutover = Math.Max(sweep.SealedTip, cursor); + long resume; + var tooOld = false; + + var live = await CreateLiveSessionAsync(options, cutover, cutover, cancellationToken).ConfigureAwait(false); + try + { + var enumerator = live.RunAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); + try + { + while (true) + { + bool hasNext; + try + { + hasNext = await enumerator.MoveNextAsync().ConfigureAwait(false); + } + catch (JetstreamV2Exception ex) when (ex.ErrorName == JetstreamV2ErrorNames.CursorTooOld) + { + _logger.LogWarning("Jetstream live cursor too old; re-entering archive backfill: {Message}", ex.Message); + tooOld = true; + break; + } + + if (!hasNext) + { + break; + } + + var evt = enumerator.Current; + + if (matcher.WantsEvent(evt)) + { + yield return evt; + } + } + } + finally + { + await enumerator.DisposeAsync().ConfigureAwait(false); + } + + resume = live.LastSeq; + } + finally + { + live.Dispose(); + } + + if (!tooOld) + { + yield break; + } + + matcher.SetAfterSeq(resume); + + if (resume <= cursor) + { + stalls++; + if (stalls >= MaxRebackfillStalls) + { + throw new JetstreamV2Exception( + $"re-backfill made no progress after {stalls} cursor-too-old cycles at seq {resume}"); + } + } + else + { + stalls = 0; + } + + cursor = resume; + } + } + + private async IAsyncEnumerable TailLiveAsync( + JetstreamV2SubscribeOptions options, + JetstreamV2Matcher matcher, + long? cursor, + long dedupFloor, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var live = await CreateLiveSessionAsync(options, cursor, dedupFloor, cancellationToken).ConfigureAwait(false); + try + { + await foreach (var evt in live.RunAsync(cancellationToken).ConfigureAwait(false)) + { + if (matcher.WantsEvent(evt)) + { + yield return evt; + } + } + } + finally + { + live.Dispose(); + } + } + + private async Task CreateLiveSessionAsync( + JetstreamV2SubscribeOptions options, + long? cursor, + long dedupFloor, + CancellationToken cancellationToken) + { + byte[]? dictionary = null; + if (_clientOptions.EnableCompression) + { + dictionary = await FetchDictionaryAsync(cancellationToken).ConfigureAwait(false); + } + + return new JetstreamV2LiveSession(new JetstreamV2LiveSessionConfig + { + BaseUri = _baseUri, + Cursor = cursor, + DedupFloor = dedupFloor, + Kinds = KindStrings(options.Kinds), + Collections = options.Collections ?? (IReadOnlyList)Array.Empty(), + Dids = options.Dids ?? (IReadOnlyList)Array.Empty(), + MaxMessageSizeBytes = options.MaxMessageSizeBytes, + ReadLimitBytes = _clientOptions.ReadLimitBytes, + BackoffMin = _clientOptions.ReconnectBackoffMin, + BackoffMax = _clientOptions.ReconnectBackoffMax, + ZstdDictionary = dictionary, + RefetchDictionary = _clientOptions.EnableCompression ? FetchDictionaryAsync : null, + Transport = _transport, + Logger = _logger, + }); + } + + private async Task FetchDictionaryAsync(CancellationToken cancellationToken) + { + try + { + return await _transport.GetZstdDictionaryAsync(null, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning("getZstdDictionary failed ({Error}); the live tail will be uncompressed", ex.Message); + return null; + } + } + + internal static IReadOnlyList KindStrings(IReadOnlyList? kinds) + { + if (kinds == null || kinds.Count == 0) + { + return Array.Empty(); + } + + var result = new List(kinds.Count); + foreach (var kind in kinds) + { + var value = kind switch + { + JetstreamV2EventKind.Commit => "commit", + JetstreamV2EventKind.Identity => "identity", + JetstreamV2EventKind.Account => "account", + JetstreamV2EventKind.Sync => "sync", + _ => throw new ArgumentException($"unknown event kind {kind}"), + }; + if (!result.Contains(value)) + { + result.Add(value); + } + } + + return result; + } + + private sealed class SweepState + { + /// The pinned sealed-archive tip (the cutover cursor), set when the first page is planned. + public long SealedTip; + } + + private async IAsyncEnumerable SweepSealedArchiveAsync( + JetstreamV2SubscribeOptions options, + JetstreamV2Matcher matcher, + long startCursor, + SweepState state, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var cursor = startCursor; + var pinned = false; + long sealedTip = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var input = new JetstreamV2PlanSnapshotInput + { + Kinds = ToListOrNull(KindStrings(options.Kinds)), + Dids = ToListOrNull(options.Dids), + Collections = ToListOrNull(options.Collections), + AfterSeq = cursor > 0 ? cursor : null, + BeforeSeq = pinned ? sealedTip : options.BeforeSeq, + }; + + var output = await _transport.PlanSnapshotAsync(input, cancellationToken).ConfigureAwait(false); + var plan = JetstreamV2PlanConverter.Convert(output); + + if (!pinned) + { + sealedTip = plan.SealedTipSeq; + pinned = true; + state.SealedTip = sealedTip; + _logger.LogInformation( + "Jetstream backfill sweep: cursor {Cursor}, sealed tip {SealedTip}", cursor, sealedTip); + } + + await foreach (var evt in DownloadPlanAsync(plan.Segments, matcher, cancellationToken).ConfigureAwait(false)) + { + yield return evt; + } + + var previous = cursor; + cursor = plan.PlannedThroughSeq; + if (cursor >= sealedTip) + { + // Whole sealed archive (startCursor, sealedTip] consumed. An empty archive is + // sealedTip == 0 and terminates here on the first page. + yield break; + } + + if (cursor <= previous) + { + // A stale, buggy, or hostile server returning a non-advancing continuation + // cursor would reissue an identical request forever; fail instead of spinning. + throw new JetstreamV2Exception( + $"planSnapshot made no progress: afterSeq={previous} plannedThroughSeq={cursor} sealedTipSeq={sealedTip}"); + } + } + } + + private async IAsyncEnumerable DownloadPlanAsync( + IReadOnlyList segments, + JetstreamV2Matcher matcher, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var windowSize = Math.Max(2, _clientOptions.DownloadConcurrency); + var window = new Queue>>(); + Task? prefetch = null; + var prefetchIndex = -1; + + for (var i = 0; i < segments.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var entry = segments[i]; + + byte[]? segmentBytes = null; + if (entry.Mode == JetstreamSegmentPlanMode.WholeSegment) + { + var download = prefetchIndex == i && prefetch != null + ? prefetch + : _transport.GetSegmentAsync(entry.Name, entry.Checksum, cancellationToken); + prefetch = null; + prefetchIndex = -1; + segmentBytes = await download.ConfigureAwait(false); + } + + if (prefetch == null && i + 1 < segments.Count && + segments[i + 1].Mode == JetstreamSegmentPlanMode.WholeSegment) + { + var next = segments[i + 1]; + prefetch = Task.Run(() => _transport.GetSegmentAsync(next.Name, next.Checksum, cancellationToken), cancellationToken); + prefetchIndex = i + 1; + } + + if (entry.Mode == JetstreamSegmentPlanMode.WholeSegment) + { + var header = JetstreamSegmentFormat.ReadHeader(segmentBytes); + for (var blockIndex = 0; blockIndex < header.BlockCount; blockIndex++) + { + var bytes = segmentBytes!; + var capturedHeader = header; + var capturedIndex = blockIndex; + window.Enqueue(Task.Run( + () => + { + var frame = JetstreamSegmentFormat.GetBlockFrame(bytes, capturedHeader, capturedIndex); + return DecodeAndConvertBlock(frame, matcher); + }, + cancellationToken)); + + while (window.Count >= windowSize) + { + foreach (var evt in await window.Dequeue().ConfigureAwait(false)) + { + yield return evt; + } + } + } + } + else + { + var name = entry.Name; + foreach (var range in entry.Blocks) + { + for (var blockIndex = range.First; ; blockIndex++) + { + var capturedIndex = blockIndex; + window.Enqueue(Task.Run( + async () => + { + var frame = await _transport.GetBlockAsync(name, capturedIndex, cancellationToken).ConfigureAwait(false); + return DecodeAndConvertBlock(frame, matcher); + }, + cancellationToken)); + + while (window.Count >= windowSize) + { + foreach (var evt in await window.Dequeue().ConfigureAwait(false)) + { + yield return evt; + } + } + + if (blockIndex == range.Last) + { + break; + } + } + } + } + } + + while (window.Count > 0) + { + foreach (var evt in await window.Dequeue().ConfigureAwait(false)) + { + yield return evt; + } + } + } + + private List DecodeAndConvertBlock(byte[] frame, JetstreamV2Matcher matcher) + { + var rows = JetstreamSegmentFormat.DecodeBlockFrame(frame); + var events = new List(rows.Count); + foreach (var row in rows) + { + if (!matcher.WantsRow(row)) + { + continue; + } + + try + { + events.Add(JetstreamV2RecordDecoder.ConvertRow(row)); + } + catch (JetstreamV2Exception ex) + { + _logger.LogWarning("Jetstream archive row skipped: {Error}", ex.Message); + } + } + + return events; + } + + private static List? ToListOrNull(IReadOnlyList? values) + { + if (values == null || values.Count == 0) + { + return null; + } + + return new List(values); + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Event.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Event.cs new file mode 100644 index 0000000..9b7f267 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Event.cs @@ -0,0 +1,56 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace CarpaNet.Jetstream; + +/// +/// A single decoded Jetstream v2 event. +/// +public sealed class JetstreamV2Event +{ + /// + /// The repository (account) DID this event belongs to. + /// + public string Did { get; set; } = string.Empty; + + /// + /// Jetstream's monotonic per-event sequence number (the stream cursor). Persist the last + /// seen value to resume later via or + /// . Cursors are instance-local: seq + /// values do not transfer between Jetstream servers. + /// + public long Seq { get; set; } + + /// + /// The event's display timestamp in microseconds since the Unix epoch: the time Jetstream + /// witnessed the event, unless an operator timestamp import overrode it. It is not the + /// record's client-supplied createdAt. is the only faithful resume position. + /// + public long TimeUs { get; set; } + + /// + /// Selects which of the payload properties below is populated. + /// + public JetstreamV2EventKind Kind { get; set; } + + /// + /// The commit payload when is . + /// + public JetstreamV2Commit? Commit { get; set; } + + /// + /// The identity payload when is . + /// + public JetstreamV2Identity? Identity { get; set; } + + /// + /// The account payload when is . + /// + public JetstreamV2Account? Account { get; set; } + + /// + /// The sync payload when is . + /// + public JetstreamV2Sync? Sync { get; set; } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2EventKind.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2EventKind.cs new file mode 100644 index 0000000..b0ca5ed --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2EventKind.cs @@ -0,0 +1,20 @@ +namespace CarpaNet.Jetstream; + +/// +/// Discriminates the firehose event type carried by a . +/// +public enum JetstreamV2EventKind +{ + /// A record create, update, or delete. is non-null. + Commit, + + /// A handle or DID document change. is non-null. + Identity, + + /// A hosting-status change. is non-null. + Account, + + /// A repo divergence requiring resync. is non-null. + /// Sync events are delivered during archive replay and on the v2 live tail (never on the v1 wire). + Sync, +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Exception.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Exception.cs new file mode 100644 index 0000000..e338d20 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Exception.cs @@ -0,0 +1,70 @@ +using System; + +namespace CarpaNet.Jetstream; + +/// +/// Well-known Jetstream v2 XRPC error names, matched structurally from error envelopes +/// (never by message substring). +/// +public static class JetstreamV2ErrorNames +{ + /// The requested seq cursor is below the server's retention floor (pre-upgrade HTTP 400). + public const string CursorTooOld = "CursorTooOld"; + + /// The negotiated zstd dictionary ID is unknown or retired (pre-upgrade HTTP 400). + public const string UnknownZstdDictionary = "UnknownZstdDictionary"; + + /// The request configuration was rejected (pre-upgrade HTTP 400). + public const string InvalidRequest = "InvalidRequest"; + + /// The client fell adversarially far behind the live tip (terminal stream error frame). + public const string ConsumerTooSlow = "ConsumerTooSlow"; + + /// The server is not ready yet (HTTP 503 during bootstrap; retryable). + public const string ServiceUnavailable = "ServiceUnavailable"; +} + +/// +/// Terminal Jetstream v2 failure: an XRPC error the managed stream cannot recover from, +/// an exhausted retry/re-backfill budget, or a protocol violation. Recoverable conditions +/// (reconnects, dictionary rotation, malformed frames, per-block download failures) are +/// handled internally and logged instead. +/// +public class JetstreamV2Exception : Exception +{ + /// + /// Creates a new exception with a message. + /// + /// The failure description. + public JetstreamV2Exception(string message) + : base(message) + { + } + + /// + /// Creates a new exception with a message and inner exception. + /// + /// The failure description. + /// The underlying failure. + public JetstreamV2Exception(string message, Exception? innerException) + : base(message, innerException) + { + } + + /// + /// Creates a new exception carrying a structured XRPC error name. + /// + /// The failure description. + /// The XRPC error name (see ). + public JetstreamV2Exception(string message, string? errorName) + : base(message) + { + ErrorName = errorName; + } + + /// + /// The structured XRPC error name when the failure came from a server error envelope + /// (see ), otherwise null. + /// + public string? ErrorName { get; } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2FrameDecoder.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2FrameDecoder.cs new file mode 100644 index 0000000..599d6ff --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2FrameDecoder.cs @@ -0,0 +1,399 @@ +using System; +using System.Globalization; +using System.Text.Json; + +namespace CarpaNet.Jetstream; + +/// +/// How a decoded live frame should be handled by the session loop. +/// +internal enum JetstreamV2FrameKind +{ + /// A decoded event; is non-null. + Event, + + /// An #info advisory (no seq, not an event): log and continue. + Info, + + /// A terminal error frame; the server closes right after sending it. + StreamError, + + /// A well-formed frame from a newer protocol revision: skip for forward compat. + Skip, + + /// A malformed frame: surface (log) but keep the connection. + Malformed, +} + +/// +/// The outcome of decoding one xrpc.v1.json frame. +/// +internal sealed class JetstreamV2FrameResult +{ + public JetstreamV2FrameKind Kind { get; private set; } + + public JetstreamV2Event? Event { get; private set; } + + /// Info name, stream error name, or malformed-frame description depending on . + public string? Name { get; private set; } + + public string? Message { get; private set; } + + public static JetstreamV2FrameResult ForEvent(JetstreamV2Event evt) => + new() { Kind = JetstreamV2FrameKind.Event, Event = evt }; + + public static JetstreamV2FrameResult ForInfo(string name, string? message) => + new() { Kind = JetstreamV2FrameKind.Info, Name = name, Message = message }; + + public static JetstreamV2FrameResult ForStreamError(string name, string? message) => + new() { Kind = JetstreamV2FrameKind.StreamError, Name = name, Message = message }; + + public static JetstreamV2FrameResult ForSkip() => + new() { Kind = JetstreamV2FrameKind.Skip }; + + public static JetstreamV2FrameResult ForMalformed(string reason) => + new() { Kind = JetstreamV2FrameKind.Malformed, Name = reason }; +} + +/// +/// Decodes xrpc.v1.json frames from the network.bsky.jetstream.subscribeEvents wire into +/// values. Unknown envelope or payload $types skip for forward +/// compatibility; a frame with no $type at all is malformed (it usually means the client hit +/// a v1 /subscribe endpoint). +/// +internal static class JetstreamV2FrameDecoder +{ + private const string PayloadTypePrefix = "network.bsky.jetstream.subscribeEvents#"; + + // Bounds on untrusted server-supplied diagnostic strings before they enter + // exception messages and logs. + private const int MaxDiagNameLength = 128; + private const int MaxDiagMessageLength = 1024; + + private const long UnixEpochTicks = 621355968000000000L; + + public static JetstreamV2FrameResult Decode(ReadOnlySpan frame) + { + JetstreamV2Envelope? envelope; + try + { + envelope = JsonSerializer.Deserialize(frame, JetstreamV2JsonContext.Default.JetstreamV2Envelope); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid frame JSON: {ex.Message}"); + } + + if (envelope == null) + { + return JetstreamV2FrameResult.ForMalformed("null frame"); + } + + switch (envelope.Type) + { + case "message": + break; + case "error": + if (string.IsNullOrEmpty(envelope.Error)) + { + return JetstreamV2FrameResult.ForMalformed("error frame missing error code"); + } + + return JetstreamV2FrameResult.ForStreamError( + Bound(envelope.Error!, MaxDiagNameLength), + envelope.Message == null ? null : Bound(envelope.Message, MaxDiagMessageLength)); + case null: + case "": + // No $type at all is not a newer protocol revision — it is a malformed frame + // (e.g. a v1 /subscribe server). Skipping it would make a wrong endpoint look + // healthy while delivering nothing. + return JetstreamV2FrameResult.ForMalformed( + "frame missing envelope $type; is the server a network.bsky.jetstream.subscribeEvents endpoint?"); + default: + // A well-formed frame with an unknown envelope $type is a newer protocol + // revision; skip rather than break. + return JetstreamV2FrameResult.ForSkip(); + } + + if (envelope.Payload == null || envelope.Payload.Value.ValueKind != JsonValueKind.Object) + { + return JetstreamV2FrameResult.ForMalformed("message frame missing payload"); + } + + var payload = envelope.Payload.Value; + if (!payload.TryGetProperty("$type", out var typeProperty) || typeProperty.ValueKind != JsonValueKind.String) + { + // A payload with NO $type is malformed, not a future addition — skipping it would + // be silent event loss. + return JetstreamV2FrameResult.ForMalformed("message payload missing $type"); + } + + var payloadType = typeProperty.GetString() ?? string.Empty; + return payloadType switch + { + PayloadTypePrefix + "commit" => DecodeCommit(payload), + PayloadTypePrefix + "identity" => DecodeIdentity(payload), + PayloadTypePrefix + "account" => DecodeAccount(payload), + PayloadTypePrefix + "sync" => DecodeSync(payload), + PayloadTypePrefix + "info" => DecodeInfo(payload), + // A nonempty unknown $type is a newer server's message kind; skip for forward compat. + _ => JetstreamV2FrameResult.ForSkip(), + }; + } + + private static JetstreamV2FrameResult DecodeCommit(JsonElement payload) + { + JetstreamV2CommitPayload? commit; + try + { + commit = payload.Deserialize(JetstreamV2JsonContext.Default.JetstreamV2CommitPayload); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid commit payload: {ex.Message}"); + } + + if (commit == null) + { + return JetstreamV2FrameResult.ForMalformed("null commit payload"); + } + + if (!TryEnvelopeFields(commit.Seq, commit.Time, out var timeUs, out var reason)) + { + return JetstreamV2FrameResult.ForMalformed(reason); + } + + // All four identifiers are lexicon-required, and a folding consumer cannot key a + // mutation without them; a frame omitting one must error rather than emit an + // unfoldable event that advances the dedup cursor. + if (string.IsNullOrEmpty(commit.Did) || string.IsNullOrEmpty(commit.Rev) || + string.IsNullOrEmpty(commit.Collection) || string.IsNullOrEmpty(commit.Rkey)) + { + return JetstreamV2FrameResult.ForMalformed("commit frame missing required did, rev, collection, or rkey"); + } + + JetstreamV2CommitOperation operation; + switch (commit.Operation) + { + case "create": + operation = JetstreamV2CommitOperation.Create; + break; + case "update": + operation = JetstreamV2CommitOperation.Update; + break; + case "delete": + operation = JetstreamV2CommitOperation.Delete; + break; + default: + return JetstreamV2FrameResult.ForMalformed($"unknown commit operation \"{commit.Operation}\""); + } + + var result = new JetstreamV2Commit + { + Operation = operation, + Collection = commit.Collection!, + Rkey = commit.Rkey!, + Rev = commit.Rev!, + Cid = commit.Cid, + }; + + if (operation != JetstreamV2CommitOperation.Delete) + { + if (commit.Record == null || commit.Record.Value.ValueKind != JsonValueKind.Object) + { + return JetstreamV2FrameResult.ForMalformed( + $"{commit.Operation} commit missing record (collection={commit.Collection} rkey={commit.Rkey})"); + } + + result.Record = commit.Record; + } + + return JetstreamV2FrameResult.ForEvent(new JetstreamV2Event + { + Did = commit.Did!, + Seq = commit.Seq, + TimeUs = timeUs, + Kind = JetstreamV2EventKind.Commit, + Commit = result, + }); + } + + private static JetstreamV2FrameResult DecodeIdentity(JsonElement payload) + { + JetstreamV2IdentityPayload? identity; + try + { + identity = payload.Deserialize(JetstreamV2JsonContext.Default.JetstreamV2IdentityPayload); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid identity payload: {ex.Message}"); + } + + // Presence-of-payload check: the outer DID is lexicon-required and is the Event.Did + // every filter and fold keys on; the wrapped upstream event must at least carry a DID. + if (identity == null || string.IsNullOrEmpty(identity.Did) || + identity.Identity == null || string.IsNullOrEmpty(identity.Identity.Did)) + { + return JetstreamV2FrameResult.ForMalformed("identity frame missing required DID or identity payload"); + } + + if (!TryEnvelopeFields(identity.Seq, identity.Time, out var timeUs, out var reason)) + { + return JetstreamV2FrameResult.ForMalformed(reason); + } + + return JetstreamV2FrameResult.ForEvent(new JetstreamV2Event + { + Did = identity.Did!, + Seq = identity.Seq, + TimeUs = timeUs, + Kind = JetstreamV2EventKind.Identity, + Identity = new JetstreamV2Identity + { + Did = identity.Identity.Did!, + Handle = string.IsNullOrEmpty(identity.Identity.Handle) ? null : identity.Identity.Handle, + Seq = identity.Identity.Seq, + Time = identity.Identity.Time ?? string.Empty, + }, + }); + } + + private static JetstreamV2FrameResult DecodeAccount(JsonElement payload) + { + JetstreamV2AccountPayload? account; + try + { + account = payload.Deserialize(JetstreamV2JsonContext.Default.JetstreamV2AccountPayload); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid account payload: {ex.Message}"); + } + + if (account == null || string.IsNullOrEmpty(account.Did) || + account.Account == null || string.IsNullOrEmpty(account.Account.Did)) + { + return JetstreamV2FrameResult.ForMalformed("account frame missing required DID or account payload"); + } + + if (!TryEnvelopeFields(account.Seq, account.Time, out var timeUs, out var reason)) + { + return JetstreamV2FrameResult.ForMalformed(reason); + } + + return JetstreamV2FrameResult.ForEvent(new JetstreamV2Event + { + Did = account.Did!, + Seq = account.Seq, + TimeUs = timeUs, + Kind = JetstreamV2EventKind.Account, + Account = new JetstreamV2Account + { + Did = account.Account.Did!, + Active = account.Account.Active, + Status = string.IsNullOrEmpty(account.Account.Status) ? null : account.Account.Status, + Seq = account.Account.Seq, + Time = account.Account.Time ?? string.Empty, + }, + }); + } + + private static JetstreamV2FrameResult DecodeSync(JsonElement payload) + { + JetstreamV2SyncPayload? sync; + try + { + sync = payload.Deserialize(JetstreamV2JsonContext.Default.JetstreamV2SyncPayload); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid sync payload: {ex.Message}"); + } + + // Archived #sync payloads from an async resync legitimately carry an empty time/seq, + // so did is the only reliable presence marker. + if (sync == null || string.IsNullOrEmpty(sync.Did) || + sync.Sync == null || string.IsNullOrEmpty(sync.Sync.Did)) + { + return JetstreamV2FrameResult.ForMalformed("sync frame missing required DID or sync payload"); + } + + if (!TryEnvelopeFields(sync.Seq, sync.Time, out var timeUs, out var reason)) + { + return JetstreamV2FrameResult.ForMalformed(reason); + } + + return JetstreamV2FrameResult.ForEvent(new JetstreamV2Event + { + Did = sync.Did!, + Seq = sync.Seq, + TimeUs = timeUs, + Kind = JetstreamV2EventKind.Sync, + Sync = new JetstreamV2Sync + { + Did = sync.Sync.Did!, + Rev = sync.Sync.Rev ?? string.Empty, + Seq = sync.Sync.Seq, + Time = sync.Sync.Time ?? string.Empty, + }, + }); + } + + private static JetstreamV2FrameResult DecodeInfo(JsonElement payload) + { + JetstreamV2InfoPayload? info; + try + { + info = payload.Deserialize(JetstreamV2JsonContext.Default.JetstreamV2InfoPayload); + } + catch (JsonException ex) + { + return JetstreamV2FrameResult.ForMalformed($"invalid info payload: {ex.Message}"); + } + + return JetstreamV2FrameResult.ForInfo( + Bound(info?.Name ?? string.Empty, MaxDiagNameLength), + info?.Message == null ? null : Bound(info.Message, MaxDiagMessageLength)); + } + + /// + /// Validates the envelope fields shared by every message kind: seq (1-based on the wire, + /// so 0 means the required field was absent) and the canonical microsecond-precision + /// datetime, parsed back to unix microseconds. + /// + private static bool TryEnvelopeFields(long seq, string? time, out long timeUs, out string reason) + { + timeUs = 0; + if (seq <= 0) + { + reason = $"frame with invalid seq {seq}"; + return false; + } + + if (string.IsNullOrEmpty(time) || + !DateTimeOffset.TryParse( + time, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + reason = $"frame time \"{time}\" is not a valid datetime"; + return false; + } + + timeUs = (parsed.UtcTicks - UnixEpochTicks) / 10; + reason = string.Empty; + return true; + } + + private static string Bound(string value, int limit) + { + if (value.Length <= limit) + { + return value; + } + + return value.Substring(0, limit) + "…"; + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Identity.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Identity.cs new file mode 100644 index 0000000..e12c12a --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Identity.cs @@ -0,0 +1,28 @@ +namespace CarpaNet.Jetstream; + +/// +/// A #identity event: a change to an account's handle or DID document, wrapping the +/// upstream com.atproto.sync.subscribeRepos event verbatim. +/// +public sealed class JetstreamV2Identity +{ + /// + /// The DID whose identity changed. + /// + public string Did { get; set; } = string.Empty; + + /// + /// The account's new handle, or null when not present in the event. + /// + public string? Handle { get; set; } + + /// + /// The upstream relay sequence number carried by the event (not Jetstream's seq). + /// + public long Seq { get; set; } + + /// + /// The RFC 3339 timestamp from the upstream event. + /// + public string Time { get; set; } = string.Empty; +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2LiveSession.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2LiveSession.cs new file mode 100644 index 0000000..79033fb --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2LiveSession.cs @@ -0,0 +1,513 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.Net.WebSockets; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using ZstdSharp; + +namespace CarpaNet.Jetstream; + +/// +/// Configuration for one . +/// +internal sealed class JetstreamV2LiveSessionConfig +{ + public Uri BaseUri { get; set; } = null!; + + /// + /// The initial wire resume point sent as ?cursor= on the first connection (the server + /// replays inclusively; the session's own seq dedup drops the overlap). Null omits the + /// parameter so the server starts at the live tip. + /// + public long? Cursor { get; set; } + + /// + /// Seeds the dedup floor: the highest seq the caller already holds. 0 means nothing was + /// delivered yet, so the first real event (seq >= 1) always passes. + /// + public long DedupFloor { get; set; } + + public IReadOnlyList Kinds { get; set; } = Array.Empty(); + + public IReadOnlyList Collections { get; set; } = Array.Empty(); + + public IReadOnlyList Dids { get; set; } = Array.Empty(); + + public long? MaxMessageSizeBytes { get; set; } + + public int ReadLimitBytes { get; set; } + + public TimeSpan BackoffMin { get; set; } + + public TimeSpan BackoffMax { get; set; } + + /// The dict-zstd dictionary blob, or null for an uncompressed tail. + public byte[]? ZstdDictionary { get; set; } + + /// Re-fetches the server's current dictionary after a rotation rejection; null disables in-place recovery. + public Func>? RefetchDictionary { get; set; } + + /// Classifies failed handshakes via a plain HTTP probe of the same URL. + public JetstreamV2Transport Transport { get; set; } = null!; + + public ILogger Logger { get; set; } = null!; +} + +/// +/// Tails the live network.bsky.jetstream.subscribeEvents websocket: dial, read, decode, +/// deduplicate the at-least-once overlap by seq, and reconnect with bounded exponential +/// backoff. Pre-upgrade CursorTooOld and InvalidRequest rejections are terminal for the +/// session (thrown as ); a rotated zstd dictionary is +/// recovered in place; everything else reconnects. +/// +internal sealed class JetstreamV2LiveSession : IDisposable +{ + private const string Subprotocol = "xrpc.v1.json"; + + private readonly JetstreamV2LiveSessionConfig _cfg; + + private Decompressor? _decompressor; + private uint _dictionaryId; + private bool _seenAny; + + public JetstreamV2LiveSession(JetstreamV2LiveSessionConfig cfg) + { + _cfg = cfg; + LastSeq = cfg.DedupFloor; + InstallDictionary(cfg.ZstdDictionary, logFailure: true); + } + + /// + /// The highest seq delivered, or the seeded dedup floor when nothing was delivered yet. + /// The engine reads it after enumeration ends to resume a re-backfill. + /// + public long LastSeq { get; private set; } + + /// + /// Runs the tail until cancellation, yielding decoded events in delivery order. + /// + public async IAsyncEnumerable RunAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var backoff = _cfg.BackoffMin; + while (!cancellationToken.IsCancellationRequested) + { + ClientWebSocket? webSocket = null; + string? failure = null; + try + { + webSocket = await ConnectAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + yield break; + } + catch (JetstreamV2Exception ex) when ( + ex.ErrorName == JetstreamV2ErrorNames.CursorTooOld || + ex.ErrorName == JetstreamV2ErrorNames.InvalidRequest) + { + // Terminal for the session: the cursor will not become valid by retrying, and + // filters are immutable per connection. The engine decides whether a + // CursorTooOld can be recovered by re-entering backfill. + throw; + } + catch (JetstreamV2Exception ex) when (ex.ErrorName == JetstreamV2ErrorNames.UnknownZstdDictionary) + { + // The server rotated its dictionary out from under us. Recoverable in place: + // refresh (or shed) the dictionary before the reconnect below. + await RefreshDictionaryAsync(cancellationToken).ConfigureAwait(false); + failure = ex.Message; + } + catch (Exception ex) + { + failure = ex.Message; + } + + if (webSocket != null) + { + var seqBefore = LastSeq; + var buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + while (true) + { + JetstreamV2Event? next = null; + try + { + var turn = await ReadNextAsync(webSocket, buffer, cancellationToken).ConfigureAwait(false); + buffer = turn.Buffer; + if (turn.SessionEnded) + { + failure = turn.Reason; + break; + } + + next = turn.Event; + } + catch (OperationCanceledException) + { + yield break; + } + catch (Exception ex) + { + failure = ex.Message; + break; + } + + // Deduplicate the at-least-once reconnect overlap: skip anything at or + // below the highest seq already delivered. + if (next == null || next.Seq <= LastSeq) + { + continue; + } + + LastSeq = next.Seq; + _seenAny = true; + yield return next; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + webSocket.Dispose(); + } + + // A session that delivered new events is healthy; reset backoff so a + // long-lived connection that finally drops reconnects promptly. + if (LastSeq != seqBefore) + { + backoff = _cfg.BackoffMin; + } + } + + if (cancellationToken.IsCancellationRequested) + { + yield break; + } + + if (failure != null) + { + _cfg.Logger.LogWarning("Jetstream live tail reconnecting: {Reason}", failure); + } + + try + { + await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + yield break; + } + + backoff = backoff.TotalMilliseconds * 2 > _cfg.BackoffMax.TotalMilliseconds + ? _cfg.BackoffMax + : TimeSpan.FromMilliseconds(backoff.TotalMilliseconds * 2); + } + } + + public void Dispose() + { + _decompressor?.Dispose(); + _decompressor = null; + } + + private readonly struct SessionTurn + { + public SessionTurn(JetstreamV2Event? evt, bool ended, string? reason, byte[] buffer) + { + Event = evt; + SessionEnded = ended; + Reason = reason; + Buffer = buffer; + } + + public JetstreamV2Event? Event { get; } + + public bool SessionEnded { get; } + + public string? Reason { get; } + + /// The (possibly re-rented, grown) receive buffer, handed back to the caller. + public byte[] Buffer { get; } + } + + /// + /// Reads frames until one yields an event or ends the session. Info advisories, unknown + /// $types, malformed frames, and stray binary frames are logged/skipped here. + /// + private async Task ReadNextAsync(ClientWebSocket webSocket, byte[] buffer, CancellationToken cancellationToken) + { + while (true) + { + var (messageType, count, grown) = await ReceiveFullMessageAsync(webSocket, buffer, cancellationToken).ConfigureAwait(false); + buffer = grown; + if (messageType == WebSocketMessageType.Close) + { + return new SessionTurn(null, true, "server closed the connection", buffer); + } + + ReadOnlyMemory frame; + if (messageType == WebSocketMessageType.Binary) + { + if (_decompressor == null) + { + // Jetstream v2 frames are text JSON unless dict-zstd was negotiated; + // ignore stray binary. + continue; + } + + byte[] decompressed; + try + { + var contentSize = Decompressor.GetDecompressedSize(buffer.AsSpan(0, count)); + if (contentSize > (ulong)_cfg.ReadLimitBytes) + { + throw new JetstreamV2Exception( + $"compressed frame would expand to {contentSize} bytes, over the {_cfg.ReadLimitBytes} byte read limit"); + } + + decompressed = _decompressor.Unwrap(buffer.AsSpan(0, count)).ToArray(); + } + catch (Exception ex) + { + // Upstream input, never crash: surface and keep the tail. + _cfg.Logger.LogWarning("Jetstream zstd frame decode failed: {Error}", ex.Message); + continue; + } + + frame = decompressed; + } + else + { + frame = new ReadOnlyMemory(buffer, 0, count); + } + + var result = JetstreamV2FrameDecoder.Decode(frame.Span); + switch (result.Kind) + { + case JetstreamV2FrameKind.Event: + return new SessionTurn(result.Event, false, null, buffer); + case JetstreamV2FrameKind.Info: + // An advisory (e.g. OutdatedCursor on a clamped timestamp resume). Not an + // event: no seq, no cursor advance. Operator-relevant, so log it. + _cfg.Logger.LogInformation("Jetstream stream info: {Name} {Message}", result.Name, result.Message); + continue; + case JetstreamV2FrameKind.StreamError: + // A terminal error frame: the server closes right after sending it. End + // the session so the reconnect loop backs off and resumes at LastSeq. + return new SessionTurn(null, true, $"stream error {result.Name}: {result.Message}", buffer); + case JetstreamV2FrameKind.Malformed: + // One bad frame must not drop the tail. + _cfg.Logger.LogWarning("Jetstream malformed frame: {Reason}", result.Name); + continue; + default: + continue; + } + } + } + + private async Task<(WebSocketMessageType MessageType, int Count, byte[] Buffer)> ReceiveFullMessageAsync( + ClientWebSocket webSocket, byte[] buffer, CancellationToken cancellationToken) + { + var total = 0; + while (true) + { + var segment = new ArraySegment(buffer, total, buffer.Length - total); + var result = await webSocket.ReceiveAsync(segment, cancellationToken).ConfigureAwait(false); + if (result.MessageType == WebSocketMessageType.Close) + { + return (WebSocketMessageType.Close, 0, buffer); + } + + total += result.Count; + if (result.EndOfMessage) + { + return (result.MessageType, total, buffer); + } + + if (total >= buffer.Length) + { + if (buffer.Length >= _cfg.ReadLimitBytes) + { + throw new JetstreamV2Exception($"message exceeds the {_cfg.ReadLimitBytes} byte read limit"); + } + + var next = ArrayPool.Shared.Rent(Math.Min(buffer.Length * 2, _cfg.ReadLimitBytes)); + Array.Copy(buffer, next, total); + ArrayPool.Shared.Return(buffer); + buffer = next; + } + } + } + + private async Task ConnectAsync(CancellationToken cancellationToken) + { + var wsUri = BuildSubscribeUri(webSocketScheme: true); + var webSocket = new ClientWebSocket(); + try + { + webSocket.Options.AddSubProtocol(Subprotocol); + await webSocket.ConnectAsync(wsUri, cancellationToken).ConfigureAwait(false); + return webSocket; + } + catch (Exception ex) when (ex is WebSocketException && !cancellationToken.IsCancellationRequested) + { + webSocket.Dispose(); + + // .NET does not expose the pre-upgrade HTTP response body, so classify the + // rejection by re-issuing the request as a plain HTTP GET: the server validates + // parameters before upgrading, so the probe reproduces the XRPC error envelope. + var envelope = await _cfg.Transport + .ProbeHandshakeErrorAsync(BuildSubscribeUri(webSocketScheme: false), cancellationToken) + .ConfigureAwait(false); + if (envelope?.Error != null && envelope.Error != JetstreamV2ErrorNames.ServiceUnavailable) + { + throw new JetstreamV2Exception($"{envelope.Error}: {envelope.Message ?? wsUri.ToString()}", envelope.Error); + } + + throw new JetstreamV2Exception($"websocket connect failed: {ex.Message}", (Exception?)ex); + } + catch + { + webSocket.Dispose(); + throw; + } + } + + private Uri BuildSubscribeUri(bool webSocketScheme) + { + var query = new List>(); + + // Once any event has been delivered, resume each new session at LastSeq — re-anchoring + // at the reconnect-time tip would silently drop events produced while disconnected. + // Before any delivery use the configured start: omit the parameter entirely for a + // from-tip start (distinct from cursor=0, which replays everything). + if (_seenAny) + { + query.Add(new KeyValuePair("cursor", LastSeq.ToString(CultureInfo.InvariantCulture))); + } + else if (_cfg.Cursor != null) + { + query.Add(new KeyValuePair("cursor", _cfg.Cursor.Value.ToString(CultureInfo.InvariantCulture))); + } + + foreach (var kind in _cfg.Kinds) + { + query.Add(new KeyValuePair("kinds", kind)); + } + + foreach (var collection in _cfg.Collections) + { + query.Add(new KeyValuePair("collections", collection)); + } + + foreach (var did in _cfg.Dids) + { + query.Add(new KeyValuePair("dids", did)); + } + + if (_cfg.MaxMessageSizeBytes is > 0) + { + query.Add(new KeyValuePair( + "maxMessageSizeBytes", _cfg.MaxMessageSizeBytes.Value.ToString(CultureInfo.InvariantCulture))); + } + + if (_decompressor != null) + { + query.Add(new KeyValuePair( + "zstdDictionary", _dictionaryId.ToString(CultureInfo.InvariantCulture))); + } + + var uri = JetstreamV2Transport.BuildXrpcUri(_cfg.BaseUri, JetstreamV2Transport.SubscribeEventsNsid, query); + if (!webSocketScheme) + { + return uri; + } + + var builder = new UriBuilder(uri); + if (builder.Scheme == "http") + { + builder.Scheme = "ws"; + } + else if (builder.Scheme == "https") + { + builder.Scheme = "wss"; + } + + return builder.Uri; + } + + private void InstallDictionary(byte[]? dictionary, bool logFailure) + { + if (dictionary == null) + { + return; + } + + if (!JetstreamZstdDictionary.TryParseId(dictionary, out var id)) + { + if (logFailure) + { + _cfg.Logger.LogWarning("Invalid zstd dictionary blob; continuing with an uncompressed live tail"); + } + + return; + } + + try + { + var decompressor = new Decompressor(); + decompressor.LoadDictionary(dictionary); + _decompressor?.Dispose(); + _decompressor = decompressor; + _dictionaryId = id; + } + catch (Exception ex) + { + if (logFailure) + { + _cfg.Logger.LogWarning("Zstd decompressor construction failed ({Error}); continuing uncompressed", ex.Message); + } + } + } + + /// + /// Recovers from a server-side dictionary rotation: re-fetch the current dictionary and + /// swap the decompressor so the next dial negotiates the new ID. When the refetch is + /// unavailable, fails, or returns the very ID just rejected (a mixed-version fleet), shed + /// the opt-in and continue uncompressed — compression is an optimization; the tail must + /// keep flowing. + /// + private async Task RefreshDictionaryAsync(CancellationToken cancellationToken) + { + if (_decompressor == null) + { + return; + } + + var rejectedId = _dictionaryId; + if (_cfg.RefetchDictionary != null) + { + var blob = await _cfg.RefetchDictionary(cancellationToken).ConfigureAwait(false); + if (blob != null && JetstreamZstdDictionary.TryParseId(blob, out var newId) && newId != rejectedId) + { + InstallDictionary(blob, logFailure: false); + if (_dictionaryId == newId) + { + _cfg.Logger.LogInformation( + "Jetstream zstd dictionary rotated: rejected {RejectedId}, now using {NewId}", rejectedId, newId); + return; + } + } + } + + _decompressor.Dispose(); + _decompressor = null; + _dictionaryId = 0; + _cfg.Logger.LogWarning( + "Jetstream zstd dictionary {RejectedId} rejected and refetch unavailable; continuing uncompressed", rejectedId); + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Matcher.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Matcher.cs new file mode 100644 index 0000000..09c4df2 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Matcher.cs @@ -0,0 +1,187 @@ +using System.Collections.Generic; + +namespace CarpaNet.Jetstream; + +/// +/// Applies the caller's exact kind/DID/collection/seq filters to archive rows and live events. +/// The snapshot planner is a one-sided transport hint (no false negatives, possible false +/// positives), so the client must re-apply exact filtering after decode; on the live tail the +/// same filters are forwarded to the server for pruning, but this matcher remains the +/// delivery authority. +/// +/// +/// Presentation contract (matching the server's wire policy): +/// +/// Kind and DID filters apply independently to all events. +/// With a collection filter set, only commit events whose collection matches are +/// delivered — but #account, #identity, and #sync always bypass the collection filter (subject to +/// the DID filter), because they are the consumer's only signal to purge a dead account's records. +/// The seq window is the exact (afterSeq, beforeSeq] bound, applied on top of the +/// planner's coarse pruning. +/// +/// +internal sealed class JetstreamV2Matcher +{ + private readonly HashSet? _kinds; + private readonly HashSet? _dids; + private readonly HashSet? _fullPaths; + private readonly List? _prefixes; + private readonly long? _beforeSeq; + private long _afterSeq; + + public JetstreamV2Matcher(JetstreamV2SubscribeOptions options) + { + _afterSeq = options.AfterSeq ?? 0; + _beforeSeq = options.BeforeSeq; + + if (options.Kinds is { Count: > 0 }) + { + _kinds = new HashSet(options.Kinds); + } + + if (options.Dids is { Count: > 0 }) + { + _dids = new HashSet(options.Dids); + } + + if (options.Collections is { Count: > 0 }) + { + foreach (var collection in options.Collections) + { + if (collection.EndsWith(".*", System.StringComparison.Ordinal)) + { + // Trim only the trailing "*", keeping the dot, so "app.bsky.feed.*" + // matches "app.bsky.feed."-prefixed NSIDs. + _prefixes ??= new List(); + _prefixes.Add(collection.Substring(0, collection.Length - 1)); + } + else + { + _fullPaths ??= new HashSet(); + _fullPaths.Add(collection); + } + } + } + } + + /// + /// Reports whether a stored archive row passes the exact filters. Runs before the + /// expensive record decode so filtered rows are never materialized. + /// + public bool WantsRow(JetstreamSegmentRow row) => + Wants(row.Seq, row.Did, PublicKind(row.Kind), row.Collection); + + /// + /// Reports whether a decoded live event passes the exact filters. + /// + public bool WantsEvent(JetstreamV2Event evt) + { + var collection = evt.Kind == JetstreamV2EventKind.Commit ? evt.Commit?.Collection ?? string.Empty : string.Empty; + return Wants(evt.Seq, evt.Did, evt.Kind, collection); + } + + /// + /// Raises the exclusive lower seq bound. Used on a re-backfill after a CursorTooOld + /// rejection: the one plan unit that straddles the resume point is admitted whole under + /// the planner's one-sided contract, and the raised floor drops its already-delivered rows + /// before decode. The bound only ever moves forward. + /// + public void SetAfterSeq(long afterSeq) => _afterSeq = afterSeq; + + private bool Wants(long seq, string did, JetstreamV2EventKind? kind, string collection) + { + if (!WantsSeq(seq)) + { + return false; + } + + if (kind == null) + { + return false; + } + + if (_kinds != null && !_kinds.Contains(kind.Value)) + { + return false; + } + + if (_dids != null && !_dids.Contains(did)) + { + return false; + } + + if (_fullPaths == null && _prefixes == null) + { + return true; + } + + // DID-level events carry no collection and always bypass the collection filter, + // subject to the DID filter applied above. + if (kind != JetstreamV2EventKind.Commit) + { + return true; + } + + // A commit lacking a collection bypasses the filter (wire parity). + if (collection.Length == 0) + { + return true; + } + + if (_fullPaths != null && _fullPaths.Contains(collection)) + { + return true; + } + + if (_prefixes != null) + { + foreach (var prefix in _prefixes) + { + if (collection.StartsWith(prefix, System.StringComparison.Ordinal)) + { + return true; + } + } + } + + return false; + } + + private bool WantsSeq(long seq) + { + // afterSeq is a resume-after bound (seq > afterSeq), but only when one was actually + // requested: 0 means "from the start of the archive" and seqs start at 1, so it + // imposes no lower bound (matching the server). + if (_afterSeq > 0 && seq <= _afterSeq) + { + return false; + } + + if (_beforeSeq != null && seq > _beforeSeq.Value) + { + return false; + } + + return true; + } + + private static JetstreamV2EventKind? PublicKind(JetstreamSegmentRowKind kind) + { + switch (kind) + { + case JetstreamSegmentRowKind.Create: + case JetstreamSegmentRowKind.Update: + case JetstreamSegmentRowKind.Delete: + case JetstreamSegmentRowKind.CreateResync: + return JetstreamV2EventKind.Commit; + case JetstreamSegmentRowKind.Identity: + return JetstreamV2EventKind.Identity; + case JetstreamSegmentRowKind.Account: + return JetstreamV2EventKind.Account; + case JetstreamSegmentRowKind.Sync: + return JetstreamV2EventKind.Sync; + default: + return null; + } + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2PlanConverter.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2PlanConverter.cs new file mode 100644 index 0000000..c2ce876 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2PlanConverter.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; + +namespace CarpaNet.Jetstream; + +/// +/// Converts and validates planSnapshot wire responses into the public plan model. The wire +/// fields come from the server, so every range is checked before use. +/// +internal static class JetstreamV2PlanConverter +{ + public static JetstreamSnapshotPlan Convert(JetstreamV2PlanSnapshotOutput output) + { + if (output.PlannedThroughSeq < 0 || output.SealedTipSeq < 0) + { + throw new JetstreamV2Exception("planSnapshot returned a negative seq"); + } + + if (output.PlannedThroughSeq > output.SealedTipSeq) + { + throw new JetstreamV2Exception( + $"planSnapshot plannedThroughSeq {output.PlannedThroughSeq} exceeds sealedTipSeq {output.SealedTipSeq}"); + } + + var segments = new List(output.Segments?.Count ?? 0); + if (output.Segments != null) + { + foreach (var dto in output.Segments) + { + segments.Add(ConvertSegment(dto)); + } + } + + return new JetstreamSnapshotPlan + { + PlannedThroughSeq = output.PlannedThroughSeq, + SealedTipSeq = output.SealedTipSeq, + Segments = segments, + }; + } + + private static JetstreamPlannedSegment ConvertSegment(JetstreamV2PlanSegmentDto dto) + { + if (string.IsNullOrEmpty(dto.Name)) + { + throw new JetstreamV2Exception($"planSnapshot segment missing name (index {dto.Index})"); + } + + if (dto.Index < 0 || dto.Index > int.MaxValue || dto.MinSeq < 0 || dto.MaxSeq < dto.MinSeq) + { + throw new JetstreamV2Exception($"planSnapshot segment \"{dto.Name}\" has an invalid index or seq range"); + } + + var segment = new JetstreamPlannedSegment + { + Name = dto.Name!, + Index = (int)dto.Index, + Checksum = dto.Checksum ?? string.Empty, + MinSeq = dto.MinSeq, + MaxSeq = dto.MaxSeq, + }; + + switch (dto.Mode) + { + case "segment": + segment.Mode = JetstreamSegmentPlanMode.WholeSegment; + break; + case "blocks": + segment.Mode = JetstreamSegmentPlanMode.Blocks; + if (dto.Blocks == null || dto.Blocks.Count == 0) + { + throw new JetstreamV2Exception($"planSnapshot segment \"{dto.Name}\" has mode=blocks but no block ranges"); + } + + var ranges = new List(dto.Blocks.Count); + foreach (var range in dto.Blocks) + { + if (range.First < 0 || range.Last < range.First || range.Last > int.MaxValue) + { + throw new JetstreamV2Exception( + $"planSnapshot segment \"{dto.Name}\" has invalid block range [{range.First},{range.Last}]"); + } + + ranges.Add(new JetstreamBlockRange { First = (int)range.First, Last = (int)range.Last }); + } + + segment.Blocks = ranges; + break; + default: + throw new JetstreamV2Exception($"planSnapshot segment \"{dto.Name}\" has unknown mode \"{dto.Mode}\""); + } + + return segment; + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2RecordDecoder.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2RecordDecoder.cs new file mode 100644 index 0000000..de90b0d --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2RecordDecoder.cs @@ -0,0 +1,319 @@ +using System; +using System.Formats.Cbor; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using CarpaNet.Cbor; + +namespace CarpaNet.Jetstream; + +/// +/// Converts decoded sealed-segment rows into values: commit +/// payloads are DAG-CBOR records converted to the atproto JSON data model (the same shape the +/// live wire carries in "record"), and marker rows wrap the upstream +/// com.atproto.sync.subscribeRepos event's DAG-CBOR. +/// +internal static class JetstreamV2RecordDecoder +{ + /// + /// Converts one archive row into the public event shape. Throws + /// for a semantically malformed row; the caller logs + /// and skips it so one bad upstream record does not lose the rest of the block. + /// + public static JetstreamV2Event ConvertRow(JetstreamSegmentRow row) + { + var evt = new JetstreamV2Event + { + Did = row.Did, + Seq = row.Seq, + TimeUs = row.DisplayTimeUs, + }; + + switch (row.Kind) + { + case JetstreamSegmentRowKind.Create: + case JetstreamSegmentRowKind.Update: + case JetstreamSegmentRowKind.Delete: + case JetstreamSegmentRowKind.CreateResync: + evt.Kind = JetstreamV2EventKind.Commit; + evt.Commit = ConvertCommit(row); + break; + case JetstreamSegmentRowKind.Identity: + evt.Kind = JetstreamV2EventKind.Identity; + evt.Identity = DecodeIdentity(row); + break; + case JetstreamSegmentRowKind.Account: + evt.Kind = JetstreamV2EventKind.Account; + evt.Account = DecodeAccount(row); + break; + case JetstreamSegmentRowKind.Sync: + evt.Kind = JetstreamV2EventKind.Sync; + evt.Sync = DecodeSync(row); + break; + default: + throw new JetstreamV2Exception($"unknown segment row kind {(int)row.Kind} (did={row.Did} seq={row.Seq})"); + } + + return evt; + } + + private static JetstreamV2Commit ConvertCommit(JetstreamSegmentRow row) + { + var commit = new JetstreamV2Commit + { + Operation = row.Kind switch + { + JetstreamSegmentRowKind.Update => JetstreamV2CommitOperation.Update, + JetstreamSegmentRowKind.Delete => JetstreamV2CommitOperation.Delete, + _ => JetstreamV2CommitOperation.Create, + }, + Collection = row.Collection, + Rkey = row.Rkey, + Rev = row.Rev, + }; + + if (row.Kind == JetstreamSegmentRowKind.Delete) + { + return commit; + } + + if (row.Payload == null || row.Payload.Length == 0) + { + throw new JetstreamV2Exception( + $"archive commit missing record payload (did={row.Did} collection={row.Collection} rkey={row.Rkey} seq={row.Seq})"); + } + + try + { + commit.Record = CborToJsonElement(row.Payload); + } + catch (Exception ex) when (ex is not JetstreamV2Exception) + { + throw new JetstreamV2Exception( + $"decode record (did={row.Did} collection={row.Collection} rkey={row.Rkey} seq={row.Seq}): {ex.Message}", ex); + } + + commit.Cid = ComputeCid(row.Payload); + return commit; + } + + /// + /// Computes the record's content identifier: the ATProto blessed CIDv1 + /// (dag-cbor, sha2-256, base32lower) over the raw DAG-CBOR payload. + /// + internal static string ComputeCid(byte[] payload) + { + using var sha = SHA256.Create(); + return ATCid.FromSha256Hash(sha.ComputeHash(payload)).Value; + } + + /// + /// Converts DAG-CBOR record bytes into the atproto JSON data model: integers stay JSON + /// numbers, byte strings become {"$bytes": base64-without-padding}, CID links become + /// {"$link": cid}. Floats are rejected — the atproto data model has integers, not + /// floats — and the top-level value must be a map. + /// + internal static JsonElement CborToJsonElement(byte[] payload) + { + var reader = new DagCborReader(payload); + if (reader.PeekState() != CborReaderState.StartMap) + { + throw new JetstreamV2Exception("record is not a CBOR map"); + } + + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + WriteValue(ref reader, writer); + } + + if (reader.BytesRemaining != 0) + { + throw new JetstreamV2Exception("record has trailing bytes after the CBOR value"); + } + + stream.Position = 0; + using var document = JsonDocument.Parse(stream); + return document.RootElement.Clone(); + } + + private static void WriteValue(ref DagCborReader reader, Utf8JsonWriter writer) + { + switch (reader.PeekState()) + { + case CborReaderState.Null: + reader.ReadNull(); + writer.WriteNullValue(); + break; + case CborReaderState.Boolean: + writer.WriteBooleanValue(reader.ReadBoolean()); + break; + case CborReaderState.UnsignedInteger: + case CborReaderState.NegativeInteger: + writer.WriteNumberValue(reader.ReadInt64()); + break; + case CborReaderState.TextString: + writer.WriteStringValue(reader.ReadTextString()); + break; + case CborReaderState.ByteString: + writer.WriteStartObject(); + writer.WriteString("$bytes", ToBase64NoPadding(reader.ReadByteString())); + writer.WriteEndObject(); + break; + case CborReaderState.Tag: + var cid = reader.ReadCidLink(); + writer.WriteStartObject(); + writer.WriteString("$link", cid.Value); + writer.WriteEndObject(); + break; + case CborReaderState.StartArray: + writer.WriteStartArray(); + var arrayCount = reader.ReadStartArray(); + var remainingItems = arrayCount ?? int.MaxValue; + while (remainingItems > 0 && reader.PeekState() != CborReaderState.EndArray) + { + WriteValue(ref reader, writer); + remainingItems--; + } + + reader.ReadEndArray(); + writer.WriteEndArray(); + break; + case CborReaderState.StartMap: + writer.WriteStartObject(); + var mapCount = reader.ReadStartMap(); + var remainingPairs = mapCount ?? int.MaxValue; + while (remainingPairs > 0 && reader.PeekState() != CborReaderState.EndMap) + { + writer.WritePropertyName(reader.ReadTextString()); + WriteValue(ref reader, writer); + remainingPairs--; + } + + reader.ReadEndMap(); + writer.WriteEndObject(); + break; + default: + // Floats and anything else are outside the atproto data model. + throw new JetstreamV2Exception($"record contains a value outside the atproto data model ({reader.PeekState()})"); + } + } + + private static string ToBase64NoPadding(byte[] bytes) => + Convert.ToBase64String(bytes).TrimEnd('='); + + private static JetstreamV2Identity DecodeIdentity(JetstreamSegmentRow row) + { + var fields = DecodeUpstreamFields(row, "identity"); + return new JetstreamV2Identity + { + Did = fields.Did ?? row.Did, + Handle = fields.Handle, + Seq = fields.Seq, + Time = fields.Time ?? string.Empty, + }; + } + + private static JetstreamV2Account DecodeAccount(JetstreamSegmentRow row) + { + var fields = DecodeUpstreamFields(row, "account"); + return new JetstreamV2Account + { + Did = fields.Did ?? row.Did, + Active = fields.Active, + Status = fields.Status, + Seq = fields.Seq, + Time = fields.Time ?? string.Empty, + }; + } + + private static JetstreamV2Sync DecodeSync(JetstreamSegmentRow row) + { + var fields = DecodeUpstreamFields(row, "sync"); + return new JetstreamV2Sync + { + Did = fields.Did ?? row.Did, + Rev = fields.Rev ?? string.Empty, + Seq = fields.Seq, + Time = fields.Time ?? string.Empty, + }; + } + + private struct UpstreamFields + { + public string? Did; + public string? Handle; + public string? Status; + public string? Rev; + public string? Time; + public long Seq; + public bool Active; + } + + /// + /// Decodes the payload of a marker row: the upstream com.atproto.sync.subscribeRepos + /// event as a DAG-CBOR map. Unknown keys (e.g. a #sync event's raw "blocks" bytes) are + /// skipped for forward compatibility. + /// + private static UpstreamFields DecodeUpstreamFields(JetstreamSegmentRow row, string kindName) + { + if (row.Payload == null || row.Payload.Length == 0) + { + throw new JetstreamV2Exception($"archive {kindName} row missing payload (did={row.Did} seq={row.Seq})"); + } + + var fields = default(UpstreamFields); + try + { + var reader = new DagCborReader(row.Payload); + var pairCount = reader.ReadStartMap(); + var remaining = pairCount ?? int.MaxValue; + while (remaining > 0 && reader.PeekState() != CborReaderState.EndMap) + { + var key = reader.ReadTextString(); + remaining--; + if (reader.PeekState() == CborReaderState.Null) + { + reader.ReadNull(); + continue; + } + + switch (key) + { + case "did": + fields.Did = reader.ReadTextString(); + break; + case "handle": + fields.Handle = reader.ReadTextString(); + break; + case "status": + fields.Status = reader.ReadTextString(); + break; + case "rev": + fields.Rev = reader.ReadTextString(); + break; + case "time": + fields.Time = reader.ReadTextString(); + break; + case "seq": + fields.Seq = reader.ReadInt64(); + break; + case "active": + fields.Active = reader.ReadBoolean(); + break; + default: + reader.SkipValue(); + break; + } + } + + reader.ReadEndMap(); + } + catch (Exception ex) when (ex is not JetstreamV2Exception) + { + throw new JetstreamV2Exception($"decode archive {kindName} payload (did={row.Did} seq={row.Seq}): {ex.Message}", ex); + } + + return fields; + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2SubscribeOptions.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2SubscribeOptions.cs new file mode 100644 index 0000000..2946774 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2SubscribeOptions.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; + +namespace CarpaNet.Jetstream; + +/// +/// Options for . The kind, DID, and collection +/// filters are independent predicates ANDed together; each is match-all when omitted. +/// +/// +/// Stream modes, selected by the seq bounds: +/// +/// Pure live (default): no /. +/// Tails the live websocket from (or the current tip when null). +/// Backfill then live: set (0 replays the whole +/// archive). Sealed history is downloaded over HTTP, then the stream cuts over to the live +/// tail with no gap. +/// Snapshot only: set with a seq bound. +/// Downloads and delivers the matched sealed range, then the stream ends without dialing +/// the websocket. +/// +/// +public sealed class JetstreamV2SubscribeOptions +{ + internal const int MaxKinds = 4; + internal const int MaxCollections = 100; + internal const int MaxDids = 10000; + + /// + /// Event kinds to receive. Null or empty means all kinds. Combine + /// with for a + /// commits-only collection stream. + /// + public IReadOnlyList? Kinds { get; set; } + + /// + /// Collection NSIDs or namespace wildcards ending in ".*" (e.g. "app.bsky.feed.*"), + /// max 100 entries. Constrains commit events only: #identity, #account, and #sync events + /// always bypass the collection filter (subject to ), because they are a + /// folding consumer's only signal to purge a deleted account's records. + /// + public IReadOnlyList? Collections { get; set; } + + /// + /// Repo DIDs to receive events for, max 10,000 entries. Applies to every event kind. + /// Null or empty means all repos. + /// + public IReadOnlyList? Dids { get; set; } + + /// + /// Exclusive lower sequence bound for an archive replay: only events with seq > AfterSeq + /// are delivered. Setting it (including 0 for the whole archive) starts with sealed-history + /// replay before the client cuts over to the live tail. Null means no backfill. + /// + public long? AfterSeq { get; set; } + + /// + /// Inclusive upper sequence bound for an archive snapshot: only events with + /// seq <= BeforeSeq are delivered. Requires — on a replay that + /// continues into the live tail the same bound would silently drop every later live event. + /// + public long? BeforeSeq { get; set; } + + /// + /// Turns a replay into a point-in-time archive snapshot: the matched sealed range is + /// downloaded and delivered, then the stream ends without starting the live tail. Requires + /// and/or . Records in the active, unsealed + /// segment (above the sealed tip) are only reachable via the live tail and are not included. + /// + public bool SnapshotOnly { get; set; } + + /// + /// Resumes a pure live tail from a previously saved cursor (typically the last delivered + /// ). The server replays inclusively and the client + /// deduplicates the overlap. Null starts at the current live tip; 0 requests replay from + /// the first retained event. Ignored when an archive replay is requested via + /// /, since that workflow computes its own + /// live cutover cursor. + /// + public long? LiveCursor { get; set; } + + /// + /// Asks the server to skip live events whose uncompressed frame (envelope included) exceeds + /// this many bytes. Null or 0 means no limit. + /// + public long? MaxMessageSizeBytes { get; set; } + + /// + /// Whether the caller asked for historical archive replay (any seq bound) versus a pure + /// live tail. + /// + internal bool BackfillRequested => AfterSeq != null || BeforeSeq != null; + + /// + /// Validates the option combination and filter limits, throwing + /// on an invalid configuration. + /// + internal void Validate() + { + if (BeforeSeq != null && !SnapshotOnly) + { + throw new ArgumentException( + $"{nameof(BeforeSeq)} requires {nameof(SnapshotOnly)}: on a replay that continues into the live tail, " + + "an upper bound would silently drop every later live event."); + } + + if (SnapshotOnly && AfterSeq == null && BeforeSeq == null) + { + throw new ArgumentException( + $"{nameof(SnapshotOnly)} requires a replay bound ({nameof(AfterSeq)} and/or {nameof(BeforeSeq)})."); + } + + if (AfterSeq is < 0) + { + throw new ArgumentException($"{nameof(AfterSeq)} must be non-negative."); + } + + if (BeforeSeq is < 0) + { + throw new ArgumentException($"{nameof(BeforeSeq)} must be non-negative."); + } + + if (LiveCursor is < 0) + { + throw new ArgumentException($"{nameof(LiveCursor)} must be non-negative."); + } + + if (MaxMessageSizeBytes is < 0) + { + throw new ArgumentException($"{nameof(MaxMessageSizeBytes)} must be non-negative."); + } + + if (Kinds != null && Kinds.Count > MaxKinds) + { + throw new ArgumentException($"{nameof(Kinds)} allows at most {MaxKinds} entries."); + } + + if (Dids != null && Dids.Count > MaxDids) + { + throw new ArgumentException($"{nameof(Dids)} allows at most {MaxDids} entries."); + } + + if (Collections != null) + { + if (Collections.Count > MaxCollections) + { + throw new ArgumentException($"{nameof(Collections)} allows at most {MaxCollections} entries."); + } + + foreach (var collection in Collections) + { + if (string.IsNullOrEmpty(collection) || collection == ".*" || collection == "*") + { + throw new ArgumentException( + $"{nameof(Collections)} entries must be exact NSIDs or namespace wildcards like \"app.bsky.feed.*\"; got \"{collection}\"."); + } + } + } + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Sync.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Sync.cs new file mode 100644 index 0000000..c0bd4a2 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Sync.cs @@ -0,0 +1,28 @@ +namespace CarpaNet.Jetstream; + +/// +/// A #sync event: the upstream signaled a repo divergence requiring a resync. The +/// authoritative replacement records follow as their own commit events. +/// +public sealed class JetstreamV2Sync +{ + /// + /// The DID of the diverged repo. + /// + public string Did { get; set; } = string.Empty; + + /// + /// The repo revision of the resync point. + /// + public string Rev { get; set; } = string.Empty; + + /// + /// The upstream relay sequence number carried by the event (not Jetstream's seq). + /// + public long Seq { get; set; } + + /// + /// The RFC 3339 timestamp from the upstream event. + /// + public string Time { get; set; } = string.Empty; +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2Transport.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2Transport.cs new file mode 100644 index 0000000..74d76ef --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2Transport.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace CarpaNet.Jetstream; + +/// +/// HTTP transport for the Jetstream v2 archive XRPC endpoints (planSnapshot, getSegment, +/// getBlock) and the public dictionary fetch. Applies bearer authentication to the archive +/// endpoints only, retries transient failures with exponential backoff, and converts XRPC +/// error envelopes into values with structured error names. +/// +internal sealed class JetstreamV2Transport +{ + internal const string PlanSnapshotNsid = "network.bsky.jetstream.planSnapshot"; + internal const string GetSegmentNsid = "network.bsky.jetstream.getSegment"; + internal const string GetBlockNsid = "network.bsky.jetstream.getBlock"; + internal const string GetZstdDictionaryNsid = "network.bsky.jetstream.getZstdDictionary"; + internal const string SubscribeEventsNsid = "network.bsky.jetstream.subscribeEvents"; + + // Bounds a single segment/block allocation: a corrupt or hostile Content-Length must not + // make the client allocate unbounded memory. Sealed segments are ~256-280 MB. + private const long MaxDownloadBytes = 1L << 30; + + private static readonly TimeSpan RetryBackoffBase = TimeSpan.FromMilliseconds(500); + private static readonly TimeSpan RetryBackoffMax = TimeSpan.FromSeconds(5); + + private readonly Uri _baseUri; + private readonly HttpClient _httpClient; + private readonly string? _apiKey; + private readonly int _maxAttempts; + private readonly ILogger _logger; + + public JetstreamV2Transport(Uri baseUri, HttpClient httpClient, string? apiKey, int maxAttempts, ILogger logger) + { + _baseUri = baseUri; + _httpClient = httpClient; + _apiKey = apiKey; + _maxAttempts = maxAttempts; + _logger = logger; + } + + /// + /// Builds an absolute /xrpc/<nsid> URI with optional query parameters. + /// + internal static Uri BuildXrpcUri(Uri baseUri, string nsid, IEnumerable>? query = null) + { + var builder = new UriBuilder(baseUri) + { + Path = "/xrpc/" + nsid, + }; + + if (query != null) + { + var parts = new List(); + foreach (var pair in query) + { + parts.Add($"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}"); + } + + if (parts.Count > 0) + { + builder.Query = string.Join("&", parts); + } + } + + return builder.Uri; + } + + public async Task PlanSnapshotAsync(JetstreamV2PlanSnapshotInput input, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(input, JetstreamV2JsonContext.Default.JetstreamV2PlanSnapshotInput); + var body = await SendAsync( + () => + { + var request = new HttpRequestMessage(HttpMethod.Post, BuildXrpcUri(_baseUri, PlanSnapshotNsid)) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + return request; + }, + authenticated: true, + cancellationToken).ConfigureAwait(false); + + JetstreamV2PlanSnapshotOutput? output; + try + { + output = JsonSerializer.Deserialize(body, JetstreamV2JsonContext.Default.JetstreamV2PlanSnapshotOutput); + } + catch (JsonException ex) + { + throw new JetstreamV2Exception($"planSnapshot returned invalid JSON: {ex.Message}", ex); + } + + if (output == null) + { + throw new JetstreamV2Exception("planSnapshot returned an empty response"); + } + + return output; + } + + public Task GetSegmentAsync(string name, string? expectedChecksum, CancellationToken cancellationToken) + { + return SendAsync( + () => new HttpRequestMessage( + HttpMethod.Get, + BuildXrpcUri(_baseUri, GetSegmentNsid, new[] { new KeyValuePair("name", name) })), + authenticated: true, + cancellationToken, + expectedChecksum); + } + + public Task GetBlockAsync(string segment, long blockIndex, CancellationToken cancellationToken) + { + return SendAsync( + () => new HttpRequestMessage( + HttpMethod.Get, + BuildXrpcUri(_baseUri, GetBlockNsid, new[] + { + new KeyValuePair("segment", segment), + new KeyValuePair("blockIndex", blockIndex.ToString(CultureInfo.InvariantCulture)), + })), + authenticated: true, + cancellationToken); + } + + public Task GetZstdDictionaryAsync(long? id, CancellationToken cancellationToken) + { + var query = id == null + ? null + : new[] { new KeyValuePair("id", id.Value.ToString(CultureInfo.InvariantCulture)) }; + + // The dictionary is public: the archive credential is deliberately never sent here. + return SendAsync( + () => new HttpRequestMessage(HttpMethod.Get, BuildXrpcUri(_baseUri, GetZstdDictionaryNsid, query)), + authenticated: false, + cancellationToken); + } + + /// + /// Classifies a failed websocket handshake by re-issuing the subscribe request as a plain + /// HTTP GET: the server validates parameters before upgrading, so an invalid cursor, + /// dictionary, or filter yields the same pre-upgrade XRPC error envelope. Returns null when + /// the failure cannot be classified (treated as transient by the caller). + /// + public async Task ProbeHandshakeErrorAsync(Uri subscribeHttpUri, CancellationToken cancellationToken) + { + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, subscribeHttpUri); + using var response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.BadRequest && + response.StatusCode != HttpStatusCode.ServiceUnavailable) + { + return null; + } + + var body = await ReadBoundedAsync(response, 4096, cancellationToken).ConfigureAwait(false); + return ParseErrorEnvelope(body); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException) + { + return null; + } + } + + private async Task SendAsync( + Func requestFactory, + bool authenticated, + CancellationToken cancellationToken, + string? expectedChecksum = null) + { + var backoff = RetryBackoffBase; + for (var attempt = 1; ; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + Exception? transient = null; + try + { + using var request = requestFactory(); + if (authenticated && !string.IsNullOrEmpty(_apiKey)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey); + } + + using var response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + VerifyChecksumHeader(response, expectedChecksum, request.RequestUri); + return await ReadBoundedAsync(response, MaxDownloadBytes, cancellationToken).ConfigureAwait(false); + } + + var status = (int)response.StatusCode; + var errorBody = await ReadBoundedAsync(response, 4096, cancellationToken).ConfigureAwait(false); + var envelope = ParseErrorEnvelope(errorBody); + if (status >= 500 || status == 429 || envelope?.Error == JetstreamV2ErrorNames.ServiceUnavailable) + { + transient = new JetstreamV2Exception( + $"HTTP {status} from {request.RequestUri}: {envelope?.Error ?? Truncate(errorBody)}", + envelope?.Error); + } + else + { + throw new JetstreamV2Exception( + envelope?.Error != null + ? $"{envelope.Error}: {envelope.Message ?? request.RequestUri!.ToString()}" + : $"HTTP {status} from {request.RequestUri}: {Truncate(errorBody)}", + envelope?.Error); + } + } + catch (HttpRequestException ex) + { + transient = ex; + } + catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + // HttpClient timeout, not a caller cancellation. + transient = ex; + } + + if (attempt >= _maxAttempts) + { + throw transient as JetstreamV2Exception + ?? new JetstreamV2Exception($"request failed after {attempt} attempts: {transient!.Message}", transient); + } + + _logger.LogWarning("Jetstream request failed (attempt {Attempt}/{Max}): {Error}; retrying in {Delay}", attempt, _maxAttempts, transient!.Message, backoff); + await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); + backoff = backoff.TotalMilliseconds * 2 > RetryBackoffMax.TotalMilliseconds + ? RetryBackoffMax + : TimeSpan.FromMilliseconds(backoff.TotalMilliseconds * 2); + } + } + + /// + /// A downloaded segment's ETag equals its plan checksum (the segment-generation pin). + /// A mismatch means a compaction rewrote the file between planning and download; the rows + /// are still valid, so log rather than fail. + /// + private void VerifyChecksumHeader(HttpResponseMessage response, string? expectedChecksum, Uri? requestUri) + { + if (string.IsNullOrEmpty(expectedChecksum)) + { + return; + } + + var etag = response.Headers.ETag?.Tag; + if (etag != null && etag.IndexOf(expectedChecksum!, StringComparison.OrdinalIgnoreCase) < 0) + { + _logger.LogWarning( + "Segment generation changed between plan and download ({Uri}): planned checksum {Checksum}, got ETag {ETag}", + requestUri, expectedChecksum, etag); + } + } + + private static async Task ReadBoundedAsync(HttpResponseMessage response, long limit, CancellationToken cancellationToken) + { + var contentLength = response.Content?.Headers.ContentLength; + if (contentLength > limit) + { + throw new JetstreamV2Exception($"response of {contentLength} bytes exceeds the {limit} byte cap"); + } + + if (response.Content == null) + { + return Array.Empty(); + } + + using var source = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + using var buffer = contentLength is > 0 and <= int.MaxValue + ? new MemoryStream((int)contentLength.Value) + : new MemoryStream(); + var chunk = new byte[81920]; + int read; + while ((read = await source.ReadAsync(chunk, 0, chunk.Length, cancellationToken).ConfigureAwait(false)) > 0) + { + if (buffer.Length + read > limit) + { + throw new JetstreamV2Exception($"response exceeded the {limit} byte cap"); + } + + buffer.Write(chunk, 0, read); + } + + return buffer.ToArray(); + } + + private static JetstreamV2XrpcError? ParseErrorEnvelope(byte[] body) + { + if (body.Length == 0) + { + return null; + } + + try + { + var envelope = JsonSerializer.Deserialize(body, JetstreamV2JsonContext.Default.JetstreamV2XrpcError); + return string.IsNullOrEmpty(envelope?.Error) ? null : envelope; + } + catch (JsonException) + { + return null; + } + } + + private static string Truncate(byte[] body) + { + var text = Encoding.UTF8.GetString(body, 0, Math.Min(body.Length, 256)); + return text.Replace('\n', ' ').Replace('\r', ' '); + } +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamV2WireModels.cs b/src/CarpaNet.Jetstream/V2/JetstreamV2WireModels.cs new file mode 100644 index 0000000..e2d9f39 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamV2WireModels.cs @@ -0,0 +1,262 @@ +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CarpaNet.Jetstream; + +/// +/// The xrpc.v1.json frame envelope (atproto proposal 0015): exactly one self-describing +/// object per text frame, discriminated by $type ("message" or "error"). +/// +internal sealed class JetstreamV2Envelope +{ + [JsonPropertyName("$type")] + public string? Type { get; set; } + + /// Message frames: the payload union, dispatched on its own $type. + [JsonPropertyName("payload")] + public JsonElement? Payload { get; set; } + + /// Error frames: the bare error type name. + [JsonPropertyName("error")] + public string? Error { get; set; } + + /// Error frames: optional description. + [JsonPropertyName("message")] + public string? Message { get; set; } +} + +/// network.bsky.jetstream.subscribeEvents#commit wire payload. +internal sealed class JetstreamV2CommitPayload +{ + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } + + [JsonPropertyName("rev")] + public string? Rev { get; set; } + + [JsonPropertyName("operation")] + public string? Operation { get; set; } + + [JsonPropertyName("collection")] + public string? Collection { get; set; } + + [JsonPropertyName("rkey")] + public string? Rkey { get; set; } + + [JsonPropertyName("cid")] + public string? Cid { get; set; } + + [JsonPropertyName("record")] + public JsonElement? Record { get; set; } +} + +/// The upstream com.atproto.sync.subscribeRepos#identity event, wrapped verbatim. +internal sealed class JetstreamV2UpstreamIdentity +{ + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("handle")] + public string? Handle { get; set; } + + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } +} + +/// network.bsky.jetstream.subscribeEvents#identity wire payload. +internal sealed class JetstreamV2IdentityPayload +{ + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } + + [JsonPropertyName("identity")] + public JetstreamV2UpstreamIdentity? Identity { get; set; } +} + +/// The upstream com.atproto.sync.subscribeRepos#account event, wrapped verbatim. +internal sealed class JetstreamV2UpstreamAccount +{ + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("active")] + public bool Active { get; set; } + + [JsonPropertyName("status")] + public string? Status { get; set; } + + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } +} + +/// network.bsky.jetstream.subscribeEvents#account wire payload. +internal sealed class JetstreamV2AccountPayload +{ + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } + + [JsonPropertyName("account")] + public JetstreamV2UpstreamAccount? Account { get; set; } +} + +/// The upstream com.atproto.sync.subscribeRepos#sync event, wrapped verbatim. +/// The raw MST block bytes are deliberately not surfaced. +internal sealed class JetstreamV2UpstreamSync +{ + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("rev")] + public string? Rev { get; set; } + + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } +} + +/// network.bsky.jetstream.subscribeEvents#sync wire payload. +internal sealed class JetstreamV2SyncPayload +{ + [JsonPropertyName("seq")] + public long Seq { get; set; } + + [JsonPropertyName("did")] + public string? Did { get; set; } + + [JsonPropertyName("time")] + public string? Time { get; set; } + + [JsonPropertyName("sync")] + public JetstreamV2UpstreamSync? Sync { get; set; } +} + +/// network.bsky.jetstream.subscribeEvents#info advisory payload (no seq, not an event). +internal sealed class JetstreamV2InfoPayload +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } +} + +/// The standard XRPC JSON error envelope returned on pre-upgrade rejections. +internal sealed class JetstreamV2XrpcError +{ + [JsonPropertyName("error")] + public string? Error { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } +} + +/// network.bsky.jetstream.planSnapshot input body. +internal sealed class JetstreamV2PlanSnapshotInput +{ + [JsonPropertyName("kinds")] + public List? Kinds { get; set; } + + [JsonPropertyName("dids")] + public List? Dids { get; set; } + + [JsonPropertyName("collections")] + public List? Collections { get; set; } + + [JsonPropertyName("afterSeq")] + public long? AfterSeq { get; set; } + + [JsonPropertyName("beforeSeq")] + public long? BeforeSeq { get; set; } +} + +/// network.bsky.jetstream.planSnapshot output body. +internal sealed class JetstreamV2PlanSnapshotOutput +{ + [JsonPropertyName("plannedThroughSeq")] + public long PlannedThroughSeq { get; set; } + + [JsonPropertyName("sealedTipSeq")] + public long SealedTipSeq { get; set; } + + [JsonPropertyName("segments")] + public List? Segments { get; set; } +} + +/// One planned segment in a planSnapshot response. +internal sealed class JetstreamV2PlanSegmentDto +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("index")] + public long Index { get; set; } + + [JsonPropertyName("checksum")] + public string? Checksum { get; set; } + + [JsonPropertyName("minSeq")] + public long MinSeq { get; set; } + + [JsonPropertyName("maxSeq")] + public long MaxSeq { get; set; } + + [JsonPropertyName("mode")] + public string? Mode { get; set; } + + [JsonPropertyName("blocks")] + public List? Blocks { get; set; } +} + +/// One inclusive block range in a planned blocks-mode segment. +internal sealed class JetstreamV2PlanBlockRangeDto +{ + [JsonPropertyName("first")] + public long First { get; set; } + + [JsonPropertyName("last")] + public long Last { get; set; } +} + +/// +/// Source-generated JSON context for the Jetstream v2 wire types. +/// +[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(JetstreamV2Envelope))] +[JsonSerializable(typeof(JetstreamV2CommitPayload))] +[JsonSerializable(typeof(JetstreamV2IdentityPayload))] +[JsonSerializable(typeof(JetstreamV2AccountPayload))] +[JsonSerializable(typeof(JetstreamV2SyncPayload))] +[JsonSerializable(typeof(JetstreamV2InfoPayload))] +[JsonSerializable(typeof(JetstreamV2XrpcError))] +[JsonSerializable(typeof(JetstreamV2PlanSnapshotInput))] +[JsonSerializable(typeof(JetstreamV2PlanSnapshotOutput))] +internal partial class JetstreamV2JsonContext : JsonSerializerContext +{ +} diff --git a/src/CarpaNet.Jetstream/V2/JetstreamZstdDictionary.cs b/src/CarpaNet.Jetstream/V2/JetstreamZstdDictionary.cs new file mode 100644 index 0000000..63746f2 --- /dev/null +++ b/src/CarpaNet.Jetstream/V2/JetstreamZstdDictionary.cs @@ -0,0 +1,35 @@ +using System.Buffers.Binary; + +namespace CarpaNet.Jetstream; + +/// +/// Parses the header of a structured zstd dictionary (RFC 8878 §5). The client fetches the +/// dictionary blob via getZstdDictionary and needs its embedded ID for the zstdDictionary +/// websocket negotiation parameter. +/// +internal static class JetstreamZstdDictionary +{ + /// The structured-dictionary magic number (RFC 8878 §5), little-endian. + private const uint Magic = 0xEC30A437; + + /// + /// Extracts the dictionary ID from a structured zstd dictionary blob. Returns false for + /// raw/content-only dictionaries (no header), which the Jetstream wire contract does not use. + /// + public static bool TryParseId(byte[]? dictionary, out uint id) + { + id = 0; + if (dictionary == null || dictionary.Length < 8) + { + return false; + } + + if (BinaryPrimitives.ReadUInt32LittleEndian(dictionary.AsSpan(0, 4)) != Magic) + { + return false; + } + + id = BinaryPrimitives.ReadUInt32LittleEndian(dictionary.AsSpan(4, 4)); + return id != 0; + } +} diff --git a/src/CarpaNet/BlueskyServices.cs b/src/CarpaNet/BlueskyServices.cs index dab0db0..d33ef67 100644 --- a/src/CarpaNet/BlueskyServices.cs +++ b/src/CarpaNet/BlueskyServices.cs @@ -70,4 +70,9 @@ public static class BlueskyServices /// Bluesky Jetstream instance 2, US-West. /// public const string Jetstream2UsWest = "https://jetstream2.us-west.bsky.network"; + + /// + /// Bluesky Jetstream v2 service, US-East. + /// + public const string JetstreamUsEast = "https://jetstream.us-east.bsky.network"; } diff --git a/tests/CarpaNet.UnitTests/CarpaNet.UnitTests.csproj b/tests/CarpaNet.UnitTests/CarpaNet.UnitTests.csproj index a6d30d4..1608792 100644 --- a/tests/CarpaNet.UnitTests/CarpaNet.UnitTests.csproj +++ b/tests/CarpaNet.UnitTests/CarpaNet.UnitTests.csproj @@ -24,6 +24,7 @@ + diff --git a/tests/CarpaNet.UnitTests/Jetstream/JetstreamSegmentFormatTests.cs b/tests/CarpaNet.UnitTests/Jetstream/JetstreamSegmentFormatTests.cs new file mode 100644 index 0000000..9f15fa4 --- /dev/null +++ b/tests/CarpaNet.UnitTests/Jetstream/JetstreamSegmentFormatTests.cs @@ -0,0 +1,372 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using CarpaNet.Jetstream; +using Xunit; +using ZstdSharp; + +namespace CarpaNet.UnitTests.Jetstream; + +/// +/// Encodes the sealed-segment wire format from the writer's side (a port of the reference +/// implementation's columnar layout) so the decoder can be exercised without fixture files. +/// +internal static class JetstreamSegmentTestData +{ + public static byte[] EncodeBlockBody(IReadOnlyList rows) + { + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream); + writer.Write((uint)rows.Count); + + foreach (var row in rows) + { + writer.Write((ulong)row.Seq); + } + + foreach (var row in rows) + { + writer.Write((ulong)row.WitnessedAt); + } + + foreach (var row in rows) + { + writer.Write((ulong)row.IndexedAt); + } + + foreach (var row in rows) + { + writer.Write((byte)row.Kind); + } + + foreach (var row in rows) + { + writer.Write((byte)Encoding.UTF8.GetByteCount(row.Collection)); + } + + foreach (var row in rows) + { + writer.Write((ushort)Encoding.UTF8.GetByteCount(row.Did)); + } + + foreach (var row in rows) + { + writer.Write((byte)Encoding.UTF8.GetByteCount(row.Rkey)); + } + + foreach (var row in rows) + { + writer.Write((byte)Encoding.UTF8.GetByteCount(row.Rev)); + } + + foreach (var row in rows) + { + writer.Write((uint)(row.Payload?.Length ?? 0)); + } + + foreach (var row in rows) + { + writer.Write(Encoding.UTF8.GetBytes(row.Collection)); + } + + foreach (var row in rows) + { + writer.Write(Encoding.UTF8.GetBytes(row.Did)); + } + + foreach (var row in rows) + { + writer.Write(Encoding.UTF8.GetBytes(row.Rkey)); + } + + foreach (var row in rows) + { + writer.Write(Encoding.UTF8.GetBytes(row.Rev)); + } + + foreach (var row in rows) + { + if (row.Payload != null) + { + writer.Write(row.Payload); + } + } + + writer.Flush(); + return stream.ToArray(); + } + + public static byte[] CompressFrame(byte[] body) + { + using var compressor = new Compressor(); + return compressor.Wrap(body).ToArray(); + } + + public static byte[] EncodeBlockFrame(IReadOnlyList rows) => + CompressFrame(EncodeBlockBody(rows)); + + /// + /// Builds a minimal but structurally valid sealed segment file: fixed header, 8-byte + /// length-prefixed block frames, and the 52-byte-entry block index at the footer offset. + /// + public static byte[] BuildSegmentFile(params IReadOnlyList[] blocks) + { + var frames = blocks.Select(EncodeBlockFrame).ToArray(); + var dataLength = frames.Sum(f => f.Length + 8); + var footerOffset = JetstreamSegmentFormat.HeaderSize + dataLength; + var file = new byte[footerOffset + blocks.Length * 52]; + var span = file.AsSpan(); + + // Header. + span[0] = (byte)'j'; + span[1] = (byte)'s'; + span[2] = (byte)'s'; + span[3] = (byte)'0'; + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(4, 8), 0xDEADBEEF); // non-zero = sealed + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(12, 2), 1); + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(14, 4), (uint)blocks.Length); + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(18, 4), (uint)blocks.Sum(b => b.Count)); + var allRows = blocks.SelectMany(b => b).ToList(); + if (allRows.Count > 0) + { + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(26, 8), (ulong)allRows.Min(r => r.Seq)); + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(34, 8), (ulong)allRows.Max(r => r.Seq)); + } + + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(58, 8), (ulong)footerOffset); + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(90, 8), (ulong)footerOffset); // block index + + // Data region: length-prefixed frames. + var offset = JetstreamSegmentFormat.HeaderSize; + for (var i = 0; i < frames.Length; i++) + { + BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(offset, 8), (ulong)frames[i].Length); + frames[i].CopyTo(span.Slice(offset + 8)); + + // Block index entry: frame offset (before the length prefix) + compressed size. + var entry = span.Slice(footerOffset + i * 52, 52); + BinaryPrimitives.WriteUInt64LittleEndian(entry.Slice(0, 8), (ulong)offset); + BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8, 4), (uint)frames[i].Length); + + offset += 8 + frames[i].Length; + } + + return file; + } + + public static JetstreamSegmentRow CommitRow(long seq, string did, string collection, string rkey, byte[]? payload) => + new() + { + Seq = seq, + WitnessedAt = 1_000_000 + seq, + Kind = payload == null ? JetstreamSegmentRowKind.Delete : JetstreamSegmentRowKind.Create, + Did = did, + Collection = collection, + Rkey = rkey, + Rev = "rev" + seq, + Payload = payload, + }; +} + +public class JetstreamSegmentFormatTests +{ + private static readonly byte[] SamplePayload = { 0xA1, 0x61, 0x61, 0x01 }; // {"a": 1} in CBOR + + [Fact] + public void DecodeBlockFrame_RoundTripsAllColumns() + { + var rows = new List + { + new() + { + Seq = 10, + WitnessedAt = 111, + IndexedAt = 222, + Kind = JetstreamSegmentRowKind.Create, + Did = "did:plc:alice", + Collection = "app.bsky.feed.post", + Rkey = "3k2a", + Rev = "aaa", + Payload = SamplePayload, + }, + new() + { + Seq = 11, + WitnessedAt = 333, + Kind = JetstreamSegmentRowKind.Delete, + Did = "did:plc:bob", + Collection = "app.bsky.feed.like", + Rkey = "3k2b", + Rev = "bbb", + }, + new() + { + Seq = 12, + WitnessedAt = 444, + Kind = JetstreamSegmentRowKind.Account, + Did = "did:plc:carol", + Payload = new byte[] { 0xA0 }, + }, + }; + + var decoded = JetstreamSegmentFormat.DecodeBlockFrame(JetstreamSegmentTestData.EncodeBlockFrame(rows)); + + Assert.Equal(3, decoded.Count); + Assert.Equal(10, decoded[0].Seq); + Assert.Equal(111, decoded[0].WitnessedAt); + Assert.Equal(222, decoded[0].IndexedAt); + Assert.Equal(222, decoded[0].DisplayTimeUs); // imported indexed_at wins + Assert.Equal(JetstreamSegmentRowKind.Create, decoded[0].Kind); + Assert.Equal("did:plc:alice", decoded[0].Did); + Assert.Equal("app.bsky.feed.post", decoded[0].Collection); + Assert.Equal("3k2a", decoded[0].Rkey); + Assert.Equal("aaa", decoded[0].Rev); + Assert.Equal(SamplePayload, decoded[0].Payload); + + Assert.Equal(JetstreamSegmentRowKind.Delete, decoded[1].Kind); + Assert.Null(decoded[1].Payload); + Assert.Equal(333, decoded[1].DisplayTimeUs); // no import → witnessed time + + Assert.Equal(JetstreamSegmentRowKind.Account, decoded[2].Kind); + Assert.Equal(string.Empty, decoded[2].Collection); + } + + [Fact] + public void DecodeBlockBody_EmptyBlock_ReturnsNoRows() + { + var decoded = JetstreamSegmentFormat.DecodeBlockBody(new byte[] { 0, 0, 0, 0 }); + Assert.Empty(decoded); + } + + [Fact] + public void DecodeBlockBody_TruncatedBody_Throws() + { + var body = JetstreamSegmentTestData.EncodeBlockBody(new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "c.o.l", "rk", SamplePayload), + }); + var truncated = body.Take(body.Length - 2).ToArray(); + + Assert.Throws(() => JetstreamSegmentFormat.DecodeBlockBody(truncated)); + } + + [Fact] + public void DecodeBlockBody_TrailingBytes_Throws() + { + var body = JetstreamSegmentTestData.EncodeBlockBody(new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "c.o.l", "rk", SamplePayload), + }); + var padded = body.Concat(new byte[] { 0xFF }).ToArray(); + + Assert.Throws(() => JetstreamSegmentFormat.DecodeBlockBody(padded)); + } + + [Fact] + public void DecodeBlockBody_InvalidKind_Throws() + { + var body = JetstreamSegmentTestData.EncodeBlockBody(new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "c.o.l", "rk", SamplePayload), + }); + + // The kind column starts after count (4) + seq/witnessed/indexed (3 × 8 per event). + body[4 + 24] = 0; + Assert.Throws(() => JetstreamSegmentFormat.DecodeBlockBody(body)); + } + + [Fact] + public void DecodeBlockBody_HostileEventCount_Throws() + { + var body = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(body, uint.MaxValue); + + Assert.Throws(() => JetstreamSegmentFormat.DecodeBlockBody(body)); + } + + [Fact] + public void ReadHeader_RejectsBadMagicActiveFileAndWrongVersion() + { + var file = JetstreamSegmentTestData.BuildSegmentFile(new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "c.o.l", "rk", SamplePayload), + }); + + var badMagic = (byte[])file.Clone(); + badMagic[0] = (byte)'x'; + Assert.Throws(() => JetstreamSegmentFormat.ReadHeader(badMagic)); + + var active = (byte[])file.Clone(); + active.AsSpan(4, 8).Clear(); // zero checksum = active file + Assert.Throws(() => JetstreamSegmentFormat.ReadHeader(active)); + + var badVersion = (byte[])file.Clone(); + badVersion[12] = 9; + Assert.Throws(() => JetstreamSegmentFormat.ReadHeader(badVersion)); + + Assert.Throws(() => JetstreamSegmentFormat.ReadHeader(new byte[10])); + } + + [Fact] + public void WholeSegmentFile_RoundTripsThroughHeaderAndBlockIndex() + { + var block0 = new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "app.bsky.feed.post", "r1", SamplePayload), + JetstreamSegmentTestData.CommitRow(2, "did:plc:a", "app.bsky.feed.post", "r2", null), + }; + var block1 = new[] + { + JetstreamSegmentTestData.CommitRow(3, "did:plc:b", "app.bsky.feed.like", "r3", SamplePayload), + }; + + var file = JetstreamSegmentTestData.BuildSegmentFile(block0, block1); + var header = JetstreamSegmentFormat.ReadHeader(file); + + Assert.Equal(2, header.BlockCount); + Assert.Equal(3, header.EventCount); + Assert.Equal(1, header.MinSeq); + Assert.Equal(3, header.MaxSeq); + + var frame0 = JetstreamSegmentFormat.GetBlockFrame(file, header, 0); + var rows0 = JetstreamSegmentFormat.DecodeBlockFrame(frame0); + Assert.Equal(2, rows0.Count); + Assert.Equal(1, rows0[0].Seq); + Assert.Equal(2, rows0[1].Seq); + + var frame1 = JetstreamSegmentFormat.GetBlockFrame(file, header, 1); + var rows1 = JetstreamSegmentFormat.DecodeBlockFrame(frame1); + Assert.Single(rows1); + Assert.Equal("did:plc:b", rows1[0].Did); + + Assert.Throws(() => JetstreamSegmentFormat.GetBlockFrame(file, header, 2)); + Assert.Throws(() => JetstreamSegmentFormat.GetBlockFrame(file, header, -1)); + } + + [Fact] + public void GetBlockFrame_CorruptIndexEntry_Throws() + { + var file = JetstreamSegmentTestData.BuildSegmentFile(new[] + { + JetstreamSegmentTestData.CommitRow(1, "did:plc:a", "c.o.l", "rk", SamplePayload), + }); + var header = JetstreamSegmentFormat.ReadHeader(file); + + // Point the block's offset past the footer: a hostile index entry must not drive an + // out-of-bounds read. + BinaryPrimitives.WriteUInt64LittleEndian( + file.AsSpan((int)header.BlockIndexOffset, 8), (ulong)file.Length + 100); + + Assert.Throws(() => JetstreamSegmentFormat.GetBlockFrame(file, header, 0)); + } + + [Fact] + public void DecodeBlockFrame_GarbageZstd_Throws() + { + Assert.Throws(() => + JetstreamSegmentFormat.DecodeBlockFrame(new byte[] { 1, 2, 3, 4, 5 })); + } +} diff --git a/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2FrameDecoderTests.cs b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2FrameDecoderTests.cs new file mode 100644 index 0000000..3b8b3df --- /dev/null +++ b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2FrameDecoderTests.cs @@ -0,0 +1,235 @@ +using System.Text; +using System.Text.Json; +using CarpaNet.Jetstream; +using Xunit; + +namespace CarpaNet.UnitTests.Jetstream; + +/// +/// Frame fixtures mirror the reference Go client's livedecode tests: the canonical +/// six-fractional-digit wire time, the proposal-0015 envelope, and the same +/// malformed-vs-skip classification. +/// +public class JetstreamV2FrameDecoderTests +{ + private const string WireTime = "1970-01-01T00:00:00.000001Z"; + + private static JetstreamV2FrameResult Decode(string frame) => + JetstreamV2FrameDecoder.Decode(Encoding.UTF8.GetBytes(frame)); + + private static string CommitFrame(long seq, string did, string op, string collection, string rkey, bool withRecord) + { + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#commit\"" + + $",\"seq\":{seq},\"did\":\"{did}\",\"time\":\"{WireTime}\"" + + $",\"rev\":\"r\",\"operation\":\"{op}\",\"collection\":\"{collection}\",\"rkey\":\"{rkey}\""; + if (withRecord) + { + frame += $",\"cid\":\"bafytest\",\"record\":{{\"$type\":\"{collection}\",\"text\":\"hi\"}}"; + } + + return frame + "}}"; + } + + [Fact] + public void Decode_CommitCreate_ProducesEvent() + { + var result = Decode(CommitFrame(42, "did:plc:a", "create", "app.bsky.feed.post", "r1", withRecord: true)); + + Assert.Equal(JetstreamV2FrameKind.Event, result.Kind); + var evt = result.Event!; + Assert.Equal(JetstreamV2EventKind.Commit, evt.Kind); + Assert.Equal(42, evt.Seq); + Assert.Equal("did:plc:a", evt.Did); + Assert.Equal(1, evt.TimeUs); // the canonical datetime parses back to unix microseconds + Assert.Equal(JetstreamV2CommitOperation.Create, evt.Commit!.Operation); + Assert.Equal("app.bsky.feed.post", evt.Commit.Collection); + Assert.Equal("r1", evt.Commit.Rkey); + Assert.Equal("bafytest", evt.Commit.Cid); + Assert.Equal("hi", evt.Commit.Record!.Value.GetProperty("text").GetString()); + } + + [Fact] + public void Decode_CommitDelete_HasNoRecordOrCid() + { + var result = Decode(CommitFrame(7, "did:plc:a", "delete", "app.bsky.feed.post", "r1", withRecord: false)); + + Assert.Equal(JetstreamV2FrameKind.Event, result.Kind); + Assert.Equal(JetstreamV2CommitOperation.Delete, result.Event!.Commit!.Operation); + Assert.Null(result.Event.Commit.Record); + Assert.Null(result.Event.Commit.Cid); + } + + [Fact] + public void Decode_CreateMissingRecord_IsMalformed() + { + var result = Decode(CommitFrame(1, "did:plc:a", "create", "app.bsky.feed.post", "r", withRecord: false)); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + Assert.Contains("missing record", result.Name); + } + + [Fact] + public void Decode_UnknownOperation_IsMalformed() + { + var result = Decode(CommitFrame(1, "did:plc:a", "upsert", "app.bsky.feed.post", "r", withRecord: true)); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + } + + [Fact] + public void Decode_InvalidSeq_IsMalformed() + { + // Seqs are 1-based on the wire; 0 means the required field was absent. Accepting it + // would hand the session an event the dedup silently swallows. + var result = Decode(CommitFrame(0, "did:plc:a", "create", "app.bsky.feed.post", "r", withRecord: true)); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + Assert.Contains("invalid seq", result.Name); + } + + [Fact] + public void Decode_Identity_WrapsUpstreamEvent() + { + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#identity\"" + + $",\"seq\":6,\"did\":\"did:plc:a\",\"time\":\"{WireTime}\"" + + ",\"identity\":{\"did\":\"did:plc:a\",\"handle\":\"alice.test\",\"seq\":600,\"time\":\"2026-05-25T00:00:00Z\"}}}"; + var result = Decode(frame); + + Assert.Equal(JetstreamV2FrameKind.Event, result.Kind); + var evt = result.Event!; + Assert.Equal(JetstreamV2EventKind.Identity, evt.Kind); + Assert.Equal(6, evt.Seq); + Assert.Equal("alice.test", evt.Identity!.Handle); + Assert.Equal(600, evt.Identity.Seq); // wrapped upstream event keeps the relay's seq + } + + [Fact] + public void Decode_AccountTombstone_KeepsStatus() + { + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#account\"" + + $",\"seq\":5,\"did\":\"did:plc:a\",\"time\":\"{WireTime}\"" + + ",\"account\":{\"did\":\"did:plc:a\",\"active\":false,\"status\":\"deleted\",\"seq\":500,\"time\":\"2026-05-25T00:00:00Z\"}}}"; + var result = Decode(frame); + + Assert.Equal(JetstreamV2FrameKind.Event, result.Kind); + var evt = result.Event!; + Assert.Equal(JetstreamV2EventKind.Account, evt.Kind); + Assert.False(evt.Account!.Active); + Assert.Equal("deleted", evt.Account.Status); + Assert.Equal(500, evt.Account.Seq); + } + + [Fact] + public void Decode_Sync_UsesEnvelopeSeqAndDropsBlocks() + { + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#sync\"" + + $",\"seq\":8,\"did\":\"did:plc:a\",\"time\":\"{WireTime}\"" + + ",\"sync\":{\"did\":\"did:plc:a\",\"rev\":\"rev1\",\"seq\":800,\"time\":\"2026-05-25T00:00:00Z\",\"blocks\":{\"$bytes\":\"AQI\"}}}}"; + var result = Decode(frame); + + Assert.Equal(JetstreamV2FrameKind.Event, result.Kind); + var evt = result.Event!; + Assert.Equal(JetstreamV2EventKind.Sync, evt.Kind); + Assert.Equal(8, evt.Seq); // envelope seq is Jetstream's + Assert.Equal("rev1", evt.Sync!.Rev); + Assert.Equal(800, evt.Sync.Seq); // wrapped upstream event keeps the relay's seq + } + + [Fact] + public void Decode_InfoFrame_IsAdvisory() + { + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#info\"" + + ",\"name\":\"OutdatedCursor\",\"message\":\"resumed from seq 5\"}}"; + var result = Decode(frame); + + Assert.Equal(JetstreamV2FrameKind.Info, result.Kind); + Assert.Equal("OutdatedCursor", result.Name); + Assert.Equal("resumed from seq 5", result.Message); + } + + [Fact] + public void Decode_ErrorFrame_IsTerminal() + { + var result = Decode("{\"$type\":\"error\",\"error\":\"ConsumerTooSlow\",\"message\":\"reconnect at cursor 9\"}"); + + Assert.Equal(JetstreamV2FrameKind.StreamError, result.Kind); + Assert.Equal("ConsumerTooSlow", result.Name); + Assert.Equal("reconnect at cursor 9", result.Message); + } + + [Fact] + public void Decode_ErrorFrameWithoutCode_IsMalformed() + { + var result = Decode("{\"$type\":\"error\",\"message\":\"oops\"}"); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + } + + [Fact] + public void Decode_UnknownPayloadType_Skips() + { + // A newer server's message kind must not break an old client. + var frame = "{\"$type\":\"message\",\"payload\":{\"$type\":\"network.bsky.jetstream.subscribeEvents#future\",\"seq\":1}}"; + var result = Decode(frame); + + Assert.Equal(JetstreamV2FrameKind.Skip, result.Kind); + } + + [Fact] + public void Decode_UnknownEnvelopeType_Skips() + { + var result = Decode("{\"$type\":\"heartbeat\"}"); + + Assert.Equal(JetstreamV2FrameKind.Skip, result.Kind); + } + + [Fact] + public void Decode_MissingEnvelopeType_IsMalformed() + { + // No $type at all means the client hit a v1 /subscribe endpoint; skipping would make + // the wrong endpoint look healthy while delivering nothing. + var result = Decode("{\"did\":\"did:plc:a\",\"time_us\":1,\"kind\":\"commit\"}"); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + Assert.Contains("subscribeEvents", result.Name); + } + + [Fact] + public void Decode_PayloadMissingType_IsMalformed() + { + // A payload with no $type is malformed, not a future addition — skipping it would be + // silent event loss. + var result = Decode("{\"$type\":\"message\",\"payload\":{\"seq\":1,\"did\":\"did:plc:a\"}}"); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + } + + [Fact] + public void Decode_InvalidJson_IsMalformed() + { + var result = Decode("{nope"); + + Assert.Equal(JetstreamV2FrameKind.Malformed, result.Kind); + } + + [Fact] + public void GetRecord_DeserializesTypedRecord() + { + var result = Decode(CommitFrame(42, "did:plc:a", "create", "app.bsky.feed.post", "r1", withRecord: true)); + var record = result.Event!.Commit!.GetRecord(JetstreamV2TestRecordContext.Default.JetstreamV2TestRecord); + + Assert.NotNull(record); + Assert.Equal("hi", record!.Text); + } +} + +public sealed class JetstreamV2TestRecord +{ + [System.Text.Json.Serialization.JsonPropertyName("text")] + public string? Text { get; set; } +} + +[System.Text.Json.Serialization.JsonSerializable(typeof(JetstreamV2TestRecord))] +public partial class JetstreamV2TestRecordContext : System.Text.Json.Serialization.JsonSerializerContext +{ +} diff --git a/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2MatcherTests.cs b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2MatcherTests.cs new file mode 100644 index 0000000..2d5aa15 --- /dev/null +++ b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2MatcherTests.cs @@ -0,0 +1,131 @@ +using CarpaNet.Jetstream; +using Xunit; + +namespace CarpaNet.UnitTests.Jetstream; + +public class JetstreamV2MatcherTests +{ + private static JetstreamV2Event Commit(long seq, string did, string collection) => new() + { + Seq = seq, + Did = did, + Kind = JetstreamV2EventKind.Commit, + Commit = new JetstreamV2Commit { Collection = collection }, + }; + + private static JetstreamV2Event Account(long seq, string did) => new() + { + Seq = seq, + Did = did, + Kind = JetstreamV2EventKind.Account, + Account = new JetstreamV2Account { Did = did, Active = false, Status = "deleted" }, + }; + + [Fact] + public void NoFilters_MatchesEverything() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions()); + + Assert.True(matcher.WantsEvent(Commit(1, "did:plc:a", "app.bsky.feed.post"))); + Assert.True(matcher.WantsEvent(Account(2, "did:plc:a"))); + } + + [Fact] + public void CollectionFilter_ExactAndWildcard() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + Collections = new[] { "app.bsky.feed.post", "app.bsky.graph.*" }, + }); + + Assert.True(matcher.WantsEvent(Commit(1, "did:plc:a", "app.bsky.feed.post"))); + Assert.True(matcher.WantsEvent(Commit(2, "did:plc:a", "app.bsky.graph.follow"))); + Assert.False(matcher.WantsEvent(Commit(3, "did:plc:a", "app.bsky.feed.like"))); + // The wildcard keeps the dot: "app.bsky.graphx" must not match "app.bsky.graph.*". + Assert.False(matcher.WantsEvent(Commit(4, "did:plc:a", "app.bsky.graphx.follow"))); + } + + [Fact] + public void CollectionFilter_DidLevelMarkersBypass() + { + // #account/#identity/#sync are the only purge signal a folding consumer gets; a + // collection filter must never suppress them. + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + Collections = new[] { "app.bsky.feed.post" }, + }); + + Assert.True(matcher.WantsEvent(Account(1, "did:plc:a"))); + } + + [Fact] + public void KindsCommitOnly_ExcludesMarkers() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + Kinds = new[] { JetstreamV2EventKind.Commit }, + }); + + Assert.True(matcher.WantsEvent(Commit(1, "did:plc:a", "app.bsky.feed.post"))); + Assert.False(matcher.WantsEvent(Account(2, "did:plc:a"))); + } + + [Fact] + public void DidFilter_AppliesToEveryKind() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + Dids = new[] { "did:plc:a" }, + }); + + Assert.True(matcher.WantsEvent(Commit(1, "did:plc:a", "c.o.l"))); + Assert.False(matcher.WantsEvent(Commit(2, "did:plc:b", "c.o.l"))); + Assert.False(matcher.WantsEvent(Account(3, "did:plc:b"))); + } + + [Fact] + public void SeqWindow_IsExclusiveInclusive() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + AfterSeq = 10, + BeforeSeq = 20, + SnapshotOnly = true, + }); + + Assert.False(matcher.WantsEvent(Commit(10, "did:plc:a", "c.o.l"))); // afterSeq is exclusive + Assert.True(matcher.WantsEvent(Commit(11, "did:plc:a", "c.o.l"))); + Assert.True(matcher.WantsEvent(Commit(20, "did:plc:a", "c.o.l"))); // beforeSeq is inclusive + Assert.False(matcher.WantsEvent(Commit(21, "did:plc:a", "c.o.l"))); + } + + [Fact] + public void AfterSeqZero_ImposesNoLowerBound() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions { AfterSeq = 0 }); + + Assert.True(matcher.WantsEvent(Commit(1, "did:plc:a", "c.o.l"))); + } + + [Fact] + public void SetAfterSeq_RaisesTheFloor() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions { AfterSeq = 0 }); + matcher.SetAfterSeq(5); + + Assert.False(matcher.WantsEvent(Commit(5, "did:plc:a", "c.o.l"))); + Assert.True(matcher.WantsEvent(Commit(6, "did:plc:a", "c.o.l"))); + } + + [Fact] + public void WantsRow_MapsSegmentKinds() + { + var matcher = new JetstreamV2Matcher(new JetstreamV2SubscribeOptions + { + Kinds = new[] { JetstreamV2EventKind.Commit }, + }); + + Assert.True(matcher.WantsRow(new JetstreamSegmentRow { Seq = 1, Kind = JetstreamSegmentRowKind.CreateResync, Did = "d" })); + Assert.False(matcher.WantsRow(new JetstreamSegmentRow { Seq = 2, Kind = JetstreamSegmentRowKind.Identity, Did = "d" })); + } +} diff --git a/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2OptionsTests.cs b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2OptionsTests.cs new file mode 100644 index 0000000..2817188 --- /dev/null +++ b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2OptionsTests.cs @@ -0,0 +1,221 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Linq; +using CarpaNet.Jetstream; +using Xunit; + +namespace CarpaNet.UnitTests.Jetstream; + +public class JetstreamV2SubscribeOptionsTests +{ + [Fact] + public void Defaults_AreValid() + { + new JetstreamV2SubscribeOptions().Validate(); + } + + [Fact] + public void BeforeSeq_RequiresSnapshotOnly() + { + var options = new JetstreamV2SubscribeOptions { BeforeSeq = 100 }; + Assert.Throws(() => options.Validate()); + + options.SnapshotOnly = true; + options.Validate(); + } + + [Fact] + public void SnapshotOnly_RequiresAReplayBound() + { + var options = new JetstreamV2SubscribeOptions { SnapshotOnly = true }; + Assert.Throws(() => options.Validate()); + + options.AfterSeq = 0; + options.Validate(); + } + + [Fact] + public void NegativeSeqs_AreRejected() + { + Assert.Throws(() => new JetstreamV2SubscribeOptions { AfterSeq = -1 }.Validate()); + Assert.Throws(() => new JetstreamV2SubscribeOptions { LiveCursor = -1 }.Validate()); + } + + [Fact] + public void FilterLimits_AreEnforced() + { + var tooManyCollections = new JetstreamV2SubscribeOptions + { + Collections = Enumerable.Range(0, 101).Select(i => $"c.o.l{i}").ToList(), + }; + Assert.Throws(() => tooManyCollections.Validate()); + + var tooManyKinds = new JetstreamV2SubscribeOptions + { + Kinds = new[] + { + JetstreamV2EventKind.Commit, JetstreamV2EventKind.Identity, JetstreamV2EventKind.Account, + JetstreamV2EventKind.Sync, JetstreamV2EventKind.Commit, + }, + }; + Assert.Throws(() => tooManyKinds.Validate()); + } + + [Fact] + public void BareWildcards_AreRejected() + { + Assert.Throws(() => + new JetstreamV2SubscribeOptions { Collections = new[] { "*" } }.Validate()); + Assert.Throws(() => + new JetstreamV2SubscribeOptions { Collections = new[] { ".*" } }.Validate()); + + new JetstreamV2SubscribeOptions { Collections = new[] { "app.bsky.feed.*" } }.Validate(); + } + + [Fact] + public void BackfillRequested_TracksSeqBounds() + { + Assert.False(new JetstreamV2SubscribeOptions().BackfillRequested); + Assert.True(new JetstreamV2SubscribeOptions { AfterSeq = 0 }.BackfillRequested); + Assert.True(new JetstreamV2SubscribeOptions { BeforeSeq = 5, SnapshotOnly = true }.BackfillRequested); + } + + [Fact] + public void KindStrings_MapAndDeduplicate() + { + var strings = JetstreamV2Engine.KindStrings(new[] + { + JetstreamV2EventKind.Commit, JetstreamV2EventKind.Sync, JetstreamV2EventKind.Commit, + }); + + Assert.Equal(new[] { "commit", "sync" }, strings); + } +} + +public class JetstreamZstdDictionaryTests +{ + [Fact] + public void TryParseId_ReadsStructuredHeader() + { + var blob = new byte[16]; + BinaryPrimitives.WriteUInt32LittleEndian(blob, 0xEC30A437); + BinaryPrimitives.WriteUInt32LittleEndian(blob.AsSpan(4), 20260811); + + Assert.True(JetstreamZstdDictionary.TryParseId(blob, out var id)); + Assert.Equal(20260811u, id); + } + + [Fact] + public void TryParseId_RejectsShortMissingMagicAndZeroId() + { + Assert.False(JetstreamZstdDictionary.TryParseId(new byte[4], out _)); + Assert.False(JetstreamZstdDictionary.TryParseId(null, out _)); + + var noMagic = new byte[16]; + Assert.False(JetstreamZstdDictionary.TryParseId(noMagic, out _)); + + var zeroId = new byte[16]; + BinaryPrimitives.WriteUInt32LittleEndian(zeroId, 0xEC30A437); + Assert.False(JetstreamZstdDictionary.TryParseId(zeroId, out _)); + } +} + +public class JetstreamV2PlanConverterTests +{ + private static JetstreamV2PlanSegmentDto ValidSegment() => new() + { + Name = "seg_000000002a.jss", + Index = 42, + Checksum = "0123456789abcdef", + MinSeq = 100, + MaxSeq = 200, + Mode = "segment", + }; + + [Fact] + public void Convert_WholeSegmentPlan() + { + var plan = JetstreamV2PlanConverter.Convert(new JetstreamV2PlanSnapshotOutput + { + PlannedThroughSeq = 200, + SealedTipSeq = 500, + Segments = new List { ValidSegment() }, + }); + + Assert.Equal(200, plan.PlannedThroughSeq); + Assert.Equal(500, plan.SealedTipSeq); + var segment = Assert.Single(plan.Segments); + Assert.Equal(JetstreamSegmentPlanMode.WholeSegment, segment.Mode); + Assert.Equal("seg_000000002a.jss", segment.Name); + } + + [Fact] + public void Convert_BlocksModeRequiresRanges() + { + var dto = ValidSegment(); + dto.Mode = "blocks"; + var output = new JetstreamV2PlanSnapshotOutput + { + PlannedThroughSeq = 200, + SealedTipSeq = 500, + Segments = new List { dto }, + }; + + Assert.Throws(() => JetstreamV2PlanConverter.Convert(output)); + + dto.Blocks = new List { new() { First = 2, Last = 5 } }; + var plan = JetstreamV2PlanConverter.Convert(output); + var range = Assert.Single(plan.Segments[0].Blocks); + Assert.Equal(2, range.First); + Assert.Equal(5, range.Last); + } + + [Fact] + public void Convert_RejectsInvalidResponses() + { + Assert.Throws(() => JetstreamV2PlanConverter.Convert( + new JetstreamV2PlanSnapshotOutput { PlannedThroughSeq = 10, SealedTipSeq = 5 })); + + Assert.Throws(() => JetstreamV2PlanConverter.Convert( + new JetstreamV2PlanSnapshotOutput { PlannedThroughSeq = -1, SealedTipSeq = 5 })); + + var inverted = ValidSegment(); + inverted.MinSeq = 300; + Assert.Throws(() => JetstreamV2PlanConverter.Convert( + new JetstreamV2PlanSnapshotOutput + { + PlannedThroughSeq = 200, + SealedTipSeq = 500, + Segments = new List { inverted }, + })); + + var unknownMode = ValidSegment(); + unknownMode.Mode = "streaming"; + Assert.Throws(() => JetstreamV2PlanConverter.Convert( + new JetstreamV2PlanSnapshotOutput + { + PlannedThroughSeq = 200, + SealedTipSeq = 500, + Segments = new List { unknownMode }, + })); + } +} + +public class JetstreamV2TransportUriTests +{ + [Fact] + public void BuildXrpcUri_EncodesRepeatedParameters() + { + var uri = JetstreamV2Transport.BuildXrpcUri( + new Uri("https://jetstream.example"), + "network.bsky.jetstream.getBlock", + new[] + { + new KeyValuePair("segment", "seg_000000002a.jss"), + new KeyValuePair("blockIndex", "7"), + }); + + Assert.Equal("https://jetstream.example/xrpc/network.bsky.jetstream.getBlock?segment=seg_000000002a.jss&blockIndex=7", uri.ToString()); + } +} diff --git a/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2RecordDecoderTests.cs b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2RecordDecoderTests.cs new file mode 100644 index 0000000..edc266d --- /dev/null +++ b/tests/CarpaNet.UnitTests/Jetstream/JetstreamV2RecordDecoderTests.cs @@ -0,0 +1,268 @@ +using System; +using System.Formats.Cbor; +using System.Security.Cryptography; +using System.Text.Json; +using CarpaNet; +using CarpaNet.Jetstream; +using Xunit; + +namespace CarpaNet.UnitTests.Jetstream; + +public class JetstreamV2RecordDecoderTests +{ + private const string SampleCid = "bafyreicqu7jhkc6ec3oq4fexqxlhkr27mjqcbaxkqz6aorpvvxwfkmmf3u"; + + private static byte[] SampleRecordCbor() + { + var writer = new CborWriter(CborConformanceMode.Canonical); + writer.WriteStartMap(5); + writer.WriteTextString("n"); + writer.WriteInt64(42); + writer.WriteTextString("arr"); + writer.WriteStartArray(3); + writer.WriteInt64(0); + writer.WriteBoolean(true); + writer.WriteNull(); + writer.WriteEndArray(); + writer.WriteTextString("bytes"); + writer.WriteByteString(new byte[] { 1, 2 }); + writer.WriteTextString("link"); + writer.WriteTag((CborTag)42); + var cidBytes = new ATCid(SampleCid).ToBytes(); + var tagged = new byte[cidBytes.Length + 1]; + cidBytes.CopyTo(tagged, 1); // DAG-CBOR CID byte strings carry a 0x00 multibase prefix + writer.WriteByteString(tagged); + writer.WriteTextString("text"); + writer.WriteTextString("hi"); + writer.WriteEndMap(); + return writer.Encode(); + } + + [Fact] + public void CborToJsonElement_ProducesAtprotoJsonShape() + { + var element = JetstreamV2RecordDecoder.CborToJsonElement(SampleRecordCbor()); + + Assert.Equal(42, element.GetProperty("n").GetInt64()); + Assert.Equal("hi", element.GetProperty("text").GetString()); + + var arr = element.GetProperty("arr"); + Assert.Equal(3, arr.GetArrayLength()); + Assert.Equal(0, arr[0].GetInt64()); + Assert.True(arr[1].GetBoolean()); + Assert.Equal(JsonValueKind.Null, arr[2].ValueKind); + + // Byte strings surface as {"$bytes": base64-without-padding}. + Assert.Equal("AQI", element.GetProperty("bytes").GetProperty("$bytes").GetString()); + + // CID links surface as {"$link": cid}. + Assert.Equal(SampleCid, element.GetProperty("link").GetProperty("$link").GetString()); + } + + [Fact] + public void CborToJsonElement_RejectsNonMapTopLevel() + { + var writer = new CborWriter(); + writer.WriteInt64(5); + + Assert.Throws(() => JetstreamV2RecordDecoder.CborToJsonElement(writer.Encode())); + } + + [Fact] + public void CborToJsonElement_RejectsFloats() + { + // The atproto data model has integers, not floats. + var writer = new CborWriter(); + writer.WriteStartMap(1); + writer.WriteTextString("f"); + writer.WriteDouble(1.5); + writer.WriteEndMap(); + + Assert.Throws(() => JetstreamV2RecordDecoder.CborToJsonElement(writer.Encode())); + } + + [Fact] + public void CborToJsonElement_RejectsTrailingBytes() + { + var payload = SampleRecordCbor(); + var padded = new byte[payload.Length + 1]; + payload.CopyTo(padded, 0); + padded[payload.Length] = 0xA0; + + Assert.Throws(() => JetstreamV2RecordDecoder.CborToJsonElement(padded)); + } + + [Fact] + public void ConvertRow_CreateCommit_DecodesRecordAndComputesCid() + { + var payload = SampleRecordCbor(); + var row = new JetstreamSegmentRow + { + Seq = 9, + WitnessedAt = 123, + Kind = JetstreamSegmentRowKind.Create, + Did = "did:plc:a", + Collection = "app.bsky.feed.post", + Rkey = "rk", + Rev = "rv", + Payload = payload, + }; + + var evt = JetstreamV2RecordDecoder.ConvertRow(row); + + Assert.Equal(JetstreamV2EventKind.Commit, evt.Kind); + Assert.Equal(9, evt.Seq); + Assert.Equal(123, evt.TimeUs); + var commit = evt.Commit!; + Assert.Equal(JetstreamV2CommitOperation.Create, commit.Operation); + Assert.Equal("hi", commit.Record!.Value.GetProperty("text").GetString()); + + using var sha = SHA256.Create(); + var expectedCid = ATCid.FromSha256Hash(sha.ComputeHash(payload)).Value; + Assert.Equal(expectedCid, commit.Cid); + } + + [Fact] + public void ConvertRow_CreateResync_RendersAsCreate() + { + var row = new JetstreamSegmentRow + { + Seq = 1, + Kind = JetstreamSegmentRowKind.CreateResync, + Did = "did:plc:a", + Collection = "c.o.l", + Rkey = "rk", + Rev = "rv", + Payload = SampleRecordCbor(), + }; + + Assert.Equal(JetstreamV2CommitOperation.Create, JetstreamV2RecordDecoder.ConvertRow(row).Commit!.Operation); + } + + [Fact] + public void ConvertRow_DeleteCommit_HasNoRecord() + { + var row = new JetstreamSegmentRow + { + Seq = 2, + Kind = JetstreamSegmentRowKind.Delete, + Did = "did:plc:a", + Collection = "c.o.l", + Rkey = "rk", + Rev = "rv", + }; + + var commit = JetstreamV2RecordDecoder.ConvertRow(row).Commit!; + Assert.Equal(JetstreamV2CommitOperation.Delete, commit.Operation); + Assert.Null(commit.Record); + Assert.Null(commit.Cid); + } + + [Fact] + public void ConvertRow_CreateWithoutPayload_Throws() + { + var row = new JetstreamSegmentRow + { + Seq = 3, + Kind = JetstreamSegmentRowKind.Create, + Did = "did:plc:a", + Collection = "c.o.l", + Rkey = "rk", + Rev = "rv", + }; + + Assert.Throws(() => JetstreamV2RecordDecoder.ConvertRow(row)); + } + + [Fact] + public void ConvertRow_Identity_DecodesUpstreamPayload() + { + var writer = new CborWriter(CborConformanceMode.Canonical); + writer.WriteStartMap(4); + writer.WriteTextString("did"); + writer.WriteTextString("did:plc:a"); + writer.WriteTextString("seq"); + writer.WriteInt64(700); + writer.WriteTextString("time"); + writer.WriteTextString("2026-05-25T00:00:00Z"); + writer.WriteTextString("handle"); + writer.WriteTextString("alice.test"); + writer.WriteEndMap(); + + var row = new JetstreamSegmentRow + { + Seq = 4, + WitnessedAt = 55, + Kind = JetstreamSegmentRowKind.Identity, + Did = "did:plc:a", + Payload = writer.Encode(), + }; + + var evt = JetstreamV2RecordDecoder.ConvertRow(row); + Assert.Equal(JetstreamV2EventKind.Identity, evt.Kind); + Assert.Equal("alice.test", evt.Identity!.Handle); + Assert.Equal(700, evt.Identity.Seq); + Assert.Equal("2026-05-25T00:00:00Z", evt.Identity.Time); + } + + [Fact] + public void ConvertRow_Account_DecodesUpstreamPayloadAndSkipsUnknownKeys() + { + var writer = new CborWriter(CborConformanceMode.Canonical); + writer.WriteStartMap(5); + writer.WriteTextString("did"); + writer.WriteTextString("did:plc:a"); + writer.WriteTextString("seq"); + writer.WriteInt64(701); + writer.WriteTextString("time"); + writer.WriteTextString("2026-05-25T00:00:00Z"); + writer.WriteTextString("active"); + writer.WriteBoolean(false); + writer.WriteTextString("status"); + writer.WriteTextString("deleted"); + writer.WriteEndMap(); + + var row = new JetstreamSegmentRow + { + Seq = 5, + Kind = JetstreamSegmentRowKind.Account, + Did = "did:plc:a", + Payload = writer.Encode(), + }; + + var evt = JetstreamV2RecordDecoder.ConvertRow(row); + Assert.False(evt.Account!.Active); + Assert.Equal("deleted", evt.Account.Status); + Assert.Equal(701, evt.Account.Seq); + } + + [Fact] + public void ConvertRow_Sync_IgnoresBlocksBytes() + { + var writer = new CborWriter(CborConformanceMode.Canonical); + writer.WriteStartMap(5); + writer.WriteTextString("did"); + writer.WriteTextString("did:plc:a"); + writer.WriteTextString("rev"); + writer.WriteTextString("rev1"); + writer.WriteTextString("seq"); + writer.WriteInt64(800); + writer.WriteTextString("time"); + writer.WriteTextString("2026-05-25T00:00:00Z"); + writer.WriteTextString("blocks"); + writer.WriteByteString(new byte[] { 1, 2, 3 }); + writer.WriteEndMap(); + + var row = new JetstreamSegmentRow + { + Seq = 6, + Kind = JetstreamSegmentRowKind.Sync, + Did = "did:plc:a", + Payload = writer.Encode(), + }; + + var evt = JetstreamV2RecordDecoder.ConvertRow(row); + Assert.Equal("rev1", evt.Sync!.Rev); + Assert.Equal(800, evt.Sync.Seq); + } +}