fsck: `Check` deadlocks instead of returning an error when a resource check fails (default `Opts{N: 1}`)

[titusz]

Summary

When a resource check fails (e.g. a stored hash tile doesn't match the derived one),
fsck.Check can block forever on an internal channel send instead of returning the
error. It also does not observe context cancellation while blocked. With the default
Opts{N: 1} this happens for any mismatch that isn't among the last resources
enumerated — i.e. the common "one corrupt tile somewhere in the log" case wedges the
very tool meant to detect it.

Observed on v1.0.2 and v1.0.4; the relevant code is unchanged on main.

Cause

In fsck/fsck.go:

• The resource-check workers run in a plain errgroup.Group{} (not
  errgroup.WithContext), so a worker returning an error cancels nothing
  (fsck.go#L109).
• A worker that hits a mismatch or fetch error returns immediately, exiting its
  for r := range f.expectedResources loop without draining the channel.
• The producer side (visit and flushPartialTiles) does bare
  f.expectedResources <- resource{...} sends with no select on ctx.Done()
  (fsck.go#L295,
  #L313).

With N: 1, the single worker's exit leaves zero consumers; once the buffered channel
(capacity N) fills, the main goroutine blocks forever on the send and never reaches
eg.Wait(). With N > 1 the window narrows but the same pattern holds if all workers
error out. A mismatch in the last resources (small tree, or corruption in a trailing
partial) drains fine and returns the error cleanly — which is presumably why tests
haven't caught it.

The blocked goroutine is also unreclaimable: a caller that gives up (timeout on its
side) leaks it for the process lifetime.

Reproduction

Self-contained, in-memory (no storage backend). Builds a valid 300-leaf tlog-tiles
log, runs Check once clean (passes), then flips one byte in the full hash tile
{0,0} and runs Check again under a 5s context: it never returns.

go.mod:

module fsck-deadlock-repro

go 1.24

require (
	github.com/transparency-dev/merkle v0.0.2
	github.com/transparency-dev/tessera v1.0.4
	golang.org/x/mod v0.25.0
)

main.go

// Reproduces: fsck.Check deadlocks (instead of returning an error) when a
// resource mismatches, with the default Opts{N: 1}. In-memory fetcher, so no
// storage backend is involved.
//
// Phase 1 (control): clean 300-leaf log -> Check returns nil.
// Phase 2: flip one byte in the full hash tile {0,0} -> Check blocks forever,
// ignoring its (already expired) context.
package main

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/transparency-dev/merkle/rfc6962"
	"github.com/transparency-dev/merkle/testonly"
	"github.com/transparency-dev/tessera/api"
	"github.com/transparency-dev/tessera/fsck"
	"golang.org/x/mod/sumdb/note"
)

const (
	origin = "repro.example/log"
	leaves = 300
	width  = 256
)

type tileKey struct {
	l, i uint64
	p    uint8
}

type bundleKey struct {
	i uint64
	p uint8
}

type memFetcher struct {
	checkpoint []byte
	tiles      map[tileKey][]byte
	bundles    map[bundleKey][]byte
}

func (m *memFetcher) ReadCheckpoint(ctx context.Context) ([]byte, error) {
	return m.checkpoint, nil
}

func (m *memFetcher) ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) {
	if b, ok := m.tiles[tileKey{l, i, p}]; ok {
		return b, nil
	}
	return nil, os.ErrNotExist
}

func (m *memFetcher) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) {
	if b, ok := m.bundles[bundleKey{i, p}]; ok {
		return b, nil
	}
	return nil, os.ErrNotExist
}

func leafHashes(bundle []byte) ([][]byte, error) {
	eb := &api.EntryBundle{}
	if err := eb.UnmarshalText(bundle); err != nil {
		return nil, err
	}
	out := make([][]byte, 0, len(eb.Entries))
	for _, e := range eb.Entries {
		h := rfc6962.DefaultHasher.HashLeaf(e)
		out = append(out, h[:])
	}
	return out, nil
}

func encodeBundle(records [][]byte) []byte {
	var out []byte
	for _, rec := range records {
		var prefix [2]byte
		binary.BigEndian.PutUint16(prefix[:], uint16(len(rec)))
		out = append(out, prefix[:]...)
		out = append(out, rec...)
	}
	return out
}

func marshalTile(nodes [][]byte) []byte {
	raw, err := api.HashTile{Nodes: nodes}.MarshalText()
	if err != nil {
		panic(err)
	}
	return raw
}

// buildLog synthesizes the full tlog-tiles artifact set for a 300-leaf log:
// full bundle 0 + partial bundle 1, full tile {0,0} + partials {0,1} and
// {1,0}, plus a signed checkpoint.
func buildLog() (*memFetcher, note.Verifier) {
	preimages := make([][]byte, leaves)
	for i := range preimages {
		preimages[i] = []byte(fmt.Sprintf("leaf-%d", i))
	}
	tree := testonly.New(rfc6962.DefaultHasher)
	tree.AppendData(preimages...)

	f := &memFetcher{
		tiles:   map[tileKey][]byte{},
		bundles: map[bundleKey][]byte{},
	}
	f.bundles[bundleKey{0, 0}] = encodeBundle(preimages[:width])
	f.bundles[bundleKey{1, leaves - width}] = encodeBundle(preimages[width:])

	hashes := func(from, to int) [][]byte {
		nodes := make([][]byte, to-from)
		for i := range nodes {
			nodes[i] = tree.LeafHash(uint64(from + i))
		}
		return nodes
	}
	f.tiles[tileKey{0, 0, 0}] = marshalTile(hashes(0, width))
	f.tiles[tileKey{0, 1, leaves - width}] = marshalTile(hashes(width, leaves))

	sub := testonly.New(rfc6962.DefaultHasher)
	sub.AppendData(preimages[:width]...)
	f.tiles[tileKey{1, 0, 1}] = marshalTile([][]byte{sub.Hash()})

	skey, vkey, err := note.GenerateKey(rand.Reader, origin)
	if err != nil {
		panic(err)
	}
	signer, err := note.NewSigner(skey)
	if err != nil {
		panic(err)
	}
	body := fmt.Sprintf("%s\n%d\n%s\n", origin, leaves, base64.StdEncoding.EncodeToString(tree.Hash()))
	if f.checkpoint, err = note.Sign(&note.Note{Text: body}, signer); err != nil {
		panic(err)
	}
	v, err := note.NewVerifier(vkey)
	if err != nil {
		panic(err)
	}
	return f, v
}

// runCheck runs Check under a 5s context and waits up to 15s for it to return.
func runCheck(f *memFetcher, v note.Verifier) (returned bool, err error) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	done := make(chan error, 1)
	go func() {
		done <- fsck.New(origin, v, f, leafHashes, fsck.Opts{N: 1}).Check(ctx)
	}()
	select {
	case err := <-done:
		return true, err
	case <-time.After(15 * time.Second):
		return false, nil
	}
}

func main() {
	f, v := buildLog()

	returned, err := runCheck(f, v)
	if !returned || err != nil {
		fmt.Printf("control broken: returned=%v err=%v\n", returned, err)
		os.Exit(1)
	}
	fmt.Println("control: clean log -> Check returned nil")

	corrupt := bytes.Clone(f.tiles[tileKey{0, 0, 0}])
	corrupt[0] ^= 0xff
	f.tiles[tileKey{0, 0, 0}] = corrupt

	returned, err = runCheck(f, v)
	switch {
	case returned && err != nil:
		fmt.Printf("ok: corrupt tile -> Check returned error: %v\n", err)
	case returned:
		fmt.Println("bug: corrupt tile -> Check returned nil")
		os.Exit(1)
	default:
		fmt.Println("DEADLOCK: Check still blocked 10s after its 5s context expired")
		buf := make([]byte, 1<<20)
		n := runtime.Stack(buf, true)
		fmt.Printf("--- goroutine dump ---\n%s\n", buf[:n])
		os.Exit(2)
	}
}

Run with go mod tidy && go run .. Output:

control: clean log -> Check returned nil
DEADLOCK: Check still blocked 10s after its 5s context expired
--- goroutine dump ---
goroutine 23 [chan send]:
github.com/transparency-dev/tessera/fsck.(*fsckTree).flushPartialTiles(...)
	.../tessera@v1.0.4/fsck/fsck.go:313
github.com/transparency-dev/tessera/fsck.(*Fsck).Check(...)
	.../tessera@v1.0.4/fsck/fsck.go:144

The worker goroutine is already gone (it exited with the mismatch error into the
errgroup); the producer is parked on the chan send in flushPartialTiles with no
consumer left.

Expected behavior

Check returns the resource-check error (as it does when the mismatch is among the
last resources), and respects ctx cancellation while blocked.