diff --git a/runner/dependency_tracker.go b/runner/dependency_tracker.go index 6ff78c3..2f55c49 100644 --- a/runner/dependency_tracker.go +++ b/runner/dependency_tracker.go @@ -22,6 +22,14 @@ var stateMutatingMethods = map[string]bool{ // DependencyTracker manages test dependencies and builds request chains for verbose output. // It tracks both explicit ref dependencies and implicit state dependencies. +// +// State dependency tracking is intentionally coarse to keep things simple: stateDependencies is a single global +// list accumulating every state-mutating test across all stateful objects. When building +// the request chain for a test, if that test depends on any stateful ref (regardless of +// which object), the entire stateDependencies list is included. This means a mutation on +// one stateful object (e.g. process_block on $chainman_A) will appear in the chain of a +// test that only uses a different stateful object (e.g. $chainman_B). This is a known +// limitation of the current model. type DependencyTracker struct { // refCreators maps reference names to the test index that created them refCreators map[string]int @@ -36,6 +44,12 @@ type DependencyTracker struct { // stateDependencies is a cumulative list of all tests affecting state (state-mutating // tests and their complete dependency chains) stateDependencies []int + + // usesStatefulRefs caches whether each test (by index) or any test it transitively depends on uses a stateful ref + usesStatefulRefs map[int]bool + + // processedTestsCount is the number of tests processed so far, used to assign sequential indices + processedTestsCount int } // NewDependencyTracker creates a new dependency tracker @@ -45,83 +59,77 @@ func NewDependencyTracker() *DependencyTracker { statefulRefs: make(map[string]bool), depChains: make(map[int][]int), stateDependencies: []int{}, + usesStatefulRefs: make(map[int]bool), } } -// BuildDependenciesForTest analyzes a test's parameters to build its complete transitive -// dependency chain. When a test uses refs created by earlier tests, this records all direct -// dependencies (tests that created those refs) and indirect dependencies (their dependencies). -// Must be called after all previous tests have been processed. -func (dt *DependencyTracker) BuildDependenciesForTest(testIndex int, test *TestCase) { - // Build dependency chain for current test based on refs it uses - var parentChains [][]int - for _, ref := range extractRefsFromParams(test.Request.Params) { - if creatorIdx, exists := dt.refCreators[ref]; exists { - // Add the creator as a direct dependency - parentChains = append(parentChains, []int{creatorIdx}) - // Add transitive dependencies (creator's dependencies) - if chain, hasChain := dt.depChains[creatorIdx]; hasChain { - parentChains = append(parentChains, chain) - } - } else { - panic(fmt.Sprintf("test %d (%s) uses undefined reference %s - no prior test created this ref", - testIndex, test.Request.ID, ref)) - } - } - dt.depChains[testIndex] = mergeSortedUnique(parentChains...) -} +// OnTestExecuted is called after a test executes. It computes and returns the request chain +// for the test, then updates internal state so subsequent tests see this test's refs and mutations. +func (dt *DependencyTracker) OnTestExecuted(test *TestCase) []int { + i := dt.processedTestsCount + refs := extractRefsFromParams(test.Request.Params) -// OnTestExecuted is called after a test executes successfully. It tracks the ref -// created by the test, marks it as stateful if needed, and updates state dependencies -// for state-mutating methods. -func (dt *DependencyTracker) OnTestExecuted(testIndex int, test *TestCase) { - // Track ref creation using the request's ref field - if test.Request.Ref != "" { - dt.refCreators[test.Request.Ref] = testIndex + requestChain := dt.buildRequestChain(i, test.Request.ID, refs) - // Mark refs from stateful methods + // Track ref creation using the request's ref field. + if test.Request.Ref != "" { + dt.refCreators[test.Request.Ref] = i if statefulCreatorMethods[test.Request.Method] { dt.statefulRefs[test.Request.Ref] = true } } - // Track state-mutating tests and their dependencies + // Track state mutations so future tests that touch stateful objects include this test in their chain. if stateMutatingMethods[test.Request.Method] { - mutatorChain := append(dt.depChains[testIndex], testIndex) + mutatorChain := append(dt.depChains[i], i) dt.stateDependencies = mergeSortedUnique(dt.stateDependencies, mutatorChain) } -} - -// BuildRequestChain builds the complete dependency chain for a test -func (dt *DependencyTracker) BuildRequestChain(testIndex int, allTests []TestCase) []int { - refDepChain := dt.depChains[testIndex] - - // Only include state dependencies if the test's dep chain contains any stateful refs - if dt.testUsesStatefulRefs(testIndex, allTests) { - return mergeSortedUnique(refDepChain, dt.stateDependencies) - } - return refDepChain + dt.processedTestsCount++ + return requestChain } -// testUsesStatefulRefs checks if a test's dependency chain includes any stateful refs -func (dt *DependencyTracker) testUsesStatefulRefs(testIndex int, allTests []TestCase) bool { - // Check all tests in the dependency chain - for _, depIdx := range dt.depChains[testIndex] { - for _, ref := range extractRefsFromParams(allTests[depIdx].Request.Params) { - if dt.statefulRefs[ref] { - return true +func (dt *DependencyTracker) buildRequestChain(i int, testID string, refs []string) []int { + var parentChains [][]int + for _, ref := range refs { + if creatorIdx, exists := dt.refCreators[ref]; exists { + parentChains = append(parentChains, []int{creatorIdx}) + if chain, hasChain := dt.depChains[creatorIdx]; hasChain { + parentChains = append(parentChains, chain) } + } else { + panic(fmt.Sprintf("test %s uses undefined reference %s - no prior test created this ref", + testID, ref)) } } + dt.depChains[i] = mergeSortedUnique(parentChains...) + + if dt.computeUsesStatefulRefs(i, refs) { + return mergeSortedUnique(dt.depChains[i], dt.stateDependencies) + } + return dt.depChains[i] +} - // Check the test itself - for _, ref := range extractRefsFromParams(allTests[testIndex].Request.Params) { +// computeUsesStatefulRefs computes and caches whether test i (or any test it depends on) +// uses a stateful ref. deps must already be cached for all indices in depChains[i]. +func (dt *DependencyTracker) computeUsesStatefulRefs(i int, refs []string) bool { + result := false + for _, ref := range refs { if dt.statefulRefs[ref] { - return true + result = true + break + } + } + if !result { + for _, depIdx := range dt.depChains[i] { + if dt.usesStatefulRefs[depIdx] { + result = true + break + } } } - return false + dt.usesStatefulRefs[i] = result + return result } // extractRefsFromParams extracts all reference names from params JSON. diff --git a/runner/dependency_tracker_test.go b/runner/dependency_tracker_test.go index ea666b8..98084fb 100644 --- a/runner/dependency_tracker_test.go +++ b/runner/dependency_tracker_test.go @@ -51,327 +51,212 @@ func TestExtractRefsFromParams(t *testing.T) { } } -func TestDependencyTracker_BuildDependencyChains(t *testing.T) { - // Create test cases to verify dependency chain building - testsJSON := `[ - { - "request": { - "id": "test0", - "method": "create_a", - "params": {}, - "ref": "$ref_a" - }, - "expected_response": {"result": {"ref": "$ref_a"}} - }, - { - "request": { - "id": "test1", - "method": "create_b", - "params": {"input": {"ref": "$ref_a"}}, - "ref": "$ref_b" - }, - "expected_response": {"result": {"ref": "$ref_b"}} - }, - { - "request": { - "id": "test2", - "method": "create_c", - "params": {}, - "ref": "$ref_c" - }, - "expected_response": {"result": {"ref": "$ref_c"}} - }, - { - "request": { - "id": "test3", - "method": "use_multiple", - "params": {"first": {"ref": "$ref_b"}, "second": {"ref": "$ref_c"}} - }, - "expected_response": {} - }, - { - "request": { - "id": "test4", - "method": "use_array", - "params": {"items": [{"ref": "$ref_a"}, {"ref": "$ref_c"}]} - }, - "expected_response": {} - } - ]` - - var testCases []TestCase - if err := json.Unmarshal([]byte(testsJSON), &testCases); err != nil { - t.Fatalf("failed to unmarshal test cases: %v", err) +func TestDependencyTracker(t *testing.T) { + type entry struct { + tc TestCase + expectedChain []int } - // Create dependency tracker and simulate test execution - tracker := NewDependencyTracker() - - for i := range testCases { - test := &testCases[i] - tracker.BuildDependenciesForTest(i, test) - tracker.OnTestExecuted(i, test) + type suite struct { + name string + cases []entry } - // Verify dependency chains - tests := []struct { - testIdx int - wantDepChain []int - description string - }{ + suites := []suite{ { - testIdx: 0, - wantDepChain: []int{}, - description: "test0 has no dependencies", - }, - { - testIdx: 1, - wantDepChain: []int{0}, - description: "test1 depends on test0", - }, - { - testIdx: 2, - wantDepChain: []int{}, - description: "test2 has no dependencies", - }, - { - testIdx: 3, - wantDepChain: []int{0, 1, 2}, - description: "test3 depends on test1 (which depends on test0) and test2", - }, - { - testIdx: 4, - wantDepChain: []int{0, 2}, - description: "test4 depends on test0 and test2 via refs nested in an array param", - }, - } - - for _, tt := range tests { - t.Run(tt.description, func(t *testing.T) { - got := tracker.depChains[tt.testIdx] - if !slices.Equal(got, tt.wantDepChain) { - t.Errorf("depChains[%d] = %v, want %v", tt.testIdx, got, tt.wantDepChain) - } - }) - } -} - -func TestDependencyTracker_StatefulRefs(t *testing.T) { - testsJSON := `[ - { - "request": { - "id": "test0", - "method": "btck_context_create", - "params": {}, - "ref": "$context" + // ref dependency chains are built from params: a test that uses a ref inherits the + // full transitive chain of tests that produced it, including refs passed via arrays + name: "ref-chains", + cases: []entry{ + { + tc: TestCase{ + Request: Request{ + ID: "ref-chains#0", + Method: "create_a", + Params: json.RawMessage(`{}`), + Ref: "$ref_a", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$ref_a"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "ref-chains#1", + Method: "create_b", + Params: json.RawMessage(`{"input": {"ref": "$ref_a"}}`), + Ref: "$ref_b", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$ref_b"}`)}, + }, + expectedChain: []int{0}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "ref-chains#2", + Method: "create_c", + Params: json.RawMessage(`{}`), + Ref: "$ref_c", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$ref_c"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "ref-chains#3", + Method: "use_multiple", + Params: json.RawMessage(`{"first": {"ref": "$ref_b"}, "second": {"ref": "$ref_c"}}`), + }, + ExpectedResponse: Response{}, + }, + expectedChain: []int{0, 1, 2}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "ref-chains#4", + Method: "use_array", + Params: json.RawMessage(`{"items": [{"ref": "$ref_a"}, {"ref": "$ref_c"}]}`), + }, + ExpectedResponse: Response{}, + }, + expectedChain: []int{0, 2}, + }, }, - "expected_response": {"result": {"ref": "$context"}} }, { - "request": { - "id": "test1", - "method": "btck_chainstate_manager_create", - "params": {"context": {"ref": "$context"}}, - "ref": "$chainman" + // stateful kernel objects (context, chainman) cause state mutations to accumulate as + // state deps; any later test that uses a stateful ref inherits those deps. state deps + // are tracked globally, not per object instance, so a mutation on one chainman bleeds + // into unrelated operations on a different chainman created afterwards — a known + // documented limitation of the dependency tracker (see state-mutations#7, state-mutations#8) + name: "state-mutations", + cases: []entry{ + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#0", + Method: "btck_context_create", + Params: json.RawMessage(`{}`), + Ref: "$context", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$context"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#1", + Method: "btck_chainstate_manager_create", + Params: json.RawMessage(`{"context": {"ref": "$context"}}`), + Ref: "$chainman", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$chainman"}`)}, + }, + expectedChain: []int{0}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#2", + Method: "btck_block_create", + Params: json.RawMessage(`{"raw_block": "deadbeef"}`), + Ref: "$block", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$block"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#3", + Method: "btck_chainstate_manager_process_block", + Params: json.RawMessage(`{"chainstate_manager": {"ref": "$chainman"}, "block": {"ref": "$block"}}`), + }, + ExpectedResponse: Response{}, + }, + expectedChain: []int{0, 1, 2}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#4", + Method: "btck_block_create", + Params: json.RawMessage(`{"raw_block": "cafebabe"}`), + Ref: "$block2", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$block2"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#5", + Method: "btck_chainstate_manager_get_active_chain", + Params: json.RawMessage(`{"chainstate_manager": {"ref": "$chainman"}}`), + Ref: "$chain", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$chain"}`)}, + }, + expectedChain: []int{0, 1, 2, 3}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#6", + Method: "btck_context_create", + Params: json.RawMessage(`{}`), + Ref: "$context_b", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$context_b"}`)}, + }, + expectedChain: []int{}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#7", + Method: "btck_chainstate_manager_create", + Params: json.RawMessage(`{"context": {"ref": "$context_b"}}`), + Ref: "$chainman_b", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$chainman_b"}`)}, + }, + expectedChain: []int{0, 1, 2, 3, 6}, + }, + { + tc: TestCase{ + Request: Request{ + ID: "state-mutations#8", + Method: "btck_chainstate_manager_get_active_chain", + Params: json.RawMessage(`{"chainstate_manager": {"ref": "$chainman_b"}}`), + Ref: "$chain_b", + }, + ExpectedResponse: Response{Result: Result(`{"ref": "$chain_b"}`)}, + }, + expectedChain: []int{0, 1, 2, 3, 6, 7}, + }, }, - "expected_response": {"result": {"ref": "$chainman"}} }, - { - "request": { - "id": "test2", - "method": "btck_block_create", - "params": {"raw_block": "deadbeef"}, - "ref": "$block" - }, - "expected_response": {"result": {"ref": "$block"}} - } - ]` - - var testCases []TestCase - if err := json.Unmarshal([]byte(testsJSON), &testCases); err != nil { - t.Fatalf("failed to unmarshal test cases: %v", err) } - tracker := NewDependencyTracker() - - for i := range testCases { - test := &testCases[i] - tracker.BuildDependenciesForTest(i, test) - tracker.OnTestExecuted(i, test) - } - - // Verify that context and chainstate_manager refs are marked as stateful - if !tracker.statefulRefs["$context"] { - t.Error("$context_ref should be marked as stateful") - } - if !tracker.statefulRefs["$chainman"] { - t.Error("$chainman_ref should be marked as stateful") - } - if tracker.statefulRefs["$block"] { - t.Error("$block_ref should NOT be marked as stateful") - } -} - -func TestDependencyTracker_StateMutations(t *testing.T) { - testsJSON := `[ - { - "request": { - "id": "test0", - "method": "btck_context_create", - "params": {}, - "ref": "$context" - }, - "expected_response": {"result": {"ref": "$context"}} - }, - { - "request": { - "id": "test1", - "method": "btck_chainstate_manager_create", - "params": {"context": {"ref": "$context"}}, - "ref": "$chainman" - }, - "expected_response": {"result": {"ref": "$chainman"}} - }, - { - "request": { - "id": "test2", - "method": "btck_block_create", - "params": {"raw_block": "deadbeef"}, - "ref": "$block" - }, - "expected_response": {"result": {"ref": "$block"}} - }, - { - "request": { - "id": "test3", - "method": "btck_chainstate_manager_process_block", - "params": {"chainstate_manager": {"ref": "$chainman"}, "block": {"ref": "$block"}} - }, - "expected_response": {} - }, - { - "request": { - "id": "test4", - "method": "btck_block_create", - "params": {"raw_block": "cafebabe"}, - "ref": "$block2" - }, - "expected_response": {"result": {"ref": "$block2"}} - } - ]` - - var testCases []TestCase - if err := json.Unmarshal([]byte(testsJSON), &testCases); err != nil { - t.Fatalf("failed to unmarshal test cases: %v", err) - } - - tracker := NewDependencyTracker() - - for i := range testCases { - test := &testCases[i] - tracker.BuildDependenciesForTest(i, test) - tracker.OnTestExecuted(i, test) - } - - // State dependencies should include test3 (process_block) and its dependencies (0, 1, 2) - expectedStateDeps := []int{0, 1, 2, 3} - if !slices.Equal(tracker.stateDependencies, expectedStateDeps) { - t.Errorf("state dependencies = %v, want %v", tracker.stateDependencies, expectedStateDeps) - } -} - -func TestDependencyTracker_BuildRequestChain(t *testing.T) { - testsJSON := `[ - { - "request": { - "id": "test0", - "method": "btck_context_create", - "params": {}, - "ref": "$context" - }, - "expected_response": {"result": {"ref": "$context"}} - }, - { - "request": { - "id": "test1", - "method": "btck_chainstate_manager_create", - "params": {"context": {"ref": "$context"}}, - "ref": "$chainman" - }, - "expected_response": {"result": {"ref": "$chainman"}} - }, - { - "request": { - "id": "test2", - "method": "btck_block_create", - "params": {"raw_block": "deadbeef"}, - "ref": "$block" - }, - "expected_response": {"result": {"ref": "$block"}} - }, - { - "request": { - "id": "test3", - "method": "btck_chainstate_manager_process_block", - "params": {"chainstate_manager": {"ref": "$chainman"}, "block": {"ref": "$block"}} - }, - "expected_response": {} - }, - { - "request": { - "id": "test4", - "method": "btck_block_create", - "params": {"raw_block": "cafebabe"}, - "ref": "$block2" - }, - "expected_response": {"result": {"ref": "$block2"}} - }, - { - "request": { - "id": "test5", - "method": "btck_chainstate_manager_get_active_chain", - "params": {"chainstate_manager": {"ref": "$chainman"}}, - "ref": "$chain" - }, - "expected_response": {"result": {"ref": "$chain"}} - } - ]` - - var testCases []TestCase - if err := json.Unmarshal([]byte(testsJSON), &testCases); err != nil { - t.Fatalf("failed to unmarshal test cases: %v", err) - } - - tracker := NewDependencyTracker() - - for i := range testCases { - test := &testCases[i] - tracker.BuildDependenciesForTest(i, test) - tracker.OnTestExecuted(i, test) - } - - tests := []struct { - testIdx int - wantChain []int - description string - }{ - { - testIdx: 4, - wantChain: []int{}, // block_create doesn't use stateful refs, so no state deps included - description: "test4 (block_create) should NOT include state dependencies", - }, - { - testIdx: 5, - wantChain: []int{0, 1, 2, 3}, // uses chainman_ref (stateful), so includes state deps - description: "test5 (get_active_chain) should include state dependencies", - }, - } - - for _, tt := range tests { - t.Run(tt.description, func(t *testing.T) { - got := tracker.BuildRequestChain(tt.testIdx, testCases) - if !slices.Equal(got, tt.wantChain) { - t.Errorf("BuildRequestChain(%d) = %v, want %v", tt.testIdx, got, tt.wantChain) + for _, s := range suites { + t.Run(s.name, func(t *testing.T) { + tracker := NewDependencyTracker() + for _, e := range s.cases { + got := tracker.OnTestExecuted(&e.tc) + t.Run(e.tc.Request.ID, func(t *testing.T) { + if !slices.Equal(got, e.expectedChain) { + t.Errorf("chain = %v, want %v", got, e.expectedChain) + } + }) } }) } diff --git a/runner/runner.go b/runner/runner.go index d121ad9..9d896d0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -144,27 +144,20 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi Message: "Skipped due to previous test failure in stateful suite", } } else { - // Build dependency chain by analyzing which refs this test uses - if verbosity != VerbosityQuiet { - depTracker.BuildDependenciesForTest(i, test) - } - // Execute the test against the handler testResult = tr.runTest(ctx, test) - // Add verbose output if requested or on failure - if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { - requestChain := depTracker.BuildRequestChain(i, suite.Tests) - verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) - if testResult.Message != "" { - testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) - } else { - testResult.Message = verboseOutput - } - } - + // Track dependencies and add verbose output if requested or on failure if verbosity != VerbosityQuiet { - depTracker.OnTestExecuted(i, test) + requestChain := depTracker.OnTestExecuted(test) + if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { + verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) + if testResult.Message != "" { + testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) + } else { + testResult.Message = verboseOutput + } + } } }