From eb2572e52695e3fe2d71a645528cd46fbb87481f Mon Sep 17 00:00:00 2001 From: stringintech Date: Tue, 19 May 2026 12:12:41 +0330 Subject: [PATCH 1/3] Report skipped tests in summaries This adds skipped counts to suite and total summaries, prints clear timeout messages when remaining suites or test cases are skipped, and makes skipped tests cause a non-zero exit because not all tests completed successfully. --- cmd/runner/main.go | 24 ++++++++++++++++++------ runner/runner.go | 24 ++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/cmd/runner/main.go b/cmd/runner/main.go index bef31e2..1709f49 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -66,16 +66,26 @@ func main() { totalPassed := 0 totalFailed := 0 totalTests := 0 + loggedTimeout := false - for _, testFile := range testFiles { - fmt.Printf("\n=== Running test suite ===\n") - + for suiteIdx, testFile := range testFiles { // Load test suite from embedded FS suite, err := runner.LoadTestSuiteFromFS(testdata.FS, testFile) if err != nil { fmt.Fprintf(os.Stderr, "Error loading test suite: %v\n", err) continue } + totalTests += len(suite.Tests) + + // Check if context is already cancelled + if ctx.Err() != nil { + if !loggedTimeout { + fmt.Printf("Skipped remaining %d test suite(s) because total execution timeout (%v) was exceeded!\n", + len(testFiles)-suiteIdx, *timeout) + loggedTimeout = true + } + continue + } // Run suite result := testRunner.RunTestSuite(ctx, *suite, verbosity) @@ -83,7 +93,6 @@ func main() { totalPassed += result.PassedTests totalFailed += result.FailedTests - totalTests += result.TotalTests // Close handler after stateful suites to prevent state leaks. // A new handler process will be spawned on-demand when the next request is sent. @@ -98,9 +107,10 @@ func main() { fmt.Printf("Total Tests: %d\n", totalTests) fmt.Printf("Passed: %d\n", totalPassed) fmt.Printf("Failed: %d\n", totalFailed) + fmt.Printf("Skipped: %d\n", totalTests-(totalPassed+totalFailed)) fmt.Printf(strings.Repeat("=", 60) + "\n") - if totalFailed > 0 { + if totalTests > totalPassed { os.Exit(1) } } @@ -110,7 +120,9 @@ func printResults(suite *runner.TestSuite, result runner.TestResult) { if suite.Description != "" { fmt.Printf("Description: %s\n", suite.Description) } - fmt.Printf("Total: %d, Passed: %d, Failed: %d\n\n", result.TotalTests, result.PassedTests, result.FailedTests) + totalSkipped := result.TotalTests - (result.PassedTests + result.FailedTests) + fmt.Printf("Total: %d, Passed: %d, Failed: %d, Skipped: %d\n\n", result.TotalTests, result.PassedTests, + result.FailedTests, totalSkipped) for i, tr := range result.TestResults { status := "✓" diff --git a/runner/runner.go b/runner/runner.go index 9d896d0..530de6b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -132,6 +132,13 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi skipTests := false for i := range suite.Tests { + // Check if context is already cancelled + if ctx.Err() != nil { + fmt.Printf("Skipped remaining %d test case(s) in suite %q because total execution timeout (%v) was exceeded!\n", + len(suite.Tests)-i, suite.Title, tr.timeout) + break + } + test := &suite.Tests[i] // Run the test case @@ -145,7 +152,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi } } else { // Execute the test against the handler - testResult = tr.runTest(ctx, test) + testResult = tr.runTest(test) // Track dependencies and add verbose output if requested or on failure if verbosity != VerbosityQuiet { @@ -168,7 +175,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi } else { result.FailedTests++ if suite.Stateful { - skipTests = true + break } } } @@ -178,18 +185,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi // runTest executes a single test case by sending a request, reading the response, // and validating the result matches expected output -func (tr *TestRunner) runTest(ctx context.Context, test *TestCase) SingleTestResult { - // Check if context is already cancelled - select { - case <-ctx.Done(): - return SingleTestResult{ - TestID: test.Request.ID, - Passed: false, - Message: fmt.Sprintf("Total execution timeout exceeded (%v)", tr.timeout), - } - default: - } - +func (tr *TestRunner) runTest(test *TestCase) SingleTestResult { err := tr.SendRequest(test.Request) if err != nil { return SingleTestResult{ From 1e6c1d88f954ed37bbdc15b1ef955d27f08f7944 Mon Sep 17 00:00:00 2001 From: stringintech Date: Tue, 19 May 2026 12:13:39 +0330 Subject: [PATCH 2/3] Print less noise at lower verbosity This changes output behavior by verbosity level: - quiet: print nothing for suites with no failures - on-failure (-v): print the suite summary, but hide passed test cases - always (-vv): print full suite and test output like before --- Makefile | 4 ++-- cmd/runner/main.go | 23 ++++++++++++++++------- runner/runner.go | 37 ++++++++++++------------------------- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 6186a1b..48f1509 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,9 @@ mock-handler: test: build @echo "Running runner unit tests..." - go test -v ./runner/... + go test ./runner/... @echo "Running conformance tests with mock handler..." - $(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv + $(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) suite-validate: @echo "Validating testdata against the suite schema..." diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 1709f49..8a384c1 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -89,7 +89,7 @@ func main() { // Run suite result := testRunner.RunTestSuite(ctx, *suite, verbosity) - printResults(suite, result) + printResults(suite, result, verbosity) totalPassed += result.PassedTests totalFailed += result.FailedTests @@ -115,18 +115,27 @@ func main() { } } -func printResults(suite *runner.TestSuite, result runner.TestResult) { - fmt.Printf("\nTest Suite: %s (%s)\n", result.SuiteTitle, result.SuiteFileName) +func printResults(suite *runner.TestSuite, result runner.TestResult, verbosity runner.VerbosityLevel) { + if verbosity < runner.VerbosityOnFailure && result.FailedTests == 0 { + return + } + + fmt.Printf("=== %s (%s) ===\n", result.SuiteTitle, result.SuiteFileName) if suite.Description != "" { - fmt.Printf("Description: %s\n", suite.Description) + fmt.Printf("%s\n", suite.Description) } totalSkipped := result.TotalTests - (result.PassedTests + result.FailedTests) - fmt.Printf("Total: %d, Passed: %d, Failed: %d, Skipped: %d\n\n", result.TotalTests, result.PassedTests, + fmt.Printf("Total: %d, Passed: %d, Failed: %d, Skipped: %d\n", result.TotalTests, result.PassedTests, result.FailedTests, totalSkipped) for i, tr := range result.TestResults { - status := "✓" - if !tr.Passed { + var status string + if tr.Passed { + if verbosity < runner.VerbosityAlways { + continue + } + status = "✓" + } else { status = "✗" } diff --git a/runner/runner.go b/runner/runner.go index 530de6b..07e6f45 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -129,8 +129,6 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi TotalTests: len(suite.Tests), } - skipTests := false - for i := range suite.Tests { // Check if context is already cancelled if ctx.Err() != nil { @@ -141,29 +139,18 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi test := &suite.Tests[i] - // Run the test case - var testResult SingleTestResult - if skipTests { - // In stateful suites, if any previous test failed, fail all subsequent tests - testResult = SingleTestResult{ - TestID: test.Request.ID, - Passed: false, - Message: "Skipped due to previous test failure in stateful suite", - } - } else { - // Execute the test against the handler - testResult = tr.runTest(test) - - // Track dependencies and add verbose output if requested or on failure - if verbosity != VerbosityQuiet { - requestChain := depTracker.OnTestExecuted(test) - if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { - verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) - if testResult.Message != "" { - testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) - } else { - testResult.Message = verboseOutput - } + // Execute the test against the handler + testResult := tr.runTest(test) + + // Track dependencies and add verbose output if requested or on failure + if verbosity != VerbosityQuiet { + requestChain := depTracker.OnTestExecuted(test) + if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { + verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) + if testResult.Message != "" { + testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) + } else { + testResult.Message = verboseOutput } } } From 06fc6bf5bcf16c516f2aa4ad29b937fc3ec41141 Mon Sep 17 00:00:00 2001 From: stringintech Date: Tue, 19 May 2026 15:53:02 +0330 Subject: [PATCH 3/3] Add failed-test recap for runner output Print each failed test case as ' ()' above the total summary section in all verbosity modes. --- cmd/runner/main.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 8a384c1..0a796b6 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -67,6 +67,7 @@ func main() { totalFailed := 0 totalTests := 0 loggedTimeout := false + failedTestLines := make([]string, 0) for suiteIdx, testFile := range testFiles { // Load test suite from embedded FS @@ -90,6 +91,7 @@ func main() { // Run suite result := testRunner.RunTestSuite(ctx, *suite, verbosity) printResults(suite, result, verbosity) + failedTestLines = append(failedTestLines, collectFailedTestLines(suite, result)...) totalPassed += result.PassedTests totalFailed += result.FailedTests @@ -101,6 +103,15 @@ func main() { } } + if len(failedTestLines) > 0 { + fmt.Printf("\n" + strings.Repeat("=", 60) + "\n") + fmt.Printf("FAILED TESTS\n") + fmt.Printf(strings.Repeat("=", 60) + "\n") + for _, line := range failedTestLines { + fmt.Printf("%s\n", line) + } + } + fmt.Printf("\n" + strings.Repeat("=", 60) + "\n") fmt.Printf("TOTAL SUMMARY\n") fmt.Printf(strings.Repeat("=", 60) + "\n") @@ -152,3 +163,14 @@ func printResults(suite *runner.TestSuite, result runner.TestResult, verbosity r fmt.Printf("\n") } + +func collectFailedTestLines(suite *runner.TestSuite, result runner.TestResult) []string { + lines := make([]string, 0, result.FailedTests) + for i, tr := range result.TestResults { + if tr.Passed { + continue + } + lines = append(lines, fmt.Sprintf("%s %s (%s)", result.SuiteFileName, tr.TestID, suite.Tests[i].Description)) + } + return lines +}