Reduce low-verbosity runner noise and improve test summaries

Makefile

@@ -20,9 +20,9 @@ mock-handler:
 
 test: build
 	@echo "Running runner unit tests..."
-	go test -v ./runner/...
+	go test ./runner/...
 	@echo "Running conformance tests with mock handler..."
-	$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv
+	$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN)
 
 suite-validate:
 	@echo "Validating testdata against the suite schema..."

cmd/runner/main.go

@@ -66,24 +66,35 @@ func main() {
 	totalPassed := 0
 	totalFailed := 0
 	totalTests := 0
+	loggedTimeout := false
+	failedTestLines := make([]string, 0)
 
-	for _, testFile := range testFiles {
-		fmt.Printf("\n=== Running test suite ===\n")
-
+	for suiteIdx, testFile := range testFiles {
 		// Load test suite from embedded FS
 		suite, err := runner.LoadTestSuiteFromFS(testdata.FS, testFile)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error loading test suite: %v\n", err)
 			continue
 		}
+		totalTests += len(suite.Tests)
+
+		// Check if context is already cancelled
+		if ctx.Err() != nil {
+			if !loggedTimeout {
+				fmt.Printf("Skipped remaining %d test suite(s) because total execution timeout (%v) was exceeded!\n",
+					len(testFiles)-suiteIdx, *timeout)
+				loggedTimeout = true
+			}
+			continue
+		}
 
 		// Run suite
 		result := testRunner.RunTestSuite(ctx, *suite, verbosity)
-		printResults(suite, result)
+		printResults(suite, result, verbosity)
+		failedTestLines = append(failedTestLines, collectFailedTestLines(suite, result)...)
 
 		totalPassed += result.PassedTests
 		totalFailed += result.FailedTests
-		totalTests += result.TotalTests
 
 		// Close handler after stateful suites to prevent state leaks.
 		// A new handler process will be spawned on-demand when the next request is sent.
@@ -92,29 +103,50 @@ func main() {
 		}
 	}
 
+	if len(failedTestLines) > 0 {
+		fmt.Printf("\n" + strings.Repeat("=", 60) + "\n")
+		fmt.Printf("FAILED TESTS\n")
+		fmt.Printf(strings.Repeat("=", 60) + "\n")
+		for _, line := range failedTestLines {
+			fmt.Printf("%s\n", line)
+		}
+	}
+
 	fmt.Printf("\n" + strings.Repeat("=", 60) + "\n")
 	fmt.Printf("TOTAL SUMMARY\n")
 	fmt.Printf(strings.Repeat("=", 60) + "\n")
 	fmt.Printf("Total Tests: %d\n", totalTests)
 	fmt.Printf("Passed:      %d\n", totalPassed)
 	fmt.Printf("Failed:      %d\n", totalFailed)
+	fmt.Printf("Skipped:     %d\n", totalTests-(totalPassed+totalFailed))
 	fmt.Printf(strings.Repeat("=", 60) + "\n")
 
-	if totalFailed > 0 {
+	if totalTests > totalPassed {
 		os.Exit(1)
 	}
 }
 
-func printResults(suite *runner.TestSuite, result runner.TestResult) {
-	fmt.Printf("\nTest Suite: %s (%s)\n", result.SuiteTitle, result.SuiteFileName)
+func printResults(suite *runner.TestSuite, result runner.TestResult, verbosity runner.VerbosityLevel) {
+	if verbosity < runner.VerbosityOnFailure && result.FailedTests == 0 {
+		return
+	}
+
+	fmt.Printf("=== %s (%s) ===\n", result.SuiteTitle, result.SuiteFileName)
 	if suite.Description != "" {
-		fmt.Printf("Description: %s\n", suite.Description)
+		fmt.Printf("%s\n", suite.Description)
 	}
-	fmt.Printf("Total: %d, Passed: %d, Failed: %d\n\n", result.TotalTests, result.PassedTests, result.FailedTests)
+	totalSkipped := result.TotalTests - (result.PassedTests + result.FailedTests)
+	fmt.Printf("Total: %d, Passed: %d, Failed: %d, Skipped: %d\n", result.TotalTests, result.PassedTests,
+		result.FailedTests, totalSkipped)
 
 	for i, tr := range result.TestResults {
-		status := "✓"
-		if !tr.Passed {
+		var status string
+		if tr.Passed {
+			if verbosity < runner.VerbosityAlways {
+				continue
+			}
+			status = "✓"
+		} else {
 			status = "✗"
 		}
 
@@ -131,3 +163,14 @@ func printResults(suite *runner.TestSuite, result runner.TestResult) {
 
 	fmt.Printf("\n")
 }
+
+func collectFailedTestLines(suite *runner.TestSuite, result runner.TestResult) []string {
+	lines := make([]string, 0, result.FailedTests)
+	for i, tr := range result.TestResults {
+		if tr.Passed {
+			continue
+		}
+		lines = append(lines, fmt.Sprintf("%s %s (%s)", result.SuiteFileName, tr.TestID, suite.Tests[i].Description))
+	}
+	return lines
+}

runner/runner.go

@@ -129,34 +129,28 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi
 		TotalTests:    len(suite.Tests),
 	}
 
-	skipTests := false
-
 	for i := range suite.Tests {
+		// Check if context is already cancelled
+		if ctx.Err() != nil {
+			fmt.Printf("Skipped remaining %d test case(s) in suite %q because total execution timeout (%v) was exceeded!\n",
+				len(suite.Tests)-i, suite.Title, tr.timeout)
+			break
+		}
+
 		test := &suite.Tests[i]
 
-		// Run the test case
-		var testResult SingleTestResult
-		if skipTests {
-			// In stateful suites, if any previous test failed, fail all subsequent tests
-			testResult = SingleTestResult{
-				TestID:  test.Request.ID,
-				Passed:  false,
-				Message: "Skipped due to previous test failure in stateful suite",
-			}
-		} else {
-			// Execute the test against the handler
-			testResult = tr.runTest(ctx, test)
-
-			// Track dependencies and add verbose output if requested or on failure
-			if verbosity != VerbosityQuiet {
-				requestChain := depTracker.OnTestExecuted(test)
-				if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) {
-					verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult)
-					if testResult.Message != "" {
-						testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput)
-					} else {
-						testResult.Message = verboseOutput
-					}
+		// Execute the test against the handler
+		testResult := tr.runTest(test)
+
+		// Track dependencies and add verbose output if requested or on failure
+		if verbosity != VerbosityQuiet {
+			requestChain := depTracker.OnTestExecuted(test)
+			if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) {
+				verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult)
+				if testResult.Message != "" {
+					testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput)
+				} else {
+					testResult.Message = verboseOutput
 				}
 			}
 		}
@@ -168,7 +162,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi
 		} else {
 			result.FailedTests++
 			if suite.Stateful {
-				skipTests = true
+				break
 			}
 		}
 	}
@@ -178,18 +172,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi
 
 // runTest executes a single test case by sending a request, reading the response,
 // and validating the result matches expected output
-func (tr *TestRunner) runTest(ctx context.Context, test *TestCase) SingleTestResult {
-	// Check if context is already cancelled
-	select {
-	case <-ctx.Done():
-		return SingleTestResult{
-			TestID:  test.Request.ID,
-			Passed:  false,
-			Message: fmt.Sprintf("Total execution timeout exceeded (%v)", tr.timeout),
-		}
-	default:
-	}
-
+func (tr *TestRunner) runTest(test *TestCase) SingleTestResult {
 	err := tr.SendRequest(test.Request)
 	if err != nil {
 		return SingleTestResult{