diff --git a/Makefile b/Makefile index 6186a1b..48f1509 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,9 @@ mock-handler: test: build @echo "Running runner unit tests..." - go test -v ./runner/... + go test ./runner/... @echo "Running conformance tests with mock handler..." - $(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv + $(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) suite-validate: @echo "Validating testdata against the suite schema..." diff --git a/cmd/runner/main.go b/cmd/runner/main.go index bef31e2..0a796b6 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -66,24 +66,35 @@ func main() { totalPassed := 0 totalFailed := 0 totalTests := 0 + loggedTimeout := false + failedTestLines := make([]string, 0) - for _, testFile := range testFiles { - fmt.Printf("\n=== Running test suite ===\n") - + for suiteIdx, testFile := range testFiles { // Load test suite from embedded FS suite, err := runner.LoadTestSuiteFromFS(testdata.FS, testFile) if err != nil { fmt.Fprintf(os.Stderr, "Error loading test suite: %v\n", err) continue } + totalTests += len(suite.Tests) + + // Check if context is already cancelled + if ctx.Err() != nil { + if !loggedTimeout { + fmt.Printf("Skipped remaining %d test suite(s) because total execution timeout (%v) was exceeded!\n", + len(testFiles)-suiteIdx, *timeout) + loggedTimeout = true + } + continue + } // Run suite result := testRunner.RunTestSuite(ctx, *suite, verbosity) - printResults(suite, result) + printResults(suite, result, verbosity) + failedTestLines = append(failedTestLines, collectFailedTestLines(suite, result)...) totalPassed += result.PassedTests totalFailed += result.FailedTests - totalTests += result.TotalTests // Close handler after stateful suites to prevent state leaks. // A new handler process will be spawned on-demand when the next request is sent. @@ -92,29 +103,50 @@ func main() { } } + if len(failedTestLines) > 0 { + fmt.Printf("\n" + strings.Repeat("=", 60) + "\n") + fmt.Printf("FAILED TESTS\n") + fmt.Printf(strings.Repeat("=", 60) + "\n") + for _, line := range failedTestLines { + fmt.Printf("%s\n", line) + } + } + fmt.Printf("\n" + strings.Repeat("=", 60) + "\n") fmt.Printf("TOTAL SUMMARY\n") fmt.Printf(strings.Repeat("=", 60) + "\n") fmt.Printf("Total Tests: %d\n", totalTests) fmt.Printf("Passed: %d\n", totalPassed) fmt.Printf("Failed: %d\n", totalFailed) + fmt.Printf("Skipped: %d\n", totalTests-(totalPassed+totalFailed)) fmt.Printf(strings.Repeat("=", 60) + "\n") - if totalFailed > 0 { + if totalTests > totalPassed { os.Exit(1) } } -func printResults(suite *runner.TestSuite, result runner.TestResult) { - fmt.Printf("\nTest Suite: %s (%s)\n", result.SuiteTitle, result.SuiteFileName) +func printResults(suite *runner.TestSuite, result runner.TestResult, verbosity runner.VerbosityLevel) { + if verbosity < runner.VerbosityOnFailure && result.FailedTests == 0 { + return + } + + fmt.Printf("=== %s (%s) ===\n", result.SuiteTitle, result.SuiteFileName) if suite.Description != "" { - fmt.Printf("Description: %s\n", suite.Description) + fmt.Printf("%s\n", suite.Description) } - fmt.Printf("Total: %d, Passed: %d, Failed: %d\n\n", result.TotalTests, result.PassedTests, result.FailedTests) + totalSkipped := result.TotalTests - (result.PassedTests + result.FailedTests) + fmt.Printf("Total: %d, Passed: %d, Failed: %d, Skipped: %d\n", result.TotalTests, result.PassedTests, + result.FailedTests, totalSkipped) for i, tr := range result.TestResults { - status := "✓" - if !tr.Passed { + var status string + if tr.Passed { + if verbosity < runner.VerbosityAlways { + continue + } + status = "✓" + } else { status = "✗" } @@ -131,3 +163,14 @@ func printResults(suite *runner.TestSuite, result runner.TestResult) { fmt.Printf("\n") } + +func collectFailedTestLines(suite *runner.TestSuite, result runner.TestResult) []string { + lines := make([]string, 0, result.FailedTests) + for i, tr := range result.TestResults { + if tr.Passed { + continue + } + lines = append(lines, fmt.Sprintf("%s %s (%s)", result.SuiteFileName, tr.TestID, suite.Tests[i].Description)) + } + return lines +} diff --git a/runner/runner.go b/runner/runner.go index 9d896d0..07e6f45 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -129,34 +129,28 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi TotalTests: len(suite.Tests), } - skipTests := false - for i := range suite.Tests { + // Check if context is already cancelled + if ctx.Err() != nil { + fmt.Printf("Skipped remaining %d test case(s) in suite %q because total execution timeout (%v) was exceeded!\n", + len(suite.Tests)-i, suite.Title, tr.timeout) + break + } + test := &suite.Tests[i] - // Run the test case - var testResult SingleTestResult - if skipTests { - // In stateful suites, if any previous test failed, fail all subsequent tests - testResult = SingleTestResult{ - TestID: test.Request.ID, - Passed: false, - Message: "Skipped due to previous test failure in stateful suite", - } - } else { - // Execute the test against the handler - testResult = tr.runTest(ctx, test) - - // Track dependencies and add verbose output if requested or on failure - if verbosity != VerbosityQuiet { - requestChain := depTracker.OnTestExecuted(test) - if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { - verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) - if testResult.Message != "" { - testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) - } else { - testResult.Message = verboseOutput - } + // Execute the test against the handler + testResult := tr.runTest(test) + + // Track dependencies and add verbose output if requested or on failure + if verbosity != VerbosityQuiet { + requestChain := depTracker.OnTestExecuted(test) + if (verbosity == VerbosityAlways) || (verbosity == VerbosityOnFailure && !testResult.Passed) { + verboseOutput := formatVerboseOutput(suite.Tests, i, requestChain, &testResult) + if testResult.Message != "" { + testResult.Message = fmt.Sprintf("%s\n%s", testResult.Message, verboseOutput) + } else { + testResult.Message = verboseOutput } } } @@ -168,7 +162,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi } else { result.FailedTests++ if suite.Stateful { - skipTests = true + break } } } @@ -178,18 +172,7 @@ func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite, verbosi // runTest executes a single test case by sending a request, reading the response, // and validating the result matches expected output -func (tr *TestRunner) runTest(ctx context.Context, test *TestCase) SingleTestResult { - // Check if context is already cancelled - select { - case <-ctx.Done(): - return SingleTestResult{ - TestID: test.Request.ID, - Passed: false, - Message: fmt.Sprintf("Total execution timeout exceeded (%v)", tr.timeout), - } - default: - } - +func (tr *TestRunner) runTest(test *TestCase) SingleTestResult { err := tr.SendRequest(test.Request) if err != nil { return SingleTestResult{