From 04ba75a3c4291ef96cf56c17dc617dbf28e51463 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Wed, 2 Sep 2026 18:39:04 +0000 Subject: [PATCH 1/3] fix cloudwatchlogs flaky tests --- generator/test_case_generator.go | 25 ++++++++++++++++++++++++ test/cloudwatchlogs/publish_logs_test.go | 19 +++++++++++------- util/awsservice/cloudwatchlogs.go | 24 +++++++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 938793aaa..5c2ccacb6 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -615,6 +615,25 @@ var partitionTests = map[string]partition{ }, } +// filterByTestDir returns a copy of configMap keeping only testConfigs whose testDir +// contains substr. Every test type key is preserved (with a possibly empty slice) so that +// main still writes a valid — if empty — matrix file for each, avoiding missing-file +// failures downstream. TEMPORARY(cloudwatchlogs-flaky): scopes CI to a single suite while +// iterating; remove with its call site in main(). +func filterByTestDir(configMap map[string][]testConfig, substr string) map[string][]testConfig { + filtered := make(map[string][]testConfig, len(configMap)) + for testType, testConfigs := range configMap { + kept := make([]testConfig, 0, len(testConfigs)) + for _, tc := range testConfigs { + if strings.Contains(tc.testDir, substr) { + kept = append(kept, tc) + } + } + filtered[testType] = kept + } + return filtered +} + func main() { useE2E := flag.Bool("e2e", false, "Use e2e test matrix generation") flag.Parse() @@ -624,6 +643,12 @@ func main() { configMap = testTypeToTestConfigE2E } + // TEMPORARY(cloudwatchlogs-flaky): scope every generated matrix to the + // cloudwatchlogs suite so CI runs only those tests while the flaky-test fix + // is validated. Delete this line (and filterByTestDir below) to restore the + // full matrix before merging. + configMap = filterByTestDir(configMap, "cloudwatchlogs") + for testType, testConfigs := range configMap { for _, partition := range partitionTests { if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) { diff --git a/test/cloudwatchlogs/publish_logs_test.go b/test/cloudwatchlogs/publish_logs_test.go index f02bd3d05..052ed5da7 100644 --- a/test/cloudwatchlogs/publish_logs_test.go +++ b/test/cloudwatchlogs/publish_logs_test.go @@ -24,11 +24,13 @@ import ( ) const ( - configOutputPath = "/opt/aws/amazon-cloudwatch-agent/bin/config.json" - logLineId1 = "foo" - logLineId2 = "bar" - logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows - sleepForFlush = 20 * time.Second // default flush interval is 5 seconds + configOutputPath = "/opt/aws/amazon-cloudwatch-agent/bin/config.json" + logLineId1 = "foo" + logLineId2 = "bar" + logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows + sleepForFlush = 20 * time.Second // default flush interval is 5 seconds + logValidationAttempts = 6 + logValidationInterval = 15 * time.Second configPathAutoRemoval = "resources/config_auto_removal.json" standardLogGroupClass = "STANDARD" infrequentAccessLogGroupClass = "INFREQUENT_ACCESS" @@ -130,12 +132,15 @@ func TestWriteLogsToCloudWatch(t *testing.T) { common.StopAgent() end := time.Now() - // check CWL to ensure we got the expected number of logs in the log stream - err = awsservice.ValidateLogs( + // Retry to absorb CloudWatch Logs propagation lag: events can take a few + // seconds to become queryable after the agent flushes. + err = awsservice.ValidateLogsWithRetry( instanceId, instanceId, &start, &end, + logValidationAttempts, + logValidationInterval, awsservice.AssertLogsCount(param.numExpectedLogs), awsservice.AssertNoDuplicateLogs(), ) diff --git a/util/awsservice/cloudwatchlogs.go b/util/awsservice/cloudwatchlogs.go index 02832b9dc..8906184c2 100644 --- a/util/awsservice/cloudwatchlogs.go +++ b/util/awsservice/cloudwatchlogs.go @@ -70,6 +70,30 @@ func ValidateLogs(logGroup, logStream string, since, until *time.Time, validator return nil } +// ValidateLogsWithRetry is ValidateLogs with bounded re-validation. GetLogsSince does not +// retry a stream that exists but whose events CloudWatch has not surfaced yet, so a single +// query right after the agent stops is racy; should re-query. +func ValidateLogsWithRetry(logGroup, logStream string, since, until *time.Time, attempts int, interval time.Duration, validators ...LogEventsValidator) error { + if attempts < 1 { + attempts = 1 + } + var err error + for attempt := 1; attempt <= attempts; attempt++ { + err = ValidateLogs(logGroup, logStream, since, until, validators...) + if err == nil { + if attempt > 1 { + log.Printf("Log validation for %s/%s passed on attempt %d/%d", logGroup, logStream, attempt, attempts) + } + return nil + } + if attempt < attempts { + log.Printf("Log validation for %s/%s failed on attempt %d/%d: %v; retrying in %s", logGroup, logStream, attempt, attempts, err, interval) + time.Sleep(interval) + } + } + return fmt.Errorf("log validation for %s/%s failed after %d attempts over %s: %w", logGroup, logStream, attempts, time.Duration(attempts-1)*interval, err) +} + // GetLogsSince makes GetLogEvents API calls, paginates through the results for the given time frame, and returns // the raw log strings func GetLogsSince(logGroup, logStream string, since, until *time.Time) ([]types.OutputLogEvent, error) { From 68608ba5d3d5c29c5d6d3e6e577a24661974f3da Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Thu, 3 Sep 2026 03:18:16 +0000 Subject: [PATCH 2/3] restore full test matrix --- generator/test_case_generator.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 5c2ccacb6..938793aaa 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -615,25 +615,6 @@ var partitionTests = map[string]partition{ }, } -// filterByTestDir returns a copy of configMap keeping only testConfigs whose testDir -// contains substr. Every test type key is preserved (with a possibly empty slice) so that -// main still writes a valid — if empty — matrix file for each, avoiding missing-file -// failures downstream. TEMPORARY(cloudwatchlogs-flaky): scopes CI to a single suite while -// iterating; remove with its call site in main(). -func filterByTestDir(configMap map[string][]testConfig, substr string) map[string][]testConfig { - filtered := make(map[string][]testConfig, len(configMap)) - for testType, testConfigs := range configMap { - kept := make([]testConfig, 0, len(testConfigs)) - for _, tc := range testConfigs { - if strings.Contains(tc.testDir, substr) { - kept = append(kept, tc) - } - } - filtered[testType] = kept - } - return filtered -} - func main() { useE2E := flag.Bool("e2e", false, "Use e2e test matrix generation") flag.Parse() @@ -643,12 +624,6 @@ func main() { configMap = testTypeToTestConfigE2E } - // TEMPORARY(cloudwatchlogs-flaky): scope every generated matrix to the - // cloudwatchlogs suite so CI runs only those tests while the flaky-test fix - // is validated. Delete this line (and filterByTestDir below) to restore the - // full matrix before merging. - configMap = filterByTestDir(configMap, "cloudwatchlogs") - for testType, testConfigs := range configMap { for _, partition := range partitionTests { if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) { From 12665e23b2404dc2a4c93ac2938508093cb93342 Mon Sep 17 00:00:00 2001 From: bhavya76 Date: Thu, 3 Sep 2026 14:22:58 +0000 Subject: [PATCH 3/3] test(cloudwatchlogs): retry IsLogGroupExists to de-flake TestLogGroupClass --- test/cloudwatchlogs/publish_logs_test.go | 4 +++- util/awsservice/cloudwatchlogs.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/cloudwatchlogs/publish_logs_test.go b/test/cloudwatchlogs/publish_logs_test.go index f02bd3d05..8f02ee568 100644 --- a/test/cloudwatchlogs/publish_logs_test.go +++ b/test/cloudwatchlogs/publish_logs_test.go @@ -29,6 +29,8 @@ const ( logLineId2 = "bar" logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows sleepForFlush = 20 * time.Second // default flush interval is 5 seconds + logGroupExistsAttempts = 6 + logGroupExistsInterval = 15 * time.Second configPathAutoRemoval = "resources/config_auto_removal.json" standardLogGroupClass = "STANDARD" infrequentAccessLogGroupClass = "INFREQUENT_ACCESS" @@ -287,7 +289,7 @@ func TestLogGroupClass(t *testing.T) { } t.Logf("Agent logs %s", string(agentLog)) - assert.True(t, awsservice.IsLogGroupExists(logGroupName, param.logGroupClass)) + assert.True(t, awsservice.IsLogGroupExistsWithRetry(logGroupName, logGroupExistsAttempts, logGroupExistsInterval, param.logGroupClass)) }) } } diff --git a/util/awsservice/cloudwatchlogs.go b/util/awsservice/cloudwatchlogs.go index 02832b9dc..aabdf9b90 100644 --- a/util/awsservice/cloudwatchlogs.go +++ b/util/awsservice/cloudwatchlogs.go @@ -152,6 +152,18 @@ func IsLogGroupExists(logGroupName string, logGroupClassArg ...types.LogGroupCla return len(describeLogGroupOutput.LogGroups) > 0 } +func IsLogGroupExistsWithRetry(logGroupName string, attempts int, interval time.Duration, logGroupClassArg ...types.LogGroupClass) bool { + for i := 0; i < attempts; i++ { + if IsLogGroupExists(logGroupName, logGroupClassArg...) { + return true + } + if i < attempts-1 { + time.Sleep(interval) + } + } + return false +} + // GetLogQueryStats for the log group between start/end (in epoch seconds) for the // query string. func GetLogQueryStats(logGroupName string, startTime, endTime int64, queryString string) (*types.QueryStatistics, error) {