From fcfbc28a23e475097ed828b14f7f632a086cf9f4 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Mon, 24 Aug 2026 16:20:47 +0000 Subject: [PATCH 1/4] Add digest validation for operator release --- .../action.yml | 80 +++++++++++++ .../workflows/operator-integration-test.yml | 105 +++++++++++++----- 2 files changed, 155 insertions(+), 30 deletions(-) create mode 100644 .github/actions/prepare-operator-minikube-image/action.yml diff --git a/.github/actions/prepare-operator-minikube-image/action.yml b/.github/actions/prepare-operator-minikube-image/action.yml new file mode 100644 index 000000000..bb6dde427 --- /dev/null +++ b/.github/actions/prepare-operator-minikube-image/action.yml @@ -0,0 +1,80 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Prepares the operator image inside the running minikube for the integration +# tests. Two modes: +# * source (default): build the image from the checked-out source (`make container`) +# * digest: pull the EXACT released image by immutable digest and load it, +# so the same test suite validates the artifact we are releasing. +# +# Outputs `img-arg`, the `make deploy` IMG= override to use (empty in source mode). +name: Prepare operator image in minikube +description: Build the operator from source, or pull+load a released image by digest. + +inputs: + operator-image-uri: + description: 'ECR repo URI of the released operator image (digest mode). Empty => build from source.' + required: false + default: '' + operator-image-digest: + description: 'Immutable sha256:... digest to validate (digest mode).' + required: false + default: '' + region: + description: 'AWS region for ECR login (digest mode).' + required: false + default: 'us-west-2' + aws-role: + description: 'IAM role ARN to assume for ECR pull (digest mode).' + required: false + default: '' + +outputs: + img-arg: + description: 'make deploy IMG= override (empty in source mode).' + value: ${{ steps.resolve.outputs.img-arg }} + +runs: + using: composite + steps: + - name: Build image from source + if: ${{ inputs.operator-image-digest == '' }} + shell: bash + run: | + eval $(minikube docker-env) + make container + docker images + + - name: Configure AWS credentials + if: ${{ inputs.operator-image-digest != '' }} + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.region }} + + - name: Login to ECR + if: ${{ inputs.operator-image-digest != '' }} + uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 + + - name: Pull released image by digest and load into minikube + if: ${{ inputs.operator-image-digest != '' }} + shell: bash + env: + LOCAL_IMG: aws/cloudwatch-agent-operator:release-validation + run: | + set -euo pipefail + RELEASED="${{ inputs.operator-image-uri }}@${{ inputs.operator-image-digest }}" + echo "Validating released operator image: ${RELEASED}" + docker pull "${RELEASED}" + docker tag "${RELEASED}" "${LOCAL_IMG}" + minikube image load "${LOCAL_IMG}" + + - name: Resolve deploy IMG arg + id: resolve + shell: bash + run: | + if [ -n "${{ inputs.operator-image-digest }}" ]; then + echo "img-arg=IMG=aws/cloudwatch-agent-operator:release-validation" >> "$GITHUB_OUTPUT" + else + echo "img-arg=" >> "$GITHUB_OUTPUT" + fi diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index b01d95682..e6ac989cf 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -11,6 +11,36 @@ on: branches: - main workflow_dispatch: + inputs: + operator-image-uri: + description: 'ECR repo URI of a released operator image to validate by digest (leave empty to build from source)' + required: false + type: string + default: '' + operator-image-digest: + description: 'Digest of the released operator image' + required: false + type: string + default: '' + region: + description: 'AWS region for ECR login (digest mode)' + required: false + type: string + default: 'us-west-2' + workflow_call: + inputs: + operator-image-uri: + required: false + type: string + default: '' + operator-image-digest: + required: false + type: string + default: '' + region: + required: false + type: string + default: 'us-west-2' concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} cancel-in-progress: true @@ -44,15 +74,18 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Deploy operator to minikube run: | - make deploy + make deploy ${{ steps.opimg.outputs.img-arg }} kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch @@ -252,15 +285,18 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Deploy operator to minikube run: | - make deploy + make deploy ${{ steps.opimg.outputs.img-arg }} - name: Test Annotations run: | @@ -305,15 +341,18 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Deploy operator to minikube run: | - make deploy + make deploy ${{ steps.opimg.outputs.img-arg }} - name: Test Annotations run: | @@ -358,15 +397,18 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Deploy operator to minikube run: | - make deploy + make deploy ${{ steps.opimg.outputs.img-arg }} - name: Test Annotations run: | @@ -411,15 +453,18 @@ jobs: run: | kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Deploy operator to minikube run: | - make deploy + make deploy ${{ steps.opimg.outputs.img-arg }} - name: Test Annotations run: | From 1646f6d9bcdd07bf7ff1cdac55e20cc0f924a2d5 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Tue, 25 Aug 2026 13:38:24 +0000 Subject: [PATCH 2/4] Add debug failure mode --- .../workflows/operator-integration-test.yml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index e6ac989cf..bc88c9c6a 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -261,6 +261,17 @@ jobs: go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals kubectl delete instrumentation sample-instrumentation + # CWQS-2726 (temporary): on failure, dump pod state/events/operator logs to + # find why injected/agent pods don't reach Running. Remove once diagnosed. + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + DeploymentAnnotationsTest: name: DeploymentAnnotationsTest runs-on: ubuntu-latest @@ -317,6 +328,15 @@ jobs: sleep 5 go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + DaemonsetAnnotationsTest: name: DaemonsetAnnotationsTest runs-on: ubuntu-latest @@ -373,6 +393,15 @@ jobs: sleep 5 go test -v -run TestAutoAnnotationForManualAnnotationRemoval ./integration-tests/manifests/annotations -timeout 30m + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + StatefulsetAnnotationsTest: name: StatefulsetAnnotationsTest runs-on: ubuntu-latest @@ -430,6 +459,15 @@ jobs: + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + NamespaceAnnotationsTest: name: NamespaceAnnotationsTest runs-on: ubuntu-latest @@ -481,3 +519,12 @@ jobs: go test -v -run TestNodeJSOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m sleep 5 go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m + + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true \ No newline at end of file From 2b510bb8d0c58af1d0c550ed6955d981637f8945 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Tue, 25 Aug 2026 16:48:25 +0000 Subject: [PATCH 3/4] fix instrumentation error --- .../action.yml | 49 ++++++++++++------- .../workflows/operator-integration-test.yml | 25 +--------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/.github/actions/prepare-operator-minikube-image/action.yml b/.github/actions/prepare-operator-minikube-image/action.yml index bb6dde427..57b3dc156 100644 --- a/.github/actions/prepare-operator-minikube-image/action.yml +++ b/.github/actions/prepare-operator-minikube-image/action.yml @@ -1,19 +1,15 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # -# Prepares the operator image inside the running minikube for the integration -# tests. Two modes: -# * source (default): build the image from the checked-out source (`make container`) -# * digest: pull the EXACT released image by immutable digest and load it, -# so the same test suite validates the artifact we are releasing. -# -# Outputs `img-arg`, the `make deploy` IMG= override to use (empty in source mode). -name: Prepare operator image in minikube -description: Build the operator from source, or pull+load a released image by digest. +# Gets the operator image into minikube and deploys it. Source mode builds from +# source (real versions baked). Digest mode pulls the released image by digest +# and overrides instrumentation images from versions.txt (like Helm/addon do). +name: Prepare and deploy operator in minikube +description: Build from source or pull a released digest, load into minikube, deploy, and (digest mode) set real instrumentation images. inputs: operator-image-uri: - description: 'ECR repo URI of the released operator image (digest mode). Empty => build from source.' + description: 'ECR repo URI of a released operator image to validate by digest. Empty => build from source.' required: false default: '' operator-image-digest: @@ -29,11 +25,6 @@ inputs: required: false default: '' -outputs: - img-arg: - description: 'make deploy IMG= override (empty in source mode).' - value: ${{ steps.resolve.outputs.img-arg }} - runs: using: composite steps: @@ -69,12 +60,32 @@ runs: docker tag "${RELEASED}" "${LOCAL_IMG}" minikube image load "${LOCAL_IMG}" - - name: Resolve deploy IMG arg - id: resolve + - name: Deploy operator shell: bash run: | + set -euo pipefail if [ -n "${{ inputs.operator-image-digest }}" ]; then - echo "img-arg=IMG=aws/cloudwatch-agent-operator:release-validation" >> "$GITHUB_OUTPUT" + make deploy IMG=aws/cloudwatch-agent-operator:release-validation else - echo "img-arg=" >> "$GITHUB_OUTPUT" + make deploy fi + + # Released images default instrumentation refs to 0.0.0; supply the real, + # published images. + - name: Override instrumentation images from source + if: ${{ inputs.operator-image-digest != '' }} + shell: bash + run: | + set -euo pipefail + ver() { grep "^$1=" versions.txt | cut -d '=' -f2; } + repo() { grep "${1}ImageRepository" main.go | head -1 | cut -d '"' -f2; } + kubectl set env deployment/cloudwatch-controller-manager -n amazon-cloudwatch \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_JAVA=$(repo autoInstrumentationJava):$(ver aws-otel-java-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_PYTHON=$(repo autoInstrumentationPython):$(ver aws-otel-python-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_DOTNET=$(repo autoInstrumentationDotNet):$(ver aws-otel-dotnet-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_NODEJS=$(repo autoInstrumentationNodeJS):$(ver aws-otel-nodejs-instrumentation) + kubectl rollout status deployment/cloudwatch-controller-manager -n amazon-cloudwatch --timeout=180s + + - name: Wait for operator ready + shell: bash + run: kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch --timeout=180s diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index bc88c9c6a..e0b85ce42 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -83,12 +83,6 @@ jobs: region: ${{ inputs.region }} aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - - name: Deploy operator to minikube - run: | - make deploy ${{ steps.opimg.outputs.img-arg }} - kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch - - - name: Test case for cloudwatch agent pod creation run: | kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset.yaml -n amazon-cloudwatch @@ -261,8 +255,7 @@ jobs: go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals kubectl delete instrumentation sample-instrumentation - # CWQS-2726 (temporary): on failure, dump pod state/events/operator logs to - # find why injected/agent pods don't reach Running. Remove once diagnosed. + # Check pod state/events/operator logs for any failure. - name: Dump diagnostics on failure if: failure() run: | @@ -305,10 +298,6 @@ jobs: region: ${{ inputs.region }} aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - - name: Deploy operator to minikube - run: | - make deploy ${{ steps.opimg.outputs.img-arg }} - - name: Test Annotations run: | kubectl get pods -A @@ -370,10 +359,6 @@ jobs: region: ${{ inputs.region }} aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - - name: Deploy operator to minikube - run: | - make deploy ${{ steps.opimg.outputs.img-arg }} - - name: Test Annotations run: | sleep 5 @@ -435,10 +420,6 @@ jobs: region: ${{ inputs.region }} aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - - name: Deploy operator to minikube - run: | - make deploy ${{ steps.opimg.outputs.img-arg }} - - name: Test Annotations run: | kubectl get pods -A @@ -500,10 +481,6 @@ jobs: region: ${{ inputs.region }} aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - - name: Deploy operator to minikube - run: | - make deploy ${{ steps.opimg.outputs.img-arg }} - - name: Test Annotations run: | kubectl get pods -A From 7708bcc95714a699635d5d9af612438a84f928fd Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Wed, 26 Aug 2026 20:14:05 +0000 Subject: [PATCH 4/4] Add digest to run name --- .github/workflows/operator-integration-test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index e0b85ce42..28e3186ed 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -2,6 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 name: Operator Integration Test +# Release-image-validation dispatches pass a correlation-id; those runs get a +# title carrying it so the checker can find the exact run it triggered. +run-name: ${{ inputs.correlation-id != '' && format('operator-image-validation [{0}]', inputs.correlation-id) || 'Operator Integration Test' }} on: push: branches: @@ -22,6 +25,11 @@ on: required: false type: string default: '' + correlation-id: + description: 'Unique per-dispatch id used to correlate this run back to its trigger' + required: false + type: string + default: '' region: description: 'AWS region for ECR login (digest mode)' required: false @@ -37,6 +45,10 @@ on: required: false type: string default: '' + correlation-id: + required: false + type: string + default: '' region: required: false type: string