Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions .github/actions/prepare-operator-minikube-image/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Gets the operator image into minikube and deploys it. Source mode builds from
# source (real versions baked). Digest mode pulls the released image by digest
# and overrides instrumentation images from versions.txt (like Helm/addon do).
name: Prepare and deploy operator in minikube
description: Build from source or pull a released digest, load into minikube, deploy, and (digest mode) set real instrumentation images.

inputs:
operator-image-uri:
description: 'ECR repo URI of a released operator image to validate by digest. Empty => build from source.'
required: false
default: ''
operator-image-digest:
description: 'Immutable sha256:... digest to validate (digest mode).'
required: false
default: ''
region:
description: 'AWS region for ECR login (digest mode).'
required: false
default: 'us-west-2'
aws-role:
description: 'IAM role ARN to assume for ECR pull (digest mode).'
required: false
default: ''

runs:
using: composite
steps:
- name: Build image from source
if: ${{ inputs.operator-image-digest == '' }}
shell: bash
run: |
eval $(minikube docker-env)
make container
docker images

- name: Configure AWS credentials
if: ${{ inputs.operator-image-digest != '' }}
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
with:
role-to-assume: ${{ inputs.aws-role }}
aws-region: ${{ inputs.region }}

- name: Login to ECR
if: ${{ inputs.operator-image-digest != '' }}
uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1

- name: Pull released image by digest and load into minikube
if: ${{ inputs.operator-image-digest != '' }}
shell: bash
env:
LOCAL_IMG: aws/cloudwatch-agent-operator:release-validation
run: |
set -euo pipefail
RELEASED="${{ inputs.operator-image-uri }}@${{ inputs.operator-image-digest }}"
echo "Validating released operator image: ${RELEASED}"
docker pull "${RELEASED}"
docker tag "${RELEASED}" "${LOCAL_IMG}"
minikube image load "${LOCAL_IMG}"

- name: Deploy operator
shell: bash
run: |
set -euo pipefail
if [ -n "${{ inputs.operator-image-digest }}" ]; then
make deploy IMG=aws/cloudwatch-agent-operator:release-validation
else
make deploy
fi

# Released images default instrumentation refs to 0.0.0; supply the real,
# published images.
- name: Override instrumentation images from source
if: ${{ inputs.operator-image-digest != '' }}
shell: bash
run: |
set -euo pipefail
ver() { grep "^$1=" versions.txt | cut -d '=' -f2; }
repo() { grep "${1}ImageRepository" main.go | head -1 | cut -d '"' -f2; }
kubectl set env deployment/cloudwatch-controller-manager -n amazon-cloudwatch \
RELATED_IMAGE_AUTO_INSTRUMENTATION_JAVA=$(repo autoInstrumentationJava):$(ver aws-otel-java-instrumentation) \
RELATED_IMAGE_AUTO_INSTRUMENTATION_PYTHON=$(repo autoInstrumentationPython):$(ver aws-otel-python-instrumentation) \
RELATED_IMAGE_AUTO_INSTRUMENTATION_DOTNET=$(repo autoInstrumentationDotNet):$(ver aws-otel-dotnet-instrumentation) \
RELATED_IMAGE_AUTO_INSTRUMENTATION_NODEJS=$(repo autoInstrumentationNodeJS):$(ver aws-otel-nodejs-instrumentation)
kubectl rollout status deployment/cloudwatch-controller-manager -n amazon-cloudwatch --timeout=180s

- name: Wait for operator ready
shell: bash
run: kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch --timeout=180s
175 changes: 128 additions & 47 deletions .github/workflows/operator-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
# SPDX-License-Identifier: Apache-2.0

name: Operator Integration Test
# Release-image-validation dispatches pass a correlation-id; those runs get a
# title carrying it so the checker can find the exact run it triggered.
run-name: ${{ inputs.correlation-id != '' && format('operator-image-validation [{0}]', inputs.correlation-id) || 'Operator Integration Test' }}
on:
push:
branches:
Expand All @@ -11,6 +14,45 @@ on:
branches:
- main
workflow_dispatch:
inputs:
operator-image-uri:
description: 'ECR repo URI of a released operator image to validate by digest (leave empty to build from source)'
required: false
type: string
default: ''
operator-image-digest:
description: 'Digest of the released operator image'
required: false
type: string
default: ''
correlation-id:
description: 'Unique per-dispatch id used to correlate this run back to its trigger'
required: false
type: string
default: ''
region:
description: 'AWS region for ECR login (digest mode)'
required: false
type: string
default: 'us-west-2'
workflow_call:
inputs:
operator-image-uri:
required: false
type: string
default: ''
operator-image-digest:
required: false
type: string
default: ''
correlation-id:
required: false
type: string
default: ''
region:
required: false
type: string
default: 'us-west-2'
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know this is not part of this PR but maybe something to consider is if we are triggering manual run and at the same time some one merges a PR it will cancel our manual run, do we want this behaviour?

@Paamicky Paamicky Aug 26, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes because a merge to main takes top priority as we care about what was just merged, rather than the manual one. A manual run of your branch won't get cancelled.

Expand Down Expand Up @@ -44,17 +86,14 @@ jobs:
sleep 10
kubectl get pods -A

- name: Build image
run: |
eval $(minikube docker-env)
make container
docker images

- name: Deploy operator to minikube
run: |
make deploy
kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch

- name: Prepare operator image
id: opimg
uses: ./.github/actions/prepare-operator-minikube-image
with:
operator-image-uri: ${{ inputs.operator-image-uri }}
operator-image-digest: ${{ inputs.operator-image-digest }}
region: ${{ inputs.region }}
aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}

- name: Test case for cloudwatch agent pod creation
run: |
Expand Down Expand Up @@ -228,6 +267,16 @@ jobs:
go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
kubectl delete instrumentation sample-instrumentation

# Check pod state/events/operator logs for any failure.
- name: Dump diagnostics on failure
if: failure()
run: |
kubectl get pods -A -o wide || true
kubectl describe pods -n default || true
kubectl describe pods -n amazon-cloudwatch || true
kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true
kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true

DeploymentAnnotationsTest:
name: DeploymentAnnotationsTest
runs-on: ubuntu-latest
Expand All @@ -252,15 +301,14 @@ jobs:
sleep 10
kubectl get pods -A

- name: Build image
run: |
eval $(minikube docker-env)
make container
docker images

- name: Deploy operator to minikube
run: |
make deploy
- name: Prepare operator image
id: opimg
uses: ./.github/actions/prepare-operator-minikube-image
with:
operator-image-uri: ${{ inputs.operator-image-uri }}
operator-image-digest: ${{ inputs.operator-image-digest }}
region: ${{ inputs.region }}
aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}

- name: Test Annotations
run: |
Expand All @@ -281,6 +329,15 @@ jobs:
sleep 5
go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m

- name: Dump diagnostics on failure
if: failure()
run: |
kubectl get pods -A -o wide || true
kubectl describe pods -n default || true
kubectl describe pods -n amazon-cloudwatch || true
kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true
kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true

DaemonsetAnnotationsTest:
name: DaemonsetAnnotationsTest
runs-on: ubuntu-latest
Expand All @@ -305,15 +362,14 @@ jobs:
sleep 10
kubectl get pods -A

- name: Build image
run: |
eval $(minikube docker-env)
make container
docker images

- name: Deploy operator to minikube
run: |
make deploy
- name: Prepare operator image
id: opimg
uses: ./.github/actions/prepare-operator-minikube-image
with:
operator-image-uri: ${{ inputs.operator-image-uri }}
operator-image-digest: ${{ inputs.operator-image-digest }}
region: ${{ inputs.region }}
aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}

- name: Test Annotations
run: |
Expand All @@ -334,6 +390,15 @@ jobs:
sleep 5
go test -v -run TestAutoAnnotationForManualAnnotationRemoval ./integration-tests/manifests/annotations -timeout 30m

- name: Dump diagnostics on failure
if: failure()
run: |
kubectl get pods -A -o wide || true
kubectl describe pods -n default || true
kubectl describe pods -n amazon-cloudwatch || true
kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true
kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true

StatefulsetAnnotationsTest:
name: StatefulsetAnnotationsTest
runs-on: ubuntu-latest
Expand All @@ -358,15 +423,14 @@ jobs:
sleep 10
kubectl get pods -A

- name: Build image
run: |
eval $(minikube docker-env)
make container
docker images

- name: Deploy operator to minikube
run: |
make deploy
- name: Prepare operator image
id: opimg
uses: ./.github/actions/prepare-operator-minikube-image
with:
operator-image-uri: ${{ inputs.operator-image-uri }}
operator-image-digest: ${{ inputs.operator-image-digest }}
region: ${{ inputs.region }}
aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}

- name: Test Annotations
run: |
Expand All @@ -388,6 +452,15 @@ jobs:



- name: Dump diagnostics on failure
if: failure()
run: |
kubectl get pods -A -o wide || true
kubectl describe pods -n default || true
kubectl describe pods -n amazon-cloudwatch || true
kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true
kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true

NamespaceAnnotationsTest:
name: NamespaceAnnotationsTest
runs-on: ubuntu-latest
Expand All @@ -411,15 +484,14 @@ jobs:
run: |
kubectl get pods -A

- name: Build image
run: |
eval $(minikube docker-env)
make container
docker images

- name: Deploy operator to minikube
run: |
make deploy
- name: Prepare operator image
id: opimg
uses: ./.github/actions/prepare-operator-minikube-image
with:
operator-image-uri: ${{ inputs.operator-image-uri }}
operator-image-digest: ${{ inputs.operator-image-digest }}
region: ${{ inputs.region }}
aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}

- name: Test Annotations
run: |
Expand All @@ -436,3 +508,12 @@ jobs:
go test -v -run TestNodeJSOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m
sleep 5
go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m

- name: Dump diagnostics on failure
if: failure()
run: |
kubectl get pods -A -o wide || true
kubectl describe pods -n default || true
kubectl describe pods -n amazon-cloudwatch || true
kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true
kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true
Loading