diff --git a/.github/actions/prepare-operator-minikube-image/action.yml b/.github/actions/prepare-operator-minikube-image/action.yml new file mode 100644 index 000000000..57b3dc156 --- /dev/null +++ b/.github/actions/prepare-operator-minikube-image/action.yml @@ -0,0 +1,91 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Gets the operator image into minikube and deploys it. Source mode builds from +# source (real versions baked). Digest mode pulls the released image by digest +# and overrides instrumentation images from versions.txt (like Helm/addon do). +name: Prepare and deploy operator in minikube +description: Build from source or pull a released digest, load into minikube, deploy, and (digest mode) set real instrumentation images. + +inputs: + operator-image-uri: + description: 'ECR repo URI of a released operator image to validate by digest. Empty => build from source.' + required: false + default: '' + operator-image-digest: + description: 'Immutable sha256:... digest to validate (digest mode).' + required: false + default: '' + region: + description: 'AWS region for ECR login (digest mode).' + required: false + default: 'us-west-2' + aws-role: + description: 'IAM role ARN to assume for ECR pull (digest mode).' + required: false + default: '' + +runs: + using: composite + steps: + - name: Build image from source + if: ${{ inputs.operator-image-digest == '' }} + shell: bash + run: | + eval $(minikube docker-env) + make container + docker images + + - name: Configure AWS credentials + if: ${{ inputs.operator-image-digest != '' }} + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.region }} + + - name: Login to ECR + if: ${{ inputs.operator-image-digest != '' }} + uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 + + - name: Pull released image by digest and load into minikube + if: ${{ inputs.operator-image-digest != '' }} + shell: bash + env: + LOCAL_IMG: aws/cloudwatch-agent-operator:release-validation + run: | + set -euo pipefail + RELEASED="${{ inputs.operator-image-uri }}@${{ inputs.operator-image-digest }}" + echo "Validating released operator image: ${RELEASED}" + docker pull "${RELEASED}" + docker tag "${RELEASED}" "${LOCAL_IMG}" + minikube image load "${LOCAL_IMG}" + + - name: Deploy operator + shell: bash + run: | + set -euo pipefail + if [ -n "${{ inputs.operator-image-digest }}" ]; then + make deploy IMG=aws/cloudwatch-agent-operator:release-validation + else + make deploy + fi + + # Released images default instrumentation refs to 0.0.0; supply the real, + # published images. + - name: Override instrumentation images from source + if: ${{ inputs.operator-image-digest != '' }} + shell: bash + run: | + set -euo pipefail + ver() { grep "^$1=" versions.txt | cut -d '=' -f2; } + repo() { grep "${1}ImageRepository" main.go | head -1 | cut -d '"' -f2; } + kubectl set env deployment/cloudwatch-controller-manager -n amazon-cloudwatch \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_JAVA=$(repo autoInstrumentationJava):$(ver aws-otel-java-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_PYTHON=$(repo autoInstrumentationPython):$(ver aws-otel-python-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_DOTNET=$(repo autoInstrumentationDotNet):$(ver aws-otel-dotnet-instrumentation) \ + RELATED_IMAGE_AUTO_INSTRUMENTATION_NODEJS=$(repo autoInstrumentationNodeJS):$(ver aws-otel-nodejs-instrumentation) + kubectl rollout status deployment/cloudwatch-controller-manager -n amazon-cloudwatch --timeout=180s + + - name: Wait for operator ready + shell: bash + run: kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch --timeout=180s diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index b01d95682..28e3186ed 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -2,6 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 name: Operator Integration Test +# Release-image-validation dispatches pass a correlation-id; those runs get a +# title carrying it so the checker can find the exact run it triggered. +run-name: ${{ inputs.correlation-id != '' && format('operator-image-validation [{0}]', inputs.correlation-id) || 'Operator Integration Test' }} on: push: branches: @@ -11,6 +14,45 @@ on: branches: - main workflow_dispatch: + inputs: + operator-image-uri: + description: 'ECR repo URI of a released operator image to validate by digest (leave empty to build from source)' + required: false + type: string + default: '' + operator-image-digest: + description: 'Digest of the released operator image' + required: false + type: string + default: '' + correlation-id: + description: 'Unique per-dispatch id used to correlate this run back to its trigger' + required: false + type: string + default: '' + region: + description: 'AWS region for ECR login (digest mode)' + required: false + type: string + default: 'us-west-2' + workflow_call: + inputs: + operator-image-uri: + required: false + type: string + default: '' + operator-image-digest: + required: false + type: string + default: '' + correlation-id: + required: false + type: string + default: '' + region: + required: false + type: string + default: 'us-west-2' concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} cancel-in-progress: true @@ -44,17 +86,14 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy - kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch - + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Test case for cloudwatch agent pod creation run: | @@ -228,6 +267,16 @@ jobs: go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals kubectl delete instrumentation sample-instrumentation + # Check pod state/events/operator logs for any failure. + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + DeploymentAnnotationsTest: name: DeploymentAnnotationsTest runs-on: ubuntu-latest @@ -252,15 +301,14 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Test Annotations run: | @@ -281,6 +329,15 @@ jobs: sleep 5 go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + DaemonsetAnnotationsTest: name: DaemonsetAnnotationsTest runs-on: ubuntu-latest @@ -305,15 +362,14 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Test Annotations run: | @@ -334,6 +390,15 @@ jobs: sleep 5 go test -v -run TestAutoAnnotationForManualAnnotationRemoval ./integration-tests/manifests/annotations -timeout 30m + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + StatefulsetAnnotationsTest: name: StatefulsetAnnotationsTest runs-on: ubuntu-latest @@ -358,15 +423,14 @@ jobs: sleep 10 kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Test Annotations run: | @@ -388,6 +452,15 @@ jobs: + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true + NamespaceAnnotationsTest: name: NamespaceAnnotationsTest runs-on: ubuntu-latest @@ -411,15 +484,14 @@ jobs: run: | kubectl get pods -A - - name: Build image - run: | - eval $(minikube docker-env) - make container - docker images - - - name: Deploy operator to minikube - run: | - make deploy + - name: Prepare operator image + id: opimg + uses: ./.github/actions/prepare-operator-minikube-image + with: + operator-image-uri: ${{ inputs.operator-image-uri }} + operator-image-digest: ${{ inputs.operator-image-digest }} + region: ${{ inputs.region }} + aws-role: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} - name: Test Annotations run: | @@ -436,3 +508,12 @@ jobs: go test -v -run TestNodeJSOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m sleep 5 go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m + + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl describe pods -n default || true + kubectl describe pods -n amazon-cloudwatch || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=200 || true \ No newline at end of file