evals: replace framework with custom ADK harness

.github/workflows/ci.yml

@@ -18,6 +18,9 @@ jobs:
         uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
           go-version-file: go.mod
+          cache-dependency-path: |
+            go.sum
+            evals/go.sum
 
       - name: Read golangci-lint version
         id: golangci-lint-version

.github/workflows/eval.yml

@@ -3,53 +3,57 @@ name: Skill Evals
 on:
   workflow_dispatch:
     inputs:
-      model:
-        description: "Model to evaluate against"
+      AWS_ACCESS_KEY_ID:
+        description: AWS_ACCESS_KEY_ID, as provisioned by doormat
+        required: true
+        type: string
+      AWS_SECRET_ACCESS_KEY:
+        description: AWS_SECRET_ACCESS_KEY, as provisioned by doormat
+        required: true
+      AWS_SESSION_TOKEN:
+        description: AWS_SESSION_TOKEN, as provisioned by doormat
         required: true
-        default: "claude-sonnet-4.6"
-        type: choice
-        options:
-          - claude-sonnet-4.6
-          - claude-opus-4.6
-          - gpt-4.1
-          - gpt-5.2
-      tasks:
-        description: "Task filter glob (blank = all)"
-        required: false
         type: string
-      tags:
-        description: "Tag filter (blank = all)"
-        required: false
+      AWS_REGION:
+        description: AWS_REGION, usually us-west-2
+        default: us-west-2
+        type: string
+      model:
+        description: "Bedrock model ID or cross-region inference profile ID"
+        required: true
+        default: "us.openai.gpt-5.6-luna"
         type: string
 
 jobs:
   eval:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      EVAL_OUTPUT: evals/results/ci.json
+      EVAL_PROVIDER: bedrock
+      EVAL_MODEL: ${{ inputs.model }}
+      AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
+      AWS_SESSION_TOKEN: ${{ inputs.AWS_SESSION_TOKEN }}
+      AWS_REGION: ${{ inputs.AWS_REGION }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
-      - name: Install waza
-        run: curl -fsSL https://raw.githubusercontent.com/microsoft/waza/main/install.sh | bash
+      - name: Set up Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: |
+            go.sum
+            evals/go.sum
 
       - name: Run evals
-        working-directory: evals/tfctl-evals
-        env:
-          # Uses the Actions-provided GITHUB_TOKEN if org has Copilot enabled.
-          # Falls back to COPILOT_TOKEN secret if that doesn't work.
-          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_TOKEN || github.token }}
-        run: |
-          ARGS="--model ${{ inputs.model }}"
-          if [ -n "${{ inputs.tasks }}" ]; then
-            ARGS="$ARGS --task '${{ inputs.tasks }}'"
-          fi
-          if [ -n "${{ inputs.tags }}" ]; then
-            ARGS="$ARGS --tags '${{ inputs.tags }}'"
-          fi
-          eval waza run evals/tfctl/eval.yaml $ARGS -o results.json
+        run: make eval/save
 
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: eval-results-${{ inputs.model }}
-          path: evals/tfctl-evals/results.json
+          path: evals/results/ci.json

.gitignore

@@ -1,3 +1,4 @@
 dist/tfctl
 .plans/
 dist/
+evals/results/

Makefile

@@ -5,6 +5,8 @@ ASSETS ?= assets
 VERSION_FILE ?= version/VERSION
 SKILL_HASHES = skills/tfctl/known_release_hashes
 SKILL_EMBEDDED = skills/tfctl/SKILL.md
+EVAL_ARGS ?=
+EVAL_OUTPUT ?= evals/results/latest.json
 CHANGELOG_FILE = CHANGELOG.md
 
 ifeq ($(GOARCH), arm64)
@@ -119,8 +121,24 @@ logotools:
 		echo "Install figlet https://www.figlet.org/" && exit 1; \
 	}
 
+.PHONY: eval/test
+eval/test:
+	@$(MAKE) -C evals test
+
+.PHONY: eval/lint
+eval/lint:
+	@$(MAKE) -C evals lint
+
 .PHONY: check
-check: fmt-check go/lint go/test
+check: fmt-check go/lint go/test eval/lint eval/test
+
+.PHONY: eval
+eval: go/install
+	@PATH="$(abspath $(dir $(BIN_PATH))):$$PATH" go -C evals run . $(EVAL_ARGS)
+
+.PHONY: eval/save
+eval/save: go/install
+	@PATH="$(abspath $(dir $(BIN_PATH))):$$PATH" go -C evals run . --output "$(abspath $(EVAL_OUTPUT))" $(EVAL_ARGS)
 
 # Help (make usage)
 .PHONY: help
@@ -151,4 +169,10 @@ help:
 	@echo "                  requires VERSION argument"
 	@echo " cleanup-release  Clean up after a release"
 	@echo "                  requires DEV_VERSION argument"
-	@echo ""
+	@echo ""
+	@echo "Evaluations:"
+	@echo " eval             Run skill evaluations"
+	@echo " eval/save        Run and save evaluation results"
+	@echo " eval/test        Test the evaluator module"
+	@echo " eval/lint        Lint the evaluator module"
+	@echo ""

evals/Makefile

@@ -0,0 +1,7 @@
+.PHONY: test
+test:
+	@go test ./...
+
+.PHONY: lint
+lint:
+	@golangci-lint run

evals/README.md

@@ -0,0 +1,107 @@
+# tfctl Skill Evaluations
+
+The evaluation runner sends each YAML task to a simple, custom, Google ADK agent with
+`skills/tfctl/SKILL.md` as its instructions. The agent has one function tool,
+`tfctl(args)`.
+
+The runner executes ordinary `tfctl` requests with a temporary configuration
+directory. A request that starts with `tfctl api`, `tfctl get`, or `tfctl create`
+is not executed. The runner records that request, stops the agent, and grades
+the recorded isolated request. This prevents a platform request and lets each
+task run independently. If the agent makes no isolated request, the runner
+grades its visible text output instead. The saved output includes model
+reasoning, but task checks do not grade it.
+
+The evaluator is an independent Go module. Run repository-level Make targets
+from the repository root; they build the current `tfctl` source before running.
+For direct `go -C evals` commands, install `tfctl` on `PATH` first.
+
+## Providers
+
+ADK Go's only OpenAI-shaped model connector
+(`google.golang.org/adk/v2/model/openaimodel`) speaks the newer Responses
+API, which neither a local OpenAI-compatible server nor AWS Bedrock speaks
+natively. Rather than run a translating proxy (e.g. LiteLLM) in front of
+either one, the runner talks to both directly through two small adapters
+in `evals/internal/model`:
+
+- `openaichat`: calls a Chat Completions endpoint (the format local model
+  servers such as llama.cpp, vLLM, and Ollama actually speak).
+- `bedrockconverse`: calls the AWS Bedrock Converse API using the standard
+  AWS SDK credential chain.
+
+Select a provider with `--provider` or `EVAL_PROVIDER`:
+
+```sh
+# A local OpenAI-compatible server
+EVAL_PROVIDER=openai EVAL_MODEL=qwen3.8-Q8 make eval
+
+# AWS Bedrock
+EVAL_PROVIDER=bedrock EVAL_MODEL=us.openai.gpt-5.6-luna make eval/save
+```
+
+For the `openai` provider, `--base-url`/`EVAL_BASE_URL` sets the Chat
+Completions base URL (default `http://127.0.0.1:8000/v1`) and
+`--api-key`/`EVAL_API_KEY` sets an optional API key. `--model`/`EVAL_MODEL`
+is the model name the server expects.
+
+For the `bedrock` provider, `--model`/`EVAL_MODEL` is a Bedrock model ID or
+cross-region inference profile ID (e.g. `us.anthropic.claude-...`).
+Credentials and region come from the standard AWS SDK default chain:
+`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, optional `AWS_SESSION_TOKEN`,
+`AWS_REGION`, or an `AWS_PROFILE`.
+
+`EVAL_PROVIDER` and `EVAL_MODEL` are required when `--provider`/`--model`
+are not set. Do not set `GOOGLE_API_KEY`; the runner does not call Gemini.
+
+## Run
+
+```sh
+EVAL_PROVIDER=openai EVAL_MODEL=qwen3.8-Q8 make eval
+EVAL_PROVIDER=bedrock EVAL_MODEL=us.openai.gpt-5.6-luna make eval/save
+```
+
+The runner accepts `--provider`, `--model`, `--base-url`, `--api-key`,
+`--output`, `--tags`, `--json`, and `--task`. Flags override the corresponding
+`EVAL_*` environment variables. `--tags` accepts comma-separated tags; `--task`
+accepts a filename glob or substring.
+
+## Tasks
+
+Tasks live in `evals/tasks/` and use this strict schema:
+
+```yaml
+task: |
+  List all workspaces and show their names.
+tags: [api-pattern, pagination]
+accept:
+  - '--all'
+  - '(?:/plans/)|(?:/applies/)'
+reject: ['\|\s*jq']
+turns: 10
+```
+
+Every `accept` expression must match the isolated invocation, or the visible
+assistant output when there is no isolated invocation. Every `reject`
+expression must not match. Expressions use Go's RE2-compatible syntax and are
+automatically case-insensitive. Use alternation such as
+`(?:first)|(?:second)` when any accepted form is sufficient. Prefer the
+smallest patterns that express required flags, paths, or request data. Invalid
+regular expressions are task validation errors. At least one check is required.
+The stable task ID comes from the filename with its numeric prefix and `.yaml`
+
+Generated files under `evals/results/` are ignored. Override the output path
+and runner arguments when needed:
+
+```sh
+make eval/save EVAL_OUTPUT=evals/results/pagination.json EVAL_ARGS='--tags pagination'
+```
+
+Run evaluator checks independently with `make eval/test` and `make eval/lint`.
+
+## CI
+
+The `Skill Evals` workflow runs against the `bedrock` provider, passes
+runner configuration through `EVAL_*`, and uploads the current JSON result
+even on a failure. AWS credentials (as provisioned by doormat) and the
+Bedrock model ID are supplied as `workflow_dispatch` inputs.

evals/go.mod

@@ -0,0 +1,64 @@
+module github.com/hashicorp/tfctl-cli/evals
+
+go 1.26.5
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.44.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.40
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.39
+	github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.58.0
+	github.com/openai/openai-go/v3 v3.49.0
+	google.golang.org/adk/v2 v2.2.0
+	google.golang.org/genai v1.66.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.22.0 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 // indirect
+	github.com/aws/smithy-go v1.28.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/jsonschema-go v0.4.3 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/safehtml v0.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
+	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/tidwall/gjson v1.19.0 // indirect
+	github.com/tidwall/match v1.2.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
+	go.opentelemetry.io/otel/log v0.20.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	google.golang.org/api v0.291.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df // indirect
+	google.golang.org/grpc v1.83.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	rsc.io/omap v1.2.0 // indirect
+	rsc.io/ordered v1.1.1 // indirect
+)