From 69d40fba60af4ec810ddb320c3efe4aef8534a79 Mon Sep 17 00:00:00 2001 From: KirschQAQ <114209152+KirschBluteX@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:16:57 -0700 Subject: [PATCH] test: cover automation preflight requirements --- automations/README.md | 5 +- .../automations/github-pr-reviewer.json | 44 ++++++++++++ .../automations/github-repo-monitor.json | 44 ++++++++++++ .../automations/upstream-fork-sync.json | 44 ++++++++++++ tests/test_automation_setup.py | 68 +++++++++++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) diff --git a/automations/README.md b/automations/README.md index 1dd88d9a..6baca1e5 100644 --- a/automations/README.md +++ b/automations/README.md @@ -44,7 +44,7 @@ two records of the same fact from drifting apart, and it is why the file is as s | The trigger kinds a deployment must support | The keys of `setup.form.triggers` | | Schedule limits and the timezone list | The `cron` and `timezone` field types | | Local validation rules | The `required` flag and `constraints` on each field | -| The preflight call | `POST /v1/validate` with the entry id, the create endpoint, and the rendered payload | +| The preflight call | `POST /v1/validate` with the entry id, the create endpoint, the rendered payload, and required MCP alternatives derived from `requires.integrations` plus the integration catalog | | The created automation's name | The entry's `name`, plus the repository that was picked | | `repos` in the create request | The repo-picker field, its declared `provider`, and a field named `ref` if there is one | | `trigger` in the create request | The key under `form.triggers`, and the fields under it named after trigger properties | @@ -56,6 +56,9 @@ two records of the same fact from drifting apart, and it is why the file is as s `tests/test_automation_setup.py` derives the preflight body and the payload-path mapping from the entry and checks them against the recorded fixtures, so these deletions stay honest rather than becoming assumptions. +The derived integration requirements include only MCP locators, authentication strategies, and required +secret names. Secret values remain in the deployment's credential store and are never included in the +preflight request. ## What an entry carries diff --git a/tests/fixtures/automations/github-pr-reviewer.json b/tests/fixtures/automations/github-pr-reviewer.json index 8532e606..25c6d0de 100644 --- a/tests/fixtures/automations/github-pr-reviewer.json +++ b/tests/fixtures/automations/github-pr-reviewer.json @@ -37,6 +37,28 @@ "schedule": "*/15 * * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, @@ -126,6 +148,28 @@ "schedule": "*/1 * * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, diff --git a/tests/fixtures/automations/github-repo-monitor.json b/tests/fixtures/automations/github-repo-monitor.json index da668f03..9a0588ee 100644 --- a/tests/fixtures/automations/github-repo-monitor.json +++ b/tests/fixtures/automations/github-repo-monitor.json @@ -38,6 +38,28 @@ "schedule": "*/15 * * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, @@ -129,6 +151,28 @@ "schedule": "*/1 * * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, diff --git a/tests/fixtures/automations/upstream-fork-sync.json b/tests/fixtures/automations/upstream-fork-sync.json index ed5c4918..879bb9e4 100644 --- a/tests/fixtures/automations/upstream-fork-sync.json +++ b/tests/fixtures/automations/upstream-fork-sync.json @@ -38,6 +38,28 @@ "schedule": "0 3 * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, @@ -128,6 +150,28 @@ "schedule": "*/1 * * * *", "timezone": "UTC" } + }, + "requirements": { + "integrations": [ + { + "id": "github", + "alternatives": [ + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "oauth2" + }, + { + "transport": "shttp", + "locator": "https://api.githubcopilot.com/mcp/", + "authStrategy": "api_key", + "secretNames": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + } + ] + } + ] } } }, diff --git a/tests/test_automation_setup.py b/tests/test_automation_setup.py index e531041c..0879ca08 100644 --- a/tests/test_automation_setup.py +++ b/tests/test_automation_setup.py @@ -35,6 +35,7 @@ BUILD_SCRIPT = ROOT / "scripts" / "build-automation-catalog.mjs" FIXTURE_DIR = ROOT / "tests" / "fixtures" / "automations" CAPABILITIES_PATH = FIXTURE_DIR / "capabilities.json" +INTEGRATION_CATALOG_DIR = ROOT / "integrations" / "catalog" # The standardized parts of a direct setup, identical for every automation and # therefore not declared in any entry. @@ -178,9 +179,76 @@ def _derive_preflight_body(entry: dict, form_values: dict) -> dict: "automationId": entry["id"], "endpoint": CREATE_PATH, "draft": _render_payload(entry, form_values), + "requirements": _derive_preflight_requirements(entry), } +def _required_secret_fields(fields: list[dict] | None) -> list[str]: + return [ + field["key"] + for field in fields or [] + if field.get("required") and field.get("type") == "password" + ] + + +def _derive_preflight_requirements(entry: dict) -> dict: + """Derive deployment checks from required integration catalog entries.""" + integrations = [] + for integration_id, requirement in entry["requires"]["integrations"].items(): + if requirement.get("required") is False: + continue + catalog_path = INTEGRATION_CATALOG_DIR / f"{integration_id}.json" + alternatives = [] + if catalog_path.exists(): + catalog_entry = _load(catalog_path) + for option in catalog_entry["connectionOptions"]: + transport = option.get("transport") + if option.get("provider") != "mcp" or not transport: + continue + auth = option["auth"] + locator = ( + transport["serverName"] + if transport["kind"] == "stdio" + else transport["url"] + ) + credential_optional = auth.get("apiKeyOptional", False) or ( + transport["kind"] != "stdio" + and transport.get("apiKeyOptional", False) + ) + if transport["kind"] == "stdio": + transport_secret_names = [ + *_required_secret_fields(transport.get("envFields")), + *_required_secret_fields(transport.get("argFields")), + ] + else: + transport_secret_names = _required_secret_fields( + transport.get("headerFields") + ) + secret_names = list( + dict.fromkeys( + [ + *( + [auth["credentialSecretName"]] + if not credential_optional + and auth.get("credentialSecretName") + else [] + ), + *transport_secret_names, + ] + ) + ) + alternative = { + "transport": transport["kind"], + "locator": locator, + "authStrategy": auth["strategy"], + } + if secret_names: + alternative["secretNames"] = secret_names + alternatives.append(alternative) + integrations.append({"id": integration_id, "alternatives": alternatives}) + return {"integrations": integrations} + + def _derive_error_map(entry: dict) -> dict[str, list[str]]: """Which form fields built each payload path.