Skip to content

test: cover automation preflight requirements #476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
KirschBluteX wants to merge 1 commit into
base: main
Choose a base branch
from
Open

test: cover automation preflight requirements #476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion automations/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ two records of the same fact from drifting apart, and it is why the file is as s
| The trigger kinds a deployment must support | The keys of `setup.form.triggers` |
| Schedule limits and the timezone list | The `cron` and `timezone` field types |
| Local validation rules | The `required` flag and `constraints` on each field |
| The preflight call | `POST /v1/validate` with the entry id, the create endpoint, and the rendered payload |
| The preflight call | `POST /v1/validate` with the entry id, the create endpoint, the rendered payload, and required MCP alternatives derived from `requires.integrations` plus the integration catalog |
| The created automation's name | The entry's `name`, plus the repository that was picked |
| `repos` in the create request | The repo-picker field, its declared `provider`, and a field named `ref` if there is one |
| `trigger` in the create request | The key under `form.triggers`, and the fields under it named after trigger properties |
Expand All @@ -56,6 +56,9 @@ two records of the same fact from drifting apart, and it is why the file is as s

`tests/test_automation_setup.py` derives the preflight body and the payload-path mapping from the entry and
checks them against the recorded fixtures, so these deletions stay honest rather than becoming assumptions.
The derived integration requirements include only MCP locators, authentication strategies, and required
secret names. Secret values remain in the deployment's credential store and are never included in the
preflight request.

## What an entry carries

Expand Down
44 changes: 44 additions & 0 deletions tests/fixtures/automations/github-pr-reviewer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,28 @@
"schedule": "*/15 * * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down Expand Up @@ -126,6 +148,28 @@
"schedule": "*/1 * * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down
44 changes: 44 additions & 0 deletions tests/fixtures/automations/github-repo-monitor.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,28 @@
"schedule": "*/15 * * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down Expand Up @@ -129,6 +151,28 @@
"schedule": "*/1 * * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down
44 changes: 44 additions & 0 deletions tests/fixtures/automations/upstream-fork-sync.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,28 @@
"schedule": "0 3 * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down Expand Up @@ -128,6 +150,28 @@
"schedule": "*/1 * * * *",
"timezone": "UTC"
}
},
"requirements": {
"integrations": [
{
"id": "github",
"alternatives": [
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "oauth2"
},
{
"transport": "shttp",
"locator": "https://api.githubcopilot.com/mcp/",
"authStrategy": "api_key",
"secretNames": [
"GITHUB_PERSONAL_ACCESS_TOKEN"
]
}
]
}
]
}
}
},
Expand Down
68 changes: 68 additions & 0 deletions tests/test_automation_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
BUILD_SCRIPT = ROOT / "scripts" / "build-automation-catalog.mjs"
FIXTURE_DIR = ROOT / "tests" / "fixtures" / "automations"
CAPABILITIES_PATH = FIXTURE_DIR / "capabilities.json"
INTEGRATION_CATALOG_DIR = ROOT / "integrations" / "catalog"

# The standardized parts of a direct setup, identical for every automation and
# therefore not declared in any entry.
Expand Down Expand Up @@ -178,9 +179,76 @@ def _derive_preflight_body(entry: dict, form_values: dict) -> dict:
"automationId": entry["id"],
"endpoint": CREATE_PATH,
"draft": _render_payload(entry, form_values),
"requirements": _derive_preflight_requirements(entry),
}


def _required_secret_fields(fields: list[dict] | None) -> list[str]:
return [
field["key"]
for field in fields or []
if field.get("required") and field.get("type") == "password"
]


def _derive_preflight_requirements(entry: dict) -> dict:
"""Derive deployment checks from required integration catalog entries."""
integrations = []
for integration_id, requirement in entry["requires"]["integrations"].items():
if requirement.get("required") is False:
continue
catalog_path = INTEGRATION_CATALOG_DIR / f"{integration_id}.json"
alternatives = []
if catalog_path.exists():
catalog_entry = _load(catalog_path)
for option in catalog_entry["connectionOptions"]:
transport = option.get("transport")
if option.get("provider") != "mcp" or not transport:
continue
auth = option["auth"]
locator = (
transport["serverName"]
if transport["kind"] == "stdio"
else transport["url"]
)
credential_optional = auth.get("apiKeyOptional", False) or (
transport["kind"] != "stdio"
and transport.get("apiKeyOptional", False)
)
if transport["kind"] == "stdio":
transport_secret_names = [
*_required_secret_fields(transport.get("envFields")),
*_required_secret_fields(transport.get("argFields")),
]
else:
transport_secret_names = _required_secret_fields(
transport.get("headerFields")
)
secret_names = list(
dict.fromkeys(
[
*(
[auth["credentialSecretName"]]
if not credential_optional
and auth.get("credentialSecretName")
else []
),
*transport_secret_names,
]
)
)
alternative = {
"transport": transport["kind"],
"locator": locator,
"authStrategy": auth["strategy"],
}
if secret_names:
alternative["secretNames"] = secret_names
alternatives.append(alternative)
integrations.append({"id": integration_id, "alternatives": alternatives})
return {"integrations": integrations}


def _derive_error_map(entry: dict) -> dict[str, list[str]]:
"""Which form fields built each payload path.

Expand Down
Loading