From ee72ae592ed90c9157d926cea6de57199f7dc495 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Wed, 20 May 2026 19:31:44 +0200 Subject: [PATCH 1/8] builddecisionscript: copy build-decision source (bug 2006684) Copy the build-decision code from fxci-config as-is, at revision 6b1ea576b59f8436b28778d5ef2bdd09c80ec348, before adapting it to run as a scriptworker task. --- builddecisionscript/README.txt | 2 + builddecisionscript/pyproject.toml | 32 + .../src/build_decision/__init__.py | 3 + builddecisionscript/src/build_decision/cli.py | 86 +++ .../src/build_decision/cron/__init__.py | 141 +++++ .../src/build_decision/cron/action.py | 55 ++ .../src/build_decision/cron/decision.py | 75 +++ .../src/build_decision/cron/schema.yml | 123 ++++ .../src/build_decision/cron/util.py | 74 +++ .../src/build_decision/decision.py | 62 ++ .../src/build_decision/git_push.py | 55 ++ .../src/build_decision/hg_push.py | 79 +++ .../src/build_decision/repository.py | 183 ++++++ .../src/build_decision/secrets.py | 28 + .../src/build_decision/util/__init__.py | 3 + .../src/build_decision/util/cli.py | 63 ++ .../src/build_decision/util/http.py | 20 + .../src/build_decision/util/keyed_by.py | 98 +++ .../src/build_decision/util/schema.py | 37 ++ .../src/build_decision/util/scopes.py | 20 + .../src/build_decision/util/trigger_action.py | 172 +++++ builddecisionscript/tests/__init__.py | 9 + builddecisionscript/tests/data/actions.json | 593 ++++++++++++++++++ builddecisionscript/tests/data/cron.yml | 32 + builddecisionscript/tests/test_cli.py | 79 +++ builddecisionscript/tests/test_cron.py | 213 +++++++ builddecisionscript/tests/test_cron_action.py | 64 ++ .../tests/test_cron_decision.py | 110 ++++ builddecisionscript/tests/test_cron_util.py | 119 ++++ builddecisionscript/tests/test_decision.py | 80 +++ builddecisionscript/tests/test_git_push.py | 79 +++ builddecisionscript/tests/test_hg_push.py | 115 ++++ builddecisionscript/tests/test_repository.py | 354 +++++++++++ builddecisionscript/tests/test_secrets.py | 28 + builddecisionscript/tests/test_util_cli.py | 28 + .../tests/test_util_keyed_by.py | 136 ++++ builddecisionscript/tests/test_util_schema.py | 11 + builddecisionscript/tests/test_util_scopes.py | 37 ++ .../tests/test_util_trigger_action.py | 261 ++++++++ 39 files changed, 3759 insertions(+) create mode 100644 builddecisionscript/README.txt create mode 100644 builddecisionscript/pyproject.toml create mode 100644 builddecisionscript/src/build_decision/__init__.py create mode 100644 builddecisionscript/src/build_decision/cli.py create mode 100644 builddecisionscript/src/build_decision/cron/__init__.py create mode 100644 builddecisionscript/src/build_decision/cron/action.py create mode 100644 builddecisionscript/src/build_decision/cron/decision.py create mode 100644 builddecisionscript/src/build_decision/cron/schema.yml create mode 100644 builddecisionscript/src/build_decision/cron/util.py create mode 100644 builddecisionscript/src/build_decision/decision.py create mode 100644 builddecisionscript/src/build_decision/git_push.py create mode 100644 builddecisionscript/src/build_decision/hg_push.py create mode 100644 builddecisionscript/src/build_decision/repository.py create mode 100644 builddecisionscript/src/build_decision/secrets.py create mode 100644 builddecisionscript/src/build_decision/util/__init__.py create mode 100644 builddecisionscript/src/build_decision/util/cli.py create mode 100644 builddecisionscript/src/build_decision/util/http.py create mode 100644 builddecisionscript/src/build_decision/util/keyed_by.py create mode 100644 builddecisionscript/src/build_decision/util/schema.py create mode 100644 builddecisionscript/src/build_decision/util/scopes.py create mode 100644 builddecisionscript/src/build_decision/util/trigger_action.py create mode 100644 builddecisionscript/tests/__init__.py create mode 100644 builddecisionscript/tests/data/actions.json create mode 100644 builddecisionscript/tests/data/cron.yml create mode 100644 builddecisionscript/tests/test_cli.py create mode 100644 builddecisionscript/tests/test_cron.py create mode 100644 builddecisionscript/tests/test_cron_action.py create mode 100644 builddecisionscript/tests/test_cron_decision.py create mode 100644 builddecisionscript/tests/test_cron_util.py create mode 100644 builddecisionscript/tests/test_decision.py create mode 100644 builddecisionscript/tests/test_git_push.py create mode 100644 builddecisionscript/tests/test_hg_push.py create mode 100644 builddecisionscript/tests/test_repository.py create mode 100644 builddecisionscript/tests/test_secrets.py create mode 100644 builddecisionscript/tests/test_util_cli.py create mode 100644 builddecisionscript/tests/test_util_keyed_by.py create mode 100644 builddecisionscript/tests/test_util_schema.py create mode 100644 builddecisionscript/tests/test_util_scopes.py create mode 100644 builddecisionscript/tests/test_util_trigger_action.py diff --git a/builddecisionscript/README.txt b/builddecisionscript/README.txt new file mode 100644 index 000000000..3eeb3b5fc --- /dev/null +++ b/builddecisionscript/README.txt @@ -0,0 +1,2 @@ +This docker image bundles the `make_decision.py` script, along with its +dependencies, for use in handling hg-push hooks. diff --git a/builddecisionscript/pyproject.toml b/builddecisionscript/pyproject.toml new file mode 100644 index 000000000..44526d9f5 --- /dev/null +++ b/builddecisionscript/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "build-decision" +version = "1.0.0" +description = "Administration of runtime configuration (Taskcluster settings) for Firefox CI" +authors = [ + { name = "Mozilla Release Engineering", email = "release+build-decision@mozilla.com" }, +] +readme = "README.txt" +classifiers = [ + "Programming Language :: Python :: 3", +] +dependencies = [ + "attrs", + "json-e", + "jsonschema>4.18", + "pyyaml", + "redo", + "referencing", + "requests", + "slugid", + "taskcluster", +] + +[dependency-groups] +dev = ["coverage", "flake8", "pytest", "pytest-cov", "pytest-mock"] + +[project.scripts] +build-decision = "build_decision.cli:main" + +[build-system] +requires = ["uv_build>=0.11.7,<0.12.0"] +build-backend = "uv_build" diff --git a/builddecisionscript/src/build_decision/__init__.py b/builddecisionscript/src/build_decision/__init__.py new file mode 100644 index 000000000..3ed169a3a --- /dev/null +++ b/builddecisionscript/src/build_decision/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. diff --git a/builddecisionscript/src/build_decision/cli.py b/builddecisionscript/src/build_decision/cli.py new file mode 100644 index 000000000..a88bdb6ed --- /dev/null +++ b/builddecisionscript/src/build_decision/cli.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import functools + +from .repository import Repository +from .secrets import get_secret +from .util.cli import CLI + +app = CLI("Build decision tasks") + + +def repo_arguments(app): + def decorator(func): + @app.argument("--repo-url", required=True) + @app.argument("--project", required=True) + @app.argument("--level", required=True) + @app.argument("--repository-type", required=True) + @app.argument("--trust-domain", required=True) + @app.argument("--github-token-secret") + @functools.wraps(func) + def wrapper(args): + repository = {} + for argument in ( + "repo_url", + "project", + "level", + "repository_type", + "trust_domain", + ): + repository[argument] = args.pop(argument) + github_token_secret = args.pop("github_token_secret", None) + if github_token_secret: + repository["github_token"] = get_secret( + github_token_secret, secret_key="token" + ) + args["repository"] = Repository(**repository) + func(args) + + return wrapper + + return decorator + + +@app.command("hg-push", help="Create an hg-push decision task.") +@repo_arguments(app) +@app.argument("--dry-run", action="store_true") +def hg_push(options): + from .hg_push import build_decision # noqa: PLC0415 + + build_decision( + repository=options["repository"], + dry_run=options["dry_run"], + ) + + +@app.command("git-push", help="Create a git-push decision task.") +@repo_arguments(app) +@app.argument("--dry-run", action="store_true") +def git_push(options): + from .git_push import build_decision # noqa: PLC0415 + + build_decision( + repository=options["repository"], + dry_run=options["dry_run"], + ) + + +@app.command("cron", help="Process `.cron.yml`.") +@repo_arguments(app) +@app.argument("--branch") +@app.argument("--force-run") +@app.argument("--dry-run", action="store_true") +def cron(options): + from .cron import run # noqa: PLC0415 + + run( + repository=options["repository"], + branch=options["branch"], + force_run=options["force_run"], + dry_run=options["dry_run"], + ) + + +main = app.main diff --git a/builddecisionscript/src/build_decision/cron/__init__.py b/builddecisionscript/src/build_decision/cron/__init__.py new file mode 100644 index 000000000..bf4547aa5 --- /dev/null +++ b/builddecisionscript/src/build_decision/cron/__init__.py @@ -0,0 +1,141 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import traceback +from pathlib import Path + +from requests.exceptions import HTTPError + +from ..repository import NoPushesError +from ..util.keyed_by import evaluate_keyed_by +from ..util.schema import Schema +from . import action, decision +from .util import calculate_time, match_utc + +# Functions to handle each `job.type` in `.cron.yml`. These are called with +# the contents of the `job` property from `.cron.yml` and should return a +# sequence of (taskId, task) tuples which will subsequently be fed to +# createTask. +JOB_TYPES = { + "decision-task": decision.run_decision_task, + "trigger-action": action.run_trigger_action, +} + +logger = logging.getLogger(__name__) + +_cron_yml_schema = Schema.from_file(Path(__file__).with_name("schema.yml")) + + +def load_jobs(repository, revision): + try: + cron_yml = repository.get_file(".cron.yml", revision=revision) + except HTTPError as e: + if e.response.status_code == 404: + return {} + raise + _cron_yml_schema.validate(cron_yml) + + # resolve keyed_by fields in each job + jobs = cron_yml["jobs"] + + return {j["name"]: j for j in jobs} + + +def should_run(job, *, time, project): + if "run-on-projects" in job: + if project not in job["run-on-projects"]: + return False + # Resolve when key here, so we don't require it before we know that we + # actually want to run on this branch. + when = evaluate_keyed_by( + job.get("when", []), + "Cron job " + job["name"], + {"project": project}, + ) + if not any(match_utc(time=time, sched=sched) for sched in when): + return False + return True + + +def run_job(job_name, job, *, repository, push_info, dry_run=False): + job_type = job["job"]["type"] + if job_type in JOB_TYPES: + JOB_TYPES[job_type]( + job_name, + job["job"], + repository=repository, + push_info=push_info, + dry_run=dry_run, + ) + else: + raise Exception(f"job type {job_type} not recognized") + + +def run(*, repository, branch, force_run, dry_run): + time = calculate_time() + + try: + push_info = repository.get_push_info(branch=branch) + except NoPushesError: + # A common cause for this is a new repository being set-up that hasn't + # had pushes made to it yet. Avoiding an exception for this allows the + # repository to be added to projects.yml before pushes are made to it. + logger.info("No pushes found; doing nothing.") + return + + jobs = load_jobs(repository, revision=push_info["revision"]) + + if force_run: + job_name = force_run + logger.info(f'force-running cron job "{job_name}"') + run_job( + job_name, + jobs[job_name], + repository=repository, + push_info=push_info, + dry_run=dry_run, + ) + return + + failed_jobs = [] + for job_name, job in sorted(jobs.items()): + if should_run(job, time=time, project=repository.project): + logger.info(f'running cron job "{job_name}"') + try: + run_job( + job_name, + job, + repository=repository, + push_info=push_info, + dry_run=dry_run, + ) + except Exception as exc: + # report the exception, but don't fail the whole cron task, as that + # would leave other jobs un-run. + failed_jobs.append((job_name, exc)) + traceback.print_exc() + logger.error( + f'cron job "{job_name}" run failed; continuing to next job' + ) + + else: + logger.info(f'not running cron job "{job_name}"') + + _format_and_raise_error_if_any(failed_jobs) + + +def _format_and_raise_error_if_any(failed_jobs): + if failed_jobs: + failed_job_names = [job_name for job_name, _ in failed_jobs] + failed_job_names_with_exceptions = ( + f'"{job_name}": "{exc}"' for job_name, exc in failed_jobs + ) + raise RuntimeError( + "Cron jobs {} couldn't be triggered properly. " + "Reason(s):\n * {}\nSee logs above for details.".format( + failed_job_names, "\n * ".join(failed_job_names_with_exceptions) + ) + ) diff --git a/builddecisionscript/src/build_decision/cron/action.py b/builddecisionscript/src/build_decision/cron/action.py new file mode 100644 index 000000000..b3a6f12ed --- /dev/null +++ b/builddecisionscript/src/build_decision/cron/action.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import os + +import taskcluster + +from ..util.http import SESSION +from ..util.trigger_action import render_action + +logger = logging.getLogger(__name__) + + +def find_decision_task(repository, revision): + """Given the parameters for this action, find the taskId of the decision + task""" + index = taskcluster.Index(taskcluster.optionsFromEnvironment(), session=SESSION) + decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision" # noqa + logger.info("Looking for index: %s", decision_index) + task_id = index.findTask(decision_index)["taskId"] + logger.info("Found decision task: %s", task_id) + return task_id + + +def run_trigger_action(job_name, job, *, repository, push_info, dry_run): + action_name = job["action-name"] + decision_task_id = find_decision_task(repository, push_info["revision"]) + + action_input = {} + + if job.get("include-cron-input") and "HOOK_PAYLOAD" in os.environ: + cron_hook_payload = json.loads(os.environ["HOOK_PAYLOAD"]) + logger.info( + "Cron Hook Payload:\n%s", + json.dumps(cron_hook_payload, indent=4, sort_keys=True), + ) + action_input.update(cron_hook_payload) + + if job.get("extra-input"): + action_input.update(job["extra-input"]) + + hook = render_action( + action_name=action_name, + task_id=None, + decision_task_id=decision_task_id, + action_input=action_input, + ) + + hook.display() + if not dry_run: + hook.submit() diff --git a/builddecisionscript/src/build_decision/cron/decision.py b/builddecisionscript/src/build_decision/cron/decision.py new file mode 100644 index 000000000..de5ef7f37 --- /dev/null +++ b/builddecisionscript/src/build_decision/cron/decision.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy +import json +import logging +import os +import shlex + +from ..decision import render_tc_yml + +logger = logging.getLogger(__name__) + + +def make_arguments(job): + arguments = [] + if "target-tasks-method" in job: + arguments.append("--target-tasks-method={}".format(job["target-tasks-method"])) + if job.get("optimize-target-tasks") is not None: + arguments.append( + "--optimize-target-tasks={}".format( + str(job["optimize-target-tasks"]).lower(), + ) + ) + if "include-push-tasks" in job: + arguments.append("--include-push-tasks") + if "rebuild-kinds" in job: + for kind in job["rebuild-kinds"]: + arguments.append(f"--rebuild-kind={kind}") + return arguments + + +def run_decision_task(job_name, job, *, repository, push_info, dry_run): + """Generate a basic decision task, based on the root .taskcluster.yml""" + push_info = copy.deepcopy(push_info) + push_info["owner"] = "cron" + + taskcluster_yml = repository.get_file( + ".taskcluster.yml", revision=push_info["revision"] + ) + + arguments = make_arguments(job) + + cron_input = {} + if job.get("include-cron-input") and "HOOK_PAYLOAD" in os.environ: + cron_hook_payload = json.loads(os.environ["HOOK_PAYLOAD"]) + logger.info( + "Cron Hook Payload:\n%s", + json.dumps(cron_hook_payload, indent=4, sort_keys=True), + ) + cron_input.update(cron_hook_payload) + + cron_info = { + "task_id": os.environ.get("TASK_ID", ""), + "job_name": job_name, + "job_symbol": job["treeherder-symbol"], + # args are shell-quoted since they are given to `bash -c` + "quoted_args": " ".join(shlex.quote(a) for a in arguments), + "input": cron_input, + } + + task = render_tc_yml( + taskcluster_yml, + taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"], + tasks_for="cron", + repository=repository.to_json(), + push=push_info, + cron=cron_info, + ) + + task.display() + if not dry_run: + task.submit() diff --git a/builddecisionscript/src/build_decision/cron/schema.yml b/builddecisionscript/src/build_decision/cron/schema.yml new file mode 100644 index 000000000..f619c2252 --- /dev/null +++ b/builddecisionscript/src/build_decision/cron/schema.yml @@ -0,0 +1,123 @@ +--- +schema: "http://json-schema.org/draft-07/schema#" +type: object +required: ["jobs"] +additionalProperties: false +properties: + jobs: + type: array + additionalItems: false + items: + type: object + required: ["name", "job"] + additionalProperties: false + properties: + name: + type: string + description: Name of the crontask (must be unique) + job: + type: object + description: Description of the job to run, keyed by 'type' + anyOf: + - {$ref: "#/definitions/job-types/decision-task"} + - {$ref: "#/definitions/job-types/trigger-action"} + run-on-projects: + type: array + title: The run-on-projects schema + description: An explanation about the purpose of this instance. + additionalItems: false + items: {type: string} + when: + anyOf: + - type: object + required: ['by-project'] + additionalProperties: false + properties: + by-project: + additionalProperties: {$ref: "#/definitions/when"} + - $ref: "#/definitions/when" +definitions: + when: + type: array + items: + type: object + additionalProperties: false + properties: + weekday: + type: string + enum: + - "Monday" + - "Tuesday" + - "Wednesday" + - "Thursday" + - "Friday" + - "Saturday" + - "Sunday" + day: + type: integer + description: Day of the month, as used by datetime. + miniumum: 1 + maximum: 31 + hour: + type: integer + miniumum: 0 + exclusiveMaximum: 24 + minute: + type: integer + miniumum: 0 + multipleOf: 15 + exclusiveMaximum: 60 + job-types: + decision-task: + required: ["type", "treeherder-symbol", "target-tasks-method"] + additionalProperties: false + properties: + type: {const: 'decision-task'} + treeherder-symbol: + type: string + description: Treeherder symbol for the cron task + target-tasks-method: + type: string + description: "--target-tasks-method 'taskgraph decision' argument" + optimize-target-tasks: + type: boolean + description: >- + If specified, this indicates whether the target + tasks are eligible for optimization. Otherwise, + the default for the project is used. + include-push-tasks: + type: boolean + description: >- + Whether tasks from the on-push graph should be re-used + in the cron graph. + rebuild-kinds: + type: array + items: {type: string} + description: Kinds that should not be re-used from the on-push graph. + include-cron-input: + type: boolean + description: >- + Whether the input to the cron hook should be added to the context + used to render .taskcluster.yml. + trigger-action: + required: ["type", "action-name"] + additionalProperties: false + properties: + type: {const: 'trigger-action'} + action-name: + type: string + description: >- + The name of the action to trigger. This will find a + push action on the corresponding commit to trigger. + include-cron-input: + type: boolean + description: >- + Whether the input to the cron hook should be used as + input to the action. + extra-input: + type: object + description: >- + Addtional input that should be passed to the action. + If both `include-cron-input` and `extra-input` are + specified, the values from `extra-input` will override + those from the cron-task input. diff --git a/builddecisionscript/src/build_decision/cron/util.py b/builddecisionscript/src/build_decision/cron/util.py new file mode 100644 index 000000000..7bcd956a2 --- /dev/null +++ b/builddecisionscript/src/build_decision/cron/util.py @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import datetime +import logging +import os + +import taskcluster + +from ..util.http import SESSION + +logger = logging.getLogger(__name__) + + +def match_utc(*, time, sched): + """Return True if time matches the given schedule. + + If minute is not specified, then every multiple of fifteen minutes will match. + Times not an even multiple of fifteen minutes will result in an exception + (since they would never run). + If hour is not specified, any hour will match. Similar for day and weekday. + """ + if sched.get("minute") and sched.get("minute") % 15 != 0: + raise Exception("cron jobs only run on multiples of 15 minutes past the hour") + + if sched.get("minute") is not None and sched.get("minute") != time.minute: + return False + + if sched.get("hour") is not None and sched.get("hour") != time.hour: + return False + + if sched.get("day") is not None and sched.get("day") != time.day: + return False + + if isinstance(sched.get("weekday"), str): + if sched.get("weekday", "").lower() != time.strftime("%A").lower(): + return False + elif sched.get("weekday") is not None: + # don't accept other values. + return False + + return True + + +def calculate_time(): + if "TASK_ID" not in os.environ: + # running in a development environment, so look for CRON_TIME or use + # the current time + if "CRON_TIME" in os.environ: + logger.warning("setting time based on $CRON_TIME") + time = datetime.datetime.utcfromtimestamp(int(os.environ["CRON_TIME"])) + logger.info("cron time: %s", time) + else: + logger.warning( + "using current time for time; try setting $CRON_TIME to a timestamp" + ) + time = datetime.datetime.utcnow() + else: + queue = taskcluster.Queue( + {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, session=SESSION + ) + task = queue.task(os.environ["TASK_ID"]) + # the task's `created` time is close to when the hook ran, although that + # may be some time ago if task execution was delayed + created = task["created"] + time = datetime.datetime.strptime(created, "%Y-%m-%dT%H:%M:%S.%fZ") + + # round down to the nearest 15m + minute = time.minute - (time.minute % 15) + time = time.replace(minute=minute, second=0, microsecond=0) + logger.info(f"calculated cron schedule time is {time}") + return time diff --git a/builddecisionscript/src/build_decision/decision.py b/builddecisionscript/src/build_decision/decision.py new file mode 100644 index 000000000..fb3c630be --- /dev/null +++ b/builddecisionscript/src/build_decision/decision.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os + +import attr +import jsone +import slugid +import taskcluster + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +def render_tc_yml(tc_yml, **context): + """ + Render .taskcluster.yml into an array of tasks. This provides a context + that is similar to that provided by actions and crons, but with `tasks-for` + set to `hg-push`. + """ + ownTaskId = slugid.nice() + context["ownTaskId"] = ownTaskId + rendered = jsone.render(tc_yml, context) + + task_count = len(rendered["tasks"]) + if task_count != 1: + logger.critical(f"Rendered result has {task_count} tasks; only one supported") + raise Exception() + + [task] = rendered["tasks"] + task_id = task.pop("taskId") + return Task(task_id, task) + + +@attr.s(frozen=True) +class Task: + task_id = attr.ib() + task_payload = attr.ib() + + def display(self): + logger.info( + "Decision Task:\n%s", + json.dumps(self.task_payload, indent=4, sort_keys=True), + ) + + def submit(self): + logger.info("Task Id: %s", self.task_id) + + if "TASKCLUSTER_PROXY_URL" in os.environ: + queue = taskcluster.Queue( + {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, + session=SESSION, + ) + else: + queue = taskcluster.Queue( + taskcluster.optionsFromEnvironment(), session=SESSION + ) + queue.createTask(self.task_id, self.task_payload) diff --git a/builddecisionscript/src/build_decision/git_push.py b/builddecisionscript/src/build_decision/git_push.py new file mode 100644 index 000000000..0afa4c964 --- /dev/null +++ b/builddecisionscript/src/build_decision/git_push.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os + +import slugid + +from .decision import render_tc_yml + +logger = logging.getLogger(__name__) + + +def build_decision(*, repository, dry_run): + logging.info("Running build-decision task") + + payload = json.loads(os.environ["HOOK_PAYLOAD"]) + logger.info("Hook Payload:\n%s", json.dumps(payload, indent=4, sort_keys=True)) + + event = { + "after": payload["sha"], + "base_ref": payload.get("base_ref"), + "before": payload["base_sha"], + "pusher": {"email": payload["owner"]}, + "ref": payload["ref"], + "repository": { + "name": repository.repo_path.split("/")[-1], + "full_name": repository.repo_path, + "html_url": repository.repo_url, + "clone_url": repository.repo_url.rstrip("/") + ".git", + }, + } + + tc_yml = repository.get_file(".taskcluster.yml", revision=event["after"]) + + _slugids = {} + + def as_slugid(name): + if name not in _slugids: + _slugids[name] = slugid.nice() + return _slugids[name] + + task = render_tc_yml( + tc_yml, + taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"], + tasks_for="git-push", + event=event, + as_slugid=as_slugid, + ) + + task.display() + if not dry_run: + task.submit() diff --git a/builddecisionscript/src/build_decision/hg_push.py b/builddecisionscript/src/build_decision/hg_push.py new file mode 100644 index 000000000..f6007231a --- /dev/null +++ b/builddecisionscript/src/build_decision/hg_push.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import time +from contextlib import contextmanager + +from .decision import render_tc_yml + +logger = logging.getLogger(__name__) + + +@contextmanager +def timed(description): + start = time.perf_counter() + yield + logging.info(f"{description} took: {time.perf_counter() - start:.1f}") + + +# Allow triggering on-push task for pushes up to 3 days old. +MAX_TIME_DRIFT = 3 * 24 * 60 * 60 + + +def get_revision_from_pulse_message(): + pulse_message = json.loads(os.environ["PULSE_MESSAGE"]) + logger.info( + "Pulse Message:\n%s", json.dumps(pulse_message, indent=4, sort_keys=True) + ) + + pulse_payload = pulse_message["payload"] + if pulse_payload["type"] != "changegroup.1": + logger.info("Not a changegroup.1 message") + return + + push_count = len(pulse_payload["data"]["pushlog_pushes"]) + if push_count != 1: + logger.info("Message has %d pushes; only one supported", push_count) + return + + head_count = len(pulse_payload["data"]["heads"]) + if head_count != 1: + logger.info("Message has %d heads; only one supported", head_count) + return + + return pulse_payload["data"]["heads"][0] + + +def build_decision(*, repository, dry_run): + logging.info("Running build-decision task") + # The hg-push hook can be triggered manually, so we throw out everything + # from the input, other than the revision, and get the pushinfo from + # hg.mozilla.org. + revision = get_revision_from_pulse_message() + + with timed("Fetching push info"): + push = repository.get_push_info(revision=revision) + + if time.time() - push["pushdate"] > MAX_TIME_DRIFT: + logger.warning("Push is too old, not triggering tasks") + return + + with timed("Fetching .taskcluster.yml"): + taskcluster_yml = repository.get_file(".taskcluster.yml", revision=revision) + + with timed("Rendering task"): + task = render_tc_yml( + taskcluster_yml, + taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"], + tasks_for="hg-push", + push=push, + repository=repository.to_json(), + ) + + task.display() + if not dry_run: + task.submit() diff --git a/builddecisionscript/src/build_decision/repository.py b/builddecisionscript/src/build_decision/repository.py new file mode 100644 index 000000000..63b6bab6d --- /dev/null +++ b/builddecisionscript/src/build_decision/repository.py @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +import attr +import redo +import yaml +from requests.exceptions import ChunkedEncodingError, ConnectionError, SSLError + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +class NoPushesError(Exception): + pass + + +@attr.s(frozen=True) +class Repository: + repo_url = attr.ib() + repository_type = attr.ib() + project = attr.ib(default=None) + level = attr.ib(default=None) + trust_domain = attr.ib(default=None) + github_token = attr.ib(default=None) + + def get_file(self, path, *, revision=None): + """ + Get `.taskcluster.yml` from 'default' (or the given revision) at the named + repo_path. Note that this does not parse the yml (so that it can be hashed + in its original form). + + If the file is not found, this returns None. + """ + headers = {} + + if self.repository_type == "hg": + if revision is None: + revision = "default" + url = f"{self.repo_url}/raw-file/{revision}/{path}" + elif self.repository_type == "git": + repo_url = self.repo_url + + ref_param = "" + # If no ref is given, the github API will default to the default branch + if revision is not None: + ref_param = f"?ref={revision}" + + if repo_url.startswith("https://github.com/"): + url = f"https://api.github.com/repos/{self.repo_path}/contents/{path}{ref_param}" + if self.github_token: + headers["Authorization"] = f"token {self.github_token}" + headers["Accept"] = "application/vnd.github.raw+json" + elif repo_url.startswith("git@github.com:"): + raise Exception( + f"Don't know how to get file from private github repo: {repo_url}" + ) + else: + raise Exception( + "Don't know how to determine get file for non-github " + f"repo: {repo_url}" + ) + else: + raise Exception(f"Unknown repository_type {self.repository_type}!") + + res = SESSION.get(url, headers=headers, timeout=60) + res.raise_for_status() + tcyml = res.text + + return yaml.safe_load(tcyml) + + @redo.retriable( + attempts=5, + sleeptime=10, + retry_exceptions=( + NoPushesError, + ChunkedEncodingError, + ConnectionError, + SSLError, + ), + ) + def get_push_info(self, *, revision=None, branch=None): + if branch and revision: + raise ValueError("Can't pass both revision and branch to get_push_info") + if self.repository_type == "hg": + if revision: + revset = revision + elif branch: + revset = branch + else: + revset = "default" + res = SESSION.get( + f"{self.repo_url}/json-pushes?version=2&changeset={revset}&full=1", + timeout=60, + ) + res.raise_for_status() + pushes = res.json()["pushes"] + if len(pushes) == 0: + # If we query immediately after a push, hg.mozilla.org might + # report that there are no pushes associated to a changeset. + # We retry, since this tends to be a transient error. + raise NoPushesError( + f"Changeset {revset} has no associated pushes. " + "Maybe the push log has not been updated?" + ) + elif len(pushes) != 1: + raise ValueError( + f"Changeset {revset} has {len(pushes)} associated pushes; " + "only one supported." + ) + [(push_id, push_info)] = pushes.items() + changesets = push_info["changesets"] + first_pushed_revision = changesets[0] + base_revision = first_pushed_revision["parents"][0] + tip_revision = changesets[-1]["node"] + if revision and revision != tip_revision: + raise ValueError( + f"Changeset {revision} is not the tip {tip_revision} of the associated push." + ) + + return { + "owner": push_info["user"], + "pushlog_id": push_id, + "pushdate": push_info["date"], + "revision": tip_revision, + "base_revision": base_revision, + } + elif self.repository_type == "git": + if revision: + raise Exception("Can't get push information for a git revision.") + if branch is None: + branch = "master" # FIXME: Use api to get default branch + repo_url = self.repo_url + headers = {} + if self.github_token: + headers["Authorization"] = f"token {self.github_token}" + if repo_url.startswith("https://github.com/"): + url = ( + f"https://api.github.com" + f"/repos/{self.repo_path}/git/ref/heads/{branch}" + ) + res = SESSION.get(url, headers=headers, timeout=60) + res.raise_for_status() + return { + "branch": branch, + "revision": res.json()["object"]["sha"], + } + elif repo_url.startswith("git@github.com:"): + raise Exception( + "Don't know how to determine revision for private github " + f"repo: {repo_url}" + ) + else: + raise Exception( + "Don't know how to determine revision for for non-github " + f"repo: {repo_url}" + ) + else: + raise Exception(f"Unknown repository_type {self.repository_type}!") + + @property + def repo_path(self): + if self.repository_type == "hg" and self.repo_url.startswith( + "https://hg.mozilla.org/" + ): + return self.repo_url.replace("https://hg.mozilla.org/", "", 1).rstrip("/") + elif self.repository_type == "git" and self.repo_url.startswith( + "https://github.com/" + ): + return self.repo_url.replace("https://github.com/", "", 1).rstrip("/") + else: + raise AttributeError(f"no repo_path available for project {self.alias}") + + def to_json(self): + return { + "url": self.repo_url, + "project": self.project, + "level": self.level, + "type": self.repository_type, + } diff --git a/builddecisionscript/src/build_decision/secrets.py b/builddecisionscript/src/build_decision/secrets.py new file mode 100644 index 000000000..38b484679 --- /dev/null +++ b/builddecisionscript/src/build_decision/secrets.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +def get_secret(secret_name, secret_key=None): + # XXX should we fall back to taskcluster api call if the proxy isn't running? + # (might be difficult and we may only hit that case if we run the docker + # image locally.) + secret_url = f"http://taskcluster/secrets/v1/secret/{secret_name}" + logging.info(f"Fetching secret at {secret_url} ...") + res = SESSION.get(secret_url, timeout=60) + # This will raise an error if the secret isn't populated or we have + # infrastructure issues. Let's die so we see there's a problem. + res.raise_for_status() + secret = res.json() + if secret_key: + # This will raise a KeyError if the secret is populated but isn't in the + # right form. Let's die so we see there's a problem and can fix it + # sooner. + return secret["secret"][secret_key] + return secret diff --git a/builddecisionscript/src/build_decision/util/__init__.py b/builddecisionscript/src/build_decision/util/__init__.py new file mode 100644 index 000000000..3ed169a3a --- /dev/null +++ b/builddecisionscript/src/build_decision/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. diff --git a/builddecisionscript/src/build_decision/util/cli.py b/builddecisionscript/src/build_decision/util/cli.py new file mode 100644 index 000000000..26dcdc08c --- /dev/null +++ b/builddecisionscript/src/build_decision/util/cli.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import sys +import traceback + +import attr + + +@attr.s(cmp=False) +class CLI: + description = attr.ib(type=str) + _commands = attr.ib(default=[], init=False) + + def command(self, *args, **kwargs): + defaults = kwargs.pop("defaults", {}) + + def decorator(func): + self._commands.append((func, args, kwargs, defaults)) + return func + + return decorator + + @staticmethod + def argument(*names, **kwargs): + def decorator(func): + if not hasattr(func, "args"): + func.args = [] + # Decorators run from bottom to top of the order they were + # specified in the source. In order to make positional arguments + # appear in the order they were specifed, we insert arguments at + # the beginning, so that the list of arguments appears in the same + # order they were specified in the source. + func.args.insert(0, (names, kwargs)) + return func + + return decorator + + def create_parser(self): + parser = argparse.ArgumentParser(description=self.description) + subparsers = parser.add_subparsers(dest="command") + subparsers.required = True + for func, args, kwargs, defaults in self._commands: + subparser = subparsers.add_parser(*args, **kwargs) + for arg in getattr(func, "args", []): + subparser.add_argument(*arg[0], **arg[1]) + subparser.set_defaults(command=func, **defaults) + return parser + + def main(self): + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO + ) + parser = self.create_parser() + args = parser.parse_args() + try: + args.command(vars(args)) + except Exception: + traceback.print_exc() + sys.exit(1) diff --git a/builddecisionscript/src/build_decision/util/http.py b/builddecisionscript/src/build_decision/util/http.py new file mode 100644 index 000000000..37f3daddb --- /dev/null +++ b/builddecisionscript/src/build_decision/util/http.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +SESSION = requests.Session() +adapter = HTTPAdapter( + max_retries=Retry( + total=3, + read=3, + connect=3, + backoff_factor=0.3, + status_forcelist=(500, 502, 503, 504), + ) +) +SESSION.mount("http://", adapter) +SESSION.mount("https://", adapter) diff --git a/builddecisionscript/src/build_decision/util/keyed_by.py b/builddecisionscript/src/build_decision/util/keyed_by.py new file mode 100644 index 000000000..711a47244 --- /dev/null +++ b/builddecisionscript/src/build_decision/util/keyed_by.py @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + + +def keymatch(attributes, target): + """ + Determine if any keys in attributes are a match to target, then return + a list of matching values. First exact matches will be checked. Failing + that, regex matches and finally a default key. + """ + # exact match + if target in attributes: + return [attributes[target]] + + # regular expression match + matches = [v for k, v in attributes.items() if re.match(k + "$", target)] + if matches: + return matches + + # default + if "default" in attributes: + return [attributes["default"]] + + return [] + + +def evaluate_keyed_by(value, item_name, attributes): + """ + For values which can either accept a literal value, or be keyed by some + attributes, perform that lookup and return the result. + + For example, given item:: + + by-test-platform: + macosx-10.11/debug: 13 + win.*: 6 + default: 12 + + a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')` + would return `12`. + + The `item_name` parameter is used to generate useful error messages. + Items can be nested as deeply as desired:: + + by-test-platform: + win.*: + by-project: + ash: .. + cedar: .. + linux: 13 + default: 12 + """ + while True: + if ( + not isinstance(value, dict) + or len(value) != 1 + or not list(value.keys())[0].startswith("by-") + ): + return value + + keyed_by = list(value.keys())[0][3:] # strip off 'by-' prefix + key = attributes.get(keyed_by) + alternatives = list(value.values())[0] + + if len(alternatives) == 1 and "default" in alternatives: + # Error out when only 'default' is specified as only alternatives, + # because we don't need to by-{keyed_by} there. + raise Exception( + f"Keyed-by '{keyed_by}' unnecessary with only value 'default' " + f"found, when determining item {item_name}" + ) + + if key is None: + if "default" in alternatives: + value = alternatives["default"] + continue + else: + raise Exception( + f"No attribute {keyed_by} and no value for 'default' found " + f"while determining item {item_name}" + ) + + matches = keymatch(alternatives, key) + if len(matches) > 1: + raise Exception( + f"Multiple matching values for {keyed_by} {key!r} found while " + f"determining item {item_name}" + ) + elif matches: + value = matches[0] + continue + + raise Exception( + f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}" + ) diff --git a/builddecisionscript/src/build_decision/util/schema.py b/builddecisionscript/src/build_decision/util/schema.py new file mode 100644 index 000000000..f6a2c6ff1 --- /dev/null +++ b/builddecisionscript/src/build_decision/util/schema.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import attr +import yaml +from jsonschema.validators import validator_for +from referencing import Registry + + +def _get_validator(schema): + # jsonschema by default allows remote references in the schema, so we + # override its default registry with one that does not do that. + registry = Registry() + cls = validator_for(schema) + cls.check_schema(schema) + return cls(schema, registry=registry) + + +@attr.s(frozen=True) +class Schema: + _schema = attr.ib() + _validator = attr.ib( + init=False, + default=attr.Factory( + lambda self: _get_validator(self._schema), takes_self=True + ), + ) + + @classmethod + def from_file(cls, path): + schema = yaml.safe_load(path.read_text()) + return cls(schema) + + def validate(self, value): + self._validator.validate(value) diff --git a/builddecisionscript/src/build_decision/util/scopes.py b/builddecisionscript/src/build_decision/util/scopes.py new file mode 100644 index 000000000..047ae43e9 --- /dev/null +++ b/builddecisionscript/src/build_decision/util/scopes.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + + +def satisfies(*, have, require): + """ + Return True if the scopes in "have" satisfy the scopes in "require". + """ + assert isinstance(have, list) + assert isinstance(require, list) + for req_scope in require: + for have_scope in have: + if have_scope == req_scope or ( + have_scope.endswith("*") and req_scope.startswith(have_scope[:-1]) + ): + break + else: + return False + return True diff --git a/builddecisionscript/src/build_decision/util/trigger_action.py b/builddecisionscript/src/build_decision/util/trigger_action.py new file mode 100644 index 000000000..4df3a8e74 --- /dev/null +++ b/builddecisionscript/src/build_decision/util/trigger_action.py @@ -0,0 +1,172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +""" +Implement triggering actions. + +For specification details see: +https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#action-context +""" + +from __future__ import annotations + +import json +import logging +import os + +import attr +import jsone +import jsonschema +import taskcluster + +from . import scopes +from .http import SESSION + +logger = logging.getLogger(__name__) + + +def _is_task_in_context(context, task_tags): + """ + A task (as defined by its tags) is said to match a tag-set if its + tags are a super-set of the tag-set. A tag-set is a set of key-value pairs. + + An action (as defined by its context) is said to be relevant for + a given task, if that task's tags match one of the tag-sets given + in the context property for the action. + """ + return any( + all( + tag in task_tags and task_tags[tag] == tag_set[tag] + for tag in tag_set.keys() + ) + for tag_set in context + ) + + +def _filter_relevant_actions(actions_json, original_task): + """ + Each action entry (from action array) must define a name, title and description. + The order of the array of actions is **significant**: actions should be displayed + in this order, and when multiple actions apply, **the first takes precedence**. + """ + relevant_actions = {} + + for action in actions_json["actions"]: + action_name = action["name"] + if action_name in relevant_actions: + continue + + if original_task is None: + if len(action["context"]) == 0: + relevant_actions[action_name] = action + else: + if _is_task_in_context(action["context"], original_task.get("tags", {})): + relevant_actions[action_name] = action + + return relevant_actions + + +def _check_decision_task_scopes(decision_task_id, hook_group_id, hook_id): + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + auth = taskcluster.Auth(taskcluster.optionsFromEnvironment(), session=SESSION) + decision_task = queue.task(decision_task_id) + decision_task_scopes = auth.expandScopes({"scopes": decision_task["scopes"]})[ + "scopes" + ] + in_tree_scope = f"in-tree:hook-action:{hook_group_id}/{hook_id}" + + if not scopes.satisfies(have=decision_task_scopes, require=[in_tree_scope]): + raise RuntimeError( + "Action is misconfigured: " + f"decision task's scopes do not include {in_tree_scope}\n" + "Decision Task {decision_task_id} has scopes:\n" + + "\n".join(f" - {scope}" for scope in decision_task_scopes) + ) + + +def render_action(*, action_name, task_id, decision_task_id, action_input): + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + + logger.debug("Fetching actions.json...") + actions_url = queue.buildUrl( + "getLatestArtifact", decision_task_id, "public/actions.json" + ) + actions_response = SESSION.get(actions_url) + actions_response.raise_for_status() + actions_json = actions_response.json() + if task_id is not None: + task_definition = queue.task(task_id) + else: + task_definition = None + + if actions_json["version"] != 1: + raise RuntimeError("Wrong version of actions.json, unable to continue") + + relevant_actions = _filter_relevant_actions(actions_json, task_definition) + + if action_name not in relevant_actions: + raise LookupError( + f"{action_name} action is not available for this task. " + f"Available: {sorted(relevant_actions.keys())}" + ) + + action = relevant_actions[action_name] + + if action["kind"] != "hook": + raise NotImplementedError( + f"Unable to submit actions with '{action['kind']}' kind." + ) + + _check_decision_task_scopes( + decision_task_id, + action["hookGroupId"], + action["hookId"], + ) + + jsonschema.validate(action_input, action["schema"]) + + context = { + "taskGroupId": decision_task_id, + "taskId": task_id or None, + "input": action_input, + } + context.update(actions_json["variables"]) + + hook_payload = jsone.render(action["hookPayload"], context) + + return Hook( + hook_group_id=action["hookGroupId"], + hook_id=action["hookId"], + hook_payload=hook_payload, + ) + + +@attr.s(frozen=True) +class Hook: + hook_group_id = attr.ib() + hook_id = attr.ib() + hook_payload = attr.ib() + + def display(self): + logger.info( + "Hook: %s/%s\nHook payload:\n%s", + self.hook_group_id, + self.hook_id, + json.dumps(self.hook_payload, indent=4, sort_keys=True), + ) + + def submit(self): + if "TASKCLUSTER_PROXY_URL" in os.environ: + hooks = taskcluster.Hooks( + {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, + session=SESSION, + ) + else: + hooks = taskcluster.Hooks( + taskcluster.optionsFromEnvironment(), session=SESSION + ) + + logger.info("Triggering hook %s/%s", self.hook_group_id, self.hook_id) + result = hooks.triggerHook(self.hook_group_id, self.hook_id, self.hook_payload) + logger.info("Task Id: %s", result["status"]["taskId"]) diff --git a/builddecisionscript/tests/__init__.py b/builddecisionscript/tests/__init__.py new file mode 100644 index 000000000..98fca351f --- /dev/null +++ b/builddecisionscript/tests/__init__.py @@ -0,0 +1,9 @@ +import os +from pathlib import Path + +TEST_DATA_DIR = Path(os.path.dirname(__file__)) / "data" + + +def fake_redo_retry(func, args, kwargs, *retry_args, **retry_kwargs): + """Mock redo.retry; can also get around @redo.retriable decorator.""" + return func(*args, **kwargs) diff --git a/builddecisionscript/tests/data/actions.json b/builddecisionscript/tests/data/actions.json new file mode 100644 index 000000000..276944e3d --- /dev/null +++ b/builddecisionscript/tests/data/actions.json @@ -0,0 +1,593 @@ +{ + "actions": [ + { + "context": [ + { + "kind": "decision-task" + }, + { + "kind": "action-callback" + }, + { + "kind": "cron-task" + } + ], + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-decision", + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "title": "Retrigger" + }, + { + "context": [ + { + "retrigger": "true" + } + ], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger", + "description": "Create a clone of the task.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger" + }, + { + "context": [ + {} + ], + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-disabled", + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger (disabled)" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "force": { + "default": false, + "description": "This task should not be re-triggered. This can be overridden by passing `true` here.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger (disabled)" + }, + { + "context": [], + "description": "Add new jobs using task labels.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "add-new-jobs", + "description": "Add new jobs using task labels.", + "name": "add-new-jobs", + "symbol": "add-new", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Add new jobs" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "add-new-jobs", + "schema": { + "properties": { + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Add new jobs" + }, + { + "context": [ + {} + ], + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "rerun", + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "name": "rerun", + "symbol": "rr", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Rerun" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "rerun", + "schema": { + "properties": {}, + "type": "object" + }, + "title": "Rerun" + }, + { + "context": [ + {} + ], + "description": "Cancel the given task", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel", + "description": "Cancel the given task", + "name": "cancel", + "symbol": "cx", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel Task" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel", + "title": "Cancel Task" + }, + { + "context": [], + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel-all", + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "name": "cancel-all", + "symbol": "cAll", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel All" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel-all", + "title": "Cancel All" + }, + { + "context": [], + "description": "Ship Fenix", + "extra": { + "actionPerm": "release-promotion" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-release-promotion/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "release-promotion", + "description": "Ship Fenix", + "name": "release-promotion", + "symbol": "${input.release_promotion_flavor}", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Ship Fenix" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "release-promotion", + "schema": { + "properties": { + "build_number": { + "default": 1, + "description": "The release build number. Starts at 1 per release version, and increments on rebuild.", + "minimum": 1, + "title": "The release build number", + "type": "integer" + }, + "do_not_optimize": { + "description": "Optional: a list of labels to avoid optimizing out of the graph (to force a rerun of, say, funsize docker-image tasks).", + "items": { + "type": "string" + }, + "type": "array" + }, + "next_version": { + "default": "", + "description": "Next version.", + "type": "string" + }, + "previous_graph_ids": { + "description": "Optional: an array of taskIds of decision or action tasks from the previous graph(s) to use to populate our `previous_graph_kinds`.", + "items": { + "type": "string" + }, + "type": "array" + }, + "rebuild_kinds": { + "description": "Optional: an array of kinds to ignore from the previous graph(s).", + "items": { + "type": "string" + }, + "type": "array" + }, + "release_promotion_flavor": { + "default": "build", + "description": "The flavor of release promotion to perform.", + "enum": [ + "ship" + ], + "type": "string" + }, + "revision": { + "description": "Optional: the revision to ship.", + "title": "Optional: revision to ship", + "type": "string" + }, + "version": { + "default": "", + "description": "Optional: override the version for release promotion. Occasionally we'll land a taskgraph fix in a later commit, but want to act on a build from a previous commit. If a version bump has landed in the meantime, relying on the in-tree version will break things.", + "type": "string" + } + }, + "required": [ + "release_promotion_flavor", + "version", + "build_number", + "next_version" + ], + "type": "object" + }, + "title": "Ship Fenix" + }, + { + "context": [], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-multiple", + "description": "Create a clone of the task.", + "name": "retrigger-multiple", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger-multiple", + "schema": { + "properties": { + "additionalProperties": false, + "requests": { + "items": { + "additionalProperties": false, + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "array" + } + }, + "type": "object" + }, + "title": "Retrigger" + } + ], + "variables": {}, + "version": 1 +} diff --git a/builddecisionscript/tests/data/cron.yml b/builddecisionscript/tests/data/cron.yml new file mode 100644 index 000000000..c51e01f7e --- /dev/null +++ b/builddecisionscript/tests/data/cron.yml @@ -0,0 +1,32 @@ +# Definitions for jobs that run periodically. For details on the format, see +# `taskcluster/taskgraph/cron/schema.py`. For documentation, see +# `taskcluster/docs/cron.rst`. +--- + +jobs: + - name: nightly + job: + type: decision-task + treeherder-symbol: Nd + target-tasks-method: nightly + when: + - {hour: 5, minute: 0} + - {hour: 17, minute: 0} + - name: fennec-production + job: + type: decision-task + treeherder-symbol: fennec-production + target-tasks-method: fennec-production + when: [] # Force hook only + - name: bump-android-components + job: + type: decision-task + treeherder-symbol: bump-ac + target-tasks-method: bump_android_components + when: [{hour: 15, minute: 30}] + - name: screenshots + job: + type: decision-task + treeherder-symbol: screenshots-D + target-tasks-method: screenshots + when: [{weekday: 'Monday', hour: 10, minute: 0}] diff --git a/builddecisionscript/tests/test_cli.py b/builddecisionscript/tests/test_cli.py new file mode 100644 index 000000000..8d02d2715 --- /dev/null +++ b/builddecisionscript/tests/test_cli.py @@ -0,0 +1,79 @@ +import sys + +import pytest + +import build_decision.cli as cli +import build_decision.cron as cron +import build_decision.hg_push as hg_push + + +def test_hg_push(mocker): + """Add hg-push cli coverage.""" + options = { + "dry_run": True, + "repository": "fakerepo", + "repo_url": "fakeurl", + "project": "fakeproject", + "level": "fakelevel", + "repository_type": "fake_repository_type", + "trust_domain": "fake_trust_domain", + } + + fake_repo = mocker.MagicMock() + + def fake_build_decision(repository, dry_run): + assert repository is fake_repo + assert dry_run + + mocker.patch.object(hg_push, "build_decision", new=fake_build_decision) + mocker.patch.object(cli, "Repository", return_value=fake_repo) + cli.hg_push(options) + + +@pytest.mark.parametrize( + "token, force_run", + ( + (True, True), + (True, False), + (False, True), + (False, False), + ), +) +def test_cron(mocker, token, force_run): + """Add cron cli coverage. + + Parametrize ``token`` for ``repo_arguments`` coverage. + """ + options = { + "dry_run": True, + "repository": "fakerepo", + "repo_url": "fakeurl", + "project": "fakeproject", + "level": "fakelevel", + "repository_type": "fake_repository_type", + "trust_domain": "fake_trust_domain", + "branch": "branch", + "force_run": force_run, + } + if token: + options["github_token_secret"] = "token_secret" + + fake_repo = mocker.MagicMock() + + def fake_run(repository, branch, force_run, dry_run): + assert repository is fake_repo + assert branch == "branch" + assert force_run == options["force_run"] + assert dry_run + + mocker.patch.object(cli, "get_secret") + mocker.patch.object(cron, "run", new=fake_run) + mocker.patch.object(cli, "Repository", return_value=fake_repo) + cli.cron(options) + + +def test_main_help(mocker): + """Call cli.main() with --help.""" + mocker.patch.object(sys, "argv", new=["--help"]) + with pytest.raises(SystemExit): + cli.main() diff --git a/builddecisionscript/tests/test_cron.py b/builddecisionscript/tests/test_cron.py new file mode 100644 index 000000000..097dc4db2 --- /dev/null +++ b/builddecisionscript/tests/test_cron.py @@ -0,0 +1,213 @@ +import pytest +import requests.exceptions +import yaml + +import build_decision.cron as cron +from build_decision.repository import NoPushesError + +from . import TEST_DATA_DIR + + +def test_load_jobs(mocker): + """Add cron load_jobs coverage.""" + with open(TEST_DATA_DIR / "cron.yml") as fh: + cron_yml = yaml.safe_load(fh) + + fake_repo = mocker.MagicMock() + fake_repo.get_file.return_value = cron_yml + expected = {} + for job in cron_yml["jobs"]: + expected[job["name"]] = job + + assert cron.load_jobs(fake_repo, "rev") == expected + + +def test_load_jobs_404(mocker): + fake_repo = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_response.status_code = 404 + fake_repo.get_file.side_effect = requests.exceptions.HTTPError( + response=fake_response + ) + assert cron.load_jobs(fake_repo, "rev") == {} + + +@pytest.mark.parametrize( + "job, match_utc_bool, project, expected", + ( + ( + # project doesn't match run-on-projects + { + "name": "name", + "run-on-projects": ["project1", "project2"], + }, + True, + "invalid-project", + False, + ), + ( + # project does match run-on-projects, time matches + { + "name": "name", + "run-on-projects": ["project1", "project2"], + "when": [{"hour": 4, "minute": 0}], + }, + True, + "project1", + True, + ), + ( + # no run-on-projects, time doesn't match + { + "name": "name", + "when": [{"hour": 4, "minute": 0}], + }, + False, + "project1", + False, + ), + ( + # no run-on-projects, time matches + { + "name": "name", + "when": [{"hour": 4, "minute": 0}], + }, + True, + "project1", + True, + ), + ), +) +def test_should_run(mocker, job, match_utc_bool, project, expected): + """Test the various branches in cron.should_run.""" + mocker.patch.object(cron, "match_utc", return_value=match_utc_bool) + assert cron.should_run(job, time="fake_time", project=project) == expected + + +@pytest.mark.parametrize( + "job_type, raises", + (("decision-task", False), ("trigger-action", False), ("unknown", Exception)), +) +def test_run_job(mocker, job_type, raises): + """Raise if we have an invalid job_type.""" + job = {"job": {"type": job_type}} + + def fake_run(*args, **kwargs): + pass + + fake_job_types = { + "decision-task": fake_run, + "trigger-action": fake_run, + } + + mocker.patch.object(cron, "JOB_TYPES", new=fake_job_types) + if raises: + with pytest.raises(raises): + cron.run_job("job_name", job, repository=None, push_info=None, dry_run=True) + else: + cron.run_job("job_name", job, repository=None, push_info=None, dry_run=True) + + +@pytest.mark.parametrize( + "force_run, jobs", + ( + ( + # Force run + "job1", + { + "job1": {}, + }, + ), + ( + # No force run, no jobs + False, + {}, + ), + ( + # No force run, one job to run + False, + { + "job1": { + "name": "job1", + "should_run": True, + }, + "job2": { + "name": "job2", + }, + }, + ), + ( + # No force run, one failing job to run + False, + { + "job1": { + "name": "job1", + }, + "job2": { + "name": "job2", + "should_run": True, + "exception": Exception, + }, + }, + ), + ), +) +def test_run(mocker, force_run, jobs): + """Add coverage for cron.run. + + ``jobs`` will look like + { + "job-name": { + "name": "job-name", + "exception": Exception, # optional, if we want a failure + "should_run": True, # optional, if we want to run + }, + ... + } + """ + fake_repo = mocker.MagicMock() + + def fake_run_job(job_name, job, **kwargs): + if job.get("exception"): + raise job["exception"]("raising") + + def fake_should_run(job, **kwargs): + return job.get("should_run", False) + + mocker.patch.object(cron, "load_jobs", return_value=jobs) + mocker.patch.object(cron, "run_job", new=fake_run_job) + mocker.patch.object(cron, "should_run", new=fake_should_run) + mocker.patch.object(cron, "_format_and_raise_error_if_any") + cron.run(repository=fake_repo, branch="branch", force_run=force_run, dry_run=True) + + +def test_run_no_pushes(mocker): + """Ensure that running cron.hook does nothing when no pushes are found, + and doesn't raise an Exception.""" + fake_repo = mocker.MagicMock() + + def fake_get_push_info(*args, **kwargs): + raise NoPushesError() + + fake_repo.get_push_info = fake_get_push_info + + mocker.patch.object(cron, "load_jobs") + cron.run(repository=fake_repo, branch="branch", force_run=False, dry_run=False) + assert cron.load_jobs.call_count == 0 + # no exceptions raised; nothing else to check! + + +def test_format_and_raise_error_if_any_with_failures(): + """Call _format_and_raise_error_if_any with failed_jobs.""" + with pytest.raises(RuntimeError): + cron._format_and_raise_error_if_any( + [ + ["one", Exception("one")], + ["two", Exception("two")], + ] + ) + + +def test_format_and_raise_error_if_any(): + """Call _format_and_raise_error_if_any without failed_jobs.""" + cron._format_and_raise_error_if_any([]) diff --git a/builddecisionscript/tests/test_cron_action.py b/builddecisionscript/tests/test_cron_action.py new file mode 100644 index 000000000..e96d161b3 --- /dev/null +++ b/builddecisionscript/tests/test_cron_action.py @@ -0,0 +1,64 @@ +import json +import os + +import pytest +import taskcluster + +import build_decision.cron.action as action + + +def test_find_decision_task(mocker): + """Mock ``Index`` and return a task id.""" + find_task = {"taskId": "found_task_id"} + fake_index = mocker.MagicMock() + fake_index.findTask.return_value = find_task + fake_repo = mocker.MagicMock() + mocker.patch.object(taskcluster, "Index", return_value=fake_index) + assert action.find_decision_task(fake_repo, "rev") == "found_task_id" + + +@pytest.mark.parametrize( + "include_cron_input, extra_input, dry_run", + ( + (False, False, False), + (True, False, True), + (False, True, False), + (True, True, True), + ), +) +def test_run_trigger_action(mocker, include_cron_input, extra_input, dry_run): + """Add coverage to cron.action.run_trigger_action.""" + expected_input = {} + job = { + "action-name": "action", + } + env = {} + if include_cron_input: + job["include-cron-input"] = True + cron_input = {"cron_input": {"one": "two"}} + env["HOOK_PAYLOAD"] = json.dumps(cron_input) + expected_input.update(cron_input) + + if extra_input: + job["extra-input"] = {"extra_input": {"three": "four"}} + expected_input.update(job["extra-input"]) + + def fake_render_action(*, action_input, **kwargs): + assert action_input == expected_input + return fake_hook + + fake_hook = mocker.MagicMock() + mocker.patch.object(os, "environ", new=env) + mocker.patch.object(action, "find_decision_task", return_value="decision_task_id") + mocker.patch.object(action, "render_action", new=fake_render_action) + action.run_trigger_action( + "action-name", + job, + repository=None, + push_info={"revision": "rev"}, + dry_run=dry_run, + ) + if not dry_run: + fake_hook.submit.assert_called_once_with() + else: + fake_hook.submit.assert_not_called() diff --git a/builddecisionscript/tests/test_cron_decision.py b/builddecisionscript/tests/test_cron_decision.py new file mode 100644 index 000000000..e0df9422e --- /dev/null +++ b/builddecisionscript/tests/test_cron_decision.py @@ -0,0 +1,110 @@ +import os + +import pytest + +import build_decision.cron.decision as decision + + +@pytest.mark.parametrize( + "job, expected", + ( + ({}, []), + ( + { + "target-tasks-method": "target", + }, + ["--target-tasks-method=target"], + ), + ( + { + "target-tasks-method": "target", + "include-push-tasks": True, + }, + ["--target-tasks-method=target", "--include-push-tasks"], + ), + ( + { + "optimize-target-tasks": ["one", "two"], + "rebuild-kinds": ["three", "four"], + }, + [ + "--optimize-target-tasks=['one', 'two']", + "--rebuild-kind=three", + "--rebuild-kind=four", + ], + ), + ), +) +def test_make_arguments(job, expected): + """Add coverage for cron.decision.make_arguments.""" + assert decision.make_arguments(job) == expected + + +@pytest.fixture +def run_decision_task(mocker): + mocker.patch.object( + os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"} + ) + job_name = "abc" + + def inner(job=None, dry_run=False, env=None): + if env: + mocker.patch.dict(os.environ, env) + + job = job or {} + job.setdefault("treeherder-symbol", "x") + + mocks = { + "hook": mocker.MagicMock(), + "repo": mocker.MagicMock(), + "render": mocker.MagicMock(), + } + mocks["repo"].get_file.return_value = {"tc": True} + mocks["render"].return_value = mocks["hook"] + + mocker.patch.object(decision, "render_tc_yml", new=mocks["render"]) + mocker.patch.object(decision, "make_arguments", return_value=["--option=arg"]) + + decision.run_decision_task( + job_name, + job, + repository=mocks["repo"], + push_info={"revision": "rev"}, + dry_run=dry_run, + ) + + return mocks + + return inner + + +@pytest.mark.parametrize("dry_run", (True, False)) +def test_dry_run(run_decision_task, dry_run): + """Add coverage for cron.decision.run_decision_task.""" + mocks = run_decision_task(dry_run=dry_run) + + if not dry_run: + mocks["hook"].submit.assert_called_once_with() + else: + mocks["hook"].submit.assert_not_called() + + +def test_cron_input(mocker, run_decision_task): + mocker.patch.object( + os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"} + ) + mock = run_decision_task()["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {} + + env = {"HOOK_PAYLOAD": '{"foo": "bar"}'} + mock = run_decision_task(env=env)["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {} + + mock = run_decision_task({"include-cron-input": True}, env=env)["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {"foo": "bar"} diff --git a/builddecisionscript/tests/test_cron_util.py b/builddecisionscript/tests/test_cron_util.py new file mode 100644 index 000000000..6b330e167 --- /dev/null +++ b/builddecisionscript/tests/test_cron_util.py @@ -0,0 +1,119 @@ +import datetime +import os + +import pytest +import taskcluster + +import build_decision.cron.util as util + +UTCNOW = datetime.datetime(2022, 4, 14, 20, 45, 50, 123345) +CREATED_STR = "2022-04-14T19:08:37.357Z" +CREATED = datetime.datetime.strptime(CREATED_STR, "%Y-%m-%dT%H:%M:%S.%fZ") + + +@pytest.mark.parametrize( + "time, sched, expected, raises", + ( + ( + # Raise on a minute that isn't a multiple of 15 + None, + {"minute": 17}, + None, + Exception, + ), + ( + # We match minute, nothing else specified! + UTCNOW, + {"minute": 45}, + True, + False, + ), + ( + # We don't match minute, nothing else specified! + UTCNOW, + {"minute": 30}, + False, + False, + ), + ( + # We don't match hour + UTCNOW, + {"hour": 17, "minute": 45}, + False, + False, + ), + ( + # We don't match day + UTCNOW, + {"day": 10}, + False, + False, + ), + ( + # We don't match weekday + UTCNOW, + {"weekday": "wednesday"}, + False, + False, + ), + ( + # Weekday isn't a string + UTCNOW, + {"weekday": {"one": "two"}}, + False, + False, + ), + ( + # Everything matches + UTCNOW, + {"weekday": "thursday", "day": 14, "hour": 20, "minute": 45}, + True, + False, + ), + ), +) +def test_match_utc(time, sched, expected, raises): + """Add coverage for cron.util.match_utc.""" + if raises: + with pytest.raises(raises): + util.match_utc(time=time, sched=sched) + else: + assert util.match_utc(time=time, sched=sched) == expected + + +@pytest.mark.parametrize( + "env, expected", + ( + ( + # No TASK_ID, no CRON_TIME: fall back to UTCNOW + {}, + datetime.datetime(2022, 4, 14, 20, 45, 0, 0), + ), + ( + # No TASK_ID, but there is CRON_TIME: use CRON_TIME + {"CRON_TIME": "1649994160"}, + datetime.datetime(2022, 4, 15, 3, 30, 0, 0), + ), + ( + # TASK_ID: use CREATED + {"TASK_ID": "task_id"}, + datetime.datetime(2022, 4, 14, 19, 0, 0, 0), + ), + ), +) +def test_calculate_time(mocker, env, expected): + """Add coverage for cron.util.calculate_time.""" + fake_queue = mocker.MagicMock() + fake_task = {"created": CREATED_STR} + fake_queue.task.return_value = fake_task + env.setdefault("TASKCLUSTER_PROXY_URL", "http://taskcluster") + + class fake_datetime(datetime.datetime): + def utcnow(): + return UTCNOW + + mocker.patch.object(os, "environ", new=env) + mocker.patch.object(datetime, "datetime", new=fake_datetime) + mocker.patch.object(taskcluster, "Queue", return_value=fake_queue) + + assert util.calculate_time() == expected diff --git a/builddecisionscript/tests/test_decision.py b/builddecisionscript/tests/test_decision.py new file mode 100644 index 000000000..e59cdb683 --- /dev/null +++ b/builddecisionscript/tests/test_decision.py @@ -0,0 +1,80 @@ +import os +from unittest.mock import MagicMock, patch + +import pytest + +import build_decision.decision as decision + + +@pytest.mark.parametrize( + "tc_yml, raises, expected", + ( + ( + { + "tasks": [ + { + "taskId": "one", + "key1": "value1", + }, + { + "taskId": "two", + "key1": "value2", + }, + ], + }, + True, + None, + ), + ( + { + "tasks": [], + }, + True, + None, + ), + ( + { + "tasks": [ + { + "taskId": "one", + "key1": "value1", + }, + ], + }, + False, + "one", + ), + ), +) +def test_render_tc_yml_exception(tc_yml, raises, expected): + """Cause render_tc_yml to raise an exception for task_count != 1""" + if raises: + with pytest.raises(Exception): + decision.render_tc_yml(tc_yml) + else: + task = decision.render_tc_yml(tc_yml) + assert task.task_id == expected + + +def test_display_task(): + """Add coverage for ``Task.display``.""" + task = decision.Task(task_id="asdf", task_payload={"foo": "bar"}) + # This will print() output; just exercise for coverage, for now. + # We can capture STDOUT or mock print later if we want more real testing. + task.display() + + +@pytest.mark.parametrize("proxy", (True, False)) +def test_submit_task(proxy): + """Add coverage for ``Task.submit``.""" + task_id = "asdf" + task_payload = {"foo": "bar"} + task = decision.Task(task_id=task_id, task_payload=task_payload) + env = {} + if proxy: + env["TASKCLUSTER_PROXY_URL"] = "http://taskcluster" + fake_queue = MagicMock() + with patch.object(decision.taskcluster, "Queue", return_value=fake_queue): + with patch.dict(os.environ, env, clear=True): + task.submit() + fake_queue.createTask.assert_called_once_with(task_id, task_payload) diff --git a/builddecisionscript/tests/test_git_push.py b/builddecisionscript/tests/test_git_push.py new file mode 100644 index 000000000..71b3c1e2c --- /dev/null +++ b/builddecisionscript/tests/test_git_push.py @@ -0,0 +1,79 @@ +import json +import os + +import pytest + +import build_decision.git_push as git_push + +HOOK_PAYLOAD = { + "base_ref": None, + "base_sha": "def456abc123def456abc123def456abc123def4", + "owner": "dev@example.com", + "ref": "refs/heads/main", + "sha": "abc123def456abc123def456abc123def456abc1", +} + + +@pytest.mark.parametrize( + "dry_run", + ( + True, + False, + ), +) +def test_build_decision(mocker, dry_run): + """Add coverage for git_push.build_decision.""" + taskcluster_root_url = "http://taskcluster.local" + + fake_repo = mocker.MagicMock() + fake_repo.repo_url = "https://github.com/mozilla-releng/fxci-config" + fake_repo.repo_path = "mozilla-releng/fxci-config" + fake_task = mocker.MagicMock() + + mocker.patch.object( + os, + "environ", + new={ + "TASKCLUSTER_ROOT_URL": taskcluster_root_url, + "HOOK_PAYLOAD": json.dumps(HOOK_PAYLOAD), + }, + ) + mock_render = mocker.patch.object(git_push, "render_tc_yml", return_value=fake_task) + + git_push.build_decision( + repository=fake_repo, + dry_run=dry_run, + ) + + fake_repo.get_file.assert_called_once_with( + ".taskcluster.yml", + revision=HOOK_PAYLOAD["sha"], + ) + + mock_render.assert_called_once() + render_kwargs = mock_render.call_args[1] + assert render_kwargs["taskcluster_root_url"] == taskcluster_root_url + assert render_kwargs["tasks_for"] == "git-push" + as_slugid = render_kwargs["as_slugid"] + assert callable(as_slugid) + # Same name returns the same slugid; different names return different slugids + assert as_slugid("foo") == as_slugid("foo") + assert as_slugid("foo") != as_slugid("bar") + assert render_kwargs["event"] == { + "ref": "refs/heads/main", + "before": HOOK_PAYLOAD["base_sha"], + "after": HOOK_PAYLOAD["sha"], + "base_ref": None, + "pusher": {"email": "dev@example.com"}, + "repository": { + "name": "fxci-config", + "full_name": "mozilla-releng/fxci-config", + "html_url": "https://github.com/mozilla-releng/fxci-config", + "clone_url": "https://github.com/mozilla-releng/fxci-config.git", + }, + } + + if dry_run: + fake_task.submit.assert_not_called() + else: + fake_task.submit.assert_called_once_with() diff --git a/builddecisionscript/tests/test_hg_push.py b/builddecisionscript/tests/test_hg_push.py new file mode 100644 index 000000000..4b69461c3 --- /dev/null +++ b/builddecisionscript/tests/test_hg_push.py @@ -0,0 +1,115 @@ +import json +import os +import time + +import pytest + +import build_decision.hg_push as hg_push + + +@pytest.mark.parametrize( + "pulse_payload, expected", + ( + ( + # None if `pulse_payload["type"] != "changegroup.1" + {"type": "unknown"}, + None, + ), + ( + # None if len(pushlog_pushes) == 0 + {"type": "changegroup.1", "data": {"pushlog_pushes": []}}, + None, + ), + ( + # None if len(pushlog_pushes) > 1 + {"type": "changegroup.1", "data": {"pushlog_pushes": ["one", "two"]}}, + None, + ), + ( + # None if len(heads) == 0 + {"type": "changegroup.1", "data": {"pushlog_pushes": ["one"], "heads": []}}, + None, + ), + ( + # None if len(heads) > 1 + { + "type": "changegroup.1", + "data": {"pushlog_pushes": ["one"], "heads": ["rev1", "rev2"]}, + }, + None, + ), + ( + # Success! + { + "type": "changegroup.1", + "data": {"pushlog_pushes": ["one"], "heads": ["rev1"]}, + }, + "rev1", + ), + ), +) +def test_get_revision_from_pulse_message(mocker, pulse_payload, expected): + """Add coverage for hg_push.get_revision_from_pulse_message.""" + pulse_message = json.dumps({"payload": pulse_payload}) + mocker.patch.object(os, "environ", new={"PULSE_MESSAGE": pulse_message}) + assert hg_push.get_revision_from_pulse_message() == expected + + +@pytest.mark.parametrize( + "push_age, dry_run", + ( + ( + # Ignore; too old + hg_push.MAX_TIME_DRIFT + 5000, + False, + ), + ( + # Don't ignore, dry run + 500, + True, + ), + ( + # Don't ignore + 1000, + False, + ), + ), +) +def test_build_decision(mocker, push_age, dry_run): + """Add coverage for hg_push.build_decision.""" + taskcluster_root_url = "http://taskcluster.local" + now_timestamp = 1649974668 + push = {"pushdate": now_timestamp - push_age} + fake_repo = mocker.MagicMock() + fake_repo.get_push_info.return_value = push + fake_task = mocker.MagicMock() + + mocker.patch.object( + os, "environ", new={"TASKCLUSTER_ROOT_URL": taskcluster_root_url} + ) + mocker.patch.object(hg_push, "get_revision_from_pulse_message", return_value="rev") + mocker.patch.object(time, "time", return_value=now_timestamp) + mock_render = mocker.patch.object(hg_push, "render_tc_yml", return_value=fake_task) + + args = { + "repository": fake_repo, + "dry_run": dry_run, + } + + hg_push.build_decision(**args) + + if not dry_run and push_age <= hg_push.MAX_TIME_DRIFT: + fake_task.submit.assert_called_once_with() + + mock_render.assert_called_once() + render_context = mock_render.call_args_list[0][1] + assert render_context.pop("repository", False) + assert render_context == { + "push": { + "pushdate": now_timestamp - push_age, + }, + "taskcluster_root_url": taskcluster_root_url, + "tasks_for": "hg-push", + } + else: + fake_task.submit.assert_not_called() diff --git a/builddecisionscript/tests/test_repository.py b/builddecisionscript/tests/test_repository.py new file mode 100644 index 000000000..7c0796baf --- /dev/null +++ b/builddecisionscript/tests/test_repository.py @@ -0,0 +1,354 @@ +import pytest +import redo +import yaml + +import build_decision.repository as repository + +from . import fake_redo_retry + + +@pytest.mark.parametrize( + "repository_type, repo_url, revision, raises, expected_url", + ( + ( + # HG, no revision + "hg", + "https://hg.mozilla.org/fake_repo", + None, + False, + "https://hg.mozilla.org/fake_repo/raw-file/default/fake_path", + ), + ( + # HG, revision + "hg", + "https://hg.mozilla.org/fake_repo", + "rev", + False, + "https://hg.mozilla.org/fake_repo/raw-file/rev/fake_path", + ), + ( + # Git, no revision + "git", + "https://github.com/org/repo", + None, + False, + "https://api.github.com/repos/org/repo/contents/fake_path", + ), + ( + # Git, no revision, trailing slash + "git", + "https://github.com/org/repo/", + None, + False, + "https://api.github.com/repos/org/repo/contents/fake_path", + ), + ( + # Git, revision + "git", + "https://github.com/org/repo", + "rev", + False, + "https://api.github.com/repos/org/repo/contents/fake_path?ref=rev", + ), + ( + # Raise on private git url + "git", + "git@github.com:org/repo", + "rev", + Exception, + None, + ), + ( + # Raise on unrecognized git url + "git", + "https://unknown-git-server.com:org/repo", + "rev", + Exception, + None, + ), + ( + # Raise on unknown repository_type + "unknown", + None, + None, + Exception, + None, + ), + ), +) +def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_url): + """Add coverage to ``Repository.get_file``.""" + + fake_session = mocker.MagicMock() + + mocker.patch.object(repository, "SESSION", new=fake_session) + mocker.patch.object(yaml, "safe_load") + + repo = repository.Repository( + repo_url=repo_url, + repository_type=repository_type, + ) + if raises: + with pytest.raises(raises): + repo.get_file("fake_path", revision=revision) + else: + repo.get_file("fake_path", revision=revision) + expected_headers = {} + if repo_url.startswith("https://github.com"): + expected_headers = {"Accept": "application/vnd.github.raw+json"} + fake_session.get.assert_called_with( + expected_url, headers=expected_headers, timeout=60 + ) + + +@pytest.mark.parametrize( + "branch, revision, pushes, raises, expected", + ( + ( + # NoPushesError on empty pushes + "branch", + None, + {"pushes": []}, + repository.NoPushesError, + None, + ), + ( + # ValueError on >1 pushes + None, + None, + {"pushes": ["one", "two"]}, + ValueError, + None, + ), + ( + # ValueError if rev and rev is not tip of changesets + None, + "secondary_rev", + None, + ValueError, + None, + ), + ( + None, + "rev", + None, + None, + { + "owner": "me", + "pushlog_id": 1, + "pushdate": "now", + "revision": "rev", + "base_revision": "baserev", + }, + ), + ( + None, + None, + { + "pushes": { + "1": { + "changesets": [{"parents": ["baserev"]}, {"node": "rev"}], + "user": "me", + "date": "now", + } + } + }, + None, + { + "owner": "me", + "pushlog_id": "1", + "pushdate": "now", + "revision": "rev", + "base_revision": "baserev", + }, + ), + ), +) +def test_hg_push_info(mocker, branch, revision, pushes, raises, expected): + """Add coverage for hg Repository.get_push_info""" + + if pushes is None: + pushes = { + "pushes": { + 1: { + "user": "me", + "date": "now", + "changesets": [{"node": "rev", "parents": ["baserev"]}], + } + } + } + + repo = repository.Repository( + repo_url="https://hg.mozilla.org/fake_repo", + repository_type="hg", + ) + + fake_session = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_session.get.return_value = fake_response + fake_response.json.return_value = pushes + + mocker.patch.object(repository, "SESSION", new=fake_session) + # We can't seem to mock the @redo.retriable decorator before it wraps the + # function, but we can reach into @redo.retriable, which calls redo.retry, + # and mock redo.retry + mocker.patch.object(redo, "retry", new=fake_redo_retry) + + if raises: + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + else: + assert repo.get_push_info(revision=revision, branch=branch) == expected + + +@pytest.mark.parametrize( + "branch, revision, repo_url, token, raises, expected", + ( + ( + # Die if git rev is specified + None, + "rev", + "https://github.com/org/repo", + None, + Exception, + None, + ), + ( + # Die on git@github + "main", + None, + "git@github.com:org/repo", + None, + Exception, + None, + ), + ( + # Die on non-github + None, + None, + "https://some-other-git-server.com:org/repo", + None, + Exception, + None, + ), + ( + # Use a token on main + "main", + None, + "https://github.com/org/repo", + "token", + None, + {"branch": "main", "revision": "rev"}, + ), + ), +) +def test_git_push_info(mocker, branch, revision, repo_url, token, raises, expected): + """Add coverage for git Repository.get_push_info""" + + repo = repository.Repository( + repo_url=repo_url, + repository_type="git", + github_token=token, + ) + + objects = { + "object": { + "sha": "rev", + }, + } + + fake_session = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_session.get.return_value = fake_response + fake_response.json.return_value = objects + + mocker.patch.object(repository, "SESSION", new=fake_session) + + # We can't seem to mock the @redo.retriable decorator before it wraps the + # function, but we can reach into @redo.retriable, which calls redo.retry, + # and mock redo.retry + mocker.patch.object(redo, "retry", new=fake_redo_retry) + + if raises: + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + else: + assert repo.get_push_info(revision=revision, branch=branch) == expected + + +@pytest.mark.parametrize( + "branch, revision, raises", + ( + ( + # Raise on both branch and revision + "branch", + "revision", + ValueError, + ), + ( + # Die on unknown repository_type + None, + None, + Exception, + ), + ), +) +def test_unknown_push_info(branch, revision, raises): + """Add coverage for non-hg non-git Repository.get_push_info""" + repo = repository.Repository( + repo_url="url", + repository_type="unknown", + ) + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + + +@pytest.mark.parametrize( + "repository_type, repo_url, raises, expected", + ( + ( + "hg", + "https://hg.mozilla.org/repo/path/", + None, + "repo/path", + ), + ( + "git", + "https://github.com/org/repo/", + None, + "org/repo", + ), + ( + "unknown", + "", + AttributeError, + None, + ), + ), +) +def test_repo_path(repository_type, repo_url, raises, expected): + """Add coverage to Repository.repo_path""" + repo = repository.Repository( + repo_url=repo_url, + repository_type=repository_type, + ) + if raises: + with pytest.raises(raises): + repo.repo_path + else: + assert repo.repo_path == expected + + +@pytest.mark.parametrize( + "kwargs, expected", + ( + ( + {"repo_url": "https://repo.url", "repository_type": "git"}, + {"url": "https://repo.url", "project": None, "level": None, "type": "git"}, + ), + ), +) +def test_to_json(kwargs, expected): + """Add coverage to ``Repository.to_json``.""" + repo = repository.Repository(**kwargs) + assert repo.to_json() == expected diff --git a/builddecisionscript/tests/test_secrets.py b/builddecisionscript/tests/test_secrets.py new file mode 100644 index 000000000..9664b6df4 --- /dev/null +++ b/builddecisionscript/tests/test_secrets.py @@ -0,0 +1,28 @@ +from unittest.mock import MagicMock, patch + +import pytest + +import build_decision.secrets as secrets + + +@pytest.mark.parametrize( + "secret_name, secret, secret_key, expected", + ( + ("secret1", {"secret": {"blah": "no peeking!!"}}, "blah", "no peeking!!"), + ( + "secret2", + {"secret": {"blah": "something"}}, + None, + {"secret": {"blah": "something"}}, + ), + ), +) +def test_get_secret(secret_name, secret, secret_key, expected): + """Mock the secrets fetch, and test which values we get back.""" + fake_res = MagicMock() + fake_res.json.return_value = secret + fake_session = MagicMock() + fake_session.get.return_value = fake_res + + with patch.object(secrets, "SESSION", new=fake_session): + assert secrets.get_secret(secret_name, secret_key=secret_key) == expected diff --git a/builddecisionscript/tests/test_util_cli.py b/builddecisionscript/tests/test_util_cli.py new file mode 100644 index 000000000..e9c2ce3cf --- /dev/null +++ b/builddecisionscript/tests/test_util_cli.py @@ -0,0 +1,28 @@ +import pytest + +import build_decision.util.cli as cli + + +@pytest.mark.parametrize("raises", (True, False)) +def test_cli_main(mocker, raises): + """Add coverage to util.cli.CLI.main.""" + + def fake_command(*args, **kwargs): + if raises: + raise Exception("raising") + + fake_parser = mocker.MagicMock() + fake_args = mocker.MagicMock() + fake_args.command = fake_command + fake_parser.parse_args.return_value = fake_args + + class test_cli(cli.CLI): + def create_parser(self): + return fake_parser + + c = test_cli("desc") + if raises: + with pytest.raises(SystemExit): + c.main() + else: + c.main() diff --git a/builddecisionscript/tests/test_util_keyed_by.py b/builddecisionscript/tests/test_util_keyed_by.py new file mode 100644 index 000000000..6b5ca25b8 --- /dev/null +++ b/builddecisionscript/tests/test_util_keyed_by.py @@ -0,0 +1,136 @@ +import pytest + +import build_decision.util.keyed_by as keyed_by + + +@pytest.mark.parametrize( + "attributes, target, expected", + ( + ({"key1": "value1"}, "key1", ["value1"]), + ({".*y1": "value1"}, "key1", ["value1"]), + ({"key1": "value1", "default": "default_value"}, "key2", ["default_value"]), + ({"key1": "value1"}, "nonexistent_key", []), + ), +) +def test_keymatch(attributes, target, expected): + """Test keyed-by logic, include regexes.""" + assert keyed_by.keymatch(attributes, target) == expected + + +@pytest.mark.parametrize( + "value, item_name, attributes, expected, exception", + ( + # `value` doesn't match the `by-*` pattern; expect `value` back + ("not_a_dict", "item_name", {}, "not_a_dict", None), + ( + {"key1": "value1", "key2": "value2"}, + "item_name", + {}, + {"key1": "value1", "key2": "value2"}, + False, + ), + ({"key1": "value1"}, "item_name", {}, {"key1": "value1"}, None), + # Directly match a single item + ( + { + "by-level": { + "1": "level1", + "3": "level3", + } + }, + "key1", + {"level": "1"}, + "level1", + False, + ), + # Exception when the only choice is `default` + ( + { + "by-level": { + "default": "default_level", + } + }, + "key1", + {"level": "1"}, + "level1", + Exception, + ), + # Exception when the attribute doesn't exist or is None and no default value + ( + { + "by-level": { + "1": "level1", + "3": "level3", + } + }, + "key1", + {}, + None, + Exception, + ), + # default value when the attribute doesn't exist or is None + ( + { + "by-level": { + "1": "level1", + "default": "default_level", + } + }, + "key1", + {}, + "default_level", + False, + ), + # Exception on more than 1 match + ( + { + "by-level": { + ".*1": "level1", + ".*21": "level21", + } + }, + "key1", + {"level": "21"}, + None, + Exception, + ), + # Exception on no match + ( + { + "by-level": { + "1": "level1", + "3": "level3", + } + }, + "key1", + {"level": "2"}, + None, + Exception, + ), + # Successful recursive match + ( + { + "by-project": { + "project1": "project1_level1", + "default": { + "by-level": { + "1": "level1", + "default": "default_level", + } + }, + }, + }, + "key1", + {}, + "default_level", + False, + ), + ), +) +def test_evaluate_keyed_by(value, item_name, attributes, expected, exception): + """Add full coverage for evaluate_keyed_by.""" + if exception: + with pytest.raises(exception): + keyed_by.evaluate_keyed_by(value, item_name, attributes) + else: + assert keyed_by.evaluate_keyed_by(value, item_name, attributes) == expected diff --git a/builddecisionscript/tests/test_util_schema.py b/builddecisionscript/tests/test_util_schema.py new file mode 100644 index 000000000..c5115711f --- /dev/null +++ b/builddecisionscript/tests/test_util_schema.py @@ -0,0 +1,11 @@ +import pytest +import referencing.exceptions + +import build_decision.util.schema as schema + + +def test_remote_ref(): + """Ensure remote references aren't resolved.""" + remote_schema = schema.Schema({"$ref": "http://example.com"}) + with pytest.raises(referencing.exceptions.Unresolvable): + remote_schema.validate("foo") diff --git a/builddecisionscript/tests/test_util_scopes.py b/builddecisionscript/tests/test_util_scopes.py new file mode 100644 index 000000000..e7c7411e4 --- /dev/null +++ b/builddecisionscript/tests/test_util_scopes.py @@ -0,0 +1,37 @@ +import pytest + +import build_decision.util.scopes as scopes + + +@pytest.mark.parametrize( + "have, require, expected", + ( + ( + # We have a subset of required scopes. + ["scope1", "scope2", "scope3"], + ["scope1", "scope3"], + True, + ), + ( + # We don't have all the required scopes. + ["scope1", "scope2", "scope3"], + ["scope1", "scope4"], + False, + ), + ( + # We have all required scopes, matching against * + ["prefix1/*", "prefix2/scope2", "prefix3/scope3-*"], + ["prefix1/scope1", "prefix2/scope2", "prefix3/scope3-4"], + True, + ), + ( + # We don't match against * + ["prefix1/*", "prefix2/scope2", "prefix3/scope3-*"], + ["prefix1/scope1", "prefix2/scope2-special", "prefix3/scope4-4"], + False, + ), + ), +) +def test_satisfies(have, require, expected): + """Add full coverage for ``scopes.satisfies``""" + assert scopes.satisfies(have=have, require=require) == expected diff --git a/builddecisionscript/tests/test_util_trigger_action.py b/builddecisionscript/tests/test_util_trigger_action.py new file mode 100644 index 000000000..1678f680d --- /dev/null +++ b/builddecisionscript/tests/test_util_trigger_action.py @@ -0,0 +1,261 @@ +import io +import json +import os + +import pytest +import requests +import taskcluster + +import build_decision.util.scopes as scopes +import build_decision.util.trigger_action as trigger_action + +from . import TEST_DATA_DIR + + +@pytest.mark.parametrize( + "context, task_tags, expected", + ( + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "different_value2", + "tag3": "required_value3", + "tag4": "required_value4", + }, + True, + ), + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "different_value2", + "tag3": "different_value3", + "tag4": "required_value4", + }, + False, + ), + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "required_value2", + "tag3": "required_value3", + }, + False, + ), + ), +) +def test_is_task_in_context(context, task_tags, expected): + """Compare context tag sets vs task tags.""" + assert trigger_action._is_task_in_context(context, task_tags) == expected + + +@pytest.mark.parametrize( + "original_task, expected_action_names", + ( + ( + None, + { + "add-new-jobs", + "cancel-all", + "release-promotion", + "retrigger-multiple", + }, + ), + ( + { + "tags": { + "kind": "cron-task", + }, + }, + { + "rerun", + "retrigger", + "cancel", + }, + ), + ), +) +def test_filter_relevant_actions(original_task, expected_action_names): + """Compare task tags against action.json's actions.""" + with open(TEST_DATA_DIR / "actions.json") as fh: + actions_json = json.load(fh) + relevant_actions = trigger_action._filter_relevant_actions( + actions_json, original_task + ) + assert set(relevant_actions.keys()) == expected_action_names + + +@pytest.mark.parametrize("raises", (None, RuntimeError)) +def test_check_decision_task_scopes(mocker, raises): + """Test how the function raises if scopes match or not.""" + + def fake_satisfies(*args, **kwargs): + # We test `scopes.satisfies` elsewhere; we're just testing the raise and + # adding coverage. + return not raises + + mocker.patch.object(trigger_action, "taskcluster") + mocker.patch.object(scopes, "satisfies", new=fake_satisfies) + + if raises: + with pytest.raises(raises): + trigger_action._check_decision_task_scopes( + "decision_task_id", "hook_group_id", "hook_id" + ) + else: + assert ( + trigger_action._check_decision_task_scopes( + "decision_task_id", "hook_group_id", "hook_id" + ) + is None + ) + + +@pytest.mark.parametrize( + "actions, action_name, task_id, action_input, raises", + ( + ( + # add-new-jobs should work for `task_id` `None` + None, + "add-new-jobs", + None, + {}, + False, + ), + ( + # retrigger should work for a non-None `task_id` + None, + "retrigger", + "task_id", + {}, + False, + ), + ( + # Die on invalid actions_json version + {"version": "invalid_version"}, + "retrigger", + "task_id", + {}, + RuntimeError, + ), + ( + # Retrigger isn't in `relevant_actions` if `task_id` is `None` + None, + "retrigger", + None, + {}, + LookupError, + ), + ( + # NotImplementedError if the action kind is not "hook" + { + "version": 1, + "actions": [ + { + "context": [], + "kind": "invalid_kind!!!", + "name": "fake_action", + } + ], + }, + "fake_action", + None, + {}, + NotImplementedError, + ), + ), +) +def test_render_action(mocker, actions, action_name, task_id, action_input, raises): + """Add coverage to ``render_action``, largely testing the raises.""" + + class fake_session: + def get(*args): + r = requests.Response() + r.status_code = 200 + r.encoding = "utf-8" + r.headers["content-type"] = "application/json" + if actions is not None: + r.raw = io.BytesIO(json.dumps(actions).encode("utf-8")) + else: + r.raw = open(TEST_DATA_DIR / "actions.json", "rb") + return r + + fake_queue = mocker.MagicMock() + fake_hook = mocker.MagicMock() + mocker.patch.object(taskcluster, "Queue", return_value=fake_queue) + mocker.patch.object(trigger_action, "Hook", new=fake_hook) + mocker.patch.object(trigger_action, "_check_decision_task_scopes") + mocker.patch.object(trigger_action, "SESSION", new=fake_session()) + + if raises: + with pytest.raises(raises): + trigger_action.render_action( + action_name=action_name, + task_id=task_id, + decision_task_id="decision_task_id", + action_input=action_input, + ) + else: + trigger_action.render_action( + action_name=action_name, + task_id=task_id, + decision_task_id="decision_task_id", + action_input=action_input, + ) + + +def test_hook_display(): + """Add coverage to Hook.display. + + Since it's only print commands, just run it. + """ + hook = trigger_action.Hook( + hook_group_id="group_id", + hook_id="id", + hook_payload={}, + ) + hook.display() + + +@pytest.mark.parametrize("has_proxy_url", (True, False)) +def test_hook_submit(mocker, has_proxy_url): + """Add coverage to Hook.submit""" + + env = {} + if has_proxy_url: + env["TASKCLUSTER_PROXY_URL"] = "fake_proxy_urL" + + mocker.patch.object(os, "environ", new=env) + mocker.patch.object(taskcluster, "Hooks") + hook = trigger_action.Hook( + hook_group_id="group_id", + hook_id="id", + hook_payload={}, + ) + hook.submit() From d59302f8a3321146b2df2a169e9a318f69f2efcb Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Wed, 20 May 2026 19:31:50 +0200 Subject: [PATCH 2/8] builddecisionscript: port to scriptworker (bug 2006684) Port the build-decision code to run as a scriptworker task. - rename the project from build-decision to builddecisionscript - adapt the original env-var and cmdline based input to be passed through the scriptworker task payload instead - add the usual scriptworker boilerplate --- builddecisionscript/README.md | 0 builddecisionscript/README.txt | 2 - builddecisionscript/docker.d/init_worker.sh | 15 ++ builddecisionscript/docker.d/worker.yml | 3 + builddecisionscript/pyproject.toml | 44 ++++-- builddecisionscript/src/build_decision/cli.py | 86 ----------- .../src/build_decision/util/cli.py | 63 -------- .../__init__.py | 0 .../cron/__init__.py | 20 ++- .../cron/action.py | 19 +-- .../cron/decision.py | 21 +-- .../cron/schema.yml | 0 .../cron/util.py | 9 +- .../data/builddecisionscript_task_schema.json | 74 ++++++++++ .../decision.py | 11 +- .../git_push.py | 17 ++- .../hg_push.py | 25 ++-- .../repository.py | 58 ++------ .../src/builddecisionscript/script.py | 100 +++++++++++++ .../secrets.py | 0 .../src/builddecisionscript/task.py | 26 ++++ .../util/__init__.py | 0 .../util/http.py | 0 .../util/keyed_by.py | 25 +--- .../util/schema.py | 5 +- .../util/scopes.py | 4 +- .../util/trigger_action.py | 33 ++--- builddecisionscript/tests/conftest.py | 68 +++++++++ builddecisionscript/tests/test_cli.py | 79 ---------- builddecisionscript/tests/test_cron.py | 15 +- builddecisionscript/tests/test_cron_action.py | 14 +- .../tests/test_cron_decision.py | 30 ++-- builddecisionscript/tests/test_cron_util.py | 8 +- builddecisionscript/tests/test_decision.py | 19 +-- builddecisionscript/tests/test_git_push.py | 14 +- builddecisionscript/tests/test_hg_push.py | 53 ++++--- builddecisionscript/tests/test_repository.py | 12 +- .../{test_util_scopes.py => test_scopes.py} | 7 +- builddecisionscript/tests/test_script.py | 136 ++++++++++++++++++ builddecisionscript/tests/test_secrets.py | 7 +- builddecisionscript/tests/test_task.py | 69 +++++++++ ...igger_action.py => test_trigger_action.py} | 35 ++--- builddecisionscript/tests/test_util_cli.py | 28 ---- .../tests/test_util_keyed_by.py | 2 +- builddecisionscript/tests/test_util_schema.py | 2 +- builddecisionscript/tox.ini | 29 ++++ pyproject.toml | 1 + uv.lock | 58 ++++++++ 48 files changed, 798 insertions(+), 548 deletions(-) create mode 100644 builddecisionscript/README.md delete mode 100644 builddecisionscript/README.txt create mode 100644 builddecisionscript/docker.d/init_worker.sh create mode 100644 builddecisionscript/docker.d/worker.yml delete mode 100644 builddecisionscript/src/build_decision/cli.py delete mode 100644 builddecisionscript/src/build_decision/util/cli.py rename builddecisionscript/src/{build_decision => builddecisionscript}/__init__.py (100%) rename builddecisionscript/src/{build_decision => builddecisionscript}/cron/__init__.py (87%) rename builddecisionscript/src/{build_decision => builddecisionscript}/cron/action.py (71%) rename builddecisionscript/src/{build_decision => builddecisionscript}/cron/decision.py (75%) rename builddecisionscript/src/{build_decision => builddecisionscript}/cron/schema.yml (100%) rename builddecisionscript/src/{build_decision => builddecisionscript}/cron/util.py (90%) create mode 100644 builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json rename builddecisionscript/src/{build_decision => builddecisionscript}/decision.py (82%) rename builddecisionscript/src/{build_decision => builddecisionscript}/git_push.py (71%) rename builddecisionscript/src/{build_decision => builddecisionscript}/hg_push.py (76%) rename builddecisionscript/src/{build_decision => builddecisionscript}/repository.py (73%) create mode 100644 builddecisionscript/src/builddecisionscript/script.py rename builddecisionscript/src/{build_decision => builddecisionscript}/secrets.py (100%) create mode 100644 builddecisionscript/src/builddecisionscript/task.py rename builddecisionscript/src/{build_decision => builddecisionscript}/util/__init__.py (100%) rename builddecisionscript/src/{build_decision => builddecisionscript}/util/http.py (100%) rename builddecisionscript/src/{build_decision => builddecisionscript}/util/keyed_by.py (73%) rename builddecisionscript/src/{build_decision => builddecisionscript}/util/schema.py (89%) rename builddecisionscript/src/{build_decision => builddecisionscript}/util/scopes.py (78%) rename builddecisionscript/src/{build_decision => builddecisionscript}/util/trigger_action.py (84%) create mode 100644 builddecisionscript/tests/conftest.py delete mode 100644 builddecisionscript/tests/test_cli.py rename builddecisionscript/tests/{test_util_scopes.py => test_scopes.py} (80%) create mode 100644 builddecisionscript/tests/test_script.py create mode 100644 builddecisionscript/tests/test_task.py rename builddecisionscript/tests/{test_util_trigger_action.py => test_trigger_action.py} (89%) delete mode 100644 builddecisionscript/tests/test_util_cli.py create mode 100644 builddecisionscript/tox.ini diff --git a/builddecisionscript/README.md b/builddecisionscript/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/builddecisionscript/README.txt b/builddecisionscript/README.txt deleted file mode 100644 index 3eeb3b5fc..000000000 --- a/builddecisionscript/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -This docker image bundles the `make_decision.py` script, along with its -dependencies, for use in handling hg-push hooks. diff --git a/builddecisionscript/docker.d/init_worker.sh b/builddecisionscript/docker.d/init_worker.sh new file mode 100644 index 000000000..a0025600a --- /dev/null +++ b/builddecisionscript/docker.d/init_worker.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -o errexit -o pipefail + +test_var_set() { + local varname=$1 + + if [[ -z "${!varname}" ]]; then + echo "error: ${varname} is not set" + exit 1 + fi +} + +test_var_set 'TASKCLUSTER_ROOT_URL' + +export VERIFY_CHAIN_OF_TRUST=false diff --git a/builddecisionscript/docker.d/worker.yml b/builddecisionscript/docker.d/worker.yml new file mode 100644 index 000000000..9051cd23b --- /dev/null +++ b/builddecisionscript/docker.d/worker.yml @@ -0,0 +1,3 @@ +work_dir: { "$eval": "WORK_DIR" } +artifact_dir: { "$eval": "ARTIFACTS_DIR" } +verbose: { "$eval": "VERBOSE == 'true'" } diff --git a/builddecisionscript/pyproject.toml b/builddecisionscript/pyproject.toml index 44526d9f5..3d6b17f6e 100644 --- a/builddecisionscript/pyproject.toml +++ b/builddecisionscript/pyproject.toml @@ -1,13 +1,16 @@ [project] -name = "build-decision" +name = "builddecisionscript" version = "1.0.0" -description = "Administration of runtime configuration (Taskcluster settings) for Firefox CI" +description = "Scriptworker script to create build decision tasks for hg-push and cron triggers" +url = "https://github.com/mozilla-releng/scriptworker-scripts/" +license = "MPL-2.0" +readme = "README.md" authors = [ - { name = "Mozilla Release Engineering", email = "release+build-decision@mozilla.com" }, + { name = "Mozilla Release Engineering", email = "release+python@mozilla.com" } ] -readme = "README.txt" classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", ] dependencies = [ "attrs", @@ -17,16 +20,39 @@ dependencies = [ "redo", "referencing", "requests", + "scriptworker-client", "slugid", "taskcluster", ] [dependency-groups] -dev = ["coverage", "flake8", "pytest", "pytest-cov", "pytest-mock"] +dev = [ + "tox", + "tox-uv", + "coverage>=4.2", + "pytest", + "pytest-asyncio<1.0", + "pytest-cov", + "pytest-mock", + "pytest-scriptworker-client", + "responses", +] -[project.scripts] -build-decision = "build_decision.cli:main" +[tool.uv.sources] +scriptworker-client = { workspace = true } +pytest-scriptworker-client = { workspace = true } [build-system] -requires = ["uv_build>=0.11.7,<0.12.0"] -build-backend = "uv_build" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = [ + "src", +] + +[tool.hatch.build.targets.wheel.sources] +"src/" = "" + +[project.scripts] +builddecisionscript = "builddecisionscript.script:main" diff --git a/builddecisionscript/src/build_decision/cli.py b/builddecisionscript/src/build_decision/cli.py deleted file mode 100644 index a88bdb6ed..000000000 --- a/builddecisionscript/src/build_decision/cli.py +++ /dev/null @@ -1,86 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. - -import functools - -from .repository import Repository -from .secrets import get_secret -from .util.cli import CLI - -app = CLI("Build decision tasks") - - -def repo_arguments(app): - def decorator(func): - @app.argument("--repo-url", required=True) - @app.argument("--project", required=True) - @app.argument("--level", required=True) - @app.argument("--repository-type", required=True) - @app.argument("--trust-domain", required=True) - @app.argument("--github-token-secret") - @functools.wraps(func) - def wrapper(args): - repository = {} - for argument in ( - "repo_url", - "project", - "level", - "repository_type", - "trust_domain", - ): - repository[argument] = args.pop(argument) - github_token_secret = args.pop("github_token_secret", None) - if github_token_secret: - repository["github_token"] = get_secret( - github_token_secret, secret_key="token" - ) - args["repository"] = Repository(**repository) - func(args) - - return wrapper - - return decorator - - -@app.command("hg-push", help="Create an hg-push decision task.") -@repo_arguments(app) -@app.argument("--dry-run", action="store_true") -def hg_push(options): - from .hg_push import build_decision # noqa: PLC0415 - - build_decision( - repository=options["repository"], - dry_run=options["dry_run"], - ) - - -@app.command("git-push", help="Create a git-push decision task.") -@repo_arguments(app) -@app.argument("--dry-run", action="store_true") -def git_push(options): - from .git_push import build_decision # noqa: PLC0415 - - build_decision( - repository=options["repository"], - dry_run=options["dry_run"], - ) - - -@app.command("cron", help="Process `.cron.yml`.") -@repo_arguments(app) -@app.argument("--branch") -@app.argument("--force-run") -@app.argument("--dry-run", action="store_true") -def cron(options): - from .cron import run # noqa: PLC0415 - - run( - repository=options["repository"], - branch=options["branch"], - force_run=options["force_run"], - dry_run=options["dry_run"], - ) - - -main = app.main diff --git a/builddecisionscript/src/build_decision/util/cli.py b/builddecisionscript/src/build_decision/util/cli.py deleted file mode 100644 index 26dcdc08c..000000000 --- a/builddecisionscript/src/build_decision/util/cli.py +++ /dev/null @@ -1,63 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. - -import argparse -import logging -import sys -import traceback - -import attr - - -@attr.s(cmp=False) -class CLI: - description = attr.ib(type=str) - _commands = attr.ib(default=[], init=False) - - def command(self, *args, **kwargs): - defaults = kwargs.pop("defaults", {}) - - def decorator(func): - self._commands.append((func, args, kwargs, defaults)) - return func - - return decorator - - @staticmethod - def argument(*names, **kwargs): - def decorator(func): - if not hasattr(func, "args"): - func.args = [] - # Decorators run from bottom to top of the order they were - # specified in the source. In order to make positional arguments - # appear in the order they were specifed, we insert arguments at - # the beginning, so that the list of arguments appears in the same - # order they were specified in the source. - func.args.insert(0, (names, kwargs)) - return func - - return decorator - - def create_parser(self): - parser = argparse.ArgumentParser(description=self.description) - subparsers = parser.add_subparsers(dest="command") - subparsers.required = True - for func, args, kwargs, defaults in self._commands: - subparser = subparsers.add_parser(*args, **kwargs) - for arg in getattr(func, "args", []): - subparser.add_argument(*arg[0], **arg[1]) - subparser.set_defaults(command=func, **defaults) - return parser - - def main(self): - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO - ) - parser = self.create_parser() - args = parser.parse_args() - try: - args.command(vars(args)) - except Exception: - traceback.print_exc() - sys.exit(1) diff --git a/builddecisionscript/src/build_decision/__init__.py b/builddecisionscript/src/builddecisionscript/__init__.py similarity index 100% rename from builddecisionscript/src/build_decision/__init__.py rename to builddecisionscript/src/builddecisionscript/__init__.py diff --git a/builddecisionscript/src/build_decision/cron/__init__.py b/builddecisionscript/src/builddecisionscript/cron/__init__.py similarity index 87% rename from builddecisionscript/src/build_decision/cron/__init__.py rename to builddecisionscript/src/builddecisionscript/cron/__init__.py index bf4547aa5..9a306264e 100644 --- a/builddecisionscript/src/build_decision/cron/__init__.py +++ b/builddecisionscript/src/builddecisionscript/cron/__init__.py @@ -2,7 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - import logging import traceback from pathlib import Path @@ -40,7 +39,6 @@ def load_jobs(repository, revision): # resolve keyed_by fields in each job jobs = cron_yml["jobs"] - return {j["name"]: j for j in jobs} @@ -60,7 +58,7 @@ def should_run(job, *, time, project): return True -def run_job(job_name, job, *, repository, push_info, dry_run=False): +def run_job(job_name, job, *, repository, push_info, cron_input=None, dry_run=False): job_type = job["job"]["type"] if job_type in JOB_TYPES: JOB_TYPES[job_type]( @@ -68,13 +66,14 @@ def run_job(job_name, job, *, repository, push_info, dry_run=False): job["job"], repository=repository, push_info=push_info, + cron_input=cron_input or {}, dry_run=dry_run, ) else: raise Exception(f"job type {job_type} not recognized") -def run(*, repository, branch, force_run, dry_run): +def run(*, repository, branch, force_run, cron_input=None, dry_run): time = calculate_time() try: @@ -96,6 +95,7 @@ def run(*, repository, branch, force_run, dry_run): jobs[job_name], repository=repository, push_info=push_info, + cron_input=cron_input, dry_run=dry_run, ) return @@ -110,6 +110,7 @@ def run(*, repository, branch, force_run, dry_run): job, repository=repository, push_info=push_info, + cron_input=cron_input, dry_run=dry_run, ) except Exception as exc: @@ -117,9 +118,7 @@ def run(*, repository, branch, force_run, dry_run): # would leave other jobs un-run. failed_jobs.append((job_name, exc)) traceback.print_exc() - logger.error( - f'cron job "{job_name}" run failed; continuing to next job' - ) + logger.error(f'cron job "{job_name}" run failed; continuing to next job') else: logger.info(f'not running cron job "{job_name}"') @@ -130,12 +129,9 @@ def run(*, repository, branch, force_run, dry_run): def _format_and_raise_error_if_any(failed_jobs): if failed_jobs: failed_job_names = [job_name for job_name, _ in failed_jobs] - failed_job_names_with_exceptions = ( - f'"{job_name}": "{exc}"' for job_name, exc in failed_jobs - ) + failed_job_names_with_exceptions = (f'"{job_name}": "{exc}"' for job_name, exc in failed_jobs) raise RuntimeError( - "Cron jobs {} couldn't be triggered properly. " - "Reason(s):\n * {}\nSee logs above for details.".format( + "Cron jobs {} couldn't be triggered properly. Reason(s):\n * {}\nSee logs above for details.".format( failed_job_names, "\n * ".join(failed_job_names_with_exceptions) ) ) diff --git a/builddecisionscript/src/build_decision/cron/action.py b/builddecisionscript/src/builddecisionscript/cron/action.py similarity index 71% rename from builddecisionscript/src/build_decision/cron/action.py rename to builddecisionscript/src/builddecisionscript/cron/action.py index b3a6f12ed..0a50fcc08 100644 --- a/builddecisionscript/src/build_decision/cron/action.py +++ b/builddecisionscript/src/builddecisionscript/cron/action.py @@ -2,10 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import json import logging -import os import taskcluster @@ -16,29 +13,23 @@ def find_decision_task(repository, revision): - """Given the parameters for this action, find the taskId of the decision - task""" + """Given repository and revision, find the taskId of the decision task.""" index = taskcluster.Index(taskcluster.optionsFromEnvironment(), session=SESSION) - decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision" # noqa + decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision" logger.info("Looking for index: %s", decision_index) task_id = index.findTask(decision_index)["taskId"] logger.info("Found decision task: %s", task_id) return task_id -def run_trigger_action(job_name, job, *, repository, push_info, dry_run): +def run_trigger_action(job_name, job, *, repository, push_info, cron_input=None, dry_run): action_name = job["action-name"] decision_task_id = find_decision_task(repository, push_info["revision"]) action_input = {} - if job.get("include-cron-input") and "HOOK_PAYLOAD" in os.environ: - cron_hook_payload = json.loads(os.environ["HOOK_PAYLOAD"]) - logger.info( - "Cron Hook Payload:\n%s", - json.dumps(cron_hook_payload, indent=4, sort_keys=True), - ) - action_input.update(cron_hook_payload) + if job.get("include-cron-input") and cron_input: + action_input.update(cron_input) if job.get("extra-input"): action_input.update(job["extra-input"]) diff --git a/builddecisionscript/src/build_decision/cron/decision.py b/builddecisionscript/src/builddecisionscript/cron/decision.py similarity index 75% rename from builddecisionscript/src/build_decision/cron/decision.py rename to builddecisionscript/src/builddecisionscript/cron/decision.py index de5ef7f37..bf8069b35 100644 --- a/builddecisionscript/src/build_decision/cron/decision.py +++ b/builddecisionscript/src/builddecisionscript/cron/decision.py @@ -2,9 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - import copy -import json import logging import os import shlex @@ -32,25 +30,18 @@ def make_arguments(job): return arguments -def run_decision_task(job_name, job, *, repository, push_info, dry_run): +def run_decision_task(job_name, job, *, repository, push_info, cron_input=None, dry_run): """Generate a basic decision task, based on the root .taskcluster.yml""" push_info = copy.deepcopy(push_info) push_info["owner"] = "cron" - taskcluster_yml = repository.get_file( - ".taskcluster.yml", revision=push_info["revision"] - ) + taskcluster_yml = repository.get_file(".taskcluster.yml", revision=push_info["revision"]) arguments = make_arguments(job) - cron_input = {} - if job.get("include-cron-input") and "HOOK_PAYLOAD" in os.environ: - cron_hook_payload = json.loads(os.environ["HOOK_PAYLOAD"]) - logger.info( - "Cron Hook Payload:\n%s", - json.dumps(cron_hook_payload, indent=4, sort_keys=True), - ) - cron_input.update(cron_hook_payload) + effective_cron_input = {} + if job.get("include-cron-input") and cron_input: + effective_cron_input.update(cron_input) cron_info = { "task_id": os.environ.get("TASK_ID", ""), @@ -58,7 +49,7 @@ def run_decision_task(job_name, job, *, repository, push_info, dry_run): "job_symbol": job["treeherder-symbol"], # args are shell-quoted since they are given to `bash -c` "quoted_args": " ".join(shlex.quote(a) for a in arguments), - "input": cron_input, + "input": effective_cron_input, } task = render_tc_yml( diff --git a/builddecisionscript/src/build_decision/cron/schema.yml b/builddecisionscript/src/builddecisionscript/cron/schema.yml similarity index 100% rename from builddecisionscript/src/build_decision/cron/schema.yml rename to builddecisionscript/src/builddecisionscript/cron/schema.yml diff --git a/builddecisionscript/src/build_decision/cron/util.py b/builddecisionscript/src/builddecisionscript/cron/util.py similarity index 90% rename from builddecisionscript/src/build_decision/cron/util.py rename to builddecisionscript/src/builddecisionscript/cron/util.py index 7bcd956a2..eb946c117 100644 --- a/builddecisionscript/src/build_decision/cron/util.py +++ b/builddecisionscript/src/builddecisionscript/cron/util.py @@ -2,7 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - import datetime import logging import os @@ -53,14 +52,10 @@ def calculate_time(): time = datetime.datetime.utcfromtimestamp(int(os.environ["CRON_TIME"])) logger.info("cron time: %s", time) else: - logger.warning( - "using current time for time; try setting $CRON_TIME to a timestamp" - ) + logger.warning("using current time for time; try setting $CRON_TIME to a timestamp") time = datetime.datetime.utcnow() else: - queue = taskcluster.Queue( - {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, session=SESSION - ) + queue = taskcluster.Queue({"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, session=SESSION) task = queue.task(os.environ["TASK_ID"]) # the task's `created` time is close to when the hook ran, although that # may be some time ago if task execution was delayed diff --git a/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json b/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json new file mode 100644 index 000000000..2c318a455 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Build Decision Task Schema", + "description": "Task schema for builddecisionscript", + "type": "object", + "properties": { + "payload": { + "type": "object", + "required": ["command", "repoUrl", "project", "level", "repositoryType", "trustDomain"], + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "enum": ["hg-push", "git-push", "cron"], + "description": "Which build decision command to run" + }, + "repoUrl": { + "type": "string", + "description": "Repository URL" + }, + "project": { + "type": "string", + "description": "Project name" + }, + "level": { + "type": "string", + "description": "Trust level" + }, + "repositoryType": { + "type": "string", + "enum": ["hg", "git"], + "description": "Repository type" + }, + "trustDomain": { + "type": "string", + "description": "Taskcluster trust domain" + }, + "githubTokenSecret": { + "type": "string", + "description": "Name of the Taskcluster secret containing a GitHub token (key: 'token')" + }, + "pulseMessage": { + "type": "object", + "description": "Pulse message payload (required for hg-push command)" + }, + "hookPayload": { + "type": "object", + "description": "Hook payload (required for git-push command)" + }, + "taskclusterYmlRepo": { + "type": "string", + "description": "Alternative hg repo URL from which to fetch .taskcluster.yml (hg-push only)" + }, + "branch": { + "type": "string", + "description": "Branch to use when fetching push info (cron only)" + }, + "forceRun": { + "type": "string", + "description": "Force-run a specific named cron job, skipping schedule checks (cron only)" + }, + "cronInput": { + "type": "object", + "description": "Additional input passed to cron jobs that set include-cron-input (cron only)" + }, + "dryRun": { + "type": "boolean", + "description": "If true, log what would happen but do not create any tasks", + "default": false + } + } + } + } +} diff --git a/builddecisionscript/src/build_decision/decision.py b/builddecisionscript/src/builddecisionscript/decision.py similarity index 82% rename from builddecisionscript/src/build_decision/decision.py rename to builddecisionscript/src/builddecisionscript/decision.py index fb3c630be..dd1aac1a5 100644 --- a/builddecisionscript/src/build_decision/decision.py +++ b/builddecisionscript/src/builddecisionscript/decision.py @@ -9,6 +9,7 @@ import attr import jsone import slugid + import taskcluster from .util.http import SESSION @@ -18,9 +19,9 @@ def render_tc_yml(tc_yml, **context): """ - Render .taskcluster.yml into an array of tasks. This provides a context - that is similar to that provided by actions and crons, but with `tasks-for` - set to `hg-push`. + Render .taskcluster.yml into a single task. The context is similar to + that provided by actions and crons, but with `tasks_for` set to the + appropriate value for the trigger type. """ ownTaskId = slugid.nice() context["ownTaskId"] = ownTaskId @@ -56,7 +57,5 @@ def submit(self): session=SESSION, ) else: - queue = taskcluster.Queue( - taskcluster.optionsFromEnvironment(), session=SESSION - ) + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) queue.createTask(self.task_id, self.task_payload) diff --git a/builddecisionscript/src/build_decision/git_push.py b/builddecisionscript/src/builddecisionscript/git_push.py similarity index 71% rename from builddecisionscript/src/build_decision/git_push.py rename to builddecisionscript/src/builddecisionscript/git_push.py index 0afa4c964..cf01c0fb3 100644 --- a/builddecisionscript/src/build_decision/git_push.py +++ b/builddecisionscript/src/builddecisionscript/git_push.py @@ -13,18 +13,17 @@ logger = logging.getLogger(__name__) -def build_decision(*, repository, dry_run): - logging.info("Running build-decision task") +def build_decision(*, repository, hook_payload, dry_run): + logging.info("Running build-decision git-push task") - payload = json.loads(os.environ["HOOK_PAYLOAD"]) - logger.info("Hook Payload:\n%s", json.dumps(payload, indent=4, sort_keys=True)) + logger.info("Hook Payload:\n%s", json.dumps(hook_payload, indent=4, sort_keys=True)) event = { - "after": payload["sha"], - "base_ref": payload.get("base_ref"), - "before": payload["base_sha"], - "pusher": {"email": payload["owner"]}, - "ref": payload["ref"], + "after": hook_payload["sha"], + "base_ref": hook_payload.get("base_ref"), + "before": hook_payload["base_sha"], + "pusher": {"email": hook_payload["owner"]}, + "ref": hook_payload["ref"], "repository": { "name": repository.repo_path.split("/")[-1], "full_name": repository.repo_path, diff --git a/builddecisionscript/src/build_decision/hg_push.py b/builddecisionscript/src/builddecisionscript/hg_push.py similarity index 76% rename from builddecisionscript/src/build_decision/hg_push.py rename to builddecisionscript/src/builddecisionscript/hg_push.py index f6007231a..82c1846fe 100644 --- a/builddecisionscript/src/build_decision/hg_push.py +++ b/builddecisionscript/src/builddecisionscript/hg_push.py @@ -2,7 +2,6 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. -import json import logging import os import time @@ -24,36 +23,33 @@ def timed(description): MAX_TIME_DRIFT = 3 * 24 * 60 * 60 -def get_revision_from_pulse_message(): - pulse_message = json.loads(os.environ["PULSE_MESSAGE"]) - logger.info( - "Pulse Message:\n%s", json.dumps(pulse_message, indent=4, sort_keys=True) - ) +def get_revision_from_pulse_message(pulse_message): + logger.info("Pulse Message:\n%s", pulse_message) pulse_payload = pulse_message["payload"] if pulse_payload["type"] != "changegroup.1": logger.info("Not a changegroup.1 message") - return + return None push_count = len(pulse_payload["data"]["pushlog_pushes"]) if push_count != 1: logger.info("Message has %d pushes; only one supported", push_count) - return + return None head_count = len(pulse_payload["data"]["heads"]) if head_count != 1: logger.info("Message has %d heads; only one supported", head_count) - return + return None return pulse_payload["data"]["heads"][0] -def build_decision(*, repository, dry_run): - logging.info("Running build-decision task") +def build_decision(*, repository, taskcluster_yml_repo, pulse_message, dry_run): + logging.info("Running build-decision hg-push task") # The hg-push hook can be triggered manually, so we throw out everything # from the input, other than the revision, and get the pushinfo from # hg.mozilla.org. - revision = get_revision_from_pulse_message() + revision = get_revision_from_pulse_message(pulse_message) with timed("Fetching push info"): push = repository.get_push_info(revision=revision) @@ -63,7 +59,10 @@ def build_decision(*, repository, dry_run): return with timed("Fetching .taskcluster.yml"): - taskcluster_yml = repository.get_file(".taskcluster.yml", revision=revision) + if taskcluster_yml_repo is None: + taskcluster_yml = repository.get_file(".taskcluster.yml", revision=revision) + else: + taskcluster_yml = taskcluster_yml_repo.get_file(".taskcluster.yml") with timed("Rendering task"): task = render_tc_yml( diff --git a/builddecisionscript/src/build_decision/repository.py b/builddecisionscript/src/builddecisionscript/repository.py similarity index 73% rename from builddecisionscript/src/build_decision/repository.py rename to builddecisionscript/src/builddecisionscript/repository.py index 63b6bab6d..eb26d9426 100644 --- a/builddecisionscript/src/build_decision/repository.py +++ b/builddecisionscript/src/builddecisionscript/repository.py @@ -29,11 +29,9 @@ class Repository: def get_file(self, path, *, revision=None): """ - Get `.taskcluster.yml` from 'default' (or the given revision) at the named - repo_path. Note that this does not parse the yml (so that it can be hashed + Get a file from 'default' (or the given revision) at the named path. + Note that this does not parse the yml (so that it can be hashed in its original form). - - If the file is not found, this returns None. """ headers = {} @@ -55,22 +53,15 @@ def get_file(self, path, *, revision=None): headers["Authorization"] = f"token {self.github_token}" headers["Accept"] = "application/vnd.github.raw+json" elif repo_url.startswith("git@github.com:"): - raise Exception( - f"Don't know how to get file from private github repo: {repo_url}" - ) + raise Exception(f"Don't know how to get file from private github repo: {repo_url}") else: - raise Exception( - "Don't know how to determine get file for non-github " - f"repo: {repo_url}" - ) + raise Exception(f"Don't know how to determine get file for non-github repo: {repo_url}") else: raise Exception(f"Unknown repository_type {self.repository_type}!") res = SESSION.get(url, headers=headers, timeout=60) res.raise_for_status() - tcyml = res.text - - return yaml.safe_load(tcyml) + return yaml.safe_load(res.text) @redo.retriable( attempts=5, @@ -102,24 +93,16 @@ def get_push_info(self, *, revision=None, branch=None): # If we query immediately after a push, hg.mozilla.org might # report that there are no pushes associated to a changeset. # We retry, since this tends to be a transient error. - raise NoPushesError( - f"Changeset {revset} has no associated pushes. " - "Maybe the push log has not been updated?" - ) + raise NoPushesError(f"Changeset {revset} has no associated pushes. Maybe the push log has not been updated?") elif len(pushes) != 1: - raise ValueError( - f"Changeset {revset} has {len(pushes)} associated pushes; " - "only one supported." - ) + raise ValueError(f"Changeset {revset} has {len(pushes)} associated pushes; only one supported.") [(push_id, push_info)] = pushes.items() changesets = push_info["changesets"] first_pushed_revision = changesets[0] base_revision = first_pushed_revision["parents"][0] tip_revision = changesets[-1]["node"] if revision and revision != tip_revision: - raise ValueError( - f"Changeset {revision} is not the tip {tip_revision} of the associated push." - ) + raise ValueError(f"Changeset {revision} is not the tip {tip_revision} of the associated push.") return { "owner": push_info["user"], @@ -138,10 +121,7 @@ def get_push_info(self, *, revision=None, branch=None): if self.github_token: headers["Authorization"] = f"token {self.github_token}" if repo_url.startswith("https://github.com/"): - url = ( - f"https://api.github.com" - f"/repos/{self.repo_path}/git/ref/heads/{branch}" - ) + url = f"https://api.github.com/repos/{self.repo_path}/git/ref/heads/{branch}" res = SESSION.get(url, headers=headers, timeout=60) res.raise_for_status() return { @@ -149,30 +129,20 @@ def get_push_info(self, *, revision=None, branch=None): "revision": res.json()["object"]["sha"], } elif repo_url.startswith("git@github.com:"): - raise Exception( - "Don't know how to determine revision for private github " - f"repo: {repo_url}" - ) + raise Exception(f"Don't know how to determine revision for private github repo: {repo_url}") else: - raise Exception( - "Don't know how to determine revision for for non-github " - f"repo: {repo_url}" - ) + raise Exception(f"Don't know how to determine revision for non-github repo: {repo_url}") else: raise Exception(f"Unknown repository_type {self.repository_type}!") @property def repo_path(self): - if self.repository_type == "hg" and self.repo_url.startswith( - "https://hg.mozilla.org/" - ): + if self.repository_type == "hg" and self.repo_url.startswith("https://hg.mozilla.org/"): return self.repo_url.replace("https://hg.mozilla.org/", "", 1).rstrip("/") - elif self.repository_type == "git" and self.repo_url.startswith( - "https://github.com/" - ): + elif self.repository_type == "git" and self.repo_url.startswith("https://github.com/"): return self.repo_url.replace("https://github.com/", "", 1).rstrip("/") else: - raise AttributeError(f"no repo_path available for project {self.alias}") + raise AttributeError(f"no repo_path available for {self.repo_url}") def to_json(self): return { diff --git a/builddecisionscript/src/builddecisionscript/script.py b/builddecisionscript/src/builddecisionscript/script.py new file mode 100644 index 000000000..b940705a6 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/script.py @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from scriptworker_client.client import sync_main + +from .repository import Repository +from .secrets import get_secret +from .task import validate_task_schema + +logger = logging.getLogger(__name__) + + +def _build_repository(payload): + github_token = None + if payload.get("githubTokenSecret"): + github_token = get_secret(payload["githubTokenSecret"], secret_key="token") + + return Repository( + repo_url=payload["repoUrl"], + repository_type=payload["repositoryType"], + project=payload["project"], + level=payload["level"], + trust_domain=payload["trustDomain"], + github_token=github_token, + ) + + +async def async_main(config, task): + validate_task_schema(task) + payload = task["payload"] + command = payload["command"] + dry_run = payload.get("dryRun", False) + + repository = _build_repository(payload) + + if command == "hg-push": + from .hg_push import build_decision # noqa: PLC0415 + + taskcluster_yml_repo = None + if payload.get("taskclusterYmlRepo"): + taskcluster_yml_repo = Repository( + repo_url=payload["taskclusterYmlRepo"], + repository_type="hg", + ) + + pulse_message = payload.get("pulseMessage") + if pulse_message is None: + raise ValueError("pulseMessage is required for hg-push command") + + build_decision( + repository=repository, + taskcluster_yml_repo=taskcluster_yml_repo, + pulse_message=pulse_message, + dry_run=dry_run, + ) + + elif command == "git-push": + from .git_push import build_decision # noqa: PLC0415 + + hook_payload = payload.get("hookPayload") + if hook_payload is None: + raise ValueError("hookPayload is required for git-push command") + + build_decision( + repository=repository, + hook_payload=hook_payload, + dry_run=dry_run, + ) + + elif command == "cron": + from .cron import run # noqa: PLC0415 + + run( + repository=repository, + branch=payload.get("branch"), + force_run=payload.get("forceRun"), + cron_input=payload.get("cronInput"), + dry_run=dry_run, + ) + + +def get_default_config(base_dir=None): + base_dir = base_dir or os.path.dirname(os.getcwd()) + return { + "work_dir": os.path.join(base_dir, "work_dir"), + "artifact_dir": os.path.join(base_dir, "artifact_dir"), + "schema_file": os.path.join(os.path.dirname(__file__), "data", "builddecisionscript_task_schema.json"), + } + + +def main(): + return sync_main(async_main, default_config=get_default_config()) + + +if __name__ == "__main__": + main() diff --git a/builddecisionscript/src/build_decision/secrets.py b/builddecisionscript/src/builddecisionscript/secrets.py similarity index 100% rename from builddecisionscript/src/build_decision/secrets.py rename to builddecisionscript/src/builddecisionscript/secrets.py diff --git a/builddecisionscript/src/builddecisionscript/task.py b/builddecisionscript/src/builddecisionscript/task.py new file mode 100644 index 000000000..b38cd3477 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/task.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os + +import jsonschema + +logger = logging.getLogger(__name__) + +_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "data", "builddecisionscript_task_schema.json") + + +def _load_schema(): + with open(_SCHEMA_PATH) as f: + return json.load(f) + + +def validate_task_schema(task): + schema = _load_schema() + try: + jsonschema.validate(task, schema) + except jsonschema.ValidationError as e: + raise ValueError(f"Invalid task payload: {e.message}") from e diff --git a/builddecisionscript/src/build_decision/util/__init__.py b/builddecisionscript/src/builddecisionscript/util/__init__.py similarity index 100% rename from builddecisionscript/src/build_decision/util/__init__.py rename to builddecisionscript/src/builddecisionscript/util/__init__.py diff --git a/builddecisionscript/src/build_decision/util/http.py b/builddecisionscript/src/builddecisionscript/util/http.py similarity index 100% rename from builddecisionscript/src/build_decision/util/http.py rename to builddecisionscript/src/builddecisionscript/util/http.py diff --git a/builddecisionscript/src/build_decision/util/keyed_by.py b/builddecisionscript/src/builddecisionscript/util/keyed_by.py similarity index 73% rename from builddecisionscript/src/build_decision/util/keyed_by.py rename to builddecisionscript/src/builddecisionscript/util/keyed_by.py index 711a47244..848c65f17 100644 --- a/builddecisionscript/src/build_decision/util/keyed_by.py +++ b/builddecisionscript/src/builddecisionscript/util/keyed_by.py @@ -54,11 +54,7 @@ def evaluate_keyed_by(value, item_name, attributes): default: 12 """ while True: - if ( - not isinstance(value, dict) - or len(value) != 1 - or not list(value.keys())[0].startswith("by-") - ): + if not isinstance(value, dict) or len(value) != 1 or not list(value.keys())[0].startswith("by-"): return value keyed_by = list(value.keys())[0][3:] # strip off 'by-' prefix @@ -68,31 +64,20 @@ def evaluate_keyed_by(value, item_name, attributes): if len(alternatives) == 1 and "default" in alternatives: # Error out when only 'default' is specified as only alternatives, # because we don't need to by-{keyed_by} there. - raise Exception( - f"Keyed-by '{keyed_by}' unnecessary with only value 'default' " - f"found, when determining item {item_name}" - ) + raise Exception(f"Keyed-by '{keyed_by}' unnecessary with only value 'default' found, when determining item {item_name}") if key is None: if "default" in alternatives: value = alternatives["default"] continue else: - raise Exception( - f"No attribute {keyed_by} and no value for 'default' found " - f"while determining item {item_name}" - ) + raise Exception(f"No attribute {keyed_by} and no value for 'default' found while determining item {item_name}") matches = keymatch(alternatives, key) if len(matches) > 1: - raise Exception( - f"Multiple matching values for {keyed_by} {key!r} found while " - f"determining item {item_name}" - ) + raise Exception(f"Multiple matching values for {keyed_by} {key!r} found while determining item {item_name}") elif matches: value = matches[0] continue - raise Exception( - f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}" - ) + raise Exception(f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}") diff --git a/builddecisionscript/src/build_decision/util/schema.py b/builddecisionscript/src/builddecisionscript/util/schema.py similarity index 89% rename from builddecisionscript/src/build_decision/util/schema.py rename to builddecisionscript/src/builddecisionscript/util/schema.py index f6a2c6ff1..2977ceb47 100644 --- a/builddecisionscript/src/build_decision/util/schema.py +++ b/builddecisionscript/src/builddecisionscript/util/schema.py @@ -2,7 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - import attr import yaml from jsonschema.validators import validator_for @@ -23,9 +22,7 @@ class Schema: _schema = attr.ib() _validator = attr.ib( init=False, - default=attr.Factory( - lambda self: _get_validator(self._schema), takes_self=True - ), + default=attr.Factory(lambda self: _get_validator(self._schema), takes_self=True), ) @classmethod diff --git a/builddecisionscript/src/build_decision/util/scopes.py b/builddecisionscript/src/builddecisionscript/util/scopes.py similarity index 78% rename from builddecisionscript/src/build_decision/util/scopes.py rename to builddecisionscript/src/builddecisionscript/util/scopes.py index 047ae43e9..b186e969f 100644 --- a/builddecisionscript/src/build_decision/util/scopes.py +++ b/builddecisionscript/src/builddecisionscript/util/scopes.py @@ -11,9 +11,7 @@ def satisfies(*, have, require): assert isinstance(require, list) for req_scope in require: for have_scope in have: - if have_scope == req_scope or ( - have_scope.endswith("*") and req_scope.startswith(have_scope[:-1]) - ): + if have_scope == req_scope or (have_scope.endswith("*") and req_scope.startswith(have_scope[:-1])): break else: return False diff --git a/builddecisionscript/src/build_decision/util/trigger_action.py b/builddecisionscript/src/builddecisionscript/util/trigger_action.py similarity index 84% rename from builddecisionscript/src/build_decision/util/trigger_action.py rename to builddecisionscript/src/builddecisionscript/util/trigger_action.py index 4df3a8e74..9c25dcf5d 100644 --- a/builddecisionscript/src/build_decision/util/trigger_action.py +++ b/builddecisionscript/src/builddecisionscript/util/trigger_action.py @@ -18,6 +18,7 @@ import attr import jsone import jsonschema + import taskcluster from . import scopes @@ -35,13 +36,7 @@ def _is_task_in_context(context, task_tags): a given task, if that task's tags match one of the tag-sets given in the context property for the action. """ - return any( - all( - tag in task_tags and task_tags[tag] == tag_set[tag] - for tag in tag_set.keys() - ) - for tag_set in context - ) + return any(all(tag in task_tags and task_tags[tag] == tag_set[tag] for tag in tag_set.keys()) for tag_set in context) def _filter_relevant_actions(actions_json, original_task): @@ -71,17 +66,14 @@ def _check_decision_task_scopes(decision_task_id, hook_group_id, hook_id): queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) auth = taskcluster.Auth(taskcluster.optionsFromEnvironment(), session=SESSION) decision_task = queue.task(decision_task_id) - decision_task_scopes = auth.expandScopes({"scopes": decision_task["scopes"]})[ - "scopes" - ] + decision_task_scopes = auth.expandScopes({"scopes": decision_task["scopes"]})["scopes"] in_tree_scope = f"in-tree:hook-action:{hook_group_id}/{hook_id}" if not scopes.satisfies(have=decision_task_scopes, require=[in_tree_scope]): raise RuntimeError( "Action is misconfigured: " f"decision task's scopes do not include {in_tree_scope}\n" - "Decision Task {decision_task_id} has scopes:\n" - + "\n".join(f" - {scope}" for scope in decision_task_scopes) + "Decision Task {decision_task_id} has scopes:\n" + "\n".join(f" - {scope}" for scope in decision_task_scopes) ) @@ -89,9 +81,7 @@ def render_action(*, action_name, task_id, decision_task_id, action_input): queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) logger.debug("Fetching actions.json...") - actions_url = queue.buildUrl( - "getLatestArtifact", decision_task_id, "public/actions.json" - ) + actions_url = queue.buildUrl("getLatestArtifact", decision_task_id, "public/actions.json") actions_response = SESSION.get(actions_url) actions_response.raise_for_status() actions_json = actions_response.json() @@ -106,17 +96,12 @@ def render_action(*, action_name, task_id, decision_task_id, action_input): relevant_actions = _filter_relevant_actions(actions_json, task_definition) if action_name not in relevant_actions: - raise LookupError( - f"{action_name} action is not available for this task. " - f"Available: {sorted(relevant_actions.keys())}" - ) + raise LookupError(f"{action_name} action is not available for this task. Available: {sorted(relevant_actions.keys())}") action = relevant_actions[action_name] if action["kind"] != "hook": - raise NotImplementedError( - f"Unable to submit actions with '{action['kind']}' kind." - ) + raise NotImplementedError(f"Unable to submit actions with '{action['kind']}' kind.") _check_decision_task_scopes( decision_task_id, @@ -163,9 +148,7 @@ def submit(self): session=SESSION, ) else: - hooks = taskcluster.Hooks( - taskcluster.optionsFromEnvironment(), session=SESSION - ) + hooks = taskcluster.Hooks(taskcluster.optionsFromEnvironment(), session=SESSION) logger.info("Triggering hook %s/%s", self.hook_group_id, self.hook_id) result = hooks.triggerHook(self.hook_group_id, self.hook_id, self.hook_payload) diff --git a/builddecisionscript/tests/conftest.py b/builddecisionscript/tests/conftest.py new file mode 100644 index 000000000..f457a0678 --- /dev/null +++ b/builddecisionscript/tests/conftest.py @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +PULSE_MESSAGE = { + "payload": { + "type": "changegroup.1", + "data": { + "pushlog_pushes": [{"time": 1234567890, "push_full_json_url": "..."}], + "heads": ["abc123def456"], + }, + } +} + +HG_PUSH_PAYLOAD = { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + "pulseMessage": PULSE_MESSAGE, +} + +CRON_PAYLOAD = { + "command": "cron", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + "branch": "default", +} + +HOOK_PAYLOAD = { + "base_ref": None, + "base_sha": "def456abc123def456abc123def456abc123def4", + "owner": "dev@example.com", + "ref": "refs/heads/main", + "sha": "abc123def456abc123def456abc123def456abc1", +} + +GIT_PUSH_PAYLOAD = { + "command": "git-push", + "repoUrl": "https://github.com/mozilla-releng/fxci-config", + "project": "fxci-config", + "level": "3", + "repositoryType": "git", + "trustDomain": "releng", + "hookPayload": HOOK_PAYLOAD, +} + + +@pytest.fixture +def hg_push_task(): + return {"payload": dict(HG_PUSH_PAYLOAD)} + + +@pytest.fixture +def cron_task(): + return {"payload": dict(CRON_PAYLOAD)} + + +@pytest.fixture +def git_push_task(): + return {"payload": dict(GIT_PUSH_PAYLOAD)} diff --git a/builddecisionscript/tests/test_cli.py b/builddecisionscript/tests/test_cli.py deleted file mode 100644 index 8d02d2715..000000000 --- a/builddecisionscript/tests/test_cli.py +++ /dev/null @@ -1,79 +0,0 @@ -import sys - -import pytest - -import build_decision.cli as cli -import build_decision.cron as cron -import build_decision.hg_push as hg_push - - -def test_hg_push(mocker): - """Add hg-push cli coverage.""" - options = { - "dry_run": True, - "repository": "fakerepo", - "repo_url": "fakeurl", - "project": "fakeproject", - "level": "fakelevel", - "repository_type": "fake_repository_type", - "trust_domain": "fake_trust_domain", - } - - fake_repo = mocker.MagicMock() - - def fake_build_decision(repository, dry_run): - assert repository is fake_repo - assert dry_run - - mocker.patch.object(hg_push, "build_decision", new=fake_build_decision) - mocker.patch.object(cli, "Repository", return_value=fake_repo) - cli.hg_push(options) - - -@pytest.mark.parametrize( - "token, force_run", - ( - (True, True), - (True, False), - (False, True), - (False, False), - ), -) -def test_cron(mocker, token, force_run): - """Add cron cli coverage. - - Parametrize ``token`` for ``repo_arguments`` coverage. - """ - options = { - "dry_run": True, - "repository": "fakerepo", - "repo_url": "fakeurl", - "project": "fakeproject", - "level": "fakelevel", - "repository_type": "fake_repository_type", - "trust_domain": "fake_trust_domain", - "branch": "branch", - "force_run": force_run, - } - if token: - options["github_token_secret"] = "token_secret" - - fake_repo = mocker.MagicMock() - - def fake_run(repository, branch, force_run, dry_run): - assert repository is fake_repo - assert branch == "branch" - assert force_run == options["force_run"] - assert dry_run - - mocker.patch.object(cli, "get_secret") - mocker.patch.object(cron, "run", new=fake_run) - mocker.patch.object(cli, "Repository", return_value=fake_repo) - cli.cron(options) - - -def test_main_help(mocker): - """Call cli.main() with --help.""" - mocker.patch.object(sys, "argv", new=["--help"]) - with pytest.raises(SystemExit): - cli.main() diff --git a/builddecisionscript/tests/test_cron.py b/builddecisionscript/tests/test_cron.py index 097dc4db2..6a7da4f17 100644 --- a/builddecisionscript/tests/test_cron.py +++ b/builddecisionscript/tests/test_cron.py @@ -1,9 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.cron as cron import pytest import requests.exceptions import yaml - -import build_decision.cron as cron -from build_decision.repository import NoPushesError +from builddecisionscript.repository import NoPushesError from . import TEST_DATA_DIR @@ -26,9 +29,7 @@ def test_load_jobs_404(mocker): fake_repo = mocker.MagicMock() fake_response = mocker.MagicMock() fake_response.status_code = 404 - fake_repo.get_file.side_effect = requests.exceptions.HTTPError( - response=fake_response - ) + fake_repo.get_file.side_effect = requests.exceptions.HTTPError(response=fake_response) assert cron.load_jobs(fake_repo, "rev") == {} @@ -182,7 +183,7 @@ def fake_should_run(job, **kwargs): def test_run_no_pushes(mocker): - """Ensure that running cron.hook does nothing when no pushes are found, + """Ensure that running cron.run does nothing when no pushes are found, and doesn't raise an Exception.""" fake_repo = mocker.MagicMock() diff --git a/builddecisionscript/tests/test_cron_action.py b/builddecisionscript/tests/test_cron_action.py index e96d161b3..9eed52cb5 100644 --- a/builddecisionscript/tests/test_cron_action.py +++ b/builddecisionscript/tests/test_cron_action.py @@ -1,10 +1,11 @@ -import json -import os +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import builddecisionscript.cron.action as action import pytest -import taskcluster -import build_decision.cron.action as action +import taskcluster def test_find_decision_task(mocker): @@ -32,11 +33,10 @@ def test_run_trigger_action(mocker, include_cron_input, extra_input, dry_run): job = { "action-name": "action", } - env = {} + cron_input = None if include_cron_input: job["include-cron-input"] = True cron_input = {"cron_input": {"one": "two"}} - env["HOOK_PAYLOAD"] = json.dumps(cron_input) expected_input.update(cron_input) if extra_input: @@ -48,7 +48,6 @@ def fake_render_action(*, action_input, **kwargs): return fake_hook fake_hook = mocker.MagicMock() - mocker.patch.object(os, "environ", new=env) mocker.patch.object(action, "find_decision_task", return_value="decision_task_id") mocker.patch.object(action, "render_action", new=fake_render_action) action.run_trigger_action( @@ -56,6 +55,7 @@ def fake_render_action(*, action_input, **kwargs): job, repository=None, push_info={"revision": "rev"}, + cron_input=cron_input, dry_run=dry_run, ) if not dry_run: diff --git a/builddecisionscript/tests/test_cron_decision.py b/builddecisionscript/tests/test_cron_decision.py index e0df9422e..c691832c1 100644 --- a/builddecisionscript/tests/test_cron_decision.py +++ b/builddecisionscript/tests/test_cron_decision.py @@ -1,9 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + import os +import builddecisionscript.cron.decision as decision import pytest -import build_decision.cron.decision as decision - @pytest.mark.parametrize( "job, expected", @@ -42,15 +45,10 @@ def test_make_arguments(job, expected): @pytest.fixture def run_decision_task(mocker): - mocker.patch.object( - os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"} - ) + mocker.patch.object(os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"}) job_name = "abc" - def inner(job=None, dry_run=False, env=None): - if env: - mocker.patch.dict(os.environ, env) - + def inner(job=None, dry_run=False, cron_input=None): job = job or {} job.setdefault("treeherder-symbol", "x") @@ -70,6 +68,7 @@ def inner(job=None, dry_run=False, env=None): job, repository=mocks["repo"], push_info={"revision": "rev"}, + cron_input=cron_input, dry_run=dry_run, ) @@ -89,22 +88,21 @@ def test_dry_run(run_decision_task, dry_run): mocks["hook"].submit.assert_not_called() -def test_cron_input(mocker, run_decision_task): - mocker.patch.object( - os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"} - ) +def test_cron_input(run_decision_task): + # No cron_input mock = run_decision_task()["render"] mock.assert_called_once() kwargs = mock.call_args_list[0][1] assert kwargs["cron"]["input"] == {} - env = {"HOOK_PAYLOAD": '{"foo": "bar"}'} - mock = run_decision_task(env=env)["render"] + # cron_input provided but include-cron-input not set in job + mock = run_decision_task(cron_input={"foo": "bar"})["render"] mock.assert_called_once() kwargs = mock.call_args_list[0][1] assert kwargs["cron"]["input"] == {} - mock = run_decision_task({"include-cron-input": True}, env=env)["render"] + # cron_input provided and include-cron-input set + mock = run_decision_task({"include-cron-input": True}, cron_input={"foo": "bar"})["render"] mock.assert_called_once() kwargs = mock.call_args_list[0][1] assert kwargs["cron"]["input"] == {"foo": "bar"} diff --git a/builddecisionscript/tests/test_cron_util.py b/builddecisionscript/tests/test_cron_util.py index 6b330e167..16b837d29 100644 --- a/builddecisionscript/tests/test_cron_util.py +++ b/builddecisionscript/tests/test_cron_util.py @@ -1,10 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + import datetime import os +import builddecisionscript.cron.util as util import pytest -import taskcluster -import build_decision.cron.util as util +import taskcluster UTCNOW = datetime.datetime(2022, 4, 14, 20, 45, 50, 123345) CREATED_STR = "2022-04-14T19:08:37.357Z" diff --git a/builddecisionscript/tests/test_decision.py b/builddecisionscript/tests/test_decision.py index e59cdb683..0c56e9e71 100644 --- a/builddecisionscript/tests/test_decision.py +++ b/builddecisionscript/tests/test_decision.py @@ -1,10 +1,12 @@ -import os +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + from unittest.mock import MagicMock, patch +import builddecisionscript.decision as decision import pytest -import build_decision.decision as decision - @pytest.mark.parametrize( "tc_yml, raises, expected", @@ -59,22 +61,15 @@ def test_render_tc_yml_exception(tc_yml, raises, expected): def test_display_task(): """Add coverage for ``Task.display``.""" task = decision.Task(task_id="asdf", task_payload={"foo": "bar"}) - # This will print() output; just exercise for coverage, for now. - # We can capture STDOUT or mock print later if we want more real testing. task.display() -@pytest.mark.parametrize("proxy", (True, False)) -def test_submit_task(proxy): +def test_submit_task(): """Add coverage for ``Task.submit``.""" task_id = "asdf" task_payload = {"foo": "bar"} task = decision.Task(task_id=task_id, task_payload=task_payload) - env = {} - if proxy: - env["TASKCLUSTER_PROXY_URL"] = "http://taskcluster" fake_queue = MagicMock() with patch.object(decision.taskcluster, "Queue", return_value=fake_queue): - with patch.dict(os.environ, env, clear=True): - task.submit() + task.submit() fake_queue.createTask.assert_called_once_with(task_id, task_payload) diff --git a/builddecisionscript/tests/test_git_push.py b/builddecisionscript/tests/test_git_push.py index 71b3c1e2c..399fefca2 100644 --- a/builddecisionscript/tests/test_git_push.py +++ b/builddecisionscript/tests/test_git_push.py @@ -1,10 +1,12 @@ -import json +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + import os +import builddecisionscript.git_push as git_push import pytest -import build_decision.git_push as git_push - HOOK_PAYLOAD = { "base_ref": None, "base_sha": "def456abc123def456abc123def456abc123def4", @@ -33,15 +35,13 @@ def test_build_decision(mocker, dry_run): mocker.patch.object( os, "environ", - new={ - "TASKCLUSTER_ROOT_URL": taskcluster_root_url, - "HOOK_PAYLOAD": json.dumps(HOOK_PAYLOAD), - }, + new={"TASKCLUSTER_ROOT_URL": taskcluster_root_url}, ) mock_render = mocker.patch.object(git_push, "render_tc_yml", return_value=fake_task) git_push.build_decision( repository=fake_repo, + hook_payload=HOOK_PAYLOAD, dry_run=dry_run, ) diff --git a/builddecisionscript/tests/test_hg_push.py b/builddecisionscript/tests/test_hg_push.py index 4b69461c3..54edfeccc 100644 --- a/builddecisionscript/tests/test_hg_push.py +++ b/builddecisionscript/tests/test_hg_push.py @@ -1,17 +1,19 @@ -import json +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + import os import time +import builddecisionscript.hg_push as hg_push import pytest -import build_decision.hg_push as hg_push - @pytest.mark.parametrize( "pulse_payload, expected", ( ( - # None if `pulse_payload["type"] != "changegroup.1" + # None if `pulse_payload["type"] != "changegroup.1"` {"type": "unknown"}, None, ), @@ -48,63 +50,70 @@ ), ), ) -def test_get_revision_from_pulse_message(mocker, pulse_payload, expected): +def test_get_revision_from_pulse_message(pulse_payload, expected): """Add coverage for hg_push.get_revision_from_pulse_message.""" - pulse_message = json.dumps({"payload": pulse_payload}) - mocker.patch.object(os, "environ", new={"PULSE_MESSAGE": pulse_message}) - assert hg_push.get_revision_from_pulse_message() == expected + pulse_message = {"payload": pulse_payload} + assert hg_push.get_revision_from_pulse_message(pulse_message) == expected @pytest.mark.parametrize( - "push_age, dry_run", + "push_age, use_tc_yml_repo, dry_run", ( ( # Ignore; too old hg_push.MAX_TIME_DRIFT + 5000, False, + False, ), ( # Don't ignore, dry run 500, + False, True, ), ( - # Don't ignore + # Don't ignore, use_tc_yml_repo 1000, + True, False, ), ), ) -def test_build_decision(mocker, push_age, dry_run): +def test_build_decision(mocker, push_age, use_tc_yml_repo, dry_run): """Add coverage for hg_push.build_decision.""" taskcluster_root_url = "http://taskcluster.local" now_timestamp = 1649974668 push = {"pushdate": now_timestamp - push_age} fake_repo = mocker.MagicMock() fake_repo.get_push_info.return_value = push + fake_tc_yml_repo = mocker.MagicMock() fake_task = mocker.MagicMock() - mocker.patch.object( - os, "environ", new={"TASKCLUSTER_ROOT_URL": taskcluster_root_url} - ) - mocker.patch.object(hg_push, "get_revision_from_pulse_message", return_value="rev") + mocker.patch.object(os, "environ", new={"TASKCLUSTER_ROOT_URL": taskcluster_root_url}) mocker.patch.object(time, "time", return_value=now_timestamp) mock_render = mocker.patch.object(hg_push, "render_tc_yml", return_value=fake_task) - args = { - "repository": fake_repo, - "dry_run": dry_run, + pulse_message = { + "payload": { + "type": "changegroup.1", + "data": {"pushlog_pushes": ["one"], "heads": ["rev"]}, + } } - hg_push.build_decision(**args) + hg_push.build_decision( + repository=fake_repo, + taskcluster_yml_repo=fake_tc_yml_repo if use_tc_yml_repo else None, + pulse_message=pulse_message, + dry_run=dry_run, + ) if not dry_run and push_age <= hg_push.MAX_TIME_DRIFT: fake_task.submit.assert_called_once_with() mock_render.assert_called_once() - render_context = mock_render.call_args_list[0][1] - assert render_context.pop("repository", False) - assert render_context == { + render_kwargs = mock_render.call_args_list[0][1] + assert render_kwargs.pop("repository", False) + assert render_kwargs == { "push": { "pushdate": now_timestamp - push_age, }, diff --git a/builddecisionscript/tests/test_repository.py b/builddecisionscript/tests/test_repository.py index 7c0796baf..54cc55b0f 100644 --- a/builddecisionscript/tests/test_repository.py +++ b/builddecisionscript/tests/test_repository.py @@ -1,9 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.repository as repository import pytest import redo import yaml -import build_decision.repository as repository - from . import fake_redo_retry @@ -96,9 +99,7 @@ def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_ expected_headers = {} if repo_url.startswith("https://github.com"): expected_headers = {"Accept": "application/vnd.github.raw+json"} - fake_session.get.assert_called_with( - expected_url, headers=expected_headers, timeout=60 - ) + fake_session.get.assert_called_with(expected_url, headers=expected_headers, timeout=60) @pytest.mark.parametrize( @@ -263,7 +264,6 @@ def test_git_push_info(mocker, branch, revision, repo_url, token, raises, expect fake_response.json.return_value = objects mocker.patch.object(repository, "SESSION", new=fake_session) - # We can't seem to mock the @redo.retriable decorator before it wraps the # function, but we can reach into @redo.retriable, which calls redo.retry, # and mock redo.retry diff --git a/builddecisionscript/tests/test_util_scopes.py b/builddecisionscript/tests/test_scopes.py similarity index 80% rename from builddecisionscript/tests/test_util_scopes.py rename to builddecisionscript/tests/test_scopes.py index e7c7411e4..f0f756039 100644 --- a/builddecisionscript/tests/test_util_scopes.py +++ b/builddecisionscript/tests/test_scopes.py @@ -1,6 +1,9 @@ -import pytest +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. -import build_decision.util.scopes as scopes +import builddecisionscript.util.scopes as scopes +import pytest @pytest.mark.parametrize( diff --git a/builddecisionscript/tests/test_script.py b/builddecisionscript/tests/test_script.py new file mode 100644 index 000000000..953dc7137 --- /dev/null +++ b/builddecisionscript/tests/test_script.py @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest +from builddecisionscript.script import _build_repository, async_main + + +def test_build_repository_hg(hg_push_task): + payload = hg_push_task["payload"] + repo = _build_repository(payload) + assert repo.repo_url == payload["repoUrl"] + assert repo.repository_type == "hg" + assert repo.project == "mozilla-central" + assert repo.level == "3" + assert repo.trust_domain == "gecko" + assert repo.github_token is None + + +def test_build_repository_fetches_github_token(hg_push_task, mocker): + payload = hg_push_task["payload"] + payload["githubTokenSecret"] = "project/releng/github-token" + mock_get_secret = mocker.patch("builddecisionscript.script.get_secret", return_value="mytoken") + + repo = _build_repository(payload) + + mock_get_secret.assert_called_once_with("project/releng/github-token", secret_key="token") + assert repo.github_token == "mytoken" + + +@pytest.mark.asyncio +async def test_async_main_hg_push(hg_push_task, mocker): + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + mock_build_decision.assert_called_once() + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["pulse_message"] == hg_push_task["payload"]["pulseMessage"] + assert call_kwargs["dry_run"] is False + assert call_kwargs["taskcluster_yml_repo"] is None + + +@pytest.mark.asyncio +async def test_async_main_hg_push_dry_run(hg_push_task, mocker): + hg_push_task["payload"]["dryRun"] = True + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["dry_run"] is True + + +@pytest.mark.asyncio +async def test_async_main_hg_push_with_taskcluster_yml_repo(hg_push_task, mocker): + hg_push_task["payload"]["taskclusterYmlRepo"] = "https://hg.mozilla.org/other-repo" + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["taskcluster_yml_repo"] is not None + assert call_kwargs["taskcluster_yml_repo"].repo_url == "https://hg.mozilla.org/other-repo" + + +@pytest.mark.asyncio +async def test_async_main_hg_push_missing_pulse_message(hg_push_task, mocker): + del hg_push_task["payload"]["pulseMessage"] + # schema requires pulseMessage to not be absent, but it's not required by schema + # the code itself raises ValueError + mocker.patch("builddecisionscript.hg_push.build_decision") + + with pytest.raises(ValueError, match="pulseMessage is required"): + await async_main({}, hg_push_task) + + +@pytest.mark.asyncio +async def test_async_main_git_push(git_push_task, mocker): + mock_build_decision = mocker.patch("builddecisionscript.git_push.build_decision") + + await async_main({}, git_push_task) + + mock_build_decision.assert_called_once() + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["hook_payload"] == git_push_task["payload"]["hookPayload"] + assert call_kwargs["dry_run"] is False + + +@pytest.mark.asyncio +async def test_async_main_git_push_dry_run(git_push_task, mocker): + git_push_task["payload"]["dryRun"] = True + mock_build_decision = mocker.patch("builddecisionscript.git_push.build_decision") + + await async_main({}, git_push_task) + + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["dry_run"] is True + + +@pytest.mark.asyncio +async def test_async_main_git_push_missing_hook_payload(git_push_task, mocker): + del git_push_task["payload"]["hookPayload"] + mocker.patch("builddecisionscript.git_push.build_decision") + + with pytest.raises(ValueError, match="hookPayload is required"): + await async_main({}, git_push_task) + + +@pytest.mark.asyncio +async def test_async_main_cron(cron_task, mocker): + mock_run = mocker.patch("builddecisionscript.cron.run") + + await async_main({}, cron_task) + + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["branch"] == "default" + assert call_kwargs["force_run"] is None + assert call_kwargs["cron_input"] is None + assert call_kwargs["dry_run"] is False + + +@pytest.mark.asyncio +async def test_async_main_cron_with_options(cron_task, mocker): + cron_task["payload"]["forceRun"] = "nightly" + cron_task["payload"]["cronInput"] = {"key": "val"} + cron_task["payload"]["dryRun"] = True + mock_run = mocker.patch("builddecisionscript.cron.run") + + await async_main({}, cron_task) + + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["force_run"] == "nightly" + assert call_kwargs["cron_input"] == {"key": "val"} + assert call_kwargs["dry_run"] is True diff --git a/builddecisionscript/tests/test_secrets.py b/builddecisionscript/tests/test_secrets.py index 9664b6df4..082098a05 100644 --- a/builddecisionscript/tests/test_secrets.py +++ b/builddecisionscript/tests/test_secrets.py @@ -1,9 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + from unittest.mock import MagicMock, patch +import builddecisionscript.secrets as secrets import pytest -import build_decision.secrets as secrets - @pytest.mark.parametrize( "secret_name, secret, secret_key, expected", diff --git a/builddecisionscript/tests/test_task.py b/builddecisionscript/tests/test_task.py new file mode 100644 index 000000000..cb51ad945 --- /dev/null +++ b/builddecisionscript/tests/test_task.py @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest +from builddecisionscript.task import validate_task_schema + + +def test_validate_hg_push_task(hg_push_task): + validate_task_schema(hg_push_task) + + +def test_validate_cron_task(cron_task): + validate_task_schema(cron_task) + + +def test_validate_cron_task_with_optional_fields(cron_task): + cron_task["payload"]["branch"] = "beta" + cron_task["payload"]["forceRun"] = "nightly" + cron_task["payload"]["cronInput"] = {"key": "value"} + cron_task["payload"]["dryRun"] = True + validate_task_schema(cron_task) + + +def test_validate_hg_push_task_with_taskcluster_yml_repo(hg_push_task): + hg_push_task["payload"]["taskclusterYmlRepo"] = "https://hg.mozilla.org/other-repo" + validate_task_schema(hg_push_task) + + +def test_validate_missing_required_field(): + task = { + "payload": { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + # missing project, level, repositoryType, trustDomain + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) + + +def test_validate_invalid_command(): + task = { + "payload": { + "command": "unknown", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) + + +def test_validate_invalid_repository_type(): + task = { + "payload": { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "svn", + "trustDomain": "gecko", + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) diff --git a/builddecisionscript/tests/test_util_trigger_action.py b/builddecisionscript/tests/test_trigger_action.py similarity index 89% rename from builddecisionscript/tests/test_util_trigger_action.py rename to builddecisionscript/tests/test_trigger_action.py index 1678f680d..005a0fefd 100644 --- a/builddecisionscript/tests/test_util_trigger_action.py +++ b/builddecisionscript/tests/test_trigger_action.py @@ -1,13 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + import io import json -import os +import builddecisionscript.util.scopes as scopes +import builddecisionscript.util.trigger_action as trigger_action import pytest import requests -import taskcluster -import build_decision.util.scopes as scopes -import build_decision.util.trigger_action as trigger_action +import taskcluster from . import TEST_DATA_DIR @@ -105,9 +108,7 @@ def test_filter_relevant_actions(original_task, expected_action_names): """Compare task tags against action.json's actions.""" with open(TEST_DATA_DIR / "actions.json") as fh: actions_json = json.load(fh) - relevant_actions = trigger_action._filter_relevant_actions( - actions_json, original_task - ) + relevant_actions = trigger_action._filter_relevant_actions(actions_json, original_task) assert set(relevant_actions.keys()) == expected_action_names @@ -125,16 +126,9 @@ def fake_satisfies(*args, **kwargs): if raises: with pytest.raises(raises): - trigger_action._check_decision_task_scopes( - "decision_task_id", "hook_group_id", "hook_id" - ) + trigger_action._check_decision_task_scopes("decision_task_id", "hook_group_id", "hook_id") else: - assert ( - trigger_action._check_decision_task_scopes( - "decision_task_id", "hook_group_id", "hook_id" - ) - is None - ) + assert trigger_action._check_decision_task_scopes("decision_task_id", "hook_group_id", "hook_id") is None @pytest.mark.parametrize( @@ -243,15 +237,8 @@ def test_hook_display(): hook.display() -@pytest.mark.parametrize("has_proxy_url", (True, False)) -def test_hook_submit(mocker, has_proxy_url): +def test_hook_submit(mocker): """Add coverage to Hook.submit""" - - env = {} - if has_proxy_url: - env["TASKCLUSTER_PROXY_URL"] = "fake_proxy_urL" - - mocker.patch.object(os, "environ", new=env) mocker.patch.object(taskcluster, "Hooks") hook = trigger_action.Hook( hook_group_id="group_id", diff --git a/builddecisionscript/tests/test_util_cli.py b/builddecisionscript/tests/test_util_cli.py deleted file mode 100644 index e9c2ce3cf..000000000 --- a/builddecisionscript/tests/test_util_cli.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest - -import build_decision.util.cli as cli - - -@pytest.mark.parametrize("raises", (True, False)) -def test_cli_main(mocker, raises): - """Add coverage to util.cli.CLI.main.""" - - def fake_command(*args, **kwargs): - if raises: - raise Exception("raising") - - fake_parser = mocker.MagicMock() - fake_args = mocker.MagicMock() - fake_args.command = fake_command - fake_parser.parse_args.return_value = fake_args - - class test_cli(cli.CLI): - def create_parser(self): - return fake_parser - - c = test_cli("desc") - if raises: - with pytest.raises(SystemExit): - c.main() - else: - c.main() diff --git a/builddecisionscript/tests/test_util_keyed_by.py b/builddecisionscript/tests/test_util_keyed_by.py index 6b5ca25b8..33e2e25e9 100644 --- a/builddecisionscript/tests/test_util_keyed_by.py +++ b/builddecisionscript/tests/test_util_keyed_by.py @@ -1,6 +1,6 @@ import pytest -import build_decision.util.keyed_by as keyed_by +import builddecisionscript.util.keyed_by as keyed_by @pytest.mark.parametrize( diff --git a/builddecisionscript/tests/test_util_schema.py b/builddecisionscript/tests/test_util_schema.py index c5115711f..23546f80f 100644 --- a/builddecisionscript/tests/test_util_schema.py +++ b/builddecisionscript/tests/test_util_schema.py @@ -1,7 +1,7 @@ import pytest import referencing.exceptions -import build_decision.util.schema as schema +import builddecisionscript.util.schema as schema def test_remote_ref(): diff --git a/builddecisionscript/tox.ini b/builddecisionscript/tox.ini new file mode 100644 index 000000000..9f4cddd7b --- /dev/null +++ b/builddecisionscript/tox.ini @@ -0,0 +1,29 @@ +[tox] +envlist = py311 + +[testenv] +setenv = + PYTHONDONTWRITEBYTECODE=1 + PYTHONPATH = {toxinidir}/tests +runner = uv-venv-lock-runner +package = editable +commands= + {posargs:py.test --cov-config=tox.ini --cov-append --cov={toxinidir}/src/builddecisionscript --cov-report term-missing tests} + +[testenv:clean] +skip_install = true +deps = coverage +commands = coverage erase +depends = + +[testenv:report] +skip_install = true +commands = coverage report -m +depends = py311 +parallel_show_output = true + +[pytest] +addopts = -vv -s --color=yes +asyncio_default_fixture_loop_scope = function +norecursedirs = .tox .git .hg sandbox +python_files = test_*.py diff --git a/pyproject.toml b/pyproject.toml index 0775ce035..41094dc11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.11" members = [ "addonscript", "balrogscript", + "builddecisionscript", "beetmoverscript", "bitrisescript", "bouncerscript", diff --git a/uv.lock b/uv.lock index ac86316c3..685a85b71 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,7 @@ members = [ "beetmoverscript", "bitrisescript", "bouncerscript", + "builddecisionscript", "configloader", "githubscript", "iscript", @@ -770,6 +771,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/6f/534205ba7590c9a8716a614f270c5c2ec419b5b7079b3f9cd31b7b5580de/brotlicffi-1.2.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2a5575653b0672638ba039b82fda56854934d7a6a24d4b8b5033f73ab43cbc1", size = 375108, upload-time = "2026-03-05T19:54:10.079Z" }, ] +[[package]] +name = "builddecisionscript" +version = "1.0.0" +source = { editable = "builddecisionscript" } +dependencies = [ + { name = "attrs" }, + { name = "json-e" }, + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "redo" }, + { name = "referencing" }, + { name = "requests" }, + { name = "scriptworker-client" }, + { name = "slugid" }, + { name = "taskcluster" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-scriptworker-client" }, + { name = "responses" }, + { name = "tox" }, + { name = "tox-uv" }, +] + +[package.metadata] +requires-dist = [ + { name = "attrs" }, + { name = "json-e" }, + { name = "jsonschema", specifier = ">4.18" }, + { name = "pyyaml" }, + { name = "redo" }, + { name = "referencing" }, + { name = "requests" }, + { name = "scriptworker-client", editable = "scriptworker_client" }, + { name = "slugid" }, + { name = "taskcluster" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=4.2" }, + { name = "pytest" }, + { name = "pytest-asyncio", specifier = "<1.0" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-scriptworker-client", editable = "scriptworker_client/packages/pytest-scriptworker-client" }, + { name = "responses" }, + { name = "tox" }, + { name = "tox-uv" }, +] + [[package]] name = "cachetools" version = "7.1.1" From 36257f2f04ea58f92ba7a2bd57a3381e74bc11ec Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Fri, 15 May 2026 17:08:56 +0200 Subject: [PATCH 3/8] builddecisionscript: no level or trust domain (bug 2006684) We'll have a single worker pool here so override the default scriptworker worker type/group/id. --- builddecisionscript/docker.d/init_worker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builddecisionscript/docker.d/init_worker.sh b/builddecisionscript/docker.d/init_worker.sh index a0025600a..8181edc98 100644 --- a/builddecisionscript/docker.d/init_worker.sh +++ b/builddecisionscript/docker.d/init_worker.sh @@ -13,3 +13,7 @@ test_var_set() { test_var_set 'TASKCLUSTER_ROOT_URL' export VERIFY_CHAIN_OF_TRUST=false + +export WORKER_TYPE="${PROJECT_NAME}${WORKER_SUFFIX}" +export WORKER_GROUP=${WORKER_TYPE} +export WORKER_ID_PREFIX="${WORKER_TYPE}-" From 84a7d536a0845d1292a406a85686e7b217179d4b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 18:17:38 +0100 Subject: [PATCH 4/8] builddecisionscript: use taskgraph.util.keyed_by (bug 2006684) We can use taskgraph directly, no need to keep a copy of these functions. --- builddecisionscript/pyproject.toml | 1 + .../src/builddecisionscript/cron/__init__.py | 2 +- .../src/builddecisionscript/util/keyed_by.py | 83 ----------- .../tests/test_util_keyed_by.py | 136 ------------------ uv.lock | 2 + 5 files changed, 4 insertions(+), 220 deletions(-) delete mode 100644 builddecisionscript/src/builddecisionscript/util/keyed_by.py delete mode 100644 builddecisionscript/tests/test_util_keyed_by.py diff --git a/builddecisionscript/pyproject.toml b/builddecisionscript/pyproject.toml index 3d6b17f6e..afb31a963 100644 --- a/builddecisionscript/pyproject.toml +++ b/builddecisionscript/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "scriptworker-client", "slugid", "taskcluster", + "taskcluster-taskgraph", ] [dependency-groups] diff --git a/builddecisionscript/src/builddecisionscript/cron/__init__.py b/builddecisionscript/src/builddecisionscript/cron/__init__.py index 9a306264e..f206e8cc2 100644 --- a/builddecisionscript/src/builddecisionscript/cron/__init__.py +++ b/builddecisionscript/src/builddecisionscript/cron/__init__.py @@ -7,9 +7,9 @@ from pathlib import Path from requests.exceptions import HTTPError +from taskgraph.util.keyed_by import evaluate_keyed_by from ..repository import NoPushesError -from ..util.keyed_by import evaluate_keyed_by from ..util.schema import Schema from . import action, decision from .util import calculate_time, match_utc diff --git a/builddecisionscript/src/builddecisionscript/util/keyed_by.py b/builddecisionscript/src/builddecisionscript/util/keyed_by.py deleted file mode 100644 index 848c65f17..000000000 --- a/builddecisionscript/src/builddecisionscript/util/keyed_by.py +++ /dev/null @@ -1,83 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import re - - -def keymatch(attributes, target): - """ - Determine if any keys in attributes are a match to target, then return - a list of matching values. First exact matches will be checked. Failing - that, regex matches and finally a default key. - """ - # exact match - if target in attributes: - return [attributes[target]] - - # regular expression match - matches = [v for k, v in attributes.items() if re.match(k + "$", target)] - if matches: - return matches - - # default - if "default" in attributes: - return [attributes["default"]] - - return [] - - -def evaluate_keyed_by(value, item_name, attributes): - """ - For values which can either accept a literal value, or be keyed by some - attributes, perform that lookup and return the result. - - For example, given item:: - - by-test-platform: - macosx-10.11/debug: 13 - win.*: 6 - default: 12 - - a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')` - would return `12`. - - The `item_name` parameter is used to generate useful error messages. - Items can be nested as deeply as desired:: - - by-test-platform: - win.*: - by-project: - ash: .. - cedar: .. - linux: 13 - default: 12 - """ - while True: - if not isinstance(value, dict) or len(value) != 1 or not list(value.keys())[0].startswith("by-"): - return value - - keyed_by = list(value.keys())[0][3:] # strip off 'by-' prefix - key = attributes.get(keyed_by) - alternatives = list(value.values())[0] - - if len(alternatives) == 1 and "default" in alternatives: - # Error out when only 'default' is specified as only alternatives, - # because we don't need to by-{keyed_by} there. - raise Exception(f"Keyed-by '{keyed_by}' unnecessary with only value 'default' found, when determining item {item_name}") - - if key is None: - if "default" in alternatives: - value = alternatives["default"] - continue - else: - raise Exception(f"No attribute {keyed_by} and no value for 'default' found while determining item {item_name}") - - matches = keymatch(alternatives, key) - if len(matches) > 1: - raise Exception(f"Multiple matching values for {keyed_by} {key!r} found while determining item {item_name}") - elif matches: - value = matches[0] - continue - - raise Exception(f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}") diff --git a/builddecisionscript/tests/test_util_keyed_by.py b/builddecisionscript/tests/test_util_keyed_by.py deleted file mode 100644 index 33e2e25e9..000000000 --- a/builddecisionscript/tests/test_util_keyed_by.py +++ /dev/null @@ -1,136 +0,0 @@ -import pytest - -import builddecisionscript.util.keyed_by as keyed_by - - -@pytest.mark.parametrize( - "attributes, target, expected", - ( - ({"key1": "value1"}, "key1", ["value1"]), - ({".*y1": "value1"}, "key1", ["value1"]), - ({"key1": "value1", "default": "default_value"}, "key2", ["default_value"]), - ({"key1": "value1"}, "nonexistent_key", []), - ), -) -def test_keymatch(attributes, target, expected): - """Test keyed-by logic, include regexes.""" - assert keyed_by.keymatch(attributes, target) == expected - - -@pytest.mark.parametrize( - "value, item_name, attributes, expected, exception", - ( - # `value` doesn't match the `by-*` pattern; expect `value` back - ("not_a_dict", "item_name", {}, "not_a_dict", None), - ( - {"key1": "value1", "key2": "value2"}, - "item_name", - {}, - {"key1": "value1", "key2": "value2"}, - False, - ), - ({"key1": "value1"}, "item_name", {}, {"key1": "value1"}, None), - # Directly match a single item - ( - { - "by-level": { - "1": "level1", - "3": "level3", - } - }, - "key1", - {"level": "1"}, - "level1", - False, - ), - # Exception when the only choice is `default` - ( - { - "by-level": { - "default": "default_level", - } - }, - "key1", - {"level": "1"}, - "level1", - Exception, - ), - # Exception when the attribute doesn't exist or is None and no default value - ( - { - "by-level": { - "1": "level1", - "3": "level3", - } - }, - "key1", - {}, - None, - Exception, - ), - # default value when the attribute doesn't exist or is None - ( - { - "by-level": { - "1": "level1", - "default": "default_level", - } - }, - "key1", - {}, - "default_level", - False, - ), - # Exception on more than 1 match - ( - { - "by-level": { - ".*1": "level1", - ".*21": "level21", - } - }, - "key1", - {"level": "21"}, - None, - Exception, - ), - # Exception on no match - ( - { - "by-level": { - "1": "level1", - "3": "level3", - } - }, - "key1", - {"level": "2"}, - None, - Exception, - ), - # Successful recursive match - ( - { - "by-project": { - "project1": "project1_level1", - "default": { - "by-level": { - "1": "level1", - "default": "default_level", - } - }, - }, - }, - "key1", - {}, - "default_level", - False, - ), - ), -) -def test_evaluate_keyed_by(value, item_name, attributes, expected, exception): - """Add full coverage for evaluate_keyed_by.""" - if exception: - with pytest.raises(exception): - keyed_by.evaluate_keyed_by(value, item_name, attributes) - else: - assert keyed_by.evaluate_keyed_by(value, item_name, attributes) == expected diff --git a/uv.lock b/uv.lock index 685a85b71..51a045edb 100644 --- a/uv.lock +++ b/uv.lock @@ -786,6 +786,7 @@ dependencies = [ { name = "scriptworker-client" }, { name = "slugid" }, { name = "taskcluster" }, + { name = "taskcluster-taskgraph" }, ] [package.dev-dependencies] @@ -813,6 +814,7 @@ requires-dist = [ { name = "scriptworker-client", editable = "scriptworker_client" }, { name = "slugid" }, { name = "taskcluster" }, + { name = "taskcluster-taskgraph" }, ] [package.metadata.requires-dev] From 9e1b9d0b92a6878cd59728a4ad671862f1866a28 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 19:05:06 +0100 Subject: [PATCH 5/8] builddecisionscript: add to CI (bug 2006684) --- taskcluster/kinds/docker-image/kind.yml | 5 +++++ taskcluster/kinds/push-image/kind.yml | 1 + taskcluster/kinds/tox/kind.yml | 4 ++++ tox.ini | 11 +++++++++++ 4 files changed, 21 insertions(+) diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 36af1f83a..0dac4e032 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -59,6 +59,11 @@ tasks: parent: base args: SCRIPT_NAME: bouncerscript + builddecisionscript: + definition: script + parent: base + args: + SCRIPT_NAME: builddecisionscript githubscript: definition: script parent: base diff --git a/taskcluster/kinds/push-image/kind.yml b/taskcluster/kinds/push-image/kind.yml index 3d2a65ae6..98a4f9628 100644 --- a/taskcluster/kinds/push-image/kind.yml +++ b/taskcluster/kinds/push-image/kind.yml @@ -50,6 +50,7 @@ tasks: bitrisescript: {} beetmoverscript: {} bouncerscript: {} + builddecisionscript: {} githubscript: {} landoscript: {} pushapkscript: {} diff --git a/taskcluster/kinds/tox/kind.yml b/taskcluster/kinds/tox/kind.yml index 72e2f15e7..09d7aa2c6 100644 --- a/taskcluster/kinds/tox/kind.yml +++ b/taskcluster/kinds/tox/kind.yml @@ -62,6 +62,9 @@ tasks: bouncerscript: resources: - bouncerscript + builddecisionscript: + resources: + - builddecisionscript configloader: resources: - configloader @@ -75,6 +78,7 @@ tasks: - addonscript/docker.d - balrogscript/docker.d - beetmoverscript/docker.d + - builddecisionscript/docker.d - bitrisescript/docker.d - bouncerscript/docker.d - githubscript/docker.d diff --git a/tox.ini b/tox.ini index 1f082e57c..e5899168a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = beetmoverscript-py311 bitrisescript-py311 bouncerscript-py311 + builddecisionscript-py311 configloader-py311 iscript-py311 githubscript-py311 @@ -49,6 +50,11 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py311 +[testenv:builddecisionscript-py311] +changedir = {toxinidir}/builddecisionscript +commands = + tox -e py311 + [testenv:configloader-py311] changedir = {toxinidir}/configloader commands = @@ -133,6 +139,11 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py314 +[testenv:builddecisionscript-py314] +changedir = {toxinidir}/builddecisionscript +commands = + tox -e py314 + [testenv:configloader-py314] changedir = {toxinidir}/configloader commands = From 3c3a87e6331dc8f9c357405217e1b771f5ad8c69 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 17:59:53 +0100 Subject: [PATCH 6/8] builddecisionscript: use TASKCLUSTER_CREDENTIALS_FD for TC operations (bug 2006684) There's no proxy in scriptworker, we talk to tc directly. Use the credentials fd passed by scriptworker for all taskcluster operations (fetching secrets, triggering hooks, creating tasks). --- .../src/builddecisionscript/cron/action.py | 3 +- .../src/builddecisionscript/decision.py | 11 ++----- .../src/builddecisionscript/secrets.py | 15 ++++------ .../builddecisionscript/util/taskcluster.py | 22 ++++++++++++++ .../util/trigger_action.py | 17 ++++------- builddecisionscript/tests/conftest.py | 29 ++++++++++++++++++ builddecisionscript/tests/test_cron_action.py | 2 +- builddecisionscript/tests/test_decision.py | 2 +- builddecisionscript/tests/test_secrets.py | 14 +++++---- builddecisionscript/tests/test_taskcluster.py | 30 +++++++++++++++++++ .../tests/test_trigger_action.py | 6 ++-- 11 files changed, 108 insertions(+), 43 deletions(-) create mode 100644 builddecisionscript/src/builddecisionscript/util/taskcluster.py create mode 100644 builddecisionscript/tests/test_taskcluster.py diff --git a/builddecisionscript/src/builddecisionscript/cron/action.py b/builddecisionscript/src/builddecisionscript/cron/action.py index 0a50fcc08..c5ee68a29 100644 --- a/builddecisionscript/src/builddecisionscript/cron/action.py +++ b/builddecisionscript/src/builddecisionscript/cron/action.py @@ -7,6 +7,7 @@ import taskcluster from ..util.http import SESSION +from ..util.taskcluster import get_taskcluster_options from ..util.trigger_action import render_action logger = logging.getLogger(__name__) @@ -14,7 +15,7 @@ def find_decision_task(repository, revision): """Given repository and revision, find the taskId of the decision task.""" - index = taskcluster.Index(taskcluster.optionsFromEnvironment(), session=SESSION) + index = taskcluster.Index(get_taskcluster_options(), session=SESSION) decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision" logger.info("Looking for index: %s", decision_index) task_id = index.findTask(decision_index)["taskId"] diff --git a/builddecisionscript/src/builddecisionscript/decision.py b/builddecisionscript/src/builddecisionscript/decision.py index dd1aac1a5..b479ff7e1 100644 --- a/builddecisionscript/src/builddecisionscript/decision.py +++ b/builddecisionscript/src/builddecisionscript/decision.py @@ -4,7 +4,6 @@ import json import logging -import os import attr import jsone @@ -13,6 +12,7 @@ import taskcluster from .util.http import SESSION +from .util.taskcluster import get_taskcluster_options logger = logging.getLogger(__name__) @@ -50,12 +50,5 @@ def display(self): def submit(self): logger.info("Task Id: %s", self.task_id) - - if "TASKCLUSTER_PROXY_URL" in os.environ: - queue = taskcluster.Queue( - {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, - session=SESSION, - ) - else: - queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + queue = taskcluster.Queue(get_taskcluster_options(), session=SESSION) queue.createTask(self.task_id, self.task_payload) diff --git a/builddecisionscript/src/builddecisionscript/secrets.py b/builddecisionscript/src/builddecisionscript/secrets.py index 38b484679..761963097 100644 --- a/builddecisionscript/src/builddecisionscript/secrets.py +++ b/builddecisionscript/src/builddecisionscript/secrets.py @@ -4,22 +4,17 @@ import logging +import taskcluster + from .util.http import SESSION +from .util.taskcluster import get_taskcluster_options logger = logging.getLogger(__name__) def get_secret(secret_name, secret_key=None): - # XXX should we fall back to taskcluster api call if the proxy isn't running? - # (might be difficult and we may only hit that case if we run the docker - # image locally.) - secret_url = f"http://taskcluster/secrets/v1/secret/{secret_name}" - logging.info(f"Fetching secret at {secret_url} ...") - res = SESSION.get(secret_url, timeout=60) - # This will raise an error if the secret isn't populated or we have - # infrastructure issues. Let's die so we see there's a problem. - res.raise_for_status() - secret = res.json() + secrets_client = taskcluster.Secrets(get_taskcluster_options(), session=SESSION) + secret = secrets_client.get(secret_name) if secret_key: # This will raise a KeyError if the secret is populated but isn't in the # right form. Let's die so we see there's a problem and can fix it diff --git a/builddecisionscript/src/builddecisionscript/util/taskcluster.py b/builddecisionscript/src/builddecisionscript/util/taskcluster.py new file mode 100644 index 000000000..40d3d7f3e --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/taskcluster.py @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import io +import json +import os + + +def get_taskcluster_options(): + """Return options for a Taskcluster client, reading credentials from the + file written by scriptworker and updated on each reclaim.""" + creds_fd = int(os.environ["TASKCLUSTER_CREDENTIALS_FD"]) + try: + with io.open(creds_fd, closefd=False) as f: + credentials = json.load(f) + finally: + os.lseek(creds_fd, 0, os.SEEK_SET) + return { + "rootUrl": os.environ["TASKCLUSTER_ROOT_URL"], + "credentials": credentials, + } diff --git a/builddecisionscript/src/builddecisionscript/util/trigger_action.py b/builddecisionscript/src/builddecisionscript/util/trigger_action.py index 9c25dcf5d..493e68812 100644 --- a/builddecisionscript/src/builddecisionscript/util/trigger_action.py +++ b/builddecisionscript/src/builddecisionscript/util/trigger_action.py @@ -13,7 +13,6 @@ import json import logging -import os import attr import jsone @@ -23,6 +22,7 @@ from . import scopes from .http import SESSION +from .taskcluster import get_taskcluster_options logger = logging.getLogger(__name__) @@ -63,8 +63,8 @@ def _filter_relevant_actions(actions_json, original_task): def _check_decision_task_scopes(decision_task_id, hook_group_id, hook_id): - queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) - auth = taskcluster.Auth(taskcluster.optionsFromEnvironment(), session=SESSION) + queue = taskcluster.Queue(get_taskcluster_options(), session=SESSION) + auth = taskcluster.Auth(get_taskcluster_options(), session=SESSION) decision_task = queue.task(decision_task_id) decision_task_scopes = auth.expandScopes({"scopes": decision_task["scopes"]})["scopes"] in_tree_scope = f"in-tree:hook-action:{hook_group_id}/{hook_id}" @@ -78,7 +78,7 @@ def _check_decision_task_scopes(decision_task_id, hook_group_id, hook_id): def render_action(*, action_name, task_id, decision_task_id, action_input): - queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + queue = taskcluster.Queue(get_taskcluster_options(), session=SESSION) logger.debug("Fetching actions.json...") actions_url = queue.buildUrl("getLatestArtifact", decision_task_id, "public/actions.json") @@ -142,14 +142,7 @@ def display(self): ) def submit(self): - if "TASKCLUSTER_PROXY_URL" in os.environ: - hooks = taskcluster.Hooks( - {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, - session=SESSION, - ) - else: - hooks = taskcluster.Hooks(taskcluster.optionsFromEnvironment(), session=SESSION) - + hooks = taskcluster.Hooks(get_taskcluster_options(), session=SESSION) logger.info("Triggering hook %s/%s", self.hook_group_id, self.hook_id) result = hooks.triggerHook(self.hook_group_id, self.hook_id, self.hook_payload) logger.info("Task Id: %s", result["status"]["taskId"]) diff --git a/builddecisionscript/tests/conftest.py b/builddecisionscript/tests/conftest.py index f457a0678..e9b1ba673 100644 --- a/builddecisionscript/tests/conftest.py +++ b/builddecisionscript/tests/conftest.py @@ -2,8 +2,37 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. +from unittest.mock import patch + import pytest +_FAKE_TC_OPTIONS = { + "rootUrl": "https://tc.example.com", + "credentials": { + "clientId": "test-client", + "accessToken": "test-token", + }, +} + + +@pytest.fixture +def fake_taskcluster_options(): + with ( + patch( + "builddecisionscript.decision.get_taskcluster_options", + return_value=_FAKE_TC_OPTIONS, + ), + patch( + "builddecisionscript.util.trigger_action.get_taskcluster_options", + return_value=_FAKE_TC_OPTIONS, + ), + patch( + "builddecisionscript.cron.action.get_taskcluster_options", + return_value=_FAKE_TC_OPTIONS, + ), + ): + yield + PULSE_MESSAGE = { "payload": { "type": "changegroup.1", diff --git a/builddecisionscript/tests/test_cron_action.py b/builddecisionscript/tests/test_cron_action.py index 9eed52cb5..fd9e8af9a 100644 --- a/builddecisionscript/tests/test_cron_action.py +++ b/builddecisionscript/tests/test_cron_action.py @@ -8,7 +8,7 @@ import taskcluster -def test_find_decision_task(mocker): +def test_find_decision_task(mocker, fake_taskcluster_options): """Mock ``Index`` and return a task id.""" find_task = {"taskId": "found_task_id"} fake_index = mocker.MagicMock() diff --git a/builddecisionscript/tests/test_decision.py b/builddecisionscript/tests/test_decision.py index 0c56e9e71..66584d702 100644 --- a/builddecisionscript/tests/test_decision.py +++ b/builddecisionscript/tests/test_decision.py @@ -64,7 +64,7 @@ def test_display_task(): task.display() -def test_submit_task(): +def test_submit_task(fake_taskcluster_options): """Add coverage for ``Task.submit``.""" task_id = "asdf" task_payload = {"foo": "bar"} diff --git a/builddecisionscript/tests/test_secrets.py b/builddecisionscript/tests/test_secrets.py index 082098a05..7edd63e49 100644 --- a/builddecisionscript/tests/test_secrets.py +++ b/builddecisionscript/tests/test_secrets.py @@ -21,11 +21,13 @@ ), ) def test_get_secret(secret_name, secret, secret_key, expected): - """Mock the secrets fetch, and test which values we get back.""" - fake_res = MagicMock() - fake_res.json.return_value = secret - fake_session = MagicMock() - fake_session.get.return_value = fake_res + fake_secrets_client = MagicMock() + fake_secrets_client.get.return_value = secret - with patch.object(secrets, "SESSION", new=fake_session): + with ( + patch("builddecisionscript.secrets.get_taskcluster_options", return_value={}), + patch("builddecisionscript.secrets.taskcluster.Secrets", return_value=fake_secrets_client), + ): assert secrets.get_secret(secret_name, secret_key=secret_key) == expected + + fake_secrets_client.get.assert_called_once_with(secret_name) diff --git a/builddecisionscript/tests/test_taskcluster.py b/builddecisionscript/tests/test_taskcluster.py new file mode 100644 index 000000000..1c8be3248 --- /dev/null +++ b/builddecisionscript/tests/test_taskcluster.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +import pytest + +from builddecisionscript.util.taskcluster import get_taskcluster_options + + +def test_get_taskcluster_options(tmp_path, monkeypatch): + credentials = {"clientId": "test-client", "accessToken": "test-token"} + creds_file = tmp_path / "credentials.json" + creds_file.write_text(json.dumps(credentials)) + + fd = os.open(str(creds_file), os.O_RDWR) + try: + monkeypatch.setenv("TASKCLUSTER_CREDENTIALS_FD", str(fd)) + monkeypatch.setenv("TASKCLUSTER_ROOT_URL", "https://tc.example.com") + + result = get_taskcluster_options() + + assert result == { + "rootUrl": "https://tc.example.com", + "credentials": credentials, + } + finally: + os.close(fd) diff --git a/builddecisionscript/tests/test_trigger_action.py b/builddecisionscript/tests/test_trigger_action.py index 005a0fefd..edcd92db3 100644 --- a/builddecisionscript/tests/test_trigger_action.py +++ b/builddecisionscript/tests/test_trigger_action.py @@ -113,7 +113,7 @@ def test_filter_relevant_actions(original_task, expected_action_names): @pytest.mark.parametrize("raises", (None, RuntimeError)) -def test_check_decision_task_scopes(mocker, raises): +def test_check_decision_task_scopes(mocker, raises, fake_taskcluster_options): """Test how the function raises if scopes match or not.""" def fake_satisfies(*args, **kwargs): @@ -185,7 +185,7 @@ def fake_satisfies(*args, **kwargs): ), ), ) -def test_render_action(mocker, actions, action_name, task_id, action_input, raises): +def test_render_action(mocker, actions, action_name, task_id, action_input, raises, fake_taskcluster_options): """Add coverage to ``render_action``, largely testing the raises.""" class fake_session: @@ -237,7 +237,7 @@ def test_hook_display(): hook.display() -def test_hook_submit(mocker): +def test_hook_submit(mocker, fake_taskcluster_options): """Add coverage to Hook.submit""" mocker.patch.object(taskcluster, "Hooks") hook = trigger_action.Hook( From 95ab38a5622fbd2b5c00bd564ce9497ca3fb1d8a Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Tue, 12 May 2026 17:34:27 +0200 Subject: [PATCH 7/8] builddecisionscript: coverage for non-404 error for .cron.yml (bug 2006684) --- builddecisionscript/tests/test_cron.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/builddecisionscript/tests/test_cron.py b/builddecisionscript/tests/test_cron.py index 6a7da4f17..4ec659b15 100644 --- a/builddecisionscript/tests/test_cron.py +++ b/builddecisionscript/tests/test_cron.py @@ -33,6 +33,14 @@ def test_load_jobs_404(mocker): assert cron.load_jobs(fake_repo, "rev") == {} +def test_load_jobs_500(mocker): + fake_repo = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_response.status_code = 500 + fake_repo.get_file.side_effect = requests.exceptions.HTTPError(response=fake_response) + with pytest.raises(requests.exceptions.HTTPError): + cron.load_jobs(fake_repo, "rev") == {} + @pytest.mark.parametrize( "job, match_utc_bool, project, expected", ( From d35e8a4a10fabbfa2a63aa69626cb21a9d776e5d Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Tue, 12 May 2026 17:36:15 +0200 Subject: [PATCH 8/8] builddecisionscript: coverage for token in Repository.get_file (bug 2006684) --- builddecisionscript/tests/test_repository.py | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/builddecisionscript/tests/test_repository.py b/builddecisionscript/tests/test_repository.py index 54cc55b0f..f475ea9e4 100644 --- a/builddecisionscript/tests/test_repository.py +++ b/builddecisionscript/tests/test_repository.py @@ -11,13 +11,14 @@ @pytest.mark.parametrize( - "repository_type, repo_url, revision, raises, expected_url", + "repository_type, repo_url, token, revision, raises, expected_url", ( ( # HG, no revision "hg", "https://hg.mozilla.org/fake_repo", None, + None, False, "https://hg.mozilla.org/fake_repo/raw-file/default/fake_path", ), @@ -25,6 +26,7 @@ # HG, revision "hg", "https://hg.mozilla.org/fake_repo", + None, "rev", False, "https://hg.mozilla.org/fake_repo/raw-file/rev/fake_path", @@ -34,6 +36,7 @@ "git", "https://github.com/org/repo", None, + None, False, "https://api.github.com/repos/org/repo/contents/fake_path", ), @@ -42,6 +45,7 @@ "git", "https://github.com/org/repo/", None, + None, False, "https://api.github.com/repos/org/repo/contents/fake_path", ), @@ -49,6 +53,16 @@ # Git, revision "git", "https://github.com/org/repo", + None, + "rev", + False, + "https://api.github.com/repos/org/repo/contents/fake_path?ref=rev", + ), + ( + # Git, revision, token + "git", + "https://github.com/org/repo", + "mytoken", "rev", False, "https://api.github.com/repos/org/repo/contents/fake_path?ref=rev", @@ -57,6 +71,7 @@ # Raise on private git url "git", "git@github.com:org/repo", + None, "rev", Exception, None, @@ -65,6 +80,7 @@ # Raise on unrecognized git url "git", "https://unknown-git-server.com:org/repo", + None, "rev", Exception, None, @@ -74,12 +90,13 @@ "unknown", None, None, + None, Exception, None, ), ), ) -def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_url): +def test_get_file(mocker, repository_type, repo_url, token, revision, raises, expected_url): """Add coverage to ``Repository.get_file``.""" fake_session = mocker.MagicMock() @@ -90,6 +107,7 @@ def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_ repo = repository.Repository( repo_url=repo_url, repository_type=repository_type, + github_token=token, ) if raises: with pytest.raises(raises): @@ -99,6 +117,8 @@ def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_ expected_headers = {} if repo_url.startswith("https://github.com"): expected_headers = {"Accept": "application/vnd.github.raw+json"} + if token: + expected_headers["Authorization"] = f"token {token}" fake_session.get.assert_called_with(expected_url, headers=expected_headers, timeout=60)