# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Support for running jobs that are invoked via the `run-task` script. """ import dataclasses import functools import os from typing import Literal, Union from typing import Optional as TOptional from mozpack import path from taskgraph.transforms.run.common import support_caches from taskgraph.util.schema import ( Schema, taskref_or_string_msgspec, ) from taskgraph.util.yaml import load_yaml from gecko_taskgraph.transforms.job import run_job_using from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout class RunTaskSchema(Schema, kw_only=True): using: Literal["run-task"] # Use the specified caches. use_caches: TOptional[Union[bool, list[str]]] = None # if true (the default), perform a checkout of gecko on the worker checkout: bool = True # Path to run command in. If a checkout is present, the path # to the checkout will be interpolated with the key `checkout` cwd: TOptional[str] = None # The sparse checkout profile to use. Value is the filename relative to # "sparse-profile-prefix" which defaults to "build/sparse-profiles/". sparse_profile: TOptional[Union[str, None]] = None # The relative path to the sparse profile. sparse_profile_prefix: TOptional[str] = None # Whether to use a shallow clone or not, default True (git only). shallow_clone: TOptional[bool] = None # if true, perform a checkout of a comm-central based branch inside the # gecko checkout comm_checkout: bool = False # The command arguments to pass to the `run-task` script, after the # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. command: Union[list[taskref_or_string_msgspec], taskref_or_string_msgspec] # Base work directory used to set up the task. workdir: TOptional[str] = None # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Only supported on # docker-worker. tooltool_downloads: Union[bool, Literal["public", "internal"]] = False # Whether to run as root. (defaults to False) run_as_root: TOptional[bool] = None def __post_init__(self): super().__post_init__() if self.tooltool_downloads is True: raise ValueError( "tooltool-downloads must be False, 'public', or 'internal'" ) def common_setup(config, job, taskdesc, command): run = job["run"] run_cwd = run.get("cwd") if run["checkout"]: repo_configs = config.repo_configs if len(repo_configs) > 1 and run["checkout"] is True: raise Exception("Must explicitly specify checkouts with multiple repos.") elif run["checkout"] is not True: repo_configs = { repo: dataclasses.replace(repo_configs[repo], **config) for (repo, config) in run["checkout"].items() } gecko_path = support_vcs_checkout(config, job, taskdesc, repo_configs) command.append(f"--gecko-checkout={gecko_path}") if config.params["repository_type"] == "git" and run.get("shallow-clone", True): command.append("--gecko-shallow-clone") if run_cwd: run_cwd = path.normpath(run_cwd.format(checkout=gecko_path)) elif run_cwd and "{checkout}" in run_cwd: raise Exception( "Found `{{checkout}}` interpolation in `cwd` for task {name} " "but the task doesn't have a checkout: {cwd}".format( cwd=run_cwd, name=job.get("name", job.get("label")) ) ) if config.params["repository_type"] == "hg" and run["sparse-profile"]: sparse_profile_prefix = run.pop( "sparse-profile-prefix", "build/sparse-profiles" ) sparse_profile_path = path.join(sparse_profile_prefix, run["sparse-profile"]) command.append(f"--gecko-sparse-profile={sparse_profile_path}") if run_cwd: command.append(f"--task-cwd={run_cwd}") support_caches(config, job, taskdesc) taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"] worker_defaults = { "checkout": True, "comm-checkout": False, "sparse-profile": None, "tooltool-downloads": False, "run-as-root": False, } load_yaml = functools.cache(load_yaml) def script_url(config, script): if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ: raise Exception("TASK_ID must be defined to use run-task on generic-worker") task_id = os.environ.get("TASK_ID", "") tc_url = "http://firefox-ci-tc.services.mozilla.com" return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}" @run_job_using( "docker-worker", "run-task", schema=RunTaskSchema, defaults=worker_defaults ) def docker_worker_run_task(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"] run_task_bin = ( "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg" ) command = [f"/builds/worker/bin/{run_task_bin}"] common_setup(config, job, taskdesc, command) if run["tooltool-downloads"]: internal = run["tooltool-downloads"] == "internal" add_tooltool(config, job, taskdesc, internal=internal) run_command = run["command"] # dict is for the case of `{'task-reference': text_type}`. if isinstance(run_command, (str, dict)): run_command = ["bash", "-cx", run_command] if run["comm-checkout"]: command.append( "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"]) ) if run["run-as-root"]: command.extend(("--user", "root", "--group", "root")) command.append("--") command.extend(run_command) worker["command"] = command @run_job_using( "generic-worker", "run-task", schema=RunTaskSchema, defaults=worker_defaults ) def generic_worker_run_task(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"] is_win = worker["os"] == "windows" is_mac = worker["os"] == "macosx" is_bitbar = worker["os"] == "linux-bitbar" is_lambda = worker["os"] == "linux-lambda" if run["tooltool-downloads"]: internal = run["tooltool-downloads"] == "internal" add_tooltool(config, job, taskdesc, internal=internal) if is_win: command = ["C:/mozilla-build/python3/python3.exe", "run-task"] elif is_mac: command = ["/usr/local/bin/python3", "run-task"] else: command = ["./run-task"] common_setup(config, job, taskdesc, command) worker.setdefault("mounts", []) run_task_bin = ( "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg" ) worker["mounts"].append({ "content": { "url": script_url(config, run_task_bin), }, "file": "./run-task", }) if ( job.get("fetches") or job.get("use-uv") or job.get("use-python", "system") != "system" ): worker["mounts"].append({ "content": { "url": script_url(config, "fetch-content"), }, "file": "./fetch-content", }) if run.get("checkout"): worker["mounts"].append({ "content": { "url": script_url(config, "robustcheckout.py"), }, "file": "./robustcheckout.py", }) run_command = run["command"] # dict is for the case of `{'task-reference': text_type}`. if isinstance(run_command, (str, dict)): if is_win: if isinstance(run_command, dict): for k in run_command.keys(): run_command[k] = f'"{run_command[k]}"' else: run_command = f'"{run_command}"' run_command = ["bash", "-cx", run_command] if run["comm-checkout"]: command.append( "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"]) ) if run["run-as-root"]: command.extend(("--user", "root", "--group", "root")) command.append("--") if is_bitbar: # Use the bitbar wrapper script which sets up the device and adb # environment variables command.append("/builds/taskcluster/script.py") elif is_lambda: command.append("/home/ltuser/taskcluster/script.py") command.extend(run_command) if is_win: taskref = False for c in command: if isinstance(c, dict): taskref = True if taskref: cmd = [] for c in command: if isinstance(c, dict): for v in c.values(): cmd.append(v) else: cmd.append(c) worker["command"] = [{"artifact-reference": " ".join(cmd)}] else: worker["command"] = [" ".join(command)] else: worker["command"] = [ ["chmod", "+x", "run-task"], command, ]