write_build_plan: rework + build src last

* don't show the cycles when generating the build plan
  (we have other places that show it now)
* interleave the different build types when generating jobs
* make the src jobs depend on the non-src jobs, as src builds
  depend on eiher msys or ucrt64 build results and otherwise
  will just stop due to missing deps. Could be improved by only
  depending on msys/ucrt64, but this is still an improvement.

.github/workflows/build.yml

@@ -91,7 +91,6 @@ jobs:
         python -m msys2_autobuild show --optional-deps "$OPTIONAL_DEPS"
 
   build:
-    needs: schedule
     timeout-minutes: 4320
 
     if: ${{ needs.schedule.outputs.build-plan != '[]' }}
@@ -101,6 +100,7 @@ jobs:
         include: ${{ fromJson(needs.schedule.outputs.build-plan) }}
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.runner }}
+    needs: ${{ matrix.needs }}
 
     steps:
 

msys2_autobuild/cmd_show_build.py

@@ -2,11 +2,22 @@ from typing import Any, List
 
 from tabulate import tabulate
 
-from .cmd_write_build_plan import show_cycles
+from .queue import Package, PackageStatus, get_buildqueue_with_status, get_cycles
-from .queue import PackageStatus, get_buildqueue_with_status
 from .utils import apply_optional_deps, gha_group
 
 
+def show_cycles(pkgs: List[Package]) -> None:
+    cycles = get_cycles(pkgs)
+    if cycles:
+        def format_package(p: Package) -> str:
+            return f"{p['name']} [{p['version_repo']} -> {p['version']}]"
+
+        with gha_group(f"Dependency Cycles ({len(cycles)})"):
+            print(tabulate([
+                (format_package(a), "<-->", format_package(b)) for (a, b) in cycles],
+                headers=["Package", "", "Package"]))
+
+
 def show_build(args: Any) -> None:
     todo = []
     waiting = []

msys2_autobuild/cmd_write_build_plan.py

@@ -1,99 +1,111 @@
 import json
 import shlex
 from typing import Any, Dict, List
+import itertools
 
-from tabulate import tabulate
+from .config import BuildType, Config, build_type_is_src
-
-from .config import BuildType, Config
 from .gh import get_current_repo, wait_for_api_limit_reset
 from .queue import (Package, PackageStatus, get_buildqueue_with_status,
-                    get_cycles, update_status)
+                    update_status)
-from .utils import apply_optional_deps, gha_group
+from .utils import apply_optional_deps
 
 
-def show_cycles(pkgs: List[Package]) -> None:
+SCHEDULE_JOB_NAME = "schedule"
-    cycles = get_cycles(pkgs)
-    if cycles:
-        def format_package(p: Package) -> str:
-            return f"{p['name']} [{p['version_repo']} -> {p['version']}]"
-
-        with gha_group(f"Dependency Cycles ({len(cycles)})"):
-            print(tabulate([
-                (format_package(a), "<-->", format_package(b)) for (a, b) in cycles],
-                headers=["Package", "", "Package"]))
 
 
-def get_job_meta() -> List[Dict[str, Any]]:
+def generate_jobs_for(build_type: BuildType, optional_deps: str, count: int):
-    hosted_runner = "windows-2022"
+    name = build_type
-    job_meta: List[Dict[str, Any]] = [
+    packages = " ".join(["base-devel"])
-        {
+    runner = ["windows-2022"] if build_type != "clangarm64" else ["Windows", "ARM64", "autobuild"]
-            "build-types": ["mingw64"],
+    build_from = itertools.cycle(["start", "end", "middle"])
-            "matrix": {
+    for i in range(count):
-                "packages": "base-devel",
+        real_name = name if i == 0 else name + "-" + str(i + 1)
-                "build-args": "--build-types mingw64",
+        build_args = ["--build-types", build_type, "--build-from", next(build_from)]
-                "name": "mingw64",
+        if optional_deps:
-                "runner": hosted_runner
+            build_args += ["--optional-deps", optional_deps]
-            }
+        yield {
-        }, {
+            "name": real_name,
-            "build-types": ["mingw32"],
+            "packages": packages,
-            "matrix": {
+            "runner": runner,
-                "packages": "base-devel",
+            "build-args": shlex.join(build_args),
-                "build-args": "--build-types mingw32",
+            "needs": [SCHEDULE_JOB_NAME],
-                "name": "mingw32",
-                "runner": hosted_runner
-            }
-        }, {
-            "build-types": ["ucrt64"],
-            "matrix": {
-                "packages": "base-devel",
-                "build-args": "--build-types ucrt64",
-                "name": "ucrt64",
-                "runner": hosted_runner
-            }
-        }, {
-            "build-types": ["clang64"],
-            "matrix": {
-                "packages": "base-devel",
-                "build-args": "--build-types clang64",
-                "name": "clang64",
-                "runner": hosted_runner
-            }
-        }, {
-            "build-types": ["clang32"],
-            "matrix": {
-                "packages": "base-devel",
-                "build-args": "--build-types clang32",
-                "name": "clang32",
-                "runner": hosted_runner
-            }
-        }, {
-            "build-types": ["clangarm64"],
-            "matrix": {
-                "packages": "base-devel",
-                "build-args": "--build-types clangarm64",
-                "name": "clangarm64",
-                "runner": ["Windows", "ARM64", "autobuild"]
-            }
-        }, {
-            "build-types": ["msys"],
-            "matrix": {
-                "packages": "base-devel",
-                "build-args": "--build-types msys",
-                "name": "msys",
-                "runner": hosted_runner
-            }
-        }, {
-            "build-types": ["msys-src", "mingw-src"],
-            "matrix": {
-                "packages": "base-devel VCS",
-                "build-args": "--build-types msys-src,mingw-src",
-                "name": "src",
-                "runner": hosted_runner
-            }
         }
-    ]
 
-    return job_meta
+
+def generate_src_jobs(needs: List[str], optional_deps: str, count: int):
+    name = "src"
+    packages = " ".join(["base-devel", "VCS"])
+    runner = ["windows-2022"]
+    build_types = [Config.MINGW_SRC_BUILD_TYPE, Config.MSYS_SRC_BUILD_TYPE]
+    build_from = itertools.cycle(["start", "end", "middle"])
+    for i in range(count):
+        real_name = name if i == 0 else name + "-" + str(i + 1)
+        build_args = ["--build-types", ",".join(build_types), "--build-from", next(build_from)]
+        if optional_deps:
+            build_args += ["--optional-deps", optional_deps]
+        yield {
+            "name": real_name,
+            "packages": packages,
+            "runner": runner,
+            "build-args": shlex.join(build_args),
+            "needs": needs + [SCHEDULE_JOB_NAME],
+        }
+
+
+# from https://docs.python.org/3/library/itertools.html
+def roundrobin(*iterables):
+    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+    # Recipe credited to George Sakkis
+    num_active = len(iterables)
+    nexts = itertools.cycle(iter(it).__next__ for it in iterables)
+    while num_active:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            # Remove the iterator we just exhausted from the cycle.
+            num_active -= 1
+            nexts = itertools.cycle(itertools.islice(nexts, num_active))
+
+
+def create_build_plan(pkgs: List[Package], optional_deps: str) -> List[Dict[str, Any]]:
+    queued_build_types: Dict[BuildType, int] = {}
+    for pkg in pkgs:
+        for build_type in pkg.get_build_types():
+            # skip if we can't build it
+            if Config.ASSETS_REPO[build_type] != get_current_repo().full_name:
+                continue
+            if pkg.get_status(build_type) == PackageStatus.WAITING_FOR_BUILD:
+                queued_build_types[build_type] = queued_build_types.get(build_type, 0) + 1
+
+    def get_job_count(build_type: BuildType) -> int:
+        queued = queued_build_types[build_type]
+        if queued > 9:
+            count = 3
+        elif queued > 3:
+            count = 2
+        else:
+            count = 1
+        return min(Config.MAXIMUM_BUILD_TYPE_JOB_COUNT.get(build_type, count), count)
+
+    # generate the build jobs
+    job_lists = []
+    for build_type, count in queued_build_types.items():
+        if build_type_is_src(build_type):
+            continue
+        count = get_job_count(build_type)
+        job_lists.append(list(generate_jobs_for(build_type, optional_deps, count)))
+    jobs = list(roundrobin(*job_lists))[:Config.MAXIMUM_JOB_COUNT]
+
+    # generate src build jobs depending on the build jobs above
+    src_build_types = [
+        b for b in [Config.MINGW_SRC_BUILD_TYPE, Config.MSYS_SRC_BUILD_TYPE]
+        if b in queued_build_types]
+    if src_build_types:
+        src_count = min(get_job_count(b) for b in src_build_types)
+        jobs.extend(list(generate_src_jobs([job["name"] for job in jobs], optional_deps, src_count)))
+
+    return jobs
 
 
 def write_build_plan(args: Any) -> None:
@@ -102,7 +114,7 @@ def write_build_plan(args: Any) -> None:
 
     apply_optional_deps(optional_deps)
 
-    def write_out(result: List[Dict[str, str]]) -> None:
+    def write_out(result: List[Dict[str, Any]]) -> None:
         with open(target_file, "wb") as h:
             h.write(json.dumps(result).encode())
 
@@ -110,59 +122,11 @@ def write_build_plan(args: Any) -> None:
 
     pkgs = get_buildqueue_with_status(full_details=True)
 
-    show_cycles(pkgs)
-
     update_status(pkgs)
 
-    queued_build_types: Dict[BuildType, int] = {}
+    jobs = create_build_plan(pkgs, optional_deps)
-    for pkg in pkgs:
-        for build_type in pkg.get_build_types():
-            if pkg.get_status(build_type) == PackageStatus.WAITING_FOR_BUILD:
-                queued_build_types[build_type] = queued_build_types.get(build_type, 0) + 1
 
-    if not queued_build_types:
+    write_out(jobs)
-        write_out([])
-        return
-
-    available_build_types = set()
-    for build_type, repo_name in Config.ASSETS_REPO.items():
-        if repo_name == get_current_repo().full_name:
-            available_build_types.add(build_type)
-
-    jobs = []
-    for job_info in get_job_meta():
-        matching_build_types = \
-            set(queued_build_types) & set(job_info["build-types"]) & available_build_types
-
-        # limit to the build type included with the lowest limit
-        job_limit = Config.MAXIMUM_JOB_COUNT
-        for build_type in matching_build_types:
-            type_limit = Config.MAXIMUM_BUILD_TYPE_JOB_COUNT.get(build_type, Config.MAXIMUM_JOB_COUNT)
-            if type_limit < job_limit:
-                job_limit = type_limit
-
-        if matching_build_types:
-            build_count = sum(queued_build_types[bt] for bt in matching_build_types)
-            job = job_info["matrix"]
-            # TODO: pin optional deps to their exact version somehow, in case something changes
-            # between this run and when the worker gets to it.
-            if optional_deps:
-                job["build-args"] = job["build-args"] + " --optional-deps " + shlex.quote(optional_deps)
-            jobs.append(job)
-            # XXX: If there is more than three builds we start two jobs with the second
-            # one having a reversed build order
-            if build_count > 3 and job_limit >= 2:
-                matrix = dict(job)
-                matrix["build-args"] = matrix["build-args"] + " --build-from end"
-                matrix["name"] = matrix["name"] + "-2"
-                jobs.append(matrix)
-            if build_count > 9 and job_limit >= 3:
-                matrix = dict(job)
-                matrix["build-args"] = matrix["build-args"] + " --build-from middle"
-                matrix["name"] = matrix["name"] + "-3"
-                jobs.append(matrix)
-
-    write_out(jobs[:Config.MAXIMUM_JOB_COUNT])
 
 
 def add_parser(subparsers) -> None: