msys2-autobuild/msys2_autobuild/build.py

import fnmatch
import json
import os
import time
import shlex
import shutil
import stat
import subprocess
import tempfile
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path, PurePath, PurePosixPath
from subprocess import check_call
from typing import Any, TypeVar
from collections.abc import Generator, Sequence

from gitea import Attachment

from .config import ArchType, BuildType, Config
from .gh import (CachedAssets, download_asset, get_asset_filename,
                 get_release, get_repo_for_build_type, upload_asset)
from .queue import Package
from .utils import SCRIPT_DIR, PathLike


class BuildError(Exception):
    pass


def get_python_path(msys2_root: PathLike, msys2_path: PathLike) -> Path:
    return Path(os.path.normpath(str(msys2_root) + str(msys2_path)))


def to_pure_posix_path(path: PathLike) -> PurePath:
    return PurePosixPath("/" + str(path).replace(":", "", 1).replace("\\", "/"))


def get_build_environ(build_type: BuildType) -> dict[str, str]:
    environ = os.environ.copy()

    # Set PACKAGER for makepkg
    packager_ref = Config.RUNNER_CONFIG[build_type]["repo"]
    if "GITHUB_SHA" in environ and "GITHUB_RUN_ID" in environ:
        packager_ref += "/" + environ["GITHUB_SHA"][:8] + "/" + environ["GITHUB_RUN_ID"]
    environ["PACKAGER"] = f"CI ({packager_ref})"

    return environ


@contextmanager
def temp_pacman_script(pacman_config: PathLike) -> Generator[PathLike, None, None]:
    """Gives a temporary pacman script which uses the passed in pacman config
    without having to pass --config to it. Required because makepkg doesn't allow
    setting the pacman conf path, but it allows setting the pacman executable path
    via the 'PACMAN' env var.
    """

    fd, filename = tempfile.mkstemp("pacman")
    os.close(fd)

    try:
        with open(filename, "w", encoding="utf-8") as h:
            cli = shlex.join(['/usr/bin/pacman', '--config', str(to_pure_posix_path(pacman_config))])
            h.write(f"""\
#!/bin/bash
set -e
exec {cli} "$@"
""")
        yield filename
    finally:
        try:
            os.unlink(filename)
        except OSError:
            pass


@contextmanager
def temp_pacman_conf(msys2_root: PathLike) -> Generator[Path, None, None]:
    """Gives a unix path to a temporary copy of pacman.conf"""

    fd, filename = tempfile.mkstemp("pacman.conf")
    os.close(fd)
    try:
        conf = get_python_path(msys2_root, "/etc/pacman.conf")
        with open(conf, "rb") as src:
            with open(filename, "wb") as dest:
                shutil.copyfileobj(src, dest)

        yield Path(filename)
    finally:
        try:
            os.unlink(filename)
        except OSError:
            pass


@contextmanager
def temp_makepkg_confd(msys2_root: PathLike, config_name: str) -> Generator[Path, None, None]:
    """Gives a path to a temporary $config_name.d file"""

    conf_dir = get_python_path(msys2_root, f"/etc/{config_name}.d")
    os.makedirs(conf_dir, exist_ok=True)
    conf_file = conf_dir / "msys2_autobuild.conf"
    try:
        open(conf_file, "wb").close()
        yield conf_file
    finally:
        try:
            os.unlink(conf_file)
        except OSError:
            pass
        try:
            os.rmdir(conf_dir)
        except OSError:
            pass


def clean_environ(environ: dict[str, str]) -> dict[str, str]:
    """Returns an environment without any CI related variables.

    This is to avoid leaking secrets to package build scripts we call.
    While in theory we trust them this can't hurt.
    """

    new_env = environ.copy()
    for key in list(new_env):
        if key.startswith(("GITHUB_", "RUNNER_")):
            del new_env[key]
    return new_env


def run_cmd(msys2_root: PathLike, args: Sequence[PathLike], **kwargs: Any) -> None:
    executable = os.path.join(msys2_root, 'usr', 'bin', 'bash.exe')
    env = clean_environ(kwargs.pop("env", os.environ.copy()))
    env["CHERE_INVOKING"] = "1"
    env["MSYSTEM"] = "MSYS"
    env["MSYS2_PATH_TYPE"] = "minimal"

    check_call([executable, '-lc'] + [shlex.join([str(a) for a in args])], env=env, **kwargs)


def make_tree_writable(topdir: PathLike) -> None:
    # Ensure all files and directories under topdir are writable
    # (and readable) by owner.
    # Taken from meson, and adjusted

    def chmod(p: PathLike) -> None:
        os.chmod(p, os.stat(p).st_mode | stat.S_IWRITE | stat.S_IREAD)

    chmod(topdir)
    for root, dirs, files in os.walk(topdir):
        for d in dirs:
            chmod(os.path.join(root, d))
        # Work around Python bug following junctions
        # https://github.com/python/cpython/issues/67596#issuecomment-1918112817
        dirs[:] = [d for d in dirs if not os.path.isjunction(os.path.join(root, d))]
        for fname in files:
            fpath = os.path.join(root, fname)
            if os.path.isfile(fpath):
                chmod(fpath)


def remove_junctions(topdir: PathLike) -> None:
    # work around a git issue where it can't handle junctions
    # https://github.com/git-for-windows/git/issues/5320
    for root, dirs, _ in os.walk(topdir):
        no_junctions = []
        for d in dirs:
            if not os.path.isjunction(os.path.join(root, d)):
                no_junctions.append(d)
            else:
                os.remove(os.path.join(root, d))
        dirs[:] = no_junctions


def reset_git_repo(path: PathLike):

    def clean():
        assert os.path.exists(path)

        # Try to avoid git hanging in a junction loop, by removing them
        # before running git clean/reset
        # https://github.com/msys2/msys2-autobuild/issues/108#issuecomment-2776420879
        try:
            remove_junctions(path)
        except OSError as e:
            print("Removing junctions failed", e)

        check_call(["git", "clean", "-xfdf"], cwd=path)
        check_call(["git", "reset", "--hard", "HEAD"], cwd=path)

    made_writable = False
    for i in range(10):
        try:
            clean()
        except subprocess.CalledProcessError:
            try:
                if not made_writable:
                    print("Trying to make files writable")
                    make_tree_writable(path)
                    remove_junctions(path)
                    made_writable = True
            except OSError as e:
                print("Making files writable failed", e)
            print(f"git clean/reset failed, sleeping for {i} seconds")
            time.sleep(i)
        else:
            break
    else:
        # run it one more time to raise
        clean()


@contextmanager
def fresh_git_repo(url: str, path: PathLike) -> Generator:
    if not os.path.exists(path):
        check_call(["git", "clone", url, path])
        check_call(["git", "config", "core.longpaths", "true"], cwd=path)
    else:
        reset_git_repo(path)
        check_call(["git", "fetch", "origin"], cwd=path)
        check_call(["git", "reset", "--hard", "origin/master"], cwd=path)
    try:
        yield
    finally:
        assert os.path.exists(path)
        reset_git_repo(path)


@contextmanager
def staging_dependencies(
        build_type: BuildType, pkg: Package, msys2_root: PathLike,
        builddir: PathLike) -> Generator[PathLike, None, None]:

    def add_to_repo(repo_root: PathLike, pacman_config: PathLike, repo_name: str,
                    assets: list[Attachment]) -> None:
        repo_dir = Path(repo_root) / repo_name
        os.makedirs(repo_dir, exist_ok=True)

        todo = []
        for asset in assets:
            asset_path = os.path.join(repo_dir, get_asset_filename(asset))
            todo.append((asset_path, asset))

        def fetch_item(item: tuple[str, Attachment]) -> tuple[str, Attachment]:
            asset_path, asset = item
            download_asset(asset, asset_path)
            return item

        package_paths = []
        with ThreadPoolExecutor(8) as executor:
            for i, item in enumerate(executor.map(fetch_item, todo)):
                asset_path, asset = item
                print(f"[{i + 1}/{len(todo)}] {get_asset_filename(asset)}")
                package_paths.append(asset_path)

        repo_name = f"autobuild-{repo_name}"
        repo_db_path = os.path.join(repo_dir, f"{repo_name}.db.tar.gz")

        with open(pacman_config, encoding="utf-8") as h:
            text = h.read()
            uri = to_pure_posix_path(repo_dir).as_uri()
            if uri not in text:
                with open(pacman_config, "w", encoding="utf-8") as h2:
                    h2.write(f"""[{repo_name}]
Server={uri}
SigLevel=Never
""")
                    h2.write(text)

        # repo-add 15 packages at a time so we don't hit the size limit for CLI arguments
        ChunkItem = TypeVar("ChunkItem")

        def chunks(lst: list[ChunkItem], n: int) -> Generator[list[ChunkItem], None, None]:
            for i in range(0, len(lst), n):
                yield lst[i:i + n]

        base_args: list[PathLike] = ["repo-add", to_pure_posix_path(repo_db_path)]
        posix_paths: list[PathLike] = [to_pure_posix_path(p) for p in package_paths]
        for chunk in chunks(posix_paths, 15):
            args = base_args + chunk
            run_cmd(msys2_root, args, cwd=repo_dir)

    cached_assets = CachedAssets()
    repo_root = os.path.join(builddir, "_REPO")
    try:
        shutil.rmtree(repo_root, ignore_errors=True)
        os.makedirs(repo_root, exist_ok=True)
        with temp_pacman_conf(msys2_root) as pacman_config:
            to_add: dict[ArchType, list[GitReleaseAsset]] = {}
            for dep_type, deps in pkg.get_depends(build_type).items():
                assets = cached_assets.get_assets(dep_type)
                for dep in deps:
                    for pattern in dep.get_build_patterns(dep_type):
                        for asset in assets:
                            if fnmatch.fnmatch(get_asset_filename(asset), pattern):
                                to_add.setdefault(dep_type, []).append(asset)
                                break
                        else:
                            if pkg.is_optional_dep(dep, dep_type):
                                # If it's there, good, if not we ignore it since it's part of a cycle
                                pass
                            else:
                                raise SystemExit(f"asset for {pattern} in {dep_type} not found")

            for dep_type, assets in to_add.items():
                add_to_repo(repo_root, pacman_config, dep_type, assets)

            with temp_pacman_script(pacman_config) as temp_pacman:
                # in case they are already installed we need to upgrade
                run_cmd(msys2_root, [to_pure_posix_path(temp_pacman), "--noconfirm", "-Suy"])
                run_cmd(msys2_root, [to_pure_posix_path(temp_pacman), "--noconfirm", "-Su"])
                yield temp_pacman
    finally:
        shutil.rmtree(repo_root, ignore_errors=True)
        # downgrade again
        run_cmd(msys2_root, ["pacman", "--noconfirm", "-Suuy"])
        run_cmd(msys2_root, ["pacman", "--noconfirm", "-Suu"])


def build_package(build_type: BuildType, pkg: Package, msys2_root: PathLike, builddir: PathLike) -> None:
    assert os.path.isabs(builddir)
    assert os.path.isabs(msys2_root)
    os.makedirs(builddir, exist_ok=True)

    repo_name = {"MINGW-packages": "W", "MSYS2-packages": "S"}.get(pkg['repo'], pkg['repo'])
    repo_dir = os.path.join(builddir, repo_name)
    to_upload: list[str] = []

    repo = get_repo_for_build_type(build_type)

    with fresh_git_repo(pkg['repo_url'], repo_dir):
        orig_pkg_dir = os.path.join(repo_dir, pkg['repo_path'])
        # Rename it to get a shorter overall build path
        # https://github.com/msys2/msys2-autobuild/issues/71
        pkg_dir = os.path.join(repo_dir, 'B')
        assert not os.path.exists(pkg_dir)
        os.rename(orig_pkg_dir, pkg_dir)

        # Fetch all keys mentioned in the PKGBUILD
        validpgpkeys = to_pure_posix_path(os.path.join(SCRIPT_DIR, 'fetch-validpgpkeys.sh'))
        run_cmd(msys2_root, ['bash', validpgpkeys], cwd=pkg_dir)

        with staging_dependencies(build_type, pkg, msys2_root, builddir) as temp_pacman:
            try:
                env = get_build_environ(build_type)
                # this makes makepkg use our custom pacman script
                env['PACMAN'] = str(to_pure_posix_path(temp_pacman))
                if build_type == Config.MINGW_SRC_BUILD_TYPE:
                    with temp_makepkg_confd(msys2_root, "makepkg_mingw.conf") as makepkg_conf:
                        with open(makepkg_conf, "w", encoding="utf-8") as h:
                            h.write("COMPRESSZST=(zstd -c -T0 --ultra -22 -)\n")

                        env['MINGW_ARCH'] = Config.MINGW_SRC_ARCH
                        run_cmd(msys2_root, [
                            'makepkg-mingw',
                            '--noconfirm',
                            '--noprogressbar',
                            '--allsource'
                        ], env=env, cwd=pkg_dir)
                elif build_type == Config.MSYS_SRC_BUILD_TYPE:
                    with temp_makepkg_confd(msys2_root, "makepkg.conf") as makepkg_conf:
                        with open(makepkg_conf, "w", encoding="utf-8") as h:
                            h.write("COMPRESSZST=(zstd -c -T0 --ultra -22 -)\n")

                        run_cmd(msys2_root, [
                            'makepkg',
                            '--noconfirm',
                            '--noprogressbar',
                            '--allsource'
                        ], env=env, cwd=pkg_dir)
                elif build_type in Config.MINGW_ARCH_LIST:
                    with temp_makepkg_confd(msys2_root, "makepkg_mingw.conf") as makepkg_conf:
                        with open(makepkg_conf, "w", encoding="utf-8") as h:
                            h.write("COMPRESSZST=(zstd -c -T0 --ultra -20 -)\n")

                        env['MINGW_ARCH'] = build_type
                        run_cmd(msys2_root, [
                            'makepkg-mingw',
                            '--noconfirm',
                            '--noprogressbar',
                            '--nocheck',
                            '--syncdeps',
                            '--rmdeps',
                            '--cleanbuild'
                        ], env=env, cwd=pkg_dir)
                elif build_type in Config.MSYS_ARCH_LIST:
                    with temp_makepkg_confd(msys2_root, "makepkg.conf") as makepkg_conf:
                        with open(makepkg_conf, "w", encoding="utf-8") as h:
                            h.write("COMPRESSZST=(zstd -c -T0 --ultra -20 -)\n")

                        run_cmd(msys2_root, [
                            'makepkg',
                            '--noconfirm',
                            '--noprogressbar',
                            '--nocheck',
                            '--syncdeps',
                            '--rmdeps',
                            '--cleanbuild'
                        ], env=env, cwd=pkg_dir)
                else:
                    assert 0

                entries = os.listdir(pkg_dir)
                for pattern in pkg.get_build_patterns(build_type):
                    found = fnmatch.filter(entries, pattern)
                    if not found:
                        raise BuildError(f"{pattern} not found, likely wrong version built")
                    to_upload.extend([os.path.join(pkg_dir, e) for e in found])

            except (subprocess.CalledProcessError, BuildError) as e:
                release = get_release(repo, "staging-failed")
                failed_data = {}
                content = json.dumps(failed_data).encode()
                upload_asset(repo, release, pkg.get_failed_name(build_type), text=True, content=content)

                raise BuildError(e)
            else:
                release = get_release(repo, "staging-" + build_type)
                for path in to_upload:
                    upload_asset(repo, release, path)