342 lines
13 KiB
Python
342 lines
13 KiB
Python
import fnmatch
|
|
import json
|
|
import os
|
|
import time
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from pathlib import Path, PurePath, PurePosixPath
|
|
from subprocess import check_call
|
|
from typing import Any, Dict, Generator, List, Sequence, TypeVar, Tuple
|
|
|
|
from github.GitReleaseAsset import GitReleaseAsset
|
|
|
|
from .config import ArchType, BuildType, Config
|
|
from .gh import (CachedAssets, download_asset, get_asset_filename,
|
|
get_current_run_urls, get_release, get_repo, upload_asset,
|
|
wait_for_api_limit_reset)
|
|
from .queue import Package
|
|
from .utils import SCRIPT_DIR, PathLike
|
|
|
|
|
|
class BuildError(Exception):
|
|
pass
|
|
|
|
|
|
def get_python_path(msys2_root: PathLike, msys2_path: PathLike) -> Path:
|
|
return Path(os.path.normpath(str(msys2_root) + str(msys2_path)))
|
|
|
|
|
|
def to_pure_posix_path(path: PathLike) -> PurePath:
|
|
return PurePosixPath("/" + str(path).replace(":", "", 1).replace("\\", "/"))
|
|
|
|
|
|
def get_build_environ(build_type: BuildType) -> Dict[str, str]:
|
|
environ = os.environ.copy()
|
|
|
|
# Set PACKAGER for makepkg
|
|
packager_ref = Config.ASSETS_REPO[build_type]
|
|
if "GITHUB_SHA" in environ and "GITHUB_RUN_ID" in environ:
|
|
packager_ref += "/" + environ["GITHUB_SHA"][:8] + "/" + environ["GITHUB_RUN_ID"]
|
|
environ["PACKAGER"] = f"CI ({packager_ref})"
|
|
|
|
return environ
|
|
|
|
|
|
@contextmanager
|
|
def temp_pacman_script(pacman_config: PathLike) -> Generator[PathLike, None, None]:
|
|
"""Gives a temporary pacman script which uses the passed in pacman config
|
|
without having to pass --config to it. Required because makepkg doesn't allow
|
|
setting the pacman conf path, but it allows setting the pacman executable path
|
|
via the 'PACMAN' env var.
|
|
"""
|
|
|
|
fd, filename = tempfile.mkstemp("pacman")
|
|
os.close(fd)
|
|
|
|
try:
|
|
with open(filename, "w", encoding="utf-8") as h:
|
|
cli = shlex.join(['/usr/bin/pacman', '--config', str(to_pure_posix_path(pacman_config))])
|
|
h.write(f"""\
|
|
#!/bin/bash
|
|
set -e
|
|
exec {cli} "$@"
|
|
""")
|
|
yield filename
|
|
finally:
|
|
try:
|
|
os.unlink(filename)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
@contextmanager
|
|
def temp_pacman_conf(msys2_root: PathLike) -> Generator[Path, None, None]:
|
|
"""Gives a unix path to a temporary copy of pacman.conf"""
|
|
|
|
fd, filename = tempfile.mkstemp("pacman.conf")
|
|
os.close(fd)
|
|
try:
|
|
conf = get_python_path(msys2_root, "/etc/pacman.conf")
|
|
with open(conf, "rb") as src:
|
|
with open(filename, "wb") as dest:
|
|
shutil.copyfileobj(src, dest)
|
|
|
|
yield Path(filename)
|
|
finally:
|
|
try:
|
|
os.unlink(filename)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def clean_environ(environ: Dict[str, str]) -> Dict[str, str]:
|
|
"""Returns an environment without any CI related variables.
|
|
|
|
This is to avoid leaking secrets to package build scripts we call.
|
|
While in theory we trust them this can't hurt.
|
|
"""
|
|
|
|
new_env = environ.copy()
|
|
for key in list(new_env):
|
|
if key.startswith(("GITHUB_", "RUNNER_")):
|
|
del new_env[key]
|
|
return new_env
|
|
|
|
|
|
def run_cmd(msys2_root: PathLike, args: Sequence[PathLike], **kwargs: Any) -> None:
|
|
executable = os.path.join(msys2_root, 'usr', 'bin', 'bash.exe')
|
|
env = clean_environ(kwargs.pop("env", os.environ.copy()))
|
|
env["CHERE_INVOKING"] = "1"
|
|
env["MSYSTEM"] = "MSYS"
|
|
env["MSYS2_PATH_TYPE"] = "minimal"
|
|
|
|
def shlex_join(split_command: Sequence[str]) -> str:
|
|
# shlex.join got added in 3.8 while we support 3.6
|
|
return ' '.join(shlex.quote(arg) for arg in split_command)
|
|
|
|
check_call([executable, '-lc'] + [shlex_join([str(a) for a in args])], env=env, **kwargs)
|
|
|
|
|
|
def reset_git_repo(path: PathLike):
|
|
|
|
def clean():
|
|
assert os.path.exists(path)
|
|
check_call(["git", "clean", "-xfdf"], cwd=path)
|
|
check_call(["git", "reset", "--hard", "HEAD"], cwd=path)
|
|
|
|
for i in range(10):
|
|
try:
|
|
clean()
|
|
except subprocess.CalledProcessError:
|
|
# sometimes git clean fails right after the build
|
|
print(f"git clean/reset failed, sleeping for {i} seconds")
|
|
time.sleep(i)
|
|
else:
|
|
break
|
|
else:
|
|
# run it one more time to raise
|
|
clean()
|
|
|
|
|
|
@contextmanager
|
|
def fresh_git_repo(url: str, path: PathLike) -> Generator:
|
|
if not os.path.exists(path):
|
|
check_call(["git", "clone", url, path])
|
|
check_call(["git", "config", "core.longpaths", "true"], cwd=path)
|
|
else:
|
|
reset_git_repo(path)
|
|
check_call(["git", "fetch", "origin"], cwd=path)
|
|
check_call(["git", "reset", "--hard", "origin/master"], cwd=path)
|
|
try:
|
|
yield
|
|
finally:
|
|
assert os.path.exists(path)
|
|
reset_git_repo(path)
|
|
|
|
|
|
@contextmanager
|
|
def staging_dependencies(
|
|
build_type: BuildType, pkg: Package, msys2_root: PathLike,
|
|
builddir: PathLike) -> Generator[PathLike, None, None]:
|
|
|
|
def add_to_repo(repo_root: PathLike, pacman_config: PathLike, repo_name: str,
|
|
assets: List[GitReleaseAsset]) -> None:
|
|
repo_dir = Path(repo_root) / repo_name
|
|
os.makedirs(repo_dir, exist_ok=True)
|
|
|
|
todo = []
|
|
for asset in assets:
|
|
asset_path = os.path.join(repo_dir, get_asset_filename(asset))
|
|
todo.append((asset_path, asset))
|
|
|
|
def fetch_item(item: Tuple[str, GitReleaseAsset]) -> Tuple[str, GitReleaseAsset]:
|
|
asset_path, asset = item
|
|
download_asset(asset, asset_path)
|
|
return item
|
|
|
|
package_paths = []
|
|
with ThreadPoolExecutor(8) as executor:
|
|
for i, item in enumerate(executor.map(fetch_item, todo)):
|
|
asset_path, asset = item
|
|
print(f"[{i + 1}/{len(todo)}] {get_asset_filename(asset)}")
|
|
package_paths.append(asset_path)
|
|
|
|
repo_name = f"autobuild-{repo_name}"
|
|
repo_db_path = os.path.join(repo_dir, f"{repo_name}.db.tar.gz")
|
|
|
|
with open(pacman_config, "r", encoding="utf-8") as h:
|
|
text = h.read()
|
|
uri = to_pure_posix_path(repo_dir).as_uri()
|
|
if uri not in text:
|
|
with open(pacman_config, "w", encoding="utf-8") as h2:
|
|
h2.write(f"""[{repo_name}]
|
|
Server={uri}
|
|
SigLevel=Never
|
|
""")
|
|
h2.write(text)
|
|
|
|
# repo-add 15 packages at a time so we don't hit the size limit for CLI arguments
|
|
ChunkItem = TypeVar("ChunkItem")
|
|
|
|
def chunks(lst: List[ChunkItem], n: int) -> Generator[List[ChunkItem], None, None]:
|
|
for i in range(0, len(lst), n):
|
|
yield lst[i:i + n]
|
|
|
|
base_args: List[PathLike] = ["repo-add", to_pure_posix_path(repo_db_path)]
|
|
posix_paths: List[PathLike] = [to_pure_posix_path(p) for p in package_paths]
|
|
for chunk in chunks(posix_paths, 15):
|
|
args = base_args + chunk
|
|
run_cmd(msys2_root, args, cwd=repo_dir)
|
|
|
|
cached_assets = CachedAssets()
|
|
repo_root = os.path.join(builddir, "_REPO")
|
|
try:
|
|
shutil.rmtree(repo_root, ignore_errors=True)
|
|
os.makedirs(repo_root, exist_ok=True)
|
|
with temp_pacman_conf(msys2_root) as pacman_config:
|
|
to_add: Dict[ArchType, List[GitReleaseAsset]] = {}
|
|
for dep_type, deps in pkg.get_depends(build_type).items():
|
|
assets = cached_assets.get_assets(dep_type)
|
|
for dep in deps:
|
|
for pattern in dep.get_build_patterns(dep_type):
|
|
for asset in assets:
|
|
if fnmatch.fnmatch(get_asset_filename(asset), pattern):
|
|
to_add.setdefault(dep_type, []).append(asset)
|
|
break
|
|
else:
|
|
if pkg.is_optional_dep(dep, dep_type):
|
|
# If it's there, good, if not we ignore it since it's part of a cycle
|
|
pass
|
|
else:
|
|
raise SystemExit(f"asset for {pattern} in {dep_type} not found")
|
|
|
|
for dep_type, assets in to_add.items():
|
|
add_to_repo(repo_root, pacman_config, dep_type, assets)
|
|
|
|
with temp_pacman_script(pacman_config) as temp_pacman:
|
|
# in case they are already installed we need to upgrade
|
|
run_cmd(msys2_root, [to_pure_posix_path(temp_pacman), "--noconfirm", "-Suy"])
|
|
run_cmd(msys2_root, [to_pure_posix_path(temp_pacman), "--noconfirm", "-Su"])
|
|
yield temp_pacman
|
|
finally:
|
|
shutil.rmtree(repo_root, ignore_errors=True)
|
|
# downgrade again
|
|
run_cmd(msys2_root, ["pacman", "--noconfirm", "-Suuy"])
|
|
run_cmd(msys2_root, ["pacman", "--noconfirm", "-Suu"])
|
|
|
|
|
|
def build_package(build_type: BuildType, pkg: Package, msys2_root: PathLike, builddir: PathLike) -> None:
|
|
assert os.path.isabs(builddir)
|
|
assert os.path.isabs(msys2_root)
|
|
os.makedirs(builddir, exist_ok=True)
|
|
|
|
repo_name = {"MINGW-packages": "M", "MSYS2-packages": "S"}.get(pkg['repo'], pkg['repo'])
|
|
repo_dir = os.path.join(builddir, repo_name)
|
|
to_upload: List[str] = []
|
|
|
|
repo = get_repo(build_type)
|
|
|
|
with fresh_git_repo(pkg['repo_url'], repo_dir):
|
|
orig_pkg_dir = os.path.join(repo_dir, pkg['repo_path'])
|
|
# Rename it to get a shorter overall build path
|
|
# https://github.com/msys2/msys2-autobuild/issues/71
|
|
pkg_dir = os.path.join(repo_dir, 'B')
|
|
assert not os.path.exists(pkg_dir)
|
|
os.rename(orig_pkg_dir, pkg_dir)
|
|
|
|
# Fetch all keys mentioned in the PKGBUILD
|
|
validpgpkeys = to_pure_posix_path(os.path.join(SCRIPT_DIR, 'fetch-validpgpkeys.sh'))
|
|
run_cmd(msys2_root, ['bash', validpgpkeys], cwd=pkg_dir)
|
|
|
|
with staging_dependencies(build_type, pkg, msys2_root, builddir) as temp_pacman:
|
|
try:
|
|
env = get_build_environ(build_type)
|
|
# this makes makepkg use our custom pacman script
|
|
env['PACMAN'] = str(to_pure_posix_path(temp_pacman))
|
|
if build_type == Config.MINGW_SRC_BUILD_TYPE:
|
|
env['MINGW_ARCH'] = Config.MINGW_SRC_ARCH
|
|
run_cmd(msys2_root, [
|
|
'makepkg-mingw',
|
|
'--noconfirm',
|
|
'--noprogressbar',
|
|
'--allsource'
|
|
], env=env, cwd=pkg_dir)
|
|
elif build_type == Config.MSYS_SRC_BUILD_TYPE:
|
|
run_cmd(msys2_root, [
|
|
'makepkg',
|
|
'--noconfirm',
|
|
'--noprogressbar',
|
|
'--allsource'
|
|
], env=env, cwd=pkg_dir)
|
|
elif build_type in Config.MINGW_ARCH_LIST:
|
|
env['MINGW_ARCH'] = build_type
|
|
run_cmd(msys2_root, [
|
|
'makepkg-mingw',
|
|
'--noconfirm',
|
|
'--noprogressbar',
|
|
'--nocheck',
|
|
'--syncdeps',
|
|
'--rmdeps',
|
|
'--cleanbuild'
|
|
], env=env, cwd=pkg_dir)
|
|
elif build_type in Config.MSYS_SRC_ARCH:
|
|
run_cmd(msys2_root, [
|
|
'makepkg',
|
|
'--noconfirm',
|
|
'--noprogressbar',
|
|
'--nocheck',
|
|
'--syncdeps',
|
|
'--rmdeps',
|
|
'--cleanbuild'
|
|
], env=env, cwd=pkg_dir)
|
|
else:
|
|
assert 0
|
|
|
|
entries = os.listdir(pkg_dir)
|
|
for pattern in pkg.get_build_patterns(build_type):
|
|
found = fnmatch.filter(entries, pattern)
|
|
if not found:
|
|
raise BuildError(f"{pattern} not found, likely wrong version built")
|
|
to_upload.extend([os.path.join(pkg_dir, e) for e in found])
|
|
|
|
except (subprocess.CalledProcessError, BuildError) as e:
|
|
wait_for_api_limit_reset()
|
|
release = get_release(repo, "staging-failed")
|
|
run_urls = get_current_run_urls()
|
|
failed_data = {}
|
|
if run_urls is not None:
|
|
failed_data["urls"] = run_urls
|
|
content = json.dumps(failed_data).encode()
|
|
upload_asset(release, pkg.get_failed_name(build_type), text=True, content=content)
|
|
|
|
raise BuildError(e)
|
|
else:
|
|
wait_for_api_limit_reset()
|
|
release = repo.get_release("staging-" + build_type)
|
|
for path in to_upload:
|
|
upload_asset(release, path)
|