CI: merge with MINGW-packages
This commit is contained in:
Jeremy Drake
parent
77d475caca
commit
85fdf26be1
@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -eo pipefail
|
||||
|
||||
# AppVeyor and Drone Continuous Integration for MSYS2
|
||||
# Author: Renato Silva <br.renatosilva@gmail.com>
|
||||
@ -25,38 +25,87 @@ list_commits || failure 'Could not detect added commits'
|
||||
list_packages || failure 'Could not detect changed files'
|
||||
message 'Processing changes' "${commits[@]}"
|
||||
test -z "${packages}" && success 'No changes in package recipes'
|
||||
define_build_order || failure 'Could not determine build order'
|
||||
|
||||
# Build
|
||||
message 'Building packages' "${packages[@]}"
|
||||
execute 'Approving recipe quality' check_recipe_quality
|
||||
|
||||
message 'Adding an empty local repository'
|
||||
repo-add $PWD/artifacts/ci.db.tar.gz
|
||||
sed -i '1s|^|[ci]\nServer = file://'"$PWD"'/artifacts/\nSigLevel = Never\n|' /etc/pacman.conf
|
||||
pacman -Sy
|
||||
|
||||
message 'Building packages'
|
||||
for package in "${packages[@]}"; do
|
||||
echo "::group::[build] ${package}"
|
||||
execute 'Fetch keys' "$DIR/fetch-validpgpkeys.sh"
|
||||
# Ensure the toolchain is installed before building the package
|
||||
execute 'Installing the toolchain' pacman -S --needed --noconfirm --noprogressbar base-devel msys2-devel
|
||||
execute 'Building binary' makepkg --noconfirm --noprogressbar --nocheck --syncdeps --rmdeps --cleanbuild
|
||||
execute 'Building source' makepkg --noconfirm --noprogressbar --allsource
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[install] ${package}"
|
||||
grep -qFx "${package}" "$(dirname "$0")/ci-dont-install-list.txt" || execute 'Installing' yes:pacman --noprogressbar --upgrade '*.pkg.tar.*'
|
||||
echo "::endgroup::"
|
||||
if [ -f $package/.ci-sequential ]; then
|
||||
cd "$package"
|
||||
for pkg in *.pkg.tar.*; do
|
||||
pkgname="$(echo "$pkg" | rev | cut -d- -f4- | rev)"
|
||||
echo "::group::[install] ${pkgname}"
|
||||
grep -qFx "${package}" "$(dirname "$0")/ci-dont-install-list.txt" || pacman --noprogressbar --upgrade --noconfirm $pkg
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[diff] ${package}"
|
||||
cd "$package"
|
||||
for pkg in *.pkg.tar.*; do
|
||||
message "File listing diff for ${pkg}"
|
||||
pkgname="$(echo "$pkg" | rev | cut -d- -f4- | rev)"
|
||||
diff -Nur <(pacman -Fl "$pkgname" | sed -e 's|^[^ ]* |/|' | sort) <(pacman -Ql "$pkgname" | sed -e 's|^[^/]*||' | sort) || true
|
||||
done
|
||||
cd - > /dev/null
|
||||
echo "::endgroup::"
|
||||
echo "::group::[diff] ${pkgname}"
|
||||
message "Package info diff for ${pkgname}"
|
||||
diff -Nur <(pacman -Si "${pkgname}") <(pacman -Qip "${pkg}") || true
|
||||
|
||||
echo "::group::[dll check] ${package}"
|
||||
execute 'Checking dll depencencies' list_dll_deps ./pkg
|
||||
execute 'Checking dll bases' list_dll_bases ./pkg
|
||||
echo "::endgroup::"
|
||||
message "File listing diff for ${pkgname}"
|
||||
diff -Nur <(pacman -Fl "$pkgname" | sed -e 's|^[^ ]* |/|' | sort) <(pacman -Ql "$pkgname" | sed -e 's|^[^/]*||' | sort) || true
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[uninstall] ${pkgname}"
|
||||
message "Uninstalling $pkgname"
|
||||
repo-add $PWD/../artifacts/ci.db.tar.gz $PWD/$pkg
|
||||
pacman -Sy
|
||||
pacman -R --recursive --unneeded --noconfirm --noprogressbar "$pkgname"
|
||||
echo "::endgroup::"
|
||||
done
|
||||
cd - > /dev/null
|
||||
else
|
||||
echo "::group::[install] ${package}"
|
||||
grep -qFx "${package}" "$(dirname "$0")/ci-dont-install-list.txt" || execute 'Installing' install_packages
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[diff] ${package}"
|
||||
cd "$package"
|
||||
for pkg in *.pkg.tar.*; do
|
||||
pkgname="$(echo "$pkg" | rev | cut -d- -f4- | rev)"
|
||||
message "Package info diff for ${pkgname}"
|
||||
diff -Nur <(pacman -Si "${pkgname}") <(pacman -Qip "${pkg}") || true
|
||||
|
||||
message "File listing diff for ${pkgname}"
|
||||
diff -Nur <(pacman -Fl "$pkgname" | sed -e 's|^[^ ]* |/|' | sort) <(pacman -Ql "$pkgname" | sed -e 's|^[^/]*||' | sort) || true
|
||||
done
|
||||
cd - > /dev/null
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[dll check] ${package}"
|
||||
execute 'Checking dll depencencies' list_dll_deps ./pkg
|
||||
execute 'Checking dll bases' list_dll_bases ./pkg
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::[uninstall] ${package}"
|
||||
repo-add $PWD/artifacts/ci.db.tar.gz "${package}"/*.pkg.tar.*
|
||||
pacman -Sy
|
||||
message "Uninstalling $package"
|
||||
cd "$package"
|
||||
export installed_packages=()
|
||||
for pkg in *.pkg.tar.*; do
|
||||
installed_packages+=("$(echo "$pkg" | rev | cut -d- -f4- | rev)")
|
||||
done
|
||||
pacman -R --recursive --unneeded --noconfirm --noprogressbar "${installed_packages[@]}"
|
||||
unset installed_packages
|
||||
cd - > /dev/null
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
|
||||
mv "${package}"/*.pkg.tar.* artifacts
|
||||
mv "${package}"/*.src.tar.* artifacts
|
||||
|
||||
@ -28,6 +28,7 @@ import shutil
|
||||
from collections import OrderedDict
|
||||
import hashlib
|
||||
import time
|
||||
import shlex
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
@ -89,6 +90,21 @@ def get_cache_key(pkgbuild_path: str) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def get_mingw_arch_list(pkgbuild_path: str) -> List[str]:
|
||||
bash = shutil.which("bash")
|
||||
assert bash is not None
|
||||
sub_commands = [
|
||||
shlex.join(['source', pkgbuild_path]),
|
||||
'echo -n "${mingw_arch[@]}"'
|
||||
]
|
||||
out = subprocess.check_output([bash, '-c', ';'.join(sub_commands)], universal_newlines=True)
|
||||
arch_list = out.strip().split()
|
||||
if not arch_list:
|
||||
arch_list = ["mingw32", "mingw64", "ucrt64", "clang64"]
|
||||
assert arch_list
|
||||
return arch_list
|
||||
|
||||
|
||||
def get_srcinfo_for_pkgbuild(args: Tuple[str, str]) -> Optional[CacheTuple]:
|
||||
pkgbuild_path, mode = args
|
||||
pkgbuild_path = os.path.abspath(pkgbuild_path)
|
||||
@ -106,9 +122,9 @@ def get_srcinfo_for_pkgbuild(args: Tuple[str, str]) -> Optional[CacheTuple]:
|
||||
srcinfos = {}
|
||||
|
||||
if mode == "mingw":
|
||||
for name in ["mingw32", "mingw64"]:
|
||||
for name in get_mingw_arch_list(pkgbuild_path):
|
||||
env = os.environ.copy()
|
||||
env["MINGW_INSTALLS"] = name
|
||||
env["MINGW_ARCH"] = name
|
||||
srcinfos[name] = subprocess.check_output(
|
||||
[bash, "/usr/bin/makepkg-mingw",
|
||||
"--printsrcinfo", "-p", git_path],
|
||||
|
||||
92
.ci/ci-get-build-order.py
Executable file
92
.ci/ci-get-build-order.py
Executable file
@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackageInfo:
|
||||
directory: str
|
||||
deps: list
|
||||
processed: bool
|
||||
|
||||
|
||||
def run(*args, **kwargs):
|
||||
return subprocess.check_output(args, **kwargs).decode("utf-8").strip()
|
||||
|
||||
|
||||
def get_toplevel():
|
||||
path = run("git", "rev-parse", "--show-toplevel")
|
||||
return run("cygpath", "-m", path)
|
||||
|
||||
|
||||
def list_changes(*git_args):
|
||||
out = run("git", "log", *git_args, "upstream/master..").splitlines()
|
||||
out += run("git", "log", *git_args, "HEAD^..").splitlines()
|
||||
return list(dict.fromkeys(x.split("::")[-1] for x in sorted(out)))
|
||||
|
||||
|
||||
def list_packages():
|
||||
changes = list_changes("--pretty=format:", "--name-only")
|
||||
return [
|
||||
x.split("/")[0]
|
||||
for x in changes
|
||||
if x.endswith("/PKGBUILD") and os.path.exists(x)
|
||||
]
|
||||
|
||||
|
||||
def get_pkginfo(package, packageset):
|
||||
props = ["depends", "makedepends", "pkgname", "provides"]
|
||||
script = f'source "{package}/PKGBUILD"\n'
|
||||
for prop in props:
|
||||
script += f"echo \"${{{prop}[@]}}\" && printf '\\0'\n"
|
||||
|
||||
shell = os.environ.get("SHELL", "bash")
|
||||
env = {**os.environ, "MINGW_PACKAGE_PREFIX": "mingw-w64"}
|
||||
results = run(shell, "-c", script, env=env).split("\0")[:-1]
|
||||
assert len(props) == len(results), "Length of props matches results"
|
||||
|
||||
info = {}
|
||||
extra_re = re.compile(r"[<>=:]") # Remove version/description
|
||||
for prop, res in zip(props, results):
|
||||
res = [extra_re.split(x, 1)[0] for x in res.strip().split() if x]
|
||||
info[prop] = res
|
||||
|
||||
deps = sorted(set((*info["depends"], *info["makedepends"])))
|
||||
pkg = PackageInfo(package, deps, False)
|
||||
for alias in (package, *info["pkgname"], *info["provides"]):
|
||||
packageset[alias] = pkg
|
||||
return pkg
|
||||
|
||||
|
||||
def get_build_order(packages, toadd=None, ordered=None):
|
||||
if toadd is None:
|
||||
toadd, ordered = {}, []
|
||||
ThreadPool(os.cpu_count()).map(lambda x: get_pkginfo(x, toadd), packages)
|
||||
|
||||
for package in packages:
|
||||
if package not in toadd:
|
||||
continue
|
||||
|
||||
pkg = toadd[package]
|
||||
if pkg.directory in ordered:
|
||||
continue
|
||||
elif pkg.processed:
|
||||
print("warning: dependency cycle detected on", package, file=sys.stderr)
|
||||
continue
|
||||
pkg.processed = True
|
||||
|
||||
get_build_order(pkg.deps, toadd, ordered)
|
||||
ordered.append(pkg.directory)
|
||||
return ordered
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(get_toplevel())
|
||||
packages = "\n".join(get_build_order(list_packages()))
|
||||
sys.stdout.buffer.write(packages.encode("utf-8")) # Prevent CRLF newlines
|
||||
@ -50,47 +50,6 @@ _list_changes() {
|
||||
_as_list "${list_name}" "${filter}" "${strip}" "$(git log "${git_options[@]}" HEAD^.. | sort -u)"
|
||||
}
|
||||
|
||||
# Get package information
|
||||
_package_info() {
|
||||
local package="${1}"
|
||||
local properties=("${@:2}")
|
||||
for property in "${properties[@]}"; do
|
||||
local -n nameref_property="${property}"
|
||||
nameref_property=($(
|
||||
source "${package}/PKGBUILD"
|
||||
declare -n nameref_property="${property}"
|
||||
echo "${nameref_property[@]}"))
|
||||
done
|
||||
}
|
||||
|
||||
# Package provides another
|
||||
_package_provides() {
|
||||
local package="${1}"
|
||||
local another="${2}"
|
||||
local pkgname provides
|
||||
_package_info "${package}" pkgname provides
|
||||
for pkg_name in "${pkgname[@]}"; do [[ "${pkg_name}" = "${another}" ]] && return 0; done
|
||||
for provided in "${provides[@]}"; do [[ "${provided}" = "${another}" ]] && return 0; done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Add package to build after required dependencies
|
||||
_build_add() {
|
||||
local package="${1}"
|
||||
local depends makedepends
|
||||
for started_package in "${started_packages[@]}"; do
|
||||
[[ "${started_package}" = "${package}" ]] && return 0
|
||||
done
|
||||
started_packages+=("${package}")
|
||||
_package_info "${package}" depends makedepends
|
||||
for dependency in "${depends[@]}" "${makedepends[@]}"; do
|
||||
for unsorted_package in "${packages[@]}"; do
|
||||
_package_provides "${unsorted_package}" "${dependency}" && _build_add "${unsorted_package}"
|
||||
done
|
||||
done
|
||||
sorted_packages+=("${package}")
|
||||
}
|
||||
|
||||
# Git configuration
|
||||
git_config() {
|
||||
local name="${1}"
|
||||
@ -114,29 +73,19 @@ execute(){
|
||||
cd - > /dev/null
|
||||
}
|
||||
|
||||
# Sort packages by dependency
|
||||
define_build_order() {
|
||||
local sorted_packages=()
|
||||
for unsorted_package in "${packages[@]}"; do
|
||||
_build_add "${unsorted_package}"
|
||||
done
|
||||
packages=("${sorted_packages[@]}")
|
||||
}
|
||||
|
||||
# Added commits
|
||||
list_commits() {
|
||||
_list_changes commits '*' '#*::' --pretty=format:'%ai::[%h] %s'
|
||||
}
|
||||
|
||||
# Changed recipes
|
||||
# Get changed packages in correct build order
|
||||
list_packages() {
|
||||
local _packages
|
||||
_list_changes _packages '*/PKGBUILD' '%/PKGBUILD' --pretty=format: --name-only || return 1
|
||||
for _package in "${_packages[@]}"; do
|
||||
local find_case_sensitive="$(find -name "${_package}" -type d -print -quit)"
|
||||
test -n "${find_case_sensitive}" && packages+=("${_package}")
|
||||
done
|
||||
return 0
|
||||
# readarray doesn't work with a plain pipe
|
||||
readarray -t packages < <("$DIR/ci-get-build-order.py")
|
||||
}
|
||||
|
||||
install_packages() {
|
||||
pacman --noprogressbar --upgrade --noconfirm *.pkg.tar.*
|
||||
}
|
||||
|
||||
# Recipe quality
|
||||
@ -149,7 +98,7 @@ check_recipe_quality() {
|
||||
saneman --format='\t%l:%c %p:%c %m' --verbose --no-terminal "${packages[@]}"
|
||||
}
|
||||
|
||||
# List DDL dependencies
|
||||
# List DLL dependencies
|
||||
list_dll_deps(){
|
||||
local target="${1}"
|
||||
echo "$(tput setaf 2)MSYS2 DLL dependencies:$(tput sgr0)"
|
||||
|
||||
13
.github/workflows/main.yml
vendored
13
.github/workflows/main.yml
vendored
@ -1,12 +1,16 @@
|
||||
name: main
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.repository != 'msys2/MSYS2-packages' || github.event_name != 'push' || github.ref != 'refs/heads/master'}}
|
||||
if: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master'}}
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
@ -15,10 +19,15 @@ jobs:
|
||||
path: temp
|
||||
fetch-depth: 0
|
||||
|
||||
# to match the autobuild environment
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MSYS
|
||||
install: git base-devel msys2-devel pactoys-git
|
||||
install: VCS base-devel msys2-devel pactoys-git
|
||||
update: true
|
||||
|
||||
- name: Add staging repo
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user