e2042058f1
We were caching based on the build type, but for most build types the repo is the same, so cache one level below instead.
297 lines
11 KiB
Python
297 lines
11 KiB
Python
import io
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from functools import lru_cache
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Generator, List, Optional, Callable
|
|
|
|
import requests
|
|
from github import Github
|
|
from github.Auth import Auth, Token
|
|
from github.GithubException import GithubException, UnknownObjectException
|
|
from github.GithubObject import GithubObject
|
|
from github.GitRelease import GitRelease
|
|
from github.GitReleaseAsset import GitReleaseAsset
|
|
from github.Repository import Repository
|
|
|
|
from .config import REQUESTS_TIMEOUT, BuildType, Config
|
|
from .utils import PathLike, get_requests_session
|
|
|
|
|
|
def get_auth(write: bool = False) -> Optional[Auth]:
|
|
if not write and os.environ.get("GITHUB_TOKEN_READONLY", ""):
|
|
return Token(os.environ["GITHUB_TOKEN_READONLY"])
|
|
elif "GITHUB_TOKEN" in os.environ:
|
|
return Token(os.environ["GITHUB_TOKEN"])
|
|
else:
|
|
if not write:
|
|
print("[Warning] 'GITHUB_TOKEN' or 'GITHUB_TOKEN_READONLY' env vars "
|
|
"not set which might lead to API rate limiting", file=sys.stderr)
|
|
return None
|
|
else:
|
|
raise Exception("'GITHUB_TOKEN' env var not set")
|
|
|
|
|
|
@contextmanager
|
|
def make_writable(obj: GithubObject) -> Generator:
|
|
# XXX: This switches the read-only token with a potentially writable one
|
|
old_requester = obj._requester # type: ignore
|
|
repo = get_current_repo(write=True)
|
|
try:
|
|
obj._requester = repo._requester # type: ignore
|
|
yield
|
|
finally:
|
|
obj._requester = old_requester # type: ignore
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def _get_repo(name: str, write: bool = False) -> Repository:
|
|
gh = get_github(write=write)
|
|
return gh.get_repo(name, lazy=True)
|
|
|
|
|
|
def get_current_repo(write: bool = False) -> Repository:
|
|
repo_full_name = os.environ.get("GITHUB_REPOSITORY", "msys2/msys2-autobuild")
|
|
return _get_repo(repo_full_name, write)
|
|
|
|
|
|
def get_repo_for_build_type(build_type: BuildType, write: bool = False) -> Repository:
|
|
return _get_repo(Config.ASSETS_REPO[build_type], write)
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def get_github(write: bool = False) -> Github:
|
|
auth = get_auth(write=write)
|
|
kwargs: Dict[str, Any] = {}
|
|
kwargs['auth'] = auth
|
|
# 100 is the maximum allowed
|
|
kwargs['per_page'] = 100
|
|
kwargs['timeout'] = sum(REQUESTS_TIMEOUT)
|
|
kwargs['seconds_between_requests'] = None
|
|
kwargs['lazy'] = True
|
|
gh = Github(**kwargs)
|
|
if auth is None and not write:
|
|
print(f"[Warning] Rate limit status: {gh.get_rate_limit().core}", file=sys.stderr)
|
|
return gh
|
|
|
|
|
|
def asset_is_complete(asset: GitReleaseAsset) -> bool:
|
|
# assets can stay around in a weird incomplete state
|
|
# in which case asset.state == "starter". GitHub shows
|
|
# them with a red warning sign in the edit UI.
|
|
return asset.state == "uploaded"
|
|
|
|
|
|
def download_text_asset(asset: GitReleaseAsset, cache=False) -> str:
|
|
assert asset_is_complete(asset)
|
|
session = get_requests_session(nocache=not cache)
|
|
with session.get(asset.browser_download_url, timeout=REQUESTS_TIMEOUT) as r:
|
|
r.raise_for_status()
|
|
return r.text
|
|
|
|
|
|
def get_current_run_urls() -> Optional[Dict[str, str]]:
|
|
# The only connection we have is the job name, so this depends
|
|
# on unique job names in all workflows
|
|
if "GITHUB_SHA" in os.environ and "GITHUB_RUN_NAME" in os.environ:
|
|
sha = os.environ["GITHUB_SHA"]
|
|
run_name = os.environ["GITHUB_RUN_NAME"]
|
|
commit = get_current_repo().get_commit(sha)
|
|
check_runs = commit.get_check_runs(
|
|
check_name=run_name, status="in_progress", filter="latest")
|
|
for run in check_runs:
|
|
html = run.html_url + "?check_suite_focus=true"
|
|
raw = commit.html_url + "/checks/" + str(run.id) + "/logs"
|
|
return {"html": html, "raw": raw}
|
|
else:
|
|
raise Exception(f"No active job found for {run_name}")
|
|
return None
|
|
|
|
|
|
def wait_for_api_limit_reset(
|
|
min_remaining_write: int = 50, min_remaining: int = 250, min_sleep: float = 60,
|
|
max_sleep: float = 300) -> None:
|
|
|
|
for write in [False, True]:
|
|
gh = get_github(write=write)
|
|
while True:
|
|
core = gh.get_rate_limit().core
|
|
reset = core.reset
|
|
now = datetime.now(timezone.utc)
|
|
diff = (reset - now).total_seconds()
|
|
print(f"{core.remaining} API calls left (write={write}), "
|
|
f"{diff} seconds until the next reset")
|
|
if core.remaining > (min_remaining_write if write else min_remaining):
|
|
break
|
|
wait = diff
|
|
if wait < min_sleep:
|
|
wait = min_sleep
|
|
elif wait > max_sleep:
|
|
wait = max_sleep
|
|
print(f"Too few API calls left, waiting for {wait} seconds")
|
|
time.sleep(wait)
|
|
|
|
|
|
def get_asset_mtime_ns(asset: GitReleaseAsset) -> int:
|
|
"""Returns the mtime of an asset in nanoseconds"""
|
|
|
|
return int(asset.updated_at.timestamp() * (1000 ** 3))
|
|
|
|
|
|
def download_asset(asset: GitReleaseAsset, target_path: str,
|
|
onverify: Callable[[str, str], None] | None = None) -> None:
|
|
assert asset_is_complete(asset)
|
|
session = get_requests_session(nocache=True)
|
|
with session.get(asset.browser_download_url, stream=True, timeout=REQUESTS_TIMEOUT) as r:
|
|
r.raise_for_status()
|
|
fd, temppath = tempfile.mkstemp()
|
|
try:
|
|
os.chmod(temppath, 0o644)
|
|
with os.fdopen(fd, "wb") as h:
|
|
for chunk in r.iter_content(4096):
|
|
h.write(chunk)
|
|
mtime_ns = get_asset_mtime_ns(asset)
|
|
os.utime(temppath, ns=(mtime_ns, mtime_ns))
|
|
if onverify is not None:
|
|
onverify(temppath, target_path)
|
|
shutil.move(temppath, target_path)
|
|
finally:
|
|
try:
|
|
os.remove(temppath)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def get_gh_asset_name(basename: PathLike, text: bool = False) -> str:
|
|
# GitHub will throw out charaters like '~' or '='. It also doesn't like
|
|
# when there is no file extension and will try to add one
|
|
return sha256(str(basename).encode("utf-8")).hexdigest() + (".bin" if not text else ".txt")
|
|
|
|
|
|
def get_asset_filename(asset: GitReleaseAsset) -> str:
|
|
if not asset.label:
|
|
return asset.name
|
|
else:
|
|
assert os.path.splitext(get_gh_asset_name(asset.label))[0] == \
|
|
os.path.splitext(asset.name)[0]
|
|
return asset.label
|
|
|
|
|
|
def is_asset_from_gha(asset: GitReleaseAsset) -> bool:
|
|
"""If the asset was uploaded from CI via GHA"""
|
|
|
|
uploader = asset.uploader
|
|
return uploader.type == "Bot" and uploader.login == "github-actions[bot]"
|
|
|
|
|
|
def is_asset_from_allowed_user(asset: GitReleaseAsset) -> bool:
|
|
"""If the asset was uploaded by an allowed user"""
|
|
|
|
uploader = asset.uploader
|
|
return uploader.type == "User" and uploader.login in Config.ALLOWED_UPLOADERS
|
|
|
|
|
|
def get_asset_uploader_name(asset: GitReleaseAsset) -> str:
|
|
"""Returns the name of the user that uploaded the asset"""
|
|
|
|
uploader = asset.uploader
|
|
return uploader.login
|
|
|
|
|
|
def get_release_assets(release: GitRelease, include_incomplete: bool = False) -> List[GitReleaseAsset]:
|
|
assets = []
|
|
for asset in release.assets:
|
|
# skip in case not fully uploaded yet (or uploading failed)
|
|
if not asset_is_complete(asset) and not include_incomplete:
|
|
continue
|
|
# We allow uploads from GHA and some special users
|
|
if not is_asset_from_gha(asset) and not is_asset_from_allowed_user(asset):
|
|
raise SystemExit(
|
|
f"ERROR: Asset '{get_asset_filename(asset)}' "
|
|
f"uploaded by {get_asset_uploader_name(asset)}'. Aborting.")
|
|
assets.append(asset)
|
|
return assets
|
|
|
|
|
|
def upload_asset(release: GitRelease, path: PathLike, replace: bool = False,
|
|
text: bool = False, content: Optional[bytes] = None) -> None:
|
|
path = Path(path)
|
|
basename = os.path.basename(str(path))
|
|
asset_name = get_gh_asset_name(basename, text)
|
|
asset_label = basename
|
|
|
|
def can_try_upload_again() -> bool:
|
|
for asset in get_release_assets(release, include_incomplete=True):
|
|
if asset_name == asset.name:
|
|
# We want to treat incomplete assets as if they weren't there
|
|
# so replace them always
|
|
if replace or not asset_is_complete(asset):
|
|
with make_writable(asset):
|
|
asset.delete_asset()
|
|
break
|
|
else:
|
|
print(f"Skipping upload for {asset_name} as {asset_label}, already exists")
|
|
return False
|
|
return True
|
|
|
|
def upload() -> None:
|
|
with make_writable(release):
|
|
if content is None:
|
|
with open(path, "rb") as fileobj:
|
|
release.upload_asset_from_memory( # type: ignore
|
|
fileobj, os.path.getsize(path), label=asset_label, name=asset_name)
|
|
else:
|
|
with io.BytesIO(content) as fileobj:
|
|
release.upload_asset_from_memory( # type: ignore
|
|
fileobj, len(content), label=asset_label, name=asset_name)
|
|
|
|
try:
|
|
upload()
|
|
except (GithubException, requests.RequestException):
|
|
if can_try_upload_again():
|
|
upload()
|
|
|
|
print(f"Uploaded {asset_name} as {asset_label}")
|
|
|
|
|
|
def get_release(repo: Repository, name: str, create: bool = True) -> GitRelease:
|
|
"""Like Repository.get_release() but creates the referenced release if needed"""
|
|
|
|
try:
|
|
return repo.get_release(name)
|
|
except UnknownObjectException:
|
|
if not create:
|
|
raise
|
|
with make_writable(repo):
|
|
return repo.create_git_release(name, name, name, prerelease=True)
|
|
|
|
|
|
class CachedAssets:
|
|
|
|
def __init__(self) -> None:
|
|
self._assets: Dict[BuildType, List[GitReleaseAsset]] = {}
|
|
self._failed: Dict[str, List[GitReleaseAsset]] = {}
|
|
|
|
def get_assets(self, build_type: BuildType) -> List[GitReleaseAsset]:
|
|
if build_type not in self._assets:
|
|
repo = get_repo_for_build_type(build_type)
|
|
release = get_release(repo, 'staging-' + build_type)
|
|
self._assets[build_type] = get_release_assets(release)
|
|
return self._assets[build_type]
|
|
|
|
def get_failed_assets(self, build_type: BuildType) -> List[GitReleaseAsset]:
|
|
repo = get_repo_for_build_type(build_type)
|
|
key = repo.full_name
|
|
if key not in self._failed:
|
|
release = get_release(repo, 'staging-failed')
|
|
self._failed[key] = get_release_assets(release)
|
|
assets = self._failed[key]
|
|
# XXX: This depends on the format of the filename
|
|
return [a for a in assets if get_asset_filename(a).startswith(build_type + "-")]
|