534 lines
19 KiB
Python
534 lines
19 KiB
Python
# Copyright 2016-2020 Christoph Reiter
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import os
|
|
import sys
|
|
import io
|
|
import json
|
|
import asyncio
|
|
import traceback
|
|
import hashlib
|
|
import functools
|
|
import gzip
|
|
import datetime
|
|
from asyncio import Event
|
|
from urllib.parse import urlparse, quote_plus
|
|
from typing import Any, Dict, Tuple, List, Set, Optional
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
import httpx
|
|
from aiolimiter import AsyncLimiter
|
|
import zstandard
|
|
|
|
from .appstate import state, Source, get_repositories, SrcInfoPackage, Package, DepType, Repository, BuildStatus, \
|
|
ExtInfo, ExtId
|
|
from .pkgextra import PkgExtra, extra_to_pkgextra_entry
|
|
from .appconfig import CYGWIN_METADATA_URL, REQUEST_TIMEOUT, AUR_METADATA_URL, ARCH_REPO_CONFIG, \
|
|
SRCINFO_URLS, UPDATE_INTERVAL, BUILD_STATUS_URLS, UPDATE_MIN_RATE, UPDATE_MIN_INTERVAL, PYPI_URLS
|
|
from .utils import version_is_newer_than, arch_version_to_msys, extract_upstream_version, logger
|
|
from . import appconfig
|
|
from .exttarfile import ExtTarFile
|
|
|
|
|
|
def get_mtime_for_response(response: httpx.Response) -> Optional[datetime.datetime]:
|
|
last_modified = response.headers.get("last-modified")
|
|
if last_modified is not None:
|
|
dt: datetime.datetime = parsedate_to_datetime(last_modified)
|
|
return dt.astimezone(datetime.timezone.utc)
|
|
return None
|
|
|
|
|
|
async def get_content_cached_mtime(url: str, *args: Any, **kwargs: Any) -> Tuple[bytes, Optional[datetime.datetime]]:
|
|
"""Returns the content of the URL response, and a datetime object for when the content was last modified"""
|
|
|
|
# cache the file locally, and store the "last-modified" date as the file mtime
|
|
cache_dir = appconfig.CACHE_DIR
|
|
if cache_dir is None:
|
|
async with httpx.AsyncClient(follow_redirects=True) as client:
|
|
r = await client.get(url, *args, **kwargs)
|
|
r.raise_for_status()
|
|
return (r.content, get_mtime_for_response(r))
|
|
|
|
os.makedirs(cache_dir, exist_ok=True)
|
|
|
|
cache_fn = quote_plus(
|
|
(urlparse(url).hostname or "") +
|
|
"." + hashlib.sha256(url.encode()).hexdigest()[:16] +
|
|
".cache")
|
|
|
|
fn = os.path.join(cache_dir, cache_fn)
|
|
if not os.path.exists(fn):
|
|
async with httpx.AsyncClient(follow_redirects=True) as client:
|
|
r = await client.get(url, *args, **kwargs)
|
|
r.raise_for_status()
|
|
with open(fn, "wb") as h:
|
|
h.write(r.content)
|
|
mtime = get_mtime_for_response(r)
|
|
if mtime is not None:
|
|
os.utime(fn, (mtime.timestamp(), mtime.timestamp()))
|
|
|
|
with open(fn, "rb") as h:
|
|
data = h.read()
|
|
file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(fn), datetime.timezone.utc)
|
|
return (data, file_mtime)
|
|
|
|
|
|
async def get_content_cached(url: str, *args: Any, **kwargs: Any) -> bytes:
|
|
return (await get_content_cached_mtime(url, *args, **kwargs))[0]
|
|
|
|
|
|
def parse_cygwin_versions(base_url: str, data: bytes) -> Tuple[Dict[str, ExtInfo], Dict[str, ExtInfo]]:
|
|
# This is kinda hacky: extract the source name from the src tarball and take
|
|
# last version line before it
|
|
version = None
|
|
source_package = None
|
|
versions: Dict[str, ExtInfo] = {}
|
|
versions_mingw64: Dict[str, ExtInfo] = {}
|
|
base_url = base_url.rsplit("/", 2)[0]
|
|
in_main = True
|
|
for line in data.decode("utf-8").splitlines():
|
|
if line.startswith("@"):
|
|
in_main = True
|
|
if line.startswith("version:"):
|
|
version = line.split(":")[-1].strip().split("-", 1)[0].split("+", 1)[0]
|
|
elif in_main and line.startswith("source:"):
|
|
in_main = False
|
|
source = line.split(":", 1)[-1].strip()
|
|
fn = source.rsplit(None, 2)[0]
|
|
source_package = fn.rsplit("/")[-1].rsplit("-", 3)[0]
|
|
src_url = base_url + "/" + fn
|
|
assert version is not None
|
|
src_url_name = src_url.rsplit("/")[-1]
|
|
if source_package.startswith("mingw64-x86_64-"):
|
|
info_name = source_package.split("-", 2)[-1]
|
|
if info_name in versions_mingw64:
|
|
existing_version = versions_mingw64[info_name][0]
|
|
if not version_is_newer_than(version, existing_version):
|
|
continue
|
|
versions_mingw64[info_name] = ExtInfo(
|
|
info_name, version, 0,
|
|
"https://cygwin.com/packages/summary/%s-src.html" % source_package,
|
|
{src_url: src_url_name})
|
|
else:
|
|
info_name = source_package
|
|
if info_name in versions:
|
|
existing_version = versions[info_name][0]
|
|
if not version_is_newer_than(version, existing_version):
|
|
continue
|
|
versions[info_name] = ExtInfo(
|
|
info_name, version, 0,
|
|
"https://cygwin.com/packages/summary/%s-src.html" % source_package,
|
|
{src_url: src_url_name})
|
|
return versions, versions_mingw64
|
|
|
|
|
|
async def update_cygwin_versions() -> None:
|
|
url = CYGWIN_METADATA_URL
|
|
if not await check_needs_update([url]):
|
|
return
|
|
logger.info("update cygwin info")
|
|
logger.info("Loading %r" % url)
|
|
data = await get_content_cached(url, timeout=REQUEST_TIMEOUT)
|
|
data = zstandard.ZstdDecompressor().decompress(data)
|
|
cygwin_versions, cygwin_versions_mingw64 = parse_cygwin_versions(url, data)
|
|
state.set_ext_infos(ExtId("cygwin", "Cygwin", True), cygwin_versions)
|
|
state.set_ext_infos(ExtId("cygwin-mingw64", "Cygwin-mingw64", False), cygwin_versions_mingw64)
|
|
|
|
|
|
async def update_build_status() -> None:
|
|
urls = BUILD_STATUS_URLS
|
|
if not await check_needs_update(urls):
|
|
return
|
|
|
|
logger.info("update build status")
|
|
responses = []
|
|
for url in urls:
|
|
logger.info("Loading %r" % url)
|
|
data, mtime = await get_content_cached_mtime(url, timeout=REQUEST_TIMEOUT)
|
|
logger.info("Done: %r, %r" % (url, str(mtime)))
|
|
responses.append((mtime, url, data))
|
|
|
|
# use the newest of all status summaries
|
|
newest = sorted(responses)[-1]
|
|
logger.info("Selected: %r" % (newest[1],))
|
|
state.build_status = BuildStatus.parse_raw(newest[2])
|
|
|
|
|
|
def parse_desc(t: str) -> Dict[str, List[str]]:
|
|
d: Dict[str, List[str]] = {}
|
|
cat = None
|
|
values: List[str] = []
|
|
for l in t.splitlines():
|
|
l = l.strip()
|
|
if cat is None:
|
|
cat = l
|
|
elif not l:
|
|
d[cat] = values
|
|
cat = None
|
|
values = []
|
|
else:
|
|
values.append(l)
|
|
if cat is not None:
|
|
d[cat] = values
|
|
return d
|
|
|
|
|
|
async def parse_repo(repo: Repository, include_files: bool = True) -> Dict[str, Source]:
|
|
sources: Dict[str, Source] = {}
|
|
|
|
def add_desc(d: Any) -> None:
|
|
source = Source.from_desc(d, repo)
|
|
if source.name not in sources:
|
|
sources[source.name] = source
|
|
else:
|
|
source = sources[source.name]
|
|
|
|
source.add_desc(d, repo)
|
|
|
|
repo_url = repo.files_url if include_files else repo.db_url
|
|
logger.info("Loading %r" % repo_url)
|
|
data = await get_content_cached(repo_url, timeout=REQUEST_TIMEOUT)
|
|
|
|
with io.BytesIO(data) as f:
|
|
with ExtTarFile.open(fileobj=f, mode="r") as tar:
|
|
packages: Dict[str, list] = {}
|
|
for info in tar:
|
|
package_name = info.name.split("/", 1)[0]
|
|
infofile = tar.extractfile(info)
|
|
if infofile is None:
|
|
continue
|
|
with infofile:
|
|
packages.setdefault(package_name, []).append(
|
|
(info.name, infofile.read()))
|
|
|
|
for package_name, infos in sorted(packages.items()):
|
|
t = ""
|
|
for name, data in sorted(infos):
|
|
if name.endswith("/desc"):
|
|
t += data.decode("utf-8")
|
|
elif name.endswith("/depends"):
|
|
t += data.decode("utf-8")
|
|
elif name.endswith("/files"):
|
|
t += data.decode("utf-8")
|
|
desc = parse_desc(t)
|
|
add_desc(desc)
|
|
|
|
return sources
|
|
|
|
|
|
async def update_arch_versions() -> None:
|
|
urls = [i[0] for i in ARCH_REPO_CONFIG]
|
|
if not await check_needs_update(urls):
|
|
return
|
|
|
|
logger.info("update versions")
|
|
arch_versions: Dict[str, ExtInfo] = {}
|
|
awaitables = []
|
|
for (url, repo) in ARCH_REPO_CONFIG:
|
|
download_url = url.rsplit("/", 1)[0]
|
|
awaitables.append(
|
|
parse_repo(
|
|
Repository(repo, "", "", "", download_url, download_url, ""),
|
|
False
|
|
)
|
|
)
|
|
|
|
# priority: real packages > real provides > aur packages > aur provides
|
|
|
|
for sources in (await asyncio.gather(*awaitables)):
|
|
for source in sources.values():
|
|
version = extract_upstream_version(arch_version_to_msys(source.version))
|
|
for p in source.packages.values():
|
|
url = "https://archlinux.org/packages/%s/%s/%s/" % (
|
|
p.repo, p.arch, p.name)
|
|
|
|
if p.name in arch_versions:
|
|
old_ver = arch_versions[p.name][0]
|
|
if version_is_newer_than(version, old_ver):
|
|
arch_versions[p.name] = ExtInfo(p.name, version, p.builddate, url, {})
|
|
else:
|
|
arch_versions[p.name] = ExtInfo(p.name, version, p.builddate, url, {})
|
|
|
|
url = "https://archlinux.org/packages/%s/%s/%s/" % (
|
|
source.repos[0], source.arches[0], source.name)
|
|
if source.name in arch_versions:
|
|
old_ver = arch_versions[source.name][0]
|
|
if version_is_newer_than(version, old_ver):
|
|
arch_versions[source.name] = ExtInfo(source.name, version, source.date, url, {})
|
|
else:
|
|
arch_versions[source.name] = ExtInfo(source.name, version, source.date, url, {})
|
|
|
|
# use provides as fallback
|
|
for p in source.packages.values():
|
|
url = "https://archlinux.org/packages/%s/%s/%s/" % (
|
|
p.repo, p.arch, p.name)
|
|
|
|
for provides in sorted(p.provides.keys()):
|
|
if provides not in arch_versions:
|
|
arch_versions[provides] = ExtInfo(provides, version, p.builddate, url, {})
|
|
|
|
logger.info("done")
|
|
state.set_ext_infos(ExtId("archlinux", "Arch Linux", False), arch_versions)
|
|
|
|
logger.info("update versions from AUR")
|
|
aur_versions: Dict[str, ExtInfo] = {}
|
|
r = await get_content_cached(AUR_METADATA_URL,
|
|
timeout=REQUEST_TIMEOUT)
|
|
items = json.loads(r)
|
|
for item in items:
|
|
name = item["Name"]
|
|
if name in aur_versions:
|
|
continue
|
|
version = item["Version"]
|
|
msys_ver = extract_upstream_version(arch_version_to_msys(version))
|
|
last_modified = item["LastModified"]
|
|
url = "https://aur.archlinux.org/packages/%s" % name
|
|
aur_versions[name] = ExtInfo(name, msys_ver, last_modified, url, {})
|
|
|
|
for item in items:
|
|
name = item["Name"]
|
|
for provides in sorted(item.get("Provides", [])):
|
|
if provides in aur_versions:
|
|
continue
|
|
version = item["Version"]
|
|
msys_ver = extract_upstream_version(arch_version_to_msys(version))
|
|
last_modified = item["LastModified"]
|
|
url = "https://aur.archlinux.org/packages/%s" % name
|
|
aur_versions[provides] = ExtInfo(provides, msys_ver, last_modified, url, {})
|
|
|
|
logger.info("done")
|
|
state.set_ext_infos(ExtId("aur", "AUR", True), aur_versions)
|
|
|
|
|
|
CacheHeaders = Dict[str, Optional[str]]
|
|
|
|
|
|
async def check_needs_update(urls: List[str], _cache: Dict[str, CacheHeaders] = {}) -> bool:
|
|
"""Raises RequestException"""
|
|
|
|
if appconfig.CACHE_DIR:
|
|
return True
|
|
|
|
async def get_cache_headers(client: httpx.AsyncClient, url: str, timeout: float) -> Tuple[str, CacheHeaders]:
|
|
"""This tries to return the cache response headers for a given URL as cheap as possible"""
|
|
|
|
old_headers = _cache.get(url, {})
|
|
last_modified = old_headers.get("last-modified")
|
|
etag = old_headers.get("etag")
|
|
fetch_headers = {}
|
|
if last_modified is not None:
|
|
fetch_headers["if-modified-since"] = last_modified
|
|
if etag is not None:
|
|
fetch_headers["if-none-match"] = etag
|
|
r = await client.head(url, timeout=timeout, headers=fetch_headers)
|
|
if r.status_code == 304:
|
|
return (url, dict(old_headers))
|
|
r.raise_for_status()
|
|
new_headers = {}
|
|
new_headers["last-modified"] = r.headers.get("last-modified")
|
|
new_headers["etag"] = r.headers.get("etag")
|
|
return (url, new_headers)
|
|
|
|
needs_update = False
|
|
async with httpx.AsyncClient(follow_redirects=True) as client:
|
|
awaitables = []
|
|
for url in urls:
|
|
awaitables.append(get_cache_headers(client, url, timeout=REQUEST_TIMEOUT))
|
|
|
|
for url, new_cache_headers in (await asyncio.gather(*awaitables)):
|
|
old_cache_headers = _cache.get(url, {})
|
|
if old_cache_headers != new_cache_headers:
|
|
needs_update = True
|
|
_cache[url] = new_cache_headers
|
|
|
|
logger.info("check needs update: %r -> %r" % (urls, needs_update))
|
|
|
|
return needs_update
|
|
|
|
|
|
async def update_source() -> None:
|
|
"""Raises RequestException"""
|
|
|
|
urls = [repo.files_url for repo in get_repositories()]
|
|
if not await check_needs_update(urls):
|
|
return
|
|
|
|
logger.info("update source")
|
|
|
|
final: Dict[str, Source] = {}
|
|
awaitables = []
|
|
for repo in get_repositories():
|
|
awaitables.append(parse_repo(repo))
|
|
for sources in await asyncio.gather(*awaitables):
|
|
for name, source in sources.items():
|
|
if name in final:
|
|
final[name].packages.update(source.packages)
|
|
else:
|
|
final[name] = source
|
|
|
|
fill_rdepends(final)
|
|
fill_provided_by(final)
|
|
state.sources = final
|
|
|
|
|
|
async def update_sourceinfos() -> None:
|
|
urls = SRCINFO_URLS
|
|
if not await check_needs_update(urls):
|
|
return
|
|
|
|
logger.info("update sourceinfos")
|
|
result: Dict[str, SrcInfoPackage] = {}
|
|
pkgextra = PkgExtra(packages={})
|
|
|
|
for url in urls:
|
|
logger.info("Loading %r" % url)
|
|
data = await get_content_cached(url, timeout=REQUEST_TIMEOUT)
|
|
json_obj = json.loads(gzip.decompress(data).decode("utf-8"))
|
|
for hash_, m in json_obj.items():
|
|
extra = m.get("extra", {})
|
|
pkgbase = None
|
|
for repo, srcinfo in m["srcinfo"].items():
|
|
for pkg in SrcInfoPackage.for_srcinfo(srcinfo, repo, m["repo"], m["path"], m["date"]):
|
|
pkgbase = pkg.pkgbase
|
|
if pkg.pkgname in result:
|
|
logger.info(f"WARN: duplicate: {pkg.pkgname} provided by "
|
|
f"{pkg.pkgbase} and {result[pkg.pkgname].pkgbase}")
|
|
result[pkg.pkgname] = pkg
|
|
if pkgbase is not None:
|
|
pkgextra.packages[pkgbase] = extra_to_pkgextra_entry(extra)
|
|
if pkgbase == "autotools":
|
|
print(pkgextra.packages[pkgbase])
|
|
|
|
state.pkgextra = pkgextra
|
|
state.sourceinfos = result
|
|
await update_pypi_versions(pkgextra)
|
|
|
|
|
|
async def update_pypi_versions(pkgextra: PkgExtra) -> None:
|
|
urls = PYPI_URLS
|
|
if not await check_needs_update(urls):
|
|
return
|
|
|
|
projects = {}
|
|
for url in urls:
|
|
logger.info("Loading %r" % url)
|
|
data = await get_content_cached(url, timeout=REQUEST_TIMEOUT)
|
|
json_obj = json.loads(gzip.decompress(data).decode("utf-8"))
|
|
projects.update(json_obj.get("projects", {}))
|
|
|
|
pypi_versions = {}
|
|
for entry in pkgextra.packages.values():
|
|
if "pypi" not in entry.references:
|
|
continue
|
|
pypi_name = entry.references["pypi"]
|
|
assert isinstance(pypi_name, str)
|
|
if pypi_name in projects:
|
|
project = projects[pypi_name]
|
|
info = project["info"]
|
|
project_urls = project.get("urls", [])
|
|
oldest_timestamp = 0
|
|
for url_entry in project_urls:
|
|
dt = datetime.datetime.fromisoformat(
|
|
url_entry["upload_time_iso_8601"].replace("Z", "+00:00"))
|
|
timestamp = int(dt.timestamp())
|
|
if oldest_timestamp == 0 or timestamp < oldest_timestamp:
|
|
oldest_timestamp = timestamp
|
|
pypi_versions[pypi_name] = ExtInfo(
|
|
pypi_name, info["version"], oldest_timestamp, info["project_url"], {})
|
|
|
|
state.set_ext_infos(ExtId("pypi", "PyPI", True), pypi_versions)
|
|
|
|
|
|
def fill_rdepends(sources: Dict[str, Source]) -> None:
|
|
deps: Dict[str, Dict[Package, Set[DepType]]] = {}
|
|
for s in sources.values():
|
|
for p in s.packages.values():
|
|
for n, r in p.depends.items():
|
|
deps.setdefault(n, dict()).setdefault(p, set()).add(DepType.NORMAL)
|
|
for n, r in p.makedepends.items():
|
|
deps.setdefault(n, dict()).setdefault(p, set()).add(DepType.MAKE)
|
|
for n, r in p.optdepends.items():
|
|
deps.setdefault(n, dict()).setdefault(p, set()).add(DepType.OPTIONAL)
|
|
for n, r in p.checkdepends.items():
|
|
deps.setdefault(n, dict()).setdefault(p, set()).add(DepType.CHECK)
|
|
|
|
for s in sources.values():
|
|
for p in s.packages.values():
|
|
rdeps = [deps.get(p.name, dict())]
|
|
for prov in p.provides:
|
|
rdeps.append(deps.get(prov, dict()))
|
|
|
|
merged: Dict[Package, Set[DepType]] = {}
|
|
for rd in rdeps:
|
|
for rp, rs in rd.items():
|
|
merged.setdefault(rp, set()).update(rs)
|
|
|
|
p.rdepends = merged
|
|
|
|
|
|
def fill_provided_by(sources: Dict[str, Source]) -> None:
|
|
|
|
provided_by: Dict[str, Set[Package]] = {}
|
|
for s in sources.values():
|
|
for p in s.packages.values():
|
|
for provides in p.provides.keys():
|
|
provided_by.setdefault(provides, set()).add(p)
|
|
for s in sources.values():
|
|
for p in s.packages.values():
|
|
if p.name in provided_by:
|
|
p.provided_by = provided_by[p.name]
|
|
|
|
|
|
_rate_limit = AsyncLimiter(UPDATE_MIN_RATE, UPDATE_MIN_INTERVAL)
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _get_update_event() -> Event:
|
|
return Event()
|
|
|
|
|
|
async def wait_for_update() -> None:
|
|
update_event = _get_update_event()
|
|
await update_event.wait()
|
|
update_event.clear()
|
|
|
|
|
|
def queue_update() -> None:
|
|
update_event = _get_update_event()
|
|
update_event.set()
|
|
|
|
|
|
async def trigger_loop() -> None:
|
|
while True:
|
|
logger.info("Sleeping for %d" % UPDATE_INTERVAL)
|
|
await asyncio.sleep(UPDATE_INTERVAL)
|
|
queue_update()
|
|
|
|
_background_tasks = set()
|
|
|
|
|
|
async def update_loop() -> None:
|
|
task = asyncio.create_task(trigger_loop())
|
|
_background_tasks.add(task)
|
|
task.add_done_callback(_background_tasks.discard)
|
|
while True:
|
|
async with _rate_limit:
|
|
logger.info("check for updates")
|
|
try:
|
|
awaitables = [
|
|
update_cygwin_versions(),
|
|
update_arch_versions(),
|
|
update_source(),
|
|
update_sourceinfos(),
|
|
update_build_status(),
|
|
]
|
|
await asyncio.gather(*awaitables)
|
|
state.ready = True
|
|
logger.info("done")
|
|
except Exception:
|
|
traceback.print_exc(file=sys.stdout)
|
|
logger.info("Waiting for next update")
|
|
await wait_for_update()
|
|
# XXX: it seems some updates don't propagate right away, so wait a bit
|
|
await asyncio.sleep(5)
|