diff --git a/app/api.py b/app/api.py index 48e7b16..667206f 100644 --- a/app/api.py +++ b/app/api.py @@ -1,13 +1,118 @@ +from functools import cmp_to_key + from fastapi import FastAPI, APIRouter, Request, Response from fastapi.responses import JSONResponse +from typing import Tuple, Dict, List, Set, Any, Iterable +from .appstate import state, SrcInfoPackage +from .utils import package_name_is_vcs, version_is_newer_than router = APIRouter() -@router.get('/') -async def index(request: Request, response: Response) -> Response: - return JSONResponse({"hello": "world"}) +def cmp_(a: Any, b: Any) -> int: + return int((a > b) - (a < b)) -api = FastAPI() + +def cmp_func(e1: Dict, e2: Dict) -> int: + # package with fewest deps first + e1_k = (len(e1["makedepends"]), sorted(e1["provides"])) + e2_k = (len(e2["makedepends"]), sorted(e2["provides"])) + e1_p, e1_m = e1["provides"], e1["makedepends"] + e2_p, e2_m = e2["provides"], e2["makedepends"] + + if e1_p & e2_m and e2_p & e1_m: + # cyclic dependency! + return cmp_(e1_k, e2_k) + elif e1_p & e2_m: + return -1 + elif e2_p & e1_m: + return 1 + else: + return cmp_(e1_k, e2_k) + + +@router.get('/buildqueue') +async def index(request: Request, response: Response, include_new: bool = True, include_update: bool = True) -> Response: + srcinfos = [] + + # packages that should be updated + if include_update: + for s in state.sources.values(): + for k, p in sorted(s.packages.items()): + if p.name in state.sourceinfos: + srcinfo = state.sourceinfos[p.name] + if package_name_is_vcs(s.name): + continue + if not version_is_newer_than(srcinfo.build_version, p.version): + continue + srcinfos.append(srcinfo) + + # packages that are new + if include_new: + available = {} + for srcinfo in state.sourceinfos.values(): + if package_name_is_vcs(srcinfo.pkgbase): + continue + available[srcinfo.pkgbase] = srcinfo + for s in state.sources.values(): + available.pop(s.name, None) + srcinfos.extend(available.values()) + + def build_key(srcinfo: SrcInfoPackage) -> Tuple[str, str]: + return (srcinfo.repo_url, srcinfo.repo_path) + + to_build: Dict[Tuple, List[SrcInfoPackage]] = {} + for srcinfo in srcinfos: + key = build_key(srcinfo) + to_build.setdefault(key, []).append(srcinfo) + + db_makedepends: Dict[str, Set[str]] = {} + for s in state.sources.values(): + for p in s.packages.values(): + md = [d[0] for d in p.depends + p.makedepends] + db_makedepends.setdefault(p.name, set()).update(md) + + def get_transitive_makedepends(packages: Iterable[str]) -> Set[str]: + todo = set(packages) + done = set() + while todo: + name = todo.pop() + if name in done: + continue + done.add(name) + # prefer depends from the GIT packages over the DB + if name in state.sourceinfos: + si = state.sourceinfos[name] + todo.update([d[0] for d in si.depends + si.makedepends]) + elif name in db_makedepends: + todo.update(db_makedepends[name]) + return done + + entries = [] + for srcinfos in to_build.values(): + packages = set() + provides: Set[str] = set() + for si in srcinfos: + packages.add(si.pkgname) + + entries.append({ + "repo_url": srcinfos[0].repo_url, + "repo_path": srcinfos[0].repo_path, + "version": srcinfos[0].build_version, + "packages": packages, + "provides": provides | packages, + "makedepends": get_transitive_makedepends(packages), + }) + + entries.sort(key=cmp_to_key(cmp_func)) + + for e in entries: + e["packages"] = sorted(e["packages"]) + del e["makedepends"] + del e["provides"] + + return JSONResponse(entries) + +api = FastAPI(title="MSYS2 Packages API", docs_url="/") api.include_router(router) diff --git a/app/appstate.py b/app/appstate.py index c4e8930..dc1f2e5 100644 --- a/app/appstate.py +++ b/app/appstate.py @@ -12,7 +12,7 @@ from urllib.parse import quote_plus, quote from typing import List, Set, Dict, Tuple, Optional, Type, Sequence, NamedTuple from .appconfig import REPOSITORIES -from .utils import vercmp, version_is_newer_than, extract_upstream_version +from .utils import vercmp, version_is_newer_than, extract_upstream_version, split_depends from .pgp import parse_signature @@ -119,7 +119,7 @@ class Repository: global state repo_packages = [] - for s in state.sources: + for s in state.sources.values(): for k, p in sorted(s.packages.items()): if p.repo == self.name and p.repo_variant == self.variant: repo_packages.append(p) @@ -153,7 +153,7 @@ class AppState: self._etag = "" self._last_update = 0.0 - self._sources: List[Source] = [] + self._sources: Dict[str, Source] = {} self._sourceinfos: Dict[str, SrcInfoPackage] = {} self._arch_versions: Dict[str, Tuple[str, str, int]] = {} self._arch_mapping: ArchMapping = ArchMapping() @@ -173,11 +173,11 @@ class AppState: return self._etag @property - def sources(self) -> List[Source]: + def sources(self) -> Dict[str, Source]: return self._sources @sources.setter - def sources(self, sources: List[Source]) -> None: + def sources(self, sources: Dict[str, Source]) -> None: self._sources = sources self._update_etag() @@ -227,16 +227,6 @@ class Package: checkdepends: List[str], sig_data: str, url: str) -> None: self.builddate = int(builddate) self.csize = csize - - def split_depends(deps: List[str]) -> List[Tuple[str, str]]: - r = [] - for d in deps: - parts = re.split("([<>=]+)", d, 1) - first = parts[0].strip() - second = "".join(parts[1:]).strip() - r.append((first, second)) - return r - self.url = url self.signature = parse_signature(base64.b64decode(sig_data)) self.depends = split_depends(depends) @@ -254,7 +244,7 @@ class Package: self.repo = repo self.repo_variant = repo_variant self.provides = dict(split_depends(provides)) - self.conflicts = conflicts + self.conflicts = dict(split_depends(conflicts)) self.replaces = replaces self.version = version self.base = base @@ -491,8 +481,10 @@ class SrcInfoPackage(object): self.repo_path = repo_path self.date = date self.epoch: Optional[str] = None - self.depends: List[str] = [] - self.makedepends: List[str] = [] + self.depends: List[Tuple[str, str]] = [] + self.makedepends: List[Tuple[str, str]] = [] + self.provides: Dict[str, str] = {} + self.conflicts: Dict[str, str] = {} self.sources: List[str] = [] @property @@ -526,6 +518,8 @@ class SrcInfoPackage(object): makedepends = [] sources = [] pkgbase = line.split(" = ", 1)[-1] + provides = [] + conflicts = [] elif line.startswith("depends = "): depends.append(line.split(" = ", 1)[-1]) elif line.startswith("makedepends = "): @@ -538,13 +532,19 @@ class SrcInfoPackage(object): pkgrel = line.split(" = ", 1)[-1] elif line.startswith("epoch = "): epoch = line.split(" = ", 1)[-1] + elif line.startswith("provides = "): + provides.append(line.split(" = ", 1)[-1]) + elif line.startswith("conflicts = "): + conflicts.append(line.split(" = ", 1)[-1]) elif line.startswith("pkgname = "): pkgname = line.split(" = ", 1)[-1] package = cls(pkgbase, pkgname, pkgver, pkgrel, repo, repo_path, date) package.epoch = epoch - package.depends = depends - package.makedepends = makedepends + package.depends = split_depends(depends) + package.makedepends = split_depends(makedepends) package.sources = sources + package.conflicts = dict(split_depends(conflicts)) + package.provides = dict(split_depends(provides)) packages.add(package) return packages diff --git a/app/fetch.py b/app/fetch.py index 1527365..1beb1ca 100644 --- a/app/fetch.py +++ b/app/fetch.py @@ -174,7 +174,7 @@ async def update_arch_versions() -> None: print("update versions from AUR") # a bit hacky, try to get the remaining versions from AUR possible_names = set() - for s in state.sources: + for s in state.sources.values(): if package_name_is_vcs(s.name): continue for p in s.packages.values(): @@ -256,9 +256,8 @@ async def update_source() -> None: else: final[name] = source - new_sources = [x[1] for x in sorted(final.items())] - fill_rdepends(new_sources) - state.sources = new_sources + fill_rdepends(final) + state.sources = final async def update_sourceinfos() -> None: @@ -278,9 +277,9 @@ async def update_sourceinfos() -> None: state.sourceinfos = result -def fill_rdepends(sources: List[Source]) -> None: +def fill_rdepends(sources: Dict[str, Source]) -> None: deps: Dict[str, Set[Tuple[Package, str]]] = {} - for s in sources: + for s in sources.values(): for p in s.packages.values(): for n, r in p.depends: deps.setdefault(n, set()).add((p, "")) @@ -291,7 +290,7 @@ def fill_rdepends(sources: List[Source]) -> None: for n, r in p.checkdepends: deps.setdefault(n, set()).add((p, "check")) - for s in sources: + for s in sources.values(): for p in s.packages.values(): rdepends = list(deps.get(p.name, set())) for prov in p.provides: diff --git a/app/utils.py b/app/utils.py index 16a9911..991a3cd 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,6 +1,7 @@ # Copyright 2016-2020 Christoph Reiter # SPDX-License-Identifier: MIT +import re from itertools import zip_longest from typing import List, Tuple, Optional @@ -126,3 +127,13 @@ def arch_version_to_msys(v: str) -> str: def version_is_newer_than(v1: str, v2: str) -> bool: return vercmp(v1, v2) == 1 + + +def split_depends(deps: List[str]) -> List[Tuple[str, str]]: + r = [] + for d in deps: + parts = re.split("([<>=]+)", d, 1) + first = parts[0].strip() + second = "".join(parts[1:]).strip() + r.append((first, second)) + return r diff --git a/app/web.py b/app/web.py index 1f805e9..62d2d18 100644 --- a/app/web.py +++ b/app/web.py @@ -124,7 +124,10 @@ async def base(request: Request, response: Response, base_name: str = None) -> R global state if base_name is not None: - res = [s for s in state.sources if s.name == base_name] + if base_name in state.sources: + res = [state.sources[base_name]] + else: + res = [] return templates.TemplateResponse("base.html", { "request": request, "sources": res, @@ -132,7 +135,7 @@ async def base(request: Request, response: Response, base_name: str = None) -> R else: return templates.TemplateResponse("baseindex.html", { "request": request, - "sources": state.sources, + "sources": state.sources.values(), }, headers=dict(response.headers)) @@ -143,7 +146,7 @@ async def group(request: Request, response: Response, group_name: Optional[str] if group_name is not None: res = [] - for s in state.sources: + for s in state.sources.values(): for k, p in sorted(s.packages.items()): if group_name in p.groups: res.append(p) @@ -155,7 +158,7 @@ async def group(request: Request, response: Response, group_name: Optional[str] }, headers=dict(response.headers)) else: groups: Dict[str, int] = {} - for s in state.sources: + for s in state.sources.values(): for k, p in sorted(s.packages.items()): for name in p.groups: groups[name] = groups.get(name, 0) + 1 @@ -171,7 +174,7 @@ async def package(request: Request, response: Response, package_name: Optional[s global state packages = [] - for s in state.sources: + for s in state.sources.values(): for k, p in sorted(s.packages.items()): if package_name is None or p.name == package_name or package_name in p.provides: if not repo or p.repo == repo: @@ -193,7 +196,7 @@ async def package(request: Request, response: Response, package_name: Optional[s @router.get('/updates', dependencies=[Depends(Etag(get_etag))]) async def updates(request: Request, response: Response) -> Response: packages: List[Package] = [] - for s in state.sources: + for s in state.sources.values(): packages.extend(s.packages.values()) packages.sort(key=lambda p: p.builddate, reverse=True) @@ -209,7 +212,7 @@ async def outofdate(request: Request, response: Response) -> Response: skipped = [] to_update = [] all_sources = [] - for s in state.sources: + for s in state.sources.values(): if package_name_is_vcs(s.name): continue @@ -253,7 +256,7 @@ async def outofdate(request: Request, response: Response) -> Response: async def queue(request: Request, response: Response) -> Response: # Create entries for all packages where the version doesn't match updates = [] - for s in state.sources: + for s in state.sources.values(): for k, p in sorted(s.packages.items()): if p.name in state.sourceinfos: srcinfo = state.sourceinfos[p.name] @@ -281,7 +284,7 @@ async def new(request: Request, response: Response) -> Response: if package_name_is_vcs(srcinfo.pkgbase): continue available[srcinfo.pkgbase] = srcinfo - for s in state.sources: + for s in state.sources.values(): available.pop(s.name, None) new = list(available.values()) @@ -299,7 +302,7 @@ async def new(request: Request, response: Response) -> Response: async def removals(request: Request, response: Response) -> Response: # get all packages in the pacman repo which are no in GIT missing = [] - for s in state.sources: + for s in state.sources.values(): for k, p in s.packages.items(): if p.name not in state.sourceinfos: missing.append((s, p)) @@ -323,7 +326,7 @@ async def python2(request: Request, response: Response) -> Response: if name.startswith("mingw-w64-x86_64-python2") or name.startswith("python2"): py2 = True if py2 and py3: - for s in state.sources: + for s in state.sources.values(): if s.name == p.base: return len(s.packages) >= 4 return True @@ -341,7 +344,7 @@ async def python2(request: Request, response: Response) -> Response: return len(done) - 1 deps: Dict[str, Tuple[Package, int, bool]] = {} - for s in state.sources: + for s in state.sources.values(): for p in s.packages.values(): if not (p.repo, p.repo_variant) in [("mingw64", ""), ("msys", "x86_64")]: continue @@ -388,12 +391,12 @@ async def search(request: Request, response: Response, q: str = "", t: str = "") if not query: pass elif qtype == "pkg": - for s in state.sources: + for s in state.sources.values(): if [p for p in parts if p.lower() in s.name.lower()] == parts: res_pkg.append(s) res_pkg.sort(key=lambda s: s.name) elif qtype == "binpkg": - for s in state.sources: + for s in state.sources.values(): for sub in s.packages.values(): if [p for p in parts if p.lower() in sub.name.lower()] == parts: res_pkg.append(sub) diff --git a/tests/test_api.py b/tests/test_api.py index d81aa32..1cb36ee 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,4 +15,4 @@ def client(): def test_api(client): - client.get('/api').raise_for_status() + client.get('/api/buildqueue').raise_for_status()