Add some API for getting the packages to build

This is just a first draft.

app/api.py

@@ -1,13 +1,118 @@
+from functools import cmp_to_key
+
 from fastapi import FastAPI, APIRouter, Request, Response
 from fastapi.responses import JSONResponse
 
+from typing import Tuple, Dict, List, Set, Any, Iterable
+from .appstate import state, SrcInfoPackage
+from .utils import package_name_is_vcs, version_is_newer_than
 
 router = APIRouter()
 
 
-@router.get('/')
-async def index(request: Request, response: Response) -> Response:
-    return JSONResponse({"hello": "world"})
+def cmp_(a: Any, b: Any) -> int:
+    return int((a > b) - (a < b))
 
-api = FastAPI()
+
+def cmp_func(e1: Dict, e2: Dict) -> int:
+    # package with fewest deps first
+    e1_k = (len(e1["makedepends"]), sorted(e1["provides"]))
+    e2_k = (len(e2["makedepends"]), sorted(e2["provides"]))
+    e1_p, e1_m = e1["provides"], e1["makedepends"]
+    e2_p, e2_m = e2["provides"], e2["makedepends"]
+
+    if e1_p & e2_m and e2_p & e1_m:
+        # cyclic dependency!
+        return cmp_(e1_k, e2_k)
+    elif e1_p & e2_m:
+        return -1
+    elif e2_p & e1_m:
+        return 1
+    else:
+        return cmp_(e1_k, e2_k)
+
+
+@router.get('/buildqueue')
+async def index(request: Request, response: Response, include_new: bool = True, include_update: bool = True) -> Response:
+    srcinfos = []
+
+    # packages that should be updated
+    if include_update:
+        for s in state.sources.values():
+            for k, p in sorted(s.packages.items()):
+                if p.name in state.sourceinfos:
+                    srcinfo = state.sourceinfos[p.name]
+                    if package_name_is_vcs(s.name):
+                        continue
+                    if not version_is_newer_than(srcinfo.build_version, p.version):
+                        continue
+                    srcinfos.append(srcinfo)
+
+    # packages that are new
+    if include_new:
+        available = {}
+        for srcinfo in state.sourceinfos.values():
+            if package_name_is_vcs(srcinfo.pkgbase):
+                continue
+            available[srcinfo.pkgbase] = srcinfo
+        for s in state.sources.values():
+            available.pop(s.name, None)
+        srcinfos.extend(available.values())
+
+    def build_key(srcinfo: SrcInfoPackage) -> Tuple[str, str]:
+        return (srcinfo.repo_url, srcinfo.repo_path)
+
+    to_build: Dict[Tuple, List[SrcInfoPackage]] = {}
+    for srcinfo in srcinfos:
+        key = build_key(srcinfo)
+        to_build.setdefault(key, []).append(srcinfo)
+
+    db_makedepends: Dict[str, Set[str]] = {}
+    for s in state.sources.values():
+        for p in s.packages.values():
+            md = [d[0] for d in p.depends + p.makedepends]
+            db_makedepends.setdefault(p.name, set()).update(md)
+
+    def get_transitive_makedepends(packages: Iterable[str]) -> Set[str]:
+        todo = set(packages)
+        done = set()
+        while todo:
+            name = todo.pop()
+            if name in done:
+                continue
+            done.add(name)
+            # prefer depends from the GIT packages over the DB
+            if name in state.sourceinfos:
+                si = state.sourceinfos[name]
+                todo.update([d[0] for d in si.depends + si.makedepends])
+            elif name in db_makedepends:
+                todo.update(db_makedepends[name])
+        return done
+
+    entries = []
+    for srcinfos in to_build.values():
+        packages = set()
+        provides: Set[str] = set()
+        for si in srcinfos:
+            packages.add(si.pkgname)
+
+        entries.append({
+            "repo_url": srcinfos[0].repo_url,
+            "repo_path": srcinfos[0].repo_path,
+            "version": srcinfos[0].build_version,
+            "packages": packages,
+            "provides": provides | packages,
+            "makedepends": get_transitive_makedepends(packages),
+        })
+
+    entries.sort(key=cmp_to_key(cmp_func))
+
+    for e in entries:
+        e["packages"] = sorted(e["packages"])
+        del e["makedepends"]
+        del e["provides"]
+
+    return JSONResponse(entries)
+
+api = FastAPI(title="MSYS2 Packages API", docs_url="/")
 api.include_router(router)

app/appstate.py

@@ -12,7 +12,7 @@ from urllib.parse import quote_plus, quote
 from typing import List, Set, Dict, Tuple, Optional, Type, Sequence, NamedTuple
 
 from .appconfig import REPOSITORIES
-from .utils import vercmp, version_is_newer_than, extract_upstream_version
+from .utils import vercmp, version_is_newer_than, extract_upstream_version, split_depends
 from .pgp import parse_signature
 
 
@@ -119,7 +119,7 @@ class Repository:
         global state
 
         repo_packages = []
-        for s in state.sources:
+        for s in state.sources.values():
             for k, p in sorted(s.packages.items()):
                 if p.repo == self.name and p.repo_variant == self.variant:
                     repo_packages.append(p)
@@ -153,7 +153,7 @@ class AppState:
 
         self._etag = ""
         self._last_update = 0.0
-        self._sources: List[Source] = []
+        self._sources: Dict[str, Source] = {}
         self._sourceinfos: Dict[str, SrcInfoPackage] = {}
         self._arch_versions: Dict[str, Tuple[str, str, int]] = {}
         self._arch_mapping: ArchMapping = ArchMapping()
@@ -173,11 +173,11 @@ class AppState:
         return self._etag
 
     @property
-    def sources(self) -> List[Source]:
+    def sources(self) -> Dict[str, Source]:
         return self._sources
 
     @sources.setter
-    def sources(self, sources: List[Source]) -> None:
+    def sources(self, sources: Dict[str, Source]) -> None:
         self._sources = sources
         self._update_etag()
 
@@ -227,16 +227,6 @@ class Package:
                  checkdepends: List[str], sig_data: str, url: str) -> None:
         self.builddate = int(builddate)
         self.csize = csize
-
-        def split_depends(deps: List[str]) -> List[Tuple[str, str]]:
-            r = []
-            for d in deps:
-                parts = re.split("([<>=]+)", d, 1)
-                first = parts[0].strip()
-                second = "".join(parts[1:]).strip()
-                r.append((first, second))
-            return r
-
         self.url = url
         self.signature = parse_signature(base64.b64decode(sig_data))
         self.depends = split_depends(depends)
@@ -254,7 +244,7 @@ class Package:
         self.repo = repo
         self.repo_variant = repo_variant
         self.provides = dict(split_depends(provides))
-        self.conflicts = conflicts
+        self.conflicts = dict(split_depends(conflicts))
         self.replaces = replaces
         self.version = version
         self.base = base
@@ -491,8 +481,10 @@ class SrcInfoPackage(object):
         self.repo_path = repo_path
         self.date = date
         self.epoch: Optional[str] = None
-        self.depends: List[str] = []
-        self.makedepends: List[str] = []
+        self.depends: List[Tuple[str, str]] = []
+        self.makedepends: List[Tuple[str, str]] = []
+        self.provides: Dict[str, str] = {}
+        self.conflicts: Dict[str, str] = {}
         self.sources: List[str] = []
 
     @property
@@ -526,6 +518,8 @@ class SrcInfoPackage(object):
                 makedepends = []
                 sources = []
                 pkgbase = line.split(" = ", 1)[-1]
+                provides = []
+                conflicts = []
             elif line.startswith("depends = "):
                 depends.append(line.split(" = ", 1)[-1])
             elif line.startswith("makedepends = "):
@@ -538,13 +532,19 @@ class SrcInfoPackage(object):
                 pkgrel = line.split(" = ", 1)[-1]
             elif line.startswith("epoch = "):
                 epoch = line.split(" = ", 1)[-1]
+            elif line.startswith("provides = "):
+                provides.append(line.split(" = ", 1)[-1])
+            elif line.startswith("conflicts = "):
+                conflicts.append(line.split(" = ", 1)[-1])
             elif line.startswith("pkgname = "):
                 pkgname = line.split(" = ", 1)[-1]
                 package = cls(pkgbase, pkgname, pkgver, pkgrel, repo, repo_path, date)
                 package.epoch = epoch
-                package.depends = depends
-                package.makedepends = makedepends
+                package.depends = split_depends(depends)
+                package.makedepends = split_depends(makedepends)
                 package.sources = sources
+                package.conflicts = dict(split_depends(conflicts))
+                package.provides = dict(split_depends(provides))
                 packages.add(package)
 
         return packages

app/fetch.py

@@ -174,7 +174,7 @@ async def update_arch_versions() -> None:
     print("update versions from AUR")
     # a bit hacky, try to get the remaining versions from AUR
     possible_names = set()
-    for s in state.sources:
+    for s in state.sources.values():
         if package_name_is_vcs(s.name):
             continue
         for p in s.packages.values():
@@ -256,9 +256,8 @@ async def update_source() -> None:
             else:
                 final[name] = source
 
-    new_sources = [x[1] for x in sorted(final.items())]
-    fill_rdepends(new_sources)
-    state.sources = new_sources
+    fill_rdepends(final)
+    state.sources = final
 
 
 async def update_sourceinfos() -> None:
@@ -278,9 +277,9 @@ async def update_sourceinfos() -> None:
     state.sourceinfos = result
 
 
-def fill_rdepends(sources: List[Source]) -> None:
+def fill_rdepends(sources: Dict[str, Source]) -> None:
     deps: Dict[str, Set[Tuple[Package, str]]] = {}
-    for s in sources:
+    for s in sources.values():
         for p in s.packages.values():
             for n, r in p.depends:
                 deps.setdefault(n, set()).add((p, ""))
@@ -291,7 +290,7 @@ def fill_rdepends(sources: List[Source]) -> None:
             for n, r in p.checkdepends:
                 deps.setdefault(n, set()).add((p, "check"))
 
-    for s in sources:
+    for s in sources.values():
         for p in s.packages.values():
             rdepends = list(deps.get(p.name, set()))
             for prov in p.provides:

app/utils.py

@@ -1,6 +1,7 @@
 # Copyright 2016-2020 Christoph Reiter
 # SPDX-License-Identifier: MIT
 
+import re
 from itertools import zip_longest
 from typing import List, Tuple, Optional
 
@@ -126,3 +127,13 @@ def arch_version_to_msys(v: str) -> str:
 
 def version_is_newer_than(v1: str, v2: str) -> bool:
     return vercmp(v1, v2) == 1
+
+
+def split_depends(deps: List[str]) -> List[Tuple[str, str]]:
+    r = []
+    for d in deps:
+        parts = re.split("([<>=]+)", d, 1)
+        first = parts[0].strip()
+        second = "".join(parts[1:]).strip()
+        r.append((first, second))
+    return r

app/web.py

@@ -124,7 +124,10 @@ async def base(request: Request, response: Response, base_name: str = None) -> R
     global state
 
     if base_name is not None:
-        res = [s for s in state.sources if s.name == base_name]
+        if base_name in state.sources:
+            res = [state.sources[base_name]]
+        else:
+            res = []
         return templates.TemplateResponse("base.html", {
             "request": request,
             "sources": res,
@@ -132,7 +135,7 @@ async def base(request: Request, response: Response, base_name: str = None) -> R
     else:
         return templates.TemplateResponse("baseindex.html", {
             "request": request,
-            "sources": state.sources,
+            "sources": state.sources.values(),
         }, headers=dict(response.headers))
 
 
@@ -143,7 +146,7 @@ async def group(request: Request, response: Response, group_name: Optional[str]
 
     if group_name is not None:
         res = []
-        for s in state.sources:
+        for s in state.sources.values():
             for k, p in sorted(s.packages.items()):
                 if group_name in p.groups:
                     res.append(p)
@@ -155,7 +158,7 @@ async def group(request: Request, response: Response, group_name: Optional[str]
         }, headers=dict(response.headers))
     else:
         groups: Dict[str, int] = {}
-        for s in state.sources:
+        for s in state.sources.values():
             for k, p in sorted(s.packages.items()):
                 for name in p.groups:
                     groups[name] = groups.get(name, 0) + 1
@@ -171,7 +174,7 @@ async def package(request: Request, response: Response, package_name: Optional[s
     global state
 
     packages = []
-    for s in state.sources:
+    for s in state.sources.values():
         for k, p in sorted(s.packages.items()):
             if package_name is None or p.name == package_name or package_name in p.provides:
                 if not repo or p.repo == repo:
@@ -193,7 +196,7 @@ async def package(request: Request, response: Response, package_name: Optional[s
 @router.get('/updates', dependencies=[Depends(Etag(get_etag))])
 async def updates(request: Request, response: Response) -> Response:
     packages: List[Package] = []
-    for s in state.sources:
+    for s in state.sources.values():
         packages.extend(s.packages.values())
     packages.sort(key=lambda p: p.builddate, reverse=True)
 
@@ -209,7 +212,7 @@ async def outofdate(request: Request, response: Response) -> Response:
     skipped = []
     to_update = []
     all_sources = []
-    for s in state.sources:
+    for s in state.sources.values():
         if package_name_is_vcs(s.name):
             continue
 
@@ -253,7 +256,7 @@ async def outofdate(request: Request, response: Response) -> Response:
 async def queue(request: Request, response: Response) -> Response:
     # Create entries for all packages where the version doesn't match
     updates = []
-    for s in state.sources:
+    for s in state.sources.values():
         for k, p in sorted(s.packages.items()):
             if p.name in state.sourceinfos:
                 srcinfo = state.sourceinfos[p.name]
@@ -281,7 +284,7 @@ async def new(request: Request, response: Response) -> Response:
         if package_name_is_vcs(srcinfo.pkgbase):
             continue
         available[srcinfo.pkgbase] = srcinfo
-    for s in state.sources:
+    for s in state.sources.values():
         available.pop(s.name, None)
     new = list(available.values())
 
@@ -299,7 +302,7 @@ async def new(request: Request, response: Response) -> Response:
 async def removals(request: Request, response: Response) -> Response:
     # get all packages in the pacman repo which are no in GIT
     missing = []
-    for s in state.sources:
+    for s in state.sources.values():
         for k, p in s.packages.items():
             if p.name not in state.sourceinfos:
                 missing.append((s, p))
@@ -323,7 +326,7 @@ async def python2(request: Request, response: Response) -> Response:
             if name.startswith("mingw-w64-x86_64-python2") or name.startswith("python2"):
                 py2 = True
             if py2 and py3:
-                for s in state.sources:
+                for s in state.sources.values():
                     if s.name == p.base:
                         return len(s.packages) >= 4
                 return True
@@ -341,7 +344,7 @@ async def python2(request: Request, response: Response) -> Response:
         return len(done) - 1
 
     deps: Dict[str, Tuple[Package, int, bool]] = {}
-    for s in state.sources:
+    for s in state.sources.values():
         for p in s.packages.values():
             if not (p.repo, p.repo_variant) in [("mingw64", ""), ("msys", "x86_64")]:
                 continue
@@ -388,12 +391,12 @@ async def search(request: Request, response: Response, q: str = "", t: str = "")
     if not query:
         pass
     elif qtype == "pkg":
-        for s in state.sources:
+        for s in state.sources.values():
             if [p for p in parts if p.lower() in s.name.lower()] == parts:
                 res_pkg.append(s)
         res_pkg.sort(key=lambda s: s.name)
     elif qtype == "binpkg":
-        for s in state.sources:
+        for s in state.sources.values():
             for sub in s.packages.values():
                 if [p for p in parts if p.lower() in sub.name.lower()] == parts:
                     res_pkg.append(sub)

tests/test_api.py

@@ -15,4 +15,4 @@ def client():
 
 
 def test_api(client):
-    client.get('/api').raise_for_status()
+    client.get('/api/buildqueue').raise_for_status()