Add a very basic gentoo package mapper

I couldn't find a proper API, so while this matches lots of packages
it also includes testing/unstable versions which makes for a lot
of false positives would we use it to detect new versions, so it's
marked as a fallback provider for now.

app/appconfig.py

@@ -39,6 +39,8 @@ PYPI_URLS = [
     "https://github.com/msys2/MSYS2-packages/releases/download/srcinfo-cache/pypi.json.gz",
 ]
 
+GENTOO_SNAPSHOT_URL = "https://mirror.leaseweb.com/gentoo/snapshots/gentoo-latest.tar.xz"
+
 # Check for updates every 5 minutes by default, at max 1 time every minute even if triggered
 UPDATE_INTERVAL = 60 * 5
 UPDATE_MIN_INTERVAL = 60

app/fetch.py

@@ -24,10 +24,12 @@ from .appstate import state, Source, get_repositories, SrcInfoPackage, Package,
     ExtInfo, ExtId
 from .pkgextra import PkgExtra, extra_to_pkgextra_entry
 from .appconfig import CYGWIN_METADATA_URL, REQUEST_TIMEOUT, AUR_METADATA_URL, ARCH_REPO_CONFIG, \
-    SRCINFO_URLS, UPDATE_INTERVAL, BUILD_STATUS_URLS, UPDATE_MIN_RATE, UPDATE_MIN_INTERVAL, PYPI_URLS
+    SRCINFO_URLS, UPDATE_INTERVAL, BUILD_STATUS_URLS, UPDATE_MIN_RATE, UPDATE_MIN_INTERVAL, PYPI_URLS, \
+    GENTOO_SNAPSHOT_URL
 from .utils import version_is_newer_than, arch_version_to_msys, extract_upstream_version, logger
 from . import appconfig
 from .exttarfile import ExtTarFile
+from .gentoo import parse_gentoo_versions
 
 
 def get_mtime_for_response(response: httpx.Response) -> datetime.datetime | None:
@@ -135,6 +137,18 @@ async def update_cygwin_versions() -> None:
     state.set_ext_infos(ExtId("cygwin-mingw64", "Cygwin-mingw64", False), cygwin_versions_mingw64)
 
 
+async def update_gentoo_versions() -> None:
+    url = GENTOO_SNAPSHOT_URL
+    if not await check_needs_update([url]):
+        return
+    logger.info("update gentoo info")
+    logger.info("Loading %r" % url)
+    data = await get_content_cached(url, timeout=REQUEST_TIMEOUT)
+    gentoo_versions = parse_gentoo_versions(data)
+    # fallback, since parsing isn't perfect and we include unstable versions
+    state.set_ext_infos(ExtId("gentoo", "Gentoo", True), gentoo_versions)
+
+
 async def update_build_status() -> None:
     urls = BUILD_STATUS_URLS
     if not await check_needs_update(urls):
@@ -515,6 +529,7 @@ async def update_loop() -> None:
             try:
                 awaitables = [
                     update_cygwin_versions(),
+                    update_gentoo_versions(),
                     update_arch_versions(),
                     update_source(),
                     update_sourceinfos(),

app/gentoo.py

@@ -0,0 +1,63 @@
+import tarfile
+import io
+import functools
+
+from .appstate import ExtInfo
+from .utils import vercmp
+
+
+def parse_gentoo_versions(data: bytes) -> dict[str, ExtInfo]:
+    packages: dict[str, dict[str, int]] = {}
+    masked = set()
+    with io.BytesIO(data) as f:
+        with tarfile.open(fileobj=f, mode="r") as tar:
+            for tarinfo in tar:
+                name = tarinfo.name
+
+                # Find package versions that are masked because they are unstable
+                # This only covers a tiny amount of packages, but it's better than nothing
+                if name.endswith("/profiles/package.mask") and tarinfo.isreg():
+                    content = tar.extractfile(tarinfo)
+                    assert content is not None
+                    for line in content.read().decode().splitlines():
+                        if line.startswith("~"):
+                            masked.add(line[1:])
+
+                # All packages
+                if name.endswith(".ebuild") and name.count("/") > 1 and tarinfo.isreg():
+                    gentoo_name = name.rsplit("/", 1)[0].split("/", 1)[-1]
+                    mtime = tarinfo.mtime
+                    package_name = gentoo_name.split("/", 1)[1]
+                    basename = name.rsplit("/", 1)[-1]
+                    version = basename[len(package_name) + 1:].rsplit(".", 1)[0]
+                    packages.setdefault(gentoo_name, {})[version] = mtime
+
+    infos = {}
+    for gentoo_name, versions in packages.items():
+
+        # Remove all masked versions and live ebuilds using 9999 as a version
+        # We are not parsing the KEYWORDS to see which versions are stable, so
+        # we also get testing packages :/
+        for version in list(versions):
+            if f"{gentoo_name}-{version}" in masked:
+                del versions[version]
+            elif "9999" in version:
+                del versions[version]
+
+        # No version left, skip
+        if not versions:
+            continue
+
+        # TODO: Not sure if the version sorting is correct for gentoo..
+        newest_version = sorted(versions, key=functools.cmp_to_key(vercmp))[-1]
+        package_name = gentoo_name.split("/", 1)[1]
+        info = ExtInfo(gentoo_name, newest_version, versions[newest_version],
+                       f"https://packages.gentoo.org/packages/{gentoo_name}", {})
+        # Add with the gentoo category and without, so we can find it by package name automatically,
+        # but can also reference it unambiguously by gentoo name
+        infos[gentoo_name] = info
+        infos[package_name] = info
+        if gentoo_name.startswith("dev-python/"):
+            infos["python-" + package_name] = info
+
+    return infos