Befator-Inc-Firmen-Netzwerk/msys2-web
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generalize external references and handle AUR separately

Browse Source 
Remove all cygwin/arch specific code from the core models to make
it easier to add more references.
This commit is contained in:
Christoph Reiter
2023-04-11 20:02:55 +02:00
parent 6b244070f8
commit 5569252cee
5 changed files with 47 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
app/appstate.py
View File

@@ -22,9 +22,12 @@ from .pgp import parse_signature
PackageKey = Tuple[str, str, str, str, str]
ExtInfo = NamedTuple('ExtInfo', [
ExtId = NamedTuple('ExtId', [
('id', str),
('name', str),
])
ExtInfo = NamedTuple('ExtInfo', [
('version', str),
('date', int),
('url', str),
@@ -75,46 +78,6 @@ def get_realname_variants(s: Source) -> List[str]:
return main + sorted(package_variants) + sorted(provides_variants)
def get_arch_info_for_base(s: Source) -> Optional[ExtInfo]:
global state
if "archlinux" in s.pkgmeta.references:
mapped = s.pkgmeta.references["archlinux"]
if mapped is None:
return None
variants = [mapped]
else:
variants = get_realname_variants(s)
for arch_name in variants:
if arch_name in state.arch_versions:
return state.arch_versions[arch_name]
return None
def get_cygwin_info_for_base(s: Source) -> Optional[ExtInfo]:
global state
# XXX: we only care about msys packages here
if s.name != s.realname:
return None
if "cygwin" in s.pkgmeta.references:
mapped = s.pkgmeta.references["cygwin"]
if mapped is None:
return None
variants = [mapped]
else:
variants = get_realname_variants(s)
for realname in variants:
if realname in state.cygwin_versions:
return state.cygwin_versions[realname]
return None
def cleanup_files(files: List[str]) -> List[str]:
"""Remove redundant directory paths and root them"""
@@ -215,8 +178,7 @@ class AppState:
self._sources: Dict[str, Source] = {}
self._sourceinfos: Dict[str, SrcInfoPackage] = {}
self._pkgmeta: PkgMeta = PkgMeta(packages={})
self._arch_versions: Dict[str, ExtInfo] = {}
self._cygwin_versions: Dict[str, ExtInfo] = {}
self._ext_infos: Dict[ExtId, Dict[str, ExtInfo]] = {}
self._build_status: BuildStatus = BuildStatus()
self._update_etag()
@@ -260,12 +222,14 @@ class AppState:
self._update_etag()
@property
def arch_versions(self) -> Dict[str, ExtInfo]:
return self._arch_versions
def ext_info_ids(self) -> List[ExtId]:
return list(self._ext_infos.keys())
@arch_versions.setter
def arch_versions(self, versions: Dict[str, ExtInfo]) -> None:
self._arch_versions = versions
def get_ext_infos(self, id: ExtId) -> Dict[str, ExtInfo]:
return self._ext_infos.get(id, {})
def set_ext_infos(self, id: ExtId, info: Dict[str, ExtInfo]) -> None:
self._ext_infos.setdefault(id, info)
self._update_etag()
@property
@@ -277,15 +241,6 @@ class AppState:
self._build_status = build_status
self._update_etag()
@property
def cygwin_versions(self) -> Dict[str, ExtInfo]:
return self._cygwin_versions
@cygwin_versions.setter
def cygwin_versions(self, cygwin_versions: Dict[str, ExtInfo]) -> None:
self._cygwin_versions = cygwin_versions
self._update_etag()
class Package:
@@ -466,7 +421,7 @@ class Source:
def upstream_version(self) -> str:
# Take the newest version of the external versions
version = None
for info in self.external_infos:
for ext_id, info in self.external_infos:
if version is None or version_is_newer_than(info.version, version):
version = info.version
return version or ""
@@ -478,7 +433,7 @@ class Source:
return state.pkgmeta.packages.get(self.name, PkgMetaEntry())
@property
def external_infos(self) -> Sequence[ExtInfo]:
def external_infos(self) -> Sequence[Tuple[ExtId, ExtInfo]]:
global state
# internal package, don't try to link it
@@ -486,13 +441,20 @@ class Source:
return []
ext = []
arch_info = get_arch_info_for_base(self)
if arch_info is not None:
ext.append(arch_info)
for ext_id in state.ext_info_ids:
if ext_id.id in self.pkgmeta.references:
mapped = self.pkgmeta.references[ext_id.id]
if mapped is None:
continue
variants = [mapped]
else:
variants = get_realname_variants(self)
cygwin_info = get_cygwin_info_for_base(self)
if cygwin_info is not None:
ext.append(cygwin_info)
infos = state.get_ext_infos(ext_id)
for realname in variants:
if realname in infos:
ext.append((ext_id, infos[realname]))
break
return sorted(ext)
@@ -500,7 +462,7 @@ class Source:
def is_outdated(self) -> bool:
msys_version = extract_upstream_version(self.version)
for info in self.external_infos:
for ext_id, info in self.external_infos:
if version_is_newer_than(info.version, msys_version):
return True
return False

25
app/fetch.py
View File

@@ -21,7 +21,8 @@ import httpx
from aiolimiter import AsyncLimiter
import zstandard
from .appstate import state, Source, get_repositories, SrcInfoPackage, Package, DepType, Repository, BuildStatus, PkgMeta, ExtInfo
from .appstate import state, Source, get_repositories, SrcInfoPackage, Package, DepType, Repository, BuildStatus, PkgMeta, \
ExtInfo, ExtId
from .appconfig import CYGWIN_METADATA_URL, REQUEST_TIMEOUT, AUR_METADATA_URL, ARCH_REPO_CONFIG, PKGMETA_URLS, \
SRCINFO_URLS, UPDATE_INTERVAL, BUILD_STATUS_URLS, UPDATE_MIN_RATE, UPDATE_MIN_INTERVAL
from .utils import version_is_newer_than, arch_version_to_msys, extract_upstream_version
@@ -100,7 +101,7 @@ def parse_cygwin_versions(base_url: str, data: bytes) -> Dict[str, ExtInfo]:
continue
src_url_name = src_url.rsplit("/")[-1]
versions[source_package] = ExtInfo(
"cygwin", "Cygwin", version, 0,
version, 0,
"https://cygwin.com/packages/summary/%s-src.html" % source_package,
{src_url: src_url_name})
return versions
@@ -115,7 +116,7 @@ async def update_cygwin_versions() -> None:
data = await get_content_cached(url, timeout=REQUEST_TIMEOUT)
data = zstandard.ZstdDecompressor().decompress(data)
cygwin_versions = parse_cygwin_versions(url, data)
state.cygwin_versions = cygwin_versions
state.set_ext_infos(ExtId("cygwin", "Cygwin"), cygwin_versions)
async def update_build_status() -> None:
@@ -222,18 +223,18 @@ async def update_arch_versions() -> None:
if p.name in arch_versions:
old_ver = arch_versions[p.name][0]
if version_is_newer_than(version, old_ver):
arch_versions[p.name] = ExtInfo("archlinux", "Arch Linux", version, p.builddate, url, {})
arch_versions[p.name] = ExtInfo(version, p.builddate, url, {})
else:
arch_versions[p.name] = ExtInfo("archlinux", "Arch Linux", version, p.builddate, url, {})
arch_versions[p.name] = ExtInfo(version, p.builddate, url, {})
url = "https://archlinux.org/packages/%s/%s/%s/" % (
source.repos[0], source.arches[0], source.name)
if source.name in arch_versions:
old_ver = arch_versions[source.name][0]
if version_is_newer_than(version, old_ver):
arch_versions[source.name] = ExtInfo("archlinux", "Arch Linux", version, source.date, url, {})
arch_versions[source.name] = ExtInfo(version, source.date, url, {})
else:
arch_versions[source.name] = ExtInfo("archlinux", "Arch Linux", version, source.date, url, {})
arch_versions[source.name] = ExtInfo(version, source.date, url, {})
# use provides as fallback
for p in source.packages.values():
@@ -242,11 +243,13 @@ async def update_arch_versions() -> None:
for provides in sorted(p.provides.keys()):
if provides not in arch_versions:
arch_versions[provides] = ExtInfo("archlinux", "Arch Linux", version, p.builddate, url, {})
arch_versions[provides] = ExtInfo(version, p.builddate, url, {})
print("done")
state.set_ext_infos(ExtId("archlinux", "Arch Linux"), arch_versions)
print("update versions from AUR")
aur_versions: Dict[str, ExtInfo] = {}
r = await get_content_cached(AUR_METADATA_URL,
timeout=REQUEST_TIMEOUT)
items = json.loads(r)
@@ -259,7 +262,7 @@ async def update_arch_versions() -> None:
msys_ver = extract_upstream_version(arch_version_to_msys(version))
last_modified = item["LastModified"]
url = "https://aur.archlinux.org/packages/%s" % name
arch_versions[name] = ExtInfo("archlinux", "Arch Linux", msys_ver, last_modified, url, {})
aur_versions[name] = ExtInfo(msys_ver, last_modified, url, {})
for item in items:
name = item["Name"]
@@ -270,10 +273,10 @@ async def update_arch_versions() -> None:
msys_ver = extract_upstream_version(arch_version_to_msys(version))
last_modified = item["LastModified"]
url = "https://aur.archlinux.org/packages/%s" % name
arch_versions[provides] = ExtInfo("archlinux", "Arch Linux", msys_ver, last_modified, url, {})
aur_versions[provides] = ExtInfo(msys_ver, last_modified, url, {})
print("done")
state.arch_versions = arch_versions
state.set_ext_infos(ExtId("aur", "AUR"), aur_versions)
async def check_needs_update(urls: List[str], _cache: Dict[str, str] = {}) -> bool:

4
app/templates/base.html
View File

@@ -77,8 +77,8 @@
{% endif%}
</dd>
{% for info in s.external_infos %}
<dt class="col-sm-3 text-sm-end">{{ info.name }}:</dt>
{% for ext_id, info in s.external_infos %}
<dt class="col-sm-3 text-sm-end">{{ ext_id.name }}:</dt>
<dd class="col-sm-9"><a href="{{ info.url }}">{{ info.version }}</a>
{% for url, label in info.other_urls|dictsort %}
| <a href="{{ url }}">{{ label }}</a>

4
app/templates/package.html
View File

@@ -62,8 +62,8 @@
{% endif%}
</dd>
{% for info in s.external_infos %}
<dt class="col-sm-3 text-sm-end">{{ info.name }}:</dt>
{% for ext_id, info in s.external_infos %}
<dt class="col-sm-3 text-sm-end">{{ ext_id.name }}:</dt>
<dd class="col-sm-9"><a href="{{ info.url }}">{{ info.version }}</a>
{% for url, label in info.other_urls|dictsort %}
| <a href="{{ url }}">{{ label }}</a>

2
app/web.py
View File

@@ -448,7 +448,7 @@ async def outofdate(request: Request, response: Response, related: Optional[str]
external_infos = s.external_infos
for info in external_infos:
for ext_id, info in external_infos:
if version_is_newer_than(info.version, msys_version):
to_update.append((s, msys_version, git_version, info.version, info.url, info.date))
break