Generation of a RPM status dashboard to compare with xs8

scripts/repo_status/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+*.dot
+*.html

scripts/repo_status/.python-version

@@ -0,0 +1 @@
+3.11.11

scripts/repo_status/XS8-normal-src.txt

@@ -0,0 +1,119 @@
+XS8-normal-src/auto-cert-kit-8.4.5-1.xs8.src.rpm
+XS8-normal-src/biosdevname-0.3.10-5.xs8.src.rpm
+XS8-normal-src/blktap-3.55.5-2.xs8.src.rpm
+XS8-normal-src/bpftool-7.4.0-2.xs8.src.rpm
+XS8-normal-src/broadcom-bnxt-en-1.10.2_223.0.183.0-2.xs8.src.rpm
+XS8-normal-src/broadcom-mpi3mr-8.1.4.0.0-2.xs8.src.rpm
+XS8-normal-src/busybox-1.22.1-7.xs8.src.rpm
+XS8-normal-src/cisco-enic-4.5.0.7-1.xs8.src.rpm
+XS8-normal-src/cisco-fnic-2.0.0.90-1.xs8.src.rpm
+XS8-normal-src/cmake3-3.26.4-3.xs8.src.rpm
+XS8-normal-src/compiler-rt18-18.1.8-3.xs8.src.rpm
+XS8-normal-src/curl-8.6.0-2.xs8.src.rpm
+XS8-normal-src/devtoolset-11-gcc-11.2.1-6.xs8.src.rpm
+XS8-normal-src/dlm-4.0.7-2.xs8.src.rpm
+XS8-normal-src/dwarves-1.26-3.xs8.src.rpm
+XS8-normal-src/e2fsprogs-1.47.0-1.xs8.src.rpm
+XS8-normal-src/edk2-20220801-1.7.7.xs8.src.rpm
+XS8-normal-src/gdisk-1.0.10-1.xs8.src.rpm
+XS8-normal-src/golang-1.19.7-1.xs8.src.rpm
+XS8-normal-src/gpumon-24.1.0-40.xs8.src.rpm
+XS8-normal-src/grub-2.06-4.0.2.xs8.src.rpm
+XS8-normal-src/guest-templates-json-2.0.13-1.xs8.src.rpm
+XS8-normal-src/host-installer-10.10.25-1.xs8.src.rpm
+XS8-normal-src/host-upgrade-plugin-3.0.3-1.xs8.src.rpm
+XS8-normal-src/hwloc-2.11.1_xen1.1.1-1.xs8.src.rpm
+XS8-normal-src/intel-i40e-2.25.11-2.xs8.src.rpm
+XS8-normal-src/intel-ice-1.15.5-2.xs8.src.rpm
+XS8-normal-src/intel-igb-5.13.20-2.xs8.src.rpm
+XS8-normal-src/intel-igc-5.10.214-3.xs8.src.rpm
+XS8-normal-src/interface-rename-2.0.6-1.xs8.src.rpm
+XS8-normal-src/ipxe-20121005-1.0.7.xs8.src.rpm
+XS8-normal-src/ipxe-efi-20180514gite7f67d5-1.0.3.xs8.src.rpm
+XS8-normal-src/irqbalance-1.0.7-15.xs8.src.rpm
+XS8-normal-src/jemalloc-5.3.0-1.xs8.src.rpm
+XS8-normal-src/kernel-4.19.19-8.0.38.xs8.src.rpm
+XS8-normal-src/kernel-livepatch-1.0.4-1.xs8.src.rpm
+XS8-normal-src/kexec-tools-2.0.15-20.xs8.src.rpm
+XS8-normal-src/libbpf-1.4.0-3.xs8.src.rpm
+XS8-normal-src/libcgroup-0.41-21.xs8.src.rpm
+XS8-normal-src/libnbd-1.20.2-1.xs8.src.rpm
+XS8-normal-src/linux-guest-loader-2.3.2-2.xs8.src.rpm
+XS8-normal-src/livepatch-build-tools-20250121-1.xs8.src.rpm
+XS8-normal-src/logrotate-3.8.6-21.xs8.src.rpm
+XS8-normal-src/lvm2-2.02.180-18.xs8.src.rpm
+XS8-normal-src/mcelog-196-3.xs8.src.rpm
+XS8-normal-src/mellanox-mlnxen-5.9_0.5.5.0-2.xs8.src.rpm
+XS8-normal-src/microsemi-smartpqi-2.1.30_031-1.xs8.src.rpm
+XS8-normal-src/mpdecimal-2.5.1-1.xs8.src.rpm
+XS8-normal-src/nagios-plugins-2.4.3-3.xs8.src.rpm
+XS8-normal-src/ncurses-6.4-4.xs8.src.rpm
+XS8-normal-src/net-snmp-5.7.2-52.xs8.src.rpm
+XS8-normal-src/newt-0.52.23-4.xs8.src.rpm
+XS8-normal-src/ninja-build-1.10.2-4.xs8.src.rpm
+XS8-normal-src/nrpe-4.1.0-11.xs8.src.rpm
+XS8-normal-src/ocaml-4.14.2-1.xs8.src.rpm
+XS8-normal-src/ocaml-findlib-1.9.6-3.xs8.src.rpm
+XS8-normal-src/opam-2.1.4-4.xs8.src.rpm
+XS8-normal-src/openssh-7.4p1-23.3.xs8.src.rpm
+XS8-normal-src/openssl-1.0.2k-26.xs8.src.rpm
+XS8-normal-src/openvswitch-2.17.7-2.xs8.src.rpm
+XS8-normal-src/plymouth-0.8.9-0.31.20140113.3.xs8.src.rpm
+XS8-normal-src/psmisc-23.6-2.xs8.src.rpm
+XS8-normal-src/pyproject-rpm-macros-1.8.0-4.xs8.src.rpm
+XS8-normal-src/python-aiocontextvars-0.2.2-3.xs8.src.rpm
+XS8-normal-src/python-charset-normalizer-2.1.0-4.xs8.src.rpm
+XS8-normal-src/python-contextvars-2.4-3.xs8.src.rpm
+XS8-normal-src/python-defusedxml-0.7.1-1.xs8.src.rpm
+XS8-normal-src/python-deprecated-1.2.14-3.xs8.src.rpm
+XS8-normal-src/python-fasteners-0.9.0-3.xs8.src.rpm
+XS8-normal-src/python-hwinfo-0.1.11-1.xs8.src.rpm
+XS8-normal-src/python-idna-3.3-4.xs8.src.rpm
+XS8-normal-src/python-immutables-0.19-5.xs8.src.rpm
+XS8-normal-src/python-netaddr-0.10.1-1.xs8.src.rpm
+XS8-normal-src/python-pam-1.8.4-1.xs8.src.rpm
+XS8-normal-src/python-prettytable-0.7.2-14.xs8.src.rpm
+XS8-normal-src/python-psutil-5.9.1-2.xs8.src.rpm
+XS8-normal-src/python-pyudev-0.21.0-2.xs8.src.rpm
+XS8-normal-src/python-requests-2.28.1-4.xs8.src.rpm
+XS8-normal-src/python-tqdm-4.50.2-1.xs8.src.rpm
+XS8-normal-src/python-typing-extensions-3.7.4.3-4.xs8.src.rpm
+XS8-normal-src/python-urllib3-1.26.20-3.xs8.src.rpm
+XS8-normal-src/python-wheel-0.31.1-5.el7_7.src.rpm
+XS8-normal-src/python-wrapt-1.14.0-4.xs8.src.rpm
+XS8-normal-src/python-zstd-1.4.5.1-4.xs8.src.rpm
+XS8-normal-src/qemu-4.2.1-5.2.12.xs8.src.rpm
+XS8-normal-src/qemu-dp-7.0.0-17.xs8.src.rpm
+XS8-normal-src/qlogic-fastlinq-8.74.0.2-1.xs8.src.rpm
+XS8-normal-src/qlogic-qla2xxx-10.02.12.01_k-1.xs8.src.rpm
+XS8-normal-src/rrd-client-lib-2.0.0-1.xs8.src.rpm
+XS8-normal-src/rsync-3.4.1-1.xs8.src.rpm
+XS8-normal-src/scapy-2.4.5-3.xs8.src.rpm
+XS8-normal-src/sm-3.2.12-3.xs8.src.rpm
+XS8-normal-src/smartmontools-7.4-2.xs8.src.rpm
+XS8-normal-src/sm-core-libs-1.1.2-1.xs8.src.rpm
+XS8-normal-src/sudo-1.9.15-4.xs8.src.rpm
+XS8-normal-src/swtpm-0.7.3-8.xs8.src.rpm
+XS8-normal-src/test-ring0-1.0.7-4.xs8.src.rpm
+XS8-normal-src/varstored-1.2.0-2.xs8.src.rpm
+XS8-normal-src/vcputune-2.0.2-1.xs8.src.rpm
+XS8-normal-src/vendor-drivers-2.0.3-1.xs8.src.rpm
+XS8-normal-src/vmss-1.2.1-1.xs8.src.rpm
+XS8-normal-src/vncterm-10.2.1-2.xs8.src.rpm
+XS8-normal-src/xapi-25.6.0-1.xs8.src.rpm
+XS8-normal-src/xcp-python-libs-3.0.4-2.xs8.src.rpm
+XS8-normal-src/xcp-python-libs-compat-2.3.5-6.xs8.src.rpm
+XS8-normal-src/xen-4.17.5-9.xs8.src.rpm
+XS8-normal-src/xencert-8.4.4-1.xs8.src.rpm
+XS8-normal-src/xen-crashdump-analyser-2.6.1-1.xs8.src.rpm
+XS8-normal-src/xen-livepatch-4-3.xs8.src.rpm
+XS8-normal-src/xenserver-config-1.0.1-3.xs8.src.rpm
+XS8-normal-src/xenserver-hwdata-20240411-1.xs8.src.rpm
+XS8-normal-src/xenserver-nagios-plugins-1.0.6-1.xs8.src.rpm
+XS8-normal-src/xenserver-plymouth-theme-1.4.0-2.xs8.src.rpm
+XS8-normal-src/xenserver-release-8.4.0-15.xs8.src.rpm
+XS8-normal-src/xenserver-status-report-2.0.11-1.xs8.src.rpm
+XS8-normal-src/xha-25.0.0-1.xs8.src.rpm
+XS8-normal-src/xsconsole-11.0.8-1.xs8.src.rpm
+XS8-normal-src/xs-obsolete-packages-8-12.xs8.src.rpm
+XS8-normal-src/xs-opam-repo-6.87.0-1.xs8.src.rpm

scripts/repo_status/create_rebase_cards.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+
+# from icecream import ic
+from tabulate import tabulate
+
+import repoquery
+from lib import (
+    collect_data_xcpng,
+    collect_data_xs8,
+    get_xs8_rpm_updates,
+    read_package_status_metadata,
+    read_previous_packages,
+)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='count', default=0)
+args = parser.parse_args()
+
+loglevel = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}.get(args.verbose, logging.DEBUG)
+logging.basicConfig(format='[%(levelname)s] %(message)s', level=loglevel)
+
+PACKAGE_STATUS = read_package_status_metadata()
+PREVIOUS_PACKAGES = read_previous_packages('XS8-normal-src.txt')
+
+xcp_set = collect_data_xcpng()
+(xs8_srpms_set, xs8_rpms_sources_set) = collect_data_xs8()
+srpm_updates = get_xs8_rpm_updates()
+
+res = []
+for n in sorted(set(xs8_srpms_set.keys()) | xs8_rpms_sources_set.keys()):
+    if n in PACKAGE_STATUS and PACKAGE_STATUS[n].status == 'ignored':
+        logging.debug(f"ignoring {n}")
+        continue
+    xs8_srpms_evr = xs8_srpms_set.get(n)
+    xs8_rpms_sources_evr = xs8_rpms_sources_set.get(n)
+    if xs8_srpms_evr is not None and xs8_rpms_sources_evr is not None:
+        xs8_evr = max(xs8_srpms_evr, xs8_rpms_sources_evr)
+    else:
+        xs8_evr = xs8_srpms_evr or xs8_rpms_sources_evr
+    xcp_evr = xcp_set.get(n)
+    if f'{n}-{xs8_evr}.xs8' in PREVIOUS_PACKAGES:
+        logging.info(f"ignoring previous package {n}")
+        continue
+    xs8_update = srpm_updates.get(f'{n}-{xs8_evr}.xs8', '?')
+    # if xcp_evr is not None and xcp_evr < xs8_evr:
+    if xcp_evr is None:
+        if not repoquery.is_pristine_upstream(str(xs8_evr)):
+            res.append((xs8_update, n, xcp_evr, xs8_evr))
+    elif xcp_evr < xs8_evr:
+        res.append((xs8_update, n, xcp_evr, xs8_evr))
+res.sort()
+print(tabulate(res, headers=['xs8 update', 'SRPM', 'XCP-ng version', 'XS8 version']))

scripts/repo_status/dnf

@@ -0,0 +1,35 @@
+#! /usr/bin/env python3
+
+import logging
+import sys
+import tempfile
+
+import repoquery
+
+ARCH = "x86_64"
+XCP_VERSION = "8.3"
+
+# Use `dnf` on xs8 and xcpng8.3 repos
+def main() -> int:
+    logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO)
+
+    args = sys.argv[1:]
+
+    with (tempfile.NamedTemporaryFile() as dnfconf,
+          tempfile.TemporaryDirectory() as yumrepod):
+
+        repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod,
+                                        sections=['base', 'updates'],
+                                        bin_arch=ARCH,
+                                        version=XCP_VERSION)
+        repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod,
+                                      sections=['base', 'normal', 'earlyaccess'],
+                                      )
+        repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod)
+
+        print(repoquery.run_dnf(args, split=False))
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/repo_status/lib.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+import csv
+import gzip
+import logging
+import os
+import tempfile
+import xml.etree.ElementTree as ET
+from collections import namedtuple
+from typing import Iterable, Iterator
+from urllib.request import urlopen
+
+import rpm  # type: ignore
+
+import repoquery
+
+ARCH = "x86_64"
+XCP_VERSION = "8.3"
+
+class EVR:
+    def __init__(self, e: str, v: str, r: str):
+        self._evr = ('0' if e in [None, 'None'] else e, v, r)
+
+    def __eq__(self, other):
+        if isinstance(other, EVR):
+            return self._evr == other._evr
+        else:
+            return self._evr == other
+
+    def __gt__(self, other):
+        if isinstance(other, EVR):
+            return rpm.labelCompare(self._evr, other._evr) > 0  # type: ignore
+        else:
+            return self._evr > other
+
+    def __lt__(self, other):
+        return other > self
+
+    def __str__(self):
+        if self._evr[0] != '0':
+            return f'{self._evr[0]}:{self._evr[1]}-{self._evr[2]}'
+        else:
+            return f'{self._evr[1]}-{self._evr[2]}'
+
+# Filters an iterator of (n, e, v, r) for newest evr of each `n`.
+# Older versions are allowed to appear before the newer ones.
+def filter_best_evr(nevrs: Iterable[tuple[str, str, str, str]]) -> Iterator[tuple[str, str, str, str]]:
+    best: dict[str, tuple[str, str, str]] = {}
+    for (n, e, v, r) in nevrs:
+        if n not in best or rpm.labelCompare(best[n], (e, v, r)) < 0:  # type: ignore
+            best[n] = (e, v, r)
+            yield (n, e, v, r)
+        # else (e, v, r) is older than a previously-seen version, drop
+
+def collect_data_xcpng() -> dict[str, EVR]:
+    with (tempfile.NamedTemporaryFile() as dnfconf,
+          tempfile.TemporaryDirectory() as yumrepod):
+        repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod,
+                                        sections=['base', 'updates'],
+                                        bin_arch=None,
+                                        version=XCP_VERSION)
+        repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod)
+
+        xcp_nevr = {
+            n: EVR(e, v, r)
+            for (n, e, v, r)
+            in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xcpng{XCP_VERSION}")
+                               for nevr in repoquery.all_srpms())}
+
+    return xcp_nevr
+
+def collect_data_xs8():
+    with (tempfile.NamedTemporaryFile() as dnfconf,
+          tempfile.TemporaryDirectory() as yumrepod):
+
+        repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod,
+                                      sections=['base', 'normal'],
+                                      )
+        repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod)
+        logging.debug("fill cache with XS info")
+        repoquery.fill_srpm_binrpms_cache()
+
+        logging.debug("get all XS SRPMs")
+        xs8_srpms = {nevr for nevr in repoquery.all_srpms()}
+        xs8_rpms_sources = {nevr for nevr in repoquery.SRPM_BINRPMS_CACHE}
+
+        xs8_srpms_set = {n: EVR(e, v, r)
+                         for (n, e, v, r)
+                         in filter_best_evr(repoquery.rpm_parse_nevr(nevr, ".xs8")
+                                            for nevr in xs8_srpms)}
+        xs8_rpms_sources_set = {n: EVR(e, v, r)
+                                for (n, e, v, r)
+                                in filter_best_evr(repoquery.rpm_parse_nevr(nevr, ".xs8")
+                                                   for nevr in xs8_rpms_sources)}
+
+    return (xs8_srpms_set, xs8_rpms_sources_set)
+
+def read_package_status_metadata():
+    with open('package_status.csv', newline='') as csvfile:
+        csvreader = csv.reader(csvfile, delimiter=';', quotechar='|')
+        headers = next(csvreader)
+        assert headers == ["SRPM_name", "status", "comment"], f"unexpected headers {headers!r}"
+        PackageStatus = namedtuple("PackageStatus", headers[1:]) # type: ignore[misc]
+        return {row[0]: PackageStatus(*row[1:])
+                for row in csvreader}
+
+def read_previous_packages(path: str):
+    with open(path) as f:
+        return [repoquery.srpm_strip_src_rpm(os.path.basename(line.strip())) for line in f]
+
+def get_xs8_rpm_updates():
+    NS = {'repo': 'http://linux.duke.edu/metadata/repo'}
+    BASE_URL = 'http://repos/repos/XS8/normal/xs8p-normal'
+
+    # read the update info path from repomd.xml
+    with urlopen(f'{BASE_URL}/repodata/repomd.xml') as f:
+        repomd = f.read()
+    data = ET.fromstring(repomd).find("repo:data[@type='updateinfo']", NS)
+    assert data is not None
+    location = data.find('repo:location', NS)
+    assert location is not None
+    path = location.attrib['href']
+
+    # read the update info file
+    res = {}
+    with urlopen(f'{BASE_URL}/{path}') as cf, gzip.open(cf, 'rb') as f:
+        updateinfo = f.read()
+    updates = ET.fromstring(updateinfo).findall('update')
+    for update in updates:
+        update_id = update.find('id')
+        assert update_id is not None
+        update_id = update_id.text
+        pkglist = update.find('pkglist')
+        assert pkglist is not None
+        collection = pkglist.find('collection')
+        assert collection is not None
+        packages = collection.findall('package')
+        for package in packages:
+            evr = EVR(package.attrib['epoch'], package.attrib['version'], package.attrib['release'])
+            rpm = f'{package.attrib["name"]}-{evr}'
+            srpm = repoquery.rpm_source_package(rpm, default=rpm)
+            res[srpm] = update_id
+    return res