From 5772f36fa39a0c768aa1b0ae8acf3759d225884b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 19:17:29 -0500 Subject: [PATCH 01/14] hack/build-image: shorten some long lines The lines were causing flake8 to complain so if we want to use the tool to validate things we should make it happy. Signed-off-by: John Mulligan --- hack/build-image | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/build-image b/hack/build-image index 515f929..535530b 100755 --- a/hack/build-image +++ b/hack/build-image @@ -358,7 +358,8 @@ def _sambacc_prefix(cli): yield ( "RUN dnf install -y /usr/bin/createrepo_c" f" && createrepo_c {distpath}" - f" && echo -e '[sambacc]\\nbaseurl=file://{distpath}\\nenabled=1\\ngpgcheck=0\\n'" + " && echo -e '[sambacc]\\n" + f"baseurl=file://{distpath}\\nenabled=1\\ngpgcheck=0\\n'" f" > {distpath}/sambacc.repo" ) yield "# --- sambacc prefix ---" @@ -372,7 +373,8 @@ def _sambacc_install(cli): yield "# --- begin modified sambacc install ---" yield ( "RUN" - f" --mount=type=bind,from=sccbuilder,source={distpath},destination={distpath}" + f" --mount=type=bind,from=sccbuilder,source={distpath}," + f"destination={distpath}" f" bash -x /usr/local/bin/install-sambacc.sh {distpath}" " ${SAMBACC_VERSION_SUFFIX}" ) From c5b0a82aba35491c27ee4444142a3581446d7be3 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 19:18:22 -0500 Subject: [PATCH 02/14] makefile: add a new check rule for the python script Avoid syntax errors and style/formatting drift by adding a very basic rule for flake8 and pyflakes to run in our CI. Signed-off-by: John Mulligan --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ee071e6..385a123 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ test-nightly-server: $(BUILDFILE_NIGHTLY_SERVER) ### Check Rules: static checks, quality tools ### -check: check-shell-scripts check-yaml +check: check-shell-scripts check-yaml check-python .PHONY: check # rule requires shellcheck and find to run check-shell-scripts: $(filter $(ALT_BIN)%,$(SHELLCHECK)) @@ -177,6 +177,15 @@ check-gitlint: $(filter $(ALT_BIN)%,$(GITLINT)) $(GITLINT) -C .gitlint --commits origin/master.. lint .PHONY: check-gitlint +check-python: _py + _py/bin/flake8 hack/build-image + _py/bin/black --check -l78 hack/build-image +.PHONY: check-python + +_py: + python3 -m venv _py + _py/bin/pip install flake8 black + ### Misc. Rules ### From bf533ee0470ebe8761cfcebe23af093ec2929fe1 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 19:30:42 -0500 Subject: [PATCH 03/14] hack/build-image: prepare for more complex actions In preparation for being able to create indexes (aka manifests) which can combine one or more "true" images the main function being purely per-target will become limiting. Create a new helper class structure that handles per target and then global actions with a simple shim class for adapting the legacy functions without needing to change them. Signed-off-by: John Mulligan --- hack/build-image | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/hack/build-image b/hack/build-image index 535530b..8f9376c 100755 --- a/hack/build-image +++ b/hack/build-image @@ -705,6 +705,32 @@ def print_tags(cli, target): print(f"{prefix}{name}") +class Builder: + def generate(self, cli): + raise NotImplementedError() + + def for_target(self, cli, target): + raise NotImplementedError() + + def for_all(self, cli): + raise NotImplementedError() + + +class SimpleBuilder(Builder): + def __init__(self, target_fn, generate_fn=None): + self.target_fn = target_fn + self.generate_fn = generate_fn or generate_images + + def generate(self, cli): + return self.generate_fn(cli) + + def for_target(self, cli, target): + return self.target_fn(cli, target) + + def for_all(self, cli): + pass + + class CLIContext: def __init__(self, cli): self._cli = cli @@ -938,14 +964,23 @@ def main(): cli.log_level = logging.DEBUG logging.basicConfig(level=cli.log_level) - _action = cli.main_action if cli.main_action else build + if not cli.main_action: + logger.debug("Using default action") + builder = SimpleBuilder(build) + elif hasattr(cli.main_action, "generate"): + logger.debug("Using builder object: %r", cli.main_action) + builder = cli.main_action() + else: + logger.debug("Legacy mode: using simple builder wrapper") + builder = SimpleBuilder(cli.main_action) imgs = [] try: - imgs = generate_images(cli) + imgs = builder.generate(cli) for img in imgs: add_special_tags(img, cli.distro_qualified) logger.info("Image %s, extra tags: %s", img, img.additional_tags) - _action(cli, img) + builder.for_target(cli, img) + builder.for_all(cli) except subprocess.CalledProcessError as err: logger.error("Failed command: %s", _cmd_to_str(err.cmd)) sys.exit(err.returncode) From bb7db4e6fc123c096ccf55f2ee0507e86047b0a4 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 19:51:41 -0500 Subject: [PATCH 04/14] hack/build-image: add a new --index action that creates indexes Add a new --index action that creates indexes/manifests based on the inputted targets. For now the indexes abstract away the architecture. To preserve the FQIN pattern our indexes and indexes alone will use 'any' in the 3rd position of the FQIN tag. This will allow consumers to eventually run something like `podman pull quay.io/samba.org/samba-server:default-fedora-any` and it will automatically select the correct image for the architecture if said arch image exists. This does NOT change the existing per-arch FQIN for "real" images. My experiments with manifest commands with podman have shown some quirky behavior so you will see a more belt-and-suspenders approach to some of these commands. I have only tested it on podman because I don't care about docker. Signed-off-by: John Mulligan --- hack/build-image | 164 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/hack/build-image b/hack/build-image index 8f9376c..568c876 100755 --- a/hack/build-image +++ b/hack/build-image @@ -38,6 +38,7 @@ Usage: import argparse import configparser import contextlib +import json import logging import os import pathlib @@ -86,6 +87,7 @@ ARCHITECTURES = { "amd64": AMD64, "arm64": ARM64, } +ANY_ARCH = "any" # DISTROS - list of supported distro bases FEDORA = "fedora" @@ -406,6 +408,42 @@ def container_push(cli, push_name): run(cli, args, check=True) +def manifest_inspect(cli, name): + """Return manifest information.""" + eng = container_engine(cli) + args = [eng, "manifest", "inspect", name] + res = run(cli, args, check=True, capture_output=True) + index_info = json.loads(res.stdout) + return index_info + + +def manifest_create(cli, name): + """Create a manifest.""" + eng = container_engine(cli) + args = [eng, "manifest", "create", name] + return run(cli, args, check=True) + + +def manifest_add( + cli, manifest_name, name, *, annotations=None, containers_storage=True +): + """Add an item to a manifest.""" + eng = container_engine(cli) + args = [eng, "manifest", "add"] + # annotations are not used for much but it could become interesting or + # useful in the future + for key, value in (annotations or {}).items(): + args.append(f"--annotation={key}={value}") + + args.append(manifest_name) + # this works around a very weird issue I saw where the image ids + # were changing after push. + if containers_storage: + name = f"containers-storage:{name}" + args.append(name) + return run(cli, args, check=True) + + def container_id(cli, target): """Construct and run a command to fetch a hexidecimal id for a container image. @@ -516,6 +554,17 @@ class TargetImage: continue yield self.image_name(tag=tag, repo_base="") + def as_index(self): + """Return a new target index object based on this image.""" + return TargetIndex( + name=self.name, + pkg_source=self.pkg_source, + distro=self.distro, + extra_tag=self.extra_tag, + repo_base=self.repo_base, + include=self, + ) + @classmethod def parse(cls, image_name): if "/" in image_name: @@ -537,6 +586,48 @@ class TargetImage: ) +class TargetIndex(TargetImage): + def __init__( + self, + name, + pkg_source, + distro, + *, + extra_tag="", + repo_base=None, + include=None, + ): + self.name = name + self.pkg_source = pkg_source + self.distro = distro + self.arch = ANY_ARCH + self.extra_tag = extra_tag + self.repo_base = repo_base + self._images = [] + if isinstance(include, list): + self._images.extend(include) + elif include: + self._images.append(include) + + def images(self): + return self._images + + def merge(self, other): + if other is None: + return self + assert self.name == other.name + assert self.pkg_source == other.pkg_source + assert self.distro == other.distro + return self.__class__( + name=self.name, + pkg_source=self.pkg_source, + distro=self.distro, + extra_tag=self.extra_tag, + repo_base=self.repo_base, + include=self._images + other.images(), + ) + + def generate_images(cli): """Given full image names or a matrix of kind/pkg_source/distro_base/arch values generate a list of target images to build/process. @@ -731,6 +822,72 @@ class SimpleBuilder(Builder): pass +class BasicBuilder(Builder): + def generate(self, cli): + return generate_images(cli) + + def for_all(self, cli): + pass + + +class IndexBuilder(BasicBuilder): + def __init__(self): + self.indexes = {} + + def for_target(self, cli, target): + idxtarget = target.as_index() + logger.info("Target index: %s", idxtarget) + key = str(idxtarget) + self.indexes[key] = idxtarget.merge(self.indexes.get(key)) + + def for_all(self, cli): + for idxtarget in self.indexes.values(): + logger.info("Creating index: %s", idxtarget) + self._clear(cli, idxtarget) + self._build(cli, idxtarget) + count = self._verify(cli, idxtarget) + logger.info("Created %s with %d manifests", idxtarget, count) + + def _clear(self, cli, idxtarget): + eng = container_engine(cli) + args = [eng, "manifest", "rm", idxtarget.image_name()] + res = run(cli, args, capture_output=True, check=False) + if res.returncode == 0: + logger.info("Removed previous manifest %s", idxtarget) + else: + logger.debug("Unable to remove previous manifest %s", idxtarget) + + def _build(self, cli, idxtarget): + manifest_create(cli, idxtarget.image_name()) + pfx = "org.samba.samba-container" + for img in idxtarget.images(): + manifest_add( + cli, + idxtarget.image_name(), + img.image_name(), + annotations={ + f"{pfx}.pkg-source": img.pkg_source, + f"{pfx}.distro": img.distro, + }, + ) + + def _verify(self, cli, idxtarget): + index_info = manifest_inspect(cli, idxtarget.image_name()) + img_count = len(index_info["manifests"]) + expecting = len(idxtarget.images()) + # I saw some weird stuff while developing this and so this + # check tries to alert on some of those conditions. + if img_count != expecting: + logger.error( + "confused manifest: saw %d entries, expected %d (%r)", + img_count, + expecting, + idxtarget.images(), + ) + raise ValueError("unexpected number of manifest entries") + return img_count + + class CLIContext: def __init__(self, cli): self._cli = cli @@ -958,6 +1115,13 @@ def main(): " for a given FQIN. Requires FQIN to already exist locally." ), ) + behaviors.add_argument( + "--index", + action="store_const", + dest="main_action", + const=IndexBuilder, + help="Create indexes (aka manifests) for named images", + ) cli = CLIContext(parser.parse_args()) if os.environ.get("BUILD_IMAGE_DEBUG") in ("1", "yes"): From c02d8688888caf644a2c531260a9b485fb70ac0b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sun, 14 Dec 2025 09:18:54 -0500 Subject: [PATCH 05/14] hack/build-image: refactor the push functions in prep for index push Swap the push function to a new Pusher class to make use of the new "main function" handling in preparation for extending push to also push indexes/manifests. Signed-off-by: John Mulligan --- hack/build-image | 53 +++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/hack/build-image b/hack/build-image index 568c876..d455e09 100755 --- a/hack/build-image +++ b/hack/build-image @@ -734,30 +734,6 @@ class QMatcher: return False -def push(cli, target): - """Command to push images.""" - if cli.push_state == "rebuild": - build(cli, target) - if cli.push_state == "exists": - try: - container_id(cli, target) - except subprocess.CalledProcessError: - build(cli, target) - - to_push = [] - push_name = target.image_name() - for tag, qual in target.additional_tags: - if tag in ("latest", "nightly"): - to_push.append((target.image_name(tag=tag), qual)) - if tag.endswith(("-latest", "-nightly")): - to_push.append((target.image_name(tag=tag), qual)) - to_push.append((push_name, QUAL_FQIN)) - qmatcher = cli.push_selected_tags or QMatcher("") - for push_name, tag_qual in to_push: - if qmatcher(tag_qual): - container_push(cli, push_name) - - def retag(cli, target): """Command to regenerate any missing unqualified tags.""" cid = container_id(cli, target) @@ -888,6 +864,33 @@ class IndexBuilder(BasicBuilder): return img_count +class Pusher(BasicBuilder): + def __init__(self): + self._to_push = [] + + def for_target(self, cli, target): + if cli.push_state == "rebuild": + build(cli, target) + if cli.push_state == "exists": + try: + container_id(cli, target) + except subprocess.CalledProcessError: + build(cli, target) + push_name = target.image_name() + for tag, qual in target.additional_tags: + if tag in ("latest", "nightly"): + self._to_push.append((target.image_name(tag=tag), qual)) + if tag.endswith(("-latest", "-nightly")): + self._to_push.append((target.image_name(tag=tag), qual)) + self._to_push.append((push_name, QUAL_FQIN)) + + def for_all(self, cli): + qmatcher = cli.push_selected_tags or QMatcher("") + for push_name, tag_qual in self._to_push: + if qmatcher(tag_qual): + container_push(cli, push_name) + + class CLIContext: def __init__(self, cli): self._cli = cli @@ -1081,7 +1084,7 @@ def main(): "--push", action="store_const", dest="main_action", - const=push, + const=Pusher, help="Push images", ) behaviors.add_argument( From a09cd2f51ee648f4ca03d4114bffdd91f1be28de Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sun, 14 Dec 2025 11:51:53 -0500 Subject: [PATCH 06/14] hack/build-image: enhance QMatcher class Enhance the QMatcher class to help be a more generic "selector" for what to push. This is in preparation for pushing indexes. Signed-off-by: John Mulligan --- hack/build-image | 82 +++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/hack/build-image b/hack/build-image index d455e09..c4ed6fe 100755 --- a/hack/build-image +++ b/hack/build-image @@ -689,41 +689,55 @@ def build(cli, target): class QMatcher: """Push only tags that meet the specified criteria: - all - all tags; - unqualified - only unqualified tags (eg. latest); - distro - only distribution base qualifed tags (eg. fedora-latest); - fqin - only fully qualified tags (eg. default-centos-amd64); - mixed - only fqin and unqualified tags; - least-qualified (default) - exactly one tag, with the least - number of qualifications + all: + all tags; + unqualified: + only unqualified tags (eg. latest); + distro/distro-qualified: + only distribution base qualifed tags (eg. fedora-latest); + fqin: + only fully qualified tags (eg. default-centos-amd64); + mixed: + only fqin and unqualified tags; + least-qualified (default): + exactly one tag, with the least number of qualifications; """ def __init__(self, key): - self.qualifications = [] + self.qualifications = set() self.count = 0 self.max_matches = 0 + if not key: + self.enable("") + return + kinds = key if isinstance(key, list) else key.split(",") + for kind in kinds: + self.enable(kind) + + def enable(self, key): + if not key: + key = "" + aliases = { + "least-qualified": [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN], + "": [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN], + "all": [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN], + "mixed": [QUAL_NONE, QUAL_FQIN], + QUAL_NONE: [QUAL_NONE], + QUAL_DISTRO: [QUAL_DISTRO], + "distro": [QUAL_DISTRO], + QUAL_FQIN: [QUAL_FQIN], + } + try: + quals = aliases[key] + except KeyError: + vals = ", ".join(aliases.keys()) + raise argparse.ArgumentTypeError( + f"value must be one of: {vals} - not {key!r}" + ) + self.qualifications.update(quals) if not key or key == "least-qualified": - self.qualifications = [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN] self.max_matches = 1 - elif key == "all": - pass - elif key == "mixed": - self.qualifications = [QUAL_NONE, QUAL_FQIN] - else: - try: - mq = { - "unqualified": QUAL_NONE, - "distro": QUAL_DISTRO, - "fqin": QUAL_FQIN, - }[key] - except KeyError: - raise argparse.ArgumentTypeError( - "value must be one of:" - " all, least-qualified, unqualified, distro, fqin;" - f" not {key}" - ) - self.qualifications = [mq] def __call__(self, qv): if self.max_matches and self.count >= self.max_matches: @@ -866,7 +880,7 @@ class IndexBuilder(BasicBuilder): class Pusher(BasicBuilder): def __init__(self): - self._to_push = [] + self._to_push = set() def for_target(self, cli, target): if cli.push_state == "rebuild": @@ -879,14 +893,14 @@ class Pusher(BasicBuilder): push_name = target.image_name() for tag, qual in target.additional_tags: if tag in ("latest", "nightly"): - self._to_push.append((target.image_name(tag=tag), qual)) + self._to_push.add((target.image_name(tag=tag), qual)) if tag.endswith(("-latest", "-nightly")): - self._to_push.append((target.image_name(tag=tag), qual)) - self._to_push.append((push_name, QUAL_FQIN)) + self._to_push.add((target.image_name(tag=tag), qual)) + self._to_push.add((push_name, QUAL_FQIN)) def for_all(self, cli): - qmatcher = cli.push_selected_tags or QMatcher("") - for push_name, tag_qual in self._to_push: + qmatcher = cli.push_kinds or QMatcher("") + for push_name, tag_qual in sorted(self._to_push): if qmatcher(tag_qual): container_push(cli, push_name) @@ -1040,6 +1054,8 @@ def main(): ), ) parser.add_argument( + "--push-kinds", + "--push-what", "--push-selected-tags", type=QMatcher, help=QMatcher.__doc__, From 9a73335a5401b369bd0d63fcb8d8ee9cacea9b23 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sun, 14 Dec 2025 12:21:26 -0500 Subject: [PATCH 07/14] hack/build-image: add support to push indexes Add support to push indexes/manifests along with "real" images. This can be invoked by specifying either "index" or "index-multiarch" to the --push-kinds/--push-what option. For example: `--push-kinds=mixed,index-multiarch`. Both options enable pushing indexes (aka manifests) but "index-multiarch" will only push those that refer to 2 or more "real" images - implying images that actually have support for multiple architectures. Signed-off-by: John Mulligan --- hack/build-image | 76 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/hack/build-image b/hack/build-image index c4ed6fe..6f03ad4 100755 --- a/hack/build-image +++ b/hack/build-image @@ -130,6 +130,7 @@ LATEST = "latest" QUAL_NONE = "unqualified" QUAL_DISTRO = "distro-qualified" QUAL_FQIN = "fqin" +QUAL_INDEX = "index" _DISCOVERED_CONTAINER_ENGINES = [] @@ -404,7 +405,24 @@ def _scan_containerfile(path, tags="sambacc"): def container_push(cli, push_name): """Construct and execute a command to push a container image.""" - args = [container_engine(cli), "push", push_name] + args = [container_engine(cli), "push"] + if cli.push_format: + args.append(f"-f{cli.push_format}") + args.append(push_name) + run(cli, args, check=True) + + +def manifest_push(cli, push_name): + """Construct and execute a command to push a manifest/index.""" + args = [ + container_engine(cli), + "manifest", + "push", + ] + if cli.push_format: + args.append(f"-f{cli.push_format}") + args.append("--all") + args.append(push_name) run(cli, args, check=True) @@ -701,12 +719,17 @@ class QMatcher: only fqin and unqualified tags; least-qualified (default): exactly one tag, with the least number of qualifications; + index: + Index/manifest "images"; + index-multiarch: + Index/manifest "images" that represent more than one architecture; """ def __init__(self, key): self.qualifications = set() self.count = 0 self.max_matches = 0 + self.minimum_index_images = 1 if not key: self.enable("") @@ -718,6 +741,7 @@ class QMatcher: def enable(self, key): if not key: key = "" + _multiarch = f"{QUAL_INDEX}-multiarch" aliases = { "least-qualified": [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN], "": [QUAL_NONE, QUAL_DISTRO, QUAL_FQIN], @@ -727,6 +751,8 @@ class QMatcher: QUAL_DISTRO: [QUAL_DISTRO], "distro": [QUAL_DISTRO], QUAL_FQIN: [QUAL_FQIN], + QUAL_INDEX: [QUAL_INDEX], + _multiarch: [QUAL_INDEX], } try: quals = aliases[key] @@ -738,6 +764,8 @@ class QMatcher: self.qualifications.update(quals) if not key or key == "least-qualified": self.max_matches = 1 + if key == _multiarch: + self.minimum_index_images = 2 def __call__(self, qv): if self.max_matches and self.count >= self.max_matches: @@ -878,6 +906,18 @@ class IndexBuilder(BasicBuilder): return img_count +def _push_weight(value): + push_name, tag_qual = value + # do all "real" fqin images first + if tag_qual == QUAL_FQIN: + return 0, push_name + # do indexes last + if tag_qual == QUAL_INDEX: + return 100, push_name + # do non-fqins after fqins + return 10, push_name + + class Pusher(BasicBuilder): def __init__(self): self._to_push = set() @@ -897,13 +937,38 @@ class Pusher(BasicBuilder): if tag.endswith(("-latest", "-nightly")): self._to_push.add((target.image_name(tag=tag), qual)) self._to_push.add((push_name, QUAL_FQIN)) + qmatcher = cli.push_kinds or QMatcher("") + if QUAL_INDEX in qmatcher.qualifications: + self._to_push.add((target.as_index().image_name(), QUAL_INDEX)) def for_all(self, cli): qmatcher = cli.push_kinds or QMatcher("") - for push_name, tag_qual in sorted(self._to_push): - if qmatcher(tag_qual): + for push_name, tag_qual in sorted(self._to_push, key=_push_weight): + if not qmatcher(tag_qual): + logger.debug("Excluding %s from push", push_name) + continue + if tag_qual == QUAL_INDEX: + self._push_index(cli, qmatcher, push_name) + else: container_push(cli, push_name) + def _push_index(self, cli, qmatcher, push_name): + try: + index_info = manifest_inspect(cli, push_name) + except subprocess.CalledProcessError: + logger.warning("No manifest present for %s, skipping", push_name) + return + img_count = len(index_info["manifests"]) + logger.debug("Manifest/index %s has %d images", push_name, img_count) + if img_count >= qmatcher.minimum_index_images: + manifest_push(cli, push_name) + else: + logger.warning( + "Skipping manifest push for %s, too few images (%d)", + push_name, + img_count, + ) + class CLIContext: def __init__(self, cli): @@ -1060,6 +1125,11 @@ def main(): type=QMatcher, help=QMatcher.__doc__, ) + parser.add_argument( + "--push-format", + choices=("oci", "v2s2"), + help="Manifest type to use when pushing", + ) parser.add_argument( "--buildfile-prefix", default=".build.", From 11458f005c55aa97ec95d0ec732124a622beb09c Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 12 Dec 2025 12:34:05 -0500 Subject: [PATCH 08/14] workflows: standardize on podman where possible Use podman in our CI because it matches what we (I?) use to test the scripts and what I use to release images. I'm not against having things work with docker but I don't like having to constantly second guess what I am doing when working on these features. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 03ce9ca..5f7588a 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: {} env: - CONTAINER_CMD: docker + CONTAINER_CMD: podman jobs: checks: runs-on: ubuntu-latest From 236e58aae7506c4682744015b35062b633bd9fea Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 12 Dec 2025 12:54:56 -0500 Subject: [PATCH 09/14] workflows: use arm builders for arm images GitHub has supported native arm64 for almost a year, but that had somehow flown under my radar until recently. Tweak the matrix to choose those builders when building for arm64 instead of relying on emulation. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 40 +++++++++++++++------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 5f7588a..f6b8791 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -45,28 +45,28 @@ jobs: matrix: package_source: [default, nightly] os: [centos, fedora, opensuse] - arch: [amd64, arm64] + arch: + - {name: amd64, host: ubuntu-24.04} + - {name: arm64, host: ubuntu-24.04-arm} exclude: # there are no nightly packages for opensuse - package_source: nightly os: opensuse - - os: centos - arch: arm64 include: - package_source: devbuilds os: centos - arch: amd64 + arch: {name: amd64, host: ubuntu-24.04} - package_source: ceph20 os: centos - arch: amd64 - runs-on: ubuntu-latest + arch: {name: amd64, host: ubuntu-24.04} + runs-on: ${{ matrix.arch.host }} env: BUILDAH_FORMAT: oci - IMG_TAG: ${{ matrix.package_source }}-${{ matrix.os }}-${{ matrix.arch }} + IMG_TAG: ${{ matrix.package_source }}-${{ matrix.os }}-${{ matrix.arch.name }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - name: Build the server image - run: make KIND=server PACKAGE_SOURCE=${{ matrix.package_source }} OS_NAME=${{ matrix.os}} BUILD_ARCH=${{ matrix.arch}} build-image + run: make KIND=server PACKAGE_SOURCE=${{ matrix.package_source }} OS_NAME=${{ matrix.os }} BUILD_ARCH=${{ matrix.arch.name }} build-image - name: Save image run: > ${{ env.CONTAINER_CMD }} save @@ -84,7 +84,9 @@ jobs: matrix: package_source: [default, nightly] os: [centos, fedora, opensuse] - arch: [amd64, arm64] + arch: + - {name: amd64, host: ubuntu-24.04} + - {name: arm64, host: ubuntu-24.04-arm} exclude: # there are no nightly packages for opensuse - package_source: nightly @@ -93,15 +95,15 @@ jobs: - package_source: default os: centos - os: centos - arch: arm64 - runs-on: ubuntu-latest + arch: {name: arm64, host: ubuntu-24.04-arm} + runs-on: ${{ matrix.arch.host }} env: BUILDAH_FORMAT: oci - IMG_TAG: ${{ matrix.package_source }}-${{ matrix.os }}-${{ matrix.arch }} + IMG_TAG: ${{ matrix.package_source }}-${{ matrix.os }}-${{ matrix.arch.name }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - name: Build the ad server image - run: make KIND=ad-server PACKAGE_SOURCE=${{ matrix.package_source }} OS_NAME=${{ matrix.os }} BUILD_ARCH=${{ matrix.arch }} build-image + run: make KIND=ad-server PACKAGE_SOURCE=${{ matrix.package_source }} OS_NAME=${{ matrix.os }} BUILD_ARCH=${{ matrix.arch.name }} build-image - name: Save image run: > ${{ env.CONTAINER_CMD }} save @@ -118,18 +120,20 @@ jobs: strategy: matrix: os: [centos, fedora, opensuse] - arch: [amd64, arm64] + arch: + - {name: amd64, host: ubuntu-24.04} + - {name: arm64, host: ubuntu-24.04-arm} exclude: - os: centos - arch: arm64 - runs-on: ubuntu-latest + arch: {name: arm64, host: ubuntu-24.04-arm} + runs-on: ${{ matrix.arch.host }} env: BUILDAH_FORMAT: oci - IMG_TAG: default-${{ matrix.os }}-${{ matrix.arch }} + IMG_TAG: default-${{ matrix.os }}-${{ matrix.arch.name}} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - name: build the client image - run: make KIND=client OS_NAME=${{ matrix.os }} BUILD_ARCH=${{ matrix.arch }} build-image + run: make KIND=client OS_NAME=${{ matrix.os }} BUILD_ARCH=${{ matrix.arch.name }} build-image # The client image is used as a base for the samba-toolbox build process. - name: Save image run: > From dd2f34bd1aec030c5acd633645c7b058b201a39a Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 12 Dec 2025 12:33:29 -0500 Subject: [PATCH 10/14] workflows: create and push indexes Indexes, aka manifests, make it easier for clients to specify a tag name that will automatically select between architectures for you when choosing an image from a registry. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index f6b8791..6475dd2 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -329,6 +329,26 @@ jobs: -i samba-client:default-fedora-arm64 -i samba-client:default-opensuse-arm64 -i samba-toolbox:default-fedora-amd64 + - name: Create manifests/indexes + run: > + ./hack/build-image + --index + --container-engine=${CONTAINER_CMD} + -i ${REPO_BASE}/samba-server:default-fedora-amd64 + -i ${REPO_BASE}/samba-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-server:default-opensuse-arm64 + -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 + -i ${REPO_BASE}/samba-server:nightly-centos-amd64 + -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 + -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 + -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 + -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-ad-server:default-opensuse-arm64 + -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 + -i ${REPO_BASE}/samba-client:default-fedora-amd64 + -i ${REPO_BASE}/samba-client:default-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-opensuse-arm64 + -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 - name: List images prior to pushing (debug) run: | ${{ env.CONTAINER_CMD }} images @@ -339,7 +359,8 @@ jobs: --container-engine=${CONTAINER_CMD} --verbose --push-state=exists - --push-selected-tags=mixed + --push-kinds=mixed,index-multiarch + --push-format=oci -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 -i ${REPO_BASE}/samba-server:default-opensuse-arm64 From baffbf8228e02ce29e068669177bdc331122c928 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 14:59:35 -0500 Subject: [PATCH 11/14] workflows: don't publish opensuse imgs We test the opensuse images but we hadn't been pushing them before but somehow some of the arm64 (and only arm64!) images got added to the list of images to publish to our quay.io registry. Stop that. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 6475dd2..fb7de19 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -314,7 +314,6 @@ jobs: --no-distro-qualified -i samba-server:default-fedora-amd64 -i samba-server:default-fedora-arm64 - -i samba-server:default-opensuse-arm64 -i samba-server:nightly-fedora-amd64 -i samba-server:nightly-fedora-arm64 -i samba-server:nightly-centos-amd64 @@ -322,12 +321,10 @@ jobs: -i samba-server:ceph20-centos-amd64 -i samba-ad-server:default-fedora-amd64 -i samba-ad-server:default-fedora-arm64 - -i samba-ad-server:default-opensuse-arm64 -i samba-ad-server:nightly-fedora-amd64 -i samba-ad-server:nightly-fedora-arm64 -i samba-client:default-fedora-amd64 -i samba-client:default-fedora-arm64 - -i samba-client:default-opensuse-arm64 -i samba-toolbox:default-fedora-amd64 - name: Create manifests/indexes run: > @@ -336,18 +333,15 @@ jobs: --container-engine=${CONTAINER_CMD} -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-ad-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 - -i ${REPO_BASE}/samba-client:default-opensuse-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 - name: List images prior to pushing (debug) run: | @@ -363,16 +357,13 @@ jobs: --push-format=oci -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-ad-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 - -i ${REPO_BASE}/samba-client:default-opensuse-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 From f8763986adb2333f10603c90cac9ff9d3d43fae4 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 13 Dec 2025 15:13:21 -0500 Subject: [PATCH 12/14] workflows: reorganize the images list a bit Try to make the list of images that we publish a bit more obvious by sorting the lists. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 45 ++++++++++++++++++++------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index fb7de19..2dad921 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -312,18 +312,23 @@ jobs: --container-engine=${CONTAINER_CMD} --repo-base=${REPO_BASE} --no-distro-qualified + -i samba-server:ceph20-centos-amd64 + -i samba-server:default-centos-amd64 + -i samba-server:default-centos-arm64 -i samba-server:default-fedora-amd64 -i samba-server:default-fedora-arm64 + -i samba-server:devbuilds-centos-amd64 + -i samba-server:nightly-centos-amd64 + -i samba-server:nightly-centos-arm64 -i samba-server:nightly-fedora-amd64 -i samba-server:nightly-fedora-arm64 - -i samba-server:nightly-centos-amd64 - -i samba-server:devbuilds-centos-amd64 - -i samba-server:ceph20-centos-amd64 -i samba-ad-server:default-fedora-amd64 -i samba-ad-server:default-fedora-arm64 + -i samba-ad-server:nightly-centos-amd64 -i samba-ad-server:nightly-fedora-amd64 -i samba-ad-server:nightly-fedora-arm64 - -i samba-client:default-fedora-amd64 + -i samba-client:default-centos-amd64 + -i samba-client:default-fedora-arm64 -i samba-client:default-fedora-arm64 -i samba-toolbox:default-fedora-amd64 - name: Create manifests/indexes @@ -331,18 +336,26 @@ jobs: ./hack/build-image --index --container-engine=${CONTAINER_CMD} + -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 + -i ${REPO_BASE}/samba-server:default-centos-amd64 + -i ${REPO_BASE}/samba-server:default-centos-arm64 -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 - -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 - -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 + -i ${REPO_BASE}/samba-server:nightly-centos-amd64 + -i ${REPO_BASE}/samba-server:nightly-centos-arm64 + -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 + -i ${REPO_BASE}/samba-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-ad-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 - -i ${REPO_BASE}/samba-client:default-fedora-amd64 + -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-centos-amd64 + -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 + # Please keep the above list of arguments SORTED. - name: List images prior to pushing (debug) run: | ${{ env.CONTAINER_CMD }} images @@ -355,15 +368,23 @@ jobs: --push-state=exists --push-kinds=mixed,index-multiarch --push-format=oci + -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 + -i ${REPO_BASE}/samba-server:default-centos-amd64 + -i ${REPO_BASE}/samba-server:default-centos-arm64 -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 - -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 - -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 - -i ${REPO_BASE}/samba-server:ceph20-centos-amd64 + -i ${REPO_BASE}/samba-server:nightly-centos-amd64 + -i ${REPO_BASE}/samba-server:nightly-centos-arm64 + -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 + -i ${REPO_BASE}/samba-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-ad-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 - -i ${REPO_BASE}/samba-client:default-fedora-amd64 + -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-centos-amd64 + -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 + # Please keep the above list of arguments SORTED. From 3a0102cbffb892dbfadd28ebfd7d1460be7fdc98 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 17 Dec 2025 13:10:59 -0500 Subject: [PATCH 13/14] workflows: enable build for ad-server nightly centos arm64 It was excluded without needing to be. Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 2dad921..7bc5171 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -94,8 +94,6 @@ jobs: # the distro packages for centos do not include an ad-dc - package_source: default os: centos - - os: centos - arch: {name: arm64, host: ubuntu-24.04-arm} runs-on: ${{ matrix.arch.host }} env: BUILDAH_FORMAT: oci @@ -325,6 +323,7 @@ jobs: -i samba-ad-server:default-fedora-amd64 -i samba-ad-server:default-fedora-arm64 -i samba-ad-server:nightly-centos-amd64 + -i samba-ad-server:nightly-centos-arm64 -i samba-ad-server:nightly-fedora-amd64 -i samba-ad-server:nightly-fedora-arm64 -i samba-client:default-centos-amd64 @@ -349,6 +348,7 @@ jobs: -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-centos-amd64 + -i ${REPO_BASE}/samba-ad-server:nightly-centos-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-client:default-centos-amd64 @@ -381,6 +381,7 @@ jobs: -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-centos-amd64 + -i ${REPO_BASE}/samba-ad-server:nightly-centos-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-client:default-centos-amd64 From 0bd96c82ee2b948cbf4aeddc7ccf43ff3378e4d9 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 18 Dec 2025 10:21:17 -0500 Subject: [PATCH 14/14] workflows: make build matrix and pushed images more consistent Signed-off-by: John Mulligan --- .github/workflows/container-image.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 7bc5171..f117eb3 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -121,9 +121,6 @@ jobs: arch: - {name: amd64, host: ubuntu-24.04} - {name: arm64, host: ubuntu-24.04-arm} - exclude: - - os: centos - arch: {name: arm64, host: ubuntu-24.04-arm} runs-on: ${{ matrix.arch.host }} env: BUILDAH_FORMAT: oci @@ -327,7 +324,8 @@ jobs: -i samba-ad-server:nightly-fedora-amd64 -i samba-ad-server:nightly-fedora-arm64 -i samba-client:default-centos-amd64 - -i samba-client:default-fedora-arm64 + -i samba-client:default-centos-arm64 + -i samba-client:default-fedora-amd64 -i samba-client:default-fedora-arm64 -i samba-toolbox:default-fedora-amd64 - name: Create manifests/indexes @@ -352,7 +350,8 @@ jobs: -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-client:default-centos-amd64 - -i ${REPO_BASE}/samba-client:default-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-centos-arm64 + -i ${REPO_BASE}/samba-client:default-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 # Please keep the above list of arguments SORTED. @@ -385,7 +384,8 @@ jobs: -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-arm64 -i ${REPO_BASE}/samba-client:default-centos-amd64 - -i ${REPO_BASE}/samba-client:default-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-centos-arm64 + -i ${REPO_BASE}/samba-client:default-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 # Please keep the above list of arguments SORTED.