[do-not-merge] Virtio-mem rebase

.buildkite/common.py

@@ -16,9 +16,8 @@
 
 # fmt: off
 DEFAULT_INSTANCES = [
-    "c5n.metal",      # Intel Skylake
     "m5n.metal",      # Intel Cascade Lake
-    "m6i.metal",      # Intel Icelake
+    "m6i.metal",      # Intel Ice Lake
     "m7i.metal-24xl", # Intel Sapphire Rapids
     "m7i.metal-48xl", # Intel Sapphire Rapids
     "m6a.metal",      # AMD Milan
@@ -79,6 +78,10 @@ def group(label, command, instances, platforms, **kwargs):
     """
     # Use the 1st character of the group name (should be an emoji)
     label1 = label[0]
+    # if the emoji is in the form ":emoji:", pick the entire slug
+    if label.startswith(":") and ":" in label[1:]:
+        label1 = label[: label.index(":", 1) + 1]
+
     steps = []
     commands = command
     if isinstance(command, str):
@@ -272,11 +275,11 @@ def __init__(self, with_build_step=True, **kwargs):
         self.per_arch["instances"] = ["m6i.metal", "m7g.metal"]
         self.per_arch["platforms"] = [("al2023", "linux_6.1")]
         self.binary_dir = args.binary_dir
-        # Build sharing
-        if with_build_step:
+        # Build sharing, if a binary dir wasn't already supplied
+        if not args.binary_dir and with_build_step:
             build_cmds, self.shared_build = shared_build()
             self.build_group_per_arch(
-                "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+                "🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build
             )
         else:
             self.shared_build = None
@@ -314,9 +317,25 @@ def _adapt_group(self, group):
         for step in group["steps"]:
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
-                step["depends_on"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
-                )
+                if "depends_on" not in step:
+                    step["depends_on"] = []
+                elif isinstance(step["depends_on"], str):
+                    step["depends_on"] = [step["depends_on"]]
+                elif isinstance(step["depends_on"], list):
+                    pass
+                else:
+                    raise ValueError(
+                        f"depends_on should be a string or a list but is {type(step['depends_on'])}"
+                    )
+
+                step["depends_on"].append(self.shared_build)
+                step["depends_on"] = [
+                    self.build_key(
+                        dep, get_arch_for_instance(step["agents"]["instance"])
+                    )
+                    for dep in step["depends_on"]
+                ]
+
         return group
 
     def build_group(self, *args, **kwargs):
@@ -332,17 +351,17 @@ def build_group(self, *args, **kwargs):
             group(*args, **combined), depends_on_build=depends_on_build
         )
 
-    def build_key(self, arch):
+    def build_key(self, key, arch):
         """Return the Buildkite key for the build step, for the specified arch"""
-        return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
+        return key.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
     def build_group_per_arch(self, label, *args, **kwargs):
         """
         Build a group, parametrizing over the architectures only.
 
         kwargs consumed by this method and not passed down to `group`:
         - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
-        - `set_key`: If True, causes the generated steps to have a "key" field
+        - `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz
         """
         depends_on_build = kwargs.pop("depends_on_build", True)
         set_key = kwargs.pop("set_key", None)
@@ -351,7 +370,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
+                    set_key, get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_coverage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Generate Buildkite pipelines dynamically"""
+
+from common import BKPipeline
+
+pipeline = BKPipeline(with_build_step=False)
+
+pipeline.build_group(
+    ":coverage: Coverage",
+    pipeline.devtool_test(
+        devtool_opts="--no-build",
+        pytest_opts="integration_tests/build/test_coverage.py",
+    ),
+)
+print(pipeline.to_json())

.buildkite/pipeline_cpu_template.py

@@ -27,7 +27,6 @@ class BkStep(str, Enum):
         ],
         BkStep.LABEL: "📖 rdmsr",
         "instances": [
-            "c5n.metal",
             "m5n.metal",
             "m6i.metal",
             "m7i.metal-24xl",
@@ -63,12 +62,10 @@ class BkStep(str, Enum):
             BkStep.TIMEOUT: 30,
         },
         "cross_instances": {
-            "m5n.metal": ["c5n.metal", "m6i.metal"],
-            "c5n.metal": ["m5n.metal", "m6i.metal"],
-            "m6i.metal": ["m5n.metal", "c5n.metal"],
+            "m5n.metal": ["m6i.metal"],
+            "m6i.metal": ["m5n.metal"],
         },
         "instances": [
-            "c5n.metal",
             "m5n.metal",
             "m6i.metal",
             "m7i.metal-24xl",

.buildkite/pipeline_cross.py

@@ -19,7 +19,6 @@
     per_instance.pop("instances")
     per_instance.pop("platforms")
     instances_x86_64 = [
-        "c5n.metal",
         "m5n.metal",
         "m6i.metal",
         "m7i.metal-24xl",
@@ -49,9 +48,8 @@
     # allow-list of what instances can be restores on what other instances (in
     # addition to itself)
     supported = {
-        "c5n.metal": ["m5n.metal", "m6i.metal"],
-        "m5n.metal": ["c5n.metal", "m6i.metal"],
-        "m6i.metal": ["c5n.metal", "m5n.metal"],
+        "m5n.metal": ["m6i.metal"],
+        "m6i.metal": ["m5n.metal"],
     }
 
     # https://github.com/firecracker-microvm/firecracker/blob/main/docs/kernel-policy.md#experimental-snapshot-compatibility-across-kernel-versions
@@ -82,7 +80,6 @@
             continue
 
         pytest_keyword_for_instance = {
-            "c5n.metal": "-k 'not None'",
             "m5n.metal": "-k 'not None'",
             "m6i.metal": "-k 'not None'",
             "m6a.metal": "",

.buildkite/pipeline_docker_popular.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Buildkite pipeline for testing popular Docker containers
+"""
+
+from common import BKPipeline, random_str
+
+pipeline = BKPipeline()
+
+ROOTFS_TAR = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz"
+
+pipeline.build_group_per_arch(
+    ":ship: Rootfs build",
+    [
+        "sudo yum install -y systemd-container",
+        "cd tools/test-popular-containers",
+        "sudo ./build_rootfs.sh",
+        f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa',
+        f'buildkite-agent artifact upload "{ROOTFS_TAR}"',
+    ],
+    depends_on_build=False,
+    set_key=ROOTFS_TAR,
+)
+
+pipeline.build_group(
+    ":whale: Docker Popular Containers",
+    [
+        "./tools/devtool download_ci_artifacts",
+        f'buildkite-agent artifact download "{ROOTFS_TAR}" .',
+        f'tar xzf "{ROOTFS_TAR}" -C tools/test-popular-containers',
+        './tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"',
+    ],
+    depends_on=ROOTFS_TAR,
+)
+
+print(pipeline.to_json())

.buildkite/pipeline_release_qa.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Buildkite pipeline for release QA
+"""
+
+from common import BKPipeline
+
+pipeline = BKPipeline(with_build_step=False)
+
+# NOTE: we need to escape $ using $$ otherwise buildkite tries to replace it instead of the shell
+
+pipeline.add_step(
+    {
+        "label": "Download release",
+        "if": 'build.env("VERSION") != "dev"',
+        "command": [
+            "aws s3 sync --no-sign-request s3://spec.ccfc.min/firecracker-ci/firecracker/$$VERSION release-$$VERSION",
+            'buildkite-agent artifact upload "release-$$VERSION/**/*"',
+        ],
+    },
+    depends_on_build=False,
+)
+
+pipeline.build_group_per_arch(
+    ":building_construction: Make release",
+    # if is a keyword for python, so we need this workaround to expand it as a kwarg
+    **{"if": 'build.env("VERSION") == "dev"'},
+    command=[
+        "./tools/devtool -y make_release",
+        "RELEASE_DIR=$$(echo release-*dev-$$(uname -m))",
+        "RELEASE_SUFFIX=$${{RELEASE_DIR#release}}",
+        "OUT_DIR=release-$$VERSION/$$(uname -m)",
+        "mkdir -p $$OUT_DIR",
+        (
+            "for f in $$RELEASE_DIR/*-$$(uname -m); do"
+            "  mv $$f $$OUT_DIR/$$(basename $$f $$RELEASE_SUFFIX);"
+            "  mv $$f.debug $$OUT_DIR/$$(basename $$f $$RELEASE_SUFFIX).debug;"
+            "done"
+        ),
+        'buildkite-agent artifact upload "release-$$VERSION/**/*"',
+    ],
+    depends_on_build=False,
+)
+
+# The devtool expects the examples to be in the same folder as the binaries to run some tests
+# (for example, uffd handler tests). Build them and upload them in the same folder.
+pipeline.build_group_per_arch(
+    ":hammer_and_wrench: Build examples",
+    command=[
+        "CARGO_TARGET=$$(uname -m)-unknown-linux-musl",
+        "./tools/devtool -y sh cargo build --target $$CARGO_TARGET --release --examples",
+        "mkdir -p release-$$VERSION/$$(uname -m)/",
+        "cp -R build/cargo_target/$$CARGO_TARGET/release/examples release-$$VERSION/$$(uname -m)/",
+        'buildkite-agent artifact upload "release-$$VERSION/**/*"',
+    ],
+    depends_on_build=False,
+)
+
+pipeline.add_step("wait", depends_on_build=False)
+
+pipeline.add_step(
+    {
+        "label": ":pipeline: PR",
+        "command": (
+            ".buildkite/pipeline_pr.py --binary-dir release-$$VERSION "
+            "| jq '(..|select(.priority? != null).priority) += 100' "
+            "| buildkite-agent pipeline upload"
+        ),
+    },
+    depends_on_build=False,
+)
+
+print(pipeline.to_json())

.mailmap

@@ -33,3 +33,5 @@ Muskaan Singla <msinglaa@amazon.com> <ec2-user@ip-172-31-1-28.us-west-2.compute.
 Egor Lazarchuk <yegorlz@amazon.co.uk>
 Nikita Zakirov <zakironi@amazon.com> <ec2-user@ip-10-0-15-219.us-west-2.compute.internal>
 Tomoya Iwata <iwata.tomoya@classmethod.jp>
+Andrea Manzini <andrea.manzini@suse.com> <ilmanzo@gmail.com>
+Colin Percival <cperciva@freebsd.org> <cperciva@tarsnap.com>

CHANGELOG.md

@@ -18,6 +18,11 @@ and this project adheres to
 
 ### Fixed
 
+- [#5418](https://github.com/firecracker-microvm/firecracker/pull/5418): Fixed
+  typo in Swagger definition of `MmdsConfig`, where the property `imds_compat`
+  was spelled as `imds_comat`. This caused auto-generated clients to create bad
+  requests.
+
 ## [1.13.0]
 
 ### Added
@@ -53,6 +58,12 @@ and this project adheres to
   guest's serial console. Not configuring it means Firecracker will continue to
   print serial output to stdout. Similarly to the logger, this configuration is
   not persisted across snapshots.
+- [#5364](https://github.com/firecracker-microvm/firecracker/pull/5364): Added
+  PCI support in Firecracker. PCI support is optional. Users can enable it
+  passing the `--enable-pci` flag when launching the Firecracker process. When
+  Firecracker process is launched with PCI support, it will create all VirtIO
+  devices using a PCI VirtIO transport. If not enabled, Firecracker will use the
+  MMIO transport instead.
 
 ### Changed
 
@@ -78,6 +89,11 @@ and this project adheres to
 
 ### Removed
 
+- [#5411](https://github.com/firecracker-microvm/firecracker/pull/5411): Removed
+  official support for Intel Skylake instances. Firecracker will continue to
+  work on those instances, but we will no longer perform automated testing on
+  them.
+
 ### Fixed
 
 - [#5222](https://github.com/firecracker-microvm/firecracker/pull/5222): Fixed