[DO NOT MERGE]: AI on cloud engines, experimental

MODULE.bazel

@@ -306,6 +306,17 @@ http_archive(
     url = "https://github.com/apalache-mc/apalache/releases/download/v0.52.2/apalache-0.52.2.tgz",
 )
 
+# Ollama CPU-only runtime, shipped in the GuestOS. Only the `ollama` binary
+# and the CPU ggml shared libraries are kept; GPU runners (CUDA/MLX/Vulkan)
+# are stripped via the `build_file` filter. See BUILD.ollama.bazel.
+http_archive(
+    name = "ollama",
+    build_file = "@//third_party:BUILD.ollama.bazel",
+    sha256 = "528acc42750f755ce57b13b8138419ceac953b6fd488045f73dc1e80246da480",
+    type = "tar.zst",
+    url = "https://github.com/ollama/ollama/releases/download/v0.21.1/ollama-linux-amd64.tar.zst",
+)
+
 # Official WebAssembly test suite.
 # To be used for testing libraries that handle canister Wasm code.
 http_archive(

ic-os/components/guestos.bzl

@@ -50,6 +50,13 @@ def component_files(mode):
         Label("guestos/ic-https-outcalls-adapter/ic-https-outcalls-adapter.socket"): "/etc/systemd/system/ic-https-outcalls-adapter.socket",
         Label("guestos/ic-https-outcalls-adapter/generate-https-outcalls-adapter-config.sh"): "/opt/ic/bin/generate-https-outcalls-adapter-config.sh",
         Label("guestos/ic-replica.service"): "/etc/systemd/system/ic-replica.service",
+        # Ollama is intentionally disabled by default. The blanket
+        # `systemctl enable` loop in the Dockerfile would otherwise enable
+        # it; an explicit `systemctl disable ollama.service` in the
+        # Dockerfile keeps it off. Start it on demand with:
+        #   systemctl enable --now ollama.service
+        Label("guestos/ollama/ollama.service"): "/etc/systemd/system/ollama.service",
+        Label("guestos/ollama/manage-ollama.sh"): "/opt/ic/bin/manage-ollama.sh",
         Label("guestos/remote-attestation-server.service"): "/etc/systemd/system/remote-attestation-server.service",
         Label("guestos/generate-ic-config/generate-ic-config.service"): "/etc/systemd/system/generate-ic-config.service",
         Label("guestos/share/ic-boundary.env"): "/opt/ic/share/ic-boundary.env",

ic-os/components/guestos/misc/sudoers

@@ -29,6 +29,6 @@ root	ALL=(ALL:ALL) NOPASSWD:ALL
 # Allow members of group sudo to execute any command
 %sudo	ALL=(ALL:ALL) NOPASSWD:ALL
 
-ic-replica ALL=(ALL:ALL) NOPASSWD: /opt/ic/bin/manageboot.sh, /opt/ic/bin/provision-ssh-keys.sh, /opt/ic/bin/read-ssh-keys.sh, /opt/ic/bin/sync_fstrim.sh, /opt/ic/bin/guestos_tool, /usr/sbin/nft
+ic-replica ALL=(ALL:ALL) NOPASSWD: /opt/ic/bin/manageboot.sh, /opt/ic/bin/provision-ssh-keys.sh, /opt/ic/bin/read-ssh-keys.sh, /opt/ic/bin/sync_fstrim.sh, /opt/ic/bin/guestos_tool, /opt/ic/bin/manage-ollama.sh, /usr/sbin/nft
 
 # See sudoers(5) for more information on "#include" directives:

ic-os/components/guestos/ollama/manage-ollama.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+# Transparently switch uid to root in order to perform the privileged function.
+# SELinux restrictions and standard permissions still apply, the script and
+# the calling user are restricted to being allowed to sudo only this.
+if [ $(id -u) != 0 ]; then
+    exec sudo "$0" "$@"
+fi
+
+ACTION="$1"
+
+case "$ACTION" in
+    start)
+        /bin/systemctl start ollama.service
+        ;;
+    stop)
+        /bin/systemctl stop ollama.service
+        ;;
+    *)
+        echo "Usage: $0 {start|stop}" >&2
+        exit 2
+        ;;
+esac

ic-os/components/guestos/ollama/ollama.service

@@ -0,0 +1,48 @@
+[Unit]
+Description=Ollama LLM runtime (disabled by default)
+Documentation=https://ollama.com
+After=network-online.target
+Wants=network-online.target
+
+# This service is intentionally disabled by default. It ships with the
+# GuestOS image so that it can be started on demand with:
+#   systemctl enable --now ollama.service
+# See the explicit `systemctl disable ollama.service` in the GuestOS
+# Dockerfile which overrides the blanket enable loop.
+
+[Service]
+Type=simple
+User=ollama
+Group=ollama
+
+# systemd will create /var/lib/ollama (mode 0750, owned by ollama:ollama)
+# before ExecStart, and ensure it exists across upgrades. This avoids
+# relying on the rootfs-baked directory, which is not visible on the
+# separately-mounted /var filesystem.
+StateDirectory=ollama
+StateDirectoryMode=0750
+
+# OLLAMA_MODELS points at the baked-in read-only model store on the
+# dm-verity-signed root partition. This makes the pre-packaged models (e.g.
+# gemma3:1b) available offline, but prevents `ollama pull` of new models at
+# runtime. To allow pulling additional models, operators can override this
+# with a drop-in (e.g. /etc/systemd/system/ollama.service.d/override.conf)
+# pointing OLLAMA_MODELS at a writable location such as /var/lib/ollama/models
+# and seeding it from /opt/ollama-models.
+Environment=HOME=/var/lib/ollama
+Environment=OLLAMA_MODELS=/opt/ollama-models
+Environment=OLLAMA_HOST=0.0.0.0:11434
+Environment=LD_LIBRARY_PATH=/opt/ollama/lib/ollama
+
+ExecStart=/opt/ollama/bin/ollama serve
+
+Restart=on-failure
+RestartSec=5s
+
+# No extra sandboxing: the GuestOS root is already dm-verity read-only and
+# the orchestrator/replica services run without sandbox flags either.
+# ProtectSystem/PrivateTmp/ReadWritePaths fail early in this environment
+# with status=226/NAMESPACE because of the dm-crypt+LVM /var layout.
+
+[Install]
+WantedBy=multi-user.target

ic-os/guestos/context/BUILD.bazel

@@ -13,6 +13,10 @@ filegroup(
         "Dockerfile",
         "docker-base.dev",
         "docker-base.prod",
+        # Used by Dockerfile.base to pre-pull the gemma3:1b ollama model into
+        # /opt/ollama-models. Only consumed by the local-base-* envs which
+        # rebuild the base image from Dockerfile.base.
+        "ollama-pull-gemma.sh",
         "packages.common",
         "packages.dev",
     ],

ic-os/guestos/context/Dockerfile

@@ -113,7 +113,8 @@ RUN systemctl disable \
     motd-news.service \
     motd-news.timer \
     fstrim.service \
-    fstrim.timer
+    fstrim.timer \
+    ollama.service
 
 # ------ GUESTOS WORK --------------------------------------------
 
@@ -126,6 +127,17 @@ RUN mkdir -p /var/lib/ic/backup \
              /var/lib/ic/crypto \
              /var/lib/ic/data
 
+# Pre-create the directories where the ollama binary and CPU runtime
+# libraries are injected at ext4-build time via the rootfs `extras` map in
+# ic-os/guestos/defs.bzl. The published guestos-base image referenced by
+# docker-base.{prod,dev} pre-dates the introduction of /opt/ollama, so
+# without this `mkdir -p` the ext4 builder's `cp` step fails with
+# "No such file or directory" for /opt/ollama/bin/ollama. Once the
+# updated base image (with `mkdir -p /opt/ollama/...` in Dockerfile.base)
+# is published and referenced from docker-base.{prod,dev}, this step
+# becomes a no-op but is harmless.
+RUN mkdir -p /opt/ollama/bin /opt/ollama/lib/ollama
+
 # Create two mount points for temporary use during setup of "var" partition
 RUN mkdir -p /mnt/var_old /mnt/var_new
 
@@ -231,6 +243,15 @@ RUN addgroup socks && \
     adduser --system --disabled-password --shell /usr/sbin/nologin -c "Dante SOCKS Proxy" socks && \
     adduser socks socks && chmod +s /usr/sbin/danted
 
+# The "ollama" account. Used to run the `ollama serve` binary when the
+# `ollama.service` unit is enabled (disabled by default; see
+# /lib/systemd/system/ollama.service).
+RUN addgroup --system ollama && \
+    adduser --system --disabled-password --home /var/lib/ollama --shell /usr/sbin/nologin --ingroup ollama -c "Ollama" ollama && \
+    mkdir -p /var/lib/ollama && \
+    chown ollama:ollama /var/lib/ollama && \
+    chmod 0750 /var/lib/ollama
+
 # ------ INSTALL SCRIPTS -----------------------------------------
 
 # Install IC binaries and other data late -- this means everything above

ic-os/guestos/context/Dockerfile.base

@@ -35,6 +35,19 @@ RUN cd /tmp/ && \
     echo "c46e5b6f53948477ff3a19d97c58307394a29fe64a01905646f026ddc32cb65b  node_exporter-1.10.2.linux-amd64.tar.gz" > node_exporter.sha256 && \
     sha256sum -c node_exporter.sha256
 
+# Download and verify ollama. Ships the binary + shared libraries needed for
+# CPU inference. Pinned to v0.21.1. The upstream bundle includes ~5 GiB of
+# GPU runners (CUDA/MLX/Vulkan); these are stripped in the main Dockerfile
+# because GuestOS nodes do CPU-only inference.
+# To update:
+#   1. Bump the version below.
+#   2. Fetch the new sha256 with `curl -L https://github.com/ollama/ollama/releases/download/v<ver>/ollama-linux-amd64.tar.zst | sha256sum`.
+#   3. Rebuild the base image and update docker-base.{prod,dev}.
+RUN cd /tmp/ && \
+    curl -L -O https://github.com/ollama/ollama/releases/download/v0.21.1/ollama-linux-amd64.tar.zst && \
+    echo "528acc42750f755ce57b13b8138419ceac953b6fd488045f73dc1e80246da480  ollama-linux-amd64.tar.zst" > ollama.sha256 && \
+    sha256sum -c ollama.sha256
+
 #
 # Second build stage:
 # - Download and cache minimal Ubuntu Server 24.04 LTS Docker image
@@ -74,3 +87,34 @@ RUN cd /tmp/ && \
     mkdir -p /etc/node_exporter && \
     tar --strip-components=1 -C /usr/local/bin/ -zvxf node_exporter-1.10.2.linux-amd64.tar.gz node_exporter-1.10.2.linux-amd64/node_exporter && \
     rm /tmp/node_exporter-1.10.2.linux-amd64.tar.gz
+
+# Install ollama (CPU-only). The upstream tarball is ~2 GiB and bundles GPU
+# runners we don't use; we extract only the binary and the CPU ggml libraries
+# into /opt/ollama. The resulting install is ~50 MiB. `curl` and `zstd` are
+# already installed from packages.common above.
+COPY --from=download /tmp/ollama-linux-amd64.tar.zst /tmp/ollama-linux-amd64.tar.zst
+# The binary and CPU ggml .so files are *also* injected later via the ext4
+# extras path with explicit 0755 mode (see rootfs map in ic-os/guestos/defs.bzl).
+# Podman's rootless squash/export drops the exec bit in this environment, so
+# relying on Dockerfile.base-only install would yield a 0644 binary at runtime.
+# We still need the binary here because `ollama-pull-gemma.sh` executes it to
+# pre-populate /opt/ollama-models.
+RUN mkdir -p /opt/ollama/bin /opt/ollama/lib/ollama /opt/ollama-models && \
+    cd /tmp/ && \
+    zstd -d ollama-linux-amd64.tar.zst -o ollama.tar && \
+    tar -xf ollama.tar bin/ollama && \
+    tar -tf ollama.tar | grep -E '^lib/ollama/libggml-(base|cpu-)[^/]*\.so' > /tmp/ollama-cpu-libs.txt && \
+    tar -xf ollama.tar -T /tmp/ollama-cpu-libs.txt && \
+    mv bin/ollama /opt/ollama/bin/ollama && \
+    mv lib/ollama/libggml-*.so* /opt/ollama/lib/ollama/ && \
+    chmod 0755 /opt/ollama/bin/ollama /opt/ollama/lib/ollama/*.so* && \
+    rm -rf /tmp/bin /tmp/lib /tmp/ollama.tar /tmp/ollama-linux-amd64.tar.zst /tmp/ollama-cpu-libs.txt && \
+    ln -s /opt/ollama/bin/ollama /usr/local/bin/ollama
+
+# Pre-pull the gemma3:1b model into /opt/ollama-models (read-only baked-in
+# model store). The on-disk layout produced by `ollama pull` is
+# /opt/ollama-models/{manifests,blobs}. Running
+# `OLLAMA_MODELS=/opt/ollama-models ollama run gemma3:1b` at runtime will
+# serve the model without any network access.
+COPY ollama-pull-gemma.sh /tmp/ollama-pull-gemma.sh
+RUN sh -eu /tmp/ollama-pull-gemma.sh && rm /tmp/ollama-pull-gemma.sh

ic-os/guestos/context/ollama-pull-gemma.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Helper used from Dockerfile.base to pre-pull the Gemma model into the
+# baked-in read-only model store at /opt/ollama-models.
+#
+# This is only invoked at base-image build time. See ../guestos/ollama.service
+# for the runtime service definition.
+
+set -eu
+
+export OLLAMA_MODELS=/opt/ollama-models
+export LD_LIBRARY_PATH=/opt/ollama/lib/ollama
+export OLLAMA_HOST=127.0.0.1:11434
+
+mkdir -p "${OLLAMA_MODELS}"
+
+/opt/ollama/bin/ollama serve >/tmp/ollama-serve.log 2>&1 &
+OLLAMA_PID=$!
+trap 'kill "${OLLAMA_PID}" 2>/dev/null || true; wait "${OLLAMA_PID}" 2>/dev/null || true' EXIT
+
+# Wait for the server to come up.
+for i in $(seq 1 60); do
+    if curl -fs "http://${OLLAMA_HOST}/" >/dev/null 2>&1; then
+        break
+    fi
+    if [ "${i}" = 60 ]; then
+        echo "ollama server failed to start within 60s" >&2
+        cat /tmp/ollama-serve.log >&2 || true
+        exit 1
+    fi
+    sleep 1
+done
+
+/opt/ollama/bin/ollama pull gemma3:1b
+
+# Sanity-check that the model is usable offline.
+/opt/ollama/bin/ollama list | grep -q '^gemma3:1b '

ic-os/guestos/defs.bzl

@@ -59,14 +59,34 @@ def image_deps(mode, malicious = False):
             # additional libraries to install
             "//rs/ic_os/release:nss_icos": "/usr/lib/x86_64-linux-gnu/libnss_icos.so.2:0644",  # Allows referring to the guest IPv6 by name guestos from host, and host as hostos from guest.
             "//rs/ic_os/release:config_tool": "/opt/ic/bin/config_tool:0755",
+
+            # Ollama CPU-only runtime. The binary and CPU ggml libraries are
+            # injected via the extras path (not via Dockerfile.base) so that
+            # they get explicit 0755 perms via `install -m` in the ext4
+            # builder, bypassing rootless-podman's tendency to drop the
+            # exec bit during squash/export. The pre-pulled gemma3:1b model
+            # in /opt/ollama-models is still baked in via Dockerfile.base
+            # (model files are read-only 0644 data, no exec-bit concern).
+            "@ollama//:bin/ollama": "/opt/ollama/bin/ollama:0755",
+            "@ollama//:lib/ollama/libggml-base.so.0.0.0": "/opt/ollama/lib/ollama/libggml-base.so.0.0.0:0755",
+            "@ollama//:lib/ollama/libggml-cpu-alderlake.so": "/opt/ollama/lib/ollama/libggml-cpu-alderlake.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-haswell.so": "/opt/ollama/lib/ollama/libggml-cpu-haswell.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-icelake.so": "/opt/ollama/lib/ollama/libggml-cpu-icelake.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-sandybridge.so": "/opt/ollama/lib/ollama/libggml-cpu-sandybridge.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-skylakex.so": "/opt/ollama/lib/ollama/libggml-cpu-skylakex.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-sse42.so": "/opt/ollama/lib/ollama/libggml-cpu-sse42.so:0755",
+            "@ollama//:lib/ollama/libggml-cpu-x64.so": "/opt/ollama/lib/ollama/libggml-cpu-x64.so:0755",
         },
 
         # Set various configuration values
         "container_context_files": Label("//ic-os/guestos/context:context-files"),
         "component_files": component_files(mode),
         "partition_table": Label("//ic-os/guestos:partitions.csv"),
+        # rootfs_size was bumped from 3G to 5G to accommodate the ollama
+        # binary + CPU runtime libraries (~50 MiB) and the pre-pulled
+        # gemma3:1b model (~815 MiB) baked into /opt/ollama-models.
         "expanded_size": "50G",
-        "rootfs_size": "3G",
+        "rootfs_size": "5G",
         "bootfs_size": "1G",
         "grub_config": Label("//ic-os/bootloader:guestos_grub.cfg"),
 

ic-os/guestos/envs/local-base-dev/BUILD.bazel

@@ -13,5 +13,8 @@ icos_build(
         "manual",
         "no-cache",
     ],
-    visibility = ["//rs:ic-os-pkg"],
+    visibility = [
+        "//rs:ic-os-pkg",
+        "//rs:system-tests-pkg",
+    ],
 )

rs/ic_os/config/tool/templates/ic.json5.template

@@ -279,6 +279,8 @@ table ip6 filter {\n\
     ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9090, 9091, 9100, 19531, 19100, 19522 } accept\n\
     # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS\n\
     ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept\n\
+    # Allow access to the ollama HTTP API (disabled by default, started on-demand via systemctl enable --now ollama).\n\
+    ip6 daddr { ::/0 } ct state { new } tcp dport { 11434 } accept\n\
     # Custom templated rules\n\
     <<IPv6_TCP_RULES>>\n\
     <<IPv6_UDP_RULES>>\n\
@@ -418,6 +420,8 @@ table ip6 filter {\n\
     ct state { established, related } accept\n\
     ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9091, 9100, 9324, 19531, 19100, 19522 } accept\n\
     ip6 saddr { ::-ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff } ct state new tcp dport 443 accept\n\
+    # Allow access to the ollama HTTP API (disabled by default, started on-demand via systemctl enable --now ollama).\n\
+    ip6 daddr { ::/0 } ct state { new } tcp dport { 11434 } accept\n\
 \n\
     <<IPv6_TCP_RULES>>\n\
     <<IPv6_UDP_RULES>>\n\