[RFC] Shrink musl docker image size by ~50% (#1670)

Emilgardis · web-flow · commit a6cffa067e9f · 2025-04-20T08:05:46.000Z
Here is a proof of concept patchset which reduces the size of the musl
images from ~2.4G to ~1.2G, on the two architectures I tried (x86_64 and
aarch64).

Using aarch64 as a primary testbed, I got these results:
* Baseline (edge): image size 2.39G (794M gzipped)
* Patch 1 (strip the binaries in musl.sh): 1.55G
* Add patch 2 (flatten the output image): 1.31G
* Add patch 3 (additional tidyup script): 1.22G (348M gzipped).

A space/bandwidth saving of 47% uncompressed, 56% compressed.

I've tested the result with my use cases (x86_64 and aarch64). The crate
I am cross-compiling depends on `ring` which has a C component.

There is almost certainly more squeezing that could be done, but the Law
of Diminishing Returns kicks in somewhere. For example, I spotted copies
of identical binaries in /usr/local/bin which could be symlinked
together to save a few more MB, but coming up with a safe and robust
solution there might take a little thought.

The majority of these savings are specific to the musl builds. I haven't
analysed any of the non-musl images but there could be scope for
improvement there as well.

If you like this approach, I'll happily copy &amp; paste the tidyup &amp;
flatten part into the other dockerfiles. OTOH, there may be other ways
to achieve this; happy to discuss.
diff --git a/docker/Dockerfile.aarch64-unknown-linux-musl b/docker/Dockerfile.aarch64-unknown-linux-musl
@@ -18,6 +18,13 @@ RUN /qemu.sh aarch64
 COPY musl.sh /
 RUN /musl.sh TARGET=aarch64-linux-musl
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=aarch64-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/aarch64-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.arm-unknown-linux-musleabi b/docker/Dockerfile.arm-unknown-linux-musleabi
@@ -22,6 +22,13 @@ RUN /musl.sh \
                       --with-float=soft \
                       --with-mode=arm"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=arm-linux-musleabi-
 ENV CROSS_SYSROOT=/usr/local/arm-linux-musleabi
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.arm-unknown-linux-musleabihf b/docker/Dockerfile.arm-unknown-linux-musleabihf
@@ -23,6 +23,13 @@ RUN /musl.sh \
                       --with-float=hard \
                       --with-mode=arm"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=arm-linux-musleabihf-
 ENV CROSS_SYSROOT=/usr/local/arm-linux-musleabihf
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.armv5te-unknown-linux-musleabi b/docker/Dockerfile.armv5te-unknown-linux-musleabi
@@ -22,6 +22,13 @@ RUN /musl.sh \
                       --with-float=soft \
                       --with-mode=arm"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=arm-linux-musleabi-
 ENV CROSS_SYSROOT=/usr/local/arm-linux-musleabi
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.armv7-unknown-linux-musleabi b/docker/Dockerfile.armv7-unknown-linux-musleabi
@@ -23,6 +23,13 @@ RUN /musl.sh \
                       --with-mode=thumb \
                       --with-mode=arm"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=arm-linux-musleabi-
 ENV CROSS_SYSROOT=/usr/local/arm-linux-musleabi
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.armv7-unknown-linux-musleabihf b/docker/Dockerfile.armv7-unknown-linux-musleabihf
@@ -23,6 +23,13 @@ RUN /musl.sh \
                       --with-mode=thumb \
                       --with-fpu=vfp"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=arm-linux-musleabihf-
 ENV CROSS_SYSROOT=/usr/local/arm-linux-musleabihf
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.i586-unknown-linux-musl b/docker/Dockerfile.i586-unknown-linux-musl
@@ -18,6 +18,13 @@ RUN /musl.sh TARGET=i586-linux-musl
 COPY qemu.sh /
 RUN /qemu.sh i386
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=i586-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/i586-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.i686-unknown-linux-musl b/docker/Dockerfile.i686-unknown-linux-musl
@@ -18,6 +18,13 @@ RUN /musl.sh TARGET=i686-linux-musl
 COPY qemu.sh /
 RUN /qemu.sh i386
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=i686-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/i686-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.loongarch64-unknown-linux-musl b/docker/Dockerfile.loongarch64-unknown-linux-musl
@@ -28,6 +28,14 @@ RUN /qemu.sh loongarch64
 COPY qemu-runner base-runner.sh /
 COPY toolchain.cmake /opt/toolchain.cmake
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+ENV PATH=/x-tools/loongarch64-unknown-linux-musl/bin/:$PATH
+
 ENV CROSS_TOOLCHAIN_PREFIX=loongarch64-unknown-linux-musl-
 ENV CROSS_SYSROOT=/x-tools/loongarch64-unknown-linux-musl/loongarch64-unknown-linux-musl/sysroot/
 ENV CROSS_TARGET_RUNNER="/qemu-runner loongarch64"
diff --git a/docker/Dockerfile.mips-unknown-linux-musl b/docker/Dockerfile.mips-unknown-linux-musl
@@ -22,6 +22,13 @@ RUN /musl.sh \
     TARGET=mips-linux-muslsf \
     "COMMON_CONFIG += -with-arch=mips32r2"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=mips-linux-muslsf-
 ENV CROSS_SYSROOT=/usr/local/mips-linux-muslsf
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.mips64-unknown-linux-muslabi64 b/docker/Dockerfile.mips64-unknown-linux-muslabi64
@@ -22,6 +22,13 @@ RUN /musl.sh \
     TARGET=mips64-linux-musl \
     "COMMON_CONFIG += -with-arch=mips64r2"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=mips64-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/mips64-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.mips64el-unknown-linux-muslabi64 b/docker/Dockerfile.mips64el-unknown-linux-muslabi64
@@ -22,6 +22,13 @@ RUN /musl.sh \
     TARGET=mips64el-linux-musl \
     "COMMON_CONFIG += -with-arch=mips64r2"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=mips64el-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/mips64el-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.mipsel-unknown-linux-musl b/docker/Dockerfile.mipsel-unknown-linux-musl
@@ -22,6 +22,13 @@ RUN /musl.sh \
     TARGET=mipsel-linux-muslsf \
     "COMMON_CONFIG += -with-arch=mips32r2"
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=mipsel-linux-muslsf-
 ENV CROSS_SYSROOT=/usr/local/mipsel-linux-muslsf
 COPY musl-symlink.sh /
diff --git a/docker/Dockerfile.riscv64gc-unknown-linux-musl b/docker/Dockerfile.riscv64gc-unknown-linux-musl
@@ -26,6 +26,14 @@ ENV PATH=/x-tools/riscv64-unknown-linux-musl/bin/:$PATH
 COPY qemu.sh /
 RUN /qemu.sh riscv64
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+ENV PATH=/x-tools/riscv64-unknown-linux-musl/bin/:$PATH
+
 COPY qemu-runner base-runner.sh /
 COPY toolchain.cmake /opt/toolchain.cmake
 
diff --git a/docker/Dockerfile.x86_64-unknown-linux-musl b/docker/Dockerfile.x86_64-unknown-linux-musl
@@ -18,6 +18,13 @@ RUN /musl.sh TARGET=x86_64-linux-musl
 COPY qemu.sh /
 RUN /qemu.sh x86_64
 
+COPY tidyup.sh /
+RUN /tidyup.sh
+
+FROM scratch AS final
+COPY --from=build / /
+CMD ["/bin/bash"]
+
 ENV CROSS_TOOLCHAIN_PREFIX=x86_64-linux-musl-
 ENV CROSS_SYSROOT=/usr/local/x86_64-linux-musl
 COPY musl-symlink.sh /
diff --git a/docker/lib.sh b/docker/lib.sh
@@ -174,3 +174,62 @@ docker_to_linux_arch() {
 
     eval "${oldstate}"
 }
+
+find_argument() {
+    # Extracts the value from an argument of the form VARIABLE=VALUE
+    local needle="$1"
+    local return_var="$2"
+    shift 2
+    local prefix="${needle}="
+    for var in "${@}"; do
+        case "$var" in
+            "$prefix"*)
+                eval "$return_var=${var#"${prefix}"}"
+                return 0 ;;
+            *)           ;;
+        esac
+    done
+    echo "Missing argument ${needle}"
+    exit 1
+}
+
+symlinkify_if_same() {
+    local file1="$1"
+    local file2="$2"
+    # Only make a symlink if the files are identical, and the destination file isn't already a symlink
+    if [ ! -L "${file2}" ] && cmp "$file1" "$file2"; then
+        ln -sf "$file1" "$file2"
+    fi
+}
+
+symlinkify_and_strip_toolchain() {
+    local target="$1"
+    local gcc_ver="$2"
+
+    local target_bin="/usr/local/${target}/bin"
+    local local_bin="/usr/local/bin"
+
+    # The first set of tools appear as /usr/local/bin/<target>-<tool> and /usr/local/<target>/bin/<tool>
+
+    # Special case: ld is itself usually hardlinked to ld.bfd
+    symlinkify_if_same "${local_bin}/ld" "${local_bin}/ld.bfd"
+
+    # Turn hard links or otherwise identical files into symlinks
+    for tool in ar  as  ld  ld.bfd  nm  objcopy  objdump  ranlib  readelf  strip; do
+        local src="${local_bin}/${target}-${tool}"
+        local dest="${target_bin}/${tool}"
+        symlinkify_if_same "${src}" "${dest}"
+        strip "${src}"
+    done
+
+    # The second set of tools only appear as /usr/local/bin/<target>-<tool>
+
+    # Special case: c++ and g++ are usually the same file
+    symlinkify_if_same "${local_bin}/${target}-c++" "${local_bin}/${target}-g++"
+    # Special case: gcc and gcc-<version>
+    symlinkify_if_same "${local_bin}/${target}-gcc" "${local_bin}/${target}-gcc-${gcc_ver}"
+
+    for tool in  addr2line  c++ c++filt  cpp  elfedit  g++  gcc  gcc-${gcc_ver}  gcc-ar  gcc-nm  gcc-ranlib  gcov  gcov-dump  gcov-tool  gfortran  gprof  size  strings; do
+        strip "${local_bin}/${target}-${tool}"
+    done
+}
diff --git a/docker/musl.sh b/docker/musl.sh
@@ -36,6 +36,9 @@ main() {
     # Don't depend on the mirrors of sabotage linux that musl-cross-make uses.
     local linux_headers_site=https://ci-mirrors.rust-lang.org/rustc/sabotage-linux-tarballs
     local linux_ver=headers-4.19.88
+    local gcc_ver=9.2.0
+    local target
+    find_argument TARGET target "${@}"
 
     # alpine GCC is built with `--enable-default-pie`, so we want to
     # ensure we use that. we want support for shared runtimes except for
@@ -44,7 +47,7 @@ main() {
     # linked, so our behavior has maximum portability, and is consistent
     # with popular musl distros.
     hide_output make install "-j$(nproc)" \
-        GCC_VER=9.2.0 \
+        GCC_VER=${gcc_ver} \
         MUSL_VER=1.2.3 \
         BINUTILS_VER=2.33.1 \
         DL_CMD='curl --retry 3 -sSfL -C - -o' \
@@ -58,6 +61,14 @@ main() {
 
     popd
 
+    symlinkify_and_strip_toolchain "${target}" "${gcc_ver}"
+
+    for dir in /usr/local/libexec/gcc/"${target}"/*; do
+        pushd "${dir}" || exit 1
+        strip cc1 cc1plus collect2 f951 lto1 lto-wrapper liblto_plugin.so.0.0.0
+        popd || exit 1
+    done
+
     rm -rf "${td}"
     rm "${0}"
 }
diff --git a/docker/tidyup.sh b/docker/tidyup.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+rm -rf /var/lib/apt/lists
+rm -rf /usr/local/doc
