diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml new file mode 100644 index 00000000..979fcdc5 --- /dev/null +++ b/.github/workflows/docker-build-push.yml @@ -0,0 +1,110 @@ +name: Docker Build and Push + +on: + push: + branches: + - main + - develop + release: + types: [created] + workflow_dispatch: + inputs: + compiler: + description: 'Compiler to use (gcc, clang, zig)' + required: false + default: 'clang' + type: choice + options: + - gcc + - clang + - zig + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + strategy: + matrix: + compiler: [gcc, clang, zig] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch,suffix=-${{ matrix.compiler }} + type=ref,event=pr,suffix=-${{ matrix.compiler }} + type=semver,pattern={{version}},suffix=-${{ matrix.compiler }} + type=semver,pattern={{major}}.{{minor}},suffix=-${{ matrix.compiler }} + type=raw,value=latest-${{ matrix.compiler }},enable={{is_default_branch}} + + - name: Build Docker image using build.sh + run: | + # Make build script executable + chmod +x ./docker/linux/build.sh + + # Run build script with cache configuration + ./docker/linux/build.sh \ + --compiler ${{ matrix.compiler }} \ + --cache-from type=gha,scope=${{ matrix.compiler }} \ + --cache-to type=gha,mode=max,scope=${{ matrix.compiler }} \ + --rebuild + + - name: Tag and push image to registry + run: | + # Get the local image name from build.sh + LOCAL_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep "clice-dev-container.*${{ matrix.compiler }}" | head -n 1) + + if [ -z "$LOCAL_IMAGE" ]; then + echo "Error: Could not find built image" + exit 1 + fi + + echo "Built image: $LOCAL_IMAGE" + + # Tag and push for each metadata tag + echo '${{ steps.meta.outputs.tags }}' | while read -r tag; do + if [ -n "$tag" ]; then + echo "Tagging as: $tag" + docker tag "$LOCAL_IMAGE" "$tag" + docker push "$tag" + fi + done + + - name: Get image digest + id: digest + run: | + # Get the digest of the pushed image + FIRST_TAG=$(echo '${{ steps.meta.outputs.tags }}' | head -n 1) + DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$FIRST_TAG" | cut -d'@' -f2) + echo "digest=$DIGEST" >> $GITHUB_OUTPUT + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.digest.outputs.digest }} + push-to-registry: true diff --git a/config/default-toolchain-version b/config/default-toolchain-version deleted file mode 100644 index 8a2b780e..00000000 --- a/config/default-toolchain-version +++ /dev/null @@ -1,6 +0,0 @@ -xmake,3.0.2 -cmake,3.31.8 -python,3.13 -gcc,14 -clang,20 -msvc diff --git a/config/default-toolchain-version.json b/config/default-toolchain-version.json new file mode 100644 index 00000000..4913612e --- /dev/null +++ b/config/default-toolchain-version.json @@ -0,0 +1,11 @@ +{ + "xmake": "3.0.3", + "cmake": "3.31.8", + "python": "3.13", + "uv": "0.9.2", + "gcc": "14.3.0", + "llvm": "20.1.8", + "glibc": "2.39", + "linux": "6.17", + "msvc": null +} diff --git a/docker/.dockerignore b/docker/.dockerignore deleted file mode 100644 index 115b4295..00000000 --- a/docker/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -build/ -out/ -.cache - -.clice/ -.llvm*/ -.xmake/ -.vscode/ -.vs/ diff --git a/docker/linux/Dockerfile b/docker/linux/Dockerfile index 1226ba68..7e89f9fa 100644 --- a/docker/linux/Dockerfile +++ b/docker/linux/Dockerfile @@ -1,265 +1,394 @@ -# build with multi-stage for cache efficiency -FROM ubuntu:24.04 AS basic-tools +# check=skip=InvalidDefaultArgInFrom,experimental=all -# allow build script to bind-mount project source into build container (host path) -ARG BUILD_SRC +# ======================================================================== +# Clice Dev Container Multi-Stage Build System +# ======================================================================== -# set non-interactive frontend to avoid prompts -ENV DEBIAN_FRONTEND=noninteractive -# ensure user-local bin is on PATH for non-apt installs (xmake, uv, python) -ENV PATH="/root/.local/bin:${PATH}" - -# install basic tools -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - # TODO: support more cache for python, xmake installation - # TODO: check why cache doesn't work after add-apt-repository, may we change it to cache? -bash -eux - <<'BASH' - - set -e - apt update - # first install minimal apt prerequisites - # software-properties-common for add-apt-repository - # gnupg for gpg to verify cmake installer - # curl, git for downloading sources - # xz-utils, unzip for extracting archives - # make for xmake installation - apt install -y --no-install-recommends \ - software-properties-common \ - curl \ - gnupg \ - git \ - xz-utils \ - unzip \ - make - - # gcc, llvm PPA - add-apt-repository -y ppa:ubuntu-toolchain-r/ppa - apt update -BASH - -# Compiler stage -FROM basic-tools AS compiler-stage - -# passed from build arg +# Arguments passed from docker image build system ARG COMPILER +ARG PACKED_IMAGE_NAME + +# Global config shared in multi-stage builds +ARG CLICE_DEV_CONTAINER_BASE_IMAGE=ubuntu:24.04 +ARG CLICE_WORKDIR=/clice +ARG RELEASE_PACKAGE_DIR=/clice-dev-container-package +ARG PACKED_RELEASE_PACKAGE_PATH=/release-pkg.7z.run +ARG ENVIRONMENT_CONFIG_FILE=/root/.bashrc +ARG BUILD_CACHE_DIR=/var/cache/clice-dev-container +ARG SETUP_SCRIPTS_DIR=${RELEASE_PACKAGE_DIR}/setup_scripts-unknown + +# APT system paths configuration +ARG APT_CACHE_DIR=/var/cache/apt +ARG APT_STATE_CACHE_DIR=/var/lib/apt + +# UV docker layer cache configuration +ARG UV_PACKAGE_DIR_NAME=uv-packages +ARG UV_TARBALL_DIR_NAME=tarball + +# Python build scripts communicate via these environment variables +ARG PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES="\ +COMPILER=${COMPILER} \ +CLICE_WORKDIR=${CLICE_WORKDIR} \ +RELEASE_PACKAGE_DIR=${RELEASE_PACKAGE_DIR} \ +PACKED_RELEASE_PACKAGE_PATH=${PACKED_RELEASE_PACKAGE_PATH} \ +ENVIRONMENT_CONFIG_FILE=${ENVIRONMENT_CONFIG_FILE} \ +BUILD_CACHE_DIR=${BUILD_CACHE_DIR}" + +# ======================================================================== +# 🐍 Base Stage: Python Environment Foundation +# ======================================================================== +FROM ${CLICE_DEV_CONTAINER_BASE_IMAGE} AS base-python-environment-for-build +LABEL description="Base image with Python and uv environment for builder stages" + +ARG CLICE_WORKDIR +ARG APT_CACHE_DIR +ARG APT_STATE_CACHE_DIR +ARG RELEASE_PACKAGE_DIR +ARG BUILD_CACHE_DIR +ARG UV_PACKAGE_DIR_NAME +ARG UV_TARBALL_DIR_NAME + +# Environment setup +ENV PATH="/root/.local/bin:${PATH}" +ENV DEBIAN_FRONTEND=noninteractive -# copy instead of bind-mount, to avoid docker build cache invalidation -COPY config/default-toolchain-version /clice/config/default-toolchain-version - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - bash -eux - <<'BASH' - set -e - - # Always install libstdc++ development files, required for both gcc and clang to link against libstdc++ - GCC_VERSION=$(grep -E '^gcc,' /clice/config/default-toolchain-version | cut -d',' -f2) - apt install -y --no-install-recommends "libstdc++-${GCC_VERSION}-dev" - - if [ "$COMPILER" = "gcc" ]; then - apt install -y --no-install-recommends "gcc-${GCC_VERSION}" "g++-${GCC_VERSION}" - update-alternatives --install /usr/bin/cc cc "/usr/bin/gcc-${GCC_VERSION}" 100 - update-alternatives --install /usr/bin/gcc gcc "/usr/bin/gcc-${GCC_VERSION}" 100 - update-alternatives --install /usr/bin/c++ c++ "/usr/bin/g++-${GCC_VERSION}" 100 - update-alternatives --install /usr/bin/g++ g++ "/usr/bin/g++-${GCC_VERSION}" 100 - elif [ "$COMPILER" = "clang" ]; then - CLANG_VERSION=$(grep -E '^clang,' /clice/config/default-toolchain-version | cut -d',' -f2) - # install clang toolchain, libstdc++ is already installed - apt install -y --no-install-recommends "clang-${CLANG_VERSION}" "clang-tools-${CLANG_VERSION}" "lld-${CLANG_VERSION}" "libclang-rt-${CLANG_VERSION}-dev" - update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${CLANG_VERSION}" 100 - update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${CLANG_VERSION}" 100 - update-alternatives --install /usr/bin/c++ c++ "/usr/bin/clang++-${CLANG_VERSION}" 100 - update-alternatives --install /usr/bin/cc cc "/usr/bin/clang-${CLANG_VERSION}" 100 - update-alternatives --install /usr/bin/ld ld "/usr/bin/lld-${CLANG_VERSION}" 100 - else - echo "Error: Unsupported compiler '$COMPILER'. Use 'gcc' or 'clang'." >&2; exit 1 - fi -BASH - -FROM compiler-stage AS build-tool-stage - -ARG XMAKE_CACHE_DIR="/docker-build-cache/xmake" -ARG CMAKE_CACHE_DIR="/docker-build-cache/cmake" -ARG UV_CACHE_DIR="/var/cache/uv" - -ENV XMAKE_CACHE_DIR=${XMAKE_CACHE_DIR} -ENV CMAKE_CACHE_DIR=${CMAKE_CACHE_DIR} -ENV UV_CACHE_DIR=${UV_CACHE_DIR} - -COPY ./pyproject.toml /clice/pyproject.toml - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - --mount=type=cache,target=${XMAKE_CACHE_DIR},sharing=locked \ - --mount=type=cache,target=${CMAKE_CACHE_DIR},sharing=locked \ - --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked \ -bash -eux - <<'BASH' - -install_xmake() { - set -e - - XMAKE_VERSION=$(grep -E '^xmake,' /clice/config/default-toolchain-version | cut -d',' -f2) - XMAKE_BASE_URL="https://github.com/xmake-io/xmake/releases/download/v${XMAKE_VERSION}" - XMAKE_FILENAME="xmake-bundle-v${XMAKE_VERSION}.linux.x86_64" - XMAKE_CACHED_FILE="${XMAKE_CACHE_DIR}/${XMAKE_FILENAME}" - - if [ ! -f "${XMAKE_CACHED_FILE}" ] ; then - rm -f "${XMAKE_CACHE_DIR}/*" - curl -fsSL --retry 3 -o "${XMAKE_CACHED_FILE}" "${XMAKE_BASE_URL}/${XMAKE_FILENAME}" - fi - - XMAKE_INSTALL_DIR="/usr/bin" - XMAKE_INSTALLED_FILE="${XMAKE_INSTALL_DIR}/${XMAKE_FILENAME}" - - cp "${XMAKE_CACHED_FILE}" "${XMAKE_INSTALLED_FILE}" - chmod +x "${XMAKE_INSTALLED_FILE}" - - update-alternatives --install /usr/bin/xmake xmake "${XMAKE_INSTALLED_FILE}" 100 - - echo "export XMAKE_ROOT=y" >> ~/.bashrc -} +# Do NOT copy all config at once, or all stages would be rebuilt when any file changes +# Only copy what is needed for this stage +COPY docker/linux/utility/pyproject.toml ${CLICE_WORKDIR}/docker/linux/utility/pyproject.toml +COPY config/docker_build_stages/default-toolchain-version.json ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json -# Attention: DO NOT install cmake via PPA with apt, which would have to install required build-essential compiler tool chain -# We SHOULD NOT install another compiler toolchain, which could cause a lot trouble -# And we should not install compiler toolchain away from compiler stage -# So we install cmake from official installer script, and cache the downloaded files -install_cmake() { +# Install minimal system dependencies, uv, and Python +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=python-build-env-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=python-build-env-apt-state \ + --mount=type=cache,target=${BUILD_CACHE_DIR},sharing=locked,id=python-build-env-cache \ + bash -eux - <<'SCRIPT' set -e - - # cached downloads live under /docker-build-cache/cmake (BuildKit cache mount) - CMAKE_VERSION=$(grep -E '^cmake,' /clice/config/default-toolchain-version | cut -d',' -f2) - ARCH="x86_64" - - BASE_URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}" - INSTALLER_FILENAME="cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" - SHA_FILENAME="cmake-${CMAKE_VERSION}-SHA-256.txt" - ASC_FILENAME="${SHA_FILENAME}.asc" - - INSTALLER_PATH="${CMAKE_CACHE_DIR}/${INSTALLER_FILENAME}" - SHA_PATH="${CMAKE_CACHE_DIR}/${SHA_FILENAME}" - ASC_PATH="${CMAKE_CACHE_DIR}/${ASC_FILENAME}" - - verify_cmake_installer() { - if ! gpg --verify "${ASC_PATH}" "${SHA_PATH}"; then - echo "Signature verification failed for ${SHA_FILENAME}." >&2 - return 1 - fi - - local expected_hash - expected_hash=$(grep "${INSTALLER_FILENAME}" "${SHA_PATH}" | awk '{print $1}') - - local actual_hash - actual_hash=$(sha256sum "${INSTALLER_PATH}" | awk '{print $1}') - if [ "${expected_hash}" != "${actual_hash}" ]; then - echo "Checksum mismatch for ${INSTALLER_FILENAME}." >&2 - return 1 - fi - - echo "Checksum for ${INSTALLER_FILENAME} is valid." - return 0 - } - - gpg --keyserver keys.openpgp.org --recv-keys C6C265324BBEBDC350B513D02D2CEF1034921684 - - if [ ! -f "${INSTALLER_PATH}" ] || ! verify_cmake_installer; then - rm -f "${CMAKE_CACHE_DIR}/*" - - curl -fsSL --retry 3 -o "${INSTALLER_PATH}" "${BASE_URL}/${INSTALLER_FILENAME}" - curl -fsSL --retry 3 -o "${SHA_PATH}" "${BASE_URL}/${SHA_FILENAME}" - curl -fsSL --retry 3 -o "${ASC_PATH}" "${BASE_URL}/${ASC_FILENAME}" - - if ! verify_cmake_installer; then - echo "Verification of the downloaded installer failed. Cleaning cache." >&2 - rm -f "${CMAKE_CACHE_DIR}/*" + + # Disable APT auto cleanup to keep apt cache + # This option would override Binary::apt::APT::Keep-Downloaded-Packages + rm -f /etc/apt/apt.conf.d/docker-clean + # It is strange that apt will accept APT::Keep-Downloaded-Packages in commandline, + # but Binary::apt::APT::Keep-Downloaded-Packages in config file + echo 'APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/99keepcache + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >> /etc/apt/apt.conf.d/99keepcache + + apt update -o DPkg::Lock::Timeout=-1 + apt install -y --no-install-recommends -o DPkg::Lock::Timeout=-1 curl jq ca-certificates + + # Get uv version from configuration + UV_VERSION=$(jq -r .uv ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json) + echo "📦 Installing uv version: $UV_VERSION" + + # Determine architecture for uv standalone build + ARCH=$(uname -m) + case "$ARCH" in + x86_64) + UV_PLATFORM="x86_64-unknown-linux-gnu" + ;; + *) + echo "Unsupported architecture: $ARCH" exit 1 - fi - fi + ;; + esac + + # Download uv standalone build from GitHub releases to package directory + UV_TARBALL_NAME="uv-${UV_PLATFORM}.tar.gz" + UV_URL="https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${UV_TARBALL_NAME}" + + # Create cache directory for UV tarball (Docker layer cache) + UV_BUILD_CACHE_DIR="${BUILD_CACHE_DIR}/uv-${UV_VERSION}" + UV_BUILD_CACHE_PACKAGE_DIR="${UV_BUILD_CACHE_DIR}/${UV_PACKAGE_DIR_NAME}" + UV_BUILD_CACHE_TARBALL_DIR="${UV_BUILD_CACHE_DIR}/${UV_TARBALL_DIR_NAME}" + UV_BUILD_CACHE_TARBALL_FILE="${UV_BUILD_CACHE_TARBALL_DIR}/${UV_TARBALL_NAME}" + mkdir -p "${UV_BUILD_CACHE_TARBALL_DIR}" + + # Create package directory for UV, to be copied to release package + UV_PACKAGE_ROOT="${RELEASE_PACKAGE_DIR}/uv-${UV_VERSION}" + UV_PACKAGE_CACHE_DIR="${UV_PACKAGE_ROOT}/${UV_PACKAGE_DIR_NAME}" + UV_PACKAGE_TARBALL_DIR="${UV_PACKAGE_ROOT}/${UV_TARBALL_DIR_NAME}" + UV_PACKAGE_TARBALL_FILE="${UV_PACKAGE_TARBALL_DIR}/${UV_TARBALL_NAME}" + mkdir -p "${UV_PACKAGE_CACHE_DIR}" "${UV_PACKAGE_TARBALL_DIR}" + + echo "🌐 Downloading uv from: $UV_URL" + echo "💾 Cache location: ${UV_BUILD_CACHE_TARBALL_FILE}" + echo "📦 Package location: ${UV_PACKAGE_TARBALL_FILE}" + + # Download to cache + echo "⬇️ Downloading uv to cache..." + curl -fsSL "$UV_URL" -o "$UV_BUILD_CACHE_TARBALL_FILE" + + # Copy to package directory for later packaging + echo "📋 Copying to package directory..." + cp "$UV_BUILD_CACHE_TARBALL_FILE" "$UV_PACKAGE_TARBALL_FILE" + + # Extract and install uv from package + echo "🔧 Installing uv..." + mkdir -p /root/.local/bin + tar -xzf "$UV_PACKAGE_TARBALL_FILE" -C /root/.local/bin --strip-components=1 + + # Verify installation + echo "✅ UV installed successfully:" + uv --version + + # Get Python version from configuration + PYTHON_VERSION=$(jq -r .python ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json) + echo "🐍 Installing Python version: $PYTHON_VERSION" + + # Install Python using UV with package_dir as cache + echo "🔧 Installing Python ${PYTHON_VERSION} to package directory..." + # This creates Python installation cache in RELEASE_PACKAGE_DIR for expand-stage + # So we use UV_PACKAGE_CACHE_DIR here + UV_CACHE_DIR=${UV_PACKAGE_CACHE_DIR} uv python install "$PYTHON_VERSION" --default + + # Save Python version to a file for expanded-image stage to read + # This allows expanded-image to know which Python version to install without jq + echo "$PYTHON_VERSION" > "${UV_PACKAGE_ROOT}/.python-version" + echo "📝 Saved Python version to: ${UV_PACKAGE_ROOT}/.python-version" + + echo "✅ Python installation cached to: ${UV_PACKAGE_CACHE_DIR}" + + # Setup Python project environment + echo "🔧 Setting up Python project environment..." + # cache to build cache, here we use docker/linux/utility pyproject.toml, only for build + UV_CACHE_DIR=${UV_BUILD_CACHE_PACKAGE_DIR} uv sync --project ${CLICE_WORKDIR}/docker/linux/utility/pyproject.toml + echo "✅ Base Python environment setup complete!" +SCRIPT + +WORKDIR ${CLICE_WORKDIR} + +# ======================================================================== +# 🏗️ Stage 1: Compiler Toolchain Builder +# ======================================================================== +FROM base-python-environment-for-build AS toolchain-builder +LABEL description="Builds custom compiler toolchain" - sh "${INSTALLER_PATH}" --skip-license --prefix=/usr/local -} - -install_python() { - set -e - PYTHON_VERSION=$(grep -E '^python,' /clice/config/default-toolchain-version | cut -d',' -f2) - curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install "${PYTHON_VERSION}" - uv sync -} - -do_install() { - set -e - - cd /clice - - export PATH="/root/.local/bin:${PATH}" - echo "export XMAKE_ROOT=y" >> ~/.bashrc - - install_cmake & - install_xmake & - install_python & - - for job in $(jobs -p); do - wait $job || exit 1 - done -} - -do_install - -BASH - -# download compile dependencies -FROM build-tool-stage AS dependency-cache-stage - -# passed from build arg -# "lto" or "non_lto" -ARG BUILD_SRC -# ARG LTO_TYPE="" - -RUN --mount=type=bind,src=./,target=/clice,rw \ -bash -eux - <<'BASH' - -# cache_xmake_packages() { -# set -e - -# export PATH="/root/.local/bin:${PATH}" -# export XMAKE_ROOT=y - -# LTO_FLAG="" -# if [ "$LTO_TYPE" = "lto" ]; then -# LTO_FLAG="--release" -# fi - -# xmake f -y -v --mode=release ${LTO_FLAG} -# xmake f -y -v --mode=debug ${LTO_FLAG} -# } - -do_prepare_dependency() { - set -e - - cd /clice - - # cache_xmake_packages & - - for job in $(jobs -p); do - wait $job || exit 1 - done -} - -do_prepare_dependency +ARG COMPILER +ARG CLICE_WORKDIR +ARG APT_CACHE_DIR +ARG APT_STATE_CACHE_DIR +ARG BUILD_CACHE_DIR +ARG PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES + +COPY config/docker_build_stages/common.py ${CLICE_WORKDIR}/config/docker_build_stages/common.py +COPY config/docker_build_stages/default-toolchain-version.json ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json +COPY config/docker_build_stages/toolchain_config.py ${CLICE_WORKDIR}/config/docker_build_stages/toolchain_config.py +COPY docker/linux/utility/build_utils.py ${CLICE_WORKDIR}/docker/linux/utility/build_utils.py +COPY docker/linux/utility/build_clice_compiler_toolchain.py ${CLICE_WORKDIR}/docker/linux/utility/build_clice_compiler_toolchain.py + +# Build the custom toolchain (Python script handles all dependencies) +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=toolchain-builder-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=toolchain-builder-apt-state \ + --mount=type=cache,target=${BUILD_CACHE_DIR},sharing=locked,id=toolchain-builder-cache \ + bash -eux - <<'SCRIPT' + + # Activate Python environment + echo "🐍 Activating Python environment..." + source ${CLICE_WORKDIR}/docker/linux/utility/.venv/bin/activate + + echo "🔨 Building custom compiler toolchain..." + eval ${PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES} \ + python docker/linux/utility/build_clice_compiler_toolchain.py + echo "✅ Toolchain build complete!" +SCRIPT + +# ======================================================================== +# 🏗️ Stage 2: Dependencies Downloader (Parallel to Stage 1) +# ======================================================================== +FROM base-python-environment-for-build AS dependencies-downloader +LABEL description="Downloads dev-container dependencies" -BASH +ARG COMPILER +ARG CLICE_WORKDIR +ARG APT_CACHE_DIR +ARG APT_STATE_CACHE_DIR +ARG BUILD_CACHE_DIR +ARG PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES + +COPY config/docker_build_stages/common.py ${CLICE_WORKDIR}/config/docker_build_stages/common.py +COPY config/docker_build_stages/default-toolchain-version.json ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json +COPY config/docker_build_stages/dependencies_config.py ${CLICE_WORKDIR}/config/docker_build_stages/dependencies_config.py +COPY docker/linux/utility/build_utils.py ${CLICE_WORKDIR}/docker/linux/utility/build_utils.py +COPY docker/linux/utility/download_dependencies.py ${CLICE_WORKDIR}/docker/linux/utility/download_dependencies.py + +# for download python dependencies +COPY tests/pyproject.toml ${CLICE_WORKDIR}/tests/pyproject.toml + +# Setup Python project environment and download dependencies +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt-state \ + --mount=type=cache,target=${BUILD_CACHE_DIR},sharing=locked,id=dependencies-downloader-cache \ + bash -eux - <<'SCRIPT' + + # Activate Python environment + echo "🐍 Activating Python environment..." + source ${CLICE_WORKDIR}/docker/linux/utility/.venv/bin/activate + + echo "📥 Downloading dependencies..." + eval ${PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES} \ + python docker/linux/utility/download_dependencies.py + echo "✅ Dependencies download complete!" +SCRIPT + +# ======================================================================== +# 🏗️ Stage 3: Release Package Creator +# ======================================================================== +FROM base-python-environment-for-build AS image-packer +LABEL description="Merges toolchain and dependencies into final release package" -FROM dependency-cache-stage AS final +ARG COMPILER +ARG CLICE_WORKDIR +ARG RELEASE_PACKAGE_DIR +ARG APT_CACHE_DIR +ARG APT_STATE_CACHE_DIR +ARG BUILD_CACHE_DIR +ARG PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES +ARG SETUP_SCRIPTS_DIR + +# For execution in this layer only +COPY config/docker_build_stages/common.py ${CLICE_WORKDIR}/config/docker_build_stages/common.py +COPY config/docker_build_stages/default-toolchain-version.json ${CLICE_WORKDIR}/config/docker_build_stages/default-toolchain-version.json +COPY config/docker_build_stages/toolchain_config.py ${CLICE_WORKDIR}/config/docker_build_stages/toolchain_config.py +COPY config/docker_build_stages/dependencies_config.py ${CLICE_WORKDIR}/config/docker_build_stages/dependencies_config.py +COPY config/docker_build_stages/package_config.py ${CLICE_WORKDIR}/config/docker_build_stages/package_config.py +COPY docker/linux/utility/container-entrypoint.sh ${CLICE_WORKDIR}/docker/linux/utility/container-entrypoint.sh +COPY docker/linux/utility/build_utils.py ${CLICE_WORKDIR}/docker/linux/utility/build_utils.py +COPY docker/linux/utility/create_release_package.py ${CLICE_WORKDIR}/docker/linux/utility/create_release_package.py + +# For package (setup scripts) +COPY config/docker_build_stages/common.py ${SETUP_SCRIPTS_DIR}/config/docker_build_stages/common.py +COPY config/docker_build_stages/default-toolchain-version.json ${SETUP_SCRIPTS_DIR}/config/docker_build_stages/default-toolchain-version.json +COPY config/docker_build_stages/dependencies_config.py ${SETUP_SCRIPTS_DIR}/config/docker_build_stages/dependencies_config.py +COPY config/docker_build_stages/toolchain_config.py ${SETUP_SCRIPTS_DIR}/config/docker_build_stages/toolchain_config.py +COPY config/docker_build_stages/package_config.py ${SETUP_SCRIPTS_DIR}/config/docker_build_stages/package_config.py +COPY docker/linux/utility/build_utils.py ${SETUP_SCRIPTS_DIR}/docker/linux/utility/build_utils.py +COPY docker/linux/utility/local_setup.py ${SETUP_SCRIPTS_DIR}/docker/linux/utility/local_setup.py + +# Copy outputs from previous stages +# Merge by RELEASE_PACKAGE_DIR structure, each component has its own directory +# No need to manually copy individual files + +# UV tarball and python +COPY --from=base-python-environment-for-build ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} +# static libstdc++ toolchain +COPY --from=toolchain-builder ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} +# other dependencies +COPY --from=dependencies-downloader ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} + +# Setup Python project environment and create final release package +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=image-packer-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=image-packer-apt-state \ + --mount=type=cache,target=${BUILD_CACHE_DIR},sharing=locked,id=dependencies-downloader-cache \ + bash -eux - <<'SCRIPT' + + # Activate Python environment + echo "🐍 Activating Python environment..." + source ${CLICE_WORKDIR}/docker/linux/utility/.venv/bin/activate + + # Create final release package + echo "📦 Creating release package..." + eval ${PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES} \ + python docker/linux/utility/create_release_package.py + echo "✅ Release package created successfully!" +SCRIPT + +# ======================================================================== +# 🏗️ Stage 4: Release Package +# ======================================================================== +FROM ${CLICE_DEV_CONTAINER_BASE_IMAGE} AS packed-image + +ARG CLICE_WORKDIR +ARG PACKED_RELEASE_PACKAGE_PATH + +# Copy only the packed release package +# All scripts, configs, and .bashrc are already inside the package +COPY --from=image-packer ${PACKED_RELEASE_PACKAGE_PATH} ${PACKED_RELEASE_PACKAGE_PATH} + +# Copy build scripts and Dockerfile +# These are needed for the final image +# Instead of using local build.sh and Dockerfile, we use the version packed here +# So we could make breaking changes to build scripts without breaking released images +COPY docker/linux/utility/common.sh ${CLICE_WORKDIR}/docker/linux/utility/common.sh +COPY docker/linux/build.sh ${CLICE_WORKDIR}/docker/linux/build.sh +COPY docker/linux/Dockerfile ${CLICE_WORKDIR}/docker/linux/Dockerfile + +# ======================================================================== +# 🏗️ Stage 5: Development Image (Expanded) +# ======================================================================== +FROM ${PACKED_IMAGE_NAME} AS expanded-image +LABEL description="Fully expanded development image" -RUN bash -eux - <<'BASH' -set -e - # clice is mounted here, so remove everything to reduce image size - rm -rf /clice +ARG COMPILER +ARG CLICE_WORKDIR +ARG RELEASE_PACKAGE_DIR +ARG PACKED_RELEASE_PACKAGE_PATH +ARG PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES +ARG UV_PACKAGE_DIR_NAME +ARG UV_TARBALL_DIR_NAME +ARG SETUP_SCRIPTS_DIR - # disable git exception in cmake build when Fetch-Content - git config --global --add safe.directory '*' -BASH +ENV PATH="/root/.local/bin:${PATH}" -WORKDIR /clice +# Expand the release image into a full development environment +# We don't mark here with --mount=type=cache because here is executed on clice developer environment +# clice developer do not have the cache from previous stages +RUN bash -eux - <<'SCRIPT' + # Extract self-extracting 7z SFX package (no p7zip-full required) + echo "📦 Extracting self-extracting release package..." + mkdir -p "${RELEASE_PACKAGE_DIR}" + + # Make the SFX archive executable and run it + chmod +x "${PACKED_RELEASE_PACKAGE_PATH}" + + # Run SFX to extract to RELEASE_PACKAGE_DIR + # -o: output directory + # -y: assume yes for all prompts + "${PACKED_RELEASE_PACKAGE_PATH}" -o"${RELEASE_PACKAGE_DIR}" -y + + echo "✅ Release package extracted!" + + # Install UV and Python from packaged files (offline installation) + echo "📦 Installing UV from package..." + + # Find UV version from directory name (uv-*) + UV_PACKAGE_ROOT=$(find "${RELEASE_PACKAGE_DIR}" -maxdepth 1 -type d -name "uv-*" | head -n 1) + + # Extract version from directory name (e.g., uv-0.9.2 -> 0.9.2) + UV_VERSION=$(basename "$UV_PACKAGE_ROOT" | sed 's/^uv-//') + echo "📋 UV version: ${UV_VERSION}" + echo "📁 UV package root: ${UV_PACKAGE_ROOT}" + + UV_PACKAGE_CACHE_DIR="${UV_PACKAGE_ROOT}/${UV_PACKAGE_DIR_NAME}" + UV_TARBALL_PATH="${UV_PACKAGE_ROOT}/${UV_TARBALL_DIR_NAME}/uv-*.tar.gz" + UV_INSTALL_DIR="/root/.local/bin" + mkdir -p "${UV_INSTALL_DIR}" + + echo "🔧 Extracting UV tarball..." + tar -xzf ${UV_TARBALL_PATH} -C ${UV_INSTALL_DIR} --strip-components=1 + echo "✅ UV installed successfully!" + + # Install Python - read version from .python-version file in UV package + PYTHON_VERSION_FILE="${UV_PACKAGE_ROOT}/.python-version" + PYTHON_VERSION=$(cat "${PYTHON_VERSION_FILE}") + echo "📋 Python version: ${PYTHON_VERSION}" + + echo "🐍 Installing Python ${PYTHON_VERSION}..." + UV_CACHE_DIR=${UV_PACKAGE_CACHE_DIR} uv python install "${PYTHON_VERSION}" --default --preview-features python-install-default + echo "✅ Python ${PYTHON_VERSION} installed successfully!" + + # Run local setup directly from packaged scripts (no download needed) + echo "🚀 Running local setup to expand environment..." + + LOCAL_SETUP_SCRIPT="${SETUP_SCRIPTS_DIR}/docker/linux/utility/local_setup.py" + + # Run local setup directly from package (no venv needed, using system Python) + eval ${PYTHON_BUILD_SCRIPT_BASE_ENV_VARIABLES} \ + python "${LOCAL_SETUP_SCRIPT}" + echo "✅ Environment expansion complete!" + + # Cleanup + echo "🧹 Cleaning up temporary files..." + rm -f "${PACKED_RELEASE_PACKAGE_PATH}" + echo "✅ Cleanup complete! Dev container ready! 🎉" +SCRIPT CMD ["/bin/bash"] diff --git a/docker/linux/build.sh b/docker/linux/build.sh index b47ecc69..f640ca00 100644 --- a/docker/linux/build.sh +++ b/docker/linux/build.sh @@ -1,54 +1,236 @@ #!/bin/bash +# ======================================================================== +# Clice Development Container Builder +# ======================================================================== + set -e +# ======================================================================== +# 🔧 Environment Setup +# ======================================================================== + +# Source common utilities +source "$(dirname "${BASH_SOURCE[0]}")/utility/common.sh" + # Save original working directory and switch to project root ORIG_PWD="$(pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -echo "${SCRIPT_DIR}" cd "${SCRIPT_DIR}/../.." PROJECT_ROOT="$(pwd)" trap 'cd "${ORIG_PWD}"' EXIT -# default configurations -COMPILER="clang" -DOCKERFILE_PATH="docker/linux/Dockerfile" +COMPILER="${DEFAULT_COMPILER}" +BUILD_STAGE="${DEFAULT_BUILD_STAGE}" +CACHE_FROM="" +CACHE_TO="" +VERSION="${DEFAULT_VERSION}" +REBUILD="false" +DEBUG="false" + +# ======================================================================== +# 📚 Usage Information +# ======================================================================== usage() { cat <] +🚀 Clice Development Container Builder + +Usage: $0 [OPTIONS] + +OPTIONS: + --compiler Target compiler (default: ${COMPILER}) + --cache-from Use cache from specified image + --cache-to Push cache to specified image and log cache operations + --version Set version tag (default: ${VERSION}) + --stage Build specific stage (packed-image or expanded-image) + --rebuild Force rebuild even if image exists + --debug Enable interactive debug mode (requires Docker 23.0+) + --help, -h Show this help message -Defaults: - --compiler ${COMPILER} +EXAMPLES: + $0 Build development container with clang + $0 --compiler gcc Build development container with gcc + $0 --stage packed-image Build only the release image + $0 --stage expanded-image Expand release image to development image + $0 --version v1.0.0 Build versioned container (v1.0.0) + $0 --rebuild Force rebuild existing image + $0 --debug Build with interactive debug mode + $0 --cache-from clice-io/clice-dev:cache Use cache from existing image + $0 --cache-to type=registry,ref=myregistry/myimage:cache Push cache + +DEBUG MODE: + --debug enables interactive debugging with docker buildx debug build + Requires Docker 23.0+ with BuildKit experimental features + On build failure, you can use debug commands to inspect the build state EOF } -# parse command line arguments +# ======================================================================== +# 🔍 Command Line Parsing +# ======================================================================== + while [ "$#" -gt 0 ]; do case "$1" in --compiler) COMPILER="$2"; shift 2;; + --cache-from) + CACHE_FROM="$2"; shift 2;; + --cache-to) + CACHE_TO="$2"; shift 2;; + --version) + VERSION="$2"; shift 2;; + --stage) + BUILD_STAGE="$2"; shift 2;; + --rebuild) + REBUILD="true"; shift 1;; + --debug) + DEBUG="true"; shift 1;; -h|--help) usage; exit 0;; *) - echo "Unknown parameter: $1" >&2; usage; exit 1;; + echo "❌ Unknown parameter: $1" >&2; usage; exit 1;; esac done -IMAGE_TAG="linux-${COMPILER}" -IMAGE_NAME="clice-io/clice-dev:${IMAGE_TAG}" +# ======================================================================== +# 🏷️ Image Naming +# ======================================================================== + +IMAGE_TAG=$(get_image_tag "${COMPILER}" "${VERSION}") +PACKED_IMAGE_NAME=$(get_packed_image_name "${COMPILER}" "${VERSION}") +EXPANDED_IMAGE_NAME=$(get_expanded_image_name "${COMPILER}" "${VERSION}") + +# Set the target image name based on build stage +if [ "$BUILD_STAGE" = "packed-image" ]; then + TARGET_IMAGE_NAME="$PACKED_IMAGE_NAME" +elif [ "$BUILD_STAGE" = "expanded-image" ]; then + TARGET_IMAGE_NAME="$EXPANDED_IMAGE_NAME" +else + TARGET_IMAGE_NAME="clice-dev_container-debug_build-$BUILD_STAGE" + BUILD_COMMAND_DEBUG_EXTRA="--on always" + echo "🔧 Debug Building Intermediate Stage: $BUILD_STAGE" >&2; +fi + +# ======================================================================== +# 🚀 Build Process +# ======================================================================== + +echo "=========================================================================" +echo "🚀 CLICE DEVELOPMENT CONTAINER BUILDER" +echo "=========================================================================" +echo "📦 Image: ${TARGET_IMAGE_NAME}" +echo "🏷️ Version: ${VERSION}" +echo "🔧 Compiler: ${COMPILER}" +echo "🐳 Dockerfile: ${DOCKERFILE_PATH}" +echo "📁 Project Root: ${PROJECT_ROOT}" +echo "⚡ Parallel Build: Enabled" +if [ -n "$CACHE_FROM" ]; then + echo "💾 Cache From: ${CACHE_FROM}" +fi +if [ -n "$CACHE_TO" ]; then + echo "💾 Cache To: ${CACHE_TO}" +fi +echo "=========================================================================" + +# ======================================================================== +# 🔄 Auto-Expansion Logic (Release → Development) +# ======================================================================== + +# Build the target image +echo "🔍 Checking for target image: ${TARGET_IMAGE_NAME}" + +# Handle REBUILD flag - clean up existing images +if [ "$REBUILD" = "true" ]; then + echo "🔄 Force rebuild requested - cleaning up existing images..." + + # Clean up target image + if docker image inspect "${TARGET_IMAGE_NAME}" >/dev/null 2>&1; then + echo "🧹 Removing existing target image: ${TARGET_IMAGE_NAME}" + docker rmi "${TARGET_IMAGE_NAME}" || true + fi +fi + +BUILD_ARGS=( + "--progress=plain" + "--target" + "${BUILD_STAGE}" + "--build-arg" + "COMPILER=${COMPILER}" + "--build-arg" + "VERSION=${VERSION}" + "--build-arg" + "PACKED_IMAGE_NAME=${PACKED_IMAGE_NAME}" + "--build-arg" + "CLICE_DIR=${CLICE_DIR}" + "--build-arg" + "BUILDKIT_INLINE_CACHE=1" +) + +if [ -n "$CACHE_FROM" ]; then + BUILD_ARGS+=("--cache-from=${CACHE_FROM}") +fi + +if [ -n "$CACHE_TO" ]; then + BUILD_ARGS+=("--cache-to=${CACHE_TO}") + echo "📝 Starting build with cache-to logging enabled..." +fi + +BUILD_ARGS+=("-t" "${TARGET_IMAGE_NAME}" "-f" "${DOCKERFILE_PATH}" ".") + +# Execute with or without debug mode +if [ "$DEBUG" = "true" ]; then + # Enable BuildKit experimental features for debug mode + echo "🐛 Debug mode enabled (BUILDX_EXPERIMENTAL=1)" + + export BUILDX_EXPERIMENTAL=1 + BUILD_COMMAND="docker buildx debug --invoke /bin/bash ${BUILD_COMMAND_DEBUG_EXTRA} build" +else + BUILD_COMMAND="docker buildx build" +fi + +echo "🔨 Build command: ${BUILD_COMMAND} ${BUILD_ARGS[*]}" +${BUILD_COMMAND} "${BUILD_ARGS[@]}" + +BUILD_SUCCESS=$? + +if [ -n "$CACHE_TO" ]; then + echo "💾 Cache operations completed. Cache pushed to: ${CACHE_TO}" +fi -echo "===========================================" -echo "Building image: ${IMAGE_NAME}" -echo "Compiler: ${COMPILER}" -echo "Dockerfile: ${DOCKERFILE_PATH}" -echo "===========================================" +# ======================================================================== +# 📊 Post-Build Information +# ======================================================================== -# build the docker image with specified arguments -# must run in clice root dir, so that we can mount the project in docker file to acquire essential files -docker buildx build --progress=plain -t "${IMAGE_NAME}" \ - --build-arg COMPILER="${COMPILER}" \ - --build-arg BUILD_SRC="${PROJECT_ROOT}" \ - -f "${DOCKERFILE_PATH}" . +BUILD_SUCCESS=$? -echo "Build complete. Image:${IMAGE_NAME}" +if [ $BUILD_SUCCESS -eq 0 ]; then + echo "=========================================================================" + echo "✅ BUILD COMPLETED SUCCESSFULLY!" + echo "=========================================================================" + echo "📦 Image Name: ${TARGET_IMAGE_NAME}" + echo "🏷️ Image Tag: ${IMAGE_TAG}" + echo "🔧 Compiler: ${COMPILER}" + echo "🏗️ Build Stage: ${BUILD_STAGE}" + + # Get image information + if command -v docker &> /dev/null; then + echo "" + echo "📊 IMAGE INFORMATION:" + docker image inspect "${TARGET_IMAGE_NAME}" --format="Size: {{.Size}} bytes ({{.VirtualSize}} virtual)" 2>/dev/null || true + docker image inspect "${TARGET_IMAGE_NAME}" --format="Created: {{.Created}}" 2>/dev/null || true + echo "" + echo "🚀 NEXT STEPS:" + echo " • Run container: ./docker/linux/run.sh --compiler ${COMPILER}" + echo " • Use container: docker run --rm -it ${TARGET_IMAGE_NAME} /bin/bash" + fi + + echo "=========================================================================" +else + echo "=========================================================================" + echo "❌ BUILD FAILED!" + echo "=========================================================================" + echo "🔍 Check the build output above for error details" + echo "=========================================================================" + exit 1 +fi diff --git a/docker/linux/run.sh b/docker/linux/run.sh index 3babc8c5..d1a22242 100644 --- a/docker/linux/run.sh +++ b/docker/linux/run.sh @@ -1,87 +1,286 @@ #!/bin/bash +# ======================================================================== +# Clice Development Container Runner +# ======================================================================== + set -e +# ======================================================================== +# 🔧 Environment Setup +# ======================================================================== + +# Source common utilities +source "$(dirname "${BASH_SOURCE[0]}")/utility/common.sh" + # Save original working directory and switch to project root ORIG_PWD="$(pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -echo "${SCRIPT_DIR}" cd "${SCRIPT_DIR}/../.." PROJECT_ROOT="$(pwd)" trap 'cd "${ORIG_PWD}"' EXIT -# default configurations -COMPILER="clang" +COMPILER="${DEFAULT_COMPILER}" RESET="false" +UPDATE="false" +VERSION="${DEFAULT_VERSION}" +COMMAND="" +CONTAINER_WORKDIR="${DEFAULT_CONTAINER_WORKDIR}" + +# ======================================================================== +# 📚 Usage Information +# ======================================================================== usage() { cat <] [--reset] +🚀 Clice Development Container Runner -Defaults: - --compiler ${COMPILER} - --reset (re-create the container) +Usage: $0 [OPTIONS] [COMMAND] + +OPTIONS: + --compiler Target compiler (default: ${COMPILER}) + --reset Remove existing container + --update Pull latest image and update + --version Use specific version (default: ${VERSION}) + --help, -h Show this help message + +EXAMPLES: + $0 Run container (build if not exists) + $0 --compiler gcc Run container with GCC compiler + $0 --reset Remove container and recreate + $0 --update Pull latest image and update + $0 bash Run specific command in container EOF } -# parse command line arguments +# ======================================================================== +# 🔍 Command Line Parsing +# ======================================================================== + while [ "$#" -gt 0 ]; do case "$1" in --compiler) COMPILER="$2"; shift 2;; --reset) RESET="true"; shift 1;; + --update) + UPDATE="true"; shift 1;; + --version) + VERSION="$2"; shift 2;; -h|--help) usage; exit 0;; - *) echo "Unknown parameter: $1"; usage; exit 1;; + --) + shift; COMMAND="$*"; break;; + -*) + echo "❌ Unknown parameter: $1" >&2; usage; exit 1;; + *) + COMMAND="$*"; break;; esac done -IMAGE_TAG="linux-${COMPILER}" -IMAGE_NAME="clice-io/clice-dev:${IMAGE_TAG}" -CONTAINER_NAME="clice-dev-linux-${COMPILER}" +# ======================================================================== +# 🏷️ Container and Image Naming +# ======================================================================== -# If the image doesn't exist, build it automatically by invoking build.sh -if ! docker image inspect "${IMAGE_NAME}" >/dev/null 2>&1; then - echo "Image ${IMAGE_NAME} not found, invoking build.sh to create it..." - ./docker/linux/build.sh --compiler "${COMPILER}" -fi +PACKED_IMAGE_NAME=$(get_packed_image_name "${COMPILER}" "${VERSION}") +EXPANDED_IMAGE_NAME=$(get_expanded_image_name "${COMPILER}" "${VERSION}") +CONTAINER_NAME=$(get_container_name "${COMPILER}" "${VERSION}") + +# ======================================================================== +# 🚀 Main Execution +# ======================================================================== + +echo "=========================================================================" +echo "🚀 CLICE DEVELOPMENT CONTAINER RUNNER" +echo "=========================================================================" +echo "🏷️ Image: ${EXPANDED_IMAGE_NAME}" +echo "🏷️ Version: ${VERSION}" +echo "🐳 Container: ${CONTAINER_NAME}" +echo "🔧 Compiler: ${COMPILER}" +echo "📁 Project Root: ${PROJECT_ROOT}" +echo "=========================================================================" + +# ======================================================================== +# 🐳 Container Management +# ======================================================================== # Handle --reset: remove the existing container if it exists if [ "${RESET}" = "true" ]; then if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - echo "Resetting container: stopping and removing existing container ${CONTAINER_NAME}..." + echo "🔄 Removing existing container: ${CONTAINER_NAME}..." docker stop "${CONTAINER_NAME}" >/dev/null 2>&1 || true docker rm "${CONTAINER_NAME}" >/dev/null 2>&1 - echo "Container ${CONTAINER_NAME} has been removed." + echo "✅ Container ${CONTAINER_NAME} has been removed." else - echo "Container ${CONTAINER_NAME} does not exist, no need to reset." + echo "ℹ️ Container ${CONTAINER_NAME} does not exist." fi + echo "🏁 Reset completed. Run again without --reset to create new container." exit 0 fi -CONTAINER_WORKDIR="/clice" +# ======================================================================== +# 🏗️ Image Management +# ======================================================================== + +# Check if we need to update/pull the packed image +UPDATE_REASON="" +if [ "$UPDATE" = "true" ]; then + UPDATE_REASON="🔄 Force updating image..." +elif ! docker image inspect "${PACKED_IMAGE_NAME}" >/dev/null 2>&1; then + UPDATE_REASON="🔄 Packed image ${PACKED_IMAGE_NAME} not found locally, pulling..." +fi + +if [ -n "$UPDATE_REASON" ]; then + + echo "${UPDATE_REASON}" + + # Remove existing expanded image before pulling (avoid conflicts) + if docker image inspect "${EXPANDED_IMAGE_NAME}" >/dev/null 2>&1; then + echo "🧹 Cleaning existing expanded image: ${EXPANDED_IMAGE_NAME}..." + if ! docker rmi "${EXPANDED_IMAGE_NAME}" >/dev/null 2>&1; then + echo "❌ Failed to remove expanded image: ${EXPANDED_IMAGE_NAME}" + echo "💡 This usually means a container is still using this image." + echo "🔧 Please run: $0 --reset to remove the container first, then try --update again." + echo "⚠️ This ensures your container data safety - we won't accidentally delete your container." + exit 1 + fi + echo "✅ Expanded image removed successfully" + fi + + echo "📥 Pulling ${PACKED_IMAGE_NAME} from registry..." + if docker pull "${PACKED_IMAGE_NAME}"; then + echo "✅ Successfully pulled image: ${PACKED_IMAGE_NAME}" + else + echo "❌ Could not pull image: ${PACKED_IMAGE_NAME}" + echo "💡 Please check if the image exists in the registry" + exit 1 + fi + + echo "🏁 Update completed." +fi + +# ======================================================================== +# 🏗️ Auto-Expand Packed Image to Development Image +# ======================================================================== -# Check if the container exists +# At this point, packed image is guaranteed to exist (either pulled or already present) +# Check if expanded development image exists, if not, expand it from packed image +if ! docker image inspect "${EXPANDED_IMAGE_NAME}" >/dev/null 2>&1; then + echo "=========================================================================" + echo "🏗️ EXPANDING PACKED IMAGE TO DEVELOPMENT IMAGE" + echo "=========================================================================" + echo "📦 Source (Packed): ${PACKED_IMAGE_NAME}" + echo "🎯 Target (Expanded): ${EXPANDED_IMAGE_NAME}" + echo "=========================================================================" + + # Run packed image container and execute its internal build.sh for expansion + # To keep the expansion process consistent and reliable, we use the build.sh script from the container itself. + # + # Since newer Docker versions don't include CLI without installation, we can't use docker buildx directly. + # Instead, we use chroot approach: + # 1. Mount host root directory to a temp folder inside container + # 2. Copy /clice to host temp directory + # 3. chroot into host root and execute build.sh + # + # Mounts: + # • / (host root) - Mount to temp directory for chroot access + + # Create temp directory on host for chroot + HOST_TEMP_DIR=$(mktemp -d -p /tmp clice-expand.XXXXXX) + + echo "📁 Created host temp directory: ${HOST_TEMP_DIR}" + echo "🔄 Preparing chroot environment..." + + if docker run --rm \ + -v "/:/host-root" \ + -e "HOST_TEMP_DIR=${HOST_TEMP_DIR}" \ + -e "COMPILER=${COMPILER}" \ + -e "VERSION=${VERSION}" \ + "${PACKED_IMAGE_NAME}" \ + /bin/bash -c ' + set -e + echo "📦 Copying /clice to host temp directory..." + cp -r /clice "/host-root${HOST_TEMP_DIR}/" + + echo "🔧 Executing build.sh via chroot..." + chroot /host-root /bin/bash -c " + cd ${HOST_TEMP_DIR}/clice && \ + ./docker/linux/build.sh --stage expanded-image --compiler ${COMPILER} --version ${VERSION} --debug + " + + echo "🧹 Cleaning up temp directory..." + rm -rf "/host-root${HOST_TEMP_DIR}" + '; then + # Clean up host temp directory (in case container cleanup failed) + rm -rf "${HOST_TEMP_DIR}" 2>/dev/null || true + echo "=========================================================================" + echo "✅ EXPANSION COMPLETED SUCCESSFULLY" + echo "=========================================================================" + echo "🎉 Development image created: ${EXPANDED_IMAGE_NAME}" + echo "📦 Ready for container creation" + echo "=========================================================================" + else + echo "=========================================================================" + echo "❌ EXPANSION FAILED" + echo "=========================================================================" + exit 1 + fi +else + echo "✅ Development image already exists: ${EXPANDED_IMAGE_NAME}" +fi + +# Check if the container exists and is using the current development image if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - echo "===========================================" - echo "Attaching to existing container: ${CONTAINER_NAME}" - echo "From image: ${IMAGE_NAME}" - echo "Project mount: ${PROJECT_ROOT} -> ${CONTAINER_WORKDIR}" - echo "===========================================" + echo "🔍 Found existing container: ${CONTAINER_NAME}" + + # Check if container uses current development image (compare image IDs) + CONTAINER_IMAGE_ID=$(docker inspect --format='{{.Image}}' "${CONTAINER_NAME}" 2>/dev/null || echo "") + EXPECTED_IMAGE_ID=$(docker inspect --format='{{.Id}}' "${EXPANDED_IMAGE_NAME}" 2>/dev/null || echo "") + + # Check target image and container match + if [ -n "$CONTAINER_IMAGE_ID" ] && [ -n "$EXPECTED_IMAGE_ID" ] && [ "$CONTAINER_IMAGE_ID" = "$EXPECTED_IMAGE_ID" ]; then + echo "✅ Container is using current development image" + echo "🚀 Starting and attaching to container..." + else + CONTAINER_IMAGE_NAME=$(docker inspect --format='{{.Config.Image}}' "${CONTAINER_NAME}" 2>/dev/null || echo "unknown") + echo "⚠️ WARNING: Container image mismatch!" + echo " 📦 Container using: ${CONTAINER_IMAGE_NAME} (ID: ${CONTAINER_IMAGE_ID})" + echo " 🎯 Expected: ${EXPANDED_IMAGE_NAME} (ID: ${EXPECTED_IMAGE_ID})" + echo "" + echo "💡 Your container is using a different image version." + echo "🛡️ To ensure data safety, please:" + echo " 1. Save any important work from the current container" + echo " 2. Run: $0 --reset to remove the outdated container" + echo " 3. Run: $0 to create a new container with the latest image" + echo "" + echo "🚀 For now, connecting to your existing container..." + fi + docker start "${CONTAINER_NAME}" >/dev/null - docker exec -it -w "${CONTAINER_WORKDIR}" "${CONTAINER_NAME}" /bin/bash + + if [ -n "$COMMAND" ]; then + docker exec -it -w "${CONTAINER_WORKDIR}" "${CONTAINER_NAME}" bash -c "$COMMAND" + else + docker exec -it -w "${CONTAINER_WORKDIR}" "${CONTAINER_NAME}" /bin/bash + fi exit 0 fi +# Create new container DOCKER_RUN_ARGS=(-it -w "${CONTAINER_WORKDIR}") DOCKER_RUN_ARGS+=(--name "${CONTAINER_NAME}") DOCKER_RUN_ARGS+=(--mount "type=bind,src=${PROJECT_ROOT},target=${CONTAINER_WORKDIR}") -echo "===========================================" -echo "Creating and running new container: ${CONTAINER_NAME}" -echo "From image: ${IMAGE_NAME}" -echo "Project mount: ${PROJECT_ROOT} -> ${CONTAINER_WORKDIR}" -echo "===========================================" +echo "=========================================================================" +echo "🚀 Creating new container: ${CONTAINER_NAME}" +echo "📦 From image: ${EXPANDED_IMAGE_NAME}" +echo "📁 Project mount: ${PROJECT_ROOT} -> ${CONTAINER_WORKDIR}" +echo "=========================================================================" -docker run "${DOCKER_RUN_ARGS[@]}" "${IMAGE_NAME}" +if [ -n "$COMMAND" ]; then + echo "🏃 Executing command: $COMMAND" + docker run --rm "${DOCKER_RUN_ARGS[@]}" "${EXPANDED_IMAGE_NAME}" bash -c "$COMMAND" +else + echo "🐚 Starting interactive shell..." + docker run "${DOCKER_RUN_ARGS[@]}" "${EXPANDED_IMAGE_NAME}" +fi \ No newline at end of file diff --git a/docker/linux/utility/build_clice_compiler_toolchain.py b/docker/linux/utility/build_clice_compiler_toolchain.py new file mode 100644 index 00000000..c730c356 --- /dev/null +++ b/docker/linux/utility/build_clice_compiler_toolchain.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +""" +Builds custom compiler toolchain (glibc, libstdc++, Linux headers) from source +using parallel execution with dependency management. +""" + +import os +import sys + +# Ensure utility directory is in Python path for imports +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from typing import Dict, List, Set + +from build_utils import ( + Job, + ParallelTaskScheduler, + run_command, + install_download_prerequisites, + install_extract_prerequisites, + download_and_verify, + extract_source, + extract_package, +) + +from config.docker_build_stages.common import TOOLCHAIN_BUILD_ENV_VARS, COMPILER +from config.docker_build_stages.toolchain_config import ( + TOOLCHAIN, + ToolchainComponent, + GccSubComponent, + LinuxSubComponent, + GlibcSubComponent, + ZigSubComponent +) + +# ======================================================================== +# 📦 Environment Setup Tasks +# ======================================================================== + +def install_build_prerequisites(component: ToolchainComponent) -> None: + """ + Note: We maintain multiple GCC versions because glibc requires + GCC < 10 to avoid linker symbol conflicts, while modern libstdc++ + benefits from the latest compiler features. + """ + # Collect all build prerequisites from sub-components + all_prerequisites = set() + for sub_component in component.sub_components: + all_prerequisites.update(sub_component.build_prerequisites) + + if not all_prerequisites: + print(f"ℹ️ [SETUP] No build prerequisites for {component.name}") + return + + print(f"🔨 [SETUP] Installing build prerequisites for {component.name}...") + print(f" 📋 Packages: {', '.join(sorted(all_prerequisites))}") + pkg_list = " ".join(sorted(all_prerequisites)) + run_command(f"apt install -y --no-install-recommends=true -o DPkg::Lock::Timeout=-1 {pkg_list}") + + # Setup GCC alternatives after installation + # Linux headers install requires gcc, even though we won't use it in linux header install + run_command("update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90") + print(f"✅ [SETUP] Build prerequisites for {component.name} installed") + +# ======================================================================== +# 📚 GNU C Library (glibc) Tasks +# ======================================================================== + +def fix_glibc_paths() -> None: + """ + 🔧 Fix Hardcoded Build Paths in glibc Installation + + glibc's build process generates various text files (.la, .pc, linker scripts) + that contain hardcoded absolute paths from the build environment. These paths + need to be cleaned up to create relocatable installations. + + This function scans all installed files and removes build-specific paths, + making the toolchain portable across different installation directories. + """ + search_path = TOOLCHAIN.sysroot_dir + print(f"🔧 [POST-PROCESS] Sanitizing hardcoded paths in {search_path}...") + + if not os.path.isdir(search_path): + print(f"❌ [ERROR] Sysroot directory not found: '{search_path}'", file=sys.stderr) + return + + files_processed = 0 + for root, _, files in os.walk(search_path): + for filename in files: + file_path = os.path.join(root, filename) + + # Check if file is text-based (skip binaries) + try: + with open(file_path, 'r', encoding='utf-8') as f: + original_content = f.read() + if '\0' in original_content: # Contains null bytes (binary) + continue + except UnicodeDecodeError: + continue # File not readable or not text + + # Look for and remove hardcoded paths + replacement_path = f"{os.path.dirname(file_path)}/" + new_content = original_content.replace(replacement_path, "") + if new_content == original_content: + continue # No changes needed + + # Apply the path fix + print(f" 🔨 Fixing paths in: {os.path.relpath(file_path, search_path)}") + print(f" ➤ Removing: '{replacement_path}'") + with open(file_path, 'w', encoding='utf-8') as f: + f.write(new_content) + files_processed += 1 + + print(f"✅ [POST-PROCESS] Path fixing complete ({files_processed} files processed)") + +def build_and_install_glibc(glibc_component: GlibcSubComponent, linux_component: LinuxSubComponent) -> None: + """ + Build Configuration: + • Uses GCC 9 (required: GCC < 10 to avoid symbol conflicts link error) + • Disables compiler warnings as errors (--disable-werror) + • Enables 64-bit libraries, disables 32-bit compatibility + """ + + print(f"🏗️ [BUILD] Starting {glibc_component.name} compilation...") + print(f" 📋 Using GCC 9 (required for glibc compatibility)") + print(f" 🎯 Target: {TOOLCHAIN.host_triplet} ({TOOLCHAIN.host_machine})") + print(f" 📁 Install: {TOOLCHAIN.sysroot_dir}/usr") + + # Prepare out-of-tree build directory + os.makedirs(glibc_component.build_dir, exist_ok=True) + + # Configure build environment with GCC 9 + compiler_env = { + 'CC': 'gcc-9', # GNU C compiler version 9 (full path) + 'CPP': 'cpp-9', # C preprocessor (explicit) + } + compiler_env.update(TOOLCHAIN_BUILD_ENV_VARS) + + # Configure glibc build + print(f"⚙️ [CONFIG] Configuring glibc build...") + configure_script = os.path.join(glibc_component.src_dir, "configure") + configure_command = f"{configure_script} --host={glibc_component.host_triplet} --prefix={TOOLCHAIN.sysroot_dir}/usr --with-headers={TOOLCHAIN.sysroot_dir}/usr/include --enable-kernel={linux_component.version} --disable-werror --disable-lib32 --enable-lib64" + run_command(configure_command, cwd=glibc_component.build_dir, env=compiler_env) + + # Compile glibc + print(f"🔨 [COMPILE] Building glibc (this may take several minutes)...") + run_command("make -j", cwd=glibc_component.build_dir, env=compiler_env) + + # Install glibc to sysroot + print(f"📦 [INSTALL] Installing glibc to sysroot...") + run_command(f"make install -j", cwd=glibc_component.build_dir, env=compiler_env) + + # Post-process to fix hardcoded paths + fix_glibc_paths() + print(f"✅ [COMPLETE] glibc build and installation finished") + +# ======================================================================== +# 🐧 Linux Kernel Headers Installation +# ======================================================================== + +def install_linux_headers(component: LinuxSubComponent) -> None: + install_path = os.path.join(TOOLCHAIN.sysroot_dir, "usr") + print(f"🐧 [INSTALL] Installing Linux kernel headers...") + print(f" 🏗️ Architecture: {TOOLCHAIN.host_machine}") + print(f" 📁 Target: {install_path}") + + # Use command-line arguments instead of environment variables + # This ensures highest priority and avoids Makefile variable conflicts + # Install to /usr within sysroot for Clang compatibility + make_args: Dict[str, str] = { + "ARCH": TOOLCHAIN.host_machine, + "INSTALL_HDR_PATH": install_path + } + + # Also preserve any global build environment variables + make_env = {} + make_env.update(TOOLCHAIN_BUILD_ENV_VARS) + + # Build the make command with arguments + args_str = " ".join([f"{key}={value}" for key, value in make_args.items()]) + + # Install sanitized kernel headers using command-line parameters + run_command(f"make {args_str} -j headers_install", cwd=component.src_dir, env=make_env) + print(f"✅ [COMPLETE] Linux kernel headers installed") + +# ======================================================================== +# 🛠️ GCC Compiler Collection Tasks +# ======================================================================== + +def download_gcc_prerequisites(component: GccSubComponent) -> None: + print(f"📦 [DOWNLOAD] Fetching {component.name} mathematical prerequisites...") + print(f" 📋 Components: GMP, MPFR, MPC") + run_command("./contrib/download_prerequisites", cwd=component.src_dir) + print(f"✅ [DOWNLOAD] GCC prerequisites ready") + +def build_and_install_libstdcpp(component: GccSubComponent) -> None: + print(f"🔧 [BUILD] Starting {component.name} C++ standard library build...") + print(f" 📋 Using GCC 14 (modern C++ support)") + print(f" 🎯 Target libraries: {', '.join(component.target_libs)}") + print(f" 🔗 Linking with glibc v{TOOLCHAIN.glibc.version}") + + # Prepare out-of-tree build directory + os.makedirs(component.build_dir, exist_ok=True) + + # Configure build environment with modern GCC + compiler_env = { + 'CC': 'gcc-14', # Modern C compiler (full path) + 'CXX': 'g++-14', # Modern C++ compiler (full path) + 'CPP': 'cpp-14', # C preprocessor (explicit) + } + compiler_env.update(TOOLCHAIN_BUILD_ENV_VARS) + + # Configure GCC for target library building + print(f"⚙️ [CONFIG] Configuring GCC for library-only build...") + configure_cmd = [ + f"{component.src_dir}/configure", + f"--host={TOOLCHAIN.host_triplet}", # Build system + f"--target={TOOLCHAIN.target_triplet}", # Target system + f"--prefix={TOOLCHAIN.sysroot_dir}/usr", # Installation prefix + f"--with-sysroot={TOOLCHAIN.sysroot_dir}", # System root for headers/libs + f"--with-glibc-version={TOOLCHAIN.glibc.version}", # glibc compatibility + "--with-gcc-major-version-only", # Use major version in paths for clang compatibility + "--disable-werror", # Don't fail on warnings + "--disable-multilib", # Single architecture only + "--disable-bootstrap", # Skip multi-stage build + "--enable-languages=c,c++", # Language support + "--enable-threads", # Threading support + "--enable-lto", # Link-time optimization + "--enable-nls", # Native language support + "--disable-shared", # Static libraries for portability + ] + run_command(" ".join(configure_cmd), cwd=component.build_dir, env=compiler_env) + + # Build only the target libraries we need + print(f"🔨 [COMPILE] Building target libraries (this will take significant time)...") + build_targets = " ".join([f"all-target-{lib}" for lib in component.target_libs]) + run_command(f"make -j {build_targets}", cwd=component.build_dir, env=compiler_env) + + # Install the built libraries + print(f"📦 [INSTALL] Installing C++ standard library and runtime libraries...") + install_targets = " ".join([f"install-target-{lib}" for lib in component.target_libs]) + run_command(f"make -j {install_targets}", cwd=component.build_dir, env=compiler_env) + print(f"✅ [COMPLETE] C++ standard library build finished") + +# ======================================================================== +# ⚡ Zig Compiler Tasks +# ======================================================================== + +def extract_zig(component: ZigSubComponent) -> None: + archive_path = os.path.join(component.cache_dir, component.tarball_name) + extract_package( + archive_path=archive_path, + target_dir=component.package_dir, + strip_top_level=True + ) + +# ======================================================================== +# 🎭 Main Build Orchestrator +# ======================================================================== + +def main() -> None: + print("🚀 ========================================================================") + print("🚀 CLICE COMPILER TOOLCHAIN BUILD SYSTEM") + print("🚀 ========================================================================") + print(f"📁 Sysroot Directory: {TOOLCHAIN.sysroot_dir}") + print(f"🎯 Target Architecture: {TOOLCHAIN.target_triplet} ({TOOLCHAIN.target_machine})") + print(f"🔧 Selected Compiler: {COMPILER}") + print("🚀 ========================================================================\n") + + # Define all jobs with dependencies + install_download_prereq_job = Job("install_download_prerequisites", install_download_prerequisites, (TOOLCHAIN,)) + install_extract_prereq_job = Job("install_extract_prerequisites", install_extract_prerequisites, (TOOLCHAIN,)) + install_build_prereq_job = Job("install_build_prerequisites", install_build_prerequisites, (TOOLCHAIN,)) + + all_jobs = [ + install_download_prereq_job, + install_extract_prereq_job, + install_build_prereq_job, + ] + + extend_jobs: List[Job] = [] + + # Conditional: Build gcc/clang toolchain (glibc + libstdc++) OR download zig + match COMPILER: + case "clang": + # Glibc Pipeline + download_glibc_job = Job("download_glibc", download_and_verify, (TOOLCHAIN.glibc,), [install_download_prereq_job]) + extract_glibc_job = Job("extract_glibc", extract_source, (TOOLCHAIN.glibc,), [download_glibc_job, install_extract_prereq_job]) + + # Linux Headers Pipeline + download_linux_job = Job("download_linux", download_and_verify, (TOOLCHAIN.linux,), [install_download_prereq_job]) + extract_linux_job = Job("extract_linux", extract_source, (TOOLCHAIN.linux,), [download_linux_job, install_extract_prereq_job]) + install_linux_headers_job = Job("install_linux_headers", install_linux_headers, (TOOLCHAIN.linux,), [extract_linux_job, install_build_prereq_job]) + + # Glibc build depends on Linux headers + build_glibc_job = Job("build_and_install_glibc", build_and_install_glibc, (TOOLCHAIN.glibc, TOOLCHAIN.linux), + [extract_glibc_job, install_build_prereq_job, install_linux_headers_job]) + + # GCC Pipeline + download_gcc_job = Job("download_gcc", download_and_verify, (TOOLCHAIN.gcc,), [install_download_prereq_job]) + extract_gcc_job = Job("extract_gcc", extract_source, (TOOLCHAIN.gcc,), [download_gcc_job, install_extract_prereq_job]) + download_gcc_prereq_job = Job("download_gcc_prerequisites", download_gcc_prerequisites, (TOOLCHAIN.gcc,), [extract_gcc_job]) + build_libstdcpp_job = Job("build_and_install_libstdcpp", build_and_install_libstdcpp, (TOOLCHAIN.gcc,), + [download_gcc_prereq_job, build_glibc_job, install_linux_headers_job, install_build_prereq_job]) + + extend_jobs = [ + download_glibc_job, + extract_glibc_job, + download_linux_job, + extract_linux_job, + install_linux_headers_job, + build_glibc_job, + download_gcc_job, + extract_gcc_job, + download_gcc_prereq_job, + build_libstdcpp_job, + ] + case "zig": + # Zig Pipeline: download, verify, and extract using standard component functions + download_zig_job = Job("download_zig", download_and_verify, (TOOLCHAIN.zig,), [install_download_prereq_job]) + extract_zig_job = Job("extract_zig", extract_zig, (TOOLCHAIN.zig,), [download_zig_job, install_extract_prereq_job]) + extend_jobs = [download_zig_job, extract_zig_job] + case _: + raise ValueError(f"Unsupported compiler: {COMPILER}") + + all_jobs.extend(extend_jobs) + + print(f"📊 Initializing parallel scheduler with {len(all_jobs)} tasks...") + total_deps = sum(len(job.dependencies) for job in all_jobs) + print(f"🔗 Total dependency edges: {total_deps}") + independent_jobs = [job.name for job in all_jobs if not job.dependencies] + print(f"⚡ Maximum parallelism: {len(independent_jobs)} initial tasks: {independent_jobs}\n") + + scheduler = ParallelTaskScheduler(all_jobs) + scheduler.run() + + print("\n🎉 ========================================================================") + print("🎉 TOOLCHAIN BUILD COMPLETED SUCCESSFULLY!") + print("🎉 ========================================================================") + print(f"✅ All components built and installed to: {TOOLCHAIN.sysroot_dir}") + print("🎉 ========================================================================") + +if __name__ == "__main__": + main() + +# Here's origin toolchain build bash, won't be updated, just for reference +# the only target is to build static link libstdc++, without full parallel build support + +# prerequests +""" +# aria2 is used for downloading files +# gawk bison are for glibc build +# bzip2 is for extracting tar.bz2 files when prepare gcc prerequisites +# rsync is required by linux kernel headers installation +apt install -y --no-install-recommends aria2 bzip2 rsync gawk bison +# gcc-9 for glibc build +# gcc-14 for libstdc++ build +apt install -y --no-install-recommends binutils gcc-9 libstdc++-9-dev gcc-14 g++-14 libstdc++-9-dev +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 90 +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 90 +update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-14 90 +update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-14 90 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 80 +update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-9 80 +""" + +# generic command +""" +export ORIGIN='$$ORIGIN' # to generate rpath relative to the binary location +export HOST=x86_64-linux-gnu # this is not essential, could be moved to python config +export TARGET=${HOST} # this is not essential, could be moved to python config +export PREFIX="${TOOLCHAIN_BUILD_ROOT}/sysroot/${HOST}/${TARGET}/glibc${versions["glibc"]}-libstdc++${versions["gcc"]}-linux${versions["linux"]}" # this is not essential, could be moved to python config +export ARCH=x86_64 # this is not essential, could be moved to python config +""" + +# build glibc +""" +# Attention: gcc version less than 10 required, or multiple definition __libc_use_alloca would be error +update-alternatives --set gcc /usr/bin/gcc-9 +update-alternatives --set cc /usr/bin/gcc-9 + +mkdir -p GLIBC_CONFIG.build_dir +cd GLIBC_CONFIG.build_dir + +../configure --host=${HOST} --prefix=${PREFIX}/usr --disable-werror --disable-lib32 --enable-lib64 +make -j +make install -j + +# This script is intended to be run after the glibc build process. +# Its purpose is to find and replace placeholder paths within the generated text-based +# files (like .la, .pc, etc.) located under the ${PREFIX} directory. +# This is a common post-build step to fix hardcoded paths from the build environment. + +# --- Configuration --- + +# 1. The root directory to search within. It's expected that the +# PREFIX environment variable is set by the build environment. +SEARCH_PATH="${PREFIX}" + +# --- Script Body --- + +# Check if the search path is valid +if [ -z "$SEARCH_PATH" ] || [ ! -d "$SEARCH_PATH" ]; then + echo "Error: SEARCH_PATH is not set or is not a valid directory: '$SEARCH_PATH'" + exit 1 +fi + +# Check if the 'file' command is available +if ! command -v file &> /dev/null; then + echo "Error: 'file' command not found. Please install it to proceed." + exit 1 +fi + +echo "Removing absolute paths from text ld scripts..." +echo "Starting search in: '$SEARCH_PATH'" +echo "========================================" + +# Find all files, then check each one to see if it's a text file containing the search string. +# Using -print0 and read -d '' handles filenames with spaces or special characters. +find "$SEARCH_PATH" -type f -print0 | while IFS= read -r -d '' file; do + + # Check if the file is a text file + MIME_TYPE=$(file -b --mime-type "$file") + if [[ "$MIME_TYPE" != text/* ]]; then + echo "--- Skipping binary file: $file (Type: $MIME_TYPE) ---" + continue + fi + + # Get the directory where the file is located. + REPLACEMENT_PATH=$(dirname "$file") + REPLACEMENT_PATH="${REPLACEMENT_PATH}/" + + # Check if the file actually contains the search string before processing + if ! grep -q "$REPLACEMENT_PATH" "$file"; then + continue + fi + + # It's a text file and contains the string, so process it. + echo -e "\n--- Processing text file: $file ---" + + + + # Use grep to show where the changes will happen. + echo " Matches found on lines:" + grep -n "$REPLACEMENT_PATH" "$file" | sed 's/^/ /g' + + echo " Deleting '$REPLACEMENT_PATH'" + + # Perform the replacement in-place using sed. + # The delimiter `|` is used to avoid conflicts if paths contain `/`. + sed -i "s|$REPLACEMENT_PATH||g" "$file" + +done + +echo "========================================" +echo "Path replacement process finished." +""" + +# build linux kernel headers(parallel with glibc build) +""" +export LINUX_SRC_URL="https://github.com/torvalds/linux/archive/refs/tags/v${versions["linux"]}.zip" +git clone https://github.com/torvalds/linux.git --depth=1 LINUX_CONFIG.src_dir # should replace with download and extract using LINUX_SRC_URL +cd LINUX_CONFIG.src_dir +make ARCH=x86_64 INSTALL_HDR_PATH=${PREFIX}/usr -j headers_install +""" + +# build libstdc++(requires glibc built and kernel headers installed) +""" +# Download prerequisites for GCC +cd GCC_CONFIG.src_dir +contrib/download_prerequisites + +# build libstdc++ +# libstdc++ could not be built separately, so we build the whole GCC but only install libstdc++ +update-alternatives --set gcc /usr/bin/gcc-14 +update-alternatives --set g++ /usr/bin/g++-14 +update-alternatives --set cc /usr/bin/gcc-14 +update-alternatives --set c++ /usr/bin/g++-14 + +mkdir -p GCC_CONFIG.build_dir +cd GCC_CONFIG.build_dir + +../configure \ + --host=${TARGET} \ + --target=${TARGET} \ + --prefix=${PREFIX}/usr \ + --with-sysroot=${PREFIX} \ + --with-glibc-version=${versions["glibc"]} \ + --disable-werror \ + --disable-multilib \ + --disable-shared \ + --disable-bootstrap \ + --enable-languages=c,c++ \ + --enable-threads \ + --enable-lto \ + --enable-nls + +make -j all-target-libgcc all-target-libstdc++-v3 all-target-libsanitizer all-target-libatomic all-target-libbacktrace all-target-libgomp all-target-libquadmath +make -j install-target-libgcc install-target-libstdc++-v3 install-target-libsanitizer install-target-libatomic install-target-libbacktrace install-target-libgomp install-target-libquadmath +""" diff --git a/docker/linux/utility/build_utils.py b/docker/linux/utility/build_utils.py new file mode 100644 index 00000000..2b4e7ba7 --- /dev/null +++ b/docker/linux/utility/build_utils.py @@ -0,0 +1,521 @@ +import shutil +import sys +import os +import tarfile +import subprocess +import hashlib +import concurrent.futures +import time + +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from config.docker_build_stages.common import Component, ToolchainSubComponent +import time +from typing import Dict, Set, Tuple, Optional, List, Callable +from graphlib import TopologicalSorter +from collections import defaultdict +from enum import Enum + +def download_file(url: str, dest: str) -> None: + if os.path.exists(dest): + print(f"File {os.path.basename(dest)} already exists. Skipping download.", flush=True) + return + + dest_dir = os.path.dirname(dest) + dest_name = os.path.basename(dest) + + print(f"Downloading {url} to {dest} (SSL verification disabled)...", flush=True) + + command = [ + "aria2c", + "--continue=true", + "--split=8", + "--max-connection-per-server=8", + "--min-split-size=1M", + "--file-allocation=falloc", # Preallocate file space + "--check-certificate=false", # Corresponds to verify=False + f'--dir="{dest_dir}"', + f'--out="{dest_name}"', + f'"{url}"' + ] + + run_command(" ".join(command)) + print("Download complete.", flush=True) + +def run_command(command: str, cwd: str = os.getcwd(), env: Dict[str, str] = {}) -> None: + print(f"--- Running command: {{{command}}} in {cwd or os.getcwd()} ---", flush=True) + + # Setup environment + process_env = os.environ.copy() + process_env["DEBIAN_FRONTEND"] = "noninteractive" + if env: + process_env.update(env) + + # Explicitly set stdout and stderr to sys.stdout/sys.stderr for real-time output + # This ensures output is visible even when running in ProcessPoolExecutor + process = subprocess.Popen( + command, + shell=True, + cwd=cwd, + env=process_env, + executable="/bin/bash", + stdout=sys.stdout, + stderr=sys.stderr, + bufsize=1, # Line buffered for real-time output + universal_newlines=True + ) + + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command) + +def verify_signature(signature_path: str, data_path: str, public_key: Optional[str] = None) -> bool: + if not os.path.exists(data_path): + raise RuntimeError(f"Data file {data_path} does not exist") + + if not os.path.exists(signature_path): + print(f"No signature file found: {os.path.basename(signature_path)}, skipping verification", flush=True) + return True + + # Detect signature type by extension and use match statement + match os.path.splitext(signature_path)[1]: + case '.minisig': + # Minisign verification using pyminisign + if not public_key: + print(f"Warning: Minisign public key not provided for {os.path.basename(signature_path)}, skipping verification", flush=True) + return True + + print(f"Verifying minisign signature for {os.path.basename(data_path)}...", flush=True) + + try: + import minisign + + # Verify using py-minisign + # verify() raises an exception if verification fails + minisign.verify(signature_path, data_path, public_key) + print("Minisign verification successful.", flush=True) + return True + + except Exception as e: + print(f"Minisign verification failed: {e}", flush=True) + return False + + case '.sig': + # GPG verification using python-gnupg + print(f"Verifying GPG signature for {os.path.basename(data_path)}...", flush=True) + + try: + import gnupg + + gpg = gnupg.GPG() + + # Verify the signature + with open(signature_path, 'rb') as sig_file: + verified = gpg.verify_file(sig_file, data_path) + + if verified.valid: + print("GPG signature verification successful.", flush=True) + return True + else: + print(f"GPG signature verification failed: {verified.status}", flush=True) + return False + + except ImportError: + print("python-gnupg library not found. Skipping signature verification.", flush=True) + return True + except Exception as e: + print(f"Error during GPG signature verification: {e}, skipping", flush=True) + return True + + case _: + print(f"Unknown signature type for {os.path.basename(signature_path)}, skipping verification", flush=True) + return True + +def verify_sha256(file_path: str, expected_hash: str) -> bool: + print(f"Verifying SHA256 for {file_path}...", flush=True) + sha256 = hashlib.sha256() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + sha256.update(chunk) + actual_hash = sha256.hexdigest() + + if actual_hash.lower() == expected_hash.lower(): + print("SHA256 verification successful.", flush=True) + return True + else: + print(f"SHA256 verification failed! Expected {expected_hash}, got {actual_hash}", flush=True) + return False + + +# === Parallel Task Scheduler Classes === + +class Job: + """Represents a single unit of work in the build process.""" + + def __init__(self, name: str, func: Callable, args: Tuple = (), dependencies: Optional[List['Job']] = None) -> None: + self.name = name # Name is only for debugging, not guaranteed unique + self.func = func + self.args = args + self.dependencies: List['Job'] = dependencies or [] + + def __repr__(self) -> str: + return f"Job(name='{self.name}', deps={[d.name for d in self.dependencies]})" + + def __hash__(self) -> int: + return id(self) # Use Python's built-in object id + + def __eq__(self, other) -> bool: + return self is other # Identity comparison + + +class TaskState(Enum): + """Task execution states for better tracking.""" + PENDING = "pending" + READY = "ready" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +def run_job(job: Job) -> Job: + """Executor function to run a job.""" + print(f"--- Starting Job: {job.name} ---", flush=True) + job.func(*job.args) + print(f"--- Finished Job: {job.name} ---", flush=True) + return job # Return the Job object itself + + +class ParallelTaskScheduler: + """ + 🚀 High-Performance Parallel Task Scheduler + + Features: + - Optimal parallel execution with minimal overhead + - Real-time dependency resolution + - Comprehensive progress tracking + - Robust error handling and recovery + - Efficient resource utilization + """ + + def __init__(self, jobs: List[Job]): + # Store all jobs as a set for fast lookups + self.all_jobs: Set[Job] = set(jobs) + + # Build Job-based dependency graph + self.dependencies: Dict[Job, Set[Job]] = { + job: set(job.dependencies) + for job in jobs + } + + # Task states use Job objects as keys + self.task_states: Dict[Job, TaskState] = {job: TaskState.PENDING for job in jobs} + self.running_futures: Dict[concurrent.futures.Future, Job] = {} # future -> Job mapping + self.completed_jobs: Set[Job] = set() + self.failed_jobs: Set[Job] = set() + + # Performance tracking uses Job objects as keys + self.start_time: Optional[float] = None + self.job_start_times: Dict[Job, float] = {} + self.job_durations: Dict[Job, float] = {} + + # Build string-based graph ONLY for TopologicalSorter (library requirement) + # Map job ID to Job object for reverse lookup + self.id_to_job: Dict[int, Job] = {id(job): job for job in jobs} + string_deps: Dict[int, Set[int]] = { + id(job): {id(dep) for dep in job.dependencies} + for job in jobs + } + + # Initialize dependency sorter with IDs + self.sorter = TopologicalSorter(string_deps) + self.sorter.prepare() + + # Reverse dependency mapping: which jobs depend on this job + self.dependents: Dict[Job, Set[Job]] = defaultdict(set) + for job, deps in self.dependencies.items(): + for dep in deps: + self.dependents[dep].add(job) + + def _get_ready_jobs(self) -> List[Job]: + """Get all jobs that are ready to run (dependencies satisfied).""" + ready_jobs: List[Job] = [] + for job_id in self.sorter.get_ready(): + job = self.id_to_job[job_id] + if self.task_states[job] == TaskState.PENDING: + ready_jobs.append(job) + return ready_jobs + + def _submit_job(self, executor: concurrent.futures.Executor, job: Job) -> concurrent.futures.Future: + """Submit a job for execution.""" + self.task_states[job] = TaskState.RUNNING + self.job_start_times[job] = time.time() + + print(f"🚀 [Scheduler] Starting job: {job.name}", flush=True) + future = executor.submit(run_job, job) + self.running_futures[future] = job + return future + + def _handle_completed_job(self, job: Job, success: bool = True) -> None: + """Handle job completion and update states.""" + duration = time.time() - self.job_start_times[job] + self.job_durations[job] = duration + + if success: + self.task_states[job] = TaskState.COMPLETED + self.completed_jobs.add(job) + self.sorter.done(id(job)) # Tell sorter using object ID + print(f"✅ [Scheduler] Job '{job.name}' completed successfully in {duration:.2f}s", flush=True) + else: + self.task_states[job] = TaskState.FAILED + self.failed_jobs.add(job) + print(f"❌ [Scheduler] Job '{job.name}' failed after {duration:.2f}s", flush=True) + + def _print_progress(self) -> None: + """Print current execution progress.""" + total = len(self.all_jobs) + completed = len(self.completed_jobs) + running = len(self.running_futures) + failed = len(self.failed_jobs) + pending = total - completed - running - failed + + elapsed = time.time() - self.start_time if self.start_time else 0 + + running_job_names = [job.name for job in self.running_futures.values()] + print(f"\n📊 [Progress] Total: {total} | ✅ Done: {completed} | 🏃 Running: {running} | ⏳ Pending: {pending} | ❌ Failed: {failed}", flush=True) + print(f"⏱️ [Time] Elapsed: {elapsed:.1f}s | Running jobs: {running_job_names}", flush=True) + + if completed > 0 and elapsed > 0: + rate = completed / elapsed + eta = (total - completed) / rate if rate > 0 else 0 + print(f"📈 [Stats] Rate: {rate:.2f} jobs/s | ETA: {eta:.1f}s", flush=True) + + def run(self, max_workers: Optional[int] = None) -> None: + """ + Execute all jobs with optimal parallel scheduling. + + Args: + max_workers: Maximum number of parallel workers (default: CPU count) + """ + print("🎯 [Scheduler] Initializing High-Performance Parallel Task Scheduler", flush=True) + print(f"📋 [Scheduler] Total jobs: {len(self.all_jobs)}", flush=True) + total_deps = sum(len(deps) for deps in self.dependencies.values()) + print(f"🔗 [Scheduler] Total dependency edges: {total_deps}", flush=True) + if total_deps > 0: + # Print dependency graph with names for debugging + debug_graph = {job.name: [dep.name for dep in deps] for job, deps in self.dependencies.items() if deps} + print(f" Dependency graph: {debug_graph}", flush=True) + + self.start_time = time.time() + + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + # Submit initial ready jobs + ready_jobs = self._get_ready_jobs() + ready_job_names = [job.name for job in ready_jobs] + print(f"🚦 [Scheduler] Initial ready jobs: {ready_job_names}", flush=True) + + for job in ready_jobs: + self._submit_job(executor, job) + + # Main execution loop + while self.running_futures: + self._print_progress() + + # Wait for at least one job to complete + done_futures, _ = concurrent.futures.wait( + self.running_futures.keys(), + return_when=concurrent.futures.FIRST_COMPLETED + ) + + # Process all completed jobs in this batch + newly_completed: List[Job] = [] + for future in done_futures: + job = self.running_futures[future] + + try: + result = future.result() # This will raise exception if job failed + self._handle_completed_job(job, success=True) + newly_completed.append(job) + except Exception as e: + print(f"💥 [Scheduler] Job '{job.name}' failed with detailed error:", flush=True) + self._handle_completed_job(job, success=False) + + # Implement fail-fast: cancel all running jobs and exit immediately + print(f"🛑 [Scheduler] FAIL-FAST: Cancelling all remaining jobs due to failure in '{job.name}'", flush=True) + for remaining_future in self.running_futures.keys(): + if remaining_future != future: + remaining_future.cancel() + remaining_job = self.running_futures[remaining_future] + print(f"❌ [Scheduler] Cancelled job: {remaining_job.name}", flush=True) + + # Clean up and raise the error immediately + raise RuntimeError(f"❌ Build failed in job '{job.name}': {str(e)}") from e + + # Clean up completed future + del self.running_futures[future] + + # Submit any newly ready jobs + if newly_completed: + ready_jobs = self._get_ready_jobs() + for job in ready_jobs: + if job not in self.running_futures.values(): + self._submit_job(executor, job) + + # Final results + total_time = time.time() - self.start_time + self._print_final_report(total_time) + + # Note: With fail-fast implementation, we won't reach here if any job failed + # The exception will be raised immediately when the first job fails + + def _print_final_report(self, total_time: float) -> None: + """Print comprehensive execution report.""" + print("\n" + "="*60, flush=True) + print("🎉 PARALLEL TASK EXECUTION COMPLETED!", flush=True) + print("="*60, flush=True) + + print(f"⏱️ Total execution time: {total_time:.2f}s", flush=True) + print(f"✅ Successfully completed: {len(self.completed_jobs)}/{len(self.all_jobs)} jobs", flush=True) + + if self.failed_jobs: + print(f"❌ Failed jobs: {len(self.failed_jobs)}", flush=True) + for job in self.failed_jobs: + print(f" - {job.name}", flush=True) + + # Show job timing analysis + if self.job_durations: + print(f"\n📊 Job Performance Analysis:", flush=True) + sorted_jobs = sorted(self.job_durations.items(), key=lambda x: x[1], reverse=True) + print(f" Slowest jobs:", flush=True) + for job, duration in sorted_jobs[:5]: + print(f" - {job.name:<30} {duration:>8.2f}s", flush=True) + + avg_duration = sum(self.job_durations.values()) / len(self.job_durations) + print(f" Average job duration: {avg_duration:.2f}s", flush=True) + + # Calculate theoretical sequential time vs actual parallel time + sequential_time = sum(self.job_durations.values()) + speedup = sequential_time / total_time if total_time > 0 else 1 + efficiency = speedup / max(len(self.running_futures), 1) * 100 + + print(f" Sequential time would be: {sequential_time:.2f}s", flush=True) + print(f" Parallel speedup: {speedup:.2f}x", flush=True) + print(f" Parallel efficiency: {efficiency:.1f}%", flush=True) + + print("="*60, flush=True) + + +# ======================================================================== +# 🛠️ Component Build Utilities +# ======================================================================== + +def install_download_prerequisites(component: Component) -> None: + print("⬇️ [SETUP] Installing download prerequisites (aria2c, gnupg)...", flush=True) + download_prerequisites = component.download_prerequisites + pkg_list = " ".join(download_prerequisites) + run_command(f"apt install -y --no-install-recommends=true -o DPkg::Lock::Timeout=-1 {pkg_list}") + print("✅ [SETUP] Download tools ready", flush=True) + +def install_extract_prerequisites(component: Component) -> None: + print("📂 [SETUP] Installing archive extraction tools...", flush=True) + extract_prerequisites = component.extract_prerequisites + pkg_list = " ".join(extract_prerequisites) + run_command(f"apt install -y --no-install-recommends=true -o DPkg::Lock::Timeout=-1 {pkg_list}") + print("✅ [SETUP] Extraction tools ready", flush=True) + +def download_and_verify(component: Component) -> None: + version = component.version + print(f"⬇️ [DOWNLOAD] Fetching {component.name} v{version}...", flush=True) + + # Ensure directories exist + os.makedirs(component.cache_dir, exist_ok=True) + + # Construct download paths and URLs + tarball_name = component.tarball_name + tarball_path = os.path.join(component.cache_dir, tarball_name) + tarball_url = component.tarball_url + + # Download main source archive + download_file(tarball_url, tarball_path) + + # Handle signature verification when available + if component.verification_name_pattern: + signature_name = component.verification_name + signature_path = os.path.join(component.cache_dir, signature_name) + signature_url = component.verification_url + + # Get public key if component has minisig_public_key attribute + public_key = getattr(component, 'minisig_public_key', None) + + try: + print(f"🔐 [VERIFY] Downloading signature for {component.name}...", flush=True) + download_file(signature_url, signature_path) + verify_signature(signature_path, tarball_path, public_key) + print(f"✅ [VERIFY] {component.name} signature verified", flush=True) + except Exception as e: + print(f"❌ [ERROR] Signature verification failed for {component.name}: {e}", file=sys.stderr, flush=True) + shutil.rmtree(component.cache_dir, ignore_errors=True) + raise + else: + print(f"⚠️ [INFO] No signature verification available for {component.name}", flush=True) + +def extract_package(archive_path: str, target_dir: str, strip_top_level: bool = True) -> None: + print(f"📂 [EXTRACT] Unpacking package...") + print(f" 📁 Source: {archive_path}") + print(f" 📁 Target: {target_dir}") + + # Ensure target directory exists + os.makedirs(target_dir, exist_ok=True) + + # Auto-detect compression format + if archive_path.endswith(".tar.xz"): + mode = "r:xz" + elif archive_path.endswith(".tar.gz") or archive_path.endswith(".tgz"): + mode = "r:gz" + elif archive_path.endswith(".tar.bz2"): + mode = "r:bz2" + elif archive_path.endswith(".tar"): + mode = "r" + else: + raise ValueError(f"Unsupported archive format: {archive_path}") + + # Extract to target directory + with tarfile.open(archive_path, mode) as tar: + tar.extractall(path=target_dir, filter='data') + + # Optionally strip single top-level directory + if strip_top_level: + extracted_items = os.listdir(target_dir) + + if len(extracted_items) == 1 and os.path.isdir(os.path.join(target_dir, extracted_items[0])): + # Single top-level directory found - strip it + top_dir_name = extracted_items[0] + top_dir_path = os.path.join(target_dir, top_dir_name) + print(f" 🔄 Stripping top-level directory: {top_dir_name}") + + # Move all contents from top_dir to parent (target_dir) + for item in os.listdir(top_dir_path): + src = os.path.join(top_dir_path, item) + dst = os.path.join(target_dir, item) + shutil.move(src, dst) + + # Remove the now-empty top-level directory + os.rmdir(top_dir_path) + + print(f"✅ [EXTRACT] package extraction complete") + +def extract_source(component: ToolchainSubComponent) -> None: + """ + Extract source code for a toolchain component. + Wrapper around extract_package for backward compatibility. + """ + version = component.version + os.makedirs(component.src_dir, exist_ok=True) + + archive_path = os.path.join(component.cache_dir, component.tarball_name) + extract_package( + archive_path=archive_path, + target_dir=component.extracted_dir, + strip_top_level=True, + ) diff --git a/docker/linux/utility/common.sh b/docker/linux/utility/common.sh new file mode 100644 index 00000000..814a9f43 --- /dev/null +++ b/docker/linux/utility/common.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# ======================================================================== +# Clice Development Container Common Variables +# ======================================================================== + +set -e + +# ======================================================================== +# ⚙️ Default Configuration +# ======================================================================== + +# These are the default values that can be overridden by command-line arguments +DEFAULT_COMPILER="clang" +DEFAULT_VERSION="latest" +DEFAULT_BUILD_STAGE="packed-image" +# where clice is located inside the docker container +CLICE_DIR="/clice" +# pwd inside the container when you open a shell +DEFAULT_CONTAINER_WORKDIR="${CLICE_DIR}" + +DOCKERFILE_PATH="docker/linux/Dockerfile" + +# ======================================================================== +# 🏷️ Naming Convention Functions +# ======================================================================== + +# Generates the base image tag. +# Usage: get_image_tag +get_image_tag() { + local compiler="$1" + local version="$2" + echo "linux-${compiler}-v${version}" +} + +# Generates the full name for the packed (release) image. +# Usage: get_packed_image_name +get_packed_image_name() { + local compiler="$1" + local version="$2" + local image_tag + image_tag=$(get_image_tag "$compiler" "$version") + echo "clice.io/clice-dev:${image_tag}" +} + +# Generates the full name for the expanded (development) image. +# Usage: get_expanded_image_name +get_expanded_image_name() { + local packed_image_name + packed_image_name=$(get_packed_image_name "$1" "$2") + echo "${packed_image_name}-expanded" +} + +# Generates the name for the development container. +# Usage: get_container_name +get_container_name() { + local compiler="$1" + local version="$2" + echo "clice_dev-linux-${compiler}-v${version}" +} diff --git a/docker/linux/utility/container-entrypoint.sh b/docker/linux/utility/container-entrypoint.sh new file mode 100644 index 00000000..66e26d8d --- /dev/null +++ b/docker/linux/utility/container-entrypoint.sh @@ -0,0 +1,33 @@ +# ======================================================================== +# 🚀 Clice Dev Container Shell Initialization +# ======================================================================== +# File: docker/linux/utility/container-entrypoint.sh +# Purpose: Bash initialization script for Clice dev container +# +# This script is sourced by .bashrc and performs: +# 1. Runs uv sync to create/update virtual environment if needed +# 2. Auto-activates virtual environment for interactive shells +# +# Usage: This file will be appended to /root/.bashrc during image build +# ======================================================================== + +# Only run in interactive shells to avoid breaking non-interactive scripts +if [[ $- == *i* ]]; then + UV_PACKAGE_ROOT=$(find "${RELEASE_PACKAGE_DIR}" -maxdepth 1 -type d -name "uv-*" | head -n 1) + UV_PACKAGE_CACHE_DIR="${UV_PACKAGE_ROOT}/${UV_PACKAGE_DIR_NAME}" + + echo "📦 Running uv sync..." + + if UV_CACHE_DIR="${UV_PACKAGE_CACHE_DIR}" uv sync --project "${CLICE_WORKDIR}/tests/pyproject.toml"; then + echo "✅ Python environment ready at ${CLICE_WORKDIR}/.venv" + else + echo "⚠️ Failed to sync Python environment (pyproject.toml might not exist)" + fi + + # Auto-activate virtual environment if it exists + if [ -f "${CLICE_WORKDIR}/.venv/bin/activate" ]; then + source "${CLICE_WORKDIR}/.venv/bin/activate" + fi +fi + +alias ll='ls -alF --color=auto' diff --git a/docker/linux/utility/create_release_package.py b/docker/linux/utility/create_release_package.py new file mode 100644 index 00000000..cf68dce2 --- /dev/null +++ b/docker/linux/utility/create_release_package.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +Stage 3: Create final release package by merging toolchain and dependencies, +generating manifest, and packaging into 7z SFX archive. +""" + +import os +import sys +import json + +# Add project root to Python path +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from config.docker_build_stages.common import ( + COMPILER, + RELEASE_PACKAGE_DIR, + PACKED_RELEASE_PACKAGE_PATH, + CLICE_WORKDIR, + DEVELOPMENT_SHELL_VARS, + TOOLCHAIN_VERSIONS, +) +from config.docker_build_stages.toolchain_config import TOOLCHAIN +from config.docker_build_stages.dependencies_config import UV +from config.docker_build_stages.package_config import ( + ALL_COMPONENTS, + BASHRC, + P7ZIP, +) + +from build_utils import ( + Job, + ParallelTaskScheduler, + run_command +) + +# ======================================================================== +# 🌍 Environment Setup Functions +# ======================================================================== + +def setup_environment_variables_and_entrypoint() -> None: + """Create .bashrc with environment variables and container entrypoint script.""" + print("🌍 Setting up .bashrc with environment variables and entrypoint script...") + + # Read container entrypoint script from BashrcComponent + entrypoint_script_path = BASHRC.entrypoint_script_source + + with open(entrypoint_script_path, 'r') as f: + entrypoint_content = f.read() + + # Create .bashrc in BASHRC component package directory + bashrc_path = BASHRC.bashrc_path + os.makedirs(os.path.dirname(bashrc_path), exist_ok=True) + + # Write complete .bashrc + with open(bashrc_path, 'w') as f: + f.write("# ========================================================================\n") + f.write("# 🚀 Clice Dev Container - Bash Configuration\n") + f.write("# ========================================================================\n") + f.write("# This file is auto-generated during image packaging.\n") + f.write("# It contains:\n") + f.write("# 1. Exported environment variables from DEVELOPMENT_SHELL_VARS\n") + f.write("# 2. Internal variables for container entrypoint (not exported)\n") + f.write("# 3. Container entrypoint script (auto Python environment setup)\n") + f.write("# ========================================================================\n\n") + + # Export environment variables from DEVELOPMENT_SHELL_VARS + f.write("# Exported environment variables (from DEVELOPMENT_SHELL_VARS)\n") + for key, value in DEVELOPMENT_SHELL_VARS.items(): + f.write(f'export {key}="{value}"\n') + f.write("\n") + + # Export compiler-specific environment variables + f.write("# Compiler environment variables\n") + match COMPILER: + case "gcc": + gcc_path = f"/usr/bin/gcc-{TOOLCHAIN_VERSIONS['gcc']}" + gxx_path = f"/usr/bin/g++-{TOOLCHAIN_VERSIONS['gcc']}" + f.write(f'export CC="{gcc_path}"\n') + f.write(f'export CXX="{gxx_path}"\n') + # For GCC/Clang, SYSROOT points to custom-built glibc/libstdc++ + sysroot = TOOLCHAIN.sysroot_dir + f.write(f'export SYSROOT="{sysroot}"\n') + case "clang": + clang_path = f"/usr/bin/clang-{TOOLCHAIN_VERSIONS['clang']}" + clangxx_path = f"/usr/bin/clang++-{TOOLCHAIN_VERSIONS['clang']}" + f.write(f'export CC="{clang_path}"\n') + f.write(f'export CXX="{clangxx_path}"\n') + # For GCC/Clang, SYSROOT points to custom-built glibc/libstdc++ + sysroot = TOOLCHAIN.sysroot_dir + f.write(f'export SYSROOT="{sysroot}"\n') + case "zig": + zig_bin = os.path.join(TOOLCHAIN.zig.package_dir, 'zig') + f.write(f'export CC="{zig_bin} cc"\n') + f.write(f'export CXX="{zig_bin} c++"\n') + # Zig uses its own bundled libc, no traditional sysroot + f.write('# Note: Zig uses its own bundled libc\n') + case _: + raise ValueError(f"Unsupported compiler: {COMPILER}") + f.write("\n") + + # Set internal variables for container entrypoint (not exported) + f.write("# Internal variables for container entrypoint (not exported to user environment)\n") + f.write(f'CLICE_WORKDIR="{CLICE_WORKDIR}"\n') + f.write(f'RELEASE_PACKAGE_DIR="{RELEASE_PACKAGE_DIR}"\n') + f.write('UV_PACKAGE_DIR_NAME="uv-packages"\n') + f.write("\n") + + # Write container entrypoint script + f.write("# ========================================================================\n") + f.write("# Container Entrypoint Script - Auto Python Environment Setup\n") + f.write("# ========================================================================\n") + f.write(entrypoint_content) + f.write("\n") + + print(f"✅ .bashrc created at {bashrc_path}") + print(f" 📝 Exported variables: {len(DEVELOPMENT_SHELL_VARS)} from DEVELOPMENT_SHELL_VARS") + for key in DEVELOPMENT_SHELL_VARS.keys(): + print(f" • {key}") + print(f" 📝 Compiler: {COMPILER}") + print(f" • CC and CXX configured for {COMPILER}") + if COMPILER in ["gcc", "clang"]: + print(f" • SYSROOT={TOOLCHAIN.sysroot_dir}") + print(" 📝 Internal variables: CLICE_WORKDIR, RELEASE_PACKAGE_DIR, UV_PACKAGE_DIR_NAME") + print(" 📝 Container entrypoint script embedded") + +# ======================================================================== +# 📋 Manifest Creation Functions +# ======================================================================== + +def create_comprehensive_manifest() -> None: + print("📋 Creating comprehensive release manifest based on ALL_COMPONENTS...") + + # Create base manifest structure + manifest = { + "release_info": { + "created_at": os.stat(RELEASE_PACKAGE_DIR).st_ctime if os.path.exists(RELEASE_PACKAGE_DIR) else None, + "stage": "final_release", + "version": "1.0.0" + }, + "components": {}, + "summary": { + "total_components": 0, + "available_components": 0, + "total_files": 0, + "total_size_mb": 0.0 + } + } + + # Process each component from ALL_COMPONENTS + for component in ALL_COMPONENTS: + package_dir = component.package_dir + + component_info = { + "name": component.name, + "type": component.__class__.__name__, + "version": getattr(component, 'version', 'unknown'), + "file_count": 0, + "size_mb": 0.0 + } + + # Calculate component statistics + file_count = sum(len(files) for _, _, files in os.walk(package_dir)) + dir_size = sum( + os.path.getsize(os.path.join(dirpath, filename)) + for dirpath, _, filenames in os.walk(package_dir) + for filename in filenames + ) / (1024 * 1024) # Convert to MB + + component_info["file_count"] = file_count + component_info["size_mb"] = round(dir_size, 2) + + # Component-specific details + match component.name: + case "apt": + # Count APT packages + apt_packages = [] + for file in os.listdir(package_dir): + if file.endswith('.deb'): + pkg_name = file.split('_')[0] + if pkg_name not in apt_packages: + apt_packages.append(pkg_name) + component_info["packages"] = sorted(apt_packages) + component_info["package_count"] = len(apt_packages) + + case "uv": + # UV and Python version information + component_info["uv_details"] = { + "uv_version": UV.version, + "python_version": UV.python_version, + } + + case "toolchain": + # Toolchain specific information + component_info["toolchain_details"] = { + "glibc_version": TOOLCHAIN.glibc.version, + "gcc_version": TOOLCHAIN.gcc.version, + "linux_version": TOOLCHAIN.linux.version, + # "llvm_version": TOOLCHAIN.llvm.version, + } + + case "clice-setup-scripts": + # Setup scripts information - executed in-place + component_info["note"] = "Executed in-place during expansion, not extracted to CLICE_WORKDIR" + component_info["structure"] = "Complete directory tree (config/, docker/linux/utility/)" + + case "bashrc": + # Bashrc information + bashrc_file = os.path.join(package_dir, ".bashrc") + component_info["bashrc_path"] = bashrc_file + component_info["bashrc_size_kb"] = round(os.path.getsize(bashrc_file) / 1024, 2) + + manifest["components"][component.name] = component_info + manifest["summary"]["total_components"] += 1 + manifest["summary"]["available_components"] += 1 + manifest["summary"]["total_files"] += component_info["file_count"] + manifest["summary"]["total_size_mb"] += component_info["size_mb"] + + # Round summary size + manifest["summary"]["total_size_mb"] = round(manifest["summary"]["total_size_mb"], 2) + + # Write manifest to release directory + manifest_file = os.path.join(RELEASE_PACKAGE_DIR, "manifest.json") + os.makedirs(RELEASE_PACKAGE_DIR, exist_ok=True) + with open(manifest_file, 'w') as f: + json.dump(manifest, f, indent=2) + + print(f"✅ Comprehensive manifest created: {manifest_file}") + print(f"📊 Components: {manifest['summary']['available_components']}/{manifest['summary']['total_components']} available") + print(f"📁 Total files: {manifest['summary']['total_files']}") + print(f"📦 Total size: {manifest['summary']['total_size_mb']} MB") + +def install_p7zip() -> None: + print("📦 Installing p7zip for archive creation...") + + packages = " ".join(P7ZIP.build_prerequisites) + run_command(f"apt install -y --no-install-recommends -o DPkg::Lock::Timeout=-1 {packages}") + print("✅ p7zip installed successfully") + +def create_final_release_package() -> None: + """Create self-extracting 7z archive containing all components.""" + print("📦 Creating self-extracting release package with 7z SFX...") + + if not os.path.exists(RELEASE_PACKAGE_DIR): + print("⚠️ No release package directory found") + return + + # Ensure parent directory exists for the packed file + packed_dir = os.path.dirname(PACKED_RELEASE_PACKAGE_PATH) + os.makedirs(packed_dir, exist_ok=True) + + print(f" 📁 Source: {RELEASE_PACKAGE_DIR}") + print(f" 📁 Target: {PACKED_RELEASE_PACKAGE_PATH}") + + + # Create self-extracting archive using 7z with SFX module + # The -sfx option creates a self-extracting executable + print(f"🔧 Creating SFX archive with settings: {P7ZIP.compression_options}...") + + seven_zip_cmd = ( + f"7z a {P7ZIP.sfx_option} {" ".join(P7ZIP.compression_options)} " + f"{PACKED_RELEASE_PACKAGE_PATH} " + f"{RELEASE_PACKAGE_DIR}/*" + ) + run_command(seven_zip_cmd) + + # Report package statistics + package_size_mb = os.path.getsize(PACKED_RELEASE_PACKAGE_PATH) / (1024 * 1024) + + print(f"✅ Self-extracting release package created: {PACKED_RELEASE_PACKAGE_PATH}") + print(f"📊 Package size: {package_size_mb:.1f} MB") + print(f"ℹ️ Extract with: {PACKED_RELEASE_PACKAGE_PATH} -o") + +# ======================================================================== +# 🚀 Main Execution +# ======================================================================== + +def main() -> None: + print("🚀 ========================================================================") + print("🚀 CLICE RELEASE PACKAGE CREATOR - STAGE 3") + print("🚀 ========================================================================") + print("📦 Creating final release package from merged stage outputs") + print("🚀 ========================================================================\n") + + # Define packaging jobs with proper dependency management + setup_bashrc_job = Job("setup_bashrc", setup_environment_variables_and_entrypoint, ()) + install_p7zip_job = Job("install_p7zip", install_p7zip, ()) + create_manifest_job = Job("create_manifest", create_comprehensive_manifest, (), [setup_bashrc_job]) + create_package_job = Job("create_package", create_final_release_package, (), [create_manifest_job, install_p7zip_job]) + + all_jobs = [ + setup_bashrc_job, + install_p7zip_job, + create_manifest_job, + create_package_job, + ] + + # Execute packaging tasks in parallel where possible + scheduler = ParallelTaskScheduler(all_jobs) + scheduler.run() + + print("\n🎉 ========================================================================") + print("🎉 STAGE 3 COMPLETED SUCCESSFULLY!") + print("🎉 ========================================================================") + print(f"✅ Final release package: {PACKED_RELEASE_PACKAGE_PATH}") + print(f"✅ Manifest: {RELEASE_PACKAGE_DIR}/manifest.json") + print(f"✅ Bashrc: {BASHRC.bashrc_path}") + print("🎉 ========================================================================") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker/linux/utility/download_dependencies.py b/docker/linux/utility/download_dependencies.py new file mode 100644 index 00000000..bc28f009 --- /dev/null +++ b/docker/linux/utility/download_dependencies.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +""" +Download all dev dependencies (APT packages, CMake, XMake, Python packages) +without installing them for Docker build cache efficiency. +""" + +import os +import shutil +import sys +import subprocess +from typing import List, Dict, Set, Optional + +# Add project root to Python path +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from build_utils import ( + Job, + ParallelTaskScheduler, + download_file, + run_command, + verify_sha256 +) +from config.docker_build_stages.common import RELEASE_PACKAGE_DIR +from config.docker_build_stages.dependencies_config import APT, UV, CMAKE, XMAKE + +# ======================================================================== +# 🛠️ Download Task Implementations +# ======================================================================== + +def install_download_prerequisites() -> None: + print("📦 Installing dependencies download prerequisites...") + + # Install all download prerequisites (universal + APT-specific) + download_prerequisites: List[str] = APT.download_prerequisites + run_command(f"apt install -y --no-install-recommends=true -o DPkg::Lock::Timeout=-1 {' '.join(download_prerequisites)}") + + print(f"✅ Installed {len(download_prerequisites)} download prerequisites") + +def get_apt_package_list(base_packages: List[str]) -> List[str]: + """Get recursive APT dependencies using apt-cache depends + awk pattern.""" + print("🔍 Resolving recursive dependencies using StackOverflow command pattern...") + + all_packages: Set[str] = set() + + for package in base_packages: + try: + # Use the exact command from StackOverflow + apt_cache_cmd: List[str] = [ + "apt-cache", "depends", "--recurse", "--no-recommends", + "--no-suggests", "--no-conflicts", "--no-breaks", + "--no-replaces", "--no-enhances", package + ] + + # Run apt-cache depends command + result: subprocess.CompletedProcess[str] = subprocess.run( + apt_cache_cmd, capture_output=True, text=True, check=True + ) + + # Use awk pattern to extract dependency packages: $1 ~ /^Depends:/{print $2} + for line in result.stdout.split('\n'): + line = line.strip() + if line.startswith('Depends:'): + # Extract the package name after "Depends: " + parts: List[str] = line.split() + if len(parts) >= 2: + pkg_name: str = parts[1] + # Remove architecture suffix and version constraints + pkg_name = pkg_name.split(':')[0].split('(')[0].split('[')[0].strip() + if pkg_name and not pkg_name.startswith('<') and pkg_name != '|': + all_packages.add(pkg_name) + + except subprocess.CalledProcessError as e: + print(f"⚠️ Warning: Could not resolve dependencies for {package}: {e}") + # Add the original package as fallback + all_packages.add(package) + + # Filter available packages (remove virtual/unavailable packages) + print(f"🔍 Found {len(all_packages)} total dependency packages, filtering available ones...") + available_packages_set: Set[str] = set(base_packages) + + for package in sorted(all_packages): + # Quick availability check + result = subprocess.run( + ["apt-cache", "show", package], + capture_output=True, text=True, check=True + ) + if result.returncode == 0: + available_packages_set.add(package) + + available_packages: List[str] = sorted(available_packages_set) + print(f"📋 Final package list: {len(available_packages)} available packages") + return available_packages + +def download_apt_packages() -> None: + print("📦 Downloading APT packages with StackOverflow command pattern...") + + # Create both download cache and package directories using component structure + os.makedirs(APT.cache_dir, exist_ok=True) + os.makedirs(APT.package_dir, exist_ok=True) + + # Stage 1: Get package list + base_packages: List[str] = list(set(APT.all_packages)) + print(f"📋 Base packages from config: {len(base_packages)} packages") + + available_packages: List[str] = get_apt_package_list(base_packages) + + # Stage 2: Download packages using apt-get download + print(f"📥 Downloading {len(available_packages)} packages to cache: {APT.cache_dir}") + + # Download all packages at once + packages_str: str = ' '.join(available_packages) + run_command(f"apt-get download {packages_str}", cwd=APT.cache_dir) + + print(f"✅ Downloaded {len(available_packages)} packages to cache") + + # Count actual .deb files in cache directory + cached_deb_count: int = len([f for f in os.listdir(APT.cache_dir) if f.endswith('.deb')]) + print(f"📋 Found {cached_deb_count} .deb files in cache directory") + + # Stage 3: Parse all packages at once with apt-cache show to get real packages and their info + print("📦 Parsing package information to identify real packages and versions...") + + # Get system architecture + arch_result: subprocess.CompletedProcess[str] = subprocess.run( + ["dpkg", "--print-architecture"], + capture_output=True, + text=True, + check=True + ) + system_arch: str = arch_result.stdout.strip() + + # Map package name -> exact filename (only for real packages) + package_to_filename: Dict[str, str] = {} + virtual_packages: List[str] = [] + + for pkg in available_packages: + try: + # Single apt-cache show call to get all info at once + show_result: subprocess.CompletedProcess[str] = subprocess.run( + ["apt-cache", "show", pkg], + capture_output=True, + text=True, + check=True + ) + + # Parse the output to extract Package, Version, and Architecture + package_name: Optional[str] = None + version: Optional[str] = None + pkg_arch: str = system_arch # Default + + for line in show_result.stdout.split('\n'): + if line.startswith('Package:'): + package_name = line.split(':', 1)[1].strip() + elif line.startswith('Version:'): + version = line.split(':', 1)[1].strip() + elif line.startswith('Architecture:'): + pkg_arch = line.split(':', 1)[1].strip() + + # Stop after first package stanza (in case of multiple versions) + if line.strip() == '' and package_name and version: + break + + # Check if this is a virtual package (Package field doesn't match query) + if not package_name or package_name != pkg: + virtual_packages.append(pkg) + print(f"📝 Skipping virtual package: {pkg} (resolves to {package_name})") + continue + + if not version: + print(f"⚠️ No version found for {pkg}") + continue + + # Construct expected filename based on Debian package naming convention + # URL-encode colons in version + encoded_version: str = version.replace(':', '%3a') + expected_filename: str = f"{pkg}_{encoded_version}_{pkg_arch}.deb" + + package_to_filename[pkg] = expected_filename + + except subprocess.CalledProcessError as e: + # If apt-cache show fails, it's likely a virtual package + virtual_packages.append(pkg) + print(f"📝 Skipping virtual package (no show output): {pkg}") + + print(f"📋 Identified {len(package_to_filename)} real packages with .deb files") + print(f"📝 Identified {len(virtual_packages)} virtual packages (no .deb files)") + + # Also verify that actual .deb files in cache match what we copied + if len(virtual_packages) + len(package_to_filename) != len(available_packages): + error_msg: str = f"File count mismatch: {len(available_packages)} available vs {len(package_to_filename)} real + {len(virtual_packages)} virtual" + print(f"❌ {error_msg}") + raise RuntimeError(error_msg) + + # Stage 4: Copy only the exact files + print("📦 Copying exact package files from cache to package directory...") + + for pkg, filename in package_to_filename.items(): + src: str = os.path.join(APT.cache_dir, filename) + dst: str = os.path.join(APT.package_dir, filename) + + shutil.copy2(src, dst) + + print(f"📊 Copied {len(package_to_filename)} real packages") + print(f"📝 Skipped {len(virtual_packages)} virtual packages (no .deb files)") + print(f"✅ Verification passed: Download count matches copy count") + print(f"📁 Cache directory: {APT.cache_dir} (preserved for future builds)") + +def download_cmake() -> None: + """Download CMake installer and verify SHA256 integrity.""" + print("🔧 Downloading CMake with verification...") + + # Create both cache and package directories using component structure + os.makedirs(CMAKE.cache_dir, exist_ok=True) + os.makedirs(CMAKE.package_dir, exist_ok=True) + + # Use CMake component configuration + cmake_filename: str = CMAKE.tarball_name + cmake_url: str = CMAKE.tarball_url + + # Download to cache directory first + cmake_cache_file: str = f"{CMAKE.cache_dir}/{cmake_filename}" + cmake_package_file: str = f"{CMAKE.package_dir}/{cmake_filename}" + + # Download CMake installer (.sh script) to cache + download_file(cmake_url, cmake_cache_file) + + # Download verification files to cache using component structure + sha_url: str = CMAKE.verification_url + sha_filename: str = CMAKE.verification_name + sha_cache_file: str = f"{CMAKE.cache_dir}/{sha_filename}" + + # Download SHA file for integrity verification + download_file(sha_url, sha_cache_file) + + # Verify CMake file integrity using build_utils + with open(sha_cache_file, 'r') as f: + sha_content: str = f.read().strip() + # Parse SHA file format: "hash filename" + for line in sha_content.split('\n'): + if cmake_filename in line: + expected_hash: str = line.split()[0] + if verify_sha256(cmake_cache_file, expected_hash): + print("✅ CMake file integrity verification successful") + else: + print("❌ CMake file integrity verification failed") + # Delete all files in cache directory on verification failure + shutil.rmtree(CMAKE.cache_dir, ignore_errors=True) + raise RuntimeError("CMake file integrity verification failed") + break + else: + print("⚠️ CMake file not found in SHA file, skipping verification") + + # Copy verified file from cache to package directory + shutil.copy2(cmake_cache_file, cmake_package_file) + + print(f"✅ CMake downloaded to cache: {cmake_cache_file}") + print(f"📦 CMake copied to package: {cmake_package_file}") + +def download_xmake() -> None: + print("🔨 Downloading XMake bundle...") + + # Create both cache and package directories using component structure + os.makedirs(XMAKE.cache_dir, exist_ok=True) + os.makedirs(XMAKE.package_dir, exist_ok=True) + + # Use XMake component configuration + xmake_filename: str = XMAKE.tarball_name + xmake_url: str = XMAKE.tarball_url + + # Download to cache directory first + xmake_cache_file: str = f"{XMAKE.cache_dir}/{xmake_filename}" + xmake_package_file: str = f"{XMAKE.package_dir}/{xmake_filename}" + + # Download XMake bundle to cache + download_file(xmake_url, xmake_cache_file) + + # Make it executable in cache + os.chmod(xmake_cache_file, 0o755) + + # Copy from cache to package directory + shutil.copy2(xmake_cache_file, xmake_package_file) + + print(f"✅ XMake downloaded to cache: {xmake_cache_file}") + print(f"📦 XMake copied to package: {xmake_package_file}") + +def download_python_packages() -> None: + print("🐍 Downloading Python packages from pyproject.toml...") + + # Create cache directory for packages + os.makedirs(UV.packages_package_dir, exist_ok=True) + + # Set UV_CACHE_DIR to packages cache directory + print(f"📥 Downloading package wheels to UV packages package dir: {UV.packages_package_dir}") + print(f"📋 Using pyproject.toml from: {UV.pyproject_file_path}") + + # Run uv sync with project root as working directory + # UV will automatically find pyproject.toml in the project root + run_command( + f"UV_CACHE_DIR={UV.packages_package_dir} uv sync --no-install-project --no-editable", + cwd=os.path.dirname(UV.pyproject_file_path) + ) + + print(f"✅ Package wheels cached to: {UV.packages_package_dir}") + print(f"📁 Packages cache will be available to later stages via cache mount") + +# LLVM downloading removed as per requirements + +# ======================================================================== +# 🚀 Main Execution +# ======================================================================== + +def main() -> None: + print("🚀 Starting Clice Dependencies Download Process...") + + # Create main cache directory + os.makedirs(RELEASE_PACKAGE_DIR, exist_ok=True) + + # Define download jobs with proper dependency management + install_download_prereq_job = Job("install_download_prerequisites", install_download_prerequisites, ()) + download_apt_job = Job("download_apt_packages", download_apt_packages, (), [install_download_prereq_job]) + download_python_job = Job("download_python_packages", download_python_packages, (), [install_download_prereq_job]) + download_cmake_job = Job("download_cmake", download_cmake, (), [install_download_prereq_job]) + download_xmake_job = Job("download_xmake", download_xmake, (), [install_download_prereq_job]) + + all_jobs = [ + install_download_prereq_job, + download_apt_job, + download_python_job, + download_cmake_job, + download_xmake_job, + ] + + # Execute downloads in parallel where possible + scheduler = ParallelTaskScheduler(all_jobs) + scheduler.run() + + print("✅ All dependencies downloaded successfully!") + print(f"📁 Cache directory: {RELEASE_PACKAGE_DIR}") + +if __name__ == "__main__": + main() diff --git a/docker/linux/utility/local_setup.py b/docker/linux/utility/local_setup.py new file mode 100644 index 00000000..0bf129d7 --- /dev/null +++ b/docker/linux/utility/local_setup.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Final setup: Install pre-downloaded packages (APT, toolchain, CMake, XMake, Python) +and deploy .bashrc configuration. +""" + +from typing import List, Optional +import os +import sys +import shutil + +# Ensure utility directory is in Python path for imports +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from config.docker_build_stages.common import ( + COMPILER, + RELEASE_PACKAGE_DIR, + CLICE_WORKDIR, + TOOLCHAIN_VERSIONS, +) +from config.docker_build_stages.dependencies_config import APT, UV, CMAKE, XMAKE, APTComponent, CMakeComponent, UVComponent, XMakeComponent +from config.docker_build_stages.toolchain_config import TOOLCHAIN, ToolchainComponent +from config.docker_build_stages.package_config import BASHRC + +from build_utils import ( + Job, + ParallelTaskScheduler, + run_command +) + +# ======================================================================== +# 🌍 Environment Setup Functions +# ======================================================================== + +def deploy_bashrc() -> None: + """Deploy .bashrc from package to /root/.bashrc.""" + print("🌍 Deploying .bashrc configuration...") + + source_bashrc = BASHRC.bashrc_path + target_bashrc = "/root/.bashrc" + + if not os.path.exists(source_bashrc): + print(f"⚠️ Warning: .bashrc not found at {source_bashrc}") + return + + # Copy .bashrc to target location + shutil.copy2(source_bashrc, target_bashrc) + + print(f"✅ .bashrc deployed to {target_bashrc}") + +# ======================================================================== +# 📦 Package Installation Functions +# ======================================================================== + +def install_apt_packages(apt_component: APTComponent) -> None: + print("📦 Installing APT development packages...") + + # Install all .deb files found in the package directory + deb_files = [f for f in os.listdir(apt_component.package_dir) if f.endswith('.deb')] + + if deb_files: + # Use dpkg to install all .deb files, with apt-get fallback for dependencies + run_command(f"dpkg -i {apt_component.package_dir}/*.deb || apt-get install -f -y") + print(f"✅ Installed {len(deb_files)} APT packages") + else: + print("⚠️ No .deb files found in APT package directory") + +def install_toolchain(toolchain_component: ToolchainComponent) -> None: + print("🔧 Installing custom toolchain...") + + match COMPILER: + case "gcc": + run_command(f'update-alternatives --install /usr/bin/cc cc "/usr/bin/gcc-{TOOLCHAIN_VERSIONS["gcc"]}" {TOOLCHAIN_VERSIONS["gcc"]}') + run_command(f'update-alternatives --install /usr/bin/gcc gcc "/usr/bin/gcc-{TOOLCHAIN_VERSIONS["gcc"]}" {TOOLCHAIN_VERSIONS["gcc"]}') + run_command(f'update-alternatives --install /usr/bin/c++ c++ "/usr/bin/g++-{TOOLCHAIN_VERSIONS["gcc"]}" {TOOLCHAIN_VERSIONS["gcc"]}') + run_command(f'update-alternatives --install /usr/bin/g++ g++ "/usr/bin/g++-{TOOLCHAIN_VERSIONS["gcc"]}" {TOOLCHAIN_VERSIONS["gcc"]}') + case "clang": + run_command(f'update-alternatives --install /usr/bin/cc cc "/usr/bin/clang-{TOOLCHAIN_VERSIONS["clang"]}" {TOOLCHAIN_VERSIONS["clang"]}') + run_command(f'update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-{TOOLCHAIN_VERSIONS["clang"]}" {TOOLCHAIN_VERSIONS["clang"]}') + run_command(f'update-alternatives --install /usr/bin/c++ c++ "/usr/bin/clang++-{TOOLCHAIN_VERSIONS["clang"]}" {TOOLCHAIN_VERSIONS["clang"]}') + run_command(f'update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-{TOOLCHAIN_VERSIONS["clang"]}" {TOOLCHAIN_VERSIONS["clang"]}') + case "zig": + # Zig binary is directly in package_dir after stripping top-level directory + zig_bin = os.path.join(toolchain_component.zig.package_dir, 'zig') + + # Setup alternatives for zig, cc, and c++ + run_command(f'update-alternatives --install /usr/bin/zig zig "{zig_bin}" 100') + # run_command(f'update-alternatives --install /usr/bin/cc cc "{zig_bin} cc" 100') + # run_command(f'update-alternatives --install /usr/bin/c++ c++ "{zig_bin} c++" 100') + print(f"✅ Zig compiler configured: {zig_bin}") + case _: + raise ValueError(f"Unsupported compiler specified: {COMPILER}") + + # clice requires to link with lld + run_command(f'update-alternatives --install /usr/bin/ld ld "/usr/bin/lld-{TOOLCHAIN_VERSIONS["clang"]}" {TOOLCHAIN_VERSIONS["clang"]}') + + print(f"✅ Toolchain available at: {toolchain_component.package_dir}") + +def install_cmake(cmake_component: CMakeComponent) -> None: + """Install CMake from pre-downloaded installer.""" + print("🔧 Installing CMake...") + + cmake_installer_filename = cmake_component.tarball_name + cmake_installer_path = os.path.join(cmake_component.package_dir, cmake_installer_filename) + + # Make installer executable and run it + run_command(f"chmod +x {cmake_installer_path}") + + # Use CMAKE component package_dir as install directory + cmake_install_dir = cmake_component.package_dir + os.makedirs(cmake_install_dir, exist_ok=True) + + # Install CMake to the component package directory + run_command(f"{cmake_installer_path} --prefix={cmake_install_dir} --skip-license") + + # Create symlinks to system PATH + cmake_bin_dir = f"{cmake_install_dir}/bin" + if os.path.exists(cmake_bin_dir): + for binary in os.listdir(cmake_bin_dir): + src = os.path.join(cmake_bin_dir, binary) + dst = f"/usr/local/bin/{binary}" + if os.path.isfile(src) and not os.path.exists(dst): + os.symlink(src, dst) + + print(f"✅ CMake installed to {cmake_install_dir}") + +def install_xmake(xmake_component: XMakeComponent) -> None: + """Install XMake from pre-downloaded package.""" + print("🔨 Installing XMake...") + + xmake_filename = xmake_component.tarball_name + xmake_path = os.path.join(xmake_component.package_dir, xmake_filename) + + # Make XMake bundle executable + run_command(f"chmod +x {xmake_path}") + + # Install XMake using update-alternatives + run_command(f"update-alternatives --install /usr/bin/xmake xmake {xmake_path} 100") + + # Run XMake + # First time we execute the bundle, it sets up its internal environment + # Environment variable is not setup yet, so we need --root option to bypass xmake root account check + run_command("xmake --root --version") + + print("✅ XMake installed successfully") + +def install_python_packages(uv_component: UVComponent) -> None: + print("🐍 Installing Python packages...") + + # Install wheel files found in the UV package directory + wheel_files = [f for f in os.listdir(uv_component.package_dir) if f.endswith('.whl')] + + if wheel_files: + # Use uv to install from the cached packages + wheel_paths = [os.path.join(uv_component.package_dir, f) for f in wheel_files] + run_command(f"uv pip install --find-links {uv_component.package_dir} --no-index --force-reinstall --no-deps {' '.join(wheel_paths)}") + print(f"✅ Installed {len(wheel_files)} Python packages") + else: + print("⚠️ No wheel files found in UV package directory") + +# ======================================================================== +# 📋 Setup Orchestration +# ======================================================================== + +def setup_git_safe_directory() -> None: + """Configure git to treat the workspace as safe.""" + print("🔧 Configuring git safe directory...") + + run_command(f"git config --global --add safe.directory {CLICE_WORKDIR}") + print("✅ Git safe directory configured") + +def main() -> None: + print("🚀 Setting up Clice Dev Container...") + + # Define setup jobs with proper dependency management + install_apt_job = Job("install_apt_packages", install_apt_packages, (APT,)) + setup_git_job = Job("setup_git_safe_directory", setup_git_safe_directory, (), [install_apt_job]) + install_toolchain_job = Job("install_toolchain", install_toolchain, (TOOLCHAIN,), [install_apt_job]) + install_cmake_job = Job("install_cmake", install_cmake, (CMAKE,)) + install_xmake_job = Job("install_xmake", install_xmake, (XMAKE,)) + install_python_job = Job("install_python_packages", install_python_packages, (UV,)) + deploy_bashrc_job = Job("deploy_bashrc", deploy_bashrc, ()) + + all_jobs = [ + install_apt_job, + setup_git_job, + install_toolchain_job, + install_cmake_job, + install_xmake_job, + install_python_job, + deploy_bashrc_job, + ] + + # Execute setup tasks in parallel where possible + scheduler = ParallelTaskScheduler(all_jobs) + scheduler.run() + + print("✅ Clice development environment setup completed successfully!") + print(f"📦 Components installed from: {RELEASE_PACKAGE_DIR}") + print(f"📝 Bashrc deployed to: /root/.bashrc") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker/linux/utility/pyproject.toml b/docker/linux/utility/pyproject.toml new file mode 100644 index 00000000..b646e5d5 --- /dev/null +++ b/docker/linux/utility/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "clice-dev-utils" +version = "0.1.0" +description = "Python dependencies for clice's development container build scripts." +requires-python = ">=3.13" +dependencies = [ + "py-minisign>=0.13.0", + "python-gnupg>=0.5.0" +] diff --git a/docs/en/dev/build.md b/docs/en/dev/build.md index 5853464d..de2564ed 100644 --- a/docs/en/dev/build.md +++ b/docs/en/dev/build.md @@ -79,4 +79,175 @@ cd llvm-project python3 /scripts/build-llvm-libs.py debug ``` -You can also refer to LLVM's official build tutorial: [Building LLVM with CMake](https://llvm.org/docs/CMake.html). +You can also refer to llvm's official build tutorial: [Building LLVM with CMake](https://llvm.org/docs/CMake.html). + +### GCC Toolchain + +clice requires GCC libstdc++ >= 14. You could use a different GCC toolchain and also link statically against its libstdc++: + +```bash +cmake .. -DCMAKE_C_FLAGS="--gcc-toolchain=/usr/local/gcc-14.3.0/" \ + -DCMAKE_CXX_FLAGS="--gcc-toolchain=/usr/local/gcc-14.3.0/" \ + -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++" +``` + +## Building + +After handling the prerequisites, you can start building clice. We provide two build methods: cmake/xmake. + +### CMake + +Below are the cmake parameters supported by clice: + +- `LLVM_INSTALL_PATH` specifies the installation path of llvm libs +- `CLICE_ENABLE_TEST` whether to build clice's unit tests + +For example: + +```bash +$ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DLLVM_INSTALL_PATH="./.llvm" -DCLICE_ENABLE_TEST=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ +$ cmake --build build +``` + +### Xmake + +Use the following command to build clice: + +```bash +$ xmake f -c --dev=true --mode=debug --toolchain=clang --llvm="./.llvm" --enable_test=true +$ xmake build --all +``` + +> --llvm is optional. If not specified, xmake will automatically download our precompiled binary + +## Dev Container + +We provide a complete Docker development container solution with pre-configured compilers, build tools, and all necessary dependencies to completely solve environment configuration issues. + +### 🚀 Quick Start + +#### Run Development Container +```bash +# Run default container +./docker/linux/run.sh + +# Run container with specific compiler +./docker/linux/run.sh --compiler gcc + +# Run container with specific version +./docker/linux/run.sh --version v1.2.3 +``` + +#### Container Management +```bash +# Reset container (remove and recreate) +./docker/linux/run.sh --reset + +# Update container image (pull latest version) +./docker/linux/run.sh --update +``` + +### 🏗️ Development Workflow + +#### Complete Development Flow Example +```bash +# 1. Start development session +./docker/linux/run.sh --compiler clang + +# 2. Build project inside container (project directory auto-mounted to /clice) +cd /clice +mkdir build && cd build + +# Build with CMake +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DLLVM_INSTALL_PATH="/usr/local/llvm" +ninja + +# Or build with XMake +xmake f --mode=debug --toolchain=clang +xmake build --all +``` + +### 📦 Container Features + +#### Pre-installed Tools and Environment +- **Compilers**: GCC 14, Clang 20 (from official LLVM PPA) +- **Build Systems**: CMake 3.28+, XMake 2.8+ +- **Development Tools**: Complete C++ development stack including debuggers, profilers, etc. +- **LLVM Libraries**: Pre-configured LLVM 20.x development libraries and headers +- **Python Environment**: Consistent Python environment managed by uv + +#### Automation Features +- **Environment Isolation**: Independent containers per compiler and version +- **Persistence**: Container state persists across sessions +- **Auto-mount**: Project directory auto-mounted to `/clice` +- **Version Awareness**: Support creating dev environment from existing release images + +### 🎯 Use Cases + +#### Daily Development +```bash +# Start development environment (auto-build if image doesn't exist) +./docker/linux/run.sh + +# Container will automatically: +# - Check and start existing container, or create new one +# - Mount project directory to /clice +# - Provide complete development environment +``` + +#### Multi-compiler Testing +```bash +# Test different compilers +./docker/linux/run.sh --compiler gcc +./docker/linux/run.sh --compiler clang + +# Each compiler has independent container and environment +``` + +#### Version Management +```bash +# Use specific version +./docker/linux/run.sh --version v1.0.0 + +# Update to latest version (can be used with --version, but not effective for released versions as their images cannot be updated) +./docker/linux/run.sh --update +``` + +### 📋 Detailed Parameters + +#### run.sh Parameters +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--compiler ` | Compiler type | `clang` | +| `--version ` | Version tag | `latest` | +| `--reset` | Remove and recreate container | - | +| `--update` | Pull latest image and update | - | + +#### Generated Image Naming Convention +- **Release image**: `clice-io/clice:linux-{compiler}-{version}` +- **Development image**: `clice-io/clice:linux-{compiler}-{version}-expanded` +- Examples: + - `clice-io/clice:linux-clang-latest` + - `clice-io/clice:linux-clang-latest-expanded` + - `clice-io/clice:linux-gcc-v1.2.3` + +### 🔧 Advanced Usage + +#### Execute Custom Commands +```bash +# Execute specific command in container (use -- separator) +./docker/linux/run.sh -- cmake --version + +# Execute multiple commands +./docker/linux/run.sh -- "cd /clice/build && cmake .." +``` + +#### Container Lifecycle Management +```bash +# Complete cleanup and rebuild +./docker/linux/run.sh --reset +``` + +> [!NOTE] +> This feature is currently in a preview stage and only supports Linux. Windows support will be provided in the future, and the functionality may be subject to change. +>>>>>>> 58bcd0e (feat: implement advanced multi-stage dev container architecture) diff --git a/docs/en/dev/dev-container-architecture.md b/docs/en/dev/dev-container-architecture.md new file mode 100644 index 00000000..fd2cba9a --- /dev/null +++ b/docs/en/dev/dev-container-architecture.md @@ -0,0 +1,610 @@ +# 🐳 Clice Container Architecture + +## Overview + +The Clice container provides a comprehensive, pre-configured environment for C++ development with all necessary toolchains, compilers, and dependencies. This document details the container architecture, build stages, file structure, caching mechanisms, and usage methods. + +## 🏗️ Multi-Stage Build Architecture + +The container uses a sophisticated multi-stage Docker build to optimize both build time and image size, adopting a parallel build strategy: + +### Architecture Flow Diagram + +```mermaid +graph TD + A[Base Image ubuntu:24.04] --> B[Python Base Environment base-python-environment] + + B --> C[Stage 1: Toolchain Builder toolchain-builder] + B --> D[Stage 2: Dependencies Downloader dependencies-downloader] + + C -->|Toolchain Build - Internal Parallel| E[Stage 3: Release Package Creator image-packer] + D -->|Dependency Download - Batch Parallel| E + + E -->|Create Compressed Archive| F[Stage 4: Final Package Image packed-image] + F -->|Auto Expand Before Runtime| G[Stage 5: Development Image expanded-image] + G --> H[Dev Container] + + subgraph "⚡ Parallel Build" + C + D + end + + subgraph "📦 Package Creation" + E + F + end + + subgraph "🏷️ Release Distribution" + I[Small Size, Easy Distribution] + F + end + + subgraph "🏷️ User Environment" + G + J[Full Featured Development] + end +``` + +### Build Stages Detailed + +#### Base Stage: Python Environment Foundation (`base-python-environment`) +**Purpose**: Establish consistent Python and uv environment foundation for all stages +**Base**: `ubuntu:24.04` + +```dockerfile +FROM ubuntu:24.04 AS base-python-environment +ENV PATH="/root/.local/bin:${PATH}" +ENV UV_CACHE_DIR=${UV_CACHE_DIR} + +# Copy project configuration to determine Python version +COPY config /clice/config +COPY docker/linux /clice/docker/linux + +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked \ + bash -eux - <<'SCRIPT' + apt update + apt install -y --no-install-recommends curl jq ca-certificates + + # Install uv for Python management + curl -LsSf https://astral.sh/uv/install.sh | sh + + # Get Python version from configuration + PYTHON_VERSION=$(jq -r .python /clice/config/default-toolchain-version.json) + uv python install "$PYTHON_VERSION" +SCRIPT +``` + +**Installed Components**: +- `curl`, `jq`, `ca-certificates` - Essential system utilities for downloading and JSON processing +- `uv` - Modern Python package and project manager for consistent environment management +- **Dynamic Python Version** - Automatically installs Python version specified in configuration files + +#### Stage 1: Toolchain Builder (`toolchain-builder`) - Parallel +**Purpose**: Build custom compiler toolchain (currently not implemented) +**Parallel Optimization**: Runs concurrently with dependencies downloader, uses internal parallel building +**Base**: `base-python-environment` + +```dockerfile +FROM base-python-environment AS toolchain-builder +# Independent cache namespaces for parallel execution +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=toolchain-builder-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=toolchain-builder-apt-state \ + --mount=type=cache,target=${CACHE_DIR_ROOT},sharing=locked,id=toolchain-builder-cache \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=toolchain-builder-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/build_clice_compiler_toolchain.py +SCRIPT +``` + +**Features**: +- **Independent Cache Namespace**: Uses `toolchain-builder-*` cache IDs for true parallel execution +- **Python-based Build System**: Uses uv for dependency management and Python scripts for build logic +- **Component Architecture**: Leverages component-based build system from build_config.py +- **Parallel Internal Processing**: Can build multiple compiler components simultaneously +- **Static Linking Support**: Can build static-linked libstdc++ for lower glibc compatibility + +#### Stage 2: Dependencies Downloader (`dependencies-downloader`) - Parallel +**Purpose**: Download all development dependencies without installing them +**Parallel Optimization**: Runs concurrently with toolchain builder, uses internal batch parallel downloads +**Base**: `base-python-environment` + +```dockerfile +FROM base-python-environment AS dependencies-downloader +# Independent cache namespaces for parallel execution +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt-state \ + --mount=type=cache,target=${CACHE_DIR_ROOT},sharing=locked,id=dependencies-downloader-cache \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=dependencies-downloader-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/download_dependencies.py +SCRIPT +``` + +**Downloaded Components**: +- **APT Packages**: Complete dependency tree resolved using component-based architecture +- **CMake**: Binary installer with SHA-256 verification +- **XMake**: Platform-specific installation bundle +- **Python Dependencies**: Development tool packages managed by uv + +**Parallel Optimization Features**: +- **Independent Cache Namespace**: Uses `dependencies-downloader-*` cache IDs +- **aria2c Multi-connection Downloads**: High-speed parallel downloads for individual files +- **Batch Processing**: APT packages downloaded in concurrent batches +- **Component-based Resolution**: Uses ALL_COMPONENTS registry for dynamic dependency management +- **Pre-resolved Dependency Trees**: Reduces download-time dependency lookups + +**Cache Structure**: +``` +${RELEASE_PACKAGE_DIR}/ +├── apt-unknown/ # APT component packages and metadata +├── uv-unknown/ # UV component packages +├── cmake-{version}/ # CMake component with version +├── xmake-{version}/ # XMake component with version +├── toolchain-unknown/ # Toolchain component container +│ ├── glibc-{version}/ # GNU C Library sub-component +│ ├── gcc-{version}/ # GNU Compiler Collection sub-component +│ ├── llvm-{version}/ # LLVM Project sub-component +│ └── linux-{version}/ # Linux Kernel Headers sub-component +└── manifest.json # Complete dependency manifest with ALL_COMPONENTS data + +${PACKED_RELEASE_PACKAGE_PATH} # Compressed archive (e.g., /release-pkg.tar.xz) +``` + +#### Stage 3: Release Package Creator (`image-packer`) +**Purpose**: Merge toolchain and dependencies into final release package for distribution +**Note**: This stage creates the compressed release package archive +**Base**: `base-python-environment` + +```dockerfile +FROM base-python-environment AS image-packer +# Merge outputs from parallel stages +COPY --from=toolchain-builder ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} +COPY --from=dependencies-downloader ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} + +# Independent cache namespace for package creation +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=packed-image-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=packed-image-apt-state \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=packed-image-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/create_release_package.py +SCRIPT +``` + +**Release Package Creation Features**: +- **Independent Cache Namespace**: Uses `packed-image-*` cache IDs for isolation +- **Python-based Merging**: Uses create_release_package.py for intelligent component merging +- **Component Integration**: Merges outputs from parallel stages using component architecture +- **Manifest Generation**: Creates comprehensive manifest.json with ALL_COMPONENTS metadata +- **Parallel Task Execution**: Uses ParallelTaskScheduler for efficient package creation + +#### Stage 4: Final Package Image (`packed-image`) +**Purpose**: Create the final distribution image with compressed release package +**Note**: This stage copies the compressed archive and environment configuration +**Base**: `base-python-environment` + +```dockerfile +FROM base-python-environment AS packed-image +COPY --from=image-packer ${PACKED_RELEASE_PACKAGE_PATH} ${PACKED_RELEASE_PACKAGE_PATH} +COPY --from=image-packer ${ENVIRONMENT_CONFIG_FILE} ${ENVIRONMENT_CONFIG_FILE} +``` + +**Final Package Features**: +- **Compressed Release Archive**: Contains `${PACKED_RELEASE_PACKAGE_PATH}` (e.g., `/release-pkg.tar.xz`) +- **Environment Configuration**: Includes pre-configured shell environment settings +- **Distribution Optimized**: Minimal size for efficient distribution and caching + +#### Stage 5: Development Image (`expanded-image`) - Final Usage +**Purpose**: Fully expanded development environment - the final usable image +**Note**: Auto-expanded from release package using Python-based setup +**Base**: Uses `${PACKED_IMAGE_NAME}` (the release image from previous stage) + +```dockerfile +FROM ${PACKED_IMAGE_NAME} AS expanded-image +RUN bash -eux - <<'SCRIPT' + # Use project-specific Python environment + uv sync --project /clice/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + + # Expand release package into full development environment + python docker/linux/utility/local_setup.py + + # Clean up build artifacts to reduce final image size + rm -rf /clice +SCRIPT +``` + +**Installed Components**: +- **Compilers**: GCC 14, Clang 20 (from official LLVM PPA) +- **Build Systems**: CMake (latest), XMake (latest) +- **Development Tools**: Complete C++ development stack including debuggers and profilers +- **Runtime Libraries**: All necessary runtime dependencies + +**Expansion Features**: +- **Python tarfile-based Extraction**: Consistent archive handling using Python's built-in tarfile module +- **Component-based Installation**: Uses component architecture for systematic tool installation +- **Size Optimization**: Removes build artifacts after expansion to minimize final image size +- **No Cache Dependencies**: Final expansion doesn't require build-time caches, suitable for end-user environments + +**Development Container**: This is the final expanded, production-ready development environment + +## 📁 Container File Structure + +### Runtime Container Structure +``` +/clice/ # Project root directory (user workspace) +├── build/ # Build output directory +├── cmake/ # CMake configuration files +├── config/ # Centralized configuration +│ ├── build_config.py # Build configuration constants and component architecture +│ └── default-toolchain-version.json # Toolchain version definitions +├── docker/linux/utility/ # Container utility scripts +│ ├── build_utils.py # Build utilities and parallel scheduler +│ ├── download_dependencies.py # Dependency downloader +│ ├── create_release_package.py # Release package creator +│ └── local_setup.py # Local environment setup +├── include/ # C++ header files +├── src/ # C++ source files +└── tests/ # Test files +``` + +### Package Structure +``` +${RELEASE_PACKAGE_DIR}/ # Component package directory (build-time) +├── apt-unknown/ # APT component packages and metadata +├── uv-unknown/ # UV component packages +├── cmake-{version}/ # CMake component (versioned) +├── xmake-{version}/ # XMake component (versioned) +├── toolchain-unknown/ # Toolchain component container +│ ├── glibc-{version}/ # GNU C Library sub-component +│ ├── gcc-{version}/ # GNU Compiler Collection sub-component +│ ├── llvm-{version}/ # LLVM Project sub-component +│ └── linux-{version}/ # Linux Kernel Headers sub-component +└── manifest.json # Complete component and dependency manifest + +${PACKED_RELEASE_PACKAGE_PATH} # Compressed release package (e.g., /release-pkg.tar.xz) + +${ENVIRONMENT_CONFIG_FILE} # Environment configuration file (e.g., /root/.bashrc) +``` + +### Dependency Manifest Structure +```json +{ + "timestamp": 1696723200, + "components": { + "apt-unknown": { + "name": "apt", + "version": "unknown", + "type": "APTComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/apt-unknown", + "packages": [ + "git", "binutils", "bison", "build-essential", "g++-14", + "gawk", "gcc-14", "gnupg", "libstdc++-14-dev", + "make", "rsync", "software-properties-common", "unzip", "xz-utils", + "aria2", "apt-rdepends", "bzip2", "xz-utils" + ], + "package_count": 125 + }, + "uv-unknown": { + "name": "uv", + "version": "unknown", + "type": "UVComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/uv-unknown" + }, + "cmake-{version}": { + "name": "cmake", + "version": "3.28.3", + "type": "CMakeComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/cmake-3.28.3", + "base_url": "https://github.com/Kitware/CMake/releases/download/v{version}", + "tarball_name": "cmake-3.28.3-linux-x86_64.sh", + "verification_name": "cmake-3.28.3-SHA-256.txt" + }, + "xmake-{version}": { + "name": "xmake", + "version": "2.8.5", + "type": "XMakeComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/xmake-2.8.5", + "base_url": "https://github.com/xmake-io/xmake/releases/download/v{version}", + "tarball_name": "xmake-bundle-v2.8.5.Linux.x86_64" + }, + "toolchain-unknown": { + "name": "toolchain", + "version": "unknown", + "type": "ToolchainComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown", + "sub_components": { + "glibc-{version}": { + "name": "glibc", + "version": "2.39", + "type": "GlibcSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/glibc-2.39", + "base_url": "https://ftpmirror.gnu.org/gnu/glibc", + "tarball_name": "glibc-2.39.tar.xz" + }, + "gcc-{version}": { + "name": "gcc", + "version": "14", + "type": "GccSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/gcc-14", + "base_url": "https://ftpmirror.gnu.org/gnu/gcc/gcc-14", + "tarball_name": "gcc-14.tar.xz" + }, + "llvm-{version}": { + "name": "llvm", + "version": "20.1.5", + "type": "LlvmSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/llvm-20.1.5", + "base_url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.5", + "tarball_name": "llvm-project-20.1.5.src.tar.xz" + }, + "linux-{version}": { + "name": "linux", + "version": "6.6", + "type": "LinuxSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/linux-6.6", + "base_url": "https://github.com/torvalds/linux/archive/refs/tags", + "tarball_name": "v6.6.tar.gz" + } + }, + "sysroot_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/sysroot/x86_64-linux-gnu/x86_64-linux-gnu/glibc2.39-libstdc++14-linux6.6" + } + }, + "build_stages": { + "dependencies_downloader": ["apt-unknown", "uv-unknown", "cmake-{version}", "xmake-{version}"], + "toolchain_builder": ["toolchain-unknown"] + }, + "environment_variables": { + "PATH": "/root/.local/bin:${PATH}", + "XMAKE_ROOT": "y" + } +} +``` + +## 🚀 Build Process + +### Build Commands +```bash +# Build with default settings (clang + latest) +./docker/linux/build.sh + +# Build with specific compiler and version +./docker/linux/build.sh --compiler gcc --version v1.2.3 +``` + +### Build Process Flow +1. **Stage 1**: Install basic system packages +2. **Stage 2**: Download all dependencies to cache +3. **Stage 3**: Install dependencies from cache to final image +4. **Finalization**: Configure environment and create development-ready container + +### Generated Images +**No distinction between dev and production builds**, unified image architecture: + +- **Image Name**: `clice-io/clice:linux-{compiler}-{version}` +- **Image Types**: + - **Release Image**: Distribution-optimized, contains compressed packages and cache, not directly usable + - **Development Image**: Fully expanded development environment, final usable image +- **Examples**: + - `clice-io/clice:linux-clang-latest` + - `clice-io/clice:linux-gcc-v1.2.3` + +**Important Notes**: +- Release image's main advantage is reducing user download image size +- Development image is the final expanded container, the environment users actually use +- Unified build process, no distinction between development and production environments + +## 🏃 Container Usage + +### Running Container +```bash +# Run with default settings +./docker/linux/run.sh + +# Run with specific compiler and version +./docker/linux/run.sh --compiler gcc --version v1.2.3 + +# Reset container (remove and recreate) +./docker/linux/run.sh --reset + +# Update container image +./docker/linux/run.sh --update +``` + +### Container Management +- **Automatic Creation**: If container doesn't exist, it's created automatically +- **Version Checking**: Container image version is validated before use +- **Workspace Mounting**: Project directory is mounted to `/clice` in container +- **Persistent Storage**: Container persists between sessions + +### Development Workflow +```bash +# 1. Build development container +./docker/linux/build.sh --compiler clang + +# 2. Start development session +./docker/linux/run.sh --compiler clang + +# 3. Inside container - build project +cd /clice +mkdir build && cd build +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug +ninja +``` + +## ⚡ Caching Strategy + +### Independent Cache Namespaces +Each build stage uses separate cache IDs to enable true parallel execution: + +#### Stage-Specific Cache IDs +- **Toolchain Builder**: `toolchain-builder-*` + - `toolchain-builder-apt` - APT package cache + - `toolchain-builder-apt-state` - APT state cache + - `toolchain-builder-cache` - General build cache + - `toolchain-builder-uv` - UV Python package cache + +- **Dependencies Downloader**: `dependencies-downloader-*` + - `dependencies-downloader-apt` - APT package cache + - `dependencies-downloader-apt-state` - APT state cache + - `dependencies-downloader-cache` - Download cache + - `dependencies-downloader-uv` - UV Python package cache + +- **Release Package Creator**: `packed-image-*` + - `packed-image-apt` - APT package cache + - `packed-image-apt-state` - APT state cache + - `packed-image-uv` - UV Python package cache + +### Docker Layer Caching +- **Python Base Environment**: Cached independently, shared across all stages +- **Stage Outputs**: Each stage's output is cached as separate Docker layers +- **Parallel Stage Isolation**: Independent caches prevent conflicts during parallel execution + +### Cache Optimization Benefits +- **True Parallel Execution**: Independent cache namespaces eliminate conflicts +- **Reduced Build Time**: Intelligent layer caching and component-based builds +- **Bandwidth Efficiency**: Downloads happen once per cache namespace +- **Offline Capability**: Complete dependency pre-resolution enables offline rebuilds +- **Selective Invalidation**: Changes to one component don't invalidate others + +## 🛡️ Security & Verification + +### Package Verification +- **CMake**: SHA-256 checksum verification of installer +- **APT Packages**: Standard APT signature verification +- **Dependency Tree**: Complete dependency resolution with `apt-rdepends` + +### Build Isolation +- **Multi-stage**: Each stage is isolated and cacheable +- **Non-root User**: Development runs as non-root user where possible +- **Clean Environment**: Each build starts from clean base + +## 🔧 Configuration Management + +### Centralized Configuration +All container configuration is managed through `config/build_config.py`: + +```python +# Version management +TOOLCHAIN_VERSIONS = { + "cmake": "3.28.3", + "xmake": "2.8.5", + "gcc": "14", + "llvm": "20" +} + +# Package lists +DEV_CONTAINER_BASIC_TOOLS = [ + "software-properties-common", + "gnupg", "git", "xz-utils", "unzip", "make" +] +``` + +### Environment Variables +- `PKG_CACHE_DIR=/pkg-cache` - Package cache directory +- `DEBIAN_FRONTEND=noninteractive` - Non-interactive package installation +- `XMAKE_ROOT=y` - XMake root privileges + +## 🚀 Performance Optimizations + +### Parallel Processing Architecture +**Parallel optimization is implemented at three levels**: + +#### Inter-Stage Parallelism (Docker Build Level) +- **Toolchain Builder** and **Dependencies Downloader** stages execute concurrently +- **Release Package Creator** waits for both parallel stages to complete +- Docker BuildKit automatically schedules parallel stage execution +- **Independent Cache Namespaces** prevent cache conflicts during parallel execution + +#### Intra-Stage Parallelism (Component Level) +**Toolchain Builder Internal Parallelism**: +- Uses `ParallelTaskScheduler` for optimal job scheduling +- Compiler components built concurrently using `ProcessPoolExecutor` +- Multi-core CPU utilization for parallel compilation +- Component dependencies resolved using topological sorting + +**Dependencies Downloader Internal Parallelism**: +- `aria2c` multi-connection downloads for individual files +- Batch processing of APT packages using parallel job execution +- Component-based parallel downloads (APT, tools, Python packages simultaneously) +- Pre-resolved dependency trees reduce download-time lookups + +**Release Package Creator Parallelism**: +- Parallel component merging using job-based task scheduler +- Concurrent manifest generation and package compression +- Optimal resource utilization during final packaging stage + +#### Cache Independence Architecture +Each stage operates with completely independent cache namespaces: +```dockerfile +# Toolchain Builder - Independent cache namespace +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=toolchain-builder-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=toolchain-builder-uv + +# Dependencies Downloader - Independent cache namespace +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=dependencies-downloader-uv + +# Release Package Creator - Independent cache namespace +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=packed-image-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=packed-image-uv +``` + +### Build Optimization +- **Layer Caching**: Aggressive Docker layer caching strategy +- **Minimal Rebuilds**: Only changed components are rebuilt +- **Size Optimization**: Multi-stage builds minimize final image size +- **Cache Separation**: Release image serves as cache layer, Development image expands quickly + +## 🔄 Maintenance & Updates + +### Version Updates +Update versions in `config/default-toolchain-version.json`: +```json +{ + "cmake": "3.28.3", + "xmake": "2.8.5", + "gcc": "14", + "llvm": "20" +} +``` + +### Adding New Dependencies +1. Update package lists in `config/build_config.py` +2. Rebuild container with `./docker/linux/build.sh --rebuild` +3. Verify with `./docker/linux/run.sh --update` + +### Container Health Checks +```bash +# Check container status +docker ps -f name=clice-linux-clang + +# Verify development environment +./docker/linux/run.sh bash -c "cmake --version && xmake --version" + +# Check package manifest +docker exec clice-linux-clang cat /pkg-cache/manifest.json +``` + +## 🎯 Best Practices + +### Development Workflow +1. Use version-specific containers for reproducible builds +2. Reset containers when switching between major versions +3. Use `--update` to pull latest images regularly +4. Mount only necessary directories to avoid performance issues + +### Container Management +1. Use descriptive version tags for release builds +2. Clean up unused containers and images periodically +3. Monitor container resource usage +4. Keep container configuration under version control + +This architecture provides a robust, efficient, and maintainable development environment for the Clice project, with optimized build times, comprehensive toolchain support, and excellent developer experience. \ No newline at end of file diff --git a/docs/zh/dev/dev-container-architecture.md b/docs/zh/dev/dev-container-architecture.md new file mode 100644 index 00000000..a45ce9e0 --- /dev/null +++ b/docs/zh/dev/dev-container-architecture.md @@ -0,0 +1,610 @@ +# 🐳 Clice 容器架构 + +## 概述 + +Clice 容器提供了一个全面、预配置的 C++ 开发环境，包含所有必要的工具链、编译器和依赖项。本文档详细说明了容器架构、构建阶段、文件结构、缓存机制和使用方法。 + +## 🏗️ 多阶段构建架构 + +容器使用复杂的多阶段 Docker 构建来优化构建时间和镜像大小，采用并行构建策略： + +### 架构流程图 + +```mermaid +graph TD + A[基础镜像 ubuntu:24.04] --> B[Python 基础环境 base-python-environment] + + B --> C[阶段 1: 工具链构建器 toolchain-builder] + B --> D[阶段 2: 依赖下载器 dependencies-downloader] + + C -->|工具链构建 - 内部并行| E[阶段 3: Release 包创建器 image-packer] + D -->|依赖下载 - 批量并行| E + + E -->|创建压缩归档| F[阶段 4: 最终包镜像 packed-image] + F -->|运行前自动展开| G[阶段 5: Development 镜像 expanded-image] + G --> H[开发容器] + + subgraph "⚡ 并行构建" + C + D + end + + subgraph "📦 包创建" + E + F + end + + subgraph "🏷️ 发布分发" + I[体积小，便于分发] + F + end + + subgraph "🏷️ 用户环境" + G + J[功能完整的开发环境] + end +``` + +### 构建阶段详解 + +#### 基础阶段：Python 环境基础 (`base-python-environment`) +**目的**：为所有阶段建立一致的 Python 和 uv 环境基础 +**基础镜像**：`ubuntu:24.04` + +```dockerfile +FROM ubuntu:24.04 AS base-python-environment +ENV PATH="/root/.local/bin:${PATH}" +ENV UV_CACHE_DIR=${UV_CACHE_DIR} + +# 复制项目配置以确定 Python 版本 +COPY config /clice/config +COPY docker/linux /clice/docker/linux + +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked \ + bash -eux - <<'SCRIPT' + apt update + apt install -y --no-install-recommends curl jq ca-certificates + + # 安装 uv 用于 Python 管理 + curl -LsSf https://astral.sh/uv/install.sh | sh + + # 从配置获取 Python 版本 + PYTHON_VERSION=$(jq -r .python /clice/config/default-toolchain-version.json) + uv python install "$PYTHON_VERSION" +SCRIPT +``` + +**安装的组件**： +- `curl`, `jq`, `ca-certificates` - 下载和 JSON 处理所需的基本系统工具 +- `uv` - 现代 Python 包和项目管理器，用于一致的环境管理 +- **动态 Python 版本** - 自动安装配置文件中指定的 Python 版本 + +#### 阶段 1：工具链构建器 (`toolchain-builder`) - 并行 +**目的**：构建自定义编译器工具链（目前暂未实现） +**并行优化**：与依赖下载器同时运行，内部使用并行构建 +**基础镜像**：`base-python-environment` + +```dockerfile +FROM base-python-environment AS toolchain-builder +# 用于并行执行的独立缓存命名空间 +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=toolchain-builder-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=toolchain-builder-apt-state \ + --mount=type=cache,target=${CACHE_DIR_ROOT},sharing=locked,id=toolchain-builder-cache \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=toolchain-builder-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/build_clice_compiler_toolchain.py +SCRIPT +``` + +**特点**： +- **独立缓存命名空间**：使用 `toolchain-builder-*` 缓存 ID 实现真正的并行执行 +- **基于 Python 的构建系统**：使用 uv 进行依赖管理，Python 脚本处理构建逻辑 +- **组件架构**：利用 build_config.py 中基于组件的构建系统 +- **并行内部处理**：可以同时构建多个编译器组件 +- **静态链接支持**：可构建静态链接的 libstdc++ 以兼容更低版本的 glibc + +#### 阶段 2：依赖下载器 (`dependencies-downloader`) - 并行 +**目的**：下载所有开发依赖项而不安装它们 +**并行优化**：与工具链构建器同时运行，内部批量并行下载 +**基础镜像**：`base-python-environment` + +```dockerfile +FROM base-python-environment AS dependencies-downloader +# 用于并行执行的独立缓存命名空间 +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt-state \ + --mount=type=cache,target=${CACHE_DIR_ROOT},sharing=locked,id=dependencies-downloader-cache \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=dependencies-downloader-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/download_dependencies.py +SCRIPT +``` + +**下载的组件**： +- **APT 包**：使用基于组件架构解析的完整依赖树 +- **CMake**：带 SHA-256 验证的二进制安装程序 +- **XMake**：平台特定的安装包 +- **Python 依赖**：由 uv 管理的开发工具包 + +**并行优化特性**： +- **独立缓存命名空间**：使用 `dependencies-downloader-*` 缓存 ID +- **aria2c 多连接下载**：单个文件的高速并行下载 +- **批处理**：APT 包并发批量下载 +- **基于组件的解析**：使用 ALL_COMPONENTS 注册表进行动态依赖管理 +- **预解析依赖树**：减少下载时的依赖查找开销 + +**缓存结构**： +``` +${RELEASE_PACKAGE_DIR}/ +├── apt-unknown/ # APT 组件包和元数据 +├── uv-unknown/ # UV 组件包 +├── cmake-{version}/ # 带版本的 CMake 组件 +├── xmake-{version}/ # 带版本的 XMake 组件 +├── toolchain-unknown/ # 工具链组件容器 +│ ├── glibc-{version}/ # GNU C 库子组件 +│ ├── gcc-{version}/ # GNU 编译器集合子组件 +│ ├── llvm-{version}/ # LLVM 项目子组件 +│ └── linux-{version}/ # Linux 内核头文件子组件 +└── manifest.json # 包含 ALL_COMPONENTS 数据的完整依赖清单 + +${PACKED_RELEASE_PACKAGE_PATH} # 压缩归档（如 /release-pkg.tar.xz） +``` + +#### 阶段 3：Release 包创建器 (`image-packer`) +**目的**：将工具链和依赖合并为用于分发的最终 release 包 +**特点**：此阶段创建压缩的 release 包归档 +**基础镜像**：`base-python-environment` + +```dockerfile +FROM base-python-environment AS image-packer +# 合并并行阶段的输出 +COPY --from=toolchain-builder ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} +COPY --from=dependencies-downloader ${RELEASE_PACKAGE_DIR} ${RELEASE_PACKAGE_DIR} + +# 用于包创建的独立缓存命名空间 +RUN --mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=packed-image-apt \ + --mount=type=cache,target=${APT_STATE_CACHE_DIR},sharing=locked,id=packed-image-apt-state \ + --mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=packed-image-uv \ + bash -eux - <<'SCRIPT' + uv sync --project /clice/docker/linux/utility/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + python docker/linux/utility/create_release_package.py +SCRIPT +``` + +**Release 包创建特性**： +- **独立缓存命名空间**：使用 `packed-image-*` 缓存 ID 进行隔离 +- **基于 Python 的合并**：使用 create_release_package.py 进行智能组件合并 +- **组件集成**：使用组件架构合并并行阶段的输出 +- **清单生成**：创建包含 ALL_COMPONENTS 元数据的综合 manifest.json +- **并行任务执行**：使用 ParallelTaskScheduler 进行高效的包创建 + +#### 阶段 4：最终包镜像 (`packed-image`) +**目的**：创建包含压缩 release 包的最终分发镜像 +**特点**：此阶段复制压缩归档和环境配置 +**基础镜像**：`base-python-environment` + +```dockerfile +FROM base-python-environment AS packed-image +COPY --from=image-packer ${PACKED_RELEASE_PACKAGE_PATH} ${PACKED_RELEASE_PACKAGE_PATH} +COPY --from=image-packer ${ENVIRONMENT_CONFIG_FILE} ${ENVIRONMENT_CONFIG_FILE} +``` + +**最终包特性**： +- **压缩 Release 归档**：包含 `${PACKED_RELEASE_PACKAGE_PATH}`（如 `/release-pkg.tar.xz`） +- **环境配置**：包含预配置的 shell 环境设置 +- **分发优化**：最小尺寸以实现高效分发和缓存 + +#### 阶段 5：Development 镜像 (`expanded-image`) - 最终使用 +**目的**：完全展开的开发环境 - 最终可用的镜像 +**特点**：使用基于 Python 的设置从 release 包自动展开 +**基础镜像**：使用 `${PACKED_IMAGE_NAME}`（来自前一阶段的 release 镜像） + +```dockerfile +FROM ${PACKED_IMAGE_NAME} AS expanded-image +RUN bash -eux - <<'SCRIPT' + # 使用项目特定的 Python 环境 + uv sync --project /clice/pyproject.toml + source /clice/docker/linux/utility/.venv/bin/activate + + # 将 release 包展开为完整的开发环境 + python docker/linux/utility/local_setup.py + + # 清理构建工件以减少最终镜像大小 + rm -rf /clice +SCRIPT +``` + +**安装的组件**： +- **编译器**：GCC 14、Clang 20（来自官方 LLVM PPA） +- **构建系统**：CMake（最新版）、XMake（最新版） +- **开发工具**：完整的 C++ 开发栈，包括调试器和分析器 +- **运行时库**：所有必要的运行时依赖 + +**展开特性**： +- **基于 Python tarfile 的提取**：使用 Python 内置 tarfile 模块进行一致的归档处理 +- **基于组件的安装**：使用组件架构进行系统的工具安装 +- **大小优化**：展开后删除构建工件以最小化最终镜像大小 +- **无缓存依赖**：最终展开不需要构建时缓存，适合最终用户环境 + +**Development 容器**：这是最终展开的、可用于生产的开发环境 + +## 📁 容器文件结构 + +### 运行时容器结构 +``` +/clice/ # 项目根目录（用户工作空间） +├── build/ # 构建输出目录 +├── cmake/ # CMake 配置文件 +├── config/ # 集中配置 +│ ├── build_config.py # 构建配置常量和组件架构 +│ └── default-toolchain-version.json # 工具链版本定义 +├── docker/linux/utility/ # 容器实用程序脚本 +│ ├── build_utils.py # 构建实用程序和并行调度器 +│ ├── download_dependencies.py # 依赖下载器 +│ ├── create_release_package.py # Release包创建器 +│ └── local_setup.py # 本地环境设置 +├── include/ # C++ 头文件 +├── src/ # C++ 源文件 +└── tests/ # 测试文件 +``` + +### 打包结构 +``` +${RELEASE_PACKAGE_DIR}/ # 组件包目录（构建时） +├── apt-unknown/ # APT 组件包和元数据 +├── uv-unknown/ # UV 组件包 +├── cmake-{version}/ # CMake 组件（带版本） +├── xmake-{version}/ # XMake 组件（带版本） +├── toolchain-unknown/ # 工具链组件容器 +│ ├── glibc-{version}/ # GNU C 库子组件 +│ ├── gcc-{version}/ # GNU 编译器集合子组件 +│ ├── llvm-{version}/ # LLVM 项目子组件 +│ └── linux-{version}/ # Linux 内核头文件子组件 +└── manifest.json # 完整组件和依赖清单 + +${PACKED_RELEASE_PACKAGE_PATH} # 压缩发布包（如 /release-pkg.tar.xz） + +${ENVIRONMENT_CONFIG_FILE} # 环境配置文件（如 /root/.bashrc） +``` + +### 依赖清单结构 +```json +{ + "timestamp": 1696723200, + "components": { + "apt-unknown": { + "name": "apt", + "version": "unknown", + "type": "APTComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/apt-unknown", + "packages": [ + "git", "binutils", "bison", "build-essential", "g++-14", + "gawk", "gcc-14", "gnupg", "libstdc++-14-dev", + "make", "rsync", "software-properties-common", "unzip", "xz-utils", + "aria2", "apt-rdepends", "bzip2", "xz-utils" + ], + "package_count": 125 + }, + "uv-unknown": { + "name": "uv", + "version": "unknown", + "type": "UVComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/uv-unknown" + }, + "cmake-{version}": { + "name": "cmake", + "version": "3.28.3", + "type": "CMakeComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/cmake-3.28.3", + "base_url": "https://github.com/Kitware/CMake/releases/download/v{version}", + "tarball_name": "cmake-3.28.3-linux-x86_64.sh", + "verification_name": "cmake-3.28.3-SHA-256.txt" + }, + "xmake-{version}": { + "name": "xmake", + "version": "2.8.5", + "type": "XMakeComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/xmake-2.8.5", + "base_url": "https://github.com/xmake-io/xmake/releases/download/v{version}", + "tarball_name": "xmake-bundle-v2.8.5.Linux.x86_64" + }, + "toolchain-unknown": { + "name": "toolchain", + "version": "unknown", + "type": "ToolchainComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown", + "sub_components": { + "glibc-{version}": { + "name": "glibc", + "version": "2.39", + "type": "GlibcSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/glibc-2.39", + "base_url": "https://ftpmirror.gnu.org/gnu/glibc", + "tarball_name": "glibc-2.39.tar.xz" + }, + "gcc-{version}": { + "name": "gcc", + "version": "14", + "type": "GccSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/gcc-14", + "base_url": "https://ftpmirror.gnu.org/gnu/gcc/gcc-14", + "tarball_name": "gcc-14.tar.xz" + }, + "llvm-{version}": { + "name": "llvm", + "version": "20.1.5", + "type": "LlvmSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/llvm-20.1.5", + "base_url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.5", + "tarball_name": "llvm-project-20.1.5.src.tar.xz" + }, + "linux-{version}": { + "name": "linux", + "version": "6.6", + "type": "LinuxSubComponent", + "package_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/linux-6.6", + "base_url": "https://github.com/torvalds/linux/archive/refs/tags", + "tarball_name": "v6.6.tar.gz" + } + }, + "sysroot_dir": "${RELEASE_PACKAGE_DIR}/toolchain-unknown/sysroot/x86_64-linux-gnu/x86_64-linux-gnu/glibc2.39-libstdc++14-linux6.6" + } + }, + "build_stages": { + "dependencies_downloader": ["apt-unknown", "uv-unknown", "cmake-{version}", "xmake-{version}"], + "toolchain_builder": ["toolchain-unknown"] + }, + "environment_variables": { + "PATH": "/root/.local/bin:${PATH}", + "XMAKE_ROOT": "y" + } +} +``` + +## 🚀 构建过程 + +### 构建命令 +```bash +# 使用默认设置构建（clang + latest） +./docker/linux/build.sh + +# 使用特定编译器和版本构建 +./docker/linux/build.sh --compiler gcc --version v1.2.3 +``` + +### 构建过程流程 +1. **阶段 1**：安装基本系统包 +2. **阶段 2**：将所有依赖下载到缓存 +3. **阶段 3**：从缓存安装依赖到最终镜像 +4. **最终化**：配置环境并创建开发就绪容器 + +### 生成的镜像 +**构建镜像不分 dev 和生产**，统一的镜像架构： + +- **镜像名称**：`clice-io/clice:linux-{compiler}-{version}` +- **镜像类型**： + - **Release 镜像**：便于分发，包含压缩包和缓存，不能直接使用 + - **Development 镜像**：完全展开的开发环境，最终使用的镜像 +- **示例**： + - `clice-io/clice:linux-clang-latest` + - `clice-io/clice:linux-gcc-v1.2.3` + +**重要说明**： +- Release 镜像主要优势是降低用户下载的镜像大小 +- Development 镜像是最终展开的容器，用户实际使用的环境 +- 构建过程统一，不区分开发和生产环境 + +## 🏃 容器使用 + +### 运行容器 +```bash +# 使用默认设置运行 +./docker/linux/run.sh + +# 使用特定编译器和版本运行 +./docker/linux/run.sh --compiler gcc --version v1.2.3 + +# 重置容器（删除并重新创建） +./docker/linux/run.sh --reset + +# 更新容器镜像 +./docker/linux/run.sh --update +``` + +### 容器管理 +- **自动创建**：如果容器不存在，会自动创建 +- **版本检查**：使用前会验证容器镜像版本 +- **工作区挂载**：项目目录挂载到容器中的 `/clice` +- **持久存储**：容器在会话之间保持持久 + +### 开发工作流程 +```bash +# 1. 构建开发容器 +./docker/linux/build.sh --compiler clang + +# 2. 开始开发会话 +./docker/linux/run.sh --compiler clang + +# 3. 在容器内 - 构建项目 +cd /clice +mkdir build && cd build +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug +ninja +``` + +## ⚡ 缓存策略 + +### 独立缓存命名空间 +每个构建阶段使用独立的缓存 ID 以实现真正的并行执行： + +#### 阶段特定缓存 ID +- **工具链构建器**：`toolchain-builder-*` + - `toolchain-builder-apt` - APT 包缓存 + - `toolchain-builder-apt-state` - APT 状态缓存 + - `toolchain-builder-cache` - 通用构建缓存 + - `toolchain-builder-uv` - UV Python 包缓存 + +- **依赖下载器**：`dependencies-downloader-*` + - `dependencies-downloader-apt` - APT 包缓存 + - `dependencies-downloader-apt-state` - APT 状态缓存 + - `dependencies-downloader-cache` - 下载缓存 + - `dependencies-downloader-uv` - UV Python 包缓存 + +- **Release 包创建器**：`packed-image-*` + - `packed-image-apt` - APT 包缓存 + - `packed-image-apt-state` - APT 状态缓存 + - `packed-image-uv` - UV Python 包缓存 + +### Docker 层缓存 +- **Python 基础环境**：独立缓存，在所有阶段间共享 +- **阶段输出**：每个阶段的输出作为独立的 Docker 层缓存 +- **并行阶段隔离**：独立缓存防止并行执行期间的冲突 + +### 缓存优化优势 +- **真正的并行执行**：独立缓存命名空间消除冲突 +- **减少构建时间**：智能层缓存和基于组件的构建 +- **带宽效率**：每个缓存命名空间下载只发生一次 +- **离线能力**：完整依赖预解析使离线重建成为可能 +- **选择性失效**：一个组件的更改不会使其他组件失效 + +## 🛡️ 安全和验证 + +### 包验证 +- **CMake**：安装程序的 SHA-256 校验和验证 +- **APT 包**：标准 APT 签名验证 +- **依赖树**：使用 `apt-rdepends` 完整依赖解析 + +### 构建隔离 +- **多阶段**：每个阶段都是隔离和可缓存的 +- **非 root 用户**：开发尽可能以非 root 用户运行 +- **清洁环境**：每次构建都从清洁基础开始 + +## 🔧 配置管理 + +### 集中配置 +所有容器配置通过 `config/build_config.py` 管理： + +```python +# 版本管理 +TOOLCHAIN_VERSIONS = { + "cmake": "3.28.3", + "xmake": "2.8.5", + "gcc": "14", + "llvm": "20" +} + +# 包列表 +DEV_CONTAINER_BASIC_TOOLS = [ + "software-properties-common", + "gnupg", "git", "xz-utils", "unzip", "make" +] +``` + +### 环境变量 +- `PKG_CACHE_DIR=/pkg-cache` - 包缓存目录 +- `DEBIAN_FRONTEND=noninteractive` - 非交互式包安装 +- `XMAKE_ROOT=y` - XMake root 权限 + +## 🚀 性能优化 + +### 并行处理架构 +**并行优化在三个层面实现**： + +#### Stage 间并行（Docker 构建层面） +- **工具链构建器** 和 **依赖下载器** 阶段并发执行 +- **Release 包创建器** 等待两个并行阶段完成 +- Docker BuildKit 自动调度并行阶段执行 +- **独立缓存命名空间** 防止并行执行期间的缓存冲突 + +#### Stage 内并行（组件层面） +**工具链构建器内部并行**： +- 使用 `ParallelTaskScheduler` 进行最优作业调度 +- 使用 `ProcessPoolExecutor` 并发构建编译器组件 +- 多核 CPU 利用率用于并行编译 +- 使用拓扑排序解析组件依赖 + +**依赖下载器内部并行**： +- `aria2c` 多连接下载单个文件 +- 使用并行作业执行的 APT 包批处理 +- 基于组件的并行下载（APT、工具、Python 包同时进行） +- 预解析依赖树减少下载时查找 + +**Release 包创建器并行**： +- 使用基于作业的任务调度器并行组件合并 +- 并发清单生成和包压缩 +- 最终打包阶段的最优资源利用 + +#### 缓存独立架构 +每个阶段使用完全独立的缓存命名空间操作： +```dockerfile +# 工具链构建器 - 独立缓存命名空间 +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=toolchain-builder-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=toolchain-builder-uv + +# 依赖下载器 - 独立缓存命名空间 +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=dependencies-downloader-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=dependencies-downloader-uv + +# Release 包创建器 - 独立缓存命名空间 +--mount=type=cache,target=${APT_CACHE_DIR},sharing=locked,id=packed-image-apt +--mount=type=cache,target=${UV_CACHE_DIR},sharing=locked,id=packed-image-uv +``` + +### 构建优化 +- **层缓存**：积极的 Docker 层缓存策略 +- **最小重建**：只重建更改的组件 +- **大小优化**：多阶段构建最小化最终镜像大小 +- **缓存分离**：Release 镜像作为缓存层，Development 镜像快速展开 + +## 🔄 维护和更新 + +### 版本更新 +更新 `config/default-toolchain-version.json` 中的版本： +```json +{ + "cmake": "3.28.3", + "xmake": "2.8.5", + "gcc": "14", + "llvm": "20" +} +``` + +### 添加新依赖 +1. 更新 `config/build_config.py` 中的包列表 +2. 使用 `./docker/linux/build.sh --rebuild` 重建容器 +3. 使用 `./docker/linux/run.sh --update` 验证 + +### 容器健康检查 +```bash +# 检查容器状态 +docker ps -f name=clice-linux-clang + +# 验证开发环境 +./docker/linux/run.sh bash -c "cmake --version && xmake --version" + +# 检查包清单 +docker exec clice-linux-clang cat /pkg-cache/manifest.json +``` + +## 🎯 最佳实践 + +### 开发工作流程 +1. 使用版本特定的容器进行可重现构建 +2. 在主要版本之间切换时重置容器 +3. 定期使用 `--update` 拉取最新镜像 +4. 仅挂载必要目录以避免性能问题 + +### 容器管理 +1. 为发布构建使用描述性版本标签 +2. 定期清理未使用的容器和镜像 +3. 监控容器资源使用情况 +4. 将容器配置保持在版本控制下 + +此架构为 Clice 项目提供了强大、高效和可维护的开发环境，具有优化的构建时间、全面的工具链支持和出色的开发者体验。 \ No newline at end of file diff --git a/xmake.lua b/xmake.lua index edc9e5bf..3d52d107 100644 --- a/xmake.lua +++ b/xmake.lua @@ -52,6 +52,8 @@ add_requires("clice-llvm", { alias = "llvm" }) add_rules("mode.release", "mode.debug", "mode.releasedbg") set_languages("c++23") add_rules("clice_build_config") +add_cxflags("--sysroot=/toolchain-build/sysroot/x86_64-linux-gnu/x86_64-linux-gnu/glibc2.39-libstdc++14.3.0-linux6.17 -v -Wl,--verbose", {force = true, tools = {"gcc", "gxx", "clang", "clangxx"}}) +add_ldflags("--verbose -static-libstdc++ -static-libgcc", {force = true}) target("clice-core", function() set_kind("$(kind)")