From 96ce38a8f655c4054c65150a41adfc18faef9bab Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Sun, 14 Dec 2025 06:36:21 +0000 Subject: [PATCH 1/7] perf(docker): optimize Dockerfiles with BuildKit cache for faster rebuilds --- .../.devcontainer/Dockerfile | 34 +++++++++++------- .../.devcontainer/Dockerfile | 35 +++++++++++-------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/sample_cpu_project/.devcontainer/Dockerfile b/src/sample_cpu_project/.devcontainer/Dockerfile index e4bb9d4..0f04f70 100644 --- a/src/sample_cpu_project/.devcontainer/Dockerfile +++ b/src/sample_cpu_project/.devcontainer/Dockerfile @@ -4,29 +4,37 @@ ARG USERNAME=devuser ARG USER_UID=1000 ARG USER_GID=$USER_UID RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - # Add sudo support - && apt-get update \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + +# Add sudo support (using BuildKit cache mounts to speed up rebuilds) +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y sudo \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME \ - && rm -rf /var/lib/apt/lists/* + && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME -# make all python tools installed by pip accesible +# Make pip-installed tools accessible and configure pip cache location ENV PATH=$PATH:/home/devuser/.local/bin +ENV PIP_CACHE_DIR=/pip-cache -RUN pip install --no-cache-dir pip --upgrade +# Install Python packages (using BuildKit cache mounts for faster rebuilds) +RUN --mount=type=cache,target=/pip-cache \ + pip install pip --upgrade COPY src/sample_cpu_project/.devcontainer/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements.txt -# install common module related pacakages -COPY src/common/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Install common module related packages +COPY src/common/requirements.txt common-requirements.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r common-requirements.txt -# install python tools +# Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . -RUN pip install --no-cache-dir -r requirements-dev.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements-dev.txt diff --git a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile index 83326a2..c631c5b 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile +++ b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile @@ -6,30 +6,37 @@ ARG USERNAME=devuser ARG USER_UID=1000 ARG USER_GID=$USER_UID RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - # Add sudo support - && apt-get update \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + +# Add sudo support (using BuildKit cache mounts to speed up rebuilds) +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y sudo \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME \ - && rm -rf /var/lib/apt/lists/* + && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME - -# make all python tools installed by pip accesible +# Make pip-installed tools accessible and configure pip cache location ENV PATH=$PATH:/home/$USERNAME/.local/bin +ENV PIP_CACHE_DIR=/pip-cache -RUN pip install pip --upgrade +# Install Python packages (using BuildKit cache mounts for faster rebuilds) +RUN --mount=type=cache,target=/pip-cache \ + pip install pip --upgrade COPY src/sample_pytorch_gpu_project/.devcontainer/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements.txt -# install common module related pacakages -COPY src/common/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Install common module related packages +COPY src/common/requirements.txt common-requirements.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r common-requirements.txt -# install python tools +# Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . -RUN pip install --no-cache-dir -r requirements-dev.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements-dev.txt RUN az extension add --name ml From 891ea72abe5c133f01300227c129c0a9010011cb Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Sun, 14 Dec 2025 06:42:41 +0000 Subject: [PATCH 2/7] perf(docker): enhance Dockerfiles with BuildKit cache for improved performance. add name to each devcontainer.json --- notebooks/.devcontainer/Dockerfile | 30 ++++++++++++------- notebooks/.devcontainer/devcontainer.json | 1 + .../.devcontainer/devcontainer.json | 1 + .../.devcontainer/devcontainer.json | 1 + 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/notebooks/.devcontainer/Dockerfile b/notebooks/.devcontainer/Dockerfile index 4dc5812..f07126b 100644 --- a/notebooks/.devcontainer/Dockerfile +++ b/notebooks/.devcontainer/Dockerfile @@ -1,24 +1,32 @@ FROM python:3.11.14 -# create non-root user and set the default user +# Create non-root user and set the default user ARG USERNAME=devuser ARG USER_UID=1000 ARG USER_GID=$USER_UID RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - # Add sudo support - && apt-get update \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + +# Add sudo support (using BuildKit cache mounts to speed up rebuilds) +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y sudo \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME \ - && rm -rf /var/lib/apt/lists/* + && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME -# make all python tools installed by pip accesible +# Make pip-installed tools accessible and configure pip cache location ENV PATH=$PATH:/home/$USERNAME/.local/bin -RUN pip install --no-cache-dir pip --upgrade +ENV PIP_CACHE_DIR=/pip-cache + +# Install development tools (linters, formatters, test runners, etc.) +RUN --mount=type=cache,target=/pip-cache \ + pip install pip --upgrade COPY requirements-dev.txt . -RUN pip install --no-cache-dir -r requirements-dev.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements-dev.txt -# install notebooks related depencencies +# Install notebooks related dependencies COPY notebooks/.devcontainer/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN --mount=type=cache,target=/pip-cache \ + pip install -r requirements.txt diff --git a/notebooks/.devcontainer/devcontainer.json b/notebooks/.devcontainer/devcontainer.json index 8e45132..d73602e 100644 --- a/notebooks/.devcontainer/devcontainer.json +++ b/notebooks/.devcontainer/devcontainer.json @@ -1,4 +1,5 @@ { + "name": "DSToolkit Notebooks Dev Container", "build": { // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../", diff --git a/src/sample_cpu_project/.devcontainer/devcontainer.json b/src/sample_cpu_project/.devcontainer/devcontainer.json index b6d537b..839d3dc 100644 --- a/src/sample_cpu_project/.devcontainer/devcontainer.json +++ b/src/sample_cpu_project/.devcontainer/devcontainer.json @@ -1,4 +1,5 @@ { + "name": "Sample CPU Project Dev Container", "build": { // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../../", diff --git a/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json b/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json index f151bda..725a88f 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json +++ b/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json @@ -1,4 +1,5 @@ { + "name": "Sample PyTorch GPU Project Dev Container", "build": { // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../../", From e2006d0872d72e0d40789c98ff836b1603374bc1 Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Sun, 14 Dec 2025 06:59:27 +0000 Subject: [PATCH 3/7] perf(docker): Use `--cache-dir` flag instead of PIP_CACHE_DIR env var to avoid runtime permission issues for non-root user --- notebooks/.devcontainer/Dockerfile | 9 ++++----- src/sample_cpu_project/.devcontainer/Dockerfile | 11 +++++------ .../.devcontainer/Dockerfile | 11 +++++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/notebooks/.devcontainer/Dockerfile b/notebooks/.devcontainer/Dockerfile index f07126b..ba95a3a 100644 --- a/notebooks/.devcontainer/Dockerfile +++ b/notebooks/.devcontainer/Dockerfile @@ -15,18 +15,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME -# Make pip-installed tools accessible and configure pip cache location +# Make pip-installed tools accessible ENV PATH=$PATH:/home/$USERNAME/.local/bin -ENV PIP_CACHE_DIR=/pip-cache # Install development tools (linters, formatters, test runners, etc.) RUN --mount=type=cache,target=/pip-cache \ - pip install pip --upgrade + pip install --cache-dir=/pip-cache pip --upgrade COPY requirements-dev.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements-dev.txt + pip install --cache-dir=/pip-cache -r requirements-dev.txt # Install notebooks related dependencies COPY notebooks/.devcontainer/requirements.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements.txt + pip install --cache-dir=/pip-cache -r requirements.txt diff --git a/src/sample_cpu_project/.devcontainer/Dockerfile b/src/sample_cpu_project/.devcontainer/Dockerfile index 0f04f70..a116921 100644 --- a/src/sample_cpu_project/.devcontainer/Dockerfile +++ b/src/sample_cpu_project/.devcontainer/Dockerfile @@ -15,26 +15,25 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME -# Make pip-installed tools accessible and configure pip cache location +# Make pip-installed tools accessible ENV PATH=$PATH:/home/devuser/.local/bin -ENV PIP_CACHE_DIR=/pip-cache # Install Python packages (using BuildKit cache mounts for faster rebuilds) RUN --mount=type=cache,target=/pip-cache \ - pip install pip --upgrade + pip install --cache-dir=/pip-cache pip --upgrade COPY src/sample_cpu_project/.devcontainer/requirements.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements.txt + pip install --cache-dir=/pip-cache -r requirements.txt # Install common module related packages COPY src/common/requirements.txt common-requirements.txt RUN --mount=type=cache,target=/pip-cache \ - pip install -r common-requirements.txt + pip install --cache-dir=/pip-cache -r common-requirements.txt # Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements-dev.txt + pip install --cache-dir=/pip-cache -r requirements-dev.txt diff --git a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile index c631c5b..c0f9cb7 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile +++ b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile @@ -17,26 +17,25 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && chmod 0440 /etc/sudoers.d/$USERNAME USER $USERNAME -# Make pip-installed tools accessible and configure pip cache location +# Make pip-installed tools accessible ENV PATH=$PATH:/home/$USERNAME/.local/bin -ENV PIP_CACHE_DIR=/pip-cache # Install Python packages (using BuildKit cache mounts for faster rebuilds) RUN --mount=type=cache,target=/pip-cache \ - pip install pip --upgrade + pip install --cache-dir=/pip-cache pip --upgrade COPY src/sample_pytorch_gpu_project/.devcontainer/requirements.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements.txt + pip install --cache-dir=/pip-cache -r requirements.txt # Install common module related packages COPY src/common/requirements.txt common-requirements.txt RUN --mount=type=cache,target=/pip-cache \ - pip install -r common-requirements.txt + pip install --cache-dir=/pip-cache -r common-requirements.txt # Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . RUN --mount=type=cache,target=/pip-cache \ - pip install -r requirements-dev.txt + pip install --cache-dir=/pip-cache -r requirements-dev.txt RUN az extension add --name ml From b92366ea64de1557b2e5319b62d14133a0d36a08 Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Sun, 14 Dec 2025 16:01:52 +0900 Subject: [PATCH 4/7] Update src/sample_cpu_project/.devcontainer/Dockerfile Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sample_cpu_project/.devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sample_cpu_project/.devcontainer/Dockerfile b/src/sample_cpu_project/.devcontainer/Dockerfile index a116921..a5d157b 100644 --- a/src/sample_cpu_project/.devcontainer/Dockerfile +++ b/src/sample_cpu_project/.devcontainer/Dockerfile @@ -16,7 +16,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ USER $USERNAME # Make pip-installed tools accessible -ENV PATH=$PATH:/home/devuser/.local/bin +ENV PATH=$PATH:/home/$USERNAME/.local/bin # Install Python packages (using BuildKit cache mounts for faster rebuilds) RUN --mount=type=cache,target=/pip-cache \ From 3a96f28c813b36748386d2fde885ecd2aa17a038 Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Sun, 14 Dec 2025 16:09:04 +0900 Subject: [PATCH 5/7] Update notebooks/.devcontainer/Dockerfile Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- notebooks/.devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/.devcontainer/Dockerfile b/notebooks/.devcontainer/Dockerfile index ba95a3a..52161e8 100644 --- a/notebooks/.devcontainer/Dockerfile +++ b/notebooks/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.11.14 -# Create non-root user and set the default user +# create non-root user and set the default user ARG USERNAME=devuser ARG USER_UID=1000 ARG USER_GID=$USER_UID From 219571e8a05d52d3f90598543e9cf668ac933918 Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Mon, 15 Dec 2025 07:08:30 +0000 Subject: [PATCH 6/7] perf(docker): update Dockerfiles to use /root/.cache/pip for pip caching --- notebooks/.devcontainer/Dockerfile | 12 ++++++------ src/sample_cpu_project/.devcontainer/Dockerfile | 16 ++++++++-------- .../.devcontainer/Dockerfile | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/notebooks/.devcontainer/Dockerfile b/notebooks/.devcontainer/Dockerfile index 52161e8..20018ab 100644 --- a/notebooks/.devcontainer/Dockerfile +++ b/notebooks/.devcontainer/Dockerfile @@ -19,13 +19,13 @@ USER $USERNAME ENV PATH=$PATH:/home/$USERNAME/.local/bin # Install development tools (linters, formatters, test runners, etc.) -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache pip --upgrade +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip pip --upgrade COPY requirements-dev.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements-dev.txt # Install notebooks related dependencies COPY notebooks/.devcontainer/requirements.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements.txt diff --git a/src/sample_cpu_project/.devcontainer/Dockerfile b/src/sample_cpu_project/.devcontainer/Dockerfile index a5d157b..dbb1830 100644 --- a/src/sample_cpu_project/.devcontainer/Dockerfile +++ b/src/sample_cpu_project/.devcontainer/Dockerfile @@ -19,21 +19,21 @@ USER $USERNAME ENV PATH=$PATH:/home/$USERNAME/.local/bin # Install Python packages (using BuildKit cache mounts for faster rebuilds) -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache pip --upgrade +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip pip --upgrade COPY src/sample_cpu_project/.devcontainer/requirements.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements.txt # Install common module related packages COPY src/common/requirements.txt common-requirements.txt -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r common-requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r common-requirements.txt # Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements-dev.txt diff --git a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile index c0f9cb7..0bffa6f 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile +++ b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile @@ -21,21 +21,21 @@ USER $USERNAME ENV PATH=$PATH:/home/$USERNAME/.local/bin # Install Python packages (using BuildKit cache mounts for faster rebuilds) -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache pip --upgrade +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip pip --upgrade COPY src/sample_pytorch_gpu_project/.devcontainer/requirements.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements.txt # Install common module related packages COPY src/common/requirements.txt common-requirements.txt -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r common-requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r common-requirements.txt # Install development tools (linters, formatters, test runners, etc.) COPY requirements-dev.txt . -RUN --mount=type=cache,target=/pip-cache \ - pip install --cache-dir=/pip-cache -r requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --cache-dir=/root/.cache/pip -r requirements-dev.txt RUN az extension add --name ml From c1bac6b2c424c87024380679b22b3385bac1c13d Mon Sep 17 00:00:00 2001 From: Kosuke Fujimoto Date: Mon, 15 Dec 2025 07:48:00 +0000 Subject: [PATCH 7/7] fix(docker): add --yes flag to az extension installation for non-interactive mode --- src/sample_pytorch_gpu_project/.devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile index 0bffa6f..b76a8b0 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile +++ b/src/sample_pytorch_gpu_project/.devcontainer/Dockerfile @@ -38,4 +38,4 @@ COPY requirements-dev.txt . RUN --mount=type=cache,target=/root/.cache/pip \ pip install --cache-dir=/root/.cache/pip -r requirements-dev.txt -RUN az extension add --name ml +RUN az extension add --name ml --yes