From 4bfbe24137c86ae628d278f925003b653417b79c Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 13 Aug 2025 12:23:49 +0100 Subject: [PATCH 01/10] add copier debug container --- .copier-answers.yml | 4 ++-- .github/CONTRIBUTING.md | 2 +- .github/workflows/ci.yml | 9 +++++++++ Dockerfile | 21 +++++++++++++++++++++ pyproject.toml | 1 + 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index c46999d..eb2056d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 4.1.0 +_commit: 4.2.0 _src_path: https://github.com/diamondlightsource/python-copier-template author_email: giles.knap@diamond.ac.uk author_name: Giles Knap @@ -9,7 +9,7 @@ component_type: service description: An example simulation IOC for testing fastCS distribution_name: fastcs-example docker: true -docker_debug: false +docker_debug: true docs_type: README git_platform: github.com github_org: DiamondLightSource diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 32f2a57..56956cf 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -24,4 +24,4 @@ It is recommended that developers use a [vscode devcontainer](https://code.visua This project was created using the [Diamond Light Source Copier Template](https://github.com/DiamondLightSource/python-copier-template) for Python projects. -For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/4.1.0/how-to.html). +For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/4.2.0/how-to.html). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb2aedf..7dc1e4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,15 @@ jobs: contents: read packages: write + debug_container: + needs: [container, test] + uses: ./.github/workflows/_debug_container.yml + with: + publish: ${{ needs.test.result == 'success' }} + permissions: + contents: read + packages: write + dist: uses: ./.github/workflows/_dist.yml diff --git a/Dockerfile b/Dockerfile index a59582b..c53bc6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,27 @@ WORKDIR /workspaces/fastcs-example RUN touch dev-requirements.txt && pip install -c dev-requirements.txt .[demo] +FROM build AS debug + + +# Set origin to use ssh +RUN git remote set-url origin git@github.com:DiamondLightSource/fastcs-example.git + + +# For this pod to understand finding user information from LDAP +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install libnss-ldapd -y +RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf + +# Make editable and debuggable +RUN pip install debugpy +RUN pip install -e . + +# Alternate entrypoint to allow devcontainer to attach +ENTRYPOINT [ "/bin/bash", "-c", "--" ] +CMD [ "while true; do sleep 30; done;" ] + + # The runtime stage copies the built venv into a slim runtime container FROM python:${PYTHON_VERSION}-slim AS runtime # Add apt-get system dependecies for runtime here if needed diff --git a/pyproject.toml b/pyproject.toml index 1c0f0a8..9413911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ lint.select = [ "C4", # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 "E", # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e "F", # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f + "N", # pep8-naming - https://docs.astral.sh/ruff/rules/#pep8-naming-n "W", # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w "I", # isort - https://docs.astral.sh/ruff/rules/#isort-i "UP", # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up From 7be83d2be12c81b758e19b73a8b9719d84b06b0d Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 14 Aug 2025 12:12:34 +0000 Subject: [PATCH 02/10] improve debug target --- Dockerfile | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index c53bc6d..655cac2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,11 @@ FROM python:${PYTHON_VERSION} AS developer # Add any system dependencies for the developer/build environment here RUN apt-get update && apt-get install -y --no-install-recommends \ + gdb \ + busybox \ graphviz \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && busybox --install -s # Set up a virtual environment and put it in PATH RUN python -m venv /venv @@ -17,33 +20,40 @@ FROM developer AS build # Requires buildkit 0.17.0 COPY --chmod=o+wrX . /workspaces/fastcs-example WORKDIR /workspaces/fastcs-example -RUN touch dev-requirements.txt && pip install -c dev-requirements.txt .[demo] - +RUN touch dev-requirements.txt && pip install debugpy -c dev-requirements.txt .[demo] FROM build AS debug - # Set origin to use ssh RUN git remote set-url origin git@github.com:DiamondLightSource/fastcs-example.git +# Make editable and debuggable +RUN pip install debugpy -e '.[dev]' +RUN chmod o+wrX /venv /workspaces + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + busybox \ + gdb \ + libnss-ldapd \ + && rm -rf /var/lib/apt/lists/* \ + && busybox --install -s # For this pod to understand finding user information from LDAP -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive apt install libnss-ldapd -y RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf -# Make editable and debuggable -RUN pip install debugpy -RUN pip install -e . - -# Alternate entrypoint to allow devcontainer to attach -ENTRYPOINT [ "/bin/bash", "-c", "--" ] -CMD [ "while true; do sleep 30; done;" ] - +# Alternate entrypoint to allow restarting the IOC +ENTRYPOINT [ "/bin/bash", "-c", "sleep infinity" ] # The runtime stage copies the built venv into a slim runtime container FROM python:${PYTHON_VERSION}-slim AS runtime -# Add apt-get system dependecies for runtime here if needed + +# Add apt-get system dependecies for runtime +RUN apt-get update && apt-get install -y --no-install-recommends \ + gdb \ + && rm -rf /var/lib/apt/lists/* + + COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH From 81db753f8a3a34dd185b4c16843521eed7e91773 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 14 Aug 2025 12:31:09 +0000 Subject: [PATCH 03/10] highlight stages with comments --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 655cac2..9e4e412 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ # The devcontainer should use the developer target and run as root with podman # or docker with user namespaces. ARG PYTHON_VERSION=3.11 + +# The developer is used by devcontainers ############################################### FROM python:${PYTHON_VERSION} AS developer # Add any system dependencies for the developer/build environment here @@ -15,13 +17,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN python -m venv /venv ENV PATH=/venv/bin:$PATH -# The build stage installs the context into the venv +# The build stage installs the context into the venv ################################### FROM developer AS build + # Requires buildkit 0.17.0 COPY --chmod=o+wrX . /workspaces/fastcs-example WORKDIR /workspaces/fastcs-example RUN touch dev-requirements.txt && pip install debugpy -c dev-requirements.txt .[demo] +# The debug stage allows for debugging and development ################################# FROM build AS debug # Set origin to use ssh @@ -45,7 +49,7 @@ RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf # Alternate entrypoint to allow restarting the IOC ENTRYPOINT [ "/bin/bash", "-c", "sleep infinity" ] -# The runtime stage copies the built venv into a slim runtime container +# The runtime stage has built venv only ################################################ FROM python:${PYTHON_VERSION}-slim AS runtime # Add apt-get system dependecies for runtime @@ -53,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ && rm -rf /var/lib/apt/lists/* - COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH From 7fe1798f3e82047708243c1b6985f6cf5458d5a8 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 14 Aug 2025 12:33:59 +0000 Subject: [PATCH 04/10] restore debug container build action --- .github/workflows/_debug_container.yml | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/_debug_container.yml diff --git a/.github/workflows/_debug_container.yml b/.github/workflows/_debug_container.yml new file mode 100644 index 0000000..31206c1 --- /dev/null +++ b/.github/workflows/_debug_container.yml @@ -0,0 +1,49 @@ +on: + workflow_call: + inputs: + publish: + type: boolean + description: If true, pushes image to container registry + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create tags for publishing debug image + id: debug-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=tag,suffix=-debug + type=raw,value=latest-debug + + - name: Build and publish debug image to container registry + if: github.ref_type == 'tag' + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false + with: + context: . + push: true + target: debug + tags: ${{ steps.debug-meta.outputs.tags }} From e713e5a0fea18cf02fa412d6ffdeb2abb631fd55 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 15 Aug 2025 15:08:45 +0100 Subject: [PATCH 05/10] fixup py debug launcher --- .vscode/launch.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7d563e0..2054795 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,22 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false + }, { "name": "Debug Unit Test", "type": "debugpy", From bc25b59f57ec6a78e51b633fea2b2d90008b956f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 18 Aug 2025 08:21:39 +0100 Subject: [PATCH 06/10] adding DEBUGGING.md --- DEBUGGING.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 DEBUGGING.md diff --git a/DEBUGGING.md b/DEBUGGING.md new file mode 100644 index 0000000..f60090b --- /dev/null +++ b/DEBUGGING.md @@ -0,0 +1,62 @@ +# Developing in Cluster + +e.g. fastcs-example + +# Deploy the develop mode chart + +First deploy the developer mode helm chart: +- checkout a branch 'develop' +- Switch the container to develop mode with developMode: true +- push the change +- ec deploy bl47p-ea-fastcs-01 develop + +This will deploy the container running sleep and with a new PVC called bl47p-ea-fastcs-01-develop, containing: +- /workspaces mounted at /workspaces in the container +- /venv mounted at /venv in the container +- both mounts above will have the original container contents copied in by an init-container + + +# Get the process started with debugpy + +Exec this in the container + +```bash +python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 --configure-subProcess true -m fastcs_example run /epics/ioc/config/controller.yaml +``` + +add `--wait-for-client` to debugpy to try debugging startup. + +Port forward the debug port on your workstation + +```bash +k port-forward -n p47-beamline bl47p-ea-fastcs-01-0 5678 +``` + +# Get VSCode connected + +mount the ...-develop PVC on your workstation and open it with vscode + +```bash +pv-mounter mount p47-beamline bl47p-ea-fastcs-01-develop ./debugger +code debugger/workspaces/fastcs-example +``` + +Launch remote debugging with a launcher configuration like this: +```json + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false + }, +``` From 54284ad6aeb67cec94897298ca8b1d2ae7332bec Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 19 Aug 2025 07:47:24 +0000 Subject: [PATCH 07/10] update to new beta fastcs version --- DEBUGGING.md | 4 +--- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/DEBUGGING.md b/DEBUGGING.md index f60090b..f05c222 100644 --- a/DEBUGGING.md +++ b/DEBUGGING.md @@ -21,11 +21,9 @@ This will deploy the container running sleep and with a new PVC called bl47p-ea- Exec this in the container ```bash -python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 --configure-subProcess true -m fastcs_example run /epics/ioc/config/controller.yaml +python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 --wait-for-client --configure-subProcess true -m fastcs_example run /epics/ioc/config/controller.yaml ``` -add `--wait-for-client` to debugpy to try debugging startup. - Port forward the debug port on your workstation ```bash diff --git a/pyproject.toml b/pyproject.toml index 9413911..55da48f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ classifiers = [ ] description = "An example simulation IOC for testing fastCS" dependencies = [ - "fastcs", + "fastcs>=0.9.2-beta.1", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" From 77daed5f5a2bd01ab4c0e1d8c812e8754b254674 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 19 Aug 2025 11:44:05 +0000 Subject: [PATCH 08/10] put -debug on image name instead of version suffix --- .github/workflows/_debug_container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_debug_container.yml b/.github/workflows/_debug_container.yml index 31206c1..3819766 100644 --- a/.github/workflows/_debug_container.yml +++ b/.github/workflows/_debug_container.yml @@ -32,10 +32,10 @@ jobs: id: debug-meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository }} + images: ghcr.io/${{ github.repository }}-debug tags: | - type=ref,event=tag,suffix=-debug - type=raw,value=latest-debug + type=ref,event=tag + type=raw,value=latest - name: Build and publish debug image to container registry if: github.ref_type == 'tag' From 72bcd73f7bb7c5d16bc5f91b101b9c5fd5ca9da2 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 12 Sep 2025 09:56:34 +0100 Subject: [PATCH 09/10] add in stdio-socket --- Dockerfile | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e4e412..00fd4d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,8 @@ FROM developer AS build # Requires buildkit 0.17.0 COPY --chmod=o+wrX . /workspaces/fastcs-example WORKDIR /workspaces/fastcs-example -RUN touch dev-requirements.txt && pip install debugpy -c dev-requirements.txt .[demo] +RUN touch dev-requirements.txt && pip install stdio-socket debugpy \ + -c dev-requirements.txt .[demo] # The debug stage allows for debugging and development ################################# FROM build AS debug diff --git a/pyproject.toml b/pyproject.toml index 55da48f..9413911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ classifiers = [ ] description = "An example simulation IOC for testing fastCS" dependencies = [ - "fastcs>=0.9.2-beta.1", + "fastcs", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" From 2e8def50d9ab635b416798c420da43f5fd830f76 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 12 Sep 2025 13:30:43 +0100 Subject: [PATCH 10/10] pull in latest stdio-socket --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 00fd4d4..43e57c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ FROM developer AS build # Requires buildkit 0.17.0 COPY --chmod=o+wrX . /workspaces/fastcs-example WORKDIR /workspaces/fastcs-example -RUN touch dev-requirements.txt && pip install stdio-socket debugpy \ +RUN touch dev-requirements.txt && pip install stdio-socket>=1.3.1 debugpy \ -c dev-requirements.txt .[demo] # The debug stage allows for debugging and development #################################