WIP 290 swarm training experiments

.github/workflows/pr-test.yaml

@@ -8,15 +8,15 @@ on:
       - main
       - dev
 
+permissions:
+  contents: read
+
 jobs:
   validate-swarm:
     runs-on: self-hosted
     timeout-minutes: 45
 
     env:
-      DATADIR: /mnt/swarm_alpha/Odelia_challange/ODELIA_Challenge_unilateral/
-      SCRATCHDIR: /mnt/scratch
-      SITE_NAME: UKA
       PYTHONUNBUFFERED: 1
 
 
@@ -47,37 +47,58 @@ jobs:
       - name: Build Docker image and dummy startup kits
         run: ./buildDockerImageAndStartupKits.sh -p tests/provision/dummy_project_for_testing.yml --use-docker-cache
 
-      - name: Prepare dummy trainings
-        continue-on-error: true
+      - name: Run integration test checking documentation on github
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh check_files_on_github
+
+      - name: Run dummy training standalone
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_dummy_training_standalone
+
+      - name: Run dummy training in simulation mode
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_dummy_training_simulation_mode
+
+      - name: Run dummy training in proof-of-concept mode
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_dummy_training_poc_mode
+
+      - name: Run 3DCNN training in simulation mode
+        continue-on-error: false
         run: |
-          ./runTestsInDocker.sh prepare_dummy_trainings
-          echo "Dummy training project prepared"
+          ./runIntegrationTests.sh run_3dcnn_simulation_mode
 
-      - name: Run dummy training
+      - name: Run integration test creating startup kits
         continue-on-error: false
         run: |
-          ./runTestsInDocker.sh run_dummy_training
-          echo "Dummy training finished"
-          echo "=== Checking log output ==="
-          ls -lh workspace/*/prod_00/client_A/logs || echo "No logs found for dummy training"
+          ./runIntegrationTests.sh create_startup_kits
 
-      - name: Run 3D CNN preflight check
+      - name: Run intergration test listing licenses
         continue-on-error: false
         run: |
-          ./runTestsInDocker.sh preflight_check
-          echo "Preflight check finished"
-          echo "=== Checking synthetic log output ==="
-          ls -lh workspace/*/prod_00/client_A/logs || echo "No logs found for preflight"
+          ./runIntegrationTests.sh run_list_licenses
 
-      - name: Run Unit Tests inside Docker
-        continue-on-error: true
+      - name: Run integration test Docker GPU preflight check
+        continue-on-error: false
         run: |
-          ./runTestsInDocker.sh run_tests
-          echo "=== [LOG CHECK] ==="
-          docker logs $(docker ps -a -q --latest) | grep -i "error" && echo "Error found in logs" || echo "No error found"
+          ./runIntegrationTests.sh run_docker_gpu_preflight_check
 
-      - name: Cleanup training artifacts
-        continue-on-error: true
+      - name: Run integration test data access preflight check
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_data_access_preflight_check
+
+      - name: Run dummy training in swarm
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_dummy_training_in_swarm
+
+      - name: Kill orphaned containers from this run (if any)
+        continue-on-error: false
+        if: always()
         run: |
-          ./runTestsInDocker.sh cleanup
-          echo "Cleanup finished"
+          ./runIntegrationTests.sh kill_server_and_clients

.github/workflows/update-apt-versions.yml

@@ -1,67 +1,53 @@
-name: Auto Update APT Versions
+name: Auto Update APT Versions (Self-hosted)
 
 on:
   schedule:
-    - cron: '0 5 * * 0'  # Every Sunday at 05:00 UTC
+    # run eveyday at 04:00 UTC
+   - cron: '0 4 * * *'
   workflow_dispatch:
 
 jobs:
   update-apt:
-    name: Update APT Package Versions in Dockerfile
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    timeout-minutes: 60
 
     steps:
       - name: Checkout repository (with submodules)
         uses: actions/checkout@v3
         with:
           submodules: true
+          fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-
-      - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install -y git
-
-      - name: Configure Git for CI
+      - name: Set up Git
         run: |
           git config --global user.email "ci@github.com"
           git config --global user.name "GitHub CI"
 
       - name: Create and switch to apt-update branch
         run: |
           git checkout -b ci/apt-update || git switch ci/apt-update
-
-      - name: Make update script executable
-        run: chmod +x scripts/ci/update_apt_versions.sh
-
       - name: Run APT update script
-        run: scripts/ci/update_apt_versions.sh
 
-      - name: Check for changes
-        id: check_changes
         run: |
-          if git diff --quiet; then
-            echo "NO_CHANGES=true" >> $GITHUB_ENV
-            echo "No changes detected."
-          else
-            echo "NO_CHANGES=false" >> $GITHUB_ENV
-            echo "Changes detected."
-          fi
+          chmod +x scripts/ci/update_apt_versions.sh
+          scripts/ci/update_apt_versions.sh
+      - name: Show git diff for debugging
+        run: git diff || true
 
-      - name: Push ci/apt-update to origin
+      - name: Push apt-update branch
         if: env.NO_CHANGES == 'false'
-        run: git push origin ci/apt-update
+        run: git push origin ci/apt-update --force
 
       - name: Create Pull Request
         if: env.NO_CHANGES == 'false'
         uses: peter-evans/create-pull-request@v5
         with:
           commit-message: "chore: update apt versions in Dockerfile_ODELIA"
           branch: ci/apt-update
+          branch-suffix: timestamp
           title: "chore: Update APT versions in Dockerfile"
           body: |
             This PR automatically updates APT package version numbers in `Dockerfile_ODELIA`
             based on a rebuild and inspection of installation logs.
           base: main
+          delete-branch: false

.gitignore

@@ -180,3 +180,6 @@ provision
 
 # Ignore provisioned files
 /workspace/
+
+# Ignore directory for caching pre-trained models
+docker_config/torch_home_cache