From 8a51382136c20b98dc066ab7c726e4d2153e2fdb Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 14:23:40 +0100 Subject: [PATCH 1/9] Repository setup & services cleanup --- .pre-commit-config.yaml | 25 ++++++++++++++ README.md | 24 ++++++++++++- compose.yml | 5 +++ containers_registry/README.md | 60 +++++++++++++++++++++++++++++++++ containers_registry/compose.yml | 10 ++++++ portainer/compose.yml | 13 +++++++ ppa_packages/Dockerfile | 10 ++++++ ppa_packages/README.md | 41 ++++++++++++++++++++++ ppa_packages/compose.yml | 10 ++++++ 9 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml create mode 100644 compose.yml create mode 100644 containers_registry/README.md create mode 100644 containers_registry/compose.yml create mode 100644 portainer/compose.yml create mode 100644 ppa_packages/Dockerfile create mode 100644 ppa_packages/README.md create mode 100644 ppa_packages/compose.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a223d7e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: pretty-format-json + - id: trailing-whitespace + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [-L, passt, -w] diff --git a/README.md b/README.md index dffbbfa..0df5da1 100644 --- a/README.md +++ b/README.md @@ -1 +1,23 @@ -# robolab_mlops \ No newline at end of file +# robolab_mlops + +Collection of all services used in Robotics Laboratory (mainly for the [Aegis robot](https://github.com/AGH-CEAI/aegis_ros)). + +## List of services with their default ports + +* [containers_registry](http://localhost:5000) +* [portainer](http://localhost:8999) +* [ppa_packages](http://localhost:80/debian) + +## Start all services + +```bash +# In the main directory +docker compose up -d +``` + +## Shutdown all services + +```bash +# In the main directory +docker compose down +``` diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..b1b6665 --- /dev/null +++ b/compose.yml @@ -0,0 +1,5 @@ +name: robolab-ops +include: + - containers_registry/compose.yml + - portainer/compose.yml + - ppa_packages/compose.yml diff --git a/containers_registry/README.md b/containers_registry/README.md new file mode 100644 index 0000000..6f26c93 --- /dev/null +++ b/containers_registry/README.md @@ -0,0 +1,60 @@ +# Containers registry + +This service allows to self-host container registry for Podman/Docker. + +Based on [this](https://infotechys.com/host-your-own-podman-registry/) tutorial. + + +## Prepare the host +```bash +export REGISTRY_PATH=/var/lib/registry +sudo mkdir -p $REGISTRY_PATH +``` + +## Run the service +```bash +podman-compose up -d +# or +docker compose up -d +``` + +## Setup the clients +The easiest way is to provide the registry address manually: +```bash +sudo vim /etc/containers/registries.conf +``` + +Assuming that `HOSTNAME` is your host IP address: +``` +[[registry]] +location = "HOSTNAME:5000" +insecure = true +``` + +> [!WARNING] +> This configuration is NOT SAFE. Consider setting up proper SSL certificates before running on your machine. + +Restart the contenerization engine: +```bash +sudo systemctl restart podman +# or +sudo systemctl restart docker +``` + +## Use it + +### Push +```bash +podman images +podman tag localhost/example:latest HOSTNAME:5000/example:latest +podman push HOSTNAME:5000/example:latest +``` + +### Pull +```bash +podman pull HOSTNAME:5000/example:latest +podman images +``` + +--- +[Back to main README](../README.md) diff --git a/containers_registry/compose.yml b/containers_registry/compose.yml new file mode 100644 index 0000000..4bb7289 --- /dev/null +++ b/containers_registry/compose.yml @@ -0,0 +1,10 @@ +services: + containers_registry: + image: docker.io/library/registry:2 + # container_name: containers_registry + privileged: true + restart: always + ports: + - "5000:5000" + volumes: + - /srv/data/hosting/volumes/containers_registry:/var/lib/registry diff --git a/portainer/compose.yml b/portainer/compose.yml new file mode 100644 index 0000000..04c41fe --- /dev/null +++ b/portainer/compose.yml @@ -0,0 +1,13 @@ +services: + portainer: + image: portainer/portainer-ce:latest + restart: always + ports: + # - "8000:8000" # Secure TLS port for remote control + - "9443:9443" + - "9000:9000" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer_data:/data +volumes: + portainer_data: diff --git a/ppa_packages/Dockerfile b/ppa_packages/Dockerfile new file mode 100644 index 0000000..0d6e904 --- /dev/null +++ b/ppa_packages/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y \ + apache2 \ + && apt autoremove -y \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* + +CMD ["apachectl", "-D", "FOREGROUND"] diff --git a/ppa_packages/README.md b/ppa_packages/README.md new file mode 100644 index 0000000..dfea973 --- /dev/null +++ b/ppa_packages/README.md @@ -0,0 +1,41 @@ +# Private Packages Repo (PPA) + +This module enables storing private *.deb packages used for automated container built. + +More [here](https://linuxconfig.org/easy-way-to-create-a-debian-package-and-local-package-repository). + +## Preparing packages + +**Copy packages:** + +Copy all `*.deb` packages into the `./packages` directory. + +**Generate package list:** + +* After moving all packages to the `./packages` directory, you need to generate the **package list**. +* On a Debian-based system (e.g., Ubuntu), run the following command inside the `./packages` directory. + * There is **no need** to run this inside the container. +```bash +dpkg-scanpackages . | gzip -c9 > Packages.gz +``` +or +```bash +sudo sh -c 'dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz' +``` + +* Done! The PPA server is now ready to serve packages. + +## Running Server + +**Docker Compose**: +```bash +docker compose up -d +``` +You can check the PPA in your webbrowser: `http://HOST/debian/`. + +### Adding new packages + +Follow `Preparing packages` step and the list of packages will automatically update. + +--- +[Back to main README](../README.md) diff --git a/ppa_packages/compose.yml b/ppa_packages/compose.yml new file mode 100644 index 0000000..a3993e6 --- /dev/null +++ b/ppa_packages/compose.yml @@ -0,0 +1,10 @@ +services: + ppa_packages: + build: + context: . + dockerfile: "${DOCKERFILE:-Dockerfile}" + ports: + - 80:80 + restart: always + volumes: + - /srv/data/hosting/volumes/ppa_packages:/var/www/html/debian From a4bb0de98311a03b21ca58cb1a44e01ba3089880 Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 14:27:57 +0100 Subject: [PATCH 2/9] CHG: updated readme --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index 0df5da1..823a347 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,44 @@ docker compose up -d # In the main directory docker compose down ``` + +--- + +### Private Packages Repo (PPA) + +How to Set up private repo [here](./ppa_packages/README.md). + +**Adding private repo** +```bash +echo "deb [trusted=yes] http://192.168.0.100/debian ./" | tee -a /etc/apt/sources.list > /dev/null +``` +### Containers registry + +[Instructions how to use self-hosted container registry.](./containers_registry/README.md) + +#### Building & pushing a particular release tag +```bash +export REGISTRY_HOSTNAME=geonosis:5000 +export AEGIS_ROS_VERSION= +export AEGIS_CONTAINER_VERSION= + +# if the building is happening on the same machine as PPA server, add argument: +# --add-host $(hostname):$(hostname -I | awk '{print $1}') +podman build . -t ceai/aegis_dev:${AEGIS_CONTAINER_VERSION} --build-arg AEGIS_ROS_TAG=${AEGIS_ROS_VERSION} +podman tag ceai/aegis_dev:${AEGIS_CONTAINER_VERSION} ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} +podman image inspect --format='{{json .Config.Labels}}' ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} + +podman push ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} +``` + +#### Pulling & entering a particular release tag +```bash +export REGISTRY_HOSTNAME=geonosis:5000 +export AEGIS_CONTAINER_VERSION= + +podman pull ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} +podman image inspect --format='{{json .Config.Labels}}' ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} + +toolbox create --image ${REGISTRY_HOSTNAME}/ceai/aegis:${AEGIS_CONTAINER_VERSION} +toolbox enter aegis-${AEGIS_CONTAINER_VERSION} +``` From 93f9106ab4037a6a41676e1c4bdd5d4465e0febb Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 14:28:31 +0100 Subject: [PATCH 3/9] CHG: updated pre-commit --- .pre-commit-config.yaml | 45 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a223d7e..1aede00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,25 +1,30 @@ repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 +- repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update + rev: v0.9.0 hooks: - - id: check-added-large-files - - id: check-case-conflict - - id: check-json - - id: check-merge-conflict - - id: check-symlinks - - id: check-toml - - id: check-xml - - id: check-yaml - - id: debug-statements - - id: destroyed-symlinks - - id: detect-private-key - - id: end-of-file-fixer - - id: mixed-line-ending - - id: pretty-format-json - - id: trailing-whitespace + - id: pre-commit-update - - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 hooks: - - id: codespell + - id: check-added-large-files + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: pretty-format-json + - id: trailing-whitespace + +- repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell args: [-L, passt, -w] From 8a6d65b2a1d70a50a77874e46ae63dc0c16e2e89 Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 17:22:16 +0100 Subject: [PATCH 4/9] Added issue and PR templates --- .github/issue_template.md | 16 ++++++++++++++++ .github/pull_request_template.md | 27 +++++++++++++++++++++++++++ .github/workflows/pre-commit.yaml | 11 +++++++++++ 3 files changed, 54 insertions(+) create mode 100644 .github/issue_template.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/pre-commit.yaml diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..e3b3945 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,16 @@ +--- + +name: "Issue template" +about: "Minimal info about an issue" +title: "" +ref: "devel" + +--- + +## Description + + + + +## References + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f73645b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,27 @@ + +## Description + + +## Motivation and context + + + + + + +## How has this been tested? + + + + + +## Checklist + + +- [ ] All TODOs in the code have been resolved or linked to a proper issue. +- [ ] Code has been (auto)formatted. +- [ ] Documentation (e.g., README, CHANGELOG, Wiki) has been updated. +- [ ] All automated checks have passed. + +--- +Clickup task: [hash](link) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..08f7311 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,11 @@ +--- +name: Pre-Commit + +on: + push: + +jobs: + pre-commit: + uses: ros-controls/ros2_control_ci/.github/workflows/reusable-pre-commit.yml@master + with: + ros_distro: humble From 1ba9afde7f0709c9075842691f4c2b67c5f34880 Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 18:36:50 +0100 Subject: [PATCH 5/9] Added ClearML --- README.md | 9 +- clearml/README.md | 13 +++ clearml/compose.yml | 194 +++++++++++++++++++++++++++++++++++++++++++ compose.yml | 3 + docker-compose.yml | 195 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 clearml/README.md create mode 100644 clearml/compose.yml create mode 100644 docker-compose.yml diff --git a/README.md b/README.md index 823a347..516a16a 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,12 @@ Collection of all services used in Robotics Laboratory (mainly for the [Aegis ro ## List of services with their default ports -* [containers_registry](http://localhost:5000) -* [portainer](http://localhost:8999) -* [ppa_packages](http://localhost:80/debian) +* ClearML - [clearml](http://localhost:8080) +* Containers registry - [containers_registry](http://localhost:5000) +* Portainer - [portainer](http://localhost:8999) +* PPA repository - [ppa_packages](http://localhost:80/debian) + +**Before first run, be sure to read the whole README!** ## Start all services diff --git a/clearml/README.md b/clearml/README.md new file mode 100644 index 0000000..328abf4 --- /dev/null +++ b/clearml/README.md @@ -0,0 +1,13 @@ +# ClearML + +Deploy instructions can be found in the [official docs](https://clear.ml/docs/latest/docs/deploying_clearml/clearml_server_linux_mac). + +## Highlight +Before first run you need to create all catalogs and setup the Elasticsearch params: + +```bash +echo "vm.max_map_count=524288" > /tmp/99-clearml.conf +sudo mv /tmp/99-clearml.conf /etc/sysctl.d/99-clearml.conf +sudo sysctl -w vm.max_map_count=524288 +sudo service docker restart +``` diff --git a/clearml/compose.yml b/clearml/compose.yml new file mode 100644 index 0000000..ed15358 --- /dev/null +++ b/clearml/compose.yml @@ -0,0 +1,194 @@ +services: + + apiserver: + command: + - apiserver + container_name: clearml-apiserver + image: clearml/server:latest + restart: unless-stopped + volumes: + - /srv/data/hosting/volumes/clearml/logs:/var/log/clearml + - /srv/data/hosting/volumes/clearml/config:/opt/clearml/config + - /srv/data/hosting/volumes/clearml/data/fileserver:/mnt/fileserver + depends_on: + - redis + - mongo + - elasticsearch + - fileserver + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + CLEARML_SERVER_DEPLOYMENT_TYPE: linux + CLEARML__apiserver__pre_populate__enabled: "true" + CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" + CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" + CLEARML__services__async_urls_delete__enabled: "true" + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} + ports: + - "8008:8008" + networks: + - backend + - frontend + + elasticsearch: + networks: + - backend + container_name: clearml-elastic + environment: + bootstrap.memory_lock: "true" + cluster.name: clearml + cluster.routing.allocation.node_initial_primaries_recoveries: "500" + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb + discovery.type: "single-node" + http.compression_level: "7" + node.name: clearml + reindex.remote.whitelist: "'*.*'" + xpack.security.enabled: "false" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + image: elasticsearch:8.17.0 + restart: unless-stopped + volumes: + - /srv/data/hosting/volumes/clearml/data/elastic_7:/usr/share/elasticsearch/data + - /usr/share/elasticsearch/logs + + fileserver: + networks: + - backend + - frontend + command: + - fileserver + container_name: clearml-fileserver + image: clearml/server:latest + environment: + CLEARML__fileserver__delete__allow_batch: "true" + restart: unless-stopped + volumes: + - /srv/data/hosting/volumes/clearml/logs:/var/log/clearml + - /srv/data/hosting/volumes/clearml/data/fileserver:/mnt/fileserver + - /srv/data/hosting/volumes/clearml/config:/opt/clearml/config + ports: + - "8081:8081" + + mongo: + networks: + - backend + container_name: clearml-mongo + image: mongo:7.0.22 + restart: unless-stopped + command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200 + volumes: + - /srv/data/hosting/volumes/clearml/data/mongo_4/db:/data/db + - /srv/data/hosting/volumes/clearml/data/mongo_4/configdb:/data/configdb + + redis: + networks: + - backend + container_name: clearml-redis + image: redis:7.4.1 + restart: unless-stopped + volumes: + - /srv/data/hosting/volumes/clearml/data/redis:/data + + webserver: + command: + - webserver + container_name: clearml-webserver + # environment: + # CLEARML_SERVER_SUB_PATH : clearml-web # Allow Clearml to be served with a URL path prefix. + image: clearml/server:latest + restart: unless-stopped + depends_on: + - apiserver + ports: + - "8080:80" + networks: + - backend + - frontend + + async_delete: + depends_on: + - apiserver + - redis + - mongo + - elasticsearch + - fileserver + container_name: clearml-async_delete + image: clearml/server:latest + networks: + - backend + restart: unless-stopped + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + PYTHONPATH: /opt/clearml/apiserver + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + entrypoint: + - python3 + - -m + - jobs.async_urls_delete + - --fileserver-host + - http://fileserver:8081 + volumes: + - /srv/data/hosting/volumes/clearml/logs:/var/log/clearml + - /srv/data/hosting/volumes/clearml/config:/opt/clearml/config + + agent-services: + networks: + - backend + container_name: clearml-agent-services + image: clearml/clearml-agent-services:latest + deploy: + restart_policy: + condition: on-failure + privileged: true + environment: + CLEARML_HOST_IP: ${CLEARML_HOST_IP} + CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-} + CLEARML_API_HOST: http://apiserver:8008 + CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-} + CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-$CLEARML_API_ACCESS_KEY} + CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-$CLEARML_API_SECRET_KEY} + CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER} + CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS} + CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0} + CLEARML_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04" + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-} + AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-} + AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-} + GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} + CLEARML_WORKER_ID: "clearml-services" + CLEARML_AGENT_DOCKER_HOST_MOUNT: "/opt/clearml/agent:/root/.clearml" + SHUTDOWN_IF_NO_ACCESS_KEY: 1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /srv/data/hosting/volumes/clearml/agent:/root/.clearml + depends_on: + - apiserver + entrypoint: > + bash -c "curl --retry 10 --retry-delay 10 --retry-connrefused 'http://apiserver:8008/debug.ping' && /usr/agent/entrypoint.sh" + +networks: + backend: + driver: bridge + frontend: + driver: bridge diff --git a/compose.yml b/compose.yml index b1b6665..637368f 100644 --- a/compose.yml +++ b/compose.yml @@ -1,5 +1,8 @@ name: robolab-ops include: + - path: clearml/compose.yml + env_file: + - /srv/data/hosting/secrets/clearml.env - containers_registry/compose.yml - portainer/compose.yml - ppa_packages/compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1b10c06 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,195 @@ +version: "3.6" +services: + + apiserver: + command: + - apiserver + container_name: clearml-apiserver + image: clearml/server:latest + restart: unless-stopped + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/config:/opt/clearml/config + - /opt/clearml/data/fileserver:/mnt/fileserver + depends_on: + - redis + - mongo + - elasticsearch + - fileserver + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + CLEARML_SERVER_DEPLOYMENT_TYPE: linux + CLEARML__apiserver__pre_populate__enabled: "true" + CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" + CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" + CLEARML__services__async_urls_delete__enabled: "true" + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} + ports: + - "8008:8008" + networks: + - backend + - frontend + + elasticsearch: + networks: + - backend + container_name: clearml-elastic + environment: + bootstrap.memory_lock: "true" + cluster.name: clearml + cluster.routing.allocation.node_initial_primaries_recoveries: "500" + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb + discovery.type: "single-node" + http.compression_level: "7" + node.name: clearml + reindex.remote.whitelist: "'*.*'" + xpack.security.enabled: "false" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + image: elasticsearch:8.17.0 + restart: unless-stopped + volumes: + - /opt/clearml/data/elastic_7:/usr/share/elasticsearch/data + - /usr/share/elasticsearch/logs + + fileserver: + networks: + - backend + - frontend + command: + - fileserver + container_name: clearml-fileserver + image: clearml/server:latest + environment: + CLEARML__fileserver__delete__allow_batch: "true" + restart: unless-stopped + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/data/fileserver:/mnt/fileserver + - /opt/clearml/config:/opt/clearml/config + ports: + - "8081:8081" + + mongo: + networks: + - backend + container_name: clearml-mongo + image: mongo:6.0.19 + restart: unless-stopped + command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200 + volumes: + - /opt/clearml/data/mongo_4/db:/data/db + - /opt/clearml/data/mongo_4/configdb:/data/configdb + + redis: + networks: + - backend + container_name: clearml-redis + image: redis:7.4.1 + restart: unless-stopped + volumes: + - /opt/clearml/data/redis:/data + + webserver: + command: + - webserver + container_name: clearml-webserver + # environment: + # CLEARML_SERVER_SUB_PATH : clearml-web # Allow Clearml to be served with a URL path prefix. + image: clearml/server:latest + restart: unless-stopped + depends_on: + - apiserver + ports: + - "8080:80" + networks: + - backend + - frontend + + async_delete: + depends_on: + - apiserver + - redis + - mongo + - elasticsearch + - fileserver + container_name: async_delete + image: clearml/server:latest + networks: + - backend + restart: unless-stopped + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + PYTHONPATH: /opt/clearml/apiserver + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + entrypoint: + - python3 + - -m + - jobs.async_urls_delete + - --fileserver-host + - http://fileserver:8081 + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/config:/opt/clearml/config + + agent-services: + networks: + - backend + container_name: clearml-agent-services + image: clearml/clearml-agent-services:latest + deploy: + restart_policy: + condition: on-failure + privileged: true + environment: + CLEARML_HOST_IP: ${CLEARML_HOST_IP} + CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-} + CLEARML_API_HOST: http://apiserver:8008 + CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-} + CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-$CLEARML_API_ACCESS_KEY} + CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-$CLEARML_API_SECRET_KEY} + CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER} + CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS} + CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0} + CLEARML_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04" + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-} + AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-} + AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-} + GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} + CLEARML_WORKER_ID: "clearml-services" + CLEARML_AGENT_DOCKER_HOST_MOUNT: "/opt/clearml/agent:/root/.clearml" + SHUTDOWN_IF_NO_ACCESS_KEY: 1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /opt/clearml/agent:/root/.clearml + depends_on: + - apiserver + entrypoint: > + bash -c "curl --retry 10 --retry-delay 10 --retry-connrefused 'http://apiserver:8008/debug.ping' && /usr/agent/entrypoint.sh" + +networks: + backend: + driver: bridge + frontend: + driver: bridge From 6802c275eed40968cb5f437348b32060200fb3bb Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 18:41:58 +0100 Subject: [PATCH 6/9] removed wrong compose file --- docker-compose.yml | 195 --------------------------------------------- 1 file changed, 195 deletions(-) delete mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 1b10c06..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,195 +0,0 @@ -version: "3.6" -services: - - apiserver: - command: - - apiserver - container_name: clearml-apiserver - image: clearml/server:latest - restart: unless-stopped - volumes: - - /opt/clearml/logs:/var/log/clearml - - /opt/clearml/config:/opt/clearml/config - - /opt/clearml/data/fileserver:/mnt/fileserver - depends_on: - - redis - - mongo - - elasticsearch - - fileserver - environment: - CLEARML_ELASTIC_SERVICE_HOST: elasticsearch - CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_MONGODB_SERVICE_HOST: mongo - CLEARML_MONGODB_SERVICE_PORT: 27017 - CLEARML_REDIS_SERVICE_HOST: redis - CLEARML_REDIS_SERVICE_PORT: 6379 - CLEARML_SERVER_DEPLOYMENT_TYPE: linux - CLEARML__apiserver__pre_populate__enabled: "true" - CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" - CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" - CLEARML__services__async_urls_delete__enabled: "true" - CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" - CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} - CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} - ports: - - "8008:8008" - networks: - - backend - - frontend - - elasticsearch: - networks: - - backend - container_name: clearml-elastic - environment: - bootstrap.memory_lock: "true" - cluster.name: clearml - cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 500mb - cluster.routing.allocation.disk.watermark.high: 500mb - cluster.routing.allocation.disk.watermark.flood_stage: 500mb - discovery.type: "single-node" - http.compression_level: "7" - node.name: clearml - reindex.remote.whitelist: "'*.*'" - xpack.security.enabled: "false" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - image: elasticsearch:8.17.0 - restart: unless-stopped - volumes: - - /opt/clearml/data/elastic_7:/usr/share/elasticsearch/data - - /usr/share/elasticsearch/logs - - fileserver: - networks: - - backend - - frontend - command: - - fileserver - container_name: clearml-fileserver - image: clearml/server:latest - environment: - CLEARML__fileserver__delete__allow_batch: "true" - restart: unless-stopped - volumes: - - /opt/clearml/logs:/var/log/clearml - - /opt/clearml/data/fileserver:/mnt/fileserver - - /opt/clearml/config:/opt/clearml/config - ports: - - "8081:8081" - - mongo: - networks: - - backend - container_name: clearml-mongo - image: mongo:6.0.19 - restart: unless-stopped - command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200 - volumes: - - /opt/clearml/data/mongo_4/db:/data/db - - /opt/clearml/data/mongo_4/configdb:/data/configdb - - redis: - networks: - - backend - container_name: clearml-redis - image: redis:7.4.1 - restart: unless-stopped - volumes: - - /opt/clearml/data/redis:/data - - webserver: - command: - - webserver - container_name: clearml-webserver - # environment: - # CLEARML_SERVER_SUB_PATH : clearml-web # Allow Clearml to be served with a URL path prefix. - image: clearml/server:latest - restart: unless-stopped - depends_on: - - apiserver - ports: - - "8080:80" - networks: - - backend - - frontend - - async_delete: - depends_on: - - apiserver - - redis - - mongo - - elasticsearch - - fileserver - container_name: async_delete - image: clearml/server:latest - networks: - - backend - restart: unless-stopped - environment: - CLEARML_ELASTIC_SERVICE_HOST: elasticsearch - CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_MONGODB_SERVICE_HOST: mongo - CLEARML_MONGODB_SERVICE_PORT: 27017 - CLEARML_REDIS_SERVICE_HOST: redis - CLEARML_REDIS_SERVICE_PORT: 6379 - PYTHONPATH: /opt/clearml/apiserver - CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" - entrypoint: - - python3 - - -m - - jobs.async_urls_delete - - --fileserver-host - - http://fileserver:8081 - volumes: - - /opt/clearml/logs:/var/log/clearml - - /opt/clearml/config:/opt/clearml/config - - agent-services: - networks: - - backend - container_name: clearml-agent-services - image: clearml/clearml-agent-services:latest - deploy: - restart_policy: - condition: on-failure - privileged: true - environment: - CLEARML_HOST_IP: ${CLEARML_HOST_IP} - CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-} - CLEARML_API_HOST: http://apiserver:8008 - CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-} - CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-$CLEARML_API_ACCESS_KEY} - CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-$CLEARML_API_SECRET_KEY} - CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER} - CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS} - CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0} - CLEARML_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04" - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} - AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-} - AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-} - AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-} - GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} - CLEARML_WORKER_ID: "clearml-services" - CLEARML_AGENT_DOCKER_HOST_MOUNT: "/opt/clearml/agent:/root/.clearml" - SHUTDOWN_IF_NO_ACCESS_KEY: 1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /opt/clearml/agent:/root/.clearml - depends_on: - - apiserver - entrypoint: > - bash -c "curl --retry 10 --retry-delay 10 --retry-connrefused 'http://apiserver:8008/debug.ping' && /usr/agent/entrypoint.sh" - -networks: - backend: - driver: bridge - frontend: - driver: bridge From a1c99f3ee31d81f23f89e4a9110c798a2472bc29 Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 19:24:38 +0100 Subject: [PATCH 7/9] Updated README --- clearml/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/clearml/README.md b/clearml/README.md index 328abf4..2b857b7 100644 --- a/clearml/README.md +++ b/clearml/README.md @@ -11,3 +11,8 @@ sudo mv /tmp/99-clearml.conf /etc/sysctl.d/99-clearml.conf sudo sysctl -w vm.max_map_count=524288 sudo service docker restart ``` + +Please continue the setup procedure for: +* [Web Login Authentication](https://clear.ml/docs/latest/docs/deploying_clearml/clearml_server_config#web-login-authentication) +* [Hashing passwords](https://clear.ml/docs/latest/docs/deploying_clearml/clearml_server_config#using-hashed-passwords) +* [Removing Fileserver Artifacts for Deleted Tasks](https://clear.ml/docs/latest/docs/deploying_clearml/clearml_server_config#removing-fileserver-artifacts-for-deleted-tasks) From 90b8731043fcb586aa3d4b00e632e9168dc9f59d Mon Sep 17 00:00:00 2001 From: macmacal Date: Mon, 15 Dec 2025 19:25:05 +0100 Subject: [PATCH 8/9] Added geonosis LAN IP --- clearml/compose.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/clearml/compose.yml b/clearml/compose.yml index ed15358..da4aef1 100644 --- a/clearml/compose.yml +++ b/clearml/compose.yml @@ -35,10 +35,14 @@ services: networks: - backend - frontend + extra_hosts: + - "geonosis:192.168.0.100" elasticsearch: networks: - backend + extra_hosts: + - "geonosis:192.168.0.100" container_name: clearml-elastic environment: bootstrap.memory_lock: "true" @@ -69,6 +73,8 @@ services: networks: - backend - frontend + extra_hosts: + - "geonosis:192.168.0.100" command: - fileserver container_name: clearml-fileserver @@ -83,9 +89,12 @@ services: ports: - "8081:8081" + mongo: networks: - backend + extra_hosts: + - "geonosis:192.168.0.100" container_name: clearml-mongo image: mongo:7.0.22 restart: unless-stopped @@ -97,6 +106,8 @@ services: redis: networks: - backend + extra_hosts: + - "geonosis:192.168.0.100" container_name: clearml-redis image: redis:7.4.1 restart: unless-stopped @@ -118,6 +129,8 @@ services: networks: - backend - frontend + extra_hosts: + - "geonosis:192.168.0.100" async_delete: depends_on: @@ -130,6 +143,8 @@ services: image: clearml/server:latest networks: - backend + extra_hosts: + - "geonosis:192.168.0.100" restart: unless-stopped environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch @@ -153,6 +168,8 @@ services: agent-services: networks: - backend + extra_hosts: + - "geonosis:192.168.0.100" container_name: clearml-agent-services image: clearml/clearml-agent-services:latest deploy: From cef52410a1e4138c4b7d47426a8a9fac215252b4 Mon Sep 17 00:00:00 2001 From: Maciej Aleksandrowicz Date: Tue, 16 Dec 2025 23:32:10 +0100 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Jakub Plachno --- README.md | 4 ++-- containers_registry/README.md | 2 +- ppa_packages/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 516a16a..605c33e 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ docker compose down ### Private Packages Repo (PPA) -How to Set up private repo [here](./ppa_packages/README.md). +You can find instructions for setting up a private repository [here](./ppa_packages/README.md). **Adding private repo** ```bash @@ -37,7 +37,7 @@ echo "deb [trusted=yes] http://192.168.0.100/debian ./" | tee -a /etc/apt/source ``` ### Containers registry -[Instructions how to use self-hosted container registry.](./containers_registry/README.md) +You can find instructions for using the self-hosted container registry [here](./containers_registry/README.md). #### Building & pushing a particular release tag ```bash diff --git a/containers_registry/README.md b/containers_registry/README.md index 6f26c93..70f8b8e 100644 --- a/containers_registry/README.md +++ b/containers_registry/README.md @@ -32,7 +32,7 @@ insecure = true ``` > [!WARNING] -> This configuration is NOT SAFE. Consider setting up proper SSL certificates before running on your machine. +> This configuration is **not safe**. Consider setting up proper SSL certificates before running on your machine. Restart the contenerization engine: ```bash diff --git a/ppa_packages/README.md b/ppa_packages/README.md index dfea973..9d58b83 100644 --- a/ppa_packages/README.md +++ b/ppa_packages/README.md @@ -1,6 +1,6 @@ # Private Packages Repo (PPA) -This module enables storing private *.deb packages used for automated container built. +This module enables storing private `*.deb` packages used for automated container built. More [here](https://linuxconfig.org/easy-way-to-create-a-debian-package-and-local-package-repository).