diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 11e1ea7..cdbf385 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -14,7 +14,7 @@ env: jobs: create_release_publish_to_dhub: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." @@ -93,6 +93,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + with: + platforms: linux/amd64,linux/arm64 # Ensure that the repo has the username and token added from the # official Volttron Dockerhub page at https://hub.docker.com/repository/docker/eclipsevolttron/volttron/general @@ -109,6 +111,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: | ${{ env.IMAGE_BASE_NAME }}:${{ env.VERSION }} diff --git a/.github/workflows/publish_develop_image.yml b/.github/workflows/publish_develop_image.yml index 8dab9d8..bde081b 100644 --- a/.github/workflows/publish_develop_image.yml +++ b/.github/workflows/publish_develop_image.yml @@ -7,11 +7,14 @@ on: branches: - develop workflow_dispatch: - pull_request: + pull_request_target: types: [opened, reopened, synchronize] env: # Image name on Volttron DockerHub account at https://hub.docker.com/orgs/eclipsevolttron - IMAGE_BASE_NAME: eclipsevolttron/volttron + IMAGE_BASE_NAME: ${{vars.IMAGE_BASE_NAME || 'eclipsevolttron/volttron'}} + IMAGE_TAG: ${{ vars.IMAGE_TAG || github.ref == 'refs/heads/develop' && 'develop' || 'test-build'}} + VOLTTRON_GIT_REPO: ${{vars.VOLTTRON_GIT_REPO || 'https://github.com/VOLTTRON/volttron.git'}} + VOLTTRON_GIT_BRANCH: ${{vars.VOLTTRON_GIT_BRANCH || 'develop'}} jobs: publish_develop_to_dhub: @@ -26,33 +29,80 @@ jobs: - name: Check out the repo uses: actions/checkout@v4 - - name: Clone submodule - run: git submodule update --init --recursive - - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + with: + platforms: linux/amd64,linux/arm64 - # Ensure that the repo has the username and token added from the - # official Volttron Dockerhub page at https://hub.docker.com/repository/docker/eclipsevolttron/volttron/general - # On instructions how to do this, see https://docs.github.com/en/actions/guides/publishing-docker-images - # https://github.com/docker/login-action - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_API_TOKEN }} + run: | + if [ -n "${{ vars.DOCKER_USER }}" -a -n "${{ secrets.DOCKER_API_TOKEN }}" ]; then + echo " " + echo "Connecting to docker" + + echo "${{ secrets.DOCKER_API_TOKEN }}" | docker login -u "${{ vars.DOCKER_USER }}" --password-stdin + status=$? + if [ $status -ne 0 ]; then + echo "Error: status $status" + exit 1 + fi + fi - name: Build and push id: docker_build uses: docker/build-push-action@v2 with: context: . + file: ${{ env.DOCKERFILE || './Dockerfile' }} + platforms: linux/amd64,linux/arm64 + build-args: | + volttron_git_branch=${{ env.VOLTTRON_GIT_BRANCH }} + volttron_repo=${{ env.VOLTTRON_GIT_REPO }} push: true tags: | - ${{ env.IMAGE_BASE_NAME }}:develop + ${{ env.IMAGE_BASE_NAME }}:${{ env.IMAGE_TAG }} + +# - name: Build Image +# id: docker_build +# run: | +# echo "Building image" +# echo "docker buildx build \ +# --build-arg volttron_git_branch=${{ env.VOLTTRON_GIT_BRANCH }} \ +# --build-arg volttron_repo=${{ env.VOLTTRON_GIT_REPO }} \ +# --platform linux/amd64,linux/arm64 \ +# -t ${{ env.IMAGE_BASE_NAME }}:${{ env.IMAGE_TAG }} \ +# -f ${{ env.DOCKERFILE || './Dockerfile' }} --no-cache --force-rm --load ." +# docker buildx build \ +# --build-arg volttron_git_branch=${{ env.VOLTTRON_GIT_BRANCH }} \ +# --build-arg volttron_repo=${{ env.VOLTTRON_GIT_REPO }} \ +# --platform linux/amd64,linux/arm64 \ +# -t ${{ env.IMAGE_BASE_NAME }}:${{ env.IMAGE_TAG }} \ +# -f ${{ env.DOCKERFILE || './Dockerfile' }} --load . +# status=$? +# if [ $status -ne 0 ]; then +# echo "Error: status $status" +# exit 1 +# fi +# +# - name: Push Image +# id: docker_push +# run: | +# echo "Pushing image" +# # docker images +# echo "docker push ${{ env.IMAGE_BASE_NAME }}:${{ env.IMAGE_TAG }}" +# echo "digest=$(docker push ${{ env.IMAGE_BASE_NAME }}:${{ env.IMAGE_TAG }} | grep '${{env.IMAGE_TAG}}: digest:')" >> $GITHUB_OUTPUT +# status=$? +# if [ $status -ne 0 ]; then +# echo "Error: status $status" +# exit 1 +# fi + +# - name: Show publish digest +# run: | +# echo "docker_push.outputs.digest: ${{ steps.docker_push.outputs.digest }}" - name: Check image publish if: steps.docker_build.outputs.digest == '' @@ -61,14 +111,4 @@ jobs: - name: Image Digest run: echo ${{ steps.docker_build.outputs.digest }} - # https://github.com/marketplace/actions/docker-hub-description - # Currently doesn't work with personal access tokens; see https://github.com/peter-evans/dockerhub-description/issues/10 - # when fixed, uncomment this step -# - name: Update DockerHub repo README -# uses: peter-evans/dockerhub-description@v2 -# with: -# username: ${{ secrets.DOCKER_USER }} -# password: ${{ secrets.DOCKER_PASSWORD }} -# readme-filepath: ./README_DHUB.md - - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.gitmodules b/.gitmodules index b8dc76a..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "volttron"] - path = volttron - url = https://github.com/volttron/volttron - branch = main diff --git a/Dockerfile b/Dockerfile index 03e0fee..d81bae1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ ARG image_repo=rust ARG image_tag=bookworm -# + FROM ${image_repo}:${image_tag} AS volttron_base + +ARG volttron_repo=https://github.com/VOLTTRON/volttron.git +ARG volttron_git_branch=main # SHELL [ "bash", "-c" ] # ENV OS_TYPE=debian ENV DIST=bookworm -ENV VOLTTRON_GIT_BRANCH=main +ENV VOLTTRON_GIT_BRANCH=${volttron_git_branch} ENV VOLTTRON_USER_HOME=/home/volttron ENV VOLTTRON_HOME=${VOLTTRON_USER_HOME}/.volttron ENV CODE_ROOT=/code @@ -15,12 +18,12 @@ ENV VOLTTRON_ROOT=${CODE_ROOT}/volttron ENV VOLTTRON_USER=volttron ENV USER_PIP_BIN=${VOLTTRON_USER_HOME}/.local/bin ENV CONFIG=/home/volttron/configs +ENV TZ=UTC ENV LOCAL_USER_ID=1000 #ENV RMQ_ROOT=${VOLTTRON_USER_HOME}/rabbitmq_server #ENV RMQ_HOME=${RMQ_ROOT}/rabbitmq_server-3.7.7 # USER root -# RUN set -eux; apt-get update; apt-get install -y --no-install-recommends \ procps \ gosu \ @@ -45,23 +48,24 @@ RUN set -eux; apt-get update; apt-get install -y --no-install-recommends \ libffi-dev \ sqlite3 # -# Set timezone -RUN echo UTC > /etc/timezone - # Set default 'python' to 'python3' RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + # # Set global.break-system-packages to true to allow pip to install packages # RUN python3 -m pip config set global.break-system-packages true + # # Upgrade pip so that we get a pre-compiled wheel for 'cryptopgraphy', which is a dependency of Volttron # See https://cryptography.io/en/latest/faq/#installing-cryptography-fails-with-error-can-not-find-rust-compiler RUN pip install --upgrade pip + # # Install rust and cargo # RUN rustup default stable + # # Create a user called 'volttron' RUN id -u $VOLTTRON_USER &>/dev/null || adduser --disabled-password --gecos "" $VOLTTRON_USER @@ -74,17 +78,23 @@ RUN mkdir -p /code && chown $VOLTTRON_USER.$VOLTTRON_USER /code && \ # Creating volttron_core stage ############################################ FROM volttron_base AS volttron_core -# + +ARG volttron_repo=https://github.com/VOLTTRON/volttron.git +ARG volttron_git_branch=main + # copy over /core, i.e. the custom startup scripts for this image RUN mkdir /startup $VOLTTRON_HOME && \ chown $VOLTTRON_USER.$VOLTTRON_USER $VOLTTRON_HOME COPY --chown=volttron:volttron ./core /startup RUN chmod +x /startup/* + # # copy over volttron repo USER $VOLTTRON_USER -COPY --chown=volttron:volttron volttron /code/volttron -WORKDIR /code/volttron + +RUN git clone --branch ${volttron_git_branch} --single-branch ${volttron_repo} ${VOLTTRON_ROOT} +WORKDIR ${VOLTTRON_ROOT} + # # Set global.break-system-packages to true to allow pip to install packages # @@ -94,15 +104,13 @@ RUN python3 -m pip config set global.break-system-packages true # See https://cryptography.io/en/latest/faq/#installing-cryptography-fails-with-error-can-not-find-rust-compiler RUN pip install --upgrade pip +# +# Now install volttron required packages +# RUN pip install -e . --user RUN echo "package installed at `date`" -# copy default configs -#COPY --chown=volttron:volttron ./platform_config.yml /platform_config.yml -#COPY --chown=volttron:volttron ./configs /home/volttron/configs - - -## +# ############################################# ## RABBITMQ SPECIFIC INSTALLATION ############################################# diff --git a/Dockerfile-dev b/Dockerfile-dev index c06231c..d2f67d6 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,19 +1,25 @@ ARG image_repo=rust ARG image_tag=bookworm -# + FROM ${image_repo}:${image_tag} AS volttron_base + +ARG volttron_repo=https://github.com/VOLTTRON/volttron.git +ARG volttron_git_branch=develop # SHELL [ "bash", "-c" ] # ENV OS_TYPE=debian ENV DIST=bookworm -ENV VOLTTRON_GIT_BRANCH=develop +ENV VOLTTRON_GIT_BRANCH=${volttron_git_branch} ENV VOLTTRON_USER_HOME=/home/volttron ENV VOLTTRON_HOME=${VOLTTRON_USER_HOME}/.volttron ENV CODE_ROOT=/code ENV VOLTTRON_ROOT=${CODE_ROOT}/volttron ENV VOLTTRON_USER=volttron ENV USER_PIP_BIN=${VOLTTRON_USER_HOME}/.local/bin +ENV CONFIG=/home/volttron/configs +ENV TZ=UTC +ENV LOCAL_USER_ID=1000 #ENV RMQ_ROOT=${VOLTTRON_USER_HOME}/rabbitmq_server #ENV RMQ_HOME=${RMQ_ROOT}/rabbitmq_server-3.7.7 # @@ -72,15 +78,22 @@ RUN mkdir -p /code && chown $VOLTTRON_USER.$VOLTTRON_USER /code && \ # Creating volttron_core stage ############################################ FROM volttron_base AS volttron_core -# -# make the volttron home directory -RUN mkdir $VOLTTRON_HOME && \ + +ARG volttron_repo=https://github.com/VOLTTRON/volttron.git +ARG volttron_git_branch=develop + +# copy over /core, i.e. the custom startup scripts for this image +RUN mkdir /startup $VOLTTRON_HOME && \ chown $VOLTTRON_USER.$VOLTTRON_USER $VOLTTRON_HOME +COPY --chown=volttron:volttron ./core /startup +RUN chmod +x /startup/* + # # copy over volttron repo USER $VOLTTRON_USER -COPY --chown=volttron:volttron volttron /code/volttron -WORKDIR /code/volttron + +RUN git clone --branch ${volttron_git_branch} --single-branch ${volttron_repo} ${VOLTTRON_ROOT} +WORKDIR ${VOLTTRON_ROOT} # # Set global.break-system-packages to true to allow pip to install packages @@ -97,11 +110,6 @@ RUN pip install --upgrade pip RUN pip install -e . --user RUN echo "package installed at `date`" -# -# Copy the startup files instead of relying on a volume due to needing to use docker build to -# create the image rather than docker-compose build -# -COPY --chown=volttron:volttron core /startup # ############################################# ## RABBITMQ SPECIFIC INSTALLATION diff --git a/README.md b/README.md index 710776a..f1b315c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ In conjunction with volume mounting of the directory, this ensures that file own # Prerequisites * Docker ^20.10.8 -* Docker-compose ^1.29.2 +* Docker compose ^1.29.2 -If you need to install docker and/or docker-compose AND you are running this image on an Ubuntu machine, you can use the script in this repo. From the root level, execute the following command: +If you need to install docker and/or docker compose AND you are running this image on an Ubuntu machine, you can use the script in this repo. From the root level, execute the following command: ```bash $ ./docker_install_ubuntu.sh @@ -26,29 +26,29 @@ To create the container and start using the platform on the container, run the f ``` # Build the image locally. Set to some tag. Then update the -docker-compose script with the updated image name that uses the tag as part of +docker compose script with the updated image name that uses the tag as part of its name. # Example below -$ docker build -t eclipsevolttron/volttron: --build-arg install_rmq=false --no-cache . +$ docker buildx build -t eclipsevolttron/volttron: --build-arg install_rmq=false --no-cache . # Create and start the container that has runs Volttron -$ docker-compose up +$ docker compose up # SSH into the container as the user 'volttron' $ docker exec -itu volttron volttron1 bash # Stop the container -$ docker-compose stop +$ docker compose stop # Start the container -$ docker-compose start +$ docker compose start -# To get a list of all containers created from docker-compose -$ docker-compose ps +# To get a list of all containers created from docker compose +$ docker compose ps # To stop and remove the container -$ docker-compose down +$ docker compose down ``` For Volttron instances using ZMQ message bus: @@ -139,9 +139,9 @@ Agents within the `platform_config.yml` file are created sequentially, it can ta # Development If you plan on extending or developing `platform_config.yml`, `configs/`, or the setup scripts in `core/`, build the -Docker image, "Dockerfile-dev", only once using `docker-compose -f docker-compose-dev.yml build --no-cache volttron1`. +Docker image, "Dockerfile-dev", only once using `docker compose -f docker-compose-dev.yml build --no-cache volttron1`. -Then start the container using `docker-compose -f docker-compose-dev.yml up`. When you want to make changes to "platform_config.yml", "configs/", or +Then start the container using `docker compose -f docker-compose-dev.yml up`. When you want to make changes to "platform_config.yml", "configs/", or "core/", simply make the changes and then rerun your container. You do not have to rebuild the image every time you make changes to those aforementioned files and folders because they are mounted into the container. The only time you should rebuild the image is when you make changes to the "volttron" source code since that is not mounted to the container but rather baked into the image during @@ -154,36 +154,31 @@ To set up your environment for development, do the following: chmod a+x core/* ``` -1. Pull in volttron from the [official volttron repo](https://github.com/VOLTTRON/volttron) using the following git command: +1. Build the image locally: +* Using docker buildx (preferred) ```bash -# Clones https://github.com/VOLTTRON/volttron.git into the 'volttron' directory -git submodule update --init --recursive +docker buildx build -f Dockerfile-dev --no-cache --force-rm -t user/inagename:tag . ``` -Why are we doing this? This repo has a directory called 'volttron', which contains the volttron codebase. In other words, this repo contains another repo in a subfolder. -When you initially clone this repo, the 'volttron' directory is empty. This directory contains the volttron codebase used to create the volttron platform. +* Dockerfile will clone the volttron repository in the image from https://github.com/VOLTTRON/volttron.git + and defaults to the main branch. Dockerfile-dev defaults to the develop branch. Other than that, they are + identical. If you would like to use a different repository, or branch, you can set the following build + arguments (specified with the --build-arg flag): + - volttron_repo: The repository to clone. Use the https URL for the repo. + - volttron_git_branch: The branch or tag to clone. Defaults to main. -OPTIONAL: This repo uses a specific version of volttron based on the commit in the 'volttron' submodule. If you want to use the latest volttron from the `develop` -branch from the volttron repo, execute the following command (NOTE: this is not required): - -```bash -# Ensure that you are in the `volttron` folder -git pull origin develop -``` - -2. Build the image locally: - -* Using docker-compose (preferred) ```bash -docker-compose -f docker-compose-dev.yml build --no-cache --force-rm +docker buildx build -f Dockerfile-dev --no-cache --force-rm \ + --build-arg volttron_repo=https://github.com/myuser/volttron-docker-fork.git \ + --build-arg volttron_git_branch=feature123 -t user/inagename:tag . ``` -3. Run the container: +2. Run the container: -* Using docker-compose (preferred) +* Using docker compose (preferred) ``` -docker-compose -f docker-compose-dev.yml up +docker compose -f docker-compose-dev.yml up ``` ## Testing diff --git a/README_DHUB.md b/README_DHUB.md index 7d33ecb..2122514 100644 --- a/README_DHUB.md +++ b/README_DHUB.md @@ -83,7 +83,7 @@ eclipsevolttron/volttron:v3.0 If you don't want to type a long `docker run` command every time you run a container, you can wrap your command in a docker-compose script. See this example from (docker-compose.yml)[https://github.com/VOLTTRON/volttron-docker/blob/main/docker-compose.yml]. Note that you still need to ensure that the paths to the bind-mounts are properly constructed. After you create your docker-compose script, -run `docker-compose run` in the same directory that holds your script. +run `docker compose run` in the same directory that holds your script. ```shell version: '3.4' diff --git a/core/setup-platform.py b/core/setup-platform.py index c1613c0..fc74172 100755 --- a/core/setup-platform.py +++ b/core/setup-platform.py @@ -1,8 +1,10 @@ +import json import subprocess import os import sys import yaml + from shutil import copy from time import sleep from volttron.platform import set_home @@ -50,10 +52,11 @@ def get_platform_configurations(platform_config_path): config = yaml.safe_load(cin) agents = config["agents"] platform_cfg = config["config"] + web_users_params = config.get("web_users") print("Platform instance name set to: {}".format(platform_cfg.get("instance-name"))) - return config, agents, platform_cfg + return config, agents, platform_cfg, web_users_params def _install_required_deps(): @@ -362,11 +365,32 @@ def final_platform_configurations(): sys.exit(0) +def create_web_map(username, password, groups=None, return_json=False): + from passlib.hash import argon2 + + groups = ( + groups if isinstance(groups, list) else [groups] if groups else ["admin", "vui"] + ) + web_users_dict = { + username: {"hashed_password": argon2.hash(password), "groups": groups} + } + return web_users_dict if not return_json else json.dumps(web_users_dict) + + +def save_web_users_json_file(username, password, groups=None): + from pathlib import Path + + with open(Path(VOLTTRON_HOME) / "web-users.json", "w") as f: + json.dump(create_web_map(username, password, groups), f) + + if __name__ == "__main__": set_home(VOLTTRON_HOME) - config_tmp, agents_tmp, platform_cfg_tmp = get_platform_configurations( + config_tmp, agents_tmp, platform_cfg_tmp, web_users = get_platform_configurations( get_platform_config_path() ) configure_platform(platform_cfg_tmp, config_tmp) install_agents(agents_tmp) + if web_users: + save_web_users_json_file(**web_users) final_platform_configurations() diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ef12cd2..3c5773d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -8,7 +8,7 @@ services: dockerfile: Dockerfile-dev args: install_rmq: 'false' - image: eclipsevolttron/volttron:v-docker-dev-latest + image: eclipsevolttron/volttron:develop ports: # host_port:container_port # http port for volttron central diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 57b8733..31ef0d3 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -16,6 +16,7 @@ services: environment: - CONFIG=/home/volttron/configs - LOCAL_USER_ID=1000 + - TZ=America/Los_Angeles volumes: volttron1-volume: diff --git a/docker-compose.yml b/docker-compose.yml index 5893f76..15b3ee3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: volttron1: container_name: volttron1 hostname: volttron1 - image: volttroncommunity/volttron:v9.0-zmq + image: eclipsevolttron/volttron:latest ports: # host_port:container_port # http port for volttron central diff --git a/run-test-docker-image.sh b/run-test-docker-image.sh index 506d241..023ead2 100755 --- a/run-test-docker-image.sh +++ b/run-test-docker-image.sh @@ -14,7 +14,7 @@ exit_cleanly() { exit_test() { echo -e "$1" docker logs --tail 25 volttron1 - docker-compose down + docker compose down exit_cleanly } @@ -62,7 +62,7 @@ attempts=5 echo "Will try at most ${attempts} attempts to start container..." while [ "${attempts}" -gt 0 ]; do echo "Attempt number ${attempts} to start container." - docker-compose --file docker-compose-test.yml up --detach + docker compose --file docker-compose-test.yml up --detach echo "Configuring and starting Volttron platform; this will take approximately several minutes........" sleep ${wait} docker ps --filter "name=volttron1" --filter "status=running" | grep 'volttron1' @@ -70,7 +70,7 @@ while [ "${attempts}" -gt 0 ]; do if [ "${tmp_code}" -eq 1 ]; then echo "Container failed to start." docker logs --tail 20 volttron1 - docker-compose down + docker compose down ((attempts=attempts-1)) else # Container was successfully created diff --git a/volttron b/volttron deleted file mode 160000 index a537ca0..0000000 --- a/volttron +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a537ca0d19fba2e6466d9d97698992d2fd664d60