From 34e22c8f2c6b3b249aafada3bfc783d7f08afa83 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Tue, 17 Jun 2025 12:55:49 +0200 Subject: [PATCH 01/27] Fixed: Vision weights were placed at incorrect path upon re-download. --- .../agents/base_vision_agent.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py index c143b4d13..62efc43cf 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py @@ -34,6 +34,7 @@ def __init__( super().__init__() self._weights_path = Path(weights_path) os.makedirs(self._weights_path, exist_ok=True) + self._is_weights_path_set = False self._init_weight_path() self.weight_path = self._weights_path self.ros2_connector = ROS2Connector(ros2_name, executor_type="single_threaded") @@ -43,9 +44,16 @@ def _init_weight_path(self): if self.WEIGHTS_FILENAME == "": raise ValueError("WEIGHTS_FILENAME is not set") - install_path = ( - self._weights_path / "vision" / "weights" / self.WEIGHTS_FILENAME - ) + # Ensure that the self._weights_path variable is set only once + # to prevent issues during weight re-downloading + if not self._is_weights_path_set: + install_path = ( + self._weights_path / "vision" / "weights" / self.WEIGHTS_FILENAME + ) + else: + install_path = self._weights_path + self._is_weights_path_set = True + # make sure the file exists if install_path.exists() and install_path.is_file(): self._weights_path = install_path From 6aa277a7b7347391c4fdc0bde1f047ef2a85f1d3 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Wed, 18 Jun 2025 12:01:07 +0200 Subject: [PATCH 02/27] Added cuda availability check to GDBoxer --- .../rai_open_set_vision/vision_markup/boxer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py index 949d18a2c..6bcb11008 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py @@ -17,6 +17,7 @@ from typing import Dict import cv2 +import torch from cv_bridge import CvBridge from groundingdino.util.inference import Model from rclpy.time import Time @@ -71,7 +72,8 @@ def __init__( if not use_cuda: self.model = Model(self.cfg_path, self.weight_path, device="cpu") else: - self.model = Model(self.cfg_path, self.weight_path) + device = "cuda" if torch.cuda.is_available() else "cpu" + self.model = Model(self.cfg_path, self.weight_path, device=device) self.bridge = CvBridge() def get_boxes( From b77b5dd2a9882ccd0af5499120cc755c6095c9fd Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Fri, 20 Jun 2025 11:22:48 +0200 Subject: [PATCH 03/27] WIP dockerfile --- docker/Dockerfile | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 494092677..b7cb44534 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,16 +13,24 @@ # limitations under the License. ARG ROS_DISTRO=jazzy - FROM osrf/ros:${ROS_DISTRO}-desktop-full + +ARG DEPENDENCIES=core_only ENV DEBIAN_FRONTEND=noninteractive +# Check whether the $DEPENDENCIES ARG has a valid value +RUN /bin/bash -c 'if [ "${DEPENDENCIES}" != "core_only" ] && [ "${DEPENDENCIES}" != "all_groups" ]; then \ + echo Error: invalid DEPENDENCIES value. Valid values "core_only", "all_groups" && exit 1; \ +fi' + + # Install dependencies RUN apt-get update && apt-get install -y \ python3 \ python3-pip \ git \ - wget + wget \ + zip # Install Poetry RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.4 @@ -32,8 +40,19 @@ ENV PATH="/root/.local/bin:$PATH" WORKDIR /rai RUN git clone https://github.com/RobotecAI/rai.git . -# Install Python dependencies with Poetry -RUN poetry install --with nomad,openset +# Temporary change to use branch with bug fixes +RUN git pull +RUN git switch dm/feat/docker-image-manipulator-demo + +# # Install Python dependencies with Poetry +# RUN poetry install --with nomad,openset + + +RUN /bin/bash -c 'if [ "${DEPENDENCIES}" = "core_only" ]; then \ + poetry install; \ +else \ + poetry install --all-groups; \ +fi' # Install ROS dependencies RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && \ From 768a7b603d0674b1db5f709164db1dde78d791d1 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 23 Jun 2025 09:59:58 +0200 Subject: [PATCH 04/27] Increased timeout for GDino and GSAM from 5.0s to 60.0s --- .../rai_open_set_vision/tools/segmentation_tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py index 16c6fc2df..e8fa31245 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py @@ -345,13 +345,15 @@ def _run( conversion_ratio = 0.001 resolved = None - resolved = get_future_result(future) + # NOTE: Image processing by GroundingDino and Grounded SAM may take a significant amount + # of time, especially when performed on the CPU. Hence, timeout is set to 60 seconds + resolved = get_future_result(future, timeout_sec=60.0) assert resolved is not None future = self._call_gsam_node(camera_img_msg, resolved) ret = [] - resolved = get_future_result(future) + resolved = get_future_result(future, timeout_sec=60.0) if resolved is not None: for img_msg in resolved.masks: ret.append(convert_ros_img_to_base64(img_msg)) From 7d0d2aeaa2933c9bce96bd6fb4d3c03d0a5486af Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 23 Jun 2025 10:08:55 +0200 Subject: [PATCH 05/27] Added fastrtps config for communication between the docker container and the host --- docker/Dockerfile | 4 ++++ fastrtps_config.xml | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 fastrtps_config.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index b7cb44534..0b8988e04 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,6 +40,10 @@ ENV PATH="/root/.local/bin:$PATH" WORKDIR /rai RUN git clone https://github.com/RobotecAI/rai.git . +# Set up the environment to use Fast DDS +ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtpsa_config.xml +ENV RMW_IMPLEMENTATION=rmw_fastrtps_cpp + # Temporary change to use branch with bug fixes RUN git pull RUN git switch dm/feat/docker-image-manipulator-demo diff --git a/fastrtps_config.xml b/fastrtps_config.xml new file mode 100644 index 000000000..288776e1b --- /dev/null +++ b/fastrtps_config.xml @@ -0,0 +1,17 @@ + + + + + CustomUdpTransport + UDPv4 + + + + + + CustomUdpTransport + + false + + + From a6c327706617dd230f69defc95e99df6bd5a6ca9 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 23 Jun 2025 14:29:28 +0200 Subject: [PATCH 06/27] Replaced apt-get with apt, updated poetry to version 2.1.3 --- docker/Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0b8988e04..1327a331e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,7 +25,7 @@ fi' # Install dependencies -RUN apt-get update && apt-get install -y \ +RUN apt update && apt install -y \ python3 \ python3-pip \ git \ @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ zip # Install Poetry -RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.4 +RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3 ENV PATH="/root/.local/bin:$PATH" # Clone and setup RAI @@ -41,17 +41,13 @@ WORKDIR /rai RUN git clone https://github.com/RobotecAI/rai.git . # Set up the environment to use Fast DDS -ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtpsa_config.xml +ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtps_config.xml ENV RMW_IMPLEMENTATION=rmw_fastrtps_cpp # Temporary change to use branch with bug fixes RUN git pull RUN git switch dm/feat/docker-image-manipulator-demo -# # Install Python dependencies with Poetry -# RUN poetry install --with nomad,openset - - RUN /bin/bash -c 'if [ "${DEPENDENCIES}" = "core_only" ]; then \ poetry install; \ else \ From b8aec4355c4e2f87c2225aa86dc3055e8faa193e Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Tue, 24 Jun 2025 08:39:11 +0200 Subject: [PATCH 07/27] Update docs for docker setup and the robotic arm demo --- docs/demos/manipulation.md | 66 ++++++++++++++++++++++++++++++++++++- docs/setup/setup_docker.md | 67 ++++++++++++++++++++++++++++++-------- 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 02ecd6a19..de593ec18 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -18,6 +18,10 @@ manipulation techniques. Make sure ROS 2 is sourced. (e.g. `source /opt/ros/humble/setup.bash`) +### Local Setup + +#### Setting up the demo + 1. Follow the RAI setup instructions in the [quick setup guide](../setup/install.md#setting-up-developer-environment). 2. Download additional dependencies: @@ -39,7 +43,7 @@ manipulation techniques. colcon build --symlink-install ``` -## Running the Demo +#### Running the demo !!! note "Remain in sourced shell" @@ -75,6 +79,66 @@ manipulation techniques. To change camera in the simulation use 1-7 keys on your keyboard once it's window is focused. +### Docker Setup + +!!! note "ROS 2 required" + + The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine. + +#### 1. Setting up the demo + +1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) + and configure communication between the container and the host ([link](<../setup/setup_docker.md#3.-set-up-communications-between-docker-and-host-(optional)>)). + +2. On the host machine, download the latest binary release for the Robotic Arm Demo: + + ```shell + ./scripts/download_demo.sh manipulation + ``` + +3. (Inside the container shell) Download additional ROS 2 dependencies: + + ```shell + vcs import < demos.repos + rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y + ``` + +4. (Inside the container shell) Build the ROS 2 workspace: + + ```shell + colcon build --symlink-install + ``` + +#### 2. Running the demo + +!!! note Source the setup shell + + Ensure ROS 2 is sourced on the host machine. Ensure that every command inside the docker container is run in a sourced shell using `source setup_shell.sh`. + +1. Launch the Robotic Arm Visualization on the host machine: + + ```shell + ./demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher + ``` + +2. (Inside the container shell) Launch the Robotic Arm Demo script inside of the docker container: + + ```shell + ros2 launch examples/manipulation-demo.launch.py + ``` + +3. (Inside the container shell) Open a new terminal for the docker container (e.g., `docker exec -it CONTAINER_ID /bin/bash`) and launch the streamlit interface: + + ```shell + streamlit run examples/manipulation-demo-streamlit.py + ``` + + Alternatively, run the simpler command-line version: + + ```shell + python examples/manipulation-demo.py + ``` + ## How it works The manipulation demo utilizes several components: diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index ef7b0ebef..8eea4766a 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -7,45 +7,86 @@ ## 1. Build the docker image -Choose the docker image based on your preferred ROS 2 version. +Choose the docker image based on your preferred ROS 2 version. You may build the selected image with only the core dependencies or, alternatively, with all the additional modules. ### 1.1. Humble +Core dependencies only: + +```bash +docker build -t rai:humble --build-arg ROS_DISTRO=humble --build-arg -f docker/Dockerfile . +``` + +All dependencies: + ```bash -docker build -t rai:humble --build-arg ROS_DISTRO=humble -f docker/Dockerfile . +docker build -t rai:humble --build-arg ROS_DISTRO=humble --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . ``` ### 1.2. Jazzy +Core dependencies only: + +```bash +docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . +``` + +All dependencies: + ```bash -docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy -f docker/Dockerfile . +docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . ``` -## 2. Run the docker container +## 2. Set up communications between docker and host (Optional) !!! tip "ROS 2 communication" If you intend to run demos on the host machine, ensure the docker container can communicate with it. Test this by running the standard ROS 2 example with one node in docker and one on the host: - [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). If - topics are not visible or cannot be subscribed to, try using - [rmw_cyclone_dds](https://github.com/ros2/rmw_cyclonedds) instead of the default - rmw_fastrtps_cpp. + [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). + +To allow the container to communicate with the host machine, configure the host environment as presented below: + +1. If not configured, set the `ROS_DOMAIN_ID` environment variable to a domain ID between 0 and 101, inclusive. Example: + + ```shell + export ROS_DOMAIN_ID=99 + ``` + +2. Install the eProsima Fast DDS middleware (should come preinstalled with ROS 2): + + ```shell + apt install ros-"${ROS_DISTRO}"-fastrtps + ``` + +3. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: + + ```shell + export FASTRTPS_DEFAULT_PROFILES_FILE=/path/to/fastrtps_config.xml + ``` + +4. Set the RMW to use eProsima Fast DDS: + + ```shell + export RMW_IMPLEMENTATION=rmw_fastrtps_cpp + ``` + +## 3. Run the docker container -### 2.1. Humble +### 3.1. Humble ```bash -docker run --net=host --ipc=host --pid=host -it rai:humble +docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:humble ``` -### 2.2. Jazzy +### 3.2. Jazzy ```bash -docker run --net=host --ipc=host --pid=host -it rai:jazzy +docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:jazzy ``` -## 3. Run the tests to confirm the setup +## 4. Run the tests to confirm the setup ```sh cd /rai From 6e4b18c921d8170ae59bb8ab9759598edb818428 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Tue, 24 Jun 2025 08:50:51 +0200 Subject: [PATCH 08/27] Removed branch switching from the dockerfile --- docker/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1327a331e..59fdae73e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,10 +44,6 @@ RUN git clone https://github.com/RobotecAI/rai.git . ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtps_config.xml ENV RMW_IMPLEMENTATION=rmw_fastrtps_cpp -# Temporary change to use branch with bug fixes -RUN git pull -RUN git switch dm/feat/docker-image-manipulator-demo - RUN /bin/bash -c 'if [ "${DEPENDENCIES}" = "core_only" ]; then \ poetry install; \ else \ From 702d19512a5a6c8298d13e3b8b43f2638c4817ec Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski <63610203+Cytrus14@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:29:05 +0200 Subject: [PATCH 09/27] Update docs/setup/setup_docker.md Co-authored-by: Bartek Boczek --- docs/setup/setup_docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 8eea4766a..79ca5ac0f 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -14,7 +14,7 @@ Choose the docker image based on your preferred ROS 2 version. You may build the Core dependencies only: ```bash -docker build -t rai:humble --build-arg ROS_DISTRO=humble --build-arg -f docker/Dockerfile . +docker build -t rai:humble --build-arg ROS_DISTRO=humble -f docker/Dockerfile . ``` All dependencies: From 4e726285e24a81a565e7269a135a0ca6d4040cfe Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Wed, 25 Jun 2025 16:35:29 +0200 Subject: [PATCH 10/27] fixed broken link in docs --- docs/demos/manipulation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index de593ec18..ab2aff3ad 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -88,7 +88,7 @@ manipulation techniques. #### 1. Setting up the demo 1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) - and configure communication between the container and the host ([link](<../setup/setup_docker.md#3.-set-up-communications-between-docker-and-host-(optional)>)). + and configure communication between the container and the host ([link](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional)). 2. On the host machine, download the latest binary release for the Robotic Arm Demo: From e2f6f9f6fee8f93741e07303f888254b284d6b6e Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 07:12:38 +0200 Subject: [PATCH 11/27] Specified that ROS 2 must be source during the setup of communication between host and docker --- docs/setup/setup_docker.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 79ca5ac0f..b7bb3f131 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -48,25 +48,31 @@ docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES= To allow the container to communicate with the host machine, configure the host environment as presented below: -1. If not configured, set the `ROS_DOMAIN_ID` environment variable to a domain ID between 0 and 101, inclusive. Example: +1. Source ROS 2 on the host machine: + + ```shell + source /opt/ros/jazzy/setup.bash # or humble + ``` + +2. If not configured, set the `ROS_DOMAIN_ID` environment variable to a domain ID between 0 and 101, inclusive. Example: ```shell export ROS_DOMAIN_ID=99 ``` -2. Install the eProsima Fast DDS middleware (should come preinstalled with ROS 2): +3. Install the eProsima Fast DDS middleware (should come preinstalled with ROS 2): ```shell apt install ros-"${ROS_DISTRO}"-fastrtps ``` -3. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: +4. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: ```shell export FASTRTPS_DEFAULT_PROFILES_FILE=/path/to/fastrtps_config.xml ``` -4. Set the RMW to use eProsima Fast DDS: +5. Set the RMW to use eProsima Fast DDS: ```shell export RMW_IMPLEMENTATION=rmw_fastrtps_cpp From 14d5f4b774e140c30110829e4594c1d1381371b1 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 07:17:02 +0200 Subject: [PATCH 12/27] Clarified that RAI git repository must be cloned to build the docker images --- docs/setup/setup_docker.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index b7bb3f131..0e7dc7433 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -7,7 +7,12 @@ ## 1. Build the docker image -Choose the docker image based on your preferred ROS 2 version. You may build the selected image with only the core dependencies or, alternatively, with all the additional modules. +Choose the docker image based on your preferred ROS 2 version. You may build the selected image with only the core dependencies or, alternatively, with all the additional modules. To build the docker image, you must clone the RAI repository: + +```bash +git clone https://github.com/RobotecAI/rai.git +cd rai +``` ### 1.1. Humble From 7c32345414549c196a316106a18527af080e90ca Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 07:18:50 +0200 Subject: [PATCH 13/27] Move fastrtps_config.xml to the 'docker' directory --- fastrtps_config.xml => docker/fastrtps_config.xml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fastrtps_config.xml => docker/fastrtps_config.xml (100%) diff --git a/fastrtps_config.xml b/docker/fastrtps_config.xml similarity index 100% rename from fastrtps_config.xml rename to docker/fastrtps_config.xml From 208d4becce1d503cf12b8597dd3bfb1d1f49f43a Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 07:42:13 +0200 Subject: [PATCH 14/27] RAI repository is now copied from the host instead of being cloned from git, added a .dockerignore file --- .dockerignore | 182 ++++++++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile | 4 +- 2 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..700b742e8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,182 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# ROS +build/ +devel/ +install/ +log/ + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +dist/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +logs/ +.vscode/ + +!examples/imgs/*.md + +src/examples/*-demo +artifact_database.pkl + +imgui.ini + +src/rai_bench/rai_bench/experiments +src/rai_interfaces + +# Demo assets +demo_assets/ diff --git a/docker/Dockerfile b/docker/Dockerfile index 59fdae73e..fee164239 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,9 +36,9 @@ RUN apt update && apt install -y \ RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3 ENV PATH="/root/.local/bin:$PATH" -# Clone and setup RAI +# Copy the RAI repository +COPY . /rai WORKDIR /rai -RUN git clone https://github.com/RobotecAI/rai.git . # Set up the environment to use Fast DDS ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtps_config.xml From eb5741910dcddfb25019a9c6f3237c01d46ac471 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 08:17:45 +0200 Subject: [PATCH 15/27] Specified that a GPU can be used within the docker container for faster inference --- docs/demos/manipulation.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index ab2aff3ad..4823d46e5 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -87,23 +87,37 @@ manipulation techniques. #### 1. Setting up the demo -1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) - and configure communication between the container and the host ([link](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional)). +1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) + and configure communication between the container and the host ([link](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional)). -2. On the host machine, download the latest binary release for the Robotic Arm Demo: +2. On the host machine, download the latest binary release for the Robotic Arm Demo: ```shell ./scripts/download_demo.sh manipulation ``` -3. (Inside the container shell) Download additional ROS 2 dependencies: +3. Run the docker container (if not already running): + + ```shell + docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:jazzy # or rai:humble + ``` + + !!! tip "NVIDIA GPU acceleration" + + If the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) is set up on your host machine, you can use the GPU within the RAI docker container for faster inference by adding the `--gpus all` option: + + ```shell + docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID --gpus all -it rai:jazzy # or rai:humble + ``` + +4. (Inside the container shell) Download additional ROS 2 dependencies: ```shell vcs import < demos.repos rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y ``` -4. (Inside the container shell) Build the ROS 2 workspace: +5. (Inside the container shell) Build the ROS 2 workspace: ```shell colcon build --symlink-install From a47ebc091032ee101cc631a16da98a5f334efc56 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 08:39:03 +0200 Subject: [PATCH 16/27] Specified that host and docker ROS 2 distributions should match --- docs/demos/manipulation.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 4823d46e5..03001c58e 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -85,6 +85,10 @@ manipulation techniques. The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine. +!!! warning "ROS 2 distributions" + + It is highly recommended that ROS 2 distribution on the host machine matches the ROS 2 distribution of the docker container. A distribution version mismatch may result in the demo not working correctly. + #### 1. Setting up the demo 1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) From 7b88d37d9150a2e64e325e6318f9009587aadf8f Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 11:43:49 +0200 Subject: [PATCH 17/27] Update command for running tests in docker --- docs/setup/setup_docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 0e7dc7433..a973dc07c 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -102,5 +102,5 @@ docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it ```sh cd /rai source setup_shell.sh -poetry run pytest +poetry run pytest tests/agents ``` From 07814d94e89e9d30ad36cefbe6ad372623a0204a Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 11:51:27 +0200 Subject: [PATCH 18/27] Fixed typo in docs --- docs/setup/setup_docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index a973dc07c..1b0401c4a 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -33,7 +33,7 @@ docker build -t rai:humble --build-arg ROS_DISTRO=humble --build-arg DEPENDENCIE Core dependencies only: ```bash -docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . +docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy -f docker/Dockerfile . ``` All dependencies: From 8afa035f82b79111be0f9ee7bdec710cf162a5ff Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 26 Jun 2025 12:15:22 +0200 Subject: [PATCH 19/27] Fixed fastrtps file path in dokcerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fee164239..696eb9ebf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,7 +41,7 @@ COPY . /rai WORKDIR /rai # Set up the environment to use Fast DDS -ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/fastrtps_config.xml +ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/docker/fastrtps_config.xml ENV RMW_IMPLEMENTATION=rmw_fastrtps_cpp RUN /bin/bash -c 'if [ "${DEPENDENCIES}" = "core_only" ]; then \ From 302aad257e0e28667d244d2509bb7315b6aead68 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 30 Jun 2025 07:24:19 +0200 Subject: [PATCH 20/27] Added sudo to apt install in docs --- docs/setup/setup_docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 1b0401c4a..24899d7b3 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -68,7 +68,7 @@ To allow the container to communicate with the host machine, configure the host 3. Install the eProsima Fast DDS middleware (should come preinstalled with ROS 2): ```shell - apt install ros-"${ROS_DISTRO}"-fastrtps + sudo apt install ros-"${ROS_DISTRO}"-fastrtps ``` 4. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: From 329020c5092caa5d87d5a0779a21f0cd6ab41ac7 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 30 Jun 2025 07:40:30 +0200 Subject: [PATCH 21/27] Updated command for executing multiple tests in docs --- docs/setup/setup_docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 24899d7b3..78d9140d0 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -102,5 +102,5 @@ docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it ```sh cd /rai source setup_shell.sh -poetry run pytest tests/agents +poetry run pytest tests/{agents,messages,tools,types} ``` From d6c81ffa06eebb542ead017048100e592d8a42e4 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 30 Jun 2025 07:48:16 +0200 Subject: [PATCH 22/27] Added warning about ROS2 version to docker setup docs --- docs/setup/setup_docker.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 78d9140d0..78623a7e2 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -51,6 +51,10 @@ docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES= host: [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). +!!! warning "ROS 2 distributions" + + It is highly recommended that ROS 2 distribution on the host machine matches the ROS 2 distribution of the docker container. A distribution version mismatch may result in the demos not working correctly. + To allow the container to communicate with the host machine, configure the host environment as presented below: 1. Source ROS 2 on the host machine: From 7a4d5bd7d5ef8f73c414613913a55dfa8c253244 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 30 Jun 2025 07:53:44 +0200 Subject: [PATCH 23/27] Made setting the FASTRTPS_DEFAULT_PROFILES_FILE env variable more explicit --- docs/setup/setup_docker.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index 78623a7e2..c186d5800 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -78,7 +78,8 @@ To allow the container to communicate with the host machine, configure the host 4. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: ```shell - export FASTRTPS_DEFAULT_PROFILES_FILE=/path/to/fastrtps_config.xml + export FASTRTPS_DEFAULT_PROFILES_FILE=$(pwd)/docker/fastrtps_config.xml + ``` 5. Set the RMW to use eProsima Fast DDS: From 5fef90e9a720c8c4e82a97b81ab6afce67d4c36b Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Mon, 30 Jun 2025 08:14:47 +0200 Subject: [PATCH 24/27] Clarified that the env variable should be exported --- docs/demos/manipulation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 03001c58e..2b2058e0f 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -83,7 +83,7 @@ manipulation techniques. !!! note "ROS 2 required" - The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine. + The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional) !!! warning "ROS 2 distributions" @@ -131,7 +131,7 @@ manipulation techniques. !!! note Source the setup shell - Ensure ROS 2 is sourced on the host machine. Ensure that every command inside the docker container is run in a sourced shell using `source setup_shell.sh`. + Ensure ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional). Ensure that every command inside the docker container is run in a sourced shell using `source setup_shell.sh`. 1. Launch the Robotic Arm Visualization on the host machine: From 2e54d0926d42f23f875208a2556ed1da2780413e Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Wed, 23 Jul 2025 11:47:08 +0200 Subject: [PATCH 25/27] Added an instruction to docs, on how to resolve a potential GPU error in docker --- docs/demos/manipulation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 2b2058e0f..14bcd3805 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -114,6 +114,18 @@ manipulation techniques. docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID --gpus all -it rai:jazzy # or rai:humble ``` + Sometimes, passing GPUs to the docker container may result in an error: + + ```shell + docker: Error response from daemon: could not select device driver "" with capabilities: [[gpu]]. + ``` + + Restarting the docker service should resolve this error: + + ```shell + sudo systemctl restart docker + ``` + 4. (Inside the container shell) Download additional ROS 2 dependencies: ```shell From c7f703d705f25ba4ae7fd1318738c8e45f8ac001 Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Wed, 23 Jul 2025 12:13:22 +0200 Subject: [PATCH 26/27] Added ROS 2 import inside docker to robotic arm demo docs --- docs/demos/manipulation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 14bcd3805..fb2a4f413 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -136,6 +136,7 @@ manipulation techniques. 5. (Inside the container shell) Build the ROS 2 workspace: ```shell + source /opt/ros/${ROS_DISTRO}/setup.bash colcon build --symlink-install ``` From 4c634304eece978fcee6de396ce4f2ccf4532fac Mon Sep 17 00:00:00 2001 From: Dominik Matejkowski Date: Thu, 24 Jul 2025 13:32:34 +0200 Subject: [PATCH 27/27] Added step with setting the OpenAI API key, clarified that the vendor may be changed --- docs/demos/manipulation.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index fb2a4f413..3e2f280a0 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -140,6 +140,19 @@ manipulation techniques. colcon build --symlink-install ``` +6. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary + to set the `$OPENAI_API_KEY` environmental variable. The command below may be utilized to set + the variable and add it to the container's `.bashrc` file: + + ```shell + export OPENAI_API_KEY=YOUR_OPEN_AI_API_KEY + echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> ~/.bashrc + ``` + + !!! note AI vendor change + + The default vendor can be changed to a different provider via the [RAI configuration tool](../setup/install.md#15-configure-rai) + #### 2. Running the demo !!! note Source the setup shell