From f3617edeb2ab8e9037a6fef7ab955e51e6688b14 Mon Sep 17 00:00:00 2001 From: Mateusz Wasilewski Date: Fri, 10 Oct 2025 13:20:14 +0200 Subject: [PATCH 1/4] Install missing deps for running GameLauncher in docker Signed-off-by: Mateusz Wasilewski --- .dockerignore | 3 +++ docker/Dockerfile | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.dockerignore b/.dockerignore index 700b742e8..b4f509535 100644 --- a/.dockerignore +++ b/.dockerignore @@ -180,3 +180,6 @@ src/rai_interfaces # Demo assets demo_assets/ + +# Docker files +Dockerfile* diff --git a/docker/Dockerfile b/docker/Dockerfile index 696eb9ebf..ed6989c05 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,3 +56,20 @@ RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && \ # Build the workspace RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && colcon build --symlink-install' + +# Install libraries required for GUI and NVIDIA support +RUN apt install -y \ + libglu1-mesa-dev \ + libxcb-xinerama0 \ + libfontconfig1-dev \ + libnvidia-gl-470 \ + libxcb-xkb-dev \ + libxkbcommon-x11-dev \ + libxkbcommon-dev \ + libxcb-xfixes0-dev \ + libxcb-xinput-dev \ + libxcb-xinput0 \ + libpcre2-16-0 \ + libunwind-dev +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=all From fb3416bf381d1d0ff5832e1f84e3b38d7d812e9b Mon Sep 17 00:00:00 2001 From: Mateusz Wasilewski Date: Fri, 10 Oct 2025 13:39:26 +0200 Subject: [PATCH 2/4] Update manipulation demo docs to run simulation in docker Signed-off-by: Mateusz Wasilewski --- docs/demos/manipulation.md | 141 ++++++++++--------------------------- 1 file changed, 38 insertions(+), 103 deletions(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 3e2f280a0..646dbd872 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -25,23 +25,23 @@ manipulation techniques. 1. Follow the RAI setup instructions in the [quick setup guide](../setup/install.md#setting-up-developer-environment). 2. Download additional dependencies: - ```shell - poetry install --with openset - vcs import < demos.repos - rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y - ``` + ```shell + poetry install --with openset + vcs import < demos.repos + rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y + ``` 3. Download the latest binary release - ```bash - ./scripts/download_demo.sh manipulation - ``` + ```bash + ./scripts/download_demo.sh manipulation + ``` 4. Build the ROS 2 workspace: - ```bash - colcon build --symlink-install - ``` + ```bash + colcon build --symlink-install + ``` #### Running the demo @@ -52,28 +52,28 @@ manipulation techniques. 1. Start the demo - ```shell - ros2 launch examples/manipulation-demo.launch.py game_launcher:=demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher - ``` + ```shell + ros2 launch examples/manipulation-demo.launch.py game_launcher:=demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher + ``` 2. In the second terminal, run the streamlit interface: - ```shell - streamlit run examples/manipulation-demo-streamlit.py - ``` + ```shell + streamlit run examples/manipulation-demo-streamlit.py + ``` - Alternatively, you can run the simpler command-line version, which also serves as an example of - how to use the RAI API for you own applications: + Alternatively, you can run the simpler command-line version, which also serves as an example of + how to use the RAI API for you own applications: - ```shell - python examples/manipulation-demo.py - ``` + ```shell + python examples/manipulation-demo.py + ``` 3. Interact with the robot arm using natural language commands. For example: - ``` - Enter a prompt: Pick up the red cube and drop it on another cube - ``` + ``` + Enter a prompt: Pick up the red cube and drop it on another cube + ``` !!! tip "Changing camera view" @@ -81,66 +81,21 @@ manipulation techniques. ### Docker Setup -!!! note "ROS 2 required" - - The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional) - -!!! warning "ROS 2 distributions" - - It is highly recommended that ROS 2 distribution on the host machine matches the ROS 2 distribution of the docker container. A distribution version mismatch may result in the demo not working correctly. - #### 1. Setting up the demo 1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) - and configure communication between the container and the host ([link](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional)). - -2. On the host machine, download the latest binary release for the Robotic Arm Demo: - - ```shell - ./scripts/download_demo.sh manipulation - ``` - -3. Run the docker container (if not already running): - - ```shell - docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:jazzy # or rai:humble - ``` - - !!! tip "NVIDIA GPU acceleration" - - If the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) is set up on your host machine, you can use the GPU within the RAI docker container for faster inference by adding the `--gpus all` option: - - ```shell - docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID --gpus all -it rai:jazzy # or rai:humble - ``` - - Sometimes, passing GPUs to the docker container may result in an error: - - ```shell - docker: Error response from daemon: could not select device driver "" with capabilities: [[gpu]]. - ``` - - Restarting the docker service should resolve this error: - - ```shell - sudo systemctl restart docker - ``` -4. (Inside the container shell) Download additional ROS 2 dependencies: +2. Run the docker container with the following command: ```shell - vcs import < demos.repos - rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y + docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix --gpus all -it rai:jazzy # or rai:humble ``` -5. (Inside the container shell) Build the ROS 2 workspace: + !!! tip "NVIDIA Container Toolkit" - ```shell - source /opt/ros/${ROS_DISTRO}/setup.bash - colcon build --symlink-install - ``` + In order to use the `--gpus all` flag, the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) must be installed on the host machine. -6. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary +3. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary to set the `$OPENAI_API_KEY` environmental variable. The command below may be utilized to set the variable and add it to the container's `.bashrc` file: @@ -153,35 +108,15 @@ manipulation techniques. The default vendor can be changed to a different provider via the [RAI configuration tool](../setup/install.md#15-configure-rai) -#### 2. Running the demo - -!!! note Source the setup shell - - Ensure ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional). Ensure that every command inside the docker container is run in a sourced shell using `source setup_shell.sh`. - -1. Launch the Robotic Arm Visualization on the host machine: +4. After this, follow the steps in the [Local Setup](#local-setup) from step 2 onwards. - ```shell - ./demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher - ``` - -2. (Inside the container shell) Launch the Robotic Arm Demo script inside of the docker container: - - ```shell - ros2 launch examples/manipulation-demo.launch.py - ``` - -3. (Inside the container shell) Open a new terminal for the docker container (e.g., `docker exec -it CONTAINER_ID /bin/bash`) and launch the streamlit interface: + !!! tip "New terminal in docker" - ```shell - streamlit run examples/manipulation-demo-streamlit.py - ``` + In order to open a new terminal in the same docker container, you can use the following command: - Alternatively, run the simpler command-line version: - - ```shell - python examples/manipulation-demo.py - ``` + ```shell + docker exec -it bash + ``` ## How it works @@ -199,8 +134,8 @@ examples/manipulation-demo.py ## Known Limitations -- `Grounding DINO` can't distinguish colors. -- VLMs tend to struggle with spatial understanding (for example left/right concepts). +- `Grounding DINO` can't distinguish colors. +- VLMs tend to struggle with spatial understanding (for example left/right concepts). !!! tip "Building from source" From 70cce0c8768655e3e6ffe5cad956fa0543a30704 Mon Sep 17 00:00:00 2001 From: Mateusz Wasilewski Date: Fri, 10 Oct 2025 17:49:12 +0200 Subject: [PATCH 3/4] Add information about xhost to docs Signed-off-by: Mateusz Wasilewski --- docs/demos/manipulation.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 646dbd872..62b7b076e 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -85,7 +85,13 @@ manipulation techniques. 1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) -2. Run the docker container with the following command: +2. Enable X11 access for the docker container: + + ```shell + xhost +local:root + ``` + +3. Run the docker container with the following command: ```shell docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix --gpus all -it rai:jazzy # or rai:humble @@ -95,7 +101,7 @@ manipulation techniques. In order to use the `--gpus all` flag, the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) must be installed on the host machine. -3. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary +4. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary to set the `$OPENAI_API_KEY` environmental variable. The command below may be utilized to set the variable and add it to the container's `.bashrc` file: @@ -108,7 +114,7 @@ manipulation techniques. The default vendor can be changed to a different provider via the [RAI configuration tool](../setup/install.md#15-configure-rai) -4. After this, follow the steps in the [Local Setup](#local-setup) from step 2 onwards. +5. After this, follow the steps in the [Local Setup](#local-setup) from step 2 onwards. !!! tip "New terminal in docker" From c86c8f46bda52e416be7b6f4b5d9920dcf9b2d46 Mon Sep 17 00:00:00 2001 From: Mateusz Wasilewski Date: Fri, 10 Oct 2025 17:52:21 +0200 Subject: [PATCH 4/4] Increase poetry timeout Signed-off-by: Mateusz Wasilewski --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index ed6989c05..53c25dc0e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,6 +17,8 @@ FROM osrf/ros:${ROS_DISTRO}-desktop-full ARG DEPENDENCIES=core_only ENV DEBIAN_FRONTEND=noninteractive +# Increase timeout for Poetry requests. +ENV POETRY_REQUESTS_TIMEOUT=300 # Check whether the $DEPENDENCIES ARG has a valid value RUN /bin/bash -c 'if [ "${DEPENDENCIES}" != "core_only" ] && [ "${DEPENDENCIES}" != "all_groups" ]; then \