Adding Humanoid Hand RL

.github/workflows/linting_auto.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9.22
+          python-version: 3.10.18
 
       - name: Install Python dependencies
         run: pip install autopep8 clang-format

autonomy/simulation/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.10)
+project(sample_msgs)
+
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+endif()
+
+find_package(ament_cmake REQUIRED)
+find_package(rosidl_default_generators REQUIRED)
+
+ament_export_dependencies(rosidl_default_runtime)
+ament_package()

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/config/extension.toml

@@ -0,0 +1,22 @@
+[package]
+
+# Note: Semantic Versioning is used: https://semver.org/
+version = "0.10.31"
+
+# Description
+title = "Isaac Lab Environments"
+description="Extension containing suite of environments for robot learning."
+readme  = "docs/README.md"
+repository = "https://github.com/isaac-sim/IsaacLab"
+category = "robotics"
+keywords = ["robotics", "rl", "il", "learning"]
+
+[dependencies]
+"isaaclab" = {}
+"isaaclab_assets" = {}
+
+[core]
+reloadable = false
+
+[[python.module]]
+name = "isaaclab_tasks"

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/modelCfg/humanoid.py

@@ -0,0 +1,44 @@
+# Code for specifying custom model parameters
+
+import isaaclab.sim as sim_utils
+from isaaclab.actuators import ImplicitActuatorCfg
+from isaaclab.assets.articulation import ArticulationCfg
+
+HAND_CFG = ArticulationCfg(
+    spawn=sim_utils.UsdFileCfg(
+        usd_path="/home/hy/Downloads/Humanoid_Wato/ModelAssets/hand.usd",
+        rigid_props=sim_utils.RigidBodyPropertiesCfg(
+            disable_gravity=False,
+            max_depenetration_velocity=5.0,
+        ),
+        activate_contact_sensors=False,
+    ),
+    init_state=ArticulationCfg.InitialStateCfg(
+        joint_pos={
+            "Revolute_1": 0.0,
+            "Revolute_2": 0.0,
+            "Revolute_3": 0.0,
+            "Revolute_4": 0.0,
+            "Revolute_5": 0.0,
+            "Revolute_6": 0.0,
+            "Revolute_7": 0.0,
+            "Revolute_8": 0.0,
+            "Revolute_9": 0.0,
+            "Revolute_10": 0.0,
+            "Revolute_11": 0.0,
+            "Revolute_12": 0.0,
+            "Revolute_13": 0.0,
+            "Revolute_14": 0.0,
+            "Revolute_15": 0.0,
+        }
+    ),
+    actuators={
+        "arm": ImplicitActuatorCfg(
+            joint_names_expr=[".*"],
+            # velocity_limit=100.0,
+            # effort_limit=87.0,
+            stiffness=0.5,
+            damping=0.5,
+        ),
+    },
+)

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/setup.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2022-2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Installation script for the 'isaaclab_tasks' python package."""
+
+import os
+import toml
+
+from setuptools import setup
+
+# Obtain the extension data from the extension.toml file
+EXTENSION_PATH = os.path.dirname(os.path.realpath(__file__))
+# Read the extension.toml file
+EXTENSION_TOML_DATA = toml.load(os.path.join(EXTENSION_PATH, "config", "extension.toml"))
+
+# Minimum dependencies required prior to installation
+INSTALL_REQUIRES = [
+    # generic
+    "numpy",
+    "torch==2.5.1",
+    "torchvision>=0.14.1",  # ensure compatibility with torch 1.13.1
+    # 5.26.0 introduced a breaking change, so we restricted it for now.
+    # See issue https://github.com/tensorflow/tensorboard/issues/6808 for details.
+    "protobuf>=3.20.2, < 5.0.0",
+    # basic logger
+    "tensorboard",
+]
+
+PYTORCH_INDEX_URL = ["https://download.pytorch.org/whl/cu118"]
+
+# Installation operation
+setup(
+    name="skyentific_poclegs",
+    author="Isaac Lab Project Developers",
+    maintainer="Isaac Lab Project Developers",
+    url=EXTENSION_TOML_DATA["package"]["repository"],
+    version=EXTENSION_TOML_DATA["package"]["version"],
+    description=EXTENSION_TOML_DATA["package"]["description"],
+    keywords=EXTENSION_TOML_DATA["package"]["keywords"],
+    include_package_data=True,
+    python_requires=">=3.10",
+    install_requires=INSTALL_REQUIRES,
+    dependency_links=PYTORCH_INDEX_URL,
+    packages=["skyentific_poclegs"],
+    classifiers=[
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3.10",
+        "Isaac Sim :: 4.5.0",
+    ],
+    zip_safe=False,
+)
+
+# From Isaac Lab source/isaaclab_tasks/setup.py

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2022-2024, The Berkeley Humanoid Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Package containing task implementations for various robotic environments."""
+
+import os
+import toml
+
+from isaaclab_tasks.utils import import_packages
+
+##
+# Register Gym environments.
+##
+
+
+# The blacklist is used to prevent importing configs from sub-packages
+_BLACKLIST_PKGS = ["utils"]
+# Import all configs in this package
+import_packages(__name__, _BLACKLIST_PKGS)

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/manipulation/config/HumanoidRLEnv/__init__.py

@@ -0,0 +1,26 @@
+import gymnasium as gym
+from . import agents
+
+gym.register(
+    id="Isaac-Reach-Humanoid-Hand-v0",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.joint_pos_env_cfg:HumanoidHandReachEnvCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:HumanoidHandReachPPORunnerCfg",
+        "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml",
+    },
+)
+
+gym.register(
+    id="Isaac-Reach-UR10-Play-v0",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.joint_pos_env_cfg:HumanoidHandReachEnvCfg_PLAY",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:HumanoidHandReachPPORunnerCfg",
+        "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml",
+    },
+)

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/manipulation/config/HumanoidRLEnv/agents/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2022-2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# From Isaac Lab source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/<project folder>/agents/__init__.py

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/manipulation/config/HumanoidRLEnv/agents/rsl_rl_ppo_cfg.py

@@ -0,0 +1,33 @@
+from isaaclab.utils import configclass
+from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg
+
+
+@configclass
+class HumanoidHandReachPPORunnerCfg(RslRlOnPolicyRunnerCfg):
+    num_steps_per_env = 24
+    max_iterations = 1000
+    save_interval = 50
+    experiment_name = "reach_humanoid_hand"
+    run_name = ""
+    resume = False
+    empirical_normalization = False
+    policy = RslRlPpoActorCriticCfg(
+        init_noise_std=1.0,
+        actor_hidden_dims=[64, 64],
+        critic_hidden_dims=[64, 64],
+        activation="elu",
+    )
+    algorithm = RslRlPpoAlgorithmCfg(
+        value_loss_coef=1.0,
+        use_clipped_value_loss=True,
+        clip_param=0.2,
+        entropy_coef=0.01,
+        num_learning_epochs=8,
+        num_mini_batches=4,
+        learning_rate=1.0e-3,
+        schedule="adaptive",
+        gamma=0.99,
+        lam=0.95,
+        desired_kl=0.01,
+        max_grad_norm=1.0,
+    )

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/manipulation/config/HumanoidRLEnv/joint_pos_env_cfg.py

@@ -0,0 +1,111 @@
+import math
+from isaaclab.utils import configclass
+import HumanoidRLPackage.HumanoidRLSetup.tasks.manipulation.mdp as mdp
+from HumanoidRLPackage.HumanoidRLSetup.tasks.manipulation.reach_env_cfg import ReachEnvCfg
+from HumanoidRLPackage.HumanoidRLSetup.modelCfg.universal_robots import UR10_CFG
+from HumanoidRLPackage.HumanoidRLSetup.modelCfg.humanoid import HAND_CFG
+
+
+@configclass
+class HumanoidHandReachEnvCfg(ReachEnvCfg):
+    def __post_init__(self):
+        # post init of parent
+        super().__post_init__()
+
+        # self.scene.robot = UR10_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
+        self.scene.robot = HAND_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
+
+        marker_scale = 0.02
+
+        self.commands.ee_pose.goal_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+        self.commands.ee_pose.current_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+
+        self.commands.ee_pose_2.goal_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+        self.commands.ee_pose_2.current_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+
+        self.commands.ee_pose_3.goal_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+        self.commands.ee_pose_3.current_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+
+        self.commands.ee_pose_4.goal_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+        self.commands.ee_pose_4.current_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+
+        self.commands.ee_pose_5.goal_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+        self.commands.ee_pose_5.current_pose_visualizer_cfg.markers["frame"].scale = (
+            marker_scale, marker_scale, marker_scale)
+
+        # override events
+        self.events.reset_robot_joints.params["position_range"] = (0.75, 1.25)
+        # override rewards
+        self.rewards.end_effector_position_tracking.params["asset_cfg"].body_names = ["TIP_B_1"]
+        self.rewards.end_effector_position_tracking_fine_grained.params["asset_cfg"].body_names = [
+            "TIP_B_1"]
+        # self.rewards.end_effector_orientation_tracking.params["asset_cfg"].body_names = ["TIP_B_1"]
+
+        self.rewards.end_effector_2_position_tracking.params["asset_cfg"].body_names = ["TIP_B_2"]
+        self.rewards.end_effector_2_position_tracking_fine_grained.params["asset_cfg"].body_names = [
+            "TIP_B_2"]
+        # self.rewards.end_effector_2_orientation_tracking.params["asset_cfg"].body_names = ["TIP_B_2"]
+
+        self.rewards.end_effector_3_position_tracking.params["asset_cfg"].body_names = ["TIP_B_3"]
+        self.rewards.end_effector_3_position_tracking_fine_grained.params["asset_cfg"].body_names = [
+            "TIP_B_3"]
+        # self.rewards.end_effector_3_orientation_tracking.params["asset_cfg"].body_names = ["TIP_B_3"]
+
+        self.rewards.end_effector_4_position_tracking.params["asset_cfg"].body_names = ["TIP_B_4"]
+        self.rewards.end_effector_4_position_tracking_fine_grained.params["asset_cfg"].body_names = [
+            "TIP_B_4"]
+        # self.rewards.end_effector_4_orientation_tracking.params["asset_cfg"].body_names = ["TIP_B_4"]
+
+        self.rewards.end_effector_5_position_tracking.params["asset_cfg"].body_names = ["TIP_B_5"]
+        self.rewards.end_effector_5_position_tracking_fine_grained.params["asset_cfg"].body_names = [
+            "TIP_B_5"]
+        # self.rewards.end_effector_5_orientation_tracking.params["asset_cfg"].body_names = ["TIP_B_5"]
+
+        # override actions
+        self.actions.arm_action = mdp.JointPositionActionCfg(
+            asset_name="robot", joint_names=[".*"], scale=0.5, use_default_offset=True
+        )
+        # override command generator body
+        # end-effector is along x-direction
+        self.commands.ee_pose.body_name = "TIP_B_1"
+        # self.commands.ee_pose.ranges.pitch = (math.pi / 2, math.pi / 2) # this rotate the pose
+
+        self.commands.ee_pose_2.body_name = "TIP_B_2"
+        # self.commands.ee_pose_2.ranges.pitch = (math.pi / 2, math.pi / 2)
+
+        self.commands.ee_pose_3.body_name = "TIP_B_3"
+        # self.commands.ee_pose_3.ranges.pitch = (math.pi / 2, math.pi / 2)
+
+        self.commands.ee_pose_4.body_name = "TIP_B_4"
+        # self.commands.ee_pose_4.ranges.pitch = (math.pi / 2, math.pi / 2)
+
+        self.commands.ee_pose_5.body_name = "TIP_B_5"
+        # self.commands.ee_pose_5.ranges.pitch = (math.pi / 2, math.pi / 2)
+
+        # self.commands.ee_pose.ranges.yaw = (-math.pi / 2, -math.pi / 2)
+        # self.commands.ee_pose_2.ranges.yaw = (-math.pi / 2, -math.pi / 2)
+
+
+@configclass
+class HumanoidHandReachEnvCfg_PLAY(HumanoidHandReachEnvCfg):
+    def __post_init__(self):
+        # post init of parent
+        super().__post_init__()
+        # make a smaller scene for play
+        self.scene.num_envs = 50
+        self.scene.env_spacing = 2.5
+        # disable randomization for play
+        self.observations.policy.enable_corruption = False
+
+# (env_isaaclab) hy@hy-LOQ-15IRX9:~/Downloads/Humanoid_Wato/HumanoidRL$ PYTHONPATH=$(pwd) /home/hy/IsaacLab/isaaclab.sh -p HumanoidRLPackage/rsl_rl_scripts/train.py --task=Isaac-Reach-Humanoid-Hand-v0 --headless
+
+# (env_isaaclab) hy@hy-LOQ-15IRX9:~/Downloads/Humanoid_Wato/HumanoidRL$ PYTHONPATH=$(pwd) /home/hy/IsaacLab/isaaclab.sh -p HumanoidRLPackage/rsl_rl_scripts/play.py --task=Isaac-Reach-Humanoid-Hand-v0 --num_envs=1

autonomy/simulation/Humanoid_RL/HumanoidRL/HumanoidRLPackage/HumanoidRLSetup/tasks/manipulation/mdp/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2022-2024, The Berkeley Humanoid Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""This sub-module contains the functions that are specific to the locomotion environments."""
+
+from isaaclab.envs.mdp import *  # noqa: F401, F403
+
+from .rewards import *  # noqa: F401, F403
+
+# From Isaac Lab source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/mdp/__init__.py