Merge commit 'a9a9b96ce78019dd5dafa379d8f097df7e761d3e' as 'modules/atom01_train/rsl_rl'

Author: tcmofashi

modules/atom01_train/rsl_rl/.github/LICENSE_HEADER.txt

@@ -0,0 +1,4 @@
+Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
+All rights reserved.
+
+SPDX-License-Identifier: BSD-3-Clause

modules/atom01_train/rsl_rl/.gitignore

@@ -0,0 +1,14 @@
+# IDEs
+.idea
+
+# builds
+*.egg-info
+build/*
+dist/*
+
+# cache
+__pycache__
+.pytest_cache
+
+# vs code
+.vscode

modules/atom01_train/rsl_rl/.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.0
+    hooks:
+      - id: ruff-check
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-symlinks
+      - id: destroyed-symlinks
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-shebang-scripts-are-executable
+      - id: detect-private-key
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies:
+        - tomli
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.1
+    hooks:
+      - id: insert-license
+        files: \.py$
+        args:
+          # - --remove-header    # Remove existing license headers. Useful when updating license.
+          - --license-filepath
+          - .github/LICENSE_HEADER.txt

modules/atom01_train/rsl_rl/CITATION.cff

@@ -0,0 +1,38 @@
+cff-version: 1.2.0
+title: "RSL-RL: A Learning Library for Robotics Research"
+message: "If you use this work, please cite the following paper."
+repository-code: "https://github.com/leggedrobotics/rsl_rl"
+license: BSD-3-Clause
+version: 3.3.0
+type: software
+authors:
+  - family-names: Schwarke
+    given-names: Clemens
+  - family-names: Mittal
+    given-names: Mayank
+  - family-names: Rudin
+    given-names: Nikita
+  - family-names: Hoeller
+    given-names: David
+keywords:
+  - reinforcement learning
+  - robotics
+  - control
+  - RSL-RL
+preferred-citation:
+  type: article
+  authors:
+    - family-names: Schwarke
+      given-names: Clemens
+    - family-names: Mittal
+      given-names: Mayank
+    - family-names: Rudin
+      given-names: Nikita
+    - family-names: Hoeller
+      given-names: David
+    - family-names: Hutter
+      given-names: Marco
+  title: "RSL-RL: A Learning Library for Robotics Research"
+  journal: "arXiv preprint"
+  doi: 10.48550/arXiv.2509.10771
+  url: "https://arxiv.org/abs/2509.10771"

modules/atom01_train/rsl_rl/CONTRIBUTORS.md

@@ -0,0 +1,42 @@
+# RSL-RL Maintainers and Contributors
+
+This is the official list of developers and contributors.
+
+To see the full list of contributors, see the revision history in the source control.
+
+Names should be added to this file as: individual names or organizations.
+
+Email addresses are tracked elsewhere to avoid spam.
+
+Please keep the lists sorted alphabetically.
+
+## Maintainers
+
+* Robotic Systems Lab, ETH Zurich
+* NVIDIA Corporation
+
+---
+
+* Clemens Schwarke
+* Mayank Mittal
+
+## Authors
+
+* Clemens Schwarke
+* David Hoeller
+* Mayank Mittal
+* Nikita Rudin
+
+## Contributors
+
+* Bikram Pandit
+* Eric Vollenweider
+* Fabian Jenelten
+* Lorenzo Terenzi
+* Marko Bjelonic
+* Matthijs van der Boon
+* Özhan Özen
+* Pascal Roth
+* Shaoshu Su
+* Zhang Chong
+* Ziqi Fan

modules/atom01_train/rsl_rl/LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2026, ETH Zurich
+Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+See licenses/dependencies for license information of dependencies of this package.

modules/atom01_train/rsl_rl/README.md

@@ -0,0 +1,109 @@
+# RSL-RL
+
+A fast and simple implementation of learning algorithms for robotics. For an overview of the library please have a look at https://arxiv.org/pdf/2509.10771.
+
+Environment repositories using the framework:
+
+* **`Isaac Lab`** (built on top of NVIDIA Isaac Sim): https://github.com/isaac-sim/IsaacLab
+* **`Legged Gym`** (built on top of NVIDIA Isaac Gym): https://leggedrobotics.github.io/legged_gym/
+* **`MuJoCo Playground`** (built on top of MuJoCo MJX and Warp): https://github.com/google-deepmind/mujoco_playground/
+* **`mjlab`** (built on top of MuJoCo Warp): https://github.com/mujocolab/mjlab
+
+The library currently supports **PPO** and **Student-Teacher Distillation** with additional features from our research. These include:
+
+* [Random Network Distillation (RND)](https://proceedings.mlr.press/v229/schwarke23a.html) - Encourages exploration by adding
+  a curiosity driven intrinsic reward.
+* [Symmetry-based Augmentation](https://arxiv.org/abs/2403.04359) - Makes the learned behaviors more symmetrical.
+
+We welcome contributions from the community. Please check our contribution guidelines for more
+information.
+
+**Maintainer**: Mayank Mittal and Clemens Schwarke <br/>
+**Affiliation**: Robotic Systems Lab, ETH Zurich & NVIDIA <br/>
+**Contact**: cschwarke@ethz.ch
+
+
+## Setup
+
+The package can be installed via PyPI with:
+
+```bash
+pip install rsl-rl-lib
+```
+
+or by cloning this repository and installing it with:
+
+```bash
+git clone https://github.com/leggedrobotics/rsl_rl
+cd rsl_rl
+pip install -e .
+```
+
+The package supports the following logging frameworks which can be configured through `logger`:
+
+* Tensorboard: https://www.tensorflow.org/tensorboard/
+* Weights & Biases: https://wandb.ai/site
+* Neptune: https://docs.neptune.ai/
+
+For a demo configuration of PPO, please check the [example_config.yaml](config/example_config.yaml) file.
+
+
+## Contribution Guidelines
+
+For documentation, we adopt the [Google Style Guide](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for docstrings. Please make sure that your code is well-documented and follows the guidelines.
+
+We use the following tools for maintaining code quality:
+
+- [pre-commit](https://pre-commit.com/): Runs a list of formatters and linters over the codebase.
+- [ruff](https://github.com/astral-sh/ruff): An extremely fast Python linter and code formatter, written in Rust.
+
+Please check [here](https://pre-commit.com/#install) for instructions to set these up. To run over the entire repository, please execute the following command in the terminal:
+
+```bash
+# for installation (only once)
+pre-commit install
+# for running
+pre-commit run --all-files
+```
+
+## Citing
+
+If you use this library for your research, please cite the following work:
+
+```text
+@article{schwarke2025rslrl,
+  title={RSL-RL: A Learning Library for Robotics Research},
+  author={Schwarke, Clemens and Mittal, Mayank and Rudin, Nikita and Hoeller, David and Hutter, Marco},
+  journal={arXiv preprint arXiv:2509.10771},
+  year={2025}
+}
+```
+
+If you use the library with curiosity-driven exploration (random network distillation), please cite:
+
+```text
+@InProceedings{schwarke2023curiosity,
+  title = 	 {Curiosity-Driven Learning of Joint Locomotion and Manipulation Tasks},
+  author =       {Schwarke, Clemens and Klemm, Victor and Boon, Matthijs van der and Bjelonic, Marko and Hutter, Marco},
+  booktitle = 	 {Proceedings of The 7th Conference on Robot Learning},
+  pages = 	 {2594--2610},
+  year = 	 {2023},
+  volume = 	 {229},
+  series = 	 {Proceedings of Machine Learning Research},
+  publisher =    {PMLR},
+  url = 	 {https://proceedings.mlr.press/v229/schwarke23a.html},
+}
+```
+
+If you use the library with symmetry augmentation, please cite:
+
+```text
+@InProceedings{mittal2024symmetry,
+  author={Mittal, Mayank and Rudin, Nikita and Klemm, Victor and Allshire, Arthur and Hutter, Marco},
+  booktitle={2024 IEEE International Conference on Robotics and Automation (ICRA)},
+  title={Symmetry Considerations for Learning Task Symmetric Robot Policies},
+  year={2024},
+  pages={7433-7439},
+  doi={10.1109/ICRA57147.2024.10611493}
+}
+```

modules/atom01_train/rsl_rl/config/example_config.yaml

@@ -0,0 +1,77 @@
+runner:
+  class_name: OnPolicyRunner
+  # General
+  num_steps_per_env: 24  # Number of steps per environment per iteration
+  max_iterations: 1500  # Number of policy updates
+  seed: 1
+  # Observations
+  obs_groups: {"policy": ["policy"], "critic": ["policy", "privileged"]} # Maps observation groups to sets. See `vec_env.py` for more information
+  # Logging parameters
+  save_interval: 50  # Check for potential saves every `save_interval` iterations
+  experiment_name: walking_experiment
+  run_name: ""
+  # Logging writer
+  logger: tensorboard  # tensorboard, neptune, wandb
+  neptune_project: legged_gym
+  wandb_project: legged_gym
+
+  # Policy
+  policy:
+    class_name: ActorCritic
+    activation: elu
+    actor_obs_normalization: false
+    critic_obs_normalization: false
+    actor_hidden_dims: [256, 256, 256]
+    critic_hidden_dims: [256, 256, 256]
+    init_noise_std: 1.0
+    noise_std_type: "scalar"  # 'scalar' or 'log'
+    state_dependent_std: false
+
+  # Algorithm
+  algorithm:
+    class_name: PPO
+    # Training
+    learning_rate: 0.001
+    num_learning_epochs: 5
+    num_mini_batches: 4  # mini batch size = num_envs * num_steps / num_mini_batches
+    schedule: adaptive  # adaptive, fixed
+    # Value function
+    value_loss_coef: 1.0
+    clip_param: 0.2
+    use_clipped_value_loss: true
+    # Surrogate loss
+    desired_kl: 0.01
+    entropy_coef: 0.01
+    gamma: 0.99
+    lam: 0.95
+    max_grad_norm: 1.0
+    # Miscellaneous
+    normalize_advantage_per_mini_batch: false
+
+    # Random network distillation
+    rnd_cfg:
+        weight: 0.0  # Initial weight of the RND reward
+        weight_schedule: null # This is a dictionary with a required key called "mode". Please check the RND module for more information
+        reward_normalization: false  # Whether to normalize RND reward
+        # Learning parameters
+        learning_rate: 0.001  # Learning rate for RND
+        # Network parameters
+        num_outputs: 1  # Number of outputs of RND network. Note: if -1, then the network will use dimensions of the observation
+        predictor_hidden_dims: [-1] # Hidden dimensions of predictor network
+        target_hidden_dims: [-1]  # Hidden dimensions of target network
+
+    # Symmetry augmentation
+    symmetry_cfg:
+      use_data_augmentation: true  # This adds symmetric trajectories to the batch
+      use_mirror_loss: false  # This adds symmetry loss term to the loss function
+      data_augmentation_func: null # String containing the module and function name to import
+      # Example: "legged_gym.envs.locomotion.anymal_c.symmetry:get_symmetric_states"
+      #
+      # .. code-block:: python
+      #
+      #     @torch.no_grad()
+      #     def get_symmetric_states(
+      #        obs: Optional[torch.Tensor] = None, actions: Optional[torch.Tensor] = None, cfg: "BaseEnvCfg" = None, obs_type: str = "policy"
+      #     ) -> Tuple[torch.Tensor, torch.Tensor]:
+      #
+      mirror_loss_coeff: 0.0 # Coefficient for symmetry loss term. If 0, no symmetry loss is used