Box2D integration: Add CarRacing-v2 (#199)

Co-authored-by: Jiayi Weng <[email protected]>

README.md

@@ -15,7 +15,7 @@
 - [x] [Toy text RL envs](https://envpool.readthedocs.io/en/latest/env/toy_text.html): Catch, FrozenLake, Taxi, NChain, CliffWalking, Blackjack
 - [x] [ViZDoom single player](https://envpool.readthedocs.io/en/latest/env/vizdoom.html)
 - [x] [DeepMind Control Suite](https://envpool.readthedocs.io/en/latest/env/dm_control.html)
-- [ ] [Box2D](https://envpool.readthedocs.io/en/latest/env/box2d.html)
+- [x] [Box2D](https://envpool.readthedocs.io/en/latest/env/box2d.html)
 - [ ] Procgen
 - [ ] Minigrid
 

docs/env/box2d.rst

@@ -50,7 +50,7 @@ The episode will terminate if the hull gets in contact with the ground or
 if the walker exceeds the right end of the terrain length.
 
 
-CarRacing-v1
+CarRacing-v2
 ------------
 
 The easiest control task to learn from pixels - a top-down racing environment.

envpool/BUILD

@@ -1,3 +1,17 @@
+# Copyright 2022 Garena Online Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 load("@pip_requirements//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])

envpool/atari/BUILD

@@ -1,3 +1,17 @@
+# Copyright 2022 Garena Online Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 load("@pip_requirements//:requirements.bzl", "requirement")
 load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
 

envpool/box2d/BUILD

@@ -1,3 +1,17 @@
+# Copyright 2022 Garena Online Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 load("@pip_requirements//:requirements.bzl", "requirement")
 load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
 
@@ -7,12 +21,17 @@ cc_library(
     name = "box2d_env",
     srcs = [
         "bipedal_walker_env.cc",
+        "car_dynamics.cc",
+        "car_racing_env.cc",
         "lunar_lander_env.cc",
         "utils.cc",
     ],
     hdrs = [
         "bipedal_walker.h",
         "bipedal_walker_env.h",
+        "car_dynamics.h",
+        "car_racing.h",
+        "car_racing_env.h",
         "lunar_lander_continuous.h",
         "lunar_lander_discrete.h",
         "lunar_lander_env.h",
@@ -21,6 +40,7 @@ cc_library(
     deps = [
         "//envpool/core:async_envpool",
         "@box2d",
+        "@opencv",
     ],
 )
 
@@ -71,6 +91,7 @@ py_test(
         requirement("gym"),
         requirement("box2d"),
         requirement("pygame"),
+        requirement("opencv-python-headless"),
         requirement("numpy"),
     ],
 )

envpool/box2d/__init__.py

@@ -18,12 +18,18 @@
 from .box2d_envpool import (
   _BipedalWalkerEnvPool,
   _BipedalWalkerEnvSpec,
+  _CarRacingEnvPool,
+  _CarRacingEnvSpec,
   _LunarLanderContinuousEnvPool,
   _LunarLanderContinuousEnvSpec,
   _LunarLanderDiscreteEnvPool,
   _LunarLanderDiscreteEnvSpec,
 )
 
+CarRacingEnvSpec, CarRacingDMEnvPool, CarRacingGymEnvPool = py_env(
+  _CarRacingEnvSpec, _CarRacingEnvPool
+)
+
 BipedalWalkerEnvSpec, BipedalWalkerDMEnvPool, BipedalWalkerGymEnvPool = py_env(
   _BipedalWalkerEnvSpec, _BipedalWalkerEnvPool
 )
@@ -41,6 +47,9 @@
 ) = py_env(_LunarLanderDiscreteEnvSpec, _LunarLanderDiscreteEnvPool)
 
 __all__ = [
+  "CarRacingEnvSpec",
+  "CarRacingDMEnvPool",
+  "CarRacingGymEnvPool",
   "BipedalWalkerEnvSpec",
   "BipedalWalkerDMEnvPool",
   "BipedalWalkerGymEnvPool",

envpool/box2d/box2d_correctness_test.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 """Unit tests for box2d environments correctness check."""
 
-from typing import Any, Dict, Tuple, no_type_check
+from typing import Any, Dict, List, Tuple, no_type_check
 
+# import cv2
 import gym
 import numpy as np
 import pygame
@@ -53,6 +54,11 @@ def test_lunar_lander_space(self) -> None:
     env1 = make_gym("LunarLanderContinuous-v2")
     self.run_space_check(env0, env1)
 
+  def test_car_racing_space(self) -> None:
+    env0 = gym.make("CarRacing-v2")
+    env1 = make_gym("CarRacing-v2")
+    self.run_space_check(env0, env1)
+
   @staticmethod
   def heuristic_lunar_lander_policy(
     s: np.ndarray, continuous: bool
@@ -114,6 +120,43 @@ def test_lunar_lander_correctness(self, num_envs: int = 30) -> None:
     self.solve_lunar_lander(num_envs, True)
     self.solve_lunar_lander(num_envs, False)
 
+  def solve_car_racing(
+    self, num_envs: int, action: List[float], target_reward: float
+  ) -> None:
+    env = make_gym("CarRacing-v2", num_envs=num_envs)
+    max_episode_steps = 100
+
+    env_id = np.arange(num_envs)
+    done = np.array([False] * num_envs)
+    obs = env.reset(env_id)
+    rewards = np.zeros(num_envs)
+    action = np.tile(action, (num_envs, 1))
+    for _ in range(max_episode_steps):
+      obs, rew, terminated, truncated, info = env.step(action, env_id)
+      env_id = info["env_id"]
+      rewards[env_id] += rew
+      # cv2.imwrite("/tmp/car_racing-{}.jpg".format(i), obs[0])
+      if np.all(done):
+        break
+      obs = obs[~done]
+      env_id = env_id[~done]
+    mean_reward = np.mean(rewards)
+    logging.info(f"{np.mean(rewards):.6f} ± {np.std(rewards):.6f}")
+
+    self.assertTrue(abs(target_reward - mean_reward) < 1, (mean_reward))
+
+  def test_car_racing_correctness(
+    self, num_envs: int = 100, render: bool = False
+  ) -> None:
+    if render:
+      pygame.init()
+      pygame.display.init()
+      self.screen = pygame.display.set_mode((600, 400))
+      self.clock = pygame.time.Clock()
+    self.solve_car_racing(num_envs, [0, 0.5, 0], 65)
+    self.solve_car_racing(num_envs, [0.1, 0.3, 0], 18.5)
+    self.solve_car_racing(num_envs, [0, 0.7, 0.1], 42.7)
+
   @staticmethod
   @no_type_check
   def heuristic_bipedal_walker_policy(

envpool/box2d/box2d_deterministic_test.py

@@ -36,12 +36,16 @@ def run_deterministic_check(
     act_space = env0.action_space
     for _ in range(5000):
       action = np.array([act_space.sample() for _ in range(num_envs)])
-      obs0 = env0.step(action)[0]
-      obs1 = env1.step(action)[0]
-      obs2 = env2.step(action)[0]
+      obs0, rew0, terminated, truncated, info0 = env0.step(action)
+      obs1, rew1, terminated, truncated, info1 = env1.step(action)
+      obs2, rew2, terminated, truncated, info2 = env2.step(action)
       np.testing.assert_allclose(obs0, obs1)
       self.assertFalse(np.allclose(obs0, obs2))
 
+  def test_car_racing(self) -> None:
+    self.run_deterministic_check("CarRacing-v2")
+    self.run_deterministic_check("CarRacing-v2", max_episode_steps=3)
+
   def test_bipedal_walker(self) -> None:
     self.run_deterministic_check("BipedalWalker-v3")
     self.run_deterministic_check("BipedalWalkerHardcore-v3")

envpool/box2d/box2d_envpool.cc

@@ -13,10 +13,14 @@
 // limitations under the License.
 
 #include "envpool/box2d/bipedal_walker.h"
+#include "envpool/box2d/car_racing.h"
 #include "envpool/box2d/lunar_lander_continuous.h"
 #include "envpool/box2d/lunar_lander_discrete.h"
 #include "envpool/core/py_envpool.h"
 
+using CarRacingEnvSpec = PyEnvSpec<box2d::CarRacingEnvSpec>;
+using CarRacingEnvPool = PyEnvPool<box2d::CarRacingEnvPool>;
+
 using BipedalWalkerEnvSpec = PyEnvSpec<box2d::BipedalWalkerEnvSpec>;
 using BipedalWalkerEnvPool = PyEnvPool<box2d::BipedalWalkerEnvPool>;
 
@@ -29,6 +33,7 @@ using LunarLanderDiscreteEnvSpec = PyEnvSpec<box2d::LunarLanderDiscreteEnvSpec>;
 using LunarLanderDiscreteEnvPool = PyEnvPool<box2d::LunarLanderDiscreteEnvPool>;
 
 PYBIND11_MODULE(box2d_envpool, m) {
+  REGISTER(m, CarRacingEnvSpec, CarRacingEnvPool)
   REGISTER(m, BipedalWalkerEnvSpec, BipedalWalkerEnvPool)
   REGISTER(m, LunarLanderContinuousEnvSpec, LunarLanderContinuousEnvPool)
   REGISTER(m, LunarLanderDiscreteEnvSpec, LunarLanderDiscreteEnvPool)