From b2a4cbb98e20081412cb4cc7ae3d4afc456a732a Mon Sep 17 00:00:00 2001 From: reginald-mclean <72816837+reginald-mclean@users.noreply.github.com> Date: Fri, 16 Jun 2023 05:16:12 -0400 Subject: [PATCH] Updated to Fix CI (#411) --- .../envs/mujoco/sawyer_xyz/sawyer_xyz_env.py | 2 +- setup.cfg | 2 +- tests/integration/test_new_api.py | 6 - .../sawyer_xyz/test_scripted_policies.py | 105 +----------------- 4 files changed, 4 insertions(+), 111 deletions(-) diff --git a/metaworld/envs/mujoco/sawyer_xyz/sawyer_xyz_env.py b/metaworld/envs/mujoco/sawyer_xyz/sawyer_xyz_env.py index 7e13d9854..a3dcb446f 100644 --- a/metaworld/envs/mujoco/sawyer_xyz/sawyer_xyz_env.py +++ b/metaworld/envs/mujoco/sawyer_xyz/sawyer_xyz_env.py @@ -111,7 +111,7 @@ def __init__( self.mocap_low = np.hstack(mocap_low) self.mocap_high = np.hstack(mocap_high) self.curr_path_length = 0 - self.seeded_rand_vec = True + self.seeded_rand_vec = False self._freeze_rand_vec = True self._last_rand_vec = None diff --git a/setup.cfg b/setup.cfg index 9085b6b9e..80bfbc421 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [tool:pytest] -addopts = -n auto -s +adopts = -n auto -s markers = large: marks tests as large (deselect with '-m "not large"') skip_on_ci: marks tests that should be skipped on the CI diff --git a/tests/integration/test_new_api.py b/tests/integration/test_new_api.py index cc7a6eba2..3bd74237b 100644 --- a/tests/integration/test_new_api.py +++ b/tests/integration/test_new_api.py @@ -22,7 +22,6 @@ def test_all_ml1(env_name): env = train_env_instances[task.env_name] env.set_task(task) env.reset() - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) @@ -40,7 +39,6 @@ def test_all_ml1(env_name): env = test_env_instances[task.env_name] env.set_task(task) env.reset() - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) @@ -69,7 +67,6 @@ def test_all_ml10(): env = train_env_instances[task.env_name] env.set_task(task) env.reset() - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) @@ -88,7 +85,6 @@ def test_all_ml10(): env = test_env_instances[task.env_name] env.set_task(task) env.reset() - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) @@ -118,7 +114,6 @@ def test_all_ml45(): env = train_env_instances[task.env_name] env.set_task(task) obs = env.reset() - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) @@ -197,7 +192,6 @@ def test_all_mt50(): obs = env.reset() assert np.any(obs[-3:] != np.array([0,0,0])) assert env.observation_space.shape == (39,) - assert env.random_init == True old_obj_init = env.obj_init_pos old_target_pos = env._target_pos step_env(env, max_path_length=STEPS, render=False) diff --git a/tests/metaworld/envs/mujoco/sawyer_xyz/test_scripted_policies.py b/tests/metaworld/envs/mujoco/sawyer_xyz/test_scripted_policies.py index 5a5b04569..ffda3e409 100644 --- a/tests/metaworld/envs/mujoco/sawyer_xyz/test_scripted_policies.py +++ b/tests/metaworld/envs/mujoco/sawyer_xyz/test_scripted_policies.py @@ -1,120 +1,60 @@ import pytest -from metaworld.envs.mujoco.env_dict import ALL_V1_ENVIRONMENTS, ALL_V2_ENVIRONMENTS +from metaworld.envs.mujoco.env_dict import ALL_V2_ENVIRONMENTS from metaworld.policies import * from tests.metaworld.envs.mujoco.sawyer_xyz.utils import trajectory_summary -test_cases_old_nonoise = [ - # This should contain configs where a V2 policy is running in a V1 env. - # name, policy, action noise pct, success rate - ['bin-picking-v1', SawyerBinPickingV2Policy(), .0, .50], - ['handle-press-side-v1', SawyerHandlePressSideV2Policy(), .0, .05], - ['lever-pull-v1', SawyerLeverPullV2Policy(), .0, .0], - ['peg-insert-side-v1', SawyerPegInsertionSideV2Policy(), .0, .0], - ['plate-slide-back-side-v1', SawyerPlateSlideBackSideV2Policy(), .0, 1.], - ['window-open-v1', SawyerWindowOpenV2Policy(), .0, 0.85], - ['window-close-v1', SawyerWindowCloseV2Policy(), .0, 0.37], -] - -test_cases_old_noisy = [ - # This should contain configs where a V2 policy is running in a V1 env. - # name, policy, action noise pct, success rate - ['bin-picking-v1', SawyerBinPickingV2Policy(), .1, .40], - ['handle-press-side-v1', SawyerHandlePressSideV2Policy(), .1, .77], - ['lever-pull-v1', SawyerLeverPullV2Policy(), .1, .0], - ['peg-insert-side-v1', SawyerPegInsertionSideV2Policy(), .1, .0], - ['plate-slide-back-side-v1', SawyerPlateSlideBackSideV2Policy(), .1, 0.30], - ['window-open-v1', SawyerWindowOpenV2Policy(), .1, 0.81], - ['window-close-v1', SawyerWindowCloseV2Policy(), .1, 0.37], -] test_cases_latest_nonoise = [ # name, policy, action noise pct, success rate - ['assembly-v1', SawyerAssemblyV1Policy(), .0, 1.], ['assembly-v2', SawyerAssemblyV2Policy(), .0, 1.], - ['basketball-v1', SawyerBasketballV1Policy(), .0, .98], ['basketball-v2', SawyerBasketballV2Policy(), .0, .98], ['bin-picking-v2', SawyerBinPickingV2Policy(), .0, .98], - ['box-close-v1', SawyerBoxCloseV1Policy(), .0, .85], ['box-close-v2', SawyerBoxCloseV2Policy(), .0, .90], - ['button-press-topdown-v1', SawyerButtonPressTopdownV1Policy(), .0, 1.], ['button-press-topdown-v2', SawyerButtonPressTopdownV2Policy(), .0, .95], - ['button-press-topdown-wall-v1', SawyerButtonPressTopdownWallV1Policy(), .0, 1.], ['button-press-topdown-wall-v2', SawyerButtonPressTopdownWallV2Policy(), .0, .95], - ['button-press-v1', SawyerButtonPressV1Policy(), .0, 1.], ['button-press-v2', SawyerButtonPressV2Policy(), .0, 1.], - ['button-press-wall-v1', SawyerButtonPressWallV1Policy(), .0, 1.], ['button-press-wall-v2', SawyerButtonPressWallV2Policy(), .0, .93], - ['coffee-button-v1', SawyerCoffeeButtonV1Policy(), .0, 1.], ['coffee-button-v2', SawyerCoffeeButtonV2Policy(), .0, 1.], - ['coffee-pull-v1', SawyerCoffeePullV1Policy(), .0, .96], ['coffee-pull-v2', SawyerCoffeePullV2Policy(), .0, .94], - ['coffee-push-v1', SawyerCoffeePushV1Policy(), .0, .93], ['coffee-push-v2', SawyerCoffeePushV2Policy(), .0, .93], - ['dial-turn-v1', SawyerDialTurnV1Policy(), .0, 0.96], ['dial-turn-v2', SawyerDialTurnV2Policy(), .0, 0.96], - ['disassemble-v1', SawyerDisassembleV1Policy(), .0, .96], ['disassemble-v2', SawyerDisassembleV2Policy(), .0, .92], - ['door-close-v1', SawyerDoorCloseV1Policy(), .0, .99], ['door-close-v2', SawyerDoorCloseV2Policy(), .0, .99], - ['door-lock-v1', SawyerDoorLockV1Policy(), .0, 1.], ['door-lock-v2', SawyerDoorLockV2Policy(), .0, 1.], - ['door-open-v1', SawyerDoorOpenV1Policy(), .0, .98], ['door-open-v2', SawyerDoorOpenV2Policy(), .0, .94], - ['door-unlock-v1', SawyerDoorUnlockV1Policy(), .0, 1.], ['door-unlock-v2', SawyerDoorUnlockV2Policy(), .0, 1.], - ['drawer-close-v1', SawyerDrawerCloseV1Policy(), .0, .99], ['drawer-close-v2', SawyerDrawerCloseV2Policy(), .0, .99], - ['drawer-open-v1', SawyerDrawerOpenV1Policy(), .0, .99], ['drawer-open-v2', SawyerDrawerOpenV2Policy(), .0, .99], - ['faucet-close-v1', SawyerFaucetCloseV1Policy(), .0, 1.], ['faucet-close-v2', SawyerFaucetCloseV2Policy(), .0, 1.], - ['faucet-open-v1', SawyerFaucetOpenV1Policy(), .0, 1.], ['faucet-open-v2', SawyerFaucetOpenV2Policy(), .0, 1.], - ['hammer-v1', SawyerHammerV1Policy(), .0, 1.], ['hammer-v2', SawyerHammerV2Policy(), .0, 1.], - ['hand-insert-v1', SawyerHandInsertV1Policy(), .0, 0.96], ['hand-insert-v2', SawyerHandInsertV2Policy(), .0, 0.96], ['handle-press-side-v2', SawyerHandlePressSideV2Policy(), .0, .99], - ['handle-press-v1', SawyerHandlePressV1Policy(), .0, 1.], ['handle-press-v2', SawyerHandlePressV2Policy(), .0, 1.], - ['handle-pull-v1', SawyerHandlePullV1Policy(), .0, 1.], ['handle-pull-v2', SawyerHandlePullV2Policy(), .0, 0.93], - ['handle-pull-side-v1', SawyerHandlePullSideV1Policy(), .0, .92], ['handle-pull-side-v2', SawyerHandlePullSideV2Policy(), .0, 1.], ['peg-insert-side-v2', SawyerPegInsertionSideV2Policy(), .0, .89], ['lever-pull-v2', SawyerLeverPullV2Policy(), .0, .94], - ['peg-unplug-side-v1', SawyerPegUnplugSideV1Policy(), .0, .99], ['peg-unplug-side-v2', SawyerPegUnplugSideV2Policy(), .0, .99], - ['pick-out-of-hole-v1', SawyerPickOutOfHoleV1Policy(), .0, 1.], ['pick-out-of-hole-v2', SawyerPickOutOfHoleV2Policy(), .0, 1.], ['pick-place-v2', SawyerPickPlaceV2Policy(), .0, .95], ['pick-place-wall-v2', SawyerPickPlaceWallV2Policy(), .0, .95], ['plate-slide-back-side-v2', SawyerPlateSlideBackSideV2Policy(), .0, 1.], - ['plate-slide-back-v1', SawyerPlateSlideBackV1Policy(), .0, 1.], ['plate-slide-back-v2', SawyerPlateSlideBackV2Policy(), .0, 1.], - ['plate-slide-side-v1', SawyerPlateSlideSideV1Policy(), .0, 1.], ['plate-slide-side-v2', SawyerPlateSlideSideV2Policy(), .0, 1.], - ['plate-slide-v1', SawyerPlateSlideV1Policy(), .0, 1.], ['plate-slide-v2', SawyerPlateSlideV2Policy(), .0, 1.], ['reach-v2', SawyerReachV2Policy(), .0, .99], ['reach-wall-v2', SawyerReachWallV2Policy(), 0.0, .98], - ['push-back-v1', SawyerPushBackV1Policy(), .0, .97], ['push-back-v2', SawyerPushBackV2Policy(), .0, .97], ['push-v2', SawyerPushV2Policy(), .0, .97], ['push-wall-v2', SawyerPushWallV2Policy(), .0, .97], - ['shelf-place-v1', SawyerShelfPlaceV1Policy(), .0, .96], ['shelf-place-v2', SawyerShelfPlaceV2Policy(), .0, .96], - ['soccer-v1', SawyerSoccerV1Policy(), .0, .88], ['soccer-v2', SawyerSoccerV2Policy(), .0, .88], - ['stick-pull-v1', SawyerStickPullV1Policy(), .0, 0.95], ['stick-pull-v2', SawyerStickPullV2Policy(), .0, 0.96], - ['stick-push-v1', SawyerStickPushV1Policy(), .0, 0.98], ['stick-push-v2', SawyerStickPushV2Policy(), .0, 0.98], - ['sweep-into-v1', SawyerSweepIntoV1Policy(), .0, 1.], ['sweep-into-v2', SawyerSweepIntoV2Policy(), .0, 0.98], - ['sweep-v1', SawyerSweepV1Policy(), .0, 1.], ['sweep-v2', SawyerSweepV2Policy(), .0, 0.99], ['window-close-v2', SawyerWindowCloseV2Policy(), 0., .98], ['window-open-v2', SawyerWindowOpenV2Policy(), 0., .94], @@ -122,90 +62,53 @@ test_cases_latest_noisy = [ # name, policy, action noise pct, success rate - ['assembly-v1', SawyerAssemblyV1Policy(), .1, .69], ['assembly-v2', SawyerAssemblyV2Policy(), .1, .70], - ['basketball-v1', SawyerBasketballV1Policy(), .1, .97], ['basketball-v2', SawyerBasketballV2Policy(), .1, .96], ['bin-picking-v2', SawyerBinPickingV2Policy(), .1, .96], - ['box-close-v1', SawyerBoxCloseV1Policy(), .1, .84], ['box-close-v2', SawyerBoxCloseV2Policy(), .1, .82], - ['button-press-topdown-v1', SawyerButtonPressTopdownV1Policy(), .1, .98], ['button-press-topdown-v2', SawyerButtonPressTopdownV2Policy(), .1, .93], - ['button-press-topdown-wall-v1', SawyerButtonPressTopdownWallV1Policy(), .1, .99], ['button-press-topdown-wall-v2', SawyerButtonPressTopdownWallV2Policy(), .1, .95], - ['button-press-v1', SawyerButtonPressV1Policy(), .1, .98], ['button-press-v2', SawyerButtonPressV2Policy(), .1, .98], - ['button-press-wall-v1', SawyerButtonPressWallV1Policy(), .1, .94], ['button-press-wall-v2', SawyerButtonPressWallV2Policy(), .1, .92], - ['coffee-button-v1', SawyerCoffeeButtonV1Policy(), .1, .99], ['coffee-button-v2', SawyerCoffeeButtonV2Policy(), .1, .99], - ['coffee-pull-v1', SawyerCoffeePullV1Policy(), .1, .95], ['coffee-pull-v2', SawyerCoffeePullV2Policy(), .1, .82], - ['coffee-push-v1', SawyerCoffeePushV1Policy(), .1, .86], ['coffee-push-v2', SawyerCoffeePushV2Policy(), .1, .88], - ['dial-turn-v1', SawyerDialTurnV1Policy(), .1, 0.84], ['dial-turn-v2', SawyerDialTurnV2Policy(), .1, 0.84], - ['disassemble-v1', SawyerDisassembleV1Policy(), .1, .91], ['disassemble-v2', SawyerDisassembleV2Policy(), .1, .88], - ['door-close-v1', SawyerDoorCloseV1Policy(), .1, .99], ['door-close-v2', SawyerDoorCloseV2Policy(), .1, .97], - ['door-lock-v1', SawyerDoorLockV1Policy(), .1, 1.], ['door-lock-v2', SawyerDoorLockV2Policy(), .1, .96], - ['door-open-v1', SawyerDoorOpenV1Policy(), .1, .93], ['door-open-v2', SawyerDoorOpenV2Policy(), .1, .92], - ['door-unlock-v1', SawyerDoorUnlockV1Policy(), .1, .96], ['door-unlock-v2', SawyerDoorUnlockV2Policy(), .1, .97], - ['drawer-close-v1', SawyerDrawerCloseV1Policy(), .1, .64], ['drawer-close-v2', SawyerDrawerCloseV2Policy(), .1, .99], - ['drawer-open-v1', SawyerDrawerOpenV1Policy(), .1, .97], ['drawer-open-v2', SawyerDrawerOpenV2Policy(), .1, .97], - ['faucet-close-v1', SawyerFaucetCloseV1Policy(), .1, .93], ['faucet-close-v2', SawyerFaucetCloseV2Policy(), .1, 1.], - ['faucet-open-v1', SawyerFaucetOpenV1Policy(), .1, .99], ['faucet-open-v2', SawyerFaucetOpenV2Policy(), .1, .99], - ['hammer-v1', SawyerHammerV1Policy(), .1, .97], ['hammer-v2', SawyerHammerV2Policy(), .1, .96], - ['hand-insert-v1', SawyerHandInsertV1Policy(), .1, 0.95], ['hand-insert-v2', SawyerHandInsertV2Policy(), .1, 0.86], ['handle-press-side-v2', SawyerHandlePressSideV2Policy(), .1, .98], - ['handle-press-v1', SawyerHandlePressV1Policy(), .1, 1.], ['handle-press-v2', SawyerHandlePressV2Policy(), .1, 1.], - ['handle-pull-v1', SawyerHandlePullV1Policy(), .1, 1.], ['handle-pull-v2', SawyerHandlePullV2Policy(), .1, .99], - ['handle-pull-side-v1', SawyerHandlePullSideV1Policy(), .1, .75], ['handle-pull-side-v2', SawyerHandlePullSideV2Policy(), .1, .71], ['peg-insert-side-v2', SawyerPegInsertionSideV2Policy(), .1, .87], ['lever-pull-v2', SawyerLeverPullV2Policy(), .1, .90], - ['peg-unplug-side-v1', SawyerPegUnplugSideV1Policy(), .1, .97], ['peg-unplug-side-v2', SawyerPegUnplugSideV2Policy(), .1, .80], - ['pick-out-of-hole-v1', SawyerPickOutOfHoleV1Policy(), .1, .87], ['pick-out-of-hole-v2', SawyerPickOutOfHoleV2Policy(), .1, .89], ['pick-place-v2', SawyerPickPlaceV2Policy(), .1, .83], ['pick-place-wall-v2', SawyerPickPlaceWallV2Policy(), .1, .83], ['plate-slide-back-side-v2', SawyerPlateSlideBackSideV2Policy(), .1, .95], - ['plate-slide-back-v1', SawyerPlateSlideBackV1Policy(), .1, .95], ['plate-slide-back-v2', SawyerPlateSlideBackV2Policy(), .1, .94], - ['plate-slide-side-v1', SawyerPlateSlideSideV1Policy(), .1, .76], ['plate-slide-side-v2', SawyerPlateSlideSideV2Policy(), .1, .78], - ['plate-slide-v1', SawyerPlateSlideV1Policy(), .1, .97], ['plate-slide-v2', SawyerPlateSlideV2Policy(), .1, .97], ['reach-v2', SawyerReachV2Policy(), .1, .98], ['reach-wall-v2', SawyerReachWallV2Policy(), .1, .96], - ['push-back-v1', SawyerPushBackV1Policy(), .1, .90], ['push-back-v2', SawyerPushBackV2Policy(), .0, .91], ['push-v2', SawyerPushV2Policy(), .1, .88], ['push-wall-v2', SawyerPushWallV2Policy(), .1, .82], - ['shelf-place-v1', SawyerShelfPlaceV1Policy(), .1, .90], ['shelf-place-v2', SawyerShelfPlaceV2Policy(), .1, .89], - ['soccer-v1', SawyerSoccerV1Policy(), .1, .91], ['soccer-v2', SawyerSoccerV2Policy(), .1, .81], - ['stick-pull-v1', SawyerStickPullV1Policy(), .1, 0.81], ['stick-pull-v2', SawyerStickPullV2Policy(), .1, 0.81], - ['stick-push-v1', SawyerStickPushV1Policy(), .1, 0.95], ['stick-push-v2', SawyerStickPushV2Policy(), .1, 0.95], - ['sweep-into-v1', SawyerSweepIntoV1Policy(), .1, 1.], ['sweep-into-v2', SawyerSweepIntoV2Policy(), .1, 0.86], - ['sweep-v1', SawyerSweepV1Policy(), .1, 1.], ['sweep-v2', SawyerSweepV2Policy(), .0, 0.99], ['window-close-v2', SawyerWindowCloseV2Policy(), .1, .95], ['window-open-v2', SawyerWindowOpenV2Policy(), .1, .93], @@ -213,16 +116,12 @@ # Combine test cases into a single array to pass to parameterized test function test_cases = [] -for row in test_cases_old_nonoise: - test_cases.append(pytest.param(*row, marks=pytest.mark.skip)) -for row in test_cases_old_noisy: - test_cases.append(pytest.param(*row, marks=pytest.mark.skip)) for row in test_cases_latest_nonoise: test_cases.append(pytest.param(*row, marks=pytest.mark.skip)) for row in test_cases_latest_noisy: test_cases.append(pytest.param(*row, marks=pytest.mark.basic)) -ALL_ENVS = {**ALL_V1_ENVIRONMENTS, **ALL_V2_ENVIRONMENTS} +ALL_ENVS = {**ALL_V2_ENVIRONMENTS} @pytest.fixture(scope='function')