StreetFighter-NoDelta.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Street Fighter No Delta\n",
    "This notebook shows how to create the environment without the delta transformation. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup StreetFighter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install gym gym-retro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import retro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "retro.data.list_games()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m retro.import ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = retro.make(game='StreetFighterIISpecialChampionEdition-Genesis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "obs = env.reset()\n",
    "done = False\n",
    "for game in range(5):\n",
    "    while not done: \n",
    "        if done: \n",
    "            obs = env.reset()\n",
    "        env.render()\n",
    "        obs, reward, done, info = env.step(env.action_space.sample())\n",
    "        print(reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://wowroms.com/en/roms/sega-genesis-megadrive/download-street-fighter-ii-special-champion-edition-europe/26496.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.action_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "env.observation_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "info['enemy_health']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "info['health']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gym import Env\n",
    "from gym.spaces import Box, MultiBinary\n",
    "import numpy as np\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StreetFighter(Env):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.observation_space = Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8)\n",
    "        self.action_space = MultiBinary(12)\n",
    "        self.game = retro.make(game='StreetFighterIISpecialChampionEdition-Genesis', use_restricted_actions=retro.Actions.FILTERED)\n",
    "        #self.score = 0\n",
    "    \n",
    "    def step(self, action):\n",
    "        obs, reward, done, info = self.game.step(action)\n",
    "        obs = self.preprocess(obs)\n",
    "        \n",
    "        # Preprocess frame from game\n",
    "        frame_delta = obs \n",
    "#         - self.previous_frame\n",
    "#         self.previous_frame = obs \n",
    "        \n",
    "        # Shape reward\n",
    "        reward = info['score'] - self.score \n",
    "        self.score = info['score']\n",
    "\n",
    "        return frame_delta, reward, done, info \n",
    "    \n",
    "    def render(self, *args, **kwargs): \n",
    "        self.game.render()\n",
    "    \n",
    "    def reset(self):\n",
    "        self.previous_frame = np.zeros(self.game.observation_space.shape)\n",
    "        \n",
    "        # Frame delta\n",
    "        obs = self.game.reset()\n",
    "        obs = self.preprocess(obs)\n",
    "        self.previous_frame = obs\n",
    "        \n",
    "        # Create initial variables\n",
    "        self.score = 0\n",
    "\n",
    "        return obs\n",
    "    \n",
    "    def preprocess(self, observation): \n",
    "        gray = cv2.cvtColor(observation, cv2.COLOR_BGR2GRAY)\n",
    "        resize = cv2.resize(gray, (84,84), interpolation=cv2.INTER_CUBIC)\n",
    "        state = np.reshape(resize, (84,84,1))\n",
    "        return state\n",
    "    \n",
    "    def close(self): \n",
    "        self.game.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "env = StreetFighter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.observation_space.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio===0.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install stable-baselines3[extra] optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import optuna for HPO\n",
    "import optuna\n",
    "# Import PPO for algos\n",
    "from stable_baselines3 import PPO\n",
    "# Evaluate Policy\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "# Import wrappers\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv, VecFrameStack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOG_DIR = './logs/'\n",
    "OPT_DIR = './opt_nodelta/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #https://github.com/araffin/rl-baselines-zoo/issues/29\n",
    "def optimize_ppo(trial):\n",
    "    \"\"\" Learning hyperparamters we want to optimise\"\"\"\n",
    "    return {\n",
    "        'n_steps': trial.suggest_int('n_steps', 2048, 8192),\n",
    "        'gamma': trial.suggest_loguniform('gamma', 0.8, 0.9999),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 1e-5, 1e-4),\n",
    "        'clip_range': trial.suggest_uniform('clip_range', 0.1, 0.4),\n",
    "        'gae_lambda': trial.suggest_uniform('gae_lambda', 0.8, .99)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_agent(trial):\n",
    "    try:\n",
    "        model_params = optimize_ppo(trial)\n",
    "        env = StreetFighter()\n",
    "        env = Monitor(env, LOG_DIR)\n",
    "        env = DummyVecEnv([lambda: env])\n",
    "        env = VecFrameStack(env, 4, channels_order='last')\n",
    "        model = PPO('CnnPolicy', env, tensorboard_log=LOG_DIR, verbose=0, **model_params)\n",
    "        model.learn(total_timesteps=100000)\n",
    "        mean_reward, _ = evaluate_policy(model, env, n_eval_episodes=20)\n",
    "        env.close()\n",
    "\n",
    "        SAVE_PATH = os.path.join(OPT_DIR, 'trial_{}_best_model'.format(trial.number))\n",
    "        model.save(SAVE_PATH)\n",
    "        return mean_reward\n",
    "    except Exception as e: \n",
    "        return -1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "study = optuna.create_study(direction='maximize')\n",
    "study.optimize(optimize_agent, n_trials=100, n_jobs=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Setup Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os for file path management\n",
    "import os \n",
    "# Import Base Callback for saving models\n",
    "from stable_baselines3.common.callbacks import BaseCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainAndLoggingCallback(BaseCallback):\n",
    "\n",
    "    def __init__(self, check_freq, save_path, verbose=1):\n",
    "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def _init_callback(self):\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self):\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
    "            self.model.save(model_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHECKPOINT_DIR = './train_nodelta/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = TrainAndLoggingCallback(check_freq=10000, save_path=CHECKPOINT_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = StreetFighter()\n",
    "env = Monitor(env, LOG_DIR)\n",
    "env = DummyVecEnv([lambda: env])\n",
    "env = VecFrameStack(env, 4, channels_order='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_params = {'n_steps': 2570.949, 'gamma': 0.906, 'learning_rate': 2e-07, 'clip_range': 0.369, 'gae_lambda': 0.891}\n",
    "#model_params = {'n_steps': 8960, 'gamma': 0.906, 'learning_rate': 2e-03, 'clip_range': 0.369, 'gae_lambda': 0.891}\n",
    "# model_params = study.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_params['n_steps'] = 40*64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO('CnnPolicy', env, tensorboard_log=LOG_DIR, verbose=1, **model_params)\n",
    "#model.load('./train_nodelta_backup/best_model_5460000.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.learn(total_timesteps=10000000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO.load('./train/best_model_90000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = StreetFighter()\n",
    "env = Monitor(env, LOG_DIR)\n",
    "env = DummyVecEnv([lambda: env])\n",
    "env = VecFrameStack(env, 4, channels_order='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "mean_reward, _ = evaluate_policy(model, env, n_eval_episodes=10, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mean_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test out the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for episode in range(1): \n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done: \n",
    "        action, _ = model.predict(obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        env.render()\n",
    "        time.sleep(0.01)\n",
    "        total_reward += reward\n",
    "    print('Total Reward for episode {} is {}'.format(total_reward, episode))\n",
    "    time.sleep(2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sonic",
   "language": "python",
   "name": "sonic"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}