q_value_iteration.py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"FrozenLake-v0\")\n",
    "action_size = env.action_space.n\n",
    "state_size = env.observation_space.n\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "qtable = np.zeros((state_size, action_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_episodes = 15000        # Total episodes\n",
    "learning_rate = 0.9           # Learning rate\n",
    "max_steps = 99                # Max steps per episode\n",
    "gamma = 0.95                  # Discounting rate\n",
    "\n",
    "# Exploration parameters\n",
    "epsilon = 1.0                 # Exploration rate\n",
    "max_epsilon = 1.0             # Exploration probability at start\n",
    "min_epsilon = 0.01            # Minimum exploration probability \n",
    "decay_rate = 0.005             # Exponential decay rate for exploration prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score over time: 0.4537333333333333\n",
      "[[3.20410763e-02 3.99237632e-02 4.37483769e-02 1.53968480e-02]\n",
      " [3.96293519e-04 2.39118085e-03 9.42928575e-04 1.35052789e-01]\n",
      " [1.20440833e-02 3.31153789e-03 7.94426476e-04 1.71878384e-03]\n",
      " [5.15678496e-04 8.49639191e-06 2.11565787e-06 9.82719673e-02]\n",
      " [2.35967885e-02 1.21116720e-02 7.14412833e-03 3.10947143e-03]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]\n",
      " [5.07706829e-07 1.86223292e-07 3.48435675e-02 9.17803932e-09]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]\n",
      " [1.91844793e-03 4.19336991e-05 2.74240418e-03 3.15613922e-02]\n",
      " [7.53020464e-05 1.91783885e-03 1.36789479e-04 1.42874044e-06]\n",
      " [7.82813427e-04 7.65632809e-04 8.03738197e-01 4.24506057e-04]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]\n",
      " [5.26317599e-04 5.14235163e-03 3.13265672e-01 9.75554322e-03]\n",
      " [1.92695865e-01 9.93993632e-01 9.84313852e-03 1.84684738e-02]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "# List of rewards\n",
    "rewards = []\n",
    "\n",
    "# 2 For life or until learning is stopped\n",
    "for episode in range(total_episodes):\n",
    "    # Reset the environment\n",
    "    state = env.reset()\n",
    "    step = 0\n",
    "    done = False\n",
    "    total_rewards = 0\n",
    "    \n",
    "    for step in range(max_steps):\n",
    "        # 3. Choose an action a in the current world state (s)\n",
    "        ## First we randomize a number\n",
    "        exp_exp_tradeoff = random.uniform(0, 1)\n",
    "        \n",
    "        ## If this number > greater than epsilon --> exploitation (taking the biggest Q value for this state)\n",
    "        if exp_exp_tradeoff > epsilon:\n",
    "            action = np.argmax(qtable[state,:])\n",
    "\n",
    "        # Else doing a random choice --> exploration\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        # Take the action (a) and observe the outcome state(s') and reward (r)\n",
    "        new_state, reward, done, info = env.step(action)\n",
    "\n",
    "        # Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * max Q(s',a') - Q(s,a)]\n",
    "        # qtable[new_state,:] : all the actions we can take from new state\n",
    "        qtable[state, action] = qtable[state, action] + learning_rate * (reward + gamma * np.max(qtable[new_state, :]) - qtable[state, action])\n",
    "        \n",
    "        total_rewards += reward\n",
    "        \n",
    "        # Our new state is state\n",
    "        state = new_state\n",
    "        \n",
    "        # If done (if we're dead) : finish episode\n",
    "        if done == True: \n",
    "            break\n",
    "        \n",
    "    # Reduce epsilon (because we need less and less exploration)\n",
    "    epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*episode) \n",
    "    rewards.append(total_rewards)\n",
    "\n",
    "print (\"Score over time: \" +  str(sum(rewards)/total_episodes))\n",
    "print(qtable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************************\n",
      "EPISODE  0\n",
      "  (Down)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFF\u001b[41mG\u001b[0m\n",
      "Number of steps 15\n",
      "****************************************************\n",
      "EPISODE  1\n",
      "  (Right)\n",
      "SFFF\n",
      "FHF\u001b[41mH\u001b[0m\n",
      "FFFH\n",
      "HFFG\n",
      "Number of steps 4\n",
      "****************************************************\n",
      "EPISODE  2\n",
      "  (Right)\n",
      "SFFF\n",
      "FHF\u001b[41mH\u001b[0m\n",
      "FFFH\n",
      "HFFG\n",
      "Number of steps 9\n",
      "****************************************************\n",
      "EPISODE  3\n",
      "  (Down)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFF\u001b[41mG\u001b[0m\n",
      "Number of steps 15\n",
      "****************************************************\n",
      "EPISODE  4\n",
      "  (Right)\n",
      "SFFF\n",
      "FHF\u001b[41mH\u001b[0m\n",
      "FFFH\n",
      "HFFG\n",
      "Number of steps 4\n"
     ]
    }
   ],
   "source": [
    "env.reset()\n",
    "\n",
    "for episode in range(5):\n",
    "    state = env.reset()\n",
    "    step = 0\n",
    "    done = False\n",
    "    print(\"****************************************************\")\n",
    "    print(\"EPISODE \", episode)\n",
    "\n",
    "    for step in range(max_steps):\n",
    "        \n",
    "        # Take the action (index) that have the maximum expected future reward given that state\n",
    "        action = np.argmax(qtable[state,:])\n",
    "        \n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        \n",
    "        if done:\n",
    "            # Here, we decide to only print the last state (to see if our agent is on the goal or fall into an hole)\n",
    "            env.render()\n",
    "            \n",
    "            # We print the number of step it took.\n",
    "            print(\"Number of steps\", step)\n",
    "            break\n",
    "        state = new_state\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}