Tic-Toc-Tae AI ( Reinforcement Learning ).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "from builtins import range, input\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"LENGTH = 3\";\n",
       "                var nbb_formatted_code = \"LENGTH = 3\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "LENGTH = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"class Agent:\\n    def __init__(self, eps=0.1, alpha=0.5):\\n        self.eps = eps  # probability of choosing random action instead of greedy\\n        self.alpha = alpha  # learning rate\\n        self.verbose = False\\n        self.state_history = []\\n\\n    def setV(self, V):\\n        self.V = V\\n\\n    def set_symbol(self, sym):\\n        self.sym = sym\\n\\n    def set_verbose(self, v):\\n        # if true, will print values for each position on the board\\n        self.verbose = v\\n\\n    def reset_history(self):\\n        self.state_history = []\\n\\n    def take_action(self, env):\\n        # choose an action based on epsilon-greedy strategy\\n        r = np.random.rand()\\n        best_state = None\\n        if r < self.eps:\\n            # take a random action\\n            if self.verbose:\\n                print(\\\"Taking a random action\\\")\\n\\n            possible_moves = []\\n            for i in range(LENGTH):\\n                for j in range(LENGTH):\\n                    if env.is_empty(i, j):\\n                        possible_moves.append((i, j))\\n            idx = np.random.choice(len(possible_moves))\\n            next_move = possible_moves[idx]\\n        else:\\n            # choose the best action based on current values of states\\n            # loop through all possible moves, get their values\\n            # keep track of the best value\\n            pos2value = {}  # for debugging\\n            next_move = None\\n            best_value = -1\\n            for i in range(LENGTH):\\n                for j in range(LENGTH):\\n                    if env.is_empty(i, j):\\n                        # what is the state if we made this move?\\n                        env.board[i, j] = self.sym\\n                        state = env.get_state()\\n                        env.board[i, j] = 0  # don't forget to change it back!\\n                        pos2value[(i, j)] = self.V[state]\\n                        if self.V[state] > best_value:\\n                            best_value = self.V[state]\\n                            best_state = state\\n                            next_move = (i, j)\\n\\n            # if verbose, draw the board w/ the values\\n            if self.verbose:\\n                print(\\\"Taking a greedy action\\\")\\n                for i in range(LENGTH):\\n                    print(\\\"------------------\\\")\\n                    for j in range(LENGTH):\\n                        if env.is_empty(i, j):\\n                            # print the value\\n                            print(\\\" %.2f|\\\" % pos2value[(i, j)], end=\\\"\\\")\\n                        else:\\n                            print(\\\"  \\\", end=\\\"\\\")\\n                            if env.board[i, j] == env.x:\\n                                print(\\\"X  |\\\", end=\\\"\\\")\\n                            elif env.board[i, j] == env.o:\\n                                print(\\\"O  |\\\", end=\\\"\\\")\\n                            else:\\n                                print(\\\"   |\\\", end=\\\"\\\")\\n                    print(\\\"\\\")\\n                print(\\\"------------------\\\")\\n\\n        # make the move\\n        env.board[next_move[0], next_move[1]] = self.sym\\n\\n    def update_state_history(self, s):\\n        # cannot put this in take_action, because take_action only happens\\n        # once every other iteration for each player\\n        # state history needs to be updated every iteration\\n        # s = env.get_state() # don't want to do this twice so pass it in\\n        self.state_history.append(s)\\n\\n    def update(self, env):\\n        # we want to BACKTRACK over the states, so that:\\n        # V(prev_state) = V(prev_state) + alpha*(V(next_state) - V(prev_state))\\n        # where V(next_state) = reward if it's the most current state\\n        #\\n        # NOTE: we ONLY do this at the end of an episode\\n        # not so for all the algorithms we will study\\n        reward = env.reward(self.sym)\\n        target = reward\\n        for prev in reversed(self.state_history):\\n            value = self.V[prev] + self.alpha * (target - self.V[prev])\\n            self.V[prev] = value\\n            target = value\\n        self.reset_history()\";\n",
       "                var nbb_formatted_code = \"class Agent:\\n    def __init__(self, eps=0.1, alpha=0.5):\\n        self.eps = eps  # probability of choosing random action instead of greedy\\n        self.alpha = alpha  # learning rate\\n        self.verbose = False\\n        self.state_history = []\\n\\n    def setV(self, V):\\n        self.V = V\\n\\n    def set_symbol(self, sym):\\n        self.sym = sym\\n\\n    def set_verbose(self, v):\\n        # if true, will print values for each position on the board\\n        self.verbose = v\\n\\n    def reset_history(self):\\n        self.state_history = []\\n\\n    def take_action(self, env):\\n        # choose an action based on epsilon-greedy strategy\\n        r = np.random.rand()\\n        best_state = None\\n        if r < self.eps:\\n            # take a random action\\n            if self.verbose:\\n                print(\\\"Taking a random action\\\")\\n\\n            possible_moves = []\\n            for i in range(LENGTH):\\n                for j in range(LENGTH):\\n                    if env.is_empty(i, j):\\n                        possible_moves.append((i, j))\\n            idx = np.random.choice(len(possible_moves))\\n            next_move = possible_moves[idx]\\n        else:\\n            # choose the best action based on current values of states\\n            # loop through all possible moves, get their values\\n            # keep track of the best value\\n            pos2value = {}  # for debugging\\n            next_move = None\\n            best_value = -1\\n            for i in range(LENGTH):\\n                for j in range(LENGTH):\\n                    if env.is_empty(i, j):\\n                        # what is the state if we made this move?\\n                        env.board[i, j] = self.sym\\n                        state = env.get_state()\\n                        env.board[i, j] = 0  # don't forget to change it back!\\n                        pos2value[(i, j)] = self.V[state]\\n                        if self.V[state] > best_value:\\n                            best_value = self.V[state]\\n                            best_state = state\\n                            next_move = (i, j)\\n\\n            # if verbose, draw the board w/ the values\\n            if self.verbose:\\n                print(\\\"Taking a greedy action\\\")\\n                for i in range(LENGTH):\\n                    print(\\\"------------------\\\")\\n                    for j in range(LENGTH):\\n                        if env.is_empty(i, j):\\n                            # print the value\\n                            print(\\\" %.2f|\\\" % pos2value[(i, j)], end=\\\"\\\")\\n                        else:\\n                            print(\\\"  \\\", end=\\\"\\\")\\n                            if env.board[i, j] == env.x:\\n                                print(\\\"X  |\\\", end=\\\"\\\")\\n                            elif env.board[i, j] == env.o:\\n                                print(\\\"O  |\\\", end=\\\"\\\")\\n                            else:\\n                                print(\\\"   |\\\", end=\\\"\\\")\\n                    print(\\\"\\\")\\n                print(\\\"------------------\\\")\\n\\n        # make the move\\n        env.board[next_move[0], next_move[1]] = self.sym\\n\\n    def update_state_history(self, s):\\n        # cannot put this in take_action, because take_action only happens\\n        # once every other iteration for each player\\n        # state history needs to be updated every iteration\\n        # s = env.get_state() # don't want to do this twice so pass it in\\n        self.state_history.append(s)\\n\\n    def update(self, env):\\n        # we want to BACKTRACK over the states, so that:\\n        # V(prev_state) = V(prev_state) + alpha*(V(next_state) - V(prev_state))\\n        # where V(next_state) = reward if it's the most current state\\n        #\\n        # NOTE: we ONLY do this at the end of an episode\\n        # not so for all the algorithms we will study\\n        reward = env.reward(self.sym)\\n        target = reward\\n        for prev in reversed(self.state_history):\\n            value = self.V[prev] + self.alpha * (target - self.V[prev])\\n            self.V[prev] = value\\n            target = value\\n        self.reset_history()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class Agent:\n",
    "    def __init__(self, eps=0.1, alpha=0.5):\n",
    "        self.eps = eps  # probability of choosing random action instead of greedy\n",
    "        self.alpha = alpha  # learning rate\n",
    "        self.verbose = False\n",
    "        self.state_history = []\n",
    "\n",
    "    def setV(self, V):\n",
    "        self.V = V\n",
    "\n",
    "    def set_symbol(self, sym):\n",
    "        self.sym = sym\n",
    "\n",
    "    def set_verbose(self, v):\n",
    "        # if true, will print values for each position on the board\n",
    "        self.verbose = v\n",
    "\n",
    "    def reset_history(self):\n",
    "        self.state_history = []\n",
    "\n",
    "    def take_action(self, env):\n",
    "        # choose an action based on epsilon-greedy strategy\n",
    "        r = np.random.rand()\n",
    "        best_state = None\n",
    "        if r < self.eps:\n",
    "            # take a random action\n",
    "            if self.verbose:\n",
    "                print(\"Taking a random action\")\n",
    "\n",
    "            possible_moves = []\n",
    "            for i in range(LENGTH):\n",
    "                for j in range(LENGTH):\n",
    "                    if env.is_empty(i, j):\n",
    "                        possible_moves.append((i, j))\n",
    "            idx = np.random.choice(len(possible_moves))\n",
    "            next_move = possible_moves[idx]\n",
    "        else:\n",
    "            # choose the best action based on current values of states\n",
    "            # loop through all possible moves, get their values\n",
    "            # keep track of the best value\n",
    "            pos2value = {}  # for debugging\n",
    "            next_move = None\n",
    "            best_value = -1\n",
    "            for i in range(LENGTH):\n",
    "                for j in range(LENGTH):\n",
    "                    if env.is_empty(i, j):\n",
    "                        # what is the state if we made this move?\n",
    "                        env.board[i, j] = self.sym\n",
    "                        state = env.get_state()\n",
    "                        env.board[i, j] = 0  # don't forget to change it back!\n",
    "                        pos2value[(i, j)] = self.V[state]\n",
    "                        if self.V[state] > best_value:\n",
    "                            best_value = self.V[state]\n",
    "                            best_state = state\n",
    "                            next_move = (i, j)\n",
    "\n",
    "            # if verbose, draw the board w/ the values\n",
    "            if self.verbose:\n",
    "                print(\"Taking a greedy action\")\n",
    "                for i in range(LENGTH):\n",
    "                    print(\"------------------\")\n",
    "                    for j in range(LENGTH):\n",
    "                        if env.is_empty(i, j):\n",
    "                            # print the value\n",
    "                            print(\" %.2f|\" % pos2value[(i, j)], end=\"\")\n",
    "                        else:\n",
    "                            print(\"  \", end=\"\")\n",
    "                            if env.board[i, j] == env.x:\n",
    "                                print(\"X  |\", end=\"\")\n",
    "                            elif env.board[i, j] == env.o:\n",
    "                                print(\"O  |\", end=\"\")\n",
    "                            else:\n",
    "                                print(\"   |\", end=\"\")\n",
    "                    print(\"\")\n",
    "                print(\"------------------\")\n",
    "\n",
    "        # make the move\n",
    "        env.board[next_move[0], next_move[1]] = self.sym\n",
    "\n",
    "    def update_state_history(self, s):\n",
    "        # cannot put this in take_action, because take_action only happens\n",
    "        # once every other iteration for each player\n",
    "        # state history needs to be updated every iteration\n",
    "        # s = env.get_state() # don't want to do this twice so pass it in\n",
    "        self.state_history.append(s)\n",
    "\n",
    "    def update(self, env):\n",
    "        # we want to BACKTRACK over the states, so that:\n",
    "        # V(prev_state) = V(prev_state) + alpha*(V(next_state) - V(prev_state))\n",
    "        # where V(next_state) = reward if it's the most current state\n",
    "        #\n",
    "        # NOTE: we ONLY do this at the end of an episode\n",
    "        # not so for all the algorithms we will study\n",
    "        reward = env.reward(self.sym)\n",
    "        target = reward\n",
    "        for prev in reversed(self.state_history):\n",
    "            value = self.V[prev] + self.alpha * (target - self.V[prev])\n",
    "            self.V[prev] = value\n",
    "            target = value\n",
    "        self.reset_history()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"# this class represents a tic-tac-toe game\\n# is a CS101-type of project\\nclass Environment:\\n    def __init__(self):\\n        self.board = np.zeros((LENGTH, LENGTH))\\n        self.x = -1  # represents an x on the board, player 1\\n        self.o = 1  # represents an o on the board, player 2\\n        self.winner = None\\n        self.ended = False\\n        self.num_states = 3 ** (LENGTH * LENGTH)\\n\\n    def is_empty(self, i, j):\\n        return self.board[i, j] == 0\\n\\n    def reward(self, sym):\\n        # no reward until game is over\\n        if not self.game_over():\\n            return 0\\n\\n        # if we get here, game is over\\n        # sym will be self.x or self.o\\n        return 1 if self.winner == sym else 0\\n\\n    def get_state(self):\\n        # returns the current state, represented as an int\\n        # from 0...|S|-1, where S = set of all possible states\\n        # |S| = 3^(BOARD SIZE), since each cell can have 3 possible values - empty, x, o\\n        # some states are not possible, e.g. all cells are x, but we ignore that detail\\n        # this is like finding the integer represented by a base-3 number\\n        k = 0\\n        h = 0\\n        for i in range(LENGTH):\\n            for j in range(LENGTH):\\n                if self.board[i, j] == 0:\\n                    v = 0\\n                elif self.board[i, j] == self.x:\\n                    v = 1\\n                elif self.board[i, j] == self.o:\\n                    v = 2\\n                h += (3 ** k) * v\\n                k += 1\\n        return h\\n\\n    def game_over(self, force_recalculate=False):\\n        # returns true if game over (a player has won or it's a draw)\\n        # otherwise returns false\\n        # also sets 'winner' instance variable and 'ended' instance variable\\n        if not force_recalculate and self.ended:\\n            return self.ended\\n\\n        # check rows\\n        for i in range(LENGTH):\\n            for player in (self.x, self.o):\\n                if self.board[i].sum() == player * LENGTH:\\n                    self.winner = player\\n                    self.ended = True\\n                    return True\\n\\n        # check columns\\n        for j in range(LENGTH):\\n            for player in (self.x, self.o):\\n                if self.board[:, j].sum() == player * LENGTH:\\n                    self.winner = player\\n                    self.ended = True\\n                    return True\\n\\n        # check diagonals\\n        for player in (self.x, self.o):\\n            # top-left -> bottom-right diagonal\\n            if self.board.trace() == player * LENGTH:\\n                self.winner = player\\n                self.ended = True\\n                return True\\n            # top-right -> bottom-left diagonal\\n            if np.fliplr(self.board).trace() == player * LENGTH:\\n                self.winner = player\\n                self.ended = True\\n                return True\\n\\n        # check if draw\\n        if np.all((self.board == 0) == False):\\n            # winner stays None\\n            self.winner = None\\n            self.ended = True\\n            return True\\n\\n        # game is not over\\n        self.winner = None\\n        return False\\n\\n    def is_draw(self):\\n        return self.ended and self.winner is None\\n\\n    # Example board\\n    # -------------\\n    # | x |   |   |\\n    # -------------\\n    # |   |   |   |\\n    # -------------\\n    # |   |   | o |\\n    # -------------\\n    def draw_board(self):\\n        for i in range(LENGTH):\\n            print(\\\"-------------\\\")\\n            for j in range(LENGTH):\\n                print(\\\"  \\\", end=\\\"\\\")\\n                if self.board[i, j] == self.x:\\n                    print(\\\"X \\\", end=\\\"\\\")\\n                elif self.board[i, j] == self.o:\\n                    print(\\\"O \\\", end=\\\"\\\")\\n                else:\\n                    print(\\\"  \\\", end=\\\"\\\")\\n            print(\\\"\\\")\\n        print(\\\"-------------\\\")\";\n",
       "                var nbb_formatted_code = \"# this class represents a tic-tac-toe game\\n# is a CS101-type of project\\nclass Environment:\\n    def __init__(self):\\n        self.board = np.zeros((LENGTH, LENGTH))\\n        self.x = -1  # represents an x on the board, player 1\\n        self.o = 1  # represents an o on the board, player 2\\n        self.winner = None\\n        self.ended = False\\n        self.num_states = 3 ** (LENGTH * LENGTH)\\n\\n    def is_empty(self, i, j):\\n        return self.board[i, j] == 0\\n\\n    def reward(self, sym):\\n        # no reward until game is over\\n        if not self.game_over():\\n            return 0\\n\\n        # if we get here, game is over\\n        # sym will be self.x or self.o\\n        return 1 if self.winner == sym else 0\\n\\n    def get_state(self):\\n        # returns the current state, represented as an int\\n        # from 0...|S|-1, where S = set of all possible states\\n        # |S| = 3^(BOARD SIZE), since each cell can have 3 possible values - empty, x, o\\n        # some states are not possible, e.g. all cells are x, but we ignore that detail\\n        # this is like finding the integer represented by a base-3 number\\n        k = 0\\n        h = 0\\n        for i in range(LENGTH):\\n            for j in range(LENGTH):\\n                if self.board[i, j] == 0:\\n                    v = 0\\n                elif self.board[i, j] == self.x:\\n                    v = 1\\n                elif self.board[i, j] == self.o:\\n                    v = 2\\n                h += (3 ** k) * v\\n                k += 1\\n        return h\\n\\n    def game_over(self, force_recalculate=False):\\n        # returns true if game over (a player has won or it's a draw)\\n        # otherwise returns false\\n        # also sets 'winner' instance variable and 'ended' instance variable\\n        if not force_recalculate and self.ended:\\n            return self.ended\\n\\n        # check rows\\n        for i in range(LENGTH):\\n            for player in (self.x, self.o):\\n                if self.board[i].sum() == player * LENGTH:\\n                    self.winner = player\\n                    self.ended = True\\n                    return True\\n\\n        # check columns\\n        for j in range(LENGTH):\\n            for player in (self.x, self.o):\\n                if self.board[:, j].sum() == player * LENGTH:\\n                    self.winner = player\\n                    self.ended = True\\n                    return True\\n\\n        # check diagonals\\n        for player in (self.x, self.o):\\n            # top-left -> bottom-right diagonal\\n            if self.board.trace() == player * LENGTH:\\n                self.winner = player\\n                self.ended = True\\n                return True\\n            # top-right -> bottom-left diagonal\\n            if np.fliplr(self.board).trace() == player * LENGTH:\\n                self.winner = player\\n                self.ended = True\\n                return True\\n\\n        # check if draw\\n        if np.all((self.board == 0) == False):\\n            # winner stays None\\n            self.winner = None\\n            self.ended = True\\n            return True\\n\\n        # game is not over\\n        self.winner = None\\n        return False\\n\\n    def is_draw(self):\\n        return self.ended and self.winner is None\\n\\n    # Example board\\n    # -------------\\n    # | x |   |   |\\n    # -------------\\n    # |   |   |   |\\n    # -------------\\n    # |   |   | o |\\n    # -------------\\n    def draw_board(self):\\n        for i in range(LENGTH):\\n            print(\\\"-------------\\\")\\n            for j in range(LENGTH):\\n                print(\\\"  \\\", end=\\\"\\\")\\n                if self.board[i, j] == self.x:\\n                    print(\\\"X \\\", end=\\\"\\\")\\n                elif self.board[i, j] == self.o:\\n                    print(\\\"O \\\", end=\\\"\\\")\\n                else:\\n                    print(\\\"  \\\", end=\\\"\\\")\\n            print(\\\"\\\")\\n        print(\\\"-------------\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# this class represents a tic-tac-toe game\n",
    "# is a CS101-type of project\n",
    "class Environment:\n",
    "    def __init__(self):\n",
    "        self.board = np.zeros((LENGTH, LENGTH))\n",
    "        self.x = -1  # represents an x on the board, player 1\n",
    "        self.o = 1  # represents an o on the board, player 2\n",
    "        self.winner = None\n",
    "        self.ended = False\n",
    "        self.num_states = 3 ** (LENGTH * LENGTH)\n",
    "\n",
    "    def is_empty(self, i, j):\n",
    "        return self.board[i, j] == 0\n",
    "\n",
    "    def reward(self, sym):\n",
    "        # no reward until game is over\n",
    "        if not self.game_over():\n",
    "            return 0\n",
    "\n",
    "        # if we get here, game is over\n",
    "        # sym will be self.x or self.o\n",
    "        return 1 if self.winner == sym else 0\n",
    "\n",
    "    def get_state(self):\n",
    "        # returns the current state, represented as an int\n",
    "        # from 0...|S|-1, where S = set of all possible states\n",
    "        # |S| = 3^(BOARD SIZE), since each cell can have 3 possible values - empty, x, o\n",
    "        # some states are not possible, e.g. all cells are x, but we ignore that detail\n",
    "        # this is like finding the integer represented by a base-3 number\n",
    "        k = 0\n",
    "        h = 0\n",
    "        for i in range(LENGTH):\n",
    "            for j in range(LENGTH):\n",
    "                if self.board[i, j] == 0:\n",
    "                    v = 0\n",
    "                elif self.board[i, j] == self.x:\n",
    "                    v = 1\n",
    "                elif self.board[i, j] == self.o:\n",
    "                    v = 2\n",
    "                h += (3 ** k) * v\n",
    "                k += 1\n",
    "        return h\n",
    "\n",
    "    def game_over(self, force_recalculate=False):\n",
    "        # returns true if game over (a player has won or it's a draw)\n",
    "        # otherwise returns false\n",
    "        # also sets 'winner' instance variable and 'ended' instance variable\n",
    "        if not force_recalculate and self.ended:\n",
    "            return self.ended\n",
    "\n",
    "        # check rows\n",
    "        for i in range(LENGTH):\n",
    "            for player in (self.x, self.o):\n",
    "                if self.board[i].sum() == player * LENGTH:\n",
    "                    self.winner = player\n",
    "                    self.ended = True\n",
    "                    return True\n",
    "\n",
    "        # check columns\n",
    "        for j in range(LENGTH):\n",
    "            for player in (self.x, self.o):\n",
    "                if self.board[:, j].sum() == player * LENGTH:\n",
    "                    self.winner = player\n",
    "                    self.ended = True\n",
    "                    return True\n",
    "\n",
    "        # check diagonals\n",
    "        for player in (self.x, self.o):\n",
    "            # top-left -> bottom-right diagonal\n",
    "            if self.board.trace() == player * LENGTH:\n",
    "                self.winner = player\n",
    "                self.ended = True\n",
    "                return True\n",
    "            # top-right -> bottom-left diagonal\n",
    "            if np.fliplr(self.board).trace() == player * LENGTH:\n",
    "                self.winner = player\n",
    "                self.ended = True\n",
    "                return True\n",
    "\n",
    "        # check if draw\n",
    "        if np.all((self.board == 0) == False):\n",
    "            # winner stays None\n",
    "            self.winner = None\n",
    "            self.ended = True\n",
    "            return True\n",
    "\n",
    "        # game is not over\n",
    "        self.winner = None\n",
    "        return False\n",
    "\n",
    "    def is_draw(self):\n",
    "        return self.ended and self.winner is None\n",
    "\n",
    "    # Example board\n",
    "    # -------------\n",
    "    # | x |   |   |\n",
    "    # -------------\n",
    "    # |   |   |   |\n",
    "    # -------------\n",
    "    # |   |   | o |\n",
    "    # -------------\n",
    "    def draw_board(self):\n",
    "        for i in range(LENGTH):\n",
    "            print(\"-------------\")\n",
    "            for j in range(LENGTH):\n",
    "                print(\"  \", end=\"\")\n",
    "                if self.board[i, j] == self.x:\n",
    "                    print(\"X \", end=\"\")\n",
    "                elif self.board[i, j] == self.o:\n",
    "                    print(\"O \", end=\"\")\n",
    "                else:\n",
    "                    print(\"  \", end=\"\")\n",
    "            print(\"\")\n",
    "        print(\"-------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"class Human:\\n    def __init__(self):\\n        pass\\n\\n    def set_symbol(self, sym):\\n        self.sym = sym\\n\\n    def take_action(self, env):\\n        while True:\\n            # break if we make a legal move\\n            move = input(\\\"Enter coordinates i,j for your next move (i,j=0..2): \\\")\\n            i, j = move.split(',')\\n            i = int(i)\\n            j = int(j)\\n            if env.is_empty(i, j):\\n                env.board[i, j] = self.sym\\n                break\\n\\n    def update(self, env):\\n        pass\\n\\n    def update_state_history(self, s):\\n        pass\";\n",
       "                var nbb_formatted_code = \"class Human:\\n    def __init__(self):\\n        pass\\n\\n    def set_symbol(self, sym):\\n        self.sym = sym\\n\\n    def take_action(self, env):\\n        while True:\\n            # break if we make a legal move\\n            move = input(\\\"Enter coordinates i,j for your next move (i,j=0..2): \\\")\\n            i, j = move.split(\\\",\\\")\\n            i = int(i)\\n            j = int(j)\\n            if env.is_empty(i, j):\\n                env.board[i, j] = self.sym\\n                break\\n\\n    def update(self, env):\\n        pass\\n\\n    def update_state_history(self, s):\\n        pass\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class Human:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def set_symbol(self, sym):\n",
    "        self.sym = sym\n",
    "\n",
    "    def take_action(self, env):\n",
    "        while True:\n",
    "            # break if we make a legal move\n",
    "            move = input(\"Enter coordinates i,j for your next move (i,j=0..2): \")\n",
    "            i, j = move.split(\",\")\n",
    "            i = int(i)\n",
    "            j = int(j)\n",
    "            if env.is_empty(i, j):\n",
    "                env.board[i, j] = self.sym\n",
    "                break\n",
    "\n",
    "    def update(self, env):\n",
    "        pass\n",
    "\n",
    "    def update_state_history(self, s):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"# recursive function that will return all\\n# possible states (as ints) and who the corresponding winner is for those states (if any)\\n# (i, j) refers to the next cell on the board to permute (we need to try -1, 0, 1)\\n# impossible games are ignored, i.e. 3x's and 3o's in a row simultaneously\\n# since that will never happen in a real game\\ndef get_state_hash_and_winner(env, i=0, j=0):\\n    results = []\\n\\n    for v in (0, env.x, env.o):\\n        env.board[i, j] = v  # if empty board it should already be 0\\n        if j == 2:\\n            # j goes back to 0, increase i, unless i = 2, then we are done\\n            if i == 2:\\n                # the board is full, collect results and return\\n                state = env.get_state()\\n                ended = env.game_over(force_recalculate=True)\\n                winner = env.winner\\n                results.append((state, winner, ended))\\n            else:\\n                results += get_state_hash_and_winner(env, i + 1, 0)\\n        else:\\n            # increment j, i stays the same\\n            results += get_state_hash_and_winner(env, i, j + 1)\\n\\n    return results\";\n",
       "                var nbb_formatted_code = \"# recursive function that will return all\\n# possible states (as ints) and who the corresponding winner is for those states (if any)\\n# (i, j) refers to the next cell on the board to permute (we need to try -1, 0, 1)\\n# impossible games are ignored, i.e. 3x's and 3o's in a row simultaneously\\n# since that will never happen in a real game\\ndef get_state_hash_and_winner(env, i=0, j=0):\\n    results = []\\n\\n    for v in (0, env.x, env.o):\\n        env.board[i, j] = v  # if empty board it should already be 0\\n        if j == 2:\\n            # j goes back to 0, increase i, unless i = 2, then we are done\\n            if i == 2:\\n                # the board is full, collect results and return\\n                state = env.get_state()\\n                ended = env.game_over(force_recalculate=True)\\n                winner = env.winner\\n                results.append((state, winner, ended))\\n            else:\\n                results += get_state_hash_and_winner(env, i + 1, 0)\\n        else:\\n            # increment j, i stays the same\\n            results += get_state_hash_and_winner(env, i, j + 1)\\n\\n    return results\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# recursive function that will return all\n",
    "# possible states (as ints) and who the corresponding winner is for those states (if any)\n",
    "# (i, j) refers to the next cell on the board to permute (we need to try -1, 0, 1)\n",
    "# impossible games are ignored, i.e. 3x's and 3o's in a row simultaneously\n",
    "# since that will never happen in a real game\n",
    "def get_state_hash_and_winner(env, i=0, j=0):\n",
    "    results = []\n",
    "\n",
    "    for v in (0, env.x, env.o):\n",
    "        env.board[i, j] = v  # if empty board it should already be 0\n",
    "        if j == 2:\n",
    "            # j goes back to 0, increase i, unless i = 2, then we are done\n",
    "            if i == 2:\n",
    "                # the board is full, collect results and return\n",
    "                state = env.get_state()\n",
    "                ended = env.game_over(force_recalculate=True)\n",
    "                winner = env.winner\n",
    "                results.append((state, winner, ended))\n",
    "            else:\n",
    "                results += get_state_hash_and_winner(env, i + 1, 0)\n",
    "        else:\n",
    "            # increment j, i stays the same\n",
    "            results += get_state_hash_and_winner(env, i, j + 1)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"\\ndef initialV_x(env, state_winner_triples):\\n    # initialize state values as follows\\n    # if x wins, V(s) = 1\\n    # if x loses or draw, V(s) = 0\\n    # otherwise, V(s) = 0.5\\n    V = np.zeros(env.num_states)\\n    for state, winner, ended in state_winner_triples:\\n        if ended:\\n            if winner == env.x:\\n                v = 1\\n            else:\\n                v = 0\\n        else:\\n            v = 0.5\\n        V[state] = v\\n    return V\\n\\n\\ndef initialV_o(env, state_winner_triples):\\n    # this is (almost) the opposite of initial V for player x\\n    # since everywhere where x wins (1), o loses (0)\\n    # but a draw is still 0 for o\\n    V = np.zeros(env.num_states)\\n    for state, winner, ended in state_winner_triples:\\n        if ended:\\n            if winner == env.o:\\n                v = 1\\n            else:\\n                v = 0\\n        else:\\n            v = 0.5\\n        V[state] = v\\n    return V\";\n",
       "                var nbb_formatted_code = \"def initialV_x(env, state_winner_triples):\\n    # initialize state values as follows\\n    # if x wins, V(s) = 1\\n    # if x loses or draw, V(s) = 0\\n    # otherwise, V(s) = 0.5\\n    V = np.zeros(env.num_states)\\n    for state, winner, ended in state_winner_triples:\\n        if ended:\\n            if winner == env.x:\\n                v = 1\\n            else:\\n                v = 0\\n        else:\\n            v = 0.5\\n        V[state] = v\\n    return V\\n\\n\\ndef initialV_o(env, state_winner_triples):\\n    # this is (almost) the opposite of initial V for player x\\n    # since everywhere where x wins (1), o loses (0)\\n    # but a draw is still 0 for o\\n    V = np.zeros(env.num_states)\\n    for state, winner, ended in state_winner_triples:\\n        if ended:\\n            if winner == env.o:\\n                v = 1\\n            else:\\n                v = 0\\n        else:\\n            v = 0.5\\n        V[state] = v\\n    return V\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def initialV_x(env, state_winner_triples):\n",
    "    # initialize state values as follows\n",
    "    # if x wins, V(s) = 1\n",
    "    # if x loses or draw, V(s) = 0\n",
    "    # otherwise, V(s) = 0.5\n",
    "    V = np.zeros(env.num_states)\n",
    "    for state, winner, ended in state_winner_triples:\n",
    "        if ended:\n",
    "            if winner == env.x:\n",
    "                v = 1\n",
    "            else:\n",
    "                v = 0\n",
    "        else:\n",
    "            v = 0.5\n",
    "        V[state] = v\n",
    "    return V\n",
    "\n",
    "\n",
    "def initialV_o(env, state_winner_triples):\n",
    "    # this is (almost) the opposite of initial V for player x\n",
    "    # since everywhere where x wins (1), o loses (0)\n",
    "    # but a draw is still 0 for o\n",
    "    V = np.zeros(env.num_states)\n",
    "    for state, winner, ended in state_winner_triples:\n",
    "        if ended:\n",
    "            if winner == env.o:\n",
    "                v = 1\n",
    "            else:\n",
    "                v = 0\n",
    "        else:\n",
    "            v = 0.5\n",
    "        V[state] = v\n",
    "    return V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"def play_game(p1, p2, env, draw=False):\\n    # loops until the game is over\\n    current_player = None\\n    while not env.game_over():\\n        # alternate between players\\n        # p1 always starts first\\n        if current_player == p1:\\n            current_player = p2\\n        else:\\n            current_player = p1\\n\\n        # draw the board before the user who wants to see it makes a move\\n        if draw:\\n            if draw == 1 and current_player == p1:\\n                env.draw_board()\\n            if draw == 2 and current_player == p2:\\n                env.draw_board()\\n\\n        # current player makes a move\\n        current_player.take_action(env)\\n\\n        # update state histories\\n        state = env.get_state()\\n        p1.update_state_history(state)\\n        p2.update_state_history(state)\\n\\n    if draw:\\n        env.draw_board()\\n\\n    # do the value function update\\n    p1.update(env)\\n    p2.update(env)\";\n",
       "                var nbb_formatted_code = \"def play_game(p1, p2, env, draw=False):\\n    # loops until the game is over\\n    current_player = None\\n    while not env.game_over():\\n        # alternate between players\\n        # p1 always starts first\\n        if current_player == p1:\\n            current_player = p2\\n        else:\\n            current_player = p1\\n\\n        # draw the board before the user who wants to see it makes a move\\n        if draw:\\n            if draw == 1 and current_player == p1:\\n                env.draw_board()\\n            if draw == 2 and current_player == p2:\\n                env.draw_board()\\n\\n        # current player makes a move\\n        current_player.take_action(env)\\n\\n        # update state histories\\n        state = env.get_state()\\n        p1.update_state_history(state)\\n        p2.update_state_history(state)\\n\\n    if draw:\\n        env.draw_board()\\n\\n    # do the value function update\\n    p1.update(env)\\n    p2.update(env)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_game(p1, p2, env, draw=False):\n",
    "    # loops until the game is over\n",
    "    current_player = None\n",
    "    while not env.game_over():\n",
    "        # alternate between players\n",
    "        # p1 always starts first\n",
    "        if current_player == p1:\n",
    "            current_player = p2\n",
    "        else:\n",
    "            current_player = p1\n",
    "\n",
    "        # draw the board before the user who wants to see it makes a move\n",
    "        if draw:\n",
    "            if draw == 1 and current_player == p1:\n",
    "                env.draw_board()\n",
    "            if draw == 2 and current_player == p2:\n",
    "                env.draw_board()\n",
    "\n",
    "        # current player makes a move\n",
    "        current_player.take_action(env)\n",
    "\n",
    "        # update state histories\n",
    "        state = env.get_state()\n",
    "        p1.update_state_history(state)\n",
    "        p2.update_state_history(state)\n",
    "\n",
    "    if draw:\n",
    "        env.draw_board()\n",
    "\n",
    "    # do the value function update\n",
    "    p1.update(env)\n",
    "    p2.update(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "200\n",
      "400\n",
      "600\n",
      "800\n",
      "1000\n",
      "1200\n",
      "1400\n",
      "1600\n",
      "1800\n",
      "2000\n",
      "2200\n",
      "2400\n",
      "2600\n",
      "2800\n",
      "3000\n",
      "3200\n",
      "3400\n",
      "3600\n",
      "3800\n",
      "4000\n",
      "4200\n",
      "4400\n",
      "4600\n",
      "4800\n",
      "5000\n",
      "5200\n",
      "5400\n",
      "5600\n",
      "5800\n",
      "6000\n",
      "6200\n",
      "6400\n",
      "6600\n",
      "6800\n",
      "7000\n",
      "7200\n",
      "7400\n",
      "7600\n",
      "7800\n",
      "8000\n",
      "8200\n",
      "8400\n",
      "8600\n",
      "8800\n",
      "9000\n",
      "9200\n",
      "9400\n",
      "9600\n",
      "9800\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"if __name__ == '__main__':\\n    # train the agent\\n    p1 = Agent()\\n    p2 = Agent()\\n\\n    # set initial V for p1 and p2\\n    env = Environment()\\n    state_winner_triples = get_state_hash_and_winner(env)\\n\\n    Vx = initialV_x(env, state_winner_triples)\\n    p1.setV(Vx)\\n    Vo = initialV_o(env, state_winner_triples)\\n    p2.setV(Vo)\\n\\n    # give each player their symbol\\n    p1.set_symbol(env.x)\\n    p2.set_symbol(env.o)\\n\\n    T = 10000\\n    for t in range(T):\\n        if t % 200 == 0:\\n            print(t)\\n        play_game(p1, p2, Environment())\";\n",
       "                var nbb_formatted_code = \"if __name__ == \\\"__main__\\\":\\n    # train the agent\\n    p1 = Agent()\\n    p2 = Agent()\\n\\n    # set initial V for p1 and p2\\n    env = Environment()\\n    state_winner_triples = get_state_hash_and_winner(env)\\n\\n    Vx = initialV_x(env, state_winner_triples)\\n    p1.setV(Vx)\\n    Vo = initialV_o(env, state_winner_triples)\\n    p2.setV(Vo)\\n\\n    # give each player their symbol\\n    p1.set_symbol(env.x)\\n    p2.set_symbol(env.o)\\n\\n    T = 10000\\n    for t in range(T):\\n        if t % 200 == 0:\\n            print(t)\\n        play_game(p1, p2, Environment())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    # train the agent\n",
    "    p1 = Agent()\n",
    "    p2 = Agent()\n",
    "\n",
    "    # set initial V for p1 and p2\n",
    "    env = Environment()\n",
    "    state_winner_triples = get_state_hash_and_winner(env)\n",
    "\n",
    "    Vx = initialV_x(env, state_winner_triples)\n",
    "    p1.setV(Vx)\n",
    "    Vo = initialV_o(env, state_winner_triples)\n",
    "    p2.setV(Vo)\n",
    "\n",
    "    # give each player their symbol\n",
    "    p1.set_symbol(env.x)\n",
    "    p2.set_symbol(env.o)\n",
    "\n",
    "    T = 10000\n",
    "    for t in range(T):\n",
    "        if t % 200 == 0:\n",
    "            print(t)\n",
    "        play_game(p1, p2, Environment())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Taking a greedy action\n",
      "------------------\n",
      " 0.56| 0.53| 0.68|\n",
      "------------------\n",
      " 0.51| 0.97| 0.64|\n",
      "------------------\n",
      " 0.64| 0.27| 0.66|\n",
      "------------------\n",
      "-------------\n",
      "            \n",
      "-------------\n",
      "      X     \n",
      "-------------\n",
      "            \n",
      "-------------\n",
      "Enter coordinates i,j for your next move (i,j=0..2): 0,0\n",
      "Taking a greedy action\n",
      "------------------\n",
      "  O  | 0.65| 0.66|\n",
      "------------------\n",
      " 0.67|  X  | 0.54|\n",
      "------------------\n",
      " 0.64| 0.95| 0.60|\n",
      "------------------\n",
      "-------------\n",
      "  O         \n",
      "-------------\n",
      "      X     \n",
      "-------------\n",
      "      X     \n",
      "-------------\n",
      "Enter coordinates i,j for your next move (i,j=0..2): 0,1\n",
      "Taking a random action\n",
      "-------------\n",
      "  O   O     \n",
      "-------------\n",
      "  X   X     \n",
      "-------------\n",
      "      X     \n",
      "-------------\n",
      "Enter coordinates i,j for your next move (i,j=0..2): 0,2\n",
      "-------------\n",
      "  O   O   O \n",
      "-------------\n",
      "  X   X     \n",
      "-------------\n",
      "      X     \n",
      "-------------\n"
     ]
    }
   ],
   "source": [
    "# play human vs. agent\n",
    "# do you think the agent learned to play the game well?\n",
    "human = Human()\n",
    "human.set_symbol(env.o)\n",
    "while True:\n",
    "    p1.set_verbose(True)\n",
    "    play_game(p1, human, Environment(), draw=2)\n",
    "    # I made the agent player 1 because I wanted to see if it would\n",
    "    # select the center as its starting move. If you want the agent\n",
    "    # to go second you can switch the human and AI.\n",
    "    answer = input(\"Play again? [Y/n]: \")\n",
    "    if answer and answer.lower()[0] == 'n':\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}