nb_ex10_3_deep-qlearn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4cdb3963-a877-4e0e-8dae-b52da700b5b7",
   "metadata": {},
   "source": [
    "## 10.4 딥러닝 기반 Q-Learning을 이용하는 강화학습"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da662815-d7d1-4e6a-84f0-a134535e23c5",
   "metadata": {},
   "source": [
    "- 관련 패키지 불러오기 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5bcb4f8d-00ee-41a0-888f-fda4045c6163",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 기본 패키지\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a1b8b60-f907-4975-b0fd-60a7cecbde32",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 강화학습 환경 패키지\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6acbe326-1d64-419c-b200-68f0d84aadf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 인공지능 패키지: 텐서플로, 케라스 \n",
    "# 호환성을 위해 텐스플로에 포함된 케라스를 불러옴 \n",
    "import tensorflow as tf # v2.4.1 at 7/25/2021\n",
    "from tensorflow import keras # v2.4.0 at 7/25/2021\n",
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e22690e9-2164-493f-b94b-ad60d1ea294a",
   "metadata": {},
   "source": [
    "- Q 함수를 위한 뉴럴넷 구성하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "acf183ad-9dad-41fa-8e9c-d7ece881c455",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_q_model(num_states, num_actions):\n",
    "    inputs = Input(shape=(num_states,))\n",
    "    layer = Dense(32, activation=\"relu\")(inputs)\n",
    "    layer = Dense(16, activation=\"relu\")(layer)\n",
    "    action = Dense(num_actions, activation=\"linear\")(layer)\n",
    "    return Model(inputs=inputs, outputs=action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "616fc130-3ccd-474a-bb69-49a497d6275c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-16 21:19:49.912474: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2021-10-16 21:19:49.959801: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:49.960254: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1050 computeCapability: 6.1\n",
      "coreClock: 1.455GHz coreCount: 5 deviceMemorySize: 2.00GiB deviceMemoryBandwidth: 104.43GiB/s\n",
      "2021-10-16 21:19:49.961762: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-10-16 21:19:49.984308: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2021-10-16 21:19:50.001283: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2021-10-16 21:19:50.004451: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2021-10-16 21:19:50.029566: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-10-16 21:19:50.035768: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-10-16 21:19:50.078290: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-10-16 21:19:50.079204: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.080175: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.080500: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0\n",
      "2021-10-16 21:19:50.083354: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2021-10-16 21:19:50.099969: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 3599995000 Hz\n",
      "2021-10-16 21:19:50.103737: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5621ca098300 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2021-10-16 21:19:50.103810: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2021-10-16 21:19:50.106040: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.106494: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1050 computeCapability: 6.1\n",
      "coreClock: 1.455GHz coreCount: 5 deviceMemorySize: 2.00GiB deviceMemoryBandwidth: 104.43GiB/s\n",
      "2021-10-16 21:19:50.106574: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-10-16 21:19:50.106605: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2021-10-16 21:19:50.106630: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2021-10-16 21:19:50.106652: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2021-10-16 21:19:50.106698: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-10-16 21:19:50.106739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-10-16 21:19:50.106763: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-10-16 21:19:50.107482: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.108518: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.108741: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0\n",
      "2021-10-16 21:19:50.109003: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-10-16 21:19:50.244328: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2021-10-16 21:19:50.244386: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1108]      0 \n",
      "2021-10-16 21:19:50.244439: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1121] 0:   N \n",
      "2021-10-16 21:19:50.245934: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.246130: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1330] Could not identify NUMA node of platform GPU id 0, defaulting to 0.  Your kernel may not have been built with NUMA support.\n",
      "2021-10-16 21:19:50.246851: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.247663: E tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:967] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2021-10-16 21:19:50.248047: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1247] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1421 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce GTX 1050, pci bus id: 0000:01:00.0, compute capability: 6.1)\n",
      "2021-10-16 21:19:50.251719: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5621cde71200 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2021-10-16 21:19:50.251732: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce GTX 1050, Compute Capability 6.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 4)]               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 32)                160       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 34        \n",
      "=================================================================\n",
      "Total params: 722\n",
      "Trainable params: 722\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = create_q_model(4,2)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4713794-8303-444c-a3ab-0e366164f05c",
   "metadata": {},
   "source": [
    "- Q함수 뉴럴넷의 학습에 필요한 코드 작성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff27de71-d01c-437e-80be-a66cb6bda466",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_env_model(id='CartPole-v1'):\n",
    "    env = gym.make(id)\n",
    "    num_states = env.observation_space.shape[0]\n",
    "    num_actions = env.action_space.n\n",
    "    model = create_q_model(num_states, num_actions)\n",
    "    return env, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bafc7395-d123-4e06-9e18-bf2bf765fa1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model):\n",
    "    state_size = 4\n",
    "    action_size = 2        \n",
    "    states = np.zeros((10,state_size), dtype=np.float32)\n",
    "    with tf.GradientTape() as tape:\n",
    "        predicts = model(states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "54ac709c-aef5-4b46-b975-7ef505b44f79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-16 21:19:52.415639: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simple processing used in training is completed!\n"
     ]
    }
   ],
   "source": [
    "env, model = get_env_model()\n",
    "train(model)\n",
    "print('Simple processing used in training is completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c96efaa3-847f-4d2b-bdee-9ec45bbf07b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simple processing used in training is completed!\n"
     ]
    }
   ],
   "source": [
    "class World_00:\n",
    "    def __init__(self):\n",
    "        self.get_env_model()\n",
    "\n",
    "    def get_env_model(self):\n",
    "        self.env = gym.make('CartPole-v1')\n",
    "        self.num_states = env.observation_space.shape[0]\n",
    "        self.num_actions = env.action_space.n\n",
    "        self.model = create_q_model(self.num_states, self.num_actions)\n",
    "        # print(self.model.summary())\n",
    "\n",
    "    def train(self):        \n",
    "        states = np.zeros((10,self.num_states), dtype=np.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            predicts = self.model(states)\n",
    "\n",
    "new_world = World_00()\n",
    "new_world.train()\n",
    "print('Simple processing used in training is completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ad2896fe-0533-4728-8fc2-625bfe355c8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def env_test_model_memory(memory, env, model, n_episodes=1000, \n",
    "        flag_render=False):\n",
    "    for e in range(n_episodes):\n",
    "        done = False\n",
    "        score = 0\n",
    "        s = env.reset()\n",
    "        while not done:\n",
    "            s_array = np.array(s).reshape((1,-1))\n",
    "            Qsa = model.predict(s_array)[0]\n",
    "            a = np.argmax(Qsa)\n",
    "            next_s, r, done, _ = env.step(a)\n",
    "            if flag_render:\n",
    "                env.render()\n",
    "            score += r\n",
    "            memory.append([s,a,r,next_s,done])\n",
    "        print(f'Episode: {e:5d} -->  Score: {score:3.1f}')\n",
    "    print('Notice that the max score is set to 500.0 in CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a3df50c1-b5e5-4590-be67-953b9cb82627",
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_rotate(l):\n",
    "    return list(zip(*l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1f615593-ff3a-4dbe-a561-a4294f98c285",
   "metadata": {},
   "outputs": [],
   "source": [
    "class World_01(World_00):\n",
    "    def __init__(self):\n",
    "        World_00.__init__(self)\n",
    "        self.memory = deque(maxlen=2000)\n",
    "        self.N_batch = 64\n",
    "        self.t_model = create_q_model(self.num_states, self.num_actions)\n",
    "        self.discount_factor = 0.99\n",
    "        self.learning_rate = 0.001\n",
    "        self.optimizer = Adam(lr=self.learning_rate)\n",
    "\n",
    "    def trial(self, flag_render=False):\n",
    "        env_test_model_memory(self.memory, self.env,\n",
    "            self.model, n_episodes=10, flag_render=flag_render)\n",
    "        print(len(self.memory))\n",
    "\n",
    "    def train_memory(self):\n",
    "        if len(self.memory) >= self.N_batch:\n",
    "            memory_batch = random.sample(self.memory, self.N_batch)\n",
    "            s_l,a_l,r_l,next_s_l,done_l = [np.array(x) for x in list_rotate(memory_batch)]\n",
    "            model_w = self.model.trainable_variables\n",
    "            with tf.GradientTape() as tape:\n",
    "                Qsa_pred_l = self.model(s_l.astype(np.float32))\n",
    "                a_l_onehot = tf.one_hot(a_l, self.num_actions)\n",
    "                Qs_a_pred_l = tf.reduce_sum(a_l_onehot * Qsa_pred_l, \n",
    "                                             axis=1)    \n",
    "\n",
    "                Qsa_tpred_l = self.t_model(next_s_l.astype(np.float32)) \n",
    "                Qsa_tpred_l = tf.stop_gradient(Qsa_tpred_l)\n",
    "\n",
    "                max_Q_next_s_a_l = np.amax(Qsa_tpred_l, axis=-1)\n",
    "                Qs_a_l = r_l + (1 - done_l) * self.discount_factor * max_Q_next_s_a_l\n",
    "                loss = tf.reduce_mean(tf.square(Qs_a_l - Qs_a_pred_l))\n",
    "                grads = tape.gradient(loss, model_w)\n",
    "                self.optimizer.apply_gradients(zip(grads, model_w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "25e0beed-f119-468b-a037-2928f646337f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:     0 -->  Score: 10.0\n",
      "Episode:     1 -->  Score: 9.0\n",
      "Episode:     2 -->  Score: 9.0\n",
      "Episode:     3 -->  Score: 9.0\n",
      "Episode:     4 -->  Score: 9.0\n",
      "Episode:     5 -->  Score: 9.0\n",
      "Episode:     6 -->  Score: 10.0\n",
      "Episode:     7 -->  Score: 10.0\n",
      "Episode:     8 -->  Score: 10.0\n",
      "Episode:     9 -->  Score: 9.0\n",
      "Notice that the max score is set to 500.0 in CartPole-v1\n",
      "94\n",
      "Completed!\n"
     ]
    }
   ],
   "source": [
    "new_world = World_01()\n",
    "new_world.trial()\n",
    "new_world.train_memory()\n",
    "new_world.env.close()\n",
    "print('Completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2734a382-5d81-4f84-aa55-5aacd29fd8fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "class World_02(World_01):\n",
    "    def __init__(self):\n",
    "        World_01.__init__(self)\n",
    "        self.epsilon = 0.2\n",
    "    \n",
    "    def update_t_model(self):\n",
    "        self.t_model.set_weights(self.model.get_weights())\n",
    "\n",
    "    def best_action(self, s):\n",
    "        if random.random() <= self.epsilon:\n",
    "            return random.randrange(self.num_actions)\n",
    "        else:\n",
    "            s_array = np.array(s).reshape((1,-1))\n",
    "            Qsa = self.model.predict(s_array)[0]\n",
    "            return np.argmax(Qsa)\n",
    "\n",
    "    def trials(self, n_episodes=100, flag_render=False):\n",
    "        memory = self.memory\n",
    "        env = self.env\n",
    "        model = self.model\n",
    "        score_l = []\n",
    "        for e in range(n_episodes):\n",
    "            done = False\n",
    "            score = 0\n",
    "            s = env.reset()\n",
    "            while not done:                \n",
    "                a = self.best_action(s)\n",
    "                next_s, r, done, _ = env.step(a)\n",
    "                if flag_render:\n",
    "                    env.render()\n",
    "                score += r\n",
    "                memory.append([s,a,r,next_s,done])\n",
    "                # self.train_memory()     \n",
    "                s = next_s\n",
    "                self.train_memory()                 \n",
    "            self.update_t_model()\n",
    "            print(f'Episode: {e:5d} -->  Score: {score:3.1f}') \n",
    "            score_l.append(score)            \n",
    "        return score_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8d0c1a94-7542-42cb-bd7d-083189161543",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:     0 -->  Score: 12.0\n",
      "Episode:     1 -->  Score: 12.0\n",
      "Episode:     2 -->  Score: 13.0\n",
      "Episode:     3 -->  Score: 9.0\n",
      "Episode:     4 -->  Score: 24.0\n",
      "Episode:     5 -->  Score: 10.0\n",
      "Episode:     6 -->  Score: 12.0\n",
      "Episode:     7 -->  Score: 9.0\n",
      "Episode:     8 -->  Score: 11.0\n",
      "Episode:     9 -->  Score: 8.0\n",
      "Episode:    10 -->  Score: 8.0\n",
      "Episode:    11 -->  Score: 8.0\n",
      "Episode:    12 -->  Score: 12.0\n",
      "Episode:    13 -->  Score: 11.0\n",
      "Episode:    14 -->  Score: 11.0\n",
      "Episode:    15 -->  Score: 10.0\n",
      "Episode:    16 -->  Score: 12.0\n",
      "Episode:    17 -->  Score: 8.0\n",
      "Episode:    18 -->  Score: 15.0\n",
      "Episode:    19 -->  Score: 10.0\n",
      "Episode:    20 -->  Score: 9.0\n",
      "Episode:    21 -->  Score: 14.0\n",
      "Episode:    22 -->  Score: 14.0\n",
      "Episode:    23 -->  Score: 14.0\n",
      "Episode:    24 -->  Score: 9.0\n",
      "Episode:    25 -->  Score: 14.0\n",
      "Episode:    26 -->  Score: 11.0\n",
      "Episode:    27 -->  Score: 12.0\n",
      "Episode:    28 -->  Score: 16.0\n",
      "Episode:    29 -->  Score: 27.0\n",
      "Episode:    30 -->  Score: 13.0\n",
      "Episode:    31 -->  Score: 12.0\n",
      "Episode:    32 -->  Score: 10.0\n",
      "Episode:    33 -->  Score: 22.0\n",
      "Episode:    34 -->  Score: 15.0\n",
      "Episode:    35 -->  Score: 63.0\n",
      "Episode:    36 -->  Score: 10.0\n",
      "Episode:    37 -->  Score: 64.0\n",
      "Episode:    38 -->  Score: 25.0\n",
      "Episode:    39 -->  Score: 11.0\n",
      "Episode:    40 -->  Score: 57.0\n",
      "Episode:    41 -->  Score: 59.0\n",
      "Episode:    42 -->  Score: 49.0\n",
      "Episode:    43 -->  Score: 51.0\n",
      "Episode:    44 -->  Score: 97.0\n",
      "Episode:    45 -->  Score: 37.0\n",
      "Episode:    46 -->  Score: 77.0\n",
      "Episode:    47 -->  Score: 34.0\n",
      "Episode:    48 -->  Score: 60.0\n",
      "Episode:    49 -->  Score: 51.0\n"
     ]
    }
   ],
   "source": [
    "new_world = World_02()\n",
    "score_l = new_world.trials(n_episodes=50)\n",
    "new_world.env.close()\n",
    "np.save('score_l.npy', score_l)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ef42bf1-4cd4-44b8-b5df-0e7f739f2d95",
   "metadata": {},
   "source": [
    "---\n",
    "### 전체코드 (분할 버전)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bc875b87-ff0e-4ac0-965c-6da3d5dafc5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 3, 5), (2, 4, 6)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = [[1,2],[3,4],[5,6]]\n",
    "list(zip(*l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d5c70ae-7c88-45bd-b098-a6ea2c771ac5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 기본 패키지\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "# 강화학습 환경 패키지\n",
    "import gym\n",
    "\n",
    "# 인공지능 패키지: 텐서플로, 케라스 \n",
    "# 호환성을 위해 텐스플로에 포함된 케라스를 불러옴 \n",
    "import tensorflow as tf\n",
    "from tensorflow import keras \n",
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48891ce7-55ae-495f-8586-6271d7c424af",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_q_model(num_states, num_actions):\n",
    "    inputs = Input(shape=(num_states,))\n",
    "    layer = Dense(32, activation=\"relu\")(inputs)\n",
    "    layer = Dense(16, activation=\"relu\")(layer)\n",
    "    action = Dense(num_actions, activation=\"linear\")(layer)\n",
    "    return Model(inputs=inputs, outputs=action)\n",
    "\n",
    "def list_rotate(l):\n",
    "    return list(zip(*l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "582c9ce2-656c-427b-8695-ba0e9698a501",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WorldFull():\n",
    "    def __init__(self):\n",
    "        self.get_env_model() #? \n",
    "        \n",
    "        self.memory = deque(maxlen=2000)\n",
    "        self.N_batch = 64\n",
    "        self.t_model = create_q_model(self.num_states, self.num_actions)\n",
    "        self.discount_factor = 0.99\n",
    "        self.learning_rate = 0.001\n",
    "        self.optimizer = Adam(lr=self.learning_rate)\n",
    "        \n",
    "        self.epsilon = 0.2\n",
    "        \n",
    "    def get_env_model(self):\n",
    "        self.env = gym.make('CartPole-v1')\n",
    "        self.num_states = self.env.observation_space.shape[0]\n",
    "        self.num_actions = self.env.action_space.n\n",
    "        self.model = create_q_model(self.num_states, self.num_actions)\n",
    "    \n",
    "    def update_t_model(self):\n",
    "        self.t_model.set_weights(self.model.get_weights())\n",
    "\n",
    "    def best_action(self, s):\n",
    "        if random.random() <= self.epsilon:\n",
    "            return random.randrange(self.num_actions)\n",
    "        else:\n",
    "            s_array = np.array(s).reshape((1,-1))\n",
    "            Qsa = self.model.predict(s_array)[0]\n",
    "            return np.argmax(Qsa)\n",
    "\n",
    "    def train_memory(self):\n",
    "        if len(self.memory) >= self.N_batch:\n",
    "            memory_batch = random.sample(self.memory, self.N_batch)\n",
    "            s_l,a_l,r_l,next_s_l,done_l = [np.array(x) for x in list_rotate(memory_batch)]\n",
    "            model_w = self.model.trainable_variables\n",
    "            with tf.GradientTape() as tape:\n",
    "                Qsa_pred_l = self.model(s_l.astype(np.float32))\n",
    "                a_l_onehot = tf.one_hot(a_l, self.num_actions)\n",
    "                Qs_a_pred_l = tf.reduce_sum(a_l_onehot * Qsa_pred_l, \n",
    "                                            axis=1)    \n",
    "\n",
    "                Qsa_tpred_l = self.t_model(next_s_l.astype(np.float32)) \n",
    "                Qsa_tpred_l = tf.stop_gradient(Qsa_tpred_l)\n",
    "\n",
    "                max_Q_next_s_a_l = np.amax(Qsa_tpred_l, axis=-1)\n",
    "                Qs_a_l = r_l + (1 - done_l) * self.discount_factor * max_Q_next_s_a_l\n",
    "                loss = tf.reduce_mean(tf.square(Qs_a_l - Qs_a_pred_l))\n",
    "                grads = tape.gradient(loss, model_w)\n",
    "                self.optimizer.apply_gradients(zip(grads, model_w))        \n",
    "        \n",
    "    def trials(self, n_episodes=100, flag_render=False):\n",
    "        memory = self.memory\n",
    "        env = self.env\n",
    "        model = self.model\n",
    "        score_l = []\n",
    "        for e in range(n_episodes):\n",
    "            done = False\n",
    "            score = 0\n",
    "            s = env.reset()\n",
    "            while not done:                \n",
    "                a = self.best_action(s)\n",
    "                next_s, r, done, _ = env.step(a)\n",
    "                if flag_render:\n",
    "                    env.render()\n",
    "                score += r\n",
    "                memory.append([s,a,r,next_s,done])\n",
    "                # self.train_memory()     \n",
    "                s = next_s\n",
    "                self.train_memory()                 \n",
    "            self.update_t_model()\n",
    "            print(f'Episode: {e:5d} -->  Score: {score:3.1f}') \n",
    "            score_l.append(score)            \n",
    "        return score_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "510f66c9-e582-40f6-9b6f-310056e67029",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_world = WorldFull()\n",
    "score_l = new_world.trials(n_episodes=100)\n",
    "new_world.env.close()\n",
    "np.save('score_l.npy', score_l)\n",
    "print('Job completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a551f15d-e800-4dc4-8541-ed9d1428f69c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:     0 -->  Score: 11.0\n",
      "Episode:     1 -->  Score: 10.0\n",
      "Episode:     2 -->  Score: 8.0\n",
      "Episode:     3 -->  Score: 8.0\n",
      "Episode:     4 -->  Score: 10.0\n",
      "Episode:     5 -->  Score: 17.0\n",
      "Episode:     6 -->  Score: 10.0\n",
      "Episode:     7 -->  Score: 9.0\n",
      "Episode:     8 -->  Score: 10.0\n",
      "Episode:     9 -->  Score: 13.0\n",
      "Episode:    10 -->  Score: 10.0\n",
      "Episode:    11 -->  Score: 10.0\n",
      "Episode:    12 -->  Score: 10.0\n",
      "Episode:    13 -->  Score: 10.0\n",
      "Episode:    14 -->  Score: 8.0\n",
      "Episode:    15 -->  Score: 12.0\n",
      "Episode:    16 -->  Score: 9.0\n",
      "Episode:    17 -->  Score: 10.0\n",
      "Episode:    18 -->  Score: 9.0\n",
      "Episode:    19 -->  Score: 8.0\n",
      "Episode:    20 -->  Score: 10.0\n",
      "Episode:    21 -->  Score: 15.0\n",
      "Episode:    22 -->  Score: 8.0\n",
      "Episode:    23 -->  Score: 8.0\n",
      "Episode:    24 -->  Score: 10.0\n",
      "Episode:    25 -->  Score: 15.0\n",
      "Episode:    26 -->  Score: 9.0\n",
      "Episode:    27 -->  Score: 9.0\n",
      "Episode:    28 -->  Score: 10.0\n",
      "Episode:    29 -->  Score: 9.0\n",
      "Episode:    30 -->  Score: 11.0\n",
      "Episode:    31 -->  Score: 8.0\n",
      "Episode:    32 -->  Score: 12.0\n",
      "Episode:    33 -->  Score: 15.0\n",
      "Episode:    34 -->  Score: 11.0\n",
      "Episode:    35 -->  Score: 11.0\n",
      "Episode:    36 -->  Score: 11.0\n",
      "Episode:    37 -->  Score: 11.0\n",
      "Episode:    38 -->  Score: 14.0\n",
      "Episode:    39 -->  Score: 49.0\n",
      "Episode:    40 -->  Score: 15.0\n",
      "Episode:    41 -->  Score: 11.0\n",
      "Episode:    42 -->  Score: 11.0\n",
      "Episode:    43 -->  Score: 16.0\n",
      "Episode:    44 -->  Score: 11.0\n",
      "Episode:    45 -->  Score: 11.0\n",
      "Episode:    46 -->  Score: 10.0\n",
      "Episode:    47 -->  Score: 75.0\n",
      "Episode:    48 -->  Score: 10.0\n",
      "Episode:    49 -->  Score: 11.0\n",
      "Episode:    50 -->  Score: 15.0\n",
      "Episode:    51 -->  Score: 12.0\n",
      "Episode:    52 -->  Score: 12.0\n",
      "Episode:    53 -->  Score: 9.0\n",
      "Episode:    54 -->  Score: 9.0\n",
      "Episode:    55 -->  Score: 10.0\n",
      "Episode:    56 -->  Score: 12.0\n",
      "Episode:    57 -->  Score: 72.0\n",
      "Episode:    58 -->  Score: 141.0\n",
      "Episode:    59 -->  Score: 52.0\n",
      "Episode:    60 -->  Score: 46.0\n",
      "Episode:    61 -->  Score: 86.0\n",
      "Episode:    62 -->  Score: 93.0\n",
      "Episode:    63 -->  Score: 64.0\n",
      "Episode:    64 -->  Score: 72.0\n",
      "Episode:    65 -->  Score: 83.0\n",
      "Episode:    66 -->  Score: 205.0\n",
      "Episode:    67 -->  Score: 109.0\n",
      "Episode:    68 -->  Score: 221.0\n",
      "Episode:    69 -->  Score: 174.0\n",
      "Episode:    70 -->  Score: 149.0\n",
      "Episode:    71 -->  Score: 221.0\n",
      "Episode:    72 -->  Score: 213.0\n",
      "Episode:    73 -->  Score: 365.0\n",
      "Episode:    74 -->  Score: 149.0\n",
      "Episode:    75 -->  Score: 132.0\n",
      "Episode:    76 -->  Score: 239.0\n",
      "Episode:    77 -->  Score: 388.0\n",
      "Episode:    78 -->  Score: 177.0\n",
      "Episode:    79 -->  Score: 197.0\n",
      "Episode:    80 -->  Score: 216.0\n",
      "Episode:    81 -->  Score: 190.0\n",
      "Episode:    82 -->  Score: 242.0\n",
      "Episode:    83 -->  Score: 239.0\n",
      "Episode:    84 -->  Score: 221.0\n",
      "Episode:    85 -->  Score: 231.0\n",
      "Episode:    86 -->  Score: 146.0\n",
      "Episode:    87 -->  Score: 154.0\n",
      "Episode:    88 -->  Score: 276.0\n",
      "Episode:    89 -->  Score: 152.0\n",
      "Episode:    90 -->  Score: 131.0\n",
      "Episode:    91 -->  Score: 146.0\n",
      "Episode:    92 -->  Score: 152.0\n",
      "Episode:    93 -->  Score: 189.0\n",
      "Episode:    94 -->  Score: 189.0\n",
      "Episode:    95 -->  Score: 163.0\n",
      "Episode:    96 -->  Score: 170.0\n",
      "Episode:    97 -->  Score: 158.0\n",
      "Episode:    98 -->  Score: 166.0\n",
      "Episode:    99 -->  Score: 180.0\n",
      "Job completed!\n"
     ]
    }
   ],
   "source": [
    "plt.plot(score_l)\n",
    "plt.title(\"Deep Q-Learning for Cartpole\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6244b5ac-3b34-4fef-914a-6b5f46654e2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Score')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABD9klEQVR4nO3deZxjZZno8d+TpCq1V3V3Va/VK3TT0LI3LQ6ICCqr4Dqio+IywzCDo851BkXHO+oMV++MDupVVFQUNxhUFEZxgWYTWZutoRt6b3ql1u5aUpWkkjz3j3NO6iSVVKW6K0ktz/fzqU8lJ+ecvKfSfZ682/OKqmKMMcYABMpdAGOMMZOHBQVjjDFpFhSMMcakWVAwxhiTZkHBGGNMmgUFY4wxaRYUjJkAIvJtEflskc79VhHZKyL9InJqMd5jMhGRc0VkX7nLMVNZUJjBRGS3iAyKSJ+IHBaRR0TkahEpy78LEWkVkZ+KSJeIRETkCRG5eIxjJsUNRFWvVtV/K9Lpvwx8RFXrVPWZiTihiFwgIg+5n32HiDwoIpcdxfl2i8gbJqJsprwsKJg3q2o9sBT4EvBJ4PulLoSIzAYeBuLAGqAZuAG4TUTeUury+IlIqJzvj/PZbDqSA0UkmGPbO4CfAz8CWoF5wP8G3nwE5y/338ZMNFW1nxn6A+wG3pC1bR2QAl7lPg/jfFPdA7QB3waqfftfCjwLHAYeAU7KOv91wGbgEPADoCpPWf4NeAEIZG3/JLATkDzHnQvsy/PaQuCXQAewC/ho1nU+6pb7IPANoNL3ugLXANvcY88F9gGfANrdYz7o2/+HwL/7yzTKvnOA/wF6gSeBfwcezlH+MNDvliUC7HC3Hw884JZ9E3BZVjm+BdztHpP9+Yr7Wf7zKP8ujgHuA7qATuCnQFPW5/pJYCMQA251/80MuuW9Fljmlvsq4ID7N/hE1rV91X3tgPs4nOszHe1ztJ8i3BfKXQD7KeOHnyMouNv3AH/nPv4qcBcwG6h3b2ZfdF87zb3pvRoIAle65wz7zv8CsNg9/s/ejTPHez4GfD7H9uXuzWVlnuMybiC+7QHgKZxvwJXACpzgcoH7+unAmUDIvYG9CHzcd7wC97jlrnbfJwF8AagALgYGgFnu/j8kMyiMtu9t7k8NcAKwlxxBIassx7qPK4DtwKfd6zoP6AOO85WjBzjL/RtUZZ1rtXu+5aO837HAG3Fu3C3AQ8BXs/7dPOt+rtW5/i0xHBRuBWqBE3Fu6m9wX/+C+5nPdd/jEeDfsj/TsT5H+ynCfaHcBbCfMn74+YPCY8BncL5VRoBjfK+9BtjlPv6W9x/Z9/oW4HW+81/te+1i3G+7Od5zu39f3/Yq9+byF3mOS99Asra/GtiTte064Ad5zvNx4Fe+5wqcl/U+g0DIt60dONN9/EMyg0LOfXGC5xDuTdx9LWdNIassXlB4LfAKvhqVe+P9nK8cPxrlXGe558tZY8tzzFuAZ7L+3XxotH9LDAeF1b5t/wF83328A7jY99oFwO7sz3S8n6P9HP2PtQeaXBYB3Tjf4GqAp0TEe01wbmzgtHVfKSL/4Du2Eqe679nre/xy1mt+ncCCHNu9bR0isgSnKQoAVa0b5RqWAgtF5LBvWxD4E4CIrAL+C1iLc40hnG+kfnuznnepasL3fADIV4Z8+7a47+U/d/b7jGYhsFdVU75tL+N8ZoWcr8v9vQCnKWYEEZkLfB0nANXjfFs/lLVboWXO/vxPdB8vdJ/7X8v1b2PUz9FMPOtoNhlE5AycG8zDODfqQWCNqja5P42+m/Fe4Hrfa02qWqOqt/pOudj3eAlO+3Eu9wJvzzHy6S9x2ud3qOoedUbg1I0RELyy7coqW72qeqOZvgW8hNMs1YDTHCNZ5yhGCuEOnKalVt+2xXn2zeUAsDjr77QE2O97Plq5t+D8bd4+yj5fdM9xkvu3eS9j/23yvWe+z/8Azg0/12t+Y32OZoJZUDAAiEiDiFyK09b9E1V93v02+l3gBvfbIyKySEQucA/7LnC1iLxaHLUicomI1PtOfY071HQ2zo33v/MU4QagAfi+iMwXkSoReTfwWeBfs74Z5yp/lf8HeALoFZFPiki1iARF5FVu0APnG3Av0C8iq4G/G99f7MioahK4A/iciNS47/3+cZzicZwmvWtFpEJEzsUZNXRbge+vwP8CPisiH3Q/94CInC0iN7m71eN0GB8WkUXAPxdw6jac9v5sn3Wvcw3wQYY//1uBfxGRFhFpxukz+EmO48f6HM0Es6Bg/kdE+nC+kX0Gp0nlg77XP4nT3v+YiPTifKM/DkBVNwB/gzNy55C73weyzv8z4I84nYM7cdrPR1DVLuBsnD6EzTg3pR8B16jqzWNcwyKcGo3/ZznOzfIUnGaSTuB7QKN7zD8B78HppP0u+YNVMXzELccrwI9xbpCxQg5U1ThwGXARzjXdCLxfVV8q9M1V9RfAu4AP4Xw7b8P5XO50d/k8ziCCHuC3OEFsLF/EuckfFpF/8m1/EOffxXrgy6r6R3f7vwMbcEYwPQ88TY5/G24QHe1zNBNMnC8Oxkw8EdkN/LWq3nsExzbgjFb6lar+74ku22QiIv8XmK+qV5a7LBNFRJbh3MQrsvpWzCRnNQUzKalqL85opaSIzC93eSaSiKwWkZPcJrd1wIeBX5W7XMYANvrITF6quhenKWO6qcdpMlqIM1T1Kww33RhTVtZ8ZIwxJs2aj4wxxqRN6eaj5uZmXbZsWbmLYYwxU8pTTz3VqaotuV4relBwszRuAPar6qXuePX/xpkGvxv4S1U95O57HU6nWxIn6dUfRjv3smXL2LBhQxFLb4wx04+IvJzvtVI0H30MJ9mY51PAelVdiTN2+VMAInICcAVO2uQLgRtzpf01xhhTPEUNCiLSClyCM9nEczlwi/v4FpxkW97221Q1pqq7cCa8rCtm+YwxxmQqdk3hqzi51f0pCuap6kEA9/dcd/siMpNn7SMzyRcAInKViGwQkQ0dHR1FKbQxxsxURQsKbh6ddlXNzjyZ95Ac20aMl1XVm1R1raqubWnJ2U9ijDHmCBWzo/ks4DJ3jd0qoEFEfgK0icgCVT0oIgtwJu+AUzPwZ1RsJX9GTWOMMUVQtJqCql6nqq2qugynA/k+VX0vzipeXo6XKxmeyXkXcIWIhEVkObASJ0OiMcaYEinHPIUvAbeLyIdxln18J4CqbhKR23EyZCZwsmMmy1A+Y4yZsaZ0mou1a9eqzVMwZvpr74vyzJ7DXLBmWuVGLBsReUpV1+Z6zdJcGGMmvduf3MvVP3mKeGLUtZbMBLCgYIyZ9PpiCVQhlrAW5WKzoGCMmfSicScYWE2h+CwoGGMmvcEhNygkLSgUmwUFY8ykNzjkBIPYkAWFYrOgYIyZ9AbjVlMoFQsKxphJLzpkfQqlYkHBGDPpDcQTAMQsKBSdBQVjzKTn9SlYTaH4LCgYYya9qI0+KhkLCsaYSc/raI4N2eS1YrOgYIwpuSd2dfPMnkMF72/zFErHgoIxpuSuv/tF/uuerQXvP2ijj0qmHKmzjTEz3EAsQThY2HfSZErTwcCCQvFZTcEYU3KDQ0liBTYFDfr6Eaz5qPgsKBhjSi46lCz4W7/XyQxWUygFCwrGmJIbjCeJF5gGO+qrKdjkteIrWlAQkSoReUJEnhORTSLyeXf750Rkv4g86/5c7DvmOhHZLiJbROSCYpXNGFM+qko0kSq4KWjQgkJJFbOjOQacp6r9IlIBPCwiv3Nfu0FVv+zfWUROAK4A1gALgXtFZJWt02zM9DKU1IzO47FY81FpFa2moI5+92mF+zPagtCXA7epakxVdwHbgXXFKp8xpjzGO7w0o6PZgkLRFbVPQUSCIvIs0A7co6qPuy99REQ2isjNIjLL3bYI2Os7fJ+7LfucV4nIBhHZ0NHRUcziG2OKYLwZTzNqCklrOCi2ogYFVU2q6ilAK7BORF4FfAs4BjgFOAh8xd1dcp0ixzlvUtW1qrq2paWlKOU2xhTPeNdGsJpCaZVk9JGqHgYeAC5U1TY3WKSA7zLcRLQPWOw7rBU4UIryGWNKx7vJDyWVVGq0FmV3fzeIiFhHcykUc/RRi4g0uY+rgTcAL4nIAt9ubwVecB/fBVwhImERWQ6sBJ4oVvmMMeUx3slo3v6N1RVWUyiBYo4+WgDcIiJBnOBzu6r+RkR+LCKn4DQN7Qb+FkBVN4nI7cBmIAFcYyOPjJl+ovHMoFBVERx9fwsKJVW0oKCqG4FTc2x/3yjHXA9cX6wyGWPKb7x9BF7zUWN1haW5KAGb0WyMKalxB4WhJBVBoboiaH0KJWBBwRhTUuOdjDYQT1JVEaQyFLDmoxKwoGCMKanoODuao0NJqiuChENWUygFCwrGmJI6kuaj6sog4VCg4CR65shZUDDGlNRgfDgQFPLNfzDu1BQqQwHraC4BCwrGmJI60ppCZdD6FErBgoIxpqSOtE/BOppLw4KCMaakjmT0kdPRbEGhFCwoGGNK6kiaj6oqnZqCjT4qPgsKxpiSGhxKUlXh3HoKSYUd9XU0J1KFJdEzR86CgjGmpKJDSRqrK4BxdDS7QQEKT7ltjowFBWNMSQ3GjyAouKOPwNJnF5sFBWNMSQ36agpj3eBTKSU6lEp3NIMttFNsFhSMMSXlDwpjNQVF3RnMzozmYEHHmKNjQcEYU1KxoRQNBTYfecNX/X0KsSFLdVFMFhSMMSU1OJSktjJEKCBjB4WhkUHBagrFZUHBGFNSg3G347iAyWje7OcqX0ez9SkUVzHXaK4SkSdE5DkR2SQin3e3zxaRe0Rkm/t7lu+Y60Rku4hsEZELilU2Y0x5qKozTyEUKCjBnZc8L6OmYEGhqIpZU4gB56nqycApwIUicibwKWC9qq4E1rvPEZETgCuANcCFwI3u+s7GmGnCG21UVWCCO6/5qKbSgkKpFC0oqKPffVrh/ihwOXCLu/0W4C3u48uB21Q1pqq7gO3AumKVzxhTetkdx4UGhSrfkNSY9SkUVVH7FEQkKCLPAu3APar6ODBPVQ8CuL/nursvAvb6Dt/nbss+51UiskFENnR0dBSz+MaYCZbdcTzWDX4wnsjYH5zRS6Z4ihoUVDWpqqcArcA6EXnVKLtLrlPkOOdNqrpWVde2tLRMUEmNMaWQDgrjbD7yVl4DG31UbCUZfaSqh4EHcPoK2kRkAYD7u93dbR+w2HdYK3CgFOUzxpSG13xUVWAq7IyO5qA7ec36FIqqmKOPWkSkyX1cDbwBeAm4C7jS3e1K4E738V3AFSISFpHlwErgiWKVzxhTetGhI+tTsNFHpRMq4rkXALe4I4gCwO2q+hsReRS4XUQ+DOwB3gmgqptE5HZgM5AArlFVm7pozDSS0XwUChAdo39geJ5CgKQ6rcnxhN0WiqloQUFVNwKn5tjeBZyf55jrgeuLVSZjTHlljD4KBugdTIy5fzAgVAYDJEJuULA+haKyGc3GmJLxDzEtpPnIW4pTRGz0UYlYUDDGlEw0o/koOPaM5qEkVRVOB3MoIIhYTaHYLCgYY0omu/mokNxH1ZXObUpECjrGHB0LCsaYkhkcysxlNNYiO4Nu85GnkGPM0bGgYIwpGa9PIRwKuPMURh9J5K3P7AkXkETPHB0LCsaYkokOJamqCBAISGFZUn19CgDhULCsHc39sQTdkXjZ3r8ULCgYY0rG3xxUaJ9CTWVm81E5awrX/3YzH77lybK9fylYUDDGlIy/OagyFCClkBjlJj/gLsjjcQJJ+SavHeyJ0tYTLdv7l4IFBWNMyQwOJamqHA4KMPoQ08F4ZvNRIXMbiikSSxCJT+8Z1RYUjDElE81qPoLRcxlFh0aOPipn81EkliQSS6A6IoHztGFBwRhTMtnNRzB6UMgefVTueQqReIJESqf1CCgLCsaYknEmo2UGhXzzDrz1nP19CuGK8s5TiMScpqOB2PRtQrKgYIwpmcGhVLqPYKxFc2KJFKrk6GguX1AYcFeCi8RHT+Q3lVlQMMaUjL+PYKw+BX9KDE85O5pTKWXALdPANO5stqBgjCkZZzSRc9sZq0/Bv8COp5xpLgaGhgNBf8xqCsYYc9RydjTnaT7yL8jjKWeai4gvEFifgjHGTICMeQoFNh9VTZLRR/6gYH0KR0BEFovI/SLyoohsEpGPuds/JyL7ReRZ9+di3zHXich2EdkiIhcUq2zGmNJLppR4IlXwkNRojuajcEWQWJlmNPv7EQamcVAo5hrNCeATqvq0iNQDT4nIPe5rN6jql/07i8gJwBXAGmAhcK+IrLJ1mo2ZHrJv8mMNSc3VfFTOmoK/HyFizUfjp6oHVfVp93Ef8CKwaJRDLgduU9WYqu4CtgPrilU+Y0xpZd/kxxqSmm/00Vj5korFXzuIWEfz0RGRZcCpwOPupo+IyEYRuVlEZrnbFgF7fYftI0cQEZGrRGSDiGzo6OgoZrGNMRMou4+gMuj8HnP0UVaWVCjPkpz9vtrBdM5/VPSgICJ1wC+Bj6tqL/At4BjgFOAg8BVv1xyHj0gwoqo3qepaVV3b0tJSnEIbYyZcvuajcc1TKCBfUrEMZIw+sprCERGRCpyA8FNVvQNAVdtUNamqKeC7DDcR7QMW+w5vBQ4Us3zGmNLJnncwHBRyf+vON0/BOab0QcGrHYRDAaspHAkREeD7wIuq+l++7Qt8u70VeMF9fBdwhYiERWQ5sBJ4oljlM8aUVvqbf4Gps/PNU4D8ndO5qCo/fnQ3T+85NP5C+3j9CC31YRt9dITOAt4HPC8iz7rbPg28W0ROwWka2g38LYCqbhKR24HNOCOXrrGRR8ZMH95NvqrANBdR3zdzz1gjlnLZ1Rnhs3duAuD1x7Xwj29cxUmtTeMrPM7chHAoQENVxbTuaC5aUFDVh8ndT3D3KMdcD1xfrDIZY8onu0+hIujcHvIFhQF37QWn0cERPoLmowOHnZXS3nrqIu7f0s5l3/gzX7viFC4/ZbTBkCNFYglqwyFqw0EbkmqMMUcruzlIRJxcRnmajyJx5ybsdySjjw70DALwj29YxZ+ufT0rWmr52eN7xl3+gZizXnRtODStm48sKBhjSmIw7tzIM2YojzIZrS+aoKEqKyiMMYw1l4NuTWFeY5j6qgrefNJCntjdTUdfbFzlj8QT1IVD1FaGrKPZGGOOVr7RRPlu8P2xBHXZQeEImo8O9gzSXBcmHHLe9+ITF6AKf9j0yrjKH3FrCjWVQRuSaowxR8vrU6iqzOw4zhsUos43c7/h0UeFf1M/0BNlYVNV+vmqeXWsaKnldy8cTG9TVT75i43c+ez+vOfxmrNqwyFLnW2MMUdrMJ4kIMOjjsANCnn6B/pjI4PCEdUUDg+yoHE4KIgIF79qAY/t7Kar32lCuvPZA/z3hr3cs7kt73kisQS1lU5H80A8ieqIubXTggUFY0xJeGsp+EcTjZbgri86SvPRODqaD/ZEWdBYnbHtohPnk0wp92xuoz+W4P/c/SIAPYNDec8TiSWpDYeoqQyRSGnZ1nUotmLOUzDGmLTBoWTGRDQYu0+hPrumEBzfPIXe6BD9sURG8xHACQsaWDqnht8+f5BdnRHa+2Isaqrm8ED+oDAQT1AbDlLrXsNALJnup5hOrKZgjCmJaDyZsWAO5G8+UtWcHc3jnafgjTzKrimICBe9agGP7Oji5j/v4p2nt3LGsllj1hRqKkPUuIFqui60Y0HBGDNuyZRywz1b6Y7ER93vlkd28+TubiBzKU5PZTD3msuDQ0mSKaW+qiJz/3EGhQOHnTkK2TUFgIvdJqSqUJBrL1xNU00lhwdyX088kSKeTFEXDlJb6QaFaTqBzZqPjDHjtr29n6+t30ZLfZj3nrk0735f/N2LVFUEufujryWap/moLzryG3e/u23k6CPn+EKbj7yJa9k1BYATFzVy4Zr5vGnNPFrqwzRWV9AbTZBMKcFAZjIGb7JajdvRDNO3plBwUBCRamCJqm4pYnmMMVNAf8xpZtnvfhPPZSCeIDqUIjqU4qO3PoMII5qPwqEAXTlu8H3ukM/6o5yncPBwlGBAmFsfHvGaiPDt952eft5U49RKegeHmFVbmbGvN1mtzh2SCk6fwnRUUPORiLwZeBb4vfv8FBG5q4jlMsZMYt63+32H8geFrn6nKeacVS1sePkQT+4+NLL5KE+fQr6aQjAgBANCPDnyhrz5QC+nfOGPbG/vS2870DPIvPowoeDYtzovKBzO0a/gTVarCTuT12D61hQK7VP4HM66B4cBVPVZYFkxCmSMmfy89vS93QN59/H6G658zVLetdZZKqWqIvOWk29Iqjc5LDsojHbMnc/u5/DAEA9sGV6R8eDhKAuaRjYd5dJY7QSFXJ3NXnlqK0PpPoXpmv+o0KCQUNWeopbEGDNleM1Ho9UUvKAwu7aSz122hpNaGzlufkPGPvmGpHo1kezRR/mOUVX+6E4886+bcLAnc+LaaBqrnSajXJ3NA27zUW04RI3bp9A/TZuPCu1TeEFE3gMERWQl8FHgkeIVyxgzmXk3xM7+GNGhkUNNAbrcoDCnNkx1ZZA7rzkrY+Ia5G8+6os6Qac+XDHitVzH7OjoZ1dnhKqKAE+9fCg92/hgT5Q3rZlf0DV5zUej1RRqKoPp2st0zX9UaE3hH4A1QAz4GdADfLxIZTLGTHL9vhFD+WoL3REnhcTsOucbeHZAACfr6WjNR9kdzeB0TseGMo/5wyanlvChs5bT1htj/+FBuiNxYolUwTWFJrf5KNcENq+pqC4coioURIRpmyl1zKAgIkHgLlX9jKqe4f78i6pGS1A+Y0wZqSqbDoxsOfZ3su47lLtfoSsSpzIUSM8AziVf85EXdLLXU/COyV6D4Z7NbZzc2sjFJzqr/T718qH04jq5hqPm0jhKUPD6UGrCQQIBoaZi+mZKHTMouEtiDohI43hOLCKLReR+EXlRRDaJyMfc7bNF5B4R2eb+nuU75joR2S4iW0TkgnFfjTFmQj26o4tLvv4w29r6Mrb3RRPpsfx789QUuvrjzKmtzFlD8HhNQdnJ5fpjztKXlaGRt6jsjua23ijP7j3Mm9bMZ/X8emoqgzz98qH0HIVcE9dyCQUD1IVDOZuPIr6OZoCa8PRdU6HQ5qMozlrL3xeRr3s/YxyTAD6hqscDZwLXiMgJwKeA9aq6EljvPsd97QqcZqoLgRvdWooxpkw63Cyibb2ZC9JEYgkWNVVTGQzkrSl0R+LMzhrvny2cJ8FdXyyRs+nIO8YfFO590Wk6euMJ8wgFA5yyuImn9hzi4OH8E9fyaayu4PDgyI7mSDyJyPBaELWVwWm7TnOhHc2/dX8KpqoHgYPu4z4ReRFYBFwOnOvudgvwAPBJd/ttqhoDdonIdpxhsI+O532NMRPHa9vvjQ6N2F5fFSIYqM7bp9BVQFDwEtzFE6mM5HK51lJIH5MVFP64qY1lc2pYObcOgNOXzuLGB3awvaOfymCAOWOUwa+ppoKenM1HCWoqnKYjYFovyVlQUFDVW0SkEljlbtqiqvkzR2URkWXAqcDjwDw3YKCqB0VkrrvbIuAx32H73G3Z57oKuApgyZIlhRbBGHMEvFm7vYMjg0JdOMTs2spRO5qXz6kZ9fz5ZijnSobnPybqdjT3RYd4dEcXHzhrWbqZ6vSls0imlD9samNBU1X6Rl6IppqK3JPXstaLrq0MTdvcR4XOaD4X2AZ8E7gR2Coi5xR4bB3wS+Djqto72q45to1YxUJVb1LVtaq6tqWlpZAiGGOOkFdTyM5P5H2Tb51Vzf58zUf9cWbXjkwv4ZdvfYTRagrhUDC98tpDWzuJJ1O88YR56ddPXeJ0U3b0xQoeeeRpqs6dFM9bS8FTEw5O25pCoX0KXwHepKqvU9VzgAuAG8Y6SEQqcALCT1X1Dndzm4gscF9fALS72/cBi32HtwIHCiyfMaYIvBtfdvNRJO58k2+dVUNnf3zEDTI6lCQSTzKnrvDmI7/e6NCIDKn+Y7z917/URlNNBactSY9XobG6glXznKakhePoTwBoqK6gZ3DkzT4SS6TTW4BbU5jhHc0V/kR4qroVyP2JucSpy30feFFV/8v30l3Ale7jK4E7fduvEJGwiCwHVgJPFFg+Y0wReDe+Ec1HUac5pXWWc9Pdn9WE5J/NPJrRmo+yF9jxHxNPpEimlAe2dPD64+aOyGp6+lInSCwocOSRp6mmgp7B+IjRUJGs5qOaadzRXGhQ2OCOPDrX/fku8NQYx5wFvA84T0SedX8uBr4EvFFEtgFvdJ+jqpuA24HNOIn3rnGHwxpjyiSS7mjOaj5yb9qts5w+g+x+hfEGhexU2GP1KcQTKZ7de5juSJzXr547Yp/Tl84GxjfyCJwJbENJTae18ERiyYz5FrXh0LQNCoWOPvo74Bqc9BYCPITTt5CXqj5M7n4CgPPzHHM9cH2BZTLGFJnXmdrnaz4aSqaIJVLUhkMsdmsK2cNSh1NcFFhT8PUpqOrYo4+SKe57qY1gQHjdypF9i2cf20zrrOqMZqVC+DOl+msGkXiCpeHhTvPacJCBeBJVHXUeht/mA72849uPEB1y/qZVFUF+cfVfcMLChjGOLK1Cg0II+JrXDOTOHxi9B8kYM+Wlawq+dvaIL4Npc12YylAgR03BTXEx1jyFHH0KsUSKRErz1xTc1drWv9jO2qWzaKwZ2ZI9v7GKhz953liXN4I/Kd4iX3bVgVgyPXENnMV2EiklnkwVvE7zk7u7GYgnueqcFYQCwo0P7OCBre2TLigU2ny0HvDXw6qBeye+OMaYySRXR3Ofb62DQEBobapmb3ZNoX84Gd5ocvUpeOfP16cQrgjQH0vw0it9nH/8yKajo5EvfXYklj0k1QkE41loZ3t7P3XhENddtJprL1zNMS21PLX70NgHllihQaFKVfu9J+7j0QcgG2OmvFwdzV7eI++b/KJZIyewdUfihAJCQ/XojRG5gkJ6LYV8M5qDAbx+4PNy9CccjXSmVN8ENlV1O5qHawQ1boDoH0e/wvb2fo6ZW5cxn+KpPYdGdGqXW6FBISIip3lPRGQtkD+RujFmWsjV0ZydrG7x7JqcQWHWGHmPIHefgtd/UZcjbbb/mCWzazimpa7gaylErtXXokMpUuo0GXnS6bPHMSx1e0c/x/rKe/rSWRweGGJnZ2Tc5Xxwawd7uvIvcHQ0Cg0KHwd+LiJ/EpGHgNuAjxSlRMaYScMLCv0xZ0F77zEM3xhbZ1XTHYlnjMbpisQLSi+Ra56CF3Ty5T7ygsJ5q+cW3MlbqKZ0n0KOmpG/pjDOJTl7Bofo6Itx7NzMoABORtfxiCWSfOL2Z/nCbzaN67hCjRoUROQMEZmvqk8Cq4H/xkl093tgV1FKZIyZFJxmk2Q6CZx3sx4ZFJyW5P2Hh2sLhSTDgzx9CqMsxQnDgWSim47AWS60MhTISIrn9Rv4awq16YV2Cqsp7OhwWt/9QWFFcx1NNRU8Pc6g8LvnX6GzP877X7NsXMcVaqyawncA76/zGuDTOKkuDgE3FaVExphJIeZOEPMmgHmdzZGsNn9vApt/vebxBgX/+ghj1RTOXtnCu9ct5swVc8Z1PYUQERqrKzL6UNLrM2dNXoPCawrb20cGhUBAOG3JLDaMMyjc8uhuVjTXcvaxzeM6rlBjBYWgqna7j98F3KSqv1TVzwLHFqVExphJwWsv9/IHeSNy0qOP3G/Oi92awi5f23hXf6yg5qNw0Lm55uxozlNTOHZuHV9820k511qYCE3VFRnNR94ILH9Hs1e2Qiew7Wh3MrZ68zo8py+dxfb2/pz5lnLZuO8wz+w5zPtes3Rcif7GY8ygICLeJ3M+cJ/vtULnOBhjpiDvhje/wbmRecHAm9Dm3SRb6sMsaqrm6T3ON96hZIreaGLMZHhwZKOPiq2pJjMoDK/PnDlPAQpfknN7ez/Lm2sJBTNvud7kumf2HE5vG8qxZrXnR4++TE1lkLef3lrQ+x6JsYLCrcCDInInzmijPwGIyLE46zQbY6Ypr2nEqyl4zUf9sSGqKgIZN7gzls3iyd3O8MpDXoqLMZLhQf55CpXBQMGTwiZaY3Vlxugjr8bkr7l4AbHQJTm3d/RnNB15Tl7cSDAg6c7mZ/Yc4rQv3MNPHnt5xL7dkTh3PXeAt522iIY8yQInwqhBwU078Qngh8DZOjygNgD8Q9FKZYwpu3RNwQsKg15QSI4YLrp22Ww6+mK83DVQcIoLgGBACAaEeHL4G3d/bKhstQTwFtoZbs6JpGsKw0GqKhREpLCaQnQoyd7uAY7JERRqKkOsWdjAhpe72XdogL/50Qb6Ygm+eu+2dDoMz38/uZd4IlW0DmZPIWs0P6aqv1LViG/bVlV9uqglM8aUlddMNFxTGB595B+eCbBuuZOA7snd3QUnw/Nkr7ncF82/FGcpNFZXZMxojuTo4wgEhJqKYEE1hV2dEVJKzpoCOE1Iz+3t4a9v2UAskeL/vPVEOvtj3PrEnvQ+PQND/PCRXbxmxRxWzas/0ksrSHF6aowxU57XwTqvwQkKfb7RR9nf5I9tqaOxuoInd3ePq6YAI5fXHC0ZXik0VVcQiSfTZfJqAzVZgbA2HCpo9FF6OGqeiXanL53F4FCSbe393PhXp/GeVy/h1ctn8+0HdxAdcpLuffKXG+nqj3PthccdzaUVxIKCMSanfrem0FBVQX04lE6K1x9NZCSHA+eb8xnLZrFh9yG6+wtLhufxsp56+mJlDgo1mfmPIrEEoYCk50d4nPTZYzcfbW/vRwRWtNTmfP3MFXNoqqngC5ev4bVuxtePnb+Stt4YP9+wl5889jK/3/QK1154XHpVuWKyEUTGmJy8mkJNOEh9VcjX0ZxgYY7Fa9Yum829L7az1b0JNtUU3nwUy6op5Dp/qTS65e4ZjNNSH2Yg7izFmT17uqaysCU5t7f3s3hWDVUVuTvOW+rDPPPZN2ac/zXHzOGMZbP42vrt9EaHOPe4Fv767BVHcVWFs5qCMSan9KStyhANvgld/VkZQz1nLHP6Fe7Z3MasmsoRq6HlE85uPipzTcHLlOoNS43EEhkL7HhqKwuvKeTrT/BkBxwR4aPnr6SzP8asmgq+8s6TizYvIZvVFIwxOQ3EkgTESf3QUFWRMaM51037xEWNhEOBETl+xjKiT2GUVddKoSkrfXb2UpyemnAwPfw2n2RK2dkZ4bUrxz/7+Oxjm/mXS47nzBVzmFNXuuVrilZTEJGbRaRdRF7wbfuciOzPWp7Te+06EdkuIltE5IJilcsYU5hI3Ok7EHFSYHuT1/K1+VeGApyyuAkovD/BOy6ezO5oLt44/LGkM6W6NYXuSDydKtuvNhwaM3X2vkMDxBOpcQVJj4jw169dwasWNY772KNRzOajHwIX5th+g6qe4v7cDSAiJwBXAGvcY250V3czxpSJf2GZeremEE+kiCdSeZt3vKGphY48gswhqdGhJPFkqqxDUtOZUgeH+P0Lr/DYzm7OOmZknqXayuCYzUfP7XPm+B5JUCiXogUFVX0I6B5zR8flwG2qGlPVXcB2YF2xymaMGVsknkwPw2yockYfRXIkh/Nb6/YrjLum4AYF75t3OYNCfVUIEdh0oIdrf/EcJ7U28vE3rBqx39I5tbzSG6XTHW2V7Zk9h/j0Hc+zvLmWNQtL+23/aJSjo/kjIrLRbV7yxlctAvb69tnnbhtBRK4SkQ0isqGjo6PYZTVmxnI6WJ2bc0N1BX3RoTHzEp22pInKUICFTdU5X8+lpjKUnvDWHx05UazUAgGhoaqCO57ejyp8492n5Uy+d447fPRP20behzbuO8z7b36C2bWV/OxvXp135NFkVOqg8C3gGOAU4CDwFXd7rm71nGvUqepNqrpWVde2tLQUpZDGGHex+nRNoYKUQltvFMh/066vquDuj57NB/5iWcHvc/axc9jZGWFbW9+YGVJLxetX+OLbT2TJnNwrD69Z2MCc2koe3JIZFLa19fHe7z1OY3UFt151JgsaCw+Qk0FJg4KqtqlqUlVTwHcZbiLaByz27doKHChl2YwxmbyOZhhuzjnQM3pQADh2bn3e5qVcLjlpIQGBu547MJyWu4zNRwBvOmEe17z+GC49aWHefQIB4ZxVLTy0rZNUavg77Dfu344Ct/7NmSwaR41psihpUBCRBb6nbwW8kUl3AVeISFhElgMrgSdKWTZjTCZ/R3ODO0zzgLu62nhu+mNpqQ/zF8c0u0HBGfFTX8bRRwCfueQE/vmC1WPu97pVLXRH4rxwwOlQPjwQ53cvvMJbT13E4tm5axiTXTGHpN4KPAocJyL7ROTDwH+IyPMishF4PfCPAKq6Cbgd2Iyz1Oc1qlr4itjGzFDvv/kJvvvQzqKcOxLPbD4COOgGhYnuCL7s5IW83DXAIzu6gPLXFAr12pXNiJBuQrrj6f3EEymuOGNJmUt25Ir2l1fVd+fY/P1R9r8euL5Y5TFmulFVHt/ZVbSROpFYIr2YTEN1ZvPRRNYUAC541Xz+5dcvcMfT+4Dyjj4ajzl1YU5c1MiDWzv4yHnHctuTezi5tZETFjaUu2hHzNJcGDNFReJJYolUxnrCEyWV0nTOH3A6kAEO9jg1hYnuCG6sruB1x7Wk03OXu6N5PM5d1cLTew7xwNYOtrb1c8W6qVtLAAsKxkxZXe74+J4iBIVBd4EXL+dPg9fRfDiasX0iXXay06lbERTCRVp/uRhed1wLKYVP3/E8NZVB3nxy/s7pqWDq/OWNMRk6+52x/cUICtmT1LyaQnckTnVFcMRawxPhDcfPo6YySF2OjKST2cmtTTRUhTjYE+WykxdOqVpOLhYUjJmiillT8BaW8TqaK0MBqt0JWBPdn+Cprgxy2ckLp9yonVAwkF4HYao3HYFlSTVmyvJmAfcODpFK6YSmVh5el3j4FlFfFWJwKFnUTuAvXP4qkqmc81YntavOWcExc+s4uXXqpLPIx4KCMVOUt+xlSqE/nkgPG50IudYlbqiuoL0vlq49FEOudBJTwcmLmzjZzRA71U3NT8AYk5GIrWdgYpuQBrx1iX0dyl5n81RvMzejs6BgzBTV1T+8wMtE9yvkykHkzWq2oDC9WVAwZorqisTwBulM9FyF4fWZ/X0KFhRmAgsKxkxRXf1xWmc5CdcmvqaQOU8BhpuPijX6yEwOFhSMmaI6++OsaHZW9JrooDCQY/RRuvloiqSgMEfGgoIxU1AqpXRHYqxoqQUmPihE4kkqg4GM0UDe6Ka6SgsK05kFBWOmoMODQ6QUFs+qIRiQiQ8KsUR6KU6PNz/BagrTmwUFY6YgbzZzc32YxuqKItQUhhfY8XjNR9anML1ZUDBmCvLyHjXXVhYnKMQSIyapeR3N9RYUpjULCsZMQV0Rp6Ywpy5MQxGCgj9ttmfZnFpCAWHpnNoJfS8zuVjIN2YK8iauzalzawoD8TGOGJ9IbGTz0bLmWjZ94QLCoeKluTDlV8zlOG8WkXYRecG3bbaI3CMi29zfs3yvXSci20Vki4hcUKxyGTMddPU7E9dm1RSr+SiZkeLCYwFh+itm89EPgQuztn0KWK+qK4H17nNE5ATgCmCNe8yNImL/+ozJoysSZ3ZNJcGA0FgdKkpHs81cnpmKFhRU9SGgO2vz5cAt7uNbgLf4tt+mqjFV3QVsB9YVq2zGTHVd/XHm1FUCzlKWvdEEqhOXcjrXkFQzM5S6o3meqh4EcH/PdbcvAvb69tvnbhtBRK4SkQ0isqGjo6OohTVmsuqKxJhTGwacoJBMaTqJ3USI5OhoNjPDZBl9lGt1kJxfe1T1JlVdq6prW1pailwsYyanrv44s301BZi4Wc1DyRTxRGpER7OZGUodFNpEZAGA+7vd3b4PWOzbrxU4UOKyGTNldPbHaK4df1BQVVJjrGw2EBu5loKZOUodFO4CrnQfXwnc6dt+hYiERWQ5sBJ4osRlM2ZKiCdS9EYTzKlzmo8aCgwKqZTyiduf47JvPjzqfpH4yLUUzMxRtE9dRG4FzgWaRWQf8K/Al4DbReTDwB7gnQCquklEbgc2AwngGlVNFqtsxkxl3trMc7Kaj8ZaU+Gr67dxxzP7Abem4QaVbOn1mS0ozEhF+9RV9d15Xjo/z/7XA9cXqzzGTBfeMpz+jmYYvabw62f28/X12ziptZGN+3p4fn8Prz9ubs59I+5SnHU2+mhGmiwdzcaYAnW5NYXmAjuan95ziGt/uZF1y2dzywfXIQLP7+vJe/5caymYmcOCgjFTjJch1etTqAuHRk2ffeP922mqruA77z2dWbWVHNNSx8Z9h/Oe3xvaaqOPZiYLCsZMMf68RwAiQkNV/lnNOzoirF02i1nuaKWTFjlNSPkcds+TnSXVzAwWFIyZYjojMSqDgYwU1k7+o5GT14aSKfZ2D7C8eTiz6YmtjbT3xWjrjY7YP5VSfvzoyyxqqqZ1Vk1xLsBMahYUjJlivBQXIsNzPvMlxdt3aJBESlnuruUMcFJrI0DO2sLdLxzk+f09/OMbV2UsxWlmDvvUjZliuvpj6aYjT741FXZ19gNk1BROWNBIMCA8n9WvMJRM8eU/bOG4efW89dScWWbMDGBBwZgj8MdNr/Brd8x/qXVH4syuzZxj0FhdkXOews6OCAArfEGhujLIyrl1PJdVU7jtyb3s7hrg2guPIxjIlXnGzAQWFIw5At/70y6+ef/2srx3Z388neLCk6/5aFdnhMbqinQns+ek1kae39+TzqwaiSX42r3bWLdsNuetzj1/wcwMFhSMOQKd/bH0JLJSUlU6czQfeUEhO332rs5IRtOR58TWJrojcfYfHgTga+u30dkf45MXHZfRV2FmHgsKxhyBjv4YhwaGGEqmSvq+vdEEsUSKufVVGdu99NnebGTP7s5IRtOR56RFTmfz8/t6uH9LOzc9tJO/evUSTl86u3iFN1OCzU4xZpyiQ0n6os7wz+5InHkNVWMcMXE6+pxhpHMbRvYpgDOr2UtkNxhPcqAnmrOmsHpBPRVB4Z4X23hgSwer59fz2UtPKHLpzVRgNQVjxsnfbNTRV9ompPZe5/1a6vMEhYHhfoXdXU4n8/KWkUEhHAqyen4Ddzy9n8F4km+85zSqKmyymrGgYMy4dbozisFpRiqldjcI5Wo+gsz8R7s63aCQo6YAziQ2gH97y6s4dm5dzn3MzGPNR8aMU6evdtBZ4pqCNws5u/ko15oKXlBYNid3UPjbc1Zw2pJZvOP01mIU1UxRFhSMGSd/85G/1lAK7X0xqiuCGSkuIPeaCjs7IsxvqMq71vLSObUszRMwzMxlzUfGjJPXjxAOBUrfp9AXY25DeMSw0caakTWF3V0RljVb/iIzPhYUjBmnzv4YDVUh5jdWlXyuQntvlLn1I1dMq6sMEZCRzUf+nEfGFKIsQUFEdovI8yLyrIhscLfNFpF7RGSb+3tWOcpmzFg6++M014dprguXPij0xUZ0MgMEApKR/+jwQJzuSDznHAVjRlPOmsLrVfUUVV3rPv8UsF5VVwLr3efGTDodfTFa6sK01IXLMCQ1OqKT2dNYXcHBHqcjeqyRR8bkM5majy4HbnEf3wK8pXxFMVPdaOsVH63O/phTU6ivLGlNIRJLEIknc9YUAM5d1cK9L7bx3Yd2DgeFHHMUjBlNuYKCAn8UkadE5Cp32zxVPQjg/s6ZlUtErhKRDSKyoaOjo0TFNVPJYzu7OO3f7mFnR39Rzt/R79QUmuvCJU11MTxHIXdN4bOXnsAlJy3g+rtf5FsP7CAYEBbbQjlmnMo1JPUsVT0gInOBe0TkpUIPVNWbgJsA1q5dq2Psbmagp14+RDKlPLv3MCtaJraj1Utx0VxXSVONk5SuVKku2vPMUfCEggG++q5TSKWU373wCsvm1NhCOWbcyvIvRlUPuL/bgV8B64A2EVkA4P5uL0fZzNS3ta0PgC3u74nkNRe11IfTqSZK1a/Q5r7PaAGoIhjg6+8+lbef1spFJy4oSbnM9FLymoKI1AIBVe1zH78J+AJwF3Al8CX3952lLpuZHra2Oc1GW14pRlBwJqs114XTNYVSpbpI1xTyNB95KoIBvvKXJ5eiSGYaKkfz0TzgV+7kmxDwM1X9vYg8CdwuIh8G9gDvLEPZzBSXSKbY4fYlbC1GUHC/rTfXhZnlBoVSpbro6ItRGQqkZy8bUwwlDwqquhMY8TVGVbuA80tdHjO9vNw9QDyR4piWWnZ0ROgZHJrQm6i/+ajJnUVcqlQX7e5QWFsExxST9UKZaWWb24/w5pMXZjyfKF7/wZy6SmoqQ9RWBkvXp9AbZV6eTmZjJooFBTOteP0Jl57kdLK+NMFNSF6Ki3DIWXugub50s5rzzWY2ZiJZUDDTyta2PhbPruaYljrqwqH0SKSJ4qW48JQy1cVos5mNmSgWFMy0srWtj1Vz6xERVs2rm/CagpfiwlOqVBfRoSS90cSYI4+MOVoWFMy0MZRMsaszwsp59QAcN7+BrW19qE7cHEcvxYWnVKkuvMAzt4TrQZuZyYKCmTZ2d0YYSirHzXdmMa+eX8/hgaF0eoiJ4KW48JQq1UVbgXMUjDlaFhTMtOF1Mq+c69QUVrk1homaxOaluGjx3Zi9x92R4g5Lzbc2szETzYKCmTa2tvURENKL0B83f2KDgtdM1FxXmd7WXFeaVBdj5T0yZqJYUDDTxta2PpbMrqGqwhkuOru2kpb68ITlQPKnuPCkg0KR+xXa+2KEAsLsmsqxdzbmKFhQMNPG1ra+dCezZ/X8+omrKfhSXHi8/oVip7po643RUh8mELDZzKa4LCgcgftfaue5vYfLXQzjE0sk2d01wKp5mamyV82rZ1t7H8nU0Y9A8qe48DTXu/mPipzqor0v99rMxkw0CwrjdM/mNj50y5O857uP8dIrveUuzhF7YX8Pf9pW+CJF3ZE4v3hqX9FG2agq929p5/GdXUd0/K7OCMmUpjuXPcfNryc6lBp1wZ1n9hzitf9xH/965wujXp8/xYVntFQXOzv6ueu5AwzGk+O9nJzv3WKdzKYELCiMw6YDPXzstmdYs7CBuqoQH/7hhoI6GLe29XHv5jYSJVqhayy/fmY/b7vxEa68+Qnuf2nsZSu2tfVx+Tcf5p9+/hwfv+3ZCb0OVeXezW1c+v8e5oM/eJJ33fQY//zz5+iNFr6c5t7uAb6+fhvAiKCwbtlsKoLCX33vce57qW3Esf/z3AHeddNj9EcT3PLoy3zoh0/mXcozO8WFJzvVRSqlfO9PO7noa3/io7c+wzn/eT8/+PMuokNHHhza+2KW98iURLlWXiur3ugQdz6zn5Xz6lk1r57ZtWN33rX3RvnrWzbQWF3BzVeeQVtvjHd+5xGu+vEGbv2bM9Odm37b2/v46r3b+O3zB1GFZXNq+Oj5K7ns5IWEguOPx/e91MbX1m/n5NZG/v7cY5nfOL5vjqmU8tV7t/L1+7bz6uWz6Y8l+Idbn+EXf/caVs9vyHnMQ1s7uOanTxOuCPLBs5bxgz/vJhAQbvjLkwu+hkgswaM7ulg1r54lc4aXh9zZ0c+1v9jIhpcPsWR2DV9+58ns7Ojn2w/u4M/bO7nqnBWEc/xd/Tbu6+HnG/YSEOFvX7eC1fMzg8Ky5lp+9fdn8Ynbn+NDP9zA209rZe2yWQBsb+/n+w/v4oxls/jO+9Zy7+Y2Pv2r53n7tx7hQ2ctx0tG2lwXZtW8Otr7MieueZrrwrT3RdnbPcC29j6+/eBOntjVzRuOn8u71y3hu3/ayef/ZzP/+YctVLvXUxMO8r4zl/K+M5dRXZn7GlWd1eN+u/Eg3ZG4DUc1JSETOduz1NauXasbNmwY93GP7+ziXTc9ln7eUh/mdatauPSkBZx1bDMVvpvd3u4BfrPxILc9uYeOvhg/v/o1rFnYCMDvXzjI1T95mhUttZzS2sSq+fWEAsK2tn62tPXx3L7D1FQE+eBZyzlhYQPfuG87mw/2Mru2Mn1zyKcyFOCsY+dw6UkLOX5+A9ffvZnbN+yjdVY1r/RECQSE96xbQlNNBdva+tnW3kd9VQWr5tWxcq4T7FbNr6OlLkx3JM7vN73Cr57ez4aXD/GXa1v597ecSHckzuXffJhQIMCvrzkr3VYeSyT509ZOfrPxAP+z8SAr59bx/Q+cwaKmar794A6+9LuXuOTEBbzxhHkAxJMpdnZE2NbWx+6uCPMaqpwAMLuGDS93c99L7USHUgQDwjtOa+Wa1x/LPS+28R+/f4mqiiCfumg17zi9Nf13f2bPIT7x8+fY2REZ87OsCApXnLGEv3/9MSxorM67XyyR5Gv3buPbD+7A373wtlMX8cW3n5j+9v/oji7+7qdPcXggd21h3fLZ3P63r8nYdvWPn+L3m15JP68Ph/jXy9bw9tMWpdNcP7Kjkz+88ApJ9//bzo4Ij+zoormukg+dvZyFbtnj7qzsra/0selAL6/0RqkICuesbOGzl57AsubaMf8mxoxFRJ5S1bU5X5uJQUFVaeuNsaWtj21tfTy/v4f7XmynL5agsboi3aE3lEyxu2sAgFMWN/HPFxzHWcc2Z5zrV8/s446n97O1rY+2XrfNubaSlfPqWLdsNh84a3m6JqKq/GFTG+tfbGOsfs+ewSEe3t6RvpmqKle/7hg+9oaVtPfG+MZ92/nF0/tIqbJkdg0r59bRG02wta0v44bWVFNBXzRBMqWsaK7lA2ct431nLk3frJ7f18M7v/MIdeFQetGYV3qj9EWdv8WbT17Apy46nrrwcKXym/dv5z//sCWjvKGAsKKllmVzamnri7G9rY9IPElzXZiLT5zP+cfP4/6X2vnZE3uIJ5zmp/NWz+WLbzsx5/KSyZQW1DRXGw5SX1X4egk9g0PpNv5QUDJGEnmiQ8n031BRDvZE2dbWx9a2fl67splzj5ubsf8Tu7r53QsHOXZuHcfNq2f1goaMv1c+T+7u5oZ7tvLIjsx+FO9vuWpePa9b1cKb1sy3hXXMhLKgUADv2/E9m9voizk3BEE4sbWRS05cwOLZNWOcAXoGhkikUszJcaM5EgPxBOtfbOfJ3d285dRFnLZkVsbrhwfihEPBjOYHVaWjP8a2tn62ujey2bUVXHLiQo5fUJ9zgZY/b+/kZ4/vQXH+LTRWV/CmNfM5O6vW5NfWG2XAvbkGRVjQVJWxbyrllKO5LkzQN4zyYM8gP31sD8fMreUtpyyyBWOA/YcH04EyILCgsZrKkHX3meKZUkFBRC4EvgYEge+p6pfy7TuRQcEYY2aK0YLCpPo6IiJB4JvARcAJwLtF5ITylsoYY2aOSRUUgHXAdlXdqapx4Dbg8jKXyRhjZozJFhQWAXt9z/e529JE5CoR2SAiGzo6Cp98ZYwxZmyTLSjk6nXM6PRQ1ZtUda2qrm1paSlRsYwxZmaYbEFhH7DY97wVOFCmshhjzIwz2YLCk8BKEVkuIpXAFcBdZS6TMcbMGJMqzYWqJkTkI8AfcIak3qyqm8pcLGOMmTEmVVAAUNW7gbvLXQ5jjJmJJt3ktfEQkQ7g5aM4RTPQOUHFmSpm4jXDzLxuu+aZY7zXvVRVc47UmdJB4WiJyIZ8s/qmq5l4zTAzr9uueeaYyOuebB3NxhhjysiCgjHGmLSZHhRuKncBymAmXjPMzOu2a545Juy6Z3SfgjHGmEwzvaZgjDHGx4KCMcaYtBkZFETkQhHZIiLbReRT5S5PMYjIYhG5X0ReFJFNIvIxd/tsEblHRLa5v2eNda6pSESCIvKMiPzGfT6tr1tEmkTkFyLykvuZv2a6XzOAiPyj++/7BRG5VUSqpuN1i8jNItIuIi/4tuW9ThG5zr2/bRGRC8bzXjMuKMyghXwSwCdU9XjgTOAa9zo/BaxX1ZXAevf5dPQx4EXf8+l+3V8Dfq+qq4GTca59Wl+ziCwCPgqsVdVX4aTGuYLped0/BC7M2pbzOt3/51cAa9xjbnTvewWZcUGBGbKQj6oeVNWn3cd9ODeJRTjXeou72y3AW8pSwCISkVbgEuB7vs3T9rpFpAE4B/g+gKrGVfUw0/iafUJAtYiEgBqcrMrT7rpV9SGgO2tzvuu8HLhNVWOqugvYjnPfK8hMDApjLuQz3YjIMuBU4HFgnqoeBCdwAHPLWLRi+SpwLZDybZvO170C6AB+4DaZfU9Eapne14yq7ge+DOwBDgI9qvpHpvl1++S7zqO6x83EoDDmQj7TiYjUAb8EPq6qveUuT7GJyKVAu6o+Ve6ylFAIOA34lqqeCkSYHk0mo3Lb0C8HlgMLgVoReW95SzUpHNU9biYGhRmzkI+IVOAEhJ+q6h3u5jYRWeC+vgBoL1f5iuQs4DIR2Y3TNHieiPyE6X3d+4B9qvq4+/wXOEFiOl8zwBuAXaraoapDwB3AXzD9r9uT7zqP6h43E4PCjFjIR0QEp435RVX9L99LdwFXuo+vBO4sddmKSVWvU9VWVV2G89nep6rvZRpft6q+AuwVkePcTecDm5nG1+zaA5wpIjXuv/fzcfrOpvt1e/Jd513AFSISFpHlwErgiYLPqqoz7ge4GNgK7AA+U+7yFOkaz8apMm4EnnV/Lgbm4IxU2Ob+nl3ushbxb3Au8Bv38bS+buAUYIP7ef8amDXdr9m97s8DLwEvAD8GwtPxuoFbcfpNhnBqAh8e7TqBz7j3ty3AReN5L0tzYYwxJm0mNh8ZY4zJw4KCMcaYNAsKxhhj0iwoGGOMSbOgYIwxJs2CgjE+IpIUkWd9P6PODBaRq0Xk/RPwvrtFpPloz2PM0bIhqcb4iEi/qtaV4X1342T77Cz1exvjZzUFYwrgfpP/vyLyhPtzrLv9cyLyT+7jj4rIZhHZKCK3udtmi8iv3W2PichJ7vY5IvJHN4Hdd/DlqxGR97rv8ayIfGc8aY+NOVoWFIzJVJ3VfPQu32u9qroO+AZOJtZsnwJOVdWTgKvdbZ8HnnG3fRr4kbv9X4GH1UlgdxewBEBEjgfeBZylqqcASeCvJvICjRlNqNwFMGaSGXRvxrnc6vt9Q47XNwI/FZFf46SaACfdyNsBVPU+t4bQiLP+wdvc7b8VkUPu/ucDpwNPOul8qGb6JnQzk5AFBWMKp3keey7BudlfBnxWRNYwehrjXOcQ4BZVve5oCmrMkbLmI2MK9y7f70f9L4hIAFisqvfjLPDTBNQBD+E2/4jIuUCnOuta+LdfhJPADpzEZu8Qkbnua7NFZGnRrsiYLFZTMCZTtYg863v+e1X1hqWGReRxnC9T7846Lgj8xG0aEuAGVT0sIp/DWRFtIzDAcKrjzwO3isjTwIM4aaBR1c0i8i/AH91AMwRcA7w8wddpTE42JNWYAtiQUTNTWPORMcaYNKspGGOMSbOagjHGmDQLCsYYY9IsKBhjjEmzoGCMMSbNgoIxxpi0/w914e+zxvBD8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score_l)\n",
    "plt.title(\"Deep Q-Learning for Cartpole\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "618ef7ae-cb31-43f9-aadf-2fc11f05c160",
   "metadata": {},
   "source": [
    "---\n",
    "### 전체코드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "09db0f4d-653d-4e26-b5da-481f76b31a70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:     0 -->  Score: 125.0\n",
      "Episode:     1 -->  Score: 30.0\n",
      "Episode:     2 -->  Score: 31.0\n",
      "Episode:     3 -->  Score: 18.0\n",
      "Episode:     4 -->  Score: 9.0\n",
      "Episode:     5 -->  Score: 13.0\n",
      "Episode:     6 -->  Score: 13.0\n",
      "Episode:     7 -->  Score: 12.0\n",
      "Episode:     8 -->  Score: 9.0\n",
      "Episode:     9 -->  Score: 17.0\n",
      "Episode:    10 -->  Score: 21.0\n",
      "Episode:    11 -->  Score: 21.0\n",
      "Episode:    12 -->  Score: 76.0\n",
      "Episode:    13 -->  Score: 120.0\n",
      "Episode:    14 -->  Score: 10.0\n",
      "Episode:    15 -->  Score: 11.0\n",
      "Episode:    16 -->  Score: 9.0\n",
      "Episode:    17 -->  Score: 8.0\n",
      "Episode:    18 -->  Score: 10.0\n",
      "Episode:    19 -->  Score: 9.0\n",
      "Episode:    20 -->  Score: 11.0\n",
      "Episode:    21 -->  Score: 14.0\n",
      "Episode:    22 -->  Score: 10.0\n",
      "Episode:    23 -->  Score: 47.0\n",
      "Episode:    24 -->  Score: 41.0\n",
      "Episode:    25 -->  Score: 63.0\n",
      "Episode:    26 -->  Score: 35.0\n",
      "Episode:    27 -->  Score: 84.0\n",
      "Episode:    28 -->  Score: 55.0\n",
      "Episode:    29 -->  Score: 47.0\n",
      "Episode:    30 -->  Score: 58.0\n",
      "Episode:    31 -->  Score: 75.0\n",
      "Episode:    32 -->  Score: 35.0\n",
      "Episode:    33 -->  Score: 38.0\n",
      "Episode:    34 -->  Score: 37.0\n",
      "Episode:    35 -->  Score: 41.0\n",
      "Episode:    36 -->  Score: 109.0\n",
      "Episode:    37 -->  Score: 54.0\n",
      "Episode:    38 -->  Score: 52.0\n",
      "Episode:    39 -->  Score: 32.0\n",
      "Episode:    40 -->  Score: 40.0\n",
      "Episode:    41 -->  Score: 41.0\n",
      "Episode:    42 -->  Score: 36.0\n",
      "Episode:    43 -->  Score: 66.0\n",
      "Episode:    44 -->  Score: 74.0\n",
      "Episode:    45 -->  Score: 36.0\n",
      "Episode:    46 -->  Score: 122.0\n",
      "Episode:    47 -->  Score: 130.0\n",
      "Episode:    48 -->  Score: 119.0\n",
      "Episode:    49 -->  Score: 239.0\n",
      "Episode:    50 -->  Score: 189.0\n",
      "Episode:    51 -->  Score: 93.0\n",
      "Episode:    52 -->  Score: 164.0\n",
      "Episode:    53 -->  Score: 149.0\n",
      "Episode:    54 -->  Score: 93.0\n",
      "Episode:    55 -->  Score: 102.0\n",
      "Episode:    56 -->  Score: 146.0\n",
      "Episode:    57 -->  Score: 88.0\n",
      "Episode:    58 -->  Score: 77.0\n",
      "Episode:    59 -->  Score: 147.0\n",
      "Episode:    60 -->  Score: 103.0\n",
      "Episode:    61 -->  Score: 119.0\n",
      "Episode:    62 -->  Score: 241.0\n",
      "Episode:    63 -->  Score: 224.0\n",
      "Episode:    64 -->  Score: 299.0\n",
      "Episode:    65 -->  Score: 206.0\n",
      "Episode:    66 -->  Score: 199.0\n",
      "Episode:    67 -->  Score: 196.0\n",
      "Episode:    68 -->  Score: 176.0\n",
      "Episode:    69 -->  Score: 205.0\n",
      "Episode:    70 -->  Score: 193.0\n",
      "Episode:    71 -->  Score: 167.0\n",
      "Episode:    72 -->  Score: 181.0\n",
      "Episode:    73 -->  Score: 168.0\n",
      "Episode:    74 -->  Score: 163.0\n",
      "Episode:    75 -->  Score: 185.0\n",
      "Episode:    76 -->  Score: 145.0\n",
      "Episode:    77 -->  Score: 179.0\n",
      "Episode:    78 -->  Score: 166.0\n",
      "Episode:    79 -->  Score: 17.0\n",
      "Episode:    80 -->  Score: 23.0\n",
      "Episode:    81 -->  Score: 18.0\n",
      "Episode:    82 -->  Score: 22.0\n",
      "Episode:    83 -->  Score: 20.0\n",
      "Episode:    84 -->  Score: 201.0\n",
      "Episode:    85 -->  Score: 313.0\n",
      "Episode:    86 -->  Score: 244.0\n",
      "Episode:    87 -->  Score: 306.0\n",
      "Episode:    88 -->  Score: 237.0\n",
      "Episode:    89 -->  Score: 319.0\n",
      "Episode:    90 -->  Score: 460.0\n",
      "Episode:    91 -->  Score: 352.0\n",
      "Episode:    92 -->  Score: 333.0\n",
      "Episode:    93 -->  Score: 297.0\n",
      "Episode:    94 -->  Score: 290.0\n",
      "Episode:    95 -->  Score: 262.0\n",
      "Episode:    96 -->  Score: 350.0\n",
      "Episode:    97 -->  Score: 336.0\n",
      "Episode:    98 -->  Score: 325.0\n",
      "Episode:    99 -->  Score: 288.0\n",
      "Job completed!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Score')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABHtklEQVR4nO2deZykVXX3v6equmvp7uq9p3v2fWEYYGQYB0FEFkFQwC1iohKjIfqSqIl5FWJMNImvmvjG5XWJxCUYFMUVNIosKpvAMMAAM8MMs+/T+1rdtd/3j+d5qqurq7qrl+qu7jrfz6c+XXWfpe5TVX1/zznn3nPEGIOiKIqiALhmuwOKoihK8aCioCiKoqRQUVAURVFSqCgoiqIoKVQUFEVRlBQqCoqiKEoKFQVFmQZE5D9E5BMFOvebROS4iAyIyOZCvEcxISKXisiJ2e5HqaKiUMKIyBERGRKRfhHpEZE/iMj7RWRWfhcislhEvicinSISEpHtInLNOMcUxQBijHm/MeafC3T6zwN/aYypNMY8Nx0nFJGrROQR+7tvF5GHReS6KZzviIhcMR19U2YXFQXljcaYKmAZ8FngY8C3ZroTIlIHPAZEgY1AA/AF4AcicsNM9ycdEfHM5vtjfTe7J3OgiLiztL0V+BHwXWAxsAD4B+CNkzj/bH82ynRjjNFHiT6AI8AVGW1bgSRwtv3ai3WnegxoBf4D8Kft/wZgJ9AD/AE4J+P8twF7gG7gO4AvR1/+GdgFuDLaPwYcAiTHcZcCJ3JsWwj8BGgHDgMfzLjOJ+x+nwa+ApSnbTfALcB++9hLgRPAR4A2+5j3pO3/X8C/pPdpjH3rgV8AfcDTwL8Aj2XpvxcYsPsSAg7a7RuA39t93w1cl9GPrwO/so/J/H7F/i7/9xi/i1XAb4FOoAP4HlCT8b1+DHgBiAB32b+ZIbu/HwWW2/2+GThlfwYfybi2L9rbTtnPvdm+07G+R30UYFyY7Q7oYxa//CyiYLcfAz5gP/8icC9QB1TZg9ln7G2vsAe9VwJu4Cb7nN608+8CltjHP+4MnFne80ngU1naV9iDy5ocx40YQNLaXcAzWHfA5cBKLHG5yt5+PrAN8NgD2EvAh9OON8ADdr/99vvEgX8CyoBrgEGg1t7/vxgpCmPt+wP7EQDOAo6TRRQy+rLafl4GHAD+zr6uy4B+YF1aP3qBi+zPwJdxrvX2+VaM8X6rgSuxBu5G4BHgixm/m5329+rP9ltiWBTuAiqATViD+hX29n+yv/Mm+z3+APxz5nc63veojwKMC7PdAX3M4pefWxSeBD6OdVcZAlalbbsQOGw//7rzj5y2fR/wmrTzvz9t2zXYd7tZ3vNA+r5p7T57cHlVjuNSA0hG+yuBYxlttwHfyXGeDwM/S3ttgMsy3mcI8KS1tQHb7Of/xUhRyLovlnjGsAdxe1tWSyGjL44ovBo4Q5pFZQ+8n0zrx3fHONdF9vmyWmw5jrkBeC7jd/NnY/2WGBaF9Wlt/wp8y35+ELgmbdtVwJHM73Si36M+pv5Qf6CSjUVAF9YdXAB4RkScbYI1sIHl675JRP4q7dhyLHPf4Xja86MZ29LpAFqytDtt7SKyFMsVBYAxpnKMa1gGLBSRnrQ2N/AogIisBf4d2IJ1jR6sO9J0jme87jTGxNNeDwK5+pBr30b7vdLPnfk+Y7EQOG6MSaa1HcX6zvI5X6f9twXLFTMKEWkCvowlQFVYd+vdGbvl2+fM73+T/Xyh/Tp9W7bfxpjfozL9aKBZGYGIXIA1wDyGNVAPARuNMTX2ozptMD4OfDptW40xJmCMuSvtlEvSni/F8h9n40HgLVlmPv0Rln/+oDHmmLFm4FSOIwhO3w5n9K3KGOPMZvo6sBfLLRXEcsdIxjkKkUK4Hcu1tDitbUmOfbNxCliS8TktBU6mvR6r3/uwPpu3jLHPZ+xznGN/Nu9k/M8m13vm+v5PYQ342balM973qEwzKgoKACISFJE3YPm67zTGvGjfjf4n8AX77hERWSQiV9mH/SfwfhF5pVhUiMi1IlKVdupb7KmmdVgD7w9zdOELQBD4log0i4hPRN4BfAL4x4w742z996U/gO1An4h8TET8IuIWkbNt0QPrDrgPGBCR9cAHJvaJTQ5jTAL4KfBJEQnY7/3uCZziKSyX3kdFpExELsWaNfSDPN/fAH8DfEJE3mN/7y4RuVhEbrd3q8IKGPeIyCLgf+dx6lYsf38mn7CvcyPwHoa//7uAvxeRRhFpwIoZ3Jnl+PG+R2WaUVFQfiEi/Vh3ZB/Hcqm8J237x7D8/U+KSB/WHf06AGPMDuDPsWbudNv7/WnG+b8P3I8VHDyE5T8fhTGmE7gYK4awB2tQ+i5wizHm2+NcwyIsiyb9sQJrsDwPy03SAXwTqLaP+Vvgj7GCtP9JbrEqBH9p9+MM8N9YA2QknwONMVHgOuD1WNf0NeDdxpi9+b65MebHwNuBP8O6O2/F+l7usXf5FNYkgl7gf7BEbDw+gzXI94jI36a1P4z1u3gI+Lwx5n67/V+AHVgzmF4EniXLb8MW0bG+R2WaEevGQVGmHxE5ArzPGPPgJI4NYs1W+pkx5h+mu2/FhIh8Dmg2xtw0232ZLkRkOdYgXpYRW1GKHLUUlKLEGNOHNVspISLNs92f6URE1ovIObbLbSvwXuBns90vRQF09pFSvBhjjmO5MuYbVVguo4VYU1X/L8OuG0WZVdR9pCiKoqRQ95GiKIqSYk67jxoaGszy5ctnuxuKoihzimeeeabDGNOYbducFoXly5ezY8eO2e6GoijKnEJEjubapu4jRVEUJYWKgqIoipJCRUFRFEVJoaKgKIqipFBRUBRFUVKoKCiKoigpVBQURVGUFCoKiqIoU+R07xAP7mmd7W5MCyoKiqIoU+TOJ4/yF3c+QyI593PJqSgoiqJMkf5wnETS0B+OzXZXpoyKgqIoyhQZiFh1hPqG5n49IRUFRVGUKRKyRaF3SC0FRVGUkicUSQAqCoqiKAoQitruI40pKIqiKOo+UhRFUVKo+0hRFEVJMTz7SEVBURSlpDHGqPtIURRFsYgmksTtlcwqCoqiKCWOE08A6Avr4jVFUZSSxnEdgVoKiqIoJY8TZC5ziwaaFUVRSp1Be+FaS7VfRUFRFKXUGbBjCi3VPnqHYhgzt9NnqygoiqJMASemsKjGTzxpGIwmxjmiuFFRUBRFmQJOTKGlxgfM/fxHKgqKoihTYDAyHFOAuT8DSUVBURRlCoRsd9GiGlsUBlUUFEVRSpaBSJwyt9BQ6QXUUlAURSlpQpE4FV4P1f4yYO6valZRUBRFmQKhSIKKcg9BvwdQS2FcRMQtIs+JyC/t13Ui8oCI7Lf/1qbte5uIHBCRfSJyVaH7piiKMlUsS8FNlc+yFFQUxudDwEtpr28FHjLGrAEesl8jImcBNwIbgauBr4mIewb6pyiKMmlCUct95HYJVT7PnF/VXFBREJHFwLXAN9OarwfusJ/fAdyQ1v4DY0zEGHMYOABsLWT/FEVRpspAJE6l13IdBX1lKgrj8EXgo0AyrW2BMeY0gP23yW5fBBxP2++E3TYCEblZRHaIyI729vaCdFpRFCVfQpE4gXLLqVHtL1P3US5E5A1AmzHmmXwPydI2KomIMeZ2Y8wWY8yWxsbGKfVRURRlqoQiCSpsS6HaXzbnVzR7Cnjui4DrROQawAcEReROoFVEWowxp0WkBWiz9z8BLEk7fjFwqoD9UxRFmTKhaJr7yO/hcEdolns0NQpmKRhjbjPGLDbGLMcKIP/WGPNO4F7gJnu3m4B77Of3AjeKiFdEVgBrgO2F6p+iKMp04KxTgPnhPiqkpZCLzwJ3i8h7gWPA2wCMMbtF5G5gDxAHbjHGzO10g4qizGsi8QSxhElZCtX+MvqG5vbitRkRBWPM74Hf2887gctz7Pdp4NMz0SdFUZSpMmjXUnACzUFfGUOxBNF4knLP3FwbPDd7rSiKUgQ4abNT7qPA3F/ApqKgKIoySUJ2Kc70dQowt2sqqCgoiqJMklCmpeBXS0FRFKVkCdkxhQonpqCioCiKUrqMthSsv3M51YWKgqIoyiRxAs3Di9fsmIKKgqIoSumRaSkE50H6bBUFRVGUSeLUZ3bWKfjK3Hg9rjldfU1FQVEUZZKEInE8LsGbtlCt2l9G76BaCoqiKCWHk/dIZDjJ81zPf6SioCiKMkkGIolUkNkhqKKgKIpSmqQX2HGY6zUVVBQURVEmiVOfOR11HymKopQoobT6zA5Bn0dFQVEUpRSxSnGOdh8NROIkk6OqCc8JVBQURVEmyUBktPuoocqLMXC8e3BS53z8QEdqUdxsoKKgKIoySQajcSrKR4rC5RsWAPDz5yZeYr5nMMo7v/UU//I/e6alf5NBRUFRFGWSWO6jkaKwqMbPhSvr+elzJzBmYi6kjoEIxsCPdpzgeNfkLI2poqKgKIoyCaLxJNFEksqMmALAW85fzNHOQZ452j2hc3YORAGIJw1f+/2BaennRFFRUBRFmQSZyfDSufrsZvxlbn7y7IkJnbN70BKFrcvrZs1aUFFQFEWZBE4pzsyYAliptF9/djO/fP404Vgi73N2hixR+LtrN+AS4au/m3lrQUVBURRlEqSqrmWxFMByIfVH4jywpzXvc3bborChpYp3bF3Cj5+ZeWtBRUFRFGUSDKTcR6NjCgDbVtbTUu2bkAupMxSl0uvB63HzgUtX43IJn/n1S9PS33xRUVAURZkEoYyqa5m4XcKbNi/ikZfb6RyI5HXO7lCUuopyAJqrfXzwstX86sUzE7I2poqKgqIoyiRwRCGQJabg8Jq1jSQNvHCyN69zdoai1NqiAHDzJatY31zFJ36+i/4ZSrKnoqAoijIJnKpruSwFgLMWBgHYnacodA9GqU8ThXKPi8+8eROt/WH+9b59U+ht/qgoKIqiTILQODEFgCpfGSsaKth1si+vc3YNRKkNlI9o27y0lj991XL++8mj7DjSNfkO54mKgqIoyiQYGGOdQjobFwbZdSo/S6FrMEp9Zfmo9r993TpqA2V876ljE+/oBFFRUBRFmQTZ6jNn4+xF1ZzoHqLHXpiWi6FognAsOcpSAEt4Ni+tZXee4jIVVBQURVEmgVN1Lb0+czbOXlgNwO5TY7uQOkPWDKW6irKs289qCXKwPTShxXCTQUVBURRlEvQMxajJclefyUY72LxrnGBzd8iaXVRX4c15nkTSsO9M/wR7OjFUFBRFUbDSVk8oJcVAdv9/JrUV5Syq8bNripbCxjwtjqmioqAoigLcePuTE5r22TEQoT7HXX0mZy8Kjjst1UmGl8tSWFzrp8rrYc/pwsYVVBQURVGA412DHGgfyHv/zlCUhjwsBbDiCoc6QmMuQHPSZtflcEm5XMKGhUG1FBRFUQpNPJEkFE3Q1hcetS0STxCJj3QrJZOG7lB+7iOwZiABvHQ6dzygezCK2yUE/WMshmsJsvd0P4kC1n8umCiIiE9EtovI8yKyW0Q+ZbfXicgDIrLf/lubdsxtInJARPaJyFWF6puiKEo6/WFrzUFrFlH4m7uf56++/9yItr5wjHjS5O0+2rho/GBzV8hauDbWbKaNC4MMxRIc7gjl9b6ToZCWQgS4zBhzLnAecLWIbANuBR4yxqwBHrJfIyJnATcCG4Grga+JSO6lgoqiKNNEn+3W6R6MjbIKXjrVx77WkXf4HbarJ19LoanKR1OVd8xFbF2hkSkusjEcbC5cXKFgomAsHAddmf0wwPXAHXb7HcAN9vPrgR8YYyLGmMPAAWBrofqnKMowxpgJ1xOeT/QNxVPP2/qGM5oaYzjVO0RrX3jE5+NkPW2ozM9SAMuFtHuMdBfdoRi1OWYeOaxuqqTMLewpYFyhoDEFEXGLyE6gDXjAGPMUsMAYcxrA/ttk774IOJ52+Am7LfOcN4vIDhHZ0d7eXsjuK0rJ8MsXTrPlXx4s+MKoYqUvLQDc1j/sQuoZjBGOJQnHkvSFh4XDqZCWr6UAcPbCIPvb+hmKZv+MO0Pjz2Yq97hYu6CKPafnqCgYYxLGmPOAxcBWETl7jN2zOdJG3boYY243xmwxxmxpbGycpp4qSmmz+1QfnaFoarArNfqGhkWhNc1SONU7lHqeHoR2LIV8YwoA5y6pIWng6w8fzLq9e3B8SwGsuMLuU30Fs+xmZPaRMaYH+D1WrKBVRFoA7L9t9m4ngCVphy0GTs1E/xSl1Gnvtwa57lIVhXC6KAwP/qd7wmntw2LRMRBFBGoD4w/iDq9d18Tbzl/Mlx/azxceeHnEtkTS0D0YzblGIZ2zWoJ0haKcyRIUnw4KOfuoUURq7Od+4ApgL3AvcJO9203APfbze4EbRcQrIiuANcD2QvVPUZRh2u073+5xkrbNV5yYgsjIwf90mqWQLhadoQi1gXI87vyHUJdL+NxbzuFt5y/mSxnC0DsUwxioy0NkNtrTWwsVVxg75+vUaAHusGcQuYC7jTG/FJEngLtF5L3AMeBtAMaY3SJyN7AHiAO3GGNK08GpKDNMylIYnJnqXsVGXziGS6Cl2j/CTXSqN4zbJSSSZsSdeefA+DOFsuEIg0uELz20n1euqONVqxvoclJc5BG43tBiF+451cflGxZMuA/jUTBRMMa8AGzO0t4JXJ7jmE8Dny5UnxRFyU7Ju4+GYgT9ZSwIekcM/qd7hmip9tE7FMuIKeS/cC0Tl0v45HUb+elzJ3h4f7stCnYyvDwS7FV6PSyvD8xJS0FRlDlAImlSd6ql6j7qHYoR9JWxIOjj5bQ1Cad6wyys9uMrc4+MKYQiqTv2yeAvd7N5SS1PHrIqqTmffz6BZoAvv2MzTVW+Sb//WGiaC0UpcTpDEZysCT0l6z6KE/R7WBD0jVincLp3iJYaHwuCXlr7p+4+Smfbyjp2neylPxxLWQr5zmY6Z3ENzdUqCoqiFADHdQTWqtpSpM+2FJqCXvojcUKROMmk4UxvmJZqPwuqhsUiGk/SOxSb0HTUbGxbWU8iadhxpHvClkIhUfeRopQ4jii4XVKy7qO+cIyVDZU0B62777b+CBVeN7GEYWGN0xa2EuENTnzhWjZesayWcreLJw91EksYKr0evJ7Zz+yjoqAoJY4jCsvrA6UrCkPD7iOwpp/6y6wBuqXaTyJpiCUsQehIpbiYmij4ytyct7SGJw51sqqxsiisBFD3kaKUPM4ahbULqlIlIUuNvrATaLZcQq194dQahZZqX8qCaO2LpOoe1E8g71Eutq2sZ9fJXo52hvJauDYTqCgoSonT3h+h0uuhpdpPTwlaCrFEksFogqC/jCbHfdQX4ZS9mnlhjT/V3tofTpXNnGqgGaxgc9LAc8d78lq4NhOo+0hRSpz2/giNVV7qKsoIRa2CMsXg254pnFoKQZ+HKq8Hf5mb1r4wIuD1uKgNlDFoWxBtfeHU/tNhKbxiqRVXiCaSaikoilIctPdHaKz0UmMvnCq1aalOMrzqQBkiYk8/jVhrFGr8iAiNVY5bKULHQJQytxD0Tf2e2lfmZvPSGgDqNKagKEox0D5gWQq1tiiUWrDZSYYX9FmDclPQZ8UU7NXMAF6Pm7qKclr7wnTZKa7HqpA2EbatrAdQS0FRlOLAcR85s19Kba2Ckwwv6Leu31rAFua0vUbBoanKmwo0T3U6ajrDolAcloLGFBSlhAnHEvSH4yMshZJzH2VYCguqvNzfGyaWSKbWKIAtFv1hRGRa4gkOW1fU8fFrNnDVxuZpO+dUyNtSEBG/iKwrZGcURZlZnDUKjZUl7D6yYwpBv3WPvCDoIxJPkjSMsBQWBL209oXpHIjQMA0zjxzcLuHPL1mZiunMNnmJgoi8EdgJ3Ge/Pk9E7i1gvxRFmQGcNQqNVV5q7CmRpZYpdZSlkJZTqCXNUmgO+mjvj9DeH5lW91Gxka+l8ElgK9ADYIzZCSwvRIcURZk5UpZClRdfmZtAubvkair0DcVxu4RAuTUNd0HVsGtoYXpMIegjaSAST06r+6jYyFcU4saY3oL2RFGUGSddFABqA+Ul5z6y0mZ7UrOJnFQXMNJSSG+fjoVrxUq+geZdIvLHgFtE1gAfBP5QuG4pijITtPdHEIE6e5CrrSgrSfeRM/MIoMleqFbp9aRcSkAqBQZMPRleMZOvpfBXwEYgAnwf6AU+XKA+KYoyQ7QPRKgLlFNm1xq2LIVScx/FRgz+gXIPVT5Pao2Cw0hLYf66j8a1FOway/caY64APl74LimKMlM4axQcagPlHOsanMUezTxOgZ10Flb7R0xHBctl5BJImvltKYwrCsaYhIgMiki1xhUUZW7z6P52njnazYevWAtkE4USdB8NxWiqqhzR9vm3nYu/fGT+J4/bRUOll7b+SGlbCjZh4EUReQAIOY3GmA8WpFeKMke5b9cZfre3jc+99ZzZ7kpWfvbcSX767Emu2dTC2gVVtPdHWNlQkdpeEyinLxwnnkjicZdGwgMnbXY6mxZXZ913QdBHKBIfJRjziXy/9f8BPgE8AjyT9lAUJY3f72vjhzuOE44lZrsrWRmwM3ze+eRRjDGpvEcOTsC5Z6h04gpOgZ18aK720VA1f60EyNNSMMbcISLlwFq7aZ8xpnR+NYqSJ05a5WNdg6xdUDXLvRmN07+fPnuS979mFdF4coQoOAvYegajNMzjufgO0XiSoViCan9+eYf+5sq1837Kbl6iICKXAncARwABlojITcaYRwrWM0WZgzirY490hIpTFCIxmqosv/h/PnoIYFSgGSiZGUj9zmrmPEVhQ0uwkN0pCvKNKfxf4HXGmH0AIrIWuAs4v1AdU5S5iHMnfrSzOGfw9IfjbFtZz8H2Ab731DHAynvk4LiPSiVTal+qwE5xZCgtBvKNKZQ5ggBgjHkZ0E9RUTJw7jyPdIbG2XN2GAjHqfJ5eOe2ZUTjSYCc7qNSIDMZnpK/KOwQkW+JyKX24z/RQLOijGIuWApVvjKuP28hVV5rICxl91FmMjwlf1H4ALAbK73Fh4A9wPsL1SlFmas4olCMlkI4liCaSFLl8xAo9/COVy6lrqJ8RJA1UO6m3OMqmbUKmQV2lPxjCh7gS8aYf4fUKuf5PzVBUSZALGHNZCl3uzjVM0QknsDrKZ757I5gVdm1hT961Tr+4pKVI8pKioi1gK1U3EdqKYwiX0vhIcCf9toPPDj93VGUuYuzBmBdcxVJAye6h2a5RyMZiIwUBY/blTUFdCnlP9KYwmjyFQWfMWbAeWE/DxSmS4oyN3HuxM9eZK2GPVpkLiQnCF7lHfuuuDZQXjLuo96hGB6X4C8rHotutslXFEIi8grnhYhsAYrrNkhRZhnHFbHJFoUjHcUVbM50H+WitqK03EdBf9kIF1qpk6/N9GHgRyJyCjDAQuDtheqUosxFnEF3WX2AKp+naC2FynFEoaak3EdxguN8HqXGmJaCiFwgIs3GmKeB9cAPgThWrebDM9A/RZkzpNwzPg/L6gMcKbJpqf15LtSqC5TTMxglmTQz0a1ZJbPAjjK+++gbgGNHXgj8HfBVoBu4vYD9UpQ5x7B7poxl9RVFaCnk5z6qCZSRNMPusPlMZoEdZXxRcBtjuuznbwduN8b8xBjzCWD1WAeKyBIR+Z2IvCQiu0XkQ3Z7nYg8ICL77b+1acfcJiIHRGSfiFw1lQtTlJkm3VJYXh/gRPcQsURylns1jCMKld6xRWFVk1Vb4DO/2puXtRCKxLn0337HPTtPTr2TM0y2AjulzriiICLOJ3Y58Nu0beN9knHgI8aYDcA24BYROQu4FXjIGLMGa6rrrQD2thuxyn5eDXzNXg+hKHOC9DvxZfUVxJOGUz3FMx+jPxzDX+Yet07CpWsb+eDla/jhjuPc+tMXxhWGHUe7OdI5yKd+sYfeOZZyu28olneG1FJhPFG4C3hYRO7Bmm30KICIrMaq05wTY8xpY8yz9vN+4CVgEXA9VsZV7L832M+vB35gjIkYYw4DB4CtE70gRZkt+iNxyj0uvB43y+utwjXFFFcYiMTHdR2BtYDtb65cy4cuX8PdO07w0Z+8QMhe45CN7Yc7cbuEnsEoX3zw5enscsHpD8fHtZxKjTE/DWPMp0XkIaAFuN8Y49wyuIC/yvdNRGQ5sBl4ClhgjDltn/+0iDTZuy0Cnkw77ITdlnmum4GbAZYuXZpvFxSl4PSHY6mZLMvrrWU8VlyhcRZ7NUx/OD9RcPjrK63yKV96aD+/2X2Gt52/hHdfuIzlaZXaALYf7mLTomo2Lgzy3SeO8o6tS4sybXgmyaRhKJYgUK6ikM646xSMMU8aY35mjEkvw/myYwWMh4hUAj8BPmyM6Rtr12xvn6U/txtjthhjtjQ2Fsc/m6KA5Z+usoOWjVVe/GXuolqr0BeOpfqXL3995Vp+8oFXcem6Jr77xBEu//eHee5Yd2p7OJbg+eO9vHJFHR953Toqyt186he7Gb5/LF4idpbY+VxaczIUtAiriJRhCcL3jDE/tZtbRaTF3t4CtNntJ4AlaYcvBk4Vsn+KMp2k34mLCMvqA0U1A2miloLD+ctq+X/v2MxjH7sMn8fF3TuOp7btPN5DNJFk64o66irK+cjr1vH4gU4e2NM6nV0vCEN2yVRdzTySgomCWEsEvwW85CTSs7kXuMl+fhNwT1r7jSLiFZEVwBpge6H6pyjTTX84NmLQXV5fUVTZUvONKeSiudrH6zY286sXzxCJWwPq9sNdiMCWZXUA/Mkrl9JU5eXe54v/fk5FITuFtBQuAt4FXCYiO+3HNcBngStFZD9wpf0aY8xu4G6stNz3AbcYY4qz+rmiZKE/HB+RV2hZQ4DjXUNFswisPxwbN+/ReFx33kJ6h2I88nIHYInC+uYg1XZxHo/bxcVrGnj8QEfRXHcuhqLW8OJT99EICiYKxpjHjDFijDnHGHOe/fiVMabTGHO5MWaN/bcr7ZhPG2NWGWPWGWN+Xai+KUohyLQUmqp8RBPJ1FTV2Way7qN0Ll7dQF1FOffsPEkskeSZo928ckXdiH1evaaB7sEYu0+NFULMTigSp70/MqU+5ktYLYWsFDSmoCilRH9aoBmgxp7/XgzJ5eKJJIPRxLh5j8ajzO3i2k0tPPhSK08d6mIolmBrhihctLoBgEcPtE/4/B//2Yu8/fYnptTHfFFRyI6KgqJMA4mkYTCaGHEnXltRPKIQilgD4ERnH2Xj+vMWEo4l+T+/egmAC5aPFIWmKh/rm6t4bH/HiPZHXm5nzxjWQziW4P49rRxqD81Iio1UTKFch8F09NNQlGlgIEteoRq73nFPEWQc7UtLwTFVXrG0lkU1fvac7mNVY8WIGs8Or17TwI4j3Sm/fWtfmPd9dwfvv/MZovHsqT8e3d/BoL3/y2f6R2w70DbAfzx8cFqnujp9K6bqeMWAioKiTAPZyjrWOqIwNPuWwnCG1KmLgsslXHfeQgC2rqjPus/FaxqJJpI8dbgTgK///iDReJJjXYPctf1Y1mPu23WGco81JO3NEIX/fuIIn/31Xn6z+8yU++8wbCmoKKSjoqAo00C2DKSpmEJo9i2FVC2FKc4+cnjT5kW4XcJr1jZk3b51eR3lHheP7e/gTG+Y728/xh9tWcy2lXV8+aH9qdKgDrFEkgdfauXaTS1U+TzsyxCF509YWXX+9b59xKcpyaDGFLKjoqAo08BwhtThQdeq6AU9RRBTyKzPPFXWLqjiidsu46qNzVm3+8vdXLC8lscOdPAfDx8kmTT81WVruO31G+gMRbn9kUMj9t9+uIveoRhXn93MugVVI0QhGk+y53Qf65urONQR4odpi+emguM+UlEYiYqCokwD2SwFt0uo9pcVRRWzfGspTISmKt+YZSwvXt3I3jP9fP+pY7z1/MUsqQtw7pIart3UwjcfPURbfzi17327zuAvc3PJmkbWNVex90xfKn7wcms/0XiSW167mguW1/LFB/czGJ36NN+hmKa5yIaKgqJMA/2R7IHc2kA5PUWQTjqbJVNoXr3Gci0ljeGW1w6XX/nbq9YRjSf5+5/tYjAaJ5k0/Gb3GV6zthF/uZv1zVX0heOc6bNE4/kTPQCcu7iGW1+/nvb+CN96dOqFH52Ygtejw2A6+mkoyjSQXnUtnWp/WVb30UwnjOsrgKUwHme1BFlc6+ePX7mUJXWBVPuKhgo+evU6HniplTd8+THufOoobf0Rrj7bckWtaw4Cw8HmF473UhsoY0mdn/OX1XHVxgV845FDKffPZAnHEvjL3GNaO6WIioKiTAO53DO1gbJR6xSePtLFuZ+6n4PtAzPWv4FInDK3zOhdscslPPDXr+Ef37hx1LabL1nF99+3jaFYgn+4ZzdlbuG1660s+uvstNtOXOH5Ez1sWlyTGryv2LCAgUicjoGprXweiibUdZQFFQVFmQb6wjHK3S58GUHL2kD5qNlHu0/20heO842HD85Y//rttNkzfVfsL3fjdmV/zwtX1XPfhy7hrecv5j0XrUhVQKsOlNFS7WPfmX6Gogn2tw1w7uLq1HEVdlGcwSlaCkO2paCMRKtLKMo0kCuvUE2gfFSJylY7t8/PnjvJX1+5lpZq/6z1b7apDpTx+bedO6rdCjb3s/tUL4mk4ZzFNaltAfvuPjTFYPNQLIGvTO+LM9FPRFGmgdyiUMZAJD5iFW9bX4RKr4ekYVoCplPpX7GyrrmKg20DPGsX9MlqKUSmGFNQ91FWVBQUZRroz1HVrNZOKZ2+qrmtP8yqxgquP3ch399+bEbWMQzMsVrE65uriCaS3LPzFM1BH01BX2pbhV0+M3MB3ERR91F2VBQUZRoYy30EI/MftfVFaKzy8RevWcVgNMF3nzha8P5NphTnbLJugTUDafepPs5JsxIAKrzWQD7VtQqW+0hFIRMVBUWZBjJrKTjUZhOF/jALgl7WNVdxxYYmvvP44WlZjDV2/+aW+2hVU0UqQH3ukpoR2wK2pRCaaqA5qpZCNlQUFGUayKyl4FATGJk+OxJP0D0Yo6nKcoe868LldA/GeOZo95jn7xmMkhinktmvXjzNHw52ZN3WH46NSNZX7Hg9blY2VACwaVEOS2GK7qNwTGMK2VBRUJRpYKxAMwznP3Kqii0IWumm1zRVAnC8ayjnuQejcV79ud9x9zg5fz7767186cH9o9qNMQxE5lZMAaxgMzDKfeTzuBGZBktBYwpZmVu/EkUpQhJJa9DNHmi23EdO/qM2WxSabFFYEPThcQknugdznv9A2wD9kTgvnc5doMYYQ2tfOKsbajCaIGlmdjXzdPCOrUtZWhdIxWUcXC4hUOaeBkshqTGFLMytX4miFCHOLJhstQoC5W7K3a5UTKGtzxYF233kdgkLa/yc6M5tKRxos1Y+nxxjn76hOJF4kshAlJ7B6IiBNFcKjmLnotUNqdKemQS8nmmxFFQURqPuI0WZIv1jVDUTEaoDw/mPnMygjqUAsLjWP66lAHCyJ7copGccdfbPp39zlYpy95SC84mkIRpPqvsoCyoKijJFxrsTT89/1NYXwSVQX5EpClOzFFr7hvMAjRIF25KpnEeiECj3pOpOT4aw1mfOiX4iijJFxqtVUBMoT4sphGmo9I7IB7S4NkBbfyQ1UGVywE6c1x+Jj0qZ4dDaN5alMH2lOIuFCu/ULIUhrbqWExUFRZkEkXiCkH0HPl6tgtpAGb22KLT2RViQtjoXLEsB4FQW91AskeRY52BqemYua8EJYC+vD4zKvjobtRQKTaB8ajEFJ+22xhRGo6KgKBOkZzDKtV9+jDd+5THCscT4loK/fNh91B+hqco7YrtTayCbC+loZ4h40nDJ2kYgd1yhtS9MpdfDpsU1KcvCoRBV12abCu/UZh8Nu49UFDIpWVFIJM24i4EUJZNwLMHN332Go50hDrWH+NJD+8cN5NZUlNEzGMMYQ3t/eEQeHxi2FI5nCTY7rqDXrLNFIUdAuq0/TFPQy+rGSk50D41wRQ3YojDX1imMhRVTUPdRIShJUXj2WDerP/4rHj+QffWnomQjmTR85EfPs/1IF//3j87jj7Ys5vZHDvHU4S6AnCuGawPlRBNJ+obidAxER1kKTVU+ytyS1VJwRGHr8jq8HldOS6GtL8KCKh+rmiowhhEupP5wDJHhRHLzgcopTkl13EcqCqMpSVEI+jwYw6iKWIoyFl96aD//88Jpbnv9eq47dyF/d80GagNl/PKF02NWNXMypR5otyqJpU9HhbHXKhxoG2BRjZ8Kr4dFNf7c7iM7n9Jqe4V0erC5z86Q6spR7GYuEpjilFTHUvCp+2gUJSkK2TJXFhOdAxHed8fTUy43qEwv9+06w0Wr67n5kpWA9Tv65HVWqcmxqppV+63f274z1kC9oMo3ap9caxUOtA+wyh7oF9X6swaardXMEZqCPlY0VOASONiWbinE51Teo3yo8HqIJcyIOhUTIazuo5yUpij4RyYpKzYeP9jJgy+18fzxntnuipJGZyjK0rqKEYP/tZtauHpjM8vqAzmPcyyFl1uzWwoAi2sCoyyFZNJwsC3E6kZbFHJYCn1DVhGfpiovXo+bpXWBVLDZGMPeM300Vo1+z7mMU31tstaCxhRyM3+cjBPA43ZR5fMUraXgmP594eLsXymSTBq6B6PUV4zMwyMifPVPXjHmpIXaCsdSsEUhh6XQbq9VcKZJnuodYiiWYFWTNR11UY2fjoHoiH3Ach0Bqamuq5sqU7+hp490s/tUH59+09mTuu5ipSItfXZNbj3OyVDUsjB09tFoStJSALugepFaCo7p3zdU2Bz7Sv70hWMkkoa6DFEAKyZQniOeAMOZUl9u7UcEGipHn2NxnTUDKd0ScAb2lKVQO3ofGF645ojCqqZKDneEiCeSfPuxw9QEynjz5sX5XegcITDF9NmpmIJaCqMoYVEoS60yLTacwSDX6lVl5ukMWTcQ2URhPGrsmEJnKEp9hRePe/S/3ZLa0WsVDraHAFLB40U1tihkuJmGk+xZLqLVjZXEEoY/HOzk/j1n+OOtS+fdHXHFFAvtaEwhNyUrCjWB8hmpjTtR4okkhzuswaBPRWEUP9pxnBu++jjGzOwak64piEK5x0WFPSgvyBJPACvVBTAi2HygbYDaQBn1ldYxOS2FjCR7joh86he7cYnw7guXT7jPxU4qpjBZSyGawO0SytzzZ0bWdFGyopCepKyYON49RDRh+TvVUhjN00e62Hm8Z8bjQZ0DkxcFGJ7xlrlGwaGpykuZW0YU2znYNpAa4AGagz7cLslqKVT5PKkylc5spYPtIa49p4Xm6tExjLlOhXdqloJTYCfXjLFSpmCiICLfFpE2EdmV1lYnIg+IyH77b23atttE5ICI7BORqwrVL4eaQDk9oeIbdB3XkYgGmrPhZAMdK6toIXAshfos8YB8qK2w4grZgsxgFY5ZVDNyWuqB9pGi4HG7aA76RlkKbf3hEWIT9JWlLJI/u2jFpPpb7EzH7CONJ2SnkJbCfwFXZ7TdCjxkjFkDPGS/RkTOAm4ENtrHfE1ECvqN1QbK6Y/EiSUmN8+5UDgrUdctqNJAcxacoGq2lBCFpCtkidFkLQWnAlsu9xFYLiRH7A60DdAVirKqsXLEPtnWKmRLsnf+slouXt0wquj9fMGxFAYm6T4KRxOaNjsHBftUjDGPAF0ZzdcDd9jP7wBuSGv/gTEmYow5DBwAthaqbzB851Zs01IPtA3QWOVlca1f3UdZSIlC18yKQmcoSqXXg9czuXuVanttTGMwtyvHqaswGI3zl99/lppAGdee0zJynyxrFVr7wqNE4cs3bubbf3rBpPo6FxiOKUwy0BxP4JvkdznfmWmpXGCMOQ1g/22y2xcB6VXJT9htoxCRm0Vkh4jsaG9vn3RHhlc1F1dc4UDbAKsbKwn6ytR9lEE4lkjNGJsN99FkrQQYthRyxRTAEoWOgQgfuft59rX28+UbN9NS7R+xz6JaP2f6wsRtC9cYY2VezbBAPG7XmNNk5zqB1OyjyQea59uMrOmiWH412aI9WaeXGGNuN8ZsMcZsaWxsnPQbOqtMi2laqjEmFVwM+st09lEG7f3DaT9m3n00VVGwfm+Zd/TpOCm0f73rDB+5cm0qXXY6i2r8JJKG072WxdQ7FLNXM8+/YPJYuF2Cv8zN4BQCzRpTyM5Mi0KriLQA2H/b7PYTwJK0/RYDpwrZEefOrZhmILX1R+iPxFOi0B+Jk9T03inO2K6jKq9n5t1HA6NXM08EZ1pp85juI0sUrtjQxP+6dHXWfTKnpTqB97FiFfOVCq970umzh2JanzkXMy0K9wI32c9vAu5Ja79RRLwisgJYA2wvZEecVabF5D5KrWBtqkxlcu2fQs74+YYTT9i8rJYT3UMzulZhqpbCDZsX8dU/fsWY00M3L6nhs2/exBfefl7OjKaZC9jaMlJclBKBcs+kLYVwNKGikINCTkm9C3gCWCciJ0TkvcBngStFZD9wpf0aY8xu4G5gD3AfcIsxZvLJ0vNg2FIoHheNIwqrGitTgUl1IQ1zxnaZXLCslkg8SfsMZZE1xliiMMnpqGAFmjODxpm4XMKNW5eOWTZzoS0KzgLHlKVQYu4jsILNk7cUNKaQi4IlxDPGvCPHpstz7P9p4NOF6k8mgXI35W5XUc0+OtA2QKXXw4Kgl6AtCr1DsRF+tWLEGDMji4Da+iN4PS42LgoCcLxraEZ86QORONFEckruo+nCV+bm1WsauOOJI7zrwmUp6ylb5tX5ToV38paCxhRyUyyB5hlHRKgJlBWd+2hVUyUiksp/X+wzkI53DbLxH3/Ds8e6C/5eztTLJVlSQhSS4RQXxTHw/vP1ZxONJ/nHe3bT3h8h6POU5AAXKHdPevaRuo9yU7KiAMWXKfVA+0AqI+ZccR+9eLKXwWiCh15qLfh7nekN0xz0pQKyMxVsTq1mLgJLAWB5QwUfvmIt9+0+w/+8eLok4wlgJcWb7DoFy31U0sNfTkr6U6kpokypvUMx2vsjqbQGQb/l2Sv2Vc2Ob3v74cx1itOPMx/fX+6modI7Y2sVppIMr1C879Ur2NASpD3LGoVSIeCdnKUQSySJJ41aCjkoaVGoLaJMqekzj4BUTKHY3UdHOy1ReP54byod8XiEY4kxi9JkwxiTshTAWug1U2sVppI2u1CUuV187i2bcElpzjwC21KYRExBaymMTWmLQkXxWApH7DvulY1Wla3Kcg8uKf5MqUc6BilzC9FEkueO9Yy7fyyR5HVfeIR/+82+Cb1PfyTOUCyRGgCX1AVGZBSdLhJJw7/et5cnDnam2orRUgA4Z3EN37rpAv7ytdnXNMx3Al73pHIfhW0h0dlH2SlpUXBqKsx0bv5sOHe9zjx0l0uo8hX/qubDnSEuW9+ESH4upPt3t3Ksa5AdRybmbmq1p6MusOf5L6n1c6pnaMIWx1gYY/jUL3bztd8f5M6njqbau0JRvB5XKt9OMfHa9U2szEiaVypUlHuIxpMTTmqp9ZnHpqRFoTZQRixhJp2TfTo50T3EgqB3hEkb9HvoCxdvTCEUidPeH+GcxTVsaA6y/UjnuMf895NHANjX2j8hMR6ej2/5zxfXBognTWqV83Rw+yOH+O4TR/GXuXnpVF+q3VnNrLn3i4vh9NkT+/9VURibkhYFp0xid2j24wrHuwZTs2ocqv1lRe0+OmLHE5bXV7B1RR3PHO0mGs9913agrZ8nD3WxoqGC/nCcU735D+jO4O+sCF5i1zSerhlI9z5/is/8ei9vOKeFP79kJYc7Q6lc/V2hyJQWrimFodJOnz3RmgpDtoj4itDyKwZKWxQCxZM++0T3EEtqR2bEDBa5++hIhzUgL28I8MoVdYRjSXad6s25/51PHqPc7eLW168HYN+Zvpz7ZpJZnH7JNE5LHYzG+eiPn+eC5bV8/m3nsnFhEGNg35l+wElxUZozfIqZgFN9bYLTUtVSGJuSFoXaiuJIihdLJDndO5TKkulQ7S+e9Nm7Tvbylq//gd40AU23FC5YUQfkjiuEInF+8swJrj2nhW0r6wHYaw+6+dDaF6baX5Zyr7XU+BCZnhTaO4/3EI4l+V+XrsZX5uasFmvF9J7Tlmh1hqaWDE8pDBWTrL4WVlEYk9IWhVT67NkVhdM9YZJm+O7XIegrHvfRXduP8czRbh470JFqO9IRorHKS4XXQ0Oll1WNFTlF4Z6dp+iPxHnntqVU+8tYWO1L3Ynng7Waefhu3etx0xz0Tcu01B1HuhGBVyy1qsMurvVT5fXwki0KU02GpxSGVE2FiVoKUcvFqbOPslPSojBcaGd2B15nYFuc6T7ye4pi8VoiafjNbmvF8hOH0kShM8SK+orU660r6nn6SNeoGUHJpOG7TxxhQ0swNfCua66akCicyVJyckltgBPTMC316SNdrFtQRbV9kyAibGgJ8tLpfsKxBIPRhIpCEVLhnZylkFqnoJXXslLaouAvDkvB8Ytncx8NxRJjBm9ngueOddMxEMFf5ubJQ8OWwOGOQZY3DPf5lSvq6A/H2ZsRK/j+9mPsPdPPzZesSM3gWdcc5GD7QN7TCduylJxcXOefcv6jeCLJs0e72bK8dkT7hpYqXjrdlyrso+6j4mO4+trELAXHfeTTNBdZKelPxeN2UeXzzLqlcKJ7CLdLaMnItV8sq5rv23WGcreL9168ggNtA7T1hxmIxOkYiLC8YdhSuHBVPR6X8Jlf7U0N9md6w3z213u5aHU9N5w3XGF1fXMVsYThUHto3PdPJK2Sk5kFapbUBjjdF+bDP3iOnz93MrXIbCLsPdNPKJrgguV1I9rPWhhkMJrg+RM9QPEtXFPSLIUJLmDTmMLYlLQoQHEkxTvePUhz0IfHPfLrSGVKncW4gjGG3+w5w0Wr67nyrAUAPHWoK7UCe3ma+2hB0Mdn3ryJxw50cOtPXsQYwyfu2UUskeT/vGnTiHn+65qrAEZZFdnoDEVIJM2o6mJvPX8xN5y3iEf3d/DhH+7k0n/7XarmQr44i+i2ZIjCBjvY/Nh+y11Wr1NSi47JWgqpKakqClkpWD2FuUJtESTFO941mJp3n051Wk2F2WLP6T6Odw3xl69dzcaFQSq9Hp481IkzvqeLAsDbtizhZM8QX3xwPx0DER5+uZ3bXr+eZRn7rWqsxOOSvOIKbamSkxmWQl2AL7z9PJJJwx8OdvLObz3Fvc+f5OZLVuV9fU8f7WZhtS+1ktxh7YIqXEIqsK5TUouP1OK1CVoKQ7EEZW6hzF3y98RZKflPpSZQTu+sWwpDo2YeQVqm1Blc1fz7fW38yTefTFkCv9l1BpfAFRsW4HG7uGB5LU8c6uRo5/AahUw+dPka/mjLYh5+uZ2NC4O89+IVo/Yp97hY2ViRlyg4d/+5Er+5XMLFaxo4d0kN9+zMv7S3MYYdR7pGWQlg3UWuaqxMTXlV91HxUeZ2Ue5xMTCJQLNaCbkpeVGYbUshHEvQ3h8ZFWSG2XEf3bPzFI8f6OSGrz3O9sNd3Lf7DFtX1KUKz29bWc+h9hDbD3exIOhNmfDpiAifftMmPnr1Or7yx68Y5RZzWNcczGutQmv/yNXMubj+3IXsPtXHgbb8ZjWd6B6itS/CBRlBZgfHheRxCUFfyRvVRUlFuXvCNRXCMS2wMxYlLwo1sxxTcO5EM6ejwuy4j/ac6uOcxdXUVZTzJ998kpdbB7hqY3Nq+4WrrIVnj+xvH+U6SqfM7eJ/XbqaFQ2591nfXMXJniH6xwmkt/aGccn4M4DecE4LLoF7s1gLyaThRPcgO4/3ELeD4E/niCc4nLXQEoVazXtUtATKPROuqTAU1frMY1HyolAbKKc/HE8NFDONs0Yhq6Uww7OPwrEEB9oHuGRNIz/9wKs4f1kt5R7XCFE4qyVIldeDMaPjCRNl3QIr2Pxya/Y7+2g8yTcfPcR3/nCE5Q0VOS0Oh6agj1etauCe50+lku3tPdPHG/7fo6z/h/u4+HO/44avPs4ffeMJjnUO8vSRbqp8Htba/cjEsRR0OmrxUuGduKUwpJbCmJS8TVxbYec/GorRUDnzwUTHUsgWU/B6XJS7XTO2gG3fmX4SScPGhUFqAuXc+d5X0hmKjvDle9wuLlhRx2/3to2YjjoZnBlIzx/v5WBbiP9+8ijt/RGW1QdYXl/B9iNdHO4Icem6Rj7xhrPyOud15y3koz9+gedP9LIg6OVPv/00CWN4z6uWs7yhgrhdL+GaLz9KucfF+ctqcbuyWwEbWqz+aTyheKnwTsJSiCU1pjAGJS8Kw6uao7MjCl2DlLtdNFWNfm8RITiDmVKdXD+O28TjdmUN7m5baYnCiixB5omwuNZPpdfDP/1yD2BZDq9aXc/RzkEefKmVxiov33nPBbx2XVPe57z67Gb+/ue7+N6TR3nxZC8DkTh3/8WFqWsCuGx9E3/9w51sP9w1an1COk1VPpqqvCVb2WwuMJnqa+GoWgpjUfKiMJz/aHaCzce7B1lU68eV427VqqkwM33bfaqXKq8nq9WSzjWbWvjt3racvvh8ERH+ZNtSjncN8q5ty9m2sm7Kvvugr4zL1jXxo2dO4HEJ33nPBSMEAaxCRnf9+TZ+v6+NV61qGPN83/7TC1LZdJXiI1DupmMgMqFjhmIJGrPchCkWKgq2pfDlh/ZjDJzqGeK2azakFmoVmuNdQ1mDzA4zmT5796k+NrQEcwqUw+LaAD+4+cJpec/bXr9hWs6Tztu3LuGBl1r5zJs38eo1jVn3cbuEyzeM/x2fvah6urunTCMV3olbChpTGJuSDzQvqQ1Q5fPw4sneVB3gT/1iN5H4zFRjO9E9mDXI7FDtnxlRSCQNe0/3j7qrnou8dl0Tz//j63jbliWz3RWlwATK3ZMqsqMxhdyUvKVQHShj5z+8LhVsfPjldm769na+/9Qx3nPR6EVX08lAJE73YGxMd03QX8axaaouNhaHO0IMxRJsnAeiAMNVuZT5TYXXM+HU2eFYAr8mw8uJfjIwYvbJJWsauGh1Pf/vtwfGnT8/VZzsqGO5j6r9nhkJNO+2K6bNB0tBKR0C5W6GYgkSSUMknuD+3WdSCe9yoe6jsdHbqQxEhFuv3sAbv/IYtz9yiI+8bt20nn/XyV5+s/sMSWM4ZtcCGMt95MQUjDEFXUC153QfZW5hTVP2OfuKUoxU2CvqT/UM8ZG7n2f7kS7WN1fxpRs3s665ilgiyfefOsY3HzvE4poAr1nXqKIwDioKWdi0uJo3nruQbz56mHdtW0bTNExJbO0L82+/2cdPnj0BgNse4JuDPlY15p7vH/SXEU8ahmKJrCklpos9p/pYu6CKco8aj8rcIWCnz37rf/yB7sEYH7x8Dd9/6hhv/MpjvO/iFdy/p5UDbQNsWVZL92CUz/56LwBVPp1RlgsVhRz87evWct+u0/z9z3fxjXedP6W79Ht2nuS2n75ILJHk5lev5JbLVqfyGo1HeqqLQomCMYY9p/q4bH3+6wEUpRhwLIV4wnDXn2/j/GW1vPvCZXzsxy/wtd8fZEVDBf/57i1csaEJEeFMb5hnjnZz8eqxpyKXMioKOVhWX8Gtr9/AP/9yD9945BDvf81wOmZjDPG0kpMel+QUjUf3t/ORu59n89IaPv+2c0elkB6P4aR4cVqmaXakMVaq6Ug8wWvXNdHaF6EzFJ03QWaldLhgRR1v3ryID1+xlqX1lhu2odLLN2/aws7jPWxcWD3C+m2u9nHtOS2z1d05gYrCGPzZRct57lg3/3rfXs5ZVM2Fq+r51Ytn+Nx9e0fMCAqUu1lWX8Hy+gCbl9bw1vOXUFdRzp5TfXzgzmdZ3VTJt/70grytg3SG02fHiCeStA9EaKry5UzNMBb94Rg/f+4k//WHIxy0K569YmkNr1lrWQhnLdQ5+crcYlGNn39/+3mj2kWEzUuzZ79VxkacxGFzkS1btpgdO3YU9D1CkTjXf/VxukJRVjVW8PSRbtY3V3HtphZEwBhrNfTRzhCHO0Ic6gjh9bi47tyFPLK/HUH42S2voqU69wyjsXjhRA/XfeVxGiq99A5FiSUM1f4yXr2mgUvXNbFpUTXL6gP4ytx0h6I8sr+dR/d3kDSG5fUVLKsPcKY3zO/3tfP0kS7iScM5i6u56cLlJJKGf7t/X6oO8YuffJ36WhWlBBCRZ4wxW7JtU0thHCq8Hr7xrvO5/iuPc6g9xP950ybefsGSnHfq+870c8cTR/jZsyfxuIQffeDCSQsCwOqmSi5b34SvzMWy+gqagz5eONHLwy+388sXTqf2a6ry0j4QwRioCZThL3Pz02dPpravb67ifa9eyVUbF3DekpqUu+uac1r4xsMHCccSKgiKohSfpSAiVwNfAtzAN40xn82170xYCg5tfWEqvB4q8lwU1TsUIxJLTMvMpWwkk4Z9rf3sbxvgSEeIY12DLKrxc+m6Rs5ZXIPbJQxFExzrGqTaXzZugRpFUUqHOWMpiIgb+CpwJXACeFpE7jXG7JndnjHhwb3aXwb+wt15u1zChpZgKud/Nvzl7lR6akVRlHwotknpW4EDxphDxpgo8APg+lnuk6IoSslQbKKwCDie9vqE3ZZCRG4WkR0isqO9vX1GO6coijLfKTZRyBa9HRH0MMbcbozZYozZ0tiYPS2yoiiKMjmKTRROAOn5jhcDo6uwK4qiKAWh2EThaWCNiKwQkXLgRuDeWe6ToihKyVBUs4+MMXER+UvgN1hTUr9tjNk9y91SFEUpGYpKFACMMb8CfjXb/VAURSlFis19pCiKoswiRbeieSKISDtwdAqnaAA6pqk7c4VSvGYozevWay4dJnrdy4wxWadvzmlRmCoisiPXUu/5SileM5Tmdes1lw7Ted3qPlIURVFSqCgoiqIoKUpdFG6f7Q7MAqV4zVCa163XXDpM23WXdExBURRFGUmpWwqKoihKGioKiqIoSoqSFAURuVpE9onIARG5dbb7UwhEZImI/E5EXhKR3SLyIbu9TkQeEJH99t95Wd1cRNwi8pyI/NJ+Pa+vW0RqROTHIrLX/s4vnO/XDCAif23/vneJyF0i4puP1y0i3xaRNhHZldaW8zpF5DZ7fNsnIldN5L1KThTSqru9HjgLeIeInDW7vSoIceAjxpgNwDbgFvs6bwUeMsasAR6yX89HPgS8lPZ6vl/3l4D7jDHrgXOxrn1eX7OILAI+CGwxxpyNlS/tRubndf8XcHVGW9brtP/PbwQ22sd8zR738qLkRIESqe5mjDltjHnWft6PNUgswrrWO+zd7gBumJUOFhARWQxcC3wzrXneXreIBIFLgG8BGGOixpge5vE1p+EB/CLiAQJYqfbn3XUbYx4BujKac13n9cAPjDERY8xh4ADWuJcXpSgK41Z3m2+IyHJgM/AUsMAYcxos4QCaZrFrheKLwEeBZFrbfL7ulUA78B3bZfZNEalgfl8zxpiTwOeBY8BpoNcYcz/z/LrTyHWdUxrjSlEUxq3uNp8QkUrgJ8CHjTF9s92fQiMibwDajDHPzHZfZhAP8Arg68aYzUCI+eEyGRPbh349sAJYCFSIyDtnt1dFwZTGuFIUhZKp7iYiZViC8D1jzE/t5lYRabG3twBts9W/AnERcJ2IHMFyDV4mIncyv6/7BHDCGPOU/frHWCIxn68Z4ArgsDGm3RgTA34KvIr5f90Oua5zSmNcKYpCSVR3ExHB8jG/ZIz597RN9wI32c9vAu6Z6b4VEmPMbcaYxcaY5Vjf7W+NMe9kHl+3MeYMcFxE1tlNlwN7mMfXbHMM2CYiAfv3fjlW7Gy+X7dDruu8F7hRRLwisgJYA2zP+6zGmJJ7ANcALwMHgY/Pdn8KdI0XY5mMLwA77cc1QD3WTIX99t+62e5rAT+DS4Ff2s/n9XUD5wE77O/750DtfL9m+7o/BewFdgH/DXjn43UDd2HFTWJYlsB7x7pO4OP2+LYPeP1E3kvTXCiKoigpStF9pCiKouRARUFRFEVJoaKgKIqipFBRUBRFUVKoKCiKoigpVBQUJQ0RSYjIzrTHmCuDReT9IvLuaXjfIyLSMNXzKMpU0SmpipKGiAwYYypn4X2PYGX77Jjp91aUdNRSUJQ8sO/kPyci2+3Harv9kyLyt/bzD4rIHhF5QUR+YLfVicjP7bYnReQcu71eRO63E9h9g7R8NSLyTvs9dorINyaS9lhRpoqKgqKMxJ/hPnp72rY+Y8xW4CtYmVgzuRXYbIw5B3i/3fYp4Dm77e+A79rt/wg8ZqwEdvcCSwFEZAPwduAiY8x5QAL4k+m8QEUZC89sd0BRiowhezDOxl1pf7+QZfsLwPdE5OdYqSbASjfyFgBjzG9tC6Eaq/7Bm+32/xGRbnv/y4HzgaetdD74mb8J3ZQiREVBUfLH5HjucC3WYH8d8AkR2cjYaYyznUOAO4wxt02lo4oyWdR9pCj58/a0v0+kbxARF7DEGPM7rAI/NUAl8Ai2+0dELgU6jFXXIr399VgJ7MBKbPZWEWmyt9WJyLKCXZGiZKCWgqKMxC8iO9Ne32eMcaalekXkKaybqXdkHOcG7rRdQwJ8wRjTIyKfxKqI9gIwyHCq408Bd4nIs8DDWGmgMcbsEZG/B+63hSYG3AIcnebrVJSs6JRURckDnTKqlArqPlIURVFSqKWgKIqipFBLQVEURUmhoqAoiqKkUFFQFEVRUqgoKIqiKClUFBRFUZQU/x/Mwle1dnBtbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 기본 패키지\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "# 강화학습 환경 패키지\n",
    "import gym\n",
    "\n",
    "# 인공지능 패키지: 텐서플로, 케라스 \n",
    "# 호환성을 위해 텐스플로에 포함된 케라스를 불러옴 \n",
    "import tensorflow as tf\n",
    "from tensorflow import keras \n",
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "def create_q_model(num_states, num_actions):\n",
    "    inputs = Input(shape=(num_states,))\n",
    "    layer = Dense(32, activation=\"relu\")(inputs)\n",
    "    layer = Dense(16, activation=\"relu\")(layer)\n",
    "    action = Dense(num_actions, activation=\"linear\")(layer)\n",
    "    return Model(inputs=inputs, outputs=action)\n",
    "\n",
    "def list_rotate(l):\n",
    "    return list(zip(*l))\n",
    "\n",
    "class WorldFull():\n",
    "    def __init__(self):\n",
    "        self.get_env_model() #? \n",
    "        \n",
    "        self.memory = deque(maxlen=2000)\n",
    "        self.N_batch = 64\n",
    "        self.t_model = create_q_model(self.num_states, self.num_actions)\n",
    "        self.discount_factor = 0.99\n",
    "        self.learning_rate = 0.001\n",
    "        self.optimizer = Adam(lr=self.learning_rate)\n",
    "        \n",
    "        self.epsilon = 0.2\n",
    "        \n",
    "    def get_env_model(self):\n",
    "        self.env = gym.make('CartPole-v1')\n",
    "        self.num_states = self.env.observation_space.shape[0]\n",
    "        self.num_actions = self.env.action_space.n\n",
    "        self.model = create_q_model(self.num_states, self.num_actions)\n",
    "    \n",
    "    def update_t_model(self):\n",
    "        self.t_model.set_weights(self.model.get_weights())\n",
    "\n",
    "    def best_action(self, s):\n",
    "        if random.random() <= self.epsilon:\n",
    "            return random.randrange(self.num_actions)\n",
    "        else:\n",
    "            s_array = np.array(s).reshape((1,-1))\n",
    "            Qsa = self.model.predict(s_array)[0]\n",
    "            return np.argmax(Qsa)\n",
    "\n",
    "    def train_memory(self):\n",
    "        if len(self.memory) >= self.N_batch:\n",
    "            memory_batch = random.sample(self.memory, self.N_batch)\n",
    "            s_l,a_l,r_l,next_s_l,done_l = [np.array(x) for x in list_rotate(memory_batch)]\n",
    "            model_w = self.model.trainable_variables\n",
    "            with tf.GradientTape() as tape:\n",
    "                Qsa_pred_l = self.model(s_l.astype(np.float32))\n",
    "                a_l_onehot = tf.one_hot(a_l, self.num_actions)\n",
    "                Qs_a_pred_l = tf.reduce_sum(a_l_onehot * Qsa_pred_l, \n",
    "                                            axis=1)    \n",
    "\n",
    "                Qsa_tpred_l = self.t_model(next_s_l.astype(np.float32)) \n",
    "                Qsa_tpred_l = tf.stop_gradient(Qsa_tpred_l)\n",
    "\n",
    "                max_Q_next_s_a_l = np.amax(Qsa_tpred_l, axis=-1)\n",
    "                Qs_a_l = r_l + (1 - done_l) * self.discount_factor * max_Q_next_s_a_l\n",
    "                loss = tf.reduce_mean(tf.square(Qs_a_l - Qs_a_pred_l))\n",
    "                grads = tape.gradient(loss, model_w)\n",
    "                self.optimizer.apply_gradients(zip(grads, model_w))        \n",
    "        \n",
    "    def trials(self, n_episodes=100, flag_render=False):\n",
    "        memory = self.memory\n",
    "        env = self.env\n",
    "        model = self.model\n",
    "        score_l = []\n",
    "        for e in range(n_episodes):\n",
    "            done = False\n",
    "            score = 0\n",
    "            s = env.reset()\n",
    "            while not done:                \n",
    "                a = self.best_action(s)\n",
    "                next_s, r, done, _ = env.step(a)\n",
    "                if flag_render:\n",
    "                    env.render()\n",
    "                score += r\n",
    "                memory.append([s,a,r,next_s,done])\n",
    "                # self.train_memory()     \n",
    "                s = next_s\n",
    "                self.train_memory()                 \n",
    "            self.update_t_model()\n",
    "            print(f'Episode: {e:5d} -->  Score: {score:3.1f}') \n",
    "            score_l.append(score)            \n",
    "        return score_l\n",
    "\n",
    "new_world = WorldFull()\n",
    "score_l = new_world.trials(n_episodes=100)\n",
    "new_world.env.close()\n",
    "np.save('score_l.npy', score_l)\n",
    "print('Job completed!')\n",
    "\n",
    "plt.plot(score_l)\n",
    "plt.title(\"Deep Q-Learning for Cartpole\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50d81da5-d43f-4e52-8d7e-34c12b6edabf",
   "metadata": {},
   "source": [
    "---\n",
    "### 참고 자료"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e6acc9-69dd-4e99-ad63-fd03ac369746",
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = q_table[p][v]\n",
    "# calculate the exponential of all elements in the input array.\n",
    "logits_exp = np.exp(logits)\n",
    "# calculate the probabilities\n",
    "probabilities = logits_exp / np.sum(logits_exp)\n",
    "# get random action\n",
    "action = np.random.choice(environment.action_space.n, p=probabilities)\n",
    "# get observation, reward and done after each step\n",
    "observation, reward, done, _ = environment.step(action)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "keras-gpu",
   "language": "python",
   "name": "keras-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}