{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import aimBotEnv\n",
    "import PPO\n",
    "import buffer\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "import time\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attempts to allocate only the GPU memory needed for allocation\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENV_PATH = './Build-MultiScene-WithLoad/Aimbot-PPO'\n",
    "WORKER_ID = 1\n",
    "BASE_PORT = 200\n",
    "\n",
    "MAX_EP = 1000\n",
    "EP_LENGTH = 100000\n",
    "GAMMA = 0.99 # discount future reward (UP?)\n",
    "EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n",
    "ACTOR_LR = 1e-5 # LR\n",
    "CRITIC_LR = 2e-5 # LR\n",
    "BATCH = 512 # learning step\n",
    "ACTOR_EPOCH = 15 # epoch\n",
    "CRITIC_EPOCH = 15 # epoch\n",
    "ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss\n",
    "ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n",
    "\n",
    "\n",
    "TRAIN = True\n",
    "SAVE_DIR = \"PPO-Model/\"+datetime.datetime.now().strftime(\"%m%d%H%M\")+\"/\"\n",
    "LOAD_DIR = None\n",
    "\n",
    "CTN_ACTION_RANGE = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "√√√√√Enviroment Initialized Success√√√√√\n",
      "√√√√√Buffer Initialized Success√√√√√\n",
      "Load Model:\n",
      "PPO-Model/09051833/183540\n",
      "CONTINUOUS_SIZE 1\n",
      "DISCRETE_SIZE 5\n",
      "STATE_SIZE 29\n"
     ]
    }
   ],
   "source": [
    "# initialize enviroment & buffer class\n",
    "env = aimBotEnv.makeEnv(envPath = ENV_PATH,\n",
    "                        workerID = WORKER_ID,\n",
    "                        basePort = BASE_PORT)\n",
    "epBuffer = buffer.buffer()\n",
    "\n",
    "STATE_SIZE = env.STATE_SIZE\n",
    "CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
    "DISCRETE_SIZE = env.DISCRETE_SIZE\n",
    "s,reward,done,loadDir = env.getSteps()\n",
    "\n",
    "# check load model or not\n",
    "if(np.any(loadDir == 0)):\n",
    "    # create a new model\n",
    "    print(\"No loadDir specified,Create a New Model\")\n",
    "    LOAD_DIR = None\n",
    "else:\n",
    "    # load model\n",
    "    loadDirDateSTR = str(int(loadDir[0]))\n",
    "    loadDirTimeSTR = str(int(loadDir[1]))\n",
    "    if len(loadDirDateSTR)!=8:\n",
    "        # fill lost 0 while converse float to string\n",
    "        for _ in range(8 - len(loadDirDateSTR)):\n",
    "            loadDirDateSTR = \"0\" + loadDirDateSTR\n",
    "    if len(loadDirTimeSTR)!=6:\n",
    "        # fill lost 0 while converse float to string\n",
    "        for _ in range(6 - len(loadDirTimeSTR)):\n",
    "            loadDirTimeSTR = \"0\" + loadDirTimeSTR\n",
    "    LOAD_DIR = \"PPO-Model/\"+loadDirDateSTR+\"/\"+loadDirTimeSTR\n",
    "    print(\"Load Model:\")\n",
    "    print(LOAD_DIR)\n",
    "\n",
    "print(\"CONTINUOUS_SIZE\",CONTINUOUS_SIZE)\n",
    "print(\"DISCRETE_SIZE\",DISCRETE_SIZE)\n",
    "print(\"STATE_SIZE\",STATE_SIZE)\n",
    "\n",
    "disActShape = [3,3,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actToKey(disAct1,disAct2,disAct3,conAct):\n",
    "    kW = 0\n",
    "    kS = 0\n",
    "    kA = 0\n",
    "    kD = 0\n",
    "    mouseShoot = 0\n",
    "    if disAct1 == 0:\n",
    "        kW = 0\n",
    "        kS = 1\n",
    "    elif disAct1 == 1:\n",
    "        kW = 0\n",
    "        kS = 0\n",
    "    elif disAct1 == 2:\n",
    "        kW = 1\n",
    "        kS = 0\n",
    "    if disAct2 == 0:\n",
    "        kA = 0\n",
    "        kD = 1\n",
    "    elif disAct2 == 1:\n",
    "        kA = 0\n",
    "        kD = 0\n",
    "    elif disAct2 == 2:\n",
    "        kA = 1\n",
    "        kD = 0\n",
    "    mouseShoot = disAct3\n",
    "    return kW,kS,kA,kD,mouseShoot,conAct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EP  0  START\n",
      "√√√√√Buffer Initialized Success√√√√√\n",
      "++++++++++++++++++++++++++++++++++++\n",
      "++++++++++++Model Loaded++++++++++++\n",
      "PPO-Model/09051833/183540\n",
      "++++++++++++++++++++++++++++++++++++\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A_Loss: -43.132581075032554 C_Loss: 4025.360986328125\n",
      "EP OVER!\n",
      "A_Loss: 1184751.5479166666 C_Loss: 1939213379930.6667\n",
      "Model's Weights Saved\n",
      "New Record! Save NN -37.67000053450465\n",
      "EP  1  START\n",
      "A_Loss: 14251923.066666666 C_Loss: 40720630.7288086\n",
      "EP OVER!\n",
      "A_Loss: 1857006.7333333334 C_Loss: 245.15654500325522\n",
      "Model's Weights Saved\n",
      "New Record! Save NN -13.100000601261854\n",
      "EP  2  START\n",
      "A_Loss: -1.0014899969100952 C_Loss: 71.29023424784343\n",
      "EP OVER!\n",
      "A_Loss: -0.31318608721097313 C_Loss: 8.897234630584716\n",
      "EP  3  START\n",
      "A_Loss: 566053.9979166667 C_Loss: 1986013.3489705403\n",
      "EP OVER!\n",
      "A_Loss: 0.6021817366282145 C_Loss: 6.035458342234294\n",
      "EP  4  START\n",
      "A_Loss: 169656457.6 C_Loss: 4317.756831359863\n",
      "EP OVER!\n",
      "A_Loss: 22155934.0 C_Loss: 63.52870483398438\n",
      "EP  5  START\n",
      "A_Loss: 0.2413090835014979 C_Loss: 10.40585823059082\n",
      "EP OVER!\n",
      "A_Loss: 0.12157159547011058 C_Loss: 14.347647285461425\n",
      "EP  6  START\n",
      "A_Loss: 215977770.66666666 C_Loss: 190435277.5966268\n",
      "EP OVER!\n",
      "A_Loss: 213349568.0 C_Loss: 1620.733740234375\n",
      "EP  7  START\n",
      "A_Loss: 100149825.6 C_Loss: 1569803.3794799806\n",
      "EP OVER!\n",
      "A_Loss: 171062395.73333332 C_Loss: 34872400594.933334\n",
      "Model's Weights Saved\n",
      "New Record! Save NN 4.119999349117279\n",
      "EP  8  START\n",
      "A_Loss: 0.7038820425669352 C_Loss: 31.980174128214518\n",
      "EP OVER!\n",
      "A_Loss: 0.2687960028648376 C_Loss: 6.94128991762797\n",
      "EP  9  START\n",
      "A_Loss: 0.1451285809278488 C_Loss: 3.5754743576049806\n",
      "EP OVER!\n",
      "A_Loss: 0.18081151247024535 C_Loss: 3.902424764633179\n",
      "EP  10  START\n",
      "A_Loss: 29139865.6 C_Loss: 174927458.90618488\n",
      "EP OVER!\n",
      "A_Loss: 16942552.4 C_Loss: 134.21190592447917\n",
      "EP  11  START\n",
      "A_Loss: 53790223.46666667 C_Loss: 647.0605305989583\n",
      "EP OVER!\n",
      "A_Loss: 27695839.866666667 C_Loss: 343.9340047200521\n",
      "EP  12  START\n",
      "A_Loss: 65695720.8 C_Loss: 61225647615.0198\n",
      "EP OVER!\n",
      "A_Loss: 0.07575627962748209 C_Loss: 10.41986296971639\n",
      "EP  13  START\n",
      "A_Loss: 51573392.266666666 C_Loss: 212022811.82775268\n",
      "EP OVER!\n",
      "A_Loss: 22073133.866666667 C_Loss: 61.18818028767904\n",
      "EP  14  START\n",
      "A_Loss: 0.13529965033133826 C_Loss: 0.966444210211436\n",
      "EP OVER!\n",
      "A_Loss: 0.1176165262858073 C_Loss: 0.5141626675923665\n",
      "EP  15  START\n",
      "A_Loss: 0.5104357699553171 C_Loss: 25.79277165730794\n",
      "EP OVER!\n",
      "A_Loss: 15181718.0 C_Loss: 9111658759031.467\n",
      "EP  16  START\n",
      "A_Loss: 30694314.533333335 C_Loss: 62842582.26665497\n",
      "EP OVER!\n",
      "A_Loss: 17702280.066666666 C_Loss: 50.18375091552734\n",
      "EP  17  START\n",
      "A_Loss: 0.08570613314708074 C_Loss: 1.7101642807324728\n",
      "EP OVER!\n",
      "A_Loss: 14154208.066666666 C_Loss: 248.65523885091145\n",
      "EP  18  START\n",
      "A_Loss: 51043696.8 C_Loss: 39972927.18101196\n",
      "EP OVER!\n",
      "A_Loss: 24850118.0 C_Loss: 6779725.618615722\n",
      "EP  19  START\n",
      "A_Loss: 75697662.4 C_Loss: 505.26551717122396\n",
      "EP OVER!\n",
      "A_Loss: 64376180.266666666 C_Loss: 299.2391876220703\n",
      "EP  20  START\n",
      "A_Loss: 55954064.8 C_Loss: 234.9048828125\n",
      "EP OVER!\n",
      "A_Loss: 57271699.733333334 C_Loss: 59254958.60783488\n",
      "EP  21  START\n",
      "A_Loss: 124046376.53333333 C_Loss: 2737571204155.7334\n",
      "EP OVER!\n",
      "A_Loss: 31774753.866666667 C_Loss: 24325611279.066666\n",
      "EP  22  START\n",
      "A_Loss: 7490019498.666667 C_Loss: 245487346653.86667\n",
      "EP OVER!\n",
      "A_Loss: 186999600.0 C_Loss: 36627312369.066666\n",
      "EP  23  START\n",
      "A_Loss: 4297551769.6 C_Loss: 1184376194184.5334\n",
      "EP OVER!\n",
      "A_Loss: 122982910044842.67 C_Loss: 7331505971746.134\n",
      "EP  24  START\n",
      "A_Loss: 2682021.3583333334 C_Loss: 2341072196027.7334\n",
      "EP OVER!\n",
      "A_Loss: 1469103463901.8667 C_Loss: 113593101.86666666\n",
      "EP  25  START\n",
      "A_Loss: 76648460.0 C_Loss: 14498072.481510418\n",
      "EP OVER!\n",
      "A_Loss: 5800877.766666667 C_Loss: 22134.297265625\n",
      "EP  26  START\n",
      "A_Loss: 34429532.53333333 C_Loss: 46926.630208333336\n",
      "EP OVER!\n",
      "A_Loss: 212612493.86666667 C_Loss: 23176.368880208334\n",
      "EP  27  START\n",
      "A_Loss: 3207813239.4666667 C_Loss: 41575.61041666667\n",
      "EP OVER!\n",
      "A_Loss: 38858107.86666667 C_Loss: 15486.2171875\n",
      "EP  28  START\n",
      "A_Loss: -58850.48046875 C_Loss: 14550529378106.69\n",
      "EP OVER!\n",
      "A_Loss: 445133260.8 C_Loss: 1749447703.4604166\n",
      "EP  29  START\n",
      "A_Loss: 80796092.8 C_Loss: 8523564.864453126\n",
      "EP OVER!\n",
      "A_Loss: 34027046.266666666 C_Loss: 3319.6830078125\n",
      "Model's Weights Saved\n",
      "New Record! Save NN 4.589999713003635\n",
      "EP  30  START\n",
      "A_Loss: 118412868.26666667 C_Loss: 23715976.025211588\n",
      "EP OVER!\n",
      "A_Loss: 94927646.93333334 C_Loss: 1067.18818359375\n",
      "EP  31  START\n",
      "A_Loss: 338770097.06666666 C_Loss: 3585200.7686075848\n",
      "EP OVER!\n",
      "A_Loss: 1479514146.1333334 C_Loss: 10348.122916666667\n",
      "Model's Weights Saved\n",
      "New Record! Save NN 38.5099995136261\n",
      "EP  32  START\n",
      "A_Loss: 119593341.86666666 C_Loss: 784.9417297363282\n",
      "EP OVER!\n",
      "A_Loss: -3.2093143463134766 C_Loss: 180.5671844482422\n",
      "EP  33  START\n",
      "A_Loss: 149599549.86666667 C_Loss: 206.16288146972656\n",
      "EP OVER!\n",
      "A_Loss: 5682631.7 C_Loss: 17.62857920328776\n",
      "EP  34  START\n",
      "A_Loss: 231213670.4 C_Loss: 272.8009948730469\n",
      "EP OVER!\n",
      "A_Loss: 167307104.0 C_Loss: 188.52051798502603\n",
      "EP  35  START\n",
      "A_Loss: 204825578.66666666 C_Loss: 335.75029296875\n",
      "EP OVER!\n",
      "A_Loss: 83991501.33333333 C_Loss: 451.3644246419271\n",
      "EP  36  START\n",
      "A_Loss: 0.6501724402109782 C_Loss: 38.280514017740884\n",
      "EP OVER!\n",
      "A_Loss: 175042737.06666666 C_Loss: 256.6915344238281\n",
      "EP  37  START\n",
      "A_Loss: 461190880.0 C_Loss: 717.4720499674479\n",
      "EP OVER!\n",
      "A_Loss: 173152186.66666666 C_Loss: 701.6123962402344\n",
      "EP  38  START\n",
      "A_Loss: 188070140.8 C_Loss: 1611377609664.9072\n",
      "EP OVER!\n",
      "A_Loss: 512005579391093.7 C_Loss: 86553492717568.0\n",
      "EP  39  START\n",
      "A_Loss: 121553.57962239583 C_Loss: 53618558110.88125\n",
      "EP OVER!\n",
      "A_Loss: 10892.0080078125 C_Loss: 132429.940625\n",
      "EP  40  START\n",
      "A_Loss: 15404.601106770833 C_Loss: 107106.1421875\n",
      "EP OVER!\n",
      "A_Loss: 1822.5163004557292 C_Loss: 182031.73854166668\n",
      "EP  41  START\n",
      "A_Loss: 599.1890706380208 C_Loss: 76118.35729166666\n",
      "EP OVER!\n",
      "A_Loss: 1033.1336588541667 C_Loss: 54971.41354166667\n",
      "EP  42  START\n",
      "A_Loss: 2602.054020182292 C_Loss: 227568.73541666666\n",
      "EP OVER!\n",
      "A_Loss: -3.2474422613779703 C_Loss: 155.22516682942708\n",
      "EP  43  START\n",
      "A_Loss: 1994.366455078125 C_Loss: 149748.48125\n",
      "EP OVER!\n",
      "A_Loss: 7441.857747395833 C_Loss: 196344.43958333333\n",
      "EP  44  START\n",
      "A_Loss: 19441.808723958333 C_Loss: 197749.58020833333\n",
      "EP OVER!\n",
      "A_Loss: 1789.1592529296875 C_Loss: 25795.180598958334\n",
      "EP  45  START\n",
      "A_Loss: 6888.085221354167 C_Loss: 149084.575\n",
      "EP OVER!\n",
      "A_Loss: 1102350.757421875 C_Loss: 180759.19166666668\n",
      "EP  46  START\n",
      "A_Loss: 536.078184000651 C_Loss: 89909.24375\n",
      "EP OVER!\n",
      "A_Loss: 794341992174.4 C_Loss: 59005630871824.9\n",
      "EP  47  START\n",
      "A_Loss: 114184075127.46666 C_Loss: 74263479521553.06\n",
      "EP OVER!\n",
      "A_Loss: 1293861572.3333333 C_Loss: 1474176015837.8667\n",
      "EP  48  START\n",
      "A_Loss: 1143.1249674479166 C_Loss: 19221068817.411232\n",
      "EP OVER!\n",
      "A_Loss: 855.1484252929688 C_Loss: 28676.431770833333\n",
      "EP  49  START\n",
      "A_Loss: 1198.7060546875 C_Loss: 41875.48359375\n",
      "EP OVER!\n",
      "A_Loss: 1215.0390218098958 C_Loss: 7167.594791666666\n",
      "EP  50  START\n",
      "A_Loss: 1840.496435546875 C_Loss: 33346.76875\n",
      "EP OVER!\n",
      "A_Loss: 97.68925425211589 C_Loss: 2453.3792643229167\n",
      "EP  51  START\n",
      "A_Loss: 2179.5446614583334 C_Loss: 25573.869270833333\n",
      "EP OVER!\n",
      "A_Loss: 3547.454052734375 C_Loss: 15107.698307291666\n",
      "EP  52  START\n",
      "A_Loss: 513.1327128092448 C_Loss: 23233.316276041667\n",
      "EP OVER!\n",
      "A_Loss: 2024.9339111328125 C_Loss: 10457.81875\n",
      "EP  53  START\n",
      "A_Loss: 321.8419494628906 C_Loss: 10830.605924479167\n",
      "EP OVER!\n",
      "A_Loss: -8.38951981862386 C_Loss: 2241.023612467448\n",
      "EP  54  START\n",
      "A_Loss: 1454.901765950521 C_Loss: 16337.481966145833\n",
      "EP OVER!\n",
      "A_Loss: 3069.5313313802085 C_Loss: 4450.756380208333\n",
      "EP  55  START\n",
      "A_Loss: 444.8317138671875 C_Loss: 15201.106184895832\n",
      "EP OVER!\n",
      "A_Loss: 1159.2985107421875 C_Loss: 35525.82317708333\n",
      "EP  56  START\n",
      "A_Loss: 1397.7694742838542 C_Loss: 8600.898404947917\n",
      "EP OVER!\n",
      "A_Loss: 160.49285074869792 C_Loss: 3749.308203125\n",
      "EP  57  START\n",
      "A_Loss: -23.67707913716634 C_Loss: 8787.21162109375\n",
      "EP OVER!\n",
      "A_Loss: 446.2368998209635 C_Loss: 8949.849674479166\n",
      "EP  58  START\n",
      "A_Loss: 1443.8223063151042 C_Loss: 4806.931754557291\n",
      "EP OVER!\n",
      "A_Loss: 1028.3776000976563 C_Loss: 10080.125455729167\n",
      "EP  59  START\n",
      "A_Loss: 979.55380859375 C_Loss: 18168.035872395834\n",
      "EP OVER!\n",
      "A_Loss: 2799.885953776042 C_Loss: 14584.458203125\n",
      "EP  60  START\n",
      "A_Loss: 83247.30833333333 C_Loss: 61687910677571.76\n",
      "EP OVER!\n",
      "A_Loss: 81.43709259033203 C_Loss: 47630220.59791667\n",
      "EP  61  START\n",
      "A_Loss: 5032.870377604167 C_Loss: 43944.79453125\n",
      "EP OVER!\n",
      "A_Loss: 9.13815720876058 C_Loss: 2573.8714192708335\n",
      "EP  62  START\n",
      "A_Loss: 523.7818908691406 C_Loss: 8441.000618489583\n",
      "EP OVER!\n",
      "A_Loss: 1593.0845377604167 C_Loss: 17598.201953125\n",
      "EP  63  START\n",
      "A_Loss: 27.476329803466797 C_Loss: 5594.57646484375\n",
      "EP OVER!\n",
      "A_Loss: 25.40858777364095 C_Loss: 11878.195572916668\n",
      "EP  64  START\n",
      "A_Loss: 3347.620393880208 C_Loss: 15555.9111328125\n",
      "EP OVER!\n",
      "A_Loss: 25.35328992207845 C_Loss: 7923.337727864583\n",
      "EP  65  START\n",
      "A_Loss: 1800.478165690104 C_Loss: 36845.799479166664\n",
      "EP OVER!\n",
      "A_Loss: 544.9222574869792 C_Loss: 2949.7829752604166\n",
      "EP  66  START\n",
      "A_Loss: 238.4148183186849 C_Loss: 2108.7259195963543\n",
      "EP OVER!\n",
      "A_Loss: 10.808652464548747 C_Loss: 1069.8261800130208\n",
      "EP  67  START\n",
      "A_Loss: 7.2742944399515785 C_Loss: 1112.0956298828125\n",
      "EP OVER!\n",
      "A_Loss: 686.372725423177 C_Loss: 1421.539111328125\n",
      "EP  68  START\n",
      "A_Loss: 38.04718589782715 C_Loss: 2771.959724934896\n",
      "EP OVER!\n",
      "A_Loss: 1133.577986653646 C_Loss: 5758.061686197917\n",
      "EP  69  START\n",
      "A_Loss: 402.1093465169271 C_Loss: 3987.9388671875\n",
      "EP OVER!\n",
      "A_Loss: -1.7969589630762737 C_Loss: 2884.10732421875\n",
      "EP  70  START\n",
      "A_Loss: 1016.8031656901042 C_Loss: 6636.073665364584\n",
      "EP OVER!\n",
      "A_Loss: 939.9079630533854 C_Loss: 3692.443391927083\n",
      "EP  71  START\n",
      "A_Loss: -107.65687510172526 C_Loss: 21740540498583.367\n",
      "EP OVER!\n",
      "A_Loss: 5876.485221354166 C_Loss: 50534.828385416666\n",
      "EP  72  START\n",
      "A_Loss: 219.98338216145834 C_Loss: 2713.9967041015625\n",
      "EP OVER!\n",
      "A_Loss: 433.9121032714844 C_Loss: 950.6825520833333\n",
      "EP  73  START\n",
      "A_Loss: 933.4014811197917 C_Loss: 1297.3184733072917\n",
      "EP OVER!\n",
      "A_Loss: 2035.8281412760416 C_Loss: 3957.7666178385416\n",
      "EP  74  START\n",
      "A_Loss: 479.3147298177083 C_Loss: 1727.3513753255208\n",
      "EP OVER!\n",
      "A_Loss: 579.0808024088542 C_Loss: 635.3917744954427\n",
      "EP  75  START\n",
      "A_Loss: 1450.3585856119792 C_Loss: 2148.2808756510417\n",
      "EP OVER!\n",
      "A_Loss: 917.6628540039062 C_Loss: 1192.1792277018228\n",
      "EP  76  START\n",
      "A_Loss: 888.0876180013021 C_Loss: 1050.4912679036458\n",
      "EP OVER!\n",
      "A_Loss: 249.14984232584635 C_Loss: 9789.355598958333\n",
      "EP  77  START\n",
      "A_Loss: 346.7358866373698 C_Loss: 1828.4739990234375\n",
      "EP OVER!\n",
      "A_Loss: 15.01647071838379 C_Loss: 1649.2421305338542\n",
      "EP  78  START\n",
      "A_Loss: 2058.658439127604 C_Loss: 1996.608447265625\n",
      "EP OVER!\n",
      "A_Loss: 155.17740580240886 C_Loss: 575.6137990315755\n",
      "EP  79  START\n",
      "A_Loss: 1246.8288167317708 C_Loss: 1408.468212890625\n",
      "EP OVER!\n",
      "A_Loss: 175.76889241536458 C_Loss: 825.604911295573\n",
      "EP  80  START\n",
      "A_Loss: 10138501.666666666 C_Loss: 33048526138440.633\n"
     ]
    }
   ],
   "source": [
    "bestScore = 200.\n",
    "stopTrainCounter = 0\n",
    "\n",
    "totalRewardHis = []\n",
    "totalActorLossHis = []\n",
    "totalCriticLossHis = []\n",
    "epHis = []\n",
    "maxTotalReward = -99999999999\n",
    "\n",
    "for ep in range(MAX_EP):\n",
    "    print(\"EP \",ep,\" START\")\n",
    "    # first time run game\n",
    "    s,_,_,_ = env.reset()\n",
    "    if (ep == 0):\n",
    "        epBuffer = buffer.buffer()\n",
    "        s = s.reshape([STATE_SIZE])\n",
    "        agent = PPO.PPO(stateSize=STATE_SIZE,\n",
    "                disActShape=disActShape,\n",
    "                conActSize=1,\n",
    "                conActRange=CTN_ACTION_RANGE,\n",
    "                criticLR=CRITIC_LR,\n",
    "                actorLR=ACTOR_LR,\n",
    "                gamma=GAMMA,\n",
    "                epsilon=EPSILON,\n",
    "                entropyWeight=ENTROPY_WHEIGHT,\n",
    "                saveDir=SAVE_DIR,\n",
    "                loadModelDir=LOAD_DIR)\n",
    "    step = 0\n",
    "    done = False\n",
    "    stopTrainCounter -= 1\n",
    "    epHis.append(ep)\n",
    "    \n",
    "    # reset total reward\n",
    "    epTotalReward = 0\n",
    "    \n",
    "    # Recorder list\n",
    "    epStepHis = []\n",
    "    epRewardHis = []\n",
    "    epActorLossHis = []\n",
    "    epCriticLossHis = []\n",
    "\n",
    "    while not done:\n",
    "        step += 1\n",
    "        if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n",
    "            epStepHis.append(step)\n",
    "            disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n",
    "            kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n",
    "            \n",
    "            nextState,thisReward,done,_ = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
    "\n",
    "            epTotalReward += thisReward\n",
    "            epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n",
    "        else:\n",
    "            disActs = np.array([[0,0,0,0,0]])\n",
    "            conActs = np.array([[0]])\n",
    "\n",
    "            nextState,thisReward,done,_ = env.step(discreteActions=disActs,continuousActions=conActs)\n",
    "            epTotalReward += thisReward\n",
    "        nextState = nextState.reshape([STATE_SIZE])\n",
    "        s = nextState\n",
    "        \n",
    "        if done:\n",
    "            print(\"EP OVER!\")\n",
    "        # update PPO after Batch step or GameOver\n",
    "        if (step+1)%BATCH == 0 or done:\n",
    "            bs = epBuffer.getStates()\n",
    "            ba = epBuffer.getActions()\n",
    "            br = epBuffer.getRewards()\n",
    "            epBuffer.clearBuffer()\n",
    "            if TRAIN:\n",
    "                epActorLoss,epCriticLoss = agent.trainCritcActor(bs,ba,br,s,CRITIC_EPOCH,ACTOR_EPOCH)\n",
    "                epActorLossHis.append(epActorLoss)\n",
    "                epCriticLossHis.append(epCriticLoss)\n",
    "        # update History Recorder\n",
    "        totalActorLossHis.append(np.mean(epActorLossHis))\n",
    "        totalCriticLossHis.append(np.mean(epCriticLossHis))\n",
    "        totalRewardHis.append(epTotalReward)\n",
    "        \n",
    "    if (epTotalReward > maxTotalReward and epTotalReward != 0):\n",
    "        maxTotalReward = epTotalReward\n",
    "        agent.saveWeights(epTotalReward)\n",
    "        print(\"New Record! Save NN\",epTotalReward)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aaa =  0\n",
      "aaa =  1\n",
      "aaa =  2\n",
      "aaa =  3\n",
      "aaa =  4\n",
      "aaa =  5\n",
      "aaa =  6\n",
      "aaa =  7\n",
      "aaa =  8\n",
      "aaa =  9\n"
     ]
    }
   ],
   "source": [
    "aaa = 0\n",
    "while aaa<10:\n",
    "    print(\"aaa = \",aaa)\n",
    "    aaa+=1"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}