{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import aimBotEnv\n", "import PPO\n", "import buffer\n", "import numpy as np\n", "\n", "import tensorflow as tf\n", "import time\n", "import datetime" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Attempts to allocate only the GPU memory needed for allocation\n", "physical_devices = tf.config.list_physical_devices('GPU')\n", "tf.config.experimental.set_memory_growth(physical_devices[0], True)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "ENV_PATH = './Build-MultiScene-WithLoad/Aimbot-PPO'\n", "WORKER_ID = 1\n", "BASE_PORT = 200\n", "\n", "MAX_EP = 1000\n", "EP_LENGTH = 100000\n", "GAMMA = 0.99 # discount future reward (UP?)\n", "EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n", "ACTOR_LR = 1e-5 # LR\n", "CRITIC_LR = 2e-5 # LR\n", "BATCH = 512 # learning step\n", "ACTOR_EPOCH = 15 # epoch\n", "CRITIC_EPOCH = 15 # epoch\n", "ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss\n", "ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n", "\n", "\n", "TRAIN = True\n", "SAVE_DIR = \"PPO-Model/\"+datetime.datetime.now().strftime(\"%m%d%H%M\")+\"/\"\n", "LOAD_DIR = None\n", "\n", "CTN_ACTION_RANGE = 10" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "√√√√√Enviroment Initialized Success√√√√√\n", "√√√√√Buffer Initialized Success√√√√√\n", "Load Model:\n", "PPO-Model/09051833/183540\n", "CONTINUOUS_SIZE 1\n", "DISCRETE_SIZE 5\n", "STATE_SIZE 29\n" ] } ], "source": [ "# initialize enviroment & buffer class\n", "env = aimBotEnv.makeEnv(envPath = ENV_PATH,\n", " workerID = WORKER_ID,\n", " basePort = BASE_PORT)\n", "epBuffer = buffer.buffer()\n", "\n", "STATE_SIZE = env.STATE_SIZE\n", "CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n", "DISCRETE_SIZE = env.DISCRETE_SIZE\n", "s,reward,done,loadDir = env.getSteps()\n", "\n", "# check load model or not\n", "if(np.any(loadDir == 0)):\n", " # create a new model\n", " print(\"No loadDir specified,Create a New Model\")\n", " LOAD_DIR = None\n", "else:\n", " # load model\n", " loadDirDateSTR = str(int(loadDir[0]))\n", " loadDirTimeSTR = str(int(loadDir[1]))\n", " if len(loadDirDateSTR)!=8:\n", " # fill lost 0 while converse float to string\n", " for _ in range(8 - len(loadDirDateSTR)):\n", " loadDirDateSTR = \"0\" + loadDirDateSTR\n", " if len(loadDirTimeSTR)!=6:\n", " # fill lost 0 while converse float to string\n", " for _ in range(6 - len(loadDirTimeSTR)):\n", " loadDirTimeSTR = \"0\" + loadDirTimeSTR\n", " LOAD_DIR = \"PPO-Model/\"+loadDirDateSTR+\"/\"+loadDirTimeSTR\n", " print(\"Load Model:\")\n", " print(LOAD_DIR)\n", "\n", "print(\"CONTINUOUS_SIZE\",CONTINUOUS_SIZE)\n", "print(\"DISCRETE_SIZE\",DISCRETE_SIZE)\n", "print(\"STATE_SIZE\",STATE_SIZE)\n", "\n", "disActShape = [3,3,2]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def actToKey(disAct1,disAct2,disAct3,conAct):\n", " kW = 0\n", " kS = 0\n", " kA = 0\n", " kD = 0\n", " mouseShoot = 0\n", " if disAct1 == 0:\n", " kW = 0\n", " kS = 1\n", " elif disAct1 == 1:\n", " kW = 0\n", " kS = 0\n", " elif disAct1 == 2:\n", " kW = 1\n", " kS = 0\n", " if disAct2 == 0:\n", " kA = 0\n", " kD = 1\n", " elif disAct2 == 1:\n", " kA = 0\n", " kD = 0\n", " elif disAct2 == 2:\n", " kA = 1\n", " kD = 0\n", " mouseShoot = disAct3\n", " return kW,kS,kA,kD,mouseShoot,conAct" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "EP 0 START\n", "√√√√√Buffer Initialized Success√√√√√\n", "++++++++++++++++++++++++++++++++++++\n", "++++++++++++Model Loaded++++++++++++\n", "PPO-Model/09051833/183540\n", "++++++++++++++++++++++++++++++++++++\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n", " return _methods._mean(a, axis=axis, dtype=dtype,\n", "c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n", " ret = ret.dtype.type(ret / rcount)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "A_Loss: -43.132581075032554 C_Loss: 4025.360986328125\n", "EP OVER!\n", "A_Loss: 1184751.5479166666 C_Loss: 1939213379930.6667\n", "Model's Weights Saved\n", "New Record! Save NN -37.67000053450465\n", "EP 1 START\n", "A_Loss: 14251923.066666666 C_Loss: 40720630.7288086\n", "EP OVER!\n", "A_Loss: 1857006.7333333334 C_Loss: 245.15654500325522\n", "Model's Weights Saved\n", "New Record! Save NN -13.100000601261854\n", "EP 2 START\n", "A_Loss: -1.0014899969100952 C_Loss: 71.29023424784343\n", "EP OVER!\n", "A_Loss: -0.31318608721097313 C_Loss: 8.897234630584716\n", "EP 3 START\n", "A_Loss: 566053.9979166667 C_Loss: 1986013.3489705403\n", "EP OVER!\n", "A_Loss: 0.6021817366282145 C_Loss: 6.035458342234294\n", "EP 4 START\n", "A_Loss: 169656457.6 C_Loss: 4317.756831359863\n", "EP OVER!\n", "A_Loss: 22155934.0 C_Loss: 63.52870483398438\n", "EP 5 START\n", "A_Loss: 0.2413090835014979 C_Loss: 10.40585823059082\n", "EP OVER!\n", "A_Loss: 0.12157159547011058 C_Loss: 14.347647285461425\n", "EP 6 START\n", "A_Loss: 215977770.66666666 C_Loss: 190435277.5966268\n", "EP OVER!\n", "A_Loss: 213349568.0 C_Loss: 1620.733740234375\n", "EP 7 START\n", "A_Loss: 100149825.6 C_Loss: 1569803.3794799806\n", "EP OVER!\n", "A_Loss: 171062395.73333332 C_Loss: 34872400594.933334\n", "Model's Weights Saved\n", "New Record! Save NN 4.119999349117279\n", "EP 8 START\n", "A_Loss: 0.7038820425669352 C_Loss: 31.980174128214518\n", "EP OVER!\n", "A_Loss: 0.2687960028648376 C_Loss: 6.94128991762797\n", "EP 9 START\n", "A_Loss: 0.1451285809278488 C_Loss: 3.5754743576049806\n", "EP OVER!\n", "A_Loss: 0.18081151247024535 C_Loss: 3.902424764633179\n", "EP 10 START\n", "A_Loss: 29139865.6 C_Loss: 174927458.90618488\n", "EP OVER!\n", "A_Loss: 16942552.4 C_Loss: 134.21190592447917\n", "EP 11 START\n", "A_Loss: 53790223.46666667 C_Loss: 647.0605305989583\n", "EP OVER!\n", "A_Loss: 27695839.866666667 C_Loss: 343.9340047200521\n", "EP 12 START\n", "A_Loss: 65695720.8 C_Loss: 61225647615.0198\n", "EP OVER!\n", "A_Loss: 0.07575627962748209 C_Loss: 10.41986296971639\n", "EP 13 START\n", "A_Loss: 51573392.266666666 C_Loss: 212022811.82775268\n", "EP OVER!\n", "A_Loss: 22073133.866666667 C_Loss: 61.18818028767904\n", "EP 14 START\n", "A_Loss: 0.13529965033133826 C_Loss: 0.966444210211436\n", "EP OVER!\n", "A_Loss: 0.1176165262858073 C_Loss: 0.5141626675923665\n", "EP 15 START\n", "A_Loss: 0.5104357699553171 C_Loss: 25.79277165730794\n", "EP OVER!\n", "A_Loss: 15181718.0 C_Loss: 9111658759031.467\n", "EP 16 START\n", "A_Loss: 30694314.533333335 C_Loss: 62842582.26665497\n", "EP OVER!\n", "A_Loss: 17702280.066666666 C_Loss: 50.18375091552734\n", "EP 17 START\n", "A_Loss: 0.08570613314708074 C_Loss: 1.7101642807324728\n", "EP OVER!\n", "A_Loss: 14154208.066666666 C_Loss: 248.65523885091145\n", "EP 18 START\n", "A_Loss: 51043696.8 C_Loss: 39972927.18101196\n", "EP OVER!\n", "A_Loss: 24850118.0 C_Loss: 6779725.618615722\n", "EP 19 START\n", "A_Loss: 75697662.4 C_Loss: 505.26551717122396\n", "EP OVER!\n", "A_Loss: 64376180.266666666 C_Loss: 299.2391876220703\n", "EP 20 START\n", "A_Loss: 55954064.8 C_Loss: 234.9048828125\n", "EP OVER!\n", "A_Loss: 57271699.733333334 C_Loss: 59254958.60783488\n", "EP 21 START\n", "A_Loss: 124046376.53333333 C_Loss: 2737571204155.7334\n", "EP OVER!\n", "A_Loss: 31774753.866666667 C_Loss: 24325611279.066666\n", "EP 22 START\n", "A_Loss: 7490019498.666667 C_Loss: 245487346653.86667\n", "EP OVER!\n", "A_Loss: 186999600.0 C_Loss: 36627312369.066666\n", "EP 23 START\n", "A_Loss: 4297551769.6 C_Loss: 1184376194184.5334\n", "EP OVER!\n", "A_Loss: 122982910044842.67 C_Loss: 7331505971746.134\n", "EP 24 START\n", "A_Loss: 2682021.3583333334 C_Loss: 2341072196027.7334\n", "EP OVER!\n", "A_Loss: 1469103463901.8667 C_Loss: 113593101.86666666\n", "EP 25 START\n", "A_Loss: 76648460.0 C_Loss: 14498072.481510418\n", "EP OVER!\n", "A_Loss: 5800877.766666667 C_Loss: 22134.297265625\n", "EP 26 START\n", "A_Loss: 34429532.53333333 C_Loss: 46926.630208333336\n", "EP OVER!\n", "A_Loss: 212612493.86666667 C_Loss: 23176.368880208334\n", "EP 27 START\n", "A_Loss: 3207813239.4666667 C_Loss: 41575.61041666667\n", "EP OVER!\n", "A_Loss: 38858107.86666667 C_Loss: 15486.2171875\n", "EP 28 START\n", "A_Loss: -58850.48046875 C_Loss: 14550529378106.69\n", "EP OVER!\n", "A_Loss: 445133260.8 C_Loss: 1749447703.4604166\n", "EP 29 START\n", "A_Loss: 80796092.8 C_Loss: 8523564.864453126\n", "EP OVER!\n", "A_Loss: 34027046.266666666 C_Loss: 3319.6830078125\n", "Model's Weights Saved\n", "New Record! Save NN 4.589999713003635\n", "EP 30 START\n", "A_Loss: 118412868.26666667 C_Loss: 23715976.025211588\n", "EP OVER!\n", "A_Loss: 94927646.93333334 C_Loss: 1067.18818359375\n", "EP 31 START\n", "A_Loss: 338770097.06666666 C_Loss: 3585200.7686075848\n", "EP OVER!\n", "A_Loss: 1479514146.1333334 C_Loss: 10348.122916666667\n", "Model's Weights Saved\n", "New Record! Save NN 38.5099995136261\n", "EP 32 START\n", "A_Loss: 119593341.86666666 C_Loss: 784.9417297363282\n", "EP OVER!\n", "A_Loss: -3.2093143463134766 C_Loss: 180.5671844482422\n", "EP 33 START\n", "A_Loss: 149599549.86666667 C_Loss: 206.16288146972656\n", "EP OVER!\n", "A_Loss: 5682631.7 C_Loss: 17.62857920328776\n", "EP 34 START\n", "A_Loss: 231213670.4 C_Loss: 272.8009948730469\n", "EP OVER!\n", "A_Loss: 167307104.0 C_Loss: 188.52051798502603\n", "EP 35 START\n", "A_Loss: 204825578.66666666 C_Loss: 335.75029296875\n", "EP OVER!\n", "A_Loss: 83991501.33333333 C_Loss: 451.3644246419271\n", "EP 36 START\n", "A_Loss: 0.6501724402109782 C_Loss: 38.280514017740884\n", "EP OVER!\n", "A_Loss: 175042737.06666666 C_Loss: 256.6915344238281\n", "EP 37 START\n", "A_Loss: 461190880.0 C_Loss: 717.4720499674479\n", "EP OVER!\n", "A_Loss: 173152186.66666666 C_Loss: 701.6123962402344\n", "EP 38 START\n", "A_Loss: 188070140.8 C_Loss: 1611377609664.9072\n", "EP OVER!\n", "A_Loss: 512005579391093.7 C_Loss: 86553492717568.0\n", "EP 39 START\n", "A_Loss: 121553.57962239583 C_Loss: 53618558110.88125\n", "EP OVER!\n", "A_Loss: 10892.0080078125 C_Loss: 132429.940625\n", "EP 40 START\n", "A_Loss: 15404.601106770833 C_Loss: 107106.1421875\n", "EP OVER!\n", "A_Loss: 1822.5163004557292 C_Loss: 182031.73854166668\n", "EP 41 START\n", "A_Loss: 599.1890706380208 C_Loss: 76118.35729166666\n", "EP OVER!\n", "A_Loss: 1033.1336588541667 C_Loss: 54971.41354166667\n", "EP 42 START\n", "A_Loss: 2602.054020182292 C_Loss: 227568.73541666666\n", "EP OVER!\n", "A_Loss: -3.2474422613779703 C_Loss: 155.22516682942708\n", "EP 43 START\n", "A_Loss: 1994.366455078125 C_Loss: 149748.48125\n", "EP OVER!\n", "A_Loss: 7441.857747395833 C_Loss: 196344.43958333333\n", "EP 44 START\n", "A_Loss: 19441.808723958333 C_Loss: 197749.58020833333\n", "EP OVER!\n", "A_Loss: 1789.1592529296875 C_Loss: 25795.180598958334\n", "EP 45 START\n", "A_Loss: 6888.085221354167 C_Loss: 149084.575\n", "EP OVER!\n", "A_Loss: 1102350.757421875 C_Loss: 180759.19166666668\n", "EP 46 START\n", "A_Loss: 536.078184000651 C_Loss: 89909.24375\n", "EP OVER!\n", "A_Loss: 794341992174.4 C_Loss: 59005630871824.9\n", "EP 47 START\n", "A_Loss: 114184075127.46666 C_Loss: 74263479521553.06\n", "EP OVER!\n", "A_Loss: 1293861572.3333333 C_Loss: 1474176015837.8667\n", "EP 48 START\n", "A_Loss: 1143.1249674479166 C_Loss: 19221068817.411232\n", "EP OVER!\n", "A_Loss: 855.1484252929688 C_Loss: 28676.431770833333\n", "EP 49 START\n", "A_Loss: 1198.7060546875 C_Loss: 41875.48359375\n", "EP OVER!\n", "A_Loss: 1215.0390218098958 C_Loss: 7167.594791666666\n", "EP 50 START\n", "A_Loss: 1840.496435546875 C_Loss: 33346.76875\n", "EP OVER!\n", "A_Loss: 97.68925425211589 C_Loss: 2453.3792643229167\n", "EP 51 START\n", "A_Loss: 2179.5446614583334 C_Loss: 25573.869270833333\n", "EP OVER!\n", "A_Loss: 3547.454052734375 C_Loss: 15107.698307291666\n", "EP 52 START\n", "A_Loss: 513.1327128092448 C_Loss: 23233.316276041667\n", "EP OVER!\n", "A_Loss: 2024.9339111328125 C_Loss: 10457.81875\n", "EP 53 START\n", "A_Loss: 321.8419494628906 C_Loss: 10830.605924479167\n", "EP OVER!\n", "A_Loss: -8.38951981862386 C_Loss: 2241.023612467448\n", "EP 54 START\n", "A_Loss: 1454.901765950521 C_Loss: 16337.481966145833\n", "EP OVER!\n", "A_Loss: 3069.5313313802085 C_Loss: 4450.756380208333\n", "EP 55 START\n", "A_Loss: 444.8317138671875 C_Loss: 15201.106184895832\n", "EP OVER!\n", "A_Loss: 1159.2985107421875 C_Loss: 35525.82317708333\n", "EP 56 START\n", "A_Loss: 1397.7694742838542 C_Loss: 8600.898404947917\n", "EP OVER!\n", "A_Loss: 160.49285074869792 C_Loss: 3749.308203125\n", "EP 57 START\n", "A_Loss: -23.67707913716634 C_Loss: 8787.21162109375\n", "EP OVER!\n", "A_Loss: 446.2368998209635 C_Loss: 8949.849674479166\n", "EP 58 START\n", "A_Loss: 1443.8223063151042 C_Loss: 4806.931754557291\n", "EP OVER!\n", "A_Loss: 1028.3776000976563 C_Loss: 10080.125455729167\n", "EP 59 START\n", "A_Loss: 979.55380859375 C_Loss: 18168.035872395834\n", "EP OVER!\n", "A_Loss: 2799.885953776042 C_Loss: 14584.458203125\n", "EP 60 START\n", "A_Loss: 83247.30833333333 C_Loss: 61687910677571.76\n", "EP OVER!\n", "A_Loss: 81.43709259033203 C_Loss: 47630220.59791667\n", "EP 61 START\n", "A_Loss: 5032.870377604167 C_Loss: 43944.79453125\n", "EP OVER!\n", "A_Loss: 9.13815720876058 C_Loss: 2573.8714192708335\n", "EP 62 START\n", "A_Loss: 523.7818908691406 C_Loss: 8441.000618489583\n", "EP OVER!\n", "A_Loss: 1593.0845377604167 C_Loss: 17598.201953125\n", "EP 63 START\n", "A_Loss: 27.476329803466797 C_Loss: 5594.57646484375\n", "EP OVER!\n", "A_Loss: 25.40858777364095 C_Loss: 11878.195572916668\n", "EP 64 START\n", "A_Loss: 3347.620393880208 C_Loss: 15555.9111328125\n", "EP OVER!\n", "A_Loss: 25.35328992207845 C_Loss: 7923.337727864583\n", "EP 65 START\n", "A_Loss: 1800.478165690104 C_Loss: 36845.799479166664\n", "EP OVER!\n", "A_Loss: 544.9222574869792 C_Loss: 2949.7829752604166\n", "EP 66 START\n", "A_Loss: 238.4148183186849 C_Loss: 2108.7259195963543\n", "EP OVER!\n", "A_Loss: 10.808652464548747 C_Loss: 1069.8261800130208\n", "EP 67 START\n", "A_Loss: 7.2742944399515785 C_Loss: 1112.0956298828125\n", "EP OVER!\n", "A_Loss: 686.372725423177 C_Loss: 1421.539111328125\n", "EP 68 START\n", "A_Loss: 38.04718589782715 C_Loss: 2771.959724934896\n", "EP OVER!\n", "A_Loss: 1133.577986653646 C_Loss: 5758.061686197917\n", "EP 69 START\n", "A_Loss: 402.1093465169271 C_Loss: 3987.9388671875\n", "EP OVER!\n", "A_Loss: -1.7969589630762737 C_Loss: 2884.10732421875\n", "EP 70 START\n", "A_Loss: 1016.8031656901042 C_Loss: 6636.073665364584\n", "EP OVER!\n", "A_Loss: 939.9079630533854 C_Loss: 3692.443391927083\n", "EP 71 START\n", "A_Loss: -107.65687510172526 C_Loss: 21740540498583.367\n", "EP OVER!\n", "A_Loss: 5876.485221354166 C_Loss: 50534.828385416666\n", "EP 72 START\n", "A_Loss: 219.98338216145834 C_Loss: 2713.9967041015625\n", "EP OVER!\n", "A_Loss: 433.9121032714844 C_Loss: 950.6825520833333\n", "EP 73 START\n", "A_Loss: 933.4014811197917 C_Loss: 1297.3184733072917\n", "EP OVER!\n", "A_Loss: 2035.8281412760416 C_Loss: 3957.7666178385416\n", "EP 74 START\n", "A_Loss: 479.3147298177083 C_Loss: 1727.3513753255208\n", "EP OVER!\n", "A_Loss: 579.0808024088542 C_Loss: 635.3917744954427\n", "EP 75 START\n", "A_Loss: 1450.3585856119792 C_Loss: 2148.2808756510417\n", "EP OVER!\n", "A_Loss: 917.6628540039062 C_Loss: 1192.1792277018228\n", "EP 76 START\n", "A_Loss: 888.0876180013021 C_Loss: 1050.4912679036458\n", "EP OVER!\n", "A_Loss: 249.14984232584635 C_Loss: 9789.355598958333\n", "EP 77 START\n", "A_Loss: 346.7358866373698 C_Loss: 1828.4739990234375\n", "EP OVER!\n", "A_Loss: 15.01647071838379 C_Loss: 1649.2421305338542\n", "EP 78 START\n", "A_Loss: 2058.658439127604 C_Loss: 1996.608447265625\n", "EP OVER!\n", "A_Loss: 155.17740580240886 C_Loss: 575.6137990315755\n", "EP 79 START\n", "A_Loss: 1246.8288167317708 C_Loss: 1408.468212890625\n", "EP OVER!\n", "A_Loss: 175.76889241536458 C_Loss: 825.604911295573\n", "EP 80 START\n", "A_Loss: 10138501.666666666 C_Loss: 33048526138440.633\n" ] } ], "source": [ "bestScore = 200.\n", "stopTrainCounter = 0\n", "\n", "totalRewardHis = []\n", "totalActorLossHis = []\n", "totalCriticLossHis = []\n", "epHis = []\n", "maxTotalReward = -99999999999\n", "\n", "for ep in range(MAX_EP):\n", " print(\"EP \",ep,\" START\")\n", " # first time run game\n", " s,_,_,_ = env.reset()\n", " if (ep == 0):\n", " epBuffer = buffer.buffer()\n", " s = s.reshape([STATE_SIZE])\n", " agent = PPO.PPO(stateSize=STATE_SIZE,\n", " disActShape=disActShape,\n", " conActSize=1,\n", " conActRange=CTN_ACTION_RANGE,\n", " criticLR=CRITIC_LR,\n", " actorLR=ACTOR_LR,\n", " gamma=GAMMA,\n", " epsilon=EPSILON,\n", " entropyWeight=ENTROPY_WHEIGHT,\n", " saveDir=SAVE_DIR,\n", " loadModelDir=LOAD_DIR)\n", " step = 0\n", " done = False\n", " stopTrainCounter -= 1\n", " epHis.append(ep)\n", " \n", " # reset total reward\n", " epTotalReward = 0\n", " \n", " # Recorder list\n", " epStepHis = []\n", " epRewardHis = []\n", " epActorLossHis = []\n", " epCriticLossHis = []\n", "\n", " while not done:\n", " step += 1\n", " if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n", " epStepHis.append(step)\n", " disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n", " kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n", " \n", " nextState,thisReward,done,_ = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n", "\n", " epTotalReward += thisReward\n", " epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n", " else:\n", " disActs = np.array([[0,0,0,0,0]])\n", " conActs = np.array([[0]])\n", "\n", " nextState,thisReward,done,_ = env.step(discreteActions=disActs,continuousActions=conActs)\n", " epTotalReward += thisReward\n", " nextState = nextState.reshape([STATE_SIZE])\n", " s = nextState\n", " \n", " if done:\n", " print(\"EP OVER!\")\n", " # update PPO after Batch step or GameOver\n", " if (step+1)%BATCH == 0 or done:\n", " bs = epBuffer.getStates()\n", " ba = epBuffer.getActions()\n", " br = epBuffer.getRewards()\n", " epBuffer.clearBuffer()\n", " if TRAIN:\n", " epActorLoss,epCriticLoss = agent.trainCritcActor(bs,ba,br,s,CRITIC_EPOCH,ACTOR_EPOCH)\n", " epActorLossHis.append(epActorLoss)\n", " epCriticLossHis.append(epCriticLoss)\n", " # update History Recorder\n", " totalActorLossHis.append(np.mean(epActorLossHis))\n", " totalCriticLossHis.append(np.mean(epCriticLossHis))\n", " totalRewardHis.append(epTotalReward)\n", " \n", " if (epTotalReward > maxTotalReward and epTotalReward != 0):\n", " maxTotalReward = epTotalReward\n", " agent.saveWeights(epTotalReward)\n", " print(\"New Record! Save NN\",epTotalReward)\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "aaa = 0\n", "aaa = 1\n", "aaa = 2\n", "aaa = 3\n", "aaa = 4\n", "aaa = 5\n", "aaa = 6\n", "aaa = 7\n", "aaa = 8\n", "aaa = 9\n" ] } ], "source": [ "aaa = 0\n", "while aaa<10:\n", " print(\"aaa = \",aaa)\n", " aaa+=1" ] } ], "metadata": { "interpreter": { "hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32" }, "kernelspec": { "display_name": "Python 3.9.7 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }