Koha9/Aimbot-PPO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
2d404cfdf2
Aimbot-PPO/Aimbot-PPO-Python/main.ipynb

660 lines
24 KiB
Plaintext
Raw Normal View History

Aimbot Enviroment very first Basic environment include Multi scene, Reward Change, Visible chart, etc....
2022-09-05 11:46:08 +00:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import aimBotEnv\n",
"import PPO\n",
"import buffer\n",
"import numpy as np\n",
"\n",
"import tensorflow as tf\n",
"import time\n",
"import datetime"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Attempts to allocate only the GPU memory needed for allocation\n",
"physical_devices = tf.config.list_physical_devices('GPU')\n",
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"ENV_PATH = './Build-MultiScene-WithLoad/Aimbot-PPO'\n",
"WORKER_ID = 1\n",
"BASE_PORT = 200\n",
"\n",
"MAX_EP = 1000\n",
"EP_LENGTH = 100000\n",
"GAMMA = 0.99 # discount future reward (UP?)\n",
"EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n",
"ACTOR_LR = 1e-5 # LR\n",
"CRITIC_LR = 2e-5 # LR\n",
"BATCH = 512 # learning step\n",
"ACTOR_EPOCH = 15 # epoch\n",
"CRITIC_EPOCH = 15 # epoch\n",
"ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss\n",
"ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n",
"\n",
"\n",
"TRAIN = True\n",
"SAVE_DIR = \"PPO-Model/\"+datetime.datetime.now().strftime(\"%m%d%H%M\")+\"/\"\n",
"LOAD_DIR = None\n",
"\n",
"CTN_ACTION_RANGE = 10"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"√√√√√Enviroment Initialized Success√√√√√\n",
"√√√√√Buffer Initialized Success√√√√√\n",
"Load Model:\n",
"PPO-Model/09051833/183540\n",
"CONTINUOUS_SIZE 1\n",
"DISCRETE_SIZE 5\n",
"STATE_SIZE 29\n"
]
}
],
"source": [
"# initialize enviroment & buffer class\n",
"env = aimBotEnv.makeEnv(envPath = ENV_PATH,\n",
" workerID = WORKER_ID,\n",
" basePort = BASE_PORT)\n",
"epBuffer = buffer.buffer()\n",
"\n",
"STATE_SIZE = env.STATE_SIZE\n",
"CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
"DISCRETE_SIZE = env.DISCRETE_SIZE\n",
"s,reward,done,loadDir = env.getSteps()\n",
"\n",
"# check load model or not\n",
"if(np.any(loadDir == 0)):\n",
" # create a new model\n",
" print(\"No loadDir specified,Create a New Model\")\n",
" LOAD_DIR = None\n",
"else:\n",
" # load model\n",
" loadDirDateSTR = str(int(loadDir[0]))\n",
" loadDirTimeSTR = str(int(loadDir[1]))\n",
" if len(loadDirDateSTR)!=8:\n",
" # fill lost 0 while converse float to string\n",
" for _ in range(8 - len(loadDirDateSTR)):\n",
" loadDirDateSTR = \"0\" + loadDirDateSTR\n",
" if len(loadDirTimeSTR)!=6:\n",
" # fill lost 0 while converse float to string\n",
" for _ in range(6 - len(loadDirTimeSTR)):\n",
" loadDirTimeSTR = \"0\" + loadDirTimeSTR\n",
" LOAD_DIR = \"PPO-Model/\"+loadDirDateSTR+\"/\"+loadDirTimeSTR\n",
" print(\"Load Model:\")\n",
" print(LOAD_DIR)\n",
"\n",
"print(\"CONTINUOUS_SIZE\",CONTINUOUS_SIZE)\n",
"print(\"DISCRETE_SIZE\",DISCRETE_SIZE)\n",
"print(\"STATE_SIZE\",STATE_SIZE)\n",
"\n",
"disActShape = [3,3,2]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def actToKey(disAct1,disAct2,disAct3,conAct):\n",
" kW = 0\n",
" kS = 0\n",
" kA = 0\n",
" kD = 0\n",
" mouseShoot = 0\n",
" if disAct1 == 0:\n",
" kW = 0\n",
" kS = 1\n",
" elif disAct1 == 1:\n",
" kW = 0\n",
" kS = 0\n",
" elif disAct1 == 2:\n",
" kW = 1\n",
" kS = 0\n",
" if disAct2 == 0:\n",
" kA = 0\n",
" kD = 1\n",
" elif disAct2 == 1:\n",
" kA = 0\n",
" kD = 0\n",
" elif disAct2 == 2:\n",
" kA = 1\n",
" kD = 0\n",
" mouseShoot = disAct3\n",
" return kW,kS,kA,kD,mouseShoot,conAct"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"EP 0 START\n",
"√√√√√Buffer Initialized Success√√√√√\n",
"++++++++++++++++++++++++++++++++++++\n",
"++++++++++++Model Loaded++++++++++++\n",
"PPO-Model/09051833/183540\n",
"++++++++++++++++++++++++++++++++++++\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n",
" return _methods._mean(a, axis=axis, dtype=dtype,\n",
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
" ret = ret.dtype.type(ret / rcount)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"A_Loss: -43.132581075032554 C_Loss: 4025.360986328125\n",
"EP OVER!\n",
"A_Loss: 1184751.5479166666 C_Loss: 1939213379930.6667\n",
"Model's Weights Saved\n",
"New Record! Save NN -37.67000053450465\n",
"EP 1 START\n",
"A_Loss: 14251923.066666666 C_Loss: 40720630.7288086\n",
"EP OVER!\n",
"A_Loss: 1857006.7333333334 C_Loss: 245.15654500325522\n",
"Model's Weights Saved\n",
"New Record! Save NN -13.100000601261854\n",
"EP 2 START\n",
"A_Loss: -1.0014899969100952 C_Loss: 71.29023424784343\n",
"EP OVER!\n",
"A_Loss: -0.31318608721097313 C_Loss: 8.897234630584716\n",
"EP 3 START\n",
"A_Loss: 566053.9979166667 C_Loss: 1986013.3489705403\n",
"EP OVER!\n",
"A_Loss: 0.6021817366282145 C_Loss: 6.035458342234294\n",
"EP 4 START\n",
"A_Loss: 169656457.6 C_Loss: 4317.756831359863\n",
"EP OVER!\n",
"A_Loss: 22155934.0 C_Loss: 63.52870483398438\n",
"EP 5 START\n",
"A_Loss: 0.2413090835014979 C_Loss: 10.40585823059082\n",
"EP OVER!\n",
"A_Loss: 0.12157159547011058 C_Loss: 14.347647285461425\n",
"EP 6 START\n",
"A_Loss: 215977770.66666666 C_Loss: 190435277.5966268\n",
"EP OVER!\n",
"A_Loss: 213349568.0 C_Loss: 1620.733740234375\n",
"EP 7 START\n",
"A_Loss: 100149825.6 C_Loss: 1569803.3794799806\n",
"EP OVER!\n",
"A_Loss: 171062395.73333332 C_Loss: 34872400594.933334\n",
"Model's Weights Saved\n",
"New Record! Save NN 4.119999349117279\n",
"EP 8 START\n",
"A_Loss: 0.7038820425669352 C_Loss: 31.980174128214518\n",
"EP OVER!\n",
"A_Loss: 0.2687960028648376 C_Loss: 6.94128991762797\n",
"EP 9 START\n",
"A_Loss: 0.1451285809278488 C_Loss: 3.5754743576049806\n",
"EP OVER!\n",
"A_Loss: 0.18081151247024535 C_Loss: 3.902424764633179\n",
"EP 10 START\n",
"A_Loss: 29139865.6 C_Loss: 174927458.90618488\n",
"EP OVER!\n",
"A_Loss: 16942552.4 C_Loss: 134.21190592447917\n",
"EP 11 START\n",
"A_Loss: 53790223.46666667 C_Loss: 647.0605305989583\n",
"EP OVER!\n",
"A_Loss: 27695839.866666667 C_Loss: 343.9340047200521\n",
"EP 12 START\n",
"A_Loss: 65695720.8 C_Loss: 61225647615.0198\n",
"EP OVER!\n",
"A_Loss: 0.07575627962748209 C_Loss: 10.41986296971639\n",
"EP 13 START\n",
"A_Loss: 51573392.266666666 C_Loss: 212022811.82775268\n",
"EP OVER!\n",
"A_Loss: 22073133.866666667 C_Loss: 61.18818028767904\n",
"EP 14 START\n",
"A_Loss: 0.13529965033133826 C_Loss: 0.966444210211436\n",
"EP OVER!\n",
"A_Loss: 0.1176165262858073 C_Loss: 0.5141626675923665\n",
"EP 15 START\n",
"A_Loss: 0.5104357699553171 C_Loss: 25.79277165730794\n",
"EP OVER!\n",
"A_Loss: 15181718.0 C_Loss: 9111658759031.467\n",
"EP 16 START\n",
"A_Loss: 30694314.533333335 C_Loss: 62842582.26665497\n",
"EP OVER!\n",
"A_Loss: 17702280.066666666 C_Loss: 50.18375091552734\n",
"EP 17 START\n",
"A_Loss: 0.08570613314708074 C_Loss: 1.7101642807324728\n",
"EP OVER!\n",
"A_Loss: 14154208.066666666 C_Loss: 248.65523885091145\n",
"EP 18 START\n",
"A_Loss: 51043696.8 C_Loss: 39972927.18101196\n",
"EP OVER!\n",
"A_Loss: 24850118.0 C_Loss: 6779725.618615722\n",
"EP 19 START\n",
"A_Loss: 75697662.4 C_Loss: 505.26551717122396\n",
"EP OVER!\n",
"A_Loss: 64376180.266666666 C_Loss: 299.2391876220703\n",
"EP 20 START\n",
"A_Loss: 55954064.8 C_Loss: 234.9048828125\n",
"EP OVER!\n",
"A_Loss: 57271699.733333334 C_Loss: 59254958.60783488\n",
"EP 21 START\n",
"A_Loss: 124046376.53333333 C_Loss: 2737571204155.7334\n",
"EP OVER!\n",
"A_Loss: 31774753.866666667 C_Loss: 24325611279.066666\n",
"EP 22 START\n",
"A_Loss: 7490019498.666667 C_Loss: 245487346653.86667\n",
"EP OVER!\n",
"A_Loss: 186999600.0 C_Loss: 36627312369.066666\n",
"EP 23 START\n",
"A_Loss: 4297551769.6 C_Loss: 1184376194184.5334\n",
"EP OVER!\n",
"A_Loss: 122982910044842.67 C_Loss: 7331505971746.134\n",
"EP 24 START\n",
"A_Loss: 2682021.3583333334 C_Loss: 2341072196027.7334\n",
"EP OVER!\n",
"A_Loss: 1469103463901.8667 C_Loss: 113593101.86666666\n",
"EP 25 START\n",
"A_Loss: 76648460.0 C_Loss: 14498072.481510418\n",
"EP OVER!\n",
"A_Loss: 5800877.766666667 C_Loss: 22134.297265625\n",
"EP 26 START\n",
"A_Loss: 34429532.53333333 C_Loss: 46926.630208333336\n",
"EP OVER!\n",
"A_Loss: 212612493.86666667 C_Loss: 23176.368880208334\n",
"EP 27 START\n",
"A_Loss: 3207813239.4666667 C_Loss: 41575.61041666667\n",
"EP OVER!\n",
"A_Loss: 38858107.86666667 C_Loss: 15486.2171875\n",
"EP 28 START\n",
"A_Loss: -58850.48046875 C_Loss: 14550529378106.69\n",
"EP OVER!\n",
"A_Loss: 445133260.8 C_Loss: 1749447703.4604166\n",
"EP 29 START\n",
"A_Loss: 80796092.8 C_Loss: 8523564.864453126\n",
"EP OVER!\n",
"A_Loss: 34027046.266666666 C_Loss: 3319.6830078125\n",
"Model's Weights Saved\n",
"New Record! Save NN 4.589999713003635\n",
"EP 30 START\n",
"A_Loss: 118412868.26666667 C_Loss: 23715976.025211588\n",
"EP OVER!\n",
"A_Loss: 94927646.93333334 C_Loss: 1067.18818359375\n",
"EP 31 START\n",
"A_Loss: 338770097.06666666 C_Loss: 3585200.7686075848\n",
"EP OVER!\n",
"A_Loss: 1479514146.1333334 C_Loss: 10348.122916666667\n",
"Model's Weights Saved\n",
"New Record! Save NN 38.5099995136261\n",
"EP 32 START\n",
"A_Loss: 119593341.86666666 C_Loss: 784.9417297363282\n",
"EP OVER!\n",
"A_Loss: -3.2093143463134766 C_Loss: 180.5671844482422\n",
"EP 33 START\n",
"A_Loss: 149599549.86666667 C_Loss: 206.16288146972656\n",
"EP OVER!\n",
"A_Loss: 5682631.7 C_Loss: 17.62857920328776\n",
"EP 34 START\n",
"A_Loss: 231213670.4 C_Loss: 272.8009948730469\n",
"EP OVER!\n",
"A_Loss: 167307104.0 C_Loss: 188.52051798502603\n",
"EP 35 START\n",
"A_Loss: 204825578.66666666 C_Loss: 335.75029296875\n",
"EP OVER!\n",
"A_Loss: 83991501.33333333 C_Loss: 451.3644246419271\n",
"EP 36 START\n",
"A_Loss: 0.6501724402109782 C_Loss: 38.280514017740884\n",
"EP OVER!\n",
"A_Loss: 175042737.06666666 C_Loss: 256.6915344238281\n",
"EP 37 START\n",
"A_Loss: 461190880.0 C_Loss: 717.4720499674479\n",
"EP OVER!\n",
"A_Loss: 173152186.66666666 C_Loss: 701.6123962402344\n",
"EP 38 START\n",
"A_Loss: 188070140.8 C_Loss: 1611377609664.9072\n",
"EP OVER!\n",
"A_Loss: 512005579391093.7 C_Loss: 86553492717568.0\n",
"EP 39 START\n",
"A_Loss: 121553.57962239583 C_Loss: 53618558110.88125\n",
"EP OVER!\n",
"A_Loss: 10892.0080078125 C_Loss: 132429.940625\n",
"EP 40 START\n",
"A_Loss: 15404.601106770833 C_Loss: 107106.1421875\n",
"EP OVER!\n",
"A_Loss: 1822.5163004557292 C_Loss: 182031.73854166668\n",
"EP 41 START\n",
"A_Loss: 599.1890706380208 C_Loss: 76118.35729166666\n",
"EP OVER!\n",
"A_Loss: 1033.1336588541667 C_Loss: 54971.41354166667\n",
"EP 42 START\n",
"A_Loss: 2602.054020182292 C_Loss: 227568.73541666666\n",
"EP OVER!\n",
"A_Loss: -3.2474422613779703 C_Loss: 155.22516682942708\n",
"EP 43 START\n",
"A_Loss: 1994.366455078125 C_Loss: 149748.48125\n",
"EP OVER!\n",
"A_Loss: 7441.857747395833 C_Loss: 196344.43958333333\n",
"EP 44 START\n",
"A_Loss: 19441.808723958333 C_Loss: 197749.58020833333\n",
"EP OVER!\n",
"A_Loss: 1789.1592529296875 C_Loss: 25795.180598958334\n",
"EP 45 START\n",
"A_Loss: 6888.085221354167 C_Loss: 149084.575\n",
"EP OVER!\n",
"A_Loss: 1102350.757421875 C_Loss: 180759.19166666668\n",
"EP 46 START\n",
"A_Loss: 536.078184000651 C_Loss: 89909.24375\n",
"EP OVER!\n",
"A_Loss: 794341992174.4 C_Loss: 59005630871824.9\n",
"EP 47 START\n",
"A_Loss: 114184075127.46666 C_Loss: 74263479521553.06\n",
"EP OVER!\n",
"A_Loss: 1293861572.3333333 C_Loss: 1474176015837.8667\n",
"EP 48 START\n",
"A_Loss: 1143.1249674479166 C_Loss: 19221068817.411232\n",
"EP OVER!\n",
"A_Loss: 855.1484252929688 C_Loss: 28676.431770833333\n",
"EP 49 START\n",
"A_Loss: 1198.7060546875 C_Loss: 41875.48359375\n",
"EP OVER!\n",
"A_Loss: 1215.0390218098958 C_Loss: 7167.594791666666\n",
"EP 50 START\n",
"A_Loss: 1840.496435546875 C_Loss: 33346.76875\n",
"EP OVER!\n",
"A_Loss: 97.68925425211589 C_Loss: 2453.3792643229167\n",
"EP 51 START\n",
"A_Loss: 2179.5446614583334 C_Loss: 25573.869270833333\n",
"EP OVER!\n",
"A_Loss: 3547.454052734375 C_Loss: 15107.698307291666\n",
"EP 52 START\n",
"A_Loss: 513.1327128092448 C_Loss: 23233.316276041667\n",
"EP OVER!\n",
"A_Loss: 2024.9339111328125 C_Loss: 10457.81875\n",
"EP 53 START\n",
"A_Loss: 321.8419494628906 C_Loss: 10830.605924479167\n",
"EP OVER!\n",
"A_Loss: -8.38951981862386 C_Loss: 2241.023612467448\n",
"EP 54 START\n",
"A_Loss: 1454.901765950521 C_Loss: 16337.481966145833\n",
"EP OVER!\n",
"A_Loss: 3069.5313313802085 C_Loss: 4450.756380208333\n",
"EP 55 START\n",
"A_Loss: 444.8317138671875 C_Loss: 15201.106184895832\n",
"EP OVER!\n",
"A_Loss: 1159.2985107421875 C_Loss: 35525.82317708333\n",
"EP 56 START\n",
"A_Loss: 1397.7694742838542 C_Loss: 8600.898404947917\n",
"EP OVER!\n",
"A_Loss: 160.49285074869792 C_Loss: 3749.308203125\n",
"EP 57 START\n",
"A_Loss: -23.67707913716634 C_Loss: 8787.21162109375\n",
"EP OVER!\n",
"A_Loss: 446.2368998209635 C_Loss: 8949.849674479166\n",
"EP 58 START\n",
"A_Loss: 1443.8223063151042 C_Loss: 4806.931754557291\n",
"EP OVER!\n",
"A_Loss: 1028.3776000976563 C_Loss: 10080.125455729167\n",
"EP 59 START\n",
"A_Loss: 979.55380859375 C_Loss: 18168.035872395834\n",
"EP OVER!\n",
"A_Loss: 2799.885953776042 C_Loss: 14584.458203125\n",
"EP 60 START\n",
"A_Loss: 83247.30833333333 C_Loss: 61687910677571.76\n",
"EP OVER!\n",
"A_Loss: 81.43709259033203 C_Loss: 47630220.59791667\n",
"EP 61 START\n",
"A_Loss: 5032.870377604167 C_Loss: 43944.79453125\n",
"EP OVER!\n",
"A_Loss: 9.13815720876058 C_Loss: 2573.8714192708335\n",
"EP 62 START\n",
"A_Loss: 523.7818908691406 C_Loss: 8441.000618489583\n",
"EP OVER!\n",
"A_Loss: 1593.0845377604167 C_Loss: 17598.201953125\n",
"EP 63 START\n",
"A_Loss: 27.476329803466797 C_Loss: 5594.57646484375\n",
"EP OVER!\n",
"A_Loss: 25.40858777364095 C_Loss: 11878.195572916668\n",
"EP 64 START\n",
"A_Loss: 3347.620393880208 C_Loss: 15555.9111328125\n",
"EP OVER!\n",
"A_Loss: 25.35328992207845 C_Loss: 7923.337727864583\n",
"EP 65 START\n",
"A_Loss: 1800.478165690104 C_Loss: 36845.799479166664\n",
"EP OVER!\n",
"A_Loss: 544.9222574869792 C_Loss: 2949.7829752604166\n",
"EP 66 START\n",
"A_Loss: 238.4148183186849 C_Loss: 2108.7259195963543\n",
"EP OVER!\n",
"A_Loss: 10.808652464548747 C_Loss: 1069.8261800130208\n",
"EP 67 START\n",
"A_Loss: 7.2742944399515785 C_Loss: 1112.0956298828125\n",
"EP OVER!\n",
"A_Loss: 686.372725423177 C_Loss: 1421.539111328125\n",
"EP 68 START\n",
"A_Loss: 38.04718589782715 C_Loss: 2771.959724934896\n",
"EP OVER!\n",
"A_Loss: 1133.577986653646 C_Loss: 5758.061686197917\n",
"EP 69 START\n",
"A_Loss: 402.1093465169271 C_Loss: 3987.9388671875\n",
"EP OVER!\n",
"A_Loss: -1.7969589630762737 C_Loss: 2884.10732421875\n",
"EP 70 START\n",
"A_Loss: 1016.8031656901042 C_Loss: 6636.073665364584\n",
"EP OVER!\n",
"A_Loss: 939.9079630533854 C_Loss: 3692.443391927083\n",
"EP 71 START\n",
"A_Loss: -107.65687510172526 C_Loss: 21740540498583.367\n",
"EP OVER!\n",
"A_Loss: 5876.485221354166 C_Loss: 50534.828385416666\n",
"EP 72 START\n",
"A_Loss: 219.98338216145834 C_Loss: 2713.9967041015625\n",
"EP OVER!\n",
"A_Loss: 433.9121032714844 C_Loss: 950.6825520833333\n",
"EP 73 START\n",
"A_Loss: 933.4014811197917 C_Loss: 1297.3184733072917\n",
"EP OVER!\n",
"A_Loss: 2035.8281412760416 C_Loss: 3957.7666178385416\n",
"EP 74 START\n",
"A_Loss: 479.3147298177083 C_Loss: 1727.3513753255208\n",
"EP OVER!\n",
"A_Loss: 579.0808024088542 C_Loss: 635.3917744954427\n",
"EP 75 START\n",
"A_Loss: 1450.3585856119792 C_Loss: 2148.2808756510417\n",
"EP OVER!\n",
"A_Loss: 917.6628540039062 C_Loss: 1192.1792277018228\n",
"EP 76 START\n",
"A_Loss: 888.0876180013021 C_Loss: 1050.4912679036458\n",
"EP OVER!\n",
"A_Loss: 249.14984232584635 C_Loss: 9789.355598958333\n",
"EP 77 START\n",
"A_Loss: 346.7358866373698 C_Loss: 1828.4739990234375\n",
"EP OVER!\n",
"A_Loss: 15.01647071838379 C_Loss: 1649.2421305338542\n",
"EP 78 START\n",
"A_Loss: 2058.658439127604 C_Loss: 1996.608447265625\n",
"EP OVER!\n",
"A_Loss: 155.17740580240886 C_Loss: 575.6137990315755\n",
"EP 79 START\n",
"A_Loss: 1246.8288167317708 C_Loss: 1408.468212890625\n",
"EP OVER!\n",
"A_Loss: 175.76889241536458 C_Loss: 825.604911295573\n",
"EP 80 START\n",
"A_Loss: 10138501.666666666 C_Loss: 33048526138440.633\n"
]
}
],
"source": [
"bestScore = 200.\n",
"stopTrainCounter = 0\n",
"\n",
"totalRewardHis = []\n",
"totalActorLossHis = []\n",
"totalCriticLossHis = []\n",
"epHis = []\n",
"maxTotalReward = -99999999999\n",
"\n",
"for ep in range(MAX_EP):\n",
" print(\"EP \",ep,\" START\")\n",
" # first time run game\n",
" s,_,_,_ = env.reset()\n",
" if (ep == 0):\n",
" epBuffer = buffer.buffer()\n",
" s = s.reshape([STATE_SIZE])\n",
" agent = PPO.PPO(stateSize=STATE_SIZE,\n",
" disActShape=disActShape,\n",
" conActSize=1,\n",
" conActRange=CTN_ACTION_RANGE,\n",
" criticLR=CRITIC_LR,\n",
" actorLR=ACTOR_LR,\n",
" gamma=GAMMA,\n",
" epsilon=EPSILON,\n",
" entropyWeight=ENTROPY_WHEIGHT,\n",
" saveDir=SAVE_DIR,\n",
" loadModelDir=LOAD_DIR)\n",
" step = 0\n",
" done = False\n",
" stopTrainCounter -= 1\n",
" epHis.append(ep)\n",
" \n",
" # reset total reward\n",
" epTotalReward = 0\n",
" \n",
" # Recorder list\n",
" epStepHis = []\n",
" epRewardHis = []\n",
" epActorLossHis = []\n",
" epCriticLossHis = []\n",
"\n",
" while not done:\n",
" step += 1\n",
" if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n",
" epStepHis.append(step)\n",
" disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n",
" kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n",
" \n",
" nextState,thisReward,done,_ = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
"\n",
" epTotalReward += thisReward\n",
" epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n",
" else:\n",
" disActs = np.array([[0,0,0,0,0]])\n",
" conActs = np.array([[0]])\n",
"\n",
" nextState,thisReward,done,_ = env.step(discreteActions=disActs,continuousActions=conActs)\n",
" epTotalReward += thisReward\n",
" nextState = nextState.reshape([STATE_SIZE])\n",
" s = nextState\n",
" \n",
" if done:\n",
" print(\"EP OVER!\")\n",
" # update PPO after Batch step or GameOver\n",
" if (step+1)%BATCH == 0 or done:\n",
" bs = epBuffer.getStates()\n",
" ba = epBuffer.getActions()\n",
" br = epBuffer.getRewards()\n",
" epBuffer.clearBuffer()\n",
" if TRAIN:\n",
" epActorLoss,epCriticLoss = agent.trainCritcActor(bs,ba,br,s,CRITIC_EPOCH,ACTOR_EPOCH)\n",
" epActorLossHis.append(epActorLoss)\n",
" epCriticLossHis.append(epCriticLoss)\n",
" # update History Recorder\n",
" totalActorLossHis.append(np.mean(epActorLossHis))\n",
" totalCriticLossHis.append(np.mean(epCriticLossHis))\n",
" totalRewardHis.append(epTotalReward)\n",
" \n",
" if (epTotalReward > maxTotalReward and epTotalReward != 0):\n",
" maxTotalReward = epTotalReward\n",
" agent.saveWeights(epTotalReward)\n",
" print(\"New Record! Save NN\",epTotalReward)\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"aaa = 0\n",
"aaa = 1\n",
"aaa = 2\n",
"aaa = 3\n",
"aaa = 4\n",
"aaa = 5\n",
"aaa = 6\n",
"aaa = 7\n",
"aaa = 8\n",
"aaa = 9\n"
]
}
],
"source": [
"aaa = 0\n",
"while aaa<10:\n",
" print(\"aaa = \",aaa)\n",
" aaa+=1"
]
}
],
"metadata": {
"interpreter": {
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
},
"kernelspec": {
"display_name": "Python 3.9.7 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue Copy Permalink