928 lines
35 KiB
Plaintext
928 lines
35 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import aimBotEnv\n",
|
|
"import PPO\n",
|
|
"import buffer\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"import tensorflow as tf\n",
|
|
"import time\n",
|
|
"import datetime"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Attempts to allocate only the GPU memory needed for allocation\n",
|
|
"physical_devices = tf.config.list_physical_devices('GPU')\n",
|
|
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ENV_PATH = './Build-MultiScene-WithLoad/Aimbot-PPO'\n",
|
|
"WORKER_ID = 1\n",
|
|
"BASE_PORT = 200\n",
|
|
"\n",
|
|
"MAX_EP = 1000\n",
|
|
"EP_LENGTH = 100000\n",
|
|
"GAMMA = 0.99 # discount future reward (UP?)\n",
|
|
"EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n",
|
|
"ACTOR_LR = 1e-5 # LR\n",
|
|
"CRITIC_LR = 2e-5 # LR\n",
|
|
"BATCH = 512 # learning step\n",
|
|
"ACTOR_EPOCH = 15 # epoch\n",
|
|
"CRITIC_EPOCH = 15 # epoch\n",
|
|
"ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss\n",
|
|
"ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n",
|
|
"\n",
|
|
"\n",
|
|
"TRAIN = True\n",
|
|
"SAVE_DIR = \"PPO-Model/\"+datetime.datetime.now().strftime(\"%m%d%H%M\")+\"/\"\n",
|
|
"LOAD_DIR = None\n",
|
|
"\n",
|
|
"CTN_ACTION_RANGE = 10"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"√√√√√Enviroment Initialized Success√√√√√\n",
|
|
"√√√√√Buffer Initialized Success√√√√√\n",
|
|
"Load Model:\n",
|
|
"PPO-Model/09051841/190108\n",
|
|
"CONTINUOUS_SIZE 1\n",
|
|
"DISCRETE_SIZE 5\n",
|
|
"STATE_SIZE 29\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# initialize enviroment & buffer class\n",
|
|
"env = aimBotEnv.makeEnv(envPath = ENV_PATH,\n",
|
|
" workerID = WORKER_ID,\n",
|
|
" basePort = BASE_PORT)\n",
|
|
"epBuffer = buffer.buffer()\n",
|
|
"\n",
|
|
"STATE_SIZE = env.STATE_SIZE\n",
|
|
"CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
|
|
"DISCRETE_SIZE = env.DISCRETE_SIZE\n",
|
|
"_,_,_,loadDir,_ = env.getSteps()\n",
|
|
"\n",
|
|
"# check load model or not\n",
|
|
"if(np.any(loadDir == 0)):\n",
|
|
" # create a new model\n",
|
|
" print(\"No loadDir specified,Create a New Model\")\n",
|
|
" LOAD_DIR = None\n",
|
|
"else:\n",
|
|
" # load model\n",
|
|
" loadDirDateSTR = str(int(loadDir[0]))\n",
|
|
" loadDirTimeSTR = str(int(loadDir[1]))\n",
|
|
" if len(loadDirDateSTR)!=8:\n",
|
|
" # fill lost 0 while converse float to string\n",
|
|
" for _ in range(8 - len(loadDirDateSTR)):\n",
|
|
" loadDirDateSTR = \"0\" + loadDirDateSTR\n",
|
|
" if len(loadDirTimeSTR)!=6:\n",
|
|
" # fill lost 0 while converse float to string\n",
|
|
" for _ in range(6 - len(loadDirTimeSTR)):\n",
|
|
" loadDirTimeSTR = \"0\" + loadDirTimeSTR\n",
|
|
" LOAD_DIR = \"PPO-Model/\"+loadDirDateSTR+\"/\"+loadDirTimeSTR\n",
|
|
" print(\"Load Model:\")\n",
|
|
" print(LOAD_DIR)\n",
|
|
"\n",
|
|
"print(\"CONTINUOUS_SIZE\",CONTINUOUS_SIZE)\n",
|
|
"print(\"DISCRETE_SIZE\",DISCRETE_SIZE)\n",
|
|
"print(\"STATE_SIZE\",STATE_SIZE)\n",
|
|
"\n",
|
|
"disActShape = [3,3,2]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def actToKey(disAct1,disAct2,disAct3,conAct):\n",
|
|
" kW = 0\n",
|
|
" kS = 0\n",
|
|
" kA = 0\n",
|
|
" kD = 0\n",
|
|
" mouseShoot = 0\n",
|
|
" if disAct1 == 0:\n",
|
|
" kW = 0\n",
|
|
" kS = 1\n",
|
|
" elif disAct1 == 1:\n",
|
|
" kW = 0\n",
|
|
" kS = 0\n",
|
|
" elif disAct1 == 2:\n",
|
|
" kW = 1\n",
|
|
" kS = 0\n",
|
|
" if disAct2 == 0:\n",
|
|
" kA = 0\n",
|
|
" kD = 1\n",
|
|
" elif disAct2 == 1:\n",
|
|
" kA = 0\n",
|
|
" kD = 0\n",
|
|
" elif disAct2 == 2:\n",
|
|
" kA = 1\n",
|
|
" kD = 0\n",
|
|
" mouseShoot = disAct3\n",
|
|
" return kW,kS,kA,kD,mouseShoot,conAct"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"EP 0 START\n",
|
|
"√√√√√Buffer Initialized Success√√√√√\n",
|
|
"++++++++++++++++++++++++++++++++++++\n",
|
|
"++++++++++++Model Loaded++++++++++++\n",
|
|
"PPO-Model/09051841/190108\n",
|
|
"++++++++++++++++++++++++++++++++++++\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n",
|
|
" return _methods._mean(a, axis=axis, dtype=dtype,\n",
|
|
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
|
|
" ret = ret.dtype.type(ret / rcount)\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"A_Loss: 128583691.2 C_Loss: 812.6740254720052\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 18745178.266666666 C_Loss: 154.92835998535156\n",
|
|
"Model's Weights Saved\n",
|
|
"New Record! Save NN -38.23000041767955\n",
|
|
"EP 1 START\n",
|
|
"A_Loss: 563364718.9333333 C_Loss: 813.9496988932292\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2345777356.8 C_Loss: 1002.7922281901042\n",
|
|
"Model's Weights Saved\n",
|
|
"New Record! Save NN 6.009999185800552\n",
|
|
"EP 2 START\n",
|
|
"A_Loss: 135096590.93333334 C_Loss: 289.0239980061849\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -1.0771097699801127 C_Loss: 94.56021474202474\n",
|
|
"EP 3 START\n",
|
|
"A_Loss: 188551280.0 C_Loss: 178.76077473958333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -0.7733836094538371 C_Loss: 24.582142766316732\n",
|
|
"EP 4 START\n",
|
|
"A_Loss: 1542771293.8666666 C_Loss: 2540.582096354167\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 17443229.466666665 C_Loss: 403.4093978881836\n",
|
|
"EP 5 START\n",
|
|
"A_Loss: 0.9811522722244262 C_Loss: 173.3322311401367\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 0.062289920759697755 C_Loss: 83.96865743001302\n",
|
|
"EP 6 START\n",
|
|
"A_Loss: 207701843.2 C_Loss: 508.06519571940106\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 198849240.53333333 C_Loss: 318.39104512532555\n",
|
|
"EP 7 START\n",
|
|
"A_Loss: 227911159.46666667 C_Loss: 280.90660603841144\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 138269843.73333332 C_Loss: 530.1331217447917\n",
|
|
"EP 8 START\n",
|
|
"A_Loss: 944055261.8666667 C_Loss: 24907129397945.176\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -4141405.066666667 C_Loss: 49846409529480.53\n",
|
|
"Model's Weights Saved\n",
|
|
"New Record! Save NN 32.569999642670155\n",
|
|
"EP 9 START\n",
|
|
"A_Loss: 13671169501.866667 C_Loss: 44895753103.339066\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 43214365218.13333 C_Loss: 37662.319010416664\n",
|
|
"EP 10 START\n",
|
|
"A_Loss: 29955732002.133335 C_Loss: 68502.11927083334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 20331074764.8 C_Loss: 38761.32864583333\n",
|
|
"EP 11 START\n",
|
|
"A_Loss: 17149350843.733334 C_Loss: 41642.053385416664\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 16021737676.8 C_Loss: 24600.219270833335\n",
|
|
"EP 12 START\n",
|
|
"A_Loss: 20431896029.866665 C_Loss: 82939.88125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 12221524718.933332 C_Loss: 41908.44921875\n",
|
|
"EP 13 START\n",
|
|
"A_Loss: 23195075515.733334 C_Loss: 72094.946875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 30219999095.466667 C_Loss: 25428.986588541666\n",
|
|
"EP 14 START\n",
|
|
"A_Loss: 27674789614.933334 C_Loss: 77286.32708333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 14385042363.733334 C_Loss: 44674.395833333336\n",
|
|
"EP 15 START\n",
|
|
"A_Loss: 6491561813.333333 C_Loss: 57468.32369791667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 14895148373.333334 C_Loss: 27674.523567708333\n",
|
|
"EP 16 START\n",
|
|
"A_Loss: 23592904157.866665 C_Loss: 54008.02786458333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 28394169002.666668 C_Loss: 35594.001302083336\n",
|
|
"EP 17 START\n",
|
|
"A_Loss: 31493442082.133335 C_Loss: 43341.80390625\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 15182564420.266666 C_Loss: 13120.031575520834\n",
|
|
"EP 18 START\n",
|
|
"A_Loss: 15298737561.6 C_Loss: 31930.973177083335\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 18384228215.466667 C_Loss: 20152.706770833334\n",
|
|
"EP 19 START\n",
|
|
"A_Loss: 19412337732.266666 C_Loss: 13991118556664.594\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 10120899857.066668 C_Loss: 39128662677.066666\n",
|
|
"EP 20 START\n",
|
|
"A_Loss: 19142413516.8 C_Loss: 129969.633203125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 5016769467.733334 C_Loss: 9036.797916666666\n",
|
|
"EP 21 START\n",
|
|
"A_Loss: 18806764066.133335 C_Loss: 14859.599609375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 8090647210.666667 C_Loss: 6834.12041015625\n",
|
|
"EP 22 START\n",
|
|
"A_Loss: 18050034210.133335 C_Loss: 15550.104427083334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 84.65721384684245 C_Loss: 9692.593880208333\n",
|
|
"EP 23 START\n",
|
|
"A_Loss: 11287560055.466667 C_Loss: 10906.8736328125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 9644667630.933332 C_Loss: 8087.964127604167\n",
|
|
"EP 24 START\n",
|
|
"A_Loss: 3744549939.2 C_Loss: 5294.3908203125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 263749590.4 C_Loss: 1567.651505533854\n",
|
|
"EP 25 START\n",
|
|
"A_Loss: 5730180334.933333 C_Loss: 3254.1029459635415\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 4258283246.9333334 C_Loss: 1893.0966227213542\n",
|
|
"EP 26 START\n",
|
|
"A_Loss: 2880076526.9333334 C_Loss: 1959.542569986979\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 3309443618.133333 C_Loss: 816.9943277994792\n",
|
|
"EP 27 START\n",
|
|
"A_Loss: 1501129864.5333333 C_Loss: 750.1781555175781\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 1310018705.0666666 C_Loss: 259.69133911132815\n",
|
|
"EP 28 START\n",
|
|
"A_Loss: 587718690.1333333 C_Loss: 433.7637430826823\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 532166058.6666667 C_Loss: 279.7773742675781\n",
|
|
"EP 29 START\n",
|
|
"A_Loss: 70524670.93333334 C_Loss: 65.66910146077474\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 73527637.06666666 C_Loss: 89.10262959798177\n",
|
|
"EP 30 START\n",
|
|
"A_Loss: 144336381.86666667 C_Loss: 11965969993490.188\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 1655646916881.0667 C_Loss: 10321726704401.066\n",
|
|
"EP 31 START\n",
|
|
"A_Loss: 41522109.46666667 C_Loss: 2869060630042.067\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -64.74283854166667 C_Loss: 16231.0900390625\n",
|
|
"EP 32 START\n",
|
|
"A_Loss: 3817272.5 C_Loss: 25434.281510416666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 279540269.8666667 C_Loss: 10897.152734375\n",
|
|
"EP 33 START\n",
|
|
"A_Loss: -88.46073099772136 C_Loss: 19774.261588541667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 400722821.04817367 C_Loss: 7724.169270833333\n",
|
|
"EP 34 START\n",
|
|
"A_Loss: 1930250.1738263448 C_Loss: 12444.766927083334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 37003849.2 C_Loss: 8640.598307291666\n",
|
|
"EP 35 START\n",
|
|
"A_Loss: 55674022.4 C_Loss: 14830.487825520833\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 78330882.4 C_Loss: 7793.051595052083\n",
|
|
"EP 36 START\n",
|
|
"A_Loss: 10209805.2 C_Loss: 11949599399.99375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 48724937985574.92 C_Loss: 17860793484356.266\n",
|
|
"EP 37 START\n",
|
|
"A_Loss: 4330507.568489583 C_Loss: 223605282927.2875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 7667.229915364584 C_Loss: 57925.12994791667\n",
|
|
"EP 38 START\n",
|
|
"A_Loss: 184.12769877115886 C_Loss: 7122.831803385417\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 15510.455403645834 C_Loss: 52148.67734375\n",
|
|
"EP 39 START\n",
|
|
"A_Loss: 5272.286588541667 C_Loss: 286373.10833333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2.6445246855417888 C_Loss: 4984.467862955729\n",
|
|
"EP 40 START\n",
|
|
"A_Loss: 5228.9494140625 C_Loss: 366411.77291666664\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -288.28272705078126 C_Loss: 12667.800716145834\n",
|
|
"EP 41 START\n",
|
|
"A_Loss: 19474.684244791668 C_Loss: 45928038184215.41\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2104.5810709635416 C_Loss: 276810.7479166667\n",
|
|
"EP 42 START\n",
|
|
"A_Loss: 61.346117909749346 C_Loss: 24438.237760416665\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 14208.790559895833 C_Loss: 220833.20625\n",
|
|
"EP 43 START\n",
|
|
"A_Loss: 5357.714680989583 C_Loss: 332224.36041666666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 85.4434425354004 C_Loss: 6307.7271484375\n",
|
|
"EP 44 START\n",
|
|
"A_Loss: 122.99782867431641 C_Loss: 19919.213411458335\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 15087.8431640625 C_Loss: 161420.19166666668\n",
|
|
"EP 45 START\n",
|
|
"A_Loss: 1674.8425455729166 C_Loss: 309222.24166666664\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 1518.2337646484375 C_Loss: 349629.71041666664\n",
|
|
"EP 46 START\n",
|
|
"A_Loss: 2379.195491536458 C_Loss: 364705.35833333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 209796462.2 C_Loss: 13010572456.520052\n",
|
|
"EP 47 START\n",
|
|
"A_Loss: -2574.44921875 C_Loss: 39260709.6\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -255952.503125 C_Loss: 1636249529955.0833\n",
|
|
"EP 48 START\n",
|
|
"A_Loss: 607266.8805664063 C_Loss: 40067520562.25208\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2080.8306803385417 C_Loss: 175271.12291666667\n",
|
|
"EP 49 START\n",
|
|
"A_Loss: 592.0791585286458 C_Loss: 303988.81666666665\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 6402.670670572917 C_Loss: 52915.89296875\n",
|
|
"EP 50 START\n",
|
|
"A_Loss: 1395209.9906412761 C_Loss: 47937.68541666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 5355160.362174479 C_Loss: 46913.726822916666\n",
|
|
"EP 51 START\n",
|
|
"A_Loss: 131.98586934407552 C_Loss: 131763.49583333332\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 545.0480163574218 C_Loss: 177076.325\n",
|
|
"EP 52 START\n",
|
|
"A_Loss: -295055.39791666664 C_Loss: 21676655815489.375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 96046800076.8 C_Loss: 2559427411968.0\n",
|
|
"EP 53 START\n",
|
|
"A_Loss: 23993694.0 C_Loss: 400391016517.3333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 93352.0390625 C_Loss: 619253805875.2\n",
|
|
"EP 54 START\n",
|
|
"A_Loss: 2732956.3833333333 C_Loss: 46527696896.0\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 39510.85885416667 C_Loss: 28781911654.4\n",
|
|
"EP 55 START\n",
|
|
"A_Loss: -29427.872395833332 C_Loss: 5438102272.0\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 90085.98177083333 C_Loss: 3624150245.866667\n",
|
|
"EP 56 START\n",
|
|
"A_Loss: -498753.30625 C_Loss: 823590289405.8667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 10569771.733333332 C_Loss: 2970159109461.3335\n",
|
|
"EP 57 START\n",
|
|
"A_Loss: -50644.94348958333 C_Loss: 646384501282.1333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 23941.277994791668 C_Loss: 53731624072.53333\n",
|
|
"EP 58 START\n",
|
|
"A_Loss: -199527.115625 C_Loss: 2830466387626.6665\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 5080.106127929687 C_Loss: 1123213416857.6\n",
|
|
"EP 59 START\n",
|
|
"A_Loss: 854754.9208333333 C_Loss: 89255951291.73334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -6642.098014322916 C_Loss: 2435903428.266667\n",
|
|
"EP 60 START\n",
|
|
"A_Loss: 21668.172135416666 C_Loss: 4344311575859.2\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -2874.193896484375 C_Loss: 1691038327.4666667\n",
|
|
"EP 61 START\n",
|
|
"A_Loss: 46610.647135416664 C_Loss: 355761074.1333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 174212.034375 C_Loss: 49708919.233333334\n",
|
|
"EP 62 START\n",
|
|
"A_Loss: 135704.40625 C_Loss: 47006909.06666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 10387.477018229167 C_Loss: 51938460.8\n",
|
|
"EP 63 START\n",
|
|
"A_Loss: 594.3608805338541 C_Loss: 16747537224969.105\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 109.09736455281576 C_Loss: 4015293.466666667\n",
|
|
"EP 64 START\n",
|
|
"A_Loss: 2528.45546875 C_Loss: 5244184.566666666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 228.57803039550782 C_Loss: 4565261.5\n",
|
|
"EP 65 START\n",
|
|
"A_Loss: 576.867138671875 C_Loss: 690111.6791666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2555.884700520833 C_Loss: 1223311.4583333333\n",
|
|
"EP 66 START\n",
|
|
"A_Loss: 1392.1302897135417 C_Loss: 4376503.633333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 3898.088427734375 C_Loss: 2490652.9833333334\n",
|
|
"EP 67 START\n",
|
|
"A_Loss: 456.2262491861979 C_Loss: 1098666.3\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -223.93345540364584 C_Loss: 2048068.5666666667\n",
|
|
"EP 68 START\n",
|
|
"A_Loss: -439.01691080729165 C_Loss: 2945374.433333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -24.373832575480144 C_Loss: 343411.81875\n",
|
|
"EP 69 START\n",
|
|
"A_Loss: 5111.536979166666 C_Loss: 10348606653210.434\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -38.84524319966634 C_Loss: 1990322.725\n",
|
|
"EP 70 START\n",
|
|
"A_Loss: 951.8831461588542 C_Loss: 223227.01875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 172.96959330240887 C_Loss: 1922374.9416666667\n",
|
|
"EP 71 START\n",
|
|
"A_Loss: 39.82610600789388 C_Loss: 533955.7208333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 620.001796468099 C_Loss: 282419.71979166666\n",
|
|
"EP 72 START\n",
|
|
"A_Loss: 10333.698046875 C_Loss: 2511066.466666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -83.75911153157553 C_Loss: 1593542.8833333333\n",
|
|
"EP 73 START\n",
|
|
"A_Loss: 1590.978955078125 C_Loss: 931809.2333333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 189.47867533365886 C_Loss: 1312441.1333333333\n",
|
|
"EP 74 START\n",
|
|
"A_Loss: -4686.18564453125 C_Loss: 8461484.133333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 152230.85208333333 C_Loss: 26483003486139.734\n",
|
|
"EP 75 START\n",
|
|
"A_Loss: 543.5623474121094 C_Loss: 676008774.1916667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 72.8592913309733 C_Loss: 646183.675\n",
|
|
"EP 76 START\n",
|
|
"A_Loss: -111.36540323893229 C_Loss: 1268551.3833333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -86.98175252278646 C_Loss: 1884845.4166666667\n",
|
|
"EP 77 START\n",
|
|
"A_Loss: 9.363872464497884 C_Loss: 574981.4\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 10.75021572113037 C_Loss: 1757512.1051269532\n",
|
|
"EP 78 START\n",
|
|
"A_Loss: -91.82536315917969 C_Loss: 1116194.3125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -2419.4617513020835 C_Loss: 9647680415.9\n",
|
|
"EP 79 START\n",
|
|
"A_Loss: 12534.184114583333 C_Loss: 125233972.73333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -12864.7326171875 C_Loss: 32251742043101.867\n",
|
|
"EP 80 START\n",
|
|
"A_Loss: 69521.21354166667 C_Loss: 239759950.93333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 1941.87587890625 C_Loss: 3182419.7\n",
|
|
"EP 81 START\n",
|
|
"A_Loss: 48.873761876424155 C_Loss: 2138957.5\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -172.59212137858074 C_Loss: 1479709.7\n",
|
|
"EP 82 START\n",
|
|
"A_Loss: -217.69344889322917 C_Loss: 1724748.0333333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 8.864885139465333 C_Loss: 2801.1948404947916\n",
|
|
"EP 83 START\n",
|
|
"A_Loss: -23.036806042989095 C_Loss: 487250.31875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -85.1435302734375 C_Loss: 459072.14375\n",
|
|
"EP 84 START\n",
|
|
"A_Loss: -159.0248555501302 C_Loss: 704506.4\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 391660.59375 C_Loss: 185407429.55\n",
|
|
"EP 85 START\n",
|
|
"A_Loss: 11304412.4 C_Loss: 436438987025.06665\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -508119.4666666667 C_Loss: 27526751540565.332\n",
|
|
"EP 86 START\n",
|
|
"A_Loss: 8.469103686014812 C_Loss: 35030720904.04556\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 79.07336120605468 C_Loss: 2628.575537109375\n",
|
|
"EP 87 START\n",
|
|
"A_Loss: 87.94131622314453 C_Loss: 3291.388395182292\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 63.07492167154948 C_Loss: 3989.5349609375\n",
|
|
"EP 88 START\n",
|
|
"A_Loss: 9.08399314880371 C_Loss: 3549.7614908854166\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 47.93385569254557 C_Loss: 1957.0295735677084\n",
|
|
"EP 89 START\n",
|
|
"A_Loss: 76.6830576578776 C_Loss: 4872.250081380208\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 77.35536549886068 C_Loss: 1778.1133951822917\n",
|
|
"EP 90 START\n",
|
|
"A_Loss: 26.445547739664715 C_Loss: 1201.9427978515625\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 37.380027770996094 C_Loss: 2120.48583984375\n",
|
|
"EP 91 START\n",
|
|
"A_Loss: 62.73122533162435 C_Loss: 1910.77255859375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 7.070840994517009 C_Loss: 2663.728580729167\n",
|
|
"Model's Weights Saved\n",
|
|
"New Record! Save NN 36.159999914467335\n",
|
|
"EP 92 START\n",
|
|
"A_Loss: 83.22457122802734 C_Loss: 4142.598046875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -2965967.5 C_Loss: 89986176198316.5\n",
|
|
"EP 93 START\n",
|
|
"A_Loss: 5039913.333333333 C_Loss: 11098771021277.867\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 15480.3697265625 C_Loss: 119004974489.6\n",
|
|
"EP 94 START\n",
|
|
"A_Loss: -16268.488020833332 C_Loss: 2171354205.866667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 25105.733463541666 C_Loss: 180378952.53333333\n",
|
|
"EP 95 START\n",
|
|
"A_Loss: -356.90814819335935 C_Loss: 4823839.716666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -103.6774673461914 C_Loss: 1091913.1416666666\n",
|
|
"EP 96 START\n",
|
|
"A_Loss: 132.41912180582682 C_Loss: 3535627.6166666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -250.67224629720053 C_Loss: 2047510.1\n",
|
|
"EP 97 START\n",
|
|
"A_Loss: 191.60374145507814 C_Loss: 2080957.1833333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 317.184033203125 C_Loss: 1221589.425\n",
|
|
"EP 98 START\n",
|
|
"A_Loss: -39.01324615478516 C_Loss: 998958.2208333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -190.1520253499349 C_Loss: 1742518.1\n",
|
|
"EP 99 START\n",
|
|
"A_Loss: 311.61248372395835 C_Loss: 620184.1416666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -264.8520751953125 C_Loss: 1779512.7666666666\n",
|
|
"EP 100 START\n",
|
|
"A_Loss: -94.42798614501953 C_Loss: 1547244.1833333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -153766.15104166666 C_Loss: 33501732032887.137\n",
|
|
"EP 101 START\n",
|
|
"A_Loss: 60208.17265625 C_Loss: 27465943313.066666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 17862.422265625 C_Loss: 988074240.0\n",
|
|
"EP 102 START\n",
|
|
"A_Loss: 28635.012890625 C_Loss: 144180219.2\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 265.3567850748698 C_Loss: 2369780.45\n",
|
|
"EP 103 START\n",
|
|
"A_Loss: -17.382435480753582 C_Loss: 726623.9833333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 16.159270922342937 C_Loss: 124937.20364583333\n",
|
|
"EP 104 START\n",
|
|
"A_Loss: 180.78487548828124 C_Loss: 2608104.4\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -472.0964823404948 C_Loss: 3132594.6666666665\n",
|
|
"EP 105 START\n",
|
|
"A_Loss: -245.632177734375 C_Loss: 1839736.6333333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -84.94388631184896 C_Loss: 788294.0833333334\n",
|
|
"EP 106 START\n",
|
|
"A_Loss: -344.7350769042969 C_Loss: 2457982.183333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 247.77904154459637 C_Loss: 191985.88020833334\n",
|
|
"EP 107 START\n",
|
|
"A_Loss: -93.01107330322266 C_Loss: 1014136.5958333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -301.968603515625 C_Loss: 2097648.5416666665\n",
|
|
"EP 108 START\n",
|
|
"A_Loss: -355.8355692545573 C_Loss: 2356025.3333333335\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -176.879106648763 C_Loss: 905950.4666666667\n",
|
|
"EP 109 START\n",
|
|
"A_Loss: 2279212.8916666666 C_Loss: 5355236439927.467\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 230154.93229166666 C_Loss: 21897054401331.2\n",
|
|
"EP 110 START\n",
|
|
"A_Loss: 204.3412821451823 C_Loss: 11179356820.975\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -44.459004465738936 C_Loss: 606090.0875\n",
|
|
"EP 111 START\n",
|
|
"A_Loss: -55.804770151774086 C_Loss: 453019.00625\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 437.7322265625 C_Loss: 531331.7\n",
|
|
"EP 112 START\n",
|
|
"A_Loss: -176.73267822265626 C_Loss: 1254341.2416666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -9.164166609446207 C_Loss: 116005.52552083334\n",
|
|
"EP 113 START\n",
|
|
"A_Loss: 57.09782918294271 C_Loss: 590456.1916666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -28.08950449625651 C_Loss: 368954.27291666664\n",
|
|
"EP 114 START\n",
|
|
"A_Loss: -141.4985616048177 C_Loss: 545265.2583333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 196.88966267903646 C_Loss: 262787.0489583333\n",
|
|
"EP 115 START\n",
|
|
"A_Loss: 9875956.133333333 C_Loss: 3146464274264.442\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 74947.0171875 C_Loss: 1476110921523.2\n",
|
|
"EP 116 START\n",
|
|
"A_Loss: -147298.26770833333 C_Loss: 7414750106920.533\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -61223.323958333334 C_Loss: 166082947754.66666\n",
|
|
"EP 117 START\n",
|
|
"A_Loss: 155204.75208333333 C_Loss: 54631106969.6\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -5608.896809895833 C_Loss: 239498007.5\n",
|
|
"EP 118 START\n",
|
|
"A_Loss: 64438.151041666664 C_Loss: 1624584972.8\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2922.034130859375 C_Loss: 6422857095816.533\n",
|
|
"EP 119 START\n",
|
|
"A_Loss: 4014.168212890625 C_Loss: 72697799.46666667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 1831.806486002604 C_Loss: 35837436.0\n",
|
|
"EP 120 START\n",
|
|
"A_Loss: 10667.054622395834 C_Loss: 26391209.166666668\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 127434.12447916667 C_Loss: 182761787114.13333\n",
|
|
"EP 121 START\n",
|
|
"A_Loss: 143913.98645833333 C_Loss: 1208985656729.6\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 16578.108658854166 C_Loss: 1837287090.1333334\n",
|
|
"EP 122 START\n",
|
|
"A_Loss: -117467.396875 C_Loss: 1478863227690.6667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -70148.46614583333 C_Loss: 5382309429111.467\n",
|
|
"EP 123 START\n",
|
|
"A_Loss: -49.01672922770182 C_Loss: 3541015449.633333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -40.59830932617187 C_Loss: 352259.06875\n",
|
|
"EP 124 START\n",
|
|
"A_Loss: -95.25742848714192 C_Loss: 858177.2\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -52.94597473144531 C_Loss: 293412.5625\n",
|
|
"EP 125 START\n",
|
|
"A_Loss: -225.69961649576823 C_Loss: 748738.2375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -4.78879960378011 C_Loss: 123586.36875\n",
|
|
"EP 126 START\n",
|
|
"A_Loss: -201.62482401529948 C_Loss: 720683.4833333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -21.21190528869629 C_Loss: 35503.817057291664\n",
|
|
"EP 127 START\n",
|
|
"A_Loss: 152.8080861409505 C_Loss: 41394907675409.0\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 3.5420272588729858 C_Loss: 95929.18802083333\n",
|
|
"EP 128 START\n",
|
|
"A_Loss: -2.3434098323186237 C_Loss: 43473.105729166666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -31.295360438028972 C_Loss: 90783.04791666666\n",
|
|
"EP 129 START\n",
|
|
"A_Loss: 361140.66041666665 C_Loss: 72744.59609375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 2418.9001627604166 C_Loss: 3928226928.5666666\n",
|
|
"EP 130 START\n",
|
|
"A_Loss: 11.188031895955403 C_Loss: 146395.30833333332\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -288.8839884440104 C_Loss: 726967.2367838542\n",
|
|
"EP 131 START\n",
|
|
"A_Loss: 228.3966074625651 C_Loss: 1645105.6708333334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 56.98198420206706 C_Loss: 80022.94166666667\n",
|
|
"EP 132 START\n",
|
|
"A_Loss: 32.61619491577149 C_Loss: 13436.4310546875\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -3.019519786039988 C_Loss: 26069.423307291665\n",
|
|
"EP 133 START\n",
|
|
"A_Loss: 5.228196366628011 C_Loss: 25157.172005208333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 13.022575950622558 C_Loss: 3747.759977213542\n",
|
|
"EP 134 START\n",
|
|
"A_Loss: -80517.8953125 C_Loss: 15650603660651.291\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 16014.4521484375 C_Loss: 9164626090.666666\n",
|
|
"EP 135 START\n",
|
|
"A_Loss: 117133.68125 C_Loss: 87002011844.26666\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 15981.832682291666 C_Loss: 47913645943.46667\n",
|
|
"EP 136 START\n",
|
|
"A_Loss: 19723.941666666666 C_Loss: 610387874.1333333\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -6.982747983932495 C_Loss: 42435512.56666667\n",
|
|
"EP 137 START\n",
|
|
"A_Loss: -177.51118876139324 C_Loss: 1209231.775\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -18.883594957987466 C_Loss: 646092.9708333333\n",
|
|
"EP 138 START\n",
|
|
"A_Loss: 152.13518269856772 C_Loss: 282594.4010416667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 18.2394229888916 C_Loss: 124778.96979166666\n",
|
|
"EP 139 START\n",
|
|
"A_Loss: 165.62351684570314 C_Loss: 132396.28125\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -245.43899637858073 C_Loss: 928183.8291666667\n",
|
|
"EP 140 START\n",
|
|
"A_Loss: -104456.69375 C_Loss: 1376307509015.4375\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 9644.121158854166 C_Loss: 66630244983.46667\n",
|
|
"EP 141 START\n",
|
|
"A_Loss: -92.3751708984375 C_Loss: 142594040.71354166\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 29.573409907023112 C_Loss: 46747.54921875\n",
|
|
"EP 142 START\n",
|
|
"A_Loss: 1008713.7708333334 C_Loss: 20930468997730.31\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: -80090.73802083333 C_Loss: 1022462403106.1333\n",
|
|
"EP 143 START\n",
|
|
"A_Loss: -23560.799739583334 C_Loss: 32153040110.933334\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 3245.091129557292 C_Loss: 950403610.1333333\n",
|
|
"EP 144 START\n",
|
|
"A_Loss: 184.1011952718099 C_Loss: 1113755.1979166667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 963.6497111002604 C_Loss: 1592140.8416666666\n",
|
|
"EP 145 START\n",
|
|
"A_Loss: 315.7770711263021 C_Loss: 494445.6979166667\n",
|
|
"EP OVER!\n",
|
|
"A_Loss: 440.2256103515625 C_Loss: 490657.3229166667\n",
|
|
"EP 146 START\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"bestScore = 200.\n",
|
|
"stopTrainCounter = 0\n",
|
|
"\n",
|
|
"totalRewardHis = []\n",
|
|
"totalActorLossHis = []\n",
|
|
"totalCriticLossHis = []\n",
|
|
"epHis = []\n",
|
|
"maxTotalReward = -99999999999\n",
|
|
"\n",
|
|
"for ep in range(MAX_EP):\n",
|
|
" print(\"EP \",ep,\" START\")\n",
|
|
" # first time run game\n",
|
|
" s,_,_,_,_ = env.reset()\n",
|
|
" if (ep == 0):\n",
|
|
" epBuffer = buffer.buffer()\n",
|
|
" s = s.reshape([STATE_SIZE])\n",
|
|
" agent = PPO.PPO(stateSize=STATE_SIZE,\n",
|
|
" disActShape=disActShape,\n",
|
|
" conActSize=1,\n",
|
|
" conActRange=CTN_ACTION_RANGE,\n",
|
|
" criticLR=CRITIC_LR,\n",
|
|
" actorLR=ACTOR_LR,\n",
|
|
" gamma=GAMMA,\n",
|
|
" epsilon=EPSILON,\n",
|
|
" entropyWeight=ENTROPY_WHEIGHT,\n",
|
|
" saveDir=SAVE_DIR,\n",
|
|
" loadModelDir=LOAD_DIR)\n",
|
|
" step = 0\n",
|
|
" done = False\n",
|
|
" stopTrainCounter -= 1\n",
|
|
" epHis.append(ep)\n",
|
|
" \n",
|
|
" # reset total reward\n",
|
|
" epTotalReward = 0\n",
|
|
" \n",
|
|
" # Recorder list\n",
|
|
" epStepHis = []\n",
|
|
" epRewardHis = []\n",
|
|
" epActorLossHis = []\n",
|
|
" epCriticLossHis = []\n",
|
|
" \n",
|
|
" # save weight immediately?\n",
|
|
" saveNow = 0;\n",
|
|
"\n",
|
|
" while not done:\n",
|
|
" step += 1\n",
|
|
" if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n",
|
|
" epStepHis.append(step)\n",
|
|
" disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n",
|
|
" kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n",
|
|
" \n",
|
|
" nextState,thisReward,done,_,saveNow = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
|
|
"\n",
|
|
" epTotalReward += thisReward\n",
|
|
" epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n",
|
|
" else:\n",
|
|
" disActs = np.array([[0,0,0,0,0]])\n",
|
|
" conActs = np.array([[0]])\n",
|
|
"\n",
|
|
" nextState,thisReward,done,_,saveNow = env.step(discreteActions=disActs,continuousActions=conActs)\n",
|
|
" epTotalReward += thisReward\n",
|
|
" nextState = nextState.reshape([STATE_SIZE])\n",
|
|
" s = nextState\n",
|
|
" \n",
|
|
" if done:\n",
|
|
" print(\"EP OVER!\")\n",
|
|
" if saveNow != 0:\n",
|
|
" print(\"SAVENOW!\")\n",
|
|
" saveNow = 0\n",
|
|
" agent.saveWeights()\n",
|
|
" # update PPO after Batch step or GameOver\n",
|
|
" if (step+1)%BATCH == 0 or done:\n",
|
|
" bs = epBuffer.getStates()\n",
|
|
" ba = epBuffer.getActions()\n",
|
|
" br = epBuffer.getRewards()\n",
|
|
" epBuffer.clearBuffer()\n",
|
|
" if TRAIN:\n",
|
|
" epActorLoss,epCriticLoss = agent.trainCritcActor(bs,ba,br,s,CRITIC_EPOCH,ACTOR_EPOCH)\n",
|
|
" epActorLossHis.append(epActorLoss)\n",
|
|
" epCriticLossHis.append(epCriticLoss)\n",
|
|
" # update History Recorder\n",
|
|
" totalActorLossHis.append(np.mean(epActorLossHis))\n",
|
|
" totalCriticLossHis.append(np.mean(epCriticLossHis))\n",
|
|
" totalRewardHis.append(epTotalReward)\n",
|
|
" \n",
|
|
" if (epTotalReward > maxTotalReward and epTotalReward != 0):\n",
|
|
" maxTotalReward = epTotalReward\n",
|
|
" agent.saveWeights(epTotalReward)\n",
|
|
" print(\"New Record! Save NN\",epTotalReward)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"aaa = 0\n",
|
|
"aaa = 1\n",
|
|
"aaa = 2\n",
|
|
"aaa = 3\n",
|
|
"aaa = 4\n",
|
|
"aaa = 5\n",
|
|
"aaa = 6\n",
|
|
"aaa = 7\n",
|
|
"aaa = 8\n",
|
|
"aaa = 9\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"aaa = 0\n",
|
|
"while aaa<10:\n",
|
|
" print(\"aaa = \",aaa)\n",
|
|
" aaa+=1"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"interpreter": {
|
|
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3.9.7 64-bit",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|