Koha9/Aimbot-PPO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整理无用变量，对环境3.6进行适配

Browse Source
This commit is contained in:
Koha9 2024-01-24 17:07:45 +09:00
parent 5aa7e0936a
commit 9d9524429c
4 changed files with 16 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Aimbot-PPO-Python/Pytorch/MultiNN-PPO.py
View File

@ -20,7 +20,7 @@ import torch.optim as optim
SIDE_CHANNEL_UUID = uuid.UUID("8bbfb62a-99b4-457c-879d-b78b69066b5e")
# tensorboard names
GAME_NAME = "Aimbot_Hybrid_Full_MNN_MultiLevel"
GAME_TYPE = "GotoOnly-Level0123-newModel"
GAME_TYPE = "GotoOnly-3.6-Level0123-newModel-Onehot"
if __name__ == "__main__":
args = parse_args()

3
Aimbot-PPO-Python/Pytorch/aimemory.py
View File

@ -58,6 +58,7 @@ class PPOMem:
# print("Win! Broadcast reward!",rewardBF[-1])
print(sum(thisRewardBF) / len(thisRewardBF))
thisRewardBF[-1] = rewardBF[-1] - self.base_win_reward
# broadcast result reward, increase all reward in this round by remainTime * self.result_broadcast_ratio
thisRewardBF = (np.asarray(thisRewardBF) + (remainTime * self.result_broadcast_ratio)).tolist()
else:
print("!!!!!DIDNT GET RESULT REWARD!!!!!!", rewardBF[-1])
@ -88,7 +89,7 @@ class PPOMem:
self.dones_bf[i].append(done[i])
self.values_bf[i].append(value_cpu[i])
if now_step % self.decision_period == 0:
# on decision period, add last skiped round's reward
# on decision period, add last skiped round's reward, only affact in decision_period != 1
self.rewards_bf[i].append(reward[i] + last_reward[i])
else:
# not on decision period, only add this round's reward

12
Aimbot-PPO-Python/Pytorch/arguments.py
View File

@ -4,7 +4,7 @@ import uuid
from distutils.util import strtobool
DEFAULT_SEED = 9331
ENV_PATH = "../Build/3.4/Aimbot-ParallelEnv"
ENV_PATH = "../Build/3.6/Aimbot-ParallelEnv"
WAND_ENTITY = "koha9"
WORKER_ID = 1
BASE_PORT = 1000
@ -16,19 +16,19 @@ TOTAL_STEPS = 3150000
BATCH_SIZE = 512
MAX_TRAINNING_DATASETS = 6000
DECISION_PERIOD = 1
LEARNING_RATE = 1.5e-4
GAMMA = 0.99
LEARNING_RATE = 5e-5
GAMMA = 0.999
GAE_LAMBDA = 0.95
EPOCHS = 3
CLIP_COEF = 0.11
LOSS_COEF = [1.0, 1.0, 1.0, 1.0] # free go attack defence
POLICY_COEF = [1.0, 1.0, 1.0, 1.0]
POLICY_COEF = [0.8, 0.8, 0.8, 0.8]
ENTROPY_COEF = [0.05, 0.05, 0.05, 0.05]
CRITIC_COEF = [0.8, 0.8, 0.8, 0.8]
CRITIC_COEF = [1.0, 1.0, 1.0, 1.0]
TARGET_LEARNING_RATE = 1e-6
FREEZE_VIEW_NETWORK = False
ANNEAL_LEARNING_RATE = True
ANNEAL_LEARNING_RATE = False
CLIP_VLOSS = True
NORM_ADV = False
TRAIN = True

21
Aimbot-PPO-Python/Pytorch/ppoagent.py
View File

@ -14,6 +14,8 @@ def layer_init(layer, std=np.sqrt(2), bias_const=0.0):
nn.init.constant_(layer.bias, bias_const)
return layer
neural_size_1 = 400
neural_size_2 = 300
class PPOAgent(nn.Module):
def __init__(
@ -31,15 +33,6 @@ class PPOAgent(nn.Module):
self.unity_action_size = env.unity_action_size
self.state_size = self.unity_observation_shape[0]
self.agent_num = env.unity_agent_num
self.target_size = self.args.target_state_size
self.time_state_size = self.args.time_state_size
self.gun_state_size = self.args.gun_state_size
self.my_state_size = self.args.my_state_size
self.ray_state_size = env.unity_observation_shape[0] - self.args.total_target_size
self.state_size_without_ray = self.args.total_target_size
self.head_input_size = (
env.unity_observation_shape[0] - self.target_size - self.time_state_size - self.gun_state_size
) # except target state input
self.unity_discrete_type = env.unity_discrete_type
self.discrete_size = env.unity_discrete_size
@ -49,9 +42,9 @@ class PPOAgent(nn.Module):
self.hidden_networks = nn.ModuleList(
[
nn.Sequential(
layer_init(nn.Linear(self.state_size, 256)),
layer_init(nn.Linear(self.state_size, neural_size_1)),
nn.LeakyReLU(),
layer_init(nn.Linear(256, 128)),
layer_init(nn.Linear(neural_size_1, neural_size_2)),
nn.LeakyReLU(),
)
for i in range(self.target_num)
@ -59,16 +52,16 @@ class PPOAgent(nn.Module):
)
self.actor_dis = nn.ModuleList(
[layer_init(nn.Linear(128, self.discrete_size), std=0.5) for i in range(self.target_num)]
[layer_init(nn.Linear(neural_size_2, self.discrete_size), std=0.5) for i in range(self.target_num)]
)
self.actor_mean = nn.ModuleList(
[layer_init(nn.Linear(128, self.continuous_size), std=0) for i in range(self.target_num)]
[layer_init(nn.Linear(neural_size_2, self.continuous_size), std=0) for i in range(self.target_num)]
)
self.actor_logstd = nn.ParameterList(
[nn.Parameter(torch.zeros(1, self.continuous_size)) for i in range(self.target_num)]
)
self.critic = nn.ModuleList(
[layer_init(nn.Linear(128, 1), std=0) for i in range(self.target_num)]
[layer_init(nn.Linear(neural_size_2, 1), std=0) for i in range(self.target_num)]
)
def get_value(self, state: torch.Tensor):