2023-07-09 17:51:44 +00:00
|
|
|
|
using System;
|
2023-08-15 10:47:14 +00:00
|
|
|
|
using System.Collections.Generic;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
using Unity.MLAgents;
|
|
|
|
|
using Unity.MLAgents.Actuators;
|
|
|
|
|
using Unity.MLAgents.Sensors;
|
|
|
|
|
using UnityEngine;
|
|
|
|
|
|
|
|
|
|
public class MLAgentsCustomController : Agent
|
|
|
|
|
{
|
2023-10-22 16:54:30 +00:00
|
|
|
|
[SerializeField] private GameObject paramContainerObj;
|
|
|
|
|
[SerializeField] private GameObject targetControllerObj;
|
|
|
|
|
[SerializeField] private GameObject environmentUIObj;
|
|
|
|
|
[SerializeField] private GameObject sideChannelObj;
|
2023-10-23 18:34:15 +00:00
|
|
|
|
[SerializeField] private GameObject worldUIControllerObj;
|
|
|
|
|
[SerializeField] private GameObject hudObj;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
|
|
|
|
|
// script
|
|
|
|
|
private AgentController agentController;
|
2023-08-15 10:47:14 +00:00
|
|
|
|
|
2023-07-09 17:51:44 +00:00
|
|
|
|
private ParameterContainer paramContainer;
|
2023-10-22 16:54:30 +00:00
|
|
|
|
private CommonParameterContainer commonParamCon;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
private TargetController targetController;
|
|
|
|
|
private EnvironmentUIControl envUIController;
|
|
|
|
|
private HUDController hudController;
|
2023-07-28 10:44:02 +00:00
|
|
|
|
private TargetUIController targetUIController;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
private RaySensors raySensors;
|
2023-08-15 10:47:14 +00:00
|
|
|
|
private MessageBoxController messageBoxController;
|
2023-10-12 13:48:02 +00:00
|
|
|
|
private AimBotSideChannelController sideChannelController;
|
2023-10-23 18:34:15 +00:00
|
|
|
|
private WorldUIController worldUICon;
|
|
|
|
|
private RewardFunction rewardFunction;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
|
|
|
|
|
// observation
|
2024-01-04 18:46:52 +00:00
|
|
|
|
private float[] myObserve = new float[5];
|
2023-07-09 17:51:44 +00:00
|
|
|
|
|
|
|
|
|
private float[] rayTagResult;
|
|
|
|
|
private float[] rayTagResultOnehot;
|
|
|
|
|
private float[] rayDisResult;
|
|
|
|
|
private float remainTime;
|
2023-12-31 00:46:27 +00:00
|
|
|
|
private float inFireBaseState;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
|
2023-10-23 18:34:15 +00:00
|
|
|
|
private int endTypeInt;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
|
|
|
|
|
private void Start()
|
|
|
|
|
{
|
|
|
|
|
agentController = transform.GetComponent<AgentController>();
|
|
|
|
|
raySensors = transform.GetComponent<RaySensors>();
|
|
|
|
|
paramContainer = paramContainerObj.GetComponent<ParameterContainer>();
|
2023-11-03 03:56:44 +00:00
|
|
|
|
commonParamCon = CommonParameterContainer.Instance;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
targetController = targetControllerObj.GetComponent<TargetController>();
|
|
|
|
|
envUIController = environmentUIObj.GetComponent<EnvironmentUIControl>();
|
2023-10-23 18:34:15 +00:00
|
|
|
|
hudController = hudObj.GetComponent<HUDController>();
|
|
|
|
|
targetUIController = hudObj.GetComponent<TargetUIController>();
|
2023-10-24 19:05:50 +00:00
|
|
|
|
messageBoxController = hudObj.GetComponent<MessageBoxController>();
|
2023-10-12 13:48:02 +00:00
|
|
|
|
sideChannelController = sideChannelObj.GetComponent<AimBotSideChannelController>();
|
2023-10-23 18:34:15 +00:00
|
|
|
|
rewardFunction = gameObject.GetComponent<RewardFunction>();
|
|
|
|
|
worldUICon = worldUIControllerObj.GetComponent<WorldUIController>();
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public override void OnEpisodeBegin()
|
|
|
|
|
{
|
|
|
|
|
agentController.UpdateLockMouse();
|
|
|
|
|
paramContainer.ResetTimeBonusReward();
|
2023-10-22 16:54:30 +00:00
|
|
|
|
if (commonParamCon.gameMode == 0)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
{
|
|
|
|
|
// train mode
|
2023-07-28 19:04:18 +00:00
|
|
|
|
Debug.Log("MLAgentCustomController.OnEpisodeBegin: train mode start");
|
2023-07-09 17:51:44 +00:00
|
|
|
|
targetController.RollNewScene();
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2023-07-28 19:04:18 +00:00
|
|
|
|
Debug.Log("MLAgentCustomController.OnEpisodeBegin: play mode start");
|
2023-07-09 17:51:44 +00:00
|
|
|
|
// play mode
|
2024-01-04 18:46:52 +00:00
|
|
|
|
targetController.PlayModeInitialize();
|
2023-07-28 10:44:02 +00:00
|
|
|
|
// reset target UI
|
|
|
|
|
targetUIController.ClearGamePressed();
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// give default Reward to Reward value will be used.
|
|
|
|
|
if (hudController.chartOn)
|
|
|
|
|
{
|
|
|
|
|
envUIController.InitChart();
|
|
|
|
|
}
|
|
|
|
|
raySensors.UpdateRayInfo(); // update raycast
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public override void CollectObservations(VectorSensor sensor)
|
|
|
|
|
{
|
|
|
|
|
//List<float> enemyLDisList = RaySensors.enemyLDisList;// All Enemy Lside Distances
|
|
|
|
|
//List<float> enemyRDisList = RaySensors.enemyRDisList;// All Enemy Rside Distances
|
|
|
|
|
/**myObserve[0] = transform.localPosition.x / raySensors.viewDistance;
|
|
|
|
|
myObserve[1] = transform.localPosition.y / raySensors.viewDistance;
|
|
|
|
|
myObserve[2] = transform.localPosition.z / raySensors.viewDistance;
|
|
|
|
|
myObserve[3] = transform.eulerAngles.y / 360f;**/
|
2024-01-04 18:46:52 +00:00
|
|
|
|
float obsNum = 0f;
|
2023-12-31 00:46:27 +00:00
|
|
|
|
float angleInRadians = transform.eulerAngles.y * Mathf.Deg2Rad;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
myObserve[0] = transform.localPosition.x;
|
|
|
|
|
myObserve[1] = transform.localPosition.y;
|
|
|
|
|
myObserve[2] = transform.localPosition.z;
|
2023-12-31 00:46:27 +00:00
|
|
|
|
myObserve[3] = MathF.Sin(angleInRadians);
|
|
|
|
|
myObserve[4] = MathF.Cos(angleInRadians);
|
2023-12-29 14:51:05 +00:00
|
|
|
|
rayTagResult = raySensors.rayTagResult;// 探测用RayTag类型结果 float[](raySensorNum,1)
|
2023-12-30 22:49:58 +00:00
|
|
|
|
rayTagResultOnehot = raySensors.rayTagResultOneHot; // 探测用RayTagonehot结果 List<int>[](raySensorNum*Tags,1)
|
2023-12-29 14:51:05 +00:00
|
|
|
|
rayDisResult = raySensors.rayDisResult; // 探测用RayDis距离结果 float[](raySensorNum,1)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
remainTime = targetController.leftTime;
|
2023-12-31 00:46:27 +00:00
|
|
|
|
inFireBaseState = targetController.GetInAreaState();
|
2023-07-09 17:51:44 +00:00
|
|
|
|
agentController.UpdateGunState();
|
|
|
|
|
//float[] focusEnemyObserve = RaySensors.focusEnemyInfo;// 最近的Enemy情报 float[](3,1) MinEnemyIndex,x,z
|
|
|
|
|
|
|
|
|
|
//sensor.AddObservation(allEnemyNum); // 敌人数量 int
|
2024-01-04 18:46:52 +00:00
|
|
|
|
sensor.AddObservation(targetController.targetState);// (5) targettype, target x,y,z, firebasesAreaDiameter
|
2023-12-31 00:46:27 +00:00
|
|
|
|
sensor.AddObservation(inFireBaseState); // (1)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
sensor.AddObservation(remainTime); // (1)
|
|
|
|
|
sensor.AddObservation(agentController.gunReadyToggle); // (1) save gun is ready?
|
2024-01-04 18:46:52 +00:00
|
|
|
|
sensor.AddObservation(myObserve); // (5)自机位置xyz+朝向 float[](5,1)
|
|
|
|
|
// count observation number
|
|
|
|
|
obsNum = targetController.targetState.Length+1+1+1+myObserve.Length;
|
|
|
|
|
Debug.Log(obsNum);
|
2023-12-31 00:16:18 +00:00
|
|
|
|
if (commonParamCon.oneHotRayTag)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
{
|
|
|
|
|
sensor.AddObservation(rayTagResultOnehot); // 探测用RayTag结果 float[](raySensorNum,1)
|
2024-01-04 18:46:52 +00:00
|
|
|
|
obsNum += rayTagResultOnehot.Length;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
sensor.AddObservation(rayTagResult);
|
2024-01-04 18:46:52 +00:00
|
|
|
|
obsNum += rayTagResult.Length;
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
2024-01-04 18:46:52 +00:00
|
|
|
|
Debug.Log(obsNum);
|
2023-12-29 14:51:05 +00:00
|
|
|
|
sensor.AddObservation(rayDisResult); // 探测用RayDis距离结果 float[](raySensorNum,1)
|
2024-01-04 18:46:52 +00:00
|
|
|
|
obsNum += rayDisResult.Length;
|
|
|
|
|
envUIController.UpdateStateText(targetController.targetState, inFireBaseState, remainTime, agentController.gunReadyToggle, myObserve, rayTagResultOnehot, rayDisResult);
|
|
|
|
|
Debug.Log(obsNum);
|
2023-07-09 17:51:44 +00:00
|
|
|
|
/*foreach(float aaa in rayDisResult)
|
|
|
|
|
{
|
|
|
|
|
Debug.Log(aaa);
|
|
|
|
|
}
|
|
|
|
|
Debug.LogWarning("------------");*/
|
|
|
|
|
//sensor.AddObservation(focusEnemyObserve); // 最近的Enemy情报 float[](3,1) MinEnemyIndex,x,z
|
|
|
|
|
//sensor.AddObservation(raySensorNum); // raySensor数量 int
|
|
|
|
|
//sensor.AddObservation(remainTime); // RemainTime int
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public override void OnActionReceived(ActionBuffers actionBuffers)
|
|
|
|
|
{
|
|
|
|
|
//获取输入
|
|
|
|
|
int vertical = actionBuffers.DiscreteActions[0];
|
|
|
|
|
int horizontal = actionBuffers.DiscreteActions[1];
|
|
|
|
|
int mouseShoot = actionBuffers.DiscreteActions[2];
|
|
|
|
|
float Mouse_X = actionBuffers.ContinuousActions[0];
|
|
|
|
|
if (vertical == 2) vertical = -1;
|
|
|
|
|
if (horizontal == 2) horizontal = -1;
|
|
|
|
|
|
|
|
|
|
//应用输入
|
|
|
|
|
agentController.CameraControl(Mouse_X, 0);
|
|
|
|
|
agentController.MoveAgent(vertical, horizontal);
|
|
|
|
|
raySensors.UpdateRayInfo(); // update raycast
|
|
|
|
|
|
|
|
|
|
//判断结束
|
|
|
|
|
float sceneReward = 0f;
|
|
|
|
|
float endReward = 0f;
|
2023-10-23 18:34:15 +00:00
|
|
|
|
(endTypeInt, sceneReward, endReward) = rewardFunction.CheckOverAndRewards();
|
|
|
|
|
float nowReward = rewardFunction.RewardCalculate(sceneReward + endReward, Mouse_X, Math.Abs(vertical) + Math.Abs(horizontal), mouseShoot);
|
2023-07-09 17:51:44 +00:00
|
|
|
|
if (hudController.chartOn)
|
|
|
|
|
{
|
2023-10-23 18:34:15 +00:00
|
|
|
|
envUIController.UpdateChart(nowReward);
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
envUIController.RemoveChart();
|
|
|
|
|
}
|
2023-10-24 19:05:50 +00:00
|
|
|
|
worldUICon.UpdateChart(targetController.targetType, endTypeInt);
|
2023-10-23 18:34:15 +00:00
|
|
|
|
//Debug.Log("reward = " + nowReward);
|
|
|
|
|
if (endTypeInt != (int)TargetController.EndType.Running)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
{
|
|
|
|
|
// Win or lose Finished
|
2023-10-23 18:34:15 +00:00
|
|
|
|
Debug.Log("Finish reward = " + nowReward);
|
2023-10-24 19:05:50 +00:00
|
|
|
|
string targetString = Enum.GetName(typeof(Targets), targetController.targetType);
|
2023-10-23 18:34:15 +00:00
|
|
|
|
switch (endTypeInt)
|
2023-07-09 17:51:44 +00:00
|
|
|
|
{
|
|
|
|
|
case (int)TargetController.EndType.Win:
|
2023-10-22 16:54:30 +00:00
|
|
|
|
sideChannelController.SendSideChannelMessage("Result", targetString + "|Win");
|
2023-08-15 10:47:14 +00:00
|
|
|
|
messageBoxController.PushMessage(
|
|
|
|
|
new List<string> { "Game Win" },
|
|
|
|
|
new List<string> { "green" });
|
2023-07-09 17:51:44 +00:00
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case (int)TargetController.EndType.Lose:
|
2023-10-12 13:48:02 +00:00
|
|
|
|
sideChannelController.SendSideChannelMessage("Result", targetString + "|Lose");
|
2023-08-15 10:47:14 +00:00
|
|
|
|
messageBoxController.PushMessage(
|
|
|
|
|
new List<string> { "Game Lose" },
|
|
|
|
|
new List<string> { "red" });
|
2023-07-09 17:51:44 +00:00
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
Debug.LogWarning("TypeError");
|
|
|
|
|
break;
|
|
|
|
|
}
|
2023-10-23 18:34:15 +00:00
|
|
|
|
SetReward(nowReward);
|
2023-07-09 17:51:44 +00:00
|
|
|
|
EndEpisode();
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
// game not over yet
|
|
|
|
|
}
|
2023-10-23 18:34:15 +00:00
|
|
|
|
SetReward(nowReward);
|
2023-07-09 17:51:44 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public override void Heuristic(in ActionBuffers actionsOut)
|
|
|
|
|
{
|
|
|
|
|
//-------------------BUILD
|
|
|
|
|
ActionSegment<float> continuousActions = actionsOut.ContinuousActions;
|
|
|
|
|
ActionSegment<int> discreteActions = actionsOut.DiscreteActions;
|
|
|
|
|
|
|
|
|
|
if (Input.GetKey(KeyCode.W) && !Input.GetKey(KeyCode.S))
|
|
|
|
|
{
|
|
|
|
|
discreteActions[0] = 1;
|
|
|
|
|
}
|
|
|
|
|
else if (Input.GetKey(KeyCode.S) && !Input.GetKey(KeyCode.W))
|
|
|
|
|
{
|
|
|
|
|
discreteActions[0] = -1;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
discreteActions[0] = 0;
|
|
|
|
|
}
|
|
|
|
|
if (Input.GetKey(KeyCode.D) && !Input.GetKey(KeyCode.A))
|
|
|
|
|
{
|
|
|
|
|
discreteActions[1] = 1;
|
|
|
|
|
}
|
|
|
|
|
else if (Input.GetKey(KeyCode.A) && !Input.GetKey(KeyCode.D))
|
|
|
|
|
{
|
|
|
|
|
discreteActions[1] = -1;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
discreteActions[1] = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Input.GetMouseButton(0))
|
|
|
|
|
{
|
|
|
|
|
// Debug.Log("mousebuttonhit");
|
|
|
|
|
discreteActions[2] = 1;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
discreteActions[2] = 0;
|
|
|
|
|
}
|
|
|
|
|
//^^^^^^^^^^^^^^^^^^^^^discrete-Control^^^^^^^^^^^^^^^^^^^^^^
|
|
|
|
|
|
|
|
|
|
//vvvvvvvvvvvvvvvvvvvvvvvvvvvvvcontinuous-Controlvvvvvvvvvvvvvvvvvvvvvv
|
|
|
|
|
float Mouse_X = Input.GetAxis("Mouse X") * agentController.mouseXSensitivity * Time.deltaTime;
|
|
|
|
|
//float Mouse_Y = Input.GetAxis("Mouse Y") * agentController.mouseYSensitivity * Time.deltaTime;
|
|
|
|
|
continuousActions[0] = Mouse_X;
|
|
|
|
|
//continuousActions[1] = nonReward;
|
|
|
|
|
//continuousActions[2] = shootReward;
|
|
|
|
|
//continuousActions[3] = shootWithoutReadyReward;
|
|
|
|
|
//continuousActions[4] = hitReward;
|
|
|
|
|
//continuousActions[5] = winReward;
|
|
|
|
|
//continuousActions[6] = loseReward;
|
|
|
|
|
//continuousActions[7] = killReward;
|
|
|
|
|
//continuousActions[1] = Mouse_Y;
|
|
|
|
|
//continuousActions[2] = timeLimit;
|
|
|
|
|
//^^^^^^^^^^^^^^^^^^^^^^^^^^^^^continuous-Control^^^^^^^^^^^^^^^^^^^^^^
|
|
|
|
|
}
|
|
|
|
|
}
|