分离所有RewardFunction相关到单独的一个Script里。

Assets/ML-Agents/Timers/Play_timers.json

@@ -1 +1 @@
-{"count":1,"self":6.87364,"total":6.9109843,"children":{"InitializeActuators":{"count":2,"self":0.0010012999999999999,"total":0.0010012999999999999,"children":null},"InitializeSensors":{"count":2,"self":0.0010003,"total":0.0010003,"children":null},"AgentSendState":{"count":99,"self":0.00050139999999999994,"total":0.0069993,"children":{"CollectObservations":{"count":99,"self":0.0049989,"total":0.0049989,"children":null},"WriteActionMask":{"count":99,"self":0,"total":0,"children":null},"RequestDecision":{"count":99,"self":0.001499,"total":0.001499,"children":null}}},"DecideAction":{"count":99,"self":0.0019998999999999998,"total":0.0019998999999999998,"children":null},"AgentAct":{"count":99,"self":0.025844,"total":0.025844,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1697993466","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-3RTM9hQ6yJRZGQKZx8Slg -hubSessionId a9694f1b-42f4-414c-9f3a-decdfec467f4 -accessToken _-EoIL8F11LvGF_SbanmzPmRWk7d3mJZ66R4Zf61pY400ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"Play","end_time_seconds":"1697993473"}}
+{"count":1,"self":14.61136,"total":14.758261,"children":{"InitializeActuators":{"count":2,"self":0.0009999,"total":0.0009999,"children":null},"InitializeSensors":{"count":2,"self":0.00099969999999999985,"total":0.00099969999999999985,"children":null},"AgentSendState":{"count":484,"self":0.0034990999999999998,"total":0.0180021,"children":{"CollectObservations":{"count":484,"self":0.0119995,"total":0.0119995,"children":null},"WriteActionMask":{"count":484,"self":0,"total":0,"children":null},"RequestDecision":{"count":484,"self":0.0025035,"total":0.0025035,"children":null}}},"DecideAction":{"count":484,"self":0.0029993,"total":0.0029993,"children":null},"AgentAct":{"count":484,"self":0.1239006,"total":0.1239006,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1698085512","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-3RTM9hQ6yJRZGQKZx8Slg -hubSessionId a9694f1b-42f4-414c-9f3a-decdfec467f4 -accessToken 0RXRt4Gr5-wz9bb63SDgNZNzv9nApR3GYqka0HInnGM00ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"Play","end_time_seconds":"1698085527"}}

Assets/ML-Agents/Timers/Train_timers.json

@@ -1 +1 @@
-{"count":1,"self":44.2918592,"total":44.7423777,"children":{"InitializeActuators":{"count":2,"self":0.0015,"total":0.0015,"children":null},"InitializeSensors":{"count":2,"self":0.0005001,"total":0.0005001,"children":null},"AgentSendState":{"count":1994,"self":0.0115036,"total":0.073176999999999992,"children":{"CollectObservations":{"count":1994,"self":0.0546691,"total":0.0546691,"children":null},"WriteActionMask":{"count":1994,"self":0.0005,"total":0.0005,"children":null},"RequestDecision":{"count":1994,"self":0.0065042999999999993,"total":0.0065042999999999993,"children":null}}},"DecideAction":{"count":1994,"self":0.0091503,"total":0.0091503,"children":null},"AgentAct":{"count":1994,"self":0.36569229999999997,"total":0.36569229999999997,"children":null}},"gauges":{"AKMAgent.CumulativeReward":{"count":1,"max":-2512.52466,"min":-2512.52466,"runningAverage":-2512.52466,"value":-2512.52466,"weightedAverage":-2512.52466}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1697992942","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-3RTM9hQ6yJRZGQKZx8Slg -hubSessionId a9694f1b-42f4-414c-9f3a-decdfec467f4 -accessToken _-EoIL8F11LvGF_SbanmzPmRWk7d3mJZ66R4Zf61pY400ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"Train","end_time_seconds":"1697992987"}}
+{"count":1,"self":14.1656544,"total":14.3145136,"children":{"InitializeActuators":{"count":2,"self":0.0015011999999999998,"total":0.0015011999999999998,"children":null},"InitializeSensors":{"count":2,"self":0.0009999,"total":0.0009999,"children":null},"AgentSendState":{"count":454,"self":0.003999,"total":0.027000399999999997,"children":{"CollectObservations":{"count":454,"self":0.0205008,"total":0.0205008,"children":null},"WriteActionMask":{"count":454,"self":0,"total":0,"children":null},"RequestDecision":{"count":454,"self":0.0025006,"total":0.0025006,"children":null}}},"DecideAction":{"count":454,"self":0.0068705,"total":0.0068705,"children":null},"AgentAct":{"count":454,"self":0.11198749999999999,"total":0.11198749999999999,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1698085370","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-3RTM9hQ6yJRZGQKZx8Slg -hubSessionId a9694f1b-42f4-414c-9f3a-decdfec467f4 -accessToken 0RXRt4Gr5-wz9bb63SDgNZNzv9nApR3GYqka0HInnGM00ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"Train","end_time_seconds":"1698085385"}}

Assets/RewardFunction.cs

@@ -0,0 +1,587 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+
+public class RewardFunction : MonoBehaviour
+{
+    public enum EndType
+    { Win, Lose, Running, Num };
+
+    [SerializeField] private GameObject parameterContainerObj;
+    [SerializeField] private GameObject sceneBlockContainerObj;
+    [SerializeField] private GameObject targetControllerObj;
+    [SerializeField] private GameObject environmentUIObj;
+    [SerializeField] private GameObject commonParameterContainerObj;
+    [SerializeField] private GameObject enemyContainerObj;
+
+    private GameObject agentObj;
+    private Camera fpsCam;
+    private CommonParameterContainer commonParamCon;
+    private SceneBlockContainer sceneBlockCon;
+    private ParameterContainer paramCon;
+    private TargetController targetCon;
+    private EnvironmentUIControl envUICon;
+    private AgentController agentCon;
+    private RaySensors raySensors;
+
+    private bool firstRewardFlag = false;
+    private float lastDistance;
+    private float lastEnemyFacingDistance = 0f; // record last enemy facing minimum distance
+    private float lastTargetFacingDistance = 0f; // record last target facing minimum distance
+    private List<float> spinRecord = new List<float>();
+
+    private void Start()
+    {
+        agentObj = gameObject;
+        agentCon = agentObj.GetComponent<AgentController>();
+        fpsCam = agentCon.fpsCam;
+        commonParamCon = commonParameterContainerObj.GetComponent<CommonParameterContainer>();
+        paramCon = parameterContainerObj.GetComponent<ParameterContainer>();
+        sceneBlockCon = sceneBlockContainerObj.GetComponent<SceneBlockContainer>();
+        targetCon = targetControllerObj.GetComponent<TargetController>();
+        envUICon = environmentUIObj.GetComponent<EnvironmentUIControl>();
+        raySensors = GetComponent<RaySensors>();
+    }
+
+    /// <summary>
+    /// Calculates the reward value.
+    /// </summary>
+    /// <param name="sceneReward">Reward value from the scene.</param>
+    /// <param name="mouseX">Movement amount of the mouse along the X-axis.</param>
+    /// <param name="movement">Movement of discrete.</param>
+    /// <param name="shootState">State of the shooting action.</param>
+    /// <returns>Returns the calculated total reward value.</returns>
+    /// <remarks>
+    /// This method calculates the total reward based on the provided parameters, 
+    /// taking into account rewards for enemy kills, shooting actions, facing reward, 
+    /// and penalties such as spin and movement.
+    /// </remarks>
+    public float RewardCalculate(float sceneReward, float mouseX, float movement, int shootState)
+    {
+        float epreward = 0f;
+        // Got kill point reward
+        if (agentCon.enemyKillCount > 0)
+        {
+            for (int i = 0; i < agentCon.enemyKillCount; i++)
+            {
+                // get
+                epreward += KillReward(agentCon.killEnemyPosition);
+            }
+            agentCon.enemyKillCount = 0;
+        }
+        else
+        {
+            agentCon.enemyKillCount = 0;
+        }
+        // Shoot action reward
+        epreward += Ballistic(shootState) + sceneReward;
+        // facing reward
+        epreward += FacingReward();
+        // Penalty
+        // spin penalty
+        spinRecord.Add(mouseX);
+        if (spinRecord.Count >= commonParamCon.spinRecordMax)
+        {
+            spinRecord.RemoveAt(0);
+        }
+        float spinPenaltyReward = Math.Abs(spinRecord.ToArray().Sum() * commonParamCon.spinPenalty);
+        if (spinPenaltyReward >= commonParamCon.spinPenaltyThreshold)
+        {
+            epreward -= spinPenaltyReward;
+        }
+        else
+        {
+            epreward -= Math.Abs(mouseX) * commonParamCon.mousePenalty;
+        }
+        // move penalty
+        if (movement != 0)
+        {
+            epreward -= commonParamCon.movePenalty;
+        }
+        return epreward;
+    }
+
+    /// <summary>
+    /// Calculates the reward value for shooting actions.
+    /// </summary>
+    /// <param name="shootState">State value of the shooting action.</param>
+    /// <returns>Returns the reward value associated with shooting.</returns>
+    /// <remarks>
+    /// This method calculates the reward value based on the shooting state and other related conditions, 
+    /// such as whether the enemy was hit, whether the shot was towards the target area, and whether the gun was ready to shoot.
+    /// </remarks>
+    private float Ballistic(int shootState)
+    {
+        Vector3 point = new Vector3(fpsCam.pixelWidth / 2, fpsCam.pixelHeight / 2, 0);// start position
+        Ray ray = fpsCam.ScreenPointToRay(point);
+        RaycastHit hit;
+        // Debug.DrawRay(centerRay.origin, centerRay.direction * 100, Color.blue);
+        // Mouse Pressed
+        if (shootState != 0 && agentCon.gunReadyToggle == true)
+        {
+            agentCon.lastShootTime = Time.time;
+            if (Physics.Raycast(ray, out hit, 100))
+            {
+                if (hit.collider.tag != agentCon.myTag && hit.collider.tag != "Wall" && hit.collider.tag != "Untagged")
+                {
+                    // kill enemy
+                    GameObject gotHitObj = hit.transform.gameObject;
+                    gotHitObj.GetComponent<States>().ReactToHit(commonParamCon.damage, gameObject);
+                    shootState = 0;
+                    return HitEnemyReward(gotHitObj.transform.position);
+                }
+            }
+            if (targetCon.targetTypeInt == (int)Targets.Attack)
+            {
+                // while if attack mode
+                float targetDis = Vector3.Distance(sceneBlockCon.nowBlock.transform.position, transform.position);
+                if (targetDis <= raySensors.viewDistance)
+                {
+                    // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
+                    if (Vector3.Distance(ray.origin + (ray.direction * targetDis), sceneBlockCon.nowBlock.transform.position) <= sceneBlockCon.nowBlock.firebasesAreaDiameter / 2)
+                    {
+                        // im shooting at target but didn't hit enemy
+                        // Debug.DrawRay(centerRay.origin, viewPoint-centerRay.origin, Color.blue);
+                        return commonParamCon.shootTargetAreaReward;
+                    }
+                }
+            }
+            shootState = 0;
+            return commonParamCon.shootReward;
+        }
+        else if (shootState != 0 && agentCon.gunReadyToggle == false)
+        {
+            // shoot without ready
+            shootState = 0;
+            return commonParamCon.shootWithoutReadyReward;
+        }
+        else
+        {
+            // do not shoot
+            shootState = 0;
+            return commonParamCon.nonReward;
+        }
+    }
+
+    /// <summary>
+    /// Retrieves the reward value based on the character's facing direction.
+    /// </summary>
+    /// <returns>Returns the reward value for the facing direction.</returns>
+    /// <remarks>
+    /// This method calculates a reward value based on the relationship between the character's facing direction and the target.
+    /// in free mode, if the character is facing an enemy, the reward is a fixed value
+    /// in attack mode, the reward depends on the distance between the character and the target, among other factors.
+    /// </remarks>
+    private float FacingReward()
+    {
+        float nowReward = 0;
+        bool isFacingtoEnemy = false;
+        float enemyFacingDistance = 0f;
+
+        Vector3 screenCenter = new Vector3(fpsCam.pixelWidth / 2, fpsCam.pixelHeight / 2, 0);
+        Vector3 screenLeft = new Vector3(0, fpsCam.pixelHeight / 2, 0);
+
+        Ray centerRay = fpsCam.ScreenPointToRay(screenCenter);
+        Ray leftRay = fpsCam.ScreenPointToRay(screenLeft);
+
+        // target fireBaseArea Position, turen y to camera's y
+        Vector3 fireBaseArea = sceneBlockCon.nowBlock.fireBasesAreaObj.transform.position;
+        fireBaseArea.y = fpsCam.transform.position.y;
+
+        // my position, turn y to camera's y
+        // Debug.DrawRay(fpsCam.transform.position, centerRay.direction * 100, Color.blue);
+        Vector3 myposition = transform.position;
+        myposition.y = fpsCam.transform.position.y;
+
+        // Target to Agent distance
+        //Debug.DrawLine(fireBaseArea, myposition, Color.red);
+        float targetDis = Vector3.Distance(fireBaseArea, myposition);
+
+        // point in centerRay and leftRay which distance is targetDis from camera center
+        Vector3 pointInCenterRay = fpsCam.transform.position + (centerRay.direction * targetDis);
+        Vector3 pointInLeftRay = fpsCam.transform.position + (leftRay.direction * targetDis);
+
+        // center of screen to target's distance
+        // Debug.DrawLine(pointInCenterRay, fireBaseArea,Color.green);
+        float camCenterToTarget = Vector3.Distance(pointInCenterRay, fireBaseArea);
+
+        // left of screen to target's distance
+        // Debug.DrawLine(pointInLeftRay, pointInCenterRay, Color.yellow);
+        float camCenterToViewEdge = Vector3.Distance(pointInLeftRay, pointInCenterRay);
+
+        switch (targetCon.targetTypeInt)
+        {
+            case (int)Targets.Free:
+                //free mode
+                RaycastHit hit;
+                if (Physics.Raycast(centerRay, out hit, 100))
+                {
+                    // facing to an enemy
+                    if (hit.collider.tag != agentCon.myTag && hit.collider.tag != "Wall")
+                    {
+                        nowReward = commonParamCon.facingReward;
+                        isFacingtoEnemy = true;
+                    }
+                }
+                if (raySensors.inViewEnemies.Count > 0 && !isFacingtoEnemy)
+                {
+                    // have enemy in view
+                    List<float> projectionDis = new List<float>();
+                    foreach (GameObject theEnemy in raySensors.inViewEnemies)
+                    {
+                        // for each enemy in view
+                        Vector3 projection = Vector3.Project(theEnemy.transform.position - transform.position, (centerRay.direction * 10));
+                        Vector3 verticalToRay = transform.position + projection - theEnemy.transform.position;
+                        projectionDis.Add(verticalToRay.magnitude);
+                        // Debug.Log("enemy!" + verticalToRay.magnitude);
+                        // Debug.DrawRay(transform.position, (centerRay.direction * 100), Color.cyan);
+                        // Debug.DrawRay(transform.position, theEnemy.transform.position - transform.position, Color.yellow);
+                        // Debug.DrawRay(transform.position, projection, Color.blue);
+                        // Debug.DrawRay(theEnemy.transform.position, verticalToRay, Color.magenta);
+                    }
+                    enemyFacingDistance = projectionDis.Min();
+                    if (enemyFacingDistance <= lastEnemyFacingDistance)
+                    {
+                        // closing to enemy
+                        nowReward = 1 / MathF.Sqrt(commonParamCon.facingInviewEnemyDisCOEF * enemyFacingDistance + 0.00001f);
+                    }
+                    else
+                    {
+                        nowReward = 0;
+                    }
+                    // enemy in view Reward
+                    lastEnemyFacingDistance = enemyFacingDistance;
+                    if (nowReward >= commonParamCon.facingReward) nowReward = commonParamCon.facingReward; // limit
+                    if (nowReward <= -commonParamCon.facingReward) nowReward = -commonParamCon.facingReward; // limit
+                    // Debug.Log("ninimum = " + nowReward);
+                }
+                break;
+
+            case (int)Targets.Attack:
+                // attack mode
+                if (targetDis <= raySensors.viewDistance)
+                {
+                    // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
+                    // while center of screen between target's distance is lower than firebasesAreaDiameter
+                    // while facing to target
+                    if (camCenterToTarget <= sceneBlockCon.nowBlock.firebasesAreaDiameter / 2)
+                    {
+                        // Debug.DrawRay(centerRay.origin, viewPoint-centerRay.origin, Color.blue);
+                        nowReward = commonParamCon.facingReward;
+                    }
+                    else
+                    {
+                        // while not facing to target
+                        nowReward = (lastTargetFacingDistance - camCenterToTarget) * commonParamCon.facingTargetReward;
+                    }
+                }
+                // update lastTargetFacingDistance
+                lastTargetFacingDistance = camCenterToTarget;
+                break;
+
+            case (int)Targets.Go:
+                // goto mode
+                if (camCenterToTarget <= camCenterToViewEdge)
+                {
+                    // fireArea is in view
+                    nowReward = commonParamCon.facingReward;
+                }
+                else
+                {
+                    nowReward = 0;
+                }
+                break;
+
+            default:
+                Debug.LogError("Wrong target type");
+                break;
+        }
+        return nowReward;
+    }
+
+    /// <summary>
+    /// Checks the game's end state and retrieves rewards.
+    /// </summary>
+    /// <returns>A tuple containing the game's end type, current reward, and final reward.
+    /// 1 = success,2 = overtime,0 = notover</returns>
+    public (int, float, float) CheckOverAndRewards()
+    {
+        int endTypeInt = 0;
+        float nowReward = 0;
+        float endReward = 0;
+        switch (targetCon.targetTypeInt)
+        {
+            case (int)Targets.Go:
+                // goto
+                (endTypeInt, nowReward, endReward) = CheckOverAndRewardsGo();
+                break;
+
+            case (int)Targets.Attack:
+                // attack
+                (endTypeInt, nowReward, endReward) = CheckOverAndRewardsAttack();
+                break;
+
+            case (int)Targets.Defence:
+                //defence
+                (endTypeInt, nowReward, endReward) = CheckOverAndRewardsDefence();
+                break;
+
+            case (int)Targets.Stay:
+                // Stay
+                // endless
+                nowReward = 0;
+                endReward = 0;
+                endTypeInt = (int)EndType.Running;
+                break;
+
+            default:
+                //free kill
+                (endTypeInt, nowReward, endReward) = CheckOverAndRewardsFreeKill();
+                break;
+        }
+        envUICon.ShowResult(endTypeInt);
+        return (endTypeInt, nowReward, endReward);
+    }
+
+    private (int, float, float) CheckOverAndRewardsGo()
+    {
+        int endTypeInt = 0;
+        float nowReward = 0;
+        float endReward = 0;
+        float nowDistance = 0;
+
+        (nowDistance, targetCon.inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
+        envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
+        float areaTargetReward = GetDistanceReward(nowDistance, targetCon.inArea);
+        //if(inArea != 0)
+        if (sceneBlockCon.nowBlock.firebasesBelong >= sceneBlockCon.nowBlock.belongMaxPoint)
+        {
+            // win
+            // let the area belongs to me
+            nowReward = areaTargetReward;
+            endReward = paramCon.goWinReward;
+            //nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
+            endTypeInt = (int)EndType.Win;
+        }
+        else if (targetCon.leftTime <= 0)
+        {
+            // time out lose
+            nowReward = areaTargetReward;
+            endReward = commonParamCon.loseReward;
+            endTypeInt = (int)EndType.Lose;
+        }
+        else
+        {
+            // keep on keeping on!
+            nowReward = areaTargetReward;
+            endReward = 0;
+            endTypeInt = (int)EndType.Running;
+        }
+        return (endTypeInt, nowReward, endReward);
+    }
+
+    private (int, float, float) CheckOverAndRewardsAttack()
+    {
+        int endTypeInt = 0;
+        float nowReward = 0;
+        float endReward = 0;
+        float nowDistance = 0;
+        (nowDistance, targetCon.inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
+        envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
+        if (sceneBlockCon.nowBlock.GetInAreaNumber(commonParamCon.group2Tag) <= 0 && targetCon.targetEnemySpawnFinish)
+        {
+            // win
+            // let the area belongs to me and kill every enmy in this area.
+            nowReward = 0;
+            endReward = paramCon.attackWinReward;
+            //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+            endTypeInt = (int)EndType.Win;
+            targetCon.targetEnemySpawnFinish = false;
+        }
+        else if (targetCon.leftTime <= 0 && targetCon.targetEnemySpawnFinish)
+        {
+            // time out lose
+            nowReward = 0;
+            endReward = commonParamCon.loseReward;
+            //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+            endTypeInt = (int)EndType.Lose;
+            targetCon.targetEnemySpawnFinish = false;
+        }
+        else
+        {
+            // keep on keeping on!
+            // nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
+            nowReward = 0;
+            endReward = 0;
+            targetCon.targetEnemySpawnFinish = true;
+            endTypeInt = (int)EndType.Running;
+        }
+        return (endTypeInt, nowReward, endReward);
+    }
+
+    private (int, float, float) CheckOverAndRewardsDefence()
+    {
+        // !!! NOT FINISHED YET!!!
+        int endTypeInt = 0;
+        float nowReward = 0;
+        float endReward = 0;
+        float nowDistance = 0;
+        (nowDistance, targetCon.inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
+        envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
+        if (targetCon.leftTime <= 0 && sceneBlockCon.nowBlock.firebasesBelong >= 0f)
+        {
+            // win
+            // time over and the area still mine
+            nowReward = paramCon.defenceWinReward;
+            //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+            endTypeInt = (int)EndType.Win;
+        }
+        else if (sceneBlockCon.nowBlock.firebasesBelong <= sceneBlockCon.nowBlock.belongMaxPoint)
+        {
+            // lost area lose
+            nowReward = commonParamCon.loseReward;
+            //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+            endTypeInt = (int)EndType.Lose;
+        }
+        else
+        {
+            // keep on keeping on!
+            // nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
+            endTypeInt = (int)EndType.Running;
+        }
+        return (endTypeInt, nowReward, endReward);
+    }
+
+    private (int, float, float) CheckOverAndRewardsFreeKill()
+    {
+        int endTypeInt = 0;
+        float nowReward = 0;
+        float endReward = 0;
+        if (enemyContainerObj.transform.childCount <= 0)
+        {
+            // win
+            // nowReward = paramCon.winReward + (paramCon.timeBonusPerSecReward * leftTime);
+            nowReward = 0;
+            endReward = paramCon.freeWinReward;
+            endTypeInt = (int)EndType.Win;
+        }
+        else if (targetCon.leftTime <= 0)
+        {
+            // lose
+            //nowReward = paramCon.loseReward;
+            nowReward = 0;
+            endReward = commonParamCon.loseReward;
+            endTypeInt = (int)EndType.Lose;
+        }
+        else
+        {
+            // keep on keeping on!
+            nowReward = 0;
+            endReward = 0;
+            endTypeInt = (int)EndType.Running;
+        }
+        return (endTypeInt, nowReward, endReward);
+    }
+
+    /// <summary>
+    /// Calculates scene reward based on distance, granting higher rewards for being closer to the target.
+    /// </summary>
+    /// <param name="nowDistance">The current distance.</param>
+    /// <param name="inarea">Whether inside an area.</param>
+    /// <returns>The reward value calculated based on distance.</returns>
+    private float GetDistanceReward(float nowDistance, int inarea)
+    {
+        if (firstRewardFlag)
+        {
+            // first distance record
+            (lastDistance, _) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
+            firstRewardFlag = false;
+        }
+        float nowSeneReward = 0f;
+        if (inarea != 0)
+        {
+            // in area
+            nowSeneReward = paramCon.inAreaReward;
+        }
+        else
+        {
+            // out of area
+            // nowSeneReward = paramCon.distanceReward * Math.Clamp(lastDistance - nowDistance, 0, 100);
+            nowSeneReward = commonParamCon.distanceReward * (lastDistance - nowDistance);
+        }
+        lastDistance = nowDistance;
+        return nowSeneReward;
+    }
+
+    /// <summary>
+    /// Calculates kill reward based on the position of the killed enemy.
+    /// </summary>
+    /// <param name="enemyPosition">The position of the killed enemy.</param>
+    /// <returns>The reward value calculated based on the kill position.</returns>
+    public float KillReward(Vector3 enemyPosition)
+    {
+        float nowKillReward = 0f;
+        if (targetCon.targetTypeInt == (int)Targets.Attack)
+        {
+            // attack mode
+            (_, int isInArea) = sceneBlockCon.nowBlock.GetDistInArea(enemyPosition);
+            if (isInArea == 1)
+            {
+                // kill in area enemy
+                nowKillReward = paramCon.killTargetEnemyReward;
+            }
+            else
+            {
+                nowKillReward = commonParamCon.killNonTargetReward;
+            }
+        }
+        else if (targetCon.targetTypeInt == (int)Targets.Free)
+        {
+            // free mode hit
+            nowKillReward = paramCon.killTargetEnemyReward;
+        }
+        else
+        {
+            // goto & defence
+            nowKillReward = commonParamCon.killNonTargetReward;
+        }
+        return nowKillReward;
+    }
+
+    /// <summary>
+    /// Calculates hit reward based on the position of the hit enemy and the current mode.
+    /// </summary>
+    /// <param name="enemyPosition">The position of the hit enemy.</param>
+    /// <returns>The reward value calculated based on the hit position and mode.</returns>
+    public float HitEnemyReward(Vector3 enemyPosition)
+    {
+        float nowHitReward = 0f;
+        if (targetCon.targetTypeInt == (int)Targets.Attack)
+        {
+            // attack mode
+            (_, int isInArea) = sceneBlockCon.nowBlock.GetDistInArea(enemyPosition);
+            if (isInArea == 1)
+            {
+                // hit in area enemy
+                nowHitReward = paramCon.hitTargetReward;
+            }
+            else
+            {
+                // hit not in area enemy
+                nowHitReward = commonParamCon.hitNonTargetReward;
+            }
+        }
+        else if (targetCon.targetTypeInt == (int)Targets.Free)
+        {
+            // free mode hit
+            nowHitReward = paramCon.hitTargetReward;
+        }
+        else
+        {
+            // goto & defence
+            nowHitReward = commonParamCon.hitNonTargetReward;
+        }
+        return nowHitReward;
+    }
+}

Assets/RewardFunction.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 04b45691af6143d4d84b5d024d062bf7
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scenes/Play.unity

@@ -1766,6 +1766,8 @@ MonoBehaviour:
   spinRecordMax: 40
   spinPenaltyThreshold: 10
   facingInviewEnemyDisCOEF: 0.5
+  group1Tag: Player
+  group2Tag: Enemy
   hitTargetRewardDefault: 25
   killTargetEnemyRewardDefault: 25
   inAreaRewardDefault: 12
@@ -3300,7 +3302,7 @@ RectTransform:
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0, y: 1}
   m_AnchorMax: {x: 1, y: 1}
-  m_AnchoredPosition: {x: 0.000030517578, y: -246.20204}
+  m_AnchoredPosition: {x: 0.000030517578, y: -246.20215}
   m_SizeDelta: {x: 0, y: 0}
   m_Pivot: {x: 0, y: 0}
 --- !u!114 &324069808
@@ -7093,6 +7095,7 @@ GameObject:
   - component: {fileID: 629870341}
   - component: {fileID: 629870347}
   - component: {fileID: 629870346}
+  - component: {fileID: 629870348}
   - component: {fileID: 629870339}
   - component: {fileID: 629870338}
   - component: {fileID: 629870344}
@@ -7261,11 +7264,12 @@ MonoBehaviour:
   hasUpgradedFromAgentParameters: 1
   MaxStep: 0
   paramContainerObj: {fileID: 707336050}
-  CommonParameterContainer: {fileID: 220347584}
+  commonParameterContainer: {fileID: 220347584}
   targetControllerObj: {fileID: 1692325237}
   environmentUIObj: {fileID: 156390936}
-  sideChannelObj: {fileID: 0}
+  sideChannelObj: {fileID: 1866094461}
-  hudUIObj: {fileID: 2082200184}
+  worldUIControllerObj: {fileID: 1052251813}
+  hudObj: {fileID: 2082200184}
   oneHotRayTag: 1
 --- !u!114 &629870347
 MonoBehaviour:
@@ -7280,12 +7284,6 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   commonParameterContainerObj: {fileID: 220347584}
-  environmentObj: {fileID: 749443372}
-  enemyContainerObj: {fileID: 891195334}
-  sceneBlockContainerObj: {fileID: 1811162388}
-  environmentUIControlObj: {fileID: 156390936}
-  targetControllerObj: {fileID: 1692325237}
-  HUDObj: {fileID: 2082200184}
   fpsCam: {fileID: 1633276149}
   moveSpeed: 9
   vX: 0
@@ -7295,7 +7293,29 @@ MonoBehaviour:
   mouseXSensitivity: 100
   mouseYSensitivity: 200
   yRotation: 0.1
+  lastShootTime: 0
+  enemyKillCount: 0
+  killEnemyPosition: {x: 0, y: 0, z: 0}
   defaultTPCamera: 1
+  myTag: 
+--- !u!114 &629870348
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 629870337}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 04b45691af6143d4d84b5d024d062bf7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  parameterContainerObj: {fileID: 707336050}
+  sceneBlockContainerObj: {fileID: 1811162388}
+  targetControllerObj: {fileID: 1692325237}
+  environmentUIObj: {fileID: 156390936}
+  commonParameterContainerObj: {fileID: 220347584}
+  enemyContainerObj: {fileID: 891195334}
 --- !u!1 &636386815
 GameObject:
   m_ObjectHideFlags: 0
@@ -12773,7 +12793,7 @@ MonoBehaviour:
   m_HandleRect: {fileID: 538300382}
   m_Direction: 2
   m_Value: 0
-  m_Size: 1
+  m_Size: 0.9999897
   m_NumberOfSteps: 0
   m_OnValueChanged:
     m_PersistentCalls:
@@ -16050,17 +16070,13 @@ MonoBehaviour:
   HUDObj: {fileID: 2082200184}
   sceneBlockContainerObj: {fileID: 1811162388}
   enemyContainerObj: {fileID: 891195334}
-  parameterContainerObj: {fileID: 707336050}
   commonParameterContainerObj: {fileID: 220347584}
   environmentUIObj: {fileID: 156390936}
-  worldUIObj: {fileID: 1052251813}
   edgeUp: {fileID: 1352953080}
   edgeDown: {fileID: 1571061029}
   edgeLeft: {fileID: 1551793502}
   edgeRight: {fileID: 1046425531}
   edgeAgent_Enemy: {fileID: 704984377}
-  group1Tag: Player
-  group2Tag: Enemy
   minEnemyAreaX: 0
   attackProb: 0.1
   gotoProb: 0.1
@@ -16072,7 +16088,9 @@ MonoBehaviour:
   - 0
   - 0
   - 0
+  inArea: 0
   targetPosition: {x: 0, y: 0, z: 0}
+  targetEnemySpawnFinish: 0
 --- !u!1 &1695161701
 GameObject:
   m_ObjectHideFlags: 0
@@ -21488,7 +21506,7 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   chartOn: 0
-  sideChannelObj: {fileID: 0}
+  sideChannelObj: {fileID: 1866094461}
   chartOnToggleObj: {fileID: 269417898}
   saveModelButton: {fileID: 0}
   chartOnTimeOutInputObj: {fileID: 149218411}

Assets/Scenes/Train.unity

@@ -3955,11 +3955,11 @@ RectTransform:
   m_Father: {fileID: 55747560}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMin: {x: 0.5, y: 0.5}
-  m_AnchorMax: {x: 0, y: 1}
+  m_AnchorMax: {x: 0.5, y: 0.5}
-  m_AnchoredPosition: {x: 250, y: -150}
+  m_AnchoredPosition: {x: 0, y: 0}
   m_SizeDelta: {x: 0, y: 0}
-  m_Pivot: {x: 0, y: 1}
+  m_Pivot: {x: 0.5, y: 0.5}
 --- !u!114 &413189341
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -4393,11 +4393,11 @@ RectTransform:
   m_Father: {fileID: 55747560}
   m_RootOrder: 1
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMin: {x: 1, y: 0.5}
-  m_AnchorMax: {x: 0, y: 1}
+  m_AnchorMax: {x: 1, y: 0.5}
-  m_AnchoredPosition: {x: 250, y: -150}
+  m_AnchoredPosition: {x: -250, y: 0}
   m_SizeDelta: {x: 0, y: 0}
-  m_Pivot: {x: 0, y: 1}
+  m_Pivot: {x: 1, y: 0.5}
 --- !u!114 &440411431
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -5859,6 +5859,7 @@ GameObject:
   - component: {fileID: 629870347}
   - component: {fileID: 629870346}
   - component: {fileID: 629870339}
+  - component: {fileID: 629870348}
   - component: {fileID: 629870338}
   - component: {fileID: 629870344}
   - component: {fileID: 629870345}
@@ -6026,11 +6027,12 @@ MonoBehaviour:
   hasUpgradedFromAgentParameters: 1
   MaxStep: 0
   paramContainerObj: {fileID: 707336050}
-  CommonParameterContainer: {fileID: 1580123439}
+  commonParameterContainer: {fileID: 1580123439}
   targetControllerObj: {fileID: 1692325237}
   environmentUIObj: {fileID: 156390936}
   sideChannelObj: {fileID: 1866094461}
-  hudUIObj: {fileID: 2082200184}
+  worldUIControllerObj: {fileID: 1052251813}
+  hudObj: {fileID: 2082200184}
   oneHotRayTag: 1
 --- !u!114 &629870347
 MonoBehaviour:
@@ -6050,7 +6052,6 @@ MonoBehaviour:
   sceneBlockContainerObj: {fileID: 1811162388}
   environmentUIControlObj: {fileID: 156390936}
   targetControllerObj: {fileID: 1692325237}
-  HUDObj: {fileID: 2082200184}
   fpsCam: {fileID: 1633276149}
   moveSpeed: 9
   vX: 0
@@ -6061,6 +6062,24 @@ MonoBehaviour:
   mouseYSensitivity: 200
   yRotation: 0.1
   defaultTPCamera: 1
+--- !u!114 &629870348
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 629870337}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 04b45691af6143d4d84b5d024d062bf7, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  commonParameterContainerObj: {fileID: 1580123439}
+  parameterContainerObj: {fileID: 707336050}
+  sceneBlockContainerObj: {fileID: 1811162388}
+  targetControllerObj: {fileID: 1692325237}
+  environmentUIObj: {fileID: 156390936}
+  enemyContainerObj: {fileID: 891195334}
 --- !u!1 &636386815
 GameObject:
   m_ObjectHideFlags: 0
@@ -7085,11 +7104,11 @@ RectTransform:
   m_Father: {fileID: 413189340}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMin: {x: 0.5, y: 0.5}
-  m_AnchorMax: {x: 0, y: 1}
+  m_AnchorMax: {x: 0.5, y: 0.5}
-  m_AnchoredPosition: {x: 2, y: -0}
+  m_AnchoredPosition: {x: 2, y: 0}
   m_SizeDelta: {x: 0, y: 0}
-  m_Pivot: {x: 0, y: 1}
+  m_Pivot: {x: 0.5, y: 0.5}
 --- !u!114 &762190081
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -7117,7 +7136,7 @@ MonoBehaviour:
     m_BestFit: 0
     m_MinSize: 10
     m_MaxSize: 40
-    m_Alignment: 0
+    m_Alignment: 4
     m_AlignByGeometry: 0
     m_RichText: 1
     m_HorizontalOverflow: 1
@@ -11162,7 +11181,7 @@ RectTransform:
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0, y: 1}
   m_AnchorMax: {x: 1, y: 1}
-  m_AnchoredPosition: {x: 0.000030517578, y: -246.2022}
+  m_AnchoredPosition: {x: 0.000030517578, y: -246.20177}
   m_SizeDelta: {x: 0, y: 0}
   m_Pivot: {x: 0, y: 0}
 --- !u!114 &1442113294
@@ -12155,6 +12174,8 @@ MonoBehaviour:
   spinRecordMax: 40
   spinPenaltyThreshold: 10
   facingInviewEnemyDisCOEF: 0.5
+  group1Tag: Player
+  group2Tag: Enemy
   hitTargetRewardDefault: 25
   killTargetEnemyRewardDefault: 25
   inAreaRewardDefault: 12
@@ -12921,17 +12942,13 @@ MonoBehaviour:
   HUDObj: {fileID: 2082200184}
   sceneBlockContainerObj: {fileID: 1811162388}
   enemyContainerObj: {fileID: 891195334}
-  parameterContainerObj: {fileID: 707336050}
   commonParameterContainerObj: {fileID: 1580123439}
   environmentUIObj: {fileID: 156390936}
-  worldUIObj: {fileID: 1052251813}
   edgeUp: {fileID: 1352953080}
   edgeDown: {fileID: 1571061029}
   edgeLeft: {fileID: 1551793502}
   edgeRight: {fileID: 1046425531}
   edgeAgent_Enemy: {fileID: 704984377}
-  group1Tag: Player
-  group2Tag: Enemy
   minEnemyAreaX: 0
   attackProb: 0.375
   gotoProb: 0.392
@@ -12943,7 +12960,9 @@ MonoBehaviour:
   - 0
   - 0
   - 0
+  inArea: 0
   targetPosition: {x: 0, y: 0, z: 0}
+  targetEnemySpawnFinish: 0
 --- !u!1 &1695161701
 GameObject:
   m_ObjectHideFlags: 0
@@ -13140,11 +13159,11 @@ RectTransform:
   m_Father: {fileID: 440411430}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
-  m_AnchorMin: {x: 0, y: 1}
+  m_AnchorMin: {x: 1, y: 0.5}
-  m_AnchorMax: {x: 0, y: 1}
+  m_AnchorMax: {x: 1, y: 0.5}
-  m_AnchoredPosition: {x: 2, y: -0}
+  m_AnchoredPosition: {x: -2, y: 0}
   m_SizeDelta: {x: 0, y: 0}
-  m_Pivot: {x: 0, y: 1}
+  m_Pivot: {x: 1, y: 0.5}
 --- !u!114 &1714055741
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -13172,7 +13191,7 @@ MonoBehaviour:
     m_BestFit: 0
     m_MinSize: 10
     m_MaxSize: 40
-    m_Alignment: 0
+    m_Alignment: 5
     m_AlignByGeometry: 0
     m_RichText: 1
     m_HorizontalOverflow: 1
@@ -16584,7 +16603,7 @@ MonoBehaviour:
   m_HandleRect: {fileID: 1766662801}
   m_Direction: 2
   m_Value: 0
-  m_Size: 1
+  m_Size: 0.9999902
   m_NumberOfSteps: 0
   m_OnValueChanged:
     m_PersistentCalls:

Assets/Script/GameScript/Character/AgentController.cs

@@ -5,22 +5,8 @@
 
 public class AgentController : MonoBehaviour
 {
-    [SerializeField]
+    [SerializeField] public GameObject commonParameterContainerObj;
-    private GameObject commonParameterContainerObj;
+    [SerializeField] public Camera fpsCam;
-    [SerializeField]
-    private GameObject environmentObj;
-    [SerializeField]
-    private GameObject enemyContainerObj;
-    [SerializeField]
-    private GameObject sceneBlockContainerObj;
-    [SerializeField]
-    private GameObject environmentUIControlObj;
-    [SerializeField]
-    private GameObject targetControllerObj;
-    [SerializeField]
-    private GameObject HUDObj;
-    [SerializeField]
-    private Camera fpsCam;
 
     [Header("GetAxis() Simulate")]
     public float moveSpeed = 9.0f;
@@ -33,45 +19,33 @@ public class AgentController : MonoBehaviour
     public float mouseYSensitivity = 200;
     public float yRotation = 0.1f;//定义一个浮点类型的量，记录‘围绕’X轴旋转的角度
 
-    private List<float> spinRecord = new List<float>();
     private bool lockMouse;
-    private float damage;
     private float fireRate;
     private bool lockCameraX;
     private bool lockCameraY;
 
     // environment
-    private float lastShootTime = 0.0f;
+    public float lastShootTime = 0.0f;
 
-    private int enemyKillCount = 0;
+    public int enemyKillCount = 0;
-    private Vector3 killEnemyPosition;
+    public Vector3 killEnemyPosition;
     public bool defaultTPCamera = true;
 
     [System.NonSerialized] public bool gunReadyToggle = true;
-    private string myTag = "";
+    public string myTag = "";
-    private float lastEnemyFacingDistance = 0f; // record last enemy facing minimum distance
-    private float lastTargetFacingDistance = 0f; // record last target facing minimum distance
 
-    // scripts
-    private RaySensors raySensors;
 
     private CharacterController playerController;
     private CommonParameterContainer commonPramCon;
-    private SceneBlockContainer blockContainer;
-    private TargetController targetCon;
 
     private void Start()
     {
         // initialize scripts
         commonPramCon = commonParameterContainerObj.GetComponent<CommonParameterContainer>();
-        blockContainer = sceneBlockContainerObj.GetComponent<SceneBlockContainer>();
+        playerController = transform.GetComponent<CharacterController>();
-        targetCon = targetControllerObj.GetComponent<TargetController>();
-        raySensors = GetComponent<RaySensors>();
-        playerController = this.transform.GetComponent<CharacterController>();
 
         // initialize Environment parameters
         lockMouse = commonPramCon.lockMouse;
-        damage = commonPramCon.damage;
         fireRate = commonPramCon.fireRate;
         lockCameraX = commonPramCon.lockCameraX;
         lockCameraY = commonPramCon.lockCameraY;
@@ -199,234 +173,7 @@ public void CameraControl(float Mouse_X, float Mouse_Y)
 
     #endregion Camera Control
 
-    #region Reward Functions
+    // Got Kill point
-
-    // ballistic 射击弹道处理，并返回获得reward
-    private float Ballistic(int shootState)
-    {
-        Vector3 point = new Vector3(fpsCam.pixelWidth / 2, fpsCam.pixelHeight / 2, 0);//发射位置
-        Ray ray = fpsCam.ScreenPointToRay(point);
-        RaycastHit hit;
-        // Debug.DrawRay(centerRay.origin, centerRay.direction * 100, Color.blue);
-        //按下鼠标左键
-        if (shootState != 0 && gunReadyToggle == true)
-        {
-            lastShootTime = Time.time;
-            if (Physics.Raycast(ray, out hit, 100))
-            {
-                if (hit.collider.tag != myTag && hit.collider.tag != "Wall" && hit.collider.tag != "Untagged")
-                {
-                    // kill enemy
-                    GameObject gotHitObj = hit.transform.gameObject;//获取受到Ray撞击的对象
-                    gotHitObj.GetComponent<States>().ReactToHit(damage, gameObject);
-                    shootState = 0;
-                    return targetCon.HitEnemyReward(gotHitObj.transform.position);
-                }
-            }
-            if (targetCon.targetTypeInt == (int)Targets.Attack)
-            {
-                // while if attack mode
-                float targetDis = Vector3.Distance(blockContainer.nowBlock.transform.position, transform.position);
-                if (targetDis <= raySensors.viewDistance)
-                {
-                    // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
-                    if (Vector3.Distance(ray.origin + (ray.direction * targetDis), blockContainer.nowBlock.transform.position) <= blockContainer.nowBlock.firebasesAreaDiameter / 2)
-                    {
-                        // im shooting at target but didn't hit enemy
-                        // Debug.DrawRay(centerRay.origin, viewPoint-centerRay.origin, Color.blue);
-                        return commonPramCon.shootTargetAreaReward;
-                    }
-                }
-            }
-            shootState = 0;
-            return commonPramCon.shootReward;
-        }
-        else if (shootState != 0 && gunReadyToggle == false)
-        {
-            // shoot without ready
-            shootState = 0;
-            return commonPramCon.shootWithoutReadyReward;
-        }
-        else
-        {
-            // do not shoot
-            shootState = 0;
-            return commonPramCon.nonReward;
-        }
-    }
-
-    private float FacingReward()
-    {
-        float nowReward = 0;
-        bool isFacingtoEnemy = false;
-        float enemyFacingDistance = 0f;
-
-        Vector3 screenCenter = new Vector3(fpsCam.pixelWidth / 2, fpsCam.pixelHeight / 2, 0);
-        Vector3 screenLeft = new Vector3(0, fpsCam.pixelHeight / 2, 0);
-
-        Ray centerRay = fpsCam.ScreenPointToRay(screenCenter);
-        Ray leftRay = fpsCam.ScreenPointToRay(screenLeft);
-
-        // target fireBaseArea Position, turen y to camera's y
-        Vector3 fireBaseArea = blockContainer.nowBlock.fireBasesAreaObj.transform.position;
-        fireBaseArea.y = fpsCam.transform.position.y;
-
-        // my position, turn y to camera's y
-        // Debug.DrawRay(fpsCam.transform.position, centerRay.direction * 100, Color.blue);
-        Vector3 myposition = transform.position;
-        myposition.y = fpsCam.transform.position.y;
-
-        // Target to Agent distance
-        //Debug.DrawLine(fireBaseArea, myposition, Color.red);
-        float targetDis = Vector3.Distance(fireBaseArea, myposition);
-
-        // point in centerRay and leftRay which distance is targetDis from camera center
-        Vector3 pointInCenterRay = fpsCam.transform.position + (centerRay.direction * targetDis);
-        Vector3 pointInLeftRay = fpsCam.transform.position + (leftRay.direction * targetDis);
-
-        // center of screen to target's distance
-        // Debug.DrawLine(pointInCenterRay, fireBaseArea,Color.green);
-        float camCenterToTarget = Vector3.Distance(pointInCenterRay, fireBaseArea);
-
-        // left of screen to target's distance
-        // Debug.DrawLine(pointInLeftRay, pointInCenterRay, Color.yellow);
-        float camCenterToViewEdge = Vector3.Distance(pointInLeftRay, pointInCenterRay);
-
-        switch (targetCon.targetTypeInt)
-        {
-            case (int)Targets.Free:
-                //free mode
-                RaycastHit hit;
-                if (Physics.Raycast(centerRay, out hit, 100))
-                {
-                    // facing to an enemy
-                    if (hit.collider.tag != myTag && hit.collider.tag != "Wall")
-                    {
-                        nowReward = commonPramCon.facingReward;
-                        isFacingtoEnemy = true;
-                    }
-                }
-                if (raySensors.inViewEnemies.Count > 0 && !isFacingtoEnemy)
-                {
-                    // have enemy in view
-                    List<float> projectionDis = new List<float>();
-                    foreach (GameObject theEnemy in raySensors.inViewEnemies)
-                    {
-                        // for each enemy in view
-                        Vector3 projection = Vector3.Project(theEnemy.transform.position - transform.position, (centerRay.direction * 10));
-                        Vector3 verticalToRay = transform.position + projection - theEnemy.transform.position;
-                        projectionDis.Add(verticalToRay.magnitude);
-                        // Debug.Log("enemy!" + verticalToRay.magnitude);
-                        // Debug.DrawRay(transform.position, (centerRay.direction * 100), Color.cyan);
-                        // Debug.DrawRay(transform.position, theEnemy.transform.position - transform.position, Color.yellow);
-                        // Debug.DrawRay(transform.position, projection, Color.blue);
-                        // Debug.DrawRay(theEnemy.transform.position, verticalToRay, Color.magenta);
-                    }
-                    enemyFacingDistance = projectionDis.Min();
-                    if (enemyFacingDistance <= lastEnemyFacingDistance)
-                    {
-                        // closing to enemy
-                        nowReward = 1 / MathF.Sqrt(commonPramCon.facingInviewEnemyDisCOEF * enemyFacingDistance + 0.00001f);
-                    }
-                    else
-                    {
-                        nowReward = 0;
-                    }
-                    // enemy in view Reward
-                    lastEnemyFacingDistance = enemyFacingDistance;
-                    if (nowReward >= commonPramCon.facingReward) nowReward = commonPramCon.facingReward; // limit
-                    if (nowReward <= -commonPramCon.facingReward) nowReward = -commonPramCon.facingReward; // limit
-                    // Debug.Log("ninimum = " + nowReward);
-                }
-                break;
-            case (int)Targets.Attack:
-                // attack mode
-                if (targetDis <= raySensors.viewDistance)
-                {
-                    // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
-                    // while center of screen between target's distance is lower than firebasesAreaDiameter
-                    // while facing to target
-                    if (camCenterToTarget <= blockContainer.nowBlock.firebasesAreaDiameter / 2)
-                    {
-                        // Debug.DrawRay(centerRay.origin, viewPoint-centerRay.origin, Color.blue);
-                        nowReward = commonPramCon.facingReward;
-                    }
-                    else
-                    {
-                        // while not facing to target
-                        nowReward = (lastTargetFacingDistance - camCenterToTarget) * commonPramCon.facingTargetReward;
-                    }
-                }
-                // update lastTargetFacingDistance
-                lastTargetFacingDistance = camCenterToTarget;
-                break;
-            case (int)Targets.Go:
-                // goto mode
-                if (camCenterToTarget <= camCenterToViewEdge)
-                {
-                    // fireArea is in view
-                    nowReward = commonPramCon.facingReward;
-                }
-                else
-                {
-                    nowReward = 0;
-                }
-                break;
-            default:
-                Debug.LogError("Wrong target type");
-                break;
-        }
-        return nowReward;
-    }
-
-    public float RewardCalculate(float sceneReward, float mouseX, float movement, int shootState)
-    {
-        float epreward = 0f;
-        // 击杀reward判断
-        if (enemyKillCount > 0)
-        {
-            for (int i = 0; i < enemyKillCount; i++)
-            {
-                // get
-                epreward += targetCon.KillReward(killEnemyPosition);
-            }
-            enemyKillCount = 0;
-        }
-        else
-        {
-            enemyKillCount = 0;
-        }
-        // 射击动作reward判断
-        epreward += Ballistic(shootState) + sceneReward;
-        // facing reward
-        epreward += FacingReward();
-        // Penalty
-        // spin penalty
-        spinRecord.Add(mouseX);
-        if (spinRecord.Count >= commonPramCon.spinRecordMax)
-        {
-            spinRecord.RemoveAt(0);
-        }
-        float spinPenaltyReward = Math.Abs(spinRecord.ToArray().Sum() * commonPramCon.spinPenalty);
-        if (spinPenaltyReward >= commonPramCon.spinPenaltyThreshold)
-        {
-            epreward -= spinPenaltyReward;
-        }
-        else
-        {
-            epreward -= Math.Abs(mouseX) * commonPramCon.mousePenalty;
-        }
-        // move penalty
-        if (movement != 0)
-        {
-            epreward -= commonPramCon.movePenalty;
-        }
-        return epreward;
-    }
-
-    #endregion Reward Functions
-
-    // GotKill 获得击杀时用于被呼出
     public void KillRecord(Vector3 killEnemyPosition)
     {
         enemyKillCount += 1;

Assets/Script/GameScript/CommonParameterContainer.cs

@@ -11,7 +11,6 @@ public class CommonParameterContainer : MonoBehaviour
 
     [Header("Env")]
     public bool lockMouse = false;
-
     public float damage = 50; // damage to enemy
     public float fireRate = 0.5f;
     public int timeLimit = 30;
@@ -21,6 +20,8 @@ public class CommonParameterContainer : MonoBehaviour
     public int spinRecordMax = 40;
     public float spinPenaltyThreshold = 10;
     public float facingInviewEnemyDisCOEF = 0.5f;
+    public string group1Tag = "Player";
+    public string group2Tag = "Enemy";
 
     [Header("Dynamic Defaut Rewards")]
     //[Tooltip("Hit Enemy reward")]

Assets/Script/GameScript/MLAgentsCustomController.cs

@@ -8,11 +8,12 @@
 public class MLAgentsCustomController : Agent
 {
     [SerializeField] private GameObject paramContainerObj;
-    [SerializeField] private GameObject CommonParameterContainer;
+    [SerializeField] private GameObject commonParameterContainer;
     [SerializeField] private GameObject targetControllerObj;
     [SerializeField] private GameObject environmentUIObj;
     [SerializeField] private GameObject sideChannelObj;
-    [SerializeField] private GameObject hudUIObj;
+    [SerializeField] private GameObject worldUIControllerObj;
+    [SerializeField] private GameObject hudObj;
 
     [Header("Env")]
     public bool oneHotRayTag = true;
@@ -29,6 +30,8 @@ public class MLAgentsCustomController : Agent
     private RaySensors raySensors;
     private MessageBoxController messageBoxController;
     private AimBotSideChannelController sideChannelController;
+    private WorldUIController worldUICon;
+    private RewardFunction rewardFunction;
 
     // observation
     private float[] myObserve = new float[4];
@@ -40,29 +43,26 @@ public class MLAgentsCustomController : Agent
     private float remainTime;
     private float inAreaState;
 
-    private int finishedState;
+    private int endTypeInt;
-    private int step = 0;
-    private int EP = 0;
 
     private void Start()
     {
         agentController = transform.GetComponent<AgentController>();
         raySensors = transform.GetComponent<RaySensors>();
         paramContainer = paramContainerObj.GetComponent<ParameterContainer>();
-        commonParamCon = CommonParameterContainer.GetComponent<CommonParameterContainer>();
+        commonParamCon = commonParameterContainer.GetComponent<CommonParameterContainer>();
         targetController = targetControllerObj.GetComponent<TargetController>();
         envUIController = environmentUIObj.GetComponent<EnvironmentUIControl>();
-        hudController = hudUIObj.GetComponent<HUDController>();
+        hudController = hudObj.GetComponent<HUDController>();
-        targetUIController = hudUIObj.GetComponent<TargetUIController>();
+        targetUIController = hudObj.GetComponent<TargetUIController>();
-        messageBoxController = hudUIObj.GetComponent<MessageBoxController>();
+        messageBoxController = worldUIControllerObj.GetComponent<MessageBoxController>();
         sideChannelController = sideChannelObj.GetComponent<AimBotSideChannelController>();
+        rewardFunction = gameObject.GetComponent<RewardFunction>();
+        worldUICon = worldUIControllerObj.GetComponent<WorldUIController>();
     }
 
-    #region On episode begin function
-
     public override void OnEpisodeBegin()
     {
-        step = 0;
         agentController.UpdateLockMouse();
         paramContainer.ResetTimeBonusReward();
         if (commonParamCon.gameMode == 0)
@@ -88,10 +88,6 @@ public override void OnEpisodeBegin()
         raySensors.UpdateRayInfo(); // update raycast
     }
 
-    #endregion On episode begin function
-
-    #region Observation sensor function
-
     public override void CollectObservations(VectorSensor sensor)
     {
         //List<float> enemyLDisList = RaySensors.enemyLDisList;// All Enemy Lside Distances
@@ -139,10 +135,6 @@ public override void CollectObservations(VectorSensor sensor)
         //sensor.AddObservation(remainTime); // RemainTime int
     }
 
-    #endregion Observation sensor function
-
-    #region Action received function
-
     public override void OnActionReceived(ActionBuffers actionBuffers)
     {
         //获取输入
@@ -161,24 +153,24 @@ public override void OnActionReceived(ActionBuffers actionBuffers)
         //判断结束
         float sceneReward = 0f;
         float endReward = 0f;
-        (finishedState, sceneReward, endReward) = targetController.CheckOverAndRewards();
+        (endTypeInt, sceneReward, endReward) = rewardFunction.CheckOverAndRewards();
-        float nowRoundReward = agentController.RewardCalculate(sceneReward + endReward, Mouse_X, Math.Abs(vertical) + Math.Abs(horizontal), mouseShoot);
+        float nowReward = rewardFunction.RewardCalculate(sceneReward + endReward, Mouse_X, Math.Abs(vertical) + Math.Abs(horizontal), mouseShoot);
         if (hudController.chartOn)
         {
-            envUIController.UpdateChart(nowRoundReward);
+            envUIController.UpdateChart(nowReward);
         }
         else
         {
             envUIController.RemoveChart();
         }
-        //Debug.Log("reward = " + nowRoundReward);
+        worldUICon.UpdateChart(targetController.targetTypeInt, endTypeInt);
-        if (finishedState != (int)TargetController.EndType.Running)
+        //Debug.Log("reward = " + nowReward);
+        if (endTypeInt != (int)TargetController.EndType.Running)
         {
             // Win or lose Finished
-            Debug.Log("Finish reward = " + nowRoundReward);
+            Debug.Log("Finish reward = " + nowReward);
-            EP += 1;
             string targetString = Enum.GetName(typeof(Targets), targetController.targetTypeInt);
-            switch (finishedState)
+            switch (endTypeInt)
             {
                 case (int)TargetController.EndType.Win:
                     sideChannelController.SendSideChannelMessage("Result", targetString + "|Win");
@@ -198,21 +190,16 @@ public override void OnActionReceived(ActionBuffers actionBuffers)
                     Debug.LogWarning("TypeError");
                     break;
             }
-            SetReward(nowRoundReward);
+            SetReward(nowReward);
             EndEpisode();
         }
         else
         {
             // game not over yet
-            step += 1;
         }
-        SetReward(nowRoundReward);
+        SetReward(nowReward);
     }
 
-    #endregion Action received function
-
-    #region Heuristic function
-
     public override void Heuristic(in ActionBuffers actionsOut)
     {
         //-------------------BUILD
@@ -270,6 +257,4 @@ public override void Heuristic(in ActionBuffers actionsOut)
         //continuousActions[2] = timeLimit;
         //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^continuous-Control^^^^^^^^^^^^^^^^^^^^^^
     }
-
-    #endregion Heuristic function
 }

Assets/Script/GameScript/TargetController.cs

@@ -5,24 +5,13 @@
 
 public class TargetController : MonoBehaviour
 {
-    [SerializeField]
+    [SerializeField] private GameObject environmentObj;
-    private GameObject environmentObj;
+    [SerializeField] private GameObject agentObj;
-    [SerializeField]
+    [SerializeField] private GameObject HUDObj;
-    private GameObject agentObj;
+    [SerializeField] private GameObject sceneBlockContainerObj;
-    [SerializeField]
+    [SerializeField] private GameObject enemyContainerObj;
-    private GameObject HUDObj;
+    [SerializeField] private GameObject commonParameterContainerObj;
-    [SerializeField]
+    [SerializeField] private GameObject environmentUIObj;
-    private GameObject sceneBlockContainerObj;
-    [SerializeField]
-    private GameObject enemyContainerObj;
-    [SerializeField]
-    private GameObject parameterContainerObj;
-    [SerializeField]
-    private GameObject commonParameterContainerObj;
-    [SerializeField]
-    private GameObject environmentUIObj;
-    [SerializeField]
-    private GameObject worldUIObj;
 
     // area
     public GameObject edgeUp;
@@ -32,11 +21,6 @@ public class TargetController : MonoBehaviour
     public GameObject edgeRight;
     public GameObject edgeAgent_Enemy;
 
-    //group
-    public string group1Tag = "Player";
-
-    public string group2Tag = "Enemy";
-
     public float minEnemyAreaX;
     [System.NonSerialized] public float maxEnemyAreaX;
     [System.NonSerialized] public float minEnemyAreaZ;
@@ -63,23 +47,20 @@ public enum EndType
     [System.NonSerialized] public int targetNum = 0;
     private Dictionary<int, float[]> oneHotRarget = new Dictionary<int, float[]>();
 
-    private int inArea = 0;
     private float freeProb;
     private float sceneBlockSize;
-    private float lastDistance;
     private int randBlockType = 0;
     private int randLevel = 0;
+    public int inArea = 0;
     public Vector3 targetPosition;
     private bool firstRewardFlag = true;
-    private bool targetEnemySpawnFinish = false;
+    public bool targetEnemySpawnFinish = false;
 
     private SceneBlockContainer sceneBlockCon;
     private EnemyContainer enemyCon;
     private EnvironmentUIControl envUICon;
-    private ParameterContainer paramCon;
     private CommonParameterContainer commonParamCon;
     private CharacterController agentCharaCon;
-    private WorldUIController worldUICon;
     private HUDController hudCon;
     private MessageBoxController messageBoxCon;
 
@@ -94,8 +75,6 @@ private void Start()
         envUICon = environmentUIObj.GetComponent<EnvironmentUIControl>();
         enemyCon = enemyContainerObj.GetComponent<EnemyContainer>();
         agentCharaCon = agentObj.GetComponent<CharacterController>();
-        paramCon = parameterContainerObj.GetComponent<ParameterContainer>();
-        worldUICon = worldUIObj.GetComponent<WorldUIController>();
         hudCon = HUDObj.GetComponent<HUDController>();
         messageBoxCon = HUDObj.GetComponent<MessageBoxController>();
 
@@ -279,7 +258,7 @@ private void RandomSpawnSceneBlock(Targets targetType)
 
         // init scene block
         sceneBlockCon.DestroyBlock();
-        sceneBlockCon.CreateNewBlock(targetType, randLevel, randBlockType, targetPosition, group1Tag, group2Tag);
+        sceneBlockCon.CreateNewBlock(targetType, randLevel, randBlockType, targetPosition, commonParamCon.group1Tag, commonParamCon.group2Tag);
         enemyCon.DestroyAllEnemys();
         enemyCon.RandomInitEnemysExcept(hudCon.enemyNum, targetPosition, sceneBlockSize);
         sceneBlockCon.nowBlock.InitBlock(environmentObj);
@@ -287,260 +266,6 @@ private void RandomSpawnSceneBlock(Targets targetType)
 
     #endregion Random SceneBlock Spawn Method
 
-    #region Reward function
-
-    /// <summary>
-    /// Checks the game's end state and retrieves rewards.
-    /// </summary>
-    /// <returns>A tuple containing the game's end type, current reward, and final reward.
-    /// 1 = success,2 = overtime,0 = notover</returns>
-    public (int, float, float) CheckOverAndRewards()
-    {
-        int endTypeInt = 0;
-        float nowReward = 0;
-        float endReward = 0;
-        float nowDistance = 0f;
-        switch (targetTypeInt)
-        {
-            case (int)Targets.Go:
-                // goto
-                (nowDistance, inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
-                envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
-                float areaTargetReward = GetDistanceReward(nowDistance, inArea);
-                //if(inArea != 0)
-                if (sceneBlockCon.nowBlock.firebasesBelong >= sceneBlockCon.nowBlock.belongMaxPoint)
-                {
-                    // win
-                    // let the area belongs to me
-                    nowReward = areaTargetReward;
-                    endReward = paramCon.goWinReward;
-                    //nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
-                    endTypeInt = (int)EndType.Win;
-                }
-                else if (leftTime <= 0)
-                {
-                    // time out lose
-                    nowReward = areaTargetReward;
-                    endReward = commonParamCon.loseReward;
-                    endTypeInt = (int)EndType.Lose;
-                }
-                else
-                {
-                    // keep on keeping on!
-                    nowReward = areaTargetReward;
-                    endReward = 0;
-                    endTypeInt = (int)EndType.Running;
-                }
-                break;
-
-            case (int)Targets.Attack:
-                // attack
-                (nowDistance, inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
-                envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
-                if (sceneBlockCon.nowBlock.GetInAreaNumber(group2Tag) <= 0 && targetEnemySpawnFinish)
-                {
-                    // win
-                    // let the area belongs to me and kill every enmy in this area.
-                    nowReward = 0;
-                    endReward = paramCon.attackWinReward;
-                    //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
-                    endTypeInt = (int)EndType.Win;
-                    targetEnemySpawnFinish = false;
-                }
-                else if (leftTime <= 0 && targetEnemySpawnFinish)
-                {
-                    // time out lose
-                    nowReward = 0;
-                    endReward = commonParamCon.loseReward;
-                    //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
-                    endTypeInt = (int)EndType.Lose;
-                    targetEnemySpawnFinish = false;
-                }
-                else
-                {
-                    // keep on keeping on!
-                    // nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
-                    nowReward = 0;
-                    endReward = 0;
-                    targetEnemySpawnFinish = true;
-                    endTypeInt = (int)EndType.Running;
-                }
-                break;
-
-            case (int)Targets.Defence:
-                //defence
-                // !!! DIDN't FINISH!!!
-                (nowDistance, inArea) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
-                envUICon.UpdateTargetGauge(sceneBlockCon.nowBlock.firebasesBelong, sceneBlockCon.nowBlock.belongMaxPoint);
-                if (leftTime <= 0 && sceneBlockCon.nowBlock.firebasesBelong >= 0f)
-                {
-                    // win
-                    // time over and the area still mine
-                    nowReward = paramCon.defenceWinReward;
-                    //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
-                    endTypeInt = (int)EndType.Win;
-                }
-                else if (sceneBlockCon.nowBlock.firebasesBelong <= sceneBlockCon.nowBlock.belongMaxPoint)
-                {
-                    // lost area lose
-                    nowReward = commonParamCon.loseReward;
-                    //nowReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
-                    endTypeInt = (int)EndType.Lose;
-                }
-                else
-                {
-                    // keep on keeping on!
-                    // nowReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
-                    endTypeInt = (int)EndType.Running;
-                }
-                break;
-
-            case (int)Targets.Stay:
-                // Stay
-                // endless
-                nowReward = 0;
-                endReward = 0;
-                endTypeInt = (int)EndType.Running;
-                break;
-
-            default:
-                //free kill
-                if (enemyContainerObj.transform.childCount <= 0)
-                {
-                    // win
-                    // nowReward = paramCon.winReward + (paramCon.timeBonusPerSecReward * leftTime);
-                    nowReward = 0;
-                    endReward = paramCon.freeWinReward;
-                    endTypeInt = (int)EndType.Win;
-                }
-                else if (leftTime <= 0)
-                {
-                    // lose
-                    //nowReward = paramCon.loseReward;
-                    nowReward = 0;
-                    endReward = commonParamCon.loseReward;
-                    endTypeInt = (int)EndType.Lose;
-                }
-                else
-                {
-                    // keep on keeping on!
-                    nowReward = 0;
-                    endReward = 0;
-                    endTypeInt = (int)EndType.Running;
-                }
-                break;
-        }
-        envUICon.ShowResult(endTypeInt);
-        worldUICon.UpdateChart(targetTypeInt, endTypeInt);
-        return (endTypeInt, nowReward, endReward);
-    }
-
-    /// <summary>
-    /// Calculates scene reward based on distance, granting higher rewards for being closer to the target.
-    /// 根据距离计算场景奖励，靠近目标则获得更高奖励。
-    /// </summary>
-    /// <param name="nowDistance">The current distance.</param>
-    /// <param name="inarea">Whether inside an area.</param>
-    /// <returns>The reward value calculated based on distance.</returns>
-    private float GetDistanceReward(float nowDistance, int inarea)
-    {
-        if (firstRewardFlag)
-        {
-            // first distance record
-            (lastDistance, _) = sceneBlockCon.GetAgentTargetDistanceAndInside(agentObj.transform.position);
-            firstRewardFlag = false;
-        }
-        float nowSeneReward = 0f;
-        if (inarea != 0)
-        {
-            // in area
-            nowSeneReward = paramCon.inAreaReward;
-        }
-        else
-        {
-            // out of area
-            // nowSeneReward = paramCon.distanceReward * Math.Clamp(lastDistance - nowDistance, 0, 100);
-            nowSeneReward = commonParamCon.distanceReward * (lastDistance - nowDistance);
-        }
-        lastDistance = nowDistance;
-        return nowSeneReward;
-    }
-
-    /// <summary>
-    /// Calculates kill reward based on the position of the killed enemy.
-    /// 根据击杀的敌人位置计算击杀奖励。
-    /// </summary>
-    /// <param name="enemyPosition">The position of the killed enemy.</param>
-    /// <returns>The reward value calculated based on the kill position.</returns>
-    public float KillReward(Vector3 enemyPosition)
-    {
-        float nowKillReward = 0f;
-        if (targetTypeInt == (int)Targets.Attack)
-        {
-            // attack mode
-            (_, int isInArea) = sceneBlockCon.nowBlock.GetDistInArea(enemyPosition);
-            if (isInArea == 1)
-            {
-                // kill in area enemy
-                nowKillReward = paramCon.killTargetEnemyReward;
-            }
-            else
-            {
-                nowKillReward = commonParamCon.killNonTargetReward;
-            }
-        }
-        else if (targetTypeInt == (int)Targets.Free)
-        {
-            // free mode hit
-            nowKillReward = paramCon.killTargetEnemyReward;
-        }
-        else
-        {
-            // goto & defence
-            nowKillReward = commonParamCon.killNonTargetReward;
-        }
-        return nowKillReward;
-    }
-
-    /// <summary>
-    /// Calculates hit reward based on the position of the hit enemy and the current mode.
-    /// 根据击中的敌人位置和当前模式计算击中Reward。
-    /// </summary>
-    /// <param name="enemyPosition">The position of the hit enemy.</param>
-    /// <returns>The reward value calculated based on the hit position and mode.</returns>
-    public float HitEnemyReward(Vector3 enemyPosition)
-    {
-        float nowHitReward = 0f;
-        if (targetTypeInt == (int)Targets.Attack)
-        {
-            // attack mode
-            (_, int isInArea) = sceneBlockCon.nowBlock.GetDistInArea(enemyPosition);
-            if (isInArea == 1)
-            {
-                // hit in area enemy
-                nowHitReward = paramCon.hitTargetReward;
-            }
-            else
-            {
-                // hit not in area enemy
-                nowHitReward = commonParamCon.hitNonTargetReward;
-            }
-        }
-        else if (targetTypeInt == (int)Targets.Free)
-        {
-            // free mode hit
-            nowHitReward = paramCon.hitTargetReward;
-        }
-        else
-        {
-            // goto & defence
-            nowHitReward = commonParamCon.hitNonTargetReward;
-        }
-        return nowHitReward;
-    }
-
-    #endregion Reward function
-
     #region Play Mode Method
 
     /// <summary>

UserSettings/EditorUserSettings.asset

@@ -9,14 +9,14 @@ EditorUserSettings:
       value: 55090405535008090b5608764626074415151c79747c74602b7c1861e4b96c6c
       flags: 0
     RecentlyUsedSceneGuid-1:
-      value: 06570c0704040b0e5a575520137b5e44154f192e292d22367c2c4866b7b3376f
-      flags: 0
-    RecentlyUsedSceneGuid-2:
       value: 5703025650035e5d090f087148715d4443161e2c7e2c20357b281b31b0e43060
       flags: 0
-    RecentlyUsedSceneGuid-3:
+    RecentlyUsedSceneGuid-2:
       value: 5453565f53020f085f5d0e72157a0844454f4c2b757d7265757a4f64b7b4313c
       flags: 0
+    RecentlyUsedSceneGuid-3:
+      value: 06570c0704040b0e5a575520137b5e44154f192e292d22367c2c4866b7b3376f
+      flags: 0
     RecentlyUsedScenePath-0:
       value: 22424703114646680e0b0227036c6c111b07142f1f2b233e2867083debf42d
       flags: 0

UserSettings/Layouts/default-2021.dwlt

@@ -14,16 +14,16 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   m_PixelRect:
     serializedVersion: 2
-    x: 65
+    x: -141
-    y: 43
+    y: 132
-    width: 1855
+    width: 1792
-    height: 1037
+    height: 1021
   m_ShowMode: 4
   m_Title: Inspector
   m_RootView: {fileID: 2}
   m_MinSize: {x: 875, y: 300}
   m_MaxSize: {x: 10000, y: 10000}
-  m_Maximized: 1
+  m_Maximized: 0
 --- !u!114 &2
 MonoBehaviour:
   m_ObjectHideFlags: 52
@@ -44,8 +44,8 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 0
-    width: 1855
+    width: 1792
-    height: 1037
+    height: 1021
   m_MinSize: {x: 875, y: 300}
   m_MaxSize: {x: 10000, y: 10000}
   m_UseTopView: 1
@@ -69,7 +69,7 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 0
-    width: 1855
+    width: 1792
     height: 30
   m_MinSize: {x: 0, y: 0}
   m_MaxSize: {x: 0, y: 0}
@@ -90,8 +90,8 @@ MonoBehaviour:
   m_Position:
     serializedVersion: 2
     x: 0
-    y: 1017
+    y: 1001
-    width: 1855
+    width: 1792
     height: 20
   m_MinSize: {x: 0, y: 0}
   m_MaxSize: {x: 0, y: 0}
@@ -115,12 +115,12 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 30
-    width: 1855
+    width: 1792
-    height: 987
+    height: 971
   m_MinSize: {x: 300, y: 200}
   m_MaxSize: {x: 24288, y: 16192}
   vertical: 0
-  controlID: 65
+  controlID: 52
 --- !u!114 &6
 MonoBehaviour:
   m_ObjectHideFlags: 52
@@ -140,12 +140,12 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 0
-    width: 991
+    width: 957
-    height: 987
+    height: 971
   m_MinSize: {x: 100, y: 200}
   m_MaxSize: {x: 8096, y: 16192}
   vertical: 1
-  controlID: 22
+  controlID: 128
 --- !u!114 &7
 MonoBehaviour:
   m_ObjectHideFlags: 52
@@ -163,8 +163,8 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 0
-    width: 991
+    width: 957
-    height: 707
+    height: 696
   m_MinSize: {x: 201, y: 221}
   m_MaxSize: {x: 4001, y: 4021}
   m_ActualView: {fileID: 13}
@@ -189,9 +189,9 @@ MonoBehaviour:
   m_Position:
     serializedVersion: 2
     x: 0
-    y: 707
+    y: 696
-    width: 991
+    width: 957
-    height: 280
+    height: 275
   m_MinSize: {x: 231, y: 271}
   m_MaxSize: {x: 10001, y: 10021}
   m_ActualView: {fileID: 15}
@@ -217,14 +217,14 @@ MonoBehaviour:
   - {fileID: 11}
   m_Position:
     serializedVersion: 2
-    x: 991
+    x: 957
     y: 0
-    width: 513
+    width: 496
-    height: 987
+    height: 971
   m_MinSize: {x: 100, y: 200}
   m_MaxSize: {x: 8096, y: 16192}
   vertical: 1
-  controlID: 66
+  controlID: 103
 --- !u!114 &10
 MonoBehaviour:
   m_ObjectHideFlags: 52
@@ -242,10 +242,10 @@ MonoBehaviour:
     serializedVersion: 2
     x: 0
     y: 0
-    width: 513
+    width: 496
-    height: 486
+    height: 547
-  m_MinSize: {x: 200, y: 200}
+  m_MinSize: {x: 202, y: 221}
-  m_MaxSize: {x: 4000, y: 4000}
+  m_MaxSize: {x: 4002, y: 4021}
   m_ActualView: {fileID: 17}
   m_Panes:
   - {fileID: 17}
@@ -267,9 +267,9 @@ MonoBehaviour:
   m_Position:
     serializedVersion: 2
     x: 0
-    y: 486
+    y: 547
-    width: 513
+    width: 496
-    height: 501
+    height: 424
   m_MinSize: {x: 102, y: 121}
   m_MaxSize: {x: 4002, y: 4021}
   m_ActualView: {fileID: 18}
@@ -293,10 +293,10 @@ MonoBehaviour:
   m_Children: []
   m_Position:
     serializedVersion: 2
-    x: 1504
+    x: 1453
     y: 0
-    width: 351
+    width: 339
-    height: 987
+    height: 971
   m_MinSize: {x: 276, y: 71}
   m_MaxSize: {x: 4001, y: 4021}
   m_ActualView: {fileID: 20}
@@ -324,10 +324,10 @@ MonoBehaviour:
     m_Tooltip: 
   m_Pos:
     serializedVersion: 2
-    x: 65
+    x: -141
-    y: 73
+    y: 162
-    width: 990
+    width: 956
-    height: 686
+    height: 675
   m_ViewDataDictionary: {fileID: 0}
   m_OverlayCanvas:
     m_LastAppliedPresetName: Default
@@ -562,9 +562,9 @@ MonoBehaviour:
   m_PlayAudio: 0
   m_AudioPlay: 0
   m_Position:
-    m_Target: {x: 55.10534, y: 48.328194, z: -133.0123}
+    m_Target: {x: 530.726, y: 365.746, z: -133.89053}
     speed: 2
-    m_Value: {x: 55.10534, y: 48.328194, z: -133.0123}
+    m_Value: {x: 530.726, y: 365.746, z: -133.89053}
   m_RenderMode: 0
   m_CameraMode:
     drawMode: 0
@@ -615,9 +615,9 @@ MonoBehaviour:
     speed: 2
     m_Value: {x: 0, y: 0, z: 0, w: 1}
   m_Size:
-    m_Target: 330.06903
+    m_Target: 473.89896
     speed: 2
-    m_Value: 330.06903
+    m_Value: 473.89896
   m_Ortho:
     m_Target: 1
     speed: 2
@@ -754,10 +754,10 @@ MonoBehaviour:
     m_Tooltip: 
   m_Pos:
     serializedVersion: 2
-    x: 65
+    x: -141
-    y: 780
+    y: 858
-    width: 990
+    width: 956
-    height: 259
+    height: 254
   m_ViewDataDictionary: {fileID: 0}
   m_OverlayCanvas:
     m_LastAppliedPresetName: Default
@@ -775,36 +775,36 @@ MonoBehaviour:
     m_SkipHidden: 0
     m_SearchArea: 1
     m_Folders:
-    - Assets/Script/UI
+    - Assets/Script/GameScript
     m_Globs: []
     m_OriginalText: 
   m_ViewMode: 1
   m_StartGridSize: 16
   m_LastFolders:
-  - Assets/Script/UI
+  - Assets/Script/GameScript
   m_LastFoldersGridSize: 16
   m_LastProjectPath: C:\Users\UCUNI\OneDrive\Unity\ML-Agents\Aimbot-ParallelEnv
   m_LockTracker:
     m_IsLocked: 0
   m_FolderTreeState:
-    scrollPos: {x: 0, y: 593}
+    scrollPos: {x: 0, y: 502}
-    m_SelectedIDs: b0980000
+    m_SelectedIDs: ce880000
-    m_LastClickedID: 39088
+    m_LastClickedID: 35022
-    m_ExpandedIDs: 0000000000980000029800000498000006980000089800000a9800000c9800000e98000010980000129800001498000016980000189800001a9800001c9800001e9800002098000022980000249800007a98000000ca9a3bffffff7f
+    m_ExpandedIDs: 00000000b0880000b2880000b4880000b6880000b8880000ba880000bc880000be880000c0880000c2880000c4880000c6880000c8880000ca880000cc880000ce880000d0880000d2880000d4880000f888000000ca9a3bffffff7f
     m_RenameOverlay:
       m_UserAcceptedRename: 0
-      m_Name: 
+      m_Name: Prefeb
-      m_OriginalName: 
+      m_OriginalName: Prefeb
       m_EditFieldRect:
         serializedVersion: 2
         x: 0
         y: 0
         width: 0
         height: 0
-      m_UserData: 0
+      m_UserData: 1000000003
       m_IsWaitingForDelay: 0
       m_IsRenaming: 0
-      m_OriginalEventType: 11
+      m_OriginalEventType: 0
       m_IsRenamingFilename: 1
       m_ClientGUIView: {fileID: 8}
     m_SearchString: 
@@ -818,7 +818,7 @@ MonoBehaviour:
     scrollPos: {x: 0, y: 0}
     m_SelectedIDs: 
     m_LastClickedID: 0
-    m_ExpandedIDs: 0000000000980000029800000498000006980000089800000a9800000c9800000e98000010980000129800001498000016980000189800001a9800001c9800001e980000209800002298000024980000
+    m_ExpandedIDs: 00000000b0880000b2880000b4880000b6880000b8880000ba880000bc880000be880000c0880000c2880000c4880000c6880000c8880000ca880000cc880000ce880000d0880000d2880000d4880000
     m_RenameOverlay:
       m_UserAcceptedRename: 0
       m_Name: 
@@ -849,18 +849,18 @@ MonoBehaviour:
     m_ExpandedInstanceIDs: c62300008a5c000000000000
     m_RenameOverlay:
       m_UserAcceptedRename: 0
-      m_Name: RayInfoUI
+      m_Name: 
-      m_OriginalName: RayInfoUI
+      m_OriginalName: 
       m_EditFieldRect:
         serializedVersion: 2
         x: 0
         y: 0
         width: 0
         height: 0
-      m_UserData: 4132
+      m_UserData: 0
       m_IsWaitingForDelay: 0
       m_IsRenaming: 0
-      m_OriginalEventType: 0
+      m_OriginalEventType: 11
       m_IsRenamingFilename: 1
       m_ClientGUIView: {fileID: 8}
     m_CreateAssetUtility:
@@ -922,20 +922,20 @@ MonoBehaviour:
     m_Tooltip: 
   m_Pos:
     serializedVersion: 2
-    x: 1056
+    x: 816
-    y: 73
+    y: 162
-    width: 511
+    width: 494
-    height: 465
+    height: 526
   m_ViewDataDictionary: {fileID: 0}
   m_OverlayCanvas:
     m_LastAppliedPresetName: Default
     m_SaveData: []
   m_SceneHierarchy:
     m_TreeViewState:
-      scrollPos: {x: 0, y: 148}
+      scrollPos: {x: 0, y: 0}
-      m_SelectedIDs: a80c0100
+      m_SelectedIDs: dc920000
-      m_LastClickedID: 68776
+      m_LastClickedID: 37596
-      m_ExpandedIDs: 6e24ffff3425ffffec31ffffac42ffff5ee3ffffacf3ffff32fbffffb6990000009a000040a2000068090100ba090100840d0100
+      m_ExpandedIDs: 82f7ffff88f8ffff30fbffff38930000
       m_RenameOverlay:
         m_UserAcceptedRename: 0
         m_Name: 
@@ -979,10 +979,10 @@ MonoBehaviour:
     m_Tooltip: 
   m_Pos:
     serializedVersion: 2
-    x: 1056
+    x: 816
-    y: 559
+    y: 709
-    width: 511
+    width: 494
-    height: 480
+    height: 403
   m_ViewDataDictionary: {fileID: 0}
   m_OverlayCanvas:
     m_LastAppliedPresetName: Default
@@ -1035,10 +1035,10 @@ MonoBehaviour:
     m_Tooltip: 
   m_Pos:
     serializedVersion: 2
-    x: 1569
+    x: 1312
-    y: 73
+    y: 162
-    width: 350
+    width: 338
-    height: 966
+    height: 950
   m_ViewDataDictionary: {fileID: 0}
   m_OverlayCanvas:
     m_LastAppliedPresetName: Default
@@ -1050,7 +1050,7 @@ MonoBehaviour:
     m_ControlHash: -371814159
     m_PrefName: Preview_InspectorPreview
   m_LastInspectedObjectInstanceID: -1
-  m_LastVerticalScrollValue: 0
+  m_LastVerticalScrollValue: 918
   m_GlobalObjectId: 
   m_InspectorMode: 0
   m_LockTracker: