From 9911ce2de055b5bf0879ab09d90f1ac535c742c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20R=C3=B6themeyer?= <26631624+TheImus@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:00:25 +0100 Subject: [PATCH 01/25] Added first tools for curiculum learning --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 169 +++++++++++++----- .../Timers/SaveTraining_single_timers.json | 2 +- Assets/Prefabs/Ball.prefab | 2 +- Assets/Scripts/Ball/ShootBall.cs | 11 +- 4 files changed, 140 insertions(+), 44 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 73d53eb..069fa40 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -40,7 +40,7 @@ public class GoalKeeperAgent : Agent /// Shows whether the action space of the agent is continuous, multi-discrete or mixed. /// private ActionSpaceType _actionSpaceType; - + void Start() { InputManager = GetComponent(); @@ -72,19 +72,62 @@ public override void OnEpisodeBegin() //Reset Car _controller.ResetCar(_startPosition, Quaternion.Euler(0f, 90f, 0f)); - //Reset Ball - _ball.localPosition = new Vector3(UnityEngine.Random.Range(-10f, 0f), UnityEngine.Random.Range(0f, 20f), UnityEngine.Random.Range(-30f, 30f)); - //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); + ResetShoot(); + + _mapData.ResetIsScored(); + } + + public static Vector3 GetLocalPositionTarget(Difficulty difficulty) + { + // Tor ist von 2 bis 8.8 + switch (difficulty) + { + case Difficulty.EASY: + return new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-2f, 2f)); + case Difficulty.MIDDLE: + return new Vector3(-53f, UnityEngine.Random.Range(0f, 2f), UnityEngine.Random.Range(-4f, 4f)); + case Difficulty.HARD: + return new Vector3(-53f, UnityEngine.Random.Range(0f, 3f), UnityEngine.Random.Range(-7f, 7f)); + default: + return new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-2f, 2f)); + } + + + } + private void ResetShoot() + { + var lesson = Academy.Instance.EnvironmentParameters.GetWithDefault("my_environment_parameter", 0.0f); + var localPositionBall = new Vector3(); + var localPositionTarget = new Vector3(); + var speed = 0.0f; + + switch (lesson) + { + case <= 0: + localPositionTarget = + GetLocalPositionTarget(Difficulty.EASY); + localPositionBall = new Vector3(UnityEngine.Random.Range(-10f, 0f), UnityEngine.Random.Range(0f, 20f), + UnityEngine.Random.Range(-30f, 30f)); + speed = UnityEngine.Random.Range(50f, 100f); + break; + + case <= 1: + localPositionTarget = + new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-3f, 3f)); + localPositionBall = new Vector3(UnityEngine.Random.Range(-5f, 0f), UnityEngine.Random.Range(0f, 10f), + UnityEngine.Random.Range(-10f, 10f)); + speed = UnityEngine.Random.Range(30f, 50f); + break; + } + + //reset velocities _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; - // Set new Taget Position - _shootAt.localPosition = new Vector3(-53f, UnityEngine.Random.Range(3f, 3f), UnityEngine.Random.Range(-7f, 7f)); - + _ball.localPosition = localPositionBall; + _shootAt.localPosition = localPositionTarget; //Throw Ball - _ball.GetComponent().ShootTarget(); - - _mapData.ResetIsScored(); + _ball.GetComponent().ShootTarget(speed); } public override void CollectObservations(VectorSensor sensor) @@ -99,16 +142,19 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: carXNormalized == NaN"); carXNormalized = -1f; } + if (float.IsNaN(carYNormalized)) { Debug.Log("Car: carYNormalized == NaN"); carYNormalized = -1f; } + if (float.IsNaN(carZNormalized)) { Debug.Log("Car: carZNormalized == NaN"); carZNormalized = -1f; } + sensor.AddObservation(new Vector3(carXNormalized, carYNormalized, carZNormalized)); //Car rotation, already normalized @@ -121,21 +167,25 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: car_rotation_x == NaN"); car_rotation_x = -1f; } + if (float.IsNaN(car_rotation_y)) { Debug.Log("Car: car_rotation_y == NaN"); car_rotation_y = -1f; } + if (float.IsNaN(car_rotation_z)) { Debug.Log("Car: car_rotation_z == NaN"); car_rotation_z = -1f; } + if (float.IsNaN(car_rotation_w)) { Debug.Log("Car: car_rotation_w == NaN"); car_rotation_w = -1f; } + sensor.AddObservation(new Quaternion(car_rotation_x, car_rotation_y, car_rotation_z, car_rotation_w)); //Car velocity Vector3 car_velocity = _rb.velocity.normalized * (_rb.velocity.magnitude / 23f); @@ -144,16 +194,19 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: car_velocity.x == NaN"); car_velocity.x = -1f; } + if (float.IsNaN(car_velocity.y)) { Debug.Log("Car: car_velocity.y == NaN"); car_velocity.y = -1f; } + if (float.IsNaN(car_velocity.z)) { Debug.Log("Car: car_velocity.z == NaN"); car_velocity.z = -1f; } + sensor.AddObservation(car_velocity); //Car angular velocity @@ -163,16 +216,19 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: _rb.angularVelocity.x == NaN"); car_angularVelocity.x = -1f; } + if (float.IsNaN(car_angularVelocity.y)) { Debug.Log("Car: _rb.angularVelocity.y == NaN"); car_angularVelocity.y = -1f; } + if (float.IsNaN(car_angularVelocity.z)) { Debug.Log("Car: _rb.angularVelocity.z == NaN"); car_angularVelocity.z = -1f; } + sensor.AddObservation(car_angularVelocity); //Ball position @@ -185,16 +241,19 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Ball: ballXNormalized == NaN"); ballXNormalized = -1f; } + if (float.IsNaN(ballYNormalized)) { Debug.Log("Ball: ballYNormalized == NaN"); ballYNormalized = -1f; } + if (float.IsNaN(ballZNormalized)) { Debug.Log("Ball: ballZNormalized == NaN"); ballZNormalized = -1f; } + sensor.AddObservation(new Vector3(ballXNormalized, ballYNormalized, ballZNormalized)); //Ball velocity @@ -204,16 +263,19 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Ball: ball_velocity.x == NaN"); ball_velocity.x = -1f; } + if (float.IsNaN(ball_velocity.y)) { Debug.Log("Ball: ball_velocity.y == NaN"); ball_velocity.y = -1f; } + if (float.IsNaN(ball_velocity.z)) { Debug.Log("Ball: ball_velocity.z == NaN"); ball_velocity.z = -1f; } + sensor.AddObservation(ball_velocity); // Boost amount @@ -223,6 +285,7 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: boostAmount == NaN"); boostAmount = -1f; } + sensor.AddObservation(boostAmount); } @@ -242,13 +305,15 @@ public override void OnActionReceived(ActionBuffers actionBuffers) ProcessMixedActions(actionBuffers); break; default: - throw new InvalidOperationException(string.Format("The method {0} does not support the {1} '{2}'.", nameof(OnActionReceived), typeof(ActionSpaceType), _actionSpaceType.ToString())); + throw new InvalidOperationException(string.Format("The method {0} does not support the {1} '{2}'.", + nameof(OnActionReceived), typeof(ActionSpaceType), _actionSpaceType.ToString())); break; } } + AssignReward(); } - + public override void Heuristic(in ActionBuffers actionsOut) { InputManager.isAgent = false; @@ -266,7 +331,7 @@ private void Reset() private void AssignReward() { AddReward(-0.001f); - + if (_rbBall.velocity.x > 0 || Time.time - _lastResetTime > _episodeLength) { // Agent scored a goal @@ -274,6 +339,7 @@ private void AssignReward() Reset(); } + if (_mapData.isScoredOrange) { // Agent got scored on @@ -288,20 +354,20 @@ private void AssignReward() /// The action buffers containing the actions. private void ProcessContinuousActions(ActionBuffers actionBuffers) { - // set inputs - InputManager.throttleInput = actionBuffers.ContinuousActions[0]; - InputManager.steerInput = actionBuffers.ContinuousActions[1]; - InputManager.yawInput = actionBuffers.ContinuousActions[1]; - InputManager.pitchInput = actionBuffers.ContinuousActions[2]; - InputManager.rollInput = 0; - if (actionBuffers.ContinuousActions[3] > 0) InputManager.rollInput = 1; - if (actionBuffers.ContinuousActions[3] < 0) InputManager.rollInput = -1; - - InputManager.isBoost = actionBuffers.ContinuousActions[4] > 0; - InputManager.isDrift = actionBuffers.ContinuousActions[5] > 0; - InputManager.isAirRoll = actionBuffers.ContinuousActions[6] > 0; - - InputManager.isJump = actionBuffers.ContinuousActions[7] > 0; + // set inputs + InputManager.throttleInput = actionBuffers.ContinuousActions[0]; + InputManager.steerInput = actionBuffers.ContinuousActions[1]; + InputManager.yawInput = actionBuffers.ContinuousActions[1]; + InputManager.pitchInput = actionBuffers.ContinuousActions[2]; + InputManager.rollInput = 0; + if (actionBuffers.ContinuousActions[3] > 0) InputManager.rollInput = 1; + if (actionBuffers.ContinuousActions[3] < 0) InputManager.rollInput = -1; + + InputManager.isBoost = actionBuffers.ContinuousActions[4] > 0; + InputManager.isDrift = actionBuffers.ContinuousActions[5] > 0; + InputManager.isAirRoll = actionBuffers.ContinuousActions[6] > 0; + + InputManager.isJump = actionBuffers.ContinuousActions[7] > 0; } /// @@ -326,6 +392,7 @@ private void ProcessMultiDiscreteActions(ActionBuffers actionBuffers) InputManager.rollInput = 0; break; } + InputManager.isBoost = actionBuffers.DiscreteActions[4] > 0; InputManager.isDrift = actionBuffers.DiscreteActions[5] > 0; InputManager.isAirRoll = actionBuffers.DiscreteActions[6] > 0; @@ -357,6 +424,7 @@ private void ProcessMixedActions(ActionBuffers actionBuffers) InputManager.rollInput = 0; break; } + InputManager.isBoost = actionBuffers.DiscreteActions[1] > 0; InputManager.isDrift = actionBuffers.DiscreteActions[2] > 0; InputManager.isAirRoll = actionBuffers.DiscreteActions[3] > 0; @@ -373,32 +441,41 @@ private ActionSpaceType DetermineActionSpaceType(ActionSpec actionSpec) int requiredNumActions = 8; // Determine action space type - if(actionSpec.NumContinuousActions > 0 && actionSpec.NumDiscreteActions == 0) + if (actionSpec.NumContinuousActions > 0 && actionSpec.NumDiscreteActions == 0) { //Propably continuous, we check the size - if(actionSpec.NumContinuousActions != requiredNumActions) + if (actionSpec.NumContinuousActions != requiredNumActions) { - throw new ArgumentException(string.Format("It seems like you tried to use a continuos action space for the agent. In this case the {0} needs 8 continuous actions.", typeof(GoalKeeperAgent))); + throw new ArgumentException(string.Format( + "It seems like you tried to use a continuos action space for the agent. In this case the {0} needs 8 continuous actions.", + typeof(GoalKeeperAgent))); } + return ActionSpaceType.Continuous; } - else if(actionSpec.NumContinuousActions == 0 && actionSpec.NumDiscreteActions > 0) + else if (actionSpec.NumContinuousActions == 0 && actionSpec.NumDiscreteActions > 0) { //Propably multi discrete, we check the size if (actionSpec.NumDiscreteActions != requiredNumActions) { - throw new ArgumentException(string.Format("It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches.", typeof(GoalKeeperAgent))); + throw new ArgumentException(string.Format( + "It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches.", + typeof(GoalKeeperAgent))); } - int[] requiredBranchSizes = { DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, 3, 2, 2, 2, 2}; - for(int i = 0; i < actionSpec.BranchSizes.Length; i++) + + int[] requiredBranchSizes = + { DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, 3, 2, 2, 2, 2 }; + for (int i = 0; i < actionSpec.BranchSizes.Length; i++) { - if(actionSpec.BranchSizes[i] != requiredBranchSizes[i]) + if (actionSpec.BranchSizes[i] != requiredBranchSizes[i]) { - throw new ArgumentException(string.Format("It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches with sizes ({1}).", - typeof(GoalKeeperAgent), + throw new ArgumentException(string.Format( + "It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches with sizes ({1}).", + typeof(GoalKeeperAgent), string.Join(", ", requiredBranchSizes))); } } + return ActionSpaceType.MultiDiscrete; } else @@ -406,18 +483,23 @@ private ActionSpaceType DetermineActionSpaceType(ActionSpec actionSpec) //Propably multi discrete, we check the size if (actionSpec.NumDiscreteActions != 5 || actionSpec.NumContinuousActions != 3) { - throw new ArgumentException(string.Format("It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches and 3 continuous actions.", typeof(GoalKeeperAgent))); + throw new ArgumentException(string.Format( + "It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches and 3 continuous actions.", + typeof(GoalKeeperAgent))); } + int[] requiredBranchSizes = { 3, 2, 2, 2, 2 }; for (int i = 0; i < actionSpec.BranchSizes.Length; i++) { if (actionSpec.BranchSizes[i] != requiredBranchSizes[i]) { - throw new ArgumentException(string.Format("It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches with sizes ({1}).", + throw new ArgumentException(string.Format( + "It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches with sizes ({1}).", typeof(GoalKeeperAgent), string.Join(", ", requiredBranchSizes))); } } + return ActionSpaceType.Mixed; } } @@ -431,4 +513,11 @@ private enum ActionSpaceType MultiDiscrete, Mixed } -} + + public enum Difficulty + { + EASY, + MIDDLE, + HARD + } +} \ No newline at end of file diff --git a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json index 71b619f..2c1ed21 100644 --- a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json +++ b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json @@ -1 +1 @@ -{"count":1,"self":16.2871648,"total":16.389739,"children":{"InitializeActuators":{"count":2,"self":0.0024779,"total":0.0024779,"children":null},"InitializeSensors":{"count":2,"self":0.0019811,"total":0.0019811,"children":null},"AgentSendState":{"count":1415,"self":0.012920899999999999,"total":0.0689809,"children":{"CollectObservations":{"count":142,"self":0.0536002,"total":0.0536002,"children":null},"WriteActionMask":{"count":142,"self":0.00096749999999999994,"total":0.00096749999999999994,"children":null},"RequestDecision":{"count":142,"self":0.0014923,"total":0.0014923,"children":null}}},"DecideAction":{"count":1415,"self":0.014373499999999999,"total":0.014373499999999999,"children":null},"AgentAct":{"count":1415,"self":0.013272299999999999,"total":0.013272299999999999,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":1,"max":10.4072571,"min":10.4072571,"runningAverage":10.4072571,"value":10.4072571,"weightedAverage":10.4072571}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1625214513","unity_version":"2020.3.11f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.11f1\\Editor\\Unity.exe -projectpath D:\/Projects\/Unity\/PG\/RoboLeague -useHub -hubIPC -cloudEnvironment production -hubSessionId fb033b90-db03-11eb-b410-0f1c5ce3a503","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1625214529"}} +{"count":1,"self":6.9039592,"total":6.9199597,"children":{"InitializeActuators":{"count":1,"self":0.0019979,"total":0.0019979,"children":null},"InitializeSensors":{"count":1,"self":0.0020004,"total":0.0020004,"children":null},"AgentSendState":{"count":273,"self":0.0030022,"total":0.0050005,"children":{"CollectObservations":{"count":28,"self":0.000998,"total":0.000998,"children":null},"WriteActionMask":{"count":28,"self":0,"total":0,"children":null},"RequestDecision":{"count":28,"self":0.0010003,"total":0.0010003,"children":null}}},"DecideAction":{"count":273,"self":0.0030006,"total":0.0030006,"children":null},"AgentAct":{"count":273,"self":0.0040012,"total":0.0040012,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":1,"max":-1.09,"min":-1.09,"runningAverage":-1.09,"value":-1.09,"weightedAverage":-1.09}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1641378180","unity_version":"2020.3.8f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.8f1\\Editor\\Unity.exe -projectPath C:\/Users\/oliht\/Documents\/Uni\/SOSE21\/Projektgruppe\/RoboLeague\/RoboLeague -debugCodeOptimization -riderPath C:\\Users\\oliht\\AppData\\Local\\JetBrains\\Toolbox\\apps\\Rider\\ch-0\\212.5284.64\\bin\\rider64.exe","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1641378187"}} \ No newline at end of file diff --git a/Assets/Prefabs/Ball.prefab b/Assets/Prefabs/Ball.prefab index d14c075..97a3b0b 100644 --- a/Assets/Prefabs/Ball.prefab +++ b/Assets/Prefabs/Ball.prefab @@ -256,7 +256,7 @@ SphereCollider: m_IsTrigger: 0 m_Enabled: 1 serializedVersion: 2 - m_Radius: 0.9275 + m_Radius: 0.9315 m_Center: {x: 0, y: 0, z: 0} --- !u!1 &2532842153811578195 GameObject: diff --git a/Assets/Scripts/Ball/ShootBall.cs b/Assets/Scripts/Ball/ShootBall.cs index 28a55ba..a1b260c 100644 --- a/Assets/Scripts/Ball/ShootBall.cs +++ b/Assets/Scripts/Ball/ShootBall.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using Unity.MLAgents; using UnityEngine; using Random = UnityEngine.Random; @@ -16,9 +17,15 @@ void Start() ShootTarget(); } - public void ShootTarget() + + public void ShootTarget(float velocity = 0.0f) { Vector3 dir = ShootAt.position - transform.position; - _rb.AddForce( dir.normalized * Random.Range(speed.x, speed.y), ForceMode.VelocityChange); + if (velocity == 0.0f) + { + velocity = Random.Range(speed.x, speed.y); + } + + _rb.AddForce( dir.normalized * velocity, ForceMode.VelocityChange); } } From 329e5d26b7b3b6836cb2ae6e29632a929e41f5e8 Mon Sep 17 00:00:00 2001 From: Kabumba Date: Wed, 5 Jan 2022 12:32:35 +0100 Subject: [PATCH 02/25] lessons und difficulties definiert --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 153 ++++++++++++++++-- 1 file changed, 139 insertions(+), 14 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 069fa40..8999b00 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -79,20 +79,55 @@ public override void OnEpisodeBegin() public static Vector3 GetLocalPositionTarget(Difficulty difficulty) { - // Tor ist von 2 bis 8.8 + // Tor ist von -8.8 bis 8.8 breit und von 0 bis 6.5 hoch switch (difficulty) { case Difficulty.EASY: - return new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-2f, 2f)); + return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 2.5f), UnityEngine.Random.Range(-2f, 2f)); case Difficulty.MIDDLE: - return new Vector3(-53f, UnityEngine.Random.Range(0f, 2f), UnityEngine.Random.Range(-4f, 4f)); + return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 4f), UnityEngine.Random.Range(-4f, 4f)); case Difficulty.HARD: - return new Vector3(-53f, UnityEngine.Random.Range(0f, 3f), UnityEngine.Random.Range(-7f, 7f)); + return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 5.6f), UnityEngine.Random.Range(-7.9f, 7.9f)); default: - return new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-2f, 2f)); + return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 2.5f), UnityEngine.Random.Range(-2f, 2f)); } + } + + public static Vector3 GetLocalPositionBall(Difficulty difficulty) + { + switch (difficulty) + { + case Difficulty.EASY: + return new Vector3(UnityEngine.Random.Range(-10f, 10f), UnityEngine.Random.Range(0.9315f, 5f), UnityEngine.Random.Range(-10f, 10f)); + case Difficulty.MIDDLE: + return new Vector3(UnityEngine.Random.Range(-20f, 20f), UnityEngine.Random.Range(0.9315f, 10f), UnityEngine.Random.Range(-20f, 20f)); + case Difficulty.HARD: + return new Vector3(UnityEngine.Random.Range(-30f, 30f), UnityEngine.Random.Range(0.9315f, 19f), UnityEngine.Random.Range(-39f, 39f)); + default: + return new Vector3(UnityEngine.Random.Range(-5f, 0f), UnityEngine.Random.Range(0f, 10f), + UnityEngine.Random.Range(-10f, 10f)); + } + + + } + + public static float GetSpeed(Difficulty difficulty) + { + switch (difficulty) + { + case Difficulty.EASY: + return UnityEngine.Random.Range(5f, 20f); + case Difficulty.MIDDLE: + return UnityEngine.Random.Range(20f, 40f); + case Difficulty.HARD: + return UnityEngine.Random.Range(40f, 60f); + default: + return UnityEngine.Random.Range(5f, 20f); + } + + } private void ResetShoot() { @@ -101,22 +136,112 @@ private void ResetShoot() var localPositionTarget = new Vector3(); var speed = 0.0f; + switch (lesson) { - case <= 0: + case 0: localPositionTarget = GetLocalPositionTarget(Difficulty.EASY); - localPositionBall = new Vector3(UnityEngine.Random.Range(-10f, 0f), UnityEngine.Random.Range(0f, 20f), - UnityEngine.Random.Range(-30f, 30f)); - speed = UnityEngine.Random.Range(50f, 100f); + localPositionBall = GetLocalPositionBall(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY); break; - case <= 1: + case 1: localPositionTarget = - new Vector3(-53f, UnityEngine.Random.Range(0f, 1f), UnityEngine.Random.Range(-3f, 3f)); - localPositionBall = new Vector3(UnityEngine.Random.Range(-5f, 0f), UnityEngine.Random.Range(0f, 10f), - UnityEngine.Random.Range(-10f, 10f)); - speed = UnityEngine.Random.Range(30f, 50f); + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY); + break; + + case 2: + localPositionTarget = + GetLocalPositionTarget(Difficulty.EASY); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.EASY); + break; + + case 3: + localPositionTarget = + GetLocalPositionTarget(Difficulty.EASY); + localPositionBall = GetLocalPositionBall(Difficulty.EASY); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 4: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.EASY); + break; + + case 5: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.EASY); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 6: + localPositionTarget = + GetLocalPositionTarget(Difficulty.EASY); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 7: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 8: + localPositionTarget = + GetLocalPositionTarget(Difficulty.HARD); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 9: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.HARD); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 10: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.HARD); + break; + + case 11: + localPositionTarget = + GetLocalPositionTarget(Difficulty.HARD); + localPositionBall = GetLocalPositionBall(Difficulty.HARD); + speed = GetSpeed(Difficulty.MIDDLE); + break; + + case 12: + localPositionTarget = + GetLocalPositionTarget(Difficulty.HARD); + localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.HARD); + break; + + case 13: + localPositionTarget = + GetLocalPositionTarget(Difficulty.MIDDLE); + localPositionBall = GetLocalPositionBall(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD); + break; + + case 14: + localPositionTarget = + GetLocalPositionTarget(Difficulty.HARD); + localPositionBall = GetLocalPositionBall(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD); break; } From 438d798b2a279ea5a65c0f8c3fefa3d6aaa89197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20R=C3=B6themeyer?= <26631624+TheImus@users.noreply.github.com> Date: Sun, 9 Jan 2022 20:00:32 +0100 Subject: [PATCH 03/25] updated speed of ball in goalkeeper scenario --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 27 +++++++++---------- .../Timers/SaveTraining_single_timers.json | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 8999b00..5b67a22 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -87,12 +87,11 @@ public static Vector3 GetLocalPositionTarget(Difficulty difficulty) case Difficulty.MIDDLE: return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 4f), UnityEngine.Random.Range(-4f, 4f)); case Difficulty.HARD: - return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 5.6f), UnityEngine.Random.Range(-7.9f, 7.9f)); + return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 5.6f), + UnityEngine.Random.Range(-7.9f, 7.9f)); default: return new Vector3(-53f, UnityEngine.Random.Range(0.9315f, 2.5f), UnityEngine.Random.Range(-2f, 2f)); } - - } public static Vector3 GetLocalPositionBall(Difficulty difficulty) @@ -100,17 +99,18 @@ public static Vector3 GetLocalPositionBall(Difficulty difficulty) switch (difficulty) { case Difficulty.EASY: - return new Vector3(UnityEngine.Random.Range(-10f, 10f), UnityEngine.Random.Range(0.9315f, 5f), UnityEngine.Random.Range(-10f, 10f)); + return new Vector3(UnityEngine.Random.Range(-10f, 10f), UnityEngine.Random.Range(0.9315f, 5f), + UnityEngine.Random.Range(-10f, 10f)); case Difficulty.MIDDLE: - return new Vector3(UnityEngine.Random.Range(-20f, 20f), UnityEngine.Random.Range(0.9315f, 10f), UnityEngine.Random.Range(-20f, 20f)); + return new Vector3(UnityEngine.Random.Range(-20f, 20f), UnityEngine.Random.Range(0.9315f, 10f), + UnityEngine.Random.Range(-20f, 20f)); case Difficulty.HARD: - return new Vector3(UnityEngine.Random.Range(-30f, 30f), UnityEngine.Random.Range(0.9315f, 19f), UnityEngine.Random.Range(-39f, 39f)); + return new Vector3(UnityEngine.Random.Range(-30f, 30f), UnityEngine.Random.Range(0.9315f, 19f), + UnityEngine.Random.Range(-39f, 39f)); default: return new Vector3(UnityEngine.Random.Range(-5f, 0f), UnityEngine.Random.Range(0f, 10f), UnityEngine.Random.Range(-10f, 10f)); } - - } public static float GetSpeed(Difficulty difficulty) @@ -122,16 +122,15 @@ public static float GetSpeed(Difficulty difficulty) case Difficulty.MIDDLE: return UnityEngine.Random.Range(20f, 40f); case Difficulty.HARD: - return UnityEngine.Random.Range(40f, 60f); + return UnityEngine.Random.Range(10f, 60f); default: return UnityEngine.Random.Range(5f, 20f); } - - } + private void ResetShoot() { - var lesson = Academy.Instance.EnvironmentParameters.GetWithDefault("my_environment_parameter", 0.0f); + var lesson = Academy.Instance.EnvironmentParameters.GetWithDefault("my_environment_parameter", 14.0f); var localPositionBall = new Vector3(); var localPositionTarget = new Vector3(); var speed = 0.0f; @@ -145,7 +144,7 @@ private void ResetShoot() localPositionBall = GetLocalPositionBall(Difficulty.EASY); speed = GetSpeed(Difficulty.EASY); break; - + case 1: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); @@ -244,7 +243,7 @@ private void ResetShoot() speed = GetSpeed(Difficulty.HARD); break; } - + //reset velocities _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; diff --git a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json index 2c1ed21..c6cc21b 100644 --- a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json +++ b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json @@ -1 +1 @@ -{"count":1,"self":6.9039592,"total":6.9199597,"children":{"InitializeActuators":{"count":1,"self":0.0019979,"total":0.0019979,"children":null},"InitializeSensors":{"count":1,"self":0.0020004,"total":0.0020004,"children":null},"AgentSendState":{"count":273,"self":0.0030022,"total":0.0050005,"children":{"CollectObservations":{"count":28,"self":0.000998,"total":0.000998,"children":null},"WriteActionMask":{"count":28,"self":0,"total":0,"children":null},"RequestDecision":{"count":28,"self":0.0010003,"total":0.0010003,"children":null}}},"DecideAction":{"count":273,"self":0.0030006,"total":0.0030006,"children":null},"AgentAct":{"count":273,"self":0.0040012,"total":0.0040012,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":1,"max":-1.09,"min":-1.09,"runningAverage":-1.09,"value":-1.09,"weightedAverage":-1.09}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1641378180","unity_version":"2020.3.8f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.8f1\\Editor\\Unity.exe -projectPath C:\/Users\/oliht\/Documents\/Uni\/SOSE21\/Projektgruppe\/RoboLeague\/RoboLeague -debugCodeOptimization -riderPath C:\\Users\\oliht\\AppData\\Local\\JetBrains\\Toolbox\\apps\\Rider\\ch-0\\212.5284.64\\bin\\rider64.exe","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1641378187"}} \ No newline at end of file +{"count":1,"self":53.385865599999995,"total":53.6266689,"children":{"InitializeActuators":{"count":1,"self":0.0019977,"total":0.0019977,"children":null},"InitializeSensors":{"count":1,"self":0.002002,"total":0.002002,"children":null},"AgentSendState":{"count":5892,"self":0.085734499999999991,"total":0.10074,"children":{"CollectObservations":{"count":590,"self":0.0059968999999999995,"total":0.0059968999999999995,"children":null},"WriteActionMask":{"count":590,"self":0.0030033,"total":0.0030033,"children":null},"RequestDecision":{"count":590,"self":0.0060053,"total":0.0060053,"children":null}}},"DecideAction":{"count":5892,"self":0.0690268,"total":0.0690268,"children":null},"AgentAct":{"count":5892,"self":0.066038,"total":0.066038,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":25,"max":1.86999989,"min":-1.42799807,"runningAverage":-0.264919668,"value":-1.102,"weightedAverage":-0.8757154}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1641754662","unity_version":"2020.3.8f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.8f1\\Editor\\Unity.exe -projectPath C:\/Users\/oliht\/Documents\/Uni\/SOSE21\/Projektgruppe\/RoboLeague\/RoboLeague -debugCodeOptimization -riderPath C:\\Users\\oliht\\AppData\\Local\\JetBrains\\Toolbox\\apps\\Rider\\ch-0\\212.5284.64\\bin\\rider64.exe","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1641754715"}} \ No newline at end of file From 9fc37b4e05910ea24d4343d4d21e495d818d060c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20R=C3=B6themeyer?= <26631624+TheImus@users.noreply.github.com> Date: Sun, 9 Jan 2022 20:06:58 +0100 Subject: [PATCH 04/25] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68a1ddc..853a562 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,6 @@ jobs: buildMethod: BuildScript.PerformBuild - uses: actions/upload-artifact@v2 with: - name: ${{ matrix.targetPlatform }} + name: ${GITHUB_REF##*/}-${{ steps.date.outputs.date }} path: builds From 7a92f1472f2b79b60250bb23deb8c05a777cc217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20R=C3=B6themeyer?= <26631624+TheImus@users.noreply.github.com> Date: Sun, 9 Jan 2022 20:15:47 +0100 Subject: [PATCH 05/25] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 853a562..b9f3d2c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,6 @@ jobs: buildMethod: BuildScript.PerformBuild - uses: actions/upload-artifact@v2 with: - name: ${GITHUB_REF##*/}-${{ steps.date.outputs.date }} + name: Build-${{ steps.date.outputs.date }} path: builds From 562f665349146a4ffe64f763cc9b50e7d39b58e2 Mon Sep 17 00:00:00 2001 From: Kabumba Date: Fri, 28 Jan 2022 17:37:35 +0100 Subject: [PATCH 06/25] random Speed difficulty angepasst --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 55 +++++++++++-------- .../GoalkeeperEnvironmentParameters.cs | 2 +- .../Goalkeeper/GoalkeeperEvironmentHandler.cs | 1 + .../Goalkeeper/SaveTraining_single.unity | 4 ++ .../Timers/SaveTraining_single_timers.json | 2 +- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 415e9fb..23a4c28 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -30,6 +30,7 @@ public class GoalKeeperAgent : Agent private CubeBoosting _boostControl; private CubeGroundControl _groundControl; private CubeAirControl _airControl; + public float difficulty; private MapData _mapData; @@ -45,6 +46,8 @@ public class GoalKeeperAgent : Agent /// private ActionSpaceType _actionSpaceType; + private Vector3 zero = Vector3.zero; + void Start() { @@ -123,134 +126,142 @@ public static Vector3 GetLocalPositionBall(Difficulty difficulty) } } - public static float GetSpeed(Difficulty difficulty) + public float GetSpeed(Difficulty difficulty, Vector3 ballPosition, Vector3 targetPosition) { + float scale; + float dist = (ballPosition - targetPosition).magnitude; + float minTime = 2f; + float maxTime = Math.Max(minTime,Math.Min(_episodeLength, dist / 5f)); switch (difficulty) { case Difficulty.EASY: - return UnityEngine.Random.Range(5f, 20f); + scale = UnityEngine.Random.Range(0.7f, 1f); + break; case Difficulty.MIDDLE: - return UnityEngine.Random.Range(20f, 40f); + scale = UnityEngine.Random.Range(0.4f, 0.7f); + break; case Difficulty.HARD: - return UnityEngine.Random.Range(10f, 60f); + scale = UnityEngine.Random.Range(0.1f, 4f); + break; default: - return UnityEngine.Random.Range(5f, 20f); + scale = UnityEngine.Random.Range(0.7f, 1f); + break; } + return dist * Math.Max(1 / minTime,scale / maxTime); } private void ResetShoot() { - var lesson = Academy.Instance.EnvironmentParameters.GetWithDefault("my_environment_parameter", 14.0f); var localPositionBall = new Vector3(); var localPositionTarget = new Vector3(); var speed = 0.0f; - switch (lesson) + switch ((int) difficulty) { case 0: localPositionTarget = GetLocalPositionTarget(Difficulty.EASY); localPositionBall = GetLocalPositionBall(Difficulty.EASY); - speed = GetSpeed(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY, localPositionBall, localPositionTarget); break; case 1: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.EASY); - speed = GetSpeed(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY, localPositionBall, localPositionTarget); break; case 2: localPositionTarget = GetLocalPositionTarget(Difficulty.EASY); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY, localPositionBall, localPositionTarget); break; case 3: localPositionTarget = GetLocalPositionTarget(Difficulty.EASY); localPositionBall = GetLocalPositionBall(Difficulty.EASY); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 4: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.EASY); + speed = GetSpeed(Difficulty.EASY, localPositionBall, localPositionTarget); break; case 5: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.EASY); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 6: localPositionTarget = GetLocalPositionTarget(Difficulty.EASY); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 7: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 8: localPositionTarget = GetLocalPositionTarget(Difficulty.HARD); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 9: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.HARD); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 10: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD, localPositionBall, localPositionTarget); break; case 11: localPositionTarget = GetLocalPositionTarget(Difficulty.HARD); localPositionBall = GetLocalPositionBall(Difficulty.HARD); - speed = GetSpeed(Difficulty.MIDDLE); + speed = GetSpeed(Difficulty.MIDDLE, localPositionBall, localPositionTarget); break; case 12: localPositionTarget = GetLocalPositionTarget(Difficulty.HARD); localPositionBall = GetLocalPositionBall(Difficulty.MIDDLE); - speed = GetSpeed(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD, localPositionBall, localPositionTarget); break; case 13: localPositionTarget = GetLocalPositionTarget(Difficulty.MIDDLE); localPositionBall = GetLocalPositionBall(Difficulty.HARD); - speed = GetSpeed(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD, localPositionBall, localPositionTarget); break; case 14: localPositionTarget = GetLocalPositionTarget(Difficulty.HARD); localPositionBall = GetLocalPositionBall(Difficulty.HARD); - speed = GetSpeed(Difficulty.HARD); + speed = GetSpeed(Difficulty.HARD, localPositionBall, localPositionTarget); break; } diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs index 4278349..924aadd 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs @@ -7,7 +7,7 @@ public class GoalkeeperEnvironmentParameters { public GoalkeeperEnvironmentParameters() { - difficulty = 1; + difficulty = 0; initialBoost = 32; canBoost = 1; canDoubleJump = 1; diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs index b94f702..f708f8a 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs @@ -16,6 +16,7 @@ public override void ResetParameter() { UpdateEnvironmentParameters(); //TODO after merge with difficulty, add difficulty parameter + environment.GetComponentInChildren().difficulty = currentParameter.difficulty; if (currentParameter.canDoubleJump == 0) { environment.GetComponentInChildren().disableDoubleJump = true; diff --git a/Assets/ML-Agents/Goalkeeper/SaveTraining_single.unity b/Assets/ML-Agents/Goalkeeper/SaveTraining_single.unity index 3e49031..5aba6cf 100644 --- a/Assets/ML-Agents/Goalkeeper/SaveTraining_single.unity +++ b/Assets/ML-Agents/Goalkeeper/SaveTraining_single.unity @@ -1057,6 +1057,10 @@ PrefabInstance: m_Modification: m_TransformParent: {fileID: 1469899324} m_Modifications: + - target: {fileID: 883974180034025411, guid: 0a3f86ef81cf99f46b33e7f9e661ee20, type: 3} + propertyPath: defaultParameter.difficulty + value: 0 + objectReference: {fileID: 0} - target: {fileID: 883974180034025414, guid: 0a3f86ef81cf99f46b33e7f9e661ee20, type: 3} propertyPath: m_Name value: GoalKeeperAgent diff --git a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json index c6cc21b..5b938f7 100644 --- a/Assets/ML-Agents/Timers/SaveTraining_single_timers.json +++ b/Assets/ML-Agents/Timers/SaveTraining_single_timers.json @@ -1 +1 @@ -{"count":1,"self":53.385865599999995,"total":53.6266689,"children":{"InitializeActuators":{"count":1,"self":0.0019977,"total":0.0019977,"children":null},"InitializeSensors":{"count":1,"self":0.002002,"total":0.002002,"children":null},"AgentSendState":{"count":5892,"self":0.085734499999999991,"total":0.10074,"children":{"CollectObservations":{"count":590,"self":0.0059968999999999995,"total":0.0059968999999999995,"children":null},"WriteActionMask":{"count":590,"self":0.0030033,"total":0.0030033,"children":null},"RequestDecision":{"count":590,"self":0.0060053,"total":0.0060053,"children":null}}},"DecideAction":{"count":5892,"self":0.0690268,"total":0.0690268,"children":null},"AgentAct":{"count":5892,"self":0.066038,"total":0.066038,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":25,"max":1.86999989,"min":-1.42799807,"runningAverage":-0.264919668,"value":-1.102,"weightedAverage":-0.8757154}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1641754662","unity_version":"2020.3.8f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.8f1\\Editor\\Unity.exe -projectPath C:\/Users\/oliht\/Documents\/Uni\/SOSE21\/Projektgruppe\/RoboLeague\/RoboLeague -debugCodeOptimization -riderPath C:\\Users\\oliht\\AppData\\Local\\JetBrains\\Toolbox\\apps\\Rider\\ch-0\\212.5284.64\\bin\\rider64.exe","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1641754715"}} \ No newline at end of file +{"count":1,"self":87.5792448,"total":87.656313099999991,"children":{"InitializeActuators":{"count":1,"self":0.001001,"total":0.001001,"children":null},"InitializeSensors":{"count":1,"self":0.0010008,"total":0.0010008,"children":null},"AgentSendState":{"count":10009,"self":0.023017299999999997,"total":0.029021599999999998,"children":{"CollectObservations":{"count":1001,"self":0.0040028,"total":0.0040028,"children":null},"WriteActionMask":{"count":1001,"self":0,"total":0,"children":null},"RequestDecision":{"count":1001,"self":0.0020015,"total":0.0020015,"children":null}}},"DecideAction":{"count":10009,"self":0.0320318,"total":0.0320318,"children":null},"AgentAct":{"count":10009,"self":0.013011199999999999,"total":0.013011199999999999,"children":null}},"gauges":{"GoalKeeping.CumulativeReward":{"count":36,"max":1.8299998,"min":-1.28499973,"runningAverage":-0.35316667,"value":1.74299991,"weightedAverage":0.5062059}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1643387710","unity_version":"2020.3.8f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.8f1\\Editor\\Unity.exe -projectpath C:\/Users\/Frederik\/RoboLeague -useHub -hubIPC -cloudEnvironment production -hubSessionId 932f6770-804c-11ec-a6f3-efad69b2e458 -accessToken JSfic1JN0fwpxKg1_eM0MmrDIy4FOox5wKmTCZGd9aI008f","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0-pre.3","scene_name":"SaveTraining_single","end_time_seconds":"1643387797"}} \ No newline at end of file From a1cbf393a119f6ca61a5a97ff1017f31b9cb73df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20R=C3=B6themeyer?= <26631624+TheImus@users.noreply.github.com> Date: Tue, 1 Feb 2022 17:18:00 +0100 Subject: [PATCH 07/25] update reward --- Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 23a4c28..666c167 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -476,12 +476,9 @@ private void Reset() /// private void AssignReward() { - AddReward(-0.001f); - if (_rbBall.velocity.x > 0 || Time.time - _lastResetTime > _episodeLength) { - // Agent scored a goal - SetReward(2f); + SetReward(1f); Reset(); } From b38eafe6101e8cdbf731ebe4b31c53bd7bd1f8ae Mon Sep 17 00:00:00 2001 From: Kabumba Date: Mon, 7 Feb 2022 16:05:03 +0100 Subject: [PATCH 08/25] Sichereres environment getten --- Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 6e775ab..5f63ead 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -48,7 +48,7 @@ public class GoalKeeperAgent : Agent void Start() { - _handler = new GoalkeeperEvironmentHandler(GameObject.Find("Environment"), defaultParameter); + _handler = new GoalkeeperEvironmentHandler(transform.root.gameObject, defaultParameter); InputManager = GetComponent(); InputManager.isAgent = true; From b00a0257a5079dbab5838de9736688342369aa01 Mon Sep 17 00:00:00 2001 From: Kabumba Date: Mon, 7 Feb 2022 16:13:04 +0100 Subject: [PATCH 09/25] Time in frames statt in seconds --- Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 5f63ead..41ac248 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -18,8 +18,9 @@ public class GoalKeeperAgent : Agent private GoalkeeperEvironmentHandler _handler; private Rigidbody _rb, _rbBall; - private float _episodeLength = 10f; - private float _lastResetTime; + private float _episodeLengthSeconds = 10f; + private float _episodeLengthFrames; + private float _lastResetFrame; private Transform _ball, _shootAt; private Vector3 _startPosition; @@ -51,6 +52,7 @@ void Start() _handler = new GoalkeeperEvironmentHandler(transform.root.gameObject, defaultParameter); InputManager = GetComponent(); InputManager.isAgent = true; + _episodeLengthFrames = _episodeLengthSeconds / Time.fixedDeltaTime; ActionSpec actionSpec = GetComponent().BrainParameters.ActionSpec; _actionSpaceType = DetermineActionSpaceType(actionSpec); @@ -70,7 +72,7 @@ void Start() _shootAt = transform.parent.Find("ShootAt"); _mapData = transform.parent.Find("World").Find("Rocket_Map").GetComponent(); - _lastResetTime = Time.time; + _lastResetFrame = Time.frameCount; _handler.UpdateEnvironmentParameters(); _handler.ResetParameter(); @@ -266,7 +268,7 @@ public override void Heuristic(in ActionBuffers actionsOut) private void Reset() { - _lastResetTime = Time.time; + _lastResetFrame = Time.frameCount; EndEpisode(); _handler.ResetParameter(); } @@ -278,10 +280,10 @@ private void AssignReward() { AddReward(-0.001f); - if (_rbBall.velocity.x > 0 || Time.time - _lastResetTime > _episodeLength) + if (_rbBall.velocity.x > 0 || Time.frameCount - _lastResetFrame > _episodeLengthFrames) { // Agent scored a goal - SetReward(2f); + SetReward(1f); Reset(); } From 1fddbdfc4b1bb7f3e4ec678714f68ada761e8915 Mon Sep 17 00:00:00 2001 From: Kabumba Date: Mon, 7 Feb 2022 17:25:03 +0100 Subject: [PATCH 10/25] s --- Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index fda2f32..ceb7683 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -478,8 +478,6 @@ private void Reset() /// private void AssignReward() { - AddReward(-0.001f); - if (_rbBall.velocity.x > 0 || Time.frameCount - _lastResetFrame > _episodeLengthFrames) { SetReward(1f); From 0d7a459222a532a323f9bed0864b6e3e0156a26c Mon Sep 17 00:00:00 2001 From: Kabumba Date: Mon, 7 Feb 2022 19:12:27 +0100 Subject: [PATCH 11/25] Checks und Normalize verallgemeinert --- Assets/ML-Agents/PGBaseAgent.cs | 118 +++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 19 deletions(-) diff --git a/Assets/ML-Agents/PGBaseAgent.cs b/Assets/ML-Agents/PGBaseAgent.cs index ff37dab..6c041ac 100644 --- a/Assets/ML-Agents/PGBaseAgent.cs +++ b/Assets/ML-Agents/PGBaseAgent.cs @@ -51,16 +51,6 @@ protected virtual void Start() mapData = transform.parent.Find("World").Find("Rocket_Map").GetComponent(); } - protected Vector3 NormalizePosition(Transform objTransform) - { - var vec = new Vector3(objTransform.localPosition.x, objTransform.localPosition.y, objTransform.localPosition.z); - - vec.x = (vec.x + 60f) / 120f; - vec.y = vec.y / 20f; - vec.z = (vec.z + 41f) / 82f; - return vec; - } - protected void AddRelativePositionNormalized(VectorSensor sensor, Transform otherTransform) { sensor.AddObservation((transform.localPosition - otherTransform.localPosition) / mapData.diag); @@ -232,26 +222,98 @@ private ActionSpaceType DetermineActionSpaceType(ActionSpec actionSpec) return ActionSpaceType.Mixed; } } + protected Vector3 NormalizeVec(Rigidbody rb, VectorType vecType, EntityType entity) + { + switch (vecType) + { + case VectorType.Position: + return NormalizePosition(rb.transform); + case VectorType.Velocity: + return NormalizeVelocity(rb, entity); + case VectorType.AngularVelocity: + return NormalizeAngularVelocity(rb, entity); + default: + return Vector3.zero; + } + } - protected bool checkVec(Vector3 vec, string name, float defaultValue) + protected Vector3 NormalizePosition(Transform objTransform) + { + var vec = new Vector3(objTransform.localPosition.x, objTransform.localPosition.y, objTransform.localPosition.z); + + vec.x = (vec.x + 60f) / 120f; + vec.y = vec.y / 20f; + vec.z = (vec.z + 41f) / 82f; + return vec; + } + + + protected Vector3 NormalizeVelocity(Rigidbody rb, EntityType entity) + { + switch (entity) + { + case EntityType.Ball: + return rb.velocity.normalized * (rb.velocity.magnitude / 60f); + case EntityType.Car: + return rb.velocity.normalized * (rb.velocity.magnitude / 23f); + default: + return Vector3.zero; + } + } + + protected Vector3 NormalizeAngularVelocity(Rigidbody rb, EntityType entity) + { + switch (entity) + { + case EntityType.Ball: + return rb.angularVelocity.normalized * (rb.angularVelocity.magnitude / 6f); + case EntityType.Car: + return rb.angularVelocity.normalized * (rb.angularVelocity.magnitude / 5.5f); + default: + return Vector3.zero; + } + } + + protected bool checkNormalizedVec(Vector3 vec, string name, VectorType type) + { + switch (type) + { + case VectorType.Position: + return checkVec(vec, name, -1f, true, false, 0f, 1f); + case VectorType.Velocity: + return checkVec(vec, name, -1f, false, true, -1f, 1f, 0f, 1f); + case VectorType.AngularVelocity: + return checkVec(vec, name, -1f, false, true, -1f, 1f, 0f, 1f); + default: + return true; + } + } + + + protected bool checkVec(Vector3 vec, string name, float defaultValue, bool legalValueInterval=true, bool legalMagnitudeInterval = true, float minLegalValue=-5f, float maxLegalValue=5f, float minLegalMagnitude=-1f, float maxLegalMagnitude=5f) { bool reset = false; for (int i = 0; i < 3; i++) { - if (float.IsNaN(vec[i]) || float.IsInfinity(vec[i])) + if (float.IsNaN(vec[i])) { - Debug.Log($"{name}[{i}] is NaN or Infinity"); + Debug.Log($"{name}[{i}] is NaN"); vec[i] = defaultValue; } - if (vec[i] < -5f || vec[i] > 5f) + if (float.IsInfinity(vec[i])) { - Debug.LogWarning($"{name}[{i}] is {vec[i]}, was expected to be in interval [-5, 5]"); + Debug.Log($"{name}[{i}] is Infinity"); + vec[i] = defaultValue; + } + if (legalValueInterval && (vec[i] < minLegalValue || vec[i] > maxLegalValue)) + { + Debug.LogWarning($"{name}[{i}] is {vec[i]}, was expected to be in interval [{minLegalValue}, {maxLegalValue}]"); reset = true; } } - if (vec.magnitude < -1f || vec.magnitude > 5f) + if (legalMagnitudeInterval && (vec.magnitude < minLegalMagnitude || vec.magnitude > maxLegalMagnitude)) { - Debug.LogWarning($"{name}.magnitude is {vec.magnitude}, was expected to be in interval [-1, 5]"); + Debug.LogWarning($"{name}.magnitude is {vec.magnitude}, was expected to be in interval [{minLegalMagnitude}, {maxLegalMagnitude}]"); reset = true; } return reset; @@ -261,9 +323,14 @@ protected void checkQuaternion(Quaternion quat, string name, float defaultValue) { for (int i = 0; i < 4; i++) { - if (float.IsNaN(quat[i]) || float.IsInfinity(quat[i])) + if (float.IsNaN(quat[i])) + { + Debug.Log(name + "[" + i + "] is NaN"); + quat[i] = defaultValue; + } + if (float.IsInfinity(quat[i])) { - Debug.Log(name + "[" + i + "] is NaN or Infinity"); + Debug.Log(name + "[" + i + "] is Infinity"); quat[i] = defaultValue; } } @@ -278,4 +345,17 @@ public enum ActionSpaceType MultiDiscrete, Mixed } + + public enum VectorType + { + Position, + Velocity, + AngularVelocity + } + + public enum EntityType + { + Ball, + Car + } } From 741de29579678b000010887f93d2cdcaa9f406e2 Mon Sep 17 00:00:00 2001 From: Kabumba Date: Mon, 7 Feb 2022 19:12:52 +0100 Subject: [PATCH 12/25] GoalkeeperAgent an PGAngent angepasst --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 396 ++---------------- 1 file changed, 24 insertions(+), 372 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index 758711e..b7fc2bf 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -16,60 +16,23 @@ public class GoalKeeperAgent : PGBaseAgent [SerializeField] public GoalkeeperEnvironmentParameters defaultParameter; private GoalkeeperEvironmentHandler _handler; - private Rigidbody _rbBall; private float _episodeLengthSeconds = 10f; private float _episodeLengthFrames; private float _lastResetFrame; + private Rigidbody _rbBall; private Transform _ball, _shootAt; private Vector3 _startPosition; - - private CubeJumping _jumpControl; - private CubeController _controller; - private CubeBoosting _boostControl; - private CubeGroundControl _groundControl; - private CubeAirControl _airControl; public float difficulty; - private MapData _mapData; - - public InputManager InputManager; - - /// - /// Contains the possible values for the discretized actions. - /// - private readonly float[] DISCRETE_ACTIONS = { -1f, -0.5f, 0f, 0.5f, 1f }; - - /// - /// Shows whether the action space of the agent is continuous, multi-discrete or mixed. - /// - private ActionSpaceType _actionSpaceType; - - private Vector3 zero = Vector3.zero; - void Start() { - + base.Start(); _handler = new GoalkeeperEvironmentHandler(transform.root.gameObject, defaultParameter); - InputManager = GetComponent(); - InputManager.isAgent = true; _episodeLengthFrames = _episodeLengthSeconds / Time.fixedDeltaTime; - - ActionSpec actionSpec = GetComponent().BrainParameters.ActionSpec; - _actionSpaceType = DetermineActionSpaceType(actionSpec); - - _rb = GetComponent(); - _airControl = GetComponentInChildren(); - _jumpControl = GetComponentInChildren(); - _controller = GetComponentInChildren(); - _boostControl = GetComponentInChildren(); - _groundControl = GetComponentInChildren(); - base.Start(); - _handler = new GoalkeeperEvironmentHandler(GameObject.Find("Environment"), defaultParameter); - - _ball = transform.parent.Find("Ball"); + _ball = _handler.environment.GetComponentInChildren().transform; _rbBall = _ball.GetComponent(); _startPosition = transform.position; @@ -90,7 +53,7 @@ public override void OnEpisodeBegin() ResetShoot(); - _mapData.ResetIsScored(); + mapData.ResetIsScored(); } public static Vector3 GetLocalPositionTarget(Difficulty difficulty) @@ -134,7 +97,7 @@ public float GetSpeed(Difficulty difficulty, Vector3 ballPosition, Vector3 targe float scale; float dist = (ballPosition - targetPosition).magnitude; float minTime = 2f; - float maxTime = Math.Max(minTime,Math.Min(_episodeLength, dist / 5f)); + float maxTime = Math.Max(minTime,Math.Min(_episodeLengthSeconds, dist / 5f)); switch (difficulty) { case Difficulty.EASY: @@ -283,150 +246,34 @@ private void ResetShoot() public override void CollectObservations(VectorSensor sensor) { //Car position - // TODO: Welche Normalisierung?? (alle /120f?) - var carXNormalized = (transform.localPosition.x + 60f) / 120f; - var carYNormalized = transform.localPosition.y / 20f; - var carZNormalized = (transform.localPosition.z + 41f) / 82f; - if (float.IsNaN(carXNormalized)) - { - Debug.Log("Car: carXNormalized == NaN"); - carXNormalized = -1f; - } - - if (float.IsNaN(carYNormalized)) - { - Debug.Log("Car: carYNormalized == NaN"); - carYNormalized = -1f; - } - - if (float.IsNaN(carZNormalized)) - { - Debug.Log("Car: carZNormalized == NaN"); - carZNormalized = -1f; - } - - sensor.AddObservation(new Vector3(carXNormalized, carYNormalized, carZNormalized)); + Vector3 carPosition = NormalizeVec(rb,VectorType.Position,EntityType.Car); + checkNormalizedVec(carPosition, "carPosition", VectorType.Position); + sensor.AddObservation(carPosition); //Car rotation, already normalized - float car_rotation_x = transform.rotation.x; - float car_rotation_y = transform.rotation.y; - float car_rotation_z = transform.rotation.z; - float car_rotation_w = transform.rotation.w; - if (float.IsNaN(car_rotation_x)) - { - Debug.Log("Car: car_rotation_x == NaN"); - car_rotation_x = -1f; - } + Quaternion carRotation = new Quaternion(transform.rotation.x, transform.rotation.y, transform.rotation.z, transform.rotation.w); + checkQuaternion(carRotation, "carRotation", -1f); + sensor.AddObservation(carRotation); - if (float.IsNaN(car_rotation_y)) - { - Debug.Log("Car: car_rotation_y == NaN"); - car_rotation_y = -1f; - } - - if (float.IsNaN(car_rotation_z)) - { - Debug.Log("Car: car_rotation_z == NaN"); - car_rotation_z = -1f; - } - - if (float.IsNaN(car_rotation_w)) - { - Debug.Log("Car: car_rotation_w == NaN"); - car_rotation_w = -1f; - } - - sensor.AddObservation(new Quaternion(car_rotation_x, car_rotation_y, car_rotation_z, car_rotation_w)); //Car velocity - Vector3 car_velocity = rb.velocity.normalized * (rb.velocity.magnitude / 23f); - if (float.IsNaN(car_velocity.x)) - { - Debug.Log("Car: car_velocity.x == NaN"); - car_velocity.x = -1f; - } - - if (float.IsNaN(car_velocity.y)) - { - Debug.Log("Car: car_velocity.y == NaN"); - car_velocity.y = -1f; - } - - if (float.IsNaN(car_velocity.z)) - { - Debug.Log("Car: car_velocity.z == NaN"); - car_velocity.z = -1f; - } - - sensor.AddObservation(car_velocity); + Vector3 carVelocity = NormalizeVec(rb, VectorType.Velocity, EntityType.Car); + checkNormalizedVec(carVelocity, "carVelocity", VectorType.Velocity); + sensor.AddObservation(carVelocity); //Car angular velocity - Vector3 car_angularVelocity = rb.angularVelocity.normalized * (rb.angularVelocity.magnitude / 5.5f); - if (float.IsNaN(car_angularVelocity.x)) - { - Debug.Log("Car: rb.angularVelocity.x == NaN"); - car_angularVelocity.x = -1f; - } - - if (float.IsNaN(car_angularVelocity.y)) - { - Debug.Log("Car: rb.angularVelocity.y == NaN"); - car_angularVelocity.y = -1f; - } - - if (float.IsNaN(car_angularVelocity.z)) - { - Debug.Log("Car: rb.angularVelocity.z == NaN"); - car_angularVelocity.z = -1f; - } - - sensor.AddObservation(car_angularVelocity); + Vector3 carAngularVelocity = NormalizeVec(rb, VectorType.AngularVelocity, EntityType.Car); + checkNormalizedVec(carAngularVelocity, "carAngularVelocity", VectorType.AngularVelocity); + sensor.AddObservation(carAngularVelocity); //Ball position - // TODO: Welche Normalisierung?? (alle /120f?) - var ballXNormalized = (_ball.localPosition.x + 60f) / 120f; - var ballYNormalized = _ball.localPosition.y / 20f; - var ballZNormalized = (_ball.localPosition.z + 41f) / 82f; - if (float.IsNaN(ballXNormalized)) - { - Debug.Log("Ball: ballXNormalized == NaN"); - ballXNormalized = -1f; - } - - if (float.IsNaN(ballYNormalized)) - { - Debug.Log("Ball: ballYNormalized == NaN"); - ballYNormalized = -1f; - } - - if (float.IsNaN(ballZNormalized)) - { - Debug.Log("Ball: ballZNormalized == NaN"); - ballZNormalized = -1f; - } - - sensor.AddObservation(new Vector3(ballXNormalized, ballYNormalized, ballZNormalized)); + Vector3 ballPosition = NormalizeVec(_rbBall, VectorType.Position, EntityType.Ball); + checkNormalizedVec(ballPosition, "ballPosition", VectorType.Position); + sensor.AddObservation(ballPosition); //Ball velocity - Vector3 ball_velocity = rb.velocity.normalized * (_rbBall.velocity.magnitude / 60f); - if (float.IsNaN(ball_velocity.x)) - { - Debug.Log("Ball: ball_velocity.x == NaN"); - ball_velocity.x = -1f; - } - - if (float.IsNaN(ball_velocity.y)) - { - Debug.Log("Ball: ball_velocity.y == NaN"); - ball_velocity.y = -1f; - } - - if (float.IsNaN(ball_velocity.z)) - { - Debug.Log("Ball: ball_velocity.z == NaN"); - ball_velocity.z = -1f; - } - - sensor.AddObservation(ball_velocity); + Vector3 ballVelocity = NormalizeVec(_rbBall, VectorType.Velocity, EntityType.Ball); + checkNormalizedVec(ballVelocity, "ballVelocity", VectorType.Velocity); + sensor.AddObservation(ballVelocity); // Boost amount float boostAmount = boostControl.boostAmount / 100f; @@ -439,35 +286,6 @@ public override void CollectObservations(VectorSensor sensor) sensor.AddObservation(boostAmount); } - public override void OnActionReceived(ActionBuffers actionBuffers) - { - if (InputManager.isAgent) - { - switch (_actionSpaceType) - { - case ActionSpaceType.Continuous: - ProcessContinuousActions(actionBuffers); - break; - case ActionSpaceType.MultiDiscrete: - ProcessMultiDiscreteActions(actionBuffers); - break; - case ActionSpaceType.Mixed: - ProcessMixedActions(actionBuffers); - break; - default: - throw new InvalidOperationException(string.Format("The method {0} does not support the {1} '{2}'.", - nameof(OnActionReceived), typeof(ActionSpaceType), _actionSpaceType.ToString())); - break; - } - } - - AssignReward(); - } - - public override void Heuristic(in ActionBuffers actionsOut) - { - InputManager.isAgent = false; - } private void Reset() { @@ -488,7 +306,7 @@ protected override void AssignReward() Reset(); } - if (_mapData.isScoredOrange) + if (mapData.isScoredOrange) { // Agent got scored on SetReward(-1f); @@ -496,172 +314,6 @@ protected override void AssignReward() } } - /// - /// Processes the actions, if is . - /// - /// The action buffers containing the actions. - private void ProcessContinuousActions(ActionBuffers actionBuffers) - { - // set inputs - InputManager.throttleInput = actionBuffers.ContinuousActions[0]; - InputManager.steerInput = actionBuffers.ContinuousActions[1]; - InputManager.yawInput = actionBuffers.ContinuousActions[1]; - InputManager.pitchInput = actionBuffers.ContinuousActions[2]; - InputManager.rollInput = 0; - if (actionBuffers.ContinuousActions[3] > 0) InputManager.rollInput = 1; - if (actionBuffers.ContinuousActions[3] < 0) InputManager.rollInput = -1; - - InputManager.isBoost = actionBuffers.ContinuousActions[4] > 0; - InputManager.isDrift = actionBuffers.ContinuousActions[5] > 0; - InputManager.isAirRoll = actionBuffers.ContinuousActions[6] > 0; - - InputManager.isJump = actionBuffers.ContinuousActions[7] > 0; - } - - /// - /// Processes the actions, if is . - /// - /// The action buffers containing the actions. - private void ProcessMultiDiscreteActions(ActionBuffers actionBuffers) - { - InputManager.throttleInput = DISCRETE_ACTIONS[actionBuffers.DiscreteActions[0]]; - InputManager.steerInput = DISCRETE_ACTIONS[actionBuffers.DiscreteActions[1]]; - InputManager.yawInput = DISCRETE_ACTIONS[actionBuffers.DiscreteActions[1]]; - InputManager.pitchInput = DISCRETE_ACTIONS[actionBuffers.DiscreteActions[2]]; - switch (actionBuffers.DiscreteActions[3]) - { - case 0: - InputManager.rollInput = -1; - break; - case 2: - InputManager.rollInput = 1; - break; - default: - InputManager.rollInput = 0; - break; - } - - InputManager.isBoost = actionBuffers.DiscreteActions[4] > 0; - InputManager.isDrift = actionBuffers.DiscreteActions[5] > 0; - InputManager.isAirRoll = actionBuffers.DiscreteActions[6] > 0; - InputManager.isJump = actionBuffers.DiscreteActions[7] > 0; - } - - /// - /// Processes the actions, if is . - /// - /// The action buffers containing the actions. - private void ProcessMixedActions(ActionBuffers actionBuffers) - { - //Continuous actions - InputManager.throttleInput = actionBuffers.ContinuousActions[0]; - InputManager.steerInput = actionBuffers.ContinuousActions[1]; - InputManager.yawInput = actionBuffers.ContinuousActions[1]; - InputManager.pitchInput = actionBuffers.ContinuousActions[2]; - - //Discrete actions - switch (actionBuffers.DiscreteActions[0]) - { - case 0: - InputManager.rollInput = -1; - break; - case 2: - InputManager.rollInput = 1; - break; - default: - InputManager.rollInput = 0; - break; - } - - InputManager.isBoost = actionBuffers.DiscreteActions[1] > 0; - InputManager.isDrift = actionBuffers.DiscreteActions[2] > 0; - InputManager.isAirRoll = actionBuffers.DiscreteActions[3] > 0; - InputManager.isJump = actionBuffers.DiscreteActions[4] > 0; - } - - /// - /// Determines the action space type for the specified . - /// - /// The for which we want to determine the action sapce type. - /// The corresponding action space type. - private ActionSpaceType DetermineActionSpaceType(ActionSpec actionSpec) - { - int requiredNumActions = 8; - - // Determine action space type - if (actionSpec.NumContinuousActions > 0 && actionSpec.NumDiscreteActions == 0) - { - //Propably continuous, we check the size - if (actionSpec.NumContinuousActions != requiredNumActions) - { - throw new ArgumentException(string.Format( - "It seems like you tried to use a continuos action space for the agent. In this case the {0} needs 8 continuous actions.", - typeof(GoalKeeperAgent))); - } - - return ActionSpaceType.Continuous; - } - else if (actionSpec.NumContinuousActions == 0 && actionSpec.NumDiscreteActions > 0) - { - //Propably multi discrete, we check the size - if (actionSpec.NumDiscreteActions != requiredNumActions) - { - throw new ArgumentException(string.Format( - "It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches.", - typeof(GoalKeeperAgent))); - } - - int[] requiredBranchSizes = - { DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, DISCRETE_ACTIONS.Length, 3, 2, 2, 2, 2 }; - for (int i = 0; i < actionSpec.BranchSizes.Length; i++) - { - if (actionSpec.BranchSizes[i] != requiredBranchSizes[i]) - { - throw new ArgumentException(string.Format( - "It seems like you tried to use a multi-discrete action space for the agent. In this case the {0} needs 8 discrete action branches with sizes ({1}).", - typeof(GoalKeeperAgent), - string.Join(", ", requiredBranchSizes))); - } - } - - return ActionSpaceType.MultiDiscrete; - } - else - { - //Propably multi discrete, we check the size - if (actionSpec.NumDiscreteActions != 5 || actionSpec.NumContinuousActions != 3) - { - throw new ArgumentException(string.Format( - "It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches and 3 continuous actions.", - typeof(GoalKeeperAgent))); - } - - int[] requiredBranchSizes = { 3, 2, 2, 2, 2 }; - for (int i = 0; i < actionSpec.BranchSizes.Length; i++) - { - if (actionSpec.BranchSizes[i] != requiredBranchSizes[i]) - { - throw new ArgumentException(string.Format( - "It seems like you tried to use a mixed action space for the agent. In this case the {0} needs 5 discrete action branches with sizes ({1}).", - typeof(GoalKeeperAgent), - string.Join(", ", requiredBranchSizes))); - } - } - - return ActionSpaceType.Mixed; - } - } - - /// - /// Helper for classifying the action space. - /// - private enum ActionSpaceType - { - Continuous, - MultiDiscrete, - Mixed - } - public enum Difficulty { EASY, From 2f46b76f715fadfe7bdfa07f73b7328edae312e6 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 13:31:47 +0100 Subject: [PATCH 13/25] shootball applies the velocity directly now and doesn't call ShootTarget() inside Start() --- Assets/Scripts/Ball/ShootBall.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Assets/Scripts/Ball/ShootBall.cs b/Assets/Scripts/Ball/ShootBall.cs index a1b260c..5fa2cf7 100644 --- a/Assets/Scripts/Ball/ShootBall.cs +++ b/Assets/Scripts/Ball/ShootBall.cs @@ -14,7 +14,6 @@ public class ShootBall : MonoBehaviour void Start() { _rb = GetComponent(); - ShootTarget(); } @@ -26,6 +25,6 @@ public void ShootTarget(float velocity = 0.0f) velocity = Random.Range(speed.x, speed.y); } - _rb.AddForce( dir.normalized * velocity, ForceMode.VelocityChange); + _rb.velocity = dir.normalized * velocity; } } From 67d5877adde81bbf46cdfd6800a4e593bdd60e56 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 13:35:44 +0100 Subject: [PATCH 14/25] added relative position to the observation space --- .../ML-Agents/Goalkeeper/GoalKeeperAgent.cs | 7 +- Assets/ML-Agents/Topscorer/Environment.prefab | 66 +++++++++++-------- .../Topscorer/Topscorer_single.unity | 20 ------ 3 files changed, 43 insertions(+), 50 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs index b7fc2bf..3b5d442 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalKeeperAgent.cs @@ -282,8 +282,13 @@ public override void CollectObservations(VectorSensor sensor) Debug.Log("Car: boostAmount == NaN"); boostAmount = -1f; } - sensor.AddObservation(boostAmount); + + // Relative position car to ball + float relativeXPos = ((_ball.localPosition.x + 60f) - (transform.localPosition.x + 60f)) / 120f; + float relativeYPos = (_ball.localPosition.y - transform.localPosition.y) / 20f; + float relativeZPos = ((_ball.localPosition.z + 41f) - (transform.localPosition.z + 41f)) / 82f; + sensor.AddObservation(new Vector3(relativeXPos, relativeYPos, relativeZPos)); } diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index e29ed7d..9345ac1 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -335,18 +335,6 @@ AudioListener: m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 6750119729059262998} m_Enabled: 0 ---- !u!114 &3243343760869561856 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 6750119729296308303} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: d2fcec0097665e74fbec36bfc73d7dc7, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &6750119729343506587 GameObject: m_ObjectHideFlags: 0 @@ -508,20 +496,6 @@ MonoBehaviour: m_Calls: [] m_LegacyBlendHint: 0 m_ComponentOwner: {fileID: 6750119729343506586} ---- !u!114 &6750119729572142156 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 9118461746507665664} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 88304332db05f5e4b80c2152484c2fa3, type: 3} - m_Name: - m_EditorClassIdentifier: - ShootAt: {fileID: 6750119728131920436} - speed: {x: 5, y: 5} --- !u!1001 &4127686330414560867 PrefabInstance: m_ObjectHideFlags: 0 @@ -532,10 +506,18 @@ PrefabInstance: - target: {fileID: 7270290807663857190, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: m_Model value: + objectReference: {fileID: 5022602860645237092, guid: b56addff173f3da41b4fa25dc61fed23, type: 3} + - target: {fileID: 7270290807663857190, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} + propertyPath: m_BrainParameters.VectorObservationSize + value: 23 + objectReference: {fileID: 0} + - target: {fileID: 7270290807663857192, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} + propertyPath: DecisionPeriod + value: 5 objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: Difficulty - value: 0 + value: 2 objectReference: {fileID: 0} - target: {fileID: 7270290807663857196, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: m_Name @@ -597,6 +579,18 @@ Transform: m_CorrespondingSourceObject: {fileID: 7270290807663857197, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} m_PrefabInstance: {fileID: 4127686330414560867} m_PrefabAsset: {fileID: 0} +--- !u!114 &3243343760869561856 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 6750119729296308303} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d2fcec0097665e74fbec36bfc73d7dc7, type: 3} + m_Name: + m_EditorClassIdentifier: --- !u!1001 &6750119728108799112 PrefabInstance: m_ObjectHideFlags: 0 @@ -724,7 +718,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 4104392332257108266, guid: 548a302a7fc95254a999ba2811266cf8, type: 3} propertyPath: m_IsActive - value: 0 + value: 1 objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: 548a302a7fc95254a999ba2811266cf8, type: 3} @@ -868,7 +862,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 2532842153811578196, guid: 2946bfcea95aa1d40872741310581650, type: 3} propertyPath: m_LocalPosition.x - value: 44.27 + value: -5.6 objectReference: {fileID: 0} - target: {fileID: 2532842153811578196, guid: 2946bfcea95aa1d40872741310581650, type: 3} propertyPath: m_LocalPosition.y @@ -919,3 +913,17 @@ Transform: m_CorrespondingSourceObject: {fileID: 2532842153811578196, guid: 2946bfcea95aa1d40872741310581650, type: 3} m_PrefabInstance: {fileID: 6750119729572142163} m_PrefabAsset: {fileID: 0} +--- !u!114 &6750119729572142156 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 9118461746507665664} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 88304332db05f5e4b80c2152484c2fa3, type: 3} + m_Name: + m_EditorClassIdentifier: + ShootAt: {fileID: 6750119728131920436} + speed: {x: 5, y: 5} diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index afdce11..7a6e5f3 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -271,25 +271,5 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} - - target: {fileID: 6750119729296308293, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: m_Model - value: - objectReference: {fileID: 5022602860645237092, guid: b56addff173f3da41b4fa25dc61fed23, type: 3} - - target: {fileID: 6750119729296308293, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: m_BrainParameters.NumStackedVectorObservations - value: 1 - objectReference: {fileID: 0} - - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: Difficulty - value: 2 - objectReference: {fileID: 0} - - target: {fileID: 7302733262041547440, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: m_IsActive - value: 1 - objectReference: {fileID: 0} - - target: {fileID: 9118461746507665671, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: m_LocalPosition.x - value: -5.6 - objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} From 109ff02ea9efba2a304bb4c4c09665063135e7bc Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 13:36:16 +0100 Subject: [PATCH 15/25] ++ --- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index b1ab8f8..5b553da 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -197,6 +197,12 @@ public override void CollectObservations(VectorSensor sensor) sensor.AddObservation(ball_position); sensor.AddObservation(ball_velocity); sensor.AddObservation(boostAmount); + + // Relative position car to ball + float relativeXPos = ((_ball.localPosition.x + 60f) - (transform.localPosition.x + 60f)) / 120f; + float relativeYPos = (_ball.localPosition.y - transform.localPosition.y) / 20f; + float relativeZPos = ((_ball.localPosition.z + 41f) - (transform.localPosition.z + 41f)) / 82f; + sensor.AddObservation(new Vector3(relativeXPos, relativeYPos, relativeZPos)); } private void Reset() From 77a84c3692fd6eb8241d9b8022ec85b8e908683d Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 13:47:03 +0100 Subject: [PATCH 16/25] (WIP) added the goalie reset parameters to the striker agent --- .../Goalkeeper/GoalkeeperEnvironmentParameters.cs | 5 +++-- .../Goalkeeper/GoalkeeperEvironmentHandler.cs | 2 +- Assets/ML-Agents/Topscorer/Environment.prefab | 8 ++++++++ Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 12 ++++++++++-- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs index 924aadd..86daa42 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEnvironmentParameters.cs @@ -7,19 +7,20 @@ public class GoalkeeperEnvironmentParameters { public GoalkeeperEnvironmentParameters() { + seed = -1; difficulty = 0; initialBoost = 32; canBoost = 1; canDoubleJump = 1; canDrift = 1; - canDoubleJump = 1; useBulletImpulse = 1; usePsyonixImpulse = 1; useSuspension = 1; useCustomBounce = 1; useWallStabilization = 1; useGroundStabilization = 1; - } + } + public float seed; public float difficulty; public float initialBoost; public float canDoubleJump; diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs index f708f8a..47908ec 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs @@ -16,7 +16,7 @@ public override void ResetParameter() { UpdateEnvironmentParameters(); //TODO after merge with difficulty, add difficulty parameter - environment.GetComponentInChildren().difficulty = currentParameter.difficulty; + //environment.GetComponentInChildren().difficulty = currentParameter.difficulty; if (currentParameter.canDoubleJump == 0) { environment.GetComponentInChildren().disableDoubleJump = true; diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index 9345ac1..9bcd680 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -515,10 +515,18 @@ PrefabInstance: propertyPath: DecisionPeriod value: 5 objectReference: {fileID: 0} + - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} + propertyPath: MaxStep + value: 2000 + objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: Difficulty value: 2 objectReference: {fileID: 0} + - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} + propertyPath: defaultParameter.difficulty + value: 2 + objectReference: {fileID: 0} - target: {fileID: 7270290807663857196, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: m_Name value: TopScorerAgent diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index 5b553da..06b3e29 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -6,11 +6,14 @@ using Unity.MLAgents.Sensors; using Unity.MLAgents.Actuators; using Unity.MLAgents.Policies; +using ML_Agents.Goalkeeper; public class TopScorerAgent : PGBaseAgent { - // Start is called before the first frame update + [SerializeField] public GoalkeeperEnvironmentParameters defaultParameter; + + private GoalkeeperEvironmentHandler _handler; public int Difficulty = 0; private Rigidbody rbBall; @@ -26,7 +29,8 @@ public class TopScorerAgent : PGBaseAgent protected override void Start() { base.Start(); - _ball = transform.parent.Find("Ball"); + _handler = new GoalkeeperEvironmentHandler(transform.root.gameObject, defaultParameter); + _ball = _handler.environment.GetComponentInChildren().transform; rbBall = _ball.GetComponent(); ball = _ball.GetComponent(); @@ -34,10 +38,14 @@ protected override void Start() _shootAt = transform.parent.Find("ShootAt"); _lastResetTime = Time.time; + + _handler.UpdateEnvironmentParameters(); + _handler.ResetParameter(); } public override void OnEpisodeBegin() { + _handler.ResetParameter(); var diff = UnityEngine.Random.Range(0, 200) % 4; switch (diff) { From e495a7d75cd1cd81675cfa9d64bfb78d59cbac1a Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 13:52:26 +0100 Subject: [PATCH 17/25] fixed seed is working now --- .../Goalkeeper/GoalkeeperEvironmentHandler.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs index 47908ec..b50db16 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs @@ -8,6 +8,7 @@ namespace ML_Agents.Goalkeeper [Serializable] public class GoalkeeperEvironmentHandler: EnvironmentHandler { + public int activeSeed = -1; public GoalkeeperEvironmentHandler(GameObject env, GoalkeeperEnvironmentParameters defaultParameter) : base(env, defaultParameter) { } @@ -15,6 +16,19 @@ public GoalkeeperEvironmentHandler(GameObject env, GoalkeeperEnvironmentParamete public override void ResetParameter() { UpdateEnvironmentParameters(); + if (currentParameter.seed >= 0) + { + activeSeed = (int)currentParameter.seed; + UnityEngine.Random.InitState((int)currentParameter.seed); + Debug.Log("fixed seed!"); + } + else + { + System.Random rand = new System.Random(); + // Sample a seed from the range 0 to 999 + activeSeed = rand.Next(1000); + UnityEngine.Random.InitState(activeSeed); + } //TODO after merge with difficulty, add difficulty parameter //environment.GetComponentInChildren().difficulty = currentParameter.difficulty; if (currentParameter.canDoubleJump == 0) From f201834226a6a34da8827d67f6e0d7074fab865b Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 17 Feb 2022 18:07:06 +0100 Subject: [PATCH 18/25] removed rewards except the goal scored one, difficulty is set to 2 --- .../Goalkeeper/GoalkeeperEvironmentHandler.cs | 1 - Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 84 ++++++++++--------- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs index b50db16..8663231 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs @@ -20,7 +20,6 @@ public override void ResetParameter() { activeSeed = (int)currentParameter.seed; UnityEngine.Random.InitState((int)currentParameter.seed); - Debug.Log("fixed seed!"); } else { diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index 06b3e29..5442121 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -46,15 +46,16 @@ protected override void Start() public override void OnEpisodeBegin() { _handler.ResetParameter(); - var diff = UnityEngine.Random.Range(0, 200) % 4; - switch (diff) - { - case 0: OnEpisodeBeginDifficulty0(); break; - case 1: OnEpisodeBeginDifficulty1(); break; - case 2: OnEpisodeBeginDifficulty2(); break; - case 3: OnEpisodeBeginDifficultyDefault(); break; - default: throw new Exception("Difficulty does not exist"); - } + // var diff = UnityEngine.Random.Range(0, 200) % 4; + // switch (diff) + // { + // case 0: OnEpisodeBeginDifficulty0(); break; + // case 1: OnEpisodeBeginDifficulty1(); break; + // case 2: OnEpisodeBeginDifficulty2(); break; + // case 3: OnEpisodeBeginDifficultyDefault(); break; + // default: throw new Exception("Difficulty does not exist"); + // } + OnEpisodeBeginDifficulty2(); ball.ResetValues(); mapData.ResetIsScored(); SetReward(0f); @@ -94,7 +95,7 @@ private void OnEpisodeBeginDifficulty2() Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), 100f); - + //Reset Ball float ball_z_pos = UnityEngine.Random.Range(0, 9) % 2 == 0 ? 6f : -6f; _ball.localPosition = new Vector3(45f, UnityEngine.Random.Range(2f, 5f), ball_z_pos + UnityEngine.Random.Range(-1f, 1f)); @@ -215,7 +216,7 @@ public override void CollectObservations(VectorSensor sensor) private void Reset() { - Debug.Log(GetCumulativeReward()); + // Debug.Log(GetCumulativeReward()); _lastResetTime = Time.time; EndEpisode(); } @@ -225,7 +226,7 @@ private void Reset() /// protected override void AssignReward() { - AddReward(-(1 / _maxStepsPerEpisode)); + // AddReward(-(1 / _maxStepsPerEpisode)); if (StepCount > _maxStepsPerEpisode)// || rb.position.x > rbBall.position.x + 5.0f) { // Agent didn't score a goal @@ -235,8 +236,9 @@ protected override void AssignReward() else { // AddShortEpisodeReward(-0.2f); - float agentBallDistanceReward = 0.00025f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag)); - AddReward(agentBallDistanceReward); + // float agentBallDistanceReward = 0.00025f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag)); + //Debug.Log(agentBallDistanceReward); + // AddReward(agentBallDistanceReward); if (mapData.isScoredBlue) { @@ -247,31 +249,31 @@ protected override void AssignReward() } } - private void OnCollisionEnter(Collision other) - { - if (other.gameObject.tag.Equals("Ball")) - { - AddReward(0.1f); - } - } - - private void AddShortEpisodeReward(float factor) - { - // adds a reward in range [0, factor] - if (StepCount > _maxStepsPerEpisode) - { - return; - } - AddReward((1f - (StepCount / _maxStepsPerEpisode)) * factor); - } - - private void AddFastShotReward(float factor) - { - // adds a reward in range [0, factor] - if (Time.time - _lastResetTime > _episodeLength) - { - return; - } - AddReward((rbBall.velocity.magnitude / _ball.GetComponent().maxVelocity) * factor); - } + // private void OnCollisionEnter(Collision other) + // { + // if (other.gameObject.tag.Equals("Ball")) + // { + // AddReward(0.1f); + // } + // } + + // private void AddShortEpisodeReward(float factor) + // { + // // adds a reward in range [0, factor] + // if (StepCount > _maxStepsPerEpisode) + // { + // return; + // } + // AddReward((1f - (StepCount / _maxStepsPerEpisode)) * factor); + // } + + // private void AddFastShotReward(float factor) + // { + // // adds a reward in range [0, factor] + // if (Time.time - _lastResetTime > _episodeLength) + // { + // return; + // } + // AddReward((rbBall.velocity.magnitude / _ball.GetComponent().maxVelocity) * factor); + // } } From b7dded82294c4d88797d2dc077fe53daf07056da Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Fri, 18 Feb 2022 12:53:02 +0100 Subject: [PATCH 19/25] set difficulty to 3 --- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 3 +-- Assets/ML-Agents/Topscorer/Topscorer_single.unity | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index 5442121..a1fcca4 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -39,7 +39,6 @@ protected override void Start() _lastResetTime = Time.time; - _handler.UpdateEnvironmentParameters(); _handler.ResetParameter(); } @@ -55,7 +54,7 @@ public override void OnEpisodeBegin() // case 3: OnEpisodeBeginDifficultyDefault(); break; // default: throw new Exception("Difficulty does not exist"); // } - OnEpisodeBeginDifficulty2(); + OnEpisodeBeginDifficultyDefault(); ball.ResetValues(); mapData.ResetIsScored(); SetReward(0f); diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index 7a6e5f3..9f1033e 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -271,5 +271,9 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} + - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} + propertyPath: defaultParameter.seed + value: -1 + objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} From cfca777362d4adb0788b323fab3361f8783d85d8 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Fri, 18 Feb 2022 17:55:42 +0100 Subject: [PATCH 20/25] max steps terminates the episode automatically, adjusted difficulty 3 --- Assets/ML-Agents/Topscorer/Environment.prefab | 8 +++-- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 30 +++++-------------- .../Topscorer/Topscorer_single.unity | 4 --- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index 9bcd680..3bd227d 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -517,7 +517,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: MaxStep - value: 2000 + value: 1000 objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: Difficulty @@ -527,6 +527,10 @@ PrefabInstance: propertyPath: defaultParameter.difficulty value: 2 objectReference: {fileID: 0} + - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} + propertyPath: defaultParameter.initialBoost + value: 32 + objectReference: {fileID: 0} - target: {fileID: 7270290807663857196, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: m_Name value: TopScorerAgent @@ -934,4 +938,4 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: ShootAt: {fileID: 6750119728131920436} - speed: {x: 5, y: 5} + speed: {x: 5, y: 10} diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index a1fcca4..9cd8adf 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -19,10 +19,6 @@ public class TopScorerAgent : PGBaseAgent private Rigidbody rbBall; private Ball ball; - private static float _episodeLength = 5f; - private static float _maxStepsPerEpisode = 120f * _episodeLength; - private float _lastResetTime; - private Transform _ball, _shootAt; private Vector3 _midFieldPosition; @@ -37,8 +33,6 @@ protected override void Start() _midFieldPosition = Vector3.zero; _shootAt = transform.parent.Find("ShootAt"); - _lastResetTime = Time.time; - _handler.ResetParameter(); } @@ -112,12 +106,12 @@ private void OnEpisodeBeginDifficulty2() private void OnEpisodeBeginDifficultyDefault() { //Reset Car - Vector3 startPosition = _midFieldPosition + new Vector3(0f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); + Vector3 startPosition = _midFieldPosition + new Vector3(20f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f + UnityEngine.Random.Range(-20f, 20f), 0f), 100f); //Reset Ball - _ball.localPosition = new Vector3(UnityEngine.Random.Range(30f, 50f), UnityEngine.Random.Range(0f, 10f), UnityEngine.Random.Range(-20f, 20f)); + _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.5f, 12f), UnityEngine.Random.Range(-20f, 20f)); //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; @@ -216,7 +210,6 @@ public override void CollectObservations(VectorSensor sensor) private void Reset() { // Debug.Log(GetCumulativeReward()); - _lastResetTime = Time.time; EndEpisode(); } @@ -226,25 +219,16 @@ private void Reset() protected override void AssignReward() { // AddReward(-(1 / _maxStepsPerEpisode)); - if (StepCount > _maxStepsPerEpisode)// || rb.position.x > rbBall.position.x + 5.0f) - { - // Agent didn't score a goal - // AddReward(-1f); - Reset(); - } - else - { // AddShortEpisodeReward(-0.2f); // float agentBallDistanceReward = 0.00025f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag)); //Debug.Log(agentBallDistanceReward); // AddReward(agentBallDistanceReward); - if (mapData.isScoredBlue) - { - // Agent scored a goal - SetReward(1f); - Reset(); - } + if (mapData.isScoredBlue) + { + // Agent scored a goal + SetReward(1f); + Reset(); } } diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index 9f1033e..7a6e5f3 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -271,9 +271,5 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} - - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: defaultParameter.seed - value: -1 - objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} From 0d7227f682de9f381828417b215dea99cdf3d006 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Sat, 19 Feb 2022 15:28:44 +0100 Subject: [PATCH 21/25] fixed boostAmount and its reset parameter, reduced max steps, work on reward function of striker --- .../Goalkeeper/GoalkeeperEvironmentHandler.cs | 2 + Assets/ML-Agents/Topscorer/Environment.prefab | 4 +- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 78 ++++++++++--------- .../Topscorer/Topscorer_single.unity | 8 ++ .../CubeController/CubeController.cs | 2 +- 5 files changed, 53 insertions(+), 41 deletions(-) diff --git a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs index 8663231..274677b 100644 --- a/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs +++ b/Assets/ML-Agents/Goalkeeper/GoalkeeperEvironmentHandler.cs @@ -9,6 +9,7 @@ namespace ML_Agents.Goalkeeper public class GoalkeeperEvironmentHandler: EnvironmentHandler { public int activeSeed = -1; + public float boost_amount = 32.0f; public GoalkeeperEvironmentHandler(GameObject env, GoalkeeperEnvironmentParameters defaultParameter) : base(env, defaultParameter) { } @@ -41,6 +42,7 @@ public override void ResetParameter() else { environment.GetComponentInChildren().boostAmount = currentParameter.initialBoost; + boost_amount = currentParameter.initialBoost; } if (currentParameter.canDrift == 0) { diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index 3bd227d..31187df 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -517,7 +517,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: MaxStep - value: 1000 + value: 800 objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: Difficulty @@ -529,7 +529,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: defaultParameter.initialBoost - value: 32 + value: 100 objectReference: {fileID: 0} - target: {fileID: 7270290807663857196, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: m_Name diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index 9cd8adf..f408b8a 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -21,6 +21,8 @@ public class TopScorerAgent : PGBaseAgent private Transform _ball, _shootAt; private Vector3 _midFieldPosition; + private bool _ballTouched = false; + private float _minDistance = 0.0f; protected override void Start() { @@ -39,6 +41,7 @@ protected override void Start() public override void OnEpisodeBegin() { _handler.ResetParameter(); + _ballTouched = false; // var diff = UnityEngine.Random.Range(0, 200) % 4; // switch (diff) // { @@ -57,7 +60,7 @@ public override void OnEpisodeBegin() private void OnEpisodeBeginDifficulty0() { Vector3 startPosition = _midFieldPosition + new Vector3(30f, 0.17f, 0f); - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), 100f); + controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); _ball.localPosition = new Vector3(45f, 0.9315f, UnityEngine.Random.Range(-5f, 5f)); rbBall.velocity = Vector3.zero; @@ -69,7 +72,7 @@ private void OnEpisodeBeginDifficulty1() // Reset Car Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-5f, 5f)); - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), 100f); + controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); //Reset Ball _ball.localPosition = new Vector3(45f, 0.93f, UnityEngine.Random.Range(1, 10) % 2 == 0 ? -7f : 7f); @@ -86,7 +89,7 @@ private void OnEpisodeBeginDifficulty1() private void OnEpisodeBeginDifficulty2() { Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), 100f); + controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); //Reset Ball @@ -108,7 +111,7 @@ private void OnEpisodeBeginDifficultyDefault() //Reset Car Vector3 startPosition = _midFieldPosition + new Vector3(20f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f + UnityEngine.Random.Range(-20f, 20f), 0f), 100f); + controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f + UnityEngine.Random.Range(-20f, 20f), 0f), _handler.boost_amount); //Reset Ball _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.5f, 12f), UnityEngine.Random.Range(-20f, 20f)); @@ -121,6 +124,7 @@ private void OnEpisodeBeginDifficultyDefault() //Throw Ball _ball.GetComponent().ShootTarget(); + _minDistance = Vector3.Distance(_ball.position, transform.position); } private void resetFromErrorState() @@ -218,45 +222,43 @@ private void Reset() /// protected override void AssignReward() { - // AddReward(-(1 / _maxStepsPerEpisode)); - // AddShortEpisodeReward(-0.2f); - // float agentBallDistanceReward = 0.00025f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag)); - //Debug.Log(agentBallDistanceReward); - // AddReward(agentBallDistanceReward); - + if (!_ballTouched) + { + AddReward(-0.01f / 800f * 5f); + } if (mapData.isScoredBlue) { // Agent scored a goal - SetReward(1f); + AddReward(1f); Reset(); } } - // private void OnCollisionEnter(Collision other) - // { - // if (other.gameObject.tag.Equals("Ball")) - // { - // AddReward(0.1f); - // } - // } - - // private void AddShortEpisodeReward(float factor) - // { - // // adds a reward in range [0, factor] - // if (StepCount > _maxStepsPerEpisode) - // { - // return; - // } - // AddReward((1f - (StepCount / _maxStepsPerEpisode)) * factor); - // } - - // private void AddFastShotReward(float factor) - // { - // // adds a reward in range [0, factor] - // if (Time.time - _lastResetTime > _episodeLength) - // { - // return; - // } - // AddReward((rbBall.velocity.magnitude / _ball.GetComponent().maxVelocity) * factor); - // } + private void OnCollisionEnter(Collision other) + { + if (other.gameObject.tag.Equals("Ball")) + { + if (!_ballTouched) + { + AddReward(0.1f); + Debug.Log(GetCumulativeReward()); + _ballTouched = true; + } + } + } + + private void RewardCloseness() + { + float currentDistance = Vector3.Distance(_ball.position, transform.position); + if (currentDistance < _minDistance) + { + _minDistance = currentDistance; + if (!_ballTouched) + { + float agentBallDistanceReward = 0.0001f * Mathf.Pow((1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag)), 0.5f); + //Debug.Log(agentBallDistanceReward); + AddReward(agentBallDistanceReward); + } + } + } } diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index 7a6e5f3..bc0e980 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -271,5 +271,13 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} + - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} + propertyPath: MaxStep + value: 800 + objectReference: {fileID: 0} + - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} + propertyPath: defaultParameter.initialBoost + value: 100 + objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} diff --git a/Assets/Scripts/CarControllers/CubeController/CubeController.cs b/Assets/Scripts/CarControllers/CubeController/CubeController.cs index 5a60214..34582ee 100644 --- a/Assets/Scripts/CarControllers/CubeController/CubeController.cs +++ b/Assets/Scripts/CarControllers/CubeController/CubeController.cs @@ -76,7 +76,7 @@ public void ResetCar(Vector3 position, Quaternion rotation, float boost = 33f) _rb.angularVelocity = Vector3.zero; GetComponent().Reset(); - GetComponent().boostAmount = 32f; + GetComponent().boostAmount = boost; } private void UpdateCarVariables() From 501e6d4bb1804a5521f3a4a31a6ebe60ef855d0f Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Sat, 19 Feb 2022 16:45:43 +0100 Subject: [PATCH 22/25] ++ --- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 10 +++++----- Assets/ML-Agents/Topscorer/Topscorer_single.unity | 8 -------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index f408b8a..ab8d2c6 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -222,10 +222,10 @@ private void Reset() /// protected override void AssignReward() { - if (!_ballTouched) - { - AddReward(-0.01f / 800f * 5f); - } + // if (!_ballTouched) + // { + // AddReward(-0.01f / 800f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag))); + // } if (mapData.isScoredBlue) { // Agent scored a goal @@ -241,7 +241,7 @@ private void OnCollisionEnter(Collision other) if (!_ballTouched) { AddReward(0.1f); - Debug.Log(GetCumulativeReward()); + // Debug.Log(GetCumulativeReward()); _ballTouched = true; } } diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index bc0e980..7a6e5f3 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -271,13 +271,5 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} - - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: MaxStep - value: 800 - objectReference: {fileID: 0} - - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} - propertyPath: defaultParameter.initialBoost - value: 100 - objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} From 578dfae6517e14665bb1209809dab6cfb6fe7796 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Sat, 19 Feb 2022 19:39:03 +0100 Subject: [PATCH 23/25] difficulty 2 and default are mixed and slightly adjusted now --- Assets/ML-Agents/Topscorer/Environment.prefab | 2 +- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 65 ++++++------------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index 31187df..4e6c0ed 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -938,4 +938,4 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: ShootAt: {fileID: 6750119728131920436} - speed: {x: 5, y: 10} + speed: {x: 5, y: 8} diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index ab8d2c6..58f4e69 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -42,53 +42,22 @@ public override void OnEpisodeBegin() { _handler.ResetParameter(); _ballTouched = false; - // var diff = UnityEngine.Random.Range(0, 200) % 4; - // switch (diff) - // { - // case 0: OnEpisodeBeginDifficulty0(); break; - // case 1: OnEpisodeBeginDifficulty1(); break; - // case 2: OnEpisodeBeginDifficulty2(); break; - // case 3: OnEpisodeBeginDifficultyDefault(); break; - // default: throw new Exception("Difficulty does not exist"); - // } - OnEpisodeBeginDifficultyDefault(); + var diff = UnityEngine.Random.Range(0, 200) % 2; + switch (diff) + { + case 0: OnEpisodeBeginDifficulty2(); break; + case 1: OnEpisodeBeginDifficultyDefault(); break; + default: throw new Exception("Difficulty does not exist"); + } + // OnEpisodeBeginDifficultyDefault(); ball.ResetValues(); mapData.ResetIsScored(); SetReward(0f); } - private void OnEpisodeBeginDifficulty0() - { - Vector3 startPosition = _midFieldPosition + new Vector3(30f, 0.17f, 0f); - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); - - _ball.localPosition = new Vector3(45f, 0.9315f, UnityEngine.Random.Range(-5f, 5f)); - rbBall.velocity = Vector3.zero; - rbBall.angularVelocity = Vector3.zero; - } - - private void OnEpisodeBeginDifficulty1() - { - // Reset Car - Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-5f, 5f)); - - controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); - - //Reset Ball - _ball.localPosition = new Vector3(45f, 0.93f, UnityEngine.Random.Range(1, 10) % 2 == 0 ? -7f : 7f); - _ball.GetComponent().velocity = Vector3.zero; - _ball.GetComponent().angularVelocity = Vector3.zero; - - // Set new Taget Position - _shootAt.localPosition = new Vector3(_ball.localPosition.x, 0f, 0f); - - //Throw Ball - _ball.GetComponent().ShootTarget(); - } - private void OnEpisodeBeginDifficulty2() { - Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); + Vector3 startPosition = _midFieldPosition + new Vector3(UnityEngine.Random.Range(20f, 25f), 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); @@ -109,12 +78,12 @@ private void OnEpisodeBeginDifficulty2() private void OnEpisodeBeginDifficultyDefault() { //Reset Car - Vector3 startPosition = _midFieldPosition + new Vector3(20f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); + Vector3 startPosition = _midFieldPosition + new Vector3(UnityEngine.Random.Range(20f, 25f), 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f + UnityEngine.Random.Range(-20f, 20f), 0f), _handler.boost_amount); //Reset Ball - _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.5f, 12f), UnityEngine.Random.Range(-20f, 20f)); + _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.75f, 13f), UnityEngine.Random.Range(-20f, 20f)); //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; @@ -222,10 +191,6 @@ private void Reset() /// protected override void AssignReward() { - // if (!_ballTouched) - // { - // AddReward(-0.01f / 800f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag))); - // } if (mapData.isScoredBlue) { // Agent scored a goal @@ -261,4 +226,12 @@ private void RewardCloseness() } } } + + private void PenalizeDistance() + { + if (!_ballTouched) + { + AddReward(-0.01f / 800f * (1 - (Vector3.Distance(_ball.position, transform.position) / mapData.diag))); + } + } } From 5a87595407b83e2d709c57664345a919fce8b02b Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Sun, 20 Feb 2022 06:47:53 +0100 Subject: [PATCH 24/25] reduced max episode length, diffuclty 2 is more varied, added seed text UI --- Assets/ML-Agents/Topscorer/Environment.prefab | 2 +- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 10 +- .../Topscorer/Topscorer_single.unity | 183 ++++++++++++++++++ 3 files changed, 191 insertions(+), 4 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/Environment.prefab b/Assets/ML-Agents/Topscorer/Environment.prefab index 4e6c0ed..b77aa7e 100644 --- a/Assets/ML-Agents/Topscorer/Environment.prefab +++ b/Assets/ML-Agents/Topscorer/Environment.prefab @@ -517,7 +517,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: MaxStep - value: 800 + value: 650 objectReference: {fileID: 0} - target: {fileID: 7270290807663857193, guid: bfdff73714e4393468e3c7968d554ba5, type: 3} propertyPath: Difficulty diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index 58f4e69..c97cc74 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -7,7 +7,7 @@ using Unity.MLAgents.Actuators; using Unity.MLAgents.Policies; using ML_Agents.Goalkeeper; - +using UnityEngine.UI; public class TopScorerAgent : PGBaseAgent { @@ -23,11 +23,13 @@ public class TopScorerAgent : PGBaseAgent private Vector3 _midFieldPosition; private bool _ballTouched = false; private float _minDistance = 0.0f; + [SerializeField] Text _seedText; protected override void Start() { base.Start(); _handler = new GoalkeeperEvironmentHandler(transform.root.gameObject, defaultParameter); + _ball = _handler.environment.GetComponentInChildren().transform; rbBall = _ball.GetComponent(); ball = _ball.GetComponent(); @@ -36,11 +38,13 @@ protected override void Start() _shootAt = transform.parent.Find("ShootAt"); _handler.ResetParameter(); + } public override void OnEpisodeBegin() { _handler.ResetParameter(); + _seedText.text = "Seed: " + _handler.activeSeed; _ballTouched = false; var diff = UnityEngine.Random.Range(0, 200) % 2; switch (diff) @@ -62,8 +66,8 @@ private void OnEpisodeBeginDifficulty2() //Reset Ball - float ball_z_pos = UnityEngine.Random.Range(0, 9) % 2 == 0 ? 6f : -6f; - _ball.localPosition = new Vector3(45f, UnityEngine.Random.Range(2f, 5f), ball_z_pos + UnityEngine.Random.Range(-1f, 1f)); + float ball_z_pos = UnityEngine.Random.Range(0, 9) % 2 == 0 ? 9f : -9f; + _ball.localPosition = new Vector3(UnityEngine.Random.Range(44f, 46f), UnityEngine.Random.Range(2f, 10f), ball_z_pos + UnityEngine.Random.Range(-1f, 1f)); //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; diff --git a/Assets/ML-Agents/Topscorer/Topscorer_single.unity b/Assets/ML-Agents/Topscorer/Topscorer_single.unity index 7a6e5f3..c171caa 100644 --- a/Assets/ML-Agents/Topscorer/Topscorer_single.unity +++ b/Assets/ML-Agents/Topscorer/Topscorer_single.unity @@ -123,6 +123,185 @@ NavMeshSettings: debug: m_Flags: 0 m_NavMeshData: {fileID: 0} +--- !u!1 &457928209 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 457928213} + - component: {fileID: 457928212} + - component: {fileID: 457928211} + - component: {fileID: 457928210} + m_Layer: 5 + m_Name: Canvas + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &457928210 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 457928209} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: dc42784cf147c0c48a680349fa168899, type: 3} + m_Name: + m_EditorClassIdentifier: + m_IgnoreReversedGraphics: 1 + m_BlockingObjects: 0 + m_BlockingMask: + serializedVersion: 2 + m_Bits: 4294967295 +--- !u!114 &457928211 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 457928209} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 0cd44c1031e13a943bb63640046fad76, type: 3} + m_Name: + m_EditorClassIdentifier: + m_UiScaleMode: 0 + m_ReferencePixelsPerUnit: 100 + m_ScaleFactor: 1 + m_ReferenceResolution: {x: 800, y: 600} + m_ScreenMatchMode: 0 + m_MatchWidthOrHeight: 0 + m_PhysicalUnit: 3 + m_FallbackScreenDPI: 96 + m_DefaultSpriteDPI: 96 + m_DynamicPixelsPerUnit: 1 + m_PresetInfoIsWorld: 0 +--- !u!223 &457928212 +Canvas: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 457928209} + m_Enabled: 1 + serializedVersion: 3 + m_RenderMode: 0 + m_Camera: {fileID: 0} + m_PlaneDistance: 100 + m_PixelPerfect: 0 + m_ReceivesEvents: 1 + m_OverrideSorting: 0 + m_OverridePixelPerfect: 0 + m_SortingBucketNormalizedSize: 0 + m_AdditionalShaderChannelsFlag: 0 + m_SortingLayerID: 0 + m_SortingOrder: 0 + m_TargetDisplay: 0 +--- !u!224 &457928213 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 457928209} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 0, y: 0, z: 0} + m_Children: + - {fileID: 841977718} + m_Father: {fileID: 0} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0, y: 0} +--- !u!1 &841977717 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 841977718} + - component: {fileID: 841977720} + - component: {fileID: 841977719} + m_Layer: 5 + m_Name: seed + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &841977718 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841977717} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 457928213} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 200, y: 50} + m_SizeDelta: {x: 300, y: 100} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &841977719 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841977717} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 48 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 3 + m_MaxSize: 48 + m_Alignment: 0 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: 'Seed: ' +--- !u!222 &841977720 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841977717} + m_CullTransparentMesh: 1 --- !u!1 &1855846174 GameObject: m_ObjectHideFlags: 0 @@ -271,5 +450,9 @@ PrefabInstance: propertyPath: m_Name value: Environment objectReference: {fileID: 0} + - target: {fileID: 6750119729296308298, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} + propertyPath: _seedText + value: + objectReference: {fileID: 841977719} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: d7d4e7d3a9271ea41b992cd47164035f, type: 3} From 456ee31d674605d360c4529f2e7bce5f66868c42 Mon Sep 17 00:00:00 2001 From: Marco Pleines Date: Thu, 24 Feb 2022 16:32:45 +0100 Subject: [PATCH 25/25] current state with easy and hard shots --- Assets/ML-Agents/Topscorer/TopScorerAgent.cs | 76 ++++++++++++++++---- Assets/Scripts/Ball/ShootBall.cs | 2 + 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs index c97cc74..618f590 100644 --- a/Assets/ML-Agents/Topscorer/TopScorerAgent.cs +++ b/Assets/ML-Agents/Topscorer/TopScorerAgent.cs @@ -46,24 +46,51 @@ public override void OnEpisodeBegin() _handler.ResetParameter(); _seedText.text = "Seed: " + _handler.activeSeed; _ballTouched = false; - var diff = UnityEngine.Random.Range(0, 200) % 2; - switch (diff) - { - case 0: OnEpisodeBeginDifficulty2(); break; - case 1: OnEpisodeBeginDifficultyDefault(); break; - default: throw new Exception("Difficulty does not exist"); - } - // OnEpisodeBeginDifficultyDefault(); + // var diff = UnityEngine.Random.Range(0, 200) % 2; + // switch (diff) + // { + // case 0: OnEpisodeBeginDifficulty2(); break; + // case 1: OnEpisodeBeginDifficultyDefault(); break; + // default: throw new Exception("Difficulty does not exist"); + // } + // OnEpisodeBeginDifficulty2(); + // OnEpisodeBegiEasy(); + OnEpisodeBeginDifficultyDefault(); ball.ResetValues(); mapData.ResetIsScored(); SetReward(0f); } + private void OnEpisodeBegiEasy() + { + Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); + controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); + Debug.Log("Agent Position"); + Debug.Log(startPosition.ToString("F6")); + + + //Reset Ball + float ball_z_pos = UnityEngine.Random.Range(0, 9) % 2 == 0 ? 6f : -6f; + _ball.localPosition = new Vector3(45f, UnityEngine.Random.Range(2f, 5f), ball_z_pos + UnityEngine.Random.Range(-1f, 1f)); + //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); + _ball.GetComponent().velocity = Vector3.zero; + _ball.GetComponent().angularVelocity = Vector3.zero; + // Debug.Log("Ball Position"); + // Debug.Log(_ball.localPosition.ToString("F6")); + + // Set new Taget Position + _shootAt.localPosition = new Vector3(_ball.localPosition.x, 0f, 0f); + + //Throw Ball + _ball.GetComponent().ShootTarget(); + } + private void OnEpisodeBeginDifficulty2() { Vector3 startPosition = _midFieldPosition + new Vector3(UnityEngine.Random.Range(20f, 25f), 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f, 0f), _handler.boost_amount); - + // Debug.Log("Agent Position"); + // Debug.Log(startPosition.ToString("F6")); //Reset Ball float ball_z_pos = UnityEngine.Random.Range(0, 9) % 2 == 0 ? 9f : -9f; @@ -75,6 +102,12 @@ private void OnEpisodeBeginDifficulty2() // Set new Taget Position _shootAt.localPosition = new Vector3(_ball.localPosition.x, 0f, 0f); + // Debug.Log("Shoot At Position"); + // Debug.Log(_shootAt.localPosition.ToString("F6")); + + // Debug.Log("Ball Position"); + // Debug.Log(_ball.localPosition.ToString("F6")); + //Throw Ball _ball.GetComponent().ShootTarget(); } @@ -82,12 +115,15 @@ private void OnEpisodeBeginDifficulty2() private void OnEpisodeBeginDifficultyDefault() { //Reset Car - Vector3 startPosition = _midFieldPosition + new Vector3(UnityEngine.Random.Range(20f, 25f), 0.17f, UnityEngine.Random.Range(-15f, 15f)); - + // Vector3 startPosition = _midFieldPosition + new Vector3(UnityEngine.Random.Range(20f, 25f), 0.17f, UnityEngine.Random.Range(-15f, 15f)); + Vector3 startPosition = _midFieldPosition + new Vector3(25f, 0.17f, UnityEngine.Random.Range(-15f, 15f)); controller.ResetCar(startPosition, Quaternion.Euler(0f, 90f + UnityEngine.Random.Range(-20f, 20f), 0f), _handler.boost_amount); + // Debug.Log("Agent Position"); + // Debug.Log(startPosition.ToString("F6")); + //Reset Ball - _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.75f, 13f), UnityEngine.Random.Range(-20f, 20f)); + _ball.localPosition = new Vector3(50.0f, UnityEngine.Random.Range(0.75f, 10f), UnityEngine.Random.Range(-20f, 20f)); //_ball.rotation = Quaternion.Euler(0f, 0f, 0f); _ball.GetComponent().velocity = Vector3.zero; _ball.GetComponent().angularVelocity = Vector3.zero; @@ -95,6 +131,12 @@ private void OnEpisodeBeginDifficultyDefault() // Set new Taget Position _shootAt.localPosition = new Vector3(UnityEngine.Random.Range(10f, 30f), UnityEngine.Random.Range(0f, 7f), rb.position.z); + // Debug.Log("Shoot At Position"); + // Debug.Log(_shootAt.localPosition.ToString("F6")); + + // Debug.Log("Ball Position"); + // Debug.Log(_ball.localPosition.ToString("F6")); + //Throw Ball _ball.GetComponent().ShootTarget(); _minDistance = Vector3.Distance(_ball.position, transform.position); @@ -195,6 +237,7 @@ private void Reset() /// protected override void AssignReward() { + RewardDirection(); if (mapData.isScoredBlue) { // Agent scored a goal @@ -209,13 +252,20 @@ private void OnCollisionEnter(Collision other) { if (!_ballTouched) { - AddReward(0.1f); + // AddReward(0.1f); // Debug.Log(GetCumulativeReward()); _ballTouched = true; } } } + private void RewardDirection() + { + float speed = rb.velocity.magnitude / 23.0f; + float lookAtTarget = Vector3.Dot((_ball.position - transform.position).normalized, rb.velocity.normalized); + AddReward((lookAtTarget * speed / 650.0f) / 10.0f); + } + private void RewardCloseness() { float currentDistance = Vector3.Distance(_ball.position, transform.position); diff --git a/Assets/Scripts/Ball/ShootBall.cs b/Assets/Scripts/Ball/ShootBall.cs index 5fa2cf7..c9f6eb8 100644 --- a/Assets/Scripts/Ball/ShootBall.cs +++ b/Assets/Scripts/Ball/ShootBall.cs @@ -24,6 +24,8 @@ public void ShootTarget(float velocity = 0.0f) { velocity = Random.Range(speed.x, speed.y); } + // Debug.Log("Speed"); + // Debug.Log(velocity.ToString("F6")); _rb.velocity = dir.normalized * velocity; }