add antGame analysis data and R Scripts and images

src/main/java/core/algo/td/QLearningOffPolicyTDControl.java

@@ -48,6 +48,8 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
         int timestampTilFood = 0;
         int rewardsPer1000 = 0;
         int foodCollected = 0;
+        int iterations = 0;
+        int foodTimestampsTotal= 0;
         while(envResult == null || !envResult.isDone()) {
             actionValues = stateActionTable.getActionValues(state);
             A action = policy.chooseAction(actionValues);
@@ -57,11 +59,10 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
             double reward = envResult.getReward();
             State nextState = envResult.getState();
             sumOfRewards += reward;
-
             rewardsPer1000+=reward;
             timestampTilFood++;
 
-            if(foodCollected == 10000){
+          /*  if(iterations == 100){
                 File file = new File(ContinuousAnt.FILE_NAME);
                 try {
                     Files.writeString(Path.of(file.getPath()),  "\n", StandardOpenOption.APPEND);
@@ -69,17 +70,46 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
                     e.printStackTrace();
                 }
                 return;
-            }
+            }*/
+
+
             if(reward == Reward.FOOD_DROP_DOWN_SUCCESS){
                 foodCollected++;
-                File file = new File(ContinuousAnt.FILE_NAME);
-                try {
-                    Files.writeString(Path.of(file.getPath()),  timestampTilFood + ",", StandardOpenOption.APPEND);
-                } catch (IOException e) {
-                    e.printStackTrace();
+                foodTimestampsTotal += timestampTilFood;
+                if(foodCollected % 1000 == 0){
+                    System.out.println(foodTimestampsTotal/1000f);
+                    File file = new File(ContinuousAnt.FILE_NAME);
+                    try {
+                        Files.writeString(Path.of(file.getPath()),  foodTimestampsTotal/1000f +",", StandardOpenOption.APPEND);
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                    foodTimestampsTotal = 0;
                 }
+                if(foodCollected == 1000){
+                    ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.15f);
+                }
+                if(foodCollected == 2000){
+                    ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.10f);
+                }
+                if(foodCollected == 3000){
+                    ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.05f);
+                }
+                if(foodCollected == 4000){
+                    System.out.println("final 0 expl");
+                    ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.00f);
+                }
+                if(foodCollected == 15000){
+                    File file = new File(ContinuousAnt.FILE_NAME);
+                    try {
+                        Files.writeString(Path.of(file.getPath()),  "\n", StandardOpenOption.APPEND);
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                    return;
+                }
+                iterations++;
                 timestampTilFood = 0;
-                rewardsPer1000 = 0;
             }
 
             // Q Update

src/main/java/evironment/antGame/Reward.java

@@ -1,13 +1,13 @@
 package evironment.antGame;
 
 public class Reward {
-    public static final double FOOD_PICK_UP_SUCCESS = 1;
+    public static final double FOOD_PICK_UP_SUCCESS = 0;
     public static final double FOOD_PICK_UP_FAIL_NO_FOOD = -1;
     public static final double FOOD_PICK_UP_FAIL_HAS_FOOD_ALREADY = -1;
 
     public static final double FOOD_DROP_DOWN_FAIL_NO_FOOD = -1;
     public static final double FOOD_DROP_DOWN_FAIL_NOT_START = -1;
-    public static final double FOOD_DROP_DOWN_SUCCESS = 40;
+    public static final double FOOD_DROP_DOWN_SUCCESS = 1;
 
     public static final double UNKNOWN_FIELD_EXPLORED = 0;
 

src/main/java/example/ContinuousAnt.java

@@ -12,8 +12,10 @@ import java.io.File;
 import java.io.IOException;
 
 public class ContinuousAnt {
-    public static final String FILE_NAME = "converge05.txt";
+    public static final String FILE_NAME = "converge22.txt";
     public static void main(String[] args) {
+        int i = 4+4+4+6+6+6+8+10+12+14+14+16+16+16+18+18+18+20+20+20+22+22+22+24+24+24+24+26+26+26+26+26+28+28+28+28+28+30+30+30+30+32+32+32+34+34+34+36+36+38+40+42;
+        System.out.println(i/52f);
         File file = new File(FILE_NAME);
         try {
             file.createNewFile();
@@ -27,9 +29,12 @@ public class ContinuousAnt {
                 AntAction.values());
         rl.setDelay(0);
         rl.setNrOfEpisodes(1);
-        rl.setDiscountFactor(0.7f);
-        rl.setLearningRate(0.2f);
-        rl.setEpsilon(0.5f);
+        //0.99 0.9 0.5
+        //0.99 0.95 0.9 0.7 0.5 0.3 0.1
+        rl.setDiscountFactor(0.05f);
+        // 0.1, 0.3, 0.5, 0.7 0.9
+        rl.setLearningRate(0.9f);
+        rl.setEpsilon(0.2f);
         rl.start();
     }
 }

src/main/java/example/JumpingDino.java

@@ -17,9 +17,9 @@ public class JumpingDino {
                 DinoAction.values());
 
         rl.setDelay(0);
-        rl.setDiscountFactor(1f);
+        rl.setDiscountFactor(9f);
         rl.setEpsilon(0.05f);
-        rl.setLearningRate(1f);
+        rl.setLearningRate(0.8f);
         rl.setNrOfEpisodes(100000);
         rl.start();
     }