create Dino Sampling state

src/main/java/core/algo/EpisodicLearning.java

@@ -6,6 +6,7 @@ import core.LearningConfig;
 import core.StepResult;
 import core.listener.LearningListener;
 import core.policy.EpsilonGreedyPolicy;
+import example.DinoSampling;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -104,10 +105,10 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
         timestamp++;
         timestampCurrentEpisode++;
         // TODO: more sophisticated way to check convergence
-        if(timestampCurrentEpisode > 300000){
+        if(timestampCurrentEpisode > 30000000){
             converged = true;
             // t
-            File file = new File("convergenceAdv.txt");
+            File file = new File(DinoSampling.FILE_NAME);
             try {
                 Files.writeString(Path.of(file.getPath()),  currentEpisode/2 + ",", StandardOpenOption.APPEND);
             } catch (IOException e) {
@@ -127,7 +128,6 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     private void startLearning(){
         dispatchLearningStart();
         while(episodesToLearn.get() > 0){
-
             dispatchEpisodeStart();
             timestampCurrentEpisode = 0;
             nextEpisode();

src/main/java/core/algo/td/QLearningOffPolicyTDControl.java

@@ -39,14 +39,23 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
         sumOfRewards = 0;
         while(envResult == null || !envResult.isDone()) {
             actionValues = stateActionTable.getActionValues(state);
-            A action = policy.chooseAction(actionValues);
-
+            A action;
+            if(currentEpisode % 2 == 0){
+                action = greedyPolicy.chooseAction(actionValues);
+            }else{
+                action = policy.chooseAction(actionValues);
+            }
+            if(converged) return;
             // Take a step
             envResult = environment.step(action);
             double reward = envResult.getReward();
             State nextState = envResult.getState();
             sumOfRewards += reward;
-
+            if(currentEpisode % 2 == 0){
+                state = nextState;
+                dispatchStepEnd();
+                continue;
+            }
             // Q Update
             double currentQValue = stateActionTable.getActionValues(state).get(action);
             // maxQ(S', a);

src/main/java/core/algo/td/SARSA.java

@@ -3,12 +3,15 @@ package core.algo.td;
 import core.*;
 import core.algo.EpisodicLearning;
 import core.policy.EpsilonGreedyPolicy;
+import core.policy.GreedyPolicy;
+import core.policy.Policy;
 
 import java.util.Map;
 
 
 public class SARSA<A extends Enum> extends EpisodicLearning<A> {
     private float alpha;
+    private Policy<A> greedyPolicy = new GreedyPolicy<>();
 
     public SARSA(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, float learningRate, int delay) {
         super(environment, actionSpace, discountFactor, delay);
@@ -32,10 +35,18 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
 
         StepResultEnvironment envResult = null;
         Map<A, Double> actionValues = stateActionTable.getActionValues(state);
-        A action = policy.chooseAction(actionValues);
+        A action;
+        if(currentEpisode % 2 == 1){
+            action = greedyPolicy.chooseAction(actionValues);
+        }else{
+            action = policy.chooseAction(actionValues);
+        }
+        //A action = policy.chooseAction(actionValues);
 
         sumOfRewards = 0;
         while(envResult == null || !envResult.isDone()) {
+
+            if(converged) return;
             // Take a step
             envResult = environment.step(action);
             sumOfRewards += envResult.getReward();
@@ -44,8 +55,20 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
 
             // Pick next action
             actionValues = stateActionTable.getActionValues(nextState);
-            A nextAction = policy.chooseAction(actionValues);
 
+            A nextAction;
+            if(currentEpisode % 2 == 1){
+                nextAction = greedyPolicy.chooseAction(actionValues);
+            }else{
+                nextAction = policy.chooseAction(actionValues);
+            }
+            //A nextAction = policy.chooseAction(actionValues);
+            if(currentEpisode % 2 == 1){
+                state = nextState;
+                action = nextAction;
+                dispatchStepEnd();
+                continue;
+            }
             // td update
             // target = reward + gamma * Q(nextState, nextAction)
             double currentQValue = stateActionTable.getActionValues(state).get(action);

src/main/java/core/policy/EpsilonPolicy.java

@@ -1,8 +1,8 @@
 package core.policy;
 
 /**
- * Chooses the action with the highest values with possibility: 1-Ɛ + Ɛ/|A|
- * With possibility of Ɛ, a random action is taken (highest values option included).
+ * Chooses the action with the highest values with possibility: 1-Epsilon + Epsilon/|A|
+ * With possibility of Epsilon, a random action is taken (highest values option included).
  *
  * @param <A> Enum class of available action in specific environment
  */

src/main/java/evironment/antGame/AntStateOriginal.java

@@ -0,0 +1,46 @@
+package evironment.antGame;
+
+import core.State;
+import lombok.AllArgsConstructor;
+
+import java.util.Objects;
+
+@AllArgsConstructor
+public class AntStateOriginal implements State {
+    private final int currentFood;
+    private final int row;
+    private final int col;
+    private final CellType type;
+    private final int smell;
+    private final int food;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AntStateOriginal that = (AntStateOriginal) o;
+        return currentFood == that.currentFood &&
+                row == that.row &&
+                col == that.col &&
+                smell == that.smell &&
+                type == that.type &&
+                food == that.food;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(currentFood, row, col, type, smell, food);
+    }
+
+    @Override
+    public String toString() {
+        return "AntStateOriginal{" +
+                "currentFood=" + currentFood +
+                ", row=" + row +
+                ", col=" + col +
+                ", type=" + type +
+                ", smell=" + smell +
+                ", food=" + food +
+                '}';
+    }
+}

src/main/java/evironment/antGame/AntWorld.java

@@ -134,9 +134,6 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
 
     @Override
     public StepResultEnvironment step(AntAction action){
-        AntObservation observation;
-        State newState;
-
         StepCalculation sc = processStep(action);
 
         // valid movement
@@ -149,11 +146,7 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
             }
         }
 
-        // get observation after action was computed
-        observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
 
-        // let the ant agent process the observation to create a valid markov state
-        newState = antAgent.feedObservation(observation);
 
         if(sc.checkCompletion) {
             sc.done = grid.isAllFoodCollected();
@@ -163,7 +156,15 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
             sc.done = true;
         }
 
-        return new StepResultEnvironment(newState, sc.reward, sc.done, sc.info);
+        return new StepResultEnvironment(generateReturnState(), sc.reward, sc.done, sc.info);
+    }
+
+    protected State generateReturnState(){
+        // get observation after action was computed
+        AntObservation observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
+
+        // let the ant agent process the observation to create a valid markov state
+        return antAgent.feedObservation(observation);
     }
 
     protected boolean isInGrid(Point pos) {

src/main/java/evironment/antGame/AntWorldContinuous.java

@@ -1,5 +1,6 @@
 package evironment.antGame;
 
+import core.State;
 import core.StepResultEnvironment;
 
 public class AntWorldContinuous extends AntWorld {
@@ -13,7 +14,6 @@ public class AntWorldContinuous extends AntWorld {
 
     @Override
     public StepResultEnvironment step(AntAction action) {
-        AntObservation observation;
         Cell currentCell = grid.getCell(myAnt.getPos());
 
         StepCalculation sc = processStep(action);
@@ -27,10 +27,13 @@ public class AntWorldContinuous extends AntWorld {
             myAnt.getPos().setLocation(sc.potentialNextPos);
         }
 
-        // get observation after action was computed
-        observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
+        return new StepResultEnvironment(generateReturnState(), sc.reward, false, sc.info);
+    }
 
-        return new StepResultEnvironment(new AntState(grid.getGrid(), observation.getPos(), observation.hasFood()), sc.reward, false, sc.info);
+    @Override
+    protected State generateReturnState(){
+        AntObservation observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
+        return new AntState(grid.getGrid(), observation.getPos(), observation.hasFood());
     }
 
 }

src/main/java/evironment/antGame/AntWorldContinuousOriginalState.java

@@ -0,0 +1,36 @@
+package evironment.antGame;
+
+import core.State;
+
+public class AntWorldContinuousOriginalState extends AntWorldContinuous {
+    public AntWorldContinuousOriginalState(int width, int height) {
+        super(width, height);
+    }
+
+    public AntWorldContinuousOriginalState() {
+        super();
+    }
+
+    @Override
+    protected State generateReturnState(){
+       return new AntStateOriginal(myAnt.hasFood()? 1:0, myAnt.getPos().x, myAnt.getPos().y, grid.getCell(myAnt.getPos()).getType(), calculateSmell(), grid.getCell(myAnt.getPos()).getFood());
+    }
+
+    /**
+     * @return total smell of neighbour food cells
+     */
+    private int calculateSmell(){
+        int smell = 0;
+        int maxX = grid.getGrid().length -1;
+        int maxY = grid.getGrid()[0].length -1;
+        int antX = myAnt.getPos().x;
+        int antY = myAnt.getPos().y;
+
+        smell += antY > 0 ? grid.getCell(antX, antY - 1).getFood() : 0;
+        smell += antY < maxY ? grid.getCell(antX, antY + 1).getFood() : 0;
+        smell += antX > 0 ? grid.getCell(antX - 1, antY).getFood() : 0;
+        smell += antX < maxX ? grid.getCell(antX + 1, antY).getFood() : 0;
+
+        return smell;
+    }
+}

src/main/java/evironment/antGame/CellType.java

@@ -1,6 +1,7 @@
 package evironment.antGame;
 
-public enum CellType {
+
+public enum CellType{
     START,
     FREE,
     OBSTACLE,

src/main/java/evironment/antGame/Grid.java

@@ -64,8 +64,8 @@ public class Grid {
             if(potFieldType != CellType.START && grid[potFood.x][potFood.y].getFood() == 0 && potFieldType != CellType.OBSTACLE) {
                 grid[potFood.x][potFood.y].setFood(1);
                 foodSpawned = true;
-                System.out.println("spawned new food at " + potFood);
-                System.out.println(initialGrid[potFood.x][potFood.y]);
+               // System.out.println("spawned new food at " + potFood);
+               // System.out.println(initialGrid[potFood.x][potFood.y]);
             }
         }
     }

src/main/java/evironment/jumpingDino/DinoWorld.java

@@ -44,7 +44,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
     @Override
     public StepResultEnvironment step(DinoAction action) {
         boolean done = false;
-        int reward = 1;
+        int reward = 0;
 
         if(action == DinoAction.JUMP){
             dino.jump();
@@ -68,7 +68,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
             spawnNewObstacle();
         }
         if(ranIntoObstacle()) {
-            reward = 0;
+            reward = -1;
             done = true;
         }
 

src/main/java/example/ContinuousAnt.java

@@ -6,12 +6,13 @@ import core.controller.RLController;
 import core.controller.RLControllerGUI;
 import evironment.antGame.AntAction;
 import evironment.antGame.AntWorldContinuous;
+import evironment.antGame.AntWorldContinuousOriginalState;
 
 public class ContinuousAnt {
     public static void main(String[] args) {
         RNG.setSeed(56);
         RLController<AntAction> rl = new RLControllerGUI<>(
-                new AntWorldContinuous(8, 8),
+                new AntWorldContinuousOriginalState(8, 8),
                 Method.Q_LEARNING_OFF_POLICY_CONTROL,
                 AntAction.values());
 

src/main/java/example/DinoSampling.java

@@ -3,7 +3,9 @@ package example;
 import core.RNG;
 import core.algo.Method;
 import core.controller.RLController;
+import core.controller.RLControllerGUI;
 import evironment.jumpingDino.DinoAction;
+import evironment.jumpingDino.DinoWorld;
 import evironment.jumpingDino.DinoWorldAdvanced;
 
 import java.io.File;
@@ -13,29 +15,35 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 
 public class DinoSampling {
+    public static final float f =0.05f;
+    public static final String FILE_NAME = "converge.txt";
     public static void main(String[] args) {
-        File file = new File("convergenceAdv.txt");
-        for(float f = 0.05f; f <=1.003 ; f+=0.05f){
+        File file = new File(FILE_NAME);
+        try {
+            file.createNewFile();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        for(float f = 0.05f; f <=1.003 ; f+=0.05f) {
             try {
                 Files.writeString(Path.of(file.getPath()), f + ",", StandardOpenOption.APPEND);
             } catch (IOException e) {
                 e.printStackTrace();
             }
-            for(int i = 1; i <= 100; i++) {
-                System.out.println("seed: " + i *13);
-                RNG.setSeed(i *13);
+            for (int i = 1; i <= 100; i++) {
+                System.out.println("seed: " + i * 13);
+                RNG.setSeed(i * 13);
 
                 RLController<DinoAction> rl = new RLController<>(
                         new DinoWorldAdvanced(),
-                        Method.MC_CONTROL_FIRST_VISIT,
+                        Method.Q_LEARNING_OFF_POLICY_CONTROL,
                         DinoAction.values());
                 rl.setDelay(0);
-                rl.setDiscountFactor(1f);
+                rl.setDiscountFactor(0.99f);
                 rl.setEpsilon(f);
-                rl.setLearningRate(1f);
-                rl.setNrOfEpisodes(50000);
+                rl.setLearningRate(0.9f);
+                rl.setNrOfEpisodes(400000);
                 rl.start();
-
             }
             try {
                 Files.writeString(Path.of(file.getPath()), "\n", StandardOpenOption.APPEND);

src/main/java/example/JumpingDino.java

@@ -5,21 +5,22 @@ import core.algo.Method;
 import core.controller.RLController;
 import evironment.jumpingDino.DinoAction;
 import evironment.jumpingDino.DinoWorld;
+import evironment.jumpingDino.DinoWorldAdvanced;
 
 public class JumpingDino {
     public static void main(String[] args) {
         RNG.setSeed(29);
 
         RLController<DinoAction> rl = new RLController<>(
-                new DinoWorld(),
+                new DinoWorldAdvanced(),
                 Method.MC_CONTROL_FIRST_VISIT,
                 DinoAction.values());
 
         rl.setDelay(0);
         rl.setDiscountFactor(1f);
-        rl.setEpsilon(0.15f);
+        rl.setEpsilon(0.05f);
         rl.setLearningRate(1f);
-        rl.setNrOfEpisodes(30000000);
+        rl.setNrOfEpisodes(100000);
         rl.start();
     }
 }