add dino jumping environment, deterministic/reproducable behaviour and save-and-load feature

- add feature to save and load learning progress (Q-Table) and current episode count
- episode end is now purely decided by environment instead of monte carlo algo capping it on 10 actions
- using linkedHashMap on all locations to ensure deterministic behaviour
- fixed major RNG issue to reproduce algorithmic behaviour
- clearing rewardHistory, to only save the last 10k rewards
- added google dino jump environment

src/main/java/core/algo/EpisodicLearning.java

@@ -3,13 +3,17 @@ package core.algo;
 import core.DiscreteActionSpace;
 import core.Environment;
 import core.listener.LearningListener;
+import lombok.Getter;
+import lombok.Setter;
 
-public abstract class EpisodicLearning<A extends Enum> extends Learning<A> implements Episodic{
+public abstract class EpisodicLearning<A extends Enum> extends Learning<A> implements Episodic {
+    @Setter
+    @Getter
     protected int currentEpisode;
     protected int episodesToLearn;
     protected volatile int episodePerSecond;
     protected int episodeSumCurrentSecond;
-    private volatile boolean meseaureEpisodeBenchMark;
+    private volatile boolean measureEpisodeBenchMark;
 
     public EpisodicLearning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay) {
         super(environment, actionSpace, discountFactor, delay);
@@ -29,6 +33,9 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
 
     protected void dispatchEpisodeEnd(double recentSumOfRewards){
         ++episodeSumCurrentSecond;
+        if(rewardHistory.size() > 10000){
+            rewardHistory.clear();
+        }
         rewardHistory.add(recentSumOfRewards);
         for(LearningListener l: learningListeners) {
             l.onEpisodeEnd(rewardHistory);
@@ -47,9 +54,9 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     }
 
     public void learn(int nrOfEpisodes){
-        meseaureEpisodeBenchMark = true;
+        measureEpisodeBenchMark = true;
         new Thread(()->{
-            while(meseaureEpisodeBenchMark){
+            while(measureEpisodeBenchMark){
                 episodePerSecond = episodeSumCurrentSecond;
                 episodeSumCurrentSecond = 0;
                 try {
@@ -65,7 +72,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
             nextEpisode();
         }
         dispatchLearningEnd();
-        meseaureEpisodeBenchMark = false;
+        measureEpisodeBenchMark = false;
     }
 
     protected abstract void nextEpisode();

src/main/java/core/algo/Learning.java

@@ -9,15 +9,17 @@ import core.policy.Policy;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.io.Serializable;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 @Getter
-public abstract class Learning<A extends Enum> {
+public abstract class Learning<A extends Enum> implements Serializable {
     protected Policy<A> policy;
     protected DiscreteActionSpace<A> actionSpace;
+    @Setter
     protected StateActionTable<A> stateActionTable;
     protected Environment<A> environment;
     protected float discountFactor;
@@ -26,7 +28,7 @@ public abstract class Learning<A extends Enum> {
     protected int delay;
     protected List<Double> rewardHistory;
 
-    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay){
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay) {
         this.environment = environment;
         this.actionSpace = actionSpace;
         this.discountFactor = discountFactor;
@@ -35,39 +37,41 @@ public abstract class Learning<A extends Enum> {
         rewardHistory = new CopyOnWriteArrayList<>();
     }
 
-    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor){
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor) {
         this(environment, actionSpace, discountFactor, LearningConfig.DEFAULT_DELAY);
     }
 
-    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay){
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
         this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, delay);
     }
 
-    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace){
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace) {
         this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_DELAY);
     }
 
+
     public abstract void learn();
 
-    public void addListener(LearningListener learningListener){
+    public void addListener(LearningListener learningListener) {
         learningListeners.add(learningListener);
     }
 
-    protected void dispatchStepEnd(){
-        for(LearningListener l: learningListeners){
+    protected void dispatchStepEnd() {
+        for (LearningListener l : learningListeners) {
             l.onStepEnd();
         }
     }
 
-    protected void dispatchLearningStart(){
-        for(LearningListener l: learningListeners){
+    protected void dispatchLearningStart() {
+        for (LearningListener l : learningListeners) {
             l.onLearningStart();
         }
     }
 
-    protected void dispatchLearningEnd(){
-        for(LearningListener l: learningListeners){
+    protected void dispatchLearningEnd() {
+        for (LearningListener l : learningListeners) {
             l.onLearningEnd();
         }
     }
+
 }

src/main/java/core/algo/MC/MonteCarloOnPolicyEGreedy.java

@@ -35,7 +35,7 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
         super(environment, actionSpace, discountFactor, delay);
         currentEpisode = 0;
         this.policy = new EpsilonGreedyPolicy<>(epsilon);
-        this.stateActionTable = new StateActionHashTable<>(this.actionSpace);
+        this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
         returnSum = new HashMap<>();
         returnCount = new HashMap<>();
     }
@@ -57,16 +57,15 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
             e.printStackTrace();
         }
         double sumOfRewards = 0;
-        for (int j = 0; j < 10; ++j) {
+        StepResultEnvironment envResult = null;
+        while(envResult == null || !envResult.isDone()){
             Map<A, Double> actionValues = stateActionTable.getActionValues(state);
             A chosenAction = policy.chooseAction(actionValues);
-            StepResultEnvironment envResult = environment.step(chosenAction);
+            envResult = environment.step(chosenAction);
             State nextState = envResult.getState();
             sumOfRewards += envResult.getReward();
             episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
 
-            if (envResult.isDone()) break;
-
             state = nextState;
 
             try {
@@ -78,13 +77,13 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
         }
 
         dispatchEpisodeEnd(sumOfRewards);
-        System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
-        Set<Pair<State, A>> stateActionPairs = new HashSet<>();
+      //  System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
+        Set<Pair<State, A>> stateActionPairs = new LinkedHashSet<>();
 
         for (StepResult<A> sr : episode) {
             stateActionPairs.add(new Pair<>(sr.getState(), sr.getAction()));
         }
-        System.out.println("stateActionPairs " + stateActionPairs.size());
+        //System.out.println("stateActionPairs " + stateActionPairs.size());
         for (Pair<State, A> stateActionPair : stateActionPairs) {
             int firstOccurenceIndex = 0;
             // find first occurance of state action pair