add epsilon convergence test and will remove unnecessary multithreaded learning

src/main/java/core/algo/EpisodicLearning.java

@@ -9,9 +9,13 @@ import core.policy.EpsilonGreedyPolicy;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -25,7 +29,8 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     protected int episodeSumCurrentSecond;
     protected double sumOfRewards;
     protected List<StepResult<A>> episode = new ArrayList<>();
-
+    protected int timestampCurrentEpisode = 0;
+    protected boolean converged;
     public EpisodicLearning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay) {
         super(environment, actionSpace, discountFactor, delay);
         initBenchMarking();
@@ -50,7 +55,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
 
     private void initBenchMarking(){
         new Thread(()->{
-            while (true){
+            while (currentlyLearning){
                 episodePerSecond = episodeSumCurrentSecond;
                 episodeSumCurrentSecond = 0;
                 try {
@@ -62,7 +67,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
         }).start();
     }
 
-    protected void dispatchEpisodeEnd(){
+    private void dispatchEpisodeEnd(){
         ++episodeSumCurrentSecond;
         if(rewardHistory.size() > 10000){
             rewardHistory.clear();
@@ -73,20 +78,20 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
         }
     }
 
-    protected void dispatchEpisodeStart(){
+    private void dispatchEpisodeStart(){
         ++currentEpisode;
         /*
-        2f 0.02 => 100
-        1.5f 0.02 => 75
-        1.4f 0.02 => fail
-        1.5f 0.1 => 16 !
-         */
-        if(this.policy instanceof EpsilonGreedyPolicy){
-            float ep = 1.5f/(float)currentEpisode;
-            if(ep < 0.10) ep = 0;
-           ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(ep);
-            System.out.println(ep);
-        }
+            2f 0.02 => 100
+            1.5f 0.02 => 75
+            1.4f 0.02 => fail
+            1.5f 0.1 => 16 !
+        */
+//        if(this.policy instanceof EpsilonGreedyPolicy){
+//            float ep = 2f/(float)currentEpisode;
+//            if(ep < 0.02) ep = 0;
+//            ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(ep);
+//            System.out.println(ep);
+//        }
         episodesToLearn.decrementAndGet();
         for(LearningListener l: learningListeners){
             l.onEpisodeStart();
@@ -97,10 +102,20 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     protected void dispatchStepEnd() {
         super.dispatchStepEnd();
         timestamp++;
+        timestampCurrentEpisode++;
         // TODO: more sophisticated way to check convergence
-        if(timestamp > 300000){
-            System.out.println("converged after: " + currentEpisode + " episode!");
-            interruptLearning();
+        if(timestampCurrentEpisode > 300000){
+            converged = true;
+            // t
+            File file = new File("convergence.txt");
+            try {
+                Files.writeString(Path.of(file.getPath()),  currentEpisode/2 + ",", StandardOpenOption.APPEND);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            System.out.println("converged after: " + currentEpisode/2 + " episode!");
+            episodesToLearn.set(0);
+            dispatchLearningEnd();
         }
     }
 
@@ -110,20 +125,23 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     }
 
     private void startLearning(){
-        learningExecutor.submit(()->{
-            dispatchLearningStart();
-            while(episodesToLearn.get() > 0){
-                dispatchEpisodeStart();
-                nextEpisode();
-                dispatchEpisodeEnd();
-            }
-            synchronized (this){
-                dispatchLearningEnd();
-                notifyAll();
-            }
-        });
+        dispatchLearningStart();
+        while(episodesToLearn.get() > 0){
+
+            dispatchEpisodeStart();
+            timestampCurrentEpisode = 0;
+            nextEpisode();
+            dispatchEpisodeEnd();
+        }
+        synchronized (this){
+            dispatchLearningEnd();
+            notifyAll();
+        }
     }
 
+    public void learnMoreEpisodes(int nrOfEpisodes){
+        episodesToLearn.addAndGet(nrOfEpisodes);
+    }
     /**
      * Stopping the while loop by setting episodesToLearn to 0.
      * The current episode can not be interrupted, so the sleep delay

src/main/java/core/algo/Learning.java

@@ -42,8 +42,7 @@ public abstract class Learning<A extends Enum>{
     @Setter
     protected int delay;
     protected List<Double> rewardHistory;
-    protected ExecutorService learningExecutor;
-    protected boolean currentlyLearning;
+    protected volatile boolean currentlyLearning;
 
     public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay) {
         this.environment = environment;
@@ -53,7 +52,6 @@ public abstract class Learning<A extends Enum>{
         currentlyLearning = false;
         learningListeners = new HashSet<>();
         rewardHistory = new CopyOnWriteArrayList<>();
-        learningExecutor = Executors.newSingleThreadExecutor();
     }
 
     public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor) {
@@ -89,8 +87,6 @@ public abstract class Learning<A extends Enum>{
 
     protected void dispatchLearningEnd() {
         currentlyLearning = false;
-        System.out.println("Checksum: " + checkSum);
-        System.out.println("Reward Checksum: " + rewardCheckSum);
         for (LearningListener l : learningListeners) {
             l.onLearningEnd();
         }

src/main/java/core/algo/mc/MonteCarloControlFirstVisitEGreedy.java

@@ -3,12 +3,17 @@ package core.algo.mc;
 import core.*;
 import core.algo.EpisodicLearning;
 import core.policy.EpsilonGreedyPolicy;
+import core.policy.GreedyPolicy;
+import core.policy.Policy;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.*;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.*;
 
 /**
@@ -35,8 +40,16 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
     private Map<Pair<State, A>, Double> returnSum;
     private Map<Pair<State, A>, Integer> returnCount;
 
+    // t
+    private float epsilon;
+    // t
+    private Policy<A> greedyPolicy = new GreedyPolicy<>();
+
+
     public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
         super(environment, actionSpace, discountFactor, delay);
+        // t
+        this.epsilon = epsilon;
         this.policy = new EpsilonGreedyPolicy<>(epsilon);
         this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
         returnSum = new HashMap<>();
@@ -58,12 +71,16 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
         }
         sumOfRewards = 0;
         StepResultEnvironment envResult = null;
-        //TODO extract to learning
-        int timestamp = 0;
+
         while(envResult == null || !envResult.isDone()) {
             Map<A, Double> actionValues = stateActionTable.getActionValues(state);
-            A chosenAction = policy.chooseAction(actionValues);
-            checkSum += chosenAction.ordinal();
+            A chosenAction;
+            if(currentEpisode % 2 == 1){
+                chosenAction = greedyPolicy.chooseAction(actionValues);
+            }else{
+                chosenAction = policy.chooseAction(actionValues);
+            }
+
             envResult = environment.step(chosenAction);
             State nextState = envResult.getState();
             sumOfRewards += envResult.getReward();
@@ -79,6 +96,11 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
             }
             timestamp++;
             dispatchStepEnd();
+            if(converged) return;
+        }
+
+        if(currentEpisode % 2 == 1){
+            return;
         }
 
         //  System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);

src/main/java/core/controller/RLController.java

@@ -67,17 +67,17 @@ public class RLController<A extends Enum> implements LearningListener {
     }
 
     protected void initListeners() {
-        learning.addListener(this);
-        new Thread(() -> {
-            while(true) {
-                printNextEpisode = true;
-                try {
-                    Thread.sleep(30 * 1000);
-                } catch (InterruptedException e) {
-                    e.printStackTrace();
+            learning.addListener(this);
+            new Thread(() -> {
+                while(learning.isCurrentlyLearning()) {
+                    printNextEpisode = true;
+                    try {
+                        Thread.sleep(30 * 1000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
                 }
-            }
-        }).start();
+            }).start();
     }
 
     private void initLearning() {
@@ -95,7 +95,13 @@ public class RLController<A extends Enum> implements LearningListener {
 
     protected void learnMoreEpisodes(int nrOfEpisodes) {
         if(learning instanceof EpisodicLearning) {
-            ((EpisodicLearning) learning).learn(nrOfEpisodes);
+            if(learning.isCurrentlyLearning()){
+                ((EpisodicLearning) learning).learnMoreEpisodes(nrOfEpisodes);
+            }else{
+                new Thread(() -> {
+                    ((EpisodicLearning) learning).learn(nrOfEpisodes);
+                }).start();
+            }
         } else {
             throw new RuntimeException("Triggering onLearnMoreEpisodes on non-episodic learning!");
         }

src/main/java/example/JumpingDino.java

@@ -3,23 +3,49 @@ package example;
 import core.RNG;
 import core.algo.Method;
 import core.controller.RLController;
+import core.controller.RLControllerGUI;
 import evironment.jumpingDino.DinoAction;
 import evironment.jumpingDino.DinoWorld;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
 public class JumpingDino {
     public static void main(String[] args) {
-        RNG.setSeed(55);
+        File file = new File("convergence.txt");
+        for(float f = 0.05f; f <=1.003 ; f+=0.05f){
+            try {
+                Files.writeString(Path.of(file.getPath()), f + ",", StandardOpenOption.APPEND);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            for(int i = 1; i <= 100; i++) {
+                System.out.println("seed: " + i *13);
+                RNG.setSeed(i *13);
 
-        RLController<DinoAction> rl = new RLController<>(
-                new DinoWorld(false, false),
-                Method.MC_CONTROL_FIRST_VISIT,
-                DinoAction.values());
+                RLController<DinoAction> rl = new RLController<>(
+                        new DinoWorld(false, false),
+                        Method.MC_CONTROL_FIRST_VISIT,
+                        DinoAction.values());
 
-        rl.setDelay(0);
-        rl.setDiscountFactor(1f);
-        rl.setEpsilon(0.15f);
-        rl.setLearningRate(1f);
-        rl.setNrOfEpisodes(400);
-        rl.start();
+
+                rl.setDelay(0);
+                rl.setDiscountFactor(1f);
+                rl.setEpsilon(f);
+                rl.setLearningRate(1f);
+                rl.setNrOfEpisodes(20000);
+                rl.start();
+
+            }
+            try {
+                Files.writeString(Path.of(file.getPath()), "\n", StandardOpenOption.APPEND);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        System.out.println("kek");
     }
 }

src/main/java/example/Results

@@ -0,0 +1,15 @@
+Method:
+Epsilon = k / currentEpisode
+set to 0 if Epsilon < b
+
+k = 1.5
+b = 0.1   => conv. 16
+
+k = 1.5
+b = 0.02  => 75
+
+k = 1.4
+b = 0.02  => fail
+
+k = 2.0
+b = 0.02  => conv. 100