add specific environment RNG

src/main/java/core/RNG.java

@@ -15,18 +15,26 @@ import java.util.Random;
  */
 public class RNG {
     private static Random rng;
+    private static Random rngEnv;
     private static int seed = 123;
     static {
         rng = new Random();
-        rng.setSeed(seed);
+        setSeed(seed, true);
     }
 
     public static Random getRandom() {
         return rng;
     }
 
-    public static void setSeed(int seed){
+    public static Random getEnvRandom() {
+        return rngEnv;
+    }
+
+    public static void setSeed(int seed, boolean setEnvSeed) {
         RNG.seed = seed;
         rng.setSeed(seed);
+        if(setEnvSeed) {
+            rngEnv.setSeed(seed);
+        }
     }
 }

src/main/java/core/algo/EpisodicLearning.java

@@ -80,18 +80,6 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
 
     private void dispatchEpisodeStart(){
         ++currentEpisode;
-        /*
-            2f 0.02 => 100
-            1.5f 0.02 => 75
-            1.4f 0.02 => fail
-            1.5f 0.1 => 16 !
-        */
-//        if(this.policy instanceof EpsilonGreedyPolicy){
-//            float ep = 2f/(float)currentEpisode;
-//            if(ep < 0.02) ep = 0;
-//            ((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(ep);
-//            System.out.println(ep);
-//        }
         episodesToLearn.decrementAndGet();
         for(LearningListener l: learningListeners){
             l.onEpisodeStart();

src/main/java/core/algo/mc/MonteCarloControlEGreedy.java

@@ -19,7 +19,7 @@ import java.util.*;
  * Change to Every-Visit by setting flag "useEveryVisit" in the constructor to true.
  * @param <A>
  */
-public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends EpisodicLearning<A> {
+public class MonteCarloControlEGreedy<A extends Enum> extends EpisodicLearning<A> {
 
     private Map<Pair<State, A>, Double> returnSum;
     private Map<Pair<State, A>, Integer> returnCount;
@@ -31,7 +31,7 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
     private Policy<A> greedyPolicy = new GreedyPolicy<>();
 
 
-    public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay, boolean useEveryVisit) {
+    public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay, boolean useEveryVisit) {
         super(environment, actionSpace, discountFactor, delay);
         isEveryVisit = useEveryVisit;
         // t
@@ -42,11 +42,11 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
         returnCount = new HashMap<>();
     }
 
-    public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
+    public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
         this(environment, actionSpace, discountFactor, epsilon, delay, false);
     }
 
-    public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
+    public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
         this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
     }
 

src/main/java/core/algo/td/QLearningOffPolicyTDControl.java

@@ -40,9 +40,9 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
         while(envResult == null || !envResult.isDone()) {
             actionValues = stateActionTable.getActionValues(state);
             A action;
-            if(currentEpisode % 2 == 0){
+            if(currentEpisode % 2 == 0) {
                 action = greedyPolicy.chooseAction(actionValues);
-            }else{
+            } else {
                 action = policy.chooseAction(actionValues);
             }
             if(converged) return;
@@ -51,7 +51,7 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
             double reward = envResult.getReward();
             State nextState = envResult.getState();
             sumOfRewards += reward;
-            if(currentEpisode % 2 == 0){
+            if(currentEpisode % 2 == 0) {
                 state = nextState;
                 dispatchStepEnd();
                 continue;

src/main/java/core/controller/RLController.java

@@ -7,7 +7,7 @@ import core.ListDiscreteActionSpace;
 import core.algo.EpisodicLearning;
 import core.algo.Learning;
 import core.algo.Method;
-import core.algo.mc.MonteCarloControlFirstVisitEGreedy;
+import core.algo.mc.MonteCarloControlEGreedy;
 import core.algo.td.QLearningOffPolicyTDControl;
 import core.algo.td.SARSA;
 import core.listener.LearningListener;
@@ -49,10 +49,10 @@ public class RLController<A extends Enum> implements LearningListener {
     public void start() {
         switch(method) {
             case MC_CONTROL_FIRST_VISIT:
-                learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
+                learning = new MonteCarloControlEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
                 break;
             case MC_CONTROL_EVERY_VISIT:
-                learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay, true);
+                learning = new MonteCarloControlEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay, true);
                 break;
 
             case SARSA_ON_POLICY_CONTROL:

src/main/java/evironment/antGame/Grid.java

@@ -29,7 +29,7 @@ public class Grid {
                 initialGrid[x][y] = new Cell(new Point(x, y), CellType.FREE);
             }
         }
-        start = new Point(RNG.getRandom().nextInt(width), RNG.getRandom().nextInt(height));
+        start = new Point(RNG.getEnvRandom().nextInt(width), RNG.getEnvRandom().nextInt(height));
         initialGrid[start.x][start.y] = new Cell(new Point(start.x, start.y), CellType.START);
         spawnNewFood(initialGrid);
         spawnObstacles();
@@ -58,8 +58,8 @@ public class Grid {
         Point potFood = new Point(0, 0);
         CellType potFieldType;
         while(!foodSpawned) {
-            potFood.x = RNG.getRandom().nextInt(width);
-            potFood.y = RNG.getRandom().nextInt(height);
+            potFood.x = RNG.getEnvRandom().nextInt(width);
+            potFood.y = RNG.getEnvRandom().nextInt(height);
             potFieldType = grid[potFood.x][potFood.y].getType();
             if(potFieldType != CellType.START && grid[potFood.x][potFood.y].getFood() == 0 && potFieldType != CellType.OBSTACLE) {
                 grid[potFood.x][potFood.y].setFood(1);

src/main/java/evironment/blackjack/cards/CardDeck.java

@@ -29,6 +29,6 @@ public class CardDeck {
             nextInt(int bound) returns random int value from (inclusive) 0
             and EXCLUSIVE! bound
           */
-        return cards.get(RNG.getRandom().nextInt(cards.size()));
+        return cards.get(RNG.getEnvRandom().nextInt(cards.size()));
     }
 }

src/main/java/evironment/jumpingDino/DinoWorldAdvanced.java

@@ -31,7 +31,7 @@ public class DinoWorldAdvanced extends DinoWorld{
     protected void spawnNewObstacle() {
         int dx;
         int xSpawn;
-        double ran = RNG.getRandom().nextDouble();
+        double ran = RNG.getEnvRandom().nextDouble();
         if(ran < 0.25){
             dx = -(int) (0.35 * Config.OBSTACLE_SPEED);
         }else if(ran < 0.5){
@@ -41,7 +41,7 @@ public class DinoWorldAdvanced extends DinoWorld{
         } else{
             dx = -(int) (3.5 * Config.OBSTACLE_SPEED);
         }
-        double ran2 = RNG.getRandom().nextDouble();
+        double ran2 = RNG.getEnvRandom().nextDouble();
         if(ran2 < 0.25) {
             // randomly spawning more right outside of the screen
             xSpawn = Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE;

src/main/java/example/BlackJack.java

@@ -9,7 +9,7 @@ import evironment.blackjack.PlayerAction;
 
 public class BlackJack {
     public static void main(String[] args) {
-        RNG.setSeed(55);
+        RNG.setSeed(55, true);
 
         RLController<PlayerAction> rl = new RLControllerGUI<>(
                 new BlackJackTable(),

src/main/java/example/DinoSampling.java

@@ -13,7 +13,6 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 
 public class DinoSampling {
-    public static final float f =0.05f;
     public static final String FILE_NAME = "advancedEveryVisit.txt";
     public static void main(String[] args) {
         File file = new File(FILE_NAME);
@@ -30,7 +29,7 @@ public class DinoSampling {
             }
             for(int i = 1; i <= 100; i++) {
                 System.out.println("seed: " + i * 13);
-                RNG.setSeed(i * 13);
+                RNG.setSeed(i * 13, true);
 
                 RLController<DinoAction> rl = new RLController<>(
                         new DinoWorldAdvanced(),

src/main/java/example/JumpingDino.java

@@ -4,12 +4,11 @@ import core.RNG;
 import core.algo.Method;
 import core.controller.RLController;
 import evironment.jumpingDino.DinoAction;
-import evironment.jumpingDino.DinoWorld;
 import evironment.jumpingDino.DinoWorldAdvanced;
 
 public class JumpingDino {
     public static void main(String[] args) {
-        RNG.setSeed(29);
+        RNG.setSeed(29, true);
 
         RLController<DinoAction> rl = new RLController<>(
                 new DinoWorldAdvanced(),

src/main/java/example/Results

@@ -1,15 +0,0 @@
-Method:
-Epsilon = k / currentEpisode
-set to 0 if Epsilon < b
-
-k = 1.5
-b = 0.1   => conv. 16
-
-k = 1.5
-b = 0.02  => 75
-
-k = 1.4
-b = 0.02  => fail
-
-k = 2.0
-b = 0.02  => conv. 100

src/main/java/example/RunningAnt.java

@@ -9,7 +9,7 @@ import evironment.antGame.AntWorld;
 
 public class RunningAnt {
     public static void main(String[] args) {
-        RNG.setSeed(56);
+        RNG.setSeed(56, true);
 
         RLController<AntAction> rl = new RLControllerGUI<>(
                 new AntWorld(8, 8),
@@ -20,7 +20,6 @@ public class RunningAnt {
         rl.setNrOfEpisodes(10000);
         rl.setDiscountFactor(0.9f);
         rl.setEpsilon(0.15f);
-
         rl.start();
     }
 }

src/main/java/example/Test.java

@@ -1,52 +0,0 @@
-package example;
-
-public class Test {
-    interface Drawable{
-        void draw();
-    }
-    interface State{
-        int getInt();
-    }
-
-    static class A implements  Drawable, State{
-        private int k;
-        public A(int a){
-            k = a;
-        }
-        @Override
-        public void draw() {
-            System.out.println("draw " + k);
-        }
-
-        @Override
-        public int getInt() {
-            System.out.println("getInt" + k);
-            return k;
-        }
-    }
-
-    static class B  implements State{
-        @Override
-        public int getInt() {
-            return 0;
-        }
-    }
-
-    public static void main(String[] args) {
-        State state = new A(24);
-        State state2 = new B();
-        state.getInt();
-
-        System.out.println(state2 instanceof Drawable);
-        drawState(state2);
-    }
-
-    static void drawState(State s){
-        if(s instanceof Drawable){
-            Drawable d = (Drawable) s;
-            d.draw();
-        }else{
-            System.out.println("invalid");
-        }
-    }
-}