add TD algorithms and started adopting to continous tasks

- add Q-Learning and SARSA
- more config variables

src/main/java/core/LearningConfig.java

@@ -2,6 +2,9 @@ package core;
 
 public class LearningConfig {
     public static final int DEFAULT_DELAY = 30;
+    public static final int DEFAULT_NR_OF_EPISODES = 10000;
     public static final float DEFAULT_EPSILON = 0.1f;
     public static final float DEFAULT_DISCOUNT_FACTOR = 1.0f;
+    // Learning rate
+    public static final float DEFAULT_ALPHA = 0.9f;
 }

src/main/java/core/algo/EpisodicLearning.java

@@ -2,6 +2,7 @@ package core.algo;
 
 import core.DiscreteActionSpace;
 import core.Environment;
+import core.LearningConfig;
 import core.StepResult;
 import core.listener.LearningListener;
 import lombok.Getter;
@@ -16,7 +17,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 public abstract class EpisodicLearning<A extends Enum> extends Learning<A> implements Episodic {
     @Setter
-    protected int currentEpisode;
+    protected int currentEpisode = 0;
     protected volatile AtomicInteger episodesToLearn = new AtomicInteger(0);
     @Getter
     protected volatile int episodePerSecond;
@@ -81,7 +82,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
 
     @Override
     public void learn(){
-      // TODO remove or learn with default episode number
+        learn(LearningConfig.DEFAULT_NR_OF_EPISODES);
     }
 
     private void startLearning(){
@@ -132,6 +133,15 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
         return episodesToLearn.get();
     }
 
+
+    public int getCurrentEpisode() {
+        return currentEpisode;
+    }
+
+    public int getEpisodesPerSecond() {
+        return episodePerSecond;
+    }
+
     @Override
     public synchronized void save(ObjectOutputStream oos) throws IOException {
       super.save(oos);

src/main/java/core/algo/MC/MonteCarloControlEGreedy.java

@@ -30,21 +30,20 @@ import java.util.*;
  *
  * @param <A>
  */
-public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<A> {
+public class MonteCarloControlEGreedy<A extends Enum> extends EpisodicLearning<A> {
 
     private Map<Pair<State, A>, Double> returnSum;
     private Map<Pair<State, A>, Integer> returnCount;
 
-    public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
+    public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
         super(environment, actionSpace, discountFactor, delay);
-        currentEpisode = 0;
         this.policy = new EpsilonGreedyPolicy<>(epsilon);
         this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
         returnSum = new HashMap<>();
         returnCount = new HashMap<>();
     }
 
-    public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
+    public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
         this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
     }
 
@@ -59,7 +58,7 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
         }
         sumOfRewards = 0;
         StepResultEnvironment envResult = null;
-        while(envResult == null || !envResult.isDone()){
+        while(envResult == null || !envResult.isDone()) {
             Map<A, Double> actionValues = stateActionTable.getActionValues(state);
             A chosenAction = policy.chooseAction(actionValues);
             envResult = environment.step(chosenAction);
@@ -77,26 +76,26 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
             dispatchStepEnd();
         }
 
-      //  System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
+        //  System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
         Set<Pair<State, A>> stateActionPairs = new LinkedHashSet<>();
 
-        for (StepResult<A> sr : episode) {
+        for(StepResult<A> sr : episode) {
             stateActionPairs.add(new ImmutablePair<>(sr.getState(), sr.getAction()));
         }
 
         //System.out.println("stateActionPairs " + stateActionPairs.size());
-        for (Pair<State, A> stateActionPair : stateActionPairs) {
+        for(Pair<State, A> stateActionPair : stateActionPairs) {
             int firstOccurenceIndex = 0;
             // find first occurance of state action pair
-            for (StepResult<A> sr : episode) {
-                if (stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())) {
+            for(StepResult<A> sr : episode) {
+                if(stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())) {
                     break;
                 }
                 firstOccurenceIndex++;
             }
 
             double G = 0;
-            for (int l = firstOccurenceIndex; l < episode.size(); ++l) {
+            for(int l = firstOccurenceIndex; l < episode.size(); ++l) {
                 G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
             }
             // slick trick to add G to the entry.
@@ -107,16 +106,6 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
         }
     }
 
-    @Override
-    public int getCurrentEpisode() {
-        return currentEpisode;
-    }
-
-    @Override
-    public int getEpisodesPerSecond(){
-        return episodePerSecond;
-    }
-
     @Override
     public void save(ObjectOutputStream oos) throws IOException {
         super.save(oos);

src/main/java/core/algo/Method.java

@@ -5,5 +5,5 @@ package core.algo;
  * which RL-algorithm should be used.
  */
 public enum Method {
-    MC_ONPOLICY_EGREEDY, TD_ONPOLICY
+    MC_CONTROL_EGREEDY, SARSA_EPISODIC, Q_LEARNING_OFF_POLICY_CONTROL
 }

src/main/java/core/algo/TD/QLearningOffPolicyTDControl.java

@@ -0,0 +1,68 @@
+package core.algo.td;
+
+import core.*;
+import core.algo.EpisodicLearning;
+import core.policy.EpsilonGreedyPolicy;
+import core.policy.GreedyPolicy;
+import core.policy.Policy;
+
+import java.util.Map;
+
+public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearning<A> {
+    private float alpha;
+    private Policy<A> greedyPolicy = new GreedyPolicy<>();
+
+    public QLearningOffPolicyTDControl(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, float learningRate, int delay) {
+        super(environment, actionSpace, discountFactor, delay);
+        alpha = learningRate;
+        this.policy = new EpsilonGreedyPolicy<>(epsilon);
+        this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
+    }
+
+    public QLearningOffPolicyTDControl(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
+        this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, LearningConfig.DEFAULT_ALPHA, delay);
+    }
+
+    @Override
+    protected void nextEpisode() {
+        State state = environment.reset();
+        try {
+            Thread.sleep(delay);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
+        StepResultEnvironment envResult = null;
+        Map<A, Double> actionValues = null;
+
+
+        sumOfRewards = 0;
+        while(envResult == null || !envResult.isDone()) {
+            actionValues = stateActionTable.getActionValues(state);
+            A action = policy.chooseAction(actionValues);
+
+            // Take a step
+            envResult = environment.step(action);
+            double reward = envResult.getReward();
+            State nextState = envResult.getState();
+            sumOfRewards += reward;
+
+            // Q Update
+            double currentQValue = stateActionTable.getActionValues(state).get(action);
+            // maxQ(S', a);
+            // Using intern "greedy policy" as a helper to determine the highest action-value
+            double highestValueNextState = stateActionTable.getActionValues(nextState).get(greedyPolicy.chooseAction(stateActionTable.getActionValues(nextState)));
+
+            double updatedQValue = currentQValue + alpha * (reward + discountFactor * highestValueNextState - currentQValue);
+            stateActionTable.setValue(state, action, updatedQValue);
+
+            state = nextState;
+            try {
+                Thread.sleep(delay);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            dispatchStepEnd();
+        }
+    }
+}

src/main/java/core/algo/TD/SARSA.java

@@ -0,0 +1,68 @@
+package core.algo.td;
+
+import core.*;
+import core.algo.EpisodicLearning;
+import core.policy.EpsilonGreedyPolicy;
+
+import java.util.Map;
+
+
+public class SARSA<A extends Enum> extends EpisodicLearning<A> {
+    private float alpha;
+
+    public SARSA(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, float learningRate, int delay) {
+        super(environment, actionSpace, discountFactor, delay);
+        alpha = learningRate;
+        this.policy = new EpsilonGreedyPolicy<>(epsilon);
+        this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
+    }
+
+    public SARSA(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
+        this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, LearningConfig.DEFAULT_ALPHA, delay);
+    }
+
+    @Override
+    protected void nextEpisode() {
+        State state = environment.reset();
+        try {
+            Thread.sleep(delay);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
+        StepResultEnvironment envResult = null;
+        Map<A, Double> actionValues = stateActionTable.getActionValues(state);
+        A action = policy.chooseAction(actionValues);
+
+        sumOfRewards = 0;
+        while(envResult == null || !envResult.isDone()) {
+            // Take a step
+            envResult = environment.step(action);
+            sumOfRewards += envResult.getReward();
+
+            State nextState = envResult.getState();
+
+            // Pick next action
+            actionValues = stateActionTable.getActionValues(nextState);
+            A nextAction = policy.chooseAction(actionValues);
+
+            // TD update
+            // target = reward + gamma * Q(nextState, nextAction)
+            double currentQValue = stateActionTable.getActionValues(state).get(action);
+            double nextQValue = stateActionTable.getActionValues(nextState).get(nextAction);
+            double reward = envResult.getReward();
+            double updatedQValue = currentQValue + alpha * (reward + discountFactor * nextQValue - currentQValue);
+            stateActionTable.setValue(state, action, updatedQValue);
+
+            state = nextState;
+            action = nextAction;
+
+            try {
+                Thread.sleep(delay);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            dispatchStepEnd();
+        }
+    }
+}

src/main/java/core/algo/TD/TemporalDifferenceOnPolicy.java

@@ -1,4 +0,0 @@
-package core.algo.TD;
-
-public class TemporalDifferenceOnPolicy {
-}

src/main/java/core/controller/RLController.java

@@ -1,18 +1,19 @@
 package core.controller;
 
-import core.*;
+import core.DiscreteActionSpace;
+import core.Environment;
+import core.LearningConfig;
+import core.ListDiscreteActionSpace;
 import core.algo.EpisodicLearning;
 import core.algo.Learning;
 import core.algo.Method;
-import core.algo.mc.MonteCarloOnPolicyEGreedy;
-import core.gui.LearningView;
-import core.gui.View;
+import core.algo.mc.MonteCarloControlEGreedy;
+import core.algo.td.QLearningOffPolicyTDControl;
+import core.algo.td.SARSA;
 import core.listener.LearningListener;
-import core.listener.ViewListener;
 import core.policy.EpsilonPolicy;
 import lombok.Setter;
 
-import javax.swing.*;
 import java.io.*;
 import java.util.List;
 
@@ -27,6 +28,8 @@ public class RLController<A extends Enum> implements LearningListener {
     @Setter
     protected float discountFactor = LearningConfig.DEFAULT_DISCOUNT_FACTOR;
     @Setter
+    protected float learningRate = LearningConfig.DEFAULT_DISCOUNT_FACTOR;
+    @Setter
     protected float epsilon = LearningConfig.DEFAULT_EPSILON;
     protected Learning<A> learning;
     protected boolean fastLearning;
@@ -45,10 +48,14 @@ public class RLController<A extends Enum> implements LearningListener {
 
     public void start() {
         switch(method) {
-            case MC_ONPOLICY_EGREEDY:
-                learning = new MonteCarloOnPolicyEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
+            case MC_CONTROL_EGREEDY:
+                learning = new MonteCarloControlEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
                 break;
-            case TD_ONPOLICY:
+            case SARSA_EPISODIC:
+                learning = new SARSA<>(environment, discreteActionSpace, discountFactor, epsilon, learningRate, delay);
+                break;
+            case Q_LEARNING_OFF_POLICY_CONTROL:
+                learning = new QLearningOffPolicyTDControl<>(environment, discreteActionSpace, discountFactor, epsilon, learningRate, delay);
                 break;
             default:
                 throw new IllegalArgumentException("Undefined method");

src/main/java/core/gui/StateActionRow.java

@@ -30,7 +30,6 @@ public class StateActionRow<A extends Enum> extends JTextArea {
 
     protected void refreshLabels(){
         if(state == null || actionValues == null) return;
-        System.out.println("refreshing");
         StringBuilder sb = new StringBuilder(state.toString()).append("\n");
         for(Map.Entry<A, Double> actionValue: actionValues.entrySet()){
             sb.append("\t").append(actionValue.getKey()).append("\t").append(actionValue.getValue()).append("\n");