add Random-, Greedy and EGreedy-Policy and first implementation of monte carlo method

- fixed bug regarding wrong generation of hashCode. hashCodes needs to be equal across equal objects. Compute hashCode on final states once and return this value instead of computing it every time .hashCode() gets called.
-

src/main/java/core/Action.java

@@ -0,0 +1,4 @@
+package core;
+
+public interface Action {
+}

src/main/java/core/Environment.java

@@ -1,5 +1,6 @@
 package core;
 
 public interface Environment<A extends Enum> {
-    StepResult step(A action);
+    StepResultEnvironment step(A action);
+    State reset();
 }

src/main/java/core/StateActionHashTable.java

@@ -55,11 +55,10 @@ public class StateActionHashTable<A extends Enum> implements StateActionTable<A>
 
     @Override
     public Map<A, Double> getActionValues(State state) {
-        Map<A, Double> actionValues = table.get(state);
-        if(actionValues == null){
-            actionValues = createDefaultActionValues();
+        if(table.get(state) == null){
+            table.put(state, createDefaultActionValues());
         }
-        return actionValues;
+        return table.get(state);
     }
 
     public static void main(String[] args) {

src/main/java/core/StepResult.java

@@ -2,14 +2,11 @@ package core;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
-import lombok.Setter;
 
-@Getter
-@Setter
 @AllArgsConstructor
-public class StepResult {
+@Getter
+public class StepResult<A extends Enum> {
     private State state;
+    private A action;
     private double reward;
-    private boolean done;
-    private String info;
 }

src/main/java/core/StepResultEnvironment.java

@@ -0,0 +1,15 @@
+package core;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+public class StepResultEnvironment {
+    private State state;
+    private double reward;
+    private boolean done;
+    private String info;
+}

src/main/java/core/algo/Learning.java

@@ -0,0 +1,27 @@
+package core.algo;
+
+import core.DiscreteActionSpace;
+import core.Environment;
+import core.StateActionTable;
+import core.policy.Policy;
+
+public abstract class Learning<A extends Enum> {
+    protected Policy<A> policy;
+    protected DiscreteActionSpace<A> actionSpace;
+    protected StateActionTable<A> stateActionTable;
+    protected Environment<A> environment;
+    protected float discountFactor;
+    protected float epsilon;
+
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon){
+        this.environment = environment;
+        this.actionSpace = actionSpace;
+        this.discountFactor = discountFactor;
+        this.epsilon = epsilon;
+    }
+    public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace){
+        this(environment, actionSpace, 1.0f, 0.1f);
+    }
+
+    public abstract void learn(int nrOfEpisodes, int delay);
+}

src/main/java/core/algo/MC/MonteCarloOnPolicyEGreedy.java

@@ -0,0 +1,75 @@
+package core.algo.MC;
+
+import core.*;
+import core.algo.Learning;
+import core.policy.EpsilonGreedyPolicy;
+import javafx.util.Pair;
+
+import java.util.*;
+
+public class MonteCarloOnPolicyEGreedy<A extends Enum> extends Learning<A> {
+
+    public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace) {
+        super(environment, actionSpace);
+        discountFactor = 1f;
+        this.policy = new EpsilonGreedyPolicy<>(0.1f);
+        this.stateActionTable = new StateActionHashTable<>(actionSpace);
+    }
+
+    @Override
+    public void learn(int nrOfEpisodes, int delay) {
+
+        Map<Pair<State, A>, Double> returnSum = new HashMap<>();
+        Map<Pair<State, A>, Integer> returnCount = new HashMap<>();
+
+        for(int i = 0; i < nrOfEpisodes; ++i) {
+
+            List<StepResult<A>> episode = new ArrayList<>();
+            State state = environment.reset();
+            for(int j=0; j < 100; ++j){
+                Map<A, Double> actionValues = stateActionTable.getActionValues(state);
+                A chosenAction = policy.chooseAction(actionValues);
+                StepResultEnvironment envResult = environment.step(chosenAction);
+                State nextState = envResult.getState();
+                episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
+
+                if(envResult.isDone()) break;
+
+                state = nextState;
+
+                try {
+                    Thread.sleep(10);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+
+            Set<Pair<State, A>> stateActionPairs = new HashSet<>();
+
+            for(StepResult<A> sr: episode){
+                stateActionPairs.add(new Pair<>(sr.getState(), sr.getAction()));
+            }
+
+            for(Pair<State, A> stateActionPair: stateActionPairs){
+                int firstOccurenceIndex = 0;
+                // find first occurance of state action pair
+                for(StepResult<A> sr: episode){
+                    if(stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())){
+                        break;
+                    }
+                    firstOccurenceIndex++;
+                }
+
+                double G = 0;
+                for(int l = firstOccurenceIndex; l < episode.size(); ++l){
+                    G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
+                }
+                // slick trick to add G to the entry.
+                // if the key does not exists, it will create a new entry with G as default value
+                returnSum.merge(stateActionPair, G, Double::sum);
+                returnCount.merge(stateActionPair, 1, Integer::sum);
+                stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
+            }
+        }
+    }
+}

src/main/java/core/algo/TD/TemporalDifferenceOnPolicy.java

@@ -0,0 +1,4 @@
+package core.algo.TD;
+
+public class TemporalDifferenceOnPolicy {
+}

src/main/java/core/policy/EpsilonGreedyPolicy.java

@@ -0,0 +1,35 @@
+package core.policy;
+
+import core.RNG;
+
+import java.util.Map;
+
+/**
+ * To prevent the agent from getting stuck only using the "best" action
+ * according to the current learning history, this policy
+ * will take random action with the probability of epsilon.
+ * (random action space includes the best action as well)
+ *
+ * @param <A> Discrete Action Enum
+ */
+public class EpsilonGreedyPolicy<A extends Enum> implements Policy<A>{
+    private float epsilon;
+    private RandomPolicy<A> randomPolicy;
+    private GreedyPolicy<A> greedyPolicy;
+
+    public EpsilonGreedyPolicy(float epsilon){
+        this.epsilon = epsilon;
+        randomPolicy = new RandomPolicy<>();
+        greedyPolicy = new GreedyPolicy<>();
+    }
+    @Override
+    public A chooseAction(Map<A, Double> actionValues) {
+        if(RNG.getRandom().nextFloat() < epsilon){
+            // Take random action
+            return randomPolicy.chooseAction(actionValues);
+        }else{
+            // Take the action with the highest value
+            return greedyPolicy.chooseAction(actionValues);
+        }
+    }
+}

src/main/java/core/policy/GreedyPolicy.java

@@ -0,0 +1,32 @@
+package core.policy;
+
+import core.RNG;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class GreedyPolicy<A extends Enum> implements Policy<A> {
+
+    @Override
+    public A chooseAction(Map<A, Double> actionValues) {
+        if(actionValues.size() == 0) throw new RuntimeException("Empty actionActionValues set");
+
+       Double highestValueAction = null;
+
+        List<A> equalHigh = new ArrayList<>();
+
+        for(Map.Entry<A, Double> actionValue : actionValues.entrySet()){
+            System.out.println(actionValue.getKey()+ " " + actionValue.getValue() );
+            if(highestValueAction == null || highestValueAction < actionValue.getValue()){
+                highestValueAction = actionValue.getValue();
+                equalHigh.clear();
+                equalHigh.add(actionValue.getKey());
+            }else if(highestValueAction.equals(actionValue.getValue())){
+                equalHigh.add(actionValue.getKey());
+            }
+        }
+
+        return equalHigh.get(RNG.getRandom().nextInt(equalHigh.size()));
+    }
+}

src/main/java/core/policy/Policy.java

@@ -0,0 +1,7 @@
+package core.policy;
+
+import java.util.Map;
+
+public interface Policy<A extends Enum> {
+    A chooseAction(Map<A, Double> actionValues);
+}

src/main/java/core/policy/RandomPolicy.java

@@ -0,0 +1,17 @@
+package core.policy;
+
+import core.RNG;
+import java.util.Map;
+
+public class RandomPolicy<A extends Enum> implements Policy<A>{
+    @Override
+    public A chooseAction(Map<A, Double> actionValues) {
+        int idx = RNG.getRandom().nextInt(actionValues.size());
+        int i = 0;
+        for(A action : actionValues.keySet()){
+            if(i++ == idx) return action;
+        }
+
+        return null;
+    }
+}