create Dino Sampling state

src/main/java/core/algo/EpisodicLearning.java

@@ -6,6 +6,7 @@ import core.LearningConfig;
 import core.StepResult;
 import core.listener.LearningListener;
 import core.policy.EpsilonGreedyPolicy;
+import example.DinoSampling;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -104,10 +105,10 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
         timestamp++;
         timestampCurrentEpisode++;
         // TODO: more sophisticated way to check convergence
-        if(timestampCurrentEpisode > 300000){
+        if(timestampCurrentEpisode > 30000000){
             converged = true;
             // t
-            File file = new File("convergenceAdv.txt");
+            File file = new File(DinoSampling.FILE_NAME);
             try {
                 Files.writeString(Path.of(file.getPath()),  currentEpisode/2 + ",", StandardOpenOption.APPEND);
             } catch (IOException e) {
@@ -127,7 +128,6 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
     private void startLearning(){
         dispatchLearningStart();
         while(episodesToLearn.get() > 0){
-
             dispatchEpisodeStart();
             timestampCurrentEpisode = 0;
             nextEpisode();

src/main/java/core/algo/td/QLearningOffPolicyTDControl.java

@@ -39,14 +39,23 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
         sumOfRewards = 0;
         while(envResult == null || !envResult.isDone()) {
             actionValues = stateActionTable.getActionValues(state);
-            A action = policy.chooseAction(actionValues);
-
+            A action;
+            if(currentEpisode % 2 == 0){
+                action = greedyPolicy.chooseAction(actionValues);
+            }else{
+                action = policy.chooseAction(actionValues);
+            }
+            if(converged) return;
             // Take a step
             envResult = environment.step(action);
             double reward = envResult.getReward();
             State nextState = envResult.getState();
             sumOfRewards += reward;
-
+            if(currentEpisode % 2 == 0){
+                state = nextState;
+                dispatchStepEnd();
+                continue;
+            }
             // Q Update
             double currentQValue = stateActionTable.getActionValues(state).get(action);
             // maxQ(S', a);

src/main/java/core/algo/td/SARSA.java

@@ -3,12 +3,15 @@ package core.algo.td;
 import core.*;
 import core.algo.EpisodicLearning;
 import core.policy.EpsilonGreedyPolicy;
+import core.policy.GreedyPolicy;
+import core.policy.Policy;
 
 import java.util.Map;
 
 
 public class SARSA<A extends Enum> extends EpisodicLearning<A> {
     private float alpha;
+    private Policy<A> greedyPolicy = new GreedyPolicy<>();
 
     public SARSA(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, float learningRate, int delay) {
         super(environment, actionSpace, discountFactor, delay);
@@ -32,10 +35,18 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
 
         StepResultEnvironment envResult = null;
         Map<A, Double> actionValues = stateActionTable.getActionValues(state);
-        A action = policy.chooseAction(actionValues);
+        A action;
+        if(currentEpisode % 2 == 1){
+            action = greedyPolicy.chooseAction(actionValues);
+        }else{
+            action = policy.chooseAction(actionValues);
+        }
+        //A action = policy.chooseAction(actionValues);
 
         sumOfRewards = 0;
         while(envResult == null || !envResult.isDone()) {
+
+            if(converged) return;
             // Take a step
             envResult = environment.step(action);
             sumOfRewards += envResult.getReward();
@@ -44,8 +55,20 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
 
             // Pick next action
             actionValues = stateActionTable.getActionValues(nextState);
-            A nextAction = policy.chooseAction(actionValues);
 
+            A nextAction;
+            if(currentEpisode % 2 == 1){
+                nextAction = greedyPolicy.chooseAction(actionValues);
+            }else{
+                nextAction = policy.chooseAction(actionValues);
+            }
+            //A nextAction = policy.chooseAction(actionValues);
+            if(currentEpisode % 2 == 1){
+                state = nextState;
+                action = nextAction;
+                dispatchStepEnd();
+                continue;
+            }
             // td update
             // target = reward + gamma * Q(nextState, nextAction)
             double currentQValue = stateActionTable.getActionValues(state).get(action);

src/main/java/core/policy/EpsilonPolicy.java

@@ -1,8 +1,8 @@
 package core.policy;
 
 /**
- * Chooses the action with the highest values with possibility: 1-Ɛ + Ɛ/|A|
- * With possibility of Ɛ, a random action is taken (highest values option included).
+ * Chooses the action with the highest values with possibility: 1-Epsilon + Epsilon/|A|
+ * With possibility of Epsilon, a random action is taken (highest values option included).
  *
  * @param <A> Enum class of available action in specific environment
  */