kono/refo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

modify q Learning to sample results and update R script

Browse Source
This commit is contained in:
kono
2020-03-28 12:35:33 +01:00
parent eca0d8db4d
commit 328fc85214
9 changed files with 97 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/core/RNG.java
+8
View File
@@ -15,18 +15,26 @@ import java.util.Random;
*/
public class RNG {
private static Random rng;
private static Random rngEnv;
private static int seed = 123;
static {
rng = new Random();
rng.setSeed(seed);
rngEnv = new Random();
rngEnv.setSeed(seed);
}
public static Random getRandom() {
return rng;
}
public static Random getRandomEnv() {
return rngEnv;
}
public static void setSeed(int seed){
RNG.seed = seed;
rng.setSeed(seed);
rngEnv = new Random();
rngEnv.setSeed(seed);
}
}
src/main/java/core/algo/EpisodicLearning.java
+2 -5
View File
@@ -105,8 +105,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
timestamp++;
timestampCurrentEpisode++;
// TODO: more sophisticated way to check convergence
if(timestampCurrentEpisode > 30000000){
converged = true;
if(false){
// t
File file = new File(DinoSampling.FILE_NAME);
try {
@@ -114,9 +113,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("converged after: " + currentEpisode/2 + " episode!");
episodesToLearn.set(0);
dispatchLearningEnd();
// System.out.println("converged after: " + currentEpisode/2 + " episode!");
}
}
src/main/java/core/algo/mc/MonteCarloControlFirstVisitEGreedy.java
+2 -17
View File
@@ -40,16 +40,9 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
private Map<Pair<State, A>, Double> returnSum;
private Map<Pair<State, A>, Integer> returnCount;
// t
private float epsilon;
// t
private Policy<A> greedyPolicy = new GreedyPolicy<>();
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
super(environment, actionSpace, discountFactor, delay);
// t
this.epsilon = epsilon;
this.policy = new EpsilonGreedyPolicy<>(epsilon);
this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
returnSum = new HashMap<>();
@@ -74,12 +67,7 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
while(envResult == null || !envResult.isDone()) {
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A chosenAction;
if(currentEpisode % 2 == 1){
chosenAction = greedyPolicy.chooseAction(actionValues);
}else{
chosenAction = policy.chooseAction(actionValues);
}
A chosenAction = policy.chooseAction(actionValues);
envResult = environment.step(chosenAction);
State nextState = envResult.getState();
@@ -96,12 +84,9 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
}
timestamp++;
dispatchStepEnd();
if(converged) return;
}
if(currentEpisode % 2 == 1){
return;
}
// System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
Set<Pair<State, A>> stateActionPairs = new LinkedHashSet<>();
src/main/java/core/algo/td/QLearningOffPolicyTDControl.java
+37 -11
View File
@@ -5,7 +5,15 @@ import core.algo.EpisodicLearning;
import core.policy.EpsilonGreedyPolicy;
import core.policy.GreedyPolicy;
import core.policy.Policy;
import evironment.antGame.Reward;
import example.ContinuousAnt;
import example.DinoSampling;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;
public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearning<A> {
@@ -37,25 +45,43 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
sumOfRewards = 0;
int timestampTilFood = 0;
int rewardsPer1000 = 0;
int foodCollected = 0;
while(envResult == null || !envResult.isDone()) {
actionValues = stateActionTable.getActionValues(state);
A action;
if(currentEpisode % 2 == 0){
action = greedyPolicy.chooseAction(actionValues);
}else{
action = policy.chooseAction(actionValues);
}
if(converged) return;
A action = policy.chooseAction(actionValues);
// Take a step
envResult = environment.step(action);
double reward = envResult.getReward();
State nextState = envResult.getState();
sumOfRewards += reward;
if(currentEpisode % 2 == 0){
state = nextState;
dispatchStepEnd();
continue;
rewardsPer1000+=reward;
timestampTilFood++;
if(foodCollected == 10000){
File file = new File(ContinuousAnt.FILE_NAME);
try {
Files.writeString(Path.of(file.getPath()), "\n", StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
return;
}
if(reward == Reward.FOOD_DROP_DOWN_SUCCESS){
foodCollected++;
File file = new File(ContinuousAnt.FILE_NAME);
try {
Files.writeString(Path.of(file.getPath()), timestampTilFood + ",", StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
timestampTilFood = 0;
rewardsPer1000 = 0;
}
// Q Update
double currentQValue = stateActionTable.getActionValues(state).get(action);
// maxQ(S', a);
src/main/java/core/algo/td/SARSA.java
+4 -21
View File
@@ -11,7 +11,6 @@ import java.util.Map;
public class SARSA<A extends Enum> extends EpisodicLearning<A> {
private float alpha;
private Policy<A> greedyPolicy = new GreedyPolicy<>();
public SARSA(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, float learningRate, int delay) {
super(environment, actionSpace, discountFactor, delay);
@@ -35,18 +34,13 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
StepResultEnvironment envResult = null;
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A action;
if(currentEpisode % 2 == 1){
action = greedyPolicy.chooseAction(actionValues);
}else{
action = policy.chooseAction(actionValues);
}
A action = policy.chooseAction(actionValues);
//A action = policy.chooseAction(actionValues);
sumOfRewards = 0;
while(envResult == null || !envResult.isDone()) {
if(converged) return;
// Take a step
envResult = environment.step(action);
sumOfRewards += envResult.getReward();
@@ -56,19 +50,8 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
// Pick next action
actionValues = stateActionTable.getActionValues(nextState);
A nextAction;
if(currentEpisode % 2 == 1){
nextAction = greedyPolicy.chooseAction(actionValues);
}else{
nextAction = policy.chooseAction(actionValues);
}
//A nextAction = policy.chooseAction(actionValues);
if(currentEpisode % 2 == 1){
state = nextState;
action = nextAction;
dispatchStepEnd();
continue;
}
A nextAction = policy.chooseAction(actionValues);
// td update
// target = reward + gamma * Q(nextState, nextAction)
double currentQValue = stateActionTable.getActionValues(state).get(action);