kono/refo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'antWorldRewardAnalysis'

Browse Source 
# Conflicts:
#	src/main/java/core/algo/EpisodicLearning.java
#	src/main/java/core/controller/RLController.java
#	src/main/java/evironment/jumpingDino/DinoWorld.java
#	src/main/java/evironment/jumpingDino/DinoWorldAdvanced.java
#	src/main/java/example/JumpingDino.java
This commit is contained in:
kono
2020-04-05 13:21:20 +02:00
parent b1d06293fe ad07c1da8f
commit 0300f3b1fd
62 changed files with 1101 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/core/RNG.java
+21 -6
View File
@@ -1,9 +1,10 @@
package core;
import java.security.SecureRandom;
import java.util.Random;
/**
* ! SecureRandom not working properly on windows/different JDKs,
* using Random again !
*
* To ensure deterministic behaviour of repeating program executions,
* this class is used for all random number generation methods.
* Do not use Math.random()!
@@ -13,19 +14,33 @@ import java.util.Random;
* execution)
*/
public class RNG {
private static SecureRandom rng;
private static Random rng;
private static Random rngEnv;
private static int seed = 123;
private static int envSeed = 13;
static {
rng = new SecureRandom();
rng.setSeed(seed);
rng = new Random();
rngEnv = new Random();
setSeed(seed, true);
}
public static Random getRandom() {
return rng;
}
public static Random getRandomEnv() {
return rngEnv;
}
public static void setSeed(int seed){
public static void setSeed(int seed, boolean setEnvRandom) {
RNG.seed = seed;
rng.setSeed(seed);
if(setEnvRandom) {
rngEnv.setSeed(13);
}
}
public static void setSeed(int seed) {
setSeed(seed, true);
}
}
src/main/java/core/algo/EpisodicLearning.java
+9 -10
View File
@@ -5,7 +5,6 @@ import core.Environment;
import core.LearningConfig;
import core.StepResult;
import core.listener.LearningListener;
import core.policy.EpsilonGreedyPolicy;
import lombok.Getter;
import lombok.Setter;
@@ -73,7 +72,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
}
}
protected void dispatchEpisodeStart(){
private void dispatchEpisodeStart(){
++currentEpisode;
episodesToLearn.decrementAndGet();
for(LearningListener l: learningListeners){
@@ -85,13 +84,20 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
protected void dispatchStepEnd() {
super.dispatchStepEnd();
timestamp++;
timestampCurrentEpisode++;
}
@Override
public void learn(){
learn(LearningConfig.DEFAULT_NR_OF_EPISODES);
}
private void startLearning(){
dispatchLearningStart();
System.out.println(episodesToLearn.get());
while(episodesToLearn.get() > 0){
dispatchEpisodeStart();
timestampCurrentEpisode = 0;
nextEpisode();
dispatchEpisodeEnd();
}
@@ -104,7 +110,6 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
public void learnMoreEpisodes(int nrOfEpisodes){
episodesToLearn.addAndGet(nrOfEpisodes);
}
/**
* Stopping the while loop by setting episodesToLearn to 0.
* The current episode can not be interrupted, so the sleep delay
@@ -127,14 +132,8 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
delay = prevDelay;
}
@Override
public void learn(){
learn(LearningConfig.DEFAULT_NR_OF_EPISODES);
}
public synchronized void learn(int nrOfEpisodes){
boolean isLearning = episodesToLearn.getAndAdd(nrOfEpisodes) != 0;
System.out.println(isLearning);
if(!isLearning)
startLearning();
}
src/main/java/core/algo/Learning.java
+1 -3
View File
@@ -16,8 +16,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
*
@@ -99,7 +97,7 @@ public abstract class Learning<A extends Enum>{
public void save(ObjectOutputStream oos) throws IOException {
oos.writeObject(rewardHistory);
oos.writeObject(stateActionTable);
// oos.writeObject(stateActionTable);
}
public void load(ObjectInputStream ois) throws IOException, ClassNotFoundException {
src/main/java/core/algo/Method.java
+1 -1
View File
@@ -5,5 +5,5 @@ package core.algo;
* which RL-algorithm should be used.
*/
public enum Method {
MC_CONTROL_FIRST_VISIT, SARSA_EPISODIC, Q_LEARNING_OFF_POLICY_CONTROL
MC_CONTROL_FIRST_VISIT, MC_CONTROL_EVERY_VISIT, SARSA_ON_POLICY_CONTROL, Q_LEARNING_OFF_POLICY_CONTROL
}
src/main/java/core/algo/mc/MonteCarloControlEGreedy.java
+134
View File
@@ -0,0 +1,134 @@
package core.algo.mc;
import core.*;
import core.algo.EpisodicLearning;
import core.policy.EpsilonGreedyPolicy;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.*;
/**
* Includes both! variants of Monte-Carlo methods
* Default method is First-Visit.
* Change to Every-Visit by setting flag "useEveryVisit" in the constructor to true.
* @param <A>
*/
public class MonteCarloControlEGreedy<A extends Enum> extends EpisodicLearning<A> {
private Map<Pair<State, A>, Double> returnSum;
private Map<Pair<State, A>, Integer> returnCount;
private boolean isEveryVisit;
public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay, boolean useEveryVisit) {
super(environment, actionSpace, discountFactor, delay);
isEveryVisit = useEveryVisit;
this.policy = new EpsilonGreedyPolicy<>(epsilon);
this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
returnSum = new HashMap<>();
returnCount = new HashMap<>();
}
public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
this(environment, actionSpace, discountFactor, epsilon, delay, false);
}
public MonteCarloControlEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
}
@Override
public void nextEpisode() {
episode = new ArrayList<>();
State state = environment.reset();
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
sumOfRewards = 0;
StepResultEnvironment envResult = null;
while(envResult == null || !envResult.isDone()) {
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A chosenAction = policy.chooseAction(actionValues);
envResult = environment.step(chosenAction);
State nextState = envResult.getState();
sumOfRewards += envResult.getReward();
rewardCheckSum += envResult.getReward();
episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
state = nextState;
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
timestamp++;
dispatchStepEnd();
}
// System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
HashMap<Pair<State, A>, List<Integer>> stateActionPairs = new LinkedHashMap<>();
int firstOccurrenceIndex = 0;
for(StepResult<A> sr : episode) {
Pair<State, A> pair = new ImmutablePair<>(sr.getState(), sr.getAction());
if(!stateActionPairs.containsKey(pair)) {
List<Integer> l = new ArrayList<>();
l.add(firstOccurrenceIndex);
stateActionPairs.put(pair, l);
}
/*
This is the only difference between First-Visit and Every-Visit.
When First-Visit is selected, only the first index of the occurrence is put into the list.
When Every-Visit is selected, every following occurrence is saved
into the list as well.
*/
else if(isEveryVisit) {
stateActionPairs.get(pair).add(firstOccurrenceIndex);
}
++firstOccurrenceIndex;
}
//System.out.println("stateActionPairs " + stateActionPairs.size());
for(Map.Entry<Pair<State, A>, List<Integer>> entry : stateActionPairs.entrySet()) {
Pair<State, A> stateActionPair = entry.getKey();
List<Integer> firstOccurrences = entry.getValue();
for(Integer firstOccurrencesIdx : firstOccurrences) {
double G = 0;
for(int l = firstOccurrencesIdx; l < episode.size(); ++l) {
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurrencesIdx));
}
// slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
}
}
}
@Override
public void save(ObjectOutputStream oos) throws IOException {
super.save(oos);
oos.writeObject(returnSum);
oos.writeObject(returnCount);
}
@Override
public void load(ObjectInputStream ois) throws IOException, ClassNotFoundException {
super.load(ois);
returnSum = (Map<Pair<State, A>, Double>) ois.readObject();
returnCount = (Map<Pair<State, A>, Integer>) ois.readObject();
}
}
src/main/java/core/algo/mc/MonteCarloControlFirstVisitEGreedy.java
-127
View File
@@ -1,127 +0,0 @@
package core.algo.mc;
import core.*;
import core.algo.EpisodicLearning;
import core.policy.EpsilonGreedyPolicy;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.*;
/**
* TODO: Major problem:
* StateActionPairs are only unique accounting for their position in the episode.
* For example:
* <p>
* startingState -> MOVE_LEFT : very first state action in the episode i = 1
* image the agent does not collect the food and does not drop it onto start, the agent will receive
* -1 for every timestamp hence (startingState -> MOVE_LEFT) will get a value of -10;
* <p>
* BUT image moving left from the starting position will have no impact on the state because
* the agent ran into a wall. The known world stays the same.
* Taking an action after that will have the exact same state but a different action
* making the value of this stateActionPair -9 because the stateAction pair took place on the second
* timestamp, summing up all remaining rewards will be -9...
* <p>
* How to encounter this problem?
*
* @param <A>
*/
public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends EpisodicLearning<A> {
private Map<Pair<State, A>, Double> returnSum;
private Map<Pair<State, A>, Integer> returnCount;
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
super(environment, actionSpace, discountFactor, delay);
this.policy = new EpsilonGreedyPolicy<>(epsilon);
this.stateActionTable = new DeterministicStateActionTable<>(this.actionSpace);
returnSum = new HashMap<>();
returnCount = new HashMap<>();
}
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
}
@Override
public void nextEpisode() {
episode = new ArrayList<>();
State state = environment.reset();
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
sumOfRewards = 0;
StepResultEnvironment envResult = null;
//TODO extract to learning
int timestamp = 0;
while(envResult == null || !envResult.isDone()) {
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A chosenAction = policy.chooseAction(actionValues);
checkSum += chosenAction.ordinal();
envResult = environment.step(chosenAction);
State nextState = envResult.getState();
sumOfRewards += envResult.getReward();
rewardCheckSum += envResult.getReward();
episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
state = nextState;
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
timestamp++;
dispatchStepEnd();
}
// System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
Set<Pair<State, A>> stateActionPairs = new LinkedHashSet<>();
for(StepResult<A> sr : episode) {
stateActionPairs.add(new ImmutablePair<>(sr.getState(), sr.getAction()));
}
//System.out.println("stateActionPairs " + stateActionPairs.size());
for(Pair<State, A> stateActionPair : stateActionPairs) {
int firstOccurenceIndex = 0;
// find first occurance of state action pair
for(StepResult<A> sr : episode) {
if(stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())) {
break;
}
firstOccurenceIndex++;
}
double G = 0;
for(int l = firstOccurenceIndex; l < episode.size(); ++l) {
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
}
// slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
}
}
@Override
public void save(ObjectOutputStream oos) throws IOException {
super.save(oos);
oos.writeObject(returnSum);
oos.writeObject(returnCount);
}
@Override
public void load(ObjectInputStream ois) throws IOException, ClassNotFoundException {
super.load(ois);
returnSum = (Map<Pair<State, A>, Double>) ois.readObject();
returnCount = (Map<Pair<State, A>, Integer>) ois.readObject();
}
}
src/main/java/core/algo/td/QLearningOffPolicyTDControl.java
+48
View File
@@ -5,7 +5,14 @@ import core.algo.EpisodicLearning;
import core.policy.EpsilonGreedyPolicy;
import core.policy.GreedyPolicy;
import core.policy.Policy;
import evironment.antGame.Reward;
import example.ContinuousAnt;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;
public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearning<A> {
@@ -37,6 +44,9 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
sumOfRewards = 0;
int timestampTilFood = 0;
int foodCollected = 0;
int foodTimestampsTotal= 0;
while(envResult == null || !envResult.isDone()) {
actionValues = stateActionTable.getActionValues(state);
A action = policy.chooseAction(actionValues);
@@ -46,6 +56,44 @@ public class QLearningOffPolicyTDControl<A extends Enum> extends EpisodicLearnin
double reward = envResult.getReward();
State nextState = envResult.getState();
sumOfRewards += reward;
timestampTilFood++;
if(reward == Reward.FOOD_DROP_DOWN_SUCCESS) {
foodCollected++;
foodTimestampsTotal += timestampTilFood;
File file = new File(ContinuousAnt.FILE_NAME);
if(foodCollected % 1000 == 0) {
System.out.println(foodTimestampsTotal / 1000f + " " + timestampCurrentEpisode);
try {
Files.writeString(Path.of(file.getPath()), foodTimestampsTotal / 1000f + ",", StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
foodTimestampsTotal = 0;
}
if(foodCollected == 1000){
((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.15f);
}
if(foodCollected == 2000){
((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.10f);
}
if(foodCollected == 3000){
((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.05f);
}
if(foodCollected == 4000){
System.out.println("Reached 0 exploration");
((EpsilonGreedyPolicy<A>) this.policy).setEpsilon(0.00f);
}
if(foodCollected == 15000){
try {
Files.writeString(Path.of(file.getPath()), "\n", StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
return;
}
timestampTilFood = 0;
}
// Q Update
double currentQValue = stateActionTable.getActionValues(state).get(action);
src/main/java/core/algo/td/SARSA.java
+8 -2
View File
@@ -3,6 +3,8 @@ package core.algo.td;
import core.*;
import core.algo.EpisodicLearning;
import core.policy.EpsilonGreedyPolicy;
import core.policy.GreedyPolicy;
import core.policy.Policy;
import java.util.Map;
@@ -32,10 +34,13 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
StepResultEnvironment envResult = null;
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A action = policy.chooseAction(actionValues);
A action = policy.chooseAction(actionValues);
//A action = policy.chooseAction(actionValues);
sumOfRewards = 0;
while(envResult == null || !envResult.isDone()) {
// Take a step
envResult = environment.step(action);
sumOfRewards += envResult.getReward();
@@ -44,7 +49,8 @@ public class SARSA<A extends Enum> extends EpisodicLearning<A> {
// Pick next action
actionValues = stateActionTable.getActionValues(nextState);
A nextAction = policy.chooseAction(actionValues);
A nextAction = policy.chooseAction(actionValues);
// td update
// target = reward + gamma * Q(nextState, nextAction)
src/main/java/core/controller/RLController.java
+17 -13
View File
@@ -7,7 +7,7 @@ import core.ListDiscreteActionSpace;
import core.algo.EpisodicLearning;
import core.algo.Learning;
import core.algo.Method;
import core.algo.mc.MonteCarloControlFirstVisitEGreedy;
import core.algo.mc.MonteCarloControlEGreedy;
import core.algo.td.QLearningOffPolicyTDControl;
import core.algo.td.SARSA;
import core.listener.LearningListener;
@@ -49,9 +49,13 @@ public class RLController<A extends Enum> implements LearningListener {
public void start() {
switch(method) {
case MC_CONTROL_FIRST_VISIT:
learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
learning = new MonteCarloControlEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
break;
case SARSA_EPISODIC:
case MC_CONTROL_EVERY_VISIT:
learning = new MonteCarloControlEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay, true);
break;
case SARSA_ON_POLICY_CONTROL:
learning = new SARSA<>(environment, discreteActionSpace, discountFactor, epsilon, learningRate, delay);
break;
case Q_LEARNING_OFF_POLICY_CONTROL:
@@ -67,17 +71,17 @@ public class RLController<A extends Enum> implements LearningListener {
}
protected void initListeners() {
learning.addListener(this);
new Thread(() -> {
while(true) {
printNextEpisode = true;
try {
Thread.sleep(30 * 1000);
} catch (InterruptedException e) {
e.printStackTrace();
learning.addListener(this);
new Thread(() -> {
while(learning.isCurrentlyLearning()) {
printNextEpisode = true;
try {
Thread.sleep(30 * 1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}).start();
}).start();
}
private void initLearning() {
src/main/java/core/policy/EpsilonPolicy.java
+2 -2
View File
@@ -1,8 +1,8 @@
package core.policy;
/**
* Chooses the action with the highest values with possibility: 1-Ɛ + Ɛ/|A|
* With possibility of Ɛ, a random action is taken (highest values option included).
* Chooses the action with the highest values with possibility: 1-Epsilon + Epsilon/|A|
* With possibility of Epsilon, a random action is taken (highest values option included).
*
* @param <A> Enum class of available action in specific environment
*/