kono/refo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add features to gui to control learning and moving learning listener interface to controller

Browse Source 
- Add metric to display episodes per second
- view not implementing learning listener anymore, controller does. Controller is controlling all view actions based upon learning events. Reacts to view events via viewListener
- add executor service for learning task
- using instance of to distinguish between episodic learning and td learning
- add feature to trigger more episodes
- add checkboxes for smoothing graph, displaying last 100 rewards only and drawing environment
- remove history panel from antworld gui
This commit is contained in:
kono
2019-12-22 17:06:54 +01:00
parent 34e7e3fdd6
commit b1246f62cc
14 changed files with 337 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/core/algo/Episodic.java
+2
View File
@@ -2,4 +2,6 @@ package core.algo;
public interface Episodic {
int getCurrentEpisode();
int getEpisodesToGo();
int getEpisodesPerSecond();
}
src/main/java/core/algo/EpisodicLearning.java
+53
View File
@@ -2,9 +2,14 @@ package core.algo;
import core.DiscreteActionSpace;
import core.Environment;
import core.listener.LearningListener;
public abstract class EpisodicLearning<A extends Enum> extends Learning<A> implements Episodic{
protected int currentEpisode;
protected int episodesToLearn;
protected volatile int episodePerSecond;
protected int episodeSumCurrentSecond;
private volatile boolean meseaureEpisodeBenchMark;
public EpisodicLearning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay) {
super(environment, actionSpace, discountFactor, delay);
@@ -22,8 +27,56 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
super(environment, actionSpace);
}
protected void dispatchEpisodeEnd(double recentSumOfRewards){
++episodeSumCurrentSecond;
rewardHistory.add(recentSumOfRewards);
for(LearningListener l: learningListeners) {
l.onEpisodeEnd(rewardHistory);
}
}
protected void dispatchEpisodeStart(){
for(LearningListener l: learningListeners){
l.onEpisodeStart();
}
}
@Override
public void learn(){
learn(0);
}
public void learn(int nrOfEpisodes){
meseaureEpisodeBenchMark = true;
new Thread(()->{
while(meseaureEpisodeBenchMark){
episodePerSecond = episodeSumCurrentSecond;
episodeSumCurrentSecond = 0;
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}).start();
episodesToLearn += nrOfEpisodes;
dispatchLearningStart();
for(int i=0; i < nrOfEpisodes; ++i){
nextEpisode();
}
dispatchLearningEnd();
meseaureEpisodeBenchMark = false;
}
protected abstract void nextEpisode();
@Override
public int getCurrentEpisode(){
return currentEpisode;
}
@Override
public int getEpisodesToGo(){
return episodesToLearn - currentEpisode;
}
}
src/main/java/core/algo/Learning.java
+14 -16
View File
@@ -24,7 +24,7 @@ public abstract class Learning<A extends Enum> {
protected Set<LearningListener> learningListeners;
@Setter
protected int delay;
private List<Double> rewardHistory;
protected List<Double> rewardHistory;
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay){
this.environment = environment;
@@ -47,29 +47,27 @@ public abstract class Learning<A extends Enum> {
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_DELAY);
}
public abstract void learn(int nrOfEpisodes);
public abstract void learn();
public void addListener(LearningListener learningListener){
learningListeners.add(learningListener);
}
protected void dispatchEpisodeEnd(double recentSumOfRewards){
rewardHistory.add(recentSumOfRewards);
for(LearningListener l: learningListeners) {
l.onEpisodeEnd(rewardHistory);
}
}
protected void dispatchEpisodeStart(){
for(LearningListener l: learningListeners){
l.onEpisodeStart();
}
}
protected void dispatchStepEnd(){
for(LearningListener l: learningListeners){
l.onStepEnd();
}
}
protected void dispatchLearningStart(){
for(LearningListener l: learningListeners){
l.onLearningStart();
}
}
protected void dispatchLearningEnd(){
for(LearningListener l: learningListeners){
l.onLearningEnd();
}
}
}
src/main/java/core/algo/MC/MonteCarloOnPolicyEGreedy.java
+59 -55
View File
@@ -11,27 +11,33 @@ import java.util.*;
* TODO: Major problem:
* StateActionPairs are only unique accounting for their position in the episode.
* For example:
*
* <p>
* startingState -> MOVE_LEFT : very first state action in the episode i = 1
* image the agent does not collect the food and drops it to the start, the agent will receive
* -1 for every timestamp hence (startingState -> MOVE_LEFT) will get a value of -10;
*
* <p>
* BUT image moving left from the starting position will have no impact on the state because
* the agent ran into a wall. The known world stays the same.
* Taking an action after that will have the exact same state but a different action
* making the value of this stateActionPair -9 because the stateAction pair took place on the second
* timestamp, summing up all remaining rewards will be -9...
*
* <p>
* How to encounter this problem?
*
* @param <A>
*/
public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<A> {
private Map<Pair<State, A>, Double> returnSum;
private Map<Pair<State, A>, Integer> returnCount;
public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
super(environment, actionSpace, discountFactor, delay);
currentEpisode = 0;
this.policy = new EpsilonGreedyPolicy<>(epsilon);
this.stateActionTable = new StateActionHashTable<>(this.actionSpace);
returnSum = new HashMap<>();
returnCount = new HashMap<>();
}
public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
@@ -40,71 +46,64 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
@Override
public void learn(int nrOfEpisodes) {
public void nextEpisode() {
++currentEpisode;
List<StepResult<A>> episode = new ArrayList<>();
State state = environment.reset();
dispatchEpisodeStart();
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
double sumOfRewards = 0;
for (int j = 0; j < 10; ++j) {
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A chosenAction = policy.chooseAction(actionValues);
StepResultEnvironment envResult = environment.step(chosenAction);
State nextState = envResult.getState();
sumOfRewards += envResult.getReward();
episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
Map<Pair<State, A>, Double> returnSum = new HashMap<>();
Map<Pair<State, A>, Integer> returnCount = new HashMap<>();
if (envResult.isDone()) break;
state = nextState;
for(int i = 0; i < nrOfEpisodes; ++i) {
++currentEpisode;
List<StepResult<A>> episode = new ArrayList<>();
State state = environment.reset();
dispatchEpisodeStart();
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
double sumOfRewards = 0;
for(int j=0; j < 10; ++j){
Map<A, Double> actionValues = stateActionTable.getActionValues(state);
A chosenAction = policy.chooseAction(actionValues);
StepResultEnvironment envResult = environment.step(chosenAction);
State nextState = envResult.getState();
sumOfRewards += envResult.getReward();
episode.add(new StepResult<>(state, chosenAction, envResult.getReward()));
dispatchStepEnd();
}
if(envResult.isDone()) break;
dispatchEpisodeEnd(sumOfRewards);
System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
Set<Pair<State, A>> stateActionPairs = new HashSet<>();
state = nextState;
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
for (StepResult<A> sr : episode) {
stateActionPairs.add(new Pair<>(sr.getState(), sr.getAction()));
}
System.out.println("stateActionPairs " + stateActionPairs.size());
for (Pair<State, A> stateActionPair : stateActionPairs) {
int firstOccurenceIndex = 0;
// find first occurance of state action pair
for (StepResult<A> sr : episode) {
if (stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())) {
break;
}
dispatchStepEnd();
firstOccurenceIndex++;
}
dispatchEpisodeEnd(sumOfRewards);
System.out.printf("Episode %d \t Reward: %f \n", i, sumOfRewards);
Set<Pair<State, A>> stateActionPairs = new HashSet<>();
for(StepResult<A> sr: episode){
stateActionPairs.add(new Pair<>(sr.getState(), sr.getAction()));
}
System.out.println("stateActionPairs " + stateActionPairs.size());
for(Pair<State, A> stateActionPair: stateActionPairs){
int firstOccurenceIndex = 0;
// find first occurance of state action pair
for(StepResult<A> sr: episode){
if(stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())){
;
break;
}
firstOccurenceIndex++;
}
double G = 0;
for(int l = firstOccurenceIndex; l < episode.size(); ++l){
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
}
// slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
double G = 0;
for (int l = firstOccurenceIndex; l < episode.size(); ++l) {
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
}
// slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
}
}
@@ -112,4 +111,9 @@ public class MonteCarloOnPolicyEGreedy<A extends Enum> extends EpisodicLearning<
public int getCurrentEpisode() {
return currentEpisode;
}
@Override
public int getEpisodesPerSecond(){
return episodePerSecond;
}
}