add fix RNG, add extended interface EpsilonPolicy and move rewardHistory to model instead of view
- only setting the seed of RNG once at the beginning and not reseeding it afterwards. Deep copying the initial AntWorld to use as blueprint for resetting the world instead of reseeding and creating pesudo random again. Reseeding the RNG has influence action selecting to always choose the same trajectory. - instance of is used to determine if policy has epsilon or not and the view will adopt to this, only showing epsilon slider if policy has epsilon
This commit is contained in:
@@ -10,8 +10,11 @@ import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import javax.swing.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
@Getter
|
||||
public abstract class Learning<A extends Enum> {
|
||||
@@ -20,38 +23,43 @@ public abstract class Learning<A extends Enum> {
|
||||
protected StateActionTable<A> stateActionTable;
|
||||
protected Environment<A> environment;
|
||||
protected float discountFactor;
|
||||
@Setter
|
||||
protected float epsilon;
|
||||
protected Set<LearningListener> learningListeners;
|
||||
@Setter
|
||||
protected int delay;
|
||||
private List<Double> rewardHistory;
|
||||
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay){
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, int delay){
|
||||
this.environment = environment;
|
||||
this.actionSpace = actionSpace;
|
||||
this.discountFactor = discountFactor;
|
||||
this.epsilon = epsilon;
|
||||
this.delay = delay;
|
||||
learningListeners = new HashSet<>();
|
||||
rewardHistory = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon){
|
||||
this(environment, actionSpace, discountFactor, epsilon, LearningConfig.DEFAULT_DELAY);
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor){
|
||||
this(environment, actionSpace, discountFactor, LearningConfig.DEFAULT_DELAY);
|
||||
}
|
||||
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay){
|
||||
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, delay);
|
||||
}
|
||||
|
||||
public Learning(Environment<A> environment, DiscreteActionSpace<A> actionSpace){
|
||||
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, LearningConfig.DEFAULT_DELAY);
|
||||
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_DELAY);
|
||||
}
|
||||
|
||||
|
||||
public abstract void learn(int nrOfEpisodes);
|
||||
|
||||
public void addListener(LearningListener learningListener){
|
||||
learningListeners.add(learningListener);
|
||||
}
|
||||
|
||||
protected void dispatchEpisodeEnd(double sum){
|
||||
protected void dispatchEpisodeEnd(double recentSumOfRewards){
|
||||
rewardHistory.add(recentSumOfRewards);
|
||||
for(LearningListener l: learningListeners) {
|
||||
l.onEpisodeEnd(sum);
|
||||
l.onEpisodeEnd(rewardHistory);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import core.*;
|
||||
import core.algo.Learning;
|
||||
import core.policy.EpsilonGreedyPolicy;
|
||||
import javafx.util.Pair;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
@@ -26,13 +28,18 @@ import java.util.*;
|
||||
*/
|
||||
public class MonteCarloOnPolicyEGreedy<A extends Enum> extends Learning<A> {
|
||||
|
||||
public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace) {
|
||||
super(environment, actionSpace);
|
||||
discountFactor = 1f;
|
||||
this.policy = new EpsilonGreedyPolicy<>(0.1f);
|
||||
this.stateActionTable = new StateActionHashTable<>(actionSpace);
|
||||
public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
|
||||
super(environment, actionSpace, discountFactor, delay);
|
||||
|
||||
this.policy = new EpsilonGreedyPolicy<>(epsilon);
|
||||
this.stateActionTable = new StateActionHashTable<>(this.actionSpace);
|
||||
}
|
||||
|
||||
public MonteCarloOnPolicyEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
|
||||
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void learn(int nrOfEpisodes) {
|
||||
|
||||
|
||||
Reference in New Issue
Block a user