kono/refo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

split Antworld into episodic and continuous task

Browse Source 
- add new simple state for jumping dino, to see if convergence is guarenteed with with state representation
- changed reward structure for ant game
This commit is contained in:
kono
2020-03-15 16:58:53 +01:00
parent 4641f50b79
commit ee1d62842d
21 changed files with 372 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/core/RNG.java
+5 -4
View File
@@ -1,9 +1,10 @@
package core;
import java.security.SecureRandom;
import java.util.Random;
/**
* ! SecureRandom not working properly on windows/different JDKs,
* using Random again !
*
* To ensure deterministic behaviour of repeating program executions,
* this class is used for all random number generation methods.
* Do not use Math.random()!
@@ -13,10 +14,10 @@ import java.util.Random;
* execution)
*/
public class RNG {
private static SecureRandom rng;
private static Random rng;
private static int seed = 123;
static {
rng = new SecureRandom();
rng = new Random();
rng.setSeed(seed);
}
src/main/java/evironment/antGame/AntWorld.java
+82 -71
View File
@@ -9,16 +9,19 @@ import evironment.antGame.gui.AntWorldComponent;
import javax.swing.*;
import java.awt.*;
/**
* Episodic AntWorld
*/
public class AntWorld implements Environment<AntAction>, Visualizable {
/**
*
*/
private Grid grid;
protected Grid grid;
/**
* Intern (backend) representation of the ant.
* The AntWorld essentially acts like the game host of the original AntGame.
*/
private Ant myAnt;
protected Ant myAnt;
/**
* The client agent. In the original AntGame the host would send jade messages
* of the current observation to each client on every tick.
@@ -32,13 +35,13 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
* through an intern grid clone (brain), for example. A history as mentioned in
* various lectures could be possible as well.
*/
private AntAgent antAgent;
protected AntAgent antAgent;
private int tick;
protected int tick;
private int maxEpisodeTicks;
public AntWorld(int width, int height, double foodDensity){
grid = new Grid(width, height, foodDensity);
public AntWorld(int width, int height) {
grid = new Grid(width, height);
antAgent = new AntAgent(width, height);
myAnt = new Ant();
maxEpisodeTicks = 1000;
@@ -46,73 +49,68 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
}
public AntWorld(){
this(Constants.DEFAULT_GRID_WIDTH, Constants.DEFAULT_GRID_HEIGHT, Constants.DEFAULT_FOOD_DENSITY);
this(Constants.DEFAULT_GRID_WIDTH, Constants.DEFAULT_GRID_HEIGHT);
}
@Override
public StepResultEnvironment step(AntAction action){
AntObservation observation;
State newState;
double reward = 0;
String info = "";
boolean done = false;
protected StepCalculation processStep(AntAction action) {
StepCalculation sc = new StepCalculation();
sc.reward = -1;
sc.info = "";
sc.done = false;
Cell currentCell = grid.getCell(myAnt.getPos());
Point potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
boolean stayOnCell = true;
sc.potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
sc.stayOnCell = true;
// flag to enable a check if all food has been collected only fired if food was dropped
// on the starting position
boolean checkCompletion = false;
sc.checkCompletion = false;
switch (action) {
switch(action) {
case MOVE_UP:
potentialNextPos.y -= 1;
stayOnCell = false;
sc.potentialNextPos.y -= 1;
sc.stayOnCell = false;
break;
case MOVE_RIGHT:
potentialNextPos.x += 1;
stayOnCell = false;
sc.potentialNextPos.x += 1;
sc.stayOnCell = false;
break;
case MOVE_DOWN:
potentialNextPos.y += 1;
stayOnCell = false;
sc.potentialNextPos.y += 1;
sc.stayOnCell = false;
break;
case MOVE_LEFT:
potentialNextPos.x -= 1;
stayOnCell = false;
sc.potentialNextPos.x -= 1;
sc.stayOnCell = false;
break;
case PICK_UP:
if(myAnt.hasFood()){
if(myAnt.hasFood()) {
// Ant tries to pick up food but can only hold one piece
reward = Reward.FOOD_PICK_UP_FAIL_HAS_FOOD_ALREADY;
}else if(currentCell.getFood() == 0){
sc.reward += Reward.FOOD_PICK_UP_FAIL_HAS_FOOD_ALREADY;
} else if(currentCell.getFood() == 0) {
// Ant tries to pick up food on cell that has no food on it
reward = Reward.FOOD_PICK_UP_FAIL_NO_FOOD;
}else if(currentCell.getFood() > 0){
sc.reward += Reward.FOOD_PICK_UP_FAIL_NO_FOOD;
} else if(currentCell.getFood() > 0) {
// Ant successfully picks up food
currentCell.setFood(currentCell.getFood() - 1);
myAnt.setHasFood(true);
reward = Reward.FOOD_PICK_UP_SUCCESS;
sc.reward = Reward.FOOD_PICK_UP_SUCCESS;
}
break;
case DROP_DOWN:
if(!myAnt.hasFood()){
if(!myAnt.hasFood()) {
// Ant had no food to drop
reward = Reward.FOOD_DROP_DOWN_FAIL_NO_FOOD;
}else{
// Drop food onto the ground
currentCell.setFood(currentCell.getFood() + 1);
sc.reward += Reward.FOOD_DROP_DOWN_FAIL_NO_FOOD;
} else {
myAnt.setHasFood(false);
// negative reward if the agent drops food on any other field
// than the starting point
if(currentCell.getType() != CellType.START){
reward = Reward.FOOD_DROP_DOWN_FAIL_NOT_START;
done = true;
}else{
reward = Reward.FOOD_DROP_DOWN_SUCCESS;
if(currentCell.getType() != CellType.START) {
sc.reward += Reward.FOOD_DROP_DOWN_FAIL_NOT_START;
// Drop food onto the ground
currentCell.setFood(currentCell.getFood() + 1);
} else {
sc.reward = Reward.FOOD_DROP_DOWN_SUCCESS;
myAnt.setPoints(myAnt.getPoints() + 1);
checkCompletion = true;
sc.checkCompletion = true;
}
}
break;
@@ -121,24 +119,33 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
}
// movement action was selected
if(!stayOnCell){
if(!isInGrid(potentialNextPos)){
stayOnCell = true;
reward = Reward.RAN_INTO_WALL;
}else if(hitObstacle(potentialNextPos)){
stayOnCell = true;
reward = Reward.RAN_INTO_OBSTACLE;
if(!sc.stayOnCell) {
if(!isInGrid(sc.potentialNextPos)) {
sc.stayOnCell = true;
sc.reward += Reward.RAN_INTO_WALL;
} else if(hitObstacle(sc.potentialNextPos)) {
sc.stayOnCell = true;
sc.reward += Reward.RAN_INTO_OBSTACLE;
}
}
return sc;
}
@Override
public StepResultEnvironment step(AntAction action){
AntObservation observation;
State newState;
StepCalculation sc = processStep(action);
// valid movement
if(!stayOnCell){
myAnt.getPos().setLocation(potentialNextPos);
if(!sc.stayOnCell) {
myAnt.getPos().setLocation(sc.potentialNextPos);
if(antAgent.getCell(myAnt.getPos()).getType() == CellType.UNKNOWN){
// the ant will move to a cell that was previously unknown
reward = Reward.UNKNOWN_FIELD_EXPLORED;
}else{
reward = 0;
// TODO: not optimal for going straight for food
// sc.reward = Reward.UNKNOWN_FIELD_EXPLORED;
}
}
@@ -148,33 +155,36 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
// let the ant agent process the observation to create a valid markov state
newState = antAgent.feedObservation(observation);
if(checkCompletion){
done = grid.isAllFoodCollected();
if(sc.checkCompletion) {
sc.done = grid.isAllFoodCollected();
}
/*
if(!done){
reward = -1;
}
*/
if(++tick == maxEpisodeTicks){
done = true;
sc.done = true;
}
StepResultEnvironment result = new StepResultEnvironment(newState, reward, done, info);
return result;
return new StepResultEnvironment(newState, sc.reward, sc.done, sc.info);
}
private boolean isInGrid(Point pos){
protected boolean isInGrid(Point pos) {
return pos.x >= 0 && pos.x < grid.getWidth() && pos.y >= 0 && pos.y < grid.getHeight();
}
private boolean hitObstacle(Point pos){
protected boolean hitObstacle(Point pos) {
return grid.getCell(pos).getType() == CellType.OBSTACLE;
}
protected class StepCalculation {
double reward;
String info;
boolean done;
Point potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
boolean stayOnCell = true;
// flag to enable a check if all food has been collected only fired if food was dropped
// on the starting position
boolean checkCompletion = false;
}
public State reset() {
grid.resetWorld();
antAgent.initUnknownWorld();
@@ -189,6 +199,7 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
public void setMaxEpisodeLength(int maxTicks){
this.maxEpisodeTicks = maxTicks;
}
public Point getSpawningPoint(){
return grid.getStartPoint();
}
src/main/java/evironment/antGame/AntWorldContinuous.java
+36
View File
@@ -0,0 +1,36 @@
package evironment.antGame;
import core.StepResultEnvironment;
public class AntWorldContinuous extends AntWorld {
public AntWorldContinuous(int width, int height) {
super(width, height);
}
public AntWorldContinuous() {
super();
}
@Override
public StepResultEnvironment step(AntAction action) {
AntObservation observation;
Cell currentCell = grid.getCell(myAnt.getPos());
StepCalculation sc = processStep(action);
// flag is set to true if food gets dropped onto starts
if(sc.checkCompletion) {
grid.spawnNewFood();
}
// valid movement
if(!sc.stayOnCell) {
myAnt.getPos().setLocation(sc.potentialNextPos);
}
// get observation after action was computed
observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
return new StepResultEnvironment(new AntState(grid.getGrid(), observation.getPos(), observation.hasFood()), sc.reward, false, sc.info);
}
}
src/main/java/evironment/antGame/Cell.java
+10
View File
@@ -7,6 +7,7 @@ import java.awt.*;
public class Cell {
@Getter
@Setter
private CellType type;
@Getter
@Setter
@@ -38,4 +39,13 @@ public class Cell {
}
return super.equals(obj);
}
@Override
public String toString() {
return "Cell{" +
"type=" + type +
", food=" + food +
", pos=" + pos +
'}';
}
}
src/main/java/evironment/antGame/Constants.java
+2 -3
View File
@@ -1,7 +1,6 @@
package evironment.antGame;
public class Constants {
public static final int DEFAULT_GRID_WIDTH = 10;
public static final int DEFAULT_GRID_HEIGHT = 10;
public static final double DEFAULT_FOOD_DENSITY = 0.1;
public static final int DEFAULT_GRID_WIDTH = 5;
public static final int DEFAULT_GRID_HEIGHT = 5;
}
src/main/java/evironment/antGame/Grid.java
+43 -12
View File
@@ -7,24 +7,18 @@ import java.awt.*;
public class Grid {
private int width;
private int height;
private double foodDensity;
private Point start;
private Cell[][] grid;
private Cell[][] initialGrid;
public Grid(int width, int height, double foodDensity){
public Grid(int width, int height) {
this.width = width;
this.height = height;
this.foodDensity = foodDensity;
grid = new Cell[width][height];
initialGrid = new Cell[width][height];
initRandomWorld();
}
public Grid(int width, int height){
this(width, height, 0);
}
public void resetWorld(){
grid = Util.deepCopyCellGrid(initialGrid);
}
@@ -32,15 +26,52 @@ public class Grid {
public void initRandomWorld(){
for(int x = 0; x < width; ++x){
for(int y = 0; y < height; ++y){
if( RNG.getRandom().nextDouble() < foodDensity){
initialGrid[x][y] = new Cell(new Point(x,y), CellType.FREE, 1);
}else{
initialGrid[x][y] = new Cell(new Point(x,y), CellType.FREE);
}
initialGrid[x][y] = new Cell(new Point(x, y), CellType.FREE);
}
}
start = new Point(RNG.getRandom().nextInt(width), RNG.getRandom().nextInt(height));
initialGrid[start.x][start.y] = new Cell(new Point(start.x, start.y), CellType.START);
spawnNewFood(initialGrid);
spawnObstacles();
}
//TODO
private void spawnObstacles() {
initialGrid[3][1].setType(CellType.OBSTACLE);
initialGrid[4][1].setType(CellType.OBSTACLE);
initialGrid[5][1].setType(CellType.OBSTACLE);
initialGrid[6][1].setType(CellType.OBSTACLE);
initialGrid[7][1].setType(CellType.OBSTACLE);
initialGrid[3][2].setType(CellType.OBSTACLE);
initialGrid[3][3].setType(CellType.OBSTACLE);
initialGrid[3][4].setType(CellType.OBSTACLE);
initialGrid[4][4].setType(CellType.OBSTACLE);
initialGrid[5][4].setType(CellType.OBSTACLE);
initialGrid[6][4].setType(CellType.OBSTACLE);
}
/**
* Spawns one additional food on a random field EXCEPT for the starting position
*/
public void spawnNewFood(Cell[][] grid) {
boolean foodSpawned = false;
Point potFood = new Point(0, 0);
CellType potFieldType;
while(!foodSpawned) {
potFood.x = RNG.getRandom().nextInt(width);
potFood.y = RNG.getRandom().nextInt(height);
potFieldType = grid[potFood.x][potFood.y].getType();
if(potFieldType != CellType.START && grid[potFood.x][potFood.y].getFood() == 0 && potFieldType != CellType.OBSTACLE) {
grid[potFood.x][potFood.y].setFood(1);
foodSpawned = true;
System.out.println("spawned new food at " + potFood);
System.out.println(initialGrid[potFood.x][potFood.y]);
}
}
}
public void spawnNewFood() {
spawnNewFood(grid);
}
public Point getStartPoint(){
src/main/java/evironment/antGame/Reward.java
+2 -2
View File
@@ -7,9 +7,9 @@ public class Reward {
public static final double FOOD_DROP_DOWN_FAIL_NO_FOOD = -1;
public static final double FOOD_DROP_DOWN_FAIL_NOT_START = -1;
public static final double FOOD_DROP_DOWN_SUCCESS = 1;
public static final double FOOD_DROP_DOWN_SUCCESS = 40;
public static final double UNKNOWN_FIELD_EXPLORED = 1;
public static final double UNKNOWN_FIELD_EXPLORED = 0;
public static final double RAN_INTO_WALL = -1;
public static final double RAN_INTO_OBSTACLE = -1;
src/main/java/evironment/jumpingDino/DinoStateSimple.java
+65
View File
@@ -0,0 +1,65 @@
package evironment.jumpingDino;
import core.State;
import core.gui.Visualizable;
import lombok.AllArgsConstructor;
import lombok.Getter;
import javax.swing.*;
import java.awt.*;
import java.io.Serializable;
import java.util.Objects;
@AllArgsConstructor
@Getter
public class DinoStateSimple implements State, Serializable, Visualizable {
protected final double scale = 0.5;
private int xDistanceToObstacle;
@Override
public String toString() {
return "DinoState{" +
"xDistanceToObstacle=" + xDistanceToObstacle +
'}';
}
@Override
public boolean equals(Object o) {
if(this == o) return true;
if(o == null || getClass() != o.getClass()) return false;
DinoStateSimple dinoState = (DinoStateSimple) o;
return xDistanceToObstacle == dinoState.xDistanceToObstacle;
}
@Override
public int hashCode() {
return Objects.hash(xDistanceToObstacle);
}
@Override
public JComponent visualize() {
return new JComponent() {
{
setPreferredSize(new Dimension(Config.FRAME_WIDTH, (int) (scale * Config.FRAME_HEIGHT)));
setVisible(true);
}
@Override
protected void paintComponent(Graphics g) {
super.paintComponents(g);
drawObjects(g);
}
};
}
public void drawObjects(Graphics g) {
g.setColor(Color.BLACK);
g.fillRect(0, (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y)), Config.FRAME_WIDTH, 2);
g.fillRect((int) (scale * Config.DINO_STARTING_X), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.DINO_SIZE)), (int) (scale * Config.DINO_SIZE), (int) (scale * Config.DINO_SIZE));
g.drawString("Distance: " + xDistanceToObstacle, (int) (scale * Config.DINO_STARTING_X), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE - 40)));
g.fillRect((int) (scale * (Config.DINO_STARTING_X + getXDistanceToObstacle())), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE)), (int) (scale * Config.OBSTACLE_SIZE), (int) (scale * Config.OBSTACLE_SIZE));
}
}
src/main/java/evironment/jumpingDino/DinoWorld.java
+1 -1
View File
@@ -76,7 +76,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
}
protected State generateReturnState(){
return new DinoState(getDistanceToObstacle(), dino.isInJump());
return new DinoStateSimple(getDistanceToObstacle());
}
protected void spawnNewObstacle(){
src/main/java/evironment/jumpingDino/DinoWorldAdvanced.java
+19 -6
View File
@@ -13,6 +13,9 @@ import java.awt.*;
*
* 350 states
* if 4 speed variants
*
* 2044
* 4 speeds, 4 distance
*/
public class DinoWorldAdvanced extends DinoWorld{
public DinoWorldAdvanced(){
@@ -21,7 +24,7 @@ public class DinoWorldAdvanced extends DinoWorld{
@Override
protected State generateReturnState() {
return new DinoStateWithSpeed(getDistanceToObstacle(), dino.isInJump(), getCurrentObstacle().getDx());
return new DinoStateWithSpeed(getDistanceToObstacle(), dino.isInJump(), currentObstacle.getDx());
}
@Override
@@ -30,16 +33,26 @@ public class DinoWorldAdvanced extends DinoWorld{
int xSpawn;
double ran = RNG.getRandom().nextDouble();
if(ran < 0.25){
dx = -(int)(0.7 * Config.OBSTACLE_SPEED);
dx = -(int) (0.35 * Config.OBSTACLE_SPEED);
}else if(ran < 0.5){
dx = -(int)(1.3 * Config.OBSTACLE_SPEED);
dx = -(int) (0.7 * Config.OBSTACLE_SPEED);
}else if(ran < 0.75){
dx = -(int)(1.6 * Config.OBSTACLE_SPEED);
} else{
dx = -2 * Config.OBSTACLE_SPEED;
dx = -(int) (3.5 * Config.OBSTACLE_SPEED);
}
double ran2 = RNG.getRandom().nextDouble();
if(ran2 < 0.25) {
// randomly spawning more right outside of the screen
xSpawn = Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE;
} else if(ran2 < 0.5) {
xSpawn = (int) (1.08 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
} else if(ran2 < 0.75) {
xSpawn = (int) (1.11 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
} else {
xSpawn = (int) (1.23 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
}
// randomly spawning more right outside of the screen
xSpawn = Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE;
currentObstacle = new Obstacle(Config.OBSTACLE_SIZE, xSpawn, Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE, dx, 0, Color.BLACK);
}
}
src/main/java/example/ContinuousAnt.java
+25
View File
@@ -0,0 +1,25 @@
package example;
import core.RNG;
import core.algo.Method;
import core.controller.RLController;
import core.controller.RLControllerGUI;
import evironment.antGame.AntAction;
import evironment.antGame.AntWorldContinuous;
public class ContinuousAnt {
public static void main(String[] args) {
RNG.setSeed(56);
RLController<AntAction> rl = new RLControllerGUI<>(
new AntWorldContinuous(8, 8),
Method.Q_LEARNING_OFF_POLICY_CONTROL,
AntAction.values());
rl.setDelay(200);
rl.setNrOfEpisodes(10000);
rl.setDiscountFactor(0.95f);
rl.setEpsilon(0.15f);
rl.start();
}
}
src/main/java/example/DinoSampling.java
+1 -2
View File
@@ -4,7 +4,6 @@ import core.RNG;
import core.algo.Method;
import core.controller.RLController;
import evironment.jumpingDino.DinoAction;
import evironment.jumpingDino.DinoWorld;
import evironment.jumpingDino.DinoWorldAdvanced;
import java.io.File;
@@ -34,7 +33,7 @@ public class DinoSampling {
rl.setDiscountFactor(1f);
rl.setEpsilon(f);
rl.setLearningRate(1f);
rl.setNrOfEpisodes(100000);
rl.setNrOfEpisodes(50000);
rl.start();
}
src/main/java/example/JumpingDino.java
+5 -7
View File
@@ -3,25 +3,23 @@ package example;
import core.RNG;
import core.algo.Method;
import core.controller.RLController;
import core.controller.RLControllerGUI;
import evironment.jumpingDino.DinoAction;
import evironment.jumpingDino.DinoWorld;
import evironment.jumpingDino.DinoWorldAdvanced;
public class JumpingDino {
public static void main(String[] args) {
RNG.setSeed(55);
RNG.setSeed(29);
RLController<DinoAction> rl = new RLControllerGUI<>(
new DinoWorldAdvanced(),
RLController<DinoAction> rl = new RLController<>(
new DinoWorld(),
Method.MC_CONTROL_FIRST_VISIT,
DinoAction.values());
rl.setDelay(100);
rl.setDelay(0);
rl.setDiscountFactor(1f);
rl.setEpsilon(0.15f);
rl.setLearningRate(1f);
rl.setNrOfEpisodes(1000000);
rl.setNrOfEpisodes(30000000);
rl.start();
}
}
src/main/java/example/RunningAnt.java
+3 -3
View File
@@ -12,13 +12,13 @@ public class RunningAnt {
RNG.setSeed(56);
RLController<AntAction> rl = new RLControllerGUI<>(
new AntWorld(3, 3, 0.1),
Method.MC_CONTROL_FIRST_VISIT,
new AntWorld(8, 8),
Method.Q_LEARNING_OFF_POLICY_CONTROL,
AntAction.values());
rl.setDelay(200);
rl.setNrOfEpisodes(10000);
rl.setDiscountFactor(1f);
rl.setDiscountFactor(0.9f);
rl.setEpsilon(0.15f);
rl.start();