split Antworld into episodic and continuous task
- add new simple state for jumping dino, to see if convergence is guarenteed with with state representation - changed reward structure for ant game
This commit is contained in:
parent
4641f50b79
commit
ee1d62842d
|
|
@ -11,7 +11,7 @@
|
|||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="false" project-jdk-name="11.0.5" project-jdk-type="JavaSDK">
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="false" project-jdk-name="11" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
0.05,447,1036,447,788,499,499,880,514,583,420,759,678,529,781,742,185,848,346,733,961,657,312,599,245,1076,1361,377,612,821,481,374,250,171,949,1079,788,465,537,431,312,565,1046,825,579,1178,547,1111,518,949,779,924,430,543,1262,642,814,508,422,1120,536,591,381,670,681,382,297,795,774,1552,691,909,748,520,639,704,575,752,418,503,292,1159,1020,570,839,696,895,511,635,830,522,628,518,408,392,605,888,790,629,547,999
|
||||
0.1,337,303,221,455,64,271,377,946,381,253,669,479,267,870,250,364,347,83,124,464,356,535,820,180,369,310,284,456,166,267,240,215,347,295,226,409,344,266,404,480,195,235,655,233,429,149,304,164,409,276,268,139,495,388,401,283,343,284,720,351,170,781,59,369,789,155,206,464,380,272,645,180,216,698,166,497,650,297,595,352,138,163,412,238,336,90,453,238,306,598,487,722,640,592,493,144,435,441,387,374
|
||||
0.15,288,400,363,158,414,302,126,175,323,303,391,202,295,329,468,198,199,179,384,193,168,443,74,153,347,458,395,209,545,199,421,587,144,200,219,204,134,200,253,193,396,108,530,728,286,118,256,351,279,308,210,713,474,415,397,328,220,201,136,515,374,154,847,578,313,250,224,392,362,551,127,211,321,176,372,572,304,290,225,541,296,315,392,235,161,359,307,360,613,194,389,278,334,153,402,224,246,260,264,226
|
||||
0.2,193,331,294,156,245,530,237,199,225,298,414,424,342,239,361,295,176,326,155,371,272,255,259,201,506,520,108,172,416,220,245,283,203,213,164,469,262,263,177,80,174,346,573,577,243,226,437,427,261,195,210,389,286,297,263,328,393,174,166,267,344,223,161,171,321,256,322,251,238,164,216,198,449,327,328,175,334,260,479,269,389,137,292,384,232,549,394,177,207,165,123,304,346,407,437,66,176,358,277,111
|
||||
0.25,155,195,340,162,250,300,204,218,278,291,184,282,95,370,265,275,168,229,207,463,273,157,233,106,338,414,397,305,283,374,347,243,78,172,284,296,187,439,149,513,689,99,93,260,382,204,141,310,170,409,436,112,283,435,284,108,210,264,335,177,121,181,124,294,129,394,183,117,211,137,254,384,263,228,423,443,349,142,115,276,268,251,275,191,387,283,124,212,191,167,490,329,293,194,294,241,432,50,105,341
|
||||
0.3,282,128,127,74,136,311,303,206,150,387,167,353,286,138,107,269,343,301,157,370,553,161,292,175,250,187,259,451,222,203,312,177,212,248,197,276,122,236,159,480,274,474,46,208,96,248,131,358,301,142,302,298,153,105,524,131,225,343,207,301,409,321,505,546,131,395,275,151,148,280,266,539,116,105,166,232,245,155,219,222,123,174,221,162,521,206,325,77,388,199,380,222,52,177,119,98,323,378,262,214
|
||||
0.35,404,130,174,176,297,284,274,139,279,225,105,115,238,191,278,162,185,316,233,535,175,131,86,245,135,262,285,427,214,224,139,170,348,371,385,151,231,258,202,260,293,109,159,323,317,68,231,170,313,168,222,216,357,225,177,125,195,361,242,133,258,158,174,101,132,272,131,213,224,124,232,701,395,321,173,144,226,260,220,211,352,190,188,422,260,151,167,241,459,237,298,225,123,274,107,242,253,134,132,288
|
||||
0.40,318,374,159,305,164,220,69,174,233,277,163,293,236,323,154,210,209,117,271,126,155,632,233,208,417,273,127,213,73,223,99,118,178,197,137,207,138,351,208,254,232,207,609,256,169,164,403,589,145,79,344,293,270,218,162,221,341,71,125,203,215,178,81,150,213,223,368,236,244,249,231,166,317,171,164,208,292,172,174,256,285,136,236,446,312,371,285,152,155,524,171,113,287,156,281,97,209,195,368,283
|
||||
0.45,198,106,298,295,298,124,168,159,194,144,110,270,253,225,126,191,184,119,255,484,528,147,181,190,345,166,158,211,304,242,92,199,219,129,86,170,167,150,190,168,514,317,154,164,186,261,276,234,72,241,130,233,269,136,217,73,219,193,353,42,127,305,361,313,86,109,254,400,327,452,165,350,192,139,499,192,147,313,225,159,122,121,228,217,79,162,183,354,360,432,258,313,325,317,265,33,175,233,203,178
|
||||
0.50,149,341,278,153,204,216,244,250,291,247,268,114,175,202,87,236,377,280,161,444,199,185,112,52,495,248,87,199,327,374,192,136,545,92,174,186,85,238,277,274,227,139,112,300,205,292,398,188,329,214,114,55,212,154,533,109,139,165,216,280,210,231,193,295,149,190,131,130,184,159,161,229,286,146,218,151,137,265,80,191,326,282,194,164,114,203,221,495,298,187,116,110,119,140,237,203,124,307,260,164
|
||||
0.55,190,161,170,533,205,104,85,228,610,294,293,380,297,333,498,303,198,248,430,180,157,131,402,706,346,282,151,102,346,227,195,98,306,391,533,312,297,149,480,191,93,110,233,104,434,244,296,124,131,275,157,165,93,94,241,96,82,417,283,150,148,142,262,216,425,223,531,430,203,153,254,221,162,472,361,286,243,202,205,433,228,266,162,156,296,487,288,196,278,195,88,400,302,216,181,283,185,137,262,120
|
||||
0.60,106,272,207,213,259,228,104,214,245,230,122,170,296,190,223,208,455,133,207,257,147,182,252,475,193,115,254,97,406,167,49,179,199,77,163,234,242,157,153,347,200,322,62,191,152,381,236,119,229,563,127,171,127,126,371,213,565,505,135,242,301,224,221,135,466,185,470,217,403,334,122,362,449,349,285,101,114,281,194,210,410,200,400,199,428,306,387,146,102,375,209,75,139,100,214,295,187,284,230,175
|
||||
0.65,136,80,343,113,146,367,303,463,507,284,211,365,259,522,44,261,122,188,1135,253,305,97,252,234,54,217,220,114,203,194,151,163,201,233,284,603,170,247,299,564,255,333,373,256,244,152,301,86,125,149,263,457,420,431,68,297,336,329,290,194,230,288,161,124,108,201,1089,296,233,737,351,253,38,688,448,148,807,273,119,131,514,98,157,159,486,231,288,164,228,47,163,287,576,141,145,197,272,248,201,787
|
||||
0.70,272,201,329,67,281,293,356,217,213,372,266,346,434,143,264,390,341,296,142,263,330,102,384,669,602,275,149,259,225,227,339,222,397,422,474,274,481,337,585,344,387,308,222,249,396,615,321,310,135,187,296,225,362,146,222,157,336,307,187,486,89,217,223,277,262,235,157,548,249,119,403,348,550,326,273,428,322,225,192,240,824,779,343,146,406,96,268,144,653,344,527,121,224,562,416,180,166,151,201,64
|
||||
0.75,815,131,219,257,95,270,177,38,283,207,451,478,422,108,331,666,573,281,139,102,179,218,87,284,274,370,173,130,235,534,420,219,174,352,234,343,337,171,302,276,1122,220,236,181,200,423,208,348,191,205,346,545,333,438,178,281,270,203,90,238,436,66,236,561,174,299,188,147,140,217,298,263,244,84,639,126,285,233,159,138,47,378,845,166,317,236,480,91,306,163,325,732,106,271,299,398,125,238,588,306
|
||||
0.80,295,191,493,205,1165,763,1031,238,69,412,56,509,356,336,97,722,106,524,239,403,493,337,270,341,112,147,448,603,356,130,336,490,854,672,253,188,375,442,308,222,625,353,216,160,265,442,263,464,143,261,686,319,160,413,237,128,147,165,391,478,632,598,333,187,463,433,253,357,134,84,671,224,273,334,185,142,666,200,112,270,113,221,1003,432,210,414,194,187,101,292,69,424,862,190,306,235,339,268,64,178
|
||||
0.85,221,61,356,234,367,238,586,474,171,168,411,157,295,304,242,118,245,361,250,856,256,194,738,391,499,213,234,411,1910,369,455,315,800,85,212,117,200,518,99,76,796,640,335,256,225,415,612,123,141,301,154,313,329,812,414,104,648,657,192,163,544,1346,305,229,131,4275,222,374,484,386,139,287,734,251,187,232,688,293,718,276,156,507,289,395,493,152,107,905,184,507,267,307,437,724,241,518,238,183,379,285
|
||||
0.90,651,606,207,546,347,1180,121,112,1135,135,203,396,245,275,354,130,320,254,255,148,291,263,448,344,237,357,211,318,700,212,234,545,240,404,277,438,391,383,402,551,392,131,740,643,393,373,288,297,403,342,288,234,319,352,869,396,277,595,87,150,833,710,129,520,619,151,150,375,860,243,282,242,296,549,426,100,890,315,505,451,184,167,643,154,56,252,147,915,463,139,227,697,386,324,292,625,66,161,420,526
|
||||
0.95,421,588,274,114,492,232,230,464,736,233,182,92,401,208,116,357,313,200,308,331,4166,440,575,1279,104,849,42,246,657,2075,203,85,1028,963,1433,696,221,169,168,276,166,238,434,713,1769,295,638,386,186,255,167,382,830,242,1249,260,592,1796,816,359,416,202,3354,928,220,343,596,57,146,705,313,309,500,138,482,57,309,590,913,380,311,212,286,324,950,894,532,525,231,2050,537,72,616,220,273,307,836,105,324,332
|
||||
1.00,898,274,43,1486,173,433,52,573,214,154,602,339,66,719,378,366,857,438,246,775,1006,551,390,182,479,365,473,765,487,1763,759,185,335,286,369,138,220,240,307,669,239,402,180,254,224,240,327,197,2168,363,364,139,485,1176,322,257,707,667,208,794,166,2786,769,147,525,584,329,667,1233,613,456,3296,386,216,684,956,217,1735,662,146,447,99,681,38,1009,133,258,935,229,878,283,499,304,155,611,532,1257,621,394,564
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
plugins {
|
||||
id 'java'
|
||||
id 'application'
|
||||
id 'com.github.johnrengelman.shadow' version '5.2.0'
|
||||
}
|
||||
|
||||
group 'net.lwenstrom.jan'
|
||||
|
|
@ -28,7 +29,7 @@ dependencies {
|
|||
}
|
||||
|
||||
// Include dependent libraries in archive.
|
||||
mainClassName = "example.JumpingDino"
|
||||
mainClassName = "example.DinoSampling"
|
||||
|
||||
|
||||
jar {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
0.05,8565,3803,4010,5419,1733,4070,7502,7880,12776,2242,10489,4384,23578,14718,17801,7770,4097,5905,4027,5969,8492,1994,1542,18577,2346,13550,2279,5072,15109,4419,10970,6150,2359,3093,7131,5932,18531,6887,22726,4925,3320,6832,13704,10372,22945,4526,12461,1831,20964,21963,2749,7112,3260,14907,9168,7133,6590,21075,5302,2546,10164,3101,3204,7850,14204,4431,12038,13430,13976,7385,7647,9634,12230,6635,9532,4181,12857,10473
|
||||
0.1,1550,4416,3473,2821,2101,3635,5168,9641,7074,5623,2571,2253,2595,4347,24575,4239,4552,6372,6638,2686,1976,7638,2479,3555,4496,2321,11712,1934,3668,9398,1312,4422,3217,9472,7053,5539,2140,2928,1576,12121,5023,2458,5947,2374,10197,1701,6186,7013,3061,3108,1856,4052,2327,2287,2026,1606,1567,2414,3756,2648,3409,4195,4803,11940,2171,1574,2656,3221,2044,2250,1528,1348,7785,3141,6622,4520,2118,2489,7125,4182,1495,2475,5037,3839,4218,1929,2302,5339,1827,5444,3710,2555,2373,2108
|
||||
0.15,2328,3189,5422,1943,2057,1630,1405,2079,8071,22741,4056,1704,2727,2339,10381,1246,1831,6658,2760,9416,4036,1243,3716,1199,8678,5480,4038,1133,3576,3852,4014,1487,5849,4275,3766,2240,3917,2164,3695,1954,1869,1854,1751,2785,2334,2821,5942,1310,1680,7867,1987,2345,1785,3898,1598,4382,2881,4874,2345,2483,3962,8058,10204,6444,2885,8494,5135,7150,1041,8020,24960,1720,3546,3751,2077,2269,6051,5372,4247,10739,7114,1169,1757,1162,2699,1765,1367,3226,2290,3695,1756,2452,7385,2599,1426,7600,15285,4712
|
||||
0.2,1406,918,2393,1369,1913,2101,5855,1967,2146,8616,1630,6279,3109,6358,2462,2387,2113,1580,1679,3521,1354,6432,1156,5562,4964,2883,1622,2395,2259,2620,5165,597,1678,1499,3081,1663,1445,915,1385,980,1794,3339,2668,2901,3156,775,1432,2226,1790,2057,2441,1090,1165,839,5338,1085,3086,1336,2527,1392,892,1498,3915,4456,852,1467,3217,3454,2225,8179,1410,1390,3432,958,714,920,1543,1432,5129,2868,1236,917,7646,12936,5340,1456,2233,1176,1113,2152,2637,4778,3093,1706,1465,4785,1633,940,2669,2219
|
||||
0.25,2467,2964,947,4283,3290,887,3352,5430,2617,2126,1285,1493,3685,1348,1680,3302,5434,2659,1528,836,1387,1164,1339,3796,8728,1008,3197,2314,1852,1302,1338,1531,2064,1034,3022,4221,1165,1820,588,2878,1330,1504,2328,1000,1758,1217,1243,1127,1385,19537,2437,4459,1488,3603,1604,974,1756,1468,2696,2511,4003,3352,1015,1150,1341,1916,1552,13896,4017,935,3175,1430,1494,1165,10492,2132,1478,5363,4016,1730,1262,4001,774,2865,4585,909,1759,993,3900,1446,2101,1542,1791,4580,3227,1322,914,1014,5030,3115
|
||||
0.3,5105,2480,2842,1427,860,5434,2914,3608,2814,2037,775,654,2607,1418,786,1093,6668,967,2877,3091,2022,1799,2963,741,1352,1126,915,2413,1938,1709,2025,1192,1787,2065,1319,4872,1162,1752,3669,971,1702,3016,718,5211,2023,2113,1912,1419,1271,1323,2039,2624,2886,1404,1914,872,1292,1102,3544,2138,953,1698,800,2866,1492,1640,1270,12637,2851,3677,2295,4025,16097,2613,2591,2223,4112,1563,2441,1120,1886,2588,1559,1690,3034,937,847,6815,1616,1398,2439,1087,1603,841,4226,2110,1412,1852,2030,1651
|
||||
0.35,2154,1462,564,1702,1024,1400,960,2879,1226,2021,3619,951,4514,878,1796,2912,2112,1128,6131,733,2392,1413,2338,1437,1694,1440,1667,1714,1025,1625,2729,1200,4780,1431,2054,4221,21191,2106,2524,1742,1743,2546,1435,2707,1590,1228,2792,6297,1181,2126,2354,4585,736,1181,797,1287,1167,2023,947,628,2975,1069,1803,1558,4170,1695,3664,2169,1026,1381,1914,1497,1547,1508,3400,1533,1706,1511,973,1514,1334,833,3001,1891,1171,1797,1867,1107,1066,1180,710,2204,2308,2090,4976,3246,3289,2952,2053,1858
|
||||
0.40,1358,2175,1946,3255,3528,1126,3074,3633,2082,3648,1478,2562,3231,2679,1471,2027,1011,2322,2608,1063,1191,2552,2487,1648,4293,1907,1158,1239,1650,2524,1264,1676,2506,2911,1078,1664,2119,1234,4422,1787,1264,1923,2271,617,1489,2358,1089,2072,3087,1224,1261,761,2000,1166,1197,3519,816,2670,3148,1543,2279,973,2901,2117,2817,2979,1035,2630,501,884,901,1088,3228,1340,2681,1404,999,880,2641,2097,1168,5745,1426,1621,2584,1248,860,867,1987,844,2770,1839,1387,8928,1918,2712,1874,4033,2830,1160
|
||||
0.45,952,3145,3339,1530,2135,1357,2356,3650,2091,3197,2500,2467,1558,1743,2915,1107,1560,1467,1984,4573,1161,1532,2155,3748,3453,1670,3841,2159,6404,1278,1013,3146,3325,1537,1202,3885,3577,847,3355,1316,773,1269,7931,759,3015,1318,1848,1707,3255,2499,1592,3410,1890,1461,1025,1330,2008,2333,838,798,4177,3636,4507,3540,3155,1250,2368,4013,1714,1045,1268,2063,3622,1994,2293,1280,2493,3504,1371,1539,1616,722,2119,6346,1275,2904,4186,3192,1409,2475,966,2020,1789,2340,2919,1735,3885,1880,2238,1097
|
||||
0.50,2374,4721,1629,1025,1505,3695,1230,1273,1615,1390,7918,3475,2994,1421,10052,1094,4076,1060,2741,10973,18456,2725,2103,909,6923,2194,8988,6814,2834,1035,6925,8001,1394,1356,1668,1536,1633,1029,3789,3673,2044,4382,1978,1337,2870,1469,929,1041,2102,1395,3240,2511,4085,1021,1881,4816,2775,2959,629,1489,2970,2494,1653,8340,1005,13202,1187,14008,2906,1174,3777,6520,2468,3466,5371,4582,2002,3097,913,3358,2075,1522,1651,2916,2235,1341,2166,2797,3426,1181,1320,3434,3799,1107,1523,13668,1183,2368,698,5306
|
||||
0.55,2647,1750,2096,2452,10203,3576,3104,4698,3485,1774,3832,1944,2167,706,5664,1154,2000,2251,2947,1433,3874,1247,783,6218,2633,3426,4620,3234,1425,1348,6754,9355,6974,9257,1669,1237,4517,1173,3650,3293,2227,1132,8711,6037,4353,2363,1788,3505,1206,1706,2020,2427,2429,2819,3784,3714,2489,943,4019,1275,651,4592,5342,1367,4862,4260,5722,1937,2618,1715,1719,14424,2459,1758,5275,619,3354,1439,1834,3520,1539,4790,1338,1385,3732,747,5776,4451,1812,1780,1459,4366,1624,2262,4316,2707,634,1521,2472
|
||||
0.60,3791,2167,4108,1529,6057,10906,1848,3916,2554,2260,10363,3852,1868,4647,2310,2445,4288,6644,4171,1847,3360,3840,9454,3431,5358,1506,3422,2613,8244,1601,3807,2675,1066,10286,1533,12266,1587,6858,1335,4647,6590,4669,6556,1865,4763,3360,1336,2594,3571,1451,3525,4525,10153,1950,1550,1622,4162,4054,3396,2066,7273,5209,1565,17691,3451,6348,8727,5952,1031,1667,1334,1739,4680,2429,4557,2444,1913,2060,5295,11317,4703,1610,5416,1454,5184,3950,1121,1682,836,1488,3644,8243,1917,9651,5226,3574,1816,11354
|
||||
0.65,2927,2667,6480,3327,1148,20622,1159,1995,11561,1387,3423,8214,4808,2243,895,12123,3130,4774,18643,1420,2069,8101,1203,2343,3404,812,7383,3894,1275,7371,2794,2107,4178,4916,906,1482,6988,5286,3549,825,14992,4329,9648,12596,1520,1953,1714,16021,1544,9407,12398,1461,3339,2375,3976,6882,1225,5720,4793,2890,978,9106,1114,2978,6228,1833,1369,1516,5869,3312,1511,1897,13421,2159,5183,5017,4529,2060,9510,1367,13258,2115,2921,3235,1910,7535,3969,1416,2256,8444,4304,2881,3293,4981,3895,1728,10426,11769
|
||||
0.70,1873,4185,17340,4931,5268,2872,995,1878,6787,5225,2391,2046,796,7451,4642,2507,4279,2829,3556,7686,2601,6621,5950,1451,7073,5008,4269,19619,6826,1939,18936,2616,1352,1546,4311,2623,9623,3558,3586,7553,6106,1623,2159,5808,3135,7249,2643,4079,2036,3324,1584,8870,2322,8866,8451,6603,9716,18438,1468,7903,7222,1878,12807,5844,1268,2276,13139,8881,11004,4889,1717,5348,7186,2271,1639,2325,6008,2738,3918,3949,1502,2461,2454,5256,3672,1521,1480,7782,2869,796,12067,11323,2872
|
||||
0.75,5316,6466,2819,1521,8721,5240,2167,7393,23803,2967,11932,9338,17412,11452,16091,1828,11745,2245,24697,1786,5518,2423,2680,7215,7624,4498,6302,19095,1255,8959,2094,11525,7324,13853,2935,23283,11041,3041,5165,22986,3130,21531,13741,6549,3985,3451,14035,5976,13429,6650,5788,19359,7972,4733,13785,3008,2927,5636,4063,1151,1713,4341,22161,2441,3818,2231,11920,17832,13841,16102,8596,3899,11222,940,10479,10952,2209,19993,12426,4239,13743,4139,10398,2877,1111,4726
|
||||
0.80,19870,3288,13724,4492,8159,5058,16764,5648,9462,19071,3914,1242,8262,4036,9421,4914,2535,5362,7298,9587,1837,15272,14922,14138,7115,17236,5123,12157,6086,1672,15573,14241,2049,2602,6802,22362,7936,7544,5330,13155,16016,4544,1489,3780,6326,7794,2808,1493,7788,12646,22312,1681,12084,4163,2197,7950,22478,5106,4382,10615,2586,12214,4799,6297,7589,4585,15734,5480,8626,7387,11932,4245,21532,1710,12737,7132,4740,14578
|
||||
0.85,5493,10568,19366,5705,15430,8183,5721,13314,3753,23888,21843,6856,2803,9434,4794,10730,4484,23990,6350,16180,8099,4298,11220,4624,5946,24895,8464,4416,6619,2800,4081,12459,1981,12488,6380,9597,10328,1901,24563,13059,3639,12988,2604,4440,22666,1775,4078,5175,1144,3759,11119,1856,10831,2229,5333,17121,9698,14919,2353,3963,8189,13920,5301,16516,2446,3985
|
||||
0.90,24592,18962,2318,17604,14327,4999,9023,5562,7541,11799,8724,12642,2147,5909,5414,12572,10018,18962,3444,16813,16492,3991,7315,17775,11087,3479,2540,9597,13169,8588,2794,24864,19491,2673,5363,4799,5852,3757,1115,5798,24249,2576
|
||||
0.95,24269,14543,6828,3800,2930,12482,8749,17286,9802,7114,3756,14507,15139,1266,3794,15184,23713,6453,15157,7045,8048,6274,4713,5024,7315,8193,3695,2114,5084,12686,19158,20940,9037,15016,15530,11063,8586,15635,7196
|
||||
1.00,11364,6363,8012,19730,8425,21388,7864,18427,3126,9522,3180,4838,4078,22968,21540,4050,5612,4990,9933,1969,20399,3691,14813,14994,1577,1587,23322,22071,13903,3584,9721
|
||||
|
|
@ -1,20 +1,2 @@
|
|||
0.05,447,1036,447,788,499,499,880,514,583,420,759,678,529,781,742,185,848,346,733,961,657,312,599,245,1076,1361,377,612,821,481,374,250,171,949,1079,788,465,537,431,312,565,1046,825,579,1178,547,1111,518,949,779,924,430,543,1262,642,814,508,422,1120,536,591,381,670,681,382,297,795,774,1552,691,909,748,520,639,704,575,752,418,503,292,1159,1020,570,839,696,895,511,635,830,522,628,518,408,392,605,888,790,629,547,999
|
||||
0.1,337,303,221,455,64,271,377,946,381,253,669,479,267,870,250,364,347,83,124,464,356,535,820,180,369,310,284,456,166,267,240,215,347,295,226,409,344,266,404,480,195,235,655,233,429,149,304,164,409,276,268,139,495,388,401,283,343,284,720,351,170,781,59,369,789,155,206,464,380,272,645,180,216,698,166,497,650,297,595,352,138,163,412,238,336,90,453,238,306,598,487,722,640,592,493,144,435,441,387,374
|
||||
0.15,288,400,363,158,414,302,126,175,323,303,391,202,295,329,468,198,199,179,384,193,168,443,74,153,347,458,395,209,545,199,421,587,144,200,219,204,134,200,253,193,396,108,530,728,286,118,256,351,279,308,210,713,474,415,397,328,220,201,136,515,374,154,847,578,313,250,224,392,362,551,127,211,321,176,372,572,304,290,225,541,296,315,392,235,161,359,307,360,613,194,389,278,334,153,402,224,246,260,264,226
|
||||
0.2,193,331,294,156,245,530,237,199,225,298,414,424,342,239,361,295,176,326,155,371,272,255,259,201,506,520,108,172,416,220,245,283,203,213,164,469,262,263,177,80,174,346,573,577,243,226,437,427,261,195,210,389,286,297,263,328,393,174,166,267,344,223,161,171,321,256,322,251,238,164,216,198,449,327,328,175,334,260,479,269,389,137,292,384,232,549,394,177,207,165,123,304,346,407,437,66,176,358,277,111
|
||||
0.25,155,195,340,162,250,300,204,218,278,291,184,282,95,370,265,275,168,229,207,463,273,157,233,106,338,414,397,305,283,374,347,243,78,172,284,296,187,439,149,513,689,99,93,260,382,204,141,310,170,409,436,112,283,435,284,108,210,264,335,177,121,181,124,294,129,394,183,117,211,137,254,384,263,228,423,443,349,142,115,276,268,251,275,191,387,283,124,212,191,167,490,329,293,194,294,241,432,50,105,341
|
||||
0.3,282,128,127,74,136,311,303,206,150,387,167,353,286,138,107,269,343,301,157,370,553,161,292,175,250,187,259,451,222,203,312,177,212,248,197,276,122,236,159,480,274,474,46,208,96,248,131,358,301,142,302,298,153,105,524,131,225,343,207,301,409,321,505,546,131,395,275,151,148,280,266,539,116,105,166,232,245,155,219,222,123,174,221,162,521,206,325,77,388,199,380,222,52,177,119,98,323,378,262,214
|
||||
0.35,404,130,174,176,297,284,274,139,279,225,105,115,238,191,278,162,185,316,233,535,175,131,86,245,135,262,285,427,214,224,139,170,348,371,385,151,231,258,202,260,293,109,159,323,317,68,231,170,313,168,222,216,357,225,177,125,195,361,242,133,258,158,174,101,132,272,131,213,224,124,232,701,395,321,173,144,226,260,220,211,352,190,188,422,260,151,167,241,459,237,298,225,123,274,107,242,253,134,132,288
|
||||
0.40,318,374,159,305,164,220,69,174,233,277,163,293,236,323,154,210,209,117,271,126,155,632,233,208,417,273,127,213,73,223,99,118,178,197,137,207,138,351,208,254,232,207,609,256,169,164,403,589,145,79,344,293,270,218,162,221,341,71,125,203,215,178,81,150,213,223,368,236,244,249,231,166,317,171,164,208,292,172,174,256,285,136,236,446,312,371,285,152,155,524,171,113,287,156,281,97,209,195,368,283
|
||||
0.45,198,106,298,295,298,124,168,159,194,144,110,270,253,225,126,191,184,119,255,484,528,147,181,190,345,166,158,211,304,242,92,199,219,129,86,170,167,150,190,168,514,317,154,164,186,261,276,234,72,241,130,233,269,136,217,73,219,193,353,42,127,305,361,313,86,109,254,400,327,452,165,350,192,139,499,192,147,313,225,159,122,121,228,217,79,162,183,354,360,432,258,313,325,317,265,33,175,233,203,178
|
||||
0.50,149,341,278,153,204,216,244,250,291,247,268,114,175,202,87,236,377,280,161,444,199,185,112,52,495,248,87,199,327,374,192,136,545,92,174,186,85,238,277,274,227,139,112,300,205,292,398,188,329,214,114,55,212,154,533,109,139,165,216,280,210,231,193,295,149,190,131,130,184,159,161,229,286,146,218,151,137,265,80,191,326,282,194,164,114,203,221,495,298,187,116,110,119,140,237,203,124,307,260,164
|
||||
0.55,190,161,170,533,205,104,85,228,610,294,293,380,297,333,498,303,198,248,430,180,157,131,402,706,346,282,151,102,346,227,195,98,306,391,533,312,297,149,480,191,93,110,233,104,434,244,296,124,131,275,157,165,93,94,241,96,82,417,283,150,148,142,262,216,425,223,531,430,203,153,254,221,162,472,361,286,243,202,205,433,228,266,162,156,296,487,288,196,278,195,88,400,302,216,181,283,185,137,262,120
|
||||
0.60,106,272,207,213,259,228,104,214,245,230,122,170,296,190,223,208,455,133,207,257,147,182,252,475,193,115,254,97,406,167,49,179,199,77,163,234,242,157,153,347,200,322,62,191,152,381,236,119,229,563,127,171,127,126,371,213,565,505,135,242,301,224,221,135,466,185,470,217,403,334,122,362,449,349,285,101,114,281,194,210,410,200,400,199,428,306,387,146,102,375,209,75,139,100,214,295,187,284,230,175
|
||||
0.65,136,80,343,113,146,367,303,463,507,284,211,365,259,522,44,261,122,188,1135,253,305,97,252,234,54,217,220,114,203,194,151,163,201,233,284,603,170,247,299,564,255,333,373,256,244,152,301,86,125,149,263,457,420,431,68,297,336,329,290,194,230,288,161,124,108,201,1089,296,233,737,351,253,38,688,448,148,807,273,119,131,514,98,157,159,486,231,288,164,228,47,163,287,576,141,145,197,272,248,201,787
|
||||
0.70,272,201,329,67,281,293,356,217,213,372,266,346,434,143,264,390,341,296,142,263,330,102,384,669,602,275,149,259,225,227,339,222,397,422,474,274,481,337,585,344,387,308,222,249,396,615,321,310,135,187,296,225,362,146,222,157,336,307,187,486,89,217,223,277,262,235,157,548,249,119,403,348,550,326,273,428,322,225,192,240,824,779,343,146,406,96,268,144,653,344,527,121,224,562,416,180,166,151,201,64
|
||||
0.75,815,131,219,257,95,270,177,38,283,207,451,478,422,108,331,666,573,281,139,102,179,218,87,284,274,370,173,130,235,534,420,219,174,352,234,343,337,171,302,276,1122,220,236,181,200,423,208,348,191,205,346,545,333,438,178,281,270,203,90,238,436,66,236,561,174,299,188,147,140,217,298,263,244,84,639,126,285,233,159,138,47,378,845,166,317,236,480,91,306,163,325,732,106,271,299,398,125,238,588,306
|
||||
0.80,295,191,493,205,1165,763,1031,238,69,412,56,509,356,336,97,722,106,524,239,403,493,337,270,341,112,147,448,603,356,130,336,490,854,672,253,188,375,442,308,222,625,353,216,160,265,442,263,464,143,261,686,319,160,413,237,128,147,165,391,478,632,598,333,187,463,433,253,357,134,84,671,224,273,334,185,142,666,200,112,270,113,221,1003,432,210,414,194,187,101,292,69,424,862,190,306,235,339,268,64,178
|
||||
0.85,221,61,356,234,367,238,586,474,171,168,411,157,295,304,242,118,245,361,250,856,256,194,738,391,499,213,234,411,1910,369,455,315,800,85,212,117,200,518,99,76,796,640,335,256,225,415,612,123,141,301,154,313,329,812,414,104,648,657,192,163,544,1346,305,229,131,4275,222,374,484,386,139,287,734,251,187,232,688,293,718,276,156,507,289,395,493,152,107,905,184,507,267,307,437,724,241,518,238,183,379,285
|
||||
0.90,651,606,207,546,347,1180,121,112,1135,135,203,396,245,275,354,130,320,254,255,148,291,263,448,344,237,357,211,318,700,212,234,545,240,404,277,438,391,383,402,551,392,131,740,643,393,373,288,297,403,342,288,234,319,352,869,396,277,595,87,150,833,710,129,520,619,151,150,375,860,243,282,242,296,549,426,100,890,315,505,451,184,167,643,154,56,252,147,915,463,139,227,697,386,324,292,625,66,161,420,526
|
||||
0.95,421,588,274,114,492,232,230,464,736,233,182,92,401,208,116,357,313,200,308,331,4166,440,575,1279,104,849,42,246,657,2075,203,85,1028,963,1433,696,221,169,168,276,166,238,434,713,1769,295,638,386,186,255,167,382,830,242,1249,260,592,1796,816,359,416,202,3354,928,220,343,596,57,146,705,313,309,500,138,482,57,309,590,913,380,311,212,286,324,950,894,532,525,231,2050,537,72,616,220,273,307,836,105,324,332
|
||||
1.00,898,274,43,1486,173,433,52,573,214,154,602,339,66,719,378,366,857,438,246,775,1006,551,390,182,479,365,473,765,487,1763,759,185,335,286,369,138,220,240,307,669,239,402,180,254,224,240,327,197,2168,363,364,139,485,1176,322,257,707,667,208,794,166,2786,769,147,525,584,329,667,1233,613,456,3296,386,216,684,956,217,1735,662,146,447,99,681,38,1009,133,258,935,229,878,283,499,304,155,611,532,1257,621,394,564
|
||||
0.05,5,5,5,5,5,5,5,5,5,5,5,5,56,56
|
||||
0.10,6,6,6,6,6,6,6,6,6
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
0.05,8,64,62,15,84,5,72,183,
|
||||
0.1,15,15,6,18,46,28,307,
|
||||
0.15,177,20,12,9,21,4,6,65,173,
|
||||
0.2,6,56,13,28,4,231,9,11,22,
|
||||
0.25,92,37,11,4,310,
|
||||
0.3,106,7,302,12,23,4,5,177,
|
||||
|
|
@ -1,6 +1,26 @@
|
|||
# Libraries
|
||||
library(ggplot2)
|
||||
library(matrixStats)
|
||||
# file.choose()
|
||||
convergence <- read.csv(file.choose(), header=FALSE, row.names=1)
|
||||
men <- rowMeans(convergence[,-1])
|
||||
ba <- barplot(names=rownames(convergence), height=men, ylim=c(0,max(men) +100), ylab = "avg. episodes until convergence", xlab = "epsilon value")
|
||||
|
||||
sds <- rowSds(sapply(convergence[,-1], `length<-`, max(lengths(convergence[,-1]))), na.rm=TRUE)
|
||||
men <- rowMeans(sapply(convergence[,-1], `length<-`, max(lengths(convergence[,-1]))), na.rm=TRUE)
|
||||
print(sds)
|
||||
|
||||
# create dummy data
|
||||
data <- data.frame(
|
||||
names=rownames(convergence),
|
||||
means=men,
|
||||
sds=sds
|
||||
)
|
||||
|
||||
ggplot(data) +
|
||||
geom_bar(aes(x=names, y=means), stat="identity", fill="skyblue", alpha=0.7) +
|
||||
geom_errorbar( aes(x=names, ymin=means-sds, ymax=means+sds), width=0.4, colour="orange", alpha=0.9, size=1.3) +
|
||||
geom_text(aes(label=as.integer(means), x =names, y=means), position=position_dodge(width=0.9), vjust=-0.25) +
|
||||
xlab("Epsilon") + ylab("avg. amount of episodes until convergence")
|
||||
|
||||
|
||||
ba <- barplot(names=rownames(convergence), height=men, ylim=c(0, max(men)*1.2), ylab = "avg. episodes until convergence", xlab = "epsilon value")
|
||||
text(x = ba, y = men, label = as.integer(men), pos = 3, cex = 0.8, col = "red")
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
package core;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* ! SecureRandom not working properly on windows/different JDKs,
|
||||
* using Random again !
|
||||
*
|
||||
* To ensure deterministic behaviour of repeating program executions,
|
||||
* this class is used for all random number generation methods.
|
||||
* Do not use Math.random()!
|
||||
|
|
@ -13,10 +14,10 @@ import java.util.Random;
|
|||
* execution)
|
||||
*/
|
||||
public class RNG {
|
||||
private static SecureRandom rng;
|
||||
private static Random rng;
|
||||
private static int seed = 123;
|
||||
static {
|
||||
rng = new SecureRandom();
|
||||
rng = new Random();
|
||||
rng.setSeed(seed);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,16 +9,19 @@ import evironment.antGame.gui.AntWorldComponent;
|
|||
import javax.swing.*;
|
||||
import java.awt.*;
|
||||
|
||||
/**
|
||||
* Episodic AntWorld
|
||||
*/
|
||||
public class AntWorld implements Environment<AntAction>, Visualizable {
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private Grid grid;
|
||||
protected Grid grid;
|
||||
/**
|
||||
* Intern (backend) representation of the ant.
|
||||
* The AntWorld essentially acts like the game host of the original AntGame.
|
||||
*/
|
||||
private Ant myAnt;
|
||||
protected Ant myAnt;
|
||||
/**
|
||||
* The client agent. In the original AntGame the host would send jade messages
|
||||
* of the current observation to each client on every tick.
|
||||
|
|
@ -32,13 +35,13 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
|
|||
* through an intern grid clone (brain), for example. A history as mentioned in
|
||||
* various lectures could be possible as well.
|
||||
*/
|
||||
private AntAgent antAgent;
|
||||
protected AntAgent antAgent;
|
||||
|
||||
private int tick;
|
||||
protected int tick;
|
||||
private int maxEpisodeTicks;
|
||||
|
||||
public AntWorld(int width, int height, double foodDensity){
|
||||
grid = new Grid(width, height, foodDensity);
|
||||
public AntWorld(int width, int height) {
|
||||
grid = new Grid(width, height);
|
||||
antAgent = new AntAgent(width, height);
|
||||
myAnt = new Ant();
|
||||
maxEpisodeTicks = 1000;
|
||||
|
|
@ -46,73 +49,68 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
|
|||
}
|
||||
|
||||
public AntWorld(){
|
||||
this(Constants.DEFAULT_GRID_WIDTH, Constants.DEFAULT_GRID_HEIGHT, Constants.DEFAULT_FOOD_DENSITY);
|
||||
this(Constants.DEFAULT_GRID_WIDTH, Constants.DEFAULT_GRID_HEIGHT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StepResultEnvironment step(AntAction action){
|
||||
AntObservation observation;
|
||||
State newState;
|
||||
double reward = 0;
|
||||
String info = "";
|
||||
boolean done = false;
|
||||
|
||||
protected StepCalculation processStep(AntAction action) {
|
||||
StepCalculation sc = new StepCalculation();
|
||||
sc.reward = -1;
|
||||
sc.info = "";
|
||||
sc.done = false;
|
||||
Cell currentCell = grid.getCell(myAnt.getPos());
|
||||
Point potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
|
||||
boolean stayOnCell = true;
|
||||
sc.potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
|
||||
sc.stayOnCell = true;
|
||||
// flag to enable a check if all food has been collected only fired if food was dropped
|
||||
// on the starting position
|
||||
boolean checkCompletion = false;
|
||||
sc.checkCompletion = false;
|
||||
|
||||
switch (action) {
|
||||
switch(action) {
|
||||
case MOVE_UP:
|
||||
potentialNextPos.y -= 1;
|
||||
stayOnCell = false;
|
||||
sc.potentialNextPos.y -= 1;
|
||||
sc.stayOnCell = false;
|
||||
break;
|
||||
case MOVE_RIGHT:
|
||||
potentialNextPos.x += 1;
|
||||
stayOnCell = false;
|
||||
sc.potentialNextPos.x += 1;
|
||||
sc.stayOnCell = false;
|
||||
break;
|
||||
case MOVE_DOWN:
|
||||
potentialNextPos.y += 1;
|
||||
stayOnCell = false;
|
||||
sc.potentialNextPos.y += 1;
|
||||
sc.stayOnCell = false;
|
||||
break;
|
||||
case MOVE_LEFT:
|
||||
potentialNextPos.x -= 1;
|
||||
stayOnCell = false;
|
||||
sc.potentialNextPos.x -= 1;
|
||||
sc.stayOnCell = false;
|
||||
break;
|
||||
case PICK_UP:
|
||||
if(myAnt.hasFood()){
|
||||
if(myAnt.hasFood()) {
|
||||
// Ant tries to pick up food but can only hold one piece
|
||||
reward = Reward.FOOD_PICK_UP_FAIL_HAS_FOOD_ALREADY;
|
||||
}else if(currentCell.getFood() == 0){
|
||||
sc.reward += Reward.FOOD_PICK_UP_FAIL_HAS_FOOD_ALREADY;
|
||||
} else if(currentCell.getFood() == 0) {
|
||||
// Ant tries to pick up food on cell that has no food on it
|
||||
reward = Reward.FOOD_PICK_UP_FAIL_NO_FOOD;
|
||||
}else if(currentCell.getFood() > 0){
|
||||
sc.reward += Reward.FOOD_PICK_UP_FAIL_NO_FOOD;
|
||||
} else if(currentCell.getFood() > 0) {
|
||||
// Ant successfully picks up food
|
||||
currentCell.setFood(currentCell.getFood() - 1);
|
||||
myAnt.setHasFood(true);
|
||||
reward = Reward.FOOD_PICK_UP_SUCCESS;
|
||||
sc.reward = Reward.FOOD_PICK_UP_SUCCESS;
|
||||
}
|
||||
break;
|
||||
case DROP_DOWN:
|
||||
if(!myAnt.hasFood()){
|
||||
if(!myAnt.hasFood()) {
|
||||
// Ant had no food to drop
|
||||
reward = Reward.FOOD_DROP_DOWN_FAIL_NO_FOOD;
|
||||
}else{
|
||||
// Drop food onto the ground
|
||||
currentCell.setFood(currentCell.getFood() + 1);
|
||||
sc.reward += Reward.FOOD_DROP_DOWN_FAIL_NO_FOOD;
|
||||
} else {
|
||||
myAnt.setHasFood(false);
|
||||
|
||||
// negative reward if the agent drops food on any other field
|
||||
// than the starting point
|
||||
if(currentCell.getType() != CellType.START){
|
||||
reward = Reward.FOOD_DROP_DOWN_FAIL_NOT_START;
|
||||
done = true;
|
||||
}else{
|
||||
reward = Reward.FOOD_DROP_DOWN_SUCCESS;
|
||||
if(currentCell.getType() != CellType.START) {
|
||||
sc.reward += Reward.FOOD_DROP_DOWN_FAIL_NOT_START;
|
||||
// Drop food onto the ground
|
||||
currentCell.setFood(currentCell.getFood() + 1);
|
||||
} else {
|
||||
sc.reward = Reward.FOOD_DROP_DOWN_SUCCESS;
|
||||
myAnt.setPoints(myAnt.getPoints() + 1);
|
||||
checkCompletion = true;
|
||||
sc.checkCompletion = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
|
@ -121,24 +119,33 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
|
|||
}
|
||||
|
||||
// movement action was selected
|
||||
if(!stayOnCell){
|
||||
if(!isInGrid(potentialNextPos)){
|
||||
stayOnCell = true;
|
||||
reward = Reward.RAN_INTO_WALL;
|
||||
}else if(hitObstacle(potentialNextPos)){
|
||||
stayOnCell = true;
|
||||
reward = Reward.RAN_INTO_OBSTACLE;
|
||||
if(!sc.stayOnCell) {
|
||||
if(!isInGrid(sc.potentialNextPos)) {
|
||||
sc.stayOnCell = true;
|
||||
sc.reward += Reward.RAN_INTO_WALL;
|
||||
} else if(hitObstacle(sc.potentialNextPos)) {
|
||||
sc.stayOnCell = true;
|
||||
sc.reward += Reward.RAN_INTO_OBSTACLE;
|
||||
}
|
||||
}
|
||||
|
||||
return sc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StepResultEnvironment step(AntAction action){
|
||||
AntObservation observation;
|
||||
State newState;
|
||||
|
||||
StepCalculation sc = processStep(action);
|
||||
|
||||
// valid movement
|
||||
if(!stayOnCell){
|
||||
myAnt.getPos().setLocation(potentialNextPos);
|
||||
if(!sc.stayOnCell) {
|
||||
myAnt.getPos().setLocation(sc.potentialNextPos);
|
||||
if(antAgent.getCell(myAnt.getPos()).getType() == CellType.UNKNOWN){
|
||||
// the ant will move to a cell that was previously unknown
|
||||
reward = Reward.UNKNOWN_FIELD_EXPLORED;
|
||||
}else{
|
||||
reward = 0;
|
||||
// TODO: not optimal for going straight for food
|
||||
// sc.reward = Reward.UNKNOWN_FIELD_EXPLORED;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -148,33 +155,36 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
|
|||
// let the ant agent process the observation to create a valid markov state
|
||||
newState = antAgent.feedObservation(observation);
|
||||
|
||||
if(checkCompletion){
|
||||
done = grid.isAllFoodCollected();
|
||||
if(sc.checkCompletion) {
|
||||
sc.done = grid.isAllFoodCollected();
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
if(!done){
|
||||
reward = -1;
|
||||
}
|
||||
*/
|
||||
if(++tick == maxEpisodeTicks){
|
||||
done = true;
|
||||
sc.done = true;
|
||||
}
|
||||
|
||||
|
||||
StepResultEnvironment result = new StepResultEnvironment(newState, reward, done, info);
|
||||
return result;
|
||||
return new StepResultEnvironment(newState, sc.reward, sc.done, sc.info);
|
||||
}
|
||||
|
||||
private boolean isInGrid(Point pos){
|
||||
protected boolean isInGrid(Point pos) {
|
||||
return pos.x >= 0 && pos.x < grid.getWidth() && pos.y >= 0 && pos.y < grid.getHeight();
|
||||
}
|
||||
|
||||
private boolean hitObstacle(Point pos){
|
||||
protected boolean hitObstacle(Point pos) {
|
||||
return grid.getCell(pos).getType() == CellType.OBSTACLE;
|
||||
}
|
||||
|
||||
protected class StepCalculation {
|
||||
double reward;
|
||||
String info;
|
||||
boolean done;
|
||||
Point potentialNextPos = new Point(myAnt.getPos().x, myAnt.getPos().y);
|
||||
boolean stayOnCell = true;
|
||||
// flag to enable a check if all food has been collected only fired if food was dropped
|
||||
// on the starting position
|
||||
boolean checkCompletion = false;
|
||||
}
|
||||
|
||||
public State reset() {
|
||||
grid.resetWorld();
|
||||
antAgent.initUnknownWorld();
|
||||
|
|
@ -189,6 +199,7 @@ public class AntWorld implements Environment<AntAction>, Visualizable {
|
|||
public void setMaxEpisodeLength(int maxTicks){
|
||||
this.maxEpisodeTicks = maxTicks;
|
||||
}
|
||||
|
||||
public Point getSpawningPoint(){
|
||||
return grid.getStartPoint();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
package evironment.antGame;
|
||||
|
||||
import core.StepResultEnvironment;
|
||||
|
||||
public class AntWorldContinuous extends AntWorld {
|
||||
public AntWorldContinuous(int width, int height) {
|
||||
super(width, height);
|
||||
}
|
||||
|
||||
public AntWorldContinuous() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StepResultEnvironment step(AntAction action) {
|
||||
AntObservation observation;
|
||||
Cell currentCell = grid.getCell(myAnt.getPos());
|
||||
|
||||
StepCalculation sc = processStep(action);
|
||||
|
||||
// flag is set to true if food gets dropped onto starts
|
||||
if(sc.checkCompletion) {
|
||||
grid.spawnNewFood();
|
||||
}
|
||||
// valid movement
|
||||
if(!sc.stayOnCell) {
|
||||
myAnt.getPos().setLocation(sc.potentialNextPos);
|
||||
}
|
||||
|
||||
// get observation after action was computed
|
||||
observation = new AntObservation(grid.getCell(myAnt.getPos()), myAnt.getPos(), myAnt.hasFood());
|
||||
|
||||
return new StepResultEnvironment(new AntState(grid.getGrid(), observation.getPos(), observation.hasFood()), sc.reward, false, sc.info);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ import java.awt.*;
|
|||
|
||||
public class Cell {
|
||||
@Getter
|
||||
@Setter
|
||||
private CellType type;
|
||||
@Getter
|
||||
@Setter
|
||||
|
|
@ -38,4 +39,13 @@ public class Cell {
|
|||
}
|
||||
return super.equals(obj);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Cell{" +
|
||||
"type=" + type +
|
||||
", food=" + food +
|
||||
", pos=" + pos +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package evironment.antGame;
|
||||
|
||||
public class Constants {
|
||||
public static final int DEFAULT_GRID_WIDTH = 10;
|
||||
public static final int DEFAULT_GRID_HEIGHT = 10;
|
||||
public static final double DEFAULT_FOOD_DENSITY = 0.1;
|
||||
public static final int DEFAULT_GRID_WIDTH = 5;
|
||||
public static final int DEFAULT_GRID_HEIGHT = 5;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,24 +7,18 @@ import java.awt.*;
|
|||
public class Grid {
|
||||
private int width;
|
||||
private int height;
|
||||
private double foodDensity;
|
||||
private Point start;
|
||||
private Cell[][] grid;
|
||||
private Cell[][] initialGrid;
|
||||
|
||||
public Grid(int width, int height, double foodDensity){
|
||||
public Grid(int width, int height) {
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.foodDensity = foodDensity;
|
||||
grid = new Cell[width][height];
|
||||
initialGrid = new Cell[width][height];
|
||||
initRandomWorld();
|
||||
}
|
||||
|
||||
public Grid(int width, int height){
|
||||
this(width, height, 0);
|
||||
}
|
||||
|
||||
public void resetWorld(){
|
||||
grid = Util.deepCopyCellGrid(initialGrid);
|
||||
}
|
||||
|
|
@ -32,15 +26,52 @@ public class Grid {
|
|||
public void initRandomWorld(){
|
||||
for(int x = 0; x < width; ++x){
|
||||
for(int y = 0; y < height; ++y){
|
||||
if( RNG.getRandom().nextDouble() < foodDensity){
|
||||
initialGrid[x][y] = new Cell(new Point(x,y), CellType.FREE, 1);
|
||||
}else{
|
||||
initialGrid[x][y] = new Cell(new Point(x,y), CellType.FREE);
|
||||
}
|
||||
initialGrid[x][y] = new Cell(new Point(x, y), CellType.FREE);
|
||||
}
|
||||
}
|
||||
start = new Point(RNG.getRandom().nextInt(width), RNG.getRandom().nextInt(height));
|
||||
initialGrid[start.x][start.y] = new Cell(new Point(start.x, start.y), CellType.START);
|
||||
spawnNewFood(initialGrid);
|
||||
spawnObstacles();
|
||||
}
|
||||
|
||||
//TODO
|
||||
private void spawnObstacles() {
|
||||
initialGrid[3][1].setType(CellType.OBSTACLE);
|
||||
initialGrid[4][1].setType(CellType.OBSTACLE);
|
||||
initialGrid[5][1].setType(CellType.OBSTACLE);
|
||||
initialGrid[6][1].setType(CellType.OBSTACLE);
|
||||
initialGrid[7][1].setType(CellType.OBSTACLE);
|
||||
initialGrid[3][2].setType(CellType.OBSTACLE);
|
||||
initialGrid[3][3].setType(CellType.OBSTACLE);
|
||||
initialGrid[3][4].setType(CellType.OBSTACLE);
|
||||
initialGrid[4][4].setType(CellType.OBSTACLE);
|
||||
initialGrid[5][4].setType(CellType.OBSTACLE);
|
||||
initialGrid[6][4].setType(CellType.OBSTACLE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawns one additional food on a random field EXCEPT for the starting position
|
||||
*/
|
||||
public void spawnNewFood(Cell[][] grid) {
|
||||
boolean foodSpawned = false;
|
||||
Point potFood = new Point(0, 0);
|
||||
CellType potFieldType;
|
||||
while(!foodSpawned) {
|
||||
potFood.x = RNG.getRandom().nextInt(width);
|
||||
potFood.y = RNG.getRandom().nextInt(height);
|
||||
potFieldType = grid[potFood.x][potFood.y].getType();
|
||||
if(potFieldType != CellType.START && grid[potFood.x][potFood.y].getFood() == 0 && potFieldType != CellType.OBSTACLE) {
|
||||
grid[potFood.x][potFood.y].setFood(1);
|
||||
foodSpawned = true;
|
||||
System.out.println("spawned new food at " + potFood);
|
||||
System.out.println(initialGrid[potFood.x][potFood.y]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void spawnNewFood() {
|
||||
spawnNewFood(grid);
|
||||
}
|
||||
|
||||
public Point getStartPoint(){
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ public class Reward {
|
|||
|
||||
public static final double FOOD_DROP_DOWN_FAIL_NO_FOOD = -1;
|
||||
public static final double FOOD_DROP_DOWN_FAIL_NOT_START = -1;
|
||||
public static final double FOOD_DROP_DOWN_SUCCESS = 1;
|
||||
public static final double FOOD_DROP_DOWN_SUCCESS = 40;
|
||||
|
||||
public static final double UNKNOWN_FIELD_EXPLORED = 1;
|
||||
public static final double UNKNOWN_FIELD_EXPLORED = 0;
|
||||
|
||||
public static final double RAN_INTO_WALL = -1;
|
||||
public static final double RAN_INTO_OBSTACLE = -1;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
package evironment.jumpingDino;
|
||||
|
||||
import core.State;
|
||||
import core.gui.Visualizable;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
import javax.swing.*;
|
||||
import java.awt.*;
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
public class DinoStateSimple implements State, Serializable, Visualizable {
|
||||
protected final double scale = 0.5;
|
||||
private int xDistanceToObstacle;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DinoState{" +
|
||||
"xDistanceToObstacle=" + xDistanceToObstacle +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if(this == o) return true;
|
||||
if(o == null || getClass() != o.getClass()) return false;
|
||||
DinoStateSimple dinoState = (DinoStateSimple) o;
|
||||
return xDistanceToObstacle == dinoState.xDistanceToObstacle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(xDistanceToObstacle);
|
||||
}
|
||||
|
||||
@Override
|
||||
public JComponent visualize() {
|
||||
return new JComponent() {
|
||||
{
|
||||
setPreferredSize(new Dimension(Config.FRAME_WIDTH, (int) (scale * Config.FRAME_HEIGHT)));
|
||||
setVisible(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void paintComponent(Graphics g) {
|
||||
super.paintComponents(g);
|
||||
drawObjects(g);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public void drawObjects(Graphics g) {
|
||||
g.setColor(Color.BLACK);
|
||||
g.fillRect(0, (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y)), Config.FRAME_WIDTH, 2);
|
||||
|
||||
g.fillRect((int) (scale * Config.DINO_STARTING_X), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.DINO_SIZE)), (int) (scale * Config.DINO_SIZE), (int) (scale * Config.DINO_SIZE));
|
||||
g.drawString("Distance: " + xDistanceToObstacle, (int) (scale * Config.DINO_STARTING_X), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE - 40)));
|
||||
|
||||
g.fillRect((int) (scale * (Config.DINO_STARTING_X + getXDistanceToObstacle())), (int) (scale * (Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE)), (int) (scale * Config.OBSTACLE_SIZE), (int) (scale * Config.OBSTACLE_SIZE));
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -76,7 +76,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
|
|||
}
|
||||
|
||||
protected State generateReturnState(){
|
||||
return new DinoState(getDistanceToObstacle(), dino.isInJump());
|
||||
return new DinoStateSimple(getDistanceToObstacle());
|
||||
}
|
||||
|
||||
protected void spawnNewObstacle(){
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ import java.awt.*;
|
|||
*
|
||||
* 350 states
|
||||
* if 4 speed variants
|
||||
*
|
||||
* 2044
|
||||
* 4 speeds, 4 distance
|
||||
*/
|
||||
public class DinoWorldAdvanced extends DinoWorld{
|
||||
public DinoWorldAdvanced(){
|
||||
|
|
@ -21,7 +24,7 @@ public class DinoWorldAdvanced extends DinoWorld{
|
|||
|
||||
@Override
|
||||
protected State generateReturnState() {
|
||||
return new DinoStateWithSpeed(getDistanceToObstacle(), dino.isInJump(), getCurrentObstacle().getDx());
|
||||
return new DinoStateWithSpeed(getDistanceToObstacle(), dino.isInJump(), currentObstacle.getDx());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -30,16 +33,26 @@ public class DinoWorldAdvanced extends DinoWorld{
|
|||
int xSpawn;
|
||||
double ran = RNG.getRandom().nextDouble();
|
||||
if(ran < 0.25){
|
||||
dx = -(int)(0.7 * Config.OBSTACLE_SPEED);
|
||||
dx = -(int) (0.35 * Config.OBSTACLE_SPEED);
|
||||
}else if(ran < 0.5){
|
||||
dx = -(int)(1.3 * Config.OBSTACLE_SPEED);
|
||||
dx = -(int) (0.7 * Config.OBSTACLE_SPEED);
|
||||
}else if(ran < 0.75){
|
||||
dx = -(int)(1.6 * Config.OBSTACLE_SPEED);
|
||||
} else{
|
||||
dx = -2 * Config.OBSTACLE_SPEED;
|
||||
dx = -(int) (3.5 * Config.OBSTACLE_SPEED);
|
||||
}
|
||||
double ran2 = RNG.getRandom().nextDouble();
|
||||
if(ran2 < 0.25) {
|
||||
// randomly spawning more right outside of the screen
|
||||
xSpawn = Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE;
|
||||
|
||||
} else if(ran2 < 0.5) {
|
||||
xSpawn = (int) (1.08 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
|
||||
} else if(ran2 < 0.75) {
|
||||
xSpawn = (int) (1.11 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
|
||||
} else {
|
||||
xSpawn = (int) (1.23 * Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE);
|
||||
}
|
||||
// randomly spawning more right outside of the screen
|
||||
xSpawn = Config.FRAME_WIDTH + Config.FRAME_WIDTH + Config.OBSTACLE_SIZE;
|
||||
currentObstacle = new Obstacle(Config.OBSTACLE_SIZE, xSpawn, Config.FRAME_HEIGHT - Config.GROUND_Y - Config.OBSTACLE_SIZE, dx, 0, Color.BLACK);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
package example;
|
||||
|
||||
import core.RNG;
|
||||
import core.algo.Method;
|
||||
import core.controller.RLController;
|
||||
import core.controller.RLControllerGUI;
|
||||
import evironment.antGame.AntAction;
|
||||
import evironment.antGame.AntWorldContinuous;
|
||||
|
||||
public class ContinuousAnt {
|
||||
public static void main(String[] args) {
|
||||
RNG.setSeed(56);
|
||||
RLController<AntAction> rl = new RLControllerGUI<>(
|
||||
new AntWorldContinuous(8, 8),
|
||||
Method.Q_LEARNING_OFF_POLICY_CONTROL,
|
||||
AntAction.values());
|
||||
|
||||
rl.setDelay(200);
|
||||
rl.setNrOfEpisodes(10000);
|
||||
rl.setDiscountFactor(0.95f);
|
||||
rl.setEpsilon(0.15f);
|
||||
|
||||
rl.start();
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,6 @@ import core.RNG;
|
|||
import core.algo.Method;
|
||||
import core.controller.RLController;
|
||||
import evironment.jumpingDino.DinoAction;
|
||||
import evironment.jumpingDino.DinoWorld;
|
||||
import evironment.jumpingDino.DinoWorldAdvanced;
|
||||
|
||||
import java.io.File;
|
||||
|
|
@ -34,7 +33,7 @@ public class DinoSampling {
|
|||
rl.setDiscountFactor(1f);
|
||||
rl.setEpsilon(f);
|
||||
rl.setLearningRate(1f);
|
||||
rl.setNrOfEpisodes(100000);
|
||||
rl.setNrOfEpisodes(50000);
|
||||
rl.start();
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,25 +3,23 @@ package example;
|
|||
import core.RNG;
|
||||
import core.algo.Method;
|
||||
import core.controller.RLController;
|
||||
import core.controller.RLControllerGUI;
|
||||
import evironment.jumpingDino.DinoAction;
|
||||
import evironment.jumpingDino.DinoWorld;
|
||||
import evironment.jumpingDino.DinoWorldAdvanced;
|
||||
|
||||
public class JumpingDino {
|
||||
public static void main(String[] args) {
|
||||
RNG.setSeed(55);
|
||||
RNG.setSeed(29);
|
||||
|
||||
RLController<DinoAction> rl = new RLControllerGUI<>(
|
||||
new DinoWorldAdvanced(),
|
||||
RLController<DinoAction> rl = new RLController<>(
|
||||
new DinoWorld(),
|
||||
Method.MC_CONTROL_FIRST_VISIT,
|
||||
DinoAction.values());
|
||||
|
||||
rl.setDelay(100);
|
||||
rl.setDelay(0);
|
||||
rl.setDiscountFactor(1f);
|
||||
rl.setEpsilon(0.15f);
|
||||
rl.setLearningRate(1f);
|
||||
rl.setNrOfEpisodes(1000000);
|
||||
rl.setNrOfEpisodes(30000000);
|
||||
rl.start();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,13 +12,13 @@ public class RunningAnt {
|
|||
RNG.setSeed(56);
|
||||
|
||||
RLController<AntAction> rl = new RLControllerGUI<>(
|
||||
new AntWorld(3, 3, 0.1),
|
||||
Method.MC_CONTROL_FIRST_VISIT,
|
||||
new AntWorld(8, 8),
|
||||
Method.Q_LEARNING_OFF_POLICY_CONTROL,
|
||||
AntAction.values());
|
||||
|
||||
rl.setDelay(200);
|
||||
rl.setNrOfEpisodes(10000);
|
||||
rl.setDiscountFactor(1f);
|
||||
rl.setDiscountFactor(0.9f);
|
||||
rl.setEpsilon(0.15f);
|
||||
|
||||
rl.start();
|
||||
|
|
|
|||
Loading…
Reference in New Issue