kono
/
refo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'epsilonBehavior' into DinoSampling 

# Conflicts:
#	src/main/java/example/DinoSampling.java
This commit is contained in:
Jan Löwenstrom 2020-04-05 12:07:39 +02:00
parent 28c40c58dd 3bdcbb39bc
commit 42dfebb048
11 changed files with 132 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.idea/misc.xml
View File

@ -11,7 +11,7 @@
</list> </list>
</option> </option>
</component> </component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="false" project-jdk-name="11.0.5" project-jdk-type="JavaSDK"> <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="false" project-jdk-name="11.0.3" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" /> <output url="file://$PROJECT_DIR$/out" />
</component> </component>
</project> </project>

28
EpsilonAnalysis.R Normal file
View File

@ -0,0 +1,28 @@
# Libraries
library(ggplot2)
library(matrixStats)
convergence <- read.csv(file.choose(), header=FALSE, row.names=1, skip=1)
sds <- rowSds(sapply(convergence[,-1], `length<-`, max(lengths(convergence[,-1]))), na.rm=TRUE)
men <- rowMeans(sapply(convergence[,-1], `length<-`, max(lengths(convergence[,-1]))), na.rm=TRUE)
print(sds)
# create dummy data
data <- data.frame(
names=rownames(convergence),
means=men,
sds=sds
)
#320 QMC
#188QMCMax
#234 MC
print(mean(men))
print(men)
convergence$group <- row.names(convergence)
convergence.m <- melt(convergence, id.vars = "groups")
ggplot(data) +
geom_bar(aes(x=names, y=means, fill=means), stat="identity", colour="black", alpha=0.8) +
geom_errorbar( aes(x=names, ymin=means, ymax=means+sds), width=0.4, colour="black", alpha=0.8, size=0.6) +
xlab("Epsilon") + ylab("?? Episoden bis Konvergenz") +
theme_bw(base_size = 24)

BIN
SimpleNoJumpEveryVisitMCA.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 78 KiB

21
convergeSimpleMCEveryNew.txt Normal file
View File

@ -0,0 +1,21 @@
#Every visit MC; just distance
0.05,36,29147,7945,707,12,595,68,4913,8034,300,804,1731,715,64899,423,703,2685,173,468,8220,4182,80071,1418,17769,216,3859,44746,218,6794,2,4182,35,258,9015,2581,62,39,17439,7346,1000,56974,21,71655,20301,4834,18144,18393,613,5,39,609,586,6,28871,1021,18051,457,13629,59217,10278,173,24407,1373,48513,31749,190,25159,130,609,609,55,924,77,4379,72708,202,2096,25965,14394,1230,2833,1791,25512,230,16227,16090,3781,76,537,229,307,2651,2,4212,3,38602,159722,703,45887,31
0.1,742,1269,9139,6764,782,25068,539,284,678,9327,2166,3469,3057,2971,1335,64551,22900,35519,62,29,15732,906,16,17,960,190,32183,2,40863,3324,37,6610,97,1578,12006,1784,3388,54,576,64,404,1638,95916,108,281,26288,1825,28,89,2359,49218,853,17821,10514,283,44669,256,6,12708,13085,3466,56628,8513,870,15,53906,20,1533,2164,1458,538,226,99,163,16,31398,76,2075,1474,36605,434,120,3744,91,2885,112,3285,69283,21176,3029,10113,2691,213,729,76694,9,1304,2,30950,74
0.15,1085,42379,2065,4847,124,27681,2257,1617,4597,1033,39095,254,120,1343,2007,1907,2992,15207,95,7835,495,16224,362,97541,141,8692,32124,1302,1768,10853,480,2,2099,25,1889,66,67,11874,290,77899,7253,31,1190,72,95,141,7732,32,26,16,5,181,361,12321,22,1631,7180,18,40,310,95444,7146,102,5210,6,19593,434,3581,3120,35,14,980,19,3816,677,1494,337,68,2764,53854,147,28,732,23755,270,4818,15,3046,1557,117,13049,323,26709,2036,431,85,113,179178,13782,705
0.2,11,119,8993,10,949,1495,20497,1325,5672,39392,35828,233,56,2707,456,12,3630,112,257,936,108,618,2353,52,2563,1632,10094,12336,24,1390,11,1130,4381,78,14658,1228,514,4706,215,15898,100,639,110,723,340,21,25,836,4,83,922,90321,22,747,11599,88,13,11,2101,43,642,1272,1742,169,614,832,681,41884,17,3159,392,9,178,2183,3271,10,378,69,35741,293,63,2733,3825,745,9,3151,2934,588,214,30150,1138,1304,98,566,1497,4474,24,944,1593,563
0.25,21,222,230,10720,108,2978,2377,1398,3310,10274,27275,1674,22,720,293,52,863,16756,76,520,28985,247,195,5507,97,5916,96,11,1346,77,157,8,14,501,40,1502,32,58,561,8,327,74,91,20,5444,914,4278,30,27,399,2186,4,75,1149,2386,22,3415,3918,56,438,8,160,451,18150,520,329,33850,16,625,128,463,73,147,33074,2545,198,1210,637,1143,180,824,171,293,16,9592,699,8,929,664,1637,2524,18383,256,31,540,233,265,998,9667,685
0.3,15,2847,8254,2332,2921,62,61322,1967,2375,4488,1142,1502,83,5177,519,128,7727,58,25,552,365,119,141,11859,7,9742,8,754,415,1534,641,8,498,12364,38,1062,75,336,3247,8,540,683,13,396,9242,194,24,935,62,83,611,4,5,5047,3134,243,1647,3294,96,257,9,1777,38112,6059,118,1702,4305,48,25907,113,715,8233,797,1721,147,39342,1111,26,11991,516,3292,24,126,1012,3290,305,5371,145,11939,65779,9057,3241,76,6924,2896,266,4380,902,634,577
0.35,68,47,70,1307,4155,1031,15740,175,829,920,2649,599,214,494,544,5026,1647,2007,48,182,294,670,156,613,20,1161,250,1710,4153,270,78,81,17,81,4816,5955,87,428,189,8,235,225,3151,7884,205,1322,25,31,787,15,3728,4,5,1314,3193,454,1175,167,176,7596,49,74,3110,389,385,2940,317,17,3846,94,99,202,88,22,1957,749,2767,57,334,326,27205,1928,14,285,3293,43,6288,43,643,36225,58,2835,1126,8335,1101,1874,2916,391,2455,13320
0.40,430,2482,4292,2558,7285,165,295,845,3480,43,2391,3,269,448,886,7400,6280,704,61,178,2706,4046,1089,139,947,855,46,415,1668,6922,235,22,5447,19,401,706,67,5379,73,527,178,3307,1254,996,276,4191,760,554,1140,15987,175,4,22,7793,4102,129,583,1056,228,27992,615,51,32,259,1508,867,2,1314,170,1135,390,3982,69,1228,1556,324,288,4165,504,504,173,313,85,2101,390,62,26544,468,793,1533,63,1838,167,3753,4042,402,6142,21336,841,72251
0.45,684,893,1075,3710,10131,9449,999,337,136,3812,80,3,3113,132,11540,277,639,315,74,55,2061,53,23,3426,513,4798,60,37,7000,76680,2757,22,1241,87,1207,4540,49,4373,48,10391,561,1698,214,507,382,251,245,998,142,1680,290,4,21,390,13326,209,916,2873,108,38,417,16,13,503,247,4716,8,23,4030,1862,104,681,114,40,638,647,3693,22959,90,249,85,485,1448,346,1575,524,429,3365,489,1124,177,3188,188,1880,110,263,108,34,408,25180
0.50,42,16830,1941,220,3775,815,2031,498,3436,2897,1061,3,2619,192,1034,672,14834,214,125,3387,1480,345,32,4599,682,350,45,572,863,45168,66,186,231,4381,596,1034,723,3776,10695,110,3889,975,809,86,579,16,40,228,180,259,54,4,1241,649,22145,372,671,1055,2444,1949,61,229,17,2392,266,1586,1820,24,2182,3007,124,270,233,46,1526,295,1276,996,352,1225,17,341,271,512,3197,474,3692,8,888,581,1091,71,4663,87,4268,1127,3638,1529,661,418
0.55,3815,777,987,185,27,1230,1594,2210,312,303,15,2,2256,3099,7131,134,3533,158,5467,1954,3945,160,281,1095,216,676,60,6476,429,267,1781,966,1156,2818,3082,16615,233,3614,95,1157,2991,1286,78,1461,783,16,40,1439,5110,22,322,4,878,5,111,1464,249,6612,631,10027,145,1804,1304,2465,992,196,32,13,204,1639,255,1151,409,6002,331,585,1023,2579,1625,730,16,249,929,265,782,327,3763,142,24,4778,130,27170,653,30,64,2757,7624,10265,242,782
0.60,72,219,1014,1553,13880,289,3735,1411,4136,112,3301,2,4284,1165,7631,675,1641,3996,728,1337,127,33,2489,3893,429,171,5,3243,128,5598,1049,1360,831,862,178,463,113,86,3557,1211,40,1309,185,34750,75,15,28,6889,743,49,15,4,582,3927,61,83,841,1639,4,984,195,2585,50,2949,2506,7500,33,14,901,2253,267,1783,254,2056,110,225,4108,3027,255,1567,24,905,91,3740,500,2585,2344,2102,160,468,258,3133,3619,5072,2158,2844,2325,343,18,2851
0.65,259,481,1669,5179,15,2300,180,118,200,5180,238,2,15,1424,6185,1967,149,291,9749,666,1616,680,3145,96,650,5056,1247,1082,310,374,1614,937,563,322,1963,161,22,38,78,821,206,1006,390,319,1613,1992,1230,535,96,320,17,4,3067,6450,552,991,17929,3373,28,204,47,143,73,632,2433,2131,78,14,412,3296,782,4784,1067,629,2132,4082,9844,6593,2112,412,401,19872,1336,368,2487,319,814,27,170,1362,100,1678,127,1800,6572,5779,693,1565,2088,798
0.70,1007,185,1973,2285,15,28,406,1468,382,1237,11692,2,55,1204,172,1042,3029,583,47,1430,2597,3431,2986,3363,4073,355,21980,1160,916,6140,893,535,13,418,578,535,3,47,3353,9003,816,3294,393,2578,2874,4252,1128,786,381,207,10,4,6397,895,2210,4982,763,54,11,1537,2,712,26,8748,123,3861,65,958,811,62,116,429,409,394,1029,1611,7386,298,1518,1322,3053,1359,5281,385,3349,4487,1939,5949,533,1506,3234,1492,2744,4,2985,7713,1492,3836,7254,342
0.75,6523,317,3968,233,15,80,2863,4066,16,3262,883,704,117,5443,2409,59,3848,3450,65,87,3764,1652,62,1646,2864,512,1226,1816,3587,378,4243,1837,10,5908,16119,4734,5,328,2639,897,353,6208,4281,1671,1520,610,451,399,3650,2542,10,3,23,66,245,2607,58,470,4,1989,2,60,3256,3603,4903,568,743,251,14701,2671,4777,698,176,26,1302,787,3275,2738,1335,2634,8791,5806,2222,13223,311,4218,8479,5535,8898,189,2425,1919,2482,4,1536,3469,2201,345,390,5228
0.80,83,2976,2164,17000,16090,4483,13563,987,14,505,282,3317,2053,387,12252,247,3519,4557,378,71,756,910,1921,2874,1862,1829,731,54,234,715,682,10015,530,633,3911,4709,265,116,104,9814,1312,601,61,19856,1102,595,10881,3728,3633,3388,9,3,811,1999,4075,2106,1888,2582,4,12564,154,383,15,2438,436,824,947,6212,3273,73,772,49,2818,11875,4322,4457,3292,3180,170,55,4707,100,5989,1835,4546,5414,808,65,7,811,1783,19664,219,4,248,1953,996,32118,7,257
0.85,2075,497,11265,1960,3624,34,366,957,14,2817,4413,18269,5069,2370,2272,12272,330,2063,4130,6284,3628,9,1938,613,15316,3474,5446,100,3894,2899,67,892,4003,348,135,1151,5953,6105,6149,5830,2078,1831,4313,626,269,877,3822,19,1106,139,9,3,18364,1119,8355,22487,3265,1415,5,13005,781,27358,728,1977,616,6789,341,7050,2368,24,1281,1913,15870,1985,619,7965,5509,538,620,2012,2022,10890,20395,1500,12,6141,8327,107,7,8407,109,45,100,4,874,632,2101,396,7,5154
0.90,902,3427,5470,42978,9193,2969,5514,2288,91,2048,1085,982,16,4210,3702,103,14119,5531,1076,2111,1360,16,10217,4417,544,607,1414,5350,1027,9041,3697,12502,43926,105,2941,31782,995,15661,773,338,153,19508,11985,33683,3006,4043,2942,2541,3817,2281,8561,6771,2305,37689,9198,19,899,13884,1818,12144,319,416,250,11815,44066,4825,783,4261,286,162,13125,138,3183,627,609,1736,1554,989,7559,3504,6641,3354,47,972,11,14096,278,15664,7,15571,389,286,1367,4,2434,25,297,3388,7,2743
0.95,493,6960,769,1563,660,4208,778,8917,98,72,21014,100,6872,7998,26519,154,4073,178,11475,3714,184,9,7250,2416,1161,38061,85,3569,25036,7121,30470,300,270,8458,668,5242,610,21,2330,15540,3238,2429,8334,103,3945,6469,166,581,27569,8,3,7695,4772,212,4868,15107,11469,132,2466,1252,4425,21010,6871,2827,155,4470,59064,5478,504,38380,5933,18,4313,4131,26218,5916,1479,4514,14507,64879,4715,2885,11,7370,2070,4241,7,11518,6675,3896,6503,26757,38,15158,1246,63,7,86,18039,5131
1.00,5753,31823,1131,5008,814,2498,3289,4148,1189,3345,455,16054,1727,4275,8701,177,354,21971,2838,2195,9,7887,682,34,1657,23710,296,29250,5562,79,7168,176,369,9651,140,19561,4518,6518,3711,20169,4594,3547,809,14814,220,774,16,2280,3,318,1921,6150,254,8587,7848,23995,2323,7108,2348,58820,3945,62059,984,10839,10909,28613,8046,20249,12086,4325,9060,176,86,47797,1107,119,12405,8481,12,4521,683,92,419,1714,584,596,102,47669,23214,665,98,8,486,2610,4602,98,3954,11155,3,14700

21
convergenceSimplyNew.txt Normal file
View File

@ -0,0 +1,21 @@
# New faster MC
0.05,3618,75,1959,674,653,1038,2851,96,774,820,83,726,2589,1424,171,99,29,484,32,2075,1906,2276,3147,94,1798,60,393,272,88,104,2763,1052,15,2521,1633,91,481,1029,3664,360,201,255,2775,496,84,15,6007,423,9,125,242,1393,80,1705,305,109,1492,215,161,102,1220,228,1157,463,1361,132,381,190,2388,4683,4670,313,140,3836,2942,1630,117,7794,26,303,291,1616,3593,418,240,44,364,17,241,2622,359,2833,278,5275,1481,1776,482,1513,567,189
0.1,333,2264,357,322,3972,270,844,546,52,838,1245,911,174,916,34,1053,944,921,68,1138,83,1652,303,638,334,406,476,185,298,213,1666,845,15,212,369,5,463,195,2294,550,155,249,2875,64,219,15,569,120,9,535,43,41,2609,2955,6,875,848,114,434,77,630,3697,33,3,897,500,343,23,117,1844,129,54,39,101,531,156,109,77,1420,21,1137,48,15,161,725,1589,44,317,2383,2432,557,140,816,482,1493,74,1008,331,210,74
0.15,324,460,1804,322,918,155,401,2090,52,62,23,377,98,397,35,112,41,901,32,396,283,111,46,638,2008,96,729,1521,79,354,67,54,131,46,482,5,636,9,474,64,226,56,28,64,1812,54,125,400,1147,40,268,65,744,158,6,105,45,77,267,271,207,364,760,3,370,734,60,31,14,115,80,39,39,443,133,70,748,27,321,68,720,135,210,153,1101,228,94,149,2324,167,1332,114,219,212,891,195,13,41,475,74
0.2,1929,64,352,105,1092,2078,81,182,52,358,23,603,111,405,160,55,170,513,77,469,65,50,33,103,588,90,121,54,1462,337,77,388,100,96,257,5,99,9,1093,64,207,136,89,15,57,15,76,61,278,57,183,101,1792,334,6,875,26,98,171,285,201,421,760,3,403,355,316,24,14,886,549,17,21,1102,1125,15,784,19,278,59,166,659,610,336,237,109,266,91,73,209,406,123,298,316,354,43,13,1066,1887,417
0.25,590,50,302,223,160,91,83,182,52,387,23,341,95,192,160,52,144,672,114,453,65,100,33,103,83,9,279,142,107,192,546,165,50,63,253,5,99,9,242,142,264,134,89,15,32,15,182,61,610,39,183,50,2132,358,6,875,26,156,400,161,152,190,190,3,383,355,25,24,40,633,32,17,34,52,806,15,88,33,53,114,953,1426,523,10,229,205,284,121,118,145,204,106,359,632,592,60,18,239,2,407
0.3,195,50,176,75,175,65,909,182,58,211,23,637,859,132,160,52,28,469,24,1094,65,523,54,103,93,607,278,985,65,224,947,370,62,36,253,5,99,9,345,142,166,134,89,15,47,1057,189,624,129,52,183,130,623,343,6,96,26,98,399,136,98,139,109,3,265,412,126,59,33,549,386,24,124,42,685,136,39,26,55,52,74,663,9,96,211,205,206,121,7,1099,482,1482,371,218,125,61,598,737,2,407
0.35,192,50,21,87,166,102,153,182,381,251,184,67,324,169,86,52,28,16,24,375,65,82,347,159,496,44,264,137,125,48,7,311,124,36,212,5,99,9,311,42,113,55,87,15,32,163,252,128,205,52,60,126,93,532,6,96,15,95,348,248,98,152,105,3,287,193,503,36,75,472,378,10,39,99,1127,127,39,34,130,78,74,263,9,96,85,226,56,26,7,762,34,56,56,511,119,83,160,247,2,130
0.40,135,26,52,322,108,130,321,257,166,349,473,59,12,553,125,52,28,125,151,101,98,69,59,215,98,50,84,143,261,279,7,311,395,44,212,30,99,9,127,48,70,55,118,37,86,95,202,674,624,109,85,238,152,97,6,96,14,64,348,111,98,118,44,3,287,60,503,25,76,123,107,10,241,42,1127,112,72,27,6,78,76,33,820,10,163,85,164,379,7,198,69,168,58,166,72,102,616,135,2,150
0.45,143,50,12,48,156,130,140,240,204,339,112,45,12,56,125,52,32,131,186,158,58,69,8,141,126,76,78,28,33,337,195,164,139,101,212,21,99,9,232,48,138,26,89,107,284,120,81,668,722,12,60,58,76,33,6,109,17,128,106,111,98,243,80,74,395,44,456,25,102,123,107,63,221,40,59,98,61,27,30,74,304,28,360,10,42,85,191,379,7,198,56,156,156,43,115,82,13,177,2,145
0.50,127,48,43,48,156,48,403,167,71,116,61,51,12,196,56,52,32,31,26,92,9,223,9,60,102,9,88,90,33,337,195,112,22,101,212,18,186,9,302,49,124,56,57,107,144,127,115,46,520,12,60,135,76,18,6,157,95,112,301,109,98,136,84,100,8,124,277,36,476,143,408,97,157,40,231,71,61,51,116,97,158,74,490,10,75,85,189,248,7,126,56,281,101,252,8,89,13,228,2,145
0.55,112,19,33,48,64,73,136,92,71,141,122,37,12,82,23,301,253,31,26,36,76,99,80,60,319,9,88,27,36,177,45,89,210,51,212,10,185,9,84,14,140,375,69,11,148,83,81,79,520,89,60,116,82,118,6,153,106,194,329,738,98,47,89,119,8,129,4,40,205,255,277,200,96,81,77,23,92,90,29,28,217,24,165,10,27,119,189,248,7,76,106,41,58,67,8,89,75,23,2,116
0.60,43,96,38,243,25,84,219,92,242,95,47,37,12,335,23,147,7,31,26,38,100,109,117,60,337,260,80,16,98,79,62,84,15,51,262,10,198,73,40,14,146,51,77,11,52,80,56,135,55,64,60,72,72,71,6,124,13,222,9,251,190,47,89,119,8,39,4,6,141,109,425,197,146,76,128,53,20,66,53,11,188,100,157,177,277,70,208,248,7,76,195,53,58,110,8,135,121,23,2,116
0.65,49,120,30,233,20,30,161,60,74,95,44,37,12,186,23,52,6,31,26,54,40,127,267,60,339,214,80,86,98,105,99,84,15,68,57,8,145,57,47,14,42,228,431,11,58,305,81,136,55,53,60,68,72,62,6,124,13,73,9,146,220,47,138,51,109,95,35,6,8,65,425,261,233,51,236,437,20,32,46,47,218,79,107,48,7,21,425,69,7,76,208,82,58,151,8,149,128,104,2,134
0.70,193,18,69,113,20,89,108,103,152,29,47,63,12,186,28,101,7,184,26,203,40,90,87,60,305,156,209,57,104,168,77,24,15,50,208,8,145,9,76,14,42,200,458,11,58,132,64,134,55,64,60,130,272,75,6,41,13,73,9,185,154,42,71,170,9,207,89,6,20,59,135,178,233,51,309,274,20,173,58,156,218,79,173,52,7,40,200,85,321,116,74,42,58,171,8,62,67,160,2,157
0.75,89,198,69,169,20,38,286,103,136,29,36,161,12,220,28,101,7,201,37,5,40,90,194,20,118,174,73,42,55,168,61,24,15,72,302,311,145,9,76,14,42,10,96,11,93,128,45,175,55,67,60,130,41,65,6,41,13,73,9,67,311,155,226,67,9,107,67,91,59,59,135,152,182,51,102,126,20,173,58,70,181,94,82,174,7,40,365,107,100,79,240,42,58,171,8,316,12,94,2,151
0.80,246,49,31,202,20,32,221,103,41,29,43,32,99,58,28,157,7,63,37,5,40,90,194,122,434,270,114,21,171,152,110,24,15,72,215,82,242,9,19,14,42,148,96,15,44,59,356,146,61,41,60,48,41,65,99,49,13,73,9,96,68,89,194,67,9,198,126,37,16,19,180,240,28,298,128,115,20,68,55,49,126,81,149,154,7,36,272,124,76,79,2,59,58,165,8,224,12,84,2,151
0.85,88,143,31,328,20,58,129,133,80,99,122,32,200,56,28,214,7,185,312,5,40,109,357,104,238,300,337,23,248,207,160,24,15,19,116,361,168,9,305,14,204,92,96,11,151,66,168,138,61,26,60,39,258,74,99,138,13,73,9,129,260,176,93,359,9,78,178,107,16,19,108,187,28,501,366,128,20,23,36,68,181,189,123,190,6,36,434,45,125,79,2,56,58,112,8,48,12,36,2,514
0.90,206,77,31,230,20,183,59,154,47,141,336,32,200,56,103,112,7,77,232,5,211,402,49,516,270,350,361,225,213,40,255,24,15,19,138,73,402,9,354,87,23,217,96,11,70,134,185,86,61,26,196,39,177,102,53,59,13,63,9,102,248,80,276,61,9,223,430,6,16,19,436,456,28,171,266,23,20,23,148,17,210,96,64,412,6,86,399,703,147,79,2,61,4,42,8,63,20,80,2,130
0.95,206,134,31,204,104,263,59,196,47,223,157,130,199,56,103,112,7,93,266,5,65,42,49,88,269,331,271,160,143,31,43,24,15,19,342,359,150,9,92,61,23,116,97,11,223,134,212,169,127,191,124,41,89,65,53,186,13,95,9,102,228,210,217,116,9,306,637,6,16,19,157,109,28,207,109,51,20,23,358,17,91,217,210,176,6,158,321,77,227,75,2,18,4,54,8,268,340,35,2,304
1.00,25,180,84,257,471,116,40,386,47,329,60,103,368,56,164,141,7,66,169,152,68,184,49,502,379,9,212,114,138,31,43,24,15,22,306,523,159,9,209,266,23,116,92,16,102,47,438,139,97,154,63,235,105,65,63,61,13,918,9,224,21,169,162,116,303,242,115,6,16,19,738,238,28,268,126,218,20,23,58,173,91,155,53,269,6,295,347,190,287,46,2,189,4,42,257,65,271,35,2,252

4
src/main/java/core/algo/EpisodicLearning.java
View File

@ -5,7 +5,6 @@ import core.Environment;
import core.LearningConfig; import core.LearningConfig;
import core.StepResult; import core.StepResult;
import core.listener.LearningListener; import core.listener.LearningListener;
import core.policy.EpsilonGreedyPolicy;
import example.DinoSampling; import example.DinoSampling;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@ -105,7 +104,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
timestamp++; timestamp++;
timestampCurrentEpisode++; timestampCurrentEpisode++;
// TODO: more sophisticated way to check convergence // TODO: more sophisticated way to check convergence
if(timestampCurrentEpisode > 30000000){ if(timestampCurrentEpisode > 50000) {
converged = true; converged = true;
// t // t
File file = new File(DinoSampling.FILE_NAME); File file = new File(DinoSampling.FILE_NAME);
@ -128,6 +127,7 @@ public abstract class EpisodicLearning<A extends Enum> extends Learning<A> imple
private void startLearning(){ private void startLearning(){
dispatchLearningStart(); dispatchLearningStart();
while(episodesToLearn.get() > 0){ while(episodesToLearn.get() > 0){
dispatchEpisodeStart(); dispatchEpisodeStart();
timestampCurrentEpisode = 0; timestampCurrentEpisode = 0;
nextEpisode(); nextEpisode();

2
src/main/java/core/algo/Method.java
View File

@ -5,5 +5,5 @@ package core.algo;
* which RL-algorithm should be used. * which RL-algorithm should be used.
*/ */
public enum Method { public enum Method {
MC_CONTROL_FIRST_VISIT, SARSA_EPISODIC, Q_LEARNING_OFF_POLICY_CONTROL MC_CONTROL_FIRST_VISIT, MC_CONTROL_EVERY_VISIT, SARSA_ON_POLICY_CONTROL, Q_LEARNING_OFF_POLICY_CONTROL
} }

88
src/main/java/core/algo/mc/MonteCarloControlFirstVisitEGreedy.java
View File

@ -8,37 +8,22 @@ import core.policy.Policy;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import java.io.*; import java.io.IOException;
import java.net.URI; import java.io.ObjectInputStream;
import java.nio.file.Files; import java.io.ObjectOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*; import java.util.*;
/** /**
* TODO: Major problem: * Includes both variants of Monte-Carlo methods
* StateActionPairs are only unique accounting for their position in the episode. * Default method is First-Visit.
* For example: * Change to Every-Visit by setting flag "useEveryVisit" in the constructor to true.
* <p>
* startingState -> MOVE_LEFT : very first state action in the episode i = 1
* image the agent does not collect the food and does not drop it onto start, the agent will receive
* -1 for every timestamp hence (startingState -> MOVE_LEFT) will get a value of -10;
* <p>
* BUT image moving left from the starting position will have no impact on the state because
* the agent ran into a wall. The known world stays the same.
* Taking an action after that will have the exact same state but a different action
* making the value of this stateActionPair -9 because the stateAction pair took place on the second
* timestamp, summing up all remaining rewards will be -9...
* <p>
* How to encounter this problem?
*
* @param <A> * @param <A>
*/ */
public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends EpisodicLearning<A> { public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends EpisodicLearning<A> {
private Map<Pair<State, A>, Double> returnSum; private Map<Pair<State, A>, Double> returnSum;
private Map<Pair<State, A>, Integer> returnCount; private Map<Pair<State, A>, Integer> returnCount;
private boolean isEveryVisit;
// t // t
private float epsilon; private float epsilon;
@ -46,8 +31,9 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
private Policy<A> greedyPolicy = new GreedyPolicy<>(); private Policy<A> greedyPolicy = new GreedyPolicy<>();
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) { public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay, boolean useEveryVisit) {
super(environment, actionSpace, discountFactor, delay); super(environment, actionSpace, discountFactor, delay);
isEveryVisit = useEveryVisit;
// t // t
this.epsilon = epsilon; this.epsilon = epsilon;
this.policy = new EpsilonGreedyPolicy<>(epsilon); this.policy = new EpsilonGreedyPolicy<>(epsilon);
@ -56,6 +42,10 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
returnCount = new HashMap<>(); returnCount = new HashMap<>();
} }
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, float discountFactor, float epsilon, int delay) {
this(environment, actionSpace, discountFactor, epsilon, delay, false);
}
public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) { public MonteCarloControlFirstVisitEGreedy(Environment<A> environment, DiscreteActionSpace<A> actionSpace, int delay) {
this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay); this(environment, actionSpace, LearningConfig.DEFAULT_DISCOUNT_FACTOR, LearningConfig.DEFAULT_EPSILON, delay);
} }
@ -104,35 +94,47 @@ public class MonteCarloControlFirstVisitEGreedy<A extends Enum> extends Episodic
} }
// System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards); // System.out.printf("Episode %d \t Reward: %f \n", currentEpisode, sumOfRewards);
Set<Pair<State, A>> stateActionPairs = new LinkedHashSet<>(); HashMap<Pair<State, A>, List<Integer>> stateActionPairs = new LinkedHashMap<>();
int firstOccurrenceIndex = 0;
for(StepResult<A> sr : episode) { for(StepResult<A> sr : episode) {
stateActionPairs.add(new ImmutablePair<>(sr.getState(), sr.getAction())); Pair<State, A> pair = new ImmutablePair<>(sr.getState(), sr.getAction());
if(!stateActionPairs.containsKey(pair)) {
List<Integer> l = new ArrayList<>();
l.add(firstOccurrenceIndex);
stateActionPairs.put(pair, l);
}
/*
This is the only difference between First-Visit and Every-Visit.
When First-Visit is selected, only the first index of the occurrence is put into the list.
When Every-Visit is selected, every following occurrence is saved
into the list as well.
*/
else if(isEveryVisit) {
stateActionPairs.get(pair).add(firstOccurrenceIndex);
}
++firstOccurrenceIndex;
} }
//System.out.println("stateActionPairs " + stateActionPairs.size()); //System.out.println("stateActionPairs " + stateActionPairs.size());
for(Pair<State, A> stateActionPair : stateActionPairs) { for(Map.Entry<Pair<State, A>, List<Integer>> entry : stateActionPairs.entrySet()) {
int firstOccurenceIndex = 0; Pair<State, A> stateActionPair = entry.getKey();
// find first occurance of state action pair List<Integer> firstOccurrences = entry.getValue();
for(StepResult<A> sr : episode) { for(Integer firstOccurrencesIdx : firstOccurrences) {
if(stateActionPair.getKey().equals(sr.getState()) && stateActionPair.getValue().equals(sr.getAction())) { double G = 0;
break; for(int l = firstOccurrencesIdx; l < episode.size(); ++l) {
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurrencesIdx));
} }
firstOccurenceIndex++; // slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
} }
double G = 0;
for(int l = firstOccurenceIndex; l < episode.size(); ++l) {
G += episode.get(l).getReward() * (Math.pow(discountFactor, l - firstOccurenceIndex));
}
// slick trick to add G to the entry.
// if the key does not exists, it will create a new entry with G as default value
returnSum.merge(stateActionPair, G, Double::sum);
returnCount.merge(stateActionPair, 1, Integer::sum);
stateActionTable.setValue(stateActionPair.getKey(), stateActionPair.getValue(), returnSum.get(stateActionPair) / returnCount.get(stateActionPair));
} }
} }
@Override @Override
public void save(ObjectOutputStream oos) throws IOException { public void save(ObjectOutputStream oos) throws IOException {
super.save(oos); super.save(oos);

6
src/main/java/core/controller/RLController.java
View File

@ -51,7 +51,11 @@ public class RLController<A extends Enum> implements LearningListener {
case MC_CONTROL_FIRST_VISIT: case MC_CONTROL_FIRST_VISIT:
learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay); learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay);
break; break;
case SARSA_EPISODIC: case MC_CONTROL_EVERY_VISIT:
learning = new MonteCarloControlFirstVisitEGreedy<>(environment, discreteActionSpace, discountFactor, epsilon, delay, true);
break;
case SARSA_ON_POLICY_CONTROL:
learning = new SARSA<>(environment, discreteActionSpace, discountFactor, epsilon, learningRate, delay); learning = new SARSA<>(environment, discreteActionSpace, discountFactor, epsilon, learningRate, delay);
break; break;
case Q_LEARNING_OFF_POLICY_CONTROL: case Q_LEARNING_OFF_POLICY_CONTROL:

4
src/main/java/evironment/jumpingDino/DinoWorld.java
View File

@ -44,7 +44,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
@Override @Override
public StepResultEnvironment step(DinoAction action) { public StepResultEnvironment step(DinoAction action) {
boolean done = false; boolean done = false;
int reward = 0; int reward = 1;
if(action == DinoAction.JUMP){ if(action == DinoAction.JUMP){
dino.jump(); dino.jump();
@ -68,7 +68,7 @@ public class DinoWorld implements Environment<DinoAction>, Visualizable {
spawnNewObstacle(); spawnNewObstacle();
} }
if(ranIntoObstacle()) { if(ranIntoObstacle()) {
reward = -1; reward = 0;
done = true; done = true;
} }

14
src/main/java/example/DinoSampling.java
View File

@ -3,9 +3,7 @@ package example;
import core.RNG; import core.RNG;
import core.algo.Method; import core.algo.Method;
import core.controller.RLController; import core.controller.RLController;
import core.controller.RLControllerGUI;
import evironment.jumpingDino.DinoAction; import evironment.jumpingDino.DinoAction;
import evironment.jumpingDino.DinoWorld;
import evironment.jumpingDino.DinoWorldAdvanced; import evironment.jumpingDino.DinoWorldAdvanced;
import java.io.File; import java.io.File;
@ -16,7 +14,7 @@ import java.nio.file.StandardOpenOption;
public class DinoSampling { public class DinoSampling {
public static final float f =0.05f; public static final float f =0.05f;
public static final String FILE_NAME = "convergenceAdvancedMCnegRew.txt"; public static final String FILE_NAME = "advancedEveryVisit.txt";
public static void main(String[] args) { public static void main(String[] args) {
File file = new File(FILE_NAME); File file = new File(FILE_NAME);
try { try {
@ -24,24 +22,24 @@ public class DinoSampling {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
for(float f = 0.05f; f <=1.003 ; f+=0.05f) { for(float f = 0.05f; f <= 1.003; f += 0.05f) {
try { try {
Files.writeString(Path.of(file.getPath()), f + ",", StandardOpenOption.APPEND); Files.writeString(Path.of(file.getPath()), f + ",", StandardOpenOption.APPEND);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
for (int i = 1; i <= 100; i++) { for(int i = 1; i <= 100; i++) {
System.out.println("seed: " + i * 13); System.out.println("seed: " + i * 13);
RNG.setSeed(i * 13); RNG.setSeed(i * 13);
RLController<DinoAction> rl = new RLController<>( RLController<DinoAction> rl = new RLController<>(
new DinoWorldAdvanced(), new DinoWorldAdvanced(),
Method.Q_LEARNING_OFF_POLICY_CONTROL, Method.MC_CONTROL_EVERY_VISIT,
DinoAction.values()); DinoAction.values());
rl.setDelay(0); rl.setDelay(0);
rl.setDiscountFactor(0.99f); rl.setDiscountFactor(1f);
rl.setEpsilon(f); rl.setEpsilon(f);
rl.setLearningRate(0.9f); rl.setLearningRate(1f);
rl.setNrOfEpisodes(400000); rl.setNrOfEpisodes(400000);
rl.start(); rl.start();
} }