diff --git a/AdvancedZ3B1MonteCarloA.png b/AdvancedZ3B1MonteCarloA.png new file mode 100644 index 0000000..e3b88b2 Binary files /dev/null and b/AdvancedZ3B1MonteCarloA.png differ diff --git a/AdvancedZ3B3MonteA.png b/AdvancedZ3B3MonteA.png new file mode 100644 index 0000000..20926c7 Binary files /dev/null and b/AdvancedZ3B3MonteA.png differ diff --git a/DiscAndLRonQLearningAdvanced.R b/DiscAndLRonQLearningAdvanced.R index 6df2cd4..f840762 100644 --- a/DiscAndLRonQLearningAdvanced.R +++ b/DiscAndLRonQLearningAdvanced.R @@ -26,31 +26,31 @@ data <- data.frame( x=seq(1, length(ta[,1])) ) ggplot(data, aes(x*1000)) + - labs(title="Diskontierungsfaktor = 0.99", - x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + + labs( x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + ylim(-1.5,0.6) + geom_line(aes(y = y, colour = "0.1"), size=1)+ geom_line(aes(y = y2, colour = "0.3"), size=1) + geom_line(aes(y = y3, colour = "0.5"), size=1) + geom_line(aes(y = y4, colour = "0.7"), size=1) + - geom_line(aes(y = y5, colour = "0.9"), size=1) + geom_line(aes(y = y5, colour = "0.9"), size=1) + + theme_bw(base_size = 24) ggplot(data, aes(x*1000)) + - labs(title="Diskontierungsfaktor = 0.9", - x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + + labs(x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + ylim(-1.5,0.6) + geom_line(aes(y = y6, colour = "0.1"), size=1) + geom_line(aes(y = y7, colour = "0.3"), size=1) + geom_line(aes(y = y8, colour = "0.5"), size=1) + geom_line(aes(y = y9, colour = "0.7"), size=1) + - geom_line(aes(y = y10, colour = "0.9"), size=1) + geom_line(aes(y = y10, colour = "0.9"), size=1) + + theme_bw(base_size = 24) ggplot(data, aes(x*1000) ) + - labs(title="Diskontierungsfaktor = 0.5", - x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + + labs(x ="Zeitstempel", y = "Ø Belohnung pro Zeitstempel", color = "Lernrate") + ylim(-1.5,0.6) + geom_line(aes(y = y11, colour = "0.1"), size=1) + geom_line(aes(y = y12, colour = "0.3"), size=1) + geom_line(aes(y = y13, colour = "0.5"), size=1) + geom_line(aes(y = y14, colour = "0.7"), size=1) + - geom_line(aes(y = y15, colour = "0.9"), size=1) \ No newline at end of file + geom_line(aes(y = y15, colour = "0.9"), size=1) + + theme_bw(base_size = 24) \ No newline at end of file diff --git a/EpsilonAnalysis.R b/EpsilonAnalysis.R index 3190686..2e260ab 100644 --- a/EpsilonAnalysis.R +++ b/EpsilonAnalysis.R @@ -20,4 +20,6 @@ convergence.m <- melt(convergence, id.vars = "groups") ggplot(data) + geom_bar(aes(x=names, y=means, fill=means), stat="identity", colour="black", alpha=0.8) + geom_errorbar( aes(x=names, ymin=means, ymax=means+sds), width=0.4, colour="black", alpha=0.8, size=0.6) + - xlab("Epsilon") + ylab("Ø Episoden bis Konvergenz") \ No newline at end of file + ylim(0,104000) + + xlab("Epsilon") + ylab("Ø Episoden bis Konvergenz") + + theme_bw(base_size = 24) \ No newline at end of file diff --git a/OptimalityDifferentDiscountFactors.R b/OptimalityDifferentDiscountFactors.R index 2207cef..7f55e68 100644 --- a/OptimalityDifferentDiscountFactors.R +++ b/OptimalityDifferentDiscountFactors.R @@ -19,8 +19,7 @@ data <- data.frame( x=seq(1, length(ta[,1])) ) ggplot(data, aes(x*1000)) + - labs(title="Durchschnittliche Zeitstempel pro gesammeltes Futter (Lernrate = 0.9)", - x ="Gesamtanzahl gesammeltes Futter", y = "Ø Zeitstempel pro Futter (log)", color = "Diskontierungsfaktor") + + labs( x ="Gesamtanzahl gesammeltes Futter", y = "Ø Zeitstempel pro Futter (log)", color = "Diskontierungsfaktor") + scale_y_log10()+ geom_hline(yintercept=23, linetype="dashed")+ geom_text(aes(0,23,label = "opt", vjust = -1)) + @@ -31,7 +30,8 @@ ggplot(data, aes(x*1000)) + geom_line(aes(y = y5, colour = "0.5"), size=1)+ geom_line(aes(y = y6, colour = "0.3"), size=1) + geom_line(aes(y = y7, colour = "0.1"), size=1) + - geom_line(aes(y = y8, colour = "0.05"), size=1) + geom_line(aes(y = y8, colour = "0.05"), size=1) + + theme_bw(base_size = 24) \ No newline at end of file diff --git a/SimpleZ1B2QLearningA.png b/SimpleZ1B2QLearningA.png new file mode 100644 index 0000000..67009ec Binary files /dev/null and b/SimpleZ1B2QLearningA.png differ diff --git a/SimpleZ1B2SarsaA.png b/SimpleZ1B2SarsaA.png new file mode 100644 index 0000000..2ac3f0f Binary files /dev/null and b/SimpleZ1B2SarsaA.png differ diff --git a/SimpleZ2B1MonteCarloA.png b/SimpleZ2B1MonteCarloA.png new file mode 100644 index 0000000..e2d8257 Binary files /dev/null and b/SimpleZ2B1MonteCarloA.png differ diff --git a/SimpleZ2B1QLearningA.png b/SimpleZ2B1QLearningA.png new file mode 100644 index 0000000..19959c7 Binary files /dev/null and b/SimpleZ2B1QLearningA.png differ diff --git a/SimpleZ2B1QLearningMaxA.png b/SimpleZ2B1QLearningMaxA.png new file mode 100644 index 0000000..1d0b097 Binary files /dev/null and b/SimpleZ2B1QLearningMaxA.png differ diff --git a/SimpleZ2B2QLearningA.png b/SimpleZ2B2QLearningA.png new file mode 100644 index 0000000..08f2d50 Binary files /dev/null and b/SimpleZ2B2QLearningA.png differ diff --git a/SimpleZ2B2SarsaA.png b/SimpleZ2B2SarsaA.png new file mode 100644 index 0000000..4bf05b2 Binary files /dev/null and b/SimpleZ2B2SarsaA.png differ diff --git a/SimpleZ2B4QLearningA.png b/SimpleZ2B4QLearningA.png new file mode 100644 index 0000000..2bde535 Binary files /dev/null and b/SimpleZ2B4QLearningA.png differ diff --git a/antGameAnalysis05DiscA.png b/antGameAnalysis05DiscA.png new file mode 100644 index 0000000..c7583ae Binary files /dev/null and b/antGameAnalysis05DiscA.png differ diff --git a/antGameAnalysis099DiscA.png b/antGameAnalysis099DiscA.png new file mode 100644 index 0000000..fcd9b83 Binary files /dev/null and b/antGameAnalysis099DiscA.png differ diff --git a/antGameAnalysis09DiscA.png b/antGameAnalysis09DiscA.png new file mode 100644 index 0000000..acffaa8 Binary files /dev/null and b/antGameAnalysis09DiscA.png differ diff --git a/convergenceAdvanced.txt b/convergenceAdvanced.txt index 5ae1340..9099183 100644 --- a/convergenceAdvanced.txt +++ b/convergenceAdvanced.txt @@ -17,5 +17,5 @@ 0.8,19870,3288,13724,4492,8159,5058,16764,5648,9462,19071,3914,1242,8262,26004,4036,9421,4914,2535,5362,7298,9587,37133,1837,35325,15272,14922,14138,7115,17236,5123,12157,37380,6086,37390,1672,15573,14241,2049,2602,6802,22362,7936,7544,5330,13155,16016,4544,1489,3780,6326,7794,31553,2808,1493,7788,12646,30464,22312,1681,12084,4163,2197,7950,22478,5106,26771,4382,10615,2586,12214,4799,6297,7589,4585,30365,32302,15734,5480,8626,7387,11932,4245,21532,1710,12737,7132,4740,14578,10680,8266,17300,4213,3264,35920,38026,10272,3984,2279,9739,33900 0.85,5493,10568,19366,5705,15430,8183,5721,13314,36667,33059,3753,40243,23888,25085,21843,6856,2803,9434,4794,29944,10730,39271,4484,23990,6350,16180,8099,4298,11220,4624,5946,24895,8464,4416,6619,2800,4081,12459,1981,12488,6380,9597,10328,1901,24563,13059,3639,12988,2604,4440,22666,1775,4078,5175,1144,3759,11119,1856,34970,10831,2229,5333,17121,9698,14919,2353,3963,8189,36145,13920,5301,16516,2446,46848,3985,,20640,151501,17556,1882,44216,39795,1638,57957,62050,3130,3693,5563,9780,3327,22969,39357,13749,37555,60070,9249,35426,4405,8340,18973 0.9,27355,24592,18962,2318,17604,35725,14327,38167,25602,50236,4999,9023,5562,7541,11799,25139,8724,12642,28509,57095,2147,5909,5414,12572,10018,68830,45393,18962,51656,25601,3444,45667,16813,57110,16492,3991,7315,17775,69277,34769,29824,11087,26371,3479,2540,9597,32593,13169,8588,2794,40136,56004,65307,24864,35523,19491,2673,5363,4799,5852,28566,42427,44011,40146,3757,1115,49574,5798,24249,2576,118943,6169,65584,7057,49505,116138,52083,1809,127776,3214,25689,103442,15260,62754,12390,3233,35309,68989,6615,30593,2503,29359,98237,11900,3240,64969,84134,25361,7384,13141 -0.95,24269,14543,6828,3800,41079,47279,27177,17286,9802,7114,3756,85275,14507,34993,15139,15184,90742,27554,23713,6453,15157,7045,8048,47550,84540,93729,68601,6274,4713,30578,5024,94239,7315,8193,46871,96466,3695,70915,62947,32258,66228,2114,5084,12686,62905,19158,20940,36270,9037,34034,15016,15530,46276,11063,8586,15635,7196,70708,50836,22464,13463,86986,43541,2001,40565,28534,44700,5625,6552,16140,2450,8492,3304,22904,20951,100472,131147,131728,43674,514,79827,181148,31431,4761,1515,2075,138139,137795,71014,170145,60000,42790,179835,18982,48085,28398,56788,126115,5442,118289,9386 +0.95,24269,14543,6828,3800,41079,47279,27177,17286,9802,7114,3756,85275,14507,34993,15139,15184,90742,27554,23713,6453,15157,7045,8048,47550,84540,93729,68601,6274,4713,30578,5024,94239,7315,8193,46871,96466,3695,70915,62947,32258,66228,2114,5084,12686,62905,19158,20940,36270,9037,34034,15016,15530,46276,11063,8586,15635,7196,70708,50836,22464,13463,86986,43541,2001,40565,28534,44700,5625,6552,16140,2450,8492,3304,22904,20951,100472,131147,131728,43674,514,79827,181148,31431,4761,1515,2075,138139,137795,71014,170145,60000,42790,179835,18982,48085,28398,56788,126115,5442,118289 1.0,11364,6363,8012,109822,19730,8425,21388,7864,18427,34072,3126,52381,35105,86487,73913,88033,76264,105864,30103,9522,31049,3180,4838,4078,133687,39236,59239,22968,21540,98395,109063,4050,5612,4990,9933,83766,140114,116077,135653,130826,130070,92207,14994,87801,1577,70868,133816,79790,1587,23322,22071,13903,3584,9721,,38605,52375,67392,10075,97733,46173,29647,2558,28151,162569,4054,10537,30871,45538,97835,45132,35042,70203,3862,100614,84525,140691,81880,80914,35187,11596,51448,2945,56551,39236,84707,64324,100588,78645,12929,32701,63306,163991,2864,34802,72929,198161,71332,98627,137754 diff --git a/convergenceAdvancedMCnegRew.txt b/convergenceAdvancedMCnegRew.txt index 9f6a895..228823b 100644 --- a/convergenceAdvancedMCnegRew.txt +++ b/convergenceAdvancedMCnegRew.txt @@ -17,3 +17,5 @@ 0.80,23371,13662,2610,3123,26600,5906,23183,11485,2044,15258,56366,4336,13184,16712,13557,11630,11685,4179,6475,22117,3277,21088,5519,4128,5046,7032,7460,5568,3386,8238,4008,3251,19140,4465,11027,7443,9430,8304,15718,5129,9187,7542,10458,3791,32996,10154,18294,2742,7202,14167,10913,14566,6472,3560,2730,7425,4317,6402,17221,24562,3051,13622,41040,7260,24759,2382,6860,3922,6252,4609,5775,11406,3102,10563,14923,5731,3119,8198,19199,11265,5756,10868,8093,5442,7096,16954,5440,5237,8315,10868,19020,3271,9158,23881,12818,17414,24735,13355,16241,8234 0.85,5692,4946,1847,10285,14826,70294,7646,9324,11638,7958,50723,11420,3453,5384,9487,29705,30433,4195,13090,27090,20988,10246,13323,4886,4881,4299,4309,6680,5921,27922,10940,16090,20098,39213,77251,59600,7327,12890,8958,48774,26761,10285,37887,23265,3874,15325,21764,5062,20689,20026,19202,5980,17445,6953,13882,16977,58621,5112,8648,42385,12739,7147,8578,4804,13216,4628,5658,7131,3338,11382,13816,57924,8512,19509,8975,14587,6045,31106,60578,6903,7097,14691,22969,6060,10760,12224,9350,24530,5787,9872,4687,7573,8933,61880,11642,15850,19393,18662,20337,5487 0.90,18330,19318,8805,23257,11711,33491,21779,18722,36910,26080,5882,10829,8848,9255,3258,32657,16622,9681,9476,6754,5958,34068,6549,16193,9290,53356,36944,15579,5563,8232,8482,6581,11648,8438,12394,17378,16809,17818,5688,31617,25372,18421,29680,26465,9368,17077,24886,7977,5025,45452,36999,17392,24618,10553,31856,84419,36957,47410,10986,42540,13596,27220,12345,22884,6044,45135,10521,22076,9204,8594,29542,26473,5245,10892,17621,15232,17679,5220,9361,20508,25484,14263,9908,89045,10082,8319,13466,10045,50753,30676,14909,52776,20422,65610,10554,11853,7408,11605,36720,23032 +0.95,24269,14543,6830,3800,41079,4807,26456,36057,12483,8749,47279,27177,17286,9802,7115,3756,85275,14507,34993,15139,32066,1275,3794,29286,15184,90742,27554,23713,6453,15157,7227,8048,47550,84540,93729,68601,6275,4713,30578,5084,94239,7315,8193,46871,96466,3764,70915,62947,32258,66228,2133,5084,12686,62905,19158,20940,36270,9037,34034,15016,15530,46276,11063,8586,15635,7196 +1.0,11364,6363,8012,19730,8425,21388,7864,18427,34072,98639,52381,35105,86487,73913,88033,76264,30103,9522,31049,3182,6600,4078,39236,59239,22968,21540,98395,4052,5612,5001,9933,83766,88344,75849,59554,38619,20400,3709,14813,92207,14994,87801,1645,70868,79790,1587,23322,22071,13903,3589,9722 diff --git a/optimalityDiffDiscA.png b/optimalityDiffDiscA.png new file mode 100644 index 0000000..13a3e33 Binary files /dev/null and b/optimalityDiffDiscA.png differ