-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathrunExp.py
383 lines (249 loc) · 23.1 KB
/
runExp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
from experimenter import run_exp
lunar = "LunarLander-v2"
cart = "CartPole-v1"
acro = "Acrobot-v1"
mountain = "MountainCar-v0"
pong = "PongNoFrameskip-v0"
seaquest = "SeaquestNoFrameskip-v0"
battlezone = "BattleZoneNoFrameskip-v0"
tree = "MineRLTreechop-v0"
diamond = "MineRLObtainDiamond-v0"
paramsQ = {"name": "Q"}
Q_s = [{"name": "Q" + str(i + 1)} for i in range(30)]
Q_no_exp_rep = {"name": "Q-Online", "USE_EXP_REP": False}
paramsQ_target_net_250 = {"name": "Q+Target_250", "TARGET_UPDATE": 250}
paramsQ_target_net_100 = {"name": "Q+Target_100", "TARGET_UPDATE": 100}
paramsQ_target_net_50 = {"name": "Q+Target_50", "TARGET_UPDATE": 50}
paramsPolyak = {"name": "Polyak", "use_polyak_averaging": True}
paramsQV = {"name": "QV", "use_QV": True}
paramsQVMAX = {"name": "QVMAX", "use_QVMAX": True}
paramsEps_2 = {"name": "EpsMid 0.2", "EPS_MID": 0.2}
paramsEps_1 = {"name": "EpsMid 0.1", "EPS_MID": 0.1}
paramsEps_05 = {"name": "EpsMid 0.05", "EPS_MID": 0.05}
paramsEps_01 = {"name": "EpsMid 0.01", "EPS_MID": 0.01}
paramsEps_005 = {"name": "EpsMid 0.005", "EPS_MID": 0.005}
paramsEps_001 = {"name": "EpsMid 0.001", "EPS_MID": 0.001}
paramsEps_0001 = {"name": "EpsMid 0.0001", "EPS_MID": 0.0001}
paramsSplit = {"name": "Q+Split", "split_Bellman": True}
params_Q_reward_noise_0_1 = {"name": "Q-RewardNoise0.1", "reward_added_noise_std": 0.1}
params_Q_reward_noise_1 = {"name": "Q-RewardNoise1", "reward_added_noise_std": 1.0}
params_Q_reward_noise_10 = {"name": "Q-RewardNoise10", "reward_added_noise_std": 10.0}
params_split_reward_noise_0_1 = {"name": "Q+Split-RewardNoise0.1", "split_Bellman": True, "reward_added_noise_std": 0.1}
params_split_reward_noise_1 = {"name": "Q+Split-RewardNoise1", "split_Bellman": True, "reward_added_noise_std": 1.0}
params_split_reward_noise_10 = {"name": "Q+Split-RewardNoise10", "split_Bellman": True, "reward_added_noise_std": 10.0}
paramsNoTarget_r_split_reward_noise_0_1 = {"name": "Q+Split-NoTarget_r-RewardNoise0.1", "split_Bellman": True, "SPLIT_BELL_NO_TARGET_r": True, "reward_added_noise_std": 0.1}
paramsNoTarget_r_split_reward_noise_1 = {"name": "Q+Split-NoTarget_r-RewardNoise1", "split_Bellman": True, "SPLIT_BELL_NO_TARGET_r": True, "reward_added_noise_std": 1.0}
paramsNoTarget_r_split_reward_noise_10 = {"name": "Q+Split-NoTarget_r-RewardNoise10", "split_Bellman": True, "SPLIT_BELL_NO_TARGET_r": True, "reward_added_noise_std": 10.0}
TDEC_pure = {"name": "Q+TDEC", "TDEC_ENABLED": True}
TDEC_pure_offset_01 = {"name": "Q+TDEC-Offset:-0.1", "TDEC_ENABLED": True, "critic_output_offset": -0.1}
TDEC_pure_offset_1 = {"name": "Q+TDEC-Offset:-1", "TDEC_ENABLED": True, "critic_output_offset": -1}
TDEC_pure_offset_10 = {"name": "Q+TDEC-Offset:-10", "TDEC_ENABLED": True, "critic_output_offset": -10}
TDEC_gamma_0_8 = {"name": "Q+TDEC-Gamma0.8", "TDEC_ENABLED": True, "TDEC_GAMMA": 0.8}
TDEC_no_target = {"name": "Q+TDEC-NoTarget", "TDEC_ENABLED": True, "TDEC_USE_TARGET_NET": False}
TDEC_mid = {"name": "Q+TDEC-DecayEps", "TDEC_ENABLED": True, "TDEC_MID": 0.1}
TDEC_abs_act = {"name": "Q+TDEC-absAct", "TDEC_ENABLED": True, "TDEC_ACT_FUNC": "absolute"}
TDEC_abs_train = {"name": "Q+TDEC-absTrain", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute"}
TDEC_mse_train = {"name": "Q+TDEC-mseTrain", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "mse"}
TDEC_abs_train_no_target = {"name": "Q+TDEC-absTrain-NoTarget", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_USE_TARGET_NET": False}
TDEC_abs_train_0_9 = {"name": "Q+TDEC-absTrain-Scale0.9", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_SCALE": 0.9}
TDEC_abs_train_0_1 = {"name": "Q+TDEC-absTrain-Scale0.1", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_SCALE": 0.1}
TDEC_abs_train_mid_0_9 = {"name": "Q+TDEC-absTrain-Mid0.9", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_MID": 0.9}
TDEC_abs_train_mid_0_5 = {"name": "Q+TDEC-absTrain-Mid0.5", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_MID": 0.5}
TDEC_abs_train_mid_0_2 = {"name": "Q+TDEC-absTrain-Mid0.2", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_MID": 0.2}
TDEC_abs_train_mid_0_1 = {"name": "Q+TDEC-absTrain-Mid0.1", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_MID": 0.1}
TDEC_abs_train_mid_0_01 = {"name": "Q+TDEC-absTrain-Mid0.01", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "absolute",
"TDEC_MID": 0.01}
TDEC_pos_act = {"name": "Q+TDEC-posAct", "TDEC_ENABLED": True, "TDEC_ACT_FUNC": "positive"}
TDEC_pos_train = {"name": "Q+TDEC-posTrain", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive"}
TDEC_pos_train_no_target = {"name": "Q+TDEC-posTrain-NoTarget", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive",
"TDEC_USE_TARGET_NET": False}
TDEC_pos_train_decay = {"name": "Q+TDEC-posTrain-DecayEps", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive",
"TDEC_MID": 0.1}
TDEC_pos_train_no_target_decay = {"name": "Q+TDEC-posTrain-NoTarget-DecayEps", "TDEC_ENABLED": True,
"TDEC_TRAIN_FUNC": "positive", "TDEC_MID": 0.1, "TDEC_USE_TARGET_NET": False}
TDEC_pos_train_no_target = {"name": "Q+TDEC-posTrain-NoTarget", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive",
"TDEC_USE_TARGET_NET": False}
TDEC_pos_train_0_9 = {"name": "Q+TDEC-posTrain-Scale0.9", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive",
"TDEC_SCALE": 0.9}
TDEC_pos_train_1 = {"name": "Q+TDEC-posTrain-Scale1", "TDEC_ENABLED": True, "TDEC_TRAIN_FUNC": "positive",
"TDEC_SCALE": 1.0}
reliabilityTest_long = Q_s
epsilonList = [paramsEps_2, paramsEps_1, paramsEps_05, paramsEps_01, paramsEps_005, paramsEps_001, paramsEps_0001]
noisyRewardList = [params_Q_reward_noise_0_1, params_Q_reward_noise_1, params_Q_reward_noise_10, params_split_reward_noise_0_1, params_split_reward_noise_1, params_split_reward_noise_10, paramsNoTarget_r_split_reward_noise_0_1, paramsNoTarget_r_split_reward_noise_1, paramsNoTarget_r_split_reward_noise_10]
splitShortList = [paramsQ, paramsSplit]
Q_list = [paramsQ]
TDEC_basic = [paramsQ, TDEC_pure, TDEC_abs_train, TDEC_pos_train, TDEC_mse_train]
TDEC_scaling_abs = [paramsQ, TDEC_abs_train, TDEC_abs_train_0_9, TDEC_abs_train_0_1, TDEC_abs_train_mid_0_9,
TDEC_abs_train_mid_0_5, TDEC_abs_train_mid_0_2, TDEC_abs_train_mid_0_1,
TDEC_abs_train_mid_0_01] # ....
TDEC_noTargets = [paramsQ, TDEC_pure, TDEC_no_target, TDEC_abs_train, TDEC_abs_train_no_target, TDEC_pos_train,
TDEC_pos_train_no_target]
TDEC_train_or_act = [paramsQ, TDEC_pure, TDEC_abs_train, TDEC_abs_act, TDEC_pos_train, TDEC_pos_act]
TDEC_list = [paramsQ, TDEC_pure, TDEC_no_target, TDEC_mid, TDEC_abs_act, TDEC_abs_train, TDEC_pos_act, TDEC_pos_train]
TDEC_smart_list = [paramsQ, TDEC_pure, TDEC_abs_train, TDEC_pos_train, TDEC_pos_train_no_target, TDEC_pos_train_0_9,
TDEC_pos_train_1]
TDEC_mse = [TDEC_mse_train]
QV_list = [paramsQ, paramsQV]
TDEC_offset_list = [TDEC_pure, TDEC_pure_offset_01, TDEC_pure_offset_1, TDEC_pure_offset_10]
paramsTracesLambda0_8 = {"name": "EligTraces0.8", "use_efficient_traces": 1, "elig_traces_lambda": 0.8}
paramsTracesLambda0_5 = {"name": "EligTraces0.5", "use_efficient_traces": 1, "elig_traces_lambda": 0.5}
paramsTracesAnneal = {"name": "EligTracesAnneal", "use_efficient_traces": 1, "elig_traces_anneal_lambda": 1}
paramsQuick = {"name": "Q_uick", "initial_steps": 100, "eval_rounds": 2, "eval_percentage:": 0.4}
QV_quick = {"name": "QV", "initial_steps": 100, "use_QV": 1, "eval_rounds": 2, "eval_percentage:": 0.4}
traces = [paramsQ, paramsTracesAnneal, paramsTracesLambda0_8, paramsTracesLambda0_5]
PER = {"name": "Q+PER", "use_PER": 1}
PER2w = {"name": "Q+PER-2worker", "use_PER": 1, "worker": 2}
PER8w = {"name": "Q+PER-8worker", "use_PER": 1, "worker": 8}
large_batch = {"name": "Q+BS256", "batch_size": 256}
# test:
#run_exp(cart, [paramsQuick, QV_quick], number_of_tests=3, length_of_tests=30000, path="test")
#run_exp(pong, [paramsQ, paramsQV, paramsSplit, PER, paramsQVMAX, paramsPolyak],
# extra_hyperparams={"initial_steps":100, "eval_percentage": 0.5},
# number_of_tests=2, length_of_tests=100, path="Pong_test")
# Test buffer:
#runExp(pong, [paramsQ, PER], number_of_tests=5, length_of_tests=8000000, path="base")
# Test PER:
#run_exp(battlezone, [paramsQ, PER], number_of_tests=2, length_of_tests=20000000, path="PER_battlezone")
# Run new Rainbow exps:
run_exp(pong, [paramsQ, paramsQV, paramsSplit, PER, paramsQVMAX, paramsPolyak], number_of_tests=5, length_of_tests=8000000, path="Pong_Trials")
run_exp(tree, [paramsQ, paramsQV, paramsSplit, PER, paramsQVMAX, paramsPolyak], number_of_tests=5, length_of_tests=8000000, path="Treechop_Trials")
# Test:
#runExp(cart, traces, number_of_tests=5, length_of_tests=20000, path="traces/", on_server=True, optimize="no", #run_metric_percentage=1, run_metric_final_percentage_weight=0)
#runExp(pong, traces, number_of_tests=5, length_of_tests=100000, path="traces_pong/", on_server=True, optimize="no", run_metric_percentage=1, run_metric_final_percentage_weight=0)
#runExp(seaquest, traces, number_of_tests=5, length_of_tests=10000000, path="traces_seaquest/", on_server=True, optimize="no", run_metric_percentage=1, run_metric_final_percentage_weight=0)
############ Exps:
#runExp(cart, reliabilityTest_long, number_of_tests=50, length_of_tests=50000, path="long_rel_test_opt", on_server=True, optimize="comet_best")
#runExp(cart, reliabilityTest_long, number_of_tests=25, length_of_tests=50000, path="long_rel_test_opt", on_server=True, optimize="no")
#runExp(cart, QV_list, number_of_tests=50, length_of_tests=50000, path="QV_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, QV_list, number_of_tests=50, length_of_tests=50000, path="QV_lunar/", on_server=True, optimize="comet_best")
#runExp(cart, TDEC_basic, number_of_tests=50, length_of_tests=50000, path="TDEC_basic_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, TDEC_basic, number_of_tests=50, length_of_tests=50000, path="TDEC_basic_lunar/", on_server=True, optimize="comet_best")
# Bellman Split basics:
#runExp(cart, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_lunar/", on_server=True, optimize="comet_best")
# Optimize for separate nets and for individual hidden layers
#runExp(cart, splitBellList, number_of_tests=50, length_of_tests=50000, path="split_netArch_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, splitBellList, number_of_tests=50, length_of_tests=50000, path="split_netArch_lunar/", on_server=True, optimize="comet_best")
# Check if added params for different architecture is the cause behind performance boost:
#runExp(cart, checkAddedParamsInfluenceList, number_of_tests=50, length_of_tests=50000, path="split_AddedParamsEffect_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, checkAddedParamsInfluenceList, number_of_tests=50, length_of_tests=50000, path="split_AddedParamsEffect_lunar/", on_server=True, optimize="comet_best")
# Check if spit approaches are more robuts to Gaussian noise (they should be in theory):
#runExp(cart, noisyRewardList, number_of_tests=50, length_of_tests=50000, path="split_GaussianNoise_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, noisyRewardList, number_of_tests=50, length_of_tests=50000, path="split_GaussianNoise_lunar/", on_server=True, optimize="comet_best")
# Optimize only lr for split
#runExp(cart, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_optOnlyLr_cart/", on_server=True, optimize="comet_best", optimize_only_lr=True)
#runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_optOnlyLr_lunar/", on_server=True, optimize="comet_best", optimize_only_lr=True)
# Optimize only Q params for split:
#runExp(cart, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_optOnlyQparams_cart/", on_server=True, optimize="comet_best", optimize_only_Q_params=True)
#runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_optOnlyQparams_lunar/", on_server=True, optimize="comet_best", optimize_only_Q_params=True)
#runExp(cart, Q_no_exp_rep_list, number_of_tests=50, length_of_tests=50000, path="no_exp_rep_cart/", on_server=True, optimize="comet_best")
#runExp(lunar, TDEC_offset_list, number_of_tests=50, length_of_tests=50000, path="Q_offsets_lunar/", on_server=True, optimize="comet_best")
############ How reliable given same hyperparameters?
#runExp(cart, reliabilityTest, number_of_tests=10, length_of_tests=50000, path="no_optimization_10/", on_server=True, optimize="no")
#runExp(cart, reliabilityTest, number_of_tests=15, length_of_tests=50000, path="no_optimization_15/", on_server=True, optimize="no")
#runExp(cart, reliabilityTest, number_of_tests=20, length_of_tests=50000, path="no_optimization_20/", on_server=True, optimize="no")
#runExp(cart, reliabilityTest, number_of_tests=30, length_of_tests=50000, path="no_optimization_30/", on_server=True, optimize="no")
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="no_optimization_50/", on_server=True, optimize="no")
#runExp(cart, reliabilityTest, number_of_tests=100, length_of_tests=50000, path="no_optimization_100/", on_server=True, optimize="no")
# still needs to be tested
########### What is the influence of more runs?
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="comet_75_sets/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=1, evals_per_optimization_step=2)
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="comet_100_sets/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=1, evals_per_optimization_step=2)
########## How many optimizations?
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="comet_1_optim/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=1, evals_per_optimization_step=2)
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="comet_2_optim/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=2, evals_per_optimization_step=2)
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="comet_3_optim/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=3, evals_per_optimization_step=2)
#runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=50000, path="split_short_comet_lunar/", on_server=True, optimize="comet_best", number_of_best_runs_to_check=5, number_of_checks_best_runs=5, final_evaluation_runs=15, number_of_hyperparam_optimizations=1, evals_per_optimization_step=2)
# Reliability Experiments:
#Standard:
#runExp(cart, reliabilityTest, number_of_tests=50, length_of_tests=50000, path="reliability_new/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=3, evals_per_optimization_step=3
# )
#More runs:
#runExp(cart, reliabilityTest, number_of_tests=100, length_of_tests=3000, path="reliability/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=1, evals_per_optimization_step=1
# )
# More evals per optimization:
#runExp(cart, reliabilityTest, number_of_tests=30, length_of_tests=3000, path="reliability_more_evals/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=1, evals_per_optimization_step=2
# )
# More optimizations:
#runExp(cart, reliabilityTest, number_of_tests=30, length_of_tests=3000, path="reliability_more_optimizations/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=2, evals_per_optimization_step=1
# )
# Both of the above:
#runExp(cart, reliabilityTest, number_of_tests=15, length_of_tests=3000, path="reliability_more_evals_and_optimizations/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=2, evals_per_optimization_step=2
# )
# Checking if running separate TPEs makes sense or if we can just keep going
#runExp(cart, reliabilityTest, number_of_tests=30, length_of_tests=3000, path="reliability_more_evals_and_more_checks/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=6, number_of_checks_best_runs=5, final_evaluation_runs=15,
# number_of_hyperparam_optimizations=1, evals_per_optimization_step=2
# )
#runExp(cart, splitShortList, number_of_tests=50, length_of_tests=50000, path="cart_split/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15
# )
#runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=100000, path="lunar_split/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15
# )
#runExp(cart, TDEC_basic, number_of_tests=50, length_of_tests=50000, path="cart_TDEC/", on_server=True, optimize="tpe_best", number_of_best_runs_to_check=3, number_of_checks_best_runs=5, #final_evaluation_runs=15
# )
#runExp(lunar, TDEC_basic, number_of_tests=50, length_of_tests=100000, path="lunar_TDEC/", on_server=True, optimize="tpe_best",
# number_of_best_runs_to_check=3, number_of_checks_best_runs=5, final_evaluation_runs=15
# )
# runExp(cart, TDEC_smart_list, number_of_tests=10, length_of_tests=50000, on_server=True, path="TDEC_smart_cart_2nd_exp/")
# runExp(lunar, TDEC_smart_list, number_of_tests=5, length_of_tests=100000, on_server=True, path="TDEC_smart_lunar/")
# runExp(lunar, QVC_list, number_of_tests=50, length_of_tests=20000, on_server=True, path="QVC_lunar/")
# cartDict = {}
# cartDict.update(runExp(cart, QVC_abs, number_of_tests=50, length_of_tests=20000, on_server=True, path="QVC_abs_cart/"))
# cartDict.update(runExp(cart, QVC_scale, number_of_tests=50, length_of_tests=20000, on_server=True, path="QVC_scale_cart/"))
############################# TDEC Experiments #############################################
####### cart
# runExp(cart, TDEC_scaling_abs, number_of_tests=50, length_of_tests=100000, path="cart_TDEC_abs_scaling/", on_server=True)
# runExp(cart, TDEC_basic, number_of_tests=50, length_of_tests=100000, path="cart_TDEC_basic/", on_server=True)
# runExp(cart, TDEC_noTargets, number_of_tests=50, length_of_tests=100000, path="cart_TDEC_no_targets/", on_server=True)
# runExp(cart, TDEC_train_or_act, number_of_tests=50, length_of_tests=100000, path="cart_TDEC_train_or_act/", on_server=True)
####### acro
# runExp(acro, TDEC_scaling_abs, number_of_tests=50, length_of_tests=100000, path="acro_TDEC_abs_scaling/", on_server=True)
# runExp(acro, TDEC_basic, number_of_tests=50, length_of_tests=100000, path="acro_TDEC_basic/", on_server=True)
# runExp(acro, TDEC_noTargets, number_of_tests=50, length_of_tests=100000, path="acro_TDEC_no_targets/", on_server=True)
# runExp(acro, TDEC_train_or_act, number_of_tests=50, length_of_tests=100000, path="acro_TDEC_train_or_act/", on_server=True)
####### mountain
# runExp(mountain, TDEC_scaling_abs, number_of_tests=50, length_of_tests=100000, path="mountain_TDEC_abs_scaling/", on_server=True)
# runExp(mountain, TDEC_basic, number_of_tests=50, length_of_tests=100000, path="mountain_TDEC_basic/", on_server=True)
# runExp(mountain, TDEC_noTargets, number_of_tests=50, length_of_tests=100000, path="mountain_TDEC_no_targets/", on_server=True)
# runExp(mountain, TDEC_train_or_act, number_of_tests=50, length_of_tests=100000, path="mountain_TDEC_train_or_act/", on_server=True)
###### lunar
# runExp(lunar, TDEC_scaling_abs, number_of_tests=50, length_of_tests=100000, path="lunar_TDEC_abs_scaling/", on_server=True)
# runExp(lunar, TDEC_basic, number_of_tests=50, length_of_tests=150000, path="lunar_TDEC_basic/", on_server=True)
# runExp(lunar, TDEC_noTargets, number_of_tests=50, length_of_tests=100000, path="lunar_TDEC_no_targets/", on_server=True)
# runExp(lunar, TDEC_train_or_act, number_of_tests=50, length_of_tests=100000, path="lunar_TDEC_train_or_act/", on_server=True)
############################# Split Experiments #############################################
# cartDict = {}
# cartDict.update(runExp(cart, splitShortList, number_of_tests=50, length_of_tests=100000, path="QsplitExp_cart/", on_server=True, randomizeParams=True))
# cartDict.update(runExp(cart, QV_split, number_of_tests=50, length_of_tests=100000, path="QVsplitExp_cart/", on_server=True, randomizeParams=True))
# cartDict.update(runExp(cart, QV_no_target, number_of_tests=50, length_of_tests=100000, path="QVnoTargetExp_cart/", on_server=True, randomizeParams=True))
# acroDict = {}
# acroDict.update(runExp(acro, splitShortList, number_of_tests=50, length_of_tests=100000, path="QsplitExp_acro/", on_server=True, randomizeParams=True))
# acroDict.update(runExp(acro, QV_split, number_of_tests=50, length_of_tests=100000, path="QVsplitExp_acro/", on_server=True, randomizeParams=True))
# acroDict.update(runExp(acro, QV_no_target, number_of_tests=50, length_of_tests=100000, path="QVnoTargetExp_acro/", on_server=True, randomizeParams=True))
# mountainDict = {}
# mountainDict.update(runExp(mountain, splitShortList, number_of_tests=50, length_of_tests=100000, path="QsplitExp_mountain/", on_server=True, randomizeParams=True))
# mountainDict.update(runExp(mountain, QV_split, number_of_tests=50, length_of_tests=100000, path="QVsplitExp_mountain/", on_server=True, randomizeParams=True))
# mountainDict.update(runExp(mountain, QV_no_target, number_of_tests=50, length_of_tests=100000, path="QVnoTargetExp_mountain/", on_server=True, randomizeParams=True))
# lunarDict = {}
# lunarDict.update(runExp(lunar, splitShortList, number_of_tests=50, length_of_tests=150000, path="QsplitExp_lunar/", on_server=True, randomizeParams=True))
# lunarDict.update(runExp(lunar, QV_split, number_of_tests=50, length_of_tests=150000, path="QVsplitExp_lunar/", on_server=True, randomizeParams=True))
# lunarDict.update(runExp(lunar, QV_no_target, number_of_tests=50, length_of_tests=150000, path="QVnoTargetExp_lunar/", on_server=True, randomizeParams=True))