starting pre-realease phase

baselines/QLearning/config/alg/qmix_smax.yaml

@@ -1,6 +1,6 @@
 "NUM_ENVS": 8
 "NUM_STEPS": 128
-"BUFFER_SIZE": 5000
+"BUFFER_SIZE": 3000
 "BUFFER_BATCH_SIZE": 32
 "TOTAL_TIMESTEPS": 20000000
 "AGENT_HIDDEN_DIM": 256
@@ -11,12 +11,12 @@
 "EPSILON_ANNEAL_TIME": 100000
 "MIXER_EMBEDDING_DIM": 64
 "MIXER_HYPERNET_HIDDEN_DIM": 256
-"MIXER_INIT_SCALE": 0.01
+"MIXER_INIT_SCALE": 0.001
 "MAX_GRAD_NORM": 10
-"TARGET_UPDATE_INTERVAL": 500
-"LR": 0.005
+"TARGET_UPDATE_INTERVAL": 200
+"LR": 0.001
 "LR_LINEAR_DECAY": False
-"EPS_ADAM": 0.001
+"EPS_ADAM": 0.00001
 "WEIGHT_DECAY_ADAM": 0.000001
 "TD_LAMBDA_LOSS": True
 "TD_LAMBDA": 0.6

baselines/QLearning/config/alg/qmix_transf_agent_smax.yaml

@@ -8,18 +8,18 @@
 "AGENT_TRANSF_NUM_LAYERS": 2
 "AGENT_TRANSF_NUM_HEADS": 8
 "AGENT_TRANSF_DIM_FF": 128
-"MIXER_EMBEDDING_DIM": 32
-"MIXER_HYPERNET_HIDDEN_DIM": 64
-"MIXER_INIT_SCALE": 0.00001
+"MIXER_EMBEDDING_DIM": 64
+"MIXER_HYPERNET_HIDDEN_DIM": 256
+"MIXER_INIT_SCALE": 0.01
 "PARAMETERS_SHARING": True
 "EPSILON_START": 1.0
 "EPSILON_FINISH": 0.05
 "EPSILON_ANNEAL_TIME": 100000
-"MAX_GRAD_NORM": 1.
-"TARGET_UPDATE_INTERVAL": 200 
+"MAX_GRAD_NORM": 10.
+"TARGET_UPDATE_INTERVAL": 1
 "LR": 0.005
 "LR_LINEAR_DECAY": False
-"LR_COSINE_WARMUP": True
+"LR_COSINE_WARMUP": False
 "LR_WARMUP": 10
 "EPS_ADAM": 0.001
 "TD_LAMBDA_LOSS": True

baselines/QLearning/config/alg/transf_qmix_mpe.yaml

@@ -1,4 +1,5 @@
 "NUM_ENVS": 8
+"N_MINI_UPDATES": 1
 "NUM_STEPS": 25
 "BUFFER_SIZE": 5000
 "BUFFER_BATCH_SIZE": 32
@@ -12,17 +13,17 @@
 "MIXER_TRANSF_NUM_LAYERS": 2
 "MIXER_TRANSF_NUM_HEADS": 8
 "MIXER_TRANSF_DIM_FF": 128
-"PARAMETERS_SHARING": True
+"USE_FAST_ATTENTION": True 
+"SCALE_INPUTS": True 
+"EMBEDDER_USE_RELU": False
 "EPSILON_START": 1.0
 "EPSILON_FINISH": 0.05
 "EPSILON_ANNEAL_TIME": 100000
 "MAX_GRAD_NORM": 10.
 "TARGET_UPDATE_INTERVAL": 200 
 "LR": 0.001
-"LR_LINEAR_DECAY": False
-"LR_COSINE_WARMUP": False
-"LR_WARMUP": 10
-"EPS_ADAM": 0.0001
+"LR_DECAY_TYPE": 
+"EPS_ADAM": 0.000001
 "TD_LAMBDA_LOSS": True
 "TD_LAMBDA": 0.6
 "GAMMA": 0.9

baselines/QLearning/config/alg/transf_qmix_smax.yaml

@@ -1,3 +1,7 @@
+# Since it's more expensive to step a transformer than a rnn in an env, 
+# more parallel environments, together with more network updates per episode to balance
+# the training. The total number of timesteps is decreased so that the total number of network updates
+# is the same of qmix.
 "NUM_ENVS": 16
 "N_MINI_UPDATES": 4
 "NUM_STEPS": 128
@@ -13,18 +17,18 @@
 "MIXER_TRANSF_NUM_LAYERS": 2
 "MIXER_TRANSF_NUM_HEADS": 8
 "MIXER_TRANSF_DIM_FF": 128
-"USE_FAST_ATTENTION": True
-"SCALE_INPUTS": True # applies batch normalization to the obs vectors 
+"USE_FAST_ATTENTION": True # assumes you have a fast_attention.py file accesible by the training script
+"SCALE_INPUTS": True # applies batch normalization to the obs vectors
 "EMBEDDER_USE_RELU": True # applies relu on the embeddings
 "EPSILON_START": 1.0
 "EPSILON_FINISH": 0.05
 "EPSILON_ANNEAL_TIME": 100000
 "MAX_GRAD_NORM": 1.
 "TARGET_UPDATE_INTERVAL": 10
-"LR_EXP_DECAY": True
 "LR": 0.005
-"LR_EXP_DECAY_RATE": 0.00002
-"LR_WARMUP": 10
+"LR_DECAY_TYPE": 'exp' # can be exp (exponential), cos (cosine), linear (linear) or None (static)
+"LR_EXP_DECAY_RATE": 0.00002 # applies only to exponential decay
+"LR_WARMUP": 10 # applies only to cosine decay
 "EPS_ADAM": 0.0000000001
 "TD_LAMBDA_LOSS": False
 "TD_LAMBDA": 0.6