diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..4113073 Binary files /dev/null and b/.DS_Store differ diff --git a/1569258663717635.mp4 b/1569258663717635.mp4 new file mode 100644 index 0000000..5823681 Binary files /dev/null and b/1569258663717635.mp4 differ diff --git a/README.md b/README.md new file mode 100755 index 0000000..d960a69 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# Learning Sturctured Commnunication + +A Tensorflow implementation of `LSC`. + + +## Code structure + +- `./graph_nets`: contains code for establishing communication sturcture. + +- `./examples/`: contains scenarios for Ising Model and Battle Game (also models). + +- `train_battle.py`: contains code for training Battle Game models + +## Requirements Installation +```shell +pip install ./ +``` +## Compile MAgent platform and run + +Before running Battle Game environment, you need to compile it. You can get more helps from: [MAgent](https://github.com/geek-ai/MAgent) + +**Steps for compiling** + +```shell +cd examples/battle_model +./build.sh +``` + +**Steps for training models under Battle Game settings** + +1. Add python path in your `~/.bashrc` or `~/.zshrc`: + + ```shell + vim ~/.zshrc + export PYTHONPATH=./examples/battle_model/python:${PYTHONPATH} + source ~/.zshrc + ``` + +2. Run training script for training: + + ```shell + ./runtiny.sh + ``` diff --git a/data/.DS_Store b/data/.DS_Store new file mode 100644 index 0000000..5e17b94 Binary files /dev/null and b/data/.DS_Store differ diff --git a/data/models/.DS_Store b/data/models/.DS_Store new file mode 100644 index 0000000..1d07e89 Binary files /dev/null and b/data/models/.DS_Store differ diff --git a/data/models/il-1/dqn_6-1995selfnomnw.data-00000-of-00001 b/data/models/il-1/dqn_6-1995selfnomnw.data-00000-of-00001 new file mode 100644 index 0000000..ba4a56b Binary files /dev/null and b/data/models/il-1/dqn_6-1995selfnomnw.data-00000-of-00001 differ diff --git a/data/models/il-1/dqn_6-1995selfnomnw.index b/data/models/il-1/dqn_6-1995selfnomnw.index new file mode 100644 index 0000000..a3688b6 Binary files /dev/null and b/data/models/il-1/dqn_6-1995selfnomnw.index differ diff --git a/data/models/il-1/dqn_6-1995selfnomnw.meta b/data/models/il-1/dqn_6-1995selfnomnw.meta new file mode 100644 index 0000000..d2ddcfa Binary files /dev/null and b/data/models/il-1/dqn_6-1995selfnomnw.meta differ diff --git a/data/tmp/gilfixtiny3b6h3000-6/gil/events.out.tfevents.1569301523.AIDA2GPU b/data/tmp/gilfixtiny3b6h3000-6/gil/events.out.tfevents.1569301523.AIDA2GPU new file mode 100644 index 0000000..f2c829b Binary files /dev/null and b/data/tmp/gilfixtiny3b6h3000-6/gil/events.out.tfevents.1569301523.AIDA2GPU differ diff --git a/examples/.DS_Store b/examples/.DS_Store new file mode 100644 index 0000000..21e9eea Binary files /dev/null and b/examples/.DS_Store differ diff --git a/examples/battle_model/.DS_Store b/examples/battle_model/.DS_Store new file mode 100755 index 0000000..81c26c5 Binary files /dev/null and b/examples/battle_model/.DS_Store differ diff --git a/examples/battle_model/.gitignore b/examples/battle_model/.gitignore new file mode 100755 index 0000000..90d48a0 --- /dev/null +++ b/examples/battle_model/.gitignore @@ -0,0 +1,2 @@ +build/ +**__pycache__ diff --git a/examples/battle_model/CMakeLists.txt b/examples/battle_model/CMakeLists.txt new file mode 100755 index 0000000..17d7bff --- /dev/null +++ b/examples/battle_model/CMakeLists.txt @@ -0,0 +1,48 @@ +cmake_minimum_required(VERSION 3.0) + +project(magent) + +IF (APPLE) + set(CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++") + link_directories("/usr/local/opt/llvm/lib") + include_directories( "/usr/local/include" ) + link_directories("/usr/local/lib/") +ENDIF() + +file(GLOB autopilot_sources src/*.cc src/gridworld/*.cc src/discrete_snake/*.cc src/utility/*.cc) +set(LIB_SRC_FILES ${autopilot_sources}) + +file(GLOB autopilot_sources src/render/*.cc src/render/*/*.cc src/render/*/*/*.cc) +set(RENDER_SRC_FILES ${autopilot_sources}) + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -std=c++11 -O3") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable -Wno-reorder -Wno-sign-compare -Wno-missing-braces") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread -DDEBUG") + +# runtime library +add_library(magent SHARED ${LIB_SRC_FILES}) +add_executable(testlib ${LIB_SRC_FILES}) + +# render +add_executable(render ${RENDER_SRC_FILES}) + +IF (APPLE) + target_link_libraries(render boost_system jsoncpp argp omp) +ELSE() + target_link_libraries(render boost_system jsoncpp) +ENDIF() + +set_target_properties( render + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/render/") + +# copy frontend and demo +add_custom_command( + TARGET render POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/src/render/frontend/" "${CMAKE_BINARY_DIR}/render" +) + +add_custom_command( + TARGET render POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/src/render/backend/demo/" "${CMAKE_BINARY_DIR}/render/" +) diff --git a/examples/battle_model/README.md b/examples/battle_model/README.md new file mode 100755 index 0000000..26bd0a7 --- /dev/null +++ b/examples/battle_model/README.md @@ -0,0 +1,13 @@ +# Battle Game + +A cooperative and competitive senario implemented based on MAgent platform + +## Code structure + +- `./algo`: contains algorithms (MFAC, AC, MFQ, IL) + +- `./python` and `./src`: contains engine files + +- `./senario_battle.py`: contains code for the logic of play Battle Game + +- `./build.sh`: you can run it to compile the MAgent platform \ No newline at end of file diff --git a/examples/battle_model/algo/__init__.py b/examples/battle_model/algo/__init__.py new file mode 100755 index 0000000..0de9aa7 --- /dev/null +++ b/examples/battle_model/algo/__init__.py @@ -0,0 +1,15 @@ +from . import q_learning + +IL = q_learning.DQN +MSGDQN = q_learning.MsgDQN +GIL=q_learning.GDQN + +def spawn_ai(algo_name, sess, env, handle, human_name, max_steps,len_nei=6): + if algo_name == 'il': + model = IL(sess, human_name, handle, env, max_steps, len_nei,memory_size=80) + elif algo_name == 'gil': + model = GIL(sess, human_name, handle, env, max_steps,len_nei, memory_size=8,isHie=False) + elif algo_name == 'msgdqn': + model=MSGDQN(sess,human_name,handle,env,max_steps,len_nei,msg_bits=3,memory_size=80000) + + return model diff --git a/examples/battle_model/algo/base.py b/examples/battle_model/algo/base.py new file mode 100755 index 0000000..43cb958 --- /dev/null +++ b/examples/battle_model/algo/base.py @@ -0,0 +1,536 @@ +import tensorflow as tf +import numpy as np +import pdb +from tensorflow.nn.rnn_cell import GRUCell + +from magent.gridworld import GridWorld +import time +import math +from graph_nets import graphs +from graph_nets import utils_np +from graph_nets import utils_tf +from graph_nets.demos import models +import sonnet as snt +import sonnet as snt +from tensorflow.contrib import rnn +import sklearn.preprocessing as preprocessing +max_len=20 +def length(sequence): + used = tf.sign(tf.reduce_max(tf.abs(sequence), reduction_indices=2)) + length = tf.reduce_sum(used, reduction_indices=1) + length = tf.cast(length, tf.int32) + return length +def make_conv_model2(nums): + return snt.Sequential([ + # tf.layers.conv2d( filters=32, kernel_size=3,activation=tf.nn.relu, name="Conv1"), + snt.nets.ConvNet2D(output_channels=[32,32],kernel_shapes=[3,3],strides=[1,1],paddings=['VALID','VALID'],activate_final=True), + snt.BatchFlatten(), + snt.nets.MLP([256,128,64,nums] , activate_final=False), + # snt.LayerNorm() + ]) +class ValueNet: + def __init__(self, sess, env, handle, name,len_nei, update_every=5, use_mf=False, learning_rate=1e-4, tau=0.99, gamma=0.95,num_bits_msg=3,msg_space=(40,40),use_msg=False,is_msg=False,use_mean_msg=False,is_hie=False,crp=False): + # assert isinstance(env, GridWorld) + self.env = env + self.name = name + self._saver = None + self.sess = sess + + self.handle = handle + self.view_space = env.get_view_space(handle) + assert len(self.view_space) == 3 + self.feature_space = env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + if is_msg: + self.num_actions=num_bits_msg + + self.num_bits_msg=num_bits_msg + self.msg_space=msg_space + + self.update_every = update_every + self.use_mf = use_mf # trigger of using mean field + self.use_msg=use_msg + self.use_mean_msg=use_mean_msg + self.temperature = 0.1 + + self.lr= learning_rate + self.tau = tau + self.gamma = gamma + self.crp=crp + with tf.variable_scope(name or "ValueNet"): + self.name_scope = tf.get_variable_scope().name + # print(self.view_space) + self.obs_input = tf.placeholder(tf.float32, (None,) + self.view_space, name="Obs-Input") + self.feat_input = tf.placeholder(tf.float32, (None,) + self.feature_space, name="Feat-Input") + self.mask = tf.placeholder(tf.float32, shape=(None,), name='Terminate-Mask') + + if self.use_mf: + self.act_prob_input = tf.placeholder(tf.float32, (None, self.num_actions), name="Act-Prob-Input") + if self.use_msg: + self.msgs_input = tf.placeholder(tf.float32, (None,)+ (self.num_bits_msg,self.msg_space[0],self.msg_space[1]), name="Msg-Input") + + if self.use_mean_msg: + self.mean_msg_input=tf.placeholder(tf.float32,(None,self.num_bits_msg),name="Mean-Msg-Input") + + # TODO: for calculating the Q-value, consider softmax usage + self.act_input = tf.placeholder(tf.int32, (None,), name="Act") + self.act_one_hot = tf.one_hot(self.act_input, depth=self.num_actions, on_value=1.0, off_value=0.0) + self.batch_num=tf.placeholder(tf.float32,(None,)) + with tf.variable_scope("Eval-Net"): + self.eval_name = tf.get_variable_scope().name + self.e_q = self._construct_net(active_func=tf.nn.relu) + self.e_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.eval_name) + + with tf.variable_scope("Target-Net"): + self.target_name = tf.get_variable_scope().name + self.t_q = self._construct_net(active_func=tf.nn.relu) + self.t_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.target_name) + + with tf.variable_scope("Update"): + self.update_op = [tf.assign(self.t_variables[i], + self.tau * self.e_variables[i] + (1. - self.tau) * self.t_variables[i]) + for i in range(len(self.t_variables))] + + with tf.variable_scope("Optimization"): + self.target_q_input = tf.placeholder(tf.float32, (None,), name="Q-Input") + self.e_q_max = tf.reduce_sum(tf.multiply(self.act_one_hot, self.e_q), axis=1) + self.loss = tf.reduce_sum(tf.square(self.target_q_input - self.e_q_max))/tf.reduce_sum(self.batch_num) + self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss) + + def _construct_net(self, active_func=None, reuse=False): + + mod=make_conv_model2(self.num_actions) + return mod(self.obs_input) + + @property + def vars(self): + return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name_scope) + + def calc_target_q(self, **kwargs): + """Calculate the target Q-value + kwargs: {'obs', 'feature', 'prob', 'dones', 'rewards'} + """ + feed_dict = { + self.obs_input: kwargs['obs'], + self.feat_input: kwargs['feature'] + } + + + t_q, e_q = self.sess.run([self.t_q, self.e_q], feed_dict=feed_dict) + act_idx = np.argmax(e_q, axis=1) + q_values = t_q[np.arange(len(t_q)), act_idx] + q_values=q_values.reshape(-1) + q_v=[] + i=0 + # pdb.set_trace() + for k in 1.-np.array(kwargs['dones']): + if k: + q_v.append(q_values[i]) + i+=1 + else: + q_v.append(0) + q_v=np.array(q_v) + target_q_value = kwargs['rewards'] + q_v* self.gamma + + return target_q_value + + def update(self): + """Q-learning update""" + self.sess.run(self.update_op) + + def act(self, **kwargs): + """Act + kwargs: {'obs', 'feature', 'prob', 'eps'} + """ + feed_dict = { + self.obs_input: kwargs['obs'], + # self.feat_input: kwargs['state'][1] + } + + if self.use_mf: + assert kwargs.get('prob', None) is not None + assert len(kwargs['prob']) == len(kwargs['obs']) + feed_dict[self.act_prob_input] = kwargs['prob'] + + if self.use_msg: + feed_dict[self.msgs_input] = kwargs['msgs'] + + + if self.use_mean_msg: + feed_dict[self.mean_msg_input]=kwargs['mean_msg'] + + + q_values = self.sess.run(self.e_q, feed_dict=feed_dict) + + + + switch = np.random.uniform() + + if switch < kwargs['eps']: + actions = np.random.choice(self.num_actions, len(kwargs['obs'])).astype(np.int32) + else: + actions = np.argmax(q_values, axis=1).astype(np.int32) + return actions + def get_q(self, **kwargs): + """Act + kwargs: {'obs', 'feature', 'prob', 'eps'} + """ + feed_dict = { + self.obs_input: kwargs['state'][0], + self.feat_input: kwargs['state'][1] + } + + if self.use_mf: + assert kwargs.get('prob', None) is not None + # print(len(kwargs['prob'])) + # print(len(kwargs['state'][0])) + assert len(kwargs['prob']) == len(kwargs['state'][0]) + feed_dict[self.act_prob_input] = kwargs['prob'] + + if self.use_msg: + feed_dict[self.msgs_input] = kwargs['msgs'] + + + if self.use_mean_msg: + feed_dict[self.mean_msg_input]=kwargs['mean_msg'] + + + q_values = self.sess.run(self.e_q, feed_dict=feed_dict) + return np.max(q_values,axis=1) + + def train(self, **kwargs): + """Train the model + kwargs: {'state': [obs, feature], 'target_q', 'prob', 'acts'} + """ + feed_dict = { + self.obs_input: kwargs['state'][0], + self.feat_input: kwargs['state'][1], + self.target_q_input: kwargs['target_q'], + self.mask: kwargs['masks'] + } + + if self.use_mf: + assert kwargs.get('prob', None) is not None + feed_dict[self.act_prob_input] = kwargs['prob'] + + if self.use_msg: + feed_dict[self.msgs_input] = kwargs['msgs'] + + if self.use_mean_msg: + feed_dict[self.mean_msg_input]=kwargs['mean_msg'] + + feed_dict[self.act_input] = kwargs['acts'] + # pdb.set_trace() + feed_dict[self.batch_num]=np.ones_like(kwargs['target_q']) + # print(self.sess.run(tf.reduce_sum(self.batch_num),feed_dict=feed_dict)) + _, loss, e_q = self.sess.run([self.train_op, self.loss, self.e_q_max], feed_dict=feed_dict) + # pdb.set_trace() + return loss, {'Eval-Q': np.round(np.mean(e_q), 6), 'Target-Q': np.round(np.mean(kwargs['target_q']), 6)} + + def get_feedback(self,**kwargs): + kwargs['dones']=np.array([not done for done in kwargs['alives']], dtype=np.bool) + q_values=self.get_q(**kwargs) + tar_q=self.calc_target_q(**kwargs) + return q_values-tar_q + + +def cons_datas(num_acts): + + + datadicts_tmpl=[{ + "obs":[[[[0.0 for i in range(6)] for i in range(13)]for i in range(13)]for i in range(3)], + "q":[[0.1 for i in range(num_acts)] for i in range(3)], + "act":[0 for i in range(3)], + "globals": [0.1, 0, 0], + "nodes": [[0.1 for i in range(256)] for j in range(3)], + "edges": [[101. for i in range(9)],[201. for i in range(9)]], + "senders": [0, 2], + "receivers": [1, 1], + "hedges":[[0.1 for i in range(9)]], + "hsenders":[0], + "hreceivers":[1], + "ledges":[[0.1 for i in range(9)]], + "lsenders":[0], + "lreceivers":[1], + "lnodes": [[0.1 for i in range(32)] for i in range(3)], + "hnodes": [[0.1 for i in range(64)] for i in range(3)] + }] + return datadicts_tmpl +class GCrpNet: + def __init__(self, sess, env, handle, name,len_nei=6, update_every=5,learning_rate=1e-4, tau=0.99, gamma=0.95,num_bits_msg=3,isHie=False,is_comm=False): + # assert isinstance(env, GridWorld) + self.env = env + self.name = name + self._saver = None + self.sess = sess + + self.isHie=isHie + self.handle = handle + self.view_space = env.get_view_space(handle) + assert len(self.view_space) == 3 + self.feature_space = env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + self.num_bits_msg=num_bits_msg + + self.update_every = update_every + + self.len_nei=len_nei + self.temperature = 0.1 + self.act_input = tf.placeholder(tf.int32, (None,), name="Act") + self.act_one_hot = tf.one_hot(self.act_input, depth=self.num_actions, on_value=1.0, off_value=0.0) + self.lr= learning_rate + self.tau = tau + self.gamma = gamma + self.is_comm=is_comm + with tf.variable_scope(name or "GHCrpNet"): + self.name_scope = tf.get_variable_scope().name + # print(self.view_space) + self.input_ph = utils_tf.placeholders_from_data_dicts(cons_datas(self.num_actions)) + with tf.variable_scope("Eval-Net"): + self.eval_name = tf.get_variable_scope().name + + self.e_graph = self._construct_net() + self.e_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.eval_name) + with tf.variable_scope("Target-Net"): + self.target_name = tf.get_variable_scope().name + self.t_graph = self._construct_net() + self.t_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.target_name) + + with tf.variable_scope("Update"): + self.update_op = [tf.assign(self.t_variables[i], + self.tau * self.e_variables[i] + (1. - self.tau) * self.t_variables[i]) + for i in range(len(self.t_variables))] + + with tf.variable_scope("Optimization"): + self.target_q_input = tf.placeholder(tf.float32, (None,), name="Q-Input") + self.e_q_max = tf.reduce_sum(tf.multiply(self.act_one_hot, self.e_graph.q), axis=1) + self.loss = tf.reduce_sum(tf.square(self.target_q_input - self.e_q_max))/tf.cast(tf.reduce_sum(self.e_graph.n_node),tf.float32) + self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss) + + def get_indexs(self,poss): + nei_len=self.len_nei + nei_space=nei_len**2 + poss_all=np.array([poss]*len(poss)) + poss_mine=np.array([ [t]*len(poss) for t in poss]) + indexs=((poss_all-poss_mine)**2).sum(axis=2) + indexs=indexs-np.ones_like(indexs)*nei_space + indexs[indexs<0]=0 + return indexs + + def _construct_net(self): + if self.is_comm: + model=models.GcommNetwork() + else: + if self.num_actions==13: + model=models.GCrpNetworkTiny() + else: + model=models.GCrpNetwork() + return model(self.input_ph) + + @property + def vars(self): + return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name_scope) + + def cons_graph(self,obs,poss,chs=None,act=None): + mini=np.zeros((52,52)) + for i in range(len(chs)): + mini[int(poss[chs[i]][0])+6,int(poss[chs[i]][1])+6]=1 + tmp=[] + for i in range(len(poss)): + tmp.append(mini[int(poss[i][0]):int(poss[i][0])+13,int(poss[i][1]):int(poss[i][1]+13)]) + tmp=np.reshape(tmp,(len(obs),13,13,1)) + obs=np.concatenate((obs,tmp),axis=-1) + #pdb.set_trace() + #for i in range(len(obs)): + # obs[i][-1]=tmp[i] + datadict={} + datadict["obs"]=obs + datadict["senders"]=[] + datadict["receivers"]=[] + datadict["edges"]=[] + datadict["lsenders"]=[] + datadict["lreceivers"]=[] + datadict["ledges"]=[] + datadict["hedges"]=[] + datadict["hsenders"]=[] + datadict["hreceivers"]=[] + + indexs=self.get_indexs(poss) + datadict['nodes']=[[0 for j in range(256)]for i in range(len(obs))] + datadict['lnodes']=[[0 for j in range(32)]for i in range(len(obs))] + datadict['hnodes']=[[0 for j in range(64)]for i in range(len(obs))] + datadict["globals"]=[0,0,0] + datadict['q']=[[0 for i in range(self.num_actions)] for i in range(len(obs))] + chs=list(set(chs)) + if chs is not None: + for ch in chs: + for ind in np.where(indexs[ch]==0)[0]: + # if ind not in chs: + datadict["senders"].append(ind) + datadict["receivers"].append(ch) + datadict["lsenders"].append(ch) + datadict["lreceivers"].append(ind) + + datadict["ledges"].append([0 for i in range(9)]) + datadict["edges"].append([0 for i in range(9)]) + for ch in chs: + for c in chs: + if ch!=c: + datadict["hedges"].append([0 for i in range(9)]) + datadict["hsenders"].append(ch) + datadict["hreceivers"].append(c) +# datadict["hedges"].append([0,0,0]) +# datadict["hsenders"].append(c) +# datadict["hreceivers"].append(ch) + + if len(datadict['edges'])==0: + datadict["edges"]=np.zeros(shape=(0,9)) + if len(datadict["hedges"])==0: + datadict["hedges"]=np.zeros(shape=(0,9)) + if len(datadict["ledges"])==0: + datadict["ledges"]=np.zeros(shape=(0,9)) + if act is not None: + datadict["act"]=act + else: + datadict["act"]=[0 for i in range(len(obs))] + # if target_q is not None: + # datadict["target_q"]=target_q + # else: + # datadict['target_q']=[[0 for i in range(21)] for i in range(len(obs))] + datadicts=[datadict] + graphtuple=utils_np.data_dicts_to_graphs_tuple(datadicts) + return graphtuple + def cons_allcomm_graph(self,obs,poss,chs=None,act=None): + datadict={} + datadict["obs"]=obs + datadict["senders"]=[] + datadict["receivers"]=[] + datadict["edges"]=[] + datadict["lsenders"]=[] + datadict["lreceivers"]=[] + datadict["ledges"]=[] + datadict["hedges"]=[] + datadict["hsenders"]=[] + datadict["hreceivers"]=[] + + indexs=self.get_indexs(poss) + datadict['nodes']=[[0 for j in range(256)]for i in range(len(obs))] + datadict['lnodes']=[[0 for j in range(32)]for i in range(len(obs))] + datadict['hnodes']=[[0 for j in range(64)]for i in range(len(obs))] + datadict["globals"]=[0,0,0] + datadict['q']=[[0 for i in range(self.num_actions)] for i in range(len(obs))] + datadict['hreceivers'].append(0) + datadict['hsenders'].append(0) + datadict['hedges'].append([0,0,0]) + for i in range(len(obs)): + for j in range(len(obs)): + datadict['senders'].append(i) + datadict['receivers'].append(j) + datadict['lsenders'].append(j) + datadict['lreceivers'].append(i) + datadict["ledges"].append([0,0,0]) + datadict["edges"].append([0,0,0]) + if len(datadict['edges'])==0: + datadict["edges"]=np.zeros(shape=(0,3)) + if len(datadict["hedges"])==0: + datadict["hedges"]=np.zeros(shape=(0,3)) + if len(datadict["ledges"])==0: + datadict["ledges"]=np.zeros(shape=(0,3)) + if act is not None: + datadict["act"]=act + else: + datadict["act"]=[0 for i in range(len(obs))] + # if target_q is not None: + # datadict["target_q"]=target_q + # else: + # datadict['target_q']=[[0 for i in range(21)] for i in range(len(obs))] + datadicts=[datadict] + graphtuple=utils_np.data_dicts_to_graphs_tuple(datadicts) + return graphtuple + + + def calc_target_q(self, **kwargs): + """Calculate the target Q-value + kwargs: {'obs', 'feature', 'prob', 'dones', 'rewards'} + """ + if self.isHie: + feed_dict=self.cons_allcomm_graph(obs=kwargs['obs'],poss=kwargs['poss'],chs=kwargs['chs']) + else: + feed_dict=self.cons_graph(obs=kwargs['obs'],poss=kwargs['poss'],chs=kwargs['chs']) + # pdb.set_trace() + t_res = self.sess.run(self.t_graph,feed_dict={self.input_ph:feed_dict}) + e_res=self.sess.run(self.e_graph, feed_dict={self.input_ph:feed_dict}) + act_idx = np.argmax(e_res.q, axis=1) + t_q=t_res.q + q_values = t_q[np.arange(len(t_q)), act_idx] + # print(len(kwargs['rewards'])) + # print(len(1. - kwargs['dones'])) + # print(len(q_values.reshape(-1))) + q_values=q_values.reshape(-1) + q_v=[] + i=0 + for k in 1.-kwargs['dones']: + if k: + q_v.append(q_values[i]) + i+=1 + else: + q_v.append(0) + q_v=np.array(q_v) + target_q_value = kwargs['rewards'] + q_v* self.gamma + + return target_q_value + + def update(self): + """Q-learning update""" + self.sess.run(self.update_op) + + def act(self, **kwargs): + """Act + kwargs: {'obs', 'feature', 'prob', 'eps'} + """ + if self.isHie: + feed_dict=self.cons_allcomm_graph(obs=kwargs["obs"],poss=kwargs["poss"],chs=kwargs["chs"]) + else: + feed_dict=self.cons_graph(obs=kwargs["obs"],poss=kwargs["poss"],chs=kwargs["chs"]) + e_res = self.sess.run(self.e_graph, feed_dict={self.input_ph:feed_dict}) + + switch = np.random.uniform() + tmp=np.random.rand() +# pdb.set_trace() + if tmp<0.005: + print(e_res.edges[0]) + if e_res.edges.shape[0]>2: + print(e_res.edges[1]) + if switch < kwargs['eps']: + actions = np.random.choice(self.num_actions,e_res.n_node).astype(np.int32) + else: + actions = np.argmax(e_res.q, axis=1).astype(np.int32) + return actions + + def get_e_res(self, **kwargs): + """Act + kwargs: {'obs', 'feature', 'prob', 'eps'} + """ + if self.isHie: + feed_dict=self.cons_allcomm_graph(obs=kwargs["obs"],poss=kwargs["poss"],chs=kwargs["chs"]) + else: + feed_dict=self.cons_graph(obs=kwargs["obs"],poss=kwargs["poss"],chs=kwargs["chs"]) + e_res = self.sess.run(self.e_graph, feed_dict={self.input_ph:feed_dict}) + return e_res + + + + def train(self, **kwargs): + """Train the model + kwargs: {'state': [obs, feature], 'target_q', 'prob', 'acts'} + """ + if self.isHie: + feed_dict=self.cons_allcomm_graph(obs=kwargs["obs"],poss=kwargs["poss"],act=kwargs["acts"],chs=kwargs["chs"]) + else: + feed_dict=self.cons_graph(obs=kwargs["obs"],poss=kwargs["poss"],act=kwargs["acts"],chs=kwargs["chs"]) + + ## acts[:len(kwargs['obs'])]=kwargs['acts'] + _,loss,e_q= self.sess.run([self.train_op,self.loss,self.e_q_max], feed_dict={self.input_ph:feed_dict,self.target_q_input:kwargs['target_q'],self.act_input:kwargs['acts']}) + # pdb.set_trace() + return loss, {'Eval-Q': np.round(np.mean(e_q), 6), 'Target-Q': np.round(np.mean(kwargs['target_q']), 6)} + diff --git a/examples/battle_model/algo/q_learning.py b/examples/battle_model/algo/q_learning.py new file mode 100755 index 0000000..32cb5fb --- /dev/null +++ b/examples/battle_model/algo/q_learning.py @@ -0,0 +1,214 @@ +import os +import tensorflow as tf +import numpy as np +import pdb + +from . import base +from . import tools + +class MsgDQN(base.ValueNet): + def __init__(self, sess, name, handle, env, len_nei,sub_len,msg_bits=5, memory_size=2**10, batch_size=64, update_every=5): + + super().__init__(sess, env, handle, name,len_nei, update_every=update_every,is_msg=True) + self.msg_bits=msg_bits + self.replay_buffer = tools.MemoryGroup(self.view_space, self.feature_space, self.msg_bits, memory_size, batch_size, sub_len) + self.sess.run(tf.global_variables_initializer()) + + def flush_buffer(self, **kwargs): + self.replay_buffer.push(**kwargs) + def test_buffer(self): + self.replay_buffer.test_buffer() + + def train(self): + self.replay_buffer.tight() + batch_num = self.replay_buffer.get_batch_num() + + # for i in range(batch_num): + # obs, feats,actions, obs_next, feat_next,rewards, dones, masks = self.replay_buffer.sample() + # target_q = self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, dones=dones) + # loss, q = super().train(state=[obs, feats], target_q=target_q, acts=actions, masks=masks) + + # if i % self.update_every == 0: + # self.update() + + # if i % 50 == 0: + # print('[*] LOSS:', loss, '/ Q:', q) + for i in range(batch_num): + obs, feat, acts, obs_next, feat_next, rewards, dones, masks, act_prob, act_prob_next, msgs, msgs_next, mean_msg, mean_msg_next,poss,poss_next,chs,chs_next= self.replay_buffer.sample() + target_q = self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, dones=dones, prob=act_prob_next,mean_msg=mean_msg_next,msgs=msgs_next) + loss, q = super().train(state=[obs, feat], target_q=target_q, prob=act_prob,acts=acts, masks=masks,mean_msg=mean_msg,msgs=msgs) + if i % self.update_every == 0: + self.update() + + if i % 50 == 0: + print('[*] LOSS:', loss, '/ Q:', q) + + def save(self, dir_path, step=0): + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name_scope) + saver = tf.train.Saver(model_vars) + + file_path = os.path.join(dir_path, "Msgdqn_{}".format(step)) + saver.save(self.sess, file_path) + + print("[*] Model saved at: {}".format(file_path)) + + def load(self, dir_path, step=0): + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name_scope) + saver = tf.train.Saver(model_vars) + + file_path = os.path.join(dir_path, "Msgdqn_{}".format(step)) + + saver.restore(self.sess, file_path) + print("[*] Loaded model from {}".format(file_path)) + +class DQN(base.ValueNet): + def __init__(self, sess, name, handle, env, sub_len,len_nei, memory_size=2**10, batch_size=64, update_every=5): + + super().__init__(sess, env, handle, name, len_nei,update_every=update_every) + + self.replay_buffer = tools.MemoryGroup(self.view_space, self.feature_space, self.num_actions, memory_size, batch_size, sub_len) + self.sess.run(tf.global_variables_initializer()) + + def flush_buffer(self, **kwargs): + self.replay_buffer.push(**kwargs) + + def train(self): + self.replay_buffer.tight() + batch_num = self.replay_buffer.get_batch_num() + + # writer=tf.summary.FileWriter('./graph1',self.sess.graph) + # for i in range(batch_num): + # obs, feats, actions,obs_next, feat_next,rewards, dones, masks = self.replay_buffer.sample() + # target_q = self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, dones=dones) + # loss, q = super().train(state=[obs, feats], target_q=target_q, acts=actions, masks=masks) + + # if i % self.update_every == 0: + # self.update() + + # if i % 50 == 0: + # print('[*] LOSS:', loss, '/ Q:', q) + for i in range(batch_num): + obs, feat, acts, obs_next, feat_next, rewards, dones, masks, act_prob, act_prob_next, msgs, msgs_next, mean_msg, mean_msg_next,poss,poss_next,chs,chs_next = self.replay_buffer.sample() + target_q=rewards.copy() + + if np.sum(dones)!=len(dones): + target_q = self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, + dones=dones, prob=act_prob_next, mean_msg=mean_msg_next, msgs=msgs_next,poss=poss_next,chs=chs_next) + # j=0 + # for ind in range(len(rewards)): + # if not dones[ind]: + # target_q[ind]=target_q_tmp[ind] + # # j+=1 + # if not (target_q==target_q_tmp).all(): + # print("??????") + # pdb.set_trace() + # target_q2 = self.calc_tanrget_q(obs=obs_next, feature=feat_next, rewards=rewards, dones=dones, prob=act_prob_next,mean_msg=mean_msg_next,msgs=msgs_next) + loss, q = super().train(state=[obs, feat], target_q=target_q, prob=act_prob,acts=acts, masks=masks,mean_msg=mean_msg,msgs=msgs) + if i % self.update_every == 0: + self.update() + + if i % 50 == 0: + print('[*] LOSS:', loss, '/ Q:', q) + + + def save(self, dir_path, step=0): + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name_scope) + saver = tf.train.Saver(model_vars) + + file_path = os.path.join(dir_path, "dqn_{}".format(step)) + saver.save(self.sess, file_path) + + print("[*] Model saved at: {}".format(file_path)) + + def load(self, dir_path, step=0): + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name_scope) + saver = tf.train.Saver(model_vars) + + file_path = os.path.join(dir_path, "dqn_{}".format(step)) + + saver.restore(self.sess, file_path) + print("[*] Loaded model from {}".format(file_path)) + + def test_buffer(self): + self.replay_buffer.test_buffer() + + + + + + + + +class GDQN(base.GCrpNet): + def __init__(self, sess, name, handle, env, sub_len,len_nei=6, memory_size=2**10, batch_size=64, update_every=5,isHie=False): + + super().__init__(sess, env, handle, name,len_nei=len_nei, update_every=update_every,isHie=isHie) + + self.replay_buffer = tools.MemoryGroup( + self.view_space, self.feature_space, self.num_actions, memory_size, batch_size, sub_len,needPoss=True,needChs=True) + self.sess.run(tf.global_variables_initializer()) + # writer=tf.summary.FileWriter('./graph/hil',self.sess.graph) + # writer.flush() + # writer.close() + def flush_buffer(self, **kwargs): + # print('in flush buffer',kwargs['chs']) + self.replay_buffer.push(**kwargs) + + def train(self): + self.replay_buffer.tight() + batch_num = self.replay_buffer.get_batch_num() + + for i in range(batch_num): + obs, feat, acts, obs_next, feat_next, rewards, dones, masks, act_prob, act_prob_next, msgs, msgs_next, mean_msg, mean_msg_next,poss,poss_next,chs,chs_next = self.replay_buffer.sample() + # pdb.set_trace() + + fl=False + if batch_num<500 and i%100==0: + fl=True + if rewards[0]>5 : + print(rewards) + target_q=rewards.copy() + if np.sum(dones)!=len(dones): + target_q = self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, + dones=dones, prob=act_prob_next, mean_msg=mean_msg_next, msgs=msgs_next,poss=poss_next,chs=chs_next,fl=fl) + # j=0 + # for ind in range(len(rewards)): + # if not dones[ind]: + # target_q[ind]=target_q_tmp[j] + # j+=1 + # target_q=self.calc_target_q(obs=obs_next, feature=feat_next, rewards=rewards, + # dones=dones, prob=act_prob_next, mean_msg=mean_msg_next, msgs=msgs_next,poss=poss_next,chs=chs_next,fl=fl) + # print(rewards) + loss, q = super().train(obs=obs, target_q=target_q,prob=act_prob, acts=acts, masks=masks, mean_msg=mean_msg, msgs=msgs,poss=poss,chs=chs,alt=0) + + # if i= self.length: + raise KeyError() + return self.data[idx] + + def sample(self, idx): + return self.data[idx % self.length] + + def pull(self): + return self.data[:self.length] + + def append(self, value): + start = 0 + num = len(value) + + if self._flag + num > self.max_len: + tail = self.max_len - self._flag + self.data[self._flag:] = value[:tail] + num -= tail + start = tail + self._flag = 0 + + self.data[self._flag:self._flag + num] = value[start:] + self._flag += num + self.length = min(self.length + len(value), self.max_len) + + def reset_new(self, start, value): + self.data[start:] = value + + +class EpisodesBufferEntry: + """Entry for episode buffer""" + def __init__(self): + self.views = [] + self.features = [] + self.actions = [] + self.rewards = [] + self.probs = [] + self.terminal = False + + def append(self, view, feature, action, reward, alive, probs=None): + self.views.append(view.copy()) + self.features.append(feature.copy()) + self.actions.append(action) + self.rewards.append(reward) + if probs is not None: + self.probs.append(probs) + if not alive: + self.terminal = True + + +class EpisodesBuffer(Buffer): + """Replay buffer to store a whole episode for all agents + one entry for one agent + """ + def __init__(self, use_mean=False): + super().__init__() + self.buffer = {} + self.use_mean = use_mean + + def push(self, **kwargs): + view, feature = kwargs['state'] + acts = kwargs['acts'] + rewards = kwargs['rewards'] + alives = kwargs['alives'] + ids = kwargs['ids'] + + if self.use_mean: + probs = kwargs['prob'] + + buffer = self.buffer + index = np.random.permutation(len(view)) + + for i in range(len(ids)): + i = index[i] + entry = buffer.get(ids[i]) + if entry is None: + entry = EpisodesBufferEntry() + buffer[ids[i]] = entry + + if self.use_mean: + entry.append(view[i], feature[i], acts[i], rewards[i], alives[i], probs=probs[i]) + else: + entry.append(view[i], feature[i], acts[i], rewards[i], alives[i]) + + def reset(self): + """ clear replay buffer """ + self.buffer = {} + + def episodes(self): + """ get episodes """ + return self.buffer.values() + + +class AgentMemory(object): + def __init__(self, obs_shape, feat_shape, act_n, max_len, msg_bits,msg_n,use_mean=False,use_msg=False,use_mean_msg=False,needPoss=False,needChs=False): + self.obs0 = MetaBuffer(obs_shape, max_len) + self.feat0 = MetaBuffer(feat_shape, max_len) + self.actions = MetaBuffer((), max_len, dtype='int32') + self.rewards = MetaBuffer((), max_len) + self.terminals = MetaBuffer((), max_len, dtype='bool') + self.use_mean = use_mean + self.use_msg=use_msg + self.use_mean_msg=use_mean_msg + self.needPoss=needPoss + self.needChs=needChs + self.is_end=False + + if self.use_mean: + self.prob = MetaBuffer((act_n,), max_len) + if self.use_msg: + self.msgs = MetaBuffer((msg_bits,msg_n[0],msg_n[1]),max_len) + if self.use_mean_msg: + self.mean_msg=MetaBuffer((msg_bits,),max_len) + if self.needPoss: + self.poss=MetaBuffer((2,),max_len) + if self.needChs: + self.chs=MetaBuffer((),max_len,dtype='int32') + + def append(self, obs0, feat0, act, reward, alive, prob=None,msgs=None,mean_msg=None,poss=None,chs=None): + self.obs0.append(np.array([obs0])) + self.feat0.append(np.array([feat0])) + self.actions.append(np.array([act], dtype=np.int32)) + self.rewards.append(np.array([reward])) + self.terminals.append(np.array([not alive], dtype=np.bool)) + + if self.use_mean: + self.prob.append(np.array([prob])) + if self.use_msg: + self.msgs.append(np.array([msgs])) + if self.use_mean_msg: + self.mean_msg.append(np.array([mean_msg])) + if self.needPoss: + self.poss.append(np.array([poss])) + if self.needChs: + # print(chs) + # pdb.set_trace() + self.chs.append(np.array([chs])) + + def pull(self): + res = { + 'obs0': self.obs0.pull(), + 'feat0': self.feat0.pull(), + 'act': self.actions.pull(), + 'poss': None if not self.needPoss else self.poss.pull(), + 'chs': None if not self.needChs else self.chs.pull(), + 'rewards': self.rewards.pull(), + 'terminals': self.terminals.pull(), + 'prob': None if not self.use_mean else self.prob.pull(), + 'msgs': None if not self.use_msg else self.msgs.pull(), + 'mean_msg': None if not self.use_mean_msg else self.mean_msg.pull() + } + + return res + +class DecodeBuffer(object): + ## TODO + ### Write the decode and verify the center's node features + ### can be decoded to the map status + def __init__(self,map_shape,feature_shape,max_len): + self.agent=dict() + self.max_len=max_len + self.map_shape=map_shape + self.feature_shape=feature_shape + self.map=MetaBuffer(map_shape,max_len) + self.feat=MetaBuffer(feature_shape,max_len) + + def _flush(self,**kwargs): + self.map.append(kwargs['map']) + self.feat.append(kwargs['feat']) + + def test_buffer(self): + if self.nb_entries/400>10: + self.agent=dict() + + def push(self,**kwargs): + self.agent[self] + + +class MemoryGroup(object): + def __init__(self, obs_shape, feat_shape, act_n, max_len, batch_size, sub_len,msg_bits=3,msg_n=10,use_mean=False,use_msg=False,use_mean_msg=False,needPoss=False,needChs=False): + self.agent = dict() + self.max_len = max_len + self.batch_size = batch_size + self.obs_shape = obs_shape + self.feat_shape = feat_shape + self.sub_len = sub_len + self.use_mean = use_mean + self.use_msg=use_msg + self.use_mean_msg=use_mean_msg + self.needPoss=needPoss + self.needChs=needChs + self.act_n = act_n + self.msg_bits=msg_bits + self.msg_n=msg_n + self.obs0 = MetaBuffer(obs_shape, max_len) + self.feat0 = MetaBuffer(feat_shape, max_len) + self.actions = MetaBuffer((), max_len, dtype='int32') + self.rewards = MetaBuffer((), max_len) + self.terminals = MetaBuffer((), max_len, dtype='bool') + self.masks = MetaBuffer((), max_len, dtype='bool') + + if use_mean: + self.prob = MetaBuffer((act_n,), max_len) + if use_msg: + self.msgs = MetaBuffer((msg_bits,msg_n[0],msg_n[1]),max_len) + if use_mean_msg: + self.mean_msg=MetaBuffer((msg_bits,),max_len) + if needPoss: + self.poss=MetaBuffer((2,),max_len) + if needChs: + self.chs=MetaBuffer((),max_len,dtype='int32') + self._new_add = 0 + + def _flush(self, **kwargs): + self.obs0.append(kwargs['obs0']) + self.feat0.append(kwargs['feat0']) + self.actions.append(kwargs['act']) + self.rewards.append(kwargs['rewards']) + self.terminals.append(kwargs['terminals']) + # print(kwargs.keys()) + if self.use_mean: + self.prob.append(kwargs['prob']) + + if self.use_msg: + self.msgs.append(kwargs['msgs']) + if self.use_mean_msg: + # import pdb; pdb.set_trace() + self.mean_msg.append(kwargs['mean_msg']) + if self.needPoss: + self.poss.append(kwargs['poss']) + if self.needChs: + self.chs.append(kwargs['chs']) + + + mask = np.where(kwargs['terminals'] == True, False, True) + # mask[-1] = False + self.masks.append(mask) + + def test_buffer(self): + if self.nb_entries/400>18: + self.agent.clear() + def push(self, **kwargs): + + # m=np.random.randint(0,1000) + # pdb.set_trace() + # if self.nb_entries/400>40: + # self.agent = dict() # clear + self.agent[self.nb_entries] = AgentMemory(self.obs_shape, self.feat_shape, self.act_n, self.sub_len, self.msg_bits,self.msg_n,use_mean=self.use_mean,use_msg=self.use_msg,use_mean_msg=self.use_mean_msg,needPoss=self.needPoss,needChs=self.needChs) + + if self.needChs: + l=len(kwargs['chs']) + for i, _id in enumerate(kwargs['ids']): + # if self.agent.get(_id) is None: + # print(self.nb_entries) + # print(kwargs['poss']) + if self.needChs and i>=l: + j=l-1 + else: + j=i + self.agent[self.nb_entries-1].append(obs0=kwargs['state'][0][i], feat0=kwargs['state'][1][i], act=kwargs['acts'][i], reward=kwargs['rewards'][i], alive=kwargs['alives'][i], + prob=kwargs['prob'][i] if self.use_mean else None,msgs=kwargs['msgs'][i] if self.use_msg else None,mean_msg=kwargs['mean_msg'][i] if self.use_mean_msg else None,poss=kwargs['poss'][i] if self.needPoss else None, + chs=kwargs['chs'][j] if self.needChs else None) + # tmp=self.agent[self.nb_entries-1] + # pdb.set_trace() + self._new_add+=1 + + def tight(self): + # ids = list(self.agent.keys()) + # np.random.shuffle(ids) + # for ele in ids: + # tmp = self.agent[ele].pull() + # self._new_add += len(tmp['obs0']) + # self._flush(**tmp) + # if self.nb_entries/400>40: + # self.agent = dict() # clear + # tmp=len(self.agent[self.nb_entries-1].masks.data) + self.agent[self.nb_entries-1].is_end=True + # None + def sample(self): + # print(self.batch_size) + # print(self.NB) + idx = np.random.choice(self.nb_entries-1, size=1) + idx=idx[0] + # tmp=self.agent[idx] + # pdb.set_trace() + while self.agent[idx].is_end and np.sum(self.agent[idx].terminals.pull())!=len(self.agent[idx].terminals.pull()): + idx = np.random.choice(self.nb_entries-1, size=1) + idx=idx[0] + next_idx = (idx + 1) % self.nb_entries + # pdb.set_trace() + obs = self.agent[idx].obs0.pull() + obs_next = self.agent[next_idx].obs0.pull() + feature = self.agent[idx].feat0.pull() + feature_next = self.agent[next_idx].feat0.pull() + actions = self.agent[idx].actions.pull() + rewards = self.agent[idx].rewards.pull() + + dones = self.agent[idx].terminals.pull() + # masks = self.agent[idx].masks #TODO + masks = np.where(dones == True, False, True) + # mask[-1] = False + rt=[ obs, feature, actions, obs_next,feature_next,rewards,dones,masks] + + + if self.use_mean: + # pdb.set_trace() + act_prob = self.agent[idx].prob.pull() + act_next_prob = self.agent[next_idx].prob.pull() + rt.append(act_prob) + rt.append(act_next_prob) + else: + rt.append(None) + rt.append(None) + + + if self.use_msg: + msgs=self.agent[idx].msgs.pull() + msgs_next=self.agent[next_idx].msgs.pull() + rt.append(msgs) + rt.append(msgs_next) + else: + rt.append(None) + rt.append(None) + + if self.use_mean_msg: + mean_msg=self.agent[idx].mean_msg.pull() + mean_msg_next=self.agent[next_idx].mean_msg.pull() + rt.append(mean_msg) + rt.append(mean_msg_next) + else: + rt.append(None) + rt.append(None) + if self.needPoss: + poss = self.agent[idx].poss.pull() + poss_next = self.agent[next_idx].poss.pull() + rt.append(poss) + rt.append(poss_next) + else: + rt.append(None) + rt.append(None) + + if self.needChs: + # pdb.set_trace() + chs = self.agent[idx].chs.pull() + chs_next = self.agent[next_idx].chs.pull() + rt.append(chs) + rt.append(chs_next) + else: + rt.append(None) + rt.append(None) + return rt + # def sample(self): + # print(self.batch_size) + # # print(self.NB) + # idx = np.random.choice(self.nb_entries, size=self.batch_size) + # next_idx = (idx + 1) % self.nb_entries + + # obs = self.obs0.sample(idx) + # obs_next = self.obs0.sample(next_idx) + # feature = self.feat0.sample(idx) + # feature_next = self.feat0.sample(next_idx) + # actions = self.actions.sample(idx) + # rewards = self.rewards.sample(idx) + + # dones = self.terminals.sample(idx) + # masks = self.masks.sample(idx) + # rt=[ obs, feature, actions, obs_next,feature_next,rewards,dones,masks] + + + # if self.use_mean: + # act_prob = self.prob.sample(idx) + # act_next_prob = self.prob.sample(next_idx) + # rt.append(act_prob) + # rt.append(act_next_prob) + # else: + # rt.append(None) + # rt.append(None) + + + # if self.use_msg: + # msgs=self.msgs.sample(idx) + # msgs_next=self.msgs.sample(next_idx) + # rt.append(msgs) + # rt.append(msgs_next) + # else: + # rt.append(None) + # rt.append(None) + + # if self.use_mean_msg: + # mean_msg=self.mean_msg.sample(idx) + # mean_msg_next=self.mean_msg.sample(next_idx) + # rt.append(mean_msg) + # rt.append(mean_msg_next) + # else: + # rt.append(None) + # rt.append(None) + # if self.needPoss: + # poss = self.poss.sample(idx) + # poss_next = self.poss.sample(next_idx) + # rt.append(poss) + # rt.append(poss_next) + # else: + # rt.append(None) + # rt.append(None) + + # if self.needChs: + # pdb.set_trace() + # chs = self.chs.sample(idx) + # chs_next = self.chs.sample(next_idx) + # rt.append(chs) + # rt.append(chs_next) + # else: + # rt.append(None) + # rt.append(None) + # return rt + + + def get_batch_num(self): + print('\n[INFO] Length of buffer and new add:', len(self.agent), self._new_add) + + #if self.nb_entries>500: + res = self._new_add * 2 + self._new_add = 0 + #else: + # res=0 + # self._new_add=0 + # pdb.set_trace() + return res + + @property + def nb_entries(self): + # if len(self.agent)%400==0: + # print(len(self.agent)) + return len(self.agent) + + +class SummaryObj: + + """ + Define a summary holder + """ + def __init__(self, log_dir, log_name, n_group=1): + self.name_set = set() + self.gra = tf.Graph() + self.n_group = n_group + + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + sess_config.gpu_options.allow_growth = True + + with self.gra.as_default(): + self.sess = tf.Session(graph=self.gra, config=sess_config) + self.train_writer = tf.summary.FileWriter(log_dir + "/" + log_name, graph=tf.get_default_graph()) + self.sess.run(tf.global_variables_initializer()) + + def register(self, name_list): + """Register summary operations with a list contains names for these operations + + Parameters + ---------- + name_list: list, contains name whose type is str + """ + + with self.gra.as_default(): + for name in name_list: + if name in self.name_set: + raise Exception("You cannot define different operations with same name: `{}`".format(name)) + self.name_set.add(name) + setattr(self, name, [tf.placeholder(tf.float32, shape=None, name='Agent_{}_{}'.format(i, name)) + for i in range(self.n_group)]) + setattr(self, name + "_op", [tf.summary.scalar('Agent_{}_{}_op'.format(i, name), getattr(self, name)[i]) + for i in range(self.n_group)]) + + def write(self, summary_dict, step): + """Write summary related to a certain step + + Parameters + ---------- + summary_dict: dict, summary value dict + step: int, global step + """ + + assert isinstance(summary_dict, dict) + for key, value in summary_dict.items(): + if key not in self.name_set: + raise Exception("Undefined operation: `{}`".format(key)) + if isinstance(value, list): + for i in range(self.n_group): + self.train_writer.add_summary(self.sess.run(getattr(self, key + "_op")[i], feed_dict={ + getattr(self, key)[i]: value[i]}), global_step=step) + else: + self.train_writer.add_summary(self.sess.run(getattr(self, key + "_op")[0], feed_dict={ + getattr(self, key)[0]: value}), global_step=step) + + +class Runner(object): + def __init__(self, sess, env, handles, map_size, max_steps, models,MsgModels, + play_handle, render_every=None, save_every=None, tau=None, log_name=None, log_dir=None, model_dir=None, train=False,len_nei=40,rewardtype='self',crp=False,crps=[False,False],is_selfplay=False,is_fix=False): + """Initialize runner + + Parameters + ---------- + sess: tf.Session + session + env: magent.GridWorld + environment handle + handles: list + group handles + map_size: int + map size of grid world + max_steps: int + the maximum of stages in a episode + render_every: int + render environment interval + save_every: int + states the interval of evaluation for self-play update + models: list + contains models + play_handle: method like + run game + tau: float + tau index for self-play update + log_name: str + define the name of log dir + log_dir: str + donates the directory of logs + model_dir: str + donates the dircetory of models + """ + self.env = env + self.models = models + self.MsgModels=MsgModels + self.max_steps = max_steps + self.handles = handles + self.map_size = map_size + self.render_every = render_every + self.save_every = save_every + self.play = play_handle + self.model_dir = model_dir + self.train = train + self.len_nei=len_nei + self.rewardtype=rewardtype + self.is_fix=is_fix + self.crp=crp + self.crps=crps + self.update_cnt=0 + self.is_selfplay=is_selfplay + if self.train: + self.summary = SummaryObj(log_name=log_name, log_dir=log_dir) + + summary_items = ['ave_agent_reward', 'total_reward', 'kill', "Sum_Reward", "Kill_Sum","var","oppo-ave-agent-reward",'oppo-total-reward','oppo-kill'] + self.summary.register(summary_items) # summary register + self.summary_items = summary_items + # self.si + # pdb.set_trace() + + assert isinstance(sess, tf.Session) + assert self.models[0].name_scope != self.models[1].name_scope + self.sess = sess + l_vars, r_vars = self.models[0].vars, self.models[1].vars + # print(len(l_vars),len(r_vars)) + if self.MsgModels[0]: + l_m_vars, r_m_vars = self.MsgModels[0].vars,self.MsgModels[1].vars + self.m_sp_op = [tf.assign(r_m_vars[i], (1. - tau) * l_m_vars[i] + tau * r_m_vars[i]) + for i in range(len(l_m_vars))] + + # assert len(l_vars) == len(r_vars) + if self.is_selfplay: + self.sp_op = [tf.assign(r_vars[i], (1. - tau) * l_vars[i] + tau * r_vars[i]) + for i in range(len(l_vars))] + + if not os.path.exists(self.model_dir): + os.makedirs(self.model_dir) + + def run(self, variant_eps, iteration, win_cnt=None,total_rewards=None): + info = {'main': None, 'opponent': None} + + # print(self.usecmc) + # pass + info['main'] = {'ave_agent_reward': 0., 'total_reward': 0., 'kill':0.,'oppo-ave-agent-reward':0.,'oppo-total-reward':0.,'oppo-kill':0} + info['opponent'] = {'ave_agent_reward': 0., 'total_reward': 0., 'kill': 0} + + max_nums, nums, agent_r_records, total_rewards = self.play(env=self.env, n_round=iteration, map_size=self.map_size, max_steps=self.max_steps, handles=self.handles, + models=self.models, MsgModels=self.MsgModels,print_every=50, eps=variant_eps, render=(iteration + 1) % self.render_every if self.render_every > 0 else False, train=self.train,len_nei=self.len_nei,rewardtype=self.rewardtype,crp=self.crp, + crps=self.crps,selfplay=self.is_selfplay,is_fix=self.is_fix) + + for i, tag in enumerate(['main', 'opponent']): + info[tag]['total_reward'] = total_rewards[i] + info[tag]['kill'] = max_nums[i] - nums[1 - i] + info[tag]['ave_agent_reward'] = agent_r_records[i] +# info[tag]['getkilled']=max_nums[i]-nums[i] + info['main']['oppo-ave-agent-reward']=info['opponent']['ave_agent_reward'] +# info['main']['getkilled']=info['opponent']['kill'] + info['main']['oppo-kill']=info['opponent']['kill'] + info['main']['oppo-total-reward']=info['opponent']['total_reward'] + + if self.train: + print('\n[INFO] {}'.format(info['main'])) + + # if self.save_every and (iteration + 1) % self.save_every == 0: + if info['main']['total_reward'] > info['opponent']['total_reward'] and self.is_selfplay: + print(Color.INFO.format('\n[INFO] Begin self-play Update ...')) + self.sess.run(self.sp_op) + if self.MsgModels[0]: + self.sess.run(self.m_sp_op) + print(Color.INFO.format('[INFO] Self-play Updated!\n')) + + print(Color.INFO.format('[INFO] Saving model ...')) + self.update_cnt+=1 + + if self.update_cnt%7==0 and self.update_cnt > 300 and iteration>1200: + self.models[0].save(self.model_dir + '-0', str(self.len_nei)+'-'+str(iteration)+self.rewardtype+('crp' if self.crp else 'nom')+('w' if self.MsgModels[0] else 'nw')) + self.models[1].save(self.model_dir + '-1', str(self.len_nei)+'-'+str(iteration)+self.rewardtype+('crp' if self.crp else 'nom')+('w' if self.MsgModels[0] else 'nw')) + if self.MsgModels[0]: + self.MsgModels[0].save(self.model_dir + '-msg0', str(self.len_nei)+'-'+str(iteration)+self.rewardtype+('crp' if self.crp else 'nom')+'w') + self.MsgModels[1].save(self.model_dir + '-msg1', str(self.len_nei)+'-'+str(iteration)+self.rewardtype+('crp' if self.crp else 'nom')+'w') + self.summary.write(info['main'], iteration) + elif not self.is_selfplay and not self.is_fix: + print(Color.INFO.format('\n[INFO] Begin self-play Update ...')) + print(Color.INFO.format('[INFO] Self-play Updated!\n')) + + print(Color.INFO.format('[INFO] Saving model ...')) + self.update_cnt+=1 + #if self.update_cnt%7==0 and self.update_cnt > 300: + # self.models[0].save(self.model_dir + '-0', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+('w' if self.MsgModels[0] else 'nw')) + # self.models[1].save(self.model_dir + '-2', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+('w' if self.MsgModels[0] else 'nw')) + # if self.MsgModels[0]: + # self.MsgModels[0].save(self.model_dir + '-msg0', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+'w') + # self.MsgModels[1].save(self.model_dir + '-msg1', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+'w') + self.summary.write(info['main'], iteration) + # self.summary.write(info['opponent'],iteration) + elif not self.is_selfplay and self.is_fix: + print(Color.INFO.format('\n[INFO] Begin self-play Update ...')) + print(Color.INFO.format('[INFO] Self-play Updated!\n')) + + print(Color.INFO.format('[INFO] Saving model ...')) + self.update_cnt+=1 + if self.update_cnt%7==0 and self.update_cnt > 500: + self.models[0].save(self.model_dir + '-0', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+('w' if self.MsgModels[0] else 'nw')) + if self.MsgModels[0]: + self.MsgModels[0].save(self.model_dir + '-msg0', str(self.len_nei)+'-'+str(iteration%5)+self.rewardtype+('crp' if self.crp else 'nom')+'w') + self.summary.write(info['main'], iteration) + # self.summary.write(info['opponent'],iteration) + + else: + print('\n[INFO] {0} \n {1}'.format(info['main'], info['opponent'])) + if info['main']['kill'] > info['opponent']['kill']: + win_cnt['main'] += 1 + elif info['main']['kill'] < info['opponent']['kill']: + win_cnt['opponent'] += 1 + else: + win_cnt['main'] += 1 + win_cnt['opponent'] += 1 + + return info['main']['ave_agent_reward'] + #,info['main']['kill'],info['main']['getkilled'] diff --git a/examples/battle_model/build.sh b/examples/battle_model/build.sh new file mode 100755 index 0000000..09eb5ed --- /dev/null +++ b/examples/battle_model/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $1 ] && [ $1 = '-c' ] # clean +then + rm -rf *.log save_model build +fi + +mkdir -p build +cd build +cmake .. + +if [[ "$OSTYPE" == "linux-gnu" ]]; then + # Linux + make -j `nproc` +elif [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + make -j `sysctl -n hw.ncpu` +fi diff --git a/examples/battle_model/python/magent/README.md b/examples/battle_model/python/magent/README.md new file mode 100755 index 0000000..47291b9 --- /dev/null +++ b/examples/battle_model/python/magent/README.md @@ -0,0 +1,7 @@ +- builtin: built-in configuration and RL algorithms +- renderer: pygame renderer +- c_lib.py: call c functions +- gridworld.py: gridworld interface +- model.py: base class for model +- environment.py: base class for environment +- utility.py: utilities diff --git a/examples/battle_model/python/magent/__init__.py b/examples/battle_model/python/magent/__init__.py new file mode 100755 index 0000000..8987530 --- /dev/null +++ b/examples/battle_model/python/magent/__init__.py @@ -0,0 +1,8 @@ +from . import model +from . import utility +from . import gridworld + +# some alias +GridWorld = gridworld.GridWorld +ProcessingModel = model.ProcessingModel +round = utility.rec_round diff --git a/examples/battle_model/python/magent/__init__.pyc b/examples/battle_model/python/magent/__init__.pyc new file mode 100755 index 0000000..1311735 Binary files /dev/null and b/examples/battle_model/python/magent/__init__.pyc differ diff --git a/examples/battle_model/python/magent/builtin/__init__.py b/examples/battle_model/python/magent/builtin/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/examples/battle_model/python/magent/builtin/__init__.pyc b/examples/battle_model/python/magent/builtin/__init__.pyc new file mode 100755 index 0000000..ff8a3c4 Binary files /dev/null and b/examples/battle_model/python/magent/builtin/__init__.pyc differ diff --git a/examples/battle_model/python/magent/builtin/common.py b/examples/battle_model/python/magent/builtin/common.py new file mode 100755 index 0000000..3ffda6a --- /dev/null +++ b/examples/battle_model/python/magent/builtin/common.py @@ -0,0 +1,45 @@ +"""Replay buffer for deep q network""" + +import numpy as np + + +class ReplayBuffer: + """a circular queue based on numpy array, supporting batch put and batch get""" + def __init__(self, shape, dtype=np.float32): + self.buffer = np.empty(shape=shape, dtype=dtype) + self.head = 0 + self.capacity = len(self.buffer) + + def put(self, data): + """put data to + + Parameters + ---------- + data: numpy array + data to add + """ + head = self.head + n = len(data) + if head + n <= self.capacity: + self.buffer[head:head+n] = data + self.head = (self.head + n) % self.capacity + else: + split = self.capacity - head + self.buffer[head:] = data[:split] + self.buffer[:n - split] = data[split:] + self.head = split + return n + + def get(self, index): + """get items + + Parameters + ---------- + index: int or numpy array + it can be any numpy supported index + """ + return self.buffer[index] + + def clear(self): + """clear replay buffer""" + self.head = 0 diff --git a/examples/battle_model/python/magent/builtin/config/__init__.py b/examples/battle_model/python/magent/builtin/config/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/examples/battle_model/python/magent/builtin/config/battle.py b/examples/battle_model/python/magent/builtin/config/battle.py new file mode 100755 index 0000000..be2eaa5 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/config/battle.py @@ -0,0 +1,33 @@ +""" battle of two armies """ + +import magent + + +def get_config(map_size): + gw = magent.gridworld + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + cfg.set({"minimap_mode": True}) + cfg.set({"embedding_size": 10}) + + small = cfg.register_agent_type( + "small", + {'width': 1, 'length': 1, 'hp': 10, 'speed': 2, + 'view_range': gw.CircleRange(6), 'attack_range': gw.CircleRange(1.5), + 'damage': 2, 'step_recover': 0.1, + + 'step_reward': -0.005, 'kill_reward': 5, 'dead_penalty': -0.1, 'attack_penalty': -0.1, + }) + + g0 = cfg.add_group(small) + g1 = cfg.add_group(small) + + a = gw.AgentSymbol(g0, index='any') + b = gw.AgentSymbol(g1, index='any') + + # reward shaping to encourage attack + cfg.add_reward_rule(gw.Event(a, 'attack', b), receiver=a, value=0.2) + cfg.add_reward_rule(gw.Event(b, 'attack', a), receiver=b, value=0.2) + + return cfg diff --git a/examples/battle_model/python/magent/builtin/config/double_attack.py b/examples/battle_model/python/magent/builtin/config/double_attack.py new file mode 100755 index 0000000..b75eee9 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/config/double_attack.py @@ -0,0 +1,42 @@ +""" +A cooperation game, tigers must attack a same deer simultaneously to get reward +""" + +import magent + + +def get_config(map_size): + gw = magent.gridworld + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + cfg.set({"embedding_size": 10}) + + deer = cfg.register_agent_type( + "deer", + {'width': 1, 'length': 1, 'hp': 5, 'speed': 1, + 'view_range': gw.CircleRange(1), 'attack_range': gw.CircleRange(0), + 'step_recover': 0.2, + 'kill_supply': 8, + }) + + tiger = cfg.register_agent_type( + "tiger", + {'width': 1, 'length': 1, 'hp': 10, 'speed': 1, + 'view_range': gw.CircleRange(4), 'attack_range': gw.CircleRange(1), + 'damage': 1, 'step_recover': -0.2, + }) + + deer_group = cfg.add_group(deer) + tiger_group = cfg.add_group(tiger) + + a = gw.AgentSymbol(tiger_group, index='any') + b = gw.AgentSymbol(tiger_group, index='any') + c = gw.AgentSymbol(deer_group, index='any') + + # tigers get reward when they attack a deer simultaneously + e1 = gw.Event(a, 'attack', c) + e2 = gw.Event(b, 'attack', c) + cfg.add_reward_rule(e1 & e2, receiver=[a, b], value=[1, 1]) + + return cfg diff --git a/examples/battle_model/python/magent/builtin/config/forest.py b/examples/battle_model/python/magent/builtin/config/forest.py new file mode 100755 index 0000000..37e7354 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/config/forest.py @@ -0,0 +1,34 @@ +""" tigers eat deer to get health point and reward""" + +import magent + + +def get_config(map_size): + gw = magent.gridworld + + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + cfg.set({"embedding_size": 10}) + + deer = cfg.register_agent_type( + "deer", + {'width': 1, 'length': 1, 'hp': 5, 'speed': 1, + 'view_range': gw.CircleRange(1), 'attack_range': gw.CircleRange(0), + 'damage': 0, 'step_recover': 0.2, + 'food_supply': 0, 'kill_supply': 8, + }) + + tiger = cfg.register_agent_type( + "tiger", + {'width': 1, 'length': 1, 'hp': 10, 'speed': 1, + 'view_range': gw.CircleRange(4), 'attack_range': gw.CircleRange(1), + 'damage': 3, 'step_recover': -0.5, + 'food_supply': 0, 'kill_supply': 0, + 'step_reward': 1, 'attack_penalty': -0.1, + }) + + deer_group = cfg.add_group(deer) + tiger_group = cfg.add_group(tiger) + + return cfg diff --git a/examples/battle_model/python/magent/builtin/config/pursuit.py b/examples/battle_model/python/magent/builtin/config/pursuit.py new file mode 100755 index 0000000..8364358 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/config/pursuit.py @@ -0,0 +1,33 @@ +import magent + + +def get_config(map_size): + gw = magent.gridworld + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + + predator = cfg.register_agent_type( + "predator", + { + 'width': 2, 'length': 2, 'hp': 1, 'speed': 1, + 'view_range': gw.CircleRange(5), 'attack_range': gw.CircleRange(2), + 'attack_penalty': -0.2 + }) + + prey = cfg.register_agent_type( + "prey", + { + 'width': 1, 'length': 1, 'hp': 1, 'speed': 1.5, + 'view_range': gw.CircleRange(4), 'attack_range': gw.CircleRange(0) + }) + + predator_group = cfg.add_group(predator) + prey_group = cfg.add_group(prey) + + a = gw.AgentSymbol(predator_group, index='any') + b = gw.AgentSymbol(prey_group, index='any') + + cfg.add_reward_rule(gw.Event(a, 'attack', b), receiver=[a, b], value=[1, -1]) + + return cfg diff --git a/examples/battle_model/python/magent/builtin/mx_model/__init__.py b/examples/battle_model/python/magent/builtin/mx_model/__init__.py new file mode 100755 index 0000000..c727776 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/mx_model/__init__.py @@ -0,0 +1,2 @@ +from .dqn import DeepQNetwork +from .a2c import AdvantageActorCritic diff --git a/examples/battle_model/python/magent/builtin/mx_model/a2c.py b/examples/battle_model/python/magent/builtin/mx_model/a2c.py new file mode 100755 index 0000000..3e5f53e --- /dev/null +++ b/examples/battle_model/python/magent/builtin/mx_model/a2c.py @@ -0,0 +1,297 @@ +"""advantage actor critic""" + +import os +import time + +import numpy as np +import mxnet as mx + +from .base import MXBaseModel + + +class AdvantageActorCritic(MXBaseModel): + def __init__(self, env, handle, name, eval_obs=None, + batch_size=64, reward_decay=0.99, learning_rate=1e-3, + train_freq=1, value_coef=0.1, ent_coef=0.1, + custom_view_space=None, custom_feature_space=None, + *args, **kwargs): + """init a model + + Parameters + ---------- + env: Environment + environment + handle: Handle (ctypes.c_int32) + handle of this group, can be got by env.get_handles + name: str + name of this model + learning_rate: float + batch_size: int + reward_decay: float + reward_decay in TD + eval_obs: numpy array + evaluation set of observation + train_freq: int + mean training times of a sample + ent_coef: float + weight of entropy loss in total loss + value_coef: float + weight of value loss in total loss + custom_feature_space: tuple + customized feature space + custom_view_space: tuple + customized feature space + """ + MXBaseModel.__init__(self, env, handle, name, "mxa2c") + # ======================== set config ======================== + self.env = env + self.handle = handle + self.view_space = custom_view_space or env.get_view_space(handle) + self.feature_space = custom_feature_space or env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + self.reward_decay = reward_decay + + self.batch_size = batch_size + self.learning_rate= learning_rate + self.train_freq = train_freq # train time of every sample (s,a,r,s') + self.eval_obs = eval_obs + + self.value_coef = value_coef + self.ent_coef = ent_coef + + self.train_ct = 0 + + # ======================= build network ======================= + self.ctx = self._get_ctx() + + self.input_view = mx.sym.var("input_view") + self.input_feature = mx.sym.var("input_feature") + + policy, value = self._create_network(self.input_view, self.input_feature) + + log_policy = mx.sym.log(policy) + out_policy = mx.sym.BlockGrad(policy) + + neg_entropy = ent_coef * mx.sym.sum(policy * log_policy, axis=1) + neg_entropy = mx.sym.MakeLoss(data=neg_entropy) + + self.sym = mx.sym.Group([log_policy, value, neg_entropy, out_policy]) + self.model = mx.mod.Module(self.sym, data_names=['input_view', 'input_feature'], + label_names=None, context=self.ctx) + + # bind (set initial batch size) + self.bind_size = batch_size + self.model.bind(data_shapes=[('input_view', (batch_size,) + self.view_space), + ('input_feature', (batch_size,) + self.feature_space)], + label_shapes=None) + + # init params + self.model.init_params(initializer=mx.init.Xavier()) + self.model.init_optimizer(optimizer='adam', optimizer_params={ + 'learning_rate': learning_rate, + 'clip_gradient': 10, + }) + + # init training buffers + self.view_buf = np.empty((1,) + self.view_space) + self.feature_buf = np.empty((1,) + self.feature_space) + self.action_buf = np.empty(1, dtype=np.int32) + self.advantage_buf, self.value_buf = np.empty(1), np.empty(1) + self.terminal_buf = np.empty(1, dtype=np.bool) + + # print("parameters", self.model.get_params()) + # mx.viz.plot_network(self.output).view() + + def _create_network(self, input_view, input_feature): + """define computation graph of network + + Parameters + ---------- + view_space: tuple + feature_space: tuple + the input shape + """ + kernel_num = [32, 32] + hidden_size = [256] + + if False: + h_conv1 = mx.sym.Convolution(data=input_view, kernel=(3, 3), + num_filter=kernel_num[0], layout="NHWC") + h_conv1 = mx.sym.Activation(data=h_conv1, act_type="relu") + h_conv2 = mx.sym.Convolution(data=h_conv1, kernel=(3, 3), + num_filter=kernel_num[1], layout="NHWC") + h_conv2 = mx.sym.Activation(data=h_conv2, act_type="relu") + else: + input_view = mx.sym.flatten(data=input_view) + h_conv2 = mx.sym.FullyConnected(input_view, num_hidden=hidden_size[0]) + h_conv2 = mx.sym.Activation(data=h_conv2, act_type="relu") + + flatten_view = mx.sym.flatten(data=h_conv2) + h_view = mx.sym.FullyConnected(data=flatten_view, num_hidden=hidden_size[0]) + h_view = mx.sym.Activation(data=h_view, act_type="relu") + + h_emb = mx.sym.FullyConnected(data=input_feature, num_hidden=hidden_size[0]) + h_emb = mx.sym.Activation(data=h_emb, act_type="relu") + + dense = h_view + h_emb + + policy = mx.sym.FullyConnected(data=dense, num_hidden=self.num_actions, no_bias=True) + policy = mx.sym.SoftmaxActivation(data=policy) + policy = mx.sym.clip(data=policy, a_min=1e-5, a_max=1 - 1e-5) + value = mx.sym.FullyConnected(data=dense, num_hidden=1) + + return policy, value + + def infer_action(self, raw_obs, ids, policy="e_greedy", eps=0): + """infer action for a batch of agents + + Parameters + ---------- + raw_obs: tuple(numpy array, numpy array) + raw observation of agents tuple(views, features) + ids: numpy array + ids of agents + + Returns + ------- + acts: numpy array of int32 + actions for agents + """ + view, feature = raw_obs[0], raw_obs[1] + n = len(view) + + ret = np.empty(n, dtype=np.int32) + self._reset_bind_size(n) + data_batch = mx.io.DataBatch(data=[mx.nd.array(view), mx.nd.array(feature)]) + self.model.forward(data_batch, is_train=False) + policy = self.model.get_outputs()[3].asnumpy() + + actions = np.arange(self.num_actions) + for i in range(n): + ret[i] = np.random.choice(actions, p=policy[i]) + return ret + + def train(self, sample_buffer, print_every=1000): + """feed new data sample and train + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + + Returns + ------- + loss: list + policy gradient loss, critic loss, entropy loss + value: float + estimated state value + """ + # calc buffer size + n = 0 + for episode in sample_buffer.episodes(): + if episode.terminal: + n += len(episode.rewards) + else: + n += len(episode.rewards) - 1 + + if n == 0: + return [0.0, 0.0, 0.0], 0.0 + + # resize to the new size + self.view_buf.resize((n,) + self.view_space) + self.feature_buf.resize((n,) + self.feature_space) + self.action_buf.resize(n) + self.value_buf.resize(n) + self.advantage_buf.resize(n) + view, feature = self.view_buf, self.feature_buf + action, value = self.action_buf, self.value_buf + advantage = self.advantage_buf + + ct = 0 + gamma = self.reward_decay + # collect episodes from multiple separate buffers to a continuous buffer + + for episode in sample_buffer.episodes(): + v, f, a, r = episode.views, episode.features, episode.actions, episode.rewards + + m = len(episode.rewards) + self._reset_bind_size(m) + data_batch = mx.io.DataBatch(data=[mx.nd.array(v), mx.nd.array(f)]) + self.model.forward(data_batch, is_train=False) + value = self.model.get_outputs()[1].asnumpy().flatten() + + delta_t = np.empty(m) + if episode.terminal: + delta_t[:m-1] = r[:m-1] + gamma * value[1:m] - value[:m-1] + delta_t[m-1] = r[m-1] + gamma * 0 - value[m-1] + else: + delta_t[:m-1] = r[:m-1] + gamma * value[1:m] - value[:m-1] + m -= 1 + v, f, a = v[:-1], f[:-1], a[:-1] + + if m == 0: + continue + + # discount advantage + keep = 0 + for i in reversed(range(m)): + keep = keep * gamma + delta_t[i] + advantage[ct+i] = keep + + view[ct:ct+m] = v + feature[ct:ct+m] = f + action[ct:ct+m] = a + ct += m + assert n == ct + + n = len(advantage) + neg_advantage = -advantage + + neg_advs_np = np.zeros((n, self.num_actions), dtype=np.float32) + neg_advs_np[np.arange(n), action] = neg_advantage + neg_advs = mx.nd.array(neg_advs_np) + + # the grads of values are exactly negative advantages + v_grads = mx.nd.array(self.value_coef * (neg_advantage[:, np.newaxis])) + data_batch = mx.io.DataBatch(data=[mx.nd.array(view), mx.nd.array(feature)]) + self._reset_bind_size(n) + self.model.forward(data_batch, is_train=True) + self.model.backward(out_grads=[neg_advs, v_grads]) + self.model.update() + log_policy, value, entropy_loss, _ = self.model.get_outputs() + + value = mx.nd.mean(value).asnumpy()[0] + log_policy = log_policy.asnumpy()[np.arange(n), action] + pg_loss = np.mean(neg_advantage * log_policy) + entropy_loss = np.mean(entropy_loss.asnumpy()) + value_loss = self.value_coef * np.mean(np.square(advantage)) + + print("sample %d %.4f %.4f %.4f %.4f" % (n, pg_loss, value_loss, entropy_loss, value)) + return [pg_loss, value_loss, entropy_loss], value + + def _reset_bind_size(self, new_size): + """reset input shape of the model + + Parameters + ---------- + new_size: int + new batch size + """ + if self.bind_size == new_size: + return + else: + self.bind_size = new_size + self.model.reshape( + data_shapes=[ + ('input_view', (new_size,) + self.view_space), + ('input_feature', (new_size,) + self.feature_space)], + ) + + def get_info(self): + """get information of the model + + Returns + ------- + info: string + """ + return "mx dqn train_time: %d" % (self.train_ct) diff --git a/examples/battle_model/python/magent/builtin/mx_model/base.py b/examples/battle_model/python/magent/builtin/mx_model/base.py new file mode 100755 index 0000000..da06313 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/mx_model/base.py @@ -0,0 +1,66 @@ +import os +import mxnet as mx + +from magent.utility import has_gpu +from magent.model import BaseModel + + +class MXBaseModel(BaseModel): + def __init__(self, env, handle, name, subclass_name): + """init a model + + Parameters + ---------- + env: magent.Environment + handle: handle (ctypes.c_int32) + name: str + subclass_name: str + name of subclass + """ + BaseModel.__init__(self, env, handle) + self.name = name + self.subclass_name = subclass_name + + def _get_ctx(self): + """return correct context , priority: gpu > cpu + + Returns + ------- + ctx: mx.context + """ + if has_gpu(): + return mx.gpu() + else: + return mx.cpu() + + def save(self, dir_name, epoch): + """save model to dir + + Parameters + ---------- + dir_name: str + name of the directory + epoch: int + """ + if not os.path.exists(dir_name): + os.mkdir(dir_name) + dir_name = os.path.join(dir_name, self.name, ) + if not os.path.exists(dir_name): + os.mkdir(dir_name) + pre = os.path.join(dir_name, self.subclass_name) + self.model.save_checkpoint(pre, epoch, save_optimizer_states=True) + + def load(self, dir_name, epoch=0, name=None): + """save model to dir + + Parameters + ---------- + dir_name: str + name of the directory + epoch: int + """ + name = name or self.name + dir_name = os.path.join(dir_name, name) + pre = os.path.join(dir_name, self.subclass_name) + _, arg_params, aux_params = mx.model.load_checkpoint(pre, epoch) + self.model.set_params(arg_params, aux_params, force_init=True) diff --git a/examples/battle_model/python/magent/builtin/mx_model/dqn.py b/examples/battle_model/python/magent/builtin/mx_model/dqn.py new file mode 100755 index 0000000..69bca5d --- /dev/null +++ b/examples/battle_model/python/magent/builtin/mx_model/dqn.py @@ -0,0 +1,389 @@ +import time + +import numpy as np +import mxnet as mx + +from .base import MXBaseModel +from ..common import ReplayBuffer +from ...utility import has_gpu + + +class DeepQNetwork(MXBaseModel): + def __init__(self, env, handle, name, + batch_size=64, learning_rate=1e-4, reward_decay=0.99, + train_freq=1, target_update=2000, memory_size=2 ** 20, eval_obs=None, + use_dueling=True, use_double=True, infer_batch_size=8192, + custom_view_space=None, custom_feature_space=None, num_gpu=1): + """init a model + + Parameters + ---------- + env: Environment + environment + handle: Handle (ctypes.c_int32) + handle of this group, can be got by env.get_handles + name: str + name of this model + learning_rate: float + batch_size: int + reward_decay: float + reward_decay in TD + train_freq: int + mean training times of a sample + target_update: int + target will update every target_update batches + memory_size: int + weight of entropy loss in total loss + eval_obs: numpy array + evaluation set of observation + use_dueling: bool + whether use dueling q network + use_double: bool + whether use double q network + num_gpu: int + number of gpu + infer_batch_size: int + batch size while inferring actions + custom_feature_space: tuple + customized feature space + custom_view_space: tuple + customized feature space + """ + MXBaseModel.__init__(self, env, handle, name, "mxdqn") + # ======================== set config ======================== + self.env = env + self.handle = handle + self.view_space = custom_view_space or env.get_view_space(handle) + self.feature_space = custom_feature_space or env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + + self.batch_size = batch_size + self.infer_batch_size = infer_batch_size + self.learning_rate = learning_rate + self.train_freq = train_freq # train time of every sample (s,a,r,s') + self.target_update = target_update # update frequency of target network + self.eval_obs = eval_obs + self.num_gpu = num_gpu + + self.use_dueling = use_dueling + self.use_double = use_double + + self.train_ct = 0 + + # ======================= build network ======================= + self.ctx = self._get_ctx() + if self.num_gpu > 1 and self.ctx == mx.gpu(): + self.ctx = [] + for i in range(self.num_gpu): + self.ctx.append(mx.gpu(i)) + + self.input_view = mx.sym.var("input_view") + self.input_feature = mx.sym.var("input_feature") + self.mask = mx.sym.var("mask") + self.action = mx.sym.var("action") + self.target = mx.sym.var("target") + + self.qvalues = self._create_network(self.input_view, self.input_feature) + self.gamma = reward_decay + self.action_onehot = mx.sym.one_hot(self.action, depth=self.num_actions) + td_error = mx.sym.square(self.target - mx.sym.sum(self.qvalues * self.action_onehot, axis=1)) + self.loss = mx.sym.sum(td_error * self.mask) / mx.sym.sum(self.mask) + self.loss = mx.sym.MakeLoss(data=self.loss) + + self.out_qvalues = mx.sym.BlockGrad(self.qvalues) + self.output = mx.sym.Group([self.out_qvalues, self.loss]) + + self.model = mx.mod.Module(self.output, + data_names=['input_view', 'input_feature'], + label_names=['action', 'target', 'mask'], context=self.ctx) + + self.target_model = mx.mod.Module(self.qvalues, + data_names=['input_view', 'input_feature'], + label_names=[], context=self.ctx) + + # bind (set initial batch size) + self.bind_size = batch_size + self.model.bind(data_shapes=[('input_view', (batch_size,) + self.view_space), + ('input_feature', (batch_size,) + self.feature_space)], + label_shapes=[('action', (batch_size,)), + ('target', (batch_size,)), + ('mask', (batch_size,))]) + self.target_model.bind(data_shapes=[('input_view', (batch_size,) + self.view_space), + ('input_feature', (batch_size,) + self.feature_space)]) + + # init params + self.model.init_params(initializer=mx.init.Xavier()) + self.model.init_optimizer(optimizer='adam', optimizer_params={ + 'learning_rate': learning_rate, + 'clip_gradient': 10.0}) + + self._copy_network(self.target_model, self.model) + + # init replay buffers + self.replay_buf_len = 0 + self.memory_size = memory_size + self.replay_buf_view = ReplayBuffer(shape=(memory_size,) + self.view_space) + self.replay_buf_feature = ReplayBuffer(shape=(memory_size,) + self.feature_space) + self.replay_buf_action = ReplayBuffer(shape=(memory_size,), dtype=np.int32) + self.replay_buf_reward = ReplayBuffer(shape=(memory_size,)) + self.replay_buf_terminal = ReplayBuffer(shape=(memory_size,), dtype=np.bool) + self.replay_buf_mask = ReplayBuffer(shape=(memory_size,)) + # if mask[i] == 0, then the item is used for padding, not for training + + # print("parameters", self.model.get_params()) + # mx.viz.plot_network(self.loss).view() + + def _create_network(self, input_view, input_feature, use_conv=True): + """define computation graph of network + + Parameters + ---------- + input_view: mx.symbol + input_feature: mx.symbol + the input tensor + """ + kernel_num = [32, 32] + hidden_size = [256] + + if use_conv: + input_view = mx.sym.transpose(data=input_view, axes=[0, 3, 1, 2]) + h_conv1 = mx.sym.Convolution(data=input_view, kernel=(3, 3), + num_filter=kernel_num[0], layout="NCHW") + h_conv1 = mx.sym.Activation(data=h_conv1, act_type="relu") + h_conv2 = mx.sym.Convolution(data=h_conv1, kernel=(3, 3), + num_filter=kernel_num[1], layout="NCHW") + h_conv2 = mx.sym.Activation(data=h_conv2, act_type="relu") + else: + input_view = mx.sym.flatten(data=input_view) + h_conv2 = mx.sym.FullyConnected(input_view, num_hidden=hidden_size[0]) + h_conv2 = mx.sym.Activation(data=h_conv2, act_type="relu") + + flatten_view = mx.sym.flatten(data=h_conv2) + h_view = mx.sym.FullyConnected(data=flatten_view, num_hidden=hidden_size[0]) + h_view = mx.sym.Activation(data=h_view, act_type="relu") + + h_emb = mx.sym.FullyConnected(data=input_feature, num_hidden=hidden_size[0]) + h_emb = mx.sym.Activation(data=h_emb, act_type="relu") + + dense = mx.sym.concat(h_view, h_emb) + + if self.use_dueling: + # state value + value = mx.sym.FullyConnected(data=dense, num_hidden=1) + advantage = mx.sym.FullyConnected(data=dense, num_hidden=self.num_actions) + + mean = mx.sym.mean(advantage, axis=1, keepdims=True) + advantage = mx.sym.broadcast_sub(advantage, mean) + qvalues = mx.sym.broadcast_add(advantage, value) + else: + qvalues = mx.sym.FullyConnected(data=dense, num_hidden=self.num_actions) + + return qvalues + + def infer_action(self, raw_obs, ids, policy="e_greedy", eps=0): + """infer action for a batch of agents + + Parameters + ---------- + raw_obs: tuple(numpy array, numpy array) + raw observation of agents tuple(views, features) + ids: numpy array + ids of agents + policy: str + can be eps-greedy or greedy + eps: float + used when policy is eps-greedy + + Returns + ------- + acts: numpy array of int32 + actions for agents + """ + view, feature = raw_obs[0], raw_obs[1] + + if policy == 'e_greedy': + eps = eps + elif policy == 'greedy': + eps = 0 + + n = len(view) + if n < self.num_gpu: + view = np.tile(view, (self.num_gpu, 1, 1, 1)) + feature = np.tile(feature, (self.num_gpu, 1)) + + batch_size = min(len(view), self.infer_batch_size) + self._reset_bind_size(batch_size) + best_actions = [] + infer_iter = mx.io.NDArrayIter(data=[view, feature], batch_size=batch_size) + for batch in infer_iter: + self.model.forward(batch, is_train=False) + qvalue_batch = self.model.get_outputs()[0] + batch_action = mx.nd.argmax(qvalue_batch, axis=1) + best_actions.append(batch_action) + best_actions = np.array([x.asnumpy() for x in best_actions]).flatten() + best_actions = best_actions[:n] + + random = np.random.randint(self.num_actions, size=(n,)) + cond = np.random.uniform(0, 1, size=(n,)) < eps + ret = np.where(cond, random, best_actions) + + return ret.astype(np.int32) + + def _calc_target(self, next_view, next_feature, rewards, terminal): + """calculate target value""" + n = len(rewards) + + data_batch = mx.io.DataBatch(data=[mx.nd.array(next_view), mx.nd.array(next_feature)]) + self._reset_bind_size(n) + if self.use_double: + self.target_model.forward(data_batch, is_train=False) + self.model.forward(data_batch, is_train=False) + t_qvalues = self.target_model.get_outputs()[0].asnumpy() + qvalues = self.model.get_outputs()[0].asnumpy() + next_value = t_qvalues[np.arange(n), np.argmax(qvalues, axis=1)] + else: + self.target_model.forward(data_batch, is_train=False) + t_qvalues = self.target_model.get_outputs()[0].asnumpy() + next_value = np.max(t_qvalues, axis=1) + + target = np.where(terminal, rewards, rewards + self.gamma * next_value) + + return target + + def _add_to_replay_buffer(self, sample_buffer): + """add samples in sample_buffer to replay buffer""" + n = 0 + for episode in sample_buffer.episodes(): + v, f, a, r = episode.views, episode.features, episode.actions, episode.rewards + + m = len(r) + + mask = np.ones((m,)) + terminal = np.zeros((m,), dtype=np.bool) + if episode.terminal: + terminal[-1] = True + else: + mask[-1] = 0 + + self.replay_buf_view.put(v) + self.replay_buf_feature.put(f) + self.replay_buf_action.put(a) + self.replay_buf_reward.put(r) + self.replay_buf_terminal.put(terminal) + self.replay_buf_mask.put(mask) + + n += m + + self.replay_buf_len = min(self.memory_size, self.replay_buf_len + n) + return n + + def train(self, sample_buffer, print_every=1000): + """ add new samples in sample_buffer to replay buffer and train + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + print_every: int + print log every print_every batches + + Returns + ------- + loss: float + bellman residual loss + value: float + estimated state value + """ + add_num = self._add_to_replay_buffer(sample_buffer) + batch_size = self.batch_size + total_loss = 0 + + n_batches = int(self.train_freq * add_num / batch_size) + if n_batches == 0: + return 0, 0 + + print("batch number: %d add: %d replay_len: %d/%d" % + (n_batches, add_num, self.replay_buf_len, self.memory_size)) + + start_time = time.time() + ct = 0 + for i in range(n_batches): + # fetch a batch + index = np.random.choice(self.replay_buf_len - 1, batch_size) + + batch_view = self.replay_buf_view.get(index) + batch_feature = self.replay_buf_feature.get(index) + batch_action = self.replay_buf_action.get(index) + batch_reward = self.replay_buf_reward.get(index) + batch_terminal = self.replay_buf_terminal.get(index) + batch_mask = self.replay_buf_mask.get(index) + + batch_next_view = self.replay_buf_view.get(index+1) + batch_next_feature = self.replay_buf_feature.get(index+1) + + batch_target = self._calc_target(batch_next_view, batch_next_feature, + batch_reward, batch_terminal) + + self._reset_bind_size(batch_size) + batch = mx.io.DataBatch(data=[mx.nd.array(batch_view), + mx.nd.array(batch_feature)], + label=[mx.nd.array(batch_action), + mx.nd.array(batch_target), + mx.nd.array(batch_mask)]) + self.model.forward(batch, is_train=True) + self.model.backward() + self.model.update() + loss = np.mean(self.model.get_outputs()[1].asnumpy()) + total_loss += loss + + if ct % self.target_update == 0: + self._copy_network(self.target_model, self.model) + + if ct % print_every == 0: + print("batch %5d, loss %.6f, eval %.6f" % (ct, loss, self._eval(batch_target))) + ct += 1 + self.train_ct += 1 + + total_time = time.time() - start_time + step_average = total_time / max(1.0, (ct / 1000.0)) + print("batches: %d, total time: %.2f, 1k average: %.2f" % (ct, total_time, step_average)) + + return total_loss / ct if ct != 0 else 0, self._eval(batch_target) + + def _reset_bind_size(self, new_size): + """reset batch size""" + if self.bind_size == new_size: + return + else: + self.bind_size = new_size + def _reshape(model, is_target): + data_shapes = [('input_view', (new_size,) + self.view_space), + ('input_feature', (new_size,) + self.feature_space)] + label_shapes = [('action', (new_size,)), + ('target', (new_size,)), + ('mask', (new_size,))] + if is_target: + label_shapes = None + model.reshape(data_shapes=data_shapes, label_shapes=label_shapes) + _reshape(self.model, False) + _reshape(self.target_model, True) + + def _copy_network(self, dest, source): + """copy to target network""" + arg_params, aux_params = source.get_params() + dest.set_params(arg_params, aux_params) + + def _eval(self, target): + """evaluate estimated q value""" + if self.eval_obs is None: + return np.mean(target) + else: + self._reset_bind_size(len(self.eval_obs[0])) + with self.ctx: + batch = mx.io.DataBatch(data=[mx.nd.array(self.eval_obs[0]), + mx.nd.array(self.eval_obs[1])]) + self.model.forward(batch, is_train=False) + return np.mean(self.model.get_outputs()[0].asnumpy()) + + def get_info(self): + return "mx dqn train_time: %d" % (self.train_ct) \ No newline at end of file diff --git a/examples/battle_model/python/magent/builtin/rule_model/__init__.py b/examples/battle_model/python/magent/builtin/rule_model/__init__.py new file mode 100755 index 0000000..544e28a --- /dev/null +++ b/examples/battle_model/python/magent/builtin/rule_model/__init__.py @@ -0,0 +1,4 @@ +from .random import RandomActor +from .rush import RushPredator +from .runaway import RunawayPrey +from .rushgather import RushGatherer diff --git a/examples/battle_model/python/magent/builtin/rule_model/__init__.pyc b/examples/battle_model/python/magent/builtin/rule_model/__init__.pyc new file mode 100755 index 0000000..3906272 Binary files /dev/null and b/examples/battle_model/python/magent/builtin/rule_model/__init__.pyc differ diff --git a/examples/battle_model/python/magent/builtin/rule_model/random.py b/examples/battle_model/python/magent/builtin/rule_model/random.py new file mode 100755 index 0000000..722682e --- /dev/null +++ b/examples/battle_model/python/magent/builtin/rule_model/random.py @@ -0,0 +1,19 @@ +"""A random agent""" + +import numpy as np + +from magent.model import BaseModel + + +class RandomActor(BaseModel): + def __init__(self, env, handle, *args, **kwargs): + BaseModel.__init__(self, env, handle) + + self.env = env + self.handle = handle + self.n_action = env.get_action_space(handle)[0] + + def infer_action(self, obs, *args, **kwargs): + num = len(obs[0]) + actions = np.random.randint(self.n_action, size=num, dtype=np.int32) + return actions diff --git a/examples/battle_model/python/magent/builtin/rule_model/random.pyc b/examples/battle_model/python/magent/builtin/rule_model/random.pyc new file mode 100755 index 0000000..19c79c9 Binary files /dev/null and b/examples/battle_model/python/magent/builtin/rule_model/random.pyc differ diff --git a/examples/battle_model/python/magent/builtin/rule_model/runaway.py b/examples/battle_model/python/magent/builtin/rule_model/runaway.py new file mode 100755 index 0000000..eaa05f1 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/rule_model/runaway.py @@ -0,0 +1,27 @@ +"""deprecated""" + +import numpy as np + +from magent.model import BaseModel +from magent.c_lib import _LIB, as_int32_c_array, as_float_c_array + + +class RunawayPrey(BaseModel): + def __init__(self, env, handle, away_handle, *args, **kwargs): + BaseModel.__init__(self, env, handle) + + self.away_channel = env.get_channel(away_handle) + self.attack_base, _ = env.get_view2attack(handle) + self.move_back = 4 + + print("attack base", self.attack_base, "away", self.away_channel) + + def infer_action(self, observations, *args, **kwargs): + obs_buf = as_float_c_array(observations[0]) + hp_buf = as_float_c_array(observations[1]) + n, height, width, n_channel = observations[0].shape + buf = np.empty((n,), dtype=np.int32) + act_buf = as_int32_c_array(buf) + _LIB.runaway_infer_action(obs_buf, hp_buf, n, height, width, n_channel, + self.attack_base, act_buf, self.away_channel, self.move_back) + return buf diff --git a/examples/battle_model/python/magent/builtin/rule_model/runaway.pyc b/examples/battle_model/python/magent/builtin/rule_model/runaway.pyc new file mode 100755 index 0000000..1c1ed4e Binary files /dev/null and b/examples/battle_model/python/magent/builtin/rule_model/runaway.pyc differ diff --git a/examples/battle_model/python/magent/builtin/rule_model/rush.py b/examples/battle_model/python/magent/builtin/rule_model/rush.py new file mode 100755 index 0000000..cec0b00 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/rule_model/rush.py @@ -0,0 +1,33 @@ +"""deprecated""" + +import ctypes +import numpy as np + +from magent.model import BaseModel +from magent.c_lib import _LIB, as_int32_c_array, as_float_c_array + + +class RushPredator(BaseModel): + def __init__(self, env, handle, attack_handle, *args, **kwargs): + BaseModel.__init__(self, env, handle) + + self.attack_channel = env.get_channel(attack_handle) + self.attack_base, self.view2attack = env.get_view2attack(handle) + + print("attack_channel", self.attack_channel) + print("view2attack", self.view2attack) + + def infer_action(self, observations, *args, **kwargs): + obs_buf = as_float_c_array(observations[0]) + hp_buf = as_float_c_array(observations[1]) + n, height, width, n_channel = observations[0].shape + buf = np.empty((n,), dtype=np.int32) + act_buf = as_int32_c_array(buf) + attack_channel = self.attack_channel + attack_base = self.attack_base + view2attack_buf = as_int32_c_array(self.view2attack) + + _LIB.rush_prey_infer_action(obs_buf, hp_buf, n, height, width, n_channel, + act_buf, attack_channel, attack_base, + view2attack_buf, ctypes.c_float(100.0)) + return buf diff --git a/examples/battle_model/python/magent/builtin/rule_model/rush.pyc b/examples/battle_model/python/magent/builtin/rule_model/rush.pyc new file mode 100755 index 0000000..11d058b Binary files /dev/null and b/examples/battle_model/python/magent/builtin/rule_model/rush.pyc differ diff --git a/examples/battle_model/python/magent/builtin/rule_model/rushgather.py b/examples/battle_model/python/magent/builtin/rule_model/rushgather.py new file mode 100755 index 0000000..dd60a86 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/rule_model/rushgather.py @@ -0,0 +1,31 @@ +"""gather agent, rush to food according to minimap""" + +import numpy as np + +from magent.model import BaseModel +from magent.c_lib import _LIB, as_int32_c_array, as_float_c_array + + +class RushGatherer(BaseModel): + def __init__(self, env, handle, *args, **kwargs): + BaseModel.__init__(self, env, handle) + + self.env = env + self.handle = handle + self.n_action = env.get_action_space(handle) + self.view_size = env.get_view_space(handle) + self.attack_base, self.view2attack = env.get_view2attack(handle) + + def infer_action(self, states, *args, **kwargs): + obs_buf = as_float_c_array(states[0]) + hp_buf = as_float_c_array(states[1]) + n, height, width, n_channel = states[0].shape + buf = np.empty((n,), dtype=np.int32) + act_buf = as_int32_c_array(buf) + attack_base = self.attack_base + + view2attack_buf = as_int32_c_array(self.view2attack) + + _LIB.gather_infer_action(obs_buf, hp_buf, n, height, width, n_channel, + act_buf, attack_base, view2attack_buf) + return buf diff --git a/examples/battle_model/python/magent/builtin/rule_model/rushgather.pyc b/examples/battle_model/python/magent/builtin/rule_model/rushgather.pyc new file mode 100755 index 0000000..e46c826 Binary files /dev/null and b/examples/battle_model/python/magent/builtin/rule_model/rushgather.pyc differ diff --git a/examples/battle_model/python/magent/builtin/tf_model/__init__.py b/examples/battle_model/python/magent/builtin/tf_model/__init__.py new file mode 100755 index 0000000..9c54406 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/tf_model/__init__.py @@ -0,0 +1,3 @@ +from .dqn import DeepQNetwork +from .drqn import DeepRecurrentQNetwork +from .a2c import AdvantageActorCritic diff --git a/examples/battle_model/python/magent/builtin/tf_model/a2c.py b/examples/battle_model/python/magent/builtin/tf_model/a2c.py new file mode 100755 index 0000000..552c583 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/tf_model/a2c.py @@ -0,0 +1,295 @@ +""" advantage actor critic """ +import os + +import numpy as np +import tensorflow as tf + +from .base import TFBaseModel + + +class AdvantageActorCritic(TFBaseModel): + def __init__(self, env, handle, name, learning_rate=1e-3, + batch_size=64, reward_decay=0.99, eval_obs=None, + train_freq=1, value_coef=0.1, ent_coef=0.08, use_comm=False, + custom_view_space=None, custom_feature_space=None): + """init a model + + Parameters + ---------- + env: Environment + environment + handle: Handle (ctypes.c_int32) + handle of this group, can be got by env.get_handles + name: str + name of this model + learning_rate: float + batch_size: int + reward_decay: float + reward_decay in TD + eval_obs: numpy array + evaluation set of observation + train_freq: int + mean training times of a sample + ent_coef: float + weight of entropy loss in total loss + value_coef: float + weight of value loss in total loss + use_comm: bool + whether use CommNet + custom_feature_space: tuple + customized feature space + custom_view_space: tuple + customized feature space + """ + TFBaseModel.__init__(self, env, handle, name, "tfa2c") + # ======================== set config ======================== + self.env = env + self.handle = handle + self.name = name + self.view_space = custom_view_space or env.get_view_space(handle) + self.feature_space = custom_feature_space or env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + self.reward_decay = reward_decay + + self.batch_size = batch_size + self.learning_rate= learning_rate + self.train_freq = train_freq # train time of every sample (s,a,r,s') + + self.value_coef = value_coef # coefficient of value in the total loss + self.ent_coef = ent_coef # coefficient of entropy in the total loss + + self.train_ct = 0 + self.use_comm = use_comm + + # ======================= build network ======================= + with tf.name_scope(self.name): + self._create_network(self.view_space, self.feature_space) + + # init tensorflow session + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + config.gpu_options.allow_growth = True + self.sess = tf.Session(config=config) + self.sess.run(tf.global_variables_initializer()) + + # init training buffers + self.view_buf = np.empty((1,) + self.view_space) + self.feature_buf = np.empty((1,) + self.feature_space) + self.action_buf = np.empty(1, dtype=np.int32) + self.reward_buf = np.empty(1, dtype=np.float32) + + def _commnet_block(self, n, hidden, skip, name, hidden_size): + """a block of CommNet + + Parameters + ---------- + n: int + number of agent + hidden: tf.tensor + hidden layer input + skip: tf.tensor + skip connection + name: str + hidden_size: int + """ + mask = (tf.ones((n, n)) - tf.eye(n)) + mask *= tf.where(n > 1, 1.0 / (tf.cast(n, tf.float32) - 1.0), 0) + + C = tf.get_variable(name + "_C", shape=(hidden_size, hidden_size)) + H = tf.get_variable(name + "_H", shape=(hidden_size, hidden_size)) + + message = tf.matmul(mask, hidden) + + return tf.tanh(tf.matmul(message, C) + tf.matmul(hidden, H) + skip) + + def _commnet(self, n, dense, hidden_size, n_step=2): + """ CommNet Learning Multiagent Communication with Backpropagation by S. Sukhbaatar et al. NIPS 2016 + + Parameters + ---------- + n: int + number of agent + hidden_size: int + n_step: int + communication step + + Returns + ------- + h: tf.tensor + hidden units after CommNet + """ + skip = dense + + h = dense + for i in range(n_step): + h = self._commnet_block(n, h, skip, "step_%d" % i, hidden_size) + + return h + + def _create_network(self, view_space, feature_space): + """define computation graph of network + + Parameters + ---------- + view_space: tuple + feature_space: tuple + the input shape + """ + # input + input_view = tf.placeholder(tf.float32, (None,) + view_space) + input_feature = tf.placeholder(tf.float32, (None,) + feature_space) + action = tf.placeholder(tf.int32, [None]) + reward = tf.placeholder(tf.float32, [None]) + num_agent = tf.placeholder(tf.int32, []) + + kernel_num = [32, 32] + hidden_size = [256] + + # fully connected + flatten_view = tf.reshape(input_view, [-1, np.prod([v.value for v in input_view.shape[1:]])]) + h_view = tf.layers.dense(flatten_view, units=hidden_size[0], activation=tf.nn.relu) + + h_emb = tf.layers.dense(input_feature, units=hidden_size[0], activation=tf.nn.relu) + + dense = tf.concat([h_view, h_emb], axis=1) + dense = tf.layers.dense(dense, units=hidden_size[0] * 2, activation=tf.nn.relu) + + if self.use_comm: + dense = self._commnet(num_agent, dense, dense.shape[-1].value) + + policy = tf.layers.dense(dense, units=self.num_actions, activation=tf.nn.softmax) + policy = tf.clip_by_value(policy, 1e-10, 1-1e-10) + value = tf.layers.dense(dense, units=1) + value = tf.reshape(value, (-1,)) + advantage = tf.stop_gradient(reward - value) + + action_mask = tf.one_hot(action, self.num_actions) + + log_policy = tf.log(policy + 1e-6) + log_prob = tf.reduce_sum(log_policy * action_mask, axis=1) + pg_loss = -tf.reduce_mean(advantage * log_prob) + vf_loss = self.value_coef * tf.reduce_mean(tf.square(reward - value)) + neg_entropy = self.ent_coef * tf.reduce_mean(tf.reduce_sum(policy * log_policy, axis=1)) + total_loss = pg_loss + vf_loss + neg_entropy + + # train op (clip gradient) + optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate) + gradients, variables = zip(*optimizer.compute_gradients(total_loss)) + gradients, _ = tf.clip_by_global_norm(gradients, 5.0) + self.train_op = optimizer.apply_gradients(zip(gradients, variables)) + + train_op = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(total_loss) + + self.input_view = input_view + self.input_feature = input_feature + self.action = action + self.reward = reward + self.num_agent = num_agent + + self.policy, self.value = policy, value + self.train_op = train_op + self.pg_loss, self.vf_loss, self.reg_loss = pg_loss, vf_loss, neg_entropy + self.total_loss = total_loss + + def infer_action(self, raw_obs, ids, *args, **kwargs): + """infer action for a batch of agents + + Parameters + ---------- + raw_obs: tuple(numpy array, numpy array) + raw observation of agents tuple(views, features) + ids: numpy array + ids of agents + + Returns + ------- + acts: numpy array of int32 + actions for agents + """ + view, feature = raw_obs[0], raw_obs[1] + n = len(view) + + policy = self.sess.run(self.policy, {self.input_view: view, + self.input_feature: feature, + self.num_agent: n}) + actions = np.arange(self.num_actions) + + ret = np.empty(n, dtype=np.int32) + for i in range(n): + ret[i] = np.random.choice(actions, p=policy[i]) + + return ret + + def train(self, sample_buffer, print_every=1000): + """feed new data sample and train + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + + Returns + ------- + loss: list + policy gradient loss, critic loss, entropy loss + value: float + estimated state value + """ + # calc buffer size + n = 0 + for episode in sample_buffer.episodes(): + n += len(episode.rewards) + + # resize to the new size + self.view_buf.resize((n,) + self.view_space) + self.feature_buf.resize((n,) + self.feature_space) + self.action_buf.resize(n) + self.reward_buf.resize(n) + view, feature = self.view_buf, self.feature_buf + action, reward = self.action_buf, self.reward_buf + + ct = 0 + gamma = self.reward_decay + # collect episodes from multiple separate buffers to a continuous buffer + for episode in sample_buffer.episodes(): + v, f, a, r = episode.views, episode.features, episode.actions, episode.rewards + m = len(episode.rewards) + + r = np.array(r) + keep = self.sess.run(self.value, feed_dict={ + self.input_view: [v[-1]], + self.input_feature: [f[-1]], + self.num_agent: 1 + })[0] + for i in reversed(range(m)): + keep = keep * gamma + r[i] + r[i] = keep + + view[ct:ct+m] = v + feature[ct:ct+m] = f + action[ct:ct+m] = a + reward[ct:ct+m] = r + ct += m + + assert n == ct + + # train + _, pg_loss, vf_loss, ent_loss, state_value = self.sess.run( + [self.train_op, self.pg_loss, self.vf_loss, self.reg_loss, self.value], feed_dict={ + self.input_view: view, + self.input_feature: feature, + self.action: action, + self.reward: reward, + self.num_agent: len(reward) + }) + print("sample", n, pg_loss, vf_loss, ent_loss) + + return [pg_loss, vf_loss, ent_loss], np.mean(state_value) + + def get_info(self): + """get information of the model + + Returns + ------- + info: string + """ + return "a2c train_time: %d" % (self.train_ct) diff --git a/examples/battle_model/python/magent/builtin/tf_model/base.py b/examples/battle_model/python/magent/builtin/tf_model/base.py new file mode 100755 index 0000000..0ddda30 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/tf_model/base.py @@ -0,0 +1,77 @@ +import os +import tensorflow as tf + +from magent.model import BaseModel + + +class TFBaseModel(BaseModel): + """base model for tensorflow model""" + def __init__(self, env, handle, name, subclass_name): + """init a model + + Parameters + ---------- + env: magent.Environment + handle: handle (ctypes.c_int32) + name: str + subclass_name: str + name of subclass + """ + BaseModel.__init__(self, env, handle) + self.name = name + self.subclass_name = subclass_name + + def save(self, dir_name, epoch): + """save model to dir + + Parameters + ---------- + dir_name: str + name of the directory + epoch: int + """ + if not os.path.exists(dir_name): + os.mkdir(dir_name) + dir_name = os.path.join(dir_name, self.name) + if not os.path.exists(dir_name): + os.mkdir(dir_name) + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name) + saver = tf.train.Saver(model_vars) + saver.save(self.sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch)) + + def load(self, dir_name, epoch=0, name=None): + """save model to dir + + Parameters + ---------- + dir_name: str + name of the directory + epoch: int + """ + if name is None or name == self.name: # the name of saved model is the same as ours + dir_name = os.path.join(dir_name, self.name) + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name) + saver = tf.train.Saver(model_vars) + saver.restore(self.sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch)) + else: # load a checkpoint with different name + backup_graph = tf.get_default_graph() + kv_dict = {} + + # load checkpoint from another saved graph + with tf.Graph().as_default(), tf.Session() as sess: + tf.train.import_meta_graph(os.path.join(dir_name, name, (self.subclass_name + "_%d") % epoch + ".meta")) + dir_name = os.path.join(dir_name, name) + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name) + sess.run(tf.global_variables_initializer()) + saver = tf.train.Saver(model_vars) + saver.restore(sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch)) + for item in tf.global_variables(): + kv_dict[item.name] = sess.run(item) + + # assign to now graph + backup_graph.as_default() + model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name) + for item in model_vars: + old_name = item.name.replace(self.name, name) + self.sess.run(tf.assign(item, kv_dict[old_name])) + diff --git a/examples/battle_model/python/magent/builtin/tf_model/dqn.py b/examples/battle_model/python/magent/builtin/tf_model/dqn.py new file mode 100755 index 0000000..48ca8c5 --- /dev/null +++ b/examples/battle_model/python/magent/builtin/tf_model/dqn.py @@ -0,0 +1,393 @@ +"""Deep q network""" + +import time + +import numpy as np +import tensorflow as tf + +from .base import TFBaseModel +from ..common import ReplayBuffer + + +class DeepQNetwork(TFBaseModel): + def __init__(self, env, handle, name, + batch_size=64, learning_rate=1e-4, reward_decay=0.99, + train_freq=1, target_update=2000, memory_size=2 ** 20, eval_obs=None, + use_dueling=True, use_double=True, use_conv=True, + custom_view_space=None, custom_feature_space=None, + num_gpu=1, infer_batch_size=8192, network_type=0): + """init a model + + Parameters + ---------- + env: Environment + environment + handle: Handle (ctypes.c_int32) + handle of this group, can be got by env.get_handles + name: str + name of this model + learning_rate: float + batch_size: int + reward_decay: float + reward_decay in TD + train_freq: int + mean training times of a sample + target_update: int + target will update every target_update batches + memory_size: int + weight of entropy loss in total loss + eval_obs: numpy array + evaluation set of observation + use_dueling: bool + whether use dueling q network + use_double: bool + whether use double q network + use_conv: bool + use convolution or fully connected layer as state encoder + num_gpu: int + number of gpu + infer_batch_size: int + batch size while inferring actions + custom_feature_space: tuple + customized feature space + custom_view_space: tuple + customized feature space + """ + TFBaseModel.__init__(self, env, handle, name, "tfdqn") + # ======================== set config ======================== + self.env = env + self.handle = handle + self.view_space = custom_view_space or env.get_view_space(handle) + self.feature_space = custom_feature_space or env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + + self.batch_size = batch_size + self.learning_rate= learning_rate + self.train_freq = train_freq # train time of every sample (s,a,r,s') + self.target_update= target_update # target network update frequency + self.eval_obs = eval_obs + self.infer_batch_size = infer_batch_size # maximum batch size when infer actions, + # change this to fit your GPU memory if you meet a OOM + + self.use_dueling = use_dueling + self.use_double = use_double + self.num_gpu = num_gpu + self.network_type = network_type + + self.train_ct = 0 + + # ======================= build network ======================= + # input place holder + self.target = tf.placeholder(tf.float32, [None]) + self.weight = tf.placeholder(tf.float32, [None]) + + self.input_view = tf.placeholder(tf.float32, (None,) + self.view_space) + self.input_feature = tf.placeholder(tf.float32, (None,) + self.feature_space) + self.action = tf.placeholder(tf.int32, [None]) + self.mask = tf.placeholder(tf.float32, [None]) + self.eps = tf.placeholder(tf.float32) # e-greedy + + # build graph + with tf.variable_scope(self.name): + with tf.variable_scope("eval_net_scope"): + self.eval_scope_name = tf.get_variable_scope().name + self.qvalues = self._create_network(self.input_view, self.input_feature, use_conv) + + if self.num_gpu > 1: # build inference graph for multiple gpus + self._build_multi_gpu_infer(self.num_gpu) + + with tf.variable_scope("target_net_scope"): + self.target_scope_name = tf.get_variable_scope().name + self.target_qvalues = self._create_network(self.input_view, self.input_feature, use_conv) + + # loss + self.gamma = reward_decay + self.actions_onehot = tf.one_hot(self.action, self.num_actions) + td_error = tf.square(self.target - tf.reduce_sum(tf.multiply(self.actions_onehot, self.qvalues), axis=1)) + self.loss = tf.reduce_sum(td_error * self.mask) / tf.reduce_sum(self.mask) + + # train op (clip gradient) + optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) + gradients, variables = zip(*optimizer.compute_gradients(self.loss)) + gradients, _ = tf.clip_by_global_norm(gradients, 5.0) + self.train_op = optimizer.apply_gradients(zip(gradients, variables)) + + # output action + def out_action(qvalues): + best_action = tf.argmax(qvalues, axis=1) + best_action = tf.to_int32(best_action) + random_action = tf.random_uniform(tf.shape(best_action), 0, self.num_actions, tf.int32) + should_explore = tf.random_uniform(tf.shape(best_action), 0, 1) < self.eps + return tf.where(should_explore, random_action, best_action) + + self.output_action = out_action(self.qvalues) + if self.num_gpu > 1: + self.infer_out_action = [out_action(qvalue) for qvalue in self.infer_qvalues] + + # target network update op + self.update_target_op = [] + t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.target_scope_name) + e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.eval_scope_name) + for i in range(len(t_params)): + self.update_target_op.append(tf.assign(t_params[i], e_params[i])) + + # init tensorflow session + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + config.gpu_options.allow_growth = True + self.sess = tf.Session(config=config) + self.sess.run(tf.global_variables_initializer()) + + # init replay buffers + self.replay_buf_len = 0 + self.memory_size = memory_size + self.replay_buf_view = ReplayBuffer(shape=(memory_size,) + self.view_space) + self.replay_buf_feature = ReplayBuffer(shape=(memory_size,) + self.feature_space) + self.replay_buf_action = ReplayBuffer(shape=(memory_size,), dtype=np.int32) + self.replay_buf_reward = ReplayBuffer(shape=(memory_size,)) + self.replay_buf_terminal = ReplayBuffer(shape=(memory_size,), dtype=np.bool) + self.replay_buf_mask = ReplayBuffer(shape=(memory_size,)) + # if mask[i] == 0, then the item is used for padding, not for training + + def _create_network(self, input_view, input_feature, use_conv=True, reuse=None): + """define computation graph of network + + Parameters + ---------- + input_view: tf.tensor + input_feature: tf.tensor + the input tensor + """ + kernel_num = [32, 32] + hidden_size = [256] + + if use_conv: # convolution + h_conv1 = tf.layers.conv2d(input_view, filters=kernel_num[0], kernel_size=3, + activation=tf.nn.relu, name="conv1", reuse=reuse) + h_conv2 = tf.layers.conv2d(h_conv1, filters=kernel_num[1], kernel_size=3, + activation=tf.nn.relu, name="conv2", reuse=reuse) + flatten_view = tf.reshape(h_conv2, [-1, np.prod([v.value for v in h_conv2.shape[1:]])]) + h_view = tf.layers.dense(flatten_view, units=hidden_size[0], activation=tf.nn.relu, + name="dense_view", reuse=reuse) + else: # fully connected + flatten_view = tf.reshape(input_view, [-1, np.prod([v.value for v in input_view.shape[1:]])]) + h_view = tf.layers.dense(flatten_view, units=hidden_size[0], activation=tf.nn.relu) + + h_emb = tf.layers.dense(input_feature, units=hidden_size[0], activation=tf.nn.relu, + name="dense_emb", reuse=reuse) + + dense = tf.concat([h_view, h_emb], axis=1) + + if self.use_dueling: + value = tf.layers.dense(dense, units=1, name="value", reuse=reuse) + advantage = tf.layers.dense(dense, units=self.num_actions, use_bias=False, + name="advantage", reuse=reuse) + + qvalues = value + advantage - tf.reduce_mean(advantage, axis=1, keep_dims=True) + else: + qvalues = tf.layers.dense(dense, units=self.num_actions, name="value", reuse=reuse) + + return qvalues + + def infer_action(self, raw_obs, ids, policy='e_greedy', eps=0): + """infer action for a batch of agents + + Parameters + ---------- + raw_obs: tuple(numpy array, numpy array) + raw observation of agents tuple(views, features) + ids: numpy array + ids of agents + policy: str + can be eps-greedy or greedy + eps: float + used when policy is eps-greedy + + Returns + ------- + acts: numpy array of int32 + actions for agents + """ + view, feature = raw_obs[0], raw_obs[1] + + if policy == 'e_greedy': + eps = eps + elif policy == 'greedy': + eps = 0 + + n = len(view) + batch_size = min(n, self.infer_batch_size) + + if self.num_gpu > 1 and n > batch_size: # infer by multi gpu in parallel + ret = self._infer_multi_gpu(view, feature, ids, eps) + else: # infer by splitting big batch in serial + ret = [] + for i in range(0, n, batch_size): + beg, end = i, i + batch_size + ret.append(self.sess.run(self.output_action, feed_dict={ + self.input_view: view[beg:end], + self.input_feature: feature[beg:end], + self.eps: eps})) + ret = np.concatenate(ret) + return ret + + def _calc_target(self, next_view, next_feature, rewards, terminal): + """calculate target value""" + n = len(rewards) + if self.use_double: + t_qvalues, qvalues = self.sess.run([self.target_qvalues, self.qvalues], + feed_dict={self.input_view: next_view, + self.input_feature: next_feature}) + next_value = t_qvalues[np.arange(n), np.argmax(qvalues, axis=1)] + else: + t_qvalues = self.sess.run(self.target_qvalues, {self.input_view: next_view, + self.input_feature: next_feature}) + next_value = np.max(t_qvalues, axis=1) + + target = np.where(terminal, rewards, rewards + self.gamma * next_value) + + return target + + def _add_to_replay_buffer(self, sample_buffer): + """add samples in sample_buffer to replay buffer""" + n = 0 + for episode in sample_buffer.episodes(): + v, f, a, r = episode.views, episode.features, episode.actions, episode.rewards + + m = len(r) + + mask = np.ones((m,)) + terminal = np.zeros((m,), dtype=np.bool) + if episode.terminal: + terminal[-1] = True + else: + mask[-1] = 0 + + self.replay_buf_view.put(v) + self.replay_buf_feature.put(f) + self.replay_buf_action.put(a) + self.replay_buf_reward.put(r) + self.replay_buf_terminal.put(terminal) + self.replay_buf_mask.put(mask) + + n += m + + self.replay_buf_len = min(self.memory_size, self.replay_buf_len + n) + return n + + def train(self, sample_buffer, print_every=1000): + """ add new samples in sample_buffer to replay buffer and train + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + print_every: int + print log every print_every batches + + Returns + ------- + loss: float + bellman residual loss + value: float + estimated state value + """ + add_num = self._add_to_replay_buffer(sample_buffer) + batch_size = self.batch_size + total_loss = 0 + + n_batches = int(self.train_freq * add_num / batch_size) + if n_batches == 0: + return 0, 0 + + print("batch number: %d add: %d replay_len: %d/%d" % + (n_batches, add_num, self.replay_buf_len, self.memory_size)) + + start_time = time.time() + ct = 0 + for i in range(n_batches): + # fetch a batch + index = np.random.choice(self.replay_buf_len - 1, batch_size) + + batch_view = self.replay_buf_view.get(index) + batch_feature = self.replay_buf_feature.get(index) + batch_action = self.replay_buf_action.get(index) + batch_reward = self.replay_buf_reward.get(index) + batch_terminal = self.replay_buf_terminal.get(index) + batch_mask = self.replay_buf_mask.get(index) + + batch_next_view = self.replay_buf_view.get(index+1) + batch_next_feature = self.replay_buf_feature.get(index+1) + + batch_target = self._calc_target(batch_next_view, batch_next_feature, + batch_reward, batch_terminal) + + ret = self.sess.run([self.train_op, self.loss], feed_dict={ + self.input_view: batch_view, + self.input_feature: batch_feature, + self.action: batch_action, + self.target: batch_target, + self.mask: batch_mask + }) + loss = ret[1] + total_loss += loss + + if ct % self.target_update == 0: + self.sess.run(self.update_target_op) + + if ct % print_every == 0: + print("batch %5d, loss %.6f, eval %.6f" % (ct, loss, self._eval(batch_target))) + ct += 1 + self.train_ct += 1 + + total_time = time.time() - start_time + step_average = total_time / max(1.0, (ct / 1000.0)) + print("batches: %d, total time: %.2f, 1k average: %.2f" % (ct, total_time, step_average)) + + return total_loss / ct if ct != 0 else 0, self._eval(batch_target) + + def _eval(self, target): + """evaluate estimated q value""" + if self.eval_obs is None: + return np.mean(target) + else: + return np.mean(self.sess.run([self.qvalues], feed_dict={ + self.input_view: self.eval_obs[0], + self.input_feature: self.eval_obs[1] + })) + + def clear_buffer(self): + """clear replay buffer""" + self.replay_buf_len = 0 + self.replay_buf_view.clear() + self.replay_buf_feature.clear() + self.replay_buf_action.clear() + self.replay_buf_reward.clear() + self.replay_buf_terminal.clear() + self.replay_buf_mask.clear() + + def _build_multi_gpu_infer(self, num_gpu): + """build inference graph for multi gpus""" + self.infer_qvalues = [] + self.infer_input_view = [] + self.infer_input_feature = [] + for i in range(num_gpu): + self.infer_input_view.append(tf.placeholder(tf.float32, (None,) + self.view_space)) + self.infer_input_feature.append(tf.placeholder(tf.float32, (None,) + self.feature_space)) + with tf.variable_scope("eval_net_scope"), tf.device("/gpu:%d" % i): + self.infer_qvalues.append(self._create_network(self.infer_input_view[i], + self.infer_input_feature[i], reuse=True)) + + def _infer_multi_gpu(self, view, feature, ids, eps): + """infer action by multi gpu in parallel """ + ret = [] + beg = 0 + while beg < len(view): + feed_dict = {self.eps: eps} + for i in range(self.num_gpu): + end = beg + self.infer_batch_size + feed_dict[self.infer_input_view[i]] = view[beg:end] + feed_dict[self.infer_input_feature[i]] = feature[beg:end] + beg += self.infer_batch_size + + ret.extend(self.sess.run(self.infer_out_action, feed_dict=feed_dict)) + return np.concatenate(ret) diff --git a/examples/battle_model/python/magent/builtin/tf_model/drqn.py b/examples/battle_model/python/magent/builtin/tf_model/drqn.py new file mode 100755 index 0000000..f8f0aaf --- /dev/null +++ b/examples/battle_model/python/magent/builtin/tf_model/drqn.py @@ -0,0 +1,581 @@ +"""Deep recurrent Q network""" + +import time +import os +import collections + +import numpy as np +import tensorflow as tf + +from .base import TFBaseModel + + +class DeepRecurrentQNetwork(TFBaseModel): + def __init__(self, env, handle, name, + batch_size=32, unroll_step=8, reward_decay=0.99, learning_rate=1e-4, + train_freq=1, memory_size=20000, target_update=2000, eval_obs=None, + use_dueling=True, use_double=True, use_episode_train=False, + custom_view_space=None, custom_feature_space=None): + """init a model + + Parameters + ---------- + env: Environment + environment + handle: Handle (ctypes.c_int32) + handle of this group, can be got by env.get_handles + name: str + name of this model + learning_rate: float + batch_size: int + reward_decay: float + reward_decay in TD + train_freq: int + mean training times of a sample + target_update: int + target will update every target_update batches + memory_size: int + weight of entropy loss in total loss + eval_obs: numpy array + evaluation set of observation + use_dueling: bool + whether use dueling q network + use_double: bool + whether use double q network + custom_feature_space: tuple + customized feature space + custom_view_space: tuple + customized feature space + """ + TFBaseModel.__init__(self, env, handle, name, "tfdrqn") + # ======================== set config ======================== + self.env = env + self.handle = handle + self.view_space = custom_view_space or env.get_view_space(handle) + self.feature_space = custom_feature_space or env.get_feature_space(handle) + self.num_actions = env.get_action_space(handle)[0] + + self.batch_size = batch_size + self.unroll_step = unroll_step + self.handle = handle + self.name = name + self.learning_rate= learning_rate + self.train_freq = train_freq # train time of every sample (s,a,r,s') + self.target_update= target_update # target network update frequency + self.eval_obs = eval_obs + + self.use_dueling = use_dueling + self.use_double = use_double + self.use_episode_train = use_episode_train + self.skip_error = 0 + self.pad_before_len = unroll_step - 1 + + self.agent_states = {} + self.train_ct = 0 + + # ======================= build network ======================= + # input place holder + self.target = tf.placeholder(tf.float32, [None]) + + self.input_view = tf.placeholder(tf.float32, (None,) + self.view_space, name="input_view") + self.input_feature = tf.placeholder(tf.float32, (None,) + self.feature_space, name="input_feature") + self.action = tf.placeholder(tf.int32, [None], name="action") + self.mask = tf.placeholder(tf.float32, [None], name="mask") + + self.batch_size_ph = tf.placeholder(tf.int32, []) + self.unroll_step_ph = tf.placeholder(tf.int32, []) + + # build graph + with tf.variable_scope(self.name): + with tf.variable_scope("eval_net_scope"): + self.eval_scope_name = tf.get_variable_scope().name + self.qvalues, self.state_in, self.rnn_state = \ + self._create_network(self.input_view, self.input_feature) + + with tf.variable_scope("target_net_scope"): + self.target_scope_name = tf.get_variable_scope().name + self.target_qvalues, self.target_state_in, self.target_rnn_state = \ + self._create_network(self.input_view, self.input_feature) + + # loss + self.gamma = reward_decay + self.actions_onehot = tf.one_hot(self.action, self.num_actions) + self.td_error = tf.square( + self.target - tf.reduce_sum(tf.multiply(self.actions_onehot, self.qvalues), axis=1) + ) + #self.loss = tf.reduce_mean(self.td_error) + self.loss = tf.reduce_sum(self.td_error * self.mask) / tf.reduce_sum(self.mask) + + # train op (clip gradient) + optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) + gradients, variables = zip(*optimizer.compute_gradients(self.loss)) + gradients, _ = tf.clip_by_global_norm(gradients, 10.0) + self.train_op = optimizer.apply_gradients(zip(gradients, variables)) + + # target network update op + self.update_target_op = [] + t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.target_scope_name) + e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.eval_scope_name) + for i in range(len(t_params)): + self.update_target_op.append(tf.assign(t_params[i], e_params[i])) + + # init tensorflow session + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + config.gpu_options.allow_growth = True + self.sess = tf.Session(config=config) + self.sess.run(tf.global_variables_initializer()) + + # init memory buffers + self.memory_size = memory_size + self.replay_buffer_lens = collections.deque(maxlen=memory_size) + self.replay_buffer = collections.deque(maxlen=memory_size) + # item format [views, features, actions, rewards, terminals, masks, len] + + # init training buffers + self.view_buf = np.empty((1,) + self.view_space) + self.feature_buf = np.empty((1,) + self.feature_space) + self.action_buf, self.reward_buf = np.empty(1, dtype=np.int32), np.empty(1) + self.terminal_buf = np.empty(1, dtype=np.bool) + + def _create_network(self, input_view, input_feature, reuse=None): + """define computation graph of network + + Parameters + ---------- + input_view: tf.tensor + input_feature: tf.tensor + the input tensor + """ + kernel_num = [32, 32] + hidden_size = [256] + + # conv + h_conv1 = tf.layers.conv2d(input_view, filters=kernel_num[0], kernel_size=3, + activation=tf.nn.relu, name="conv1", reuse=reuse) + h_conv2 = tf.layers.conv2d(h_conv1, filters=kernel_num[1], kernel_size=3, + activation=tf.nn.relu, name="conv2", reuse=reuse) + flatten_view = tf.reshape(h_conv2, [-1, np.prod([v.value for v in h_conv2.shape[1:]])]) + h_view = tf.layers.dense(flatten_view, units=hidden_size[0], activation=tf.nn.relu, + name="dense_view", reuse=reuse) + + h_emb = tf.layers.dense(input_feature, units=hidden_size[0], activation=tf.nn.relu, + name="dense_emb", reuse=reuse) + + dense = tf.concat([h_view, h_emb], axis=1) + + # RNN + state_size = hidden_size[0] * 2 + rnn_cell = tf.contrib.rnn.GRUCell(num_units=state_size) + + rnn_in = tf.reshape(dense, shape=[self.batch_size_ph, self.unroll_step_ph, state_size]) + state_in = rnn_cell.zero_state(self.batch_size_ph, tf.float32) + rnn, rnn_state = tf.nn.dynamic_rnn( + cell=rnn_cell, inputs=rnn_in, dtype=tf.float32, initial_state=state_in + ) + rnn = tf.reshape(rnn, shape=[-1, state_size]) + + if self.use_dueling: + value = tf.layers.dense(dense, units=1, name="dense_value", reuse=reuse) + advantage = tf.layers.dense(dense, units=self.num_actions, use_bias=False, + name="dense_advantage", reuse=reuse) + + qvalues = value + advantage - tf.reduce_mean(advantage, axis=1, keep_dims=True) + else: + qvalues = tf.layers.dense(rnn, units=self.num_actions) + + self.state_size = state_size + return qvalues, state_in, rnn_state + + def _get_agent_states(self, ids): + """get hidden state of agents""" + n = len(ids) + states = np.empty([n, self.state_size]) + default = np.zeros([self.state_size]) + for i in range(n): + states[i] = self.agent_states.get(ids[i], default) + return states + + def _set_agent_states(self, ids, states): + """set hidden state for agents""" + if len(ids) <= len(self.agent_states) * 0.5: + self.agent_states = {} + for i in range(len(ids)): + self.agent_states[ids[i]] = states[i] + + def infer_action(self, raw_obs, ids, policy='e_greedy', eps=0): + """infer action for a batch of agents + + Parameters + ---------- + raw_obs: tuple(numpy array, numpy array) + raw observation of agents tuple(views, features) + ids: numpy array + ids of agents + policy: str + can be eps-greedy or greedy + eps: float + used when policy is eps-greedy + + Returns + ------- + acts: numpy array of int32 + actions for agents + """ + view, feature = raw_obs[0], raw_obs[1] + n = len(ids) + + states = self._get_agent_states(ids) + qvalues, states = self.sess.run([self.qvalues, self.rnn_state], feed_dict={ + self.input_view: view, + self.input_feature: feature, + self.state_in: states, + self.batch_size_ph: n, + self.unroll_step_ph: 1 + }) + self._set_agent_states(ids, states) + best_actions = np.argmax(qvalues, axis=1) + + if policy == 'e_greedy': + random = np.random.randint(self.num_actions, size=(n,)) + cond = np.random.uniform(0, 1, size=(n,)) < eps + ret = np.where(cond, random, best_actions) + elif policy == 'greedy': + ret = best_actions + + return ret.astype(np.int32) + + def _calc_target(self, next_view, next_feature, rewards, terminal, batch_size, unroll_step): + """calculate target value""" + n = len(rewards) + if self.use_double: + t_qvalues, qvalues = self.sess.run([self.target_qvalues, self.qvalues], feed_dict={ + self.input_view: next_view, + self.input_feature: next_feature, + # self.state_in: state_in, + # self.target_state_in: state_in, + self.batch_size_ph: batch_size, + self.unroll_step_ph: unroll_step}) + # ignore the first value (the first value is for computing correct hidden state) + # t_qvalues = t_qvalues.reshape([-1, unroll_step, self.num_actions]) + # t_qvalues = t_qvalues[:, 1:, :].reshape([-1, self.num_actions]) + # qvalues = qvalues.reshape([-1, unroll_step, self.num_actions]) + # qvalues = qvalues[:, 1:, :].reshape([-1, self.num_actions]) + next_value = t_qvalues[np.arange(n), np.argmax(qvalues, axis=1)] + else: + t_qvalues = self.sess.run(self.target_qvalues, feed_dict={ + self.input_view: next_view, + self.input_feature: next_feature, + # self.target_state_in: state_in, + self.batch_size_ph: batch_size, + self.unroll_step_ph: unroll_step}) + # t_qvalues = t_qvalues.reshape([-1, unroll_step, self.num_actions]) + # t_qvalues = t_qvalues[:,1:,:].reshape([-1, self.num_actions]) + next_value = np.max(t_qvalues, axis=1) + + target = np.where(terminal, rewards, rewards + self.gamma * next_value) + + return target + + def _add_to_replay_buffer(self, sample_buffer): + """add samples in sample_buffer to replay buffer""" + n = 0 + for episode in sample_buffer.episodes(): + v, f, a, r = episode.views, episode.features, episode.actions, episode.rewards + + m = len(r) + + mask = np.ones((m,)) + terminal = np.zeros((m,), dtype=np.bool) + if episode.terminal: + terminal[-1] = True + else: + mask[-1] = 0 + + item = [v, f, a, r, terminal, mask, m] + self.replay_buffer_lens.append(m) + self.replay_buffer.append(item) + + n += m + return n + + def train(self, sample_buffer, print_every=500): + """ add new samples in sample_buffer to replay buffer and train + do not keep hidden state (split episode into short sequences) + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + print_every: int + print log every print_every batches + + Returns + ------- + loss: float + bellman residual loss + value: float + estimated state value + """ + add_num = self._add_to_replay_buffer(sample_buffer) + + batch_size = self.batch_size + unroll_step = self.unroll_step + + # calc sample weight of episodes (i.e. their lengths) + replay_buffer = self.replay_buffer + replay_lens_sum = np.sum(self.replay_buffer_lens) + weight = np.array(self.replay_buffer_lens, dtype=np.float32) / replay_lens_sum + + n_batches = self.train_freq * add_num / (batch_size * (unroll_step - self.skip_error)) + if n_batches == 0: + return 0, 0 + + max_ = batch_size * unroll_step + batch_view = np.zeros((max_+1,) + self.view_space, dtype=np.float32) + batch_feature = np.zeros((max_+1,) + self.feature_space, dtype=np.float32) + batch_action = np.zeros((max_,), dtype=np.int32) + batch_reward = np.zeros((max_,), dtype=np.float32) + batch_terminal = np.zeros((max_,), dtype=np.bool) + batch_mask = np.zeros((max_,), dtype=np.float32) + + # calc batch number + n_batches = int(self.train_freq * add_num / (batch_size * (unroll_step - self.skip_error))) + print("batches: %d add: %d replay_len: %d/%d" % + (n_batches, add_num, len(self.replay_buffer), self.memory_size)) + + ct = 0 + total_loss = 0 + start_time = time.time() + # train batches + for i in range(n_batches): + indexes = np.random.choice(len(replay_buffer), self.batch_size, p=weight) + + batch_mask[:] = 0 + + for j in range(batch_size): + item = replay_buffer[indexes[j]] + v, f, a, r, t = item[0], item[1], item[2], item[3], item[4] + length = len(v) + + start = np.random.randint(length) + real_step = min(length - start, unroll_step) + + beg = j * unroll_step + batch_view[beg:beg+real_step] = v[start:start+real_step] + batch_feature[beg:beg+real_step] = f[start:start+real_step] + batch_action[beg:beg+real_step] = a[start:start+real_step] + batch_reward[beg:beg+real_step] = r[start:start+real_step] + batch_terminal[beg:beg+real_step] = t[start:start+real_step] + batch_mask[beg:beg+real_step] = 1.0 + + if not t[start+real_step-1]: + batch_mask[beg+real_step-1] = 0 + + # collect trajectories from different IDs to a single buffer + target = self._calc_target(batch_view[1:], batch_feature[1:], + batch_reward, batch_terminal, batch_size, unroll_step) + + ret = self.sess.run([self.train_op, self.loss], feed_dict={ + self.input_view: batch_view[:-1], + self.input_feature: batch_feature[:-1], + self.action: batch_action, + self.target: target, + self.mask: batch_mask, + self.batch_size_ph: batch_size, + self.unroll_step_ph: unroll_step, + }) + loss = ret[1] + total_loss += loss + + if ct % self.target_update == 0: + self.sess.run(self.update_target_op) + + if ct % print_every == 0: + print("batch %5d, loss %.6f, qvalue %.6f" % (ct, loss, self._eval(target))) + ct += 1 + self.train_ct += 1 + + total_time = time.time() - start_time + step_average = total_time / max(1.0, (ct / 1000.0)) + print("batches: %d, total time: %.2f, 1k average: %.2f" % (ct, total_time, step_average)) + + return total_loss / ct if ct != 0 else 0, self._eval(target) + + def train_keep_hidden(self, sample_buffer, print_every=500): + """ add new samples in sample_buffer to replay buffer and train + keep hidden state (split episode into small sequence, but keep hidden states) + this means must train some episodes continuously not fully random. + to use this training scheme, you should also modify self._calc_target + + Parameters + ---------- + sample_buffer: magent.utility.EpisodesBuffer + buffer contains samples + print_every: int + print log every print_every batches + + Returns + ------- + loss: float + bellman residual loss + value: float + estimated state value + """ + + add_num = self._add_to_replay_buffer(sample_buffer) + + batch_size = self.batch_size + unroll_step = self.unroll_step + + # calc sample weight of episodes (i.e. their lengths) + replay_buffer = self.replay_buffer + replay_lens_sum = np.sum(self.replay_buffer_lens) + weight = np.array(self.replay_buffer_lens, dtype=np.float32) / replay_lens_sum + + max_len = self._div_round(np.max(self.replay_buffer_lens), unroll_step) + n_batches = self.train_freq * add_num / (batch_size * unroll_step) + if n_batches == 0: + return 0, 0 + + # allocate buffer + max_ = batch_size * max_len + batch_view = np.zeros((max_+1,) + self.view_space, dtype=np.float32) + batch_feature = np.zeros((max_+1,) + self.feature_space, dtype=np.float32) + batch_action = np.zeros((max_,), dtype=np.int32) + batch_reward = np.zeros((max_,), dtype=np.float32) + batch_terminal = np.zeros((max_,), dtype=np.bool) + batch_mask = np.zeros((max_,), dtype=np.float32) + batch_hidden = np.zeros((batch_size, self.state_size), dtype=np.float32) + batch_pick = np.zeros((batch_size, max_len), dtype=np.bool) + pick_buffer = np.arange(max_, dtype=np.int32) + + print("batches: %d add: %d replay_len: %d, %d/%d" % + (n_batches, add_num, replay_lens_sum, len(self.replay_buffer), self.memory_size)) + + start_time = time.time() + total_loss = 0 + ct = 0 + while ct < n_batches: + # random sample agent episodes (sequence) + indexs = np.random.choice(len(replay_buffer), self.batch_size, p=weight) + train_length = 0 + to_sort = [] + + # collect length and sort + for j, index in enumerate(indexs): + length = replay_buffer[index][-1] + length = self._div_round(length, unroll_step) + train_length = max(train_length, length) + to_sort.append([index, length]) + to_sort.sort(key=lambda x: -x[1]) + + # merge short episodes to long episodes (use greedy method) + merged = [False for _ in range(batch_size)] + rows = [] + for j in range(len(to_sort)): + if merged[j]: + continue + row = [to_sort[j][0]] + now_len = to_sort[j][1] + if True: # use compress + for k in range(j+1, batch_size): + if now_len + to_sort[k][1] <= train_length: + row.append(to_sort[k][0]) + now_len += to_sort[k][1] + merged[k] = True + rows.append(row) + n_rows = len(rows) + + batch_reset = np.zeros((train_length, batch_size), dtype=np.bool) + batch_mask[:] = 0 + + # copy from replay buffer to batch buffer + for j, row in enumerate(rows): + beg = j * max_len + init_beg = beg + # fill a row + for index in row: + v, f, a, r, terminal, mask, x = replay_buffer[index] + + batch_reset[(beg - init_beg)/unroll_step, j] = True + batch_view[beg:beg+x] = v + batch_feature[beg:beg+x] = f + batch_action[beg:beg+x] = a + batch_reward[beg:beg+x] = r + batch_terminal[beg:beg+x] = terminal + batch_mask[beg:beg+x] = mask + + beg += self._div_round(x, unroll_step) + + # train steps + for j in range((train_length + unroll_step - 1) / unroll_step): + batch_pick[:] = False + batch_pick[:n_rows, j * unroll_step:(j+1) * unroll_step] = True + + pick = pick_buffer[batch_pick.reshape(-1)].reshape(n_rows, unroll_step) + next_pick = np.empty((n_rows, unroll_step + 1), dtype=np.int32) # next pick choose one more state than pick + next_pick[:, :unroll_step] = pick + next_pick[:, unroll_step] = pick[:, -1] + 1 + pick = pick.reshape(-1) + next_pick = next_pick.reshape(-1) + + steps = len(pick) / n_rows + assert steps > 0 + if np.sum(batch_mask[pick]) < 1: + continue + + batch_hidden[batch_reset[j]] = np.zeros_like(batch_hidden[0]) + + batch_target = self._calc_target(batch_view[next_pick], batch_feature[next_pick], + batch_reward[pick], batch_terminal[pick], batch_hidden[:n_rows], + n_rows, steps + 1) + ret = self.sess.run( + [self.train_op, self.loss, self.rnn_state], + feed_dict={ + self.input_view: batch_view[pick], + self.input_feature: batch_feature[pick], + self.action: batch_action[pick], + self.target: batch_target, + self.mask: batch_mask[pick], + self.state_in: batch_hidden[:n_rows], + self.batch_size_ph: n_rows, + self.unroll_step_ph: steps + }) + loss, batch_hidden[:n_rows] = ret[1], ret[2] + total_loss += loss + + if ct % self.target_update == 0: + self.sess.run(self.update_target_op) + + if ct % print_every == 0: + print("batches %5d, mask %d/%d (%d), loss %.6f, qvalue %.6f" % + (ct, sum(batch_mask), n_rows * train_length, n_rows, loss, self._eval(batch_target))) + ct += 1 + self.train_ct += 1 + + total_time = time.time() - start_time + step_average = total_time / max(1.0, (ct / 1000.0)) + print("batches: %d, total time: %.2f, 1k average: %.2f" % (ct, total_time, step_average)) + + return round(total_loss / ct if ct != 0 else 0, 6), self._eval(batch_target) + + @staticmethod + def _div_round(x, divisor): + """round up to nearest integer that are divisible by divisor""" + return (x + divisor - 1) / divisor * divisor + + def _eval(self, target): + """evaluate estimated q value""" + if self.eval_obs is None: + return np.mean(target) + else: + return np.mean(self.sess.run(self.target_qvalues, feed_dict={ + self.input_view: self.eval_obs[0], + self.input_feature: self.eval_obs[1], + self.batch_size_ph: self.eval_obs[0].shape[0], + self.unroll_step_ph: 1 + })) + + def get_info(self): + """get information of model""" + return "tfdrqn train_time: %d" % (self.train_ct) diff --git a/examples/battle_model/python/magent/c_lib.py b/examples/battle_model/python/magent/c_lib.py new file mode 100755 index 0000000..bcfd9eb --- /dev/null +++ b/examples/battle_model/python/magent/c_lib.py @@ -0,0 +1,42 @@ +""" some utility for call C++ code""" + +from __future__ import absolute_import + +import os +import ctypes +import platform +import multiprocessing + + +def _load_lib(): + """ Load library in build/lib. """ + cur_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) + lib_path = os.path.join(cur_path, "../../build/") + if platform.system() == 'Darwin': + path_to_so_file = os.path.join(lib_path, "libmagent.dylib") + elif platform.system() == 'Linux': + path_to_so_file = os.path.join(lib_path, "libmagent.so") + else: + raise BaseException("unsupported system: " + platform.system()) + lib = ctypes.CDLL(path_to_so_file, ctypes.RTLD_GLOBAL) + return lib + + +def as_float_c_array(buf): + """numpy to ctypes array""" + return buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) + + +def as_int32_c_array(buf): + """numpy to ctypes array""" + return buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)) + + +def as_bool_c_array(buf): + """numpy to ctypes array""" + return buf.ctypes.data_as(ctypes.POINTER(ctypes.c_bool)) + + +if 'OMP_NUM_THREADS' not in os.environ: + os.environ['OMP_NUM_THREADS'] = str(multiprocessing.cpu_count() // 2) +_LIB = _load_lib() diff --git a/examples/battle_model/python/magent/c_lib.pyc b/examples/battle_model/python/magent/c_lib.pyc new file mode 100755 index 0000000..17c9aad Binary files /dev/null and b/examples/battle_model/python/magent/c_lib.pyc differ diff --git a/examples/battle_model/python/magent/discrete_snake.py b/examples/battle_model/python/magent/discrete_snake.py new file mode 100755 index 0000000..7e31c6f --- /dev/null +++ b/examples/battle_model/python/magent/discrete_snake.py @@ -0,0 +1,210 @@ +""" Deprecated!! """ + +from __future__ import absolute_import + +import ctypes +import os +import importlib + +import numpy as np + +from .c_lib import _LIB, as_float_c_array, as_int32_c_array +from .environment import Environment + + +class DiscreteSnake(Environment): + """deprecated""" + OBS_VIEW_INDEX = 0 + OBS_FEATURE_INDEX = 1 + + def __init__(self, config, **kwargs): + Environment.__init__(self) + + # for global settings + game = ctypes.c_void_p() + _LIB.env_new_game(ctypes.byref(game), b"DiscreteSnake") + self.game = game + + config_value_type = { + 'map_width': int, 'map_height': int, + 'view_width': int, 'view_height': int, + 'max_dead_penalty': float, 'corpse_value': float, + 'embedding_size': int, 'total_resource': int, + 'render_dir': str, + } + + # config general setting + for key in config.config_dict: + print("discrete_snake.py L37 : ", key, config.config_dict[key]) + value_type = config_value_type[key] + if value_type is int: + _LIB.env_config_game(self.game, key, ctypes.byref(ctypes.c_int(config.config_dict[key]))) + elif value_type is bool: + _LIB.env_config_game(self.game, key, ctypes.byref(ctypes.c_bool(config.config_dict[key]))) + elif value_type is float: + _LIB.env_config_game(self.game, key, ctypes.byref(ctypes.c_float(config.config_dict[key]))) + elif value_type is str: + _LIB.env_config_game(self.game, key, ctypes.c_char_p(config.config_dict[key])) + + # init observation buffer (for acceleration) + self._init_obs_buf() + + # init view size, feature size, action space + buf = np.empty((3,), dtype=np.int32) + _LIB.env_get_info(self.game, 0, b"view_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.view_space = [buf[0], buf[1], buf[2]] + _LIB.env_get_info(self.game, 0, b"feature_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.feature_space = buf[0] + _LIB.env_get_info(self.game, 0, b"action_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.action_space = buf[0] + + def reset(self): + _LIB.env_reset(self.game) + + def _add_object(self, obj_id, method, **kwargs): + if method == "random": + _LIB.discrete_snake_add_object(self.game, obj_id, int(kwargs["n"]), b"random", 0) + else: + print("unsupported type of method") + exit(-1) + + def add_walls(self, method, **kwargs): + # handle = -1 for walls + self._add_object(-1, method, **kwargs) + + def add_food(self, method, **kwargs): + # handles = -2 for food + self._add_object(-2, method, **kwargs) + + def add_agent(self, method, *args, **kwargs): + self._add_object(0, method, **kwargs) + + # ====== RUN ====== + def _get_obs_buf(self, key, shape, dtype): + if self.obs_bufs[key] is None: + group_buf = self.obs_bufs[key] = [1] # (buf_id, buf1, buf2, ...) + group_buf.append(np.zeros(shape=shape, dtype=dtype)) + group_buf.append(np.zeros(shape=shape, dtype=dtype)) + ret = group_buf[1] + else: + group_buf = self.obs_bufs[key] + turn = group_buf[0] + ret = group_buf[turn] + if shape != ret.shape: + ret.resize(shape, refcheck=False) + group_buf[0] = (turn-1 + 1) % 2 + 1 + + return ret + + def _init_obs_buf(self): + self.obs_bufs = [None, None] + + def get_observation(self, handle=0): + view_space = self.view_space + feature_space = self.feature_space + + n = self.get_num(handle) + view_buf = self._get_obs_buf(self.OBS_VIEW_INDEX, [n] + view_space, np.float32) + feature_buf = self._get_obs_buf(self.OBS_FEATURE_INDEX, (n, feature_space), np.float32) + + bufs = (ctypes.POINTER(ctypes.c_float) * 2)() + bufs[0] = as_float_c_array(view_buf) + bufs[1] = as_float_c_array(feature_buf) + _LIB.env_get_observation(self.game, handle, bufs) + + return view_buf, feature_buf + + def set_action(self, handle, actions): + assert isinstance(actions, np.ndarray) + assert actions.dtype == np.int32 + _LIB.env_set_action(self.game, handle, actions.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + + def step(self): + done = ctypes.c_int32() + _LIB.env_step(self.game, ctypes.byref(done)) + return done + + def get_reward(self, handle=0): + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.float32) + _LIB.env_get_reward(self.game, handle, + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))) + return buf + + def clear_dead(self): + _LIB.discrete_snake_clear_dead(self.game) + + # ====== INFO ====== + def get_num(self, handle=0): + num = ctypes.c_int32() + _LIB.env_get_info(self.game, handle, "num", ctypes.byref(num)) + return num.value + + def get_action_space(self, handle=0): + return self.action_space + + def get_view_space(self, handle=0): + return self.view_space + + def get_feature_space(self, handle=0): + return self.feature_space + + def get_agent_id(self, handle=0): + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.int32) + _LIB.env_get_info(self.game, handle, b"id", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + return buf + + def get_head(self, handle=0): + n = self.get_num(handle) + buf = np.empty((n, 2), dtype=np.int32) + _LIB.env_get_info(self.game, handle, b"head", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + return buf + + def get_alive(self, handle=0): + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.bool) + _LIB.env_get_info(self.game, handle, b"alive", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_bool))) + return buf + + def get_length(self, handle=0): + n = self.get_num(handle) + buf = np.empty((n, ), dtype=np.int32) + _LIB.env_get_info(self.game, handle, b"length", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int))) + return buf + + def get_food_num(self): + num = ctypes.c_int32() + _LIB.env_get_info(self.game, -2, "num", ctypes.byref(num)) # -2 for food + return num.value + + # ====== RENDER ====== + def set_render_dir(self, name): + if not os.path.exists(name): + os.mkdir(name) + _LIB.env_config_game(self.game, b"render_dir", name) + + def render(self): + _LIB.env_render(self.game) + + def render_next_file(self): + _LIB.env_render_next_file(self.game) + + def __del__(self): + _LIB.env_delete_game(self.game) + + +class Config: + def __init__(self): + self.config_dict = {} + + def set(self, args): + for key in args: + self.config_dict[key] = args[key] \ No newline at end of file diff --git a/examples/battle_model/python/magent/environment.py b/examples/battle_model/python/magent/environment.py new file mode 100755 index 0000000..4181437 --- /dev/null +++ b/examples/battle_model/python/magent/environment.py @@ -0,0 +1,45 @@ +""" base class for environment """ + + +class Environment: + """see subclass for detailed comment""" + def __init__(self): + pass + + def reset(self): + pass + + # ====== RUN ====== + def get_observation(self, handle): + pass + + def set_action(self, handle, actions): + pass + + def step(self): + pass + + def render(self): + pass + + def render_next_file(self): + pass + + def get_reward(self, handle): + pass + def set_chs(self,handle,chs): + pass + + # ====== INFO ====== + def get_num(self, handle): + pass + + def get_action_space(self, handle): + pass + + def get_view_space(self, handle): + pass + + def get_feature_space(self, handle): + pass + diff --git a/examples/battle_model/python/magent/environment.pyc b/examples/battle_model/python/magent/environment.pyc new file mode 100755 index 0000000..ab2402f Binary files /dev/null and b/examples/battle_model/python/magent/environment.pyc differ diff --git a/examples/battle_model/python/magent/gridworld.py b/examples/battle_model/python/magent/gridworld.py new file mode 100755 index 0000000..2b45162 --- /dev/null +++ b/examples/battle_model/python/magent/gridworld.py @@ -0,0 +1,812 @@ +"""gridworld interface""" +from __future__ import absolute_import + +import ctypes +import os +import importlib + +import numpy as np + +from .c_lib import _LIB, as_float_c_array, as_int32_c_array +from .environment import Environment + + +class GridWorld(Environment): + # constant + OBS_INDEX_VIEW = 0 + OBS_INDEX_HP = 1 + + def __init__(self, config, **kwargs): + """ + Parameters + ---------- + config: str or Config Object + if config is a string, then it is a name of builtin config, + builtin config are stored in python/magent/builtin/config + kwargs are the arguments to the config + if config is a Config Object, then parameters are stored in that object + """ + Environment.__init__(self) + + # if is str, load built in configuration + if isinstance(config, str): + # built-in config are stored in python/magent/builtin/config + try: + demo_game = importlib.import_module('magent.builtin.config.' + config) + config = getattr(demo_game, 'get_config')(**kwargs) + except AttributeError: + raise BaseException('unknown built-in game "' + config + '"') + + # create new game + game = ctypes.c_void_p() + _LIB.env_new_game(ctypes.byref(game), b"GridWorld") + self.game = game + + # set global configuration + config_value_type = { + 'map_width': int, 'map_height': int, + 'food_mode': bool, 'turn_mode': bool, 'minimap_mode': bool, + 'revive_mode': bool, 'goal_mode': bool, + 'embedding_size': int, + 'render_dir': str, + } + + for key in config.config_dict: + value_type = config_value_type[key] + if value_type is int: + _LIB.env_config_game(self.game, key.encode("ascii"), ctypes.byref(ctypes.c_int(config.config_dict[key]))) + elif value_type is bool: + _LIB.env_config_game(self.game, key.encode("ascii"), ctypes.byref(ctypes.c_bool(config.config_dict[key]))) + elif value_type is float: + _LIB.env_config_game(self.game, key.encode("ascii"), ctypes.byref(ctypes.c_float(config.config_dict[key]))) + elif value_type is str: + _LIB.env_config_game(self.game, key.encode("ascii"), ctypes.c_char_p(config.config_dict[key])) + + # register agent types + for name in config.agent_type_dict: + type_args = config.agent_type_dict[name] + + # special pre-process for view range and attack range + for key in [x for x in type_args.keys()]: + if key == "view_range": + val = type_args[key] + del type_args[key] + type_args["view_radius"] = val.radius + type_args["view_angle"] = val.angle + elif key == "attack_range": + val = type_args[key] + del type_args[key] + type_args["attack_radius"] = val.radius + type_args["attack_angle"] = val.angle + + length = len(type_args) + keys = (ctypes.c_char_p * length)(*[key.encode("ascii") for key in type_args.keys()]) + values = (ctypes.c_float * length)(*type_args.values()) + + _LIB.gridworld_register_agent_type(self.game, name.encode("ascii"), length, keys, values) + + # serialize event expression, send to C++ engine + self._serialize_event_exp(config) + + # init group handles + self.group_handles = [] + for item in config.groups: + handle = ctypes.c_int32() + _LIB.gridworld_new_group(self.game, item.encode("ascii"), ctypes.byref(handle)) + self.group_handles.append(handle) + + # init observation buffer (for acceleration) + self._init_obs_buf() + + # init view space, feature space, action space + self.view_space = {} + self.feature_space = {} + self.action_space = {} + buf = np.empty((3,), dtype=np.int32) + for handle in self.group_handles: + _LIB.env_get_info(self.game, handle, b"view_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.view_space[handle.value] = (buf[0], buf[1], buf[2]) + _LIB.env_get_info(self.game, handle, b"feature_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.feature_space[handle.value] = (buf[0],) + _LIB.env_get_info(self.game, handle, b"action_space", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + self.action_space[handle.value] = (buf[0],) + + def reset(self): + """reset environment""" + _LIB.env_reset(self.game) + + def add_walls(self, method, **kwargs): + """add wall to environment + + Parameters + ---------- + method: str + can be 'random' or 'custom' + if method is 'random', then kwargs["n"] is a int + if method is 'custom', then kwargs["pos"] is a list of coordination + + Examples + -------- + # add 1000 walls randomly + >>> env.add_walls(method="random", n=1000) + + # add 3 walls to (1,2), (4,5) and (9, 8) in map + >>> env.add_walls(method="custom", pos=[(1,2), (4,5), (9,8)]) + """ + # handle = -1 for walls + kwargs["dir"] = 0 + self.add_agents(-1, method, **kwargs) + + # ====== AGENT ====== + def new_group(self, name): + """register a new group into environment""" + handle = ctypes.c_int32() + _LIB.gridworld_new_group(self.game, ctypes.c_char_p(name.encode("ascii")), ctypes.byref(handle)) + return handle + + def add_agents(self, handle, method, **kwargs): + """add agents to environment + + Parameters + ---------- + handle: group handle + method: str + can be 'random' or 'custom' + if method is 'random', then kwargs["n"] is a int + if method is 'custom', then kwargs["pos"] is a list of coordination + + Examples + -------- + # add 1000 walls randomly + >>> env.add_agents(handle, method="random", n=1000) + + # add 3 agents to (1,2), (4,5) and (9, 8) in map + >>> env.add_agents(handle, method="custom", pos=[(1,2), (4,5), (9,8)]) + """ + if method == "random": + _LIB.gridworld_add_agents(self.game, handle, int(kwargs["n"]), b"random", 0, 0, 0) + elif method == "custom": + n = len(kwargs["pos"]) + pos = np.array(kwargs["pos"], dtype=np.int32) + if len(pos) <= 0: + return + if pos.shape[1] == 3: # if has dir + xs, ys, dirs = pos[:, 0], pos[:, 1], pos[:, 2] + else: # if do not has dir, use zero padding + xs, ys, dirs = pos[:, 0], pos[:, 1], np.zeros((n,), dtype=np.int32) + # copy again, to make these arrays continuous in memory + xs, ys, dirs = np.array(xs), np.array(ys), np.array(dirs) + _LIB.gridworld_add_agents(self.game, handle, n, b"custom", as_int32_c_array(xs), + as_int32_c_array(ys), as_int32_c_array(dirs)) + elif method == "fill": + x, y = kwargs["pos"][0], kwargs["pos"][1] + width, height = kwargs["size"][0], kwargs["size"][1] + dir = kwargs.get("dir", np.zeros_like(x)) + bind = np.array([x, y, width, height, dir], dtype=np.int32) + _LIB.gridworld_add_agents(self.game, handle, 0, b"fill", as_int32_c_array(bind), + 0, 0, 0) + elif method == "maze": + # TODO: implement maze add + x_start, y_start, x_end, y_end = kwargs["pos"][0], kwargs["pos"][1], kwargs["pos"][2], kwargs["pos"][3] + thick = kwargs["pos"][4] + bind = np.array([x_start, y_start, x_end, y_end, thick], dtype=np.int32) + _LIB.gridworld_add_agents(self.game, handle, 0, b"maze", as_int32_c_array(bind), + 0, 0, 0) + else: + print("Unknown type of position") + exit(-1) + + # ====== RUN ====== + def _get_obs_buf(self, group, key, shape, dtype): + """get buffer to receive observation from c++ engine""" + obs_buf = self.obs_bufs[key] + if group in obs_buf: + ret = obs_buf[group] + if shape != ret.shape: + ret.resize(shape, refcheck=False) + else: + ret = obs_buf[group] = np.empty(shape=shape, dtype=dtype) + + return ret + + def _init_obs_buf(self): + """init observation buffer""" + self.obs_bufs = [] + self.obs_bufs.append({}) + self.obs_bufs.append({}) + + def get_observation(self, handle): + """ get observation of a whole group + + Parameters + ---------- + handle : group handle + + Returns + ------- + obs : tuple (views, features) + views is a numpy array, whose shape is n * view_width * view_height * n_channel + features is a numpy array, whose shape is n * feature_size + for agent i, (views[i], features[i]) is its observation at this step + """ + view_space = self.view_space[handle.value] + feature_space = self.feature_space[handle.value] + no = handle.value + + n = self.get_num(handle) + view_buf = self._get_obs_buf(no, self.OBS_INDEX_VIEW, (n,) + view_space, np.float32) + feature_buf = self._get_obs_buf(no, self.OBS_INDEX_HP, (n,) + feature_space, np.float32) + + bufs = (ctypes.POINTER(ctypes.c_float) * 2)() + bufs[0] = as_float_c_array(view_buf) + bufs[1] = as_float_c_array(feature_buf) + _LIB.env_get_observation(self.game, handle, bufs) + + return view_buf, feature_buf + + def set_chs(self,handle,is_chs): + assert isinstance(is_chs, np.ndarray) + assert is_chs.dtype == np.int32 + _LIB.env_set_chs(self.game, handle, is_chs.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + + + def set_action(self, handle, actions): + """ set actions for whole group + + Parameters + ---------- + handle: group handle + actions: numpy array + the dtype of actions must be int32 + """ + assert isinstance(actions, np.ndarray) + assert actions.dtype == np.int32 + _LIB.env_set_action(self.game, handle, actions.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + + def step(self): + """simulation one step after set actions + + Returns + ------- + done: bool + whether the game is done + """ + done = ctypes.c_int32() + _LIB.env_step(self.game, ctypes.byref(done)) + return bool(done) + + def get_reward(self, handle): + """ get reward for a whole group + + Returns + ------- + rewards: numpy array (float32) + reward for all the agents in the group + """ + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.float32) + _LIB.env_get_reward(self.game, handle, + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))) + return buf + + def clear_dead(self): + """ clear dead agents in the engine + must be called after step() + """ + _LIB.gridworld_clear_dead(self.game) + + # ====== INFO ====== + def get_handles(self): + """ get all group handles in the environment """ + return self.group_handles + + def get_num(self, handle): + """ get the number of agents in a group""" + num = ctypes.c_int32() + _LIB.env_get_info(self.game, handle, b'num', ctypes.byref(num)) + return num.value + + def get_action_space(self, handle): + """get action space + + Returns + ------- + action_space : tuple + """ + return self.action_space[handle.value] + + def get_view_space(self, handle): + """get view space + + Returns + ------- + view_space : tuple + """ + return self.view_space[handle.value] + + def get_feature_space(self, handle): + """ get feature space + + Returns + ------- + feature_space : tuple + """ + return self.feature_space[handle.value] + + def get_agent_id(self, handle): + """ get agent id + + Returns + ------- + ids : numpy array (int32) + id of all the agents in the group + """ + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.int32) + _LIB.env_get_info(self.game, handle, b"id", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + return buf + + def get_alive(self, handle): + """ get alive status of agents in a group + + Returns + ------- + alives: numpy array (bool) + whether the agents are alive + """ + n = self.get_num(handle) + buf = np.empty((n,), dtype=np.bool) + _LIB.env_get_info(self.game, handle, b"alive", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_bool))) + return buf + + def get_pos(self, handle): + """ get position of agents in a group + + Returns + ------- + pos: numpy array (int) + the shape of pos is (n, 2) + """ + n = self.get_num(handle) + buf = np.empty((n, 2), dtype=np.int32) + _LIB.env_get_info(self.game, handle, b"pos", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + return buf + def get_hps(self,handle): + n=self.get_num(handle) + buf = np.empty((n,),dtype=np.float32) + _LIB.env_get_info(self.game, handle, b"hps", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))) + return buf + + def get_mean_info(self, handle): + """ deprecated """ + buf = np.empty(2 + self.action_space[handle.value][0], dtype=np.float32) + _LIB.env_get_info(self.game, handle, b"mean_info", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))) + return buf + + def get_view2attack(self, handle): + """ get a matrix with the same size of view_range, + if element >= 0, then it means it is a attackable point, and the corresponding + action number is the value of that element + Returns + ------- + attack_back: int + buf: numpy array + map attack action into view + """ + size = self.get_view_space(handle)[0:2] + buf = np.empty(size, dtype=np.int32) + attack_base = ctypes.c_int32() + _LIB.env_get_info(self.game, handle, b"view2attack", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + _LIB.env_get_info(self.game, handle, b"attack_base", + ctypes.byref(attack_base)) + return attack_base.value, buf + + def get_global_minimap(self, height, width): + """ compress global map into a minimap of given size + Parameters + ---------- + height: int + the height of minimap + width: int + the width of minimap + + Returns + ------- + minimap : numpy array + the shape (n_group + 1, height, width) + """ + buf = np.empty((height, width, len(self.group_handles)), dtype=np.float32) + buf[0, 0, 0] = height + buf[0, 0, 1] = width + _LIB.env_get_info(self.game, -1, b"global_minimap", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))) + return buf + + def set_seed(self, seed): + """ set random seed of the engine""" + _LIB.env_config_game(self.game, b"seed", ctypes.byref(ctypes.c_int(seed))) + + # ====== RENDER ====== + def set_render_dir(self, name): + """ set directory to save render file""" + if not os.path.exists(name): + os.mkdir(name) + _LIB.env_config_game(self.game, b"render_dir", name.encode("ascii")) + + def render(self): + """ render a step """ + _LIB.env_render(self.game) + + def _get_groups_info(self): + """ private method, for interactive application""" + n = len(self.group_handles) + buf = np.empty((n, 5), dtype=np.int32) + _LIB.env_get_info(self.game, -1, b"groups_info", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + return buf + + def _get_walls_info(self): + """ private method, for interactive application""" + n = 100 * 100 + buf = np.empty((n, 2), dtype=np.int32) + _LIB.env_get_info(self.game, -1, b"walls_info", + buf.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))) + n = buf[0, 0] # the first line is the number of walls + return buf[1:1+n] + + def _get_render_info(self, x_range, y_range): + """ private method, for interactive application""" + n = 0 + for handle in self.group_handles: + n += self.get_num(handle) + + buf = np.empty((n+1, 4), dtype=np.int32) + buf[0] = x_range[0], y_range[0], x_range[1], y_range[1] + _LIB.env_get_info(self.game, -1, b"render_window_info", + buf.ctypes.data_as(ctypes.POINTER((ctypes.c_int32)))) + + # the first line is for the number of agents in the window range + info_line = buf[0] + agent_ct, attack_event_ct = info_line[0], info_line[1] + buf = buf[1:1 + info_line[0]] + + agent_info = {} + for item in buf: + agent_info[item[0]] = [item[1], item[2], item[3]] + + buf = np.empty((attack_event_ct, 3), dtype=np.int32) + _LIB.env_get_info(self.game, -1, b"attack_event", + buf.ctypes.data_as(ctypes.POINTER((ctypes.c_int32)))) + attack_event = buf + + return agent_info, attack_event + + def __del__(self): + _LIB.env_delete_game(self.game) + + # ====== SPECIAL RULE ====== + def set_goal(self, handle, method, *args, **kwargs): + """ deprecated """ + if method == "random": + _LIB.gridworld_set_goal(self.game, handle, b"random", 0, 0) + else: + raise NotImplementedError + + # ====== PRIVATE ====== + def _serialize_event_exp(self, config): + """serialize event expression and sent them to game engine""" + game = self.game + + # collect agent symbol + symbol2int = {} + config.symbol_ct = 0 + + def collect_agent_symbol(node, config): + for item in node.inputs: + if isinstance(item, EventNode): + collect_agent_symbol(item, config) + elif isinstance(item, AgentSymbol): + if item not in symbol2int: + symbol2int[item] = config.symbol_ct + config.symbol_ct += 1 + + for rule in config.reward_rules: + on = rule[0] + receiver = rule[1] + for symbol in receiver: + if symbol not in symbol2int: + symbol2int[symbol] = config.symbol_ct + config.symbol_ct += 1 + collect_agent_symbol(on, config) + + # collect event node + event2int = {} + config.node_ct = 0 + + def collect_event_node(node, config): + if node not in event2int: + event2int[node] = config.node_ct + config.node_ct += 1 + for item in node.inputs: + if isinstance(item, EventNode): + collect_event_node(item, config) + + for rule in config.reward_rules: + collect_event_node(rule[0], config) + + # send to C++ engine + for sym in symbol2int: + no = symbol2int[sym] + _LIB.gridworld_define_agent_symbol(game, no, sym.group, sym.index) + + for event in event2int: + no = event2int[event] + inputs = np.zeros_like(event.inputs, dtype=np.int32) + for i, item in enumerate(event.inputs): + if isinstance(item, EventNode): + inputs[i] = event2int[item] + elif isinstance(item, AgentSymbol): + inputs[i] = symbol2int[item] + else: + inputs[i] = item + n_inputs = len(inputs) + _LIB.gridworld_define_event_node(game, no, event.op, as_int32_c_array(inputs), n_inputs) + + for rule in config.reward_rules: + # rule = [on, receiver, value, terminal] + on = event2int[rule[0]] + + receiver = np.zeros_like(rule[1], dtype=np.int32) + for i, item in enumerate(rule[1]): + receiver[i] = symbol2int[item] + if len(rule[2]) == 1 and rule[2][0] == 'auto': + value = np.zeros(receiver, dtype=np.float32) + else: + value = np.array(rule[2], dtype=np.float32) + n_receiver = len(receiver) + _LIB.gridworld_add_reward_rule(game, on, as_int32_c_array(receiver), + as_float_c_array(value), n_receiver, rule[3]) + + +''' +the following classes are for reward description +''' +class EventNode: + """an AST node of the event expression""" + OP_AND = 0 + OP_OR = 1 + OP_NOT = 2 + + OP_KILL = 3 + OP_AT = 4 + OP_IN = 5 + OP_COLLIDE = 6 + OP_ATTACK = 7 + OP_DIE = 8 + OP_IN_A_LINE = 9 + OP_ALIGN = 10 + + # can extend more operation below + + def __init__(self): + # for non-leaf node + self.op = None + # for leaf node + self.predicate = None + + self.inputs = [] + + def __call__(self, subject, predicate, *args): + node = EventNode() + node.predicate = predicate + if predicate == 'kill': + node.op = EventNode.OP_KILL + node.inputs = [subject, args[0]] + elif predicate == 'at': + node.op = EventNode.OP_AT + coor = args[0] + node.inputs = [subject, coor[0], coor[1]] + elif predicate == 'in': + node.op = EventNode.OP_IN + coor = args[0] + x1, y1 = min(coor[0][0], coor[1][0]), min(coor[0][1], coor[1][1]) + x2, y2 = max(coor[0][0], coor[1][0]), max(coor[0][1], coor[1][1]) + node.inputs = [subject, x1, y1, x2, y2] + elif predicate == 'attack': + node.op = EventNode.OP_ATTACK + node.inputs = [subject, args[0]] + elif predicate == 'kill': + node.op = EventNode.OP_KILL + node.inputs = [subject, args[0]] + elif predicate == 'collide': + node.op = EventNode.OP_COLLIDE + node.inputs = [subject, args[0]] + elif predicate == 'die': + node.op = EventNode.OP_DIE + node.inputs = [subject] + elif predicate == 'in_a_line': + node.op = EventNode.OP_IN_A_LINE + node.inputs = [subject] + elif predicate == 'align': + node.op = EventNode.OP_ALIGN + node.inputs = [subject] + else: + raise Exception("invalid predicate of event " + predicate) + return node + + def __and__(self, other): + node = EventNode() + node.op = EventNode.OP_AND + node.inputs = [self, other] + return node + + def __or__(self, other): + node = EventNode() + node.op = EventNode.OP_OR + node.inputs = [self, other] + return node + + def __invert__(self): + node = EventNode() + node.op = EventNode.OP_NOT + node.inputs = [self] + return node + +Event = EventNode() + +class AgentSymbol: + """symbol to represent some agents""" + def __init__(self, group, index): + """ define a agent symbol, it can be the object or subject of EventNode + + group: group handle + it is the return value of cfg.add_group() + index: int or str + int: a deterministic integer id + str: can be 'all' or 'any', represents all or any agents in a group + """ + self.group = group if group is not None else -1 + if index == 'any': + self.index = -1 + elif index == 'all': + self.index = -2 + else: + assert isinstance(self.index, int), "index must be a deterministic int" + self.index = index + + def __str__(self): + return 'agent(%d,%d)' % (self.group, self.index) + + +class Config: + """configuration class of gridworld game""" + def __init__(self): + self.config_dict = {} + self.agent_type_dict = {} + self.groups = [] + self.reward_rules = [] + + def set(self, args): + """ set parameters of global configuration + + Parameters + ---------- + args : dict + key value pair of the configuration + """ + for key in args: + self.config_dict[key] = args[key] + + def register_agent_type(self, name, attr): + """ register an agent type + + Parameters + ---------- + name : str + name of the type (should be unique) + attr: dict + key value pair of the agent type + see notes below to know the available attributes + + Notes + ----- + height: int, height of agent body + width: int, width of agent body + speed: float, maximum speed, i.e. the radius of move circle of the agent + hp: float, maximum health point of the agent + view_range: gw.CircleRange or gw.SectorRange + + damage: float, attack damage + step_recover: float, step recover of health point (can be negative) + kill_supply: float, the hp gain when kill this type of agents + + step_reward: float, reward get in every step + kill_reward: float, reward gain when kill this type of agent + dead_penalty: float, reward get when dead + attack_penalty: float, reward get when perform an attack (this is used to make agents do not attack blank grid) + """ + if name in self.agent_type_dict: + raise Exception("type name %s already exists" % name) + self.agent_type_dict[name] = attr + return name + + def add_group(self, agent_type): + """ add a group to the configuration + + Returns + ------- + group_handle : int + a handle for the new added group + """ + no = len(self.groups) + self.groups.append(agent_type) + return no + + def add_reward_rule(self, on, receiver, value, terminal=False): + """ add a reward rule + + Some note: + 1. if the receiver is not a deterministic agent, + it must be one of the agents involved in the triggering event + + Parameters + ---------- + on: Expr + a bool expression of the trigger event + receiver: (list of) AgentSymbol + receiver of this reward rule + value: (list of) float + value to assign + terminal: bool + whether this event will terminate the game + """ + if not (isinstance(receiver, tuple) or isinstance(receiver, list)): + assert not (isinstance(value, tuple) or isinstance(value, tuple)) + receiver = [receiver] + value = [value] + if len(receiver) != len(value): + raise Exception("the length of receiver and value should be equal") + self.reward_rules.append([on, receiver, value, terminal]) + + +class CircleRange: + def __init__(self, radius): + """ define a circle range for attack or view + + Parameters + ---------- + radius : float + """ + self.radius = radius + self.angle = 360 + + def __str__(self): + return 'circle(%g)' % self.radius + + +class SectorRange: + def __init__(self, radius, angle): + """ define a sector range for attack or view + + Parameters + ---------- + radius : float + angle : float + angle should be less than 180 + """ + self.radius = radius + self.angle = angle + if self.angle >= 180: + raise Exception("the angle of a sector should be smaller than 180 degree") + + def __str__(self): + return 'sector(%g, %g)' % (self.radius, self.angle) diff --git a/examples/battle_model/python/magent/gridworld.pyc b/examples/battle_model/python/magent/gridworld.pyc new file mode 100755 index 0000000..a1d5311 Binary files /dev/null and b/examples/battle_model/python/magent/gridworld.pyc differ diff --git a/examples/battle_model/python/magent/model.py b/examples/battle_model/python/magent/model.py new file mode 100755 index 0000000..bddb37a --- /dev/null +++ b/examples/battle_model/python/magent/model.py @@ -0,0 +1,338 @@ +""" base model classes""" + +try: + import thread +except ImportError: + import _thread as thread + +import multiprocessing +import multiprocessing.connection +import sys + +import numpy as np + +class BaseModel: + def __init__(self, env, handle, *args, **kwargs): + """ init + + Parameters + ---------- + env: Environment + env + handle: GroupHandle + handle of this group, handles are returned by env.get_handles() + """ + pass + + def infer_action(self, raw_obs, ids, *args, **kwargs): + """ infer action for a group of agents + + Parameters + ---------- + raw_obs: tuple + raw_obs is a tuple of (view, feature) + view is a numpy array, its shape is n * view_width * view_height * n_channel + it contains the spatial local observation for all the agents + feature is a numpy array, its shape is n * feature_size + it contains the non-spatial feature for all the agents + ids: numpy array of int32 + the unique id of every agents + args: + additional custom args + kwargs: + additional custom args + """ + pass + + def train(self, sample_buffer, **kwargs): + """ feed new samples and train + + Parameters + ---------- + sample_buffer: EpisodesBuffer + a buffer contains transitions of agents + + Returns + ------- + loss and estimated mean state value + """ + return 0, 0 # loss, mean value + + def save(self, *args, **kwargs): + """ save the model """ + pass + + def load(self, *args, **kwargs): + """ load the model """ + pass + + +class NDArrayPackage: + """wrapper for transferring numpy arrays by bytes""" + def __init__(self, *args): + if isinstance(args[0], np.ndarray): + self.data = args + self.info = [(x.shape, x.dtype) for x in args] + else: + self.data = None + self.info = args[0] + + self.max_len = (1 << 30) / 4 + + def send_to(self, conn, use_thread=False): + assert self.data is not None + + def send_thread(): + for x in self.data: + if np.prod(x.shape) > self.max_len: + seg = int(self.max_len // np.prod(x.shape[1:])) + for pt in range(0, len(x), seg): + conn.send_bytes(x[pt:pt+seg]) + else: + conn.send_bytes(x) + + if use_thread: + thread.start_new_thread(send_thread, ()) + else: + send_thread() + + def recv_from(self, conn): + bufs = [] + for info in self.info: + buf = np.empty(shape=(int(np.prod(info[0])),), dtype=info[1]) + + item_size = int(np.prod(info[0][1:])) + if np.prod(info[0]) > self.max_len: + seg = int(self.max_len // item_size) + for pt in range(0, int(np.prod(info[0])), seg * item_size): + conn.recv_bytes_into(buf[pt:pt+seg * item_size]) + else: + conn.recv_bytes_into(buf) + bufs.append(buf.reshape(info[0])) + return bufs + + +class ProcessingModel(BaseModel): + """ + start a sub-processing to host a model, + use pipe or socket for communication + """ + def __init__(self, env, handle, name, port, sample_buffer_capacity=1000, + RLModel=None, **kwargs): + """ + Parameters + ---------- + env: environment + handle: group handle + name: str + name of the model (be used when store model) + port: int + port of socket or suffix of pipe + sample_buffer_capacity: int + the maximum number of samples (s,r,a,s') to collect in a game round + RLModel: BaseModel + the RL algorithm class + kwargs: dict + arguments for RLModel + """ + BaseModel.__init__(self, env, handle) + + assert RLModel is not None + + kwargs['env'] = env + kwargs['handle'] = handle + kwargs['name'] = name + addr = 'magent-pipe-' + str(port) # named pipe + # addr = ('localhost', port) # socket + proc = multiprocessing.Process( + target=model_client, + args=(addr, sample_buffer_capacity, RLModel, kwargs), + ) + + proc.start() + listener = multiprocessing.connection.Listener(addr) + self.conn = listener.accept() + + def sample_step(self, rewards, alives, block=True): + """record a step (should be followed by check_done) + + Parameters + ---------- + block: bool + if it is True, the function call will block + if it is False, the caller must call check_done() afterward + to check/consume the return message + """ + package = NDArrayPackage(rewards, alives) + self.conn.send(["sample", package.info]) + package.send_to(self.conn) + + if block: + self.check_done() + + def infer_action(self, raw_obs, ids, policy='e_greedy', eps=0, block=True): + """ infer action + + Parameters + ---------- + policy: str + can be 'e_greedy' or 'greedy' + eps: float + used when policy is 'e_greedy' + block: bool + if it is True, the function call will block, and return actions + if it is False, the function call won't block, the caller + must call fetch_action() to get actions + + Returns + ------- + actions: numpy array (int32) + see above + """ + + package = NDArrayPackage(raw_obs[0], raw_obs[1], ids) + self.conn.send(["act", policy, eps, package.info]) + package.send_to(self.conn, use_thread=True) + + if block: + info = self.conn.recv() + return NDArrayPackage(info).recv_from(self.conn)[0] + else: + return None + + def fetch_action(self): + """ fetch actions , fetch action after calling infer_action(block=False) + + Returns + ------- + actions: numpy array (int32) + """ + info = self.conn.recv() + return NDArrayPackage(info).recv_from(self.conn)[0] + + def train(self, print_every=5000, block=True): + """ train new data samples according to the model setting + + Parameters + ---------- + print_every: int + print training log info every print_every batches + + """ + self.conn.send(['train', print_every]) + + if block: + return self.fetch_train() + + def fetch_train(self): + """ fetch result of train after calling train(block=False) + + Returns + ------- + loss: float + mean loss + value: float + mean state value + """ + return self.conn.recv() + + def save(self, save_dir, epoch, block=True): + """ save model + + Parameters + ---------- + block: bool + if it is True, the function call will block + if it is False, the caller must call check_done() afterward + to check/consume the return message + """ + + self.conn.send(["save", save_dir, epoch]) + if block: + self.check_done() + + def load(self, save_dir, epoch, name=None, block=True): + """ load model + + Parameters + ---------- + name: str + name of the model (set when stored name is not the same as self.name) + block: bool + if it is True, the function call will block + if it is False, the caller must call check_done() afterward + to check/consume the return message + """ + self.conn.send(["load", save_dir, epoch, name]) + if block: + self.check_done() + + def check_done(self): + """ check return message of sub processing """ + assert self.conn.recv() == 'done' + + def quit(self): + """ quit """ + self.conn.send(["quit"]) + + +def model_client(addr, sample_buffer_capacity, RLModel, model_args): + """target function for sub-processing to host a model + + Parameters + ---------- + addr: socket address + sample_buffer_capacity: int + the maximum number of samples (s,r,a,s') to collect in a game round + RLModel: BaseModel + the RL algorithm class + args: dict + arguments to RLModel + """ + import magent.utility + + model = RLModel(**model_args) + sample_buffer = magent.utility.EpisodesBuffer(capacity=sample_buffer_capacity) + + conn = multiprocessing.connection.Client(addr) + + while True: + cmd = conn.recv() + if cmd[0] == 'act': + policy = cmd[1] + eps = cmd[2] + array_info = cmd[3] + + view, feature, ids = NDArrayPackage(array_info).recv_from(conn) + obs = (view, feature) + + acts = model.infer_action(obs, ids, policy=policy, eps=eps) + package = NDArrayPackage(acts) + conn.send(package.info) + package.send_to(conn) + elif cmd[0] == 'train': + print_every = cmd[1] + total_loss, value = model.train(sample_buffer, print_every=print_every) + sample_buffer = magent.utility.EpisodesBuffer(sample_buffer_capacity) + conn.send((total_loss, value)) + elif cmd[0] == 'sample': + array_info = cmd[1] + rewards, alives = NDArrayPackage(array_info).recv_from(conn) + sample_buffer.record_step(ids, obs, acts, rewards, alives) + conn.send("done") + elif cmd[0] == 'save': + savedir = cmd[1] + n_iter = cmd[2] + model.save(savedir, n_iter) + conn.send("done") + elif cmd[0] == 'load': + savedir = cmd[1] + n_iter = cmd[2] + name = cmd[3] + model.load(savedir, n_iter, name) + conn.send("done") + elif cmd[0] == 'quit': + break + else: + print("Error: Unknown command %s" % cmd[0]) + break diff --git a/examples/battle_model/python/magent/model.pyc b/examples/battle_model/python/magent/model.pyc new file mode 100755 index 0000000..40ab2e0 Binary files /dev/null and b/examples/battle_model/python/magent/model.pyc differ diff --git a/examples/battle_model/python/magent/renderer/__init__.py b/examples/battle_model/python/magent/renderer/__init__.py new file mode 100755 index 0000000..a87c571 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/__init__.py @@ -0,0 +1,2 @@ +from .base_renderer import BaseRenderer +from .pygame_renderer import PyGameRenderer diff --git a/examples/battle_model/python/magent/renderer/base_renderer.py b/examples/battle_model/python/magent/renderer/base_renderer.py new file mode 100755 index 0000000..7b4d4f5 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/base_renderer.py @@ -0,0 +1,12 @@ +from abc import ABCMeta, abstractmethod + + +class BaseRenderer: + __metaclass__ = ABCMeta + + def __init__(self): + pass + + @abstractmethod + def start(self, *args, **kwargs): + pass diff --git a/examples/battle_model/python/magent/renderer/pygame_renderer.py b/examples/battle_model/python/magent/renderer/pygame_renderer.py new file mode 100755 index 0000000..16c307c --- /dev/null +++ b/examples/battle_model/python/magent/renderer/pygame_renderer.py @@ -0,0 +1,384 @@ +from __future__ import absolute_import +from __future__ import division + +import math + +import pygame +import numpy as np + +from magent.renderer.base_renderer import BaseRenderer +from magent.renderer.server import BaseServer + + +class PyGameRenderer(BaseRenderer): + def __init__(self): + super(PyGameRenderer, self).__init__() + + def start( + self, + server, + animation_total=2, + animation_stop=0, + resolution=None, + fps_soft_bound=60, + background_rgb=(255, 255, 255), + attack_line_rgb=(0, 0, 0), + attack_dot_rgb=(0, 0, 0), + attack_dot_size=0.3, + text_rgb=(0, 0, 0), + text_size=16, + text_spacing=3, + banner_size=32, + banner_spacing=3, + bigscreen_size=72, + bigscreen_spacing=0, + grid_rgba=(pygame.Color(0, 0, 0), 30), + grid_size=7.5, + grid_min_size=2, + grid_max_size=100, + zoom_rate=1 / 30, + move_rate=4, + full_screen=False + ): + def draw_line(surface, color, a, b): + pygame.draw.line( + surface, color, + (int(round(a[0])), int(round(a[1]))), + (int(round(b[0])), int(round(b[1]))) + ) + + def draw_rect(surface, color, a, w, h): + pygame.draw.rect(surface, color, pygame.Rect(*map(int, ( + round(a[0]), round(a[1]), + round(w + a[0] - round(a[0])), + round(h + a[1] - round(a[1])))))) + + def draw_rect_matrix(matrix, color, a, w, h, resolution): + x, y, w, h = map(int, (round(a[0]), round(a[1]), round(w + a[0] - round(a[0])), round(h + a[1] - round(a[1])))) + matrix[max(x, 0):min(x + w, resolution[0]), max(y, 0):min(h + y, resolution[1]), :] = color + + def draw_line_matrix(matrix, color, a, b, resolution): + a = (min(max(0, a[0]), resolution[0] - 1), min(max(0, a[1]), resolution[1] - 1)) + b = (min(max(0, b[0]), resolution[0] - 1), min(max(0, b[1]), resolution[1] - 1)) + a = map(int, (round(a[0]), round(a[1]))) + b = map(int, (round(b[0]), round(b[1]))) + if a[0] == b[0]: + if a[1] > b[1]: + matrix[a[0], b[1]:a[1] + 1] = color + else: + matrix[a[0], a[1]:b[1] + 1] = color + elif a[1] == b[1]: + if a[0] > b[0]: + matrix[b[0]:a[0] + 1, a[1]] = color + else: + matrix[a[0]:b[0] + 1, a[1]] = color + else: + raise NotImplementedError + + if not isinstance(server, BaseServer): + raise BaseException('property server must be an instance of BaseServer') + + pygame.init() + pygame.display.init() + + if resolution is None: + info = pygame.display.Info() + resolution = info.current_w, info.current_h + + clock = pygame.time.Clock() + + if full_screen: + canvas = pygame.display.set_mode(resolution, pygame.DOUBLEBUF | pygame.FULLSCREEN, 0) + else: + canvas = pygame.display.set_mode(resolution, pygame.DOUBLEBUF, 0) + + pygame.display.set_caption('MAgent Renderer Window') + text_formatter = pygame.font.SysFont(None, text_size, True) + banner_formatter = pygame.font.SysFont(None, banner_size, True) + bigscreen_formatter = pygame.font.SysFont(None, bigscreen_size, True) + + map_size, groups, static_info = server.get_info() + view_position = [map_size[0] / 2 * grid_size - resolution[0] / 2, + map_size[1] / 2 * grid_size - resolution[1] / 2] + frame_id = 0 + + walls = static_info['wall'] + + old_data = None + new_data = None + + need_static_update = True + #show_grid = False + animation_progress = 0 + + grid_map = np.zeros((resolution[0], resolution[1], 3), dtype=np.int16) + + while True: + done = False + status = server.get_status(frame_id) + triggered = False + # calculate the relative moues coordinates in the gridworld + mouse_x, mouse_y = pygame.mouse.get_pos() + mouse_x = int((mouse_x + view_position[0]) / grid_size) + mouse_y = int((mouse_y + view_position[1]) / grid_size) + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + done = True + elif event.type == pygame.KEYDOWN: + #if event.key == pygame.K_g: + # show_grid = not show_grid + #else: + # triggered = server.keydown(frame_id, event.key, mouse_x, mouse_y) + triggered = server.keydown(frame_id, event.key, mouse_x, mouse_y) + elif event.type == pygame.MOUSEBUTTONDOWN: + if event.button == 4 or event.button == 5: + center_before = ( + (view_position[0] + resolution[0] / 2) / grid_size, + (view_position[1] + resolution[1] / 2) / grid_size + ) + if event.button == 5: + grid_size = max(grid_size - grid_size * zoom_rate, grid_min_size) + need_static_update = True + else: + grid_size = min(grid_size + grid_size * zoom_rate, grid_max_size) + need_static_update = True + center_after = ( + (view_position[0] + resolution[0] / 2) / grid_size, + (view_position[1] + resolution[1] / 2) / grid_size + ) + view_position[0] += (center_before[0] - center_after[0]) * grid_size + view_position[1] += (center_before[1] - center_after[1]) * grid_size + else: + triggered = server.mousedown(frame_id, pygame.mouse.get_pressed(), mouse_x, mouse_y) + + pressed = pygame.key.get_pressed() + if pressed[pygame.K_ESCAPE]: + pygame.quit() + done = True + + if pressed[pygame.K_COMMA] or pressed[pygame.K_PERIOD]: + # center before means the center before zoom operation + # center after means the center after zoom operation + # we need to keep that the above two are consistent during zoom operation + # and hence we need to adjust view_position simultaneously + center_before = ( + (view_position[0] + resolution[0] / 2) / grid_size, + (view_position[1] + resolution[1] / 2) / grid_size + ) + if pressed[pygame.K_COMMA]: + grid_size = max(grid_size - grid_size * zoom_rate, grid_min_size) + need_static_update = True + else: + grid_size = min(grid_size + grid_size * zoom_rate, grid_max_size) + need_static_update = True + center_after = ( + (view_position[0] + resolution[0] / 2) / grid_size, + (view_position[1] + resolution[1] / 2) / grid_size + ) + view_position[0] += (center_before[0] - center_after[0]) * grid_size + view_position[1] += (center_before[1] - center_after[1]) * grid_size + + if pressed[pygame.K_LEFT]: + view_position[0] -= move_rate * grid_size + need_static_update = True + if pressed[pygame.K_RIGHT]: + view_position[0] += move_rate * grid_size + need_static_update = True + if pressed[pygame.K_UP]: + view_position[1] -= move_rate * grid_size + need_static_update = True + if pressed[pygame.K_DOWN]: + view_position[1] += move_rate * grid_size + need_static_update = True + + if done: + break + + # x_range: which vertical gridlines should be shown on the display + # y_range: which horizontal gridlines should be shown on the display + x_range = ( + max(0, int(math.floor(max(0, view_position[0]) / grid_size))), + min(map_size[0], int(math.ceil(max(0, view_position[0] + resolution[0]) / grid_size))) + ) + + y_range = ( + max(0, int(math.floor(max(0, view_position[1]) / grid_size))), + min(map_size[1], int(math.ceil(max(0, view_position[1] + resolution[1]) / grid_size))) + ) + + canvas.fill(background_rgb) + + #if show_grid: + # if need_static_update or True: + # grids = pygame.Surface(resolution) + # grids.set_alpha(grid_rgba[1]) + # grids.fill(background_rgb) + # + # for i in range(x_range[0], x_range[1] + 1): + # draw_line( + # canvas, grid_rgba[0], + # (i * grid_size - view_position[0], max(0, view_position[1]) - view_position[1]), + # ( + # i * grid_size - view_position[0], + # min(view_position[1] + resolution[1], map_size[1] * grid_size) - view_position[1] + # ) + # ) + # for i in range(y_range[0], y_range[1] + 1): + # draw_line( + # canvas, grid_rgba[0], + # (max(0, view_position[0]) - view_position[0], i * grid_size - view_position[1]), + # ( + # min(view_position[0] + resolution[0], map_size[0] * grid_size) - view_position[0], + # i * grid_size - view_position[1] + # ) + # ) + + if new_data is None or animation_progress > animation_total + animation_stop: + buffered_new_data = server.get_data( + frame_id, + (view_position[0] / grid_size, (view_position[0] + resolution[0]) / grid_size), + (view_position[1] / grid_size, (view_position[1] + resolution[1]) / grid_size) + ) + if buffered_new_data is None: + buffered_new_data = new_data + old_data = new_data + new_data = buffered_new_data + frame_id += 1 + animation_progress = 0 + + if new_data is not None: + if old_data is None and animation_progress == 0: + animation_progress = animation_total + + if need_static_update: + pygame.pixelcopy.surface_to_array(grid_map, canvas) + for wall in walls: + x, y = wall[0], wall[1] + if x >= x_range[0] and x <= x_range[1] and y >= y_range[0] and y <= y_range[1]: + draw_rect_matrix(grid_map, (127, 127, 127), + (x * grid_size - view_position[0], y * grid_size - view_position[1]), + grid_size, grid_size, resolution) + pygame.pixelcopy.array_to_surface(canvas, grid_map) + + rate = min(1.0, animation_progress / animation_total) + for key in new_data[0]: + new_prop = new_data[0][key] + old_prop = old_data[0][key] if old_data is not None and key in old_data[0] else None + new_group = groups[new_prop[2]] + old_group = groups[old_prop[2]] if old_prop is not None else None + now_prop = [a * (1 - rate) + b * rate for a, b in + zip(old_prop, new_prop)] if old_prop is not None else new_prop + now_group = [a * (1 - rate) + b * rate for a, b in + zip(old_group, new_group)] if old_group is not None else new_group + + draw_rect( + canvas, (int(now_group[2]), int(now_group[3]), int(now_group[4])), + ( + now_prop[0] * grid_size - view_position[0], + now_prop[1] * grid_size - view_position[1] + ), + now_group[0] * grid_size, + now_group[1] * grid_size + ) + + for key, event_x, event_y in new_data[1]: + if not key in new_data[0]: + continue + new_prop = new_data[0][key] + old_prop = old_data[0][key] if old_data is not None and key in old_data[0] else None + new_group = groups[new_prop[2]] + old_group = groups[old_prop[2]] if old_prop is not None else None + now_prop = [a * (1 - rate) + b * rate for a, b in + zip(old_prop, new_prop)] if old_prop is not None else new_prop + now_group = [a * (1 - rate) + b * rate for a, b in + zip(old_group, new_group)] if old_group is not None else new_group + draw_line( + canvas, attack_line_rgb, + ( + now_prop[0] * grid_size - view_position[0] + now_group[0] / 2 * grid_size, + now_prop[1] * grid_size - view_position[1] + now_group[1] / 2 * grid_size + ), + ( + event_x * grid_size - view_position[0] + grid_size / 2, + event_y * grid_size - view_position[1] + grid_size / 2 + ) + ) + draw_rect( + canvas, attack_dot_rgb, + ( + event_x * grid_size - view_position[0] + grid_size / 2 - attack_dot_size * grid_size / 2, + event_y * grid_size - view_position[1] + grid_size / 2 - attack_dot_size * grid_size / 2, + ), + attack_dot_size * grid_size, + attack_dot_size * grid_size + ) + + if status or triggered or animation_progress < animation_total + animation_stop: + animation_progress += 1 + + text_fps = text_formatter.render('FPS: {}'.format(int(clock.get_fps())), True, text_rgb) + text_window = text_formatter.render( + 'Window: (%.1f, %.1f, %.1f, %.1f)' % ( + view_position[0], view_position[1], + view_position[0] + resolution[0], + view_position[1] + resolution[1] + ), True, text_rgb + ) + + text_grids = text_formatter.render('Numbers: %d' % len(new_data[0]), True, text_rgb) + text_mouse = text_formatter.render('Mouse: (%d, %d)' % (mouse_x, mouse_y), True, text_rgb) + + canvas.blit(text_fps, (0, 0)) + canvas.blit(text_window, (0, (text_size + text_spacing) / 1.5)) + canvas.blit(text_grids, (0, (text_size + text_spacing) / 1.5 * 2)) + canvas.blit(text_mouse, (0, (text_size + text_spacing) / 1.5 * 3)) + + height_now = 0 + for texts in server.get_banners(frame_id, resolution): + content = [] + width, height = 0, 0 + for text in texts: + text = banner_formatter.render(text[0], True, pygame.Color(*text[1])) + content.append((text, width)) + width += text.get_width() + height = max(height, text.get_height()) + start = (resolution[0] - width) / 2.0 + for b in content: + canvas.blit(b[0], (start + b[1], height_now)) + height_now += height + banner_spacing + + endscreen_texts = server.get_endscreen(frame_id) + if endscreen_texts: + total_height = 0 + endscreen_contents = [] + endscreen = pygame.Surface(resolution) + endscreen.set_alpha(230) + endscreen.fill(background_rgb) + for texts in endscreen_texts: + content = [] + height = 0 + for text in texts: + text = bigscreen_formatter.render(text[0], True, pygame.Color(*text[1])) + content.append([text]) + height = max(height, text.get_height()) + total_height += height + bigscreen_spacing + endscreen_contents.append(content) + + total_height -= bigscreen_spacing + for content in endscreen_contents: + height, total_width = 0, 0 + for b in content: + b.append(total_width) + total_width += b[0].get_width() + height = max(height, b[0].get_height()) + width_now = (resolution[0] - total_width) / 2.0 + for b in content: + endscreen.blit(b[0], (width_now + b[1], resolution[1] / 2.0 - total_height + height)) + height_now += height + bigscreen_spacing + canvas.blit(endscreen, (0, 0)) + if need_static_update: + need_static_update = False + + pygame.display.update() + clock.tick(fps_soft_bound) diff --git a/examples/battle_model/python/magent/renderer/server/__init__.py b/examples/battle_model/python/magent/renderer/server/__init__.py new file mode 100755 index 0000000..22f1063 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/__init__.py @@ -0,0 +1,5 @@ +from .base_server import BaseServer +from .sample_server import SampleServer +from .random_server import RandomServer +from .battle_server import BattleServer +from .arrange_server import ArrangeServer diff --git a/examples/battle_model/python/magent/renderer/server/arrange_server.py b/examples/battle_model/python/magent/renderer/server/arrange_server.py new file mode 100755 index 0000000..2081b87 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/arrange_server.py @@ -0,0 +1,372 @@ +import time + +import numpy as np +import random +import magent +from magent.builtin.tf_model import DeepQNetwork +from magent.renderer.server import BaseServer +from magent.utility import FontProvider + + +def remove_wall(d, cur_pos, wall_set, unit): + if d == 0: + for i in range(0, unit): + for j in range(0, unit): + temp = (cur_pos[0] + i, cur_pos[1] + unit + j) + if temp in wall_set: + wall_set.remove(temp) + elif d == 1: + for i in range(0, unit): + for j in range(0, unit): + temp = (cur_pos[0] - unit + i, cur_pos[1] + j) + if temp in wall_set: + wall_set.remove(temp) + elif d == 2: + for i in range(0, unit): + for j in range(0, unit): + temp = (cur_pos[0] + i, cur_pos[1] - unit + j) + if temp in wall_set: + wall_set.remove(temp) + elif d == 3: + for i in range(0, unit): + for j in range(0, unit): + temp = (cur_pos[0] + unit + i, cur_pos[1] + j) + if temp in wall_set: + wall_set.remove(temp) + + +def dfs(x, y, width, height, unit, wall_set): + pos = set() + trace = list() + pos.add((x, y)) + trace.append((x, y)) + + max_x = x + width + max_y = y + height + + d = random.choice(range(4)) + pos_list = [] + flag = 0 + while len(trace) > 0: + if flag == 4: + cur_pos = trace[-1] + trace.pop() + if random.choice(range(2)) == 0: + remove_wall(d, cur_pos, wall_set, unit) + flag = 0 + if len(trace) == 0: + break + cur_pos = list(trace[-1]) + if d == 0: + cur_pos[1] = max(y, cur_pos[1] - 2 * unit) + elif d == 1: + cur_pos[0] = min(max_x, cur_pos[0] + 2 * unit) + elif d == 2: + cur_pos[1] = min(max_y, cur_pos[1] + 2 * unit) + elif d == 3: + cur_pos[0] = max(x, cur_pos[0] - 2 * unit) + if tuple(cur_pos) in pos: + d = (d + 1) % 4 + flag += 1 + else: + remove_wall(d, cur_pos, wall_set, unit) + trace.append(tuple(cur_pos)) + pos.add(tuple(cur_pos)) + d = random.choice(range(4)) + + +def clean_pos_set_convert_to_list(pos_set, pos_list): + for v in pos_list: + if v in pos_set: + pos_set.remove(v) + return list(pos_set) + + +def draw_line(x, y, width, height): + pos_set = [] + for r in range(height): + for c in range(width): + pos_set.append((x + c, y + r)) + return pos_set + + +def open_the_door(x_s, y_s, w, h, unit): + pos_list = [] + n_door = 15 + random_horizon_list_x = [x_s + (2 * np.random.choice(w // 2 // unit, n_door) + 1) * unit, x_s + (2 * np.random.choice(w // 2 // unit, n_door) - 1) * unit] + random_vertical_list_y = [y_s + (2 * np.random.choice(h // 2 // unit, n_door) + 1) * unit, y_s + (2 * np.random.choice(h // 2 // unit, n_door) + 1) * unit] + + y_e = y_s + h - unit + for v in random_horizon_list_x[0]: + pos_list.extend([(v, y_s), (v + 1, y_s), (v, y_s + 1), (v + 1, y_s + 1)]) + for v in random_horizon_list_x[1]: + pos_list.extend([(v, y_e), (v + 1, y_e), (v, y_e + 1), (v + 1, y_e + 1)]) + + x_e = x_s + w - unit + for v in random_vertical_list_y[0]: + pos_list.extend([(x_s, v), (x_s, v + 1), (x_s + 1, v), (x_s + 1, v + 1)]) + for v in random_vertical_list_y[1]: + pos_list.extend([(x_e, v), (x_e, v + 1), (x_e + 1, v), (x_e + 1, v + 1)]) + + return pos_list + + +def create_maze(pos, width, height, unit, font_area): + # draw block: with rect: left(x), top(y), width, height + pos_set = [] + for i in range(height): + if i % 2 == 0: + pos_set.extend(draw_line(pos[0], pos[1] + i * unit, width * unit, unit)) + pos_set.extend(draw_line(pos[0], pos[1] + font_area[1] + i * unit, width * unit, unit)) + pos_set.extend(draw_line(pos[0] + i * unit, pos[1] + height * unit, unit, font_area[1])) + pos_set.extend(draw_line(pos[0] + font_area[0] + i * unit, pos[1] + height * unit, unit, font_area[1])) + + for i in range(width): + if i % 2 == 0: + pos_set.extend(draw_line(pos[0] + i * unit, pos[1], unit, height * unit)) + pos_set.extend(draw_line(pos[0] + i * unit, pos[1] + font_area[1], unit, height * unit)) + pos_set.extend(draw_line(pos[0], pos[1] + i * unit, height * unit, unit)) + pos_set.extend(draw_line(pos[0] + font_area[0], pos[1] + i * unit, height * unit, unit)) + + pos_set = set(pos_set) + + dfs(pos[0] + 2, pos[1] + 2, (width - 1) * unit, (height - 1) * unit, unit, pos_set) # north + dfs(pos[0] + 2, pos[1] + (height - 2) * unit, (height - 1) * unit, (width + 3) * unit, unit, pos_set) # west + dfs(pos[0] + height * unit, pos[1] + font_area[1] - unit, (width - height) * unit, (height - 1) * unit, unit, pos_set) # south + dfs(pos[0] + font_area[0] - unit, pos[1] + (height - 2) * unit, (height - 1) * unit, font_area[1] - (height + 1) * unit, unit, pos_set) # east + + temp = [] + temp.extend(open_the_door(pos[0], pos[1], font_area[0] + height * unit, font_area[1] + height * unit, unit)) + res = clean_pos_set_convert_to_list(pos_set, temp) + return res + + +def load_config(map_size): + gw = magent.gridworld + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + cfg.set({"minimap_mode": True}) + cfg.set({"embedding_size": 12}) + + goal = cfg.register_agent_type( + "goal", + {'width': 1, 'length': 1, + + 'can_absorb': True + } + ) + + agent = cfg.register_agent_type( + "agent", + {'width': 1, 'length': 1, 'hp': 10, 'speed': 2, + 'view_range': gw.CircleRange(6), + 'damage': 2, 'step_recover': -10.0/400, + + 'step_reward': 0, + }) + + g_goal = cfg.add_group(goal) + g_agent = cfg.add_group(agent) + + g = gw.AgentSymbol(g_goal, 'any') + a = gw.AgentSymbol(g_agent, 'any') + + cfg.add_reward_rule(gw.Event(a, 'collide', g), receiver=a, value=10) + + return cfg + + +def generate_map(mode, env, map_size, goal_handle, handles, messages, font): + # pre-process message + max_len = 8 + new = [] + for msg in messages: + if len(msg) > max_len: + for i in range(0, len(msg), max_len): + new.append(msg[i:i+max_len]) + else: + new.append(msg) + messages = new + + center_x, center_y = map_size // 2, map_size // 2 + + # create maze + if mode == 1: + radius = 90 + pos_list = create_maze([center_x - radius, center_y - radius], radius + 1, 15, 2, font_area=[radius * 2 - 28, radius * 2 - 28]) + env.add_walls(method="custom", pos=pos_list) + + def add_square(pos, side, gap): + side = int(side) + for x in range(center_x - side//2, center_x + side//2 + 1, gap): + pos.append([x, center_y - side//2]) + pos.append([x, center_y + side//2]) + for y in range(center_y - side//2, center_y + side//2 + 1, gap): + pos.append([center_x - side//2, y]) + pos.append([center_x + side//2, y]) + + def draw(base_x, base_y, scale, data): + w, h = len(data), len(data[0]) + pos = [] + for i in range(w): + for j in range(h): + if data[i][j] == 1: + start_x = i * scale + base_y + start_y = j * scale + base_x + for x in range(start_x, start_x + scale): + for y in range(start_y, start_y + scale): + pos.append([y, x]) + + env.add_agents(goal_handle, method="custom", pos=pos) + + base_y = (map_size - len(messages) * font.height) // 2 + for message in messages: + base_x = (map_size - len(message) * font.width) // 2 + scale = 1 + for x in message: + data = font.get(x) + draw(base_x, base_y, scale, data) + base_x += font.width + base_y += font.height + 1 + + alpha_goal_num = env.get_num(goal_handle) + + # agent + pos = [] + + add_square(pos, map_size * 0.95, 1) + add_square(pos, map_size * 0.90, 1) + add_square(pos, map_size * 0.85, 1) + add_square(pos, map_size * 0.80, 1) + + pos = np.array(pos) + pos = pos[np.random.choice(np.arange(len(pos)), int(alpha_goal_num * 1.6), replace=False)] + + env.add_agents(handles[0], method="custom", pos=pos) + + +class ArrangeServer(BaseServer): + def get_banners(self, frame_id, resolution): + return [] + + def keydown(self, frame_id, key, mouse_x, mouse_y): + return False + + def get_status(self, frame_id): + if self.done: + return None + else: + return True + + def get_endscreen(self, frame_id): + return [] + + def mousedown(self, frame_id, key, mouse_x, mouse_y): + return False + + def get_info(self): + ret = self.env._get_groups_info() + ret[1] = ret[0] + return (self.map_size, self.map_size), ret, {'wall': self.env._get_walls_info()} + + def __init__(self, path="data/arrange_model", messages=None, mode=1): + # some parameter + map_size = 250 + eps = 0.15 + + # init the game + env = magent.GridWorld(load_config(map_size)) + font = FontProvider('data/font_8x8/basic.txt') + + handles = env.get_handles() + food_handle, handles = handles[0], handles[1:] + models = [] + models.append(DeepQNetwork(env, handles[0], 'arrange', use_conv=True)) + + # load model + models[0].load(path, 10) + + # init environment + env.reset() + generate_map(mode, env, map_size, food_handle, handles, messages, font) + + # save to member variable + self.env = env + self.food_handle = food_handle + self.handles = handles + self.eps = eps + self.models = models + self.done = False + self.map_size = map_size + self.new_rule_ct = 0 + self.pos_reward_ct = set() + self.num = None + + self.ct = 0 + + def step(self): + handles = self.handles + models = self.models + env = self.env + + center_x = self.map_size // 2 + center_y = self.map_size + + for j in range(2): + obs = [env.get_observation(handle) for handle in handles] + ids = [env.get_agent_id(handle) for handle in handles] + + for i in range(len(handles)): + if self.new_rule_ct > 0: + obs[i][1][:, 10:12] = 0 + else: + obs[i][1][:, 10:12] = 1 + acts = models[i].infer_action(obs[i], ids[i], 'e_greedy', eps=self.eps) + env.set_action(handles[i], acts) + + done = env.step() + + goal_num = env.get_num(self.food_handle) + rewards = env.get_reward(handles[0]) + + for id_, r in zip(ids[0], rewards): + if r > 0.05 and id_ not in self.pos_reward_ct: + self.pos_reward_ct.add(id_) + + if 1.0 * len(self.pos_reward_ct) / goal_num >= 0.99: + self.new_rule_ct += 1 + + self.num = [env.get_num(handle) for handle in [self.food_handle] + handles] + env.clear_dead() + + if done: + break + + return done + + def get_data(self, frame_id, x_range, y_range): + start = time.time() + + if not self.done: + self.done = self.step() + print(self.done) + if self.done: + print("done!") + + pos, event = self.env._get_render_info(x_range, y_range) + print(" fps ", 1 / (time.time() - start)) + return pos, event + + def add_agents(self, x, y, g): + pos = [] + for i in range(-3, 3): + for j in range(-3, 3): + pos.append((x + i, y + j)) + self.env.add_agents(self.handles[g], method="custom", pos=pos) + + def get_map_size(self): + return self.map_size, self.map_size + + def get_numbers(self): + return self.num diff --git a/examples/battle_model/python/magent/renderer/server/base_server.py b/examples/battle_model/python/magent/renderer/server/base_server.py new file mode 100755 index 0000000..624b116 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/base_server.py @@ -0,0 +1,41 @@ +from abc import ABCMeta, abstractmethod + + +class BaseServer: + __metaclass__ = ABCMeta + + @abstractmethod + def get_info(self): + pass + + @abstractmethod + def get_data(self, frame_id, x_range, y_range): + pass + + @abstractmethod + def add_agents(self, x, y, g): + pass + + @abstractmethod + def get_map_size(self): + pass + + @abstractmethod + def get_banners(self, frame_id, resolution): + pass + + @abstractmethod + def get_status(self, frame_id): + pass + + @abstractmethod + def keydown(self, frame_id, key, mouse_x, mouse_y): + pass + + @abstractmethod + def mousedown(self, frame_id, key, mouse_x, mouse_y): + pass + + @abstractmethod + def get_endscreen(self, frame_id): + pass \ No newline at end of file diff --git a/examples/battle_model/python/magent/renderer/server/battle_server.py b/examples/battle_model/python/magent/renderer/server/battle_server.py new file mode 100755 index 0000000..980823d --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/battle_server.py @@ -0,0 +1,239 @@ +import math +import time + +import matplotlib.pyplot as plt +import numpy as np + +import magent +from magent.builtin.tf_model import DeepQNetwork +from magent.renderer.server import BaseServer + + +def load_config(map_size): + gw = magent.gridworld + cfg = gw.Config() + + cfg.set({"map_width": map_size, "map_height": map_size}) + cfg.set({"minimap_mode": True}) + + cfg.set({"embedding_size": 10}) + + small = cfg.register_agent_type( + "small", + {'width': 1, 'length': 1, 'hp': 10, 'speed': 2, + 'view_range': gw.CircleRange(6), 'attack_range': gw.CircleRange(1.5), + 'damage': 2, 'step_recover': 0.1, + 'step_reward': -0.001, 'kill_reward': 100, 'dead_penalty': -0.05, 'attack_penalty': -1, + }) + + g0 = cfg.add_group(small) + g1 = cfg.add_group(small) + + a = gw.AgentSymbol(g0, index='any') + b = gw.AgentSymbol(g1, index='any') + + cfg.add_reward_rule(gw.Event(a, 'attack', b), receiver=a, value=2) + cfg.add_reward_rule(gw.Event(b, 'attack', a), receiver=b, value=2) + + return cfg + + +def generate_map(env, map_size, handles): + width = map_size + height = map_size + + init_num = 20 + + gap = 3 + leftID, rightID = 0, 1 + + # left + pos = [] + for y in range(10, 45): + pos.append((width / 2 - 5, y)) + pos.append((width / 2 - 4, y)) + for y in range(50, height // 2 + 25): + pos.append((width / 2 - 5, y)) + pos.append((width / 2 - 4, y)) + + for y in range(height // 2 - 25, height - 50): + pos.append((width / 2 + 5, y)) + pos.append((width / 2 + 4, y)) + for y in range(height - 45, height - 10): + pos.append((width / 2 + 5, y)) + pos.append((width / 2 + 4, y)) + env.add_walls(pos=pos, method="custom") + + n = init_num + side = int(math.sqrt(n)) * 2 + pos = [] + for x in range(width // 2 - gap - side, width // 2 - gap - side + side, 2): + for y in range((height - side) // 2, (height - side) // 2 + side, 2): + pos.append([x, y, 0]) + env.add_agents(handles[leftID], method="custom", pos=pos) + + # right + n = init_num + side = int(math.sqrt(n)) * 2 + pos = [] + for x in range(width // 2 + gap, width // 2 + gap + side, 2): + for y in range((height - side) // 2, (height - side) // 2 + side, 2): + pos.append([x, y, 0]) + env.add_agents(handles[rightID], method="custom", pos=pos) + + +class BattleServer(BaseServer): + def __init__(self, path="data/battle_model", total_step=1000, add_counter=10, add_interval=50): + # some parameter + map_size = 125 + eps = 0.05 + + # init the game + env = magent.GridWorld(load_config(map_size)) + + handles = env.get_handles() + models = [] + models.append(DeepQNetwork(env, handles[0], 'trusty-battle-game-l', use_conv=True)) + models.append(DeepQNetwork(env, handles[1], 'trusty-battle-game-r', use_conv=True)) + + # load model + models[0].load(path, 0, 'trusty-battle-game-l') + models[1].load(path, 0, 'trusty-battle-game-r') + + # init environment + env.reset() + generate_map(env, map_size, handles) + + # save to member variable + self.env = env + self.handles = handles + self.eps = eps + self.models = models + self.map_size = map_size + self.total_step = total_step + self.add_interval = add_interval + self.add_counter = add_counter + self.done = False + print(env.get_view2attack(handles[0])) + plt.show() + + def get_info(self): + return (self.map_size, self.map_size), self.env._get_groups_info(), {'wall': self.env._get_walls_info()} + + def step(self): + handles = self.handles + models = self.models + env = self.env + + obs = [env.get_observation(handle) for handle in handles] + ids = [env.get_agent_id(handle) for handle in handles] + + counter = [] + for i in range(len(handles)): + acts = models[i].infer_action(obs[i], ids[i], 'e_greedy', eps=self.eps) + env.set_action(handles[i], acts) + counter.append(np.zeros(shape=env.get_action_space(handles[i]))) + for j in acts: + counter[-1][j] += 1 + # plt.clf() + # for c in counter: + # plt.bar(range(len(c)), c / np.sum(c)) + # plt.draw() + # plt.pause(1e-8) + + # code for checking the correctness of observation + # for channel in range(7): + # x = magent.round(list(obs[1][0][0][:,:,channel]), 2) + # for row in x: + # print row + # print("-------------") + # input() + + done = env.step() + env.clear_dead() + + return done + + def get_data(self, frame_id, x_range, y_range): + start = time.time() + if self.done: + return None + self.done = self.step() + pos, event = self.env._get_render_info(x_range, y_range) + print(" fps ", 1 / (time.time() - start)) + return pos, event + + def add_agents(self, x, y, g): + pos = [] + for i in range(-5, 5): + for j in range(-5, 5): + pos.append((x + i, y + j)) + self.env.add_agents(self.handles[g], method="custom", pos=pos) + + pos = [] + x = np.random.randint(0, self.map_size - 1) + y = np.random.randint(0, self.map_size - 1) + for i in range(-5, 5): + for j in range(-5, 6): + pos.append((x + i, y + j)) + self.env.add_agents(self.handles[g ^ 1], method="custom", pos=pos) + + def get_map_size(self): + return self.map_size, self.map_size + + def get_banners(self, frame_id, resolution): + red = '{}'.format(self.env.get_num(self.handles[0])), (200, 0, 0) + vs = ' vs ', (0, 0, 0) + blue = '{}'.format(self.env.get_num(self.handles[1])), (0, 0, 200) + result = [(red, vs, blue)] + + tmp = '{} chance(s) remained'.format( + max(0, self.add_counter)), (0, 0, 0) + result.append((tmp,)) + + tmp = '{} / {} steps'.format(frame_id, self.total_step), (0, 0, 0) + result.append((tmp,)) + if frame_id % self.add_interval == 0 and frame_id < self.total_step and self.add_counter > 0: + tmp = 'Please press your left mouse button to add agents', (0, 0, 0) + result.append((tmp,)) + return result + + def get_status(self, frame_id): + if frame_id % self.add_interval == 0 and self.add_counter > 0: + return False + elif frame_id >= self.total_step or self.done: + return None + else: + return True + + def keydown(self, frame_id, key, mouse_x, mouse_y): + return False + + def mousedown(self, frame_id, pressed, mouse_x, mouse_y): + if frame_id % self.add_interval == 0 and frame_id < self.total_step and pressed[0] \ + and self.add_counter > 0 and not self.done: + self.add_counter -= 1 + pos = [] + for i in range(-5, 5): + for j in range(-5, 5): + pos.append((mouse_x + i, mouse_y + j)) + self.env.add_agents(self.handles[0], method="custom", pos=pos) + + pos = [] + x = np.random.randint(0, self.map_size - 1) + y = np.random.randint(0, self.map_size - 1) + for i in range(-5, 6): + for j in range(-5, 5): + pos.append((x + i, y + j)) + self.env.add_agents(self.handles[1], method="custom", pos=pos) + return True + return False + + def get_endscreen(self, frame_id): + if frame_id == self.total_step or self.done: + if self.env.get_num(self.handles[0]) > self.env.get_num(self.handles[1]): + return [(("You", (200, 0, 0)), (" win! :)", (0, 0, 0)))] + else: + return [(("You", (200, 0, 0)), (" lose. :(", (0, 0, 0)))] + else: + return [] diff --git a/examples/battle_model/python/magent/renderer/server/random_server.py b/examples/battle_model/python/magent/renderer/server/random_server.py new file mode 100755 index 0000000..d7b3cb1 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/random_server.py @@ -0,0 +1,69 @@ +import random + +from .base_server import BaseServer + + +class RandomServer(BaseServer): + def __init__(self, agent_number=1000, group_number=20, map_size=100, shape_range=3, speed=5, event_range=100): + self._data = {} + self._map_size = map_size + self._number = agent_number + for i in range(agent_number): + self._data.setdefault(i, [ + random.randint(0, map_size - 1), + random.randint(0, map_size - 1), + random.randint(0, group_number - 1) + ]) + self._group = [] + for i in range(group_number): + self._group.append([ + random.randint(1, shape_range), + random.randint(1, shape_range), + random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255) + ]) + self._speed = speed + self._event_range = event_range + self._map_size = map_size + + def get_group_info(self): + return self._group + + def get_static_info(self): + return {"wall": []} + + def get_data(self, frame_id, x_range, y_range): + result = {} + event = [] + for i in self._data: + olddata = self._data[i] + data = [0, 0, 0] + data[0] = olddata[0] + random.randint(-self._speed, self._speed) + data[1] = olddata[1] + random.randint(-self._speed, self._speed) + data[0] = min(max(data[0], 0), self._map_size - 1) + data[1] = min(max(data[1], 0), self._map_size - 1) + data[2] = olddata[2] + self._data[i] = data + if (x_range[0] <= data[0] <= x_range[1] and y_range[0] <= data[1] <= y_range[1]) or \ + (x_range[0] <= olddata[0] <= x_range[1] and y_range[0] <= olddata[1] <= y_range[1]): + result.setdefault(i, olddata) + event_number = random.randint(0, self._event_range) + for i in range(event_number): + agent_id, _ = random.choice(self._data.items()) + event.append( + ( + agent_id, + random.randint(0, self._map_size - 1), + random.randint(0, self._map_size - 1) + ) + ) + return result, event + + def add_agents(self, x, y, g): + self._data.setdefault(self._number, (x, y, g)) + self._number += 1 + + def get_map_size(self): + return self._map_size, self._map_size + diff --git a/examples/battle_model/python/magent/renderer/server/sample_server.py b/examples/battle_model/python/magent/renderer/server/sample_server.py new file mode 100755 index 0000000..c4d9cb6 --- /dev/null +++ b/examples/battle_model/python/magent/renderer/server/sample_server.py @@ -0,0 +1,27 @@ +from .base_server import BaseServer + + +class SampleServer(BaseServer): + def get_group_info(self): + return [[1, 1, 0, 0, 0]] + + def get_static_info(self): + return {"walls": []} + + def get_data(self, frame_id, x_range, y_range): + if frame_id == 0: + return {1: [10, 10, 0]}, [(1, 0, 0)] + elif frame_id == 1: + return {1: [9, 10, 0]}, [(1, 0, 0)] + elif frame_id == 2: + return {1: [8, 10, 0]}, [(1, 0, 0)] + elif frame_id == 3: + return {1: [14, 12, 0]}, [(1, 0, 0)] + else: + return {1: [10, 10, 0]}, [(1, 0, 0)] + + def add_agents(self, x, y, g): + pass + + def get_map_size(self): + return [50, 50] diff --git a/examples/battle_model/python/magent/utility.py b/examples/battle_model/python/magent/utility.py new file mode 100755 index 0000000..52a4e96 --- /dev/null +++ b/examples/battle_model/python/magent/utility.py @@ -0,0 +1,305 @@ +""" some utilities """ + +import math +import collections +import platform + +import numpy as np +import logging +import collections +import os + +from magent.builtin.rule_model import RandomActor + + +class EpisodesBufferEntry: + """Entry for episode buffer""" + def __init__(self): + self.views = [] + self.features = [] + self.actions = [] + self.rewards = [] + self.terminal = False + + def append(self, view, feature, action, reward, alive): + self.views.append(view.copy()) + self.features.append(feature.copy()) + self.actions.append(action) + self.rewards.append(reward) + if not alive: + self.terminal = True + + +class EpisodesBuffer: + """Replay buffer to store a whole episode for all agents + one entry for one agent + """ + def __init__(self, capacity): + self.buffer = {} + self.capacity = capacity + self.is_full = False + + def record_step(self, ids, obs, acts, rewards, alives): + """record transitions (s, a, r, terminal) in a step""" + buffer = self.buffer + index = np.random.permutation(len(ids)) + + if self.is_full: # extract loop invariant in else part + for i in range(len(ids)): + entry = buffer.get(ids[i]) + if entry is None: + continue + entry.append(obs[0][i], obs[1][i], acts[i], rewards[i], alives[i]) + else: + for i in range(len(ids)): + i = index[i] + entry = buffer.get(ids[i]) + if entry is None: + if self.is_full: + continue + else: + entry = EpisodesBufferEntry() + buffer[ids[i]] = entry + if len(buffer) >= self.capacity: + self.is_full = True + + entry.append(obs[0][i], obs[1][i], acts[i], rewards[i], alives[i]) + + def reset(self): + """ clear replay buffer """ + self.buffer = {} + self.is_full = False + + def episodes(self): + """ get episodes """ + return self.buffer.values() + + +# decay schedulers +def exponential_decay(now_step, total_step, final_value, rate): + """exponential decay scheduler""" + decay = math.exp(math.log(final_value)/total_step ** rate) + return max(final_value, 1 * decay ** (now_step ** rate)) + + +def linear_decay(now_step, total_step, final_value): + """linear decay scheduler""" + decay = (1 - final_value) / total_step + return max(final_value, 1 - decay * now_step) + + +def piecewise_decay(now_step, anchor, anchor_value): + """piecewise linear decay scheduler + + Parameters + --------- + now_step : int + current step + anchor : list of integer + step anchor + anchor_value: list of float + value at corresponding anchor + """ + i = 0 + while i < len(anchor) and now_step >= anchor[i]: + i += 1 + + if i == len(anchor): + return anchor_value[-1] + else: + return anchor_value[i-1] + (now_step - anchor[i-1]) * \ + ((anchor_value[i] - anchor_value[i-1]) / (anchor[i] - anchor[i-1])) + + +# eval observation set generator +def sample_observation(env, handles, n_obs=-1, step=-1): + """Sample observations by random actors. + These samples can be used for evaluation + + Parameters + ---------- + env : environment + handles: list of handle + n_obs : int + number of observation + step : int + maximum step + + Returns + ------- + ret : list of raw observation + raw observation for every group + the format of raw observation is tuple(view, feature) + """ + models = [RandomActor(env, handle) for handle in handles] + + n = len(handles) + views = [[] for _ in range(n)] + features = [[] for _ in range(n)] + + done = False + step_ct = 0 + while not done: + obs = [env.get_observation(handle) for handle in handles] + ids = [env.get_agent_id(handle) for handle in handles] + + for i in range(n): + act = models[i].infer_action(obs[i], ids[i]) + env.set_action(handles[i], act) + + done = env.step() + env.clear_dead() + + # record steps + for i in range(n): + views[i].append(obs[i][0]) + features[i].append(features[i][1]) + + if step != -1 and step_ct > step: + break + + if step_ct % 100 == 0: + print("sample step %d" % step_ct) + + step_ct += 1 + + for i in range(n): + views[i] = np.array(views[i], dtype=np.float32).reshape((-1,) + + env.get_view_space(handles[i])) + features[i] = np.array(features[i], dtype=np.float32).reshape((-1,) + + env.get_feature_space(handles[i])) + + if n_obs != -1: + for i in range(n): + views[i] = views[i][np.random.choice(np.arange(views[i].shape[0]), n_obs)] + features[i] = features[i][np.random.choice(np.arange(features[i].shape[0]), n_obs)] + + ret = [(v, f) for v, f in zip(views, features)] + return ret + + +def init_logger(filename): + """ initialize logger config + + Parameters + ---------- + filename : str + filename of the log + """ + logging.basicConfig(level=logging.INFO, filename=filename + ".log") + console = logging.StreamHandler() + console.setLevel(logging.INFO) + logging.getLogger('').addHandler(console) + + +def rec_round(x, ndigits=2): + """ round x recursively + + Parameters + ---------- + x: float, int, list, list of list, ... + variable to round, support many types + ndigits: int + precision in decimal digits + """ + if isinstance(x, collections.Iterable): + return [rec_round(item, ndigits) for item in x] + return round(x, ndigits) + + +def has_gpu(): + """ check where has a nvidia gpu """ + ret = os.popen("nvidia-smi -L 2>/dev/null").read() + return ret.find("GPU") != -1 + + +def download_file(filename, url): + """download url to filename""" + print("Download %s from %s..." % (filename, url)) + + ret = os.system("wget -O %s '%s'" % (filename, url)) + + if ret != 0: + print("ERROR: wget fails!") + print("If you are an OSX user, you can install wget by 'brew install wget' and retry.") + exit(-1) + else: + print("download done!") + + +def download_model(url): + """download model from url""" + name = url.split('/')[-1] + name = os.path.join('data', name) + download_file(name, url) + def do_commond(cmd): + print(cmd) + os.system(cmd) + do_commond("tar xzf %s -C data" % name) + do_commond("rm %s" % name) + + +def check_model(name): + """check whether a model is downloaded""" + infos = { + 'against': + (('data/battle_model/battle/tfdqn_0.index',), + 'https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/_data/magent/against-0.tar.gz'), + + 'battle-game': + (("data/battle_model/trusty-battle-game-l/tfdqn_0.index", + "data/battle_model/trusty-battle-game-r/tfdqn_0.index"), + 'https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/_data/magent/battle_model.tar.gz'), + + 'arrange': + (('data/arrange_model/arrange/tfdqn_10.index',), + 'https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/_data/magent/arrange_game.tar.gz',) + } + + if name not in infos: + raise RuntimeError("Unknown model name") + + info = infos[name] + missing = False + for check in info[0]: + if not os.path.exists(check): + missing = True + if missing: + download_model(info[1]) + + +class FontProvider: + """provide pixel font""" + def __init__(self, filename): + data = [] + # read raw + with open(filename) as fin: + for line in fin.readlines(): + char = [] + for x in line.split(','): + char.append(eval(x)) + data.append(char) + + height = 8 + width = 8 + + # expand bit compress + expand_data = [] + for char in data: + expand_char = [[0 for _ in range(width)] for _ in range(height)] + for i in range(width): + for j in range(height): + set = char[i] & (1 << j) + if set: + expand_char[i][j] = 1 + expand_data.append(expand_char) + + self.data = expand_data + self.width = width + self.height = height + + def get(self, i): + if isinstance(i, int): + return self.data[i] + else: + return self.data[ord(i)] diff --git a/examples/battle_model/python/magent/utility.pyc b/examples/battle_model/python/magent/utility.pyc new file mode 100755 index 0000000..09358d9 Binary files /dev/null and b/examples/battle_model/python/magent/utility.pyc differ diff --git a/examples/battle_model/senario_battle.py b/examples/battle_model/senario_battle.py new file mode 100755 index 0000000..838f36d --- /dev/null +++ b/examples/battle_model/senario_battle.py @@ -0,0 +1,376 @@ +import random +import math +import numpy as np +import time +import tensorflow as tf +import pdb +import json +import pickle +import pandas as pd + + + +def generate_map(env, map_size, handles): + """ generate a map, which consists of two squares of agents""" + width = height = map_size + init_num = map_size * map_size * 0.04 + gap = 3 + leftID=0 + rightID=1 + + # left + n = init_num + side = int(math.sqrt(n)) * 2 + pos = [] + for x in range(width//2 - gap - side, width//2 - gap - side + side, 2): + for y in range((height - side)//2, (height - side)//2 + side, 2): + pos.append([x, y, 0]) + env.add_agents(handles[leftID], method="custom", pos=pos) + + # right + n = init_num + side = int(math.sqrt(n)) * 2 + gap=3 + pos = [] + for x in range(width//2 + gap, width//2 + gap + side, 2): + for y in range((height - side)//2, (height - side)//2 + side, 2): + pos.append([x, y, 0]) + env.add_agents(handles[rightID], method="custom", pos=pos) + print('done') + + + + +def get_distance_matrix(env,handle,nei_len): + poss=env.get_pos(handle) + + nei_space=nei_len**2 + poss_all=np.array([poss]*len(poss)) + poss_mine=np.array([ [t]*len(poss) for t in poss]) + indexs=((poss_all-poss_mine)**2).sum(axis=2) + indexs=indexs-np.ones_like(indexs)*nei_space + indexs[indexs<0]=0 + return indexs,poss + + + + + + + +def check_isbetter(W,indexs,chs): + chs=np.array(chs) + for ch in chs: + # pdb.set_trace() + a=np.where(indexs[ch]==0)[0] + tmp=[val for val in a if val in chs and val!=ch] + if len(tmp)>0: + for t in tmp: + if W[ch]<=W[t]: + chs=np.delete(chs,np.where(chs==ch)[0][0]) + break + return chs + +def form_chs(W,indexs,un): + ch=[] + uns=np.array(un) + while len(uns)!=0: + # + a=uns[np.argmax(W[uns])] + ch.append(a) + uns=np.array(list(set(uns)-set(np.where(indexs[a]==0)[0]))) + # pdb.set_trace() + # p.delete(uns,) + return list(set(ch)) + + +def ids_to_ind(id,ids): + ind=[] + for i in id: + a=np.where(ids==i)[0] + if list(a) == []: + continue + else: + ind.append(a[0]) + return ind + +def ind_to_id(ind,ids): + id=[] + for i in ind: + id.append(ids[i]) + return id + +def get_chs(indexs,hps,chs_id,ids,weight=None): + chs=ids_to_ind(chs_id,ids) + # pdb.set_trace() + # s=time.time() + if list(weight): + W=weight + else: + W=calc_w(indexs,hps) + # pdb.set_trace() + # downgrade nearby chs based on their W + chs=check_isbetter(W,indexs,chs) + # find undecided nodes + un=set([i for i in range(len(indexs))]) + for ch in chs: + un=un.intersection(set(np.where(indexs[ch]>0)[0])) + un=list(un) + + un=form_chs(W,indexs,un) + # print('getchs',time.time()-s) + chs=list(set(chs).union(set(un))) + # pdb.set_trace() + chs_id = ind_to_id(chs,ids) + return chs,chs_id + + +def get_neis(senders,indexs,poss): + # i=0 + neis=[] + for r in indexs[senders]: + for i in np.where(r<0)[0]: + neis.append(i) + # neis.append(i for i in np.where(r<0)[0]) + # neis=np.where(index[senders]<0)[0] + return neis + + + +def play(env, n_round, map_size, max_steps, handles, models, print_every, eps=1.0, render=False, train=False,len_nei=40,MsgModels=None,num_bits_msg=5,msg_space=10,rewardtype='self',use_concate_mean_global=False,crps=[False,False],crp=False,selfplay=False,is_fix=False): + """play a ground and train""" + env.reset() + # print(crp) + + generate_map(env, map_size, handles) + + step_ct = 0 + done = False + + n_group = len(handles) + state = [None for _ in range(n_group)] + acts = [None for _ in range(n_group)] + ids = [None for _ in range(n_group)] + sender_ids = [None for _ in range(n_group)] + ids_=[None for _ in range(n_group)] + + alives = [None for _ in range(n_group)] + alives_ = [None for _ in range(n_group)] + rewards = [None for _ in range(n_group)] + sender_rewards = [None for _ in range(n_group)] + nums = [env.get_num(handle) for handle in handles] + max_nums = nums.copy() + + loss = [None for _ in range(n_group)] + eval_q = [None for _ in range(n_group)] + n_action = [env.get_action_space(handles[0])[0], env.get_action_space(handles[1])[0]] + + print("\n\n[*] ROUND #{0}, EPS: {1:.2f} NUMBER: {2}".format(n_round, eps, nums)) + mean_rewards = [[] for _ in range(n_group)] + total_rewards = [[] for _ in range(n_group)] + + + for i in range(n_group): + state[i] = list(env.get_observation(handles[i])) + msg=[[],[]] + l_msg=[[],[]] + msgs=[[],[]] + mean_msg=[[],[]] + senders=[[],[]] + feedbacks=[[],[]] + sender_rewards_adv=[[],[]] + sender_rewards_all=[[],[]] + state_senders=[[],[]] + chs=[[],[]] + chs_id=[[],[]] + hps=[[],[]] + posss=[[],[]] + indexss=[[],[]] + q_values=None + q_values_pre=None + # index0=[] + first=True + + models[0].test_buffer() + models[1].test_buffer() + if MsgModels[0]: + MsgModels[0].test_buffer() + while not done and step_ct < max_steps: + + # take actions for every model + # start=time.time() + for i in range(n_group): + state[i] = list(env.get_observation(handles[i])) + ids[i] = env.get_agent_id(handles[i]) + hps[i]=env.get_hps(handles[i]) + posss[i] = env.get_pos(handles[i]) + indexss[i],poss=get_distance_matrix(env,handles[i],len_nei) + + if MsgModels[i] : + # pdb.set_trace() + # print('msgmodel msgmodel.....') + # if i==0: + # index0=indexs + msg[i]=MsgModels[i].act(obs=state[i][0],eps=eps) + + chs[i],chs_id[i]=get_chs(indexss[i],hps[i],chs_id[i],ids[i],msg[i]) + # print(len(chs[0])) + # pdb.set_trace() + if crp: + if i==1: + acts[i] = models[i].act(obs=state[i][0], eps=eps, msgs=msgs[i], mean_msg=mean_msg[i], + crp=crp,poss=posss[i],chs=chs[i]) + else: + acts[i] = models[i].act(obs=state[i][0], eps=eps, msgs=msgs[i], mean_msg=mean_msg[i], + crp=crp,poss=posss[i],chs=chs[i]) + else: + if i==1: + acts[i] = models[i].act(obs=state[i][0], eps=eps, msgs=msgs[i], mean_msg=mean_msg[i], + crp=crp,poss=posss[i]) + else: + acts[i] = models[i].act(obs=state[i][0], eps=eps, msgs=msgs[i], mean_msg=mean_msg[i], + crp=crp,poss=posss[i]) +# e_res=models[i].get_e_res(state=state[i], eps=eps, msgs=msgs[i], mean_msg=mean_msg[i], +# crp=crp,poss=posss[i]) + # cluster + # poss=np.array([posss[i] for j in range(len(state[i][0]))])) + # print('nn',time.time()-s0) + # print('get acts',time.time()-start) + + #pdb.set_trace() + for i in range(n_group): + env.set_action(handles[i], acts[i]) + + # simulate one step + done = env.step() + sender_alives=[[],[]] + + for i in range(n_group): + rewards[i] = env.get_reward(handles[i]) + alives[i] = env.get_alive(handles[i]) + # sender_rewards[i]=rewards[i][senders[i]] + + if not first: + pre_reward=buffer['rewards'] + + buf_chs=[None for i in range(len(posss[0]))] + i=0 + for p in posss[0]: + buf_chs[i]=chs[0][i%len(chs[0])] + i+=1 + # print(chs[0]) + buffer = { + 'state': state[0], 'acts': acts[0], + 'alives': alives[0], 'ids': ids[0], 'poss': posss[0],'chs':chs[0], + } + buffer1 = { + 'state': state[1], 'acts': acts[1], + 'alives': alives[1], 'ids': ids[1], 'poss': posss[1],'chs':chs[1], + } + + # print(buffer['alives']) + buffer['msgs'] = msgs[0] + buffer['mean_msg']=mean_msg[0] + buffer['rewards']=rewards[0] + + buffer1['msgs'] = msgs[1] + buffer1['mean_msg']=mean_msg[1] + buffer1['rewards']=rewards[1] + + + # print(buffer) + + if not first: + buffer_=buffer.copy() + buffer_['obs']=buffer_['state'][0] + buffer_['feature']=buffer_['state'][1] + r_indexs=[] + for j in range(len(ids[0])): + if ids[0][j] in ids_[0]: + r_indexs.append(j) + buffer_['rewards'] = pre_reward[r_indexs] + buffer_['dones']=np.array([not done for done in buffer_['alives']], dtype=np.bool) + + # print('----------------------------') + # print(len(buffer_['dones'])) + # print(len(buffer_['rewards'])) + # print(len(buffer_['obs'])) + + # q_values_pre=models[0].calc_target_q(**buffer_) + # # if not first: + # feedbacks[0]=q_values[r_indexs]-q_values_pre + + sender_rewards_adv[0]=np.array([sum(feedbacks[0])/len(rewards[0]) for k in range(len(senders[0]))]) + # q_values=models[0].get_q(**buffer) + # sender_rewards_all[0]=np.array([sum(rewards[0])/len(rewards[0]) for k in range(len(senders[0]))]) + sender_alives[0]=alives[0][senders[0]] + + # if use_concate_mean_global: + # msg[0]=l_msg[0][senders[0]] + if rewardtype == 'self': + buffer_msg = { + 'state': state[0], 'acts': msg[0], 'rewards': rewards[0], + 'alives': alives[0], 'ids': ids[0] + } + + for i in range(n_group): + ids_[i]=ids[i] + # act_onehot=np.eye(n_action[i])[np.array(acts[i])] + indexs,poss=get_distance_matrix(env,handles[i],len_nei) + + # pdb.set_trace() + + if first: + first=False + # stat info + + + + + if render: + env.render() + + # clear dead agents + env.clear_dead() + nums = [env.get_num(handle) for handle in handles] + +# if done: + +# if nums[1]==0: +# buffer['alives']=[False for i in range(len(alives[0]))] +# buffer['rewards'] = [10 for i in range(len(ids[0]))] +# elif nums[0]==0: +# buffer['rewards']=[-10 for i in range(len(ids[0]))] + + if train: + models[0].flush_buffer(**buffer) + models[1].flush_buffer(**buffer1) + if not first and MsgModels[0]: + MsgModels[0].flush_buffer(**buffer_msg) + + for i in range(n_group): + sum_reward = sum(rewards[i]) + if nums[i]!=0: + rewards[i] = sum_reward/nums[i] + else: + rewards[i] = sum_reward + mean_rewards[i].append(rewards[i]) + total_rewards[i].append(sum_reward) + info = {"Ave-Reward": np.round(rewards, decimals=6), "NUM": nums} + step_ct += 1 + + if step_ct % print_every == 0: + print("> step #{}, info: {}".format(step_ct, info)) + + + if train: + models[0].train() + if not selfplay and not is_fix: + models[1].train() + if MsgModels[0]: + MsgModels[0].train() + for i in range(n_group): + mean_rewards[i] = sum(mean_rewards[i]) / len(mean_rewards[i]) + total_rewards[i] = sum(total_rewards[i]) + return max_nums, nums, mean_rewards, total_rewards + diff --git a/examples/battle_model/src/Environment.h b/examples/battle_model/src/Environment.h new file mode 100755 index 0000000..9cef0dd --- /dev/null +++ b/examples/battle_model/src/Environment.h @@ -0,0 +1,42 @@ +/** + * \file Environment.h + * \brief Base class for environment + */ + +#ifndef MAGNET_ENVIRONMENT_H +#define MAGNET_ENVIRONMENT_H + +namespace magent { +namespace environment { + +typedef int GroupHandle; + +class Environment { +public: + Environment() = default; + virtual ~Environment() = default; + + // game + virtual void set_config(const char *key, void *p_value) = 0; + + // run step + virtual void reset() = 0; + virtual void get_observation(GroupHandle group, float **linear_buffers) = 0; + virtual void set_action(GroupHandle group, const int *actions) = 0; + virtual void set_chs(GroupHandle group,const int *chs)=0; + virtual void step(int *done) = 0; + virtual void get_reward(GroupHandle group, float *buffer) = 0; + + // info getter + virtual void get_info(GroupHandle group, const char *name, void *buffer) = 0; + + // render + virtual void render() = 0; +}; + +typedef Environment* EnvHandle; + +} // namespace environment +} // namespace magent + +#endif //MAGNET_ENVIRONMENT_H diff --git a/examples/battle_model/src/discrete_snake/DiscreteSnake.cc b/examples/battle_model/src/discrete_snake/DiscreteSnake.cc new file mode 100755 index 0000000..a37de69 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/DiscreteSnake.cc @@ -0,0 +1,408 @@ +/** + * \file DiscreteSnake.cc + * \brief core game engine of the discrete snake + */ + +#include +#include +#include +#include +#include +#include "DiscreteSnake.h" + + +namespace magent { +namespace discrete_snake { + +DiscreteSnake::DiscreteSnake() { + width = height = 100; + view_width = view_height = 21; + total_resource = (int)(100 * 100 * 0.1); + embedding_size = 16; + max_dead_penalty = -10; + corpse_value = 1; + initial_length = 3; + head_mask = nullptr; + + first_render = true; +} + +DiscreteSnake::~DiscreteSnake() { + if (head_mask != nullptr) + delete head_mask; + + for (auto agent : agents) + delete agent; + + for (auto food : foods) + delete food; +} + +void DiscreteSnake::reset() { + id_counter = 0; + render_generator.next_file(); + + map.reset(width, height); + head_mask = new int [width * height]; + + //free agents + for (int i = 0; i < agents.size(); i++) { + delete agents[i]; + } +} + +void DiscreteSnake::set_config(const char *key, void *p_value) { + float fvalue = *(float *)p_value; + int ivalue = *(int *)p_value; + bool bvalue = *(bool *)p_value; + const char *strvalue = (const char *)p_value; + + if (strequ(key, "map_width")) + width = ivalue; + else if (strequ(key, "map_height")) + height = ivalue; + + else if (strequ(key, "view_width")) + view_width = ivalue; + else if (strequ(key, "view_height")) + view_height = ivalue; + else if (strequ(key, "max_dead_penalty")) + max_dead_penalty = fvalue; + else if (strequ(key, "corpse_value")) + corpse_value = fvalue; + else if (strequ(key, "initial_length")) + initial_length = ivalue; + else if (strequ(key, "total_resource")) + total_resource = ivalue; + + else if (strequ(key, "embedding_size")) + embedding_size = ivalue; + else if (strequ(key, "render_dir")) + render_generator.set_render("save_dir", strvalue); + + else if (strequ(key, "seed")) + srand(ivalue); + + else + LOG(FATAL) << "invalid argument in DiscreteSnake::set_config: " << key; +} + +void DiscreteSnake::add_object(int obj_id, int n, const char *method, const int *linear_buffer) { + if (obj_id == -1) { // wall + + } else if (obj_id == -2) { // food + std::vector pos; + if (strequ(method, "random")) { // snake + for (int i = 0; i < n; i++) { + Food *food = new Food(1, 1, corpse_value); + + map.get_random_blank(pos, 1); + foods.insert(food); + map.add_food(food, pos[0].x, pos[0].y); + } + } else { + LOG(FATAL) << "unsupported method in DiscreteSnake::add_object : " << method; + } + } else if (obj_id == 0) { // snake + if (strequ(method, "random")) { // snake + std::vector pos; + for (int i = 0; i < n; i++) { + Agent *agent = new Agent(id_counter); + Direction dir = (Direction) (random() % (int) DIR_NUM); + + map.get_random_blank(pos, initial_length); + + agent->set_dir(dir); + agent->init_body(pos); + + map.add_agent(agent); + agents.push_back(agent); + } + } else { + LOG(FATAL) << "unsupported method in DiscreteSnake::add_object : " << method; + } + } +} + +void DiscreteSnake::get_observation(GroupHandle group, float **linear_buffer) { + int n_channel = CHANNEL_NUM; // wall food self other id + int n_action = (int)ACT_NUM; + int feature_size = embedding_size + n_action + 1; // embedding + last_action + length + + float (*view_buffer)[view_height][view_width][n_channel]; + float (*feature_buffer)[feature_size]; + + view_buffer = (decltype(view_buffer))linear_buffer[0]; + feature_buffer = (decltype(feature_buffer))linear_buffer[1]; + + size_t agent_size = agents.size(); + + memset(view_buffer, 0, sizeof(float) * agent_size * view_height * view_width * n_channel); + memset(feature_buffer, 0, sizeof(float) * agent_size * feature_size); + + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + + map.extract_view(agent, (float *)view_buffer[i], + view_height, view_width, n_channel, id_counter); + agent->get_embedding(feature_buffer[i], embedding_size); + feature_buffer[i][embedding_size + agent->get_action()] = 1; + feature_buffer[i][embedding_size + n_action] = agent->get_length(); + } +} + +void DiscreteSnake::set_chs(GroupHandle group,const int *chs){ + +} +void DiscreteSnake::set_action(GroupHandle group, const int *actions) { + #pragma omp parallel for + for (int i = 0; i < agents.size(); i++) { + Agent *agent = agents[i]; + Action act = (Action) actions[i]; + agent->set_action(act); + } +} + +void DiscreteSnake::step(int *done) { + #pragma omp declare reduction (merge : std::vector : omp_out.insert(omp_out.end(), omp_in.begin(), omp_in.end())) + #pragma omp declare reduction (merge : std::vector : omp_out.insert(omp_out.end(), omp_in.begin(), omp_in.end())) + #pragma omp declare reduction (merge : std::set : omp_out.insert(omp_in.begin(), omp_in.end())) + + const Action dir2inverse[] = { + ACT_LEFT, ACT_UP, ACT_RIGHT, ACT_DOWN, + }; + + const int delta[][2] = { + {1, 0}, {0, 1}, {-1, 0}, {0, -1}, + }; + + const double eps = 1e-6; + + // update body + LOG(TRACE) << "update body. "; + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + + Action act = agent->get_action(); + Direction dir = agent->get_dir(); + + if (act != ACT_NOOP && (int)act != (int)dir && act != dir2inverse[dir]) { + dir = (Direction)act; + agent->set_dir(dir); + } + + // push new head + Position head = agent->get_head(); + head.x += delta[dir][0]; + head.y += delta[dir][1]; + agent->push_head(head); + + // pop old tail + if (agent->get_total_reward() + 1 + initial_length - eps < agent->get_length()) + map.move_tail(agent); + } + + memset(head_mask, 0, sizeof(int) * width * height); + for (int i = 0; i < agent_size; i++) + head_mask[map.pos2int(agents[i]->get_head())]++; + + // check head (food or wall or other) + LOG(TRACE) << "check head. "; + std::vector eat_list; + std::vector dead_list; + std::set double_head_list; + + int added_length = 0; + #pragma omp parallel for reduction(merge: dead_list) reduction(merge: eat_list) reduction(merge: double_head_list) reduction(+: added_length) + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + Food *eaten = nullptr; + + float reward = 0; + bool dead = false; + + PositionInteger head_int = map.pos2int(agent->get_head()); + if (head_mask[head_int] > 1) { + dead = true; + double_head_list.insert(head_int); + } else { + map.move_head(agent, head_int, reward, dead, eaten); + } + + if (dead) { + dead_list.push_back(agent); + agent->set_dead(); + //agent->add_reward(-std::min(-max_dead_penalty, (Reward)(agent->get_length() - initial_length))); + agent->add_reward(-max_dead_penalty); + } else { + if (eaten != nullptr) { + eat_list.push_back(eaten); + agent->add_reward(reward); + } + // calc total length for resource balancing + added_length += agent->get_length() - initial_length; + } + } + + // delete eaten foods + LOG(TRACE) << "delete eaten food. "; + #pragma omp parallel for + for (int i = 0; i < eat_list.size(); i++) { + map.remove_food(eat_list[i]); + delete eat_list[i]; + } + for (int i = 0; i < eat_list.size(); i++) { + foods.erase(eat_list[i]); + } + + // make dead agents as food + LOG(TRACE) << "make food. "; + std::vector new_foods; + #pragma omp parallel for reduction(merge: new_foods) + for (int i = 0; i < dead_list.size(); i++) { + int add = (int)dead_list[i]->get_length() - initial_length; + map.make_food(dead_list[i], corpse_value, new_foods, add); + } + foods.insert(new_foods.begin(), new_foods.end()); + + // double head, balance total resource + int add = total_resource - added_length - (int)foods.size(); + if (add > 0) { + for (auto pos_int : double_head_list) { + Food *food = new Food(1, 1, corpse_value); + Position pos = map.int2pos(pos_int); + + if (map.add_food(food, pos.x, pos.y)) { + foods.insert(food); + if (--add == 0) + break; + } + } + } + add_object(-2, add, "random", nullptr); + + // find new position for dead agents + /*LOG(TRACE) << "find new position. "; + for (int i = 0; i < dead_list.size(); i++) { + Agent *agent = dead_list[i]; + Direction dir = (Direction)(random() % (int)DIR_NUM); + std::vector pos; + + bool found = map.get_random_blank(pos, initial_length); + if (!found) + LOG(FATAL) << "filled map"; + + agent->set_dir(dir); + agent->init_body(pos); + map.add_agent(agent); + }*/ + + /*if (foods.size() != map.get_food_num()) { + printf("%d %d\n", foods.size(), map.get_food_num()); + exit(0); + }*/ + + *done = 0; +} + +void DiscreteSnake::get_reward(GroupHandle group, float *buffer) { + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + buffer[i] = agents[i]->get_reward(); + } +} + +void DiscreteSnake::clear_dead() { + size_t pt = 0; + size_t agent_size = agents.size(); + for (int j = 0; j < agent_size; j++) { + Agent *agent = agents[j]; + if (agent->is_dead()) { + delete agent; + } else { + agent->init_reward(); + agents[pt++] = agent; + + } + } + agents.resize(pt); +} + +/** + * info getter + */ +void DiscreteSnake::get_info(GroupHandle group, const char *name, void *void_buffer) { + int *int_buffer = (int *)void_buffer; + float *float_buffer = (float *)void_buffer; + bool *bool_buffer = (bool *)void_buffer; + + if (strequ(name, "id")) { + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + int_buffer[i] = agents[i]->get_id(); + } + } else if (strequ(name, "num")) { + if (group == 0) { + int_buffer[0] = (int) agents.size(); + } else if (group == -1) { // wall + int_buffer[0] = 0; + } else if (group == -2) { // food + int_buffer[0] = (int) foods.size(); + } + } else if (strequ(name, "length")) { + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + int_buffer[i] = (int) agents[i]->get_length(); + } + } else if (strequ(name, "alive")) { + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + bool_buffer[i] = !agents[i]->is_dead(); + } + } else if (strequ(name, "head")) { + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + int_buffer[2 * i] = agents[i]->get_body().front().x; + int_buffer[2 * i + 1] = agents[i]->get_body().front().y; + } + } else if (strequ(name, "action_space")) { + int_buffer[0] = (int)ACT_NUM; + } else if (strequ(name, "view_space")) { + int_buffer[0] = view_height; + int_buffer[1] = view_width; + int_buffer[2] = CHANNEL_NUM; + } else if (strequ(name, "feature_space")) { + int n_action = (int)ACT_NUM; + int_buffer[0] = embedding_size + n_action + 1; // embedding + last_action + length + } else { + std::cerr << name << std::endl; + LOG(FATAL) << "unsupported info name in DiscreteSnake::get_info : " << name; + } +} + +/** + * render + */ +void DiscreteSnake::render() { + if (first_render) { + first_render = false; + render_generator.gen_config(map, width, height); + } + + render_generator.render_a_frame(agents, foods); +} + +void DiscreteSnake::render_next_file(){ + render_generator.next_file(); +} + +} // namespace discrete_snake +} // namespace magent \ No newline at end of file diff --git a/examples/battle_model/src/discrete_snake/DiscreteSnake.h b/examples/battle_model/src/discrete_snake/DiscreteSnake.h new file mode 100755 index 0000000..a395297 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/DiscreteSnake.h @@ -0,0 +1,167 @@ +/** + * \file DiscreteSnake.h + * \brief core game engine of the discrete snake + */ + +#ifndef MAGNET_DISCRETE_SNACK_H +#define MAGNET_DISCRETE_SNACK_H + +#include +#include +#include +#include "snake_def.h" +#include "Map.h" +#include "RenderGenerator.h" + +namespace magent { +namespace discrete_snake { + +class DiscreteSnake : public Environment { +public: + DiscreteSnake(); + ~DiscreteSnake() override; + + // game + void set_config(const char *key, void * p_value) override; + + // run step + void reset() override; + void get_observation(GroupHandle group, float **buffer) override; + void set_action(GroupHandle group, const int *actions) override; + void set_chs(GroupHandle group, const int *chs) override; + + + void step(int *done) override; + void get_reward(GroupHandle group, float *buffer) override; + + // info getter + void get_info(GroupHandle group, const char *name, void *void_buffer) override; + + // render + void render() override; + void render_next_file(); + + void clear_dead(); + + void add_object(int obj_id, int n, const char *method, const int *linear_buffer); + +private: + Map map; + std::vector agents; + std::set foods; + int *head_mask; + + int id_counter; + bool first_render; + + /* config */ + int width, height; + int view_width, view_height; + Reward max_dead_penalty, corpse_value; + int embedding_size; + int initial_length; + int total_resource; + + /* render */ + RenderGenerator render_generator; +}; + +class Food { +public: + Food(int w, int h, float v) : width(w), height(h), value(v) { + } + + void set_xy(int x, int y) { + this->x = x; + this->y = y; + } + void get_xy(int &x, int &y) const { + x = this->x; + y = this->y; + } + + void get_size(int &w, int &h) const { + w = width; + h = height; + } + float get_value() const { return value; } + + +private: + int x, y; + int width, height; + float value; +}; + +class Agent { +public: + explicit Agent(int &id_counter): group(0), dead(false), in_event_calc(false), dir(DIR_NUM), last_action(ACT_NUM), + next_reward(0), total_reward(0) { + id = group; + id = id_counter++; + } + + void init_body(std::vector &pos) { + body.clear(); + body.insert(body.begin(), pos.begin(), pos.end()); + total_reward = 0; + } + + void set_dir(Direction dir) { this->dir = dir; } + Direction get_dir() const { return dir; } + + void set_action(Action act) { this->last_action = act; } + Action get_action() const { return this->last_action; } + + void init_reward() { next_reward = 0; } + void add_reward(Reward r) { + next_reward += r; + total_reward += r; + } + Reward get_reward() const { return next_reward; } + Reward get_total_reward() const { return total_reward; } + + Position get_head() const { return body.front(); } + void push_head(Position head) { body.push_front(head); } + Position pop_tail() { + Position ret = body.back(); + body.pop_back(); + return ret; + } + + int get_id() const { return id; } + void get_embedding(float *buf, int size) { + if (embedding.empty()) { + int t = id; + for (int i = 0; i < size; i++, t >>= 1) { + embedding.push_back((float)(t & 1)); + } + } + memcpy(buf, &embedding[0], sizeof(float) * size); + } + + std::deque &get_body() { return body; } + size_t get_length() const { return body.size(); } + + void set_dead() { dead = true; } + bool is_dead() { return dead; } + +private: + std::deque body; + Direction dir; + + Reward next_reward; + Reward total_reward; + Action last_action; + bool dead; + bool in_event_calc; + + int id; + std::vector embedding; + GroupHandle group; +}; + +} // namespace discrete_snake +} // namespace magent + +#endif //MAGNET_DISCRETE_SNACK_H diff --git a/examples/battle_model/src/discrete_snake/Map.cc b/examples/battle_model/src/discrete_snake/Map.cc new file mode 100755 index 0000000..ebd2732 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/Map.cc @@ -0,0 +1,293 @@ +/** + * \file Map.cc + * \brief The map for the game engine + */ + +#include +#include +#include + +#include "DiscreteSnake.h" +#include "Map.h" + +namespace magent { +namespace discrete_snake { + +Map::Map() : slots(nullptr) { +} + +Map::~Map() { + delete [] slots; +} + +void Map::reset(int width, int height) { + if (slots != nullptr) + delete [] slots; + slots = new Slot[width * height]; + + for (int i = 0; i < width * height; i++) { + slots[i].occ_type = OCC_NONE; + } + + map_width = width; + map_height = height; + + // init border + for (int i = 0; i < map_width; i++) { + add_wall(Position{i, 0}); + add_wall(Position{i, map_height - 1}); + } + for (int i = 0; i < map_height; i++) { + add_wall(Position{0, i}); + add_wall(Position{map_width - 1, i}); + } +} + +int Map::add_wall(Position pos) { + PositionInteger pos_int = pos2int(pos); + if (slots[pos_int].occ_type != OCC_NONE) + return 1; + slots[pos_int].occ_type = OCC_WALL; + return 0; +} + +int Map::get_food_num() { + int sum = 0; + int size = map_width * map_height; + #pragma omp parallel for reduction(+:sum) + for (int i = 0; i < size; i++) + if (slots[i].occ_type == OCC_FOOD) { + sum += 1; + } + return sum; +} + +bool Map::get_random_blank(std::vector &pos, int n) { + int tries = 0; + + pos.resize(n); + while(tries < map_width * map_height) { + int last_dir = 100; + PositionInteger pos_int; + + int x = (int)random() % map_width; + int y = (int)random() % map_height; + + int i; + for (i = 0; i < n; i++) { + pos_int = pos2int(x, y); + if (slots[pos_int].occ_type != OCC_NONE) + break; + + pos[i] = Position{x, y}; + + int start = (int)random() % 100; + for (int j = 0; j < 4; j++) { // 4 direction + int new_x = x, new_y = y; + int dir = (start + j) % 4; + if (abs(dir - last_dir) == 2) + continue; + switch (dir) { + case 0: new_x -= 1; break; + case 1: new_y -= 1; break; + case 2: new_x += 1; break; + case 3: new_y += 1; break; + default: + LOG(FATAL) << "invalid dir in Map::get_random_blank"; + } + if (slots[pos_int].occ_type == OCC_NONE) { + x = new_x; y = new_y; + last_dir = dir; + break; + } + } + } + if (i == n) { + return true; + } + tries++; + } + return false; +} + +void Map::add_agent(Agent *agent) { + for (auto pos : agent->get_body()) { + PositionInteger pos_int = pos2int(pos); + slots[pos_int].occ_type = OCC_AGENT; + slots[pos_int].occupier = agent; + slots[pos_int].occ_ct = 1; + } +} + +void Map::extract_view(const Agent* agent, float *linear_buffer, int height, int width, int channel, + int id_counter) { + Position pos = agent->get_head(); + + float (*buffer)[width][channel] = (decltype(buffer)) linear_buffer; + + int x_start = pos.x - width / 2; + int y_start = pos.y - height / 2; + int x_end = x_start + width - 1; + int y_end = y_start + height - 1; + + x_start = std::max(0, std::min(map_width-1, x_start)); + x_end = std::max(0, std::min(map_width-1, x_end)); + y_start = std::max(0, std::min(map_height-1, y_start)); + y_end = std::max(0, std::min(map_height-1, y_end)); + + int view_x_start = 0 + x_start - (pos.x - width/2); + int view_y_start = 0 + y_start - (pos.y - height/2); + + int view_x = view_x_start; + for (int x = x_start; x <= x_end; x++) { + int view_y = view_y_start; + for (int y = y_start; y <= y_end; y++) { + PositionInteger pos_int = pos2int(x, y); + Agent *occupier; + switch (slots[pos_int].occ_type) { + case OCC_NONE: + break; + case OCC_WALL: + buffer[view_y][view_x][CHANNEL_WALL] = 1; + break; + case OCC_FOOD: + buffer[view_y][view_x][CHANNEL_FOOD] = 1; + break; + case OCC_AGENT: + occupier = (Agent*)slots[pos_int].occupier; + if (occupier == agent) { + buffer[view_y][view_x][CHANNEL_SELF] = 1; + } else { + buffer[view_y][view_x][CHANNEL_OTHER] = 1; + } + buffer[view_y][view_x][CHANNEL_ID] = (float) (occupier->get_id() + 1) / id_counter; + break; + } + view_y++; + } + view_x++; + } +} + +void Map::move_tail(Agent *agent) { + Position tail = agent->pop_tail(); + PositionInteger tail_int = pos2int(tail); + + int remain = --slots[tail_int].occ_ct; + + if (remain == 0) { + slots[tail_int].occ_type = OCC_NONE; + } +} + +void Map::move_head(Agent *agent, PositionInteger head_int, Reward &reward, bool &dead, Food *&eaten) { + Food *food; + int x, y, width, height; // for food + + switch (slots[head_int].occ_type) { + case OCC_NONE: + slots[head_int].occ_type = OCC_AGENT; + slots[head_int].occupier = agent; + slots[head_int].occ_ct = 1; + reward = 0; + dead = false; + break; + case OCC_AGENT: + if (slots[head_int].occupier != agent) { + Agent *other = (Agent *)slots[head_int].occupier; + dead = true; + } else { + slots[head_int].occ_ct++; + } + break; + case OCC_WALL: + dead = true; + break; + case OCC_FOOD: + food = (Food *) slots[head_int].occupier; + eaten = food; + int x, y; + food->get_xy(x, y); + + reward = food->get_value(); + + slots[head_int].occ_type = OCC_AGENT; + slots[head_int].occupier = agent; + slots[head_int].occ_ct = 1; + break; + } +} + +void Map::make_food(Agent *agent, Reward value, std::vector &foods, int add) { + bool skip_head = true; + int ct = 0; + for (Position pos : agent->get_body()) { + if (skip_head) { + skip_head = false; + } else { + PositionInteger pos_int = pos2int(pos); + + if (slots[pos_int].occ_type == OCC_AGENT) { + if (ct < add) { + Food *food = new Food(1, 1, value); + slots[pos_int].occ_type = OCC_FOOD; + slots[pos_int].occupier = food; + food->set_xy(pos.x, pos.y); + foods.push_back(food); + ct++; + } else { + slots[pos_int].occ_type = OCC_NONE; + } + } + } + } +} + +bool Map::add_food(Food *food, int x, int y) { + int width, height; + PositionInteger pos_int; + food->set_xy(x, y); + food->get_size(width, height); + + // check clean + for (int i = 0; i < width; i++) + for (int j = 0; j < height; j++) { + pos_int = pos2int(x + i, y + j); + if (slots[pos_int].occ_type != OCC_NONE) + return false; + } + + // mark food + for (int i = 0; i < width; i++) + for (int j = 0; j < height; j++) { + pos_int = pos2int(x + i, y + j); + slots[pos_int].occ_type = OCC_FOOD; + slots[pos_int].occupier = food; + } + return true; +} + +void Map::remove_food(Food *food) { + int x, y, width, height; + food->get_xy(x, y); + food->get_size(width, height); + // clear food + for (int i = 0; i < width; i++) + for (int j = 0; j < height; j++) { + PositionInteger pos_int = pos2int(x + i, y + j); + if (slots[pos_int].occ_type == OCC_FOOD) { + slots[pos_int].occ_type = OCC_NONE; + slots[pos_int].occupier = nullptr; + } + } +} + +void Map::get_wall(std::vector &walls) const { + for (int i = 0; i < map_width * map_height; i++) { + if (slots[i].occ_type == OCC_WALL) + walls.push_back(int2pos(i)); + } +} + +} // namespace discrete_snake +} // namespace magent \ No newline at end of file diff --git a/examples/battle_model/src/discrete_snake/Map.h b/examples/battle_model/src/discrete_snake/Map.h new file mode 100755 index 0000000..77c0510 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/Map.h @@ -0,0 +1,79 @@ +/** + * \file Map.h + * \brief The map for the game engine + */ + +#ifndef MAGNET_DISCRETE_SNAKE_MAP_H +#define MAGNET_DISCRETE_SNAKE_MAP_H + +#include +#include +#include +#include "snake_def.h" + +namespace magent { +namespace discrete_snake { + +typedef enum {OCC_NONE, OCC_WALL, OCC_FOOD, OCC_AGENT} OccType; + +struct Slot { + OccType occ_type; + void *occupier; + int occ_ct; +}; + +class Map { +public: + Map(); + ~Map(); + + void reset(int wdith, int height); + + void add_agent(Agent *agent); + int add_wall(Position pos); + + void get_wall(std::vector &walls) const; + + bool get_random_blank(std::vector &pos, int n); + void extract_view(const Agent* agent, float *linear_buffer, int height, int width, int channel, + int id_counter); + + void move_tail(Agent *agent); + void move_head(Agent *agent, PositionInteger head_int, Reward &reward, bool &dead, Food *&eaten); + + bool add_food(Food *food, int x, int y); + void remove_food(Food *food); + void make_food(Agent *agent, Reward value, std::vector &foods, int add); + int get_food_num(); + + /** + * Utility + */ + bool in_board(int x, int y) const { + return x >= 0 && x < map_width && y >= 0 && y < map_height; + } + + PositionInteger pos2int(Position pos) const { + return pos2int(pos.x, pos.y); + } + + PositionInteger pos2int(int x, int y) const { + return (PositionInteger)x * map_height + y; + //return (PositionInteger)y * map_width + x; + } + + Position int2pos(PositionInteger pos) const { + return Position{(int)(pos / map_height), (int)(pos % map_height)}; + //return Position{(int)(pos % map_width), (int)(pos / map_width)}; + } + +private: + Slot *slots; + + int map_width, map_height; +}; + +} // namespace discrete_snake +} // namespace magent + +#endif //MAGNET_DISCRETE_SNAKE_MAP_H diff --git a/examples/battle_model/src/discrete_snake/RenderGenerator.cc b/examples/battle_model/src/discrete_snake/RenderGenerator.cc new file mode 100755 index 0000000..073a5f4 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/RenderGenerator.cc @@ -0,0 +1,206 @@ +/** + * \file RenderGenerator.cc + * \brief Generate data for render + */ + +#include +#include +#include +#include + +#include "RenderGenerator.h" +#include "DiscreteSnake.h" +#include "../utility/utility.h" + +namespace magent { +namespace discrete_snake { + +RenderGenerator::RenderGenerator() { + save_dir = ""; + file_ct = frame_ct = 0; + frame_per_file = 10000; + id_ct = 0; +} + +void RenderGenerator::next_file() { + file_ct++; + frame_ct = 0; +} + +void RenderGenerator::set_render(const char *key, const char *value) { + if (strequ(key, "save_dir")) + save_dir = std::string(value); + else if (strequ(key, "frame_per_file")) + sscanf(value, "%d", &frame_per_file); +} + +template +void print_json(std::ofstream &os, const char *key, T value, bool last=false) { + os << "\"" << key << "\": " << value; + if (last) + os << std::endl; + else + os << "," << std::endl; +} + +std::string rgba_string(int r, int g, int b, float alpha) { + std::stringstream ss; + ss << "\"rgba(" << r << "," << g << "," << b << "," << alpha << ")\""; + return ss.str(); +}; + +void RenderGenerator::gen_config(const Map &map, int w, int h) { + /***** config *****/ + std::ofstream f_config(save_dir + "/" + "config.json"); + + int colors[][3] = { + {192, 64, 64}, + {64, 64, 192}, + {64, 192, 64}, + }; + int i; + + f_config << "{" << std::endl; + print_json(f_config, "width", w); + print_json(f_config, "height", h); + print_json(f_config, "static-file", "\"static.map\""); + print_json(f_config, "obstacle-style", rgba_string(127, 127, 127, 1)); + print_json(f_config, "dynamic-file-directory", "\".\""); + print_json(f_config, "attack-style", rgba_string(63, 63, 63, 0.8)); + print_json(f_config, "minimap-width", 300); + print_json(f_config, "minimap-height", 250); + + // groups + f_config << "\"group\" : [" << std::endl; + + // food + i = 1; + f_config << "{" << std::endl; + print_json(f_config, "height", 1); + print_json(f_config, "width", 1); + print_json(f_config, "style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 1)); + print_json(f_config, "anchor", "[0, 0]"); + print_json(f_config, "max-speed", 0); + print_json(f_config, "speed-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.01)); + print_json(f_config, "vision-radius", 0); + print_json(f_config, "vision-angle", 0); + print_json(f_config, "vision-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.2)); + print_json(f_config, "attack-radius", 0); + print_json(f_config, "attack-angle", 0); + print_json(f_config, "attack-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.1)); + print_json(f_config, "broadcast-radius", 1, true); + f_config << "}," << std::endl; + + // snake head + i = 0; + f_config << "{" << std::endl; + print_json(f_config, "height", 1); + print_json(f_config, "width", 1); + print_json(f_config, "style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 1)); + print_json(f_config, "anchor", "[0, 0]"); + print_json(f_config, "max-speed", 0); + print_json(f_config, "speed-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.01)); + print_json(f_config, "vision-radius", 0); + print_json(f_config, "vision-angle", 0); + print_json(f_config, "vision-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.2)); + print_json(f_config, "attack-radius", 0); + print_json(f_config, "attack-angle", 0); + print_json(f_config, "attack-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.1)); + print_json(f_config, "broadcast-radius", 1, true); + f_config << "}," << std::endl; + + // snake body + i = 2; + f_config << "{" << std::endl; + print_json(f_config, "height", 1); + print_json(f_config, "width", 1); + print_json(f_config, "style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.9)); + print_json(f_config, "anchor", "[0, 0]"); + print_json(f_config, "max-speed", 0); + print_json(f_config, "speed-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.01)); + print_json(f_config, "vision-radius", 0); + print_json(f_config, "vision-angle", 0); + print_json(f_config, "vision-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.2)); + print_json(f_config, "attack-radius", 0); + print_json(f_config, "attack-angle", 0); + print_json(f_config, "attack-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.1)); + print_json(f_config, "broadcast-radius", 1, true); + f_config << "}" << std::endl; + + f_config << "]" << std::endl; + f_config << "}" << std::endl; + + /***** static *****/ + std::ofstream f_static(save_dir + "/" + "static.map"); + std::vector walls; + map.get_wall(walls); + + // walls + f_static << walls.size() << std::endl; + for (int i = 0; i < walls.size(); i++) { + f_static << walls[i].x << " " << walls[i].y << std::endl; + } +} + + +void RenderGenerator::render_a_frame(const std::vector &agents, const std::set &foods) { + if (save_dir == "") { + return; + } + + std::string filename = save_dir + "/" + "video_" + std::to_string(file_ct) + ".txt"; + std::ofstream fout(filename.c_str(), frame_ct == 0 ? std::ios::out : std::ios::app); + + int num_snake = 0; + for (int i = 0; i < agents.size(); i++) { + if (agents[i]->is_dead()) + continue; + num_snake += agents[i]->get_body().size(); + } + + int num_food = foods.size(); +// for (auto food : foods) { +// int w, h; +// food->get_size(w, h); +// num_food += w * h; +// } + + fout << "F" << " " << num_snake + num_food << " " << 0 << " " << 0 << std::endl; + + const int dir2angle[] = {0, 90, 180, 270}; + int hp = 100; + int dir = dir2angle[3]; + + // food + for (auto food : foods) { + int x, y; + int width, height; + food->get_xy(x, y); + fout << id_ct++ << " " << hp << " " << dir << " " << x << " " << y << " " << 1 << std::endl; +// food->get_size(width, height); +// for (int i = 0; i < width; i++) +// for (int j = 0; j < height; j++) +// fout << id++ << " " << hp << " " << dir << " " << x+i << " " << y+j << " " << 1 << std::endl; + } + + // agent + for (auto agent : agents) { + if (agent->is_dead()) + continue; + auto &body = agent->get_body(); + int tmp = body.size(); + for (auto pos_iter = body.rbegin(); pos_iter != body.rend(); pos_iter++ ) { + int color = --tmp == 0 ? 0 : 2; + fout << id_ct++ << " " << hp << " " << dir << " " << pos_iter->x << " " << pos_iter->y + << " " << color << std::endl; + } + } + + if (frame_ct++ > frame_per_file) { + frame_ct = 0; + file_ct++; + } +} + +} // namespace discrete_snake +} // namespace magent diff --git a/examples/battle_model/src/discrete_snake/RenderGenerator.h b/examples/battle_model/src/discrete_snake/RenderGenerator.h new file mode 100755 index 0000000..bba1b9d --- /dev/null +++ b/examples/battle_model/src/discrete_snake/RenderGenerator.h @@ -0,0 +1,45 @@ +/** + * \file RenderGenerator.h + * \brief Generate data for render + */ + +#ifndef MAGNET_DISCRETE_SNAKE_RENDER_H +#define MAGNET_DISCRETE_SNAKE_RENDER_H + +#include + +#include "snake_def.h" +#include "Map.h" + +namespace magent { +namespace discrete_snake { + +class RenderGenerator { +public: + RenderGenerator(); + + void next_file(); + + void set_render(const char *key, const char *value); + void gen_config(const Map &map, int w, int h); + + void render_a_frame(const std::vector &agents, const std::set &foods); + + + std::string get_save_dir() { + return save_dir; + } + +private: + std::string save_dir; + + int file_ct; + int frame_ct; + int frame_per_file; + unsigned int id_ct; +}; + +} // namespace magent +} // namespace discrete_snake + +#endif //MAGNET_DISCRETE_SNAKE_RENDER_H diff --git a/examples/battle_model/src/discrete_snake/snake_def.h b/examples/battle_model/src/discrete_snake/snake_def.h new file mode 100755 index 0000000..6dfc298 --- /dev/null +++ b/examples/battle_model/src/discrete_snake/snake_def.h @@ -0,0 +1,37 @@ +/** + * \file snake_def.h + * \brief some global definition for discrete_snake + */ + + +#ifndef MAGNET_DISCRETE_SNAKE_SNAKEDEF_H +#define MAGNET_DISCRETE_SNAKE_SNAKEDEF_H + +#include "../utility/utility.h" +#include "../Environment.h" + +namespace magent { +namespace discrete_snake { + +using ::magent::environment::Environment; +using ::magent::environment::GroupHandle; +using ::magent::utility::strequ; + +struct Position { + int x; + int y; +}; + +typedef int PositionInteger; + +typedef enum { ACT_RIGHT, ACT_DOWN, ACT_LEFT, ACT_UP, ACT_NOOP, ACT_NUM } Action; +typedef enum { RIGHT, DOWN, LEFT, UP, DIR_NUM } Direction; +enum { CHANNEL_WALL, CHANNEL_FOOD, CHANNEL_SELF, CHANNEL_OTHER, CHANNEL_ID, CHANNEL_NUM}; +typedef float Reward; + +class Agent; +class Food; +} // namespace discrete_snake +} // namespace magent + +#endif //MAGNET_DISCRETE_SNAKE_SNAKEDEF_H diff --git a/examples/battle_model/src/gridworld/AgentType.cc b/examples/battle_model/src/gridworld/AgentType.cc new file mode 100755 index 0000000..d5710df --- /dev/null +++ b/examples/battle_model/src/gridworld/AgentType.cc @@ -0,0 +1,126 @@ +/** + * \file AgentType.cc + * \brief implementation of AgentType (mainly initialization) + */ + +#include "AgentType.h" + +namespace magent { +namespace gridworld { + + +#define AGENT_TYPE_SET_INT(name) \ + if (strequ(keys[i], #name)) {\ + name = (int)(values[i] + 0.5);\ + is_set = true;\ + } + +#define AGENT_TYPE_SET_FLOAT(name) \ + if (strequ(keys[i], #name)) {\ + name = values[i];\ + is_set = true;\ + } + +#define AGENT_TYPE_SET_BOOL(name)\ + if (strequ(keys[i], #name)) {\ + name = bool(int(values[i] + 0.5));\ + is_set = true;\ + } + +AgentType::AgentType(int n, std::string name, const char **keys, float *values, bool turn_mode) { + this->name = name; + + // default value + attack_in_group = false; + width = length = 1; + speed = 1.0; hp = 1.0; + + view_radius = 1; view_angle = 360; + attack_radius = 0; attack_angle = 0; + + hear_radius = speak_radius = 0.0f; + speak_ability = 0; + + damage = trace = eat_ability = step_recover = kill_supply = food_supply = 0; + + attack_in_group = false; can_absorb = false; + step_reward = kill_reward = dead_penalty = attack_penalty = 0.0; + can_absorb = false; + + // init member vars from str (reflection) + bool is_set; + for (int i = 0; i < n; i++) { + is_set = false; + AGENT_TYPE_SET_INT(width); + AGENT_TYPE_SET_INT(length); + + AGENT_TYPE_SET_FLOAT(speed); + AGENT_TYPE_SET_FLOAT(hp); + + AGENT_TYPE_SET_FLOAT(view_radius); AGENT_TYPE_SET_FLOAT(view_angle); + AGENT_TYPE_SET_FLOAT(attack_radius);AGENT_TYPE_SET_FLOAT(attack_angle); + + AGENT_TYPE_SET_FLOAT(hear_radius); AGENT_TYPE_SET_FLOAT(speak_radius); + AGENT_TYPE_SET_INT(speak_ability); + + AGENT_TYPE_SET_FLOAT(damage); AGENT_TYPE_SET_FLOAT(trace); + AGENT_TYPE_SET_FLOAT(eat_ability); + AGENT_TYPE_SET_FLOAT(step_recover); AGENT_TYPE_SET_FLOAT(kill_supply); + AGENT_TYPE_SET_FLOAT(food_supply); + + AGENT_TYPE_SET_BOOL(attack_in_group); AGENT_TYPE_SET_BOOL(can_absorb); + + AGENT_TYPE_SET_FLOAT(step_reward); AGENT_TYPE_SET_FLOAT(kill_reward); + AGENT_TYPE_SET_FLOAT(dead_penalty); AGENT_TYPE_SET_FLOAT(attack_penalty); + + AGENT_TYPE_SET_FLOAT(view_x_offset); AGENT_TYPE_SET_FLOAT(view_y_offset); + AGENT_TYPE_SET_FLOAT(att_x_offset); AGENT_TYPE_SET_FLOAT(att_y_offset); + AGENT_TYPE_SET_FLOAT(turn_x_offset); AGENT_TYPE_SET_FLOAT(turn_y_offset); + + if (!is_set) { + LOG(FATAL) << "invalid agent config in AgentType::AgentType : " << keys[i]; + } + } + + // NOTE: do not support SectorRange with angle >= 180, only support circle range when angle >= 180 + int parity = width % 2; // use parity to make range center-symmetric + if (view_angle >= 180) { + if (fabs(view_angle - 360) > 1e-5) { + LOG(FATAL) << "only supports ranges with angle = 360, when angle > 180."; + } + view_range = new CircleRange(view_radius, 0, parity); + } else { + view_range = new SectorRange(view_angle, view_radius, parity); + } + + if (attack_angle >= 180) { + if (fabs(attack_angle - 360) > 1e-5) { + LOG(FATAL) << "only supports ranges with angle = 360, when angle > 180."; + } + attack_range = new CircleRange(attack_radius, width / 2.0f, parity); + } else { + attack_range = new SectorRange(attack_angle, attack_radius, parity); + } + + move_range = new CircleRange(speed, 0, 1); + view_x_offset = width / 2; view_y_offset = length / 2; + att_x_offset = width / 2; att_y_offset = length / 2; + turn_x_offset = 0; turn_y_offset = 0; + + move_base = 0; + turn_base = move_range->get_count(); + + if (turn_mode) { + attack_base = turn_base + 2; + } else { + attack_base = turn_base; + } + int n_action = attack_base + attack_range->get_count(); + for (int i = 0; i < n_action; i++) { + action_space.push_back(i); + } + // action space layout : move turn attack ... +} + +} // namespace magent +} // namespace gridworld diff --git a/examples/battle_model/src/gridworld/AgentType.h b/examples/battle_model/src/gridworld/AgentType.h new file mode 100755 index 0000000..662f57e --- /dev/null +++ b/examples/battle_model/src/gridworld/AgentType.h @@ -0,0 +1,55 @@ +/** + * \file AgentType.h + * \brief implementation of AgentType (mainly initialization) + */ + +#ifndef MAGENT_GRIDWORLD_AGENTTYPE_H +#define MAGENT_GRIDWORLD_AGENTTYPE_H + +#include + +#include "grid_def.h" +#include "Range.h" + +namespace magent { +namespace gridworld { + +class AgentType { +public: + AgentType(int n, std::string name, const char **keys, float *values, bool turn_mode); + + // user defined setting + int width, length; + float speed, hp; + float view_radius, view_angle; + float attack_radius, attack_angle; + + float hear_radius, speak_radius; + + int speak_ability; + float damage, trace, eat_ability, step_recover, kill_supply, food_supply; + bool attack_in_group; + int is_ch; + Reward step_reward, kill_reward, dead_penalty, attack_penalty; + + int view_x_offset, view_y_offset; + int att_x_offset, att_y_offset; + int turn_x_offset, turn_y_offset; + + /** special for demo **/ + bool can_absorb; + + /***** system calculated setting *****/ + std::string name; + int n_channel; // obstacle, group1, group_hp1, group2, group_hp2 + Range *view_range, *attack_range, *move_range; + + int move_base, turn_base, attack_base; + std::vector action_space; +}; + + +} // namespace magent +} // namespace gridworld + +#endif //MAGENT_GRIDWORLD_AGENTTYPE_H diff --git a/examples/battle_model/src/gridworld/GridWorld.cc b/examples/battle_model/src/gridworld/GridWorld.cc new file mode 100755 index 0000000..f7930e9 --- /dev/null +++ b/examples/battle_model/src/gridworld/GridWorld.cc @@ -0,0 +1,996 @@ +/** + * \file GridWorld.cc + * \brief core game engine of the gridworld + */ + +#include +#include +#include +#include +#include + +#include "GridWorld.h" + +namespace magent { +namespace gridworld { + +GridWorld::GridWorld() { + first_render = true; + + food_mode = false; + turn_mode = false; + minimap_mode = false; + goal_mode = false; + large_map_mode = false; + mean_mode = false; + + reward_des_initialized = false; + embedding_size = 0; + random_engine.seed(0); + + counter_x = counter_y = nullptr; +} + +GridWorld::~GridWorld() { + for (int i = 0; i < groups.size(); i++) { + std::vector &agents = groups[i].get_agents(); + + // free agents + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int j = 0; j < agent_size; j++) { + delete agents[j]; + } + + // free ranges + AgentType &type = groups[i].get_type(); + if (type.view_range != nullptr) { + delete type.view_range; + type.view_range = nullptr; + } + if (type.attack_range != nullptr) { + delete type.attack_range; + type.attack_range = nullptr; + } + if (type.move_range != nullptr) { + delete type.move_range; + type.move_range = nullptr; + } + } + + if (counter_x != nullptr) + delete [] counter_x; + if (counter_y != nullptr) + delete [] counter_y; + + if (large_map_mode) { + delete [] move_buffers; + delete [] turn_buffers; + } +} + +void GridWorld::reset() { + id_counter = 0; + + if (width * height > 99 * 99) { + large_map_mode = true; + if (width * height > 1000 * 1000) { + NUM_SEP_BUFFER = 16; + } else { + NUM_SEP_BUFFER = 8; + } + move_buffers = new std::vector[NUM_SEP_BUFFER]; + turn_buffers = new std::vector[NUM_SEP_BUFFER]; + } + + // reset map + map.reset(width, height, food_mode); + + if (counter_x != nullptr) + delete [] counter_x; + if (counter_y != nullptr) + delete [] counter_y; + counter_x = new int [width]; + counter_y = new int [height]; + + render_generator.next_file(); + stat_recorder.reset(); + + for (int i = 0;i < groups.size(); i++) { + std::vector &agents = groups[i].get_agents(); + + // free agents + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int j = 0; j < agent_size; j++) { + delete agents[j]; + } + + groups[i].clear(); + groups[i].get_type().n_channel = group2channel((GroupHandle)groups.size()); + } + + if (!reward_des_initialized) { + init_reward_description(); + reward_des_initialized = true; + } +} + +void GridWorld::set_config(const char *key, void *p_value) { + float fvalue = *(float *)p_value; + int ivalue = *(int *)p_value; + bool bvalue = *(bool *)p_value; + const char *strvalue = (const char *)p_value; + + if (strequ(key, "map_width")) + width = ivalue; + else if (strequ(key, "map_height")) + height = ivalue; + + else if (strequ(key, "food_mode")) // dead agent will leave food in the map + food_mode = bvalue; + else if (strequ(key, "turn_mode")) // has two more actions -- turn left and turn right + turn_mode = bvalue; + else if (strequ(key, "minimap_mode")) // add minimap into observation + minimap_mode = bvalue; + else if (strequ(key, "goal_mode")) // deprecated every agents has a specific goal + goal_mode = bvalue; + else if (strequ(key, "embedding_size")) // embedding size in the observation.feature + embedding_size = ivalue; + + else if (strequ(key, "render_dir")) // the directory of saved videos + render_generator.set_render("save_dir", strvalue); + else if (strequ(key, "seed")) // random seed + random_engine.seed((unsigned long)ivalue); + + else + LOG(FATAL) << "invalid argument in GridWorld::set_config : " << key; +} + +void GridWorld::register_agent_type(const char *name, int n, const char **keys, float *values) { + std::string str(name); + + if (agent_types.find(str) != agent_types.end()) + LOG(FATAL) << "duplicated name of agent type in GridWorld::register_agent_type : " << str; + + agent_types.insert(std::make_pair(str, AgentType(n, str, keys, values, turn_mode))); +} + +void GridWorld::new_group(const char* agent_name, GroupHandle *group) { + *group = (GroupHandle)groups.size(); + + auto iter = agent_types.find(std::string(agent_name)); + if (iter == agent_types.end()) { + LOG(FATAL) << "invalid name of agent type in new_group : " << agent_name; + } + + groups.push_back(Group(iter->second)); +} + +void add_or_error(int ret, int x, int y, int &id_counter, Group &g, Agent *agent) { + if (ret != 0) { + LOG(WARNING) << "invalid position in add_agents (" << x << ", " << y << "), already occupied, ignored.\n"; + } else { + id_counter++; + g.add_agent(agent); + } +}; + +void GridWorld::add_agents(GroupHandle group, int n, const char *method, + const int *pos_x, const int *pos_y, const int *pos_dir) { + int ret; + + if (group == -1) { // group == -1 for wall + if (strequ(method, "random")) { + for (int i = 0; i < n; i++) { + Position pos = map.get_random_blank(random_engine); + ret = map.add_wall(pos); + if (ret != 0) { + LOG(WARNING) << "invalid position in add_wall (" << pos_x[i] << ", " << pos_y[i] << "), " + << "already occupied, ignored."; + } + } + } else if (strequ(method, "custom")) { + for (int i = 0; i < n; i++) { + Position pos = Position{pos_x[i], pos_y[i]}; + ret = map.add_wall(pos); + if (ret != 0) { + LOG(WARNING) << "invalid position in add_wall (" << pos_x[i] << ", " << pos_y[i] << "), " + << "already occupied, ignored."; + } + } + } else if (strequ(method, "fill")) { + // parameter int xs[4] = {x, y, width, height} + int x_start = pos_x[0], y_start = pos_x[1]; + int x_end = x_start + pos_x[2], y_end = y_start + pos_x[3]; + for (int x = x_start; x < x_end; x++) + for (int y = y_start; y < y_end; y++) { + ret = map.add_wall(Position{x, y}); + if (ret != 0) { + LOG(WARNING) << "invalid position in add_wall (" << x << ", " << y << "), " + << "already occupied, ignored."; + } + } + } else { + LOG(FATAL) << "unsupported method in GridWorld::add_agents : " << method; + } + } else { // group >= 0 for agents + if (group > groups.size()) { + LOG(FATAL) << "invalid group handle in GridWorld::add_agents : " << group; + } + Group &g = groups[group]; + int base_channel_id = group2channel(group); + int width = g.get_type().width, length = g.get_type().length; + AgentType &agent_type = g.get_type(); + + if (strequ(method, "random")) { + for (int i = 0; i < n; i++) { + Agent *agent = new Agent(agent_type, id_counter, group); + Direction dir = turn_mode ? (Direction)(random_engine() % DIR_NUM) : NORTH; + Position pos; + + if (dir == NORTH || dir == SOUTH) { + pos = map.get_random_blank(random_engine, width, length); + } else { + pos = map.get_random_blank(random_engine, length, width); + } + + agent->set_dir(dir); + agent->set_pos(pos); + + ret = map.add_agent(agent, base_channel_id); + add_or_error(ret, pos.x, pos.y, id_counter, g, agent); + } + } else if (strequ(method, "custom")) { + for (int i = 0; i < n; i++) { + Agent *agent = new Agent(agent_type, id_counter, group); + + if (pos_dir[i] >= DIR_NUM) { + LOG(FATAL) << "invalid direction in GridWorld::add_agent"; + } + + agent->set_dir(turn_mode ? (Direction) pos_dir[i] : NORTH); + agent->set_pos((Position) {pos_x[i], pos_y[i]}); + + ret = map.add_agent(agent, base_channel_id); + add_or_error(ret, pos_x[i], pos_y[i], id_counter, g, agent); + } + } else if (strequ(method, "fill")) { + // parameter int xs[4] = {x, y, width, height} + int x_start = pos_x[0], y_start = pos_x[1]; + int x_end = x_start + pos_x[2], y_end = y_start + pos_x[3]; + Direction dir = turn_mode ? (Direction)pos_x[4] : NORTH; + int m_width, m_height; + + if (dir == NORTH || dir == SOUTH) { + m_width = width; + m_height = length; + } else if (dir == WEST || dir == EAST) { + m_width = length; + m_height = width; + } else { + LOG(FATAL) << "invalid direction in GridWorld::add_agent"; + } + + for (int x = x_start; x < x_end; x += m_width) + for (int y = y_start; y < y_end; y += m_height) { + Agent *agent = new Agent(agent_type, id_counter, group); + + agent->set_pos(Position{x, y}); + agent->set_dir(dir); + + ret = map.add_agent(agent, base_channel_id); + add_or_error(ret, x, y, id_counter, g, agent); + } + } else { + LOG(FATAL) << "unsupported method in GridWorld::add_agents : " << method; + } + } +} + +void GridWorld::get_observation(GroupHandle group, float **linear_buffers) { + Group &g = groups[group]; + AgentType &type = g.get_type(); + + const int n_channel = g.get_type().n_channel; + const int view_width = g.get_type().view_range->get_width(); + const int view_height = g.get_type().view_range->get_height(); + const int n_group = (int)groups.size(); + const int n_action = (int)type.action_space.size(); + const int feature_size = get_feature_size(group); + + std::vector &agents = g.get_agents(); + size_t agent_size = agents.size(); + + // transform buffers + NDPointer view_buffer(linear_buffers[0], {{-1, view_height, view_width, n_channel}}); + NDPointer feature_buffer(linear_buffers[1], {{-1, feature_size}}); + + memset(view_buffer.data, 0, sizeof(float) * agent_size * view_height * view_width * n_channel); + memset(feature_buffer.data, 0, sizeof(float) * agent_size * feature_size); + + // gather view info from AgentType + const Range *range = type.view_range; + int view_x_offset = type.view_x_offset, view_y_offset = type.view_y_offset; + int view_left_top_x, view_left_top_y, view_right_bottom_x, view_right_bottom_y; + range->get_range_rela_offset(view_left_top_x, view_left_top_y, + view_right_bottom_x, view_right_bottom_y); + + // to make channel layout in observation symmetric to every group + std::vector channel_trans = make_channel_trans(group, + group2channel(0), + type.n_channel, + n_group); + + // build minimap + NDPointer minimap(nullptr, {{view_height, view_width, n_group}}); + int scale_h = (height + view_height - 1) / view_height; + int scale_w = (width + view_width - 1) / view_width; + + if (minimap_mode) { + minimap.data = new float [view_height * view_width * n_group]; + memset(minimap.data, 0, sizeof(float) * view_height * view_width * n_group); + + std::vector group_sizes; + for (int i = 0; i < n_group; i++) + group_sizes.push_back(groups[i].get_size() > 0 ? (int)groups[i].get_size() : 1); + + // by agents + #pragma omp parallel for + for (int i = 0; i < n_group; i++) { + std::vector &agents_ = groups[i].get_agents(); + AgentType type_ = agents[0]->get_type(); + size_t total_ct = 0; + for (int j = 0; j < agents_.size(); j++) { + if (type_.can_absorb && agents_[j]->is_absorbed()) // ignore absorbed goal + continue; + Position pos = agents_[j]->get_pos(); + int x = pos.x / scale_w, y = pos.y / scale_h; + minimap.at(y, x, i)++; + total_ct++; + } + // scale + for (int j = 0; j < view_height; j++) { + for (int k = 0; k < view_width; k++) { + minimap.at(j, k, i) /= total_ct; + } + } + } + } + + // fill local view for every agents + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + // get spatial view + map.extract_view(agent, view_buffer.data + i*view_height*view_width*n_channel, &channel_trans[0], range, + n_channel, view_width, view_height, view_x_offset, view_y_offset, + view_left_top_x, view_left_top_y, view_right_bottom_x, view_right_bottom_y); + + if (minimap_mode) { + int self_x = agent->get_pos().x / scale_w; + int self_y = agent->get_pos().y / scale_h; + for (int j = 0; j < n_group; j++) { + int minimap_channel = channel_trans[group2channel(j)] + 2; + // copy minimap to channel + for (int k = 0; k < view_height; k++) { + for (int l = 0; l < view_width; l++) { + view_buffer.at(i, k, l, minimap_channel)= minimap.at(k, l, j); + } + } + view_buffer.at(i, self_y, self_x, minimap_channel) += 1; + } + } + + // get non-spatial feature + agent->get_embedding(feature_buffer.data + i*feature_size, embedding_size); + Position pos = agent->get_pos(); + // last action + feature_buffer.at(i, embedding_size + agent->get_action()) = 1; + // last reward + feature_buffer.at(i, embedding_size + n_action) = agent->get_last_reward(); + if (minimap_mode) { // absolute coordination + feature_buffer.at(i, embedding_size + n_action + 1) = (float) pos.x / width; + feature_buffer.at(i, embedding_size + n_action + 2) = (float) pos.y / height; + } + } + + if (minimap_mode) + delete [] minimap.data; +} + +void GridWorld::set_chs(GroupHandle group, const int *chs){ + std::vector &agents = groups[group].get_agents(); + size_t agent_size = agents.size(); + // LOG(TRACE)<set_ch(chs[i]); + } +} +void GridWorld::set_action(GroupHandle group, const int *actions) { + std::vector &agents = groups[group].get_agents(); + const AgentType &type = groups[group].get_type(); + // action space layout : move turn attack ... + const int bandwidth = (width + NUM_SEP_BUFFER - 1) / NUM_SEP_BUFFER; + + size_t agent_size = agents.size(); + + if (large_map_mode) { // divide actions in group, in order to compute them in parallel + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + Action act = (Action) actions[i]; + agent->set_action(act); + + if (act < type.turn_base) { // move + int x = agent->get_pos().x; + int x_ = x % bandwidth; + if (x_ < 4 || x_ > bandwidth - 4) { + move_buffer_bound.push_back(MoveAction{agent, act - type.move_base}); + } else { + int to = agent->get_pos().x / bandwidth; + move_buffers[to].push_back(MoveAction{agent, act - type.move_base}); + } + } else if (act < type.attack_base) { // turn + int x = agent->get_pos().x; + int x_ = x % bandwidth; + if (x_ < 4 || x_ > bandwidth - 4) { + turn_buffer_bound.push_back(TurnAction{agent, act - type.move_base}); + } else { + int to = agent->get_pos().x / bandwidth; + turn_buffers[to].push_back(TurnAction{agent, act - type.move_base}); + } + } else { // attack + attack_buffer.push_back(AttackAction{agent, act - type.attack_base}); + } + } + } else { + for (int i = 0; i < agent_size; i++) { + Agent *agent = agents[i]; + Action act = (Action) actions[i]; + agent->set_action(act); + + if (act < type.turn_base) { // move + move_buffer_bound.push_back(MoveAction{agent, act - type.move_base}); + } else if (act < type.attack_base) { // turn + turn_buffer_bound.push_back(TurnAction{agent, act - type.move_base}); + exit(0); + } else { // attack + attack_buffer.push_back(AttackAction{agent, act - type.attack_base}); + } + } + } +} + +void GridWorld::step(int *done) { + #pragma omp declare reduction (merge : std::vector : omp_out.insert(omp_out.end(), omp_in.begin(), omp_in.end())) + const bool stat = false; + + LOG(TRACE) << "gridworld step begin. "; + size_t attack_size = attack_buffer.size(); + size_t group_size = groups.size(); + + // shuffle attacks + for (int i = 0; i < attack_size; i++) { + int j = (int)random_engine() % (i+1); + std::swap(attack_buffer[i], attack_buffer[j]); + } + + LOG(TRACE) << "attack. "; + std::vector render_attack_buffer; + std::map attack_obj_counter; // for statistic info + // attack + #pragma omp parallel for reduction(merge: render_attack_buffer) + for (int i = 0; i < attack_size; i++) { + Agent *agent = attack_buffer[i].agent; + if (agent->is_dead()) + continue; + + int obj_x, obj_y; + PositionInteger obj_pos = map.get_attack_obj(attack_buffer[i], obj_x, obj_y); + LOG(TRACE) << "attack 1 "; + if (!first_render) + render_attack_buffer.emplace_back(RenderAttackEvent{agent->get_id(), obj_x, obj_y}); + + if (obj_pos == -1) { // attack blank block + + agent->add_reward(agent->get_type().attack_penalty); + continue; + } + LOG(TRACE) << "attack 0 "; + + if (stat){ + LOG(TRACE)<<"attack2"; + attack_obj_counter[obj_pos]++; + } + + float reward = 0.0; + GroupHandle dead_group = -1; + #pragma omp critical + { + reward = map.do_attack(agent, obj_pos, dead_group); + if (dead_group != -1) { + groups[dead_group].inc_dead_ct(); + } + } + agent->add_reward(reward + agent->get_type().attack_penalty); + } + + attack_buffer.clear(); + if (!first_render) + render_generator.set_attack_event(render_attack_buffer); + + if (stat) { + for (auto iter : attack_obj_counter) { + if (iter.second > 1) { + stat_recorder.both_attack++; + } + } + } + + // starve + LOG(TRACE) << "starve. "; + for (int i = 0; i < group_size; i++) { + Group &group = groups[i]; + std::vector &agents = group.get_agents(); + int starve_ct = 0; + size_t agent_size = agents.size(); + + #pragma omp parallel for reduction(+: starve_ct) + for (int j = 0; j < agent_size; j++) { + Agent *agent = agents[j]; + + if (agent->is_dead()) + continue; + + // alive agents + bool starve = agent->starve(); + if (starve) { + map.remove_agent(agent); + starve_ct++; + } + } + group.set_dead_ct(group.get_dead_ct() + starve_ct); + } + + if (turn_mode) { + // do turn + auto do_turn_for_a_buffer = [] (std::vector &turn_buf, Map &map) { + //std::random_shuffle(turn_buf.begin(), turn_buf.end()); + size_t turn_size = turn_buf.size(); + for (int i = 0; i < turn_size; i++) { + Action act = turn_buf[i].action; + Agent *agent = turn_buf[i].agent; + + if (agent->is_dead()) + continue; + + int dir = act * 2 - 1; + map.do_turn(agent, dir); + } + turn_buf.clear(); + }; + + if (large_map_mode) { + LOG(TRACE) << "turn parallel. "; + #pragma omp parallel for + for (int i = 0; i < NUM_SEP_BUFFER; i++) { // turn in separate areas, do them in parallel + do_turn_for_a_buffer(turn_buffers[i], map); + } + } + LOG(TRACE) << "turn boundary. "; + do_turn_for_a_buffer(turn_buffer_bound, map); + } + + // do move + auto do_move_for_a_buffer = [] (std::vector &move_buf, Map &map) { + //std::random_shuffle(move_buf.begin(), move_buf.end()); + size_t move_size = move_buf.size(); + for (int j = 0; j < move_size; j++) { + Action act = move_buf[j].action; + Agent *agent = move_buf[j].agent; + + if (agent->is_dead() || agent->is_absorbed()) + continue; + + int dx, dy; + int delta[2]; + agent->get_type().move_range->num2delta(act, dx, dy); + switch(agent->get_dir()) { + case NORTH: + delta[0] = dx; delta[1] = dy; break; + case SOUTH: + delta[0] = -dx; delta[1] = -dy; break; + case WEST: + delta[0] = dy; delta[1] = -dx; break; + case EAST: + delta[0] = -dy; delta[1] = dx; break; + default: + LOG(FATAL) << "invalid direction in GridWorld::step when do move"; + } + + map.do_move(agent, delta); + } + move_buf.clear(); + }; + + if (large_map_mode) { + LOG(TRACE) << "move parallel. "; + #pragma omp parallel for + for (int i = 0; i < NUM_SEP_BUFFER; i++) { // move in separate areas, do them in parallel + do_move_for_a_buffer(move_buffers[i], map); + } + } + LOG(TRACE) << "move boundary. "; + do_move_for_a_buffer(move_buffer_bound, map); + + LOG(TRACE) << "calc_reward. "; + calc_reward(); + + LOG(TRACE) << "game over check. "; + int live_ct = 0; // default game over condition: all the agents in an arbitrary group die + for (int i = 0; i < groups.size(); i++) { + if (groups[i].get_alive_num() > 0) + live_ct++; + } + *done = (int)(live_ct < groups.size()); + + size_t rule_size = reward_rules.size(); + for (int i = 0; i < rule_size; i++) { + if (reward_rules[i].trigger && reward_rules[i].is_terminal) + *done = (int)true; + } +} + +void GridWorld::clear_dead() { + size_t group_size = groups.size(); + + #pragma omp parallel for + for (int i = 0; i < group_size; i++) { + Group &group = groups[i]; + group.init_reward(); + std::vector &agents = group.get_agents(); + + // clear dead agents + size_t agent_size = agents.size(); + int dead_ct = 0; + unsigned int pt = 0; + float sum_x = 0, sum_y = 0; + + for (int j = 0; j < agent_size; j++) { + Agent *agent = agents[j]; + if (agent->is_dead()) { + delete agent; + dead_ct++; + } else { + agent->init_reward(); + agent->set_index(pt); + agents[pt++] = agent; + + //Position pos = agent->get_pos(); + //sum_x += pos.x; sum_y += pos.y; + } + } + agents.resize(pt); + group.set_dead_ct(0); + } +} + +void GridWorld::set_goal(GroupHandle group, const char *method, const int *linear_buffer) { + // deprecated + if (strequ(method, "random")) { + std::vector &agents = groups[group].get_agents(); + for (int i = 0; i < agents.size(); i++) { + int x = (int)random_engine() % width; + int y = (int)random_engine() % height; + agents[i]->set_goal(Position{x, y}, 0); + } + } else { + LOG(FATAL) << "invalid goal type in GridWorld::set_goal"; + } +} + +void GridWorld::calc_reward() { + size_t rule_size = reward_rules.size(); + for (int i = 0; i < groups.size(); i++) + groups[i].set_recursive_base(0); + + for (int i = 0; i < rule_size; i++) { + reward_rules[i].trigger = false; + std::vector &input_symbols = reward_rules[i].input_symbols; + std::vector &infer_obj = reward_rules[i].infer_obj; + calc_rule(input_symbols, infer_obj, reward_rules[i], 0); + } +} + +void GridWorld::get_reward(GroupHandle group, float *buffer) { + std::vector &agents = groups[group].get_agents(); + + size_t agent_size = agents.size(); + Reward group_reward = groups[group].get_reward(); + + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + buffer[i] = agents[i]->get_reward() + group_reward; + } +} + +/** + * info getter + */ +void GridWorld::get_info(GroupHandle group, const char *name, void *void_buffer) { + // for more information from the engine, add items here + + std::vector &agents = groups[group].get_agents(); + int *int_buffer = (int *)void_buffer; + float *float_buffer = (float *)void_buffer; + bool *bool_buffer = (bool *)void_buffer; + + if (strequ(name, "num")) { // int + int_buffer[0] = groups[group].get_num(); + } else if (strequ(name, "id")) { // int + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + int_buffer[i] = agents[i]->get_id(); + } + } else if (strequ(name, "pos")) { // int + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + int_buffer[2 * i] = agents[i]->get_pos().x; + int_buffer[2 * i + 1] = agents[i]->get_pos().y; + } + } else if (strequ(name,"hps")){ //float + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; iget_hp(); + } + + }else if(strequ(name,"chs")){ + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; iget_ch(); + } + } + else if (strequ(name, "alive")) { // bool + size_t agent_size = agents.size(); + #pragma omp parallel for + for (int i = 0; i < agent_size; i++) { + bool_buffer[i] = !agents[i]->is_dead(); + } + } else if (strequ(name, "global_minimap")) { + size_t n_group = groups.size(); + + int view_height = (int)lround(float_buffer[0]); + int view_width = (int)lround(float_buffer[1]); + memset(float_buffer, 0, sizeof(float) * view_height * view_width * n_group); + + NDPointer minimap(float_buffer, {view_height, view_width, (int)n_group}); + + int scale_h = (height + view_height - 1) / view_height; + int scale_w = (width + view_width - 1) / view_width; + + for (size_t i = 0; i < n_group; i++) { + size_t channel = (i - group + n_group) % n_group; + std::vector &agents_ = groups[i].get_agents(); + for (size_t j = 0; j < agents_.size(); j++) { + Position pos = agents_[j]->get_pos(); + int x = pos.x / scale_w, y = pos.y / scale_h; + minimap.at(y, x, channel)++; + } + // scale + for (size_t j = 0; j < view_height; j++) { + for (size_t k = 0; k < view_width; k++) { + minimap.at(j, k, channel) /= agents_.size(); + } + } + } + } else if (strequ(name, "mean_info")) { + size_t agent_size = agents.size(); + int n_action = (int)groups[group].get_type().action_space.size(); + int *action_counter = new int[n_action]; + float sum_x, sum_y; + sum_x = sum_y = 0; + memset(action_counter, 0, sizeof(int) * n_action); + #pragma omp parallel for reduction(+: sum_x) reduction(+: sum_y) + for (int i = 0; i < agent_size; i++) { + Position pos = agents[i]->get_pos(); + sum_x += pos.x; + sum_y += pos.y; + int x = agents[i]->get_action(); + #pragma omp atomic + action_counter[x]++; + } + + assert (agent_size != 0); + float_buffer[0] = sum_x / agent_size; + float_buffer[1] = sum_y / agent_size; + for (int i = 0; i < n_action; i++) + float_buffer[2 + i] = (float)(1.0 * action_counter[i] / agent_size); + } else if (strequ(name, "walls_info")) { + std::vector walls; + map.get_wall(walls); + + NDPointer coor(int_buffer, {-1, 2}); + for (int i = 0; i < walls.size(); i++) { + coor.at(i+1, 0) = walls[i].x; + coor.at(i+1, 1) = walls[i].y; + } + coor.at(0, 0) = (int)walls.size(); + } else if (strequ(name, "render_window_info")) { + first_render = false; + + int ct = 1; + int range_x1, range_y1, range_x2, range_y2; + + // read parameter + range_x1 = int_buffer[0]; + range_y1 = int_buffer[1]; + range_x2 = int_buffer[2]; + range_y2 = int_buffer[3]; + + // fill valid agents + NDPointer ret(int_buffer, {-1, 4}); + for (int i = 0; i < groups.size(); i++) { + std::vector &agents = groups[i].get_agents(); + for (int j = 0; j < agents.size(); j++) { + int id = agents[j]->get_id(); + Position pos = agents[j]->get_pos(); + + if (pos.x < range_x1 || pos.x > range_x2 || pos.y < range_y1 || + pos.y > range_y2) + continue; + + if (agents[j]->get_type().can_absorb && !agents[j]->is_absorbed()) + continue; + + ret.at(ct, 0) = id; + ret.at(ct, 1) = pos.x; + ret.at(ct, 2) = pos.y; + ret.at(ct, 3) = i; + ct++; + } + } + + // the first line of ret returns counter info + ret.at(0, 0) = ct - 1; + ret.at(0, 1) = (int)render_generator.get_attack_event().size(); + } else if (strequ(name, "attack_event")) { + std::vector &attack_events = render_generator.get_attack_event(); + NDPointer ret(int_buffer, {-1, 3}); + for (int i = 0; i < attack_events.size(); i++) { + ret.at(i, 0) = attack_events[i].id; + ret.at(i, 1) = attack_events[i].x; + ret.at(i, 2) = attack_events[i].y; + } + } else if (strequ(name, "action_space")) { // int + int_buffer[0] = (int)groups[group].get_type().action_space.size(); + } else if (strequ(name, "view_space")) { // int + // the following is necessary! user can call get_view_space before reset + groups[group].get_type().n_channel = group2channel((GroupHandle)groups.size()); + int_buffer[0] = groups[group].get_type().view_range->get_height(); + int_buffer[1] = groups[group].get_type().view_range->get_width(); + int_buffer[2] = groups[group].get_type().n_channel; + } else if (strequ(name, "feature_space")) { + int_buffer[0] = get_feature_size(group); + } else if (strequ(name, "view2attack")) { + const AgentType &type = groups[group].get_type(); + const Range *range = type.attack_range; + const Range *view_range = type.view_range; + const int view_width = view_range->get_width(), view_height = view_range->get_height(); + + NDPointer ret(int_buffer, {view_height, view_width}); + memset(ret.data, -1, sizeof(int) * view_height * view_width); + int x1, y1, x2, y2; + + view_range->get_range_rela_offset(x1, y1, x2, y2); + for (int i = 0; i < range->get_count(); i++) { + int dx, dy; + range->num2delta(i, dx, dy); + //dx -= type.att_x_offset; dy -= type.att_y_offset; + + ret.at(dy - y1, dx - x1) = i; + } + } else if (strequ(name, "attack_base")) { + int_buffer[0] = groups[group].get_type().attack_base; + } else if (strequ(name, "groups_info")) { + const int colors[][3] = { + {192, 64, 64}, + {64, 64, 192}, + {64, 192, 64}, + {64, 64, 64}, + }; + NDPointer info(int_buffer, {-1, 5}); + + for (int i = 0; i < groups.size(); i++) { + info.at(i, 0) = groups[i].get_type().width; + info.at(i, 1) = groups[i].get_type().length; + info.at(i, 2) = colors[i][0]; + info.at(i, 3) = colors[i][1]; + info.at(i, 4) = colors[i][2]; + } + } else if (strequ(name, "both_attack")) { + int_buffer[0] = stat_recorder.both_attack; + } else { + LOG(FATAL) << "unsupported info name in GridWorld::get_info : " << name; + } +} + +// private utility +std::vector GridWorld::make_channel_trans( + GroupHandle group, + int base, int n_channel, int n_group) { + std::vector trans((unsigned int)n_channel); + for (int i = 0; i < base; i++) + trans[i] = i; + for (int i = 0; i < groups.size(); i++) { + int cycle_group = (group + i) % n_group; + trans[group2channel(cycle_group)] = base; + if (minimap_mode) { + base += 3; + } else { + base += 2; + } + } + return trans; +} + +int GridWorld::group2channel(GroupHandle group) { + int base = 1; + int scale = 2; + if (food_mode) + base++; + if (minimap_mode) + scale++; + + return base + group * scale; // wall + additional + (has, hp) + (has, hp) + ... +} + +int GridWorld::get_feature_size(GroupHandle group) { + // feature space layout : [embedding, last_action (one hot), last_reward] + int feature_space = embedding_size + (int)groups[group].get_type().action_space.size() + 1; + if (goal_mode) + feature_space += 2; + if (minimap_mode) // x, y coordinate + feature_space += 2; + return feature_space; +} + +/** + * render + */ +void GridWorld::render() { + if (render_generator.get_save_dir() == "___debug___") + map.render(); + else { + if (first_render) { + first_render = false; + render_generator.gen_config(groups, width, height); + } + render_generator.render_a_frame(groups, map); + } +} + +} // namespace magent +} // namespace gridworld + +// reward counter for align +// memset(counter_x, 0, width * sizeof(int)); +// memset(counter_y, 0, height * sizeof(int)); +// for (int i = 0; i < group_size; i++) { +// std::vector &agents = groups[i].get_agents(); +// size_t agent_size = agents.size(); +// #pragma omp parallel for +// for (int j = 0; j < agent_size; j++) { +// if (agents[j]->is_dead()) +// continue; +// Position pos = agents[j]->get_pos(); +// #pragma omp atomic +// counter_x[pos.x]++; +// #pragma omp atomic +// counter_y[pos.y]++; +// } +// } diff --git a/examples/battle_model/src/gridworld/GridWorld.h b/examples/battle_model/src/gridworld/GridWorld.h new file mode 100755 index 0000000..1dffab8 --- /dev/null +++ b/examples/battle_model/src/gridworld/GridWorld.h @@ -0,0 +1,338 @@ +/** + * \file GridWorld.h + * \brief core game engine of the gridworld + */ + +#ifndef MAGNET_GRIDWORLD_GRIDWORLD_H +#define MAGNET_GRIDWORLD_GRIDWORLD_H + +#include +#include +#include +#include +#include + +#include "../Environment.h" +#include "grid_def.h" +#include "Map.h" +#include "Range.h" +#include "AgentType.h" +#include "RenderGenerator.h" +#include "RewardEngine.h" + +namespace magent { +namespace gridworld { + + +// the statistical recorder +struct StatRecorder { + int both_attack; + + void reset() { + both_attack = 0; + } +}; + + +// the main engine +class GridWorld: public Environment { +public: + GridWorld(); + ~GridWorld() override; + + // game + void reset() override; + void set_config(const char *key, void *p_value) override; + + // run step + void get_observation(GroupHandle group, float **linear_buffers) override; + void set_action(GroupHandle group, const int *actions) override; + void set_chs(GroupHandle group, const int *is_chs) override; + void step(int *done) override; + void get_reward(GroupHandle group, float *buffer) override; + + // info getter + void get_info(GroupHandle group, const char *name, void *buffer) override; + + // render + void render() override; + + // special run step + void clear_dead(); + void set_goal(GroupHandle group, const char *method, const int *linear_buffer); + + // agent + void register_agent_type(const char *name, int n, const char **keys, float *values); + void new_group(const char *agent_name, GroupHandle *group); + void add_agents(GroupHandle group, int n, const char *method, + const int *pos_x, const int *pos_y, const int *pos_dir); + + // reward description + void define_agent_symbol(int no, int group, int index); + void define_event_node(int no, int op, int *inputs, int n_inputs); + void add_reward_rule(int on, int *receivers, float *values, int n_receiver, + bool is_terminal, bool auto_value); + +private: + // reward description + void init_reward_description(); + void calc_reward(); + void calc_rule(std::vector &input_symbols, std::vector &infer_obj, + RewardRule &rule, int now); + bool calc_event_node(EventNode *node, RewardRule &rule); + void collect_related_symbol(EventNode &node); + + // utility + // to make channel layout in observation symmetric to every group + std::vector make_channel_trans( + GroupHandle group, int base, int n_channel, int n_group); + int group2channel(GroupHandle group); + int get_feature_size(GroupHandle group); + + // game config + int width, height; + bool food_mode; // default = False + bool turn_mode; // default = False + bool minimap_mode; // default = False + bool goal_mode; // default = False + bool large_map_mode; // default = False + bool mean_mode; + int embedding_size; // default = 0 + + // game states : map, agent and group + Map map; + std::map agent_types; + std::vector groups; + std::default_random_engine random_engine; + + // reward description + std::vector agent_symbols; + std::vector event_nodes; + std::vector reward_rules; + bool reward_des_initialized; + + // action buffer + std::vector attack_buffer; + // split the events to small regions and boundary for parallel + int NUM_SEP_BUFFER; + std::vector *move_buffers, move_buffer_bound; + std::vector *turn_buffers, turn_buffer_bound; + + // render + RenderGenerator render_generator; + int id_counter; + bool first_render; + + // statistic recorder + StatRecorder stat_recorder; + int *counter_x, *counter_y; +}; + + +class Agent { +public: + Agent(AgentType &type, int id, GroupHandle group) : dead(false), absorbed(false), group(group), + next_reward(0), + type(type), + last_op(OP_NULL), op_obj(nullptr), index(0) { + this->id = id; + dir = Direction(rand() % 4); + hp = type.hp; + last_action = static_cast(type.action_space.size()); // dangerous here ! + next_reward = 0; + + init_reward(); + } + + Position &get_pos() { return pos; } + const Position &get_pos() const { return pos; } + void set_pos(Position pos) { this->pos = pos; } + + Direction get_dir() const { return dir; } + void set_dir(Direction dir) { this->dir = dir; } + + AgentType &get_type() { return type; } + const AgentType &get_type() const { return type; } + + int get_id() const { return id; } + void get_embedding(float *buf, int size) { + // embedding are binary form of id + if (embedding.empty()) { + int t = id; + for (int i = 0; i < size; i++, t >>= 1) { + embedding.push_back((float)(t & 1)); + } + } + memcpy(buf, &embedding[0], sizeof(float) * size); + } + + void init_reward() { + last_reward = next_reward; + last_op = OP_NULL; + next_reward = type.step_reward; + op_obj = nullptr; + be_involved = false; + } + Reward get_reward() { return next_reward; } + Reward get_last_reward() { return last_reward; } + void add_reward(Reward add) { next_reward += add; } + + void set_involved(bool value) { be_involved = value; } + bool get_involved() { return be_involved; } + + void set_action(Action act) { last_action = act; } + void set_ch(int isch){is_ch=isch;type.is_ch=is_ch;} + int get_ch() const{return is_ch;} + Action get_action() { return last_action; } + + void add_hp(float add) { hp = std::min(type.hp, hp + add); } + float get_hp() const { return hp; } + void set_hp(float value) { hp = value; } + + bool is_dead() const { return dead; } + void set_dead(bool value) { dead = value; } + bool is_absorbed() const { return absorbed; } + void set_absorbed(bool value) { absorbed = value; } + + bool starve() { + if (type.step_recover > 0) { + add_hp(type.step_recover); + } + else + be_attack(-type.step_recover); + return dead; + } + + void be_attack(float damage) { + hp -= damage; + if (hp < 0.0) { + dead = true; + next_reward = type.dead_penalty; + } + } + + GroupHandle get_group() const { return group; } + int get_index() const { return index; } + void set_index(int i) { index = i; } + + EventOp get_last_op() const { return last_op; } + void set_last_op(EventOp op){ last_op = op; } + + void *get_op_obj() const { return op_obj; } + void set_op_obj(void *obj) { op_obj = obj; } + + void get_goal(Position ¢er, int &radius) const { + center = goal; + radius = goal_radius; + } + void set_goal(Position center, int radius) { + goal = center; + goal_radius = radius; + } + +private: + int id; + bool dead; + bool absorbed; + + Position pos; + Direction dir; + float hp; + + EventOp last_op; + void *op_obj; + + Action last_action; + Reward next_reward, last_reward; + AgentType &type; + GroupHandle group; + int index; + + bool be_involved; + + std::vector embedding; + Position goal; + int goal_radius; + int is_ch; + int ch_radius=6; +}; + + +class Group { +public: + Group(AgentType &type) : type(type), dead_ct(0), next_reward(0), + center_x(0), center_y(0), recursive_base(0) { + } + + void add_agent(Agent *agent) { + agents.push_back(agent); + } + + int get_num() { return (int)agents.size(); } + int get_alive_num() { return get_num() - dead_ct; } + size_t get_size() { return agents.size(); } + + std::vector &get_agents() { return agents; } + AgentType &get_type() { return type; } + + void set_dead_ct(int ct) { dead_ct = ct; } + int get_dead_ct() const { return dead_ct; } + void inc_dead_ct() { dead_ct++; } + + void clear() { + agents.clear(); + dead_ct = 0; + } + + void init_reward() { next_reward = 0; } + Reward get_reward() { return next_reward; } + void add_reward(Reward add) { next_reward += add; } + + // use a base to eliminate duplicates in Gridworld::calc_reward + int get_recursive_base() { + return recursive_base; + } + void set_recursive_base(int base) { recursive_base = base; } + + void set_center(float cx, float cy) { center_x = cx; center_y = cy; } + void get_center(float &cx,float &cy) { cx = center_x; cy = center_y; } + void refresh_center() { + float sum_x = 0, sum_y = 0; + for (int i = 0; i < agents.size(); i++) { + sum_x += agents[i]->get_pos().x; + sum_y += agents[i]->get_pos().y; + } + center_x = sum_x / agents.size(); + center_y = sum_y / agents.size(); + } + +private: + AgentType &type; + std::vector agents; + int dead_ct; + + Reward next_reward; // group reward + float center_x, center_y; + + int recursive_base; +}; + +struct MoveAction { + Agent *agent; + int action; +}; + +struct TurnAction { + Agent *agent; + int action; +}; + +struct AttackAction { + Agent *agent; + int action; +}; + +} // namespace magent +} // namespace gridworld + +#endif //MAGNET_GRIDWORLD_GRIDWORLD_H \ No newline at end of file diff --git a/examples/battle_model/src/gridworld/Map.cc b/examples/battle_model/src/gridworld/Map.cc new file mode 100755 index 0000000..e1d7293 --- /dev/null +++ b/examples/battle_model/src/gridworld/Map.cc @@ -0,0 +1,683 @@ +/** + * \file Map.cc + * \brief The map for the game engine + */ + +#include +#include +#include +#include "Map.h" +#include "GridWorld.h" + +namespace magent { +namespace gridworld { + +inline void abs_to_rela(int c_x, int c_y, Direction dir, int abs_x, int abs_y, int &rela_x, int &rela_y); +inline void rela_to_abs(int c_x, int c_y, Direction dir, int rela_x, int rela_y, int &abs_x, int &abs_y); +inline void save_to_real(const Agent *agent, int &real_x, int &real_y); +inline void real_to_save(const Agent *agent, int real_x, int real_y, Direction new_dir, int &save_x, int &save_y); +inline void get_size_for_dir(Agent *agent, int &width, int &height); + +#define MAP_INNER_Y_ADD w + +void Map::reset(int width, int height, bool food_mode) { + this->w = width; + this->h = height; + this->food_mode = food_mode; + + if (slots != nullptr) + delete [] slots; + slots = new MapSlot[w * h]; + + if (channel_ids != nullptr) + delete [] channel_ids; + channel_ids = new int[w * h]; + + memset(channel_ids, -1, sizeof(int) * w * h); + + // init border + for (int i = 0; i < w; i++) { + add_wall(Position{i, 0}); + add_wall(Position{i, h-1}); + } + for (int i = 0; i < h; i++) { + add_wall(Position{0, i}); + add_wall(Position{w-1, i}); + } +} + +Position Map::get_random_blank(std::default_random_engine &random_engine, int width, int height) { + int tries = 0; + while (true) { + int x = (int) random_engine() % (w - width); + int y = (int) random_engine() % (h - height); + + if (is_blank_area(x, y, width, height)) { + return Position{x, y}; + } + + if (tries++ > w * h) { + LOG(FATAL) << "cannot find a blank position in a filled map"; + } + } +} + +int Map::add_agent(Agent *agent, Position pos, int width, int height, int base_channel_id) { + if (is_blank_area(pos.x, pos.y, width, height)) { + // fill in map + fill_area(pos.x, pos.y, width, height, agent, OCC_AGENT, base_channel_id); + return 0; + } else { + return 1; + } +} + +int Map::add_agent(Agent *agent, int base_channel_id) { + Direction dir = agent->get_dir(); + Position pos = agent->get_pos(); + int width = agent->get_type().width, length = agent->get_type().length; + + int m_width, m_height; + + if (dir == NORTH || dir == SOUTH) { + m_width = width; + m_height = length; + } else { + m_width = length; + m_height = width; + } + + if (is_blank_area(pos.x, pos.y, m_width, m_height)) { + // fill in map + fill_area(pos.x, pos.y, m_width, m_height, agent, OCC_AGENT, base_channel_id); + return 0; + } else { + return 1; + } +} + +void Map::remove_agent(Agent *agent) { + Position pos = agent->get_pos(); + int width, height; + get_size_for_dir(agent, width, height); + + // clear map + clear_area(pos.x, pos.y, width, height); +} + +int Map::add_wall(Position pos) { + PositionInteger pos_int = pos2int(pos); + if (slots[pos_int].slot_type == BLANK && slots[pos_int].occupier != nullptr) + return 1; + slots[pos_int].slot_type = OBSTACLE; + set_channel_id(pos_int, wall_channel_id); + return 0; +} + +void Map::average_pooling_group(float *group_buffer, int x0, int y0, int width, int height) { + for (int x = x0; x < x0 + width; x++) { + for (int y = y0; y < y0 + height; y++) { + PositionInteger pos_int = pos2int(x, y); + if (slots[pos_int].occupier != nullptr && slots[pos_int].occ_type == OCC_AGENT) { + Agent *agent = (Agent *)slots[pos_int].occupier; + group_buffer[agent->get_group()]++; + } + } + } +} + +void Map::extract_view(const Agent *agent, float *linear_buffer, const int *channel_trans, const Range *range, + int n_channel, int width, int height, int view_x_offset, int view_y_offset, + int view_left_top_x, int view_left_top_y, + int view_right_bottom_x, int view_right_bottom_y) const { + // convert coordinates between absolute map and relative view + Direction dir = agent->get_dir(); + + int agent_x, agent_y; + int eye_x, eye_y; + int x1, y1, x2, y2; + + save_to_real(agent, agent_x, agent_y); + rela_to_abs(agent_x, agent_y, dir, view_x_offset, view_y_offset, eye_x, eye_y); + rela_to_abs(eye_x, eye_y, dir, view_left_top_x, view_left_top_y, x1, y1); + rela_to_abs(eye_x, eye_y, dir, view_right_bottom_x, view_right_bottom_y, x2, y2); + + // find the coordinate of start point and end point in map + int start_x, start_y, end_x, end_y; + start_x = std::max(std::min(x1, x2), 0); + end_x = std::min(std::max(x1, x2), w - 1); + start_y = std::max(std::min(y1, y2), 0); + end_y = std::min(std::max(y1, y2), h - 1); + + NDPointer buffer(linear_buffer, {height, width, n_channel}); + + // build projection from map coordinate to view buffer coordinate + int view_x, view_y; + int view_rela_x, view_rela_y; + abs_to_rela(eye_x, eye_y, dir, start_x, start_y, view_rela_x, view_rela_y); + view_x = view_rela_x - view_left_top_x; + view_y = view_rela_y - view_left_top_y; + + int *p_view_inner, *p_view_outer; + int d_view_inner, d_view_outer; + switch (dir) { + case NORTH: + p_view_inner = &view_y; p_view_outer = &view_x; + d_view_inner = 1; d_view_outer = 1; + break; + case SOUTH: + p_view_inner = &view_y; p_view_outer = &view_x; + d_view_inner = -1; d_view_outer = -1; + break; + case EAST: + p_view_inner = &view_x; p_view_outer = &view_y; + d_view_inner = 1; d_view_outer = -1; + break; + case WEST: + p_view_inner = &view_x; p_view_outer = &view_y; + d_view_inner = -1; d_view_outer = 1; + break; + default: + LOG(FATAL) << "invalid direction in Map::extract_view"; + } + + int start_inner = *p_view_inner; + + // scan the map + for (int x = start_x; x <= end_x; x++) { + PositionInteger pos_int = pos2int(x, start_y); + for (int y = start_y; y <= end_y; y++) { + int channel_id = channel_ids[pos_int]; + + if (channel_id != -1 && range->is_in(view_y, view_x)) { + channel_id = channel_trans[channel_id]; + buffer.at(view_y, view_x, channel_id) = 1; + if (slots[pos_int].occupier != nullptr && slots[pos_int].occ_type == OCC_AGENT) { // is agent + Agent *p = ((Agent *) slots[pos_int].occupier); + buffer.at(view_y, view_x, channel_id + 1) = p->get_hp() / p->get_type().hp; // normalize hp + } + } + + *p_view_inner += d_view_inner; + pos_int += MAP_INNER_Y_ADD; + } + *p_view_inner = start_inner; + *p_view_outer += d_view_outer; + } +} + +PositionInteger Map::get_attack_obj(const AttackAction &attack, int &obj_x, int &obj_y) const { + const Agent *agent = attack.agent; + const AgentType *type = &attack.agent->get_type(); + Direction dir = agent->get_dir(); + + int att_x_offset = type->att_x_offset, att_y_offset = type->att_y_offset; + int agent_x, agent_y; + int rela_x, rela_y; +// for(int i=0;i<13;i++){ +// agent->get_type().attack_range->num2delta(i, rela_x, rela_y); +// printf("AttackAction = %d, ", i); +// printf("x is %d, y is %d\n",rela_x,rela_y); +// } +// exit(0); + agent->get_type().attack_range->num2delta(attack.action, rela_x, rela_y); +// printf("AttackAction = %d, ", attack.action); +// printf("x is %d, y is %d\n",rela_x,rela_y); + save_to_real(agent, agent_x, agent_y); + rela_to_abs(agent_x, agent_y, dir, att_x_offset + rela_x, att_y_offset + rela_y, obj_x, obj_y); + + if (!in_board(obj_x, obj_y)) { + return -1; + } + + PositionInteger pos_int = pos2int(obj_x, obj_y); + + if (slots[pos_int].occupier == nullptr) { + return -1; + } + + switch (slots[pos_int].occ_type) { + case OCC_AGENT: + { + Agent *obj = (Agent *) slots[pos_int].occupier; + + if (!type->attack_in_group && agent->get_group() == obj->get_group()) { // same type + return -1; + } else { + return pos_int; + } + break; + } + case OCC_FOOD: + return pos_int; + default: + LOG(FATAL) << "invalid occ_type in Map::get_attack_obj"; + + } + return -1; +} + +// do attack for agent, return kill_reward and dead_group +Reward Map::do_attack(Agent *agent, PositionInteger pos_int, GroupHandle &dead_group) { + // !! all the check should be done at Map::get_attack_obj + + if (slots[pos_int].occupier == nullptr) // dead + return 0.0; + + switch(slots[pos_int].occ_type) { + case OCC_AGENT: + { + Agent *obj = ((Agent *)slots[pos_int].occupier); + + obj->be_attack(agent->get_type().damage); + if (obj->is_dead()) { + agent->set_last_op(OP_KILL); + agent->set_op_obj(obj); + + // remove dead people + remove_agent(obj); + dead_group = obj->get_group(); + agent->add_hp(obj->get_type().kill_supply); + + // add food + if (food_mode) { + slots[pos_int].occ_type = OCC_FOOD; + Food *food = new Food; + *food = obj->get_type().food_supply; + slots[pos_int].occupier = food; + set_channel_id(pos_int, food_channel_id); + } + return obj->get_type().kill_reward; + } else { + agent->set_last_op(OP_ATTACK); + agent->set_op_obj(obj); + return 0.0; + } + break; + } + case OCC_FOOD: // deprecated ? + { + Food *food = (Food *)slots[pos_int].occupier; + float add = std::min(agent->get_type().eat_ability, *food); + agent->add_hp(add); + *food -= add; + if (*food < 0.1) { + slots[pos_int].occupier = nullptr; + set_channel_id(pos_int, -1); + delete food; + } + break; + } + default: + LOG(FATAL) << "invalid occ_type in Map::do_attack"; + } + + return 0.0; +} + +// do move for agent, dx = delta[0], dy = delta[1] +Reward Map::do_move(Agent *agent, const int delta[2]) { + Position &pos = agent->get_pos(); + const int new_x = pos.x + delta[0], new_y = pos.y + delta[1]; + int width, height; + get_size_for_dir(agent, width, height); + + bool blank = is_blank_area(new_x, new_y, width, height, agent); + if (blank) { + PositionInteger old_pos_int = pos2int(pos); + + // backup old + void *occupier = slots[old_pos_int].occupier; + OccupyType occ_type = slots[old_pos_int].occ_type; + int channel_id = channel_ids[old_pos_int]; + + clear_area(pos.x, pos.y, width, height); + fill_area(new_x, new_y, width, height, occupier, occ_type, channel_id); + + pos.x = new_x; + pos.y = new_y; + } else { + void *collide = get_collide(new_x, new_y, width, height, agent); + if (collide != nullptr) { + /*Agent *obj = (Agent *)collide; + if (agent->get_group() != obj->get_group()) + printf("%d %d\n", agent->get_group(), obj->get_group());*/ + + Agent *obj = (Agent *)collide; + if (obj->get_type().can_absorb) { // special condition + if (!obj->is_absorbed()) { + obj->set_absorbed(true); + obj->set_hp(obj->get_hp() * 2); + agent->set_dead(true); + remove_agent(agent); + agent->set_last_op(OP_COLLIDE); + agent->set_op_obj(collide); + } + } else { + agent->set_last_op(OP_COLLIDE); + agent->set_op_obj(collide); + } + } else { + } + } + return 0.0; +} + +// to turn, wise = 1 for clockwise, wise = -1 for counter-clockwise +Reward Map::do_turn(Agent *agent, int wise) { + int width, height; + + Position &pos = agent->get_pos(); + PositionInteger pos_int = pos2int(pos); + Direction dir = agent->get_dir(); + Direction new_dir = (Direction) ((dir + wise + DIR_NUM) % DIR_NUM); + + get_size_for_dir(agent, width, height); + + int agent_x, agent_y; + int anchor_x, anchor_y; + int new_x, new_y; + int save_x, save_y; + save_to_real(agent, agent_x, agent_y); + rela_to_abs(agent_x, agent_y, dir, agent->get_type().turn_x_offset, agent->get_type().turn_y_offset, + anchor_x, anchor_y); + + int dx = agent_x - anchor_x, dy = agent_y - anchor_y; + if (wise == -1) { // wise = 1 for clockwise, = -1 for counter-clockwise + new_x = anchor_x - dy; + new_y = anchor_y + dx; + } else{ + new_x = anchor_x + dy; + new_y = anchor_y - dx; + } + + real_to_save(agent, new_x, new_y, new_dir, save_x, save_y); + + if (is_blank_area(save_x, save_y, height, width, agent)) { + // backup old + void *occupier = slots[pos_int].occupier; + OccupyType occ_type = slots[pos_int].occ_type; + int channel_id = channel_ids[pos_int]; + + // clear old + clear_area(pos.x, pos.y, width, height); + + // fill new + agent->set_dir(new_dir); + + fill_area(save_x, save_y, height, width, occupier, occ_type, channel_id); + pos.x = save_x; pos.y = save_y; + } + return 0.0; +} + +int Map::get_align(Agent *agent) { + Position pos = agent->get_pos(); + GroupHandle group = agent->get_group(); + PositionInteger max_size = w * h; + + // scan x axis + PositionInteger pos_int = pos2int(pos); + int x_align = -1; + do { // positive direction + x_align++; + pos_int++; + } while (slots[pos_int].occupier != nullptr && // NOTE: do not check boundary, since the map has walls + ((Agent *)slots[pos_int].occupier)->get_group() == group); + + pos_int = pos2int(pos); + do { // negtive direction + x_align++; + pos_int--; + } while (slots[pos_int].occupier != nullptr && + ((Agent *)slots[pos_int].occupier)->get_group() == group); + + // scan y axis + pos_int = pos2int(pos); + int y_align = -1; + do { // positive direction + y_align++; + pos_int += MAP_INNER_Y_ADD; + } while (slots[pos_int].occupier != nullptr && + ((Agent *)slots[pos_int].occupier)->get_group() == group); + + pos_int = pos2int(pos); + do { // negtive direction + y_align++; + pos_int -= MAP_INNER_Y_ADD; + } while (slots[pos_int].occupier != nullptr && + ((Agent *)slots[pos_int].occupier)->get_group() == group); + + return std::max(x_align, y_align); +} + +/** + * Utility to operate map + */ + +// check if rectangle (x,y) - (x + width, y + height) is a blank area +// the rectangle can only contains blank slots and itself +inline bool Map::is_blank_area(int x, int y, int width, int height, void *self) { + if (x < 0 || y < 0 || x + width >= w || y + height >= h) + return false; + bool blank = true; + for (int i = 0; i < width && blank; i++) { + PositionInteger pos_int = pos2int(x + i, y); + for (int j = 0; j < height && blank; j++) { + if (slots[pos_int].slot_type != BLANK || + (slots[pos_int].occupier != nullptr && + slots[pos_int].occupier != self)) { + blank = false; + } + pos_int += MAP_INNER_Y_ADD; + } + } + return blank; +} + +// fill a rectangle (x, y) - (x + width, y + height) with specific occupier +inline void Map::fill_area(int x, int y, int width, int height, void *occupier, OccupyType occ_type, int channel_id) { + for (int i = 0; i < width; i++) { + PositionInteger pos_int = pos2int(x + i, y); + for (int j = 0; j < height; j++) { + slots[pos_int].occupier = occupier; + slots[pos_int].occ_type = occ_type; + set_channel_id(pos_int, channel_id); + pos_int += MAP_INNER_Y_ADD; + } + } +} + +// get original occupier in the rectangle who results in a collide with a move intention +inline void * Map::get_collide(int x, int y, int width, int height, void *self) { + if (x < 0 || y < 0 || x + width >= w || y + height >= h) + return nullptr; + void *collide_obj = nullptr; + for (int i = 0; i < width && collide_obj == nullptr; i++) { + PositionInteger pos_int = pos2int(x + i, y); + for (int j = 0; j < height && collide_obj == nullptr; j++) { + if (slots[pos_int].occ_type == OCC_AGENT && + slots[pos_int].occupier != self) { + collide_obj = slots[pos_int].occupier; + } + pos_int += MAP_INNER_Y_ADD; + } + } + return collide_obj; +} + +// clear a rectangle +inline void Map::clear_area(int x, int y, int width, int height) { + for (int i = 0; i < width; i++) { + PositionInteger pos_int = pos2int(x + i, y); + for (int j = 0; j < height; j++) { + slots[pos_int].occupier = nullptr; + set_channel_id(pos_int, -1); + pos_int += MAP_INNER_Y_ADD; + } + } +} + +inline void rela_to_abs(int c_x, int c_y, Direction dir, int rela_x, int rela_y, int &abs_x, int &abs_y) { + switch (dir) { + case NORTH: + abs_x = c_x + rela_x; abs_y = c_y + rela_y; + break; + case SOUTH: + abs_x = c_x - rela_x; abs_y = c_y - rela_y; + break; + case WEST: + abs_x = c_x + rela_y; abs_y = c_y - rela_x; + break; + case EAST: + abs_x = c_x - rela_y; abs_y = c_y + rela_x; + break; + default: + LOG(FATAL) << "invalid direction in rela_to_abs"; + } +} + +inline void abs_to_rela(int c_x, int c_y, Direction dir, int abs_x, int abs_y, int &rela_x, int &rela_y) { + switch (dir) { + case NORTH: + rela_x = abs_x - c_x; rela_y = abs_y - c_y; + break; + case SOUTH: + rela_x = c_x - abs_x; rela_y = c_y - abs_y; + break; + case WEST: + rela_y = abs_x - c_x; rela_x = c_y - abs_y; + break; + case EAST: + rela_y = c_x - abs_x; rela_x = abs_y - c_y; + break; + default: + LOG(FATAL) << "invalid direction in abs_to_rela"; + } +} + +inline void save_to_real(const Agent *agent, int &real_x, int &real_y) { + Direction dir = agent->get_dir(); + Position pos = agent->get_pos(); + int width = agent->get_type().width, length = agent->get_type().length; + + switch(dir) { + case NORTH: + real_x = pos.x; real_y = pos.y; + break; + case SOUTH: + real_x = pos.x + width - 1; real_y = pos.y + length - 1; + break; + case WEST: + real_x = pos.x; real_y = pos.y + width - 1; + break; + case EAST: + real_x = pos.x + length - 1; real_y = pos.y; + break; + default: + LOG(FATAL) << "invalid direction in save_to_real"; + } +} + +inline void real_to_save(const Agent *agent, int real_x, int real_y, Direction new_dir, int &save_x, int &save_y) { + int width = agent->get_type().width, length = agent->get_type().length; + + switch(new_dir) { + case NORTH: + save_x = real_x; save_y = real_y; + break; + case SOUTH: + save_x = real_x - width + 1; save_y = real_y - length + 1; + break; + case WEST: + save_x = real_x; save_y = real_y - width + 1; + break; + case EAST: + save_x = real_x - length + 1; save_y = real_y; + break; + default: + LOG(FATAL) << "invalid direction in real_to_save"; + } +} + +inline void get_size_for_dir(Agent *agent, int &width, int &height) { + Direction dir = agent->get_dir(); + + if (dir == NORTH || dir == SOUTH) { + width = agent->get_type().width; + height = agent->get_type().length; + } else { + width = agent->get_type().length; + height = agent->get_type().width; + } +} + +void Map::get_wall(std::vector &walls) const { + for (int i = 0; i < w * h; i++) { + if (slots[i].slot_type == OBSTACLE) { + walls.push_back(int2pos(i)); + } + } +} + +/** + * Render for debug, print the map to terminal screen + */ +void Map::render() { + for (int x = 0; x < w; x++) + printf("="); puts(""); + printf(" "); + for (int x = 0; x < w; x++) + printf("%2d ", x); puts(""); + + for (int y = 0; y < h; y++) { + printf("%2d ", y); + for (int x = 0; x < w; x++) { + MapSlot s = slots[pos2int(x, y)]; + char buf[4] = {0, 0, 0}; + + switch (s.slot_type) { + case BLANK: + if (s.occupier == nullptr) { + buf[0] = ' '; + } else { + switch (s.occ_type) { + case OCC_AGENT: { + Agent &agent = *((Agent *)s.occupier); + switch (agent.get_dir()) { + case EAST: buf[0] = '>'; break; + case WEST: buf[0] = '<'; break; + case NORTH: buf[0] = '^'; break; + case SOUTH: buf[0] = 'v'; break; + default: + LOG(FATAL) << "invalid direction in Map::render"; + } + buf[1] = (char)toupper(agent.get_type().name[0]); + } + break; + case OCC_FOOD: + buf[0] = '+'; + break; + default: + LOG(FATAL) << "invalid occ type in Map::render"; + } + } + break; + case OBSTACLE: + buf[0] = '#'; + break; + default: + LOG(FATAL) << "invalid slot type in Map::render"; + + } + printf("%3s", buf); + } + printf("\n"); + } + + for (int x = 0; x < w; x++) + printf("="); puts("\n"); +} + +} // namespace magent +} // namespace gridworld diff --git a/examples/battle_model/src/gridworld/Map.h b/examples/battle_model/src/gridworld/Map.h new file mode 100755 index 0000000..123d583 --- /dev/null +++ b/examples/battle_model/src/gridworld/Map.h @@ -0,0 +1,115 @@ +/** + * \file Map.h + * \brief The map for the game engine + */ + +#ifndef MAGNET_GRIDWORLD_MAP_H +#define MAGNET_GRIDWORLD_MAP_H + +#include +#include +#include "grid_def.h" +#include "../Environment.h" +#include "Range.h" + +namespace magent { +namespace gridworld { + +typedef enum {BLANK, OBSTACLE} SlotType; +typedef enum {OCC_AGENT, OCC_FOOD} OccupyType; + +typedef float Food; + +class MapSlot { +public: + MapSlot() : slot_type(BLANK), occupier(nullptr) {} + SlotType slot_type; + OccupyType occ_type; + void *occupier; +}; + + +class Map { +public: + Map(): slots(nullptr), channel_ids(nullptr), w(-1), h(-1), + wall_channel_id(0), food_channel_id(1) { + } + + ~Map() { + delete [] slots; + delete [] channel_ids; + } + + void reset(int width, int height, bool food_mode); + + Position get_random_blank(std::default_random_engine &random_engine, int width=1, int height=1); + + + int add_agent(Agent *agent, Position pos, int width, int height, int base_channel_id); + int add_agent(Agent *agent, int base_channel_id); + + int add_wall(Position pos); + void remove_agent(Agent *agent); + + void average_pooling_group(float *group_buffer, int x0, int y0, int width, int height); + void extract_view(const Agent *agent, float *linear_buffer, const int *channel_trans, const Range *range, + int n_channel, int width, int height, int view_x_offset, int view_y_offset, + int view_left_top_x, int view_left_top_y, + int view_right_bottom_x, int view_right_bottom_y) const; + + PositionInteger get_attack_obj(const AttackAction &attack, int &obj_x, int &obj_y) const; + Reward do_attack(Agent *agent, PositionInteger pos_int, GroupHandle &dead_group); + + Reward do_move(Agent *agent, const int delta[2]); + Reward do_turn(Agent *agent, int wise); + + int get_align(Agent *agent); + + void render(); + void get_wall(std::vector &walls) const; + +private: + MapSlot* slots; + int *channel_ids; // channel_id is supposed to be a member of MapSlot, extract it out from MapSlot for faster access of memory + int w, h; + const int wall_channel_id, food_channel_id; + bool food_mode; + + /** + * Utility + */ + bool in_board(int x, int y) const { + return x >= 0 && x < w && y >= 0 && y < h; + } + + PositionInteger pos2int(Position pos) const { + return pos2int(pos.x, pos.y); + } + + PositionInteger pos2int(int x, int y) const { + //return (PositionInteger)x * h + y; + return (PositionInteger)y * w + x; + } + + Position int2pos(PositionInteger pos) const { + //return Position{(int)(pos / h), (int)(pos % h)}; + return Position{(int)(pos % w), (int)(pos / w)}; + } + + void set_channel_id(PositionInteger pos, int id) { + channel_ids[pos] = id; + } + + void dfs(std::default_random_engine &random_engine, int x, int y, int thick, int mode); + + inline bool is_blank_area(int x, int y, int width, int height, void *self = nullptr); + inline void clear_area(int x, int y, int width, int height); + inline void fill_area(int x, int y, int width, int height, + void *occupier, OccupyType occ_type, int channel_id); + inline void *get_collide(int x, int y, int width, int height, void *self); +}; + +} // namespace gridworld +} // namespace magent + +#endif //MAGNET_GRIDWORLD_MAP_H diff --git a/examples/battle_model/src/gridworld/Range.h b/examples/battle_model/src/gridworld/Range.h new file mode 100755 index 0000000..b383f77 --- /dev/null +++ b/examples/battle_model/src/gridworld/Range.h @@ -0,0 +1,199 @@ +/** + * \file Range.h + * \brief View and attack range in GridWorld + */ + +#ifndef MAGNET_GRIDWORLD_RANGE_H +#define MAGNET_GRIDWORLD_RANGE_H + +#include +#include +#include + +namespace magent { +namespace gridworld { + +static const double PI = 3.1415926536; + +class Range { +public: + Range() : width(-1), height(-1), count(0) { + is_in_range = nullptr; + dx = dy = nullptr; + } + + Range(const Range &other) : width(other.width), height(other.height), count(other.count) { + is_in_range = new bool[width * height]; + dx = new int[width * height]; + dy = new int[width * height]; + + memcpy(is_in_range, other.is_in_range, sizeof(bool) * width * height); + memcpy(dx, other.dx, sizeof(bool) * width * height); + memcpy(dy, other.dy, sizeof(bool) * width * height); + } + + ~Range() { + if (is_in_range != nullptr) + delete [] is_in_range; + if (dx != nullptr) + delete [] dx; + if (dy != nullptr) + delete [] dy; + } + + bool is_in(int row, int col) const { + return is_in_range[row * width + col]; + } + + int get_width() const { return width; } + int get_height() const { return height; } + + void get_range_rela_offset(int &x1, int &y1, int &x2, int &y2) const { + x1 = this->x1; y1 = this->y1; + x2 = this->x2; y2 = this->y2; + } + int add_rela_offset(int x_off, int y_off) { + x1 += x_off; x2 += x_off; + y1 += y_off; y2 += y_off; + return -1; + } + + int get_count() const { return count; } + void num2delta(int n, int &dx, int &dy) const { + // do not check boundary + dx = this->dx[n]; + dy = this->dy[n]; +// for (int i = 0; i < width * height; i++){ +// printf("dir %d: (%d, %d)\n", i, this->dx[i], this->dy[i]); +// } +// exit(0); + } + + void print_self() { + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + if (is_in_range[i * width + j]) { + printf("1"); + } else + printf("0"); + } + printf("\n"); + } + int ct = 0; + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + if (is_in_range[i * width + j]) { + printf("%2d,%2d ", dx[ct], dy[ct]); + ct++; + } else { + printf("%2d,%2d ", 0, 0); + } + } + printf("\n"); + } + printf("\n"); + } + +protected: + int width, height; + int count; + int x1, y1, x2, y2; + bool *is_in_range; + int *dx; + int *dy; +}; + +// sector range +// stored as a rectangle with sector mask +class SectorRange : public Range { +public: + SectorRange(float angle, float radius, int parity) { + height = (int)(radius + 0.5); + width = (int)(2 * radius * sin(angle / 2 * (PI / 180)) + 0.5); + if (width % 2 != parity) { // fit to parity, pick ceil + width--; + } + + is_in_range = new bool[width * height]; + dx = new int[width * height]; + dy = new int[width * height]; + + const double eps = 0.00001; + + count = 0; + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + double dis_x, dis_y; + + dis_x = std::fabs(j - (width-1)/2.0); + dis_y = std::fabs(height - i); + + double dis = sqrt(dis_x * dis_x + dis_y * dis_y); + + if (dis < radius + 0.2 + eps && dis_x / dis_y + < tan(angle / 2 * PI / 180) + eps) { + is_in_range[i * width + j] = true; + dx[count] = j - width/2; + dy[count] = i - height; + count++; + } else { + is_in_range[i * width + j] = false; + } + } + } + + x1 = -width / 2; y1 = -height; + x2 = (width-1) / 2; y2 = -1; + } +}; + + +// circle range +// stored as a rectangle with circular mask +class CircleRange : public Range { +public: + CircleRange (float radius, float inner_radius, int parity) { + const double eps = 1e-8; + + width = (2 * int(radius + eps) + parity); + int center = (int)(radius); + + if (width % 2 != parity) { // fit to parity, pick ceil + width++; + } + height = width; + + is_in_range = new bool[width * width]; + dx = new int[width * width]; + dy = new int[width * width]; + + count = 0; + double delta = (parity == 0 ? 0.5 : 0); + for (int i = 0; i < width; i++) { + for (int j = 0; j < width; j++) { + double dis_x = fabs(j - center + delta); + double dis_y = fabs(i - center + delta); + double dis = sqrt(dis_x * dis_x + dis_y * dis_y); + + if (dis < radius + eps) { + if (dis > inner_radius - eps) { // if inc_center is false, exclude the center + is_in_range[i * width + j] = true; + dx[count] = j - center; + dy[count] = i - center; + count++; + } + } else { + is_in_range[i * width + j] = false; + } + } + } + + x1 = y1 = -center; + x2 = y2 = width - center - 1; + } +}; + +} // namespace gridworld +} // namespace magent + +#endif //MAGNET_GRIDWORLD_RANGE_H diff --git a/examples/battle_model/src/gridworld/RenderGenerator.cc b/examples/battle_model/src/gridworld/RenderGenerator.cc new file mode 100755 index 0000000..e97b95a --- /dev/null +++ b/examples/battle_model/src/gridworld/RenderGenerator.cc @@ -0,0 +1,188 @@ +/** + * \file RenderGenerator.cc + * \brief Generate data for render + */ + +#include +#include +#include + +#include "RenderGenerator.h" +#include "GridWorld.h" + +namespace magent { +namespace gridworld { + +RenderGenerator::RenderGenerator() { + save_dir = ""; + file_ct = frame_ct = 0; + frame_per_file = 10000; +} + +void RenderGenerator::next_file() { + file_ct++; + frame_ct = 0; +} + +void RenderGenerator::set_attack_event(std::vector &attack_events) { + this->attack_events = attack_events; +} + +std::vector &RenderGenerator::get_attack_event() { + return this->attack_events; +} + +void RenderGenerator::set_render(const char *key, const char *value) { + if (strequ(key, "save_dir")) + save_dir = std::string(value); + else if (strequ(key, "frame_per_file")) + sscanf(value, "%d", &frame_per_file); +} + +template +void print_json(std::ofstream &os, const char *key, T value, bool last=false) { + os << "\"" << key << "\": " << value; + if (last) + os << std::endl; + else + os << "," << std::endl; +} + +std::string rgba_string(int r, int g, int b, float alpha) { + std::stringstream ss; + ss << "\"rgba(" << r << "," << g << "," << b << "," << alpha << ")\""; + return ss.str(); +}; + +void RenderGenerator::gen_config(std::vector &group, int w, int h) { + /***** config *****/ + std::ofstream f_config(save_dir + "/" + "config.json"); + + int colors[][3] = { + {192, 64, 64}, + {64, 64, 192}, + {64, 192, 64}, + {64, 64, 64}, + }; + + f_config << "{" << std::endl; + print_json(f_config, "width", w); + print_json(f_config, "height", h); + print_json(f_config, "static-file", "\"static.map\""); + print_json(f_config, "obstacle-style", rgba_string(127, 127, 127, 1)); + print_json(f_config, "dynamic-file-directory", "\".\""); + print_json(f_config, "attack-style", rgba_string(63, 63, 63, 0.8)); + print_json(f_config, "minimap-width", 300); + print_json(f_config, "minimap-height", 250); + + // groups + f_config << "\"group\" : [" << std::endl; + for (int i = 0; i < group.size(); i++) { + AgentType &type = group[i].get_type(); + f_config << "{" << std::endl; + + print_json(f_config, "height", type.length); + print_json(f_config, "width", type.width); + print_json(f_config, "style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 1)); + print_json(f_config, "anchor", "[0, 0]"); + print_json(f_config, "max-speed", (int)type.speed); + print_json(f_config, "speed-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.01)); + print_json(f_config, "vision-radius", type.view_radius); + print_json(f_config, "vision-angle", type.view_angle); + print_json(f_config, "vision-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.2)); + print_json(f_config, "attack-radius", type.attack_radius); + print_json(f_config, "attack-angle", type.attack_angle); + print_json(f_config, "attack-style", rgba_string(colors[i][0], colors[i][1], colors[i][2], 0.1)); + print_json(f_config, "broadcast-radius", 1, true); + + if (i == group.size() - 1) + f_config << "}" << std::endl; + else + f_config << "}," << std::endl; + } + f_config << "]" << std::endl; + f_config << "}" << std::endl; +} + + +void RenderGenerator::render_a_frame(std::vector &groups, const Map &map) { + if (save_dir == "") { + return; + } + + std::string filename = save_dir + "/" + "video_" + std::to_string(file_ct) + ".txt"; + std::ofstream fout(filename.c_str(), frame_ct == 0 ? std::ios::out : std::ios::app); + + // walls + if (frame_ct ==0) { + std::vector walls; + map.get_wall(walls); + + // walls + fout << "W" << " " << walls.size() << std::endl; + for (int i = 0; i < walls.size(); i++) { + fout << walls[i].x << " " << walls[i].y << std::endl; + } + } + + // count agents + int num_agents = 0; + for (int i = 0; i < groups.size(); i++) { + num_agents += groups[i].get_agents().size(); + bool can_absorb = groups[i].get_type().can_absorb; + if (can_absorb) { + const std::vector &agents = groups[i].get_agents(); + for (int j = 0; j < agents.size(); j++) { + if (!agents[j]->is_absorbed()) { + num_agents--; + } + } + } + } + int num_attacks = (int)attack_events.size(); + + // frame info + fout << "F" << " " << num_agents << " " << num_attacks << " " << 0 << std::endl; + + // agent + const int dir2angle[] = {0, 90, 180, 270}; + for (int i = 0; i < groups.size(); i++) { + const std::vector &agents = groups[i].get_agents(); + + bool can_absorb = agents[0]->get_type().can_absorb; + // when type.can_absorb is true, do not render it if it is not absorbed + + for (int j = 0; j < agents.size(); j++) { + const Agent &agent = *agents[j]; + + if (can_absorb && !agent.is_absorbed()) + continue; + + Position pos = agent.get_pos(); + int id = agent.get_id(); + int hp = std::max(0, int(100 * agent.get_hp() / agent.get_type().hp)); + hp = std::min(hp, 100); + int dir = dir2angle[(int)agent.get_dir()]; + int is_ch = agent.get_ch(); + fout << id << " " << hp << " " << dir << " " << pos.x << " " << pos.y << " " << i << " " << is_ch< frame_per_file) { + frame_ct = 0; + file_ct++; + } +} + +} // namespace gridworld +} // namespace magent diff --git a/examples/battle_model/src/gridworld/RenderGenerator.h b/examples/battle_model/src/gridworld/RenderGenerator.h new file mode 100755 index 0000000..6776e39 --- /dev/null +++ b/examples/battle_model/src/gridworld/RenderGenerator.h @@ -0,0 +1,56 @@ +/** + * \file RenderGenerator.h + * \brief Generate data for render + */ + +#ifndef MAGNET_GRIDWORLD_RENDER_H +#define MAGNET_GRIDWORLD_RENDER_H + +#include +#include + +#include "grid_def.h" +#include "Map.h" + +namespace magent { +namespace gridworld { + +struct RenderAttackEvent{ + int id; + int x, y; +}; + +class RenderGenerator { +public: + RenderGenerator(); + + // move to next file + void next_file(); + + // getter and setter for attack_events + void set_attack_event(std::vector &attack_events); + std::vector &get_attack_event(); + + void set_render(const char *key, const char *value); + void gen_config(std::vector &group, int w, int h); + + void render_a_frame(std::vector &groups, const Map &map); + + std::string get_save_dir() { + return save_dir; + } + +private: + std::string save_dir; + + int file_ct; + int frame_ct; + int frame_per_file; + + std::vector attack_events; +}; + +} // namespace gridworld +} // namespace magent + +#endif //MAGNET_GRIDWORLD_RENDER_H diff --git a/examples/battle_model/src/gridworld/RewardEngine.cc b/examples/battle_model/src/gridworld/RewardEngine.cc new file mode 100755 index 0000000..912e412 --- /dev/null +++ b/examples/battle_model/src/gridworld/RewardEngine.cc @@ -0,0 +1,448 @@ +/** + * \file reward_description.cc + * \brief implementation of reward description + */ + +#include "assert.h" + +#include "RewardEngine.h" +#include "GridWorld.h" + +namespace magent { +namespace gridworld { + +bool AgentSymbol::bind_with_check(void *entity) { + // bind agent symbol to entity with correctness check + Agent *agent = (Agent *)entity; + if (group != agent->get_group()) + return false; + if (index != -1 && index != agent->get_index()) + return false; + this->entity = agent; + return true; +} + +/** + * some interface for python bind + */ +void GridWorld::define_agent_symbol(int no, int group, int index) { +// LOG(TRACE) << "define agent symbol %d (group=%d index=%d)\n", no, group, index); + if (no >= agent_symbols.size()) { + agent_symbols.resize((unsigned)no + 1); + } + agent_symbols[no].group = group; + agent_symbols[no].index = index; +} + +void GridWorld::define_event_node(int no, int op, int *inputs, int n_inputs) { +// TRACE_PRINT("define event node %d op=%d inputs=[", no, op); +// for (int i = 0; i < n_inputs; i++) +// TRACE_PRINT("%d, ", inputs[i]); +// TRACE_PRINT("]\n"); + if (no >= event_nodes.size()) { + event_nodes.resize((unsigned)no + 1); + } + + event_nodes[no].op = (EventOp)op; // be careful here + for (int i = 0; i < n_inputs; i++) + event_nodes[no].raw_parameter.push_back(inputs[i]); +} + +void GridWorld::add_reward_rule(int on, int *receivers, float *values, int n_receiver, + bool is_terminal, bool auto_value) { +// TRACE_PRINT("define rule on=%d rec,val=[", on); +// for (int i = 0; i < n_receiver; i++) +// TRACE_PRINT("%d %g, ", receiver[i], value[i]); +// TRACE_PRINT("]\n"); + + RewardRule rule; + + rule.raw_parameter.push_back(on); + for (int i = 0; i < n_receiver; i++) { + rule.raw_parameter.push_back(receivers[i]); + rule.values.push_back(values[i]); + } + rule.is_terminal = is_terminal; + rule.auto_value = auto_value; + + reward_rules.push_back(rule); +} + +void GridWorld::collect_related_symbol(EventNode &node) { + switch (node.op) { + // Logic operation + case OP_AND: case OP_OR: + collect_related_symbol(*node.node_input[0]); + collect_related_symbol(*node.node_input[1]); + // union related symbols + node.related_symbols.insert(node.node_input[0]->related_symbols.begin(), node.node_input[0]->related_symbols.end()); + node.related_symbols.insert(node.node_input[1]->related_symbols.begin(), node.node_input[1]->related_symbols.end()); + // union infer map + node.infer_map.insert(node.node_input[0]->infer_map.begin(), node.node_input[0]->infer_map.end()); + node.infer_map.insert(node.node_input[1]->infer_map.begin(), node.node_input[1]->infer_map.end()); + break; + case OP_NOT: + collect_related_symbol(*node.node_input[0]); + // union related symbols + node.related_symbols.insert(node.node_input[0]->related_symbols.begin(), node.node_input[0]->related_symbols.end()); + // union infer map + node.infer_map.insert(node.node_input[0]->infer_map.begin(), node.node_input[0]->infer_map.end()); + break; + // Binary-agent operation + case OP_KILL: case OP_COLLIDE: case OP_ATTACK: + node.related_symbols.insert(node.symbol_input[0]); + node.related_symbols.insert(node.symbol_input[1]); + break; + // Unary-agent operation + case OP_AT: case OP_IN: case OP_DIE: case OP_IN_A_LINE: case OP_ALIGN: + node.related_symbols.insert(node.symbol_input[0]); + break; + default: + LOG(FATAL) << "invalid event op in GridWorld::collect_related_symbol"; + } +} + +void GridWorld::init_reward_description() { + // from serial data to pointer + for (int i = 0; i < event_nodes.size(); i++) { + EventNode &node = event_nodes[i]; + switch (node.op) { + case OP_AND: case OP_OR: + node.node_input.push_back(&event_nodes[node.raw_parameter[0]]); + node.node_input.push_back(&event_nodes[node.raw_parameter[1]]); + break; + case OP_NOT: + node.node_input.push_back(&event_nodes[node.raw_parameter[0]]); + break; + case OP_KILL: case OP_COLLIDE: case OP_ATTACK: + node.symbol_input.push_back(&agent_symbols[node.raw_parameter[0]]); + node.symbol_input.push_back(&agent_symbols[node.raw_parameter[1]]); + node.infer_map.insert(std::make_pair(node.symbol_input[0], node.symbol_input[1])); + break; + case OP_AT: + node.symbol_input.push_back(&agent_symbols[node.raw_parameter[0]]); + node.int_input.push_back(node.raw_parameter[1]); + node.int_input.push_back(node.raw_parameter[2]); + break; + case OP_IN: + node.symbol_input.push_back(&agent_symbols[node.raw_parameter[0]]); + node.int_input.push_back(node.raw_parameter[1]); + node.int_input.push_back(node.raw_parameter[2]); + node.int_input.push_back(node.raw_parameter[3]); + node.int_input.push_back(node.raw_parameter[4]); + break; + case OP_DIE: case OP_IN_A_LINE: case OP_ALIGN: + node.symbol_input.push_back(&agent_symbols[node.raw_parameter[0]]); + break; + default: + LOG(FATAL) << "invalid event op in GridWorld::init_reward_description"; + } + } + + for (int i = 0; i < reward_rules.size(); i++) { + RewardRule &rule = reward_rules[i]; + rule.on = &event_nodes[rule.raw_parameter[0]]; + for (int j = 1; j < rule.raw_parameter.size(); j++) { + rule.receivers.push_back(&agent_symbols[rule.raw_parameter[j]]); + } + } + + // calc related symbols + for (int i = 0; i < event_nodes.size(); i++) { + collect_related_symbol(event_nodes[i]); + } + + // for every reward rule, find all the input and build inference graph + for (int i = 0; i < reward_rules.size(); i++) { + std::vector input_symbols; + std::vector infer_obj; + std::set added; + + EventNode &on = *reward_rules[i].on; + // first pass, scan to find infer pair, add them + for (auto sub_iter = on.related_symbols.begin(); sub_iter != on.related_symbols.end(); + sub_iter++) { + if (added.find(*sub_iter) != added.end()) // already be inferred + continue; + + auto obj_iter = on.infer_map.find(*sub_iter); + if (obj_iter != on.infer_map.end()) { // can infer object + input_symbols.push_back(*sub_iter); + infer_obj.push_back(obj_iter->second); + + added.insert(*sub_iter); + added.insert(obj_iter->second); + } + } + + // second pass, add remaining symbols + for (auto sub_iter = on.related_symbols.begin(); sub_iter != on.related_symbols.end(); + sub_iter++) { + if (added.find(*sub_iter) == added.end()) { + input_symbols.push_back(*sub_iter); + infer_obj.push_back(nullptr); + } + } + + reward_rules[i].input_symbols = input_symbols; + reward_rules[i].infer_obj = infer_obj; + } + + /** + * semantic check + * 1. the object of attack, collide, kill cannot be a group + * 2. any non-deterministic receiver must be involved in the triggering event + */ + for (int i = 0; i < reward_rules.size(); i++) { + // TODO: omitted temporally + } + + // print rules for debug + /*for (int i = 0; i < reward_rules.size(); i++) { + printf("on: %d\n", (int)reward_rules[i].on->op); + printf("input symbols: "); + for (int j = 0; j < reward_rules[i].input_symbols.size(); j++) { + printf("(%d,%d) ", reward_rules[i].input_symbols[j]->group, + reward_rules[i].input_symbols[j]->index); + if (reward_rules[i].infer_obj[j] != nullptr) { + printf("-> (%d,%d) ", reward_rules[i].infer_obj[j]->group, + reward_rules[i].infer_obj[j]->index); + } + } + printf("\n"); + }*/ +} + +bool GridWorld::calc_event_node(EventNode *node, RewardRule &rule) { + bool ret; + switch (node->op) { + case OP_ATTACK: case OP_KILL: case OP_COLLIDE: { + Agent *sub, *obj; + // object must be an agent, cannot be a group ! + assert(!node->symbol_input[1]->is_all()); + obj = (Agent *)node->symbol_input[1]->entity; + if (node->symbol_input[0]->is_all()) { + const std::vector &agents = groups[node->symbol_input[0]->group].get_agents(); + ret = true; + for (int i = 0; i < agents.size(); i++) { + sub = agents[i]; + + if (!(sub->get_last_op() == node->op && sub->get_op_obj() == obj)) { + ret = false; + break; + } + } + } else { + sub = (Agent *)node->symbol_input[0]->entity; + ret = sub->get_last_op() == node->op && sub->get_op_obj() == obj; + } + } + break; + case OP_ALIGN: { + // subject must be an agent, cannot be a group! + assert(!node->symbol_input[0]->is_all()); + + Agent *sub = (Agent *)node->symbol_input[0]->entity; + + // int align = map.get_align(sub); + Position pos = sub->get_pos(); + assert(pos.x < width && pos.y < height); + int align = counter_x[pos.x] + counter_y[pos.y]; + + if (rule.auto_value) { + assert(rule.values.size() == 1); + rule.values[0] = align - 1; + ret = true; + } else { + ret = align > 1; + } + } + break; + case OP_IN_A_LINE: { + assert(node->symbol_input[0]->is_all()); + std::vector &agents = groups[node->symbol_input[0]->group].get_agents(); + if (agents.size() < 2) { + ret = true; + } else { + ret = false; + // check if they are in a line, condition : 1. x is the same 2. max_y - min_y + 1 = #agent + int dx, dy; + dx = agents[0]->get_pos().x - agents[1]->get_pos().x; + dy = agents[0]->get_pos().y - agents[1]->get_pos().y; + bool in_line = true; + if (dx == 0 && dy != 0) { + int min_y, max_y; + int base_x = agents[0]->get_pos().x; + min_y = max_y = agents[0]->get_pos().y; + for (int i = 1; i < agents.size() && in_line; i++) { + Position pos = agents[i]->get_pos(); + min_y = std::min(pos.y, min_y); max_y = std::max(pos.y, max_y); + in_line = (base_x == pos.x); + } + ret = in_line && max_y - min_y + 1 == agents.size(); + } else if (dx != 0 && dy == 0) { + int min_x, max_x; + int base_y = agents[0]->get_pos().y; + min_x = max_x = agents[0]->get_pos().x; + for (int i = 1; i < agents.size() && in_line; i++) { + Position pos = agents[i]->get_pos(); + min_x = std::min(pos.x, min_x); max_x = std::max(pos.x, max_x); + in_line = (base_y == pos.y); + } + ret = in_line && max_x - min_x + 1 == agents.size(); + } + } + } + break; + case OP_AT: { + std::vector &int_input = node->int_input; + if (node->symbol_input[0]->is_all()) { + const std::vector &agents = groups[node->symbol_input[0]->group].get_agents(); + ret = true; + for (int i = 0; i < agents.size(); i++) { + Position pos = agents[i]->get_pos(); + if (!(pos.x == int_input[0] && pos.y == int_input[1])) { + ret = false; + break; + } + } + } else { + Agent *sub = (Agent *)node->symbol_input[0]->entity; + Position pos = sub->get_pos(); + ret = (pos.x == int_input[0] && pos.y == int_input[1]); + } + } + break; + case OP_IN: { + std::vector &int_input = node->int_input; + if (node->symbol_input[0]->is_all()) { + const std::vector &agents = groups[node->symbol_input[0]->group].get_agents(); + ret = true; + for (int i = 0; i < agents.size(); i++) { + Position pos = agents[i]->get_pos(); + if (!((pos.x > int_input[0] && pos.x < int_input[2] + && pos.y > int_input[1] && pos.y < int_input[3]))) { + ret = false; + break; + } + } + } else { + Agent *sub = (Agent *)node->symbol_input[0]->entity; + Position pos = sub->get_pos(); + ret = ((pos.x > int_input[0] && pos.x < int_input[2] + && pos.y > int_input[1] && pos.y < int_input[3])); + } + } + break; + case OP_DIE: { + if (node->symbol_input[0]->is_all()) { + const std::vector &agents = groups[node->symbol_input[0]->group].get_agents(); + ret = true; + for (int i = 0; i < agents.size(); i++) { + if (!agents[i]->is_dead()) { + ret = false; + break; + } + } + } else { + Agent *sub = (Agent *)node->symbol_input[0]->entity; + ret = sub->is_dead(); + } + } + break; + + case OP_AND: + ret = calc_event_node(node->node_input[0], rule) + && calc_event_node(node->node_input[1], rule); + break; + case OP_OR: + ret = calc_event_node(node->node_input[0], rule) + || calc_event_node(node->node_input[1], rule); + break; + case OP_NOT: + ret = !calc_event_node(node->node_input[0], rule); + break; + + default: + LOG(FATAL) << "invalid op of EventNode in GridWorld::calc_event_node"; + } + + return ret; +} + +void GridWorld::calc_rule(std::vector &input_symbols, + std::vector &infer_obj, + RewardRule &rule, int now) { + if (now == input_symbols.size()) { // DFS last layer + if (calc_event_node(rule.on, rule)) { // if it is true, assign reward + rule.trigger = true; + const std::vector &receivers = rule.receivers; + for (int i = 0; i < receivers.size(); i++) { + AgentSymbol *sym = receivers[i]; + if (sym->is_all()) { + groups[sym->group].add_reward(rule.values[i]); + } else { + Agent* agent = (Agent *)sym->entity; + agent->add_reward(rule.values[i]); + } + } + } + } else { // scan every possible permutation + AgentSymbol *sym = input_symbols[now]; + if (sym->is_any()) { + const std::vector &agents = groups[sym->group].get_agents(); + int base = groups[sym->group].get_recursive_base() + 1; + base = 0; + + for (int i = base; i < agents.size(); i++) { + groups[sym->group].set_recursive_base(i); + sym->entity = (void *)agents[i]; + + if (agents[i]->get_involved()) + continue; + agents[i]->set_involved(true); + + if (infer_obj[now] != nullptr) { // can infer in this step + void *entity = agents[i]->get_op_obj(); + if (entity != nullptr && infer_obj[now]->bind_with_check(entity)) { + calc_rule(input_symbols, infer_obj, rule, now + 1); + } + } else { + calc_rule(input_symbols, infer_obj, rule, now + 1); + } + agents[i]->set_involved(false); + } + } else if (sym->is_all()) { + if (infer_obj[now] != nullptr) { + Group &g = groups[sym->group]; + if (g.get_agents().size() > 0) { + void *entity = g.get_agents()[0]->get_op_obj(); // pick first agent to infer + if (entity != nullptr && infer_obj[now]->bind_with_check(entity)) { + calc_rule(input_symbols, infer_obj, rule, now + 1); + } + } + } else { + calc_rule(input_symbols, infer_obj, rule, now + 1); + } + } else { // deterministic ID + Group &g = groups[sym->group]; + if (sym->index < g.get_size()) { + Agent *agent = g.get_agents()[sym->index]; + sym->entity = (void *)agent; + + if (infer_obj[now] != nullptr) { + if (agent->get_op_obj() != nullptr) { + if (infer_obj[now]->bind_with_check(agent->get_op_obj())) { + calc_rule(input_symbols, infer_obj, rule, now + 1); + } + } + } + } + } + } +} + + + +} // namespace gridworld +} // namespace magent \ No newline at end of file diff --git a/examples/battle_model/src/gridworld/RewardEngine.h b/examples/battle_model/src/gridworld/RewardEngine.h new file mode 100755 index 0000000..6ec5f47 --- /dev/null +++ b/examples/battle_model/src/gridworld/RewardEngine.h @@ -0,0 +1,66 @@ +/** + * \file reward_description.h + * \brief Data structure for reward description + */ + +#ifndef MAGNET_GRIDWORLD_REWARD_DESCRIPTION_H +#define MAGNET_GRIDWORLD_REWARD_DESCRIPTION_H + +#include +#include +#include +#include "grid_def.h" + +namespace magent { +namespace gridworld { + +class AgentSymbol { +public: + int group; + int index; // -1 for any, -2 for all + void *entity; + + bool is_all() { + return index == -2; + } + + bool is_any() { + return index == -1; + } + + bool bind_with_check(void *entity); +}; + +class EventNode { +public: + EventOp op; + std::vector symbol_input; + std::vector node_input; + std::vector int_input; + + std::set related_symbols; + std::map infer_map; + + std::vector raw_parameter; // serialized parameter from python end +}; + +class RewardRule { +public: + std::vector input_symbols; + std::vector infer_obj; + EventNode *on; + + std::vector receivers; + std::vector values; + bool is_terminal; + bool auto_value; + + std::vector raw_parameter; // serialized parameter from python end + + bool trigger; +}; + +} // namespace gridworld +} // namespace magent + +#endif //MAGNET_GRIDWORLD_REWARD_DESCRIPTION_H diff --git a/examples/battle_model/src/gridworld/grid_def.h b/examples/battle_model/src/gridworld/grid_def.h new file mode 100755 index 0000000..4de2d1c --- /dev/null +++ b/examples/battle_model/src/gridworld/grid_def.h @@ -0,0 +1,57 @@ +/** + * \file grid_def.h + * \brief some global definition for gridworld + */ + +#ifndef MAGNET_GRIDWORLD_GRIDDEF_H +#define MAGNET_GRIDWORLD_GRIDDEF_H + +#include "../Environment.h" +#include "../utility/utility.h" + +namespace magent { +namespace gridworld { + +typedef enum {EAST, SOUTH, WEST, NORTH, DIR_NUM} Direction; + +typedef enum { + OP_AND, OP_OR, OP_NOT, + /***** split *****/ + OP_KILL, OP_AT, OP_IN, OP_COLLIDE, OP_ATTACK, OP_DIE, + OP_IN_A_LINE, OP_ALIGN, + OP_NULL, +} EventOp; + + +struct Position { + int x, y; +}; +typedef long long PositionInteger; + +typedef float Reward; +typedef int Action; + +// some forward declaration +class Agent; +class AgentType; +class Group; + +struct MoveAction; +struct TurnAction; +struct AttackAction; + +// reward description +class AgentSymbol; +class RewardRule; +class EventNode; + +using ::magent::environment::Environment; +using ::magent::environment::GroupHandle; +using ::magent::utility::strequ; +using ::magent::utility::NDPointer; + +} // namespace gridworld +} // namespace magent + + +#endif //MAGNET_GRIDWORLD_GRIDDEF_H diff --git a/examples/battle_model/src/gridworld/test.cc b/examples/battle_model/src/gridworld/test.cc new file mode 100755 index 0000000..1e91dda --- /dev/null +++ b/examples/battle_model/src/gridworld/test.cc @@ -0,0 +1,159 @@ +/** + * \file test.cc + * \brief unit test for some function + */ + +#if 0 +#include "../Environment.h" +#include "GridWorld.h" +#include "../runtime_api.h" + +using ::magent::gridworld::SectorRange; +using ::magent::gridworld::CircleRange; + +void test_sector_range() { + printf(" ==== test sector range ==== \n"); + + /*SectorRange s1(120, 3, 0); + SectorRange s2(120, 5, 0); + + s1.print_self(); + s2.print_self();*/ + + CircleRange r1(4, 1, 0); + CircleRange r2(4, 0, 1); + + r1.print_self(); + r2.print_self(); +} + +void get_observation(EnvHandle game, GroupHandle group, float **pview_buf, float **php_buf) { + int buf[3]; + int height, width, n_channel; + env_get_int_info(game, group, "view_size", buf); + height = buf[0], width = buf[1], n_channel = buf[2]; + int n; + env_get_num(game, group, &n); + + float *view_buf = new float[n * height * width * n_channel]; + float *hp_buf = new float[n]; + + float *bufs[] = {view_buf, hp_buf}; + env_get_observation(game, group, bufs); + + *pview_buf = view_buf; + *php_buf = hp_buf; +} + +void get_position(EnvHandle game, GroupHandle group, int **ppos_x, int **ppos_y) { + int n; + env_get_num(game, group, &n); + int *pos = new int [n * 2]; + int *pos_x = new int [n]; + int *pos_y = new int [n]; + env_get_int_info(game, group, "pos", pos); + for (int i = 0; i < n; i++) { + pos_x[i] = pos[i * 2]; + pos_y[i] = pos[i * 2 + 1]; + } + *ppos_x = pos_x; + *ppos_y = pos_y; + delete [] pos; +} + +void test_extract_view() { + EnvHandle game; + env_new_game(&game, "GridWorld"); + + // config + env_set_config(game, "map_width", 20); + env_set_config(game, "map_height", 20); + + // register type + const char *type_keys[] = { + "width", "length", "speed", "hp", + "view_radius", "view_angle", "attack_radius", "attack_angle", + "hear_radius", "speak_radius", + "speak_ability", "damage", "trace", "step_recover", "kill_supply", + "attack_in_group", + "step_reward", "kill_reward", "dead_penalty", + }; + + float type_deer[] = { + 2, 2, 2, 10, + 3, 360, 0, 0, + 0, 0, + 0, 3, 0, -0.5f, 10, + 0, + 0, 0, -1, + }; + + float type_tiger[] = { + 1, 1, 2, 10, + 3, 120, 2, 120, + 0, 0, + 0, 3, 0, -0.5f, 10, + 0, + 0, 0, -1, + }; + + int n = sizeof(type_keys) / sizeof(type_keys[1]); + gridworld_register_agent_type(game, "deer", n, type_keys, type_deer); + gridworld_register_agent_type(game, "tiger", n, type_keys, type_tiger); + + // new group + GroupHandle deer_handle, tiger_handle; + gridworld_new_group(game, "deer", &deer_handle); + gridworld_new_group(game, "tiger", &tiger_handle); + + env_reset(game); + + gridworld_add_agents(game, -1, 25, "random", nullptr, nullptr, nullptr); + gridworld_add_agents(game, deer_handle, 20, "random", nullptr, nullptr, nullptr); + gridworld_add_agents(game, tiger_handle, 10, "random", nullptr, nullptr, nullptr); + env_set_render(game, "save_dir", "___debug___"); + env_render(game); + + int observation_handle = deer_handle; + + int buf[3]; + int t_height, t_width, t_n_channel; + env_get_int_info(game, observation_handle, "view_size", buf); + t_height = buf[0], t_width = buf[1], t_n_channel = buf[2]; + + float (*view_buf)[t_height][t_width][t_n_channel]; + float *hp_buf; + int *pos_x, *pos_y; + + env_get_num(game, observation_handle, &n); + get_position(game, observation_handle, &pos_x, &pos_y); + get_observation(game, observation_handle, (float **)&view_buf, &hp_buf); + + printf("observation\n"); + for (int i = 0; i < n; i++) { + printf("======\n"); + printf("%d %d\n", pos_x[i], pos_y[i]); + for (int j = 0; j < t_height; j++) { + for (int k = 0; k < t_n_channel; k++) { + for (int l = 0; l < t_width; l++) { + printf("%1.1f ", (view_buf[i][j][l][k])); + } + printf(" "); + } + printf("\n"); + } + } + + printf("hp"); + for (int i = 0; i < n; i++) { + printf("%.2f ", hp_buf[i]); + } +} +#endif + +int main() { + //test_sector_range(); + //test_extract_view(); + + return 0; +} diff --git a/examples/battle_model/src/render/backend/data.cc b/examples/battle_model/src/render/backend/data.cc new file mode 100755 index 0000000..c62d6d5 --- /dev/null +++ b/examples/battle_model/src/render/backend/data.cc @@ -0,0 +1,430 @@ +#include +#include +#include +#ifdef __linux__ + #include +#else + #include +#endif +#include + +#include "data.h" +#include "utility/logger.h" +#include "utility/exception.h" + +namespace magent { +namespace render { + + +Window::Window(int xmin, int ymin, int xmax, int ymax) : wmin(xmin, ymin), wmax(xmax, ymax) { + +} + +bool Window::accept(int x, int y) const { + return wmin.x <= x && wmin.y <= y && x <= wmax.x && y <= wmax.y; +} + +bool Window::accept(int x, int y, int w, int h) const { + return ((wmin.x <= x && x <= wmax.x) || (wmin.x <= x + w && x + w <= wmax.x)) + && ((wmin.y <= y && y <= wmax.y) || (wmin.y <= y + h && y + h <= wmax.y)); +} + +Coordinate::Coordinate(int x, int y) : x(x), y(y) { + +} + +Coordinate::Coordinate() : x(0), y(0) { + +} + +const unsigned int &Frame::getAgentsNumber() const { + return nAgents; +} + +const unsigned int &Frame::getEventsNumber() const { + return nEvents; +} + +const AgentData &Frame::getAgent(unsigned int id) const { + return agents[id]; +} + +const EventData &Frame::getEvent(unsigned int id) const { + return events[id]; +} + +void Frame::load(std::istream & handle) { + if (!handle) { + throw RenderException("invalid file handle"); + } + if (!(handle >> nAgents >> nEvents >> nBreads)) { + throw RenderException("cannot read nAgents and nEvents and nBreads"); + } + if (agents != nullptr) { + delete[](agents); + agents = nullptr; + } + + if (events != nullptr) { + delete[](events); + events = nullptr; + } + + if (breads != nullptr) { + delete[](breads); + breads = nullptr; + } + + agents = new render::AgentData[nAgents]; + std::unordered_map map; + try { + for (unsigned int i = 0; i < nAgents; i++) { + if (!(handle >> agents[i].id >> agents[i].hp >> agents[i].direction + >> agents[i].position.x >> agents[i].position.y >> agents[i].groupID>>agents[i].ch)) { + throw RenderException("cannot read the next agent, map file may be broken"); + } + map[agents[i].id] = i; + } + } catch (const RenderException &e) { + delete[](agents); + agents = nullptr; + nAgents = 0; + throw; + } + + events = new render::EventData[nEvents]; + try{ + for (unsigned int i = 0, j = 0; i < nEvents; i++) { + int id; + if (!(handle >> events[i].type >> id >> events[i].position.x >> events[i].position.y)) { + throw RenderException("cannot read the next event, map file may be broken"); + } + if (map.find(id) != map.end()) { + events[i].agent = &agents[map[id]]; + } + } + } catch (const RenderException &e) { + delete[](events); + events = nullptr; + nEvents = 0; + throw; + } + + breads = new render::BreadData[nBreads]; + try{ + for (unsigned int i = 0; i < nBreads; i++) { + if (!(handle >> breads[i].position.x >> breads[i].position.y >> breads[i].hp)) { + throw RenderException("cannot read the next food, map file may be broken"); + } + } + } catch (const RenderException &e) { + delete[](breads); + breads = nullptr; + nBreads = 0; + throw; + } +} + +Frame::~Frame() { + if (agents != nullptr) { + delete[](agents); + agents = nullptr; + } + + if (events != nullptr) { + delete[](events); + events = nullptr; + } + + if (breads != nullptr) { + delete[](breads); + breads = nullptr; + } +} + +Frame::Frame() : nEvents(0), nAgents(0), nBreads(0), agents(nullptr), events(nullptr), breads(nullptr) { + +} + +const unsigned int &Frame::getBreadsNumber() const { + return nBreads; +} + +const BreadData &Frame::getBread(unsigned int id) const { + return breads[id]; +} + +void Frame::releaseMemory() { + agents = nullptr; + events = nullptr; + breads = nullptr; +} + +Buffer::~Buffer() { + maxSize = 0; + if (frames != nullptr) { + delete[](frames); + frames = nullptr; + } + if (obstacles != nullptr) { + delete[](obstacles); + obstacles = nullptr; + } +} + +const Frame &Buffer::operator [](unsigned int id)const { + return frames[id]; +} + +void Buffer::resize(unsigned int size) { + auto memory = new Frame[size]; + memcpy(static_cast(memory), static_cast(frames), sizeof(*frames) * std::min(maxSize, size)); + for (unsigned int i = 0; i < maxSize; i++) { + frames[i].releaseMemory(); + } + delete[](frames); + frames = memory; + maxSize = size; +} + +Buffer::Buffer(unsigned int maxSize) : maxSize(maxSize), frames(new Frame [maxSize]), nObstacles(0), obstacles(nullptr) { + +} + +const unsigned int & Buffer::getFramesNumber()const { + return nFrames; +} + +void Buffer::load(std::istream &handle) { + if (!handle) { + throw RenderException("invalid handle of the map data file"); + } + + std::string tmp; + if (!(handle >> tmp >> nObstacles)) { + throw RenderException("cannot read the number of obstacles in the data file"); + } + + if (tmp != "W") { + throw RenderException("invalid tag of walls"); + } + + if (obstacles != nullptr) { + delete[](obstacles); + obstacles = nullptr; + } + try { + obstacles = new Coordinate[nObstacles]; + for (unsigned int i = 0; i < nObstacles; i++) { + if (!(handle >> obstacles[i].x >> obstacles[i].y)) { + throw RenderException("cannot read the information of the next obstacle in the data file"); + } + } + } catch (const RenderException &e) { + delete[](obstacles); + obstacles = nullptr; + throw; + } + + nFrames = 0; + while (!handle.eof()) { + if (nFrames == maxSize) { + resize(maxSize * 2); + } + if (!(handle >> tmp)) { + break; + } + if (tmp != "F") { + throw RenderException("invalid frame flag, the map file may be broken"); + } + frames[nFrames++].load(handle); + } +} + +const unsigned int &Buffer::getObstaclesNumber() const { + return nObstacles; +} + +const Coordinate &Buffer::getObstacle(unsigned int id) const { + return obstacles[id]; +} + +void Config::load(std::istream &handle) { + if (!handle) { + throw RenderException("invalid handle of the map configuration file"); + } + Json::Reader reader; + Json::Value root; + if (reader.parse(handle, root)) { + if (root["width"].isUInt()) { + width = root["width"].asUInt(); + } else { + throw RenderException("property width must be an UInt"); + } + if (root["height"].isUInt()) { + height = root["height"].asUInt(); + } else { + throw RenderException("property height must be an UInt"); + } + // if (root["static-file"].isString()) { + // } else { + // throw RenderException("property height must be a String"); + // } + if (root["minimap-width"].isUInt()) { + miniMAPWidth = root["minimap-width"].asUInt(); + } else { + throw RenderException("property minimap-width must be an UInt"); + } + if (root["minimap-height"].isUInt()) { + miniMAPHeight = root["minimap-height"].asUInt(); + } else { + throw RenderException("property minimap-height must be an UInt"); + } + if (!root["obstacle-style"].isString()) { + throw RenderException("property obstacle-style must be an String"); + } + if (!root["dynamic-file-directory"].isString()) { + throw RenderException("property dynamic-file-directory must be an String"); + } else { + dataPath = root["dynamic-file-directory"].asString(); + root.removeMember("dynamic-file-directory"); + } + if (!root["attack-style"].isString()) { + throw RenderException("property attack-style must be an UInt"); + } + Json::Value & groups = root["group"]; + if (!groups) { + throw RenderException("lacks of property groups"); + } + if (groups.empty()) { + throw RenderException("property groups must contain at least one group"); + } + nStyles = groups.size(); + styles = new Style[nStyles]; + try { + unsigned int index = 0; + for (auto group : groups) { + if (group["height"].isUInt()) { + styles[index].height = group["height"].asUInt(); + } else { + throw RenderException("property height in the group must be an UInt"); + } + if (group["width"].isUInt()) { + styles[index].width = group["width"].asUInt(); + } else { + throw RenderException("property width in the group must be an UInt"); + } + if (group["style"].isString()) { + if (sscanf( + group["style"].asString().c_str(), + "rgba(%d,%d,%d,%*f)", + &styles[index].red, + &styles[index].blue, + &styles[index].green + ) != 3) { + throw RenderException("property style in the group must be rgba(r: int, g: int, b: int, a: float)"); + } + } else { + throw RenderException("property style in the group must be an String"); + } + if (!group["anchor"].isArray() || group["anchor"].size() != 2u) { + throw RenderException("property turn must be an array of size 2"); + } else { + // FIXME: maybe load the group information here + } + if (group["max-speed"].isUInt()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property max-speed must be an UInt"); + } + if (group["vision-radius"].isNumeric()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property vision-radius must be a number"); + } + if (group["vision-angle"].isUInt()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property vision-angle must be an UInt"); + } + if (group["vision-style"].isString()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property vision-style must be an UInt"); + } + if (group["attack-radius"].isNumeric()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property attack-radius must be a number"); + } + if (group["attack-angle"].isUInt()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property attack-angle must be an UInt"); + } + if (group["broadcast-radius"].isNumeric()) { + // FIXME: maybe load the group information here + } else { + throw RenderException("property broadcast-radius must be a number"); + } + index++; + } + } catch (const RenderException &e) { + delete[](styles); + styles = nullptr; + throw; + } + Json::FastWriter writer; + frontendJSON = writer.write(root); + } else { + throw RenderException("validation to JSON configuration file failed"); + } +} + +Config::~Config() { + if (styles != nullptr){ + delete[](styles); + styles = nullptr; + } +} + +const std::string & Config::getFrontendJSON()const { + return frontendJSON; +} + +const std::string &Config::getDataPath() { + return dataPath; +} + +const Style &Config::getStyle(unsigned int id) const { + return styles[id]; +} + +const unsigned int &Config::getStylesNumber() const { + return nStyles; +} + +Config::Config() + : height(0), width(0), nStyles(0), styles(nullptr){ + +} + +const unsigned int &Config::getHeight() const { + return height; +} + +const unsigned int &Config::getWidth() const { + return width; +} + +const unsigned int &Config::getMiniMAPHeight() const { + return miniMAPHeight; +} + +const unsigned int &Config::getMiniMAPWidth() const { + return miniMAPWidth; +} + + +} // namespace render +} // namespace magent \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/data.h b/examples/battle_model/src/render/backend/data.h new file mode 100755 index 0000000..4f51799 --- /dev/null +++ b/examples/battle_model/src/render/backend/data.h @@ -0,0 +1,140 @@ +#ifndef MAGNET_RENDER_BACKEND_HANDLE_DATA_H_ +#define MAGNET_RENDER_BACKEND_HANDLE_DATA_H_ + +#include +#include "utility/utility.h" +#include +#include +#include + +namespace magent { +namespace render { + +struct Coordinate { + int x, y; + + Coordinate(); + Coordinate(int x, int y); +}; + +struct AgentData : public render::Unique { + Coordinate position; + int id, hp, direction; + unsigned int groupID; + int ch; +}; + +struct BreadData : public render::Unique { + Coordinate position; + int hp; +}; + +struct EventData : public render::Unique { + int type; + const AgentData * agent; + Coordinate position; +}; + +struct Window { + Coordinate wmin, wmax; + + Window(int xmin, int ymin, int xmax, int ymax); + + bool accept(int, int)const; + + bool accept(int, int, int, int)const; +}; + +class Frame : public render::Unique { +private: + unsigned int nAgents, nEvents, nBreads; + render::AgentData * agents; + render::EventData * events; + render::BreadData * breads; + +public: + explicit Frame(); + + void load(std::istream & /*handle*/); + + const unsigned int & getAgentsNumber() const; + + const unsigned int & getEventsNumber() const; + + const unsigned int & getBreadsNumber() const; + + const render::AgentData & getAgent(unsigned int id) const; + + const render::EventData & getEvent(unsigned int id) const; + + const render::BreadData & getBread(unsigned int id) const; + + ~Frame() override; + + void releaseMemory(); +}; + +struct Style { + unsigned int height, width, red, blue, green; +}; + +class Buffer : public render::Unique { +private: + unsigned int nFrames, maxSize, nObstacles; + Frame * frames; + Coordinate * obstacles; + + void resize(unsigned int size); + +public: + explicit Buffer(unsigned int maxSize = 1000); + + void load(std::istream & /*handle*/); + + const Frame & operator [](unsigned int /*id*/)const; + + ~Buffer() override; + + const unsigned int & getFramesNumber()const; + + const unsigned int & getObstaclesNumber()const; + + const Coordinate & getObstacle(unsigned int id)const; + +}; + +class Config : public render::Unique { +private: + unsigned int height, width, miniMAPHeight, miniMAPWidth, nStyles; + Style * styles; + std::string frontendJSON; + std::string dataPath; + +public: + explicit Config(); + + void load(std::istream & /*handle*/); + + const unsigned int & getMiniMAPHeight()const; + + const unsigned int & getMiniMAPWidth()const; + + const unsigned int & getHeight()const; + + const unsigned int & getWidth()const; + + const unsigned int & getStylesNumber()const; + + const Style & getStyle(unsigned int id)const; + + ~Config() override; + + const std::string & getFrontendJSON()const; + + const std::string &getDataPath(); +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_HANDLE_DATA_H_ diff --git a/examples/battle_model/src/render/backend/demo/config.json b/examples/battle_model/src/render/backend/demo/config.json new file mode 100755 index 0000000..aa1c2b1 --- /dev/null +++ b/examples/battle_model/src/render/backend/demo/config.json @@ -0,0 +1,40 @@ +{ + "width": 500, + "height": 500, + "static-file": "static.map", + "obstacle-style": "rgba(170,170,170,1)", + "dynamic-file-directory": "data", + "attack-style": "rbga(0,0,0,0.8)", + "minimap-width": 300, + "minimap-height": 250, + "group" : [ + { + "height": 1, + "width": 1, + "style": "rgba(150,100,100,1)", + "anchor": [0, 0], + "max-speed": 1, + "vision-radius": 5, + "vision-angle": 360, + "vision-style": "rgba(150,100,100,0.05)", + "attack-radius": 3, + "attack-angle": 60, + "attack-style": "rgba(150,100,100,0.1)", + "broadcast-radius": 1 + }, + { + "height": 1, + "width": 1, + "style": "rgba(0,200,0,1)", + "anchor": [0, 0], + "max-speed": 1, + "vision-radius": 4, + "vision-angle": 360, + "vision-style": "rgba(0,0,200,0.05)", + "attack-radius": 1, + "attack-angle": 180, + "attack-style": "rgba(0,0,200,0.1)", + "broadcast-radius": 1 + } + ] +} diff --git a/examples/battle_model/src/render/backend/demo/static.map b/examples/battle_model/src/render/backend/demo/static.map new file mode 100755 index 0000000..7ccb50f --- /dev/null +++ b/examples/battle_model/src/render/backend/demo/static.map @@ -0,0 +1,11997 @@ +11996 +0 0 +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 +10 0 +11 0 +12 0 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +21 0 +22 0 +23 0 +24 0 +25 0 +26 0 +27 0 +28 0 +29 0 +30 0 +31 0 +32 0 +33 0 +34 0 +35 0 +36 0 +37 0 +38 0 +39 0 +40 0 +41 0 +42 0 +43 0 +44 0 +45 0 +46 0 +47 0 +48 0 +49 0 +50 0 +51 0 +52 0 +53 0 +54 0 +55 0 +56 0 +57 0 +58 0 +59 0 +60 0 +61 0 +62 0 +63 0 +64 0 +65 0 +66 0 +67 0 +68 0 +69 0 +70 0 +71 0 +72 0 +73 0 +74 0 +75 0 +76 0 +77 0 +78 0 +79 0 +80 0 +81 0 +82 0 +83 0 +84 0 +85 0 +86 0 +87 0 +88 0 +89 0 +90 0 +91 0 +92 0 +93 0 +94 0 +95 0 +96 0 +97 0 +98 0 +99 0 +100 0 +101 0 +102 0 +103 0 +104 0 +105 0 +106 0 +107 0 +108 0 +109 0 +110 0 +111 0 +112 0 +113 0 +114 0 +115 0 +116 0 +117 0 +118 0 +119 0 +120 0 +121 0 +122 0 +123 0 +124 0 +125 0 +126 0 +127 0 +128 0 +129 0 +130 0 +131 0 +132 0 +133 0 +134 0 +135 0 +136 0 +137 0 +138 0 +139 0 +140 0 +141 0 +142 0 +143 0 +144 0 +145 0 +146 0 +147 0 +148 0 +149 0 +150 0 +151 0 +152 0 +153 0 +154 0 +155 0 +156 0 +157 0 +158 0 +159 0 +160 0 +161 0 +162 0 +163 0 +164 0 +165 0 +166 0 +167 0 +168 0 +169 0 +170 0 +171 0 +172 0 +173 0 +174 0 +175 0 +176 0 +177 0 +178 0 +179 0 +180 0 +181 0 +182 0 +183 0 +184 0 +185 0 +186 0 +187 0 +188 0 +189 0 +190 0 +191 0 +192 0 +193 0 +194 0 +195 0 +196 0 +197 0 +198 0 +199 0 +200 0 +201 0 +202 0 +203 0 +204 0 +205 0 +206 0 +207 0 +208 0 +209 0 +210 0 +211 0 +212 0 +213 0 +214 0 +215 0 +216 0 +217 0 +218 0 +219 0 +220 0 +221 0 +222 0 +223 0 +224 0 +225 0 +226 0 +227 0 +228 0 +229 0 +230 0 +231 0 +232 0 +233 0 +234 0 +235 0 +236 0 +237 0 +238 0 +239 0 +240 0 +241 0 +242 0 +243 0 +244 0 +245 0 +246 0 +247 0 +248 0 +249 0 +250 0 +251 0 +252 0 +253 0 +254 0 +255 0 +256 0 +257 0 +258 0 +259 0 +260 0 +261 0 +262 0 +263 0 +264 0 +265 0 +266 0 +267 0 +268 0 +269 0 +270 0 +271 0 +272 0 +273 0 +274 0 +275 0 +276 0 +277 0 +278 0 +279 0 +280 0 +281 0 +282 0 +283 0 +284 0 +285 0 +286 0 +287 0 +288 0 +289 0 +290 0 +291 0 +292 0 +293 0 +294 0 +295 0 +296 0 +297 0 +298 0 +299 0 +300 0 +301 0 +302 0 +303 0 +304 0 +305 0 +306 0 +307 0 +308 0 +309 0 +310 0 +311 0 +312 0 +313 0 +314 0 +315 0 +316 0 +317 0 +318 0 +319 0 +320 0 +321 0 +322 0 +323 0 +324 0 +325 0 +326 0 +327 0 +328 0 +329 0 +330 0 +331 0 +332 0 +333 0 +334 0 +335 0 +336 0 +337 0 +338 0 +339 0 +340 0 +341 0 +342 0 +343 0 +344 0 +345 0 +346 0 +347 0 +348 0 +349 0 +350 0 +351 0 +352 0 +353 0 +354 0 +355 0 +356 0 +357 0 +358 0 +359 0 +360 0 +361 0 +362 0 +363 0 +364 0 +365 0 +366 0 +367 0 +368 0 +369 0 +370 0 +371 0 +372 0 +373 0 +374 0 +375 0 +376 0 +377 0 +378 0 +379 0 +380 0 +381 0 +382 0 +383 0 +384 0 +385 0 +386 0 +387 0 +388 0 +389 0 +390 0 +391 0 +392 0 +393 0 +394 0 +395 0 +396 0 +397 0 +398 0 +399 0 +400 0 +401 0 +402 0 +403 0 +404 0 +405 0 +406 0 +407 0 +408 0 +409 0 +410 0 +411 0 +412 0 +413 0 +414 0 +415 0 +416 0 +417 0 +418 0 +419 0 +420 0 +421 0 +422 0 +423 0 +424 0 +425 0 +426 0 +427 0 +428 0 +429 0 +430 0 +431 0 +432 0 +433 0 +434 0 +435 0 +436 0 +437 0 +438 0 +439 0 +440 0 +441 0 +442 0 +443 0 +444 0 +445 0 +446 0 +447 0 +448 0 +449 0 +450 0 +451 0 +452 0 +453 0 +454 0 +455 0 +456 0 +457 0 +458 0 +459 0 +460 0 +461 0 +462 0 +463 0 +464 0 +465 0 +466 0 +467 0 +468 0 +469 0 +470 0 +471 0 +472 0 +473 0 +474 0 +475 0 +476 0 +477 0 +478 0 +479 0 +480 0 +481 0 +482 0 +483 0 +484 0 +485 0 +486 0 +487 0 +488 0 +489 0 +490 0 +491 0 +492 0 +493 0 +494 0 +495 0 +496 0 +497 0 +498 0 +499 0 +0 1 +3 1 +12 1 +47 1 +68 1 +89 1 +122 1 +162 1 +181 1 +201 1 +206 1 +222 1 +231 1 +240 1 +268 1 +325 1 +366 1 +396 1 +405 1 +442 1 +470 1 +490 1 +499 1 +0 2 +4 2 +11 2 +13 2 +38 2 +47 2 +51 2 +99 2 +169 2 +178 2 +218 2 +278 2 +289 2 +335 2 +336 2 +412 2 +454 2 +457 2 +494 2 +499 2 +0 3 +1 3 +16 3 +26 3 +84 3 +93 3 +136 3 +142 3 +192 3 +219 3 +249 3 +255 3 +323 3 +351 3 +378 3 +396 3 +430 3 +453 3 +486 3 +499 3 +0 4 +12 4 +68 4 +94 4 +104 4 +121 4 +137 4 +141 4 +150 4 +153 4 +159 4 +179 4 +220 4 +305 4 +314 4 +342 4 +352 4 +360 4 +379 4 +384 4 +385 4 +455 4 +470 4 +475 4 +482 4 +499 4 +0 5 +33 5 +51 5 +57 5 +64 5 +74 5 +105 5 +116 5 +138 5 +143 5 +186 5 +194 5 +243 5 +259 5 +264 5 +306 5 +402 5 +422 5 +443 5 +491 5 +499 5 +0 6 +5 6 +6 6 +103 6 +115 6 +117 6 +118 6 +125 6 +149 6 +153 6 +158 6 +173 6 +179 6 +203 6 +270 6 +284 6 +287 6 +355 6 +371 6 +383 6 +424 6 +430 6 +493 6 +499 6 +0 7 +3 7 +45 7 +66 7 +77 7 +95 7 +118 7 +134 7 +202 7 +221 7 +240 7 +255 7 +278 7 +309 7 +313 7 +334 7 +353 7 +376 7 +464 7 +466 7 +499 7 +0 8 +8 8 +14 8 +45 8 +71 8 +95 8 +103 8 +108 8 +122 8 +128 8 +156 8 +223 8 +245 8 +260 8 +266 8 +335 8 +375 8 +401 8 +404 8 +455 8 +483 8 +499 8 +0 9 +8 9 +80 9 +124 9 +134 9 +151 9 +201 9 +213 9 +252 9 +283 9 +367 9 +382 9 +402 9 +459 9 +465 9 +499 9 +0 10 +8 10 +113 10 +161 10 +232 10 +277 10 +303 10 +314 10 +321 10 +342 10 +362 10 +391 10 +446 10 +499 10 +0 11 +17 11 +18 11 +38 11 +43 11 +98 11 +144 11 +149 11 +162 11 +182 11 +198 11 +275 11 +281 11 +316 11 +386 11 +389 11 +422 11 +463 11 +483 11 +485 11 +499 11 +0 12 +17 12 +77 12 +94 12 +124 12 +148 12 +171 12 +209 12 +255 12 +260 12 +269 12 +291 12 +292 12 +307 12 +378 12 +413 12 +460 12 +471 12 +499 12 +0 13 +16 13 +49 13 +62 13 +101 13 +133 13 +154 13 +195 13 +198 13 +201 13 +212 13 +225 13 +261 13 +324 13 +357 13 +399 13 +412 13 +499 13 +0 14 +32 14 +51 14 +87 14 +96 14 +159 14 +188 14 +234 14 +238 14 +307 14 +315 14 +334 14 +348 14 +351 14 +375 14 +380 14 +421 14 +469 14 +481 14 +490 14 +494 14 +499 14 +0 15 +11 15 +16 15 +24 15 +29 15 +63 15 +81 15 +124 15 +127 15 +144 15 +174 15 +183 15 +214 15 +234 15 +261 15 +296 15 +322 15 +382 15 +416 15 +462 15 +489 15 +494 15 +499 15 +0 16 +9 16 +14 16 +20 16 +81 16 +88 16 +115 16 +149 16 +225 16 +269 16 +280 16 +308 16 +340 16 +352 16 +362 16 +383 16 +388 16 +431 16 +452 16 +462 16 +463 16 +473 16 +499 16 +0 17 +12 17 +54 17 +75 17 +108 17 +133 17 +145 17 +165 17 +184 17 +201 17 +204 17 +219 17 +228 17 +241 17 +247 17 +267 17 +338 17 +347 17 +381 17 +396 17 +409 17 +449 17 +470 17 +496 17 +499 17 +0 18 +31 18 +51 18 +70 18 +152 18 +176 18 +192 18 +195 18 +203 18 +209 18 +264 18 +299 18 +392 18 +418 18 +422 18 +427 18 +476 18 +486 18 +499 18 +0 19 +23 19 +40 19 +51 19 +99 19 +109 19 +120 19 +128 19 +178 19 +191 19 +210 19 +213 19 +248 19 +252 19 +298 19 +325 19 +348 19 +426 19 +431 19 +461 19 +476 19 +482 19 +499 19 +0 20 +47 20 +49 20 +101 20 +116 20 +192 20 +225 20 +231 20 +240 20 +271 20 +276 20 +284 20 +366 20 +369 20 +391 20 +444 20 +449 20 +471 20 +499 20 +0 21 +2 21 +3 21 +23 21 +24 21 +35 21 +43 21 +77 21 +88 21 +89 21 +110 21 +128 21 +204 21 +259 21 +285 21 +323 21 +384 21 +390 21 +417 21 +432 21 +499 21 +0 22 +36 22 +93 22 +135 22 +137 22 +138 22 +193 22 +210 22 +229 22 +263 22 +278 22 +309 22 +359 22 +361 22 +366 22 +378 22 +398 22 +410 22 +437 22 +440 22 +456 22 +460 22 +499 22 +0 23 +10 23 +32 23 +58 23 +103 23 +141 23 +161 23 +169 23 +184 23 +213 23 +230 23 +236 23 +304 23 +331 23 +334 23 +335 23 +356 23 +375 23 +405 23 +449 23 +453 23 +457 23 +499 23 +0 24 +56 24 +132 24 +195 24 +208 24 +258 24 +330 24 +338 24 +375 24 +435 24 +436 24 +450 24 +499 24 +0 25 +15 25 +20 25 +53 25 +78 25 +141 25 +184 25 +210 25 +249 25 +257 25 +296 25 +319 25 +338 25 +347 25 +354 25 +356 25 +361 25 +389 25 +432 25 +439 25 +467 25 +499 25 +0 26 +3 26 +7 26 +25 26 +40 26 +53 26 +71 26 +109 26 +112 26 +139 26 +161 26 +162 26 +169 26 +215 26 +238 26 +299 26 +312 26 +318 26 +324 26 +329 26 +364 26 +388 26 +414 26 +443 26 +451 26 +476 26 +489 26 +490 26 +499 26 +0 27 +68 27 +86 27 +122 27 +129 27 +179 27 +180 27 +284 27 +369 27 +391 27 +441 27 +499 27 +0 28 +2 28 +58 28 +59 28 +65 28 +94 28 +99 28 +105 28 +131 28 +140 28 +144 28 +163 28 +179 28 +206 28 +259 28 +263 28 +317 28 +327 28 +336 28 +338 28 +361 28 +428 28 +433 28 +462 28 +479 28 +499 28 +0 29 +26 29 +59 29 +64 29 +71 29 +92 29 +106 29 +109 29 +170 29 +303 29 +338 29 +347 29 +378 29 +402 29 +415 29 +499 29 +0 30 +57 30 +86 30 +97 30 +102 30 +112 30 +120 30 +133 30 +139 30 +152 30 +165 30 +193 30 +239 30 +252 30 +283 30 +288 30 +323 30 +345 30 +359 30 +360 30 +375 30 +377 30 +408 30 +416 30 +446 30 +499 30 +0 31 +20 31 +30 31 +42 31 +51 31 +95 31 +109 31 +118 31 +138 31 +149 31 +173 31 +186 31 +192 31 +223 31 +234 31 +345 31 +402 31 +407 31 +414 31 +423 31 +426 31 +432 31 +435 31 +499 31 +0 32 +49 32 +101 32 +103 32 +140 32 +163 32 +182 32 +214 32 +216 32 +245 32 +249 32 +261 32 +343 32 +347 32 +380 32 +401 32 +426 32 +476 32 +489 32 +499 32 +0 33 +8 33 +9 33 +22 33 +40 33 +46 33 +72 33 +114 33 +169 33 +187 33 +242 33 +247 33 +275 33 +339 33 +355 33 +368 33 +377 33 +412 33 +445 33 +476 33 +491 33 +495 33 +499 33 +0 34 +2 34 +37 34 +50 34 +56 34 +59 34 +67 34 +102 34 +103 34 +114 34 +120 34 +170 34 +259 34 +278 34 +298 34 +309 34 +341 34 +350 34 +366 34 +418 34 +427 34 +451 34 +457 34 +477 34 +486 34 +497 34 +498 34 +499 34 +0 35 +24 35 +51 35 +54 35 +68 35 +69 35 +79 35 +99 35 +102 35 +138 35 +146 35 +167 35 +183 35 +208 35 +217 35 +241 35 +258 35 +259 35 +264 35 +266 35 +285 35 +335 35 +367 35 +381 35 +407 35 +413 35 +425 35 +439 35 +489 35 +499 35 +0 36 +17 36 +24 36 +49 36 +56 36 +66 36 +78 36 +100 36 +111 36 +118 36 +134 36 +180 36 +243 36 +249 36 +263 36 +268 36 +306 36 +339 36 +340 36 +342 36 +353 36 +359 36 +382 36 +404 36 +421 36 +455 36 +471 36 +495 36 +499 36 +0 37 +19 37 +23 37 +40 37 +48 37 +115 37 +146 37 +160 37 +192 37 +196 37 +223 37 +235 37 +257 37 +267 37 +411 37 +477 37 +499 37 +0 38 +5 38 +33 38 +50 38 +79 38 +149 38 +171 38 +218 38 +232 38 +263 38 +285 38 +298 38 +327 38 +346 38 +348 38 +394 38 +416 38 +466 38 +499 38 +0 39 +58 39 +62 39 +72 39 +75 39 +78 39 +107 39 +113 39 +128 39 +157 39 +167 39 +199 39 +273 39 +279 39 +288 39 +341 39 +364 39 +384 39 +418 39 +420 39 +499 39 +0 40 +9 40 +11 40 +19 40 +41 40 +69 40 +126 40 +185 40 +192 40 +203 40 +204 40 +216 40 +269 40 +336 40 +341 40 +348 40 +360 40 +409 40 +440 40 +445 40 +471 40 +478 40 +483 40 +499 40 +0 41 +21 41 +46 41 +57 41 +74 41 +98 41 +136 41 +153 41 +169 41 +225 41 +264 41 +275 41 +318 41 +354 41 +363 41 +429 41 +473 41 +499 41 +0 42 +35 42 +50 42 +96 42 +102 42 +109 42 +139 42 +140 42 +158 42 +189 42 +195 42 +200 42 +224 42 +317 42 +335 42 +347 42 +385 42 +413 42 +422 42 +444 42 +461 42 +493 42 +499 42 +0 43 +6 43 +13 43 +73 43 +140 43 +150 43 +166 43 +173 43 +181 43 +212 43 +244 43 +253 43 +262 43 +327 43 +336 43 +339 43 +390 43 +394 43 +416 43 +424 43 +433 43 +465 43 +499 43 +0 44 +44 44 +49 44 +95 44 +107 44 +170 44 +176 44 +180 44 +231 44 +296 44 +305 44 +313 44 +319 44 +397 44 +402 44 +405 44 +429 44 +436 44 +458 44 +479 44 +499 44 +0 45 +1 45 +52 45 +106 45 +136 45 +141 45 +163 45 +164 45 +165 45 +180 45 +278 45 +298 45 +310 45 +330 45 +383 45 +422 45 +424 45 +460 45 +461 45 +462 45 +499 45 +0 46 +35 46 +98 46 +108 46 +134 46 +198 46 +214 46 +218 46 +254 46 +255 46 +273 46 +285 46 +288 46 +343 46 +348 46 +360 46 +385 46 +422 46 +426 46 +435 46 +469 46 +482 46 +492 46 +499 46 +0 47 +3 47 +9 47 +27 47 +62 47 +70 47 +95 47 +104 47 +139 47 +148 47 +183 47 +194 47 +203 47 +300 47 +329 47 +360 47 +367 47 +376 47 +408 47 +421 47 +423 47 +446 47 +498 47 +499 47 +0 48 +18 48 +67 48 +103 48 +190 48 +203 48 +252 48 +270 48 +278 48 +343 48 +346 48 +378 48 +393 48 +396 48 +420 48 +444 48 +446 48 +449 48 +468 48 +499 48 +0 49 +15 49 +25 49 +27 49 +50 49 +52 49 +83 49 +86 49 +90 49 +112 49 +188 49 +200 49 +209 49 +317 49 +326 49 +328 49 +334 49 +390 49 +412 49 +418 49 +423 49 +428 49 +436 49 +447 49 +474 49 +499 49 +0 50 +33 50 +145 50 +156 50 +328 50 +439 50 +451 50 +455 50 +457 50 +482 50 +489 50 +499 50 +0 51 +45 51 +52 51 +79 51 +108 51 +114 51 +213 51 +218 51 +262 51 +268 51 +281 51 +289 51 +325 51 +333 51 +389 51 +398 51 +402 51 +452 51 +455 51 +474 51 +488 51 +490 51 +499 51 +0 52 +9 52 +26 52 +47 52 +104 52 +109 52 +163 52 +172 52 +181 52 +277 52 +328 52 +350 52 +367 52 +419 52 +440 52 +441 52 +448 52 +475 52 +499 52 +0 53 +80 53 +131 53 +145 53 +228 53 +229 53 +280 53 +293 53 +302 53 +316 53 +348 53 +352 53 +376 53 +429 53 +448 53 +494 53 +499 53 +0 54 +4 54 +64 54 +98 54 +103 54 +139 54 +164 54 +202 54 +259 54 +276 54 +280 54 +332 54 +392 54 +427 54 +437 54 +478 54 +499 54 +0 55 +26 55 +47 55 +58 55 +138 55 +149 55 +166 55 +173 55 +176 55 +214 55 +234 55 +239 55 +251 55 +252 55 +275 55 +305 55 +326 55 +330 55 +350 55 +400 55 +436 55 +473 55 +499 55 +0 56 +13 56 +24 56 +70 56 +80 56 +118 56 +122 56 +137 56 +165 56 +200 56 +235 56 +270 56 +281 56 +286 56 +296 56 +310 56 +342 56 +344 56 +357 56 +387 56 +397 56 +423 56 +453 56 +487 56 +495 56 +499 56 +0 57 +22 57 +52 57 +56 57 +137 57 +142 57 +158 57 +169 57 +197 57 +232 57 +244 57 +288 57 +301 57 +313 57 +323 57 +343 57 +484 57 +488 57 +499 57 +0 58 +5 58 +9 58 +32 58 +37 58 +52 58 +83 58 +154 58 +167 58 +203 58 +212 58 +228 58 +262 58 +271 58 +293 58 +297 58 +305 58 +313 58 +320 58 +321 58 +323 58 +344 58 +351 58 +366 58 +375 58 +382 58 +390 58 +476 58 +499 58 +0 59 +13 59 +51 59 +94 59 +104 59 +137 59 +143 59 +216 59 +233 59 +255 59 +274 59 +294 59 +307 59 +308 59 +312 59 +392 59 +448 59 +481 59 +499 59 +0 60 +18 60 +27 60 +45 60 +61 60 +97 60 +100 60 +106 60 +120 60 +185 60 +271 60 +295 60 +299 60 +366 60 +374 60 +383 60 +430 60 +436 60 +446 60 +463 60 +491 60 +494 60 +499 60 +0 61 +8 61 +33 61 +38 61 +45 61 +47 61 +101 61 +107 61 +117 61 +171 61 +177 61 +202 61 +220 61 +227 61 +248 61 +273 61 +320 61 +328 61 +335 61 +368 61 +372 61 +398 61 +455 61 +493 61 +499 61 +0 62 +42 62 +52 62 +54 62 +70 62 +111 62 +114 62 +118 62 +299 62 +340 62 +400 62 +413 62 +442 62 +450 62 +492 62 +499 62 +0 63 +8 63 +47 63 +50 63 +91 63 +93 63 +102 63 +121 63 +127 63 +140 63 +174 63 +193 63 +199 63 +235 63 +265 63 +290 63 +299 63 +312 63 +432 63 +472 63 +499 63 +0 64 +26 64 +32 64 +88 64 +99 64 +125 64 +145 64 +157 64 +171 64 +184 64 +193 64 +221 64 +236 64 +256 64 +277 64 +292 64 +310 64 +314 64 +318 64 +333 64 +379 64 +385 64 +418 64 +420 64 +423 64 +430 64 +446 64 +475 64 +479 64 +480 64 +487 64 +491 64 +499 64 +0 65 +33 65 +77 65 +148 65 +157 65 +163 65 +166 65 +226 65 +267 65 +276 65 +301 65 +323 65 +364 65 +450 65 +490 65 +499 65 +0 66 +18 66 +37 66 +80 66 +105 66 +110 66 +171 66 +303 66 +347 66 +362 66 +383 66 +401 66 +425 66 +428 66 +435 66 +447 66 +487 66 +498 66 +499 66 +0 67 +16 67 +21 67 +74 67 +75 67 +90 67 +201 67 +203 67 +213 67 +215 67 +227 67 +251 67 +252 67 +320 67 +325 67 +349 67 +470 67 +481 67 +483 67 +499 67 +0 68 +6 68 +25 68 +27 68 +57 68 +74 68 +86 68 +112 68 +131 68 +174 68 +184 68 +214 68 +279 68 +331 68 +332 68 +335 68 +337 68 +348 68 +379 68 +399 68 +451 68 +457 68 +499 68 +0 69 +8 69 +25 69 +40 69 +61 69 +62 69 +64 69 +68 69 +76 69 +88 69 +105 69 +110 69 +142 69 +190 69 +211 69 +242 69 +246 69 +247 69 +262 69 +269 69 +296 69 +310 69 +324 69 +391 69 +408 69 +460 69 +466 69 +499 69 +0 70 +54 70 +121 70 +130 70 +196 70 +249 70 +256 70 +277 70 +289 70 +301 70 +322 70 +336 70 +345 70 +355 70 +398 70 +452 70 +499 70 +0 71 +86 71 +89 71 +185 71 +196 71 +233 71 +237 71 +246 71 +277 71 +294 71 +308 71 +380 71 +388 71 +390 71 +421 71 +422 71 +429 71 +439 71 +499 71 +0 72 +4 72 +13 72 +28 72 +67 72 +68 72 +91 72 +104 72 +108 72 +124 72 +129 72 +155 72 +163 72 +168 72 +194 72 +230 72 +240 72 +244 72 +245 72 +271 72 +281 72 +290 72 +291 72 +312 72 +315 72 +339 72 +363 72 +370 72 +403 72 +430 72 +438 72 +486 72 +490 72 +498 72 +499 72 +0 73 +30 73 +82 73 +112 73 +116 73 +132 73 +178 73 +193 73 +242 73 +295 73 +308 73 +309 73 +313 73 +342 73 +346 73 +348 73 +368 73 +373 73 +434 73 +454 73 +459 73 +488 73 +490 73 +499 73 +0 74 +42 74 +88 74 +98 74 +102 74 +137 74 +152 74 +165 74 +220 74 +281 74 +319 74 +336 74 +339 74 +381 74 +386 74 +465 74 +492 74 +499 74 +0 75 +18 75 +48 75 +51 75 +53 75 +56 75 +122 75 +239 75 +254 75 +337 75 +347 75 +359 75 +370 75 +439 75 +470 75 +492 75 +499 75 +0 76 +8 76 +58 76 +88 76 +132 76 +184 76 +210 76 +230 76 +233 76 +244 76 +270 76 +321 76 +362 76 +369 76 +373 76 +385 76 +412 76 +425 76 +443 76 +499 76 +0 77 +14 77 +81 77 +100 77 +183 77 +190 77 +226 77 +230 77 +300 77 +314 77 +350 77 +398 77 +471 77 +487 77 +499 77 +0 78 +12 78 +26 78 +28 78 +88 78 +91 78 +110 78 +144 78 +147 78 +149 78 +150 78 +159 78 +181 78 +196 78 +326 78 +334 78 +357 78 +372 78 +390 78 +443 78 +499 78 +0 79 +10 79 +12 79 +37 79 +116 79 +117 79 +130 79 +165 79 +201 79 +215 79 +261 79 +271 79 +297 79 +302 79 +303 79 +331 79 +352 79 +355 79 +374 79 +385 79 +394 79 +407 79 +430 79 +443 79 +470 79 +478 79 +487 79 +495 79 +499 79 +0 80 +14 80 +34 80 +92 80 +94 80 +106 80 +147 80 +198 80 +251 80 +253 80 +276 80 +311 80 +314 80 +366 80 +394 80 +395 80 +456 80 +462 80 +469 80 +486 80 +487 80 +499 80 +0 81 +16 81 +28 81 +43 81 +64 81 +153 81 +168 81 +264 81 +292 81 +294 81 +299 81 +315 81 +373 81 +428 81 +475 81 +495 81 +497 81 +499 81 +0 82 +21 82 +41 82 +45 82 +46 82 +64 82 +87 82 +101 82 +136 82 +148 82 +156 82 +166 82 +195 82 +203 82 +294 82 +301 82 +319 82 +412 82 +434 82 +454 82 +461 82 +499 82 +0 83 +2 83 +13 83 +111 83 +117 83 +126 83 +169 83 +189 83 +213 83 +228 83 +229 83 +274 83 +279 83 +287 83 +309 83 +379 83 +389 83 +438 83 +490 83 +499 83 +0 84 +144 84 +145 84 +210 84 +218 84 +242 84 +274 84 +329 84 +330 84 +343 84 +350 84 +361 84 +377 84 +429 84 +448 84 +452 84 +499 84 +0 85 +5 85 +12 85 +17 85 +37 85 +64 85 +97 85 +174 85 +185 85 +193 85 +195 85 +218 85 +255 85 +261 85 +264 85 +300 85 +313 85 +318 85 +381 85 +386 85 +389 85 +398 85 +434 85 +435 85 +488 85 +490 85 +499 85 +0 86 +112 86 +117 86 +151 86 +153 86 +236 86 +311 86 +327 86 +337 86 +357 86 +376 86 +387 86 +435 86 +437 86 +447 86 +478 86 +480 86 +489 86 +496 86 +499 86 +0 87 +16 87 +30 87 +58 87 +115 87 +146 87 +229 87 +304 87 +322 87 +324 87 +331 87 +337 87 +362 87 +400 87 +418 87 +488 87 +489 87 +495 87 +499 87 +0 88 +12 88 +16 88 +28 88 +34 88 +67 88 +144 88 +145 88 +152 88 +227 88 +301 88 +322 88 +340 88 +342 88 +365 88 +374 88 +457 88 +482 88 +499 88 +0 89 +54 89 +69 89 +73 89 +75 89 +139 89 +167 89 +168 89 +232 89 +251 89 +296 89 +318 89 +388 89 +417 89 +467 89 +488 89 +498 89 +499 89 +0 90 +3 90 +7 90 +39 90 +56 90 +62 90 +116 90 +170 90 +172 90 +175 90 +205 90 +207 90 +239 90 +274 90 +291 90 +345 90 +359 90 +366 90 +466 90 +469 90 +497 90 +499 90 +0 91 +58 91 +67 91 +109 91 +145 91 +200 91 +202 91 +235 91 +287 91 +291 91 +301 91 +306 91 +337 91 +399 91 +440 91 +450 91 +470 91 +499 91 +0 92 +8 92 +10 92 +24 92 +53 92 +79 92 +104 92 +180 92 +241 92 +256 92 +330 92 +369 92 +386 92 +424 92 +432 92 +449 92 +499 92 +0 93 +82 93 +85 93 +95 93 +98 93 +159 93 +189 93 +198 93 +204 93 +259 93 +300 93 +362 93 +374 93 +463 93 +499 93 +0 94 +9 94 +25 94 +69 94 +102 94 +114 94 +122 94 +140 94 +168 94 +258 94 +305 94 +342 94 +347 94 +348 94 +351 94 +359 94 +417 94 +422 94 +423 94 +442 94 +497 94 +499 94 +0 95 +20 95 +67 95 +68 95 +87 95 +166 95 +186 95 +206 95 +240 95 +243 95 +255 95 +286 95 +331 95 +405 95 +422 95 +428 95 +437 95 +497 95 +499 95 +0 96 +18 96 +75 96 +137 96 +190 96 +226 96 +251 96 +281 96 +312 96 +339 96 +364 96 +374 96 +379 96 +386 96 +390 96 +400 96 +428 96 +434 96 +466 96 +470 96 +471 96 +499 96 +0 97 +6 97 +39 97 +49 97 +62 97 +77 97 +93 97 +189 97 +193 97 +195 97 +198 97 +254 97 +262 97 +299 97 +315 97 +330 97 +356 97 +377 97 +378 97 +395 97 +420 97 +455 97 +492 97 +493 97 +499 97 +0 98 +10 98 +24 98 +46 98 +84 98 +87 98 +142 98 +168 98 +192 98 +215 98 +216 98 +274 98 +286 98 +346 98 +362 98 +373 98 +374 98 +390 98 +395 98 +421 98 +462 98 +470 98 +484 98 +499 98 +0 99 +5 99 +9 99 +19 99 +33 99 +88 99 +149 99 +192 99 +211 99 +255 99 +362 99 +363 99 +364 99 +408 99 +463 99 +498 99 +499 99 +0 100 +8 100 +30 100 +33 100 +42 100 +46 100 +57 100 +59 100 +80 100 +92 100 +102 100 +120 100 +161 100 +174 100 +202 100 +267 100 +282 100 +289 100 +340 100 +378 100 +388 100 +445 100 +447 100 +449 100 +465 100 +470 100 +489 100 +499 100 +0 101 +11 101 +58 101 +171 101 +174 101 +182 101 +207 101 +226 101 +252 101 +273 101 +284 101 +320 101 +331 101 +337 101 +376 101 +421 101 +428 101 +432 101 +435 101 +437 101 +495 101 +496 101 +499 101 +0 102 +75 102 +128 102 +183 102 +256 102 +270 102 +330 102 +345 102 +372 102 +375 102 +396 102 +427 102 +453 102 +487 102 +499 102 +0 103 +3 103 +61 103 +163 103 +175 103 +186 103 +190 103 +271 103 +273 103 +305 103 +377 103 +399 103 +409 103 +426 103 +463 103 +471 103 +499 103 +0 104 +22 104 +31 104 +35 104 +44 104 +96 104 +114 104 +126 104 +134 104 +175 104 +198 104 +199 104 +216 104 +218 104 +239 104 +308 104 +313 104 +314 104 +315 104 +323 104 +331 104 +355 104 +401 104 +402 104 +409 104 +451 104 +462 104 +496 104 +499 104 +0 105 +5 105 +54 105 +121 105 +122 105 +141 105 +169 105 +171 105 +177 105 +200 105 +214 105 +224 105 +232 105 +272 105 +295 105 +311 105 +346 105 +376 105 +381 105 +444 105 +462 105 +463 105 +480 105 +499 105 +0 106 +38 106 +72 106 +93 106 +179 106 +230 106 +237 106 +320 106 +383 106 +388 106 +407 106 +420 106 +426 106 +442 106 +463 106 +499 106 +0 107 +19 107 +43 107 +48 107 +62 107 +72 107 +89 107 +90 107 +136 107 +223 107 +249 107 +282 107 +289 107 +311 107 +314 107 +328 107 +347 107 +448 107 +465 107 +486 107 +499 107 +0 108 +21 108 +37 108 +47 108 +49 108 +56 108 +105 108 +159 108 +196 108 +211 108 +264 108 +275 108 +308 108 +350 108 +368 108 +436 108 +486 108 +496 108 +499 108 +0 109 +33 109 +80 109 +117 109 +161 109 +183 109 +208 109 +240 109 +280 109 +338 109 +359 109 +365 109 +402 109 +419 109 +437 109 +499 109 +0 110 +29 110 +37 110 +47 110 +261 110 +289 110 +299 110 +375 110 +377 110 +378 110 +443 110 +448 110 +499 110 +0 111 +4 111 +30 111 +52 111 +57 111 +85 111 +107 111 +111 111 +115 111 +121 111 +138 111 +191 111 +209 111 +242 111 +265 111 +269 111 +286 111 +319 111 +401 111 +428 111 +473 111 +492 111 +496 111 +499 111 +0 112 +20 112 +56 112 +67 112 +89 112 +100 112 +102 112 +123 112 +150 112 +171 112 +184 112 +257 112 +299 112 +344 112 +349 112 +366 112 +370 112 +376 112 +380 112 +386 112 +387 112 +393 112 +412 112 +430 112 +464 112 +483 112 +493 112 +499 112 +0 113 +2 113 +27 113 +31 113 +45 113 +49 113 +99 113 +103 113 +245 113 +253 113 +293 113 +303 113 +326 113 +340 113 +361 113 +369 113 +389 113 +415 113 +468 113 +485 113 +489 113 +499 113 +0 114 +38 114 +118 114 +135 114 +145 114 +188 114 +212 114 +222 114 +224 114 +227 114 +238 114 +261 114 +322 114 +389 114 +411 114 +427 114 +454 114 +456 114 +495 114 +499 114 +0 115 +28 115 +36 115 +40 115 +48 115 +91 115 +99 115 +103 115 +104 115 +119 115 +121 115 +123 115 +156 115 +176 115 +182 115 +199 115 +219 115 +233 115 +235 115 +252 115 +340 115 +348 115 +353 115 +379 115 +415 115 +418 115 +433 115 +499 115 +0 116 +15 116 +58 116 +93 116 +165 116 +202 116 +214 116 +245 116 +257 116 +283 116 +305 116 +364 116 +395 116 +400 116 +402 116 +413 116 +420 116 +422 116 +479 116 +499 116 +0 117 +9 117 +109 117 +169 117 +182 117 +188 117 +201 117 +216 117 +225 117 +245 117 +268 117 +283 117 +296 117 +330 117 +367 117 +369 117 +372 117 +374 117 +375 117 +400 117 +432 117 +453 117 +462 117 +499 117 +0 118 +11 118 +14 118 +36 118 +58 118 +92 118 +119 118 +150 118 +158 118 +218 118 +232 118 +242 118 +334 118 +351 118 +356 118 +366 118 +375 118 +436 118 +446 118 +453 118 +464 118 +499 118 +0 119 +29 119 +55 119 +77 119 +79 119 +82 119 +89 119 +101 119 +109 119 +117 119 +135 119 +143 119 +146 119 +262 119 +269 119 +281 119 +288 119 +311 119 +314 119 +319 119 +321 119 +327 119 +379 119 +384 119 +442 119 +461 119 +469 119 +499 119 +0 120 +2 120 +73 120 +76 120 +107 120 +136 120 +150 120 +151 120 +165 120 +193 120 +208 120 +223 120 +224 120 +252 120 +312 120 +316 120 +356 120 +389 120 +406 120 +410 120 +435 120 +457 120 +465 120 +499 120 +0 121 +13 121 +74 121 +102 121 +136 121 +137 121 +138 121 +140 121 +169 121 +171 121 +200 121 +222 121 +324 121 +380 121 +388 121 +401 121 +422 121 +430 121 +434 121 +445 121 +492 121 +499 121 +0 122 +78 122 +95 122 +101 122 +145 122 +154 122 +202 122 +236 122 +247 122 +361 122 +366 122 +469 122 +470 122 +477 122 +492 122 +498 122 +499 122 +0 123 +52 123 +56 123 +57 123 +60 123 +92 123 +139 123 +171 123 +181 123 +183 123 +204 123 +208 123 +246 123 +288 123 +295 123 +299 123 +321 123 +331 123 +376 123 +389 123 +399 123 +404 123 +422 123 +448 123 +482 123 +499 123 +0 124 +8 124 +48 124 +54 124 +101 124 +108 124 +152 124 +167 124 +197 124 +216 124 +238 124 +240 124 +250 124 +267 124 +331 124 +342 124 +360 124 +375 124 +380 124 +400 124 +445 124 +461 124 +465 124 +499 124 +0 125 +33 125 +82 125 +83 125 +95 125 +115 125 +154 125 +161 125 +166 125 +176 125 +182 125 +202 125 +206 125 +221 125 +241 125 +244 125 +248 125 +272 125 +273 125 +284 125 +320 125 +377 125 +389 125 +391 125 +394 125 +395 125 +404 125 +421 125 +446 125 +450 125 +461 125 +499 125 +0 126 +3 126 +23 126 +42 126 +66 126 +73 126 +76 126 +113 126 +139 126 +141 126 +160 126 +179 126 +252 126 +280 126 +323 126 +327 126 +399 126 +402 126 +437 126 +499 126 +0 127 +8 127 +85 127 +119 127 +123 127 +173 127 +216 127 +255 127 +276 127 +320 127 +339 127 +369 127 +379 127 +383 127 +418 127 +453 127 +499 127 +0 128 +7 128 +14 128 +22 128 +57 128 +73 128 +138 128 +139 128 +143 128 +171 128 +190 128 +251 128 +253 128 +262 128 +346 128 +390 128 +394 128 +412 128 +441 128 +494 128 +499 128 +0 129 +6 129 +66 129 +69 129 +190 129 +210 129 +232 129 +333 129 +355 129 +356 129 +384 129 +434 129 +453 129 +460 129 +471 129 +499 129 +0 130 +6 130 +25 130 +38 130 +73 130 +74 130 +105 130 +109 130 +122 130 +133 130 +148 130 +220 130 +233 130 +237 130 +323 130 +325 130 +379 130 +429 130 +459 130 +465 130 +484 130 +487 130 +499 130 +0 131 +1 131 +25 131 +47 131 +81 131 +111 131 +120 131 +143 131 +151 131 +161 131 +192 131 +197 131 +290 131 +351 131 +368 131 +404 131 +408 131 +476 131 +480 131 +499 131 +0 132 +41 132 +75 132 +91 132 +114 132 +169 132 +185 132 +188 132 +212 132 +244 132 +262 132 +277 132 +309 132 +320 132 +321 132 +333 132 +351 132 +401 132 +409 132 +440 132 +442 132 +444 132 +499 132 +0 133 +26 133 +31 133 +52 133 +67 133 +83 133 +92 133 +120 133 +155 133 +172 133 +188 133 +209 133 +212 133 +228 133 +237 133 +247 133 +266 133 +313 133 +339 133 +372 133 +417 133 +418 133 +423 133 +428 133 +461 133 +482 133 +499 133 +0 134 +20 134 +24 134 +51 134 +57 134 +66 134 +89 134 +121 134 +132 134 +161 134 +195 134 +207 134 +221 134 +224 134 +228 134 +234 134 +245 134 +266 134 +315 134 +339 134 +371 134 +375 134 +402 134 +408 134 +417 134 +426 134 +495 134 +499 134 +0 135 +11 135 +15 135 +29 135 +33 135 +46 135 +61 135 +70 135 +71 135 +83 135 +167 135 +170 135 +173 135 +196 135 +201 135 +209 135 +218 135 +224 135 +320 135 +342 135 +343 135 +368 135 +414 135 +477 135 +499 135 +0 136 +3 136 +77 136 +90 136 +144 136 +163 136 +178 136 +186 136 +221 136 +237 136 +248 136 +277 136 +280 136 +354 136 +372 136 +377 136 +412 136 +479 136 +499 136 +0 137 +49 137 +59 137 +119 137 +128 137 +131 137 +141 137 +182 137 +186 137 +190 137 +215 137 +227 137 +245 137 +265 137 +291 137 +338 137 +417 137 +428 137 +460 137 +476 137 +478 137 +499 137 +0 138 +35 138 +47 138 +60 138 +67 138 +70 138 +82 138 +92 138 +134 138 +145 138 +151 138 +155 138 +163 138 +165 138 +230 138 +370 138 +388 138 +392 138 +448 138 +462 138 +469 138 +499 138 +0 139 +2 139 +37 139 +46 139 +94 139 +109 139 +125 139 +131 139 +138 139 +156 139 +183 139 +233 139 +260 139 +437 139 +481 139 +483 139 +499 139 +0 140 +14 140 +54 140 +75 140 +79 140 +85 140 +163 140 +206 140 +225 140 +232 140 +265 140 +277 140 +286 140 +295 140 +319 140 +344 140 +375 140 +398 140 +401 140 +402 140 +404 140 +417 140 +450 140 +465 140 +466 140 +485 140 +499 140 +0 141 +98 141 +100 141 +119 141 +134 141 +152 141 +155 141 +182 141 +240 141 +266 141 +293 141 +296 141 +326 141 +341 141 +344 141 +358 141 +379 141 +386 141 +400 141 +402 141 +411 141 +413 141 +415 141 +421 141 +447 141 +499 141 +0 142 +44 142 +95 142 +103 142 +104 142 +123 142 +124 142 +154 142 +202 142 +236 142 +244 142 +257 142 +271 142 +289 142 +329 142 +337 142 +345 142 +375 142 +377 142 +395 142 +424 142 +473 142 +499 142 +0 143 +14 143 +18 143 +40 143 +77 143 +146 143 +173 143 +272 143 +282 143 +285 143 +304 143 +317 143 +400 143 +409 143 +411 143 +450 143 +479 143 +499 143 +0 144 +54 144 +68 144 +77 144 +102 144 +106 144 +122 144 +156 144 +281 144 +293 144 +310 144 +335 144 +356 144 +381 144 +407 144 +410 144 +426 144 +438 144 +443 144 +484 144 +499 144 +0 145 +22 145 +36 145 +107 145 +142 145 +152 145 +160 145 +161 145 +171 145 +184 145 +188 145 +191 145 +216 145 +238 145 +253 145 +256 145 +269 145 +317 145 +338 145 +381 145 +387 145 +389 145 +408 145 +423 145 +472 145 +499 145 +0 146 +2 146 +28 146 +68 146 +122 146 +130 146 +168 146 +365 146 +383 146 +385 146 +394 146 +398 146 +408 146 +420 146 +470 146 +499 146 +0 147 +32 147 +105 147 +110 147 +138 147 +148 147 +152 147 +182 147 +184 147 +193 147 +238 147 +268 147 +272 147 +299 147 +316 147 +330 147 +355 147 +368 147 +370 147 +398 147 +422 147 +430 147 +499 147 +0 148 +7 148 +35 148 +37 148 +71 148 +80 148 +124 148 +150 148 +168 148 +192 148 +217 148 +325 148 +337 148 +340 148 +366 148 +377 148 +381 148 +390 148 +400 148 +407 148 +417 148 +451 148 +462 148 +497 148 +499 148 +0 149 +1 149 +30 149 +40 149 +79 149 +100 149 +109 149 +121 149 +181 149 +192 149 +318 149 +383 149 +406 149 +409 149 +458 149 +463 149 +489 149 +499 149 +0 150 +1 150 +25 150 +103 150 +110 150 +119 150 +123 150 +142 150 +169 150 +185 150 +205 150 +212 150 +241 150 +251 150 +256 150 +258 150 +286 150 +324 150 +325 150 +350 150 +456 150 +499 150 +0 151 +13 151 +20 151 +34 151 +122 151 +153 151 +284 151 +286 151 +312 151 +338 151 +371 151 +393 151 +403 151 +418 151 +420 151 +425 151 +462 151 +481 151 +499 151 +0 152 +28 152 +45 152 +49 152 +58 152 +85 152 +90 152 +121 152 +135 152 +165 152 +183 152 +197 152 +221 152 +239 152 +249 152 +266 152 +308 152 +328 152 +333 152 +342 152 +355 152 +365 152 +391 152 +408 152 +422 152 +424 152 +450 152 +479 152 +487 152 +499 152 +0 153 +21 153 +72 153 +85 153 +88 153 +152 153 +154 153 +159 153 +225 153 +286 153 +342 153 +353 153 +373 153 +454 153 +471 153 +488 153 +499 153 +0 154 +9 154 +47 154 +48 154 +67 154 +91 154 +104 154 +126 154 +136 154 +269 154 +301 154 +331 154 +341 154 +362 154 +373 154 +394 154 +400 154 +402 154 +434 154 +499 154 +0 155 +16 155 +25 155 +35 155 +89 155 +93 155 +116 155 +124 155 +147 155 +155 155 +185 155 +235 155 +252 155 +307 155 +312 155 +335 155 +354 155 +366 155 +371 155 +411 155 +413 155 +424 155 +441 155 +474 155 +499 155 +0 156 +65 156 +66 156 +81 156 +84 156 +85 156 +146 156 +167 156 +191 156 +234 156 +257 156 +285 156 +286 156 +293 156 +294 156 +315 156 +320 156 +324 156 +340 156 +347 156 +352 156 +396 156 +399 156 +406 156 +434 156 +437 156 +446 156 +449 156 +450 156 +486 156 +499 156 +0 157 +2 157 +6 157 +21 157 +25 157 +61 157 +82 157 +87 157 +135 157 +147 157 +192 157 +202 157 +231 157 +273 157 +288 157 +296 157 +316 157 +317 157 +337 157 +348 157 +357 157 +371 157 +422 157 +426 157 +457 157 +489 157 +499 157 +0 158 +22 158 +32 158 +57 158 +67 158 +124 158 +157 158 +209 158 +221 158 +249 158 +260 158 +268 158 +291 158 +331 158 +353 158 +367 158 +416 158 +482 158 +498 158 +499 158 +0 159 +29 159 +51 159 +87 159 +98 159 +103 159 +132 159 +188 159 +247 159 +251 159 +258 159 +265 159 +275 159 +332 159 +380 159 +419 159 +467 159 +470 159 +476 159 +499 159 +0 160 +24 160 +28 160 +31 160 +179 160 +183 160 +192 160 +206 160 +211 160 +242 160 +245 160 +276 160 +290 160 +320 160 +335 160 +358 160 +367 160 +372 160 +462 160 +482 160 +499 160 +0 161 +1 161 +16 161 +42 161 +51 161 +54 161 +109 161 +126 161 +137 161 +143 161 +145 161 +156 161 +157 161 +165 161 +261 161 +311 161 +335 161 +353 161 +354 161 +379 161 +382 161 +441 161 +499 161 +0 162 +23 162 +29 162 +32 162 +52 162 +59 162 +60 162 +113 162 +141 162 +145 162 +152 162 +153 162 +186 162 +192 162 +242 162 +257 162 +311 162 +324 162 +328 162 +345 162 +410 162 +434 162 +439 162 +442 162 +454 162 +456 162 +466 162 +485 162 +486 162 +492 162 +499 162 +0 163 +24 163 +35 163 +49 163 +54 163 +96 163 +191 163 +251 163 +262 163 +280 163 +296 163 +424 163 +425 163 +442 163 +499 163 +0 164 +12 164 +61 164 +111 164 +181 164 +186 164 +226 164 +245 164 +275 164 +294 164 +305 164 +323 164 +360 164 +401 164 +450 164 +469 164 +472 164 +478 164 +480 164 +499 164 +0 165 +4 165 +14 165 +31 165 +61 165 +92 165 +95 165 +168 165 +171 165 +181 165 +190 165 +199 165 +214 165 +221 165 +243 165 +310 165 +312 165 +335 165 +397 165 +402 165 +472 165 +497 165 +499 165 +0 166 +23 166 +24 166 +39 166 +116 166 +133 166 +135 166 +186 166 +202 166 +220 166 +293 166 +335 166 +366 166 +372 166 +382 166 +407 166 +412 166 +453 166 +499 166 +0 167 +41 167 +52 167 +70 167 +76 167 +109 167 +113 167 +126 167 +162 167 +169 167 +210 167 +274 167 +323 167 +360 167 +374 167 +393 167 +395 167 +435 167 +488 167 +491 167 +499 167 +0 168 +3 168 +9 168 +11 168 +39 168 +49 168 +72 168 +88 168 +96 168 +104 168 +151 168 +162 168 +169 168 +197 168 +277 168 +337 168 +357 168 +358 168 +368 168 +394 168 +444 168 +446 168 +458 168 +499 168 +0 169 +31 169 +43 169 +87 169 +97 169 +104 169 +107 169 +116 169 +118 169 +181 169 +203 169 +215 169 +234 169 +248 169 +252 169 +269 169 +278 169 +307 169 +313 169 +319 169 +350 169 +379 169 +392 169 +415 169 +431 169 +457 169 +496 169 +499 169 +0 170 +10 170 +18 170 +28 170 +47 170 +58 170 +60 170 +79 170 +112 170 +175 170 +181 170 +228 170 +259 170 +262 170 +269 170 +337 170 +385 170 +410 170 +439 170 +460 170 +471 170 +499 170 +0 171 +5 171 +56 171 +73 171 +101 171 +155 171 +159 171 +192 171 +216 171 +245 171 +264 171 +278 171 +299 171 +302 171 +311 171 +367 171 +384 171 +392 171 +399 171 +426 171 +464 171 +483 171 +499 171 +0 172 +5 172 +16 172 +46 172 +78 172 +113 172 +115 172 +152 172 +168 172 +180 172 +190 172 +214 172 +314 172 +332 172 +346 172 +415 172 +450 172 +468 172 +499 172 +0 173 +4 173 +26 173 +46 173 +67 173 +72 173 +76 173 +79 173 +117 173 +148 173 +179 173 +218 173 +231 173 +317 173 +321 173 +372 173 +381 173 +404 173 +455 173 +469 173 +471 173 +474 173 +490 173 +499 173 +0 174 +39 174 +60 174 +74 174 +96 174 +106 174 +162 174 +164 174 +175 174 +194 174 +240 174 +261 174 +271 174 +313 174 +341 174 +344 174 +358 174 +363 174 +380 174 +393 174 +406 174 +426 174 +459 174 +473 174 +481 174 +485 174 +499 174 +0 175 +25 175 +32 175 +69 175 +114 175 +158 175 +164 175 +252 175 +256 175 +320 175 +348 175 +383 175 +394 175 +397 175 +401 175 +428 175 +432 175 +442 175 +445 175 +471 175 +482 175 +493 175 +499 175 +0 176 +24 176 +29 176 +44 176 +55 176 +104 176 +105 176 +108 176 +111 176 +141 176 +167 176 +191 176 +206 176 +235 176 +311 176 +339 176 +462 176 +499 176 +0 177 +92 177 +144 177 +158 177 +220 177 +242 177 +243 177 +278 177 +286 177 +322 177 +333 177 +339 177 +392 177 +395 177 +406 177 +445 177 +453 177 +454 177 +461 177 +499 177 +0 178 +14 178 +53 178 +73 178 +110 178 +122 178 +131 178 +149 178 +180 178 +183 178 +236 178 +251 178 +270 178 +340 178 +499 178 +0 179 +24 179 +26 179 +47 179 +93 179 +165 179 +183 179 +200 179 +207 179 +230 179 +231 179 +233 179 +242 179 +248 179 +270 179 +340 179 +342 179 +359 179 +415 179 +424 179 +453 179 +472 179 +499 179 +0 180 +1 180 +2 180 +30 180 +64 180 +77 180 +79 180 +87 180 +115 180 +142 180 +149 180 +153 180 +189 180 +194 180 +202 180 +215 180 +230 180 +272 180 +276 180 +281 180 +326 180 +397 180 +431 180 +445 180 +448 180 +499 180 +0 181 +1 181 +28 181 +66 181 +91 181 +130 181 +156 181 +165 181 +194 181 +231 181 +301 181 +314 181 +315 181 +327 181 +334 181 +366 181 +438 181 +450 181 +467 181 +490 181 +495 181 +499 181 +0 182 +14 182 +43 182 +141 182 +143 182 +144 182 +169 182 +180 182 +181 182 +251 182 +273 182 +291 182 +324 182 +332 182 +342 182 +412 182 +432 182 +436 182 +459 182 +490 182 +499 182 +0 183 +59 183 +70 183 +109 183 +111 183 +149 183 +175 183 +200 183 +209 183 +313 183 +331 183 +344 183 +359 183 +361 183 +368 183 +400 183 +414 183 +441 183 +473 183 +499 183 +0 184 +20 184 +45 184 +71 184 +112 184 +130 184 +137 184 +157 184 +161 184 +241 184 +242 184 +245 184 +251 184 +273 184 +336 184 +341 184 +346 184 +422 184 +451 184 +479 184 +489 184 +491 184 +499 184 +0 185 +37 185 +45 185 +112 185 +131 185 +155 185 +209 185 +220 185 +247 185 +254 185 +259 185 +297 185 +363 185 +453 185 +480 185 +499 185 +0 186 +23 186 +30 186 +65 186 +86 186 +92 186 +94 186 +100 186 +126 186 +142 186 +179 186 +238 186 +261 186 +266 186 +281 186 +296 186 +303 186 +328 186 +329 186 +347 186 +354 186 +366 186 +368 186 +376 186 +378 186 +397 186 +414 186 +461 186 +499 186 +0 187 +40 187 +71 187 +158 187 +162 187 +171 187 +207 187 +274 187 +278 187 +282 187 +306 187 +309 187 +348 187 +350 187 +404 187 +407 187 +440 187 +453 187 +499 187 +0 188 +4 188 +12 188 +20 188 +71 188 +92 188 +140 188 +203 188 +220 188 +252 188 +301 188 +317 188 +320 188 +368 188 +397 188 +400 188 +425 188 +499 188 +0 189 +11 189 +30 189 +36 189 +61 189 +94 189 +164 189 +170 189 +174 189 +234 189 +282 189 +309 189 +347 189 +383 189 +386 189 +426 189 +435 189 +463 189 +466 189 +485 189 +499 189 +0 190 +14 190 +19 190 +24 190 +95 190 +108 190 +123 190 +130 190 +144 190 +156 190 +158 190 +193 190 +212 190 +233 190 +264 190 +279 190 +348 190 +349 190 +353 190 +403 190 +412 190 +423 190 +430 190 +434 190 +459 190 +487 190 +490 190 +499 190 +0 191 +23 191 +30 191 +46 191 +48 191 +62 191 +125 191 +145 191 +168 191 +179 191 +203 191 +217 191 +221 191 +232 191 +259 191 +267 191 +288 191 +290 191 +293 191 +304 191 +319 191 +360 191 +382 191 +438 191 +444 191 +445 191 +457 191 +459 191 +487 191 +499 191 +0 192 +44 192 +74 192 +92 192 +127 192 +136 192 +148 192 +151 192 +167 192 +204 192 +224 192 +245 192 +247 192 +253 192 +271 192 +289 192 +291 192 +301 192 +310 192 +344 192 +348 192 +356 192 +372 192 +398 192 +400 192 +419 192 +456 192 +467 192 +470 192 +491 192 +499 192 +0 193 +33 193 +54 193 +83 193 +160 193 +226 193 +244 193 +254 193 +303 193 +332 193 +366 193 +408 193 +422 193 +425 193 +440 193 +443 193 +449 193 +458 193 +488 193 +492 193 +499 193 +0 194 +3 194 +16 194 +33 194 +34 194 +92 194 +108 194 +124 194 +129 194 +141 194 +145 194 +163 194 +178 194 +217 194 +235 194 +292 194 +338 194 +367 194 +378 194 +399 194 +416 194 +432 194 +499 194 +0 195 +3 195 +12 195 +33 195 +65 195 +118 195 +188 195 +211 195 +243 195 +286 195 +293 195 +294 195 +310 195 +337 195 +342 195 +379 195 +388 195 +440 195 +477 195 +490 195 +495 195 +499 195 +0 196 +10 196 +46 196 +64 196 +78 196 +89 196 +96 196 +105 196 +142 196 +216 196 +217 196 +224 196 +239 196 +241 196 +246 196 +280 196 +302 196 +386 196 +443 196 +449 196 +451 196 +452 196 +477 196 +486 196 +499 196 +0 197 +4 197 +18 197 +42 197 +79 197 +87 197 +95 197 +100 197 +174 197 +185 197 +239 197 +242 197 +292 197 +392 197 +397 197 +398 197 +416 197 +427 197 +429 197 +435 197 +441 197 +446 197 +450 197 +463 197 +471 197 +499 197 +0 198 +51 198 +66 198 +89 198 +144 198 +206 198 +212 198 +221 198 +222 198 +232 198 +241 198 +253 198 +257 198 +306 198 +322 198 +323 198 +345 198 +352 198 +382 198 +413 198 +416 198 +425 198 +442 198 +453 198 +499 198 +0 199 +24 199 +42 199 +59 199 +64 199 +72 199 +86 199 +98 199 +113 199 +125 199 +127 199 +155 199 +157 199 +176 199 +217 199 +227 199 +234 199 +294 199 +358 199 +370 199 +420 199 +433 199 +473 199 +487 199 +499 199 +0 200 +8 200 +21 200 +22 200 +36 200 +41 200 +51 200 +54 200 +56 200 +68 200 +164 200 +169 200 +205 200 +216 200 +237 200 +255 200 +282 200 +351 200 +373 200 +410 200 +428 200 +430 200 +433 200 +499 200 +0 201 +6 201 +62 201 +82 201 +92 201 +106 201 +123 201 +190 201 +270 201 +306 201 +350 201 +369 201 +390 201 +391 201 +407 201 +415 201 +416 201 +451 201 +480 201 +499 201 +0 202 +3 202 +13 202 +67 202 +99 202 +171 202 +172 202 +206 202 +214 202 +345 202 +349 202 +367 202 +399 202 +406 202 +469 202 +483 202 +495 202 +499 202 +0 203 +19 203 +39 203 +98 203 +131 203 +170 203 +198 203 +208 203 +209 203 +218 203 +261 203 +262 203 +266 203 +267 203 +272 203 +284 203 +293 203 +308 203 +335 203 +345 203 +377 203 +381 203 +397 203 +423 203 +440 203 +444 203 +499 203 +0 204 +57 204 +64 204 +86 204 +133 204 +157 204 +177 204 +189 204 +248 204 +266 204 +271 204 +273 204 +275 204 +339 204 +381 204 +420 204 +450 204 +455 204 +470 204 +471 204 +498 204 +499 204 +0 205 +11 205 +18 205 +37 205 +42 205 +84 205 +135 205 +136 205 +193 205 +242 205 +276 205 +282 205 +311 205 +363 205 +365 205 +399 205 +442 205 +448 205 +457 205 +499 205 +0 206 +5 206 +50 206 +63 206 +64 206 +72 206 +85 206 +91 206 +106 206 +131 206 +144 206 +157 206 +173 206 +204 206 +209 206 +276 206 +295 206 +317 206 +335 206 +363 206 +425 206 +427 206 +438 206 +445 206 +446 206 +499 206 +0 207 +65 207 +70 207 +85 207 +89 207 +107 207 +161 207 +165 207 +196 207 +200 207 +208 207 +274 207 +318 207 +406 207 +423 207 +431 207 +466 207 +499 207 +0 208 +1 208 +44 208 +46 208 +76 208 +81 208 +109 208 +197 208 +216 208 +240 208 +269 208 +281 208 +306 208 +313 208 +326 208 +328 208 +343 208 +376 208 +377 208 +381 208 +390 208 +413 208 +450 208 +462 208 +469 208 +495 208 +499 208 +0 209 +8 209 +41 209 +49 209 +69 209 +104 209 +135 209 +148 209 +185 209 +226 209 +290 209 +343 209 +391 209 +435 209 +458 209 +499 209 +0 210 +4 210 +11 210 +35 210 +68 210 +114 210 +116 210 +136 210 +178 210 +193 210 +230 210 +303 210 +332 210 +340 210 +352 210 +363 210 +407 210 +449 210 +499 210 +0 211 +20 211 +113 211 +209 211 +282 211 +321 211 +327 211 +379 211 +409 211 +422 211 +448 211 +462 211 +469 211 +479 211 +491 211 +499 211 +0 212 +2 212 +14 212 +84 212 +113 212 +124 212 +128 212 +131 212 +147 212 +175 212 +177 212 +211 212 +234 212 +248 212 +276 212 +289 212 +302 212 +319 212 +340 212 +344 212 +366 212 +374 212 +415 212 +435 212 +469 212 +471 212 +496 212 +499 212 +0 213 +17 213 +42 213 +122 213 +132 213 +156 213 +178 213 +209 213 +238 213 +288 213 +293 213 +309 213 +352 213 +360 213 +399 213 +406 213 +409 213 +439 213 +499 213 +0 214 +12 214 +44 214 +68 214 +122 214 +168 214 +222 214 +242 214 +304 214 +308 214 +311 214 +328 214 +379 214 +381 214 +404 214 +427 214 +456 214 +499 214 +0 215 +22 215 +26 215 +82 215 +98 215 +139 215 +202 215 +221 215 +229 215 +241 215 +244 215 +264 215 +281 215 +292 215 +320 215 +371 215 +392 215 +413 215 +452 215 +462 215 +466 215 +483 215 +491 215 +499 215 +0 216 +3 216 +57 216 +90 216 +97 216 +103 216 +143 216 +160 216 +162 216 +185 216 +198 216 +201 216 +227 216 +257 216 +267 216 +269 216 +328 216 +330 216 +337 216 +411 216 +430 216 +458 216 +482 216 +490 216 +499 216 +0 217 +8 217 +9 217 +34 217 +83 217 +108 217 +113 217 +127 217 +137 217 +188 217 +197 217 +211 217 +222 217 +237 217 +244 217 +259 217 +271 217 +279 217 +299 217 +301 217 +311 217 +315 217 +316 217 +318 217 +321 217 +323 217 +338 217 +346 217 +371 217 +416 217 +418 217 +436 217 +442 217 +473 217 +481 217 +499 217 +0 218 +1 218 +70 218 +77 218 +104 218 +149 218 +150 218 +182 218 +236 218 +244 218 +271 218 +273 218 +321 218 +349 218 +488 218 +499 218 +0 219 +89 219 +102 219 +117 219 +216 219 +222 219 +229 219 +273 219 +277 219 +320 219 +391 219 +419 219 +422 219 +452 219 +484 219 +499 219 +0 220 +30 220 +51 220 +63 220 +68 220 +79 220 +82 220 +88 220 +112 220 +125 220 +165 220 +172 220 +175 220 +182 220 +186 220 +198 220 +199 220 +223 220 +233 220 +245 220 +250 220 +265 220 +270 220 +288 220 +292 220 +354 220 +362 220 +400 220 +428 220 +430 220 +492 220 +499 220 +0 221 +14 221 +47 221 +54 221 +79 221 +87 221 +129 221 +142 221 +157 221 +158 221 +189 221 +200 221 +203 221 +249 221 +251 221 +298 221 +302 221 +304 221 +342 221 +360 221 +420 221 +456 221 +457 221 +472 221 +486 221 +499 221 +0 222 +22 222 +47 222 +56 222 +74 222 +110 222 +120 222 +123 222 +159 222 +180 222 +182 222 +188 222 +267 222 +305 222 +306 222 +309 222 +442 222 +444 222 +449 222 +451 222 +459 222 +462 222 +464 222 +499 222 +0 223 +57 223 +68 223 +98 223 +122 223 +126 223 +161 223 +190 223 +254 223 +315 223 +347 223 +377 223 +404 223 +426 223 +441 223 +461 223 +463 223 +490 223 +499 223 +0 224 +32 224 +39 224 +87 224 +91 224 +119 224 +144 224 +150 224 +161 224 +181 224 +199 224 +216 224 +237 224 +246 224 +249 224 +263 224 +283 224 +291 224 +296 224 +319 224 +328 224 +354 224 +373 224 +402 224 +419 224 +464 224 +474 224 +476 224 +499 224 +0 225 +48 225 +105 225 +112 225 +222 225 +230 225 +244 225 +282 225 +288 225 +334 225 +341 225 +392 225 +425 225 +426 225 +452 225 +499 225 +0 226 +43 226 +56 226 +61 226 +99 226 +109 226 +173 226 +176 226 +183 226 +192 226 +195 226 +197 226 +217 226 +248 226 +263 226 +284 226 +289 226 +303 226 +346 226 +404 226 +409 226 +434 226 +499 226 +0 227 +39 227 +73 227 +118 227 +141 227 +150 227 +153 227 +210 227 +236 227 +247 227 +282 227 +361 227 +386 227 +393 227 +408 227 +474 227 +485 227 +491 227 +493 227 +499 227 +0 228 +38 228 +48 228 +74 228 +97 228 +143 228 +196 228 +206 228 +217 228 +251 228 +279 228 +282 228 +293 228 +347 228 +368 228 +399 228 +416 228 +438 228 +448 228 +494 228 +499 228 +0 229 +10 229 +17 229 +39 229 +66 229 +97 229 +109 229 +116 229 +128 229 +147 229 +164 229 +202 229 +288 229 +337 229 +370 229 +390 229 +419 229 +470 229 +475 229 +492 229 +499 229 +0 230 +19 230 +22 230 +65 230 +91 230 +98 230 +116 230 +138 230 +153 230 +156 230 +163 230 +172 230 +189 230 +197 230 +219 230 +234 230 +241 230 +246 230 +255 230 +312 230 +330 230 +383 230 +478 230 +499 230 +0 231 +10 231 +37 231 +69 231 +71 231 +118 231 +119 231 +155 231 +219 231 +275 231 +277 231 +332 231 +339 231 +344 231 +361 231 +373 231 +374 231 +418 231 +479 231 +498 231 +499 231 +0 232 +23 232 +88 232 +106 232 +130 232 +134 232 +147 232 +151 232 +166 232 +175 232 +211 232 +227 232 +255 232 +268 232 +281 232 +322 232 +327 232 +376 232 +392 232 +393 232 +406 232 +475 232 +498 232 +499 232 +0 233 +6 233 +17 233 +120 233 +174 233 +216 233 +217 233 +253 233 +286 233 +292 233 +304 233 +386 233 +435 233 +439 233 +490 233 +499 233 +0 234 +47 234 +78 234 +88 234 +106 234 +159 234 +179 234 +193 234 +246 234 +257 234 +280 234 +283 234 +296 234 +299 234 +335 234 +377 234 +400 234 +457 234 +472 234 +499 234 +0 235 +23 235 +48 235 +91 235 +109 235 +152 235 +166 235 +174 235 +176 235 +233 235 +272 235 +320 235 +354 235 +366 235 +371 235 +374 235 +381 235 +414 235 +416 235 +433 235 +468 235 +472 235 +499 235 +0 236 +23 236 +62 236 +66 236 +80 236 +157 236 +184 236 +199 236 +203 236 +217 236 +242 236 +297 236 +338 236 +347 236 +355 236 +421 236 +463 236 +476 236 +486 236 +499 236 +0 237 +21 237 +29 237 +45 237 +47 237 +72 237 +78 237 +93 237 +98 237 +102 237 +140 237 +143 237 +147 237 +149 237 +188 237 +227 237 +289 237 +302 237 +318 237 +343 237 +347 237 +367 237 +386 237 +410 237 +434 237 +438 237 +458 237 +486 237 +490 237 +499 237 +0 238 +4 238 +5 238 +11 238 +96 238 +108 238 +113 238 +132 238 +150 238 +151 238 +190 238 +191 238 +241 238 +244 238 +264 238 +265 238 +284 238 +330 238 +347 238 +371 238 +386 238 +388 238 +402 238 +436 238 +446 238 +477 238 +484 238 +499 238 +0 239 +6 239 +12 239 +47 239 +66 239 +74 239 +78 239 +80 239 +150 239 +177 239 +178 239 +179 239 +199 239 +240 239 +244 239 +247 239 +282 239 +331 239 +333 239 +337 239 +369 239 +382 239 +403 239 +408 239 +423 239 +436 239 +499 239 +0 240 +10 240 +25 240 +39 240 +48 240 +67 240 +122 240 +134 240 +143 240 +162 240 +167 240 +172 240 +173 240 +203 240 +206 240 +209 240 +210 240 +216 240 +239 240 +246 240 +277 240 +362 240 +371 240 +379 240 +419 240 +447 240 +460 240 +479 240 +499 240 +0 241 +3 241 +24 241 +49 241 +85 241 +91 241 +121 241 +126 241 +147 241 +167 241 +169 241 +215 241 +217 241 +262 241 +264 241 +266 241 +308 241 +371 241 +388 241 +483 241 +488 241 +499 241 +0 242 +23 242 +26 242 +30 242 +112 242 +127 242 +132 242 +170 242 +174 242 +178 242 +204 242 +225 242 +243 242 +282 242 +285 242 +312 242 +320 242 +349 242 +463 242 +499 242 +0 243 +40 243 +63 243 +126 243 +161 243 +170 243 +182 243 +221 243 +225 243 +291 243 +293 243 +324 243 +342 243 +355 243 +486 243 +499 243 +0 244 +7 244 +20 244 +47 244 +59 244 +60 244 +63 244 +73 244 +122 244 +161 244 +200 244 +276 244 +280 244 +316 244 +321 244 +337 244 +362 244 +366 244 +499 244 +0 245 +19 245 +65 245 +87 245 +91 245 +105 245 +108 245 +221 245 +255 245 +306 245 +330 245 +338 245 +428 245 +442 245 +474 245 +486 245 +499 245 +0 246 +13 246 +40 246 +72 246 +78 246 +104 246 +152 246 +170 246 +171 246 +247 246 +277 246 +287 246 +398 246 +400 246 +402 246 +445 246 +449 246 +457 246 +495 246 +499 246 +0 247 +6 247 +35 247 +37 247 +44 247 +54 247 +63 247 +76 247 +137 247 +167 247 +179 247 +189 247 +194 247 +215 247 +223 247 +249 247 +257 247 +262 247 +296 247 +317 247 +355 247 +373 247 +387 247 +413 247 +449 247 +466 247 +474 247 +499 247 +0 248 +24 248 +55 248 +63 248 +70 248 +72 248 +104 248 +135 248 +141 248 +149 248 +161 248 +187 248 +203 248 +206 248 +238 248 +243 248 +257 248 +262 248 +319 248 +322 248 +338 248 +390 248 +402 248 +423 248 +446 248 +451 248 +460 248 +483 248 +499 248 +0 249 +4 249 +14 249 +27 249 +34 249 +41 249 +62 249 +69 249 +88 249 +130 249 +157 249 +170 249 +195 249 +197 249 +215 249 +221 249 +226 249 +227 249 +255 249 +293 249 +303 249 +324 249 +325 249 +335 249 +368 249 +386 249 +404 249 +467 249 +488 249 +499 249 +0 250 +28 250 +83 250 +153 250 +206 250 +237 250 +248 250 +252 250 +271 250 +338 250 +345 250 +388 250 +436 250 +441 250 +443 250 +453 250 +462 250 +466 250 +484 250 +499 250 +0 251 +15 251 +53 251 +83 251 +104 251 +109 251 +160 251 +177 251 +202 251 +217 251 +223 251 +233 251 +241 251 +329 251 +346 251 +352 251 +354 251 +388 251 +391 251 +406 251 +423 251 +466 251 +489 251 +496 251 +498 251 +499 251 +0 252 +18 252 +28 252 +80 252 +92 252 +99 252 +111 252 +133 252 +144 252 +196 252 +207 252 +231 252 +242 252 +295 252 +309 252 +353 252 +444 252 +447 252 +449 252 +459 252 +483 252 +488 252 +499 252 +0 253 +8 253 +70 253 +127 253 +156 253 +183 253 +193 253 +217 253 +327 253 +331 253 +362 253 +392 253 +477 253 +499 253 +0 254 +17 254 +21 254 +70 254 +123 254 +128 254 +150 254 +155 254 +184 254 +200 254 +232 254 +278 254 +313 254 +324 254 +346 254 +386 254 +416 254 +442 254 +499 254 +0 255 +26 255 +48 255 +180 255 +198 255 +204 255 +207 255 +230 255 +234 255 +318 255 +342 255 +364 255 +395 255 +455 255 +489 255 +490 255 +499 255 +0 256 +34 256 +62 256 +85 256 +88 256 +91 256 +96 256 +98 256 +174 256 +178 256 +209 256 +210 256 +253 256 +280 256 +302 256 +328 256 +382 256 +405 256 +409 256 +423 256 +453 256 +486 256 +490 256 +493 256 +495 256 +499 256 +0 257 +6 257 +57 257 +58 257 +71 257 +74 257 +82 257 +137 257 +158 257 +179 257 +182 257 +238 257 +259 257 +343 257 +349 257 +356 257 +359 257 +427 257 +433 257 +446 257 +452 257 +455 257 +464 257 +499 257 +0 258 +6 258 +66 258 +91 258 +98 258 +102 258 +203 258 +204 258 +282 258 +291 258 +357 258 +363 258 +432 258 +443 258 +482 258 +494 258 +499 258 +0 259 +5 259 +51 259 +78 259 +84 259 +117 259 +155 259 +157 259 +179 259 +193 259 +227 259 +232 259 +233 259 +236 259 +243 259 +320 259 +330 259 +348 259 +365 259 +389 259 +411 259 +496 259 +499 259 +0 260 +105 260 +130 260 +173 260 +197 260 +207 260 +223 260 +237 260 +243 260 +333 260 +348 260 +351 260 +366 260 +383 260 +417 260 +423 260 +499 260 +0 261 +15 261 +22 261 +51 261 +58 261 +62 261 +64 261 +83 261 +89 261 +100 261 +124 261 +138 261 +153 261 +156 261 +162 261 +167 261 +189 261 +214 261 +277 261 +286 261 +299 261 +304 261 +306 261 +312 261 +317 261 +347 261 +350 261 +403 261 +463 261 +499 261 +0 262 +9 262 +24 262 +38 262 +70 262 +78 262 +119 262 +196 262 +198 262 +210 262 +240 262 +254 262 +268 262 +300 262 +319 262 +323 262 +343 262 +347 262 +348 262 +374 262 +389 262 +468 262 +472 262 +492 262 +499 262 +0 263 +3 263 +15 263 +80 263 +110 263 +118 263 +210 263 +218 263 +234 263 +283 263 +286 263 +302 263 +320 263 +417 263 +423 263 +430 263 +465 263 +476 263 +483 263 +491 263 +494 263 +499 263 +0 264 +7 264 +49 264 +70 264 +86 264 +94 264 +147 264 +151 264 +167 264 +177 264 +198 264 +210 264 +219 264 +227 264 +246 264 +261 264 +263 264 +280 264 +289 264 +290 264 +292 264 +314 264 +339 264 +405 264 +436 264 +472 264 +499 264 +0 265 +8 265 +9 265 +18 265 +33 265 +40 265 +42 265 +99 265 +107 265 +155 265 +183 265 +245 265 +304 265 +323 265 +417 265 +423 265 +426 265 +431 265 +436 265 +442 265 +444 265 +499 265 +0 266 +9 266 +32 266 +77 266 +79 266 +88 266 +93 266 +158 266 +200 266 +202 266 +207 266 +236 266 +256 266 +346 266 +355 266 +372 266 +397 266 +411 266 +440 266 +464 266 +469 266 +499 266 +0 267 +28 267 +81 267 +104 267 +108 267 +119 267 +130 267 +161 267 +179 267 +252 267 +264 267 +297 267 +315 267 +329 267 +330 267 +348 267 +411 267 +423 267 +444 267 +447 267 +449 267 +499 267 +0 268 +7 268 +8 268 +10 268 +13 268 +60 268 +74 268 +78 268 +83 268 +118 268 +125 268 +165 268 +174 268 +199 268 +214 268 +233 268 +234 268 +258 268 +266 268 +271 268 +277 268 +317 268 +330 268 +334 268 +335 268 +337 268 +363 268 +400 268 +407 268 +412 268 +424 268 +425 268 +465 268 +496 268 +499 268 +0 269 +25 269 +29 269 +43 269 +92 269 +93 269 +100 269 +136 269 +154 269 +160 269 +201 269 +206 269 +305 269 +307 269 +319 269 +325 269 +375 269 +403 269 +499 269 +0 270 +31 270 +47 270 +48 270 +77 270 +128 270 +140 270 +149 270 +172 270 +217 270 +250 270 +258 270 +263 270 +268 270 +306 270 +326 270 +371 270 +380 270 +409 270 +423 270 +430 270 +468 270 +473 270 +499 270 +0 271 +45 271 +73 271 +75 271 +84 271 +99 271 +150 271 +180 271 +235 271 +241 271 +242 271 +259 271 +264 271 +265 271 +295 271 +320 271 +321 271 +438 271 +461 271 +499 271 +0 272 +31 272 +37 272 +126 272 +164 272 +245 272 +296 272 +314 272 +325 272 +337 272 +435 272 +436 272 +443 272 +446 272 +499 272 +0 273 +38 273 +46 273 +55 273 +66 273 +69 273 +75 273 +81 273 +85 273 +108 273 +126 273 +183 273 +229 273 +259 273 +273 273 +290 273 +304 273 +360 273 +377 273 +414 273 +457 273 +472 273 +487 273 +499 273 +0 274 +20 274 +39 274 +74 274 +90 274 +107 274 +112 274 +140 274 +226 274 +240 274 +286 274 +296 274 +306 274 +314 274 +349 274 +356 274 +377 274 +394 274 +405 274 +431 274 +441 274 +495 274 +499 274 +0 275 +14 275 +39 275 +68 275 +78 275 +96 275 +108 275 +123 275 +153 275 +176 275 +201 275 +209 275 +303 275 +320 275 +362 275 +412 275 +431 275 +461 275 +499 275 +0 276 +9 276 +88 276 +91 276 +98 276 +131 276 +135 276 +216 276 +231 276 +235 276 +259 276 +265 276 +284 276 +393 276 +406 276 +484 276 +486 276 +499 276 +0 277 +35 277 +63 277 +170 277 +232 277 +248 277 +278 277 +281 277 +304 277 +352 277 +356 277 +360 277 +375 277 +392 277 +397 277 +426 277 +437 277 +465 277 +473 277 +474 277 +499 277 +0 278 +41 278 +83 278 +119 278 +136 278 +144 278 +167 278 +188 278 +235 278 +267 278 +287 278 +318 278 +349 278 +358 278 +382 278 +389 278 +428 278 +434 278 +444 278 +457 278 +499 278 +0 279 +53 279 +59 279 +118 279 +148 279 +214 279 +233 279 +250 279 +252 279 +290 279 +306 279 +323 279 +345 279 +364 279 +397 279 +455 279 +472 279 +474 279 +499 279 +0 280 +62 280 +95 280 +106 280 +115 280 +126 280 +140 280 +166 280 +192 280 +197 280 +233 280 +240 280 +257 280 +317 280 +318 280 +321 280 +340 280 +345 280 +377 280 +477 280 +480 280 +482 280 +499 280 +0 281 +20 281 +51 281 +54 281 +56 281 +57 281 +64 281 +99 281 +103 281 +110 281 +112 281 +133 281 +155 281 +175 281 +181 281 +196 281 +225 281 +274 281 +290 281 +294 281 +377 281 +380 281 +393 281 +399 281 +406 281 +418 281 +423 281 +435 281 +436 281 +459 281 +480 281 +495 281 +499 281 +0 282 +31 282 +42 282 +68 282 +101 282 +135 282 +141 282 +144 282 +213 282 +251 282 +295 282 +337 282 +355 282 +373 282 +406 282 +454 282 +472 282 +491 282 +496 282 +499 282 +0 283 +7 283 +48 283 +112 283 +165 283 +191 283 +212 283 +234 283 +264 283 +290 283 +298 283 +325 283 +332 283 +354 283 +387 283 +395 283 +404 283 +406 283 +409 283 +424 283 +431 283 +460 283 +464 283 +488 283 +499 283 +0 284 +22 284 +37 284 +95 284 +101 284 +102 284 +113 284 +128 284 +137 284 +190 284 +204 284 +219 284 +226 284 +231 284 +293 284 +322 284 +345 284 +372 284 +405 284 +425 284 +438 284 +441 284 +460 284 +499 284 +0 285 +22 285 +58 285 +94 285 +106 285 +125 285 +150 285 +176 285 +225 285 +232 285 +245 285 +246 285 +331 285 +372 285 +383 285 +402 285 +414 285 +499 285 +0 286 +70 286 +74 286 +89 286 +93 286 +113 286 +168 286 +170 286 +229 286 +248 286 +257 286 +275 286 +301 286 +359 286 +384 286 +445 286 +476 286 +499 286 +0 287 +5 287 +58 287 +89 287 +136 287 +149 287 +153 287 +185 287 +236 287 +250 287 +263 287 +274 287 +278 287 +363 287 +370 287 +414 287 +437 287 +450 287 +464 287 +466 287 +499 287 +0 288 +52 288 +53 288 +123 288 +134 288 +167 288 +179 288 +189 288 +191 288 +210 288 +234 288 +280 288 +297 288 +298 288 +347 288 +350 288 +384 288 +389 288 +420 288 +468 288 +485 288 +499 288 +0 289 +7 289 +18 289 +47 289 +52 289 +62 289 +143 289 +153 289 +177 289 +181 289 +182 289 +186 289 +318 289 +354 289 +358 289 +417 289 +418 289 +423 289 +499 289 +0 290 +25 290 +28 290 +34 290 +84 290 +118 290 +196 290 +199 290 +208 290 +210 290 +214 290 +236 290 +269 290 +302 290 +331 290 +369 290 +423 290 +424 290 +441 290 +456 290 +476 290 +499 290 +0 291 +9 291 +35 291 +38 291 +47 291 +62 291 +82 291 +106 291 +112 291 +129 291 +132 291 +200 291 +286 291 +296 291 +300 291 +306 291 +312 291 +328 291 +350 291 +373 291 +386 291 +387 291 +391 291 +412 291 +437 291 +454 291 +474 291 +499 291 +0 292 +32 292 +45 292 +48 292 +59 292 +66 292 +72 292 +103 292 +124 292 +128 292 +141 292 +164 292 +309 292 +330 292 +369 292 +385 292 +399 292 +409 292 +427 292 +431 292 +433 292 +492 292 +497 292 +499 292 +0 293 +5 293 +18 293 +30 293 +51 293 +52 293 +68 293 +83 293 +99 293 +146 293 +155 293 +160 293 +168 293 +183 293 +184 293 +197 293 +222 293 +260 293 +264 293 +279 293 +303 293 +305 293 +323 293 +368 293 +404 293 +414 293 +420 293 +450 293 +482 293 +493 293 +498 293 +499 293 +0 294 +45 294 +82 294 +88 294 +104 294 +105 294 +117 294 +120 294 +137 294 +161 294 +170 294 +181 294 +193 294 +202 294 +229 294 +241 294 +253 294 +304 294 +337 294 +375 294 +397 294 +416 294 +435 294 +457 294 +482 294 +485 294 +488 294 +499 294 +0 295 +36 295 +58 295 +74 295 +76 295 +104 295 +128 295 +170 295 +172 295 +190 295 +201 295 +232 295 +261 295 +287 295 +288 295 +290 295 +306 295 +318 295 +403 295 +408 295 +465 295 +473 295 +477 295 +480 295 +499 295 +0 296 +4 296 +15 296 +126 296 +158 296 +167 296 +183 296 +264 296 +351 296 +431 296 +439 296 +488 296 +499 296 +0 297 +16 297 +22 297 +55 297 +195 297 +210 297 +229 297 +242 297 +248 297 +286 297 +350 297 +360 297 +363 297 +402 297 +403 297 +408 297 +416 297 +451 297 +453 297 +499 297 +0 298 +14 298 +24 298 +28 298 +32 298 +44 298 +55 298 +81 298 +172 298 +173 298 +183 298 +197 298 +215 298 +248 298 +255 298 +268 298 +306 298 +318 298 +353 298 +357 298 +373 298 +383 298 +399 298 +402 298 +405 298 +445 298 +454 298 +457 298 +499 298 +0 299 +19 299 +58 299 +138 299 +171 299 +186 299 +226 299 +251 299 +310 299 +334 299 +353 299 +364 299 +365 299 +372 299 +374 299 +401 299 +412 299 +437 299 +442 299 +457 299 +462 299 +491 299 +499 299 +0 300 +6 300 +26 300 +30 300 +63 300 +99 300 +115 300 +177 300 +181 300 +183 300 +198 300 +222 300 +237 300 +242 300 +249 300 +251 300 +271 300 +279 300 +282 300 +310 300 +362 300 +377 300 +395 300 +401 300 +410 300 +413 300 +435 300 +469 300 +499 300 +0 301 +29 301 +38 301 +112 301 +117 301 +137 301 +162 301 +173 301 +246 301 +290 301 +313 301 +362 301 +403 301 +412 301 +453 301 +455 301 +466 301 +474 301 +486 301 +499 301 +0 302 +19 302 +76 302 +77 302 +88 302 +92 302 +149 302 +182 302 +188 302 +196 302 +203 302 +252 302 +256 302 +259 302 +262 302 +303 302 +307 302 +352 302 +366 302 +388 302 +389 302 +422 302 +423 302 +487 302 +499 302 +0 303 +42 303 +90 303 +91 303 +120 303 +141 303 +142 303 +147 303 +153 303 +164 303 +185 303 +194 303 +205 303 +226 303 +229 303 +249 303 +254 303 +260 303 +275 303 +303 303 +351 303 +377 303 +396 303 +400 303 +461 303 +480 303 +499 303 +0 304 +3 304 +14 304 +52 304 +68 304 +79 304 +85 304 +97 304 +98 304 +103 304 +126 304 +131 304 +147 304 +157 304 +163 304 +175 304 +186 304 +219 304 +222 304 +247 304 +254 304 +268 304 +292 304 +304 304 +306 304 +344 304 +353 304 +362 304 +440 304 +499 304 +0 305 +57 305 +75 305 +82 305 +84 305 +103 305 +124 305 +187 305 +255 305 +270 305 +310 305 +335 305 +348 305 +364 305 +371 305 +402 305 +407 305 +408 305 +428 305 +452 305 +499 305 +0 306 +36 306 +107 306 +120 306 +123 306 +128 306 +134 306 +137 306 +154 306 +217 306 +247 306 +252 306 +275 306 +280 306 +366 306 +379 306 +383 306 +423 306 +426 306 +452 306 +453 306 +499 306 +0 307 +14 307 +30 307 +56 307 +145 307 +215 307 +295 307 +333 307 +338 307 +358 307 +359 307 +391 307 +397 307 +407 307 +448 307 +491 307 +499 307 +0 308 +2 308 +22 308 +40 308 +110 308 +114 308 +169 308 +180 308 +207 308 +265 308 +290 308 +328 308 +334 308 +346 308 +373 308 +389 308 +395 308 +401 308 +409 308 +476 308 +499 308 +0 309 +28 309 +43 309 +48 309 +90 309 +120 309 +131 309 +140 309 +154 309 +186 309 +217 309 +221 309 +222 309 +269 309 +274 309 +290 309 +299 309 +310 309 +344 309 +373 309 +396 309 +409 309 +436 309 +439 309 +470 309 +498 309 +499 309 +0 310 +3 310 +44 310 +86 310 +96 310 +134 310 +141 310 +154 310 +194 310 +197 310 +217 310 +251 310 +290 310 +302 310 +349 310 +368 310 +372 310 +445 310 +455 310 +499 310 +0 311 +6 311 +19 311 +38 311 +72 311 +97 311 +133 311 +134 311 +166 311 +183 311 +260 311 +301 311 +314 311 +333 311 +346 311 +363 311 +375 311 +416 311 +464 311 +478 311 +483 311 +498 311 +499 311 +0 312 +49 312 +71 312 +89 312 +90 312 +110 312 +121 312 +132 312 +195 312 +200 312 +228 312 +265 312 +282 312 +287 312 +295 312 +300 312 +355 312 +377 312 +381 312 +384 312 +390 312 +391 312 +420 312 +493 312 +499 312 +0 313 +8 313 +32 313 +35 313 +40 313 +78 313 +91 313 +94 313 +98 313 +120 313 +210 313 +225 313 +252 313 +290 313 +293 313 +310 313 +327 313 +336 313 +354 313 +373 313 +393 313 +419 313 +455 313 +499 313 +0 314 +2 314 +24 314 +100 314 +166 314 +195 314 +225 314 +286 314 +293 314 +297 314 +422 314 +445 314 +458 314 +473 314 +488 314 +498 314 +499 314 +0 315 +12 315 +57 315 +108 315 +131 315 +161 315 +166 315 +169 315 +177 315 +180 315 +234 315 +298 315 +300 315 +345 315 +499 315 +0 316 +12 316 +46 316 +55 316 +131 316 +152 316 +234 316 +255 316 +261 316 +279 316 +290 316 +300 316 +333 316 +393 316 +411 316 +438 316 +450 316 +499 316 +0 317 +29 317 +50 317 +53 317 +95 317 +99 317 +109 317 +131 317 +170 317 +188 317 +192 317 +222 317 +236 317 +270 317 +275 317 +276 317 +335 317 +352 317 +394 317 +403 317 +422 317 +442 317 +496 317 +497 317 +499 317 +0 318 +6 318 +30 318 +33 318 +64 318 +74 318 +82 318 +128 318 +134 318 +161 318 +170 318 +289 318 +313 318 +341 318 +345 318 +372 318 +381 318 +385 318 +393 318 +433 318 +456 318 +457 318 +482 318 +499 318 +0 319 +29 319 +49 319 +50 319 +60 319 +82 319 +117 319 +135 319 +154 319 +170 319 +198 319 +204 319 +213 319 +215 319 +250 319 +276 319 +303 319 +320 319 +332 319 +353 319 +383 319 +390 319 +413 319 +425 319 +433 319 +499 319 +0 320 +45 320 +90 320 +101 320 +113 320 +142 320 +148 320 +161 320 +165 320 +176 320 +202 320 +215 320 +228 320 +234 320 +249 320 +320 320 +330 320 +346 320 +350 320 +374 320 +375 320 +379 320 +396 320 +413 320 +419 320 +487 320 +499 320 +0 321 +6 321 +14 321 +38 321 +60 321 +111 321 +135 321 +138 321 +145 321 +150 321 +172 321 +229 321 +244 321 +247 321 +283 321 +317 321 +323 321 +336 321 +370 321 +428 321 +439 321 +464 321 +467 321 +470 321 +499 321 +0 322 +23 322 +50 322 +74 322 +90 322 +98 322 +104 322 +106 322 +118 322 +156 322 +180 322 +196 322 +264 322 +289 322 +290 322 +306 322 +319 322 +328 322 +332 322 +342 322 +345 322 +351 322 +399 322 +415 322 +437 322 +445 322 +499 322 +0 323 +20 323 +40 323 +43 323 +52 323 +87 323 +129 323 +159 323 +170 323 +207 323 +212 323 +223 323 +245 323 +265 323 +307 323 +309 323 +401 323 +445 323 +499 323 +0 324 +39 324 +56 324 +115 324 +130 324 +131 324 +177 324 +181 324 +184 324 +204 324 +231 324 +243 324 +252 324 +293 324 +338 324 +341 324 +368 324 +388 324 +430 324 +436 324 +460 324 +461 324 +499 324 +0 325 +2 325 +29 325 +32 325 +129 325 +191 325 +317 325 +328 325 +331 325 +395 325 +401 325 +439 325 +452 325 +461 325 +498 325 +499 325 +0 326 +33 326 +48 326 +62 326 +74 326 +87 326 +144 326 +216 326 +219 326 +224 326 +262 326 +274 326 +292 326 +347 326 +386 326 +428 326 +429 326 +499 326 +0 327 +2 327 +24 327 +71 327 +84 327 +86 327 +95 327 +115 327 +116 327 +132 327 +137 327 +143 327 +166 327 +196 327 +272 327 +303 327 +338 327 +342 327 +348 327 +357 327 +438 327 +481 327 +499 327 +0 328 +6 328 +13 328 +39 328 +48 328 +57 328 +103 328 +192 328 +208 328 +250 328 +348 328 +349 328 +357 328 +389 328 +425 328 +442 328 +499 328 +0 329 +108 329 +135 329 +160 329 +170 329 +197 329 +202 329 +239 329 +300 329 +322 329 +331 329 +342 329 +351 329 +400 329 +434 329 +486 329 +499 329 +0 330 +8 330 +29 330 +49 330 +70 330 +80 330 +110 330 +141 330 +145 330 +336 330 +386 330 +403 330 +464 330 +492 330 +497 330 +499 330 +0 331 +16 331 +32 331 +53 331 +60 331 +94 331 +121 331 +153 331 +157 331 +174 331 +178 331 +203 331 +210 331 +375 331 +394 331 +415 331 +483 331 +499 331 +0 332 +23 332 +53 332 +134 332 +138 332 +182 332 +190 332 +226 332 +246 332 +249 332 +294 332 +317 332 +340 332 +358 332 +362 332 +432 332 +441 332 +451 332 +458 332 +461 332 +479 332 +480 332 +499 332 +0 333 +4 333 +50 333 +90 333 +99 333 +137 333 +139 333 +167 333 +173 333 +190 333 +193 333 +202 333 +207 333 +228 333 +268 333 +276 333 +291 333 +308 333 +338 333 +376 333 +387 333 +390 333 +413 333 +424 333 +431 333 +458 333 +497 333 +499 333 +0 334 +17 334 +21 334 +49 334 +69 334 +92 334 +121 334 +139 334 +154 334 +172 334 +177 334 +180 334 +267 334 +365 334 +387 334 +499 334 +0 335 +39 335 +44 335 +76 335 +89 335 +113 335 +124 335 +138 335 +142 335 +167 335 +177 335 +180 335 +218 335 +233 335 +257 335 +260 335 +276 335 +355 335 +391 335 +406 335 +433 335 +469 335 +499 335 +0 336 +32 336 +48 336 +72 336 +94 336 +187 336 +192 336 +226 336 +227 336 +279 336 +288 336 +296 336 +302 336 +305 336 +314 336 +318 336 +322 336 +324 336 +410 336 +432 336 +487 336 +490 336 +499 336 +0 337 +17 337 +20 337 +75 337 +94 337 +123 337 +153 337 +214 337 +220 337 +237 337 +250 337 +270 337 +278 337 +286 337 +315 337 +332 337 +351 337 +377 337 +416 337 +436 337 +459 337 +473 337 +499 337 +0 338 +2 338 +110 338 +148 338 +152 338 +157 338 +186 338 +203 338 +230 338 +272 338 +280 338 +305 338 +312 338 +351 338 +394 338 +438 338 +447 338 +460 338 +486 338 +498 338 +499 338 +0 339 +17 339 +38 339 +46 339 +83 339 +92 339 +124 339 +128 339 +170 339 +266 339 +267 339 +302 339 +313 339 +318 339 +333 339 +369 339 +373 339 +391 339 +425 339 +441 339 +446 339 +456 339 +482 339 +491 339 +499 339 +0 340 +18 340 +25 340 +50 340 +56 340 +69 340 +75 340 +76 340 +84 340 +132 340 +150 340 +155 340 +163 340 +165 340 +182 340 +238 340 +250 340 +273 340 +284 340 +302 340 +305 340 +318 340 +340 340 +365 340 +403 340 +411 340 +415 340 +427 340 +499 340 +0 341 +4 341 +38 341 +41 341 +42 341 +47 341 +60 341 +95 341 +96 341 +103 341 +116 341 +117 341 +118 341 +119 341 +137 341 +159 341 +166 341 +169 341 +186 341 +209 341 +213 341 +224 341 +235 341 +245 341 +258 341 +264 341 +286 341 +293 341 +350 341 +376 341 +383 341 +499 341 +0 342 +15 342 +19 342 +24 342 +32 342 +55 342 +65 342 +66 342 +111 342 +112 342 +211 342 +213 342 +244 342 +251 342 +278 342 +314 342 +334 342 +349 342 +357 342 +365 342 +376 342 +390 342 +422 342 +424 342 +431 342 +441 342 +457 342 +460 342 +485 342 +499 342 +0 343 +34 343 +63 343 +69 343 +71 343 +79 343 +90 343 +93 343 +95 343 +97 343 +104 343 +116 343 +126 343 +144 343 +159 343 +204 343 +259 343 +283 343 +307 343 +378 343 +411 343 +427 343 +434 343 +438 343 +468 343 +476 343 +488 343 +499 343 +0 344 +4 344 +68 344 +126 344 +165 344 +182 344 +209 344 +295 344 +304 344 +325 344 +415 344 +457 344 +461 344 +494 344 +499 344 +0 345 +55 345 +68 345 +87 345 +98 345 +151 345 +160 345 +211 345 +214 345 +226 345 +274 345 +288 345 +294 345 +370 345 +376 345 +386 345 +403 345 +414 345 +424 345 +432 345 +440 345 +443 345 +474 345 +494 345 +499 345 +0 346 +36 346 +99 346 +120 346 +122 346 +140 346 +173 346 +176 346 +177 346 +190 346 +201 346 +257 346 +281 346 +399 346 +416 346 +420 346 +424 346 +426 346 +446 346 +447 346 +465 346 +468 346 +495 346 +499 346 +0 347 +41 347 +64 347 +77 347 +88 347 +123 347 +134 347 +142 347 +155 347 +179 347 +194 347 +204 347 +209 347 +255 347 +268 347 +285 347 +350 347 +362 347 +383 347 +385 347 +388 347 +405 347 +443 347 +450 347 +468 347 +469 347 +499 347 +0 348 +28 348 +36 348 +45 348 +48 348 +54 348 +58 348 +124 348 +131 348 +153 348 +163 348 +170 348 +183 348 +207 348 +252 348 +259 348 +261 348 +308 348 +322 348 +332 348 +338 348 +362 348 +393 348 +413 348 +417 348 +424 348 +433 348 +442 348 +444 348 +456 348 +471 348 +499 348 +0 349 +24 349 +37 349 +42 349 +43 349 +53 349 +74 349 +129 349 +148 349 +162 349 +195 349 +225 349 +234 349 +238 349 +262 349 +263 349 +277 349 +284 349 +286 349 +287 349 +323 349 +333 349 +359 349 +424 349 +425 349 +446 349 +492 349 +497 349 +499 349 +0 350 +3 350 +4 350 +25 350 +75 350 +96 350 +118 350 +127 350 +152 350 +170 350 +198 350 +199 350 +243 350 +296 350 +343 350 +368 350 +382 350 +384 350 +451 350 +455 350 +489 350 +499 350 +0 351 +14 351 +26 351 +38 351 +69 351 +73 351 +103 351 +210 351 +260 351 +262 351 +274 351 +275 351 +290 351 +332 351 +351 351 +458 351 +470 351 +475 351 +478 351 +495 351 +499 351 +0 352 +10 352 +31 352 +40 352 +44 352 +87 352 +122 352 +129 352 +168 352 +170 352 +199 352 +203 352 +226 352 +239 352 +247 352 +279 352 +299 352 +344 352 +361 352 +371 352 +380 352 +387 352 +390 352 +391 352 +417 352 +446 352 +472 352 +499 352 +0 353 +3 353 +7 353 +8 353 +13 353 +28 353 +41 353 +96 353 +101 353 +117 353 +155 353 +169 353 +189 353 +202 353 +208 353 +229 353 +256 353 +258 353 +264 353 +274 353 +279 353 +353 353 +367 353 +436 353 +437 353 +443 353 +455 353 +461 353 +499 353 +0 354 +1 354 +55 354 +58 354 +78 354 +88 354 +102 354 +126 354 +164 354 +186 354 +213 354 +229 354 +246 354 +266 354 +298 354 +364 354 +384 354 +447 354 +457 354 +465 354 +497 354 +499 354 +0 355 +40 355 +44 355 +62 355 +66 355 +69 355 +88 355 +105 355 +106 355 +110 355 +236 355 +237 355 +270 355 +311 355 +329 355 +359 355 +362 355 +383 355 +409 355 +460 355 +487 355 +498 355 +499 355 +0 356 +17 356 +19 356 +107 356 +160 356 +180 356 +192 356 +248 356 +254 356 +327 356 +379 356 +385 356 +458 356 +480 356 +489 356 +499 356 +0 357 +9 357 +31 357 +40 357 +43 357 +54 357 +99 357 +119 357 +123 357 +136 357 +152 357 +229 357 +230 357 +239 357 +285 357 +300 357 +322 357 +363 357 +367 357 +402 357 +415 357 +425 357 +445 357 +454 357 +479 357 +485 357 +499 357 +0 358 +15 358 +32 358 +65 358 +78 358 +84 358 +102 358 +131 358 +171 358 +188 358 +203 358 +231 358 +251 358 +254 358 +271 358 +272 358 +309 358 +365 358 +367 358 +384 358 +417 358 +487 358 +499 358 +0 359 +15 359 +20 359 +33 359 +40 359 +44 359 +48 359 +64 359 +72 359 +106 359 +220 359 +268 359 +279 359 +280 359 +322 359 +344 359 +350 359 +386 359 +432 359 +466 359 +472 359 +491 359 +499 359 +0 360 +26 360 +31 360 +54 360 +64 360 +73 360 +85 360 +102 360 +124 360 +175 360 +186 360 +187 360 +188 360 +210 360 +237 360 +240 360 +244 360 +247 360 +416 360 +418 360 +442 360 +460 360 +485 360 +499 360 +0 361 +13 361 +52 361 +182 361 +229 361 +258 361 +322 361 +336 361 +356 361 +385 361 +412 361 +417 361 +444 361 +461 361 +499 361 +0 362 +21 362 +112 362 +180 362 +192 362 +218 362 +238 362 +305 362 +321 362 +364 362 +371 362 +397 362 +407 362 +441 362 +484 362 +494 362 +499 362 +0 363 +22 363 +34 363 +84 363 +94 363 +131 363 +178 363 +185 363 +190 363 +207 363 +226 363 +245 363 +250 363 +256 363 +269 363 +350 363 +357 363 +360 363 +366 363 +408 363 +456 363 +499 363 +0 364 +6 364 +7 364 +12 364 +23 364 +100 364 +109 364 +114 364 +132 364 +154 364 +185 364 +203 364 +216 364 +248 364 +264 364 +275 364 +288 364 +305 364 +312 364 +395 364 +442 364 +457 364 +466 364 +499 364 +0 365 +12 365 +74 365 +78 365 +81 365 +144 365 +204 365 +227 365 +236 365 +263 365 +269 365 +273 365 +320 365 +353 365 +359 365 +384 365 +406 365 +415 365 +499 365 +0 366 +24 366 +34 366 +41 366 +46 366 +76 366 +149 366 +158 366 +163 366 +166 366 +167 366 +217 366 +225 366 +245 366 +248 366 +283 366 +287 366 +341 366 +390 366 +404 366 +409 366 +426 366 +437 366 +460 366 +464 366 +481 366 +489 366 +499 366 +0 367 +5 367 +14 367 +17 367 +52 367 +73 367 +111 367 +133 367 +135 367 +139 367 +143 367 +192 367 +209 367 +271 367 +292 367 +318 367 +325 367 +362 367 +382 367 +385 367 +421 367 +499 367 +0 368 +44 368 +58 368 +68 368 +182 368 +194 368 +198 368 +202 368 +211 368 +214 368 +260 368 +301 368 +303 368 +346 368 +348 368 +392 368 +401 368 +423 368 +472 368 +479 368 +482 368 +489 368 +499 368 +0 369 +14 369 +84 369 +89 369 +134 369 +135 369 +175 369 +222 369 +242 369 +255 369 +259 369 +335 369 +343 369 +423 369 +424 369 +429 369 +430 369 +469 369 +495 369 +499 369 +0 370 +5 370 +10 370 +30 370 +40 370 +46 370 +123 370 +132 370 +162 370 +215 370 +216 370 +284 370 +293 370 +295 370 +324 370 +343 370 +360 370 +409 370 +414 370 +457 370 +459 370 +498 370 +499 370 +0 371 +7 371 +22 371 +38 371 +43 371 +70 371 +122 371 +159 371 +174 371 +220 371 +221 371 +222 371 +249 371 +265 371 +281 371 +287 371 +294 371 +299 371 +325 371 +334 371 +339 371 +348 371 +355 371 +404 371 +409 371 +432 371 +440 371 +472 371 +499 371 +0 372 +1 372 +63 372 +68 372 +165 372 +180 372 +242 372 +269 372 +322 372 +389 372 +390 372 +397 372 +449 372 +499 372 +0 373 +11 373 +55 373 +61 373 +94 373 +120 373 +147 373 +168 373 +210 373 +219 373 +242 373 +249 373 +259 373 +284 373 +318 373 +400 373 +407 373 +499 373 +0 374 +10 374 +15 374 +83 374 +92 374 +131 374 +143 374 +162 374 +233 374 +264 374 +277 374 +316 374 +334 374 +340 374 +414 374 +424 374 +464 374 +481 374 +494 374 +499 374 +0 375 +5 375 +7 375 +91 375 +126 375 +129 375 +158 375 +168 375 +183 375 +225 375 +272 375 +283 375 +394 375 +433 375 +458 375 +459 375 +499 375 +0 376 +10 376 +27 376 +69 376 +82 376 +107 376 +141 376 +181 376 +204 376 +251 376 +253 376 +254 376 +262 376 +277 376 +281 376 +363 376 +373 376 +377 376 +391 376 +393 376 +427 376 +461 376 +476 376 +499 376 +0 377 +36 377 +39 377 +64 377 +83 377 +96 377 +109 377 +164 377 +198 377 +222 377 +223 377 +232 377 +240 377 +245 377 +268 377 +277 377 +283 377 +290 377 +304 377 +320 377 +325 377 +331 377 +366 377 +369 377 +373 377 +387 377 +388 377 +438 377 +451 377 +454 377 +455 377 +470 377 +494 377 +497 377 +499 377 +0 378 +37 378 +64 378 +89 378 +168 378 +185 378 +186 378 +216 378 +219 378 +230 378 +235 378 +251 378 +255 378 +392 378 +406 378 +481 378 +499 378 +0 379 +2 379 +14 379 +30 379 +81 379 +128 379 +153 379 +178 379 +208 379 +223 379 +235 379 +263 379 +267 379 +289 379 +344 379 +350 379 +406 379 +418 379 +421 379 +423 379 +431 379 +452 379 +462 379 +494 379 +499 379 +0 380 +20 380 +67 380 +71 380 +96 380 +135 380 +151 380 +152 380 +157 380 +181 380 +194 380 +232 380 +249 380 +270 380 +300 380 +309 380 +325 380 +339 380 +361 380 +362 380 +369 380 +400 380 +423 380 +474 380 +499 380 +0 381 +5 381 +51 381 +69 381 +75 381 +88 381 +90 381 +93 381 +102 381 +114 381 +136 381 +138 381 +146 381 +183 381 +200 381 +223 381 +241 381 +439 381 +440 381 +447 381 +457 381 +495 381 +496 381 +499 381 +0 382 +21 382 +32 382 +52 382 +58 382 +71 382 +89 382 +99 382 +150 382 +169 382 +210 382 +238 382 +279 382 +281 382 +328 382 +336 382 +340 382 +380 382 +402 382 +412 382 +438 382 +443 382 +463 382 +499 382 +0 383 +17 383 +31 383 +41 383 +70 383 +76 383 +121 383 +172 383 +173 383 +176 383 +278 383 +313 383 +334 383 +355 383 +379 383 +384 383 +422 383 +426 383 +430 383 +447 383 +448 383 +450 383 +465 383 +484 383 +499 383 +0 384 +14 384 +24 384 +26 384 +44 384 +67 384 +118 384 +125 384 +153 384 +154 384 +179 384 +230 384 +237 384 +265 384 +270 384 +280 384 +289 384 +385 384 +419 384 +426 384 +427 384 +457 384 +499 384 +0 385 +36 385 +47 385 +50 385 +87 385 +88 385 +94 385 +121 385 +158 385 +177 385 +240 385 +249 385 +272 385 +291 385 +329 385 +353 385 +385 385 +390 385 +395 385 +499 385 +0 386 +5 386 +29 386 +37 386 +46 386 +64 386 +98 386 +111 386 +149 386 +155 386 +159 386 +163 386 +180 386 +187 386 +228 386 +320 386 +330 386 +337 386 +432 386 +492 386 +499 386 +0 387 +80 387 +88 387 +97 387 +105 387 +109 387 +124 387 +131 387 +133 387 +149 387 +165 387 +173 387 +195 387 +210 387 +219 387 +301 387 +354 387 +398 387 +403 387 +414 387 +456 387 +499 387 +0 388 +33 388 +60 388 +92 388 +113 388 +162 388 +181 388 +192 388 +213 388 +243 388 +246 388 +259 388 +371 388 +414 388 +467 388 +478 388 +490 388 +499 388 +0 389 +2 389 +21 389 +33 389 +64 389 +75 389 +93 389 +94 389 +237 389 +247 389 +274 389 +277 389 +317 389 +355 389 +357 389 +361 389 +373 389 +400 389 +416 389 +437 389 +452 389 +471 389 +499 389 +0 390 +14 390 +61 390 +107 390 +145 390 +151 390 +201 390 +278 390 +345 390 +439 390 +482 390 +499 390 +0 391 +1 391 +28 391 +48 391 +89 391 +100 391 +109 391 +155 391 +186 391 +277 391 +289 391 +366 391 +399 391 +404 391 +420 391 +445 391 +499 391 +0 392 +6 392 +54 392 +73 392 +76 392 +96 392 +105 392 +126 392 +205 392 +228 392 +231 392 +253 392 +271 392 +302 392 +315 392 +348 392 +394 392 +422 392 +455 392 +475 392 +479 392 +499 392 +0 393 +37 393 +46 393 +70 393 +103 393 +198 393 +245 393 +256 393 +302 393 +335 393 +361 393 +385 393 +394 393 +415 393 +418 393 +431 393 +434 393 +444 393 +462 393 +467 393 +499 393 +0 394 +3 394 +30 394 +37 394 +65 394 +79 394 +97 394 +107 394 +126 394 +166 394 +182 394 +207 394 +213 394 +250 394 +288 394 +305 394 +345 394 +352 394 +397 394 +400 394 +499 394 +0 395 +20 395 +26 395 +71 395 +103 395 +111 395 +134 395 +153 395 +182 395 +211 395 +262 395 +276 395 +403 395 +412 395 +447 395 +478 395 +494 395 +499 395 +0 396 +51 396 +53 396 +85 396 +133 396 +193 396 +203 396 +224 396 +270 396 +283 396 +346 396 +375 396 +388 396 +398 396 +489 396 +499 396 +0 397 +2 397 +18 397 +43 397 +100 397 +165 397 +183 397 +189 397 +198 397 +214 397 +267 397 +308 397 +331 397 +337 397 +370 397 +392 397 +397 397 +438 397 +446 397 +465 397 +478 397 +489 397 +499 397 +0 398 +7 398 +34 398 +49 398 +58 398 +118 398 +138 398 +146 398 +150 398 +156 398 +178 398 +179 398 +189 398 +192 398 +233 398 +234 398 +264 398 +265 398 +280 398 +286 398 +299 398 +309 398 +358 398 +366 398 +409 398 +410 398 +423 398 +450 398 +458 398 +491 398 +492 398 +499 398 +0 399 +14 399 +73 399 +103 399 +106 399 +111 399 +112 399 +113 399 +119 399 +121 399 +155 399 +161 399 +180 399 +202 399 +229 399 +244 399 +287 399 +311 399 +353 399 +361 399 +363 399 +404 399 +411 399 +413 399 +438 399 +439 399 +447 399 +450 399 +490 399 +499 399 +0 400 +14 400 +39 400 +46 400 +54 400 +71 400 +84 400 +86 400 +135 400 +141 400 +144 400 +175 400 +184 400 +257 400 +267 400 +314 400 +369 400 +372 400 +430 400 +437 400 +441 400 +478 400 +480 400 +499 400 +0 401 +7 401 +15 401 +36 401 +74 401 +107 401 +115 401 +133 401 +197 401 +233 401 +252 401 +263 401 +318 401 +324 401 +342 401 +344 401 +347 401 +365 401 +382 401 +431 401 +435 401 +445 401 +484 401 +492 401 +499 401 +0 402 +5 402 +42 402 +67 402 +98 402 +129 402 +171 402 +175 402 +189 402 +209 402 +216 402 +244 402 +251 402 +346 402 +359 402 +363 402 +393 402 +410 402 +414 402 +499 402 +0 403 +43 403 +95 403 +96 403 +100 403 +106 403 +149 403 +151 403 +164 403 +167 403 +178 403 +185 403 +224 403 +234 403 +237 403 +261 403 +290 403 +298 403 +304 403 +315 403 +349 403 +371 403 +385 403 +398 403 +426 403 +440 403 +451 403 +456 403 +471 403 +487 403 +499 403 +0 404 +4 404 +27 404 +64 404 +73 404 +143 404 +184 404 +199 404 +206 404 +217 404 +223 404 +233 404 +256 404 +258 404 +272 404 +279 404 +295 404 +335 404 +352 404 +355 404 +368 404 +372 404 +384 404 +425 404 +443 404 +451 404 +479 404 +481 404 +498 404 +499 404 +0 405 +4 405 +37 405 +126 405 +130 405 +134 405 +225 405 +268 405 +273 405 +291 405 +366 405 +367 405 +372 405 +382 405 +429 405 +431 405 +447 405 +499 405 +0 406 +17 406 +22 406 +36 406 +46 406 +55 406 +63 406 +106 406 +140 406 +176 406 +207 406 +212 406 +228 406 +243 406 +264 406 +276 406 +286 406 +297 406 +315 406 +322 406 +348 406 +349 406 +384 406 +388 406 +412 406 +415 406 +437 406 +446 406 +479 406 +491 406 +499 406 +0 407 +78 407 +81 407 +101 407 +134 407 +156 407 +189 407 +203 407 +211 407 +212 407 +216 407 +256 407 +289 407 +291 407 +292 407 +332 407 +336 407 +413 407 +414 407 +419 407 +433 407 +482 407 +491 407 +499 407 +0 408 +15 408 +96 408 +103 408 +114 408 +126 408 +135 408 +151 408 +159 408 +166 408 +208 408 +368 408 +381 408 +396 408 +398 408 +411 408 +499 408 +0 409 +11 409 +27 409 +60 409 +109 409 +116 409 +125 409 +130 409 +159 409 +182 409 +227 409 +230 409 +256 409 +266 409 +267 409 +294 409 +325 409 +347 409 +358 409 +455 409 +463 409 +493 409 +499 409 +0 410 +3 410 +21 410 +35 410 +48 410 +49 410 +51 410 +99 410 +136 410 +158 410 +161 410 +171 410 +194 410 +210 410 +214 410 +216 410 +225 410 +228 410 +242 410 +290 410 +299 410 +341 410 +348 410 +386 410 +414 410 +417 410 +421 410 +433 410 +447 410 +466 410 +499 410 +0 411 +24 411 +27 411 +64 411 +74 411 +79 411 +105 411 +109 411 +121 411 +125 411 +148 411 +200 411 +239 411 +259 411 +289 411 +298 411 +351 411 +390 411 +401 411 +413 411 +454 411 +464 411 +482 411 +487 411 +499 411 +0 412 +15 412 +20 412 +32 412 +41 412 +75 412 +93 412 +102 412 +123 412 +139 412 +147 412 +161 412 +166 412 +203 412 +236 412 +252 412 +260 412 +270 412 +285 412 +289 412 +301 412 +302 412 +353 412 +395 412 +400 412 +433 412 +464 412 +465 412 +499 412 +0 413 +35 413 +45 413 +68 413 +74 413 +137 413 +139 413 +168 413 +204 413 +208 413 +221 413 +290 413 +305 413 +327 413 +331 413 +344 413 +347 413 +358 413 +365 413 +449 413 +452 413 +468 413 +499 413 +0 414 +54 414 +77 414 +79 414 +86 414 +124 414 +173 414 +187 414 +199 414 +214 414 +228 414 +252 414 +268 414 +277 414 +313 414 +327 414 +356 414 +362 414 +389 414 +419 414 +489 414 +499 414 +0 415 +1 415 +44 415 +70 415 +100 415 +166 415 +170 415 +183 415 +251 415 +305 415 +320 415 +388 415 +412 415 +436 415 +440 415 +443 415 +445 415 +499 415 +0 416 +54 416 +70 416 +105 416 +120 416 +163 416 +206 416 +222 416 +224 416 +230 416 +234 416 +258 416 +298 416 +311 416 +315 416 +329 416 +331 416 +376 416 +457 416 +471 416 +497 416 +499 416 +0 417 +4 417 +8 417 +56 417 +95 417 +136 417 +188 417 +252 417 +273 417 +381 417 +391 417 +421 417 +435 417 +439 417 +442 417 +458 417 +499 417 +0 418 +27 418 +30 418 +64 418 +66 418 +97 418 +110 418 +148 418 +232 418 +255 418 +256 418 +262 418 +321 418 +325 418 +399 418 +425 418 +430 418 +431 418 +435 418 +444 418 +499 418 +0 419 +40 419 +52 419 +60 419 +158 419 +161 419 +199 419 +202 419 +217 419 +232 419 +283 419 +297 419 +305 419 +359 419 +447 419 +490 419 +499 419 +0 420 +21 420 +66 420 +158 420 +178 420 +218 420 +221 420 +265 420 +304 420 +322 420 +347 420 +351 420 +363 420 +364 420 +367 420 +384 420 +441 420 +447 420 +454 420 +462 420 +490 420 +499 420 +0 421 +15 421 +66 421 +187 421 +188 421 +197 421 +218 421 +285 421 +289 421 +340 421 +368 421 +399 421 +408 421 +422 421 +468 421 +482 421 +484 421 +499 421 +0 422 +11 422 +46 422 +50 422 +125 422 +206 422 +276 422 +283 422 +284 422 +290 422 +306 422 +325 422 +332 422 +354 422 +365 422 +371 422 +440 422 +472 422 +499 422 +0 423 +11 423 +12 423 +24 423 +30 423 +61 423 +94 423 +190 423 +194 423 +263 423 +315 423 +378 423 +389 423 +412 423 +416 423 +432 423 +483 423 +498 423 +499 423 +0 424 +1 424 +26 424 +32 424 +40 424 +87 424 +90 424 +134 424 +171 424 +195 424 +201 424 +228 424 +229 424 +243 424 +249 424 +250 424 +257 424 +302 424 +311 424 +365 424 +389 424 +412 424 +413 424 +414 424 +459 424 +480 424 +495 424 +499 424 +0 425 +5 425 +8 425 +12 425 +55 425 +70 425 +77 425 +81 425 +98 425 +99 425 +109 425 +141 425 +179 425 +183 425 +196 425 +202 425 +215 425 +220 425 +258 425 +273 425 +285 425 +291 425 +304 425 +323 425 +326 425 +400 425 +432 425 +463 425 +499 425 +0 426 +5 426 +12 426 +19 426 +28 426 +39 426 +79 426 +95 426 +123 426 +124 426 +128 426 +130 426 +147 426 +179 426 +185 426 +297 426 +425 426 +426 426 +446 426 +461 426 +476 426 +493 426 +499 426 +0 427 +13 427 +89 427 +162 427 +165 427 +190 427 +204 427 +211 427 +224 427 +239 427 +244 427 +282 427 +302 427 +309 427 +316 427 +327 427 +344 427 +355 427 +366 427 +433 427 +436 427 +458 427 +462 427 +467 427 +485 427 +488 427 +499 427 +0 428 +17 428 +18 428 +22 428 +59 428 +64 428 +70 428 +172 428 +208 428 +239 428 +250 428 +284 428 +324 428 +478 428 +480 428 +490 428 +499 428 +0 429 +1 429 +15 429 +96 429 +118 429 +120 429 +142 429 +182 429 +192 429 +218 429 +219 429 +232 429 +243 429 +247 429 +267 429 +291 429 +302 429 +338 429 +339 429 +342 429 +382 429 +383 429 +407 429 +485 429 +489 429 +492 429 +499 429 +0 430 +56 430 +63 430 +67 430 +69 430 +85 430 +91 430 +108 430 +126 430 +182 430 +189 430 +197 430 +229 430 +235 430 +237 430 +254 430 +266 430 +269 430 +281 430 +288 430 +296 430 +339 430 +346 430 +386 430 +441 430 +488 430 +499 430 +0 431 +5 431 +65 431 +133 431 +192 431 +193 431 +194 431 +196 431 +232 431 +286 431 +301 431 +313 431 +318 431 +330 431 +373 431 +380 431 +418 431 +435 431 +448 431 +461 431 +493 431 +499 431 +0 432 +31 432 +74 432 +84 432 +88 432 +91 432 +123 432 +199 432 +224 432 +256 432 +260 432 +267 432 +319 432 +356 432 +382 432 +390 432 +403 432 +462 432 +478 432 +497 432 +499 432 +0 433 +9 433 +68 433 +83 433 +87 433 +100 433 +145 433 +181 433 +208 433 +220 433 +233 433 +239 433 +263 433 +312 433 +361 433 +447 433 +490 433 +499 433 +0 434 +70 434 +71 434 +72 434 +135 434 +136 434 +155 434 +158 434 +191 434 +256 434 +268 434 +272 434 +312 434 +327 434 +334 434 +337 434 +379 434 +392 434 +417 434 +428 434 +441 434 +452 434 +497 434 +499 434 +0 435 +19 435 +47 435 +95 435 +144 435 +207 435 +217 435 +282 435 +332 435 +340 435 +350 435 +376 435 +377 435 +389 435 +393 435 +417 435 +418 435 +425 435 +476 435 +499 435 +0 436 +4 436 +35 436 +71 436 +83 436 +85 436 +123 436 +152 436 +166 436 +168 436 +201 436 +238 436 +282 436 +297 436 +305 436 +318 436 +326 436 +329 436 +342 436 +374 436 +382 436 +415 436 +417 436 +418 436 +419 436 +447 436 +452 436 +475 436 +499 436 +0 437 +1 437 +46 437 +55 437 +78 437 +104 437 +107 437 +142 437 +161 437 +162 437 +246 437 +259 437 +283 437 +305 437 +327 437 +337 437 +464 437 +472 437 +499 437 +0 438 +7 438 +39 438 +62 438 +65 438 +86 438 +97 438 +104 438 +166 438 +190 438 +196 438 +208 438 +211 438 +249 438 +271 438 +275 438 +292 438 +294 438 +318 438 +394 438 +422 438 +435 438 +452 438 +462 438 +499 438 +0 439 +21 439 +68 439 +90 439 +174 439 +247 439 +273 439 +334 439 +403 439 +410 439 +418 439 +452 439 +482 439 +499 439 +0 440 +50 440 +66 440 +67 440 +113 440 +133 440 +141 440 +156 440 +194 440 +205 440 +213 440 +227 440 +228 440 +305 440 +362 440 +404 440 +423 440 +424 440 +499 440 +0 441 +21 441 +163 441 +181 441 +185 441 +232 441 +251 441 +268 441 +306 441 +311 441 +317 441 +319 441 +360 441 +379 441 +393 441 +402 441 +412 441 +434 441 +455 441 +499 441 +0 442 +24 442 +25 442 +37 442 +47 442 +50 442 +69 442 +95 442 +118 442 +127 442 +232 442 +258 442 +284 442 +306 442 +330 442 +388 442 +420 442 +437 442 +489 442 +499 442 +0 443 +34 443 +52 443 +92 443 +140 443 +159 443 +207 443 +219 443 +222 443 +269 443 +286 443 +342 443 +369 443 +376 443 +406 443 +445 443 +452 443 +491 443 +499 443 +0 444 +5 444 +15 444 +16 444 +18 444 +41 444 +63 444 +83 444 +103 444 +117 444 +122 444 +131 444 +199 444 +206 444 +221 444 +250 444 +305 444 +306 444 +322 444 +357 444 +358 444 +370 444 +440 444 +499 444 +0 445 +28 445 +52 445 +68 445 +71 445 +90 445 +119 445 +121 445 +125 445 +128 445 +204 445 +214 445 +225 445 +252 445 +257 445 +285 445 +317 445 +332 445 +350 445 +376 445 +416 445 +433 445 +436 445 +442 445 +464 445 +498 445 +499 445 +0 446 +44 446 +64 446 +106 446 +115 446 +161 446 +183 446 +230 446 +261 446 +276 446 +310 446 +336 446 +383 446 +419 446 +499 446 +0 447 +7 447 +12 447 +33 447 +51 447 +53 447 +117 447 +158 447 +215 447 +277 447 +286 447 +289 447 +359 447 +397 447 +409 447 +447 447 +494 447 +499 447 +0 448 +11 448 +13 448 +17 448 +78 448 +83 448 +150 448 +178 448 +180 448 +278 448 +307 448 +337 448 +344 448 +423 448 +442 448 +447 448 +484 448 +488 448 +492 448 +499 448 +0 449 +10 449 +21 449 +25 449 +54 449 +62 449 +78 449 +83 449 +114 449 +142 449 +182 449 +185 449 +210 449 +234 449 +247 449 +270 449 +326 449 +339 449 +355 449 +410 449 +462 449 +477 449 +478 449 +496 449 +499 449 +0 450 +17 450 +25 450 +74 450 +93 450 +130 450 +148 450 +158 450 +162 450 +181 450 +206 450 +207 450 +225 450 +260 450 +276 450 +303 450 +314 450 +385 450 +422 450 +464 450 +490 450 +499 450 +0 451 +15 451 +19 451 +46 451 +51 451 +70 451 +106 451 +112 451 +113 451 +138 451 +150 451 +207 451 +217 451 +243 451 +278 451 +336 451 +338 451 +383 451 +409 451 +448 451 +451 451 +470 451 +491 451 +499 451 +0 452 +41 452 +84 452 +85 452 +157 452 +162 452 +220 452 +257 452 +293 452 +325 452 +347 452 +352 452 +355 452 +366 452 +408 452 +429 452 +457 452 +459 452 +475 452 +499 452 +0 453 +35 453 +38 453 +104 453 +106 453 +143 453 +167 453 +205 453 +222 453 +283 453 +290 453 +345 453 +355 453 +404 453 +425 453 +472 453 +499 453 +0 454 +27 454 +38 454 +50 454 +68 454 +74 454 +77 454 +123 454 +126 454 +133 454 +164 454 +172 454 +200 454 +206 454 +217 454 +262 454 +273 454 +278 454 +279 454 +301 454 +334 454 +389 454 +391 454 +423 454 +428 454 +466 454 +469 454 +499 454 +0 455 +17 455 +30 455 +67 455 +113 455 +137 455 +189 455 +200 455 +205 455 +243 455 +273 455 +277 455 +323 455 +378 455 +395 455 +417 455 +447 455 +499 455 +0 456 +61 456 +94 456 +95 456 +104 456 +112 456 +113 456 +151 456 +157 456 +193 456 +208 456 +210 456 +271 456 +279 456 +334 456 +340 456 +368 456 +407 456 +415 456 +438 456 +462 456 +490 456 +499 456 +0 457 +45 457 +59 457 +61 457 +64 457 +93 457 +97 457 +120 457 +152 457 +233 457 +237 457 +304 457 +305 457 +345 457 +349 457 +398 457 +407 457 +409 457 +411 457 +431 457 +462 457 +469 457 +485 457 +498 457 +499 457 +0 458 +8 458 +25 458 +48 458 +56 458 +97 458 +113 458 +197 458 +233 458 +247 458 +250 458 +255 458 +269 458 +291 458 +345 458 +381 458 +419 458 +420 458 +445 458 +458 458 +468 458 +497 458 +499 458 +0 459 +6 459 +16 459 +66 459 +80 459 +111 459 +123 459 +177 459 +199 459 +222 459 +264 459 +265 459 +282 459 +333 459 +399 459 +409 459 +424 459 +465 459 +499 459 +0 460 +2 460 +6 460 +19 460 +42 460 +87 460 +96 460 +119 460 +150 460 +171 460 +199 460 +248 460 +250 460 +251 460 +277 460 +301 460 +322 460 +323 460 +325 460 +384 460 +426 460 +499 460 +0 461 +30 461 +31 461 +42 461 +46 461 +57 461 +96 461 +163 461 +187 461 +236 461 +295 461 +303 461 +392 461 +401 461 +410 461 +430 461 +434 461 +442 461 +443 461 +490 461 +499 461 +0 462 +17 462 +47 462 +95 462 +129 462 +206 462 +207 462 +235 462 +284 462 +311 462 +347 462 +350 462 +354 462 +398 462 +413 462 +468 462 +499 462 +0 463 +1 463 +15 463 +71 463 +72 463 +87 463 +93 463 +174 463 +178 463 +214 463 +232 463 +297 463 +308 463 +345 463 +351 463 +370 463 +437 463 +438 463 +499 463 +0 464 +9 464 +36 464 +43 464 +53 464 +79 464 +106 464 +109 464 +215 464 +231 464 +250 464 +254 464 +256 464 +276 464 +403 464 +413 464 +459 464 +499 464 +0 465 +22 465 +81 465 +169 465 +194 465 +212 465 +283 465 +296 465 +326 465 +377 465 +380 465 +424 465 +443 465 +483 465 +493 465 +499 465 +0 466 +4 466 +9 466 +14 466 +21 466 +26 466 +107 466 +112 466 +122 466 +138 466 +139 466 +152 466 +174 466 +192 466 +193 466 +227 466 +258 466 +281 466 +290 466 +365 466 +381 466 +392 466 +406 466 +430 466 +456 466 +474 466 +488 466 +497 466 +499 466 +0 467 +34 467 +41 467 +86 467 +93 467 +95 467 +120 467 +176 467 +186 467 +198 467 +207 467 +213 467 +225 467 +232 467 +263 467 +294 467 +369 467 +372 467 +384 467 +471 467 +499 467 +0 468 +35 468 +40 468 +47 468 +53 468 +81 468 +100 468 +107 468 +145 468 +156 468 +163 468 +165 468 +219 468 +252 468 +306 468 +333 468 +386 468 +451 468 +456 468 +477 468 +487 468 +499 468 +0 469 +4 469 +68 469 +85 469 +88 469 +131 469 +154 469 +187 469 +189 469 +205 469 +208 469 +269 469 +285 469 +313 469 +326 469 +374 469 +423 469 +448 469 +499 469 +0 470 +54 470 +82 470 +87 470 +129 470 +157 470 +189 470 +191 470 +226 470 +267 470 +310 470 +330 470 +337 470 +339 470 +357 470 +379 470 +384 470 +397 470 +446 470 +463 470 +485 470 +495 470 +499 470 +0 471 +28 471 +35 471 +40 471 +42 471 +83 471 +113 471 +122 471 +222 471 +227 471 +256 471 +258 471 +262 471 +297 471 +312 471 +361 471 +363 471 +486 471 +487 471 +498 471 +499 471 +0 472 +47 472 +60 472 +119 472 +138 472 +153 472 +159 472 +208 472 +211 472 +224 472 +240 472 +313 472 +333 472 +449 472 +499 472 +0 473 +14 473 +71 473 +119 473 +154 473 +169 473 +187 473 +279 473 +314 473 +340 473 +341 473 +392 473 +395 473 +439 473 +452 473 +459 473 +477 473 +499 473 +0 474 +38 474 +75 474 +82 474 +107 474 +146 474 +160 474 +214 474 +239 474 +267 474 +319 474 +320 474 +325 474 +342 474 +347 474 +377 474 +472 474 +478 474 +499 474 +0 475 +13 475 +64 475 +82 475 +94 475 +144 475 +208 475 +209 475 +257 475 +269 475 +276 475 +312 475 +320 475 +330 475 +350 475 +495 475 +499 475 +0 476 +57 476 +66 476 +111 476 +145 476 +199 476 +216 476 +256 476 +275 476 +284 476 +316 476 +360 476 +388 476 +416 476 +440 476 +443 476 +452 476 +460 476 +499 476 +0 477 +22 477 +64 477 +79 477 +116 477 +120 477 +145 477 +207 477 +221 477 +226 477 +232 477 +258 477 +259 477 +294 477 +296 477 +324 477 +333 477 +376 477 +393 477 +408 477 +476 477 +499 477 +0 478 +4 478 +17 478 +33 478 +37 478 +41 478 +45 478 +51 478 +79 478 +104 478 +108 478 +125 478 +140 478 +171 478 +175 478 +184 478 +187 478 +194 478 +208 478 +251 478 +254 478 +255 478 +268 478 +272 478 +277 478 +292 478 +362 478 +384 478 +394 478 +403 478 +428 478 +461 478 +499 478 +0 479 +9 479 +10 479 +57 479 +105 479 +115 479 +128 479 +179 479 +181 479 +220 479 +230 479 +256 479 +277 479 +289 479 +290 479 +298 479 +351 479 +388 479 +390 479 +425 479 +442 479 +499 479 +0 480 +22 480 +57 480 +90 480 +108 480 +143 480 +146 480 +284 480 +298 480 +311 480 +319 480 +330 480 +357 480 +380 480 +384 480 +388 480 +482 480 +499 480 +0 481 +14 481 +45 481 +104 481 +144 481 +149 481 +167 481 +205 481 +224 481 +294 481 +305 481 +329 481 +359 481 +412 481 +446 481 +447 481 +499 481 +0 482 +51 482 +154 482 +155 482 +162 482 +187 482 +251 482 +263 482 +295 482 +305 482 +313 482 +339 482 +353 482 +369 482 +424 482 +426 482 +499 482 +0 483 +23 483 +35 483 +70 483 +121 483 +134 483 +170 483 +178 483 +185 483 +189 483 +206 483 +220 483 +251 483 +257 483 +260 483 +266 483 +306 483 +317 483 +321 483 +329 483 +388 483 +414 483 +455 483 +462 483 +478 483 +499 483 +0 484 +16 484 +20 484 +104 484 +142 484 +209 484 +215 484 +229 484 +245 484 +246 484 +253 484 +337 484 +428 484 +499 484 +0 485 +3 485 +40 485 +45 485 +50 485 +78 485 +122 485 +152 485 +202 485 +206 485 +235 485 +269 485 +278 485 +316 485 +387 485 +389 485 +398 485 +403 485 +414 485 +421 485 +439 485 +450 485 +457 485 +475 485 +495 485 +499 485 +0 486 +33 486 +54 486 +63 486 +209 486 +229 486 +350 486 +489 486 +499 486 +0 487 +77 487 +133 487 +147 487 +151 487 +193 487 +205 487 +241 487 +252 487 +260 487 +289 487 +363 487 +379 487 +432 487 +436 487 +438 487 +448 487 +479 487 +480 487 +499 487 +0 488 +171 488 +192 488 +209 488 +232 488 +240 488 +252 488 +263 488 +268 488 +286 488 +333 488 +335 488 +343 488 +348 488 +370 488 +478 488 +499 488 +0 489 +18 489 +41 489 +43 489 +44 489 +64 489 +97 489 +136 489 +187 489 +190 489 +200 489 +205 489 +222 489 +355 489 +356 489 +382 489 +408 489 +422 489 +450 489 +499 489 +0 490 +16 490 +19 490 +62 490 +114 490 +147 490 +154 490 +201 490 +232 490 +269 490 +347 490 +382 490 +458 490 +471 490 +477 490 +491 490 +499 490 +0 491 +22 491 +89 491 +106 491 +116 491 +167 491 +184 491 +201 491 +216 491 +228 491 +237 491 +255 491 +269 491 +278 491 +287 491 +291 491 +339 491 +366 491 +373 491 +375 491 +416 491 +420 491 +458 491 +481 491 +497 491 +499 491 +0 492 +4 492 +9 492 +31 492 +38 492 +59 492 +137 492 +178 492 +205 492 +230 492 +386 492 +483 492 +485 492 +497 492 +499 492 +0 493 +18 493 +88 493 +101 493 +129 493 +167 493 +174 493 +180 493 +184 493 +192 493 +198 493 +199 493 +232 493 +265 493 +272 493 +306 493 +335 493 +358 493 +372 493 +435 493 +445 493 +453 493 +478 493 +499 493 +0 494 +36 494 +55 494 +85 494 +98 494 +108 494 +116 494 +136 494 +194 494 +195 494 +217 494 +238 494 +258 494 +275 494 +278 494 +296 494 +387 494 +428 494 +444 494 +457 494 +499 494 +0 495 +14 495 +43 495 +55 495 +65 495 +67 495 +136 495 +163 495 +167 495 +176 495 +193 495 +199 495 +214 495 +270 495 +356 495 +363 495 +392 495 +401 495 +428 495 +461 495 +467 495 +470 495 +499 495 +0 496 +45 496 +69 496 +70 496 +96 496 +134 496 +160 496 +178 496 +224 496 +227 496 +253 496 +255 496 +275 496 +283 496 +323 496 +335 496 +363 496 +424 496 +433 496 +450 496 +463 496 +499 496 +0 497 +5 497 +6 497 +75 497 +128 497 +130 497 +157 497 +167 497 +179 497 +271 497 +322 497 +359 497 +397 497 +400 497 +401 497 +465 497 +493 497 +499 497 +0 498 +44 498 +50 498 +61 498 +96 498 +110 498 +127 498 +142 498 +154 498 +157 498 +195 498 +218 498 +251 498 +280 498 +294 498 +309 498 +327 498 +344 498 +407 498 +420 498 +461 498 +499 498 +0 499 +1 499 +2 499 +3 499 +4 499 +5 499 +6 499 +7 499 +8 499 +9 499 +10 499 +11 499 +12 499 +13 499 +14 499 +15 499 +16 499 +17 499 +18 499 +19 499 +20 499 +21 499 +22 499 +23 499 +24 499 +25 499 +26 499 +27 499 +28 499 +29 499 +30 499 +31 499 +32 499 +33 499 +34 499 +35 499 +36 499 +37 499 +38 499 +39 499 +40 499 +41 499 +42 499 +43 499 +44 499 +45 499 +46 499 +47 499 +48 499 +49 499 +50 499 +51 499 +52 499 +53 499 +54 499 +55 499 +56 499 +57 499 +58 499 +59 499 +60 499 +61 499 +62 499 +63 499 +64 499 +65 499 +66 499 +67 499 +68 499 +69 499 +70 499 +71 499 +72 499 +73 499 +74 499 +75 499 +76 499 +77 499 +78 499 +79 499 +80 499 +81 499 +82 499 +83 499 +84 499 +85 499 +86 499 +87 499 +88 499 +89 499 +90 499 +91 499 +92 499 +93 499 +94 499 +95 499 +96 499 +97 499 +98 499 +99 499 +100 499 +101 499 +102 499 +103 499 +104 499 +105 499 +106 499 +107 499 +108 499 +109 499 +110 499 +111 499 +112 499 +113 499 +114 499 +115 499 +116 499 +117 499 +118 499 +119 499 +120 499 +121 499 +122 499 +123 499 +124 499 +125 499 +126 499 +127 499 +128 499 +129 499 +130 499 +131 499 +132 499 +133 499 +134 499 +135 499 +136 499 +137 499 +138 499 +139 499 +140 499 +141 499 +142 499 +143 499 +144 499 +145 499 +146 499 +147 499 +148 499 +149 499 +150 499 +151 499 +152 499 +153 499 +154 499 +155 499 +156 499 +157 499 +158 499 +159 499 +160 499 +161 499 +162 499 +163 499 +164 499 +165 499 +166 499 +167 499 +168 499 +169 499 +170 499 +171 499 +172 499 +173 499 +174 499 +175 499 +176 499 +177 499 +178 499 +179 499 +180 499 +181 499 +182 499 +183 499 +184 499 +185 499 +186 499 +187 499 +188 499 +189 499 +190 499 +191 499 +192 499 +193 499 +194 499 +195 499 +196 499 +197 499 +198 499 +199 499 +200 499 +201 499 +202 499 +203 499 +204 499 +205 499 +206 499 +207 499 +208 499 +209 499 +210 499 +211 499 +212 499 +213 499 +214 499 +215 499 +216 499 +217 499 +218 499 +219 499 +220 499 +221 499 +222 499 +223 499 +224 499 +225 499 +226 499 +227 499 +228 499 +229 499 +230 499 +231 499 +232 499 +233 499 +234 499 +235 499 +236 499 +237 499 +238 499 +239 499 +240 499 +241 499 +242 499 +243 499 +244 499 +245 499 +246 499 +247 499 +248 499 +249 499 +250 499 +251 499 +252 499 +253 499 +254 499 +255 499 +256 499 +257 499 +258 499 +259 499 +260 499 +261 499 +262 499 +263 499 +264 499 +265 499 +266 499 +267 499 +268 499 +269 499 +270 499 +271 499 +272 499 +273 499 +274 499 +275 499 +276 499 +277 499 +278 499 +279 499 +280 499 +281 499 +282 499 +283 499 +284 499 +285 499 +286 499 +287 499 +288 499 +289 499 +290 499 +291 499 +292 499 +293 499 +294 499 +295 499 +296 499 +297 499 +298 499 +299 499 +300 499 +301 499 +302 499 +303 499 +304 499 +305 499 +306 499 +307 499 +308 499 +309 499 +310 499 +311 499 +312 499 +313 499 +314 499 +315 499 +316 499 +317 499 +318 499 +319 499 +320 499 +321 499 +322 499 +323 499 +324 499 +325 499 +326 499 +327 499 +328 499 +329 499 +330 499 +331 499 +332 499 +333 499 +334 499 +335 499 +336 499 +337 499 +338 499 +339 499 +340 499 +341 499 +342 499 +343 499 +344 499 +345 499 +346 499 +347 499 +348 499 +349 499 +350 499 +351 499 +352 499 +353 499 +354 499 +355 499 +356 499 +357 499 +358 499 +359 499 +360 499 +361 499 +362 499 +363 499 +364 499 +365 499 +366 499 +367 499 +368 499 +369 499 +370 499 +371 499 +372 499 +373 499 +374 499 +375 499 +376 499 +377 499 +378 499 +379 499 +380 499 +381 499 +382 499 +383 499 +384 499 +385 499 +386 499 +387 499 +388 499 +389 499 +390 499 +391 499 +392 499 +393 499 +394 499 +395 499 +396 499 +397 499 +398 499 +399 499 +400 499 +401 499 +402 499 +403 499 +404 499 +405 499 +406 499 +407 499 +408 499 +409 499 +410 499 +411 499 +412 499 +413 499 +414 499 +415 499 +416 499 +417 499 +418 499 +419 499 +420 499 +421 499 +422 499 +423 499 +424 499 +425 499 +426 499 +427 499 +428 499 +429 499 +430 499 +431 499 +432 499 +433 499 +434 499 +435 499 +436 499 +437 499 +438 499 +439 499 +440 499 +441 499 +442 499 +443 499 +444 499 +445 499 +446 499 +447 499 +448 499 +449 499 +450 499 +451 499 +452 499 +453 499 +454 499 +455 499 +456 499 +457 499 +458 499 +459 499 +460 499 +461 499 +462 499 +463 499 +464 499 +465 499 +466 499 +467 499 +468 499 +469 499 +470 499 +471 499 +472 499 +473 499 +474 499 +475 499 +476 499 +477 499 +478 499 +479 499 +480 499 +481 499 +482 499 +483 499 +484 499 +485 499 +486 499 +487 499 +488 499 +489 499 +490 499 +491 499 +492 499 +493 499 +494 499 +495 499 +496 499 +497 499 +498 499 +499 499 diff --git a/examples/battle_model/src/render/backend/protocol.h b/examples/battle_model/src/render/backend/protocol.h new file mode 100755 index 0000000..e85c096 --- /dev/null +++ b/examples/battle_model/src/render/backend/protocol.h @@ -0,0 +1,45 @@ +#ifndef MAGNET_RENDER_BACKEND_PROTOCOL_H_ +#define MAGNET_RENDER_BACKEND_PROTOCOL_H_ + +#include + +#include "utility/exception.h" +#include "utility/utility.h" +#include "data.h" + +namespace magent { +namespace render { + +enum Type { + LOAD, + PICK +}; + +typedef const std::pair Result; + +template +class Base : public render::Unique { +private: + virtual T encode(const render::AgentData &)const = 0; + + virtual T encode(const render::EventData &)const = 0; + + virtual T encode(const render::BreadData &)const = 0; + + virtual T encode(const render::Config &, unsigned int)const = 0; + +public: + + virtual T encode(const render::Frame &, const render::Config &, + const render::Buffer &, const render::Window &)const = 0; + + virtual T encodeError(const T &)const = 0; + + virtual const std::pair decode(const T &)const = 0; + +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_PROTOCOL_H_ \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/render.cc b/examples/battle_model/src/render/backend/render.cc new file mode 100755 index 0000000..4156231 --- /dev/null +++ b/examples/battle_model/src/render/backend/render.cc @@ -0,0 +1,12 @@ +#include "server.h" +#include "websocket.h" +#include "text.h" + +int main(int argc, char *argv[]) { + magent::render::RenderConfig config; + magent::render::parse(argc, argv, config); + magent::render::Logger::verbose = !config.quiet; + magent::render::TextServer server(config, 256); + + server.run(); +} diff --git a/examples/battle_model/src/render/backend/server.h b/examples/battle_model/src/render/backend/server.h new file mode 100755 index 0000000..79b431b --- /dev/null +++ b/examples/battle_model/src/render/backend/server.h @@ -0,0 +1,120 @@ +#ifndef MAGNET_RENDER_BACKEND_SERVER_H_ +#define MAGNET_RENDER_BACKEND_SERVER_H_ + +#include + +#include "utility/utility.h" +#include "protocol.h" +#include "socket.h" +#include "utility/config.h" +#include "utility/logger.h" + +namespace magent { +namespace render { + +template +class TextServer : protected S { +private: + static_assert(std::is_base_of, P>::value, "P is not derived from magent::render::Base"); + static_assert(std::is_base_of, S>::value, "S is not derived from magent::render::ISocket"); + + P protocol; + render::Buffer buffer; + render::Config config; + + void receive(const std::string & message) override { + try { + std::pair data = protocol.decode(message); + switch (data.first) { + case magent::render::LOAD: + load( + static_cast * const>(data.second)->first, + static_cast * const>(data.second)->second + ); + delete(static_cast * const>(data.second)); + break; + case magent::render::PICK: + const std::pair &coordinate = + *static_cast * const>(data.second); + pick(coordinate.first, coordinate.second); + delete(static_cast * const>(data.second)); + break; + } + } catch (const render::RenderException &e) { + render::Logger::STDERR.log(e.what()); + reply(e.what()); + return; + } + } + +protected: + void pick(int frame, const render::Window & window) { + S::reply(protocol.encode(buffer[frame], config, buffer, window)); + } + + void load(const std::string & conf_path, const std::string & data_path) { + std::ifstream handleConf(conf_path); + try { + config.load(handleConf); + render::Logger::STDERR.log(config.getDataPath() + '/' + data_path); + std::ifstream handleData(config.getDataPath() + '/' + data_path); + buffer.load(handleData); + reply(buffer.getFramesNumber()); + } catch (const magent::render::RenderException &e) { + render::Logger::STDERR.log(e.what()); + reply(e.what()); + } + handleConf.close(); + } + + /** + * Reply frame data to the frontend. + * @param frame: the frame to be send to the frontend. + */ + void reply(const render::Frame & frame) { + S::reply(protocol.encode(frame)); + } + + /** + * Reply error message to the frontend. + * @param message: the error message to be send to the frontend. + */ + void reply(const std::string & message) override { + S::reply(protocol.encodeError(message)); + } + + /** + * Reply frame initial message to the frontend. + * @param message: the initial data to be send to the frontend. + */ + void reply(unsigned int nFrame) { + S::reply(protocol.encode(config, nFrame)); + } + + void open() override { + + } + + void close() override { + + } + + void error() override { + + } + +public: + explicit TextServer(const RenderConfig &config, unsigned int maxBufferSize) + : buffer(maxBufferSize), config(), S(config.port) { + + } + + void run() override { + S::run(); + } +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_SERVER_H_ diff --git a/examples/battle_model/src/render/backend/socket.h b/examples/battle_model/src/render/backend/socket.h new file mode 100755 index 0000000..31f5387 --- /dev/null +++ b/examples/battle_model/src/render/backend/socket.h @@ -0,0 +1,37 @@ +#ifndef MAGNET_RENDER_BACKEND_SOCKET_H_ +#define MAGNET_RENDER_BACKEND_SOCKET_H_ + +#include +#include + +#include "utility/utility.h" + +namespace magent { +namespace render { + +template +class ISocket : public render::Unique { +protected: + const T args; + +public: + explicit ISocket(const T &args) : args(args) { + } + + virtual void reply(const std::string &) = 0; + + virtual void receive(const std::string &) = 0; + + virtual void open() = 0; + + virtual void close() = 0; + + virtual void error() = 0; + + virtual void run() = 0; +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_SOCKET_H_ diff --git a/examples/battle_model/src/render/backend/text.cc b/examples/battle_model/src/render/backend/text.cc new file mode 100755 index 0000000..02d03ad --- /dev/null +++ b/examples/battle_model/src/render/backend/text.cc @@ -0,0 +1,190 @@ +#ifndef MAGNET_RENDER_BACKEND_TEXT_CPP_ +#define MAGNET_RENDER_BACKEND_TEXT_CPP_ + +#include +#include +#include +#include "text.h" +#include "utility/logger.h" + +namespace magent { +namespace render { + +std::string Text::encode(const magent::render::AgentData &agent)const { + return std::to_string(agent.id) + + ' ' + std::to_string(agent.position.x) + + ' ' + std::to_string(agent.position.y) + + ' ' + std::to_string(agent.groupID) + + ' ' + std::to_string(agent.direction) + + ' ' + std::to_string(agent.hp) + + ' '+ std::to_string(agent.ch); +} + +std::string Text::encode(const magent::render::EventData &event)const { + return std::to_string(event.type) + + ' ' + std::to_string(event.agent->id) + + ' ' + std::to_string(event.position.x) + + ' ' + std::to_string(event.position.y); +} + +std::string Text::encode(const render::Config &config, unsigned int nFrame)const { + return 'i' + std::to_string(nFrame) + '|' + config.getFrontendJSON(); +} + +std::string Text::encodeError(const std::string &message)const { + return 'e' + message; +} + +Result Text::decode(const std::string &data)const { + switch (data[0]) { + case 'l': { + long pos = data.find_first_of(','); + if (pos == std::string::npos) { + throw RenderException("invalid load operation"); + } + return { + Type::LOAD, + new std::pair( + data.substr(1, static_cast(pos - 1)), + data.substr(static_cast(pos + 1), data.length() - pos) + ) + }; + } + case 'p': { + int frameID, xmin, xmax, ymin, ymax; + if (sscanf(data.substr(1).c_str(), "%d%d%d%d%d", &frameID, &xmin, &ymin, &xmax, &ymax) != 5) { + throw RenderException("invalid pick operation"); + } + return { + Type::PICK, + new std::pair(frameID, render::Window(xmin, ymin, xmax, ymax)) + }; + } + default: + throw RenderException("invalid message"); + } +} + +std::string Text::encode(const magent::render::Frame &frame, + const magent::render::Config &config, + const magent::render::Buffer &buffer, + const magent::render::Window &window)const { + std::string result("f"); + std::unordered_map hasEvent; + for (unsigned int i = 0, size = frame.getEventsNumber(), first = 1; i < size; i++) { + const render::EventData &data = frame.getEvent(i); + const render::Style &style = config.getStyle(data.agent->groupID); + unsigned int width = style.width; + unsigned int height = style.height; + if (data.agent->direction % 180 != 0) std::swap(width, height); + if (window.accept(data.position.x, data.position.y) + || window.accept(data.agent->position.x, data.agent->position.y, width, height)) { + hasEvent[data.agent->id] = true; + if (first == 0u) result.append("|"); + result.append(encode(data)); + first = 0; + } + } + result.append(";"); + + unsigned int mapHeight = config.getHeight(); + unsigned int mapWidth = config.getWidth(); + unsigned int miniMAPHeight = config.getMiniMAPHeight(); + unsigned int miniMAPWidth = config.getMiniMAPWidth(); + const unsigned int & nStyles = config.getStylesNumber(); + auto minimap = new unsigned int*[miniMAPHeight * miniMAPWidth]; + auto agentsCounter = new unsigned int[nStyles]; + for (unsigned int i = 0; i < miniMAPHeight * miniMAPWidth; i++) { + minimap[i] = new unsigned int[nStyles]; + std::fill(minimap[i], minimap[i] + nStyles, 0); + } + memset(agentsCounter, 0, sizeof(*agentsCounter) * nStyles); + for (unsigned int i = 0, size = frame.getAgentsNumber(), first = 1; i < size; i++) { + const magent::render::AgentData & data = frame.getAgent(i); + const render::Style &style = config.getStyle(data.groupID); + unsigned int width = style.width; + unsigned int height = style.height; + if (data.direction % 180 != 0) std::swap(width, height); + if (hasEvent[data.id] || window.accept(data.position.x, data.position.y, width, height)) { + if (first == 0) result.append("|"); + result.append(encode(data)); + first = 0; + } + agentsCounter[data.groupID]++; + + auto miniPositionX = static_cast(1.0 * data.position.x / mapWidth * miniMAPWidth); + auto miniPositionY = static_cast(1.0 * data.position.y / mapHeight * miniMAPHeight); + minimap[miniPositionY * miniMAPWidth + miniPositionX][data.groupID]++; + } + result.append(";"); + + for (unsigned int i = 0, size = frame.getBreadsNumber(), first = 1; i < size; i++) { + const magent::render::BreadData & data = frame.getBread(i); + if (window.accept(data.position.x, data.position.y)) { + if (first == 0u) result.append("|"); + result.append(encode(data)); + first = 0; + } + } + result.append(";"); + + for (unsigned int i = 0, size = buffer.getObstaclesNumber(), first = 1; i < size; i++) { + const render::Coordinate &now = buffer.getObstacle(i); + if (window.accept(now.x, now.y)) { + if (first == 0u) result.append("|"); + result.append(std::to_string(now.x)); + result.append(" "); + result.append(std::to_string(now.y)); + first = 0; + } + } + result.append(";"); + + for (unsigned int i = 0, first = 1; i < miniMAPHeight * miniMAPWidth; i++) { + if (first == 0u) result.append(" "); + double red = 0, blue = 0, green = 0; + unsigned int sum = 0; + for (unsigned int j = 0; j < nStyles; j++) { + sum += minimap[i][j]; + } + for (unsigned int j = 0; j < nStyles; j++) { + red += 1.0 * config.getStyle(j).red * minimap[i][j] / sum; + blue += 1.0 * config.getStyle(j).blue * minimap[i][j] / sum; + green += 1.0 * config.getStyle(j).green * minimap[i][j] / sum; + } + unsigned int value = 0; + if (sum == 0u) { + value = (0xFFu << 24) | (0xFFu << 16) | (0xFFu << 8) | (0xFFu << 0); + } else { + value |= static_cast(red) << 24; + value |= static_cast(blue) << 16; + value |= static_cast(green) << 8; + value |= static_cast(0xFFu) << 0; + } + + result.append(std::to_string(value)); + first = 0; + delete[](minimap[i]); + } + delete[](minimap); + + result.append(";"); + for (unsigned int i = 0, first = 1; i < nStyles; i++) { + if (first == 0u) result.append(" "); + result.append(std::to_string(agentsCounter[i])); + first = 0; + } + + return result; +} + +std::string Text::encode(const magent::render::BreadData &bread) const { + return std::to_string(bread.position.x) + + ' ' + std::to_string(bread.position.y) + + ' ' + std::to_string(bread.hp); +} + +} // namespace render +} // namespace magent + +#endif // MAGNET_RENDER_BACKEND_TEXT_CPP_ \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/text.h b/examples/battle_model/src/render/backend/text.h new file mode 100755 index 0000000..65faeb2 --- /dev/null +++ b/examples/battle_model/src/render/backend/text.h @@ -0,0 +1,36 @@ +#ifndef MAGNET_RENDER_BACKEND_PROTOCOL_TEXT_H_ +#define MAGNET_RENDER_BACKEND_PROTOCOL_TEXT_H_ + +#include + +#include "protocol.h" +#include "data.h" + +namespace magent { +namespace render { + +class Text : public Base { +private: + std::string encode(const render::AgentData & /*unused*/)const override; + + std::string encode(const render::EventData & /*unused*/)const override; + + std::string encode(const render::BreadData & /*unused*/)const override ; + +public: + std::string encode(const render::Config & /*unused*/, unsigned int /*unused*/)const override; + + std::string encode(const render::Frame & /*unused*/, + const render::Config & /*unused*/, + const render::Buffer & /*unused*/, + const render::Window & /*unused*/)const override; + + std::string encodeError(const std::string & /*unused*/)const override; + + Result decode(const std::string & /*unused*/)const override; +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_PROTOCOL_TEXT_H_ diff --git a/examples/battle_model/src/render/backend/utility/config.cc b/examples/battle_model/src/render/backend/utility/config.cc new file mode 100755 index 0000000..7c4cb6f --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/config.cc @@ -0,0 +1,57 @@ +#include +#include +#include "config.h" + +namespace magent { +namespace render { + +const char MATRIX_ARGP_ARGS_DOCUMENT[] = "arg1 arg2 ..."; +const char MATRIX_ARGP_DOCUMENT[] = "render: The backend server of magnet platform."; +const argp_option MATRIX_ARGP_OPTIONS[] ={ + {"port" , 'P', "PORT" , 0, "Specify the port to be used by the server(the default port is 9030)." , 1}, + {"quiet" , 'Q', nullptr, 0, "Quiet mode will be used and almost all warning, diagnostic and exception message will be suppressed.", 2}, + {nullptr} +}; + +error_t __parser(int key, char *arg, struct argp_state *state) { + RenderConfig &config = *static_cast(state->input); + switch (key) { + case 'P': { + int port = 0; + for (size_t i = 0, size = strlen(arg); i < size; i++) { + if (isdigit(arg[i]) == 0) { + std::cout << "port must be positive integer between (0..65535)." << std::endl; + return ARGP_ERR_UNKNOWN; + } + port = port * 10 + (arg[i] - '0'); + if (port < 0 || port >= 65536) { + std::cout << "port must be positive integer between (0..65535)." << std::endl; + return ARGP_ERR_UNKNOWN; + } + } + config.port = static_cast(port); + break; + } + case 'Q': { + config.quiet = true; + break; + } + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +const argp MATRIX_ARGP = { + MATRIX_ARGP_OPTIONS, + __parser, + MATRIX_ARGP_ARGS_DOCUMENT, + MATRIX_ARGP_DOCUMENT +}; + +void parse(int argc, char *argv[], RenderConfig &config) { + argp_parse(&MATRIX_ARGP, argc, argv, ARGP_NO_ARGS | ARGP_IN_ORDER, 0, &config); +} + +} // namespace render +} // namespace magnet \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/utility/config.h b/examples/battle_model/src/render/backend/utility/config.h new file mode 100755 index 0000000..202a077 --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/config.h @@ -0,0 +1,21 @@ +#ifndef MATRIX_RENDER_BACKEND_UTILITY_CONFIG_H_ +#define MATRIX_RENDER_BACKEND_UTILITY_CONFIG_H_ + +#include +#include +#include + +namespace magent { +namespace render { + +struct RenderConfig { + uint16_t port = 9030; + bool quiet = false; +}; + +void parse(int argc, char *argv[], RenderConfig &config); + +} // namespace magent +} // namespace ARGPParser + +#endif //MATRIX_RENDER_BACKEND_UTILITY_CONFIG_H_ \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/utility/exception.h b/examples/battle_model/src/render/backend/utility/exception.h new file mode 100755 index 0000000..5b386ef --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/exception.h @@ -0,0 +1,30 @@ +#ifndef MAGNET_RENDER_BACKEND_UTILITY_EXCEPTION_H_ +#define MAGNET_RENDER_BACKEND_UTILITY_EXCEPTION_H_ + +#include +#include +#include "utility.h" + +namespace magent { +namespace render { + +class BaseException : public std::runtime_error { +public: + explicit BaseException(const std::string &message) : std::runtime_error(message) {} + + explicit BaseException(const char *message) : std::runtime_error(message) {} +}; + +class RenderException : public BaseException { +public: + explicit RenderException(const std::string &message) : + BaseException("magent::render::RenderException: \"" + message + "\"") {} + + explicit RenderException(const char *message) : + BaseException("magent::render::RenderException: \"" + std::string(message) + "\"") {} +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_UTILITY_EXCEPTION_H_ diff --git a/examples/battle_model/src/render/backend/utility/logger.cc b/examples/battle_model/src/render/backend/utility/logger.cc new file mode 100755 index 0000000..92fc718 --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/logger.cc @@ -0,0 +1,27 @@ +#include "logger.h" + +namespace magent { +namespace render { + +Logger Logger::STDOUT(std::cout); +Logger Logger::STDERR(std::cerr); +bool Logger::verbose = true; + +void Logger::log(const std::string &rhs) { + if (verbose) { + std::time_t time = std::time(nullptr); + std::unique_lock locker(mutex); + ostream << std::put_time(std::localtime(&time), "[%Y-%m-%d %H:%M:%S] [") + << std::this_thread::get_id() << "] " << rhs << std::endl; + } +} + +void Logger::raw(const std::string &rhs) { + if (verbose) { + std::unique_lock locker(mutex); + ostream << rhs << std::endl; + } +} + +} // namespace magent +} // namespace render \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/utility/logger.h b/examples/battle_model/src/render/backend/utility/logger.h new file mode 100755 index 0000000..a1dacf2 --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/logger.h @@ -0,0 +1,37 @@ +#ifndef MATRIX_RENDER_BACKEND_UTILITY_LOGGER_H_ +#define MATRIX_RENDER_BACKEND_UTILITY_LOGGER_H_ + +#include +#include +#include +#include +#include +#include +#include "utility.h" + +namespace magent { +namespace render { + +/** + * Logger is a thread-safe class to output message + */ +class Logger : public Unique { +private: + std::mutex mutex; + std::ostream & ostream; + + explicit Logger(std::ostream &ostream) : ostream(ostream) {} + +public: + static Logger STDOUT; + static Logger STDERR; + static bool verbose; + + void log(const std::string &rhs); + void raw(const std::string &rhs); +}; + +} // namespace render +} // namespace magent + +#endif //MATRIX_RENDER_BACKEND_UTILITY_LOGGER_H_ diff --git a/examples/battle_model/src/render/backend/utility/utility.h b/examples/battle_model/src/render/backend/utility/utility.h new file mode 100755 index 0000000..a82a2d0 --- /dev/null +++ b/examples/battle_model/src/render/backend/utility/utility.h @@ -0,0 +1,25 @@ +#ifndef MAGNET_RENDER_BACKEND_UTILITY_UNIQUE_H_ +#define MAGNET_RENDER_BACKEND_UTILITY_UNIQUE_H_ + +namespace magent { +namespace render { + +class Unique { +public: + Unique() = default; + + Unique(const Unique &) = delete; + + Unique(const Unique &&) = delete; + + Unique &operator =(const Unique &) = delete; + + Unique &operator =(const Unique &&) = delete; + + virtual ~Unique() = default; +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_UTILITY_UNIQUE_H_ diff --git a/examples/battle_model/src/render/backend/websocket.cc b/examples/battle_model/src/render/backend/websocket.cc new file mode 100755 index 0000000..3690d8a --- /dev/null +++ b/examples/battle_model/src/render/backend/websocket.cc @@ -0,0 +1,80 @@ +#include "websocket.h" + +namespace magent { +namespace render { + +void WebSocket::__on_open(const websocketpp::connection_hdl & /*unused*/) { + ws.stop_listening(); + Logger::STDERR.log("successfully connected to the client"); + open(); +} + +void WebSocket::__on_close(const websocketpp::connection_hdl & /*unused*/) { + Logger::STDERR.log("the client closed the connection"); + close(); +#ifdef DEBUG + ws.listen(args); + ws.start_accept(); +#else + std::string answer; + while (true) { + std::cout << "Enter yes or no to continue or stop the server: "; + std::cin >> answer; + if (answer == "YES" || answer == "Yes" || answer == "yes") { + ws.listen(args); + ws.start_accept(); + break; + } + if (answer == "No" || answer == "no" || answer == "NO") { + ws.stop(); + break; + } + std::cout << "Please enter yes or no to continue or stop the server."; + } +#endif +} + +void WebSocket::__on_error(const websocketpp::connection_hdl & /*unused*/) { + error(); +} + +void WebSocket::__on_message(const websocketpp::connection_hdl &connection_hdl, WSServer::message_ptr message) { + this->connection_hdl = &connection_hdl; + Logger::STDERR.log("message receive: " + message->get_payload()); + receive(message->get_payload()); + this->connection_hdl = nullptr; +} + +WebSocket::WebSocket(uint16_t port) : ISocket(port), connection_hdl(nullptr) { + ws.set_reuse_addr(true); + ws.init_asio(); + ws.clear_access_channels(websocketpp::log::alevel::all); + ws.clear_error_channels(websocketpp::log::alevel::all); + ws.set_open_handler(websocketpp::lib::bind(&WebSocket::__on_open, this, websocketpp::lib::placeholders::_1)); + ws.set_close_handler(websocketpp::lib::bind(&WebSocket::__on_close, this, websocketpp::lib::placeholders::_1)); + ws.set_fail_handler(websocketpp::lib::bind(&WebSocket::__on_error, this, websocketpp::lib::placeholders::_1)); + ws.set_message_handler(websocketpp::lib::bind( + &WebSocket::__on_message, this, + websocketpp::lib::placeholders::_1, + websocketpp::lib::placeholders::_2 + )); +} + +void WebSocket::reply(const std::string &message) { + websocketpp::lib::error_code errcode; + ws.send(*connection_hdl, message, websocketpp::frame::opcode::text, errcode); +} + +void WebSocket::run() { + try { + Logger::STDERR.log("Listening on port " + std::to_string(args)); + ws.listen(args); + ws.start_accept(); + ws.run(); + } catch (const websocketpp::exception &e) { + Logger::STDERR.log("cannot listen on port " + std::to_string(args) + " reason: " + e.what()); + } +} + +} // namespace render +} // namespace magent \ No newline at end of file diff --git a/examples/battle_model/src/render/backend/websocket.h b/examples/battle_model/src/render/backend/websocket.h new file mode 100755 index 0000000..8f7fe09 --- /dev/null +++ b/examples/battle_model/src/render/backend/websocket.h @@ -0,0 +1,39 @@ +#ifndef MAGNET_RENDER_BACKEND_WEBSOCKET_H_ +#define MAGNET_RENDER_BACKEND_WEBSOCKET_H_ + +#include +#include + +#include "utility/logger.h" +#include "socket.h" + +namespace magent { +namespace render { + +class WebSocket : public ISocket { +private: + typedef websocketpp::server WSServer; + + WSServer ws; + const websocketpp::connection_hdl * connection_hdl; + + void __on_open(const websocketpp::connection_hdl & /*connection*/); + + void __on_close(const websocketpp::connection_hdl & /*connection*/); + + void __on_error(const websocketpp::connection_hdl & /*connection*/); + + void __on_message(const websocketpp::connection_hdl & connection_hdl, WSServer::message_ptr message); + +public: + explicit WebSocket(uint16_t port); + + void reply(const std::string & message) override; + + void run() override; +}; + +} // namespace render +} // namespace magent + +#endif //MAGNET_RENDER_BACKEND_WEBSOCKET_H_ diff --git a/examples/battle_model/src/render/frontend/index.html b/examples/battle_model/src/render/frontend/index.html new file mode 100755 index 0000000..64e6ce7 --- /dev/null +++ b/examples/battle_model/src/render/frontend/index.html @@ -0,0 +1,106 @@ + + + + + magent Viewer + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + + \ No newline at end of file diff --git a/examples/battle_model/src/render/frontend/js/render-handle.js b/examples/battle_model/src/render/frontend/js/render-handle.js new file mode 100755 index 0000000..2ac2a70 --- /dev/null +++ b/examples/battle_model/src/render/frontend/js/render-handle.js @@ -0,0 +1,717 @@ +var _gridCTX = undefined; +var _staticCTX = undefined; +var _agentRangeCTX = undefined; +var _agentBodyCTX = undefined; +var _agentHPCTX = undefined; +var _eventCTX = undefined; +var _statusStaticCTX = undefined; +var _minimapCTX = undefined; +var _minimapPostionCTX = undefined; + +var _socket = undefined; +var _offsetX = undefined; +var _offsetY = undefined; +var _mapFPS = undefined; +var _mapIPS = undefined; +var _mapProcessedFrame = undefined; +var _mapProcessedImage = undefined; +var _mapCurrentImage = undefined; +var _mapTotalImage = undefined; +var _mapStatTime = undefined; +var _mapAnimateTick = undefined; +var _mapStatus = undefined; +var _mapData = undefined; +var _mapLastData = undefined; +var _mapStyles = undefined; +var _mapSpeed = undefined; +var _mapForcedPause = undefined; + +var _isBrowserSizeChanged = undefined; +var _isWindowChanged = undefined; +var _isGridSizeChanged = undefined; +$('#magnet-file-form-conf').val('config.json') +$('#magnet-file-form-map').val('video_1.txt') +function _onresize() { + _isBrowserSizeChanged = true; + _isGridSizeChanged = true; + _isWindowChanged = true; +} + +function _drawGrid() { + _gridCTX.clearRect(0, 0, _gridCTX.canvas.width, _gridCTX.canvas.height); + _gridCTX.beginPath(); + _gridCTX.strokeStyle = "#ffffff"; + var i; + var magentX = Math.floor(_offsetX) - _offsetX; + var magentY = Math.floor(_offsetY) - _offsetY; + for (i = 0; i * gridSize < _gridCTX.canvas.height; i++) { + _gridCTX.moveTo(0, i * gridSize + magentY / gridSize); + _gridCTX.lineTo(_gridCTX.canvas.width, i * gridSize + magentY / gridSize); + } + for (i = 0; i * gridSize < _gridCTX.canvas.width; i++) { + _gridCTX.moveTo(i * gridSize + magentX / gridSize, 0); + _gridCTX.lineTo(i * gridSize + magentX / gridSize, _gridCTX.canvas.height); + } + _gridCTX.stroke(); +} + +function _drawStatusFigure() { + // _drawGrid(); + _statusStaticCTX.clearRect(0, 0, _statusStaticCTX.canvas.width, _statusStaticCTX.canvas.height); + _statusStaticCTX.beginPath(); + if (_socket !== undefined) { + if (_socket.readyState === _socket.CONNECTING) { + _statusStaticCTX.strokeText('render Status: CONNECTING', STATUS_PADDING_LEFT, STATUS_PADDING_TOP); + } else if (_socket.readyState === _socket.OPEN) { + _statusStaticCTX.strokeText('render Status: OPEN', STATUS_PADDING_LEFT, STATUS_PADDING_TOP); + if (_mapStyles !== undefined) { + _statusStaticCTX.strokeText('IPS: ' + _mapIPS.toString(), STATUS_PADDING_LEFT, STATUS_PADDING_TOP + STATUS_SPACING * 2); + _statusStaticCTX.strokeText('Window: (' + + Math.floor(_offsetY).toString() + ', ' + + Math.floor(_offsetY).toString() + ', ' + + (Math.ceil(_offsetX + _gridCTX.canvas.width / gridSize)).toString() + ', ' + + (Math.ceil(_offsetY + _gridCTX.canvas.height / gridSize)).toString() + ')', + STATUS_PADDING_LEFT, STATUS_PADDING_TOP + STATUS_SPACING * 3 + ); + } + } else if (_socket.readyState === _socket.CLOSING) { + _statusStaticCTX.strokeText('render Status: CLOSING', STATUS_PADDING_LEFT, STATUS_PADDING_TOP); + } else { + _statusStaticCTX.strokeText('render Status: CLOSED', STATUS_PADDING_LEFT, STATUS_PADDING_TOP); + } + } else { + _statusStaticCTX.strokeText('render Status: PREPARING', STATUS_PADDING_LEFT, STATUS_PADDING_TOP); + } + _statusStaticCTX.strokeText('FPS: ' + _mapFPS.toString(), STATUS_PADDING_LEFT, STATUS_PADDING_TOP + STATUS_SPACING); +} + +function _drawNumbers() { + _statusDynamicCTX.clearRect(0, 0, _statusDynamicCTX.canvas.width, _statusDynamicCTX.canvas.height); + for (var i = 0; i < _mapData[5].length; i++) { + _statusDynamicCTX.strokeText( + 'Group ' + i.toString() + ' number: ' + _mapData[5][i], + STATUS_PADDING_LEFT, + STATUS_PADDING_TOP + STATUS_SPACING * (4 + i) + ); + } +} + +function _onkeydown(event) { + var windowCenterX, windowCenterY; + if (event.keyCode === 37) { + _offsetX = _offsetX - Math.max(1, Math.round(MOVE_SPACING * 10 / gridSize)); // Left + _isWindowChanged = true; + } else if (event.keyCode === 38) { + _offsetY = _offsetY - Math.max(1, Math.round(MOVE_SPACING * 10 / gridSize)); // Up + _isWindowChanged = true; + } else if (event.keyCode === 39) { + _offsetX = _offsetX + Math.max(1, Math.round(MOVE_SPACING * 10 / gridSize)); + _isWindowChanged = true; + } else if (event.keyCode === 40) { + _offsetY = _offsetY + Math.max(1, Math.round(MOVE_SPACING * 10 / gridSize)); + _isWindowChanged = true; + } else if (event.keyCode === 69) { + $("#magnet-file-modal").modal('show'); + } else if (event.keyCode === 72) { + $("#magnet-help-modal").modal('show'); + } else if (event.keyCode === 83) { + $("#magnet-settings-modal").modal('show'); + } else if (event.keyCode === 188) { + windowCenterX = _gridCTX.canvas.width / 2 / gridSize + _offsetX; + windowCenterY = _gridCTX.canvas.height / 2 / gridSize + _offsetY; + gridSize = Math.max(1, gridSize - 1); + _offsetX = _offsetX + windowCenterX - (_gridCTX.canvas.width / 2 / gridSize + _offsetX); + _offsetY = _offsetY + windowCenterY - (_gridCTX.canvas.height / 2 / gridSize + _offsetY); + _isGridSizeChanged = true; + _isWindowChanged = true; + } else if (event.keyCode === 190) { + windowCenterX = _gridCTX.canvas.width / 2 / gridSize + _offsetX; + windowCenterY = _gridCTX.canvas.height / 2 / gridSize + _offsetY; + gridSize = Math.min(100, gridSize + 1); + _offsetX = _offsetX + windowCenterX - (_gridCTX.canvas.width / 2 / gridSize + _offsetX); + _offsetY = _offsetY + windowCenterY - (_gridCTX.canvas.height / 2 / gridSize + _offsetY); + _isGridSizeChanged = true; + _isWindowChanged = true; + } else if (event.keyCode === 80) { + _mapForcedPause ^= 1; + } +} + +function _collectTicks() { + var timenow = performance.now(); + var elapsed = timenow - _mapStatTime; + _mapIPS = elapsed < STATUS_FPS_PERIOD ? 'N/A' : Math.round(1000 * _mapProcessedImage / elapsed); + _mapFPS = elapsed < STATUS_FPS_PERIOD ? 'N/A' : Math.round(1000 * _mapProcessedFrame / elapsed); + _mapStatTime = timenow; + _mapProcessedFrame = 0; + _mapProcessedImage = 0; + _drawStatusFigure(); + // _drawGrid(); + setTimeout(_collectTicks, STATUS_FPS_PERIOD); +} + +function run() { + _statusStaticCTX = document.getElementById('magnet-canvas-status-physical-information').getContext('2d'); + _statusStaticCTX.canvas.height = STATUS_HEIGHT; + _statusStaticCTX.canvas.width = STATUS_WIDTH; + _statusDynamicCTX = document.getElementById('magnet-canvas-status-statistics').getContext('2d'); + _statusDynamicCTX.canvas.height = STATUS_HEIGHT; + _statusDynamicCTX.canvas.width = STATUS_WIDTH; + + _mapStatTime = performance.now(); + + _collectTicks(); + + window.addEventListener('resize', _onresize); + window.addEventListener('keydown', _onkeydown); + $('#magnet-file-form-submit').click(function (e) { + e.preventDefault(); + if (_socket.readyState !== _socket.OPEN) { + $.jGrowl('This client is not connected to the server, please waiting to be connected', { + position: 'bottom-right' + }); + } else { + _mapStatus = 'STOP'; + _socket.send('l' + $('#magnet-file-form-conf').val() + ',' + $('#magnet-file-form-map').val()); + $('#magnet-file-modal').modal('hide'); + } + }); + $('#magnet-file-modal').on('shown.bs.modal', function() { + window.removeEventListener('keydown', _onkeydown); + }).on('hide.bs.modal', function() { + window.addEventListener('keydown', _onkeydown); + }); + $('#magnet-help-modal').on('shown.bs.modal', function() { + window.removeEventListener('keydown', _onkeydown); + }).on('hide.bs.modal', function() { + window.addEventListener('keydown', _onkeydown); + }); + $('#magnet-settings-modal').on('shown.bs.modal', function() { + window.removeEventListener('keydown', _onkeydown); + }).on('hide.bs.modal', function() { + window.addEventListener('keydown', _onkeydown); + }); + // _drawGrid(); + + var _connect = function () { + _socket = new WebSocket(SOCKET_HOST); + _socket.onopen = function () { + console.log('successfully connected to the backend server'); + $("#magnet-file-modal").modal('show'); + // _drawGrid(); + _drawStatusFigure(); + }; + _socket.onclose = function () { + console.log('connection closed.'); + _mapStatus = 'STOP'; + $.jGrowl('Connection lost from the server.', { + position: 'bottom-right' + }); + $('#magnet-settings-progress').unbind().attr('disabled', true); + $('#magnet-settings-speed').unbind().attr('disabled', true); + setTimeout(_connect, SOCKET_RECONNECT_PERIOD); + // _drawGrid(); + _drawStatusFigure(); + }; + _socket.onerror = function () { + console.log('connection lost.'); + _drawStatusFigure(); + }; + _socket.onmessage = function (data) { + data = data.data; + var op = data[0]; + data = data.substr(1); + // console.log(op) + switch (op) { + case 'i': + var pos = data.indexOf('|'); + _mapStyles = eval('(' + data.substr(pos + 1) + ')'); + _mapTotalImage = parseInt(data.substr(0, pos)); + _mapProcessedImage = 0; + _mapCurrentImage = 0; + _mapLastData = undefined; + _mapData = undefined; + _offsetX = 0; + _offsetY = 0; + gridSize = 10; + _mapAnimateTick = 0; + _mapSpeed = 80; + _mapForcedPause = false; + + _gridCTX = document.getElementById('magnet-canvas-grid').getContext('2d'); + // console.log(_gridCTX) + _drawGrid(); + _agentRangeCTX = document.getElementById('magnet-canvas-agent-range').getContext('2d'); + _eventCTX = document.getElementById('magnet-canvas-event').getContext('2d'); + _agentBodyCTX = document.getElementById('magnet-canvas-agent-body').getContext('2d'); + _staticCTX = document.getElementById('magnet-canvas-static').getContext('2d'); + _agentHPCTX = document.getElementById('magnet-canvas-agent-hp').getContext('2d'); + _minimapCTX = document.getElementById('magnet-canvas-minimap').getContext('2d'); + _minimapCTX.canvas.height = _mapStyles['minimap-height']; + _minimapCTX.canvas.width = _mapStyles['minimap-width']; + _minimapPostionCTX = document.getElementById('magnet-canvas-minimap-position').getContext('2d'); + _minimapPostionCTX.canvas.height = _mapStyles['minimap-height']; + _minimapPostionCTX.canvas.width = _mapStyles['minimap-width']; + + _onresize(); + $('#magnet-settings-progress') + .removeAttr('disabled') + .attr('min', 0) + .attr('max', _mapTotalImage - 1) + .attr('value', 0) + .change(function () { + _mapLastData = undefined; + _mapData = undefined; + _mapAnimateTick = 0; + _mapCurrentImage = parseInt($('#magnet-settings-progress').val()); + _socket.send('p' + _mapCurrentImage.toString() + ' ' + + Math.floor(_offsetX).toString() + ' ' + Math.floor(_offsetY).toString() + ' ' + + Math.ceil(_offsetX + window.innerWidth / gridSize).toString() + ' ' + + Math.ceil(_offsetY + window.innerHeight / gridSize).toString()); + _mapStatus = 'PAUSE'; + }) + .bind('slide', function () { + _mapLastData = undefined; + _mapData = undefined; + _mapAnimateTick = 0; + _mapCurrentImage = parseInt($('#magnet-settings-progress').val()); + + _socket.send('p' + _mapCurrentImage.toString() + ' ' + + Math.floor(_offsetX).toString() + ' ' + Math.floor(_offsetY).toString() + ' ' + + Math.ceil(_offsetX + window.innerWidth / gridSize).toString() + ' ' + + Math.ceil(_offsetY + window.innerHeight / gridSize).toString()); + _mapStatus = 'PAUSE'; + }); + $('#magnet-settings-speed') + .removeAttr('disabled') + .attr('min', 0) + .attr('max', ANIMATE_STEP) + .attr('value', 80) + .bind('change', function () { + _mapSpeed = parseInt($('#magnet-settings-speed').val()); + }); + _mapStatus = 'PAUSE'; + + _animate(); + + _socket.send('p' + _mapCurrentImage.toString() + ' ' + + Math.floor(_offsetX).toString() + ' ' + Math.floor(_offsetY).toString() + ' ' + + Math.ceil(_offsetX + window.innerWidth / gridSize).toString() + ' ' + + Math.ceil(_offsetY + window.innerHeight / gridSize).toString()); + break; + case 'f': + _mapStatus = 'PLAY'; + _mapLastData = _mapData; + _mapData = [[], [], [], [], [], [],[]]; + + data = data.split(';'); + + data[0] = data[0].split('|'); // Events + for (var itEvents = 0, nEvents = data[0].length; itEvents < nEvents; itEvents++) { + if (data[0][itEvents] !== '') { + data[0][itEvents] = data[0][itEvents].split(' '); + _mapData[0].push([ + parseInt(data[0][itEvents][0]), + parseInt(data[0][itEvents][1]), + parseInt(data[0][itEvents][2]), + parseInt(data[0][itEvents][3]) + ]); + } + } + + data[1] = data[1].split('|'); // Agents + for (var itAgents = 0, nAgents = data[1].length; itAgents < nAgents; itAgents++) { + if (data[1][itAgents] === '') continue; + data[1][itAgents] = data[1][itAgents].split(' '); + var x = parseInt(data[1][itAgents][1]); + var y = parseInt(data[1][itAgents][2]); + var group = parseInt(data[1][itAgents][3]); + var dir = parseInt(data[1][itAgents][4]); + var hp = parseInt(data[1][itAgents][5]); + var ch=parseInt(data[1][itAgents][6]); + if (!_mapStyles['group'].hasOwnProperty(group)) { + $.jGrowl('group ' + group.toString() + ' is not found in the configuration file', { + position: 'bottom-right' + }); + _mapStatus = 'STOP'; + break; + } + if (dir === 90) { + x = x + _mapStyles['group'][group]['height'] - 1; + y = y + _mapStyles['group'][group]['width'] - 1; + } else if (dir === 180) { + y = y + _mapStyles['group'][group]['height'] - 1; + } else if (dir === 0) { + x = x + _mapStyles['group'][group]['width'] - 1; + } + _mapData[1][data[1][itAgents][0]] = [x, y, group, dir, hp] + } + + data[2] = data[2].split('|'); + for (var itBreads = 0, nBreads = data[2].length; itBreads < nBreads; itBreads++) { + if (data[2][itBreads] !== '') { + data[2][itBreads] = data[2][itBreads].split(' '); + _mapData[2].push([ + parseInt(data[2][itBreads][0]), + parseInt(data[2][itBreads][1]), + parseInt(data[2][itBreads][2]) + ]); + } + } + data[3] = data[3].split('|'); + for (var itObstacles = 0, nObstacles = data[3].length; itObstacles < nObstacles; itObstacles++) { + if (data[3][itObstacles] !== '') { + data[3][itObstacles] = data[3][itObstacles].split(' '); + _mapData[3].push([ + parseInt(data[3][itObstacles][0]), + parseInt(data[3][itObstacles][1]) + ]); + } + } + data[4] = data[4].split(' '); + for (var itPixels = 0, nPixels = data[4].length; itPixels < nPixels; itPixels++) { + if (data[4][itPixels] !== '') { + _mapData[4].push(parseInt(data[4][itPixels])); + } + } + + _mapData[5] = data[5].split(' '); + break; + case 'e': + $.jGrowl(data, {position: 'bottom-right'}); + break; + default: + $.jGrowl('invalid message from backend. Please check the version of frontend and backend', { + position: 'bottom-right' + }); + break; + } + }; + }; + _connect(); +} + +function _getOriginGridCoordinate(curData, oldData, style) { + var theta, anchorX, anchorY; + var rate = Math.min(_mapAnimateTick, ANIMATE_STEP) / ANIMATE_STEP; + var nowAngle = (curData[3] + 90) / 180 * Math.PI; + var nowAnchorX = curData[0] + style['anchor'][0] * Math.cos(nowAngle) - style['anchor'][1] * Math.sin(nowAngle) - _offsetX; + var nowAnchorY = curData[1] + style['anchor'][0] * Math.sin(nowAngle) + style['anchor'][1] * Math.cos(nowAngle) - _offsetY; + + if (oldData !== undefined) { + var lastAngle = (oldData[3] + 90) / 180 * Math.PI; + if (Math.abs(lastAngle - nowAngle) < Math.PI) { + theta = (nowAngle - lastAngle) * rate + lastAngle; + } else if (lastAngle > nowAngle) { + theta = (nowAngle - lastAngle + Math.PI * 2) * rate + lastAngle; + } else { + theta = (nowAngle - lastAngle - Math.PI * 2) * rate + lastAngle; + } + var preAnchorX = oldData[0] + style['anchor'][0] * Math.cos(lastAngle) - style['anchor'][1] * Math.sin(lastAngle) - _offsetX; + var preAnchorY = oldData[1] + style['anchor'][0] * Math.sin(lastAngle) + style['anchor'][1] * Math.cos(lastAngle) - _offsetY; + + anchorX = nowAnchorX * rate + preAnchorX * (1 - rate); + anchorY = nowAnchorY * rate + preAnchorY * (1 - rate); + } else { + theta = nowAngle; + anchorX = nowAnchorX; + anchorY = nowAnchorY; + } + var originXMaster = anchorX * Math.cos(theta) + anchorY * Math.sin(theta) - style['anchor'][0] + Math.sqrt(2) / 2 * Math.cos(Math.PI / 4 - theta) - 0.5; + var originYMaster = -anchorX * Math.sin(theta) + anchorY * Math.cos(theta) - style['anchor'][1] + Math.sqrt(2) / 2 * Math.sin(Math.PI / 4 - theta) - 0.5; + return [originXMaster, originYMaster, theta]; +} + +function _drawAgent() { + var counter = 0; + _agentBodyCTX.clearRect(0, 0, _agentBodyCTX.canvas.width, _agentBodyCTX.canvas.height); + _agentRangeCTX.clearRect(0, 0, _agentRangeCTX.canvas.width, _agentRangeCTX.canvas.height); + _agentHPCTX.clearRect(0, 0, _agentHPCTX.canvas.width, _agentHPCTX.canvas.height); + // _drawGrid(); + for (var agentID in _mapData[1]) { + if (_mapData[1].hasOwnProperty(agentID)) { + var style = _mapStyles['group'][_mapData[1][agentID][2]]; + // console.log(style) + var result = _getOriginGridCoordinate( + _mapData[1][agentID], + _mapLastData !== undefined ? _mapLastData[1][agentID] : undefined, + style + ); + var originXMaster = result[0]; + var originYMaster = result[1]; + + _agentBodyCTX.beginPath(); + _agentBodyCTX.rotate(result[2]); + _agentBodyCTX.fillStyle = style['style']; + _agentBodyCTX.rect( + originXMaster * gridSize, + originYMaster * gridSize, + style['height'] * gridSize, + style['width'] * gridSize + ); + _agentBodyCTX.fill(); + _agentBodyCTX.rotate(-result[2]); + + if (gridSize >= 4) { + //if (parseInt(style['vision-angle']) <= 180) { + // _agentRangeCTX.beginPath(); + // _agentRangeCTX.fillStyle = style['vision-style']; + // _agentRangeCTX.rotate(result[2]); + // _agentRangeCTX.moveTo( + // originXMaster * gridSize + style['height'] * gridSize / 2, + // originYMaster * gridSize + style['width'] * gridSize / 2 + // ); + // _agentRangeCTX.arc( + // originXMaster * gridSize + style['height'] * gridSize / 2, + // originYMaster * gridSize + style['width'] * gridSize / 2, + // style['vision-radius'] * gridSize, + // -(parseInt(style['vision-angle'])) / 360 * Math.PI - Math.PI / 2, + // (parseInt(style['vision-angle'])) / 360 * Math.PI - Math.PI / 2, + // false + // ); + // _agentRangeCTX.fill(); + // _agentRangeCTX.rotate(-result[2]); + //} + + _agentRangeCTX.beginPath(); + _agentRangeCTX.fillStyle = style['attack-style']; + _agentRangeCTX.rotate(result[2]); + _agentRangeCTX.moveTo( + originXMaster * gridSize + style['height'] * gridSize / 2, + originYMaster * gridSize + style['width'] * gridSize / 2 + ); + _agentRangeCTX.arc( + originXMaster * gridSize + style['height'] * gridSize / 2, + originYMaster * gridSize + style['width'] * gridSize / 2, + style['attack-radius'] * gridSize, + -(parseInt(style['attack-angle'])) / 360 * Math.PI - Math.PI / 2, + (parseInt(style['attack-angle'])) / 360 * Math.PI - Math.PI / 2, + false + ); + _agentRangeCTX.fill(); + _agentRangeCTX.rotate(-result[2]); + // if (_mapData[1][agentID][6]==1){ + // _agentRangeCTX.arc( + // originXMaster * gridSize + style['height'] * gridSize / 2, + // originYMaster * gridSize + style['width'] * gridSize / 2, + // 6 * gridSize, + // -(parseInt(style['attack-angle'])) / 360 * Math.PI , + // (parseInt(style['attack-angle'])) / 360 * Math.PI , + // false + // ); + // } + console.log(_mapData[1][agentID][5]) + if (gridSize >= 6) { + _agentHPCTX.beginPath(); + _agentHPCTX.rotate(result[2]); + _agentHPCTX.rect( + originXMaster * gridSize, + originYMaster * gridSize, + gridSize / 4, + style['width'] * gridSize + ); + _agentHPCTX.strokeStyle = "rgba(0,0,0,1)"; + //_agentHPCTX.stroke(); + _agentHPCTX.fillStyle = "rgba(255,255,255,1)"; + _agentHPCTX.fill(); + + _agentHPCTX.beginPath(); + _agentHPCTX.fillStyle = style['style']; + var hp; + var rate = Math.min(_mapAnimateTick, ANIMATE_STEP) / ANIMATE_STEP; + if (_mapLastData !== undefined && _mapLastData[1].hasOwnProperty(agentID)) { + hp = _mapLastData[1][agentID][4] * (1 - rate) + _mapData[1][agentID][4] * rate; + } else { + hp = _mapData[1][agentID][4]; + } + _agentHPCTX.rect( + originXMaster * gridSize, + originYMaster * gridSize + (100 - hp) / 100 * style['width'] * gridSize, + gridSize / 4, + hp / 100 * style['width'] * gridSize + ); + + _agentHPCTX.fill(); + _agentHPCTX.rotate(-result[2]); + } + } + } + } +} + +function _drawObstacles() { + _staticCTX.clearRect(0, 0, _staticCTX.canvas.width, _staticCTX.canvas.height); + _staticCTX.beginPath(); + _staticCTX.fillStyle = _mapStyles['obstacle-style']; + for (var i = 0; i < _mapData[3].length; i++) { + _staticCTX.rect( + (_mapData[3][i][0] - _offsetX) * gridSize, + (_mapData[3][i][1] - _offsetY) * gridSize, + gridSize, gridSize); + } + _staticCTX.fill(); +} + +function _drawEvent() { + _eventCTX.clearRect(0, 0, _eventCTX.canvas.width, _eventCTX.canvas.height); + if (gridSize >= 4) { + _eventCTX.beginPath(); + _eventCTX.strokeStyle = _mapStyles['attack-style']; + for (var i = 0; i < _mapData[0].length; i++) { + if (_mapData[0][i][0] === 0) { + var id = _mapData[0][i][1]; + if (_mapData[1][id] === undefined) { + continue; + } + var style = _mapStyles['group'][_mapData[1][id][2]]; + var result = _getOriginGridCoordinate( + _mapData[1][id], + _mapLastData !== undefined ? _mapLastData[1][id] : undefined, + style + ); + _eventCTX.rotate(result[2]); + _eventCTX.moveTo( + result[0] * gridSize + style['height'] * gridSize / 2, + result[1] * gridSize + style['width'] * gridSize / 2 + ); + _eventCTX.rotate(-result[2]); + _eventCTX.lineTo( + _mapData[0][i][2] * gridSize + gridSize / 2 - _offsetX * gridSize, + _mapData[0][i][3] * gridSize + gridSize / 2 - _offsetY * gridSize + ); + } + } + _eventCTX.stroke(); + _eventCTX.beginPath(); + _eventCTX.fillStyle = _mapStyles['attack-style']; + for (i = 0; i < _mapData[0].length; i++) { + if (_mapData[0][i][0] === 0) { + id = _mapData[0][i][1]; + if (_mapData[1][id] === undefined) { + continue; + } + style = _mapStyles['group'][_mapData[1][id][2]]; + result = _getOriginGridCoordinate( + _mapData[1][id], + _mapLastData !== undefined ? _mapLastData[1][id] : undefined, + style + ); + _eventCTX.rect( + _mapData[0][i][2] * gridSize + gridSize / 2 - _offsetX * gridSize - gridSize / 8, + _mapData[0][i][3] * gridSize + gridSize / 2 - _offsetY * gridSize - gridSize / 8, + gridSize / 4, + gridSize / 4 + ); + } + } + _eventCTX.fill(); + } +} + +function _drawMiniMAP() { + if ($('#magnet-settings-minimap').is(':checked')) { + var imgData = _minimapCTX.createImageData(_mapStyles['minimap-width'], _mapStyles['minimap-height']); + for (var i = 0, size = _mapData[4].length; i < size; i++) { + imgData.data[i * 4] = (_mapData[4][i] >> 24) & 255; + imgData.data[i * 4 + 1] = (_mapData[4][i] >> 16) & 255; + imgData.data[i * 4 + 2] = (_mapData[4][i] >> 8 ) & 255; + imgData.data[i * 4 + 3] = _mapData[4][i] & 255; + } + _minimapCTX.putImageData(imgData, 0, 0); + _minimapCTX.strokeRect(0, 0, _mapStyles['minimap-width'], _mapStyles['minimap-height']); + } else { + _minimapCTX.clearRect(0, 0, _minimapCTX.canvas.width, _minimapCTX.canvas.height); + } +} + +function _drawMiniMAPPosition() { + _minimapPostionCTX.clearRect(0, 0, _minimapPostionCTX.canvas.width, _minimapPostionCTX.canvas.height); + if ($('#magnet-settings-minimap').is(':checked')) { + var offsetXMini = _offsetX / _mapStyles['width'] * _minimapPostionCTX.canvas.width; + var offsetYMini = _offsetY / _mapStyles['height'] * _minimapPostionCTX.canvas.height; + var xLength = Math.min(_mapStyles['width'], _gridCTX.canvas.width / gridSize) / _mapStyles['width'] * _minimapPostionCTX.canvas.width; + var yLength = Math.min(_mapStyles['height'], _gridCTX.canvas.height / gridSize) / _mapStyles['height'] * _minimapPostionCTX.canvas.height; + + _minimapPostionCTX.strokeStyle = "rgba(0,0,255,255)"; + _minimapPostionCTX.strokeRect(offsetXMini, offsetYMini, xLength, yLength); + } +} + +function _animate() { + _mapProcessedFrame++; + if ((_mapStatus !== 'STOP' && _mapStatus !== 'PLAY') || _mapForcedPause) { // Pause + window.requestAnimationFrame(_animate); + } else if (_mapStatus === 'PLAY' && _mapData !== undefined) { + if (_isBrowserSizeChanged) { + _gridCTX.canvas.width = window.innerWidth; + _gridCTX.canvas.height = window.innerHeight; + + _agentRangeCTX.canvas.width = window.innerWidth; + _agentRangeCTX.canvas.height = window.innerHeight; + + _eventCTX.canvas.width = window.innerWidth; + _eventCTX.canvas.height = window.innerHeight; + + _agentBodyCTX.canvas.width = window.innerWidth; + _agentBodyCTX.canvas.height = window.innerHeight; + + _agentHPCTX.canvas.width = window.innerWidth; + _agentHPCTX.canvas.height = window.innerHeight; + + _staticCTX.canvas.width = window.innerWidth; + _staticCTX.canvas.height = window.innerHeight; + + _isBrowserSizeChanged = false; + } + if (_isGridSizeChanged) { + _drawGrid(); + _isGridSizeChanged = false; + } + if (_isWindowChanged) { + _drawObstacles(); + _drawMiniMAPPosition(); + if (_mapAnimateTick === 0) { + _isWindowChanged = false; + } + } + if (_mapAnimateTick === 0) { + _drawNumbers(); + _drawMiniMAP(); + } + + if (_mapAnimateTick < TOTAL_STEP) { + _drawAgent(_mapData[1], _mapLastData !== undefined ? _mapLastData[1] : undefined); + _drawEvent(); + _mapAnimateTick += _mapSpeed; + } else { + _mapProcessedImage++; + _mapCurrentImage++; + _mapAnimateTick = 0; + if (_mapCurrentImage >= _mapTotalImage) { + _mapCurrentImage = _mapTotalImage - 1; + _socket.send('p' + _mapCurrentImage.toString() + ' ' + + Math.floor(_offsetX).toString() + ' ' + Math.floor(_offsetY).toString() + ' ' + + Math.ceil(_offsetX + window.innerWidth / gridSize).toString() + ' ' + + Math.ceil(_offsetY + window.innerHeight / gridSize).toString()); + _mapStatus = 'WAITING'; + } else { + $('#magnet-settings-progress').val(_mapCurrentImage); + _socket.send('p' + _mapCurrentImage.toString() + ' ' + + Math.floor(_offsetX).toString() + ' ' + Math.floor(_offsetY).toString() + ' ' + + Math.ceil(_offsetX + window.innerWidth / gridSize).toString() + ' ' + + Math.ceil(_offsetY + window.innerHeight / gridSize).toString()); + _mapStatus = 'WAITING'; + } + } + window.requestAnimationFrame(_animate); + } else { + _minimapCTX.clearRect(0, 0, _minimapCTX.canvas.width, _minimapCTX.canvas.height); + _staticCTX.clearRect(0, 0, _staticCTX.canvas.width, _staticCTX.canvas.height); + _gridCTX.clearRect(0, 0, _gridCTX.canvas.width, _gridCTX.canvas.height); + _agentRangeCTX.clearRect(0, 0, _agentRangeCTX.canvas.width, _agentRangeCTX.canvas.height); + _agentHPCTX.clearRect(0, 0, _agentHPCTX.canvas.width, _agentHPCTX.canvas.height); + _agentBodyCTX.clearRect(0, 0, _agentBodyCTX.canvas.width, _agentBodyCTX.canvas.height); + _eventCTX.clearRect(0, 0, _eventCTX.canvas.width, _eventCTX.canvas.height); + _minimapPostionCTX.clearRect(0, 0, _minimapPostionCTX.canvas.width, _minimapPostionCTX.canvas.height); + } +} diff --git a/examples/battle_model/src/render/frontend/js/render-parameter.js b/examples/battle_model/src/render/frontend/js/render-parameter.js new file mode 100755 index 0000000..37be426 --- /dev/null +++ b/examples/battle_model/src/render/frontend/js/render-parameter.js @@ -0,0 +1,17 @@ +var SOCKET_HOST = 'ws://localhost:9030'; +var SOCKET_RECONNECT_PERIOD = 2000; + +var STATUS_SPACING = 15; +var STATUS_PADDING_TOP = 15; +var STATUS_PADDING_LEFT = 10; +var STATUS_WIDTH = 150; +var STATUS_HEIGHT = 200; +var STATUS_FPS_PERIOD = 1000; + +var MOVE_SPACING = 0.1; + +var ANIMATE_STEP = 500; +var TOTAL_STEP = 700; + +var gridSize = 20; +var gridColor = 'rgba(0,0,0,0.1)'; \ No newline at end of file diff --git a/examples/battle_model/src/render/frontend/js/render.js b/examples/battle_model/src/render/frontend/js/render.js new file mode 100755 index 0000000..dc6a9da --- /dev/null +++ b/examples/battle_model/src/render/frontend/js/render.js @@ -0,0 +1,3 @@ +$(document).ready(function () { + run(); +}); \ No newline at end of file diff --git a/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.css b/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.css new file mode 100755 index 0000000..0d42ed0 --- /dev/null +++ b/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.css @@ -0,0 +1,28 @@ +/*! ======================================================================== + * Bootstrap Toggle: bootstrap-toggle.css v2.2.0 + * http://www.bootstraptoggle.com + * ======================================================================== + * Copyright 2014 Min Hur, The New York Times Company + * Licensed under MIT + * ======================================================================== */ +.checkbox label .toggle,.checkbox-inline .toggle{margin-left:-20px;margin-right:5px} +.toggle{position:relative;overflow:hidden} +.toggle input[type=checkbox]{display:none} +.toggle-group{position:absolute;width:200%;top:0;bottom:0;left:0;transition:left .35s;-webkit-transition:left .35s;-moz-user-select:none;-webkit-user-select:none} +.toggle.off .toggle-group{left:-100%} +.toggle-on{position:absolute;top:0;bottom:0;left:0;right:50%;margin:0;border:0;border-radius:0} +.toggle-off{position:absolute;top:0;bottom:0;left:50%;right:0;margin:0;border:0;border-radius:0} +.toggle-handle{position:relative;margin:0 auto;padding-top:0;padding-bottom:0;height:100%;width:0;border-width:0 1px} +.toggle.btn{min-width:59px;min-height:34px} +.toggle-on.btn{padding-right:24px} +.toggle-off.btn{padding-left:24px} +.toggle.btn-lg{min-width:79px;min-height:45px} +.toggle-on.btn-lg{padding-right:31px} +.toggle-off.btn-lg{padding-left:31px} +.toggle-handle.btn-lg{width:40px} +.toggle.btn-sm{min-width:50px;min-height:30px} +.toggle-on.btn-sm{padding-right:20px} +.toggle-off.btn-sm{padding-left:20px} +.toggle.btn-xs{min-width:35px;min-height:22px} +.toggle-on.btn-xs{padding-right:12px} +.toggle-off.btn-xs{padding-left:12px} \ No newline at end of file diff --git a/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.js b/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.js new file mode 100755 index 0000000..3711320 --- /dev/null +++ b/examples/battle_model/src/render/frontend/vendor/bootstrap-toggle.min.js @@ -0,0 +1,9 @@ +/*! ======================================================================== + * Bootstrap Toggle: bootstrap-toggle.js v2.2.0 + * http://www.bootstraptoggle.com + * ======================================================================== + * Copyright 2014 Min Hur, The New York Times Company + * Licensed under MIT + * ======================================================================== */ ++function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.toggle"),f="object"==typeof b&&b;e||d.data("bs.toggle",e=new c(this,f)),"string"==typeof b&&e[b]&&e[b]()})}var c=function(b,c){this.$element=a(b),this.options=a.extend({},this.defaults(),c),this.render()};c.VERSION="2.2.0",c.DEFAULTS={on:"On",off:"Off",onstyle:"primary",offstyle:"default",size:"normal",style:"",width:null,height:null},c.prototype.defaults=function(){return{on:this.$element.attr("data-on")||c.DEFAULTS.on,off:this.$element.attr("data-off")||c.DEFAULTS.off,onstyle:this.$element.attr("data-onstyle")||c.DEFAULTS.onstyle,offstyle:this.$element.attr("data-offstyle")||c.DEFAULTS.offstyle,size:this.$element.attr("data-size")||c.DEFAULTS.size,style:this.$element.attr("data-style")||c.DEFAULTS.style,width:this.$element.attr("data-width")||c.DEFAULTS.width,height:this.$element.attr("data-height")||c.DEFAULTS.height}},c.prototype.render=function(){this._onstyle="btn-"+this.options.onstyle,this._offstyle="btn-"+this.options.offstyle;var b="large"===this.options.size?"btn-lg":"small"===this.options.size?"btn-sm":"mini"===this.options.size?"btn-xs":"",c=a('