From 9518dbf453b83ee6542ef618262a1b4477301be8 Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Sat, 22 Jan 2022 20:13:41 +0100 Subject: [PATCH 1/2] PoC: Multiple-agent types scheduling and datacollection Proof of concept. This code is not meant for production. This commit adds support for multiple agent-types, including scheduling and data collection. The scheduler is based on the RandomActivationByBreed scheduler from the wolf_sheep example. It supports RandomActivation by type, in the order that they are initialized. See https://github.com/projectmesa/mesa/blob/246c69d592a82e89c75d555c79736c3f619d434a/examples/wolf_sheep/wolf_sheep/schedule.py The DataCollection module is modified to support multiple agent types. It uses a dictionary structure for all agent reporters and records, with the agent class as the key. The ChartModule has been extended to enable data collection both Agent and Model variables, and agents of different types. Contains currently a breaking change by requiring "Type" key in the ChartModule input dictionary, which can either be "Model" or "Agent". If "Agent", a key "Agent_type" is also required. To-do: - Scheduler: Allow custom order of agent types in staging (and not just the order in which their created). - DataCollector: Ensure it still works with only a single/default agent type - ChartVisualisation: 1) Try to reduce breaking changes, 2) ensure it still works with a single/default agent-type, 3) Allow different kinds of statistical processing then mean (median, sum, min, max, standard deviation, etc.) (maybe implement this in DataCollector) - Cleanup - Testing - Documentation (including examples) --- mesa/datacollection.py | 55 +++++++++----- mesa/time.py | 73 ++++++++++++++++++- .../modules/ChartVisualization.py | 37 ++++++++-- 3 files changed, 138 insertions(+), 27 deletions(-) diff --git a/mesa/datacollection.py b/mesa/datacollection.py index 469c67f383f..4b917a56fe8 100644 --- a/mesa/datacollection.py +++ b/mesa/datacollection.py @@ -95,20 +95,31 @@ class attributes of model {"Model_Function":[function, [param_1, param_2]]} """ - self.model_reporters = {} - self.agent_reporters = {} + self.agent_types = agent_reporters.keys() + self.model_reporters = {} self.model_vars = {} - self._agent_records = {} - self.tables = {} if model_reporters is not None: for name, reporter in model_reporters.items(): self._new_model_reporter(name, reporter) - if agent_reporters is not None: - for name, reporter in agent_reporters.items(): - self._new_agent_reporter(name, reporter) + self.agent_reporters = {} + self._agent_records = {} + self.agent_attr_index = {} + self.agent_name_index = {} + + for agent_type in self.agent_types: + self.agent_reporters[agent_type] = {} + self._agent_records[agent_type] = {} + self.agent_name_index[agent_type] = {} + self.tables = {} + + for agent_type in self.agent_types: + if agent_reporters[agent_type] is not None: + for name, reporter in agent_reporters[agent_type].items(): + self.agent_name_index[agent_type][name] = reporter + self._new_agent_reporter(name, reporter, agent_type) if tables is not None: for name, columns in tables.items(): @@ -127,7 +138,7 @@ def _new_model_reporter(self, name, reporter): self.model_reporters[name] = reporter self.model_vars[name] = [] - def _new_agent_reporter(self, name, reporter): + def _new_agent_reporter(self, name, reporter, agent_type): """Add a new agent-level reporter to collect. Args: @@ -140,7 +151,7 @@ def _new_agent_reporter(self, name, reporter): attribute_name = reporter reporter = partial(self._getattr, reporter) reporter.attribute_name = attribute_name - self.agent_reporters[name] = reporter + self.agent_reporters[agent_type][name] = reporter def _new_table(self, table_name, table_columns): """Add a new table that objects can write to. @@ -153,13 +164,16 @@ def _new_table(self, table_name, table_columns): new_table = {column: [] for column in table_columns} self.tables[table_name] = new_table - def _record_agents(self, model): + def _record_agents(self, model, agent_type): """Record agents data in a mapping of functions and agents.""" - rep_funcs = self.agent_reporters.values() + rep_funcs = self.agent_reporters[agent_type].values() if all([hasattr(rep, "attribute_name") for rep in rep_funcs]): prefix = ["model.schedule.steps", "unique_id"] attributes = [func.attribute_name for func in rep_funcs] + self.agent_attr_index[agent_type] = {k: v for v, k in enumerate(prefix + attributes)} + print(self.agent_attr_index[agent_type]) get_reports = attrgetter(*prefix + attributes) + else: def get_reports(agent): @@ -167,7 +181,7 @@ def get_reports(agent): reports = tuple(rep(agent) for rep in rep_funcs) return _prefix + reports - agent_records = map(get_reports, model.schedule.agents) + agent_records = map(get_reports, model.schedule.agents_by_type[agent_type].values()) return agent_records def _reporter_decorator(self, reporter): @@ -190,9 +204,10 @@ def collect(self, model): else: self.model_vars[var].append(self._reporter_decorator(reporter)) - if self.agent_reporters: - agent_records = self._record_agents(model) - self._agent_records[model.schedule.steps] = list(agent_records) + for agent_type in self.agent_types: + if self.agent_reporters[agent_type]: + agent_records = self._record_agents(model, agent_type) + self._agent_records[agent_type][model.schedule.steps] = list(agent_records) def add_table_row(self, table_name, row, ignore_missing=False): """Add a row dictionary to a specific table. @@ -229,21 +244,21 @@ def get_model_vars_dataframe(self): """ return pd.DataFrame(self.model_vars) - def get_agent_vars_dataframe(self): + def get_agent_vars_dataframe(self, agent_type): """Create a pandas DataFrame from the agent variables. The DataFrame has one column for each variable, with two additional columns for tick and agent_id. """ - all_records = itertools.chain.from_iterable(self._agent_records.values()) - rep_names = [rep_name for rep_name in self.agent_reporters] + all_records = itertools.chain.from_iterable(self._agent_records[agent_type].values()) + rep_names = [rep_name for rep_name in self.agent_reporters[agent_type]] df = pd.DataFrame.from_records( data=all_records, - columns=["Step", "AgentID"] + rep_names, + columns=["Step", f"{agent_type}_AgentID"] + rep_names, ) - df = df.set_index(["Step", "AgentID"]) + df = df.set_index(["Step", f"{agent_type}_AgentID"]) return df def get_table_dataframe(self, table_name): diff --git a/mesa/time.py b/mesa/time.py index 0caaea82a51..66cdecccc9e 100644 --- a/mesa/time.py +++ b/mesa/time.py @@ -21,7 +21,7 @@ model has taken. """ -from collections import OrderedDict +from collections import OrderedDict, defaultdict # mypy from typing import Dict, Iterator, List, Optional, Union @@ -198,3 +198,74 @@ def step(self) -> None: self.time += self.stage_time self.steps += 1 + +class RandomActivationByType(RandomActivation): + """ + A scheduler which activates each type of agent once per step, in random + order, with the order reshuffled every step. + + This is equivalent to the NetLogo 'ask breed...' and is generally the + default behavior for an ABM. + + Assumes that all agents have a step() method. + """ + + def __init__(self, model): + super().__init__(model) + self.agents_by_type = defaultdict(dict) + + def add(self, agent): + """ + Add an Agent object to the schedule + + Args: + agent: An Agent to be added to the schedule. + """ + + self._agents[agent.unique_id] = agent + agent_class = type(agent) + self.agents_by_type[agent_class][agent.unique_id] = agent + + def remove(self, agent): + """ + Remove all instances of a given agent from the schedule. + """ + + del self._agents[agent.unique_id] + + agent_class = type(agent) + del self.agents_by_type[agent_class][agent.unique_id] + + def step(self, by_type=True): + """ + Executes the step of each agent type, one at a time, in random order. + + Args: + by_type: If True, run all agents of a single type before running + the next one. + """ + if by_type: + for agent_class in self.agents_by_type: + self.step_type(agent_class) + self.steps += 1 + self.time += 1 + else: + super().step() + + def step_type(self, type): + """ + Shuffle order and run all agents of a given type. + + Args: + type: Class object of the type to run. + """ + agent_keys = list(self.agents_by_type[type].keys()) + self.model.random.shuffle(agent_keys) + for agent_key in agent_keys: + self.agents_by_type[type][agent_key].step() + + def get_type_count(self, type_class): + """ + Returns the current number of agents of certain type in the queue. + """ + return len(self.agents_by_type[type_class].values()) diff --git a/mesa/visualization/modules/ChartVisualization.py b/mesa/visualization/modules/ChartVisualization.py index 350d83a1fc0..a26bf52e84c 100644 --- a/mesa/visualization/modules/ChartVisualization.py +++ b/mesa/visualization/modules/ChartVisualization.py @@ -6,6 +6,7 @@ """ import json +import statistics from mesa.visualization.ModularVisualization import VisualizationElement @@ -34,8 +35,6 @@ class ChartModule(VisualizationElement): data_collector_name="datacollector") TODO: - Have it be able to handle agent-level variables as well. - More Pythonic customization; in particular, have both series-level and chart-level options settable in Python, and passed to the front-end the same way that "Color" is currently. @@ -78,9 +77,35 @@ def render(self, model): for s in self.series: name = s["Label"] - try: - val = data_collector.model_vars[name][-1] # Latest value - except (IndexError, KeyError): - val = 0 + entity = s["Type"] + if entity == "Model": + try: + val = data_collector.model_vars[name][-1] # Latest value + except (IndexError, KeyError): + val = 0 + elif entity == "Agent": + agent_dict = {e.__name__: e for e in list(model.schedule.agents_by_type.keys())} + agent_type = agent_dict[s["Agent_type"]] + try: + # Get the reporter from the name + reporter = model.datacollector.agent_name_index[agent_type][name] + + # Get the index of the reporter + attr_index = model.datacollector.agent_attr_index[agent_type][reporter] + + # Create a dictionary with all attributes from all agents + attr_dict = model.datacollector._agent_records[agent_type] + + # Get the values from all agents in a list + values_tuples = list(attr_dict.values())[-1] + + # Get the correct value using the attribute index + values = [value_tuple[attr_index] for value_tuple in values_tuples] + + # Calculate the mean among all agents + val = statistics.mean(values) + + except (IndexError, KeyError): + val = 0 current_values.append(val) return current_values From bc74fa320529e72b5f39d3f3a4acdbad0ce4398e Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Sun, 23 Jan 2022 11:41:47 +0100 Subject: [PATCH 2/2] Move metric aggregation function to datacollector Moves the metric aggegration code to the datacollector, in a new function get_agent_metric(). It returns a single value from the agent variables, using either a function in the statistics module or one of the built-in functions "min", "max", "sum" or "len". In the ChartModule an optional "Metric" can now be called, which defaults to "mean". --- mesa/datacollection.py | 35 +++++++++++++++++++ .../modules/ChartVisualization.py | 24 +++---------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/mesa/datacollection.py b/mesa/datacollection.py index 4b917a56fe8..ae7a002bdd4 100644 --- a/mesa/datacollection.py +++ b/mesa/datacollection.py @@ -40,6 +40,7 @@ from operator import attrgetter import pandas as pd import types +import statistics class DataCollector: @@ -271,3 +272,37 @@ def get_table_dataframe(self, table_name): if table_name not in self.tables: raise Exception("No such table.") return pd.DataFrame(self.tables[table_name]) + + def get_agent_metric(self, var_name, agent_type, metric="mean"): + """Get a single aggegrated value from an agent variable. + + Args: + var_name: The name of the variable to aggegrate. + agent_type: Agent type class + metric: Statistics metric to be used (default: mean) + all functions from built-in statistics module are supported + as well as "min", "max", "sum" and "len" + + """ + # Get the reporter from the name + reporter = self.agent_name_index[agent_type][var_name] + + # Get the index of the reporter + attr_index = self.agent_attr_index[agent_type][reporter] + + # Create a dictionary with all attributes from all agents + attr_dict = self._agent_records[agent_type] + + # Get the values from all agents in a list + values_tuples = list(attr_dict.values())[-1] + + # Get the correct value using the attribute index + values = [value_tuple[attr_index] for value_tuple in values_tuples] + + # Calculate the metric among all agents (mean by default) + if metric in ["min", "max", "sum", "len"]: + value = eval(f"{metric}(values)") + else: + stat_function = getattr(statistics, metric) + value = stat_function(values) + return value diff --git a/mesa/visualization/modules/ChartVisualization.py b/mesa/visualization/modules/ChartVisualization.py index a26bf52e84c..70f14be56d3 100644 --- a/mesa/visualization/modules/ChartVisualization.py +++ b/mesa/visualization/modules/ChartVisualization.py @@ -6,7 +6,6 @@ """ import json -import statistics from mesa.visualization.ModularVisualization import VisualizationElement @@ -86,25 +85,12 @@ def render(self, model): elif entity == "Agent": agent_dict = {e.__name__: e for e in list(model.schedule.agents_by_type.keys())} agent_type = agent_dict[s["Agent_type"]] + if "Metric" in s.keys(): + metric = s["Metric"] + else: + metric = "mean" try: - # Get the reporter from the name - reporter = model.datacollector.agent_name_index[agent_type][name] - - # Get the index of the reporter - attr_index = model.datacollector.agent_attr_index[agent_type][reporter] - - # Create a dictionary with all attributes from all agents - attr_dict = model.datacollector._agent_records[agent_type] - - # Get the values from all agents in a list - values_tuples = list(attr_dict.values())[-1] - - # Get the correct value using the attribute index - values = [value_tuple[attr_index] for value_tuple in values_tuples] - - # Calculate the mean among all agents - val = statistics.mean(values) - + val = data_collector.get_agent_metric(name, agent_type, metric) except (IndexError, KeyError): val = 0 current_values.append(val)