Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove MetricResult.config. #513

Merged
merged 45 commits into from
Sep 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
3321668
Use SerializableDataclass to serialize and deserialize MetricConfig o…
tetsuok Aug 30, 2022
a134e0d
Fix isort
tetsuok Aug 30, 2022
49baf8d
Bring back cls_name
tetsuok Aug 30, 2022
0e7ebd4
Merge branch 'main' into issue-427-metric-config
tetsuok Aug 31, 2022
72c0a8e
Merge branch 'main' into issue-427-metric-config
tetsuok Sep 5, 2022
5b33546
Replace metric_config_from_dict with get_metric_config_serializer
tetsuok Sep 5, 2022
793096f
Fix isort issue
tetsuok Sep 5, 2022
05f6e9d
Merge branch 'main' into issue-427-metric-config
tetsuok Sep 5, 2022
3471647
Fix TypeError
tetsuok Sep 5, 2022
0b464b5
Format with Black
tetsuok Sep 5, 2022
ace4515
Merge branch 'main' into issue-427-metric-config
tetsuok Sep 10, 2022
301f123
Ignore typecheck for dataclass
tetsuok Sep 10, 2022
3654a1e
Update explainaboard/metrics/nlg_meta_evaluation.py
tetsuok Sep 10, 2022
8f74cbd
Add type annotations to to_metric
tetsuok Sep 10, 2022
02d48f6
Merge branch 'main' into issue-427-metric-config
tetsuok Sep 11, 2022
bdf5698
Remove external_stats from tests
tetsuok Sep 11, 2022
64823af
Merge branch 'main' into issue-427-metric-config
Sep 14, 2022
c35232e
add MetricValue
odashi Sep 14, 2022
9f1592c
Merge branch 'issue-427-metric-config' of github.com:tetsuok/Explaina…
odashi Sep 14, 2022
1d8c8be
fix linter errors
odashi Sep 14, 2022
62232ee
fix serialization bug
odashi Sep 14, 2022
7143ec2
Merge branch 'issue-427-metric-config' into refactor-metricresult
odashi Sep 16, 2022
cfcf8dc
add some tests.
odashi Sep 19, 2022
bb18379
Merge branch 'main' into refactor-metricresult
odashi Sep 19, 2022
67b6d13
fix bugs and introduce shape checking.
odashi Sep 19, 2022
de3dac4
Fix batching in most metrics
neubig Sep 19, 2022
d07e733
Simplification
neubig Sep 20, 2022
d2823e7
Merge branch 'main' into refactor-metricresult
odashi Sep 20, 2022
e11cc81
Merge branch 'fix-calc-metric-from-aggregate' into refactor-metricresult
odashi Sep 20, 2022
c5fe886
fix bugs
odashi Sep 20, 2022
b850461
Merge branch 'main' into refactor-metricresult
odashi Sep 20, 2022
caed18f
Change RuntimeError to assertion.
odashi Sep 20, 2022
0cf56d9
Merge branch 'fix-calc-metric-from-aggregate' into refactor-metricresult
odashi Sep 20, 2022
9f19f8f
Merge branch 'main' into refactor-metricresult
odashi Sep 23, 2022
ad02ab2
remove metric registry
odashi Sep 24, 2022
70ee795
Merge branch 'main' into refactor-metricresult
odashi Sep 27, 2022
db05c47
list to dict
odashi Sep 27, 2022
b80d5fe
list[Performance] -> dict[str, Performance]
odashi Sep 27, 2022
3a84252
fix
odashi Sep 27, 2022
167fc6b
fix
odashi Sep 27, 2022
6b6ae0e
Remove additional config argument from Metric
odashi Sep 28, 2022
a02ba04
Merge branch 'dict-performance' into dict-metricresult
odashi Sep 28, 2022
e854d21
Merge branch 'refactor-metricresult' into dict-metricresult
odashi Sep 29, 2022
0fa40c0
Merge branch 'main' into dict-metricresult
odashi Sep 29, 2022
164a9b8
fix
odashi Sep 29, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion explainaboard/metrics/external_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,4 +181,4 @@ def evaluate_from_stats(
ci[0], ci[1], confidence_alpha
)

return MetricResult(self.config, metric_values)
return MetricResult(metric_values)
19 changes: 3 additions & 16 deletions explainaboard/metrics/metric.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,28 +112,17 @@ def deserialize(cls, data: dict[str, SerializableData]) -> Serializable:
class MetricResult(Serializable):
"""A result of computing a metric over some data."""

_config: MetricConfig
_values: dict[str, MetricValue]

def __init__(self, config: MetricConfig, values: dict[str, MetricValue]) -> None:
def __init__(self, values: dict[str, MetricValue]) -> None:
"""Initializes MetricResult object.

Args:
config: Config of the Metric that calculated this result.
values: Values calculated by the Metric.
"""
self._config = config
self._values = values

@property
def config(self) -> MetricConfig:
"""Obtains underlying MetricConfig.

Returns:
A MetricConfig object related to this MetricResult.
"""
return self._config

def get_value(self, cls: type[MetricValueT], name: str) -> MetricValueT | None:
"""Obtains a value with specific type and name.

Expand All @@ -152,7 +141,6 @@ def get_value(self, cls: type[MetricValueT], name: str) -> MetricValueT | None:
def serialize(self) -> dict[str, SerializableData]:
"""See Serializable.serialize."""
return {
"config": self.config,
"values": self._values,
}

Expand All @@ -171,8 +159,7 @@ def deserialize(cls, data: dict[str, SerializableData]) -> Serializable:
):
raise ValueError("`values` has incompatible data.")

# See mypy/issues/4717
return cls(narrow(MetricConfig, data["config"]), values) # type: ignore
return cls(values)


@dataclass
Expand Down Expand Up @@ -611,7 +598,7 @@ def evaluate_from_stats(
ci[0], ci[1], confidence_alpha
)

return MetricResult(self.config, metric_values)
return MetricResult(metric_values)

def evaluate(
self,
Expand Down
7 changes: 2 additions & 5 deletions explainaboard/metrics/metric_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,8 @@ class MetricResultTest(unittest.TestCase):
def test_serialize(self) -> None:
score = Score(1.0)
ci = ConfidenceInterval(1.0, 2.0, 0.5)
config = _DummyMetricConfig(name="foo")
result = MetricResult(config, {"bar": score, "baz": ci})
result = MetricResult({"bar": score, "baz": ci})
serialized: dict[str, SerializableData] = {
"config": config,
"values": {"bar": score, "baz": ci},
}
self.assertEqual(result.serialize(), serialized)
Expand All @@ -126,14 +124,13 @@ def test_deserialize(self) -> None:
"values": {"bar": score, "baz": ci},
}
restored = narrow(MetricResult, MetricResult.deserialize(serialized))
self.assertIs(restored.config, config)
self.assertEqual(restored._values, {"bar": score, "baz": ci})

def test_get_value(self) -> None:
score = Score(1.0)
ci = ConfidenceInterval(1.0, 2.0, 0.5)

result = MetricResult(_DummyMetricConfig(name="foo"), {"bar": score, "baz": ci})
result = MetricResult({"bar": score, "baz": ci})

# get_value() should return existing objects.
self.assertIsNone(result.get_value(Score, "foo"))
Expand Down