diff --git a/backend/src/impl/db_utils/system_db_utils.py b/backend/src/impl/db_utils/system_db_utils.py index cb0d89f3..142ea758 100644 --- a/backend/src/impl/db_utils/system_db_utils.py +++ b/backend/src/impl/db_utils/system_db_utils.py @@ -352,7 +352,7 @@ def create_system( """ def _validate_and_create_system(): - system = {} + system = {"results": {}} user = get_user() system["creator"] = user.id @@ -435,10 +435,21 @@ def _validate_and_create_system(): ] # -- add the analysis results to the system object - sys_info_dict = overall_statistics.sys_info.to_dict() - system.system_info = SystemInfo.from_dict(sys_info_dict) + sys_info = overall_statistics.sys_info + system.system_info = SystemInfo.from_dict(sys_info.to_dict()) system.metric_stats = binarized_stats + if sys_info.analysis_levels and sys_info.results.overall: + # store overall metrics in the DB so they can be queried + for level, result in zip( + sys_info.analysis_levels, sys_info.results.overall + ): + system.results[level.name] = {} + for metric_result in result: + system.results[level.name][ + metric_result.metric_name + ] = metric_result.value + def db_operations(session: ClientSession) -> str: # Insert system system.created_at = system.last_modified = datetime.utcnow() diff --git a/backend/src/impl/default_controllers_impl.py b/backend/src/impl/default_controllers_impl.py index 8581dec4..d11490fd 100644 --- a/backend/src/impl/default_controllers_impl.py +++ b/backend/src/impl/default_controllers_impl.py @@ -111,6 +111,13 @@ def user_get() -> modelUser: def tasks_get() -> list[TaskCategory]: + """Returns a list of task categories and metadata for each + task. + NOTE: supported_metrics only returns metrics for the example + level. This is because the SDK does not provide a way to configure + the list of metrics for other analysis levels. This should be fixed + in the future. + """ _categories = get_task_categories() categories: list[TaskCategory] = [] for _category in _categories: @@ -267,6 +274,13 @@ def systems_get( creator: str | None, shared_users: list[str] | None, ) -> SystemsReturn: + """Returns a systems according to the provided filters + + Args: + sort_field: created_at or a field within results. `results` has two levels: + analysis level and metric so the field should be provided as a dot separated + value (e.g. example.F1) + """ if not sort_field: sort_field = "created_at" if not sort_direction: @@ -274,7 +288,7 @@ def systems_get( if sort_direction not in ["asc", "desc"]: abort_with_error_message(400, "sort_direction needs to be one of asc or desc") if sort_field != "created_at": - sort_field = f"system_info.results.overall.{sort_field}.value" + sort_field = f"results.{sort_field}" dir = ASCENDING if sort_direction == "asc" else DESCENDING diff --git a/frontend/src/components/SystemsTable/SystemTableContent.tsx b/frontend/src/components/SystemsTable/SystemTableContent.tsx index 90ae28c1..62f85857 100644 --- a/frontend/src/components/SystemsTable/SystemTableContent.tsx +++ b/frontend/src/components/SystemsTable/SystemTableContent.tsx @@ -15,7 +15,6 @@ import { SystemModel } from "../../models"; import { DeleteOutlined, EditOutlined } from "@ant-design/icons"; import { PrivateIcon, useUser } from ".."; import { generateLeaderboardURL } from "../../utils"; -import { getOverallMap } from "../Analysis/utils"; const { Text } = Typography; interface Props { @@ -48,11 +47,9 @@ export function SystemTableContent({ }: Props) { const { userInfo } = useUser(); const metricColumns: ColumnsType = metricNames.map((metric) => ({ - dataIndex: metric, - render: (_, record) => - getOverallMap(record.system_info.results.overall)[metric]?.value, + dataIndex: ["results", ...metric.split(".")], title: metric, - width: 100, + width: 135, ellipsis: true, align: "center", })); diff --git a/frontend/src/components/SystemsTable/index.tsx b/frontend/src/components/SystemsTable/index.tsx index 65296943..fae19be3 100644 --- a/frontend/src/components/SystemsTable/index.tsx +++ b/frontend/src/components/SystemsTable/index.tsx @@ -62,19 +62,26 @@ export function SystemsTable() { const { state: loginState, userInfo } = useUser(); const userId = userInfo?.id; - /** generate metrics options list */ - function getMetricsNames() { + /** generate metrics options list. Metric names use the pattern + * `${analysisLevel}.{metricName}`. */ + function getMetricsNames(): string[] { const metricNames = new Set(); for (const sys of systems) { - for (const resultsLevel of sys.system_info.results.overall) { - resultsLevel.forEach((name) => metricNames.add(name.metric_name)); + for (const [level, metrics] of Object.entries(sys.results)) { + Object.keys(metrics).forEach((name) => + metricNames.add(`${level}.${name}`) + ); } } // if a task is selected, add all supported metrics to the options list if (filters.task) { const task = findTask(taskCategories, filters.task); if (task) - task.supported_metrics.forEach((metric) => metricNames.add(metric)); + // The backend only returns metric names for the example level so it is + // hardcoded here. This should be fixed in the future. + task.supported_metrics.forEach((metricName) => + metricNames.add(`example.${metricName}`) + ); } return Array.from(metricNames); } @@ -98,8 +105,9 @@ export function SystemsTable() { ) { if (taskFilter) { const initialTask = findTask(taskCategories, taskFilter); - if (initialTask && initialTask.supported_metrics.length > 0) - return initialTask.supported_metrics[0]; + if (initialTask && initialTask.supported_metrics.length > 0) { + return `example.${initialTask.supported_metrics[0]}`; + } } return "created_at"; } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 34da1938..e5d1f100 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.0" info: title: "ExplainaBoard" description: "Backend APIs for ExplainaBoard" - version: "0.2.9" + version: "0.2.10" contact: email: "explainaboard@gmail.com" license: @@ -307,10 +307,12 @@ paths: default: 20 - in: query name: sort_field - description: supports `created_at` (default) and metric names. + description: > + Supports `created_at` (default) and metric names. Metric names should be + provided as {analysis_level}.{metric_name} schema: type: string - example: Accuracy + example: example.Accuracy - in: query name: sort_direction schema: @@ -983,6 +985,13 @@ components: description: the user who created the system preferred_username: type: string + results: + type: object + example: {"span": {"F1": 0.9221}} + additionalProperties: + type: object + additionalProperties: + type: number shared_users: type: array items: @@ -1017,6 +1026,7 @@ components: - source_language - target_language - task + - results - system_info - metric_stats - $ref: "#/components/schemas/Time"