Skip to content

Commit

Permalink
Capture dataset-related events (#385)
Browse files Browse the repository at this point in the history
* add events to dataset activities

* properly raise exception in listing dataset history call

* record events on dataset testing
  • Loading branch information
nankolena authored Jan 3, 2024
1 parent f861c84 commit 48bc2cd
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 1 deletion.
9 changes: 9 additions & 0 deletions kolena/_api/v1/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@ class Event(str, Enum):
# test run
EXECUTE_TEST_RUN = "sdk-test-run-executed"

# dataset
REGISTER_DATASET = "sdk-dataset-registered"
FETCH_DATASET = "sdk-dataset-fetched"
FETCH_DATASET_HISTORY = "sdk-dataset-history-fetched"

# dataset evaluation
FETCH_DATASET_MODEL_RESULT = "sdk-dataset-model-result-fetched"
TEST_DATASET_MODEL = "sdk-dataset-model-tested"

@dataclass(frozen=True)
class RecordEventRequest:
event_name: str
Expand Down
7 changes: 6 additions & 1 deletion kolena/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import pandas as pd
import requests

from kolena._api.v1.event import EventAPI
from kolena._api.v2.dataset import CommitData
from kolena._api.v2.dataset import EntityData
from kolena._api.v2.dataset import ListCommitHistoryRequest
Expand All @@ -40,6 +41,7 @@
from kolena._utils.batched_load import upload_data_frame
from kolena._utils.consts import BatchSize
from kolena._utils.endpoints import get_dataset_url
from kolena._utils.instrumentation import with_event
from kolena._utils.serde import from_dict
from kolena._utils.state import API_V2
from kolena.dataset.common import COL_DATAPOINT
Expand Down Expand Up @@ -227,6 +229,7 @@ def resolve_id_fields(
return id_fields


@with_event(event_name=EventAPI.Event.REGISTER_DATASET)
def register_dataset(
name: str,
df: Union[Iterator[pd.DataFrame], pd.DataFrame],
Expand Down Expand Up @@ -296,6 +299,7 @@ def _iter_dataset(
yield _to_deserialized_dataframe(df_batch, column=COL_DATAPOINT)


@with_event(event_name=EventAPI.Event.FETCH_DATASET)
def fetch_dataset(
name: str,
commit: str = None,
Expand All @@ -315,7 +319,7 @@ def _list_commits(name: str, descending: bool = False, offset: int = 0, limit: i
"""
request = ListCommitHistoryRequest(name=name, descending=descending, offset=offset, limit=limit)
response = krequests.put(Path.LIST_COMMITS, json=asdict(request))
response.raise_for_status()
krequests.raise_for_status(response)
return from_dict(ListCommitHistoryResponse, response.json())


Expand All @@ -339,6 +343,7 @@ def _iter_commits(
break


@with_event(event_name=EventAPI.Event.FETCH_DATASET_HISTORY)
def fetch_dataset_history(
name: str,
descending: bool = False,
Expand Down
4 changes: 4 additions & 0 deletions kolena/dataset/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import pandas as pd

from kolena._api.v1.event import EventAPI
from kolena._api.v2.model import LoadResultsRequest
from kolena._api.v2.model import Path
from kolena._api.v2.model import UploadResultsRequest
Expand All @@ -32,6 +33,7 @@
from kolena._utils.batched_load import init_upload
from kolena._utils.batched_load import upload_data_frame
from kolena._utils.consts import BatchSize
from kolena._utils.instrumentation import with_event
from kolena._utils.state import API_V2
from kolena.dataset.common import COL_DATAPOINT
from kolena.dataset.common import COL_DATAPOINT_ID_OBJECT
Expand Down Expand Up @@ -100,6 +102,7 @@ def _upload_results(
krequests.raise_for_status(response)


@with_event(EventAPI.Event.FETCH_DATASET_MODEL_RESULT)
def fetch_results(
dataset: str,
model: str,
Expand Down Expand Up @@ -133,6 +136,7 @@ def _validate_configs(configs: List[TYPE_EVALUATION_CONFIG]) -> None:
raise IncorrectUsageError("duplicate eval configs are invalid")


@with_event(EventAPI.Event.TEST_DATASET_MODEL)
def test(
dataset: str,
model: str,
Expand Down

0 comments on commit 48bc2cd

Please sign in to comment.