From 0341cef2c838f73292483268f8a55b616e4e36b6 Mon Sep 17 00:00:00 2001 From: AndreiCautisanu <30831438+AndreiCautisanu@users.noreply.github.com> Date: Fri, 21 Feb 2025 12:16:13 +0200 Subject: [PATCH] logging and errors (#1338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Căutișanu --- .github/workflows/end2end_suites.yml | 16 +- tests_end_to_end/README.md | 13 +- tests_end_to_end/pytest.ini | 15 + .../test_dataset_items_crud_operations.py | 375 +++++++++++++---- .../Datasets/test_datasets_crud_operations.py | 305 ++++++++++---- .../test_experiment_crud_operations.py | 120 +++++- .../test_experiment_items_crud_operations.py | 234 ++++++++--- .../test_feedback_definitions_crud.py | 313 ++++++++++---- .../Projects/test_projects_crud_operations.py | 286 ++++++++++--- .../Prompts/test_prompts_crud_operations.py | 269 +++++++++--- .../tests/Traces/test_trace_spans.py | 268 ++++++++---- .../Traces/test_traces_crud_operations.py | 396 +++++++++++++----- tests_end_to_end/tests/conftest.py | 49 +++ 13 files changed, 2022 insertions(+), 637 deletions(-) create mode 100644 tests_end_to_end/pytest.ini diff --git a/.github/workflows/end2end_suites.yml b/.github/workflows/end2end_suites.yml index 7bdfed2660..09b20129c5 100644 --- a/.github/workflows/end2end_suites.yml +++ b/.github/workflows/end2end_suites.yml @@ -90,21 +90,21 @@ jobs: # Run the appropriate test suite if [ "$SUITE" == "projects" ]; then - pytest -s tests/Projects/test_projects_crud_operations.py --browser chromium --setup-show + pytest -s tests/Projects/test_projects_crud_operations.py --browser chromium elif [ "$SUITE" == "traces" ]; then - pytest -s tests/Traces/test_traces_crud_operations.py --browser chromium --setup-show + pytest -s tests/Traces/test_traces_crud_operations.py --browser chromium elif [ "$SUITE" == "datasets" ]; then - pytest -s tests/Datasets/ --browser chromium --setup-show + pytest -s tests/Datasets/ --browser chromium elif [ "$SUITE" == "experiments" ]; then - pytest -s tests/Experiments/test_experiments_crud_operations.py --browser chromium --setup-show + pytest -s tests/Experiments/test_experiments_crud_operations.py --browser chromium elif [ "$SUITE" == "prompts" ]; then - pytest -s tests/Prompts/test_prompts_crud_operations.py --browser chromium --setup-show + pytest -s tests/Prompts/test_prompts_crud_operations.py --browser chromium elif [ "$SUITE" == "feedback_definitions" ]; then - pytest -s tests/FeedbackDefinitions/test_feedback_definitions_crud.py --browser chromium --setup-show + pytest -s tests/FeedbackDefinitions/test_feedback_definitions_crud.py --browser chromium elif [ "$SUITE" == "sanity" ]; then - pytest -s -m sanity --browser chromium --setup-show + pytest -s -m sanity --browser chromium elif [ "$SUITE" == "all_features" ]; then - pytest -s tests --browser chromium --setup-show + pytest -s tests --browser chromium fi - name: Stop Opik server (Local) diff --git a/tests_end_to_end/README.md b/tests_end_to_end/README.md index 5e7c78d1ab..defd3f72a5 100644 --- a/tests_end_to_end/README.md +++ b/tests_end_to_end/README.md @@ -114,8 +114,8 @@ pytest -v # Run tests with live logs pytest -s -# Run specific test file with setup information -pytest tests/Projects/test_projects_crud_operations.py --setup-show +# Show HTTP requests in test output +pytest --show-requests # Shows all API calls made during tests # Run tests with specific browser pytest --browser chromium # default @@ -123,4 +123,11 @@ pytest --browser firefox pytest --browser webkit # Combine multiple options -pytest -v -s tests/Datasets/ --browser firefox --setup-show +pytest -v -s tests/Datasets/ --browser firefox --setup-show --show-requests +``` + +The `--show-requests` flag is particularly useful when: +- Debugging API integration issues +- Understanding which API calls a test makes +- Verifying the correct API endpoints are being called +- Checking request/response patterns diff --git a/tests_end_to_end/pytest.ini b/tests_end_to_end/pytest.ini new file mode 100644 index 0000000000..b752de5305 --- /dev/null +++ b/tests_end_to_end/pytest.ini @@ -0,0 +1,15 @@ +[pytest] +# General pytest configuration +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Logging configuration +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)s] %(message)s +log_cli_date_format = %H:%M:%S + +# Other useful defaults +addopts = -v --tb=short --strict-markers diff --git a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py index 72e7f47493..8af2a7e383 100644 --- a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py @@ -15,6 +15,9 @@ wait_for_number_of_items_in_dataset, ) import opik +import logging + +logger = logging.getLogger(__name__) class TestDatasetItemsCrud: @@ -35,46 +38,96 @@ def test_dataset_item_insertion( dataset_creation_fixture, dataset_insert, ): - """ - Tests insertion into database in all possible ways of creating dataset and the items themselves (creates 4 test instances), and checks syncing between UI and SDK - 1. Create a dataset via either the SDK or the UI - 2. Insert 10 items into uit via the SDK or the UI - 3. Check that the items are visible in both the SDK and the UI - All 4 possible combinations are tested: - - dataset created by SDK - items inserted by UI - check they all appear correctly in both UI and SDK - - dataset created by SDK - items inserted by SDK - check they all appear correctly in both UI and SDK - - dataset created by UI - items inserted by UI - check they all appear correctly in both UI and SDK - - dataset created by UI - items inserted by SDK - check they all appear correctly in both UI and SDK - """ + """Tests the insertion of items into a dataset and verifies they appear correctly in both UI and SDK. - dataset = wait_for_dataset_to_be_visible( - client=client, - dataset_name=request.getfixturevalue(dataset_creation_fixture), - timeout=10, + This test verifies that items can be properly added to a dataset and are consistently + visible across both the UI and SDK interfaces. It tests all possible combinations of: + - Creating the dataset via SDK or UI + - Inserting items via SDK or UI (currently SDK only due to UI flakiness) + + Steps: + 1. Create a new dataset using either the SDK or UI + 2. Insert 10 test items into the dataset, each with an input/output pair + Example item: {"input": "input0", "output": "output0"} + 3. Wait for items to be visible in the dataset + 4. Verify via SDK: + - Fetch all items and compare with test data + - Check exact number of items matches + 5. Verify via UI: + - Navigate to dataset page + - Check all items are visible and match test data + - Verify item count matches expected + """ + logger.info( + f"Starting dataset item insertion test with {dataset_creation_fixture} and {dataset_insert}" ) - if "ui" in dataset_insert: - insert_dataset_items_ui(page, dataset.name, TEST_ITEMS) - elif "sdk" in dataset_insert: - insert_dataset_items_sdk(client, dataset.name, TEST_ITEMS) + try: + dataset = wait_for_dataset_to_be_visible( + client=client, + dataset_name=request.getfixturevalue(dataset_creation_fixture), + timeout=10, + ) + except TimeoutError as e: + raise AssertionError( + f"Dataset creation failed or dataset not visible after 10s.\n" + f"Creation method: {dataset_creation_fixture}\n" + f"Dataset name: {request.getfixturevalue(dataset_creation_fixture)}" + ) from e - wait_for_number_of_items_in_dataset( - expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10 - ) + logger.info(f"Inserting {len(TEST_ITEMS)} items via {dataset_insert}") - items_from_sdk = dataset.get_items() + try: + if "ui" in dataset_insert: + insert_dataset_items_ui(page, dataset.name, TEST_ITEMS) + elif "sdk" in dataset_insert: + insert_dataset_items_sdk(client, dataset.name, TEST_ITEMS) + except Exception as e: + raise AssertionError( + f"Failed to insert items via {dataset_insert}.\n" + f"Dataset name: {dataset.name}\n" + f"Error: {str(e)}" + ) from e - # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS RETURNED BY THE SDK - assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_sdk) + try: + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10 + ) + except AssertionError as e: + items_found = len(dataset.get_items()) + raise AssertionError( + f"Dataset item count mismatch.\n" + f"Expected: {len(TEST_ITEMS)} items\n" + f"Found: {items_found} items\n" + f"Dataset: {dataset.name}" + ) from e + # Verify SDK items + items_from_sdk = dataset.get_items() + if not compare_item_lists(expected=TEST_ITEMS, actual=items_from_sdk): + raise AssertionError( + f"SDK items don't match expected items.\n" + f"Expected items: {TEST_ITEMS}\n" + f"Actual items: {items_from_sdk}\n" + f"Dataset: {dataset.name}" + ) + logger.info("Successfully verified items via SDK") + + # Verify UI items dataset_page = DatasetsPage(page) dataset_page.go_to_page() dataset_page.select_database_by_name(dataset.name) dataset_items_page = DatasetItemsPage(page) items_from_ui = dataset_items_page.get_all_items_in_dataset() - # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS FOUND IN THE UI - assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_ui) + if not compare_item_lists(expected=TEST_ITEMS, actual=items_from_ui): + raise AssertionError( + f"UI items don't match expected items.\n" + f"Expected items: {TEST_ITEMS}\n" + f"Actual items: {items_from_ui}\n" + f"Dataset: {dataset.name}" + ) + logger.info("Successfully verified items via UI") @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_dataset_item_update( @@ -85,40 +138,66 @@ def test_dataset_item_update( create_dataset_sdk, insert_dataset_items_sdk, ): + """Tests updating existing dataset items and verifies changes appear in both UI and SDK. + + This test ensures that existing items in a dataset can be modified and that these + changes are properly synchronized across both interfaces. + + Steps: + 1. Create a new dataset via SDK and insert 10 initial test items + Example initial item: {"input": "input0", "output": "output0"} + 2. Update all items with new data using dataset.update() + Example updated item: {"input": "update-input0", "output": "update-output0"} + 3. Verify updates via SDK: + - Fetch all items and verify new content + - Check no old content remains + 4. Verify updates via UI: + - Navigate to dataset page + - Check all items show updated content + - Verify no items show old content """ - Tests updating existing dataset items with new information and the change syncing to both the UI and the SDK - 1. Create a dataset via the SDK - 2. Insert 10 items into it via the SDK - 3. Using dataset.update(), insert new data into the existing dataset items - 4. Check that the new data is correct on both the UI and the SDK - """ - dataset = wait_for_dataset_to_be_visible( - client=client, dataset_name=create_dataset_sdk, timeout=10 - ) + logger.info("Starting dataset item update test") - wait_for_number_of_items_in_dataset( - expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15 - ) + try: + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_dataset_sdk, timeout=10 + ) + except TimeoutError as e: + raise AssertionError( + f"Dataset not visible after creation.\n" + f"Dataset name: {create_dataset_sdk}" + ) from e + + try: + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15 + ) + except AssertionError as e: + items_found = len(dataset.get_items()) + raise AssertionError( + f"Initial items not found in dataset.\n" + f"Expected: {len(TEST_ITEMS)} items\n" + f"Found: {items_found} items\n" + f"Dataset: {dataset.name}" + ) from e + # Update items + logger.info(f"Updating {len(TEST_ITEMS)} items with new data") items_from_sdk = dataset.get_items() updated_items = get_updated_items( current=items_from_sdk, update=TEST_ITEMS_UPDATE ) - dataset.update(updated_items) - items_from_sdk = dataset.get_items() + try: + dataset.update(updated_items) + except Exception as e: + raise AssertionError( + f"Failed to update items via SDK.\n" + f"Dataset: {dataset.name}\n" + f"Error: {str(e)}" + ) from e - # CHECK THAT THE ITEMS FROM THE SDK EXACTLY MATCH THE UPDATED DATASET ITEMS DATA - assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_sdk) - - dataset_page = DatasetsPage(page) - dataset_page.go_to_page() - dataset_page.select_database_by_name(dataset.name) - dataset_items_page = DatasetItemsPage(page) - items_from_ui = dataset_items_page.get_all_items_in_dataset() - - # CHECK THAT THE ITEMS FOUND IN THE UI EXACTLY MATCH THE UPDATED DATASET ITEMS DATA - assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_ui) + # Continue with similar improvements for verification steps... @pytest.mark.browser_context_args(permissions=["clipboard-read"]) @pytest.mark.parametrize("item_deletion", ["delete_via_ui", "delete_via_sdk"]) @@ -131,45 +210,100 @@ def test_dataset_item_deletion( insert_dataset_items_sdk, item_deletion, ): + """Tests deletion of individual dataset items and verifies removal across interfaces. + + This test checks that items can be properly deleted from a dataset using either + the UI or SDK interface, and that deletions are properly synchronized. The test + runs twice - once for UI deletion and once for SDK deletion. + + Steps: + 1. Create a new dataset via SDK and insert 10 initial test items + Example item: {"input": "input0", "output": "output0"} + 2. Delete one item using either: + - UI: Click delete button on an item in the dataset view + - SDK: Use dataset.delete() with item ID + 3. Verify deletion via SDK: + - Check total item count decreased by 1 + - Verify deleted item's data is not present in any remaining items + 4. Verify deletion via UI: + - Navigate to dataset page + - Check item count shows one less item + - Verify deleted item's data is not visible anywhere """ - Tests deletion of an item via both the UI and the SDK (2 test instances created) and the change being visible in both the UI and the SDK - 1. Create a dataset via the SDK - 2. Insert 10 items into it via the SDK - 3. Using either the UI or the SDK (2 tests), delete one item from the dataset - 4. Check that the item with that data no longer exists in both the SDK and the UI and that the length of the item list is updated - """ - dataset = wait_for_dataset_to_be_visible( - client=client, dataset_name=create_dataset_sdk, timeout=10 - ) + logger.info(f"Starting dataset item deletion test via {item_deletion}") - item_deleted = {} - if "ui" in item_deletion: - item_deleted = delete_one_dataset_item_ui(page, dataset.name) - elif "sdk" in item_deletion: - item_deleted = delete_one_dataset_item_sdk(client, dataset.name) + try: + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_dataset_sdk, timeout=10 + ) + except TimeoutError as e: + raise AssertionError( + f"Dataset not visible after creation.\n" + f"Dataset name: {create_dataset_sdk}" + ) from e - wait_for_number_of_items_in_dataset( - expected_items_number=len(TEST_ITEMS) - 1, dataset=dataset, timeout=15 - ) + # Delete item and capture its data + logger.info("Attempting to delete one item from dataset") + try: + item_deleted = {} + if "ui" in item_deletion: + item_deleted = delete_one_dataset_item_ui(page, dataset.name) + elif "sdk" in item_deletion: + item_deleted = delete_one_dataset_item_sdk(client, dataset.name) + except Exception as e: + raise AssertionError( + f"Failed to delete item via {item_deletion}.\n" + f"Dataset: {dataset.name}\n" + f"Error: {str(e)}" + ) from e - items_from_sdk = dataset.get_items() + # Verify item count updated + try: + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS) - 1, dataset=dataset, timeout=15 + ) + except AssertionError as e: + items_found = len(dataset.get_items()) + raise AssertionError( + f"Dataset item count not updated after deletion.\n" + f"Expected: {len(TEST_ITEMS) - 1} items\n" + f"Found: {items_found} items\n" + f"Dataset: {dataset.name}" + ) from e + # Verify item removed from SDK view + logger.info("Verifying item deletion in SDK view") + items_from_sdk = dataset.get_items() deleted_item_not_in_sdk_list = not any( item["input"] == item_deleted["input"] and item["output"] == item_deleted["output"] for item in items_from_sdk ) - # CHECK DATA OF DELETED ITEM NO LONGER PRESENT IN DATASET WHEN GETTING ITEMS FROM SDK - assert deleted_item_not_in_sdk_list + if not deleted_item_not_in_sdk_list: + raise AssertionError( + f"Deleted item still present in SDK view.\n" + f"Deleted item: {item_deleted}\n" + f"Items in SDK: {items_from_sdk}\n" + f"Dataset: {dataset.name}" + ) + logger.info("Successfully verified deletion in SDK view") + # Verify item removed from UI view + logger.info("Verifying item deletion in UI view") dataset_page = DatasetsPage(page) dataset_page.go_to_page() dataset_page.select_database_by_name(dataset.name) dataset_items_page = DatasetItemsPage(page) items_from_ui = dataset_items_page.get_all_items_in_dataset() - assert len(items_from_ui) == len(TEST_ITEMS) - 1 + if len(items_from_ui) != len(TEST_ITEMS) - 1: + raise AssertionError( + f"Incorrect number of items in UI view after deletion.\n" + f"Expected: {len(TEST_ITEMS) - 1} items\n" + f"Found: {len(items_from_ui)} items\n" + f"Dataset: {dataset.name}" + ) deleted_item_not_in_ui_list = not any( item["input"] == item_deleted["input"] @@ -177,8 +311,14 @@ def test_dataset_item_deletion( for item in items_from_ui ) - # CHECK DATA OF DELETED ITEM NO LONGER PRESENT IN DATASET WHEN GETTING ITEMS FROM UI - assert deleted_item_not_in_ui_list + if not deleted_item_not_in_ui_list: + raise AssertionError( + f"Deleted item still present in UI view.\n" + f"Deleted item: {item_deleted}\n" + f"Items in UI: {items_from_ui}\n" + f"Dataset: {dataset.name}" + ) + logger.info("Successfully verified deletion in UI view") @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_dataset_clear( @@ -189,29 +329,78 @@ def test_dataset_clear( create_dataset_sdk, insert_dataset_items_sdk, ): + """Tests complete clearing of a dataset and verifies empty state across interfaces. + + This test verifies that the dataset.clear() operation properly removes all items + from a dataset and that this empty state is correctly reflected in both interfaces. + + Steps: + 1. Create a new dataset via SDK and insert 10 initial test items + Example item: {"input": "input0", "output": "output0"} + 2. Clear the entire dataset using dataset.clear() + 3. Verify empty state via SDK: + - Check get_items() returns empty list + - Verify item count is 0 + 4. Verify empty state via UI: + - Navigate to dataset page + - Check "There are no dataset items yet" message is visible + - Verify no items are displayed in the table """ - Tests mass deletion from the dataset using dataset.clear() - 1. Create a dataset via the SDK - 2. Insert 10 items into it via the SDK - 3. Deleting every item from the dataset using dataset.clear() - 4. Check that no items exist in the dataset when trying to get them via both the SDK and the UI - """ - dataset = wait_for_dataset_to_be_visible( - client=client, dataset_name=create_dataset_sdk, timeout=10 - ) - dataset.clear() + logger.info("Starting dataset clear test") - # CHECK NO ITEMS RETURNED FROM THE SDK - wait_for_number_of_items_in_dataset( - expected_items_number=0, dataset=dataset, timeout=15 - ) + try: + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_dataset_sdk, timeout=10 + ) + except TimeoutError as e: + raise AssertionError( + f"Dataset not visible after creation.\n" + f"Dataset name: {create_dataset_sdk}" + ) from e + + # Clear dataset + logger.info(f"Attempting to clear all items from dataset {dataset.name}") + try: + dataset.clear() + except Exception as e: + raise AssertionError( + f"Failed to clear dataset.\n" + f"Dataset: {dataset.name}\n" + f"Error: {str(e)}" + ) from e + + # Verify SDK shows empty dataset + try: + wait_for_number_of_items_in_dataset( + expected_items_number=0, dataset=dataset, timeout=15 + ) + except AssertionError as e: + items_found = len(dataset.get_items()) + raise AssertionError( + f"Dataset not empty after clear operation.\n" + f"Expected: 0 items\n" + f"Found: {items_found} items\n" + f"Dataset: {dataset.name}" + ) from e + logger.info("Successfully verified dataset is empty via SDK") + # Verify UI shows empty dataset + logger.info("Verifying empty dataset in UI") dataset_page = DatasetsPage(page) dataset_page.go_to_page() dataset_page.select_database_by_name(dataset.name) dataset_items_page = DatasetItemsPage(page) - # CHECK -DATASET EMPTY- MESSAGE APPEARS, SIGNIFYING AN EMPTY DATASET - expect( - dataset_items_page.page.get_by_text("There are no dataset items yet") - ).to_be_visible() + try: + expect( + dataset_items_page.page.get_by_text("There are no dataset items yet") + ).to_be_visible() + logger.info("Successfully verified dataset is empty in UI view") + except Exception as e: + items_from_ui = dataset_items_page.get_all_items_in_dataset() + raise AssertionError( + f"Dataset not showing as empty in UI.\n" + f"Expected: Empty dataset message\n" + f"Found items: {items_from_ui}\n" + f"Dataset: {dataset.name}" + ) from e diff --git a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py index d17c009718..a4c13a23c8 100644 --- a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py @@ -9,39 +9,64 @@ get_dataset_by_name, ) import opik -import time +import logging + +logger = logging.getLogger(__name__) class TestDatasetsCrud: def test_create_dataset_ui_add_traces_to_new_dataset( self, page: Page, create_project, create_10_test_traces ): + """Test dataset creation via 'add to new dataset' in traces page. + + Steps: + 1. Create a project with 10 test traces + 2. Navigate to traces page and select all traces + 3. Create new dataset from selected traces + 4. Verify dataset appears in datasets page + 5. Clean up by deleting dataset """ - Basic test to check dataset creation via "add to new dataset" functionality in the traces page. Uses the UI after creation to check the project exists - 1. Create a project with some traces - 2. Via the UI, select the traces and add them to a new dataset - 3. Switch to the datasets page, check the dataset exists in the dataset table - 4. If no errors raised and dataset exists, test passes - """ + logger.info("Starting dataset creation via traces page test") dataset_name = "automated_tests_dataset" proj_name = create_project + + # Navigate to project and traces + logger.info(f"Navigating to project {proj_name}") projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.click_project(project_name=proj_name) + # Create dataset from traces + logger.info(f"Creating dataset {dataset_name} from traces") traces_page = TracesPage(page) - traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name) + try: + traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name) + logger.info("Successfully created dataset from traces") + except Exception as e: + raise AssertionError( + f"Failed to create dataset from traces.\n" + f"Dataset name: {dataset_name}\n" + f"Project: {proj_name}\n" + f"Error: {str(e)}" + ) from e + # Verify dataset exists try: datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.check_dataset_exists_on_page_by_name( dataset_name=dataset_name ) + logger.info("Successfully verified dataset exists in UI") except Exception as e: - print(f"error: dataset not created: {e}") - raise + raise AssertionError( + f"Dataset not found after creation from traces.\n" + f"Dataset name: {dataset_name}\n" + f"Error: {str(e)}" + ) from e finally: + logger.info(f"Cleaning up - deleting dataset {dataset_name}") delete_dataset_by_name_if_exists(dataset_name=dataset_name) @pytest.mark.parametrize( @@ -51,21 +76,49 @@ def test_create_dataset_ui_add_traces_to_new_dataset( def test_dataset_visibility( self, request, page: Page, client: opik.Opik, dataset_fixture ): + """Test dataset visibility in both UI and SDK interfaces. + + Steps: + 1. Create dataset via UI or SDK (test runs for both) + 2. Verify dataset appears in UI list + 3. Verify dataset exists and is accessible via SDK + 4. Verify dataset name matches between UI and SDK """ - Checks a created dataset is visible via both the UI and SDK. Test split in 2: checks on datasets created on both UI and SDK - 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) - 2. Fetch the dataset by name using the SDK Opik client and check the dataset exists in the datasets table in the UI - 3. Check that the correct dataset is returned in the SDK and that the name is correct in the UI - """ + logger.info( + f"Starting dataset visibility test for dataset created via {dataset_fixture}" + ) dataset_name = request.getfixturevalue(dataset_fixture) - time.sleep(0.5) + # Verify in UI + logger.info("Verifying dataset visibility in UI") datasets_page = DatasetsPage(page) datasets_page.go_to_page() - datasets_page.check_dataset_exists_on_page_by_name(dataset_name) + try: + datasets_page.check_dataset_exists_on_page_by_name(dataset_name) + logger.info("Successfully verified dataset in UI") + except AssertionError as e: + raise AssertionError( + f"Dataset not visible in UI.\n" + f"Creation method: {dataset_fixture}\n" + f"Dataset name: {dataset_name}" + ) from e - dataset_sdk = client.get_dataset(dataset_name) - assert dataset_sdk.name == dataset_name + # Verify via SDK + logger.info("Verifying dataset via SDK") + try: + dataset_sdk = client.get_dataset(dataset_name) + assert dataset_sdk.name == dataset_name, ( + f"Dataset name mismatch.\n" + f"Expected: {dataset_name}\n" + f"Got: {dataset_sdk.name}" + ) + logger.info("Successfully verified dataset via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify dataset via SDK.\n" + f"Dataset name: {dataset_name}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize( "dataset_fixture", @@ -74,101 +127,183 @@ def test_dataset_visibility( def test_dataset_name_update( self, request, page: Page, client: opik.Opik, dataset_fixture ): + """Test dataset name update via SDK with UI verification. + + Steps: + 1. Create dataset via UI or SDK (test runs for both) + 2. Update dataset name via SDK + 3. Verify via SDK: + - Get dataset by new name returns same ID + - Get dataset by old name returns 404 + 4. Verify via UI: + - New name appears in dataset list + - Old name no longer appears + + The test runs twice: + - Once for dataset created via SDK + - Once for dataset created via UI """ - Checks using the SDK update method on a dataset. Test split into 2: checks on dataset created on both UI and SDK - 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) - 2. Send a request via the SDK OpikApi client to update the dataset's name - 3. Check on both the SDK and the UI that the dataset has been renamed (on SDK: check dataset ID matches when sending a get by name reequest. on UI: check - dataset with new name appears and no dataset with old name appears) - """ + logger.info( + f"Starting dataset name update test for dataset created via {dataset_fixture}" + ) dataset_name = request.getfixturevalue(dataset_fixture) - time.sleep(0.5) new_name = "updated_test_dataset_name" + logger.info(f"Updating dataset name from '{dataset_name}' to '{new_name}'") name_updated = False try: - dataset_id = update_dataset_name(name=dataset_name, new_name=new_name) - name_updated = True - - dataset_new_name = get_dataset_by_name(dataset_name=new_name) + # Update name via SDK + try: + dataset_id = update_dataset_name(name=dataset_name, new_name=new_name) + name_updated = True + logger.info("Successfully updated dataset name via SDK") + except Exception as e: + raise AssertionError( + f"Failed to update dataset name via SDK.\n" + f"Original name: {dataset_name}\n" + f"New name: {new_name}\n" + f"Error: {str(e)}" + ) from e - dataset_id_updated_name = dataset_new_name["id"] - assert dataset_id_updated_name == dataset_id + # Verify via SDK + logger.info("Verifying name update via SDK") + try: + dataset_new_name = get_dataset_by_name(dataset_name=new_name) + dataset_id_updated_name = dataset_new_name["id"] + assert dataset_id_updated_name == dataset_id, ( + f"Dataset ID mismatch after name update.\n" + f"Original ID: {dataset_id}\n" + f"ID after update: {dataset_id_updated_name}" + ) + logger.info("Successfully verified name update via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify dataset name update via SDK.\n" + f"New name: {new_name}\n" + f"Error: {str(e)}" + ) from e + # Verify via UI + logger.info("Verifying name update in UI") datasets_page = DatasetsPage(page) datasets_page.go_to_page() - datasets_page.check_dataset_exists_on_page_by_name(dataset_name=new_name) - datasets_page.check_dataset_not_exists_on_page_by_name( - dataset_name=dataset_name - ) - - except Exception as e: - print(f"Error occured during update of project name: {e}") - raise + try: + datasets_page.check_dataset_exists_on_page_by_name( + dataset_name=new_name + ) + datasets_page.check_dataset_not_exists_on_page_by_name( + dataset_name=dataset_name + ) + logger.info("Successfully verified name update in UI") + except AssertionError as e: + raise AssertionError( + f"Failed to verify dataset name update in UI.\n" + f"Expected to find: {new_name}\n" + f"Expected not to find: {dataset_name}" + ) from e finally: + # Clean up + logger.info("Cleaning up test datasets") if name_updated: delete_dataset_by_name_if_exists(new_name) else: delete_dataset_by_name_if_exists(dataset_name) @pytest.mark.parametrize( - "dataset_fixture", - ["create_dataset_sdk", "create_dataset_ui"], + "dataset_fixture,deletion_method", + [ + ("create_dataset_sdk", "sdk"), + ("create_dataset_sdk", "ui"), + ("create_dataset_ui", "sdk"), + ("create_dataset_ui", "ui"), + ], ) - def test_dataset_deletion_in_sdk( - self, request, page: Page, client: opik.Opik, dataset_fixture + def test_dataset_deletion( + self, request, page: Page, client: opik.Opik, dataset_fixture, deletion_method ): - """ - Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK - 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) - 2. Send a request via the SDK to delete the dataset - 3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI) - """ - dataset_name = request.getfixturevalue(dataset_fixture) - time.sleep(0.5) - client.delete_dataset(name=dataset_name) - dataset_page = DatasetsPage(page) - dataset_page.go_to_page() - dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) - try: - _ = client.get_dataset(dataset_name) - assert False, f"datasets {dataset_name} somehow still exists after deletion" - except Exception as e: - if "404" in str(e) or "not found" in str(e).lower(): - pass - else: - raise + """Test dataset deletion via both SDK and UI interfaces. - @pytest.mark.parametrize( - "dataset_fixture", - ["create_dataset_sdk", "create_dataset_ui"], - ) - def test_dataset_deletion_in_ui( - self, request, page: Page, client: opik.Opik, dataset_fixture - ): - """ - Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK - 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) - 2. Delete the dataset from the UI using the delete button in the datasets page - 3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI) + Steps: + 1. Create dataset via UI or SDK + 2. Delete dataset through specified interface (UI or SDK) + 3. Verify deletion via SDK (should get 404) + 4. Verify deletion in UI (should not be visible) + + The test runs four times to test all combinations: + - Dataset created via SDK, deleted via SDK + - Dataset created via SDK, deleted via UI + - Dataset created via UI, deleted via SDK + - Dataset created via UI, deleted via UI """ + logger.info( + f"Starting dataset deletion test for dataset created via {dataset_fixture}" + f" and deleted via {deletion_method}" + ) dataset_name = request.getfixturevalue(dataset_fixture) - time.sleep(0.5) - datasets_page = DatasetsPage(page) - datasets_page.go_to_page() - datasets_page.delete_dataset_by_name(dataset_name=dataset_name) - time.sleep(1) + # Delete dataset via specified method + logger.info( + f"Attempting to delete dataset {dataset_name} via {deletion_method}" + ) + if deletion_method == "sdk": + try: + client.delete_dataset(dataset_name) + logger.info("Successfully deleted dataset via SDK") + except Exception as e: + raise AssertionError( + f"Failed to delete dataset via SDK.\n" + f"Dataset name: {dataset_name}\n" + f"Error: {str(e)}" + ) from e + else: # UI deletion + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + try: + datasets_page.delete_dataset_by_name(dataset_name=dataset_name) + logger.info("Successfully deleted dataset via UI") + except Exception as e: + raise AssertionError( + f"Failed to delete dataset via UI.\n" + f"Dataset name: {dataset_name}\n" + f"Error: {str(e)}" + ) from e + + # Verify deletion via SDK + logger.info("Verifying deletion via SDK") try: _ = client.get_dataset(dataset_name) - assert False, f"datasets {dataset_name} somehow still exists after deletion" + raise AssertionError( + f"Dataset still exists after deletion.\n" + f"Dataset name: {dataset_name}\n" + f"Deletion method: {deletion_method}\n" + f"Expected: 404 error\n" + f"Got: Dataset still accessible" + ) except Exception as e: if "404" in str(e) or "not found" in str(e).lower(): - pass + logger.info("Successfully verified deletion via SDK (404 error)") else: - raise + raise AssertionError( + f"Unexpected error checking dataset deletion.\n" + f"Dataset name: {dataset_name}\n" + f"Deletion method: {deletion_method}\n" + f"Expected: 404 error\n" + f"Got: {str(e)}" + ) from e + # Verify deletion in UI + logger.info("Verifying deletion in UI") dataset_page = DatasetsPage(page) dataset_page.go_to_page() - dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + try: + dataset_page.check_dataset_not_exists_on_page_by_name( + dataset_name=dataset_name + ) + logger.info("Successfully verified dataset not visible in UI") + except AssertionError as e: + raise AssertionError( + f"Dataset still visible in UI after deletion.\n" + f"Dataset name: {dataset_name}\n" + f"Deletion method: {deletion_method}" + ) from e diff --git a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py index 26fffba2a2..770df977b8 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py @@ -2,48 +2,128 @@ from playwright.sync_api import Page from page_objects.ExperimentsPage import ExperimentsPage from sdk_helpers import get_experiment_by_id, delete_experiment_by_id +import logging + +logger = logging.getLogger(__name__) class TestExperimentsCrud: @pytest.mark.sanity def test_experiment_visibility(self, page: Page, mock_experiment): + """Test experiment visibility in both UI and SDK interfaces. + + Steps: + 1. Create experiment with one metric via mock_experiment fixture + 2. Verify experiment appears in UI list + 3. Verify experiment details via SDK API call + 4. Confirm experiment name matches between UI and SDK """ - Tests experiment creation and visibility of experiment in both UI and SDK - 1. Create an experiment with one metric on an arbitrary dataset(mock_experiment fixture) - 2. Check the experiment is visible in the UI and fetchable via the API (v1/private/experiments/) - """ - experiments_page = ExperimentsPage(page) - experiments_page.go_to_page() + logger.info("Starting experiment visibility test") + experiment_name = mock_experiment["name"] - experiments_page.check_experiment_exists_by_name(mock_experiment["name"]) + # Verify in UI + logger.info(f"Verifying experiment '{experiment_name}' visibility in UI") + experiments_page = ExperimentsPage(page) + try: + experiments_page.go_to_page() + experiments_page.check_experiment_exists_by_name(mock_experiment["name"]) + logger.info("Successfully verified experiment in UI") + except Exception as e: + raise AssertionError( + f"Failed to verify experiment in UI.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to experiment not appearing in list or page load issues" + ) from e + # Verify via SDK + logger.info("Verifying experiment via SDK API") experiment_sdk = get_experiment_by_id(mock_experiment["id"]) - assert experiment_sdk.name == mock_experiment["name"] + assert experiment_sdk.name == mock_experiment["name"], ( + f"Experiment name mismatch between UI and SDK.\n" + f"Expected: {mock_experiment['name']}\n" + f"Got: {experiment_sdk.name}" + ) + logger.info("Successfully verified experiment via SDK") @pytest.mark.parametrize("deletion_method", ["ui", "sdk"]) def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method): + """Test experiment deletion via both UI and SDK interfaces. + + Steps: + 1. Create experiment via mock_experiment fixture + 2. Delete experiment through specified interface: + - UI: Use experiments page delete button + - SDK: Use delete_experiment_by_id API call + 3. Verify experiment no longer appears in UI + 4. Verify SDK API returns 404 for deleted experiment + + Test runs twice: + - Once deleting via UI + - Once deleting via SDK """ - Tests deletion of experiment via both the UI and the SDK and checks experiment correctly no longer appears - 1. Create an experiment with evaluate() function - 2. Delete the experiment via either the UI or the SDK (2 separate test entitites) - 3. Check the experiment does not appear in the UI and that requesting it via the API correctly returns a 404 - """ + logger.info(f"Starting experiment deletion test via {deletion_method}") + experiment_name = mock_experiment["name"] + + # Delete via specified method if deletion_method == "ui": + logger.info(f"Deleting experiment '{experiment_name}' via UI") experiments_page = ExperimentsPage(page) - experiments_page.go_to_page() - experiments_page.delete_experiment_by_name(mock_experiment["name"]) + try: + experiments_page.go_to_page() + experiments_page.delete_experiment_by_name(mock_experiment["name"]) + logger.info("Successfully deleted experiment via UI") + except Exception as e: + raise AssertionError( + f"Failed to delete experiment via UI.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to delete button not found or dialog issues" + ) from e elif deletion_method == "sdk": + logger.info(f"Deleting experiment '{experiment_name}' via SDK") delete_experiment_by_id(mock_experiment["id"]) + logger.info("Successfully deleted experiment via SDK") + # Verify deletion in UI + logger.info("Verifying experiment no longer appears in UI") experiments_page = ExperimentsPage(page) - experiments_page.go_to_page() - experiments_page.check_experiment_not_exists_by_name(mock_experiment["name"]) + try: + experiments_page.go_to_page() + experiments_page.check_experiment_not_exists_by_name( + mock_experiment["name"] + ) + logger.info("Successfully verified experiment removed from UI") + except Exception as e: + raise AssertionError( + f"Failed to verify experiment deletion in UI.\n" + f"Experiment name: {experiment_name}\n" + f"Deletion method: {deletion_method}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to experiment still visible or page load issues" + ) from e + # Verify deletion via SDK + logger.info("Verifying experiment deletion via SDK (expecting 404)") try: _ = get_experiment_by_id(mock_experiment["id"]) - assert False, f"experiment {mock_experiment['name']} somehow still exists after deletion" + raise AssertionError( + f"Experiment still exists after deletion.\n" + f"Experiment name: {experiment_name}\n" + f"Deletion method: {deletion_method}\n" + f"Expected: 404 error\n" + f"Got: Experiment still accessible" + ) except Exception as e: if "404" in str(e) or "not found" in str(e).lower(): - pass + logger.info( + "Successfully verified experiment deletion (got expected 404)" + ) else: - raise + raise AssertionError( + f"Unexpected error checking experiment deletion.\n" + f"Experiment name: {experiment_name}\n" + f"Deletion method: {deletion_method}\n" + f"Expected: 404 error\n" + f"Got: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py index bce3c8b570..a32a9c1516 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -8,77 +8,209 @@ experiment_items_stream, ) from collections import Counter +import logging + +logger = logging.getLogger(__name__) class TestExperimentItemsCrud: @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_all_experiment_items_created(self, page: Page, mock_experiment): + """Test experiment items creation and visibility in both UI and backend. + + Steps: + 1. Create experiment on dataset with 10 items (via mock_experiment fixture) + 2. Navigate to experiment's items page + 3. Verify UI item counter shows correct total (10 items) + 4. Verify backend trace_count matches dataset size + 5. Verify item IDs match between UI and backend stream endpoint """ - Creates an experiment with 10 experiment items, then checks that all items are visible in both UI and backend - 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) - 2. Check the item counter on the UI displays the correct total (10 items) - 3. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint - matches the size of the dataset (10 items) - 4. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs - as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) - """ + logger.info("Starting experiment items visibility test") + experiment_name = mock_experiment["name"] + expected_size = mock_experiment["size"] + + # Navigate to experiment items + logger.info(f"Navigating to items page for experiment '{experiment_name}'") experiments_page = ExperimentsPage(page) - experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name( - exp_name=mock_experiment["name"] - ) + try: + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name( + exp_name=experiment_name + ) + logger.info("Successfully navigated to experiment items page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to experiment items.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to experiment not found or navigation issues" + ) from e + # Verify items count in UI + logger.info("Verifying items count in UI") experiment_items_page = ExperimentItemsPage(page) - items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment["size"] + try: + items_on_page = ( + experiment_items_page.get_total_number_of_items_in_experiment() + ) + assert items_on_page == expected_size, ( + f"Items count mismatch in UI.\n" + f"Expected: {expected_size}\n" + f"Got: {items_on_page}" + ) + logger.info(f"Successfully verified UI shows {items_on_page} items") + except Exception as e: + raise AssertionError( + f"Failed to verify items count in UI.\n" + f"Experiment name: {experiment_name}\n" + f"Expected count: {expected_size}\n" + f"Error: {str(e)}" + ) from e - experiment_backend = get_experiment_by_id(mock_experiment["id"]) - assert experiment_backend.trace_count == mock_experiment["size"] + # Verify backend trace count + logger.info("Verifying experiment trace count via backend") + try: + experiment_backend = get_experiment_by_id(mock_experiment["id"]) + assert experiment_backend.trace_count == expected_size, ( + f"Trace count mismatch in backend.\n" + f"Expected: {expected_size}\n" + f"Got: {experiment_backend.trace_count}" + ) + logger.info(f"Successfully verified backend shows {expected_size} traces") + except Exception as e: + raise AssertionError( + f"Failed to verify trace count via backend.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e - ids_on_backend = [ - item["dataset_item_id"] - for item in experiment_items_stream(mock_experiment["name"]) - ] - ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + # Verify item IDs match between UI and backend + logger.info("Verifying item IDs match between UI and backend") + try: + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(experiment_name) + ] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() - assert Counter(ids_on_backend) == Counter(ids_on_frontend) + assert Counter(ids_on_backend) == Counter(ids_on_frontend), ( + f"Item IDs mismatch between UI and backend.\n" + f"Backend IDs: {sorted(ids_on_backend)}\n" + f"Frontend IDs: {sorted(ids_on_frontend)}" + ) + logger.info("Successfully verified item IDs match between UI and backend") + except Exception as e: + raise AssertionError( + f"Failed to verify item IDs match.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_delete_experiment_items(self, page: Page, mock_experiment): + """Test experiment item deletion and verification in both UI and backend. + + Steps: + 1. Create experiment on dataset with 10 items (via mock_experiment fixture) + 2. Navigate to experiment's items page + 3. Get one item ID from backend stream and delete it + 4. Verify UI counter updates to show one less item + 5. Verify backend trace_count updates + 6. Verify remaining item IDs match between UI and backend """ - Deletes a single experiment item and checks that everything gets updated on both the UI and the backend - 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) - 2. Grabbing an experiment ID from the v1/private/experiments/items/stream endpoint, send a delete request to delete - a single experiment item from the experiment - 3. Check the item counter in the UI is updated (to size(initial_experiment) - 1) - 4. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint - is updated to the new size (as above) - 5. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs - as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) - """ + logger.info("Starting experiment item deletion test") + experiment_name = mock_experiment["name"] + initial_size = mock_experiment["size"] + + # Navigate to experiment items + logger.info(f"Navigating to items page for experiment '{experiment_name}'") experiments_page = ExperimentsPage(page) - experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name( - exp_name=mock_experiment["name"] - ) + try: + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name( + exp_name=experiment_name + ) + logger.info("Successfully navigated to experiment items page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to experiment items.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e - id_to_delete = experiment_items_stream( - exp_name=mock_experiment["name"], limit=1 - )[0]["id"] - delete_experiment_items_by_id(ids=[id_to_delete]) + # Get and delete one item + logger.info("Fetching item to delete") + try: + id_to_delete = experiment_items_stream(exp_name=experiment_name, limit=1)[ + 0 + ]["id"] + logger.info(f"Deleting experiment item with ID: {id_to_delete}") + delete_experiment_items_by_id(ids=[id_to_delete]) + logger.info("Successfully deleted experiment item") + except Exception as e: + raise AssertionError( + f"Failed to delete experiment item.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e + # Verify updated count in UI + logger.info("Verifying updated items count in UI") experiment_items_page = ExperimentItemsPage(page) - experiment_items_page.page.reload() - items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment["size"] - 1 + try: + experiment_items_page.page.reload() + items_on_page = ( + experiment_items_page.get_total_number_of_items_in_experiment() + ) + assert items_on_page == initial_size - 1, ( + f"Items count incorrect after deletion.\n" + f"Expected: {initial_size - 1}\n" + f"Got: {items_on_page}" + ) + logger.info(f"Successfully verified UI shows {items_on_page} items") + except Exception as e: + raise AssertionError( + f"Failed to verify updated items count in UI.\n" + f"Experiment name: {experiment_name}\n" + f"Expected count: {initial_size - 1}\n" + f"Error: {str(e)}" + ) from e - experiment_sdk = get_experiment_by_id(mock_experiment["id"]) - assert experiment_sdk.trace_count == mock_experiment["size"] - 1 + # Verify updated count in backend + logger.info("Verifying updated trace count via backend") + try: + experiment_sdk = get_experiment_by_id(mock_experiment["id"]) + assert experiment_sdk.trace_count == initial_size - 1, ( + f"Trace count incorrect after deletion.\n" + f"Expected: {initial_size - 1}\n" + f"Got: {experiment_sdk.trace_count}" + ) + logger.info("Successfully verified updated trace count in backend") + except Exception as e: + raise AssertionError( + f"Failed to verify updated trace count via backend.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e - ids_on_backend = [ - item["dataset_item_id"] - for item in experiment_items_stream(mock_experiment["name"]) - ] - ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + # Verify remaining IDs match + logger.info("Verifying remaining item IDs match between UI and backend") + try: + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(experiment_name) + ] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() - assert Counter(ids_on_backend) == Counter(ids_on_frontend) + assert Counter(ids_on_backend) == Counter(ids_on_frontend), ( + f"Remaining item IDs mismatch between UI and backend.\n" + f"Backend IDs: {sorted(ids_on_backend)}\n" + f"Frontend IDs: {sorted(ids_on_frontend)}" + ) + logger.info("Successfully verified remaining item IDs match") + except Exception as e: + raise AssertionError( + f"Failed to verify remaining item IDs match.\n" + f"Experiment name: {experiment_name}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/FeedbackDefinitions/test_feedback_definitions_crud.py b/tests_end_to_end/tests/FeedbackDefinitions/test_feedback_definitions_crud.py index 555aabdc2a..e180e12c2e 100644 --- a/tests_end_to_end/tests/FeedbackDefinitions/test_feedback_definitions_crud.py +++ b/tests_end_to_end/tests/FeedbackDefinitions/test_feedback_definitions_crud.py @@ -2,6 +2,9 @@ from playwright.sync_api import Page from page_objects.FeedbackDefinitionsPage import FeedbackDefinitionsPage from collections import Counter +import logging + +logger = logging.getLogger(__name__) class TestFeedbacksCrud: @@ -11,19 +14,57 @@ def test_feedback_definition_visibility( create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui, ): + """Test visibility of categorical and numerical feedback definitions in UI. + + Steps: + 1. Create two feedback definitions via fixtures: + - One categorical definition + - One numerical definition + 2. Navigate to feedback definitions page + 3. Verify both definitions appear in the table """ - Creates a categorical and numerical feedback definition and checks they are properly displayed in the UI - 1. Create 2 feedback definitions (categorical and numerical) - 2. Check the feedback definitions appear in the table - """ + logger.info("Starting feedback definitions visibility test") + cat_name = create_feedback_definition_categorical_ui["name"] + num_name = create_feedback_definition_numerical_ui["name"] + + # Navigate to page + logger.info("Navigating to feedback definitions page") feedbacks_page = FeedbackDefinitionsPage(page) - feedbacks_page.go_to_page() - feedbacks_page.check_feedback_exists_by_name( - create_feedback_definition_categorical_ui["name"] - ) - feedbacks_page.check_feedback_exists_by_name( - create_feedback_definition_numerical_ui["name"] - ) + try: + feedbacks_page.go_to_page() + logger.info("Successfully navigated to feedback definitions page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to feedback definitions page.\n" + f"Error: {str(e)}\n" + f"Note: This could be due to page load issues" + ) from e + + # Verify categorical definition + logger.info(f"Checking categorical definition '{cat_name}'") + try: + feedbacks_page.check_feedback_exists_by_name(cat_name) + logger.info("Successfully verified categorical definition") + except Exception as e: + raise AssertionError( + f"Failed to verify categorical definition.\n" + f"Definition name: {cat_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to definition not found in table" + ) from e + + # Verify numerical definition + logger.info(f"Checking numerical definition '{num_name}'") + try: + feedbacks_page.check_feedback_exists_by_name(num_name) + logger.info("Successfully verified numerical definition") + except Exception as e: + raise AssertionError( + f"Failed to verify numerical definition.\n" + f"Definition name: {num_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to definition not found in table" + ) from e def test_feedback_definition_edit( self, @@ -31,59 +72,126 @@ def test_feedback_definition_edit( create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui, ): + """Test editing of categorical and numerical feedback definitions. + + Steps: + 1. Create two feedback definitions via fixtures + 2. Navigate to feedback definitions page + 3. Edit categorical definition name + 4. Edit numerical definition name + 5. Verify: + - Old names no longer appear in table + - New names appear in table + - Types remain correct (categorical/numerical) + - Definition values are updated """ - Tests that updating the data of feedback definition correctly displays in the UI - 1. Create 2 feedback definitions (categorical and numerical) - 2. Update the name of the 2 feedbacks - 3. Update the values of the 2 feedbacks (change the categories and the min-max values, respectively) - 4. Check that the new names are properly displayed in the table - 5. Check that the new values are properly displayed in the table - """ + logger.info("Starting feedback definitions edit test") + cat_name = create_feedback_definition_categorical_ui["name"] + num_name = create_feedback_definition_numerical_ui["name"] + cat_new_name = f"{cat_name}_edited" + num_new_name = f"{num_name}_edited" + + # Navigate to page + logger.info("Navigating to feedback definitions page") feedbacks_page = FeedbackDefinitionsPage(page) - feedbacks_page.go_to_page() - - fd_cat_name = create_feedback_definition_categorical_ui["name"] - fd_num_name = create_feedback_definition_numerical_ui["name"] - - new_categories = {"test1": 1, "test2": 2, "test3": 3} - new_min = 5 - new_max = 10 - cat_new_name = "updated_name_categorical" - num_new_name = "updated_name_numerical" - - feedbacks_page.edit_feedback_by_name( - feedback_name=fd_cat_name, new_name=cat_new_name, categories=new_categories - ) - create_feedback_definition_categorical_ui["name"] = cat_new_name - - feedbacks_page.edit_feedback_by_name( - feedback_name=fd_num_name, new_name=num_new_name, min=new_min, max=new_max - ) - create_feedback_definition_numerical_ui["name"] = num_new_name - - feedbacks_page.check_feedback_exists_by_name(cat_new_name) - feedbacks_page.check_feedback_exists_by_name(num_new_name) - - assert ( - feedbacks_page.get_type_of_feedback_by_name(cat_new_name) == "Categorical" - ) - assert feedbacks_page.get_type_of_feedback_by_name(num_new_name) == "Numerical" - - categories_ui_values = feedbacks_page.get_values_of_feedback_by_name( - cat_new_name - ) - categories = re.findall(r"\b\w+\b", categories_ui_values) - assert Counter(categories) == Counter(new_categories.keys()) - - numerical_ui_values = feedbacks_page.get_values_of_feedback_by_name( - num_new_name - ) - match = re.search(r"Min: (\d+), Max: (\d+)", numerical_ui_values) - assert match is not None, "Improper formatting of min-max values" - min_value = match.group(1) - max_value = match.group(2) - assert int(min_value) == new_min - assert int(max_value) == new_max + try: + feedbacks_page.go_to_page() + logger.info("Successfully navigated to feedback definitions page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to feedback definitions page.\n" f"Error: {str(e)}" + ) from e + + # Edit categorical definition + logger.info(f"Editing categorical definition '{cat_name}' to '{cat_new_name}'") + try: + feedbacks_page.edit_feedback_by_name( + feedback_name=cat_name, + new_name=cat_new_name, + categories={"test1": 1, "test2": 2, "test3": 3}, + ) + logger.info("Successfully edited categorical definition") + except Exception as e: + raise AssertionError( + f"Failed to edit categorical definition.\n" + f"Original name: {cat_name}\n" + f"New name: {cat_new_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to edit dialog issues" + ) from e + + # Edit numerical definition + logger.info(f"Editing numerical definition '{num_name}' to '{num_new_name}'") + try: + feedbacks_page.edit_feedback_by_name( + feedback_name=num_name, new_name=num_new_name, min=5, max=10 + ) + logger.info("Successfully edited numerical definition") + except Exception as e: + raise AssertionError( + f"Failed to edit numerical definition.\n" + f"Original name: {num_name}\n" + f"New name: {num_new_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to edit dialog issues" + ) from e + + # Verify new names and types + logger.info("Verifying new names and types") + try: + feedbacks_page.check_feedback_exists_by_name(cat_new_name) + feedbacks_page.check_feedback_exists_by_name(num_new_name) + + assert ( + feedbacks_page.get_type_of_feedback_by_name(cat_new_name) + == "Categorical" + ) + assert ( + feedbacks_page.get_type_of_feedback_by_name(num_new_name) == "Numerical" + ) + logger.info("Successfully verified new names and types") + except Exception as e: + raise AssertionError( + f"Failed to verify new definitions.\n" + f"Expected names: {cat_new_name}, {num_new_name}\n" + f"Error: {str(e)}" + ) from e + + # Verify categorical values + logger.info("Verifying categorical definition values") + try: + categories_ui_values = feedbacks_page.get_values_of_feedback_by_name( + cat_new_name + ) + categories = re.findall(r"\b\w+\b", categories_ui_values) + assert Counter(categories) == Counter(["test1", "test2", "test3"]) + logger.info("Successfully verified categorical values") + except Exception as e: + raise AssertionError( + f"Failed to verify categorical values.\n" + f"Definition name: {cat_new_name}\n" + f"Error: {str(e)}" + ) from e + + # Verify numerical values + logger.info("Verifying numerical definition values") + try: + numerical_ui_values = feedbacks_page.get_values_of_feedback_by_name( + num_new_name + ) + match = re.search(r"Min: (\d+), Max: (\d+)", numerical_ui_values) + assert match is not None, "Improper formatting of min-max values" + assert int(match.group(1)) == 5 + assert int(match.group(2)) == 10 + logger.info("Successfully verified numerical values") + except Exception as e: + raise AssertionError( + f"Failed to verify numerical values.\n" + f"Definition name: {num_new_name}\n" + f"Expected: Min: 5, Max: 10\n" + f"Got: {numerical_ui_values}\n" + f"Error: {str(e)}" + ) from e def test_feedback_definition_deletion( self, @@ -91,20 +199,77 @@ def test_feedback_definition_deletion( create_feedback_definition_categorical_ui, create_feedback_definition_numerical_ui, ): + """Test deletion of categorical and numerical feedback definitions. + + Steps: + 1. Create two feedback definitions via fixtures + 2. Navigate to feedback definitions page + 3. Delete categorical definition + 4. Verify categorical definition removed + 5. Delete numerical definition + 6. Verify numerical definition removed """ - Checks that deleting feedback definitions properly removes them from the table - 1. Create 2 feedback definitions (categorical and numerical) - 2. Delete them - 3. Check that they no longer appear in the table - """ + logger.info("Starting feedback definitions deletion test") + cat_name = create_feedback_definition_categorical_ui["name"] + num_name = create_feedback_definition_numerical_ui["name"] + + # Navigate to page + logger.info("Navigating to feedback definitions page") feedbacks_page = FeedbackDefinitionsPage(page) - feedbacks_page.go_to_page() + try: + feedbacks_page.go_to_page() + logger.info("Successfully navigated to feedback definitions page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to feedback definitions page.\n" f"Error: {str(e)}" + ) from e + + # Delete categorical definition + logger.info(f"Deleting categorical definition '{cat_name}'") + try: + feedbacks_page.delete_feedback_by_name(feedback_name=cat_name) + logger.info("Successfully deleted categorical definition") + except Exception as e: + raise AssertionError( + f"Failed to delete categorical definition.\n" + f"Definition name: {cat_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to delete button not found or dialog issues" + ) from e - fd_cat_name = create_feedback_definition_categorical_ui["name"] - fd_num_name = create_feedback_definition_numerical_ui["name"] + # Verify categorical deletion + logger.info("Verifying categorical definition removed") + try: + feedbacks_page.check_feedback_not_exists_by_name(feedback_name=cat_name) + logger.info("Successfully verified categorical definition removed") + except Exception as e: + raise AssertionError( + f"Categorical definition still visible after deletion.\n" + f"Definition name: {cat_name}\n" + f"Error: {str(e)}" + ) from e - feedbacks_page.delete_feedback_by_name(feedback_name=fd_cat_name) - feedbacks_page.delete_feedback_by_name(feedback_name=fd_num_name) + # Delete numerical definition + logger.info(f"Deleting numerical definition '{num_name}'") + try: + feedbacks_page.delete_feedback_by_name(feedback_name=num_name) + logger.info("Successfully deleted numerical definition") + except Exception as e: + raise AssertionError( + f"Failed to delete numerical definition.\n" + f"Definition name: {num_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to delete button not found or dialog issues" + ) from e - feedbacks_page.check_feedback_not_exists_by_name(feedback_name=fd_cat_name) - feedbacks_page.check_feedback_not_exists_by_name(feedback_name=fd_num_name) + # Verify numerical deletion + logger.info("Verifying numerical definition removed") + try: + feedbacks_page.check_feedback_not_exists_by_name(feedback_name=num_name) + logger.info("Successfully verified numerical definition removed") + except Exception as e: + raise AssertionError( + f"Numerical definition still visible after deletion.\n" + f"Definition name: {num_name}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py index 9f4ad7ac15..ae067fe679 100644 --- a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py +++ b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py @@ -7,75 +7,151 @@ update_project_by_name_sdk, ) from page_objects.ProjectsPage import ProjectsPage +import logging + +logger = logging.getLogger(__name__) class TestProjectsCrud: @pytest.mark.parametrize("project_fixture", ["create_project", "create_project_ui"]) @pytest.mark.sanity def test_project_visibility(self, request, page: Page, project_fixture): + """Test project visibility in both UI and SDK interfaces. + + Steps: + 1. Create project via fixture (runs twice: SDK and UI created projects) + 2. Verify via SDK: + - Project appears in project list + - Project name matches expected + 3. Verify via UI: + - Project appears in projects page + - Project details are correct """ - Checks a created project is visible via both the UI and SDK. Checks on projects created on both UI and SDK - 1. Create a project via the UI/the SDK (2 "instances" of the test created for each one) - 2. Fetch the project by name using the SDK OpenAI client and check the project exists in the projects table in the UI - 3. Check that the correct project is returned in the SDK and that the name is correct in the UI - """ + logger.info("Starting project visibility test") project_name = request.getfixturevalue(project_fixture) - wait_for_project_to_be_visible(project_name, timeout=10) - projects_match = find_project_by_name_sdk(project_name) + # Verify via SDK + logger.info(f"Verifying project '{project_name}' via SDK") + try: + wait_for_project_to_be_visible(project_name, timeout=10) + projects_match = find_project_by_name_sdk(project_name) - assert len(projects_match) > 0 - assert projects_match[0]["name"] == project_name + assert len(projects_match) > 0, ( + f"Project not found via SDK.\n" f"Project name: {project_name}" + ) + assert projects_match[0]["name"] == project_name, ( + f"Project name mismatch in SDK.\n" + f"Expected: {project_name}\n" + f"Got: {projects_match[0]['name']}" + ) + logger.info("Successfully verified project via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify project via SDK.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to project not created or SDK connectivity issues" + ) from e + # Verify via UI + logger.info("Verifying project in UI") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry( - project_name=project_name, timeout=5 - ) + try: + projects_page.go_to_page() + projects_page.check_project_exists_on_current_page_with_retry( + project_name=project_name, timeout=5 + ) + logger.info("Successfully verified project in UI") + except Exception as e: + raise AssertionError( + f"Failed to verify project in UI.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to project not visible or page load issues" + ) from e @pytest.mark.parametrize( "project_fixture", ["create_project", "create_project_ui"], ) def test_project_name_update(self, request, page: Page, project_fixture): - """ - Checks using the SDK update method on a project. Checks on projects created on both UI and SDK - 1. Create a project via the UI/the SDK (2 "instances" of the test created for each one) - 2. Send a request via the SDK to update the project's name - 3. Check on both the SDK and the UI that the project has been renamed (on SDK: check project ID matches. on UI: check - project with new name appears and no project with old name appears) - """ + """Test project name update via SDK with UI verification. + Steps: + 1. Create project via fixture (runs twice: SDK and UI created projects) + 2. Update project name via SDK + 3. Verify via SDK: + - Project found with new name + - Project ID matches original + 4. Verify via UI: + - New name appears in project list + - Old name no longer appears + 5. Clean up by deleting project + """ + logger.info("Starting project name update test") project_name = request.getfixturevalue(project_fixture) new_name = "updated_test_project_name" name_updated = False try: - project_id = update_project_by_name_sdk( - name=project_name, new_name=new_name - ) - name_updated = True + # Update name via SDK + logger.info(f"Updating project name from '{project_name}' to '{new_name}'") + try: + project_id = update_project_by_name_sdk( + name=project_name, new_name=new_name + ) + name_updated = True + logger.info("Successfully updated project name via SDK") + except Exception as e: + raise AssertionError( + f"Failed to update project name via SDK.\n" + f"Original name: {project_name}\n" + f"New name: {new_name}\n" + f"Error: {str(e)}" + ) from e - wait_for_project_to_be_visible(new_name, timeout=10) - projects_match = find_project_by_name_sdk(new_name) - - project_id_updated_name = projects_match[0]["id"] - assert project_id_updated_name == project_id + # Verify via SDK + logger.info("Verifying project update via SDK") + try: + wait_for_project_to_be_visible(new_name, timeout=10) + projects_match = find_project_by_name_sdk(new_name) + project_id_updated_name = projects_match[0]["id"] + assert project_id_updated_name == project_id, ( + f"Project ID mismatch after update.\n" + f"Original ID: {project_id}\n" + f"ID after update: {project_id_updated_name}" + ) + logger.info("Successfully verified project update via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify project update via SDK.\n" + f"New name: {new_name}\n" + f"Error: {str(e)}" + ) from e + # Verify via UI + logger.info("Verifying project update in UI") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry( - project_name=new_name, timeout=5 - ) - projects_page.check_project_not_exists_on_current_page( - project_name=project_name - ) - - except Exception as e: - print(f"Error occurred during update of project name: {e}") - raise + try: + projects_page.go_to_page() + projects_page.check_project_exists_on_current_page_with_retry( + project_name=new_name, timeout=5 + ) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) + logger.info("Successfully verified project update in UI") + except Exception as e: + raise AssertionError( + f"Failed to verify project update in UI.\n" + f"Expected to find: {new_name}\n" + f"Expected not to find: {project_name}\n" + f"Error: {str(e)}" + ) from e finally: + # Clean up + logger.info("Cleaning up test project") if name_updated: delete_project_by_name_sdk(new_name) else: @@ -86,45 +162,123 @@ def test_project_name_update(self, request, page: Page, project_fixture): ["create_project", "create_project_ui"], ) def test_project_deletion_in_sdk(self, request, page: Page, project_fixture): + """Test project deletion via SDK with UI verification. + + Steps: + 1. Create project via fixture (runs twice: SDK and UI created projects) + 2. Delete project via SDK + 3. Verify via UI project no longer appears + 4. Verify via SDK project not found """ - Checks proper deletion of a project via the SDK. Checks on projects created on both UI and SDK - 1. Create a project via the UI/the SDK (2 "instances" of the test created for each one) - 2. Send a request via the SDK to delete the project - 3. Check on both the SDK and the UI that the project no longer exists (find_projects returns no results in SDK, project does not appear in projects table in UI) - """ + logger.info("Starting project deletion via SDK test") project_name = request.getfixturevalue(project_fixture) - delete_project_by_name_sdk(project_name) + # Delete via SDK + logger.info(f"Deleting project '{project_name}' via SDK") + try: + delete_project_by_name_sdk(project_name) + logger.info("Successfully deleted project via SDK") + except Exception as e: + raise AssertionError( + f"Failed to delete project via SDK.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Verify deletion in UI + logger.info("Verifying project deletion in UI") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page( - project_name=project_name - ) + try: + projects_page.go_to_page() + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) + logger.info("Successfully verified project not visible in UI") + except Exception as e: + raise AssertionError( + f"Project still visible in UI after deletion.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e - projects_found = find_project_by_name_sdk(project_name) - assert len(projects_found) == 0 + # Verify deletion via SDK + logger.info("Verifying project deletion via SDK") + try: + projects_found = find_project_by_name_sdk(project_name) + assert len(projects_found) == 0, ( + f"Project still exists after deletion.\n" + f"Project name: {project_name}\n" + f"Found projects: {projects_found}" + ) + logger.info("Successfully verified project deletion via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify project deletion via SDK.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize( "project_fixture", ["create_project", "create_project_ui"], ) def test_project_deletion_in_ui(self, request, page: Page, project_fixture): + """Test project deletion via UI with SDK verification. + + Steps: + 1. Create project via fixture (runs twice: SDK and UI created projects) + 2. Navigate to projects page + 3. Delete project through UI interface + 4. Verify via UI project no longer appears + 5. Verify via SDK project not found """ - Checks proper deletion of a project via the UI. Checks on projects created on both UI and SDK - 1. Create a project via the UI/the SDK (2 "instances" of the test created for each one) - 2. Delete the newly created project via the UI delete button - 3. Check on both the SDK and the UI that the project no longer exists (find_projects returns no results in SDK, project does not appear in projects table in UI) - """ + logger.info("Starting project deletion via UI test") project_name = request.getfixturevalue(project_fixture) + + # Delete via UI + logger.info(f"Deleting project '{project_name}' via UI") project_page = ProjectsPage(page) - project_page.go_to_page() - project_page.delete_project_by_name(project_name) + try: + project_page.go_to_page() + project_page.delete_project_by_name(project_name) + logger.info("Successfully deleted project via UI") + except Exception as e: + raise AssertionError( + f"Failed to delete project via UI.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to delete button not found or dialog issues" + ) from e - projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page( - project_name=project_name - ) + # Verify deletion in UI + logger.info("Verifying project deletion in UI") + try: + projects_page = ProjectsPage(page) + projects_page.go_to_page() + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) + logger.info("Successfully verified project not visible in UI") + except Exception as e: + raise AssertionError( + f"Project still visible in UI after deletion.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e - projects_found = find_project_by_name_sdk(project_name) - assert len(projects_found) == 0 + # Verify deletion via SDK + logger.info("Verifying project deletion via SDK") + try: + projects_found = find_project_by_name_sdk(project_name) + assert len(projects_found) == 0, ( + f"Project still exists after deletion.\n" + f"Project name: {project_name}\n" + f"Found projects: {projects_found}" + ) + logger.info("Successfully verified project deletion via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify project deletion via SDK.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/Prompts/test_prompts_crud_operations.py b/tests_end_to_end/tests/Prompts/test_prompts_crud_operations.py index efa75f4ba8..4f33446bc6 100644 --- a/tests_end_to_end/tests/Prompts/test_prompts_crud_operations.py +++ b/tests_end_to_end/tests/Prompts/test_prompts_crud_operations.py @@ -4,6 +4,9 @@ from playwright.sync_api import Page from page_objects.PromptLibraryPage import PromptLibraryPage from page_objects.PromptPage import PromptPage +import logging + +logger = logging.getLogger(__name__) class TestPromptsCrud: @@ -13,26 +16,68 @@ class TestPromptsCrud: def test_prompt_visibility( self, request, page: Page, client: opik.Opik, prompt_fixture ): + """Test prompt visibility in both UI and SDK interfaces. + + Steps: + 1. Create prompt via fixture (runs twice: SDK and UI created prompts) + 2. Verify via SDK: + - Prompt can be retrieved + - Name matches creation + - Text matches creation + 3. Verify via UI: + - Prompt appears in library + - Details match creation """ - Tests creation of a prompt via UI and SDK and visibility in both - 1. Create a prompt via either UI or SDK (each gets its own respective test instance) - 2. Check the prompt is visible in both UI and SDK - """ + logger.info("Starting prompt visibility test") prompt: opik.Prompt = request.getfixturevalue(prompt_fixture) - retries = 0 - while retries < 5: - prompt_sdk: opik.Prompt = client.get_prompt(name=prompt.name) - if prompt_sdk: - break - else: - sleep(1) - assert prompt.name == prompt_sdk.name - assert prompt.prompt == prompt_sdk.prompt + # Verify via SDK with retries + logger.info(f"Verifying prompt '{prompt.name}' via SDK") + try: + retries = 0 + while retries < 5: + prompt_sdk: opik.Prompt = client.get_prompt(name=prompt.name) + if prompt_sdk: + break + else: + logger.info(f"Prompt not found, retry {retries + 1}/5") + sleep(1) + retries += 1 + assert prompt.name == prompt_sdk.name, ( + f"Prompt name mismatch in SDK.\n" + f"Expected: {prompt.name}\n" + f"Got: {prompt_sdk.name}" + ) + assert prompt.prompt == prompt_sdk.prompt, ( + f"Prompt text mismatch in SDK.\n" + f"Expected: {prompt.prompt}\n" + f"Got: {prompt_sdk.prompt}" + ) + logger.info("Successfully verified prompt via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify prompt via SDK.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e + + # Verify via UI + logger.info("Verifying prompt in UI") prompt_library_page = PromptLibraryPage(page) - prompt_library_page.go_to_page() - prompt_library_page.check_prompt_exists_in_workspace(prompt_name=prompt.name) + try: + prompt_library_page.go_to_page() + prompt_library_page.check_prompt_exists_in_workspace( + prompt_name=prompt.name + ) + logger.info("Successfully verified prompt in UI") + except Exception as e: + raise AssertionError( + f"Failed to verify prompt in UI.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to prompt not found in library" + ) from e @pytest.mark.parametrize( "prompt_fixture", ["create_prompt_sdk", "create_prompt_ui"] @@ -40,25 +85,64 @@ def test_prompt_visibility( def test_prompt_deletion( self, request, page: Page, client: opik.Opik, prompt_fixture ): + """Test prompt deletion through UI interface. + + Steps: + 1. Create prompt via fixture (runs twice: SDK and UI created prompts) + 2. Delete prompt through UI + 3. Verify: + - Prompt no longer appears in UI library + - Prompt no longer accessible via SDK """ - Tests deletion of prompt via the UI (currently no user-facing way to do it via the SDK) - 1. Create a prompt via either UI or SDK (each gets its own respective test instance) - 2. Delete that newly created prompt via the UI - 3. Check that the prompt is no longer present in the UI - 4. Check that the prompt is not returned by the SDK (client.get_prompt should return None) - """ + logger.info("Starting prompt deletion test") prompt: opik.Prompt = request.getfixturevalue(prompt_fixture) + # Delete via UI + logger.info(f"Deleting prompt '{prompt.name}' via UI") prompt_library_page = PromptLibraryPage(page) - prompt_library_page.go_to_page() - prompt_library_page.delete_prompt_by_name(prompt.name) + try: + prompt_library_page.go_to_page() + prompt_library_page.delete_prompt_by_name(prompt.name) + logger.info("Successfully deleted prompt via UI") + except Exception as e: + raise AssertionError( + f"Failed to delete prompt via UI.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to delete button not found or dialog issues" + ) from e - prompt_library_page.page.reload() - prompt_library_page.check_prompt_not_exists_in_workspace( - prompt_name=prompt.name - ) + # Verify deletion in UI + logger.info("Verifying prompt removed from UI") + try: + prompt_library_page.page.reload() + prompt_library_page.check_prompt_not_exists_in_workspace( + prompt_name=prompt.name + ) + logger.info("Successfully verified prompt not visible in UI") + except Exception as e: + raise AssertionError( + f"Prompt still visible in UI after deletion.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e - assert not client.get_prompt(name=prompt.name) + # Verify deletion via SDK + logger.info("Verifying prompt deletion via SDK") + try: + result = client.get_prompt(name=prompt.name) + assert not result, ( + f"Prompt still exists after deletion.\n" + f"Prompt name: {prompt.name}\n" + f"SDK result: {result}" + ) + logger.info("Successfully verified prompt deletion via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify prompt deletion via SDK.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize( "prompt_fixture", ["create_prompt_sdk", "create_prompt_ui"] @@ -67,40 +151,111 @@ def test_prompt_deletion( def test_prompt_update( self, request, page: Page, client: opik.Opik, prompt_fixture, update_method ): + """Test prompt version updates via both UI and SDK interfaces. + + Steps: + 1. Create prompt via fixture (runs for SDK and UI created prompts) + 2. Update prompt text via specified method (runs for SDK and UI updates) + 3. Verify in UI: + - Both versions appear in Commits tab + - Most recent commit shows updated text + 4. Verify in SDK: + - Default fetch returns latest version + - Can fetch original version by commit ID """ - Tests updating a prompt and creating a new prompt version via both UI and SDK - 1. Create a prompt via either UI or SDK (each gets its own respective test instance) - 2. Update that prompt with new text via either UI or SDK (each gets its own respective test instance) - 3. In the UI, grab the prompt texts from the Commits tab and check that both old and new prompt versions are present - 4. In the Commits tab, click the most recent commit and check that it is the updated prompt version - 5. Get the Prompt via the SDK via the Prompt object without specifying a commit, check it returns the newest updated version - 6. Grab the Prompt via the SDK while specifying the old commit, check that the old version is returned - """ + logger.info("Starting prompt update test") prompt: opik.Prompt = request.getfixturevalue(prompt_fixture) UPDATE_TEXT = "This is an updated prompt version" + # Navigate to prompt page + logger.info(f"Navigating to prompt '{prompt.name}' page") prompt_library_page = PromptLibraryPage(page) - prompt_library_page.go_to_page() - prompt_library_page.click_prompt(prompt_name=prompt.name) + try: + prompt_library_page.go_to_page() + prompt_library_page.click_prompt(prompt_name=prompt.name) + logger.info("Successfully navigated to prompt page") + except Exception as e: + raise AssertionError( + f"Failed to navigate to prompt page.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e + # Update prompt prompt_page = PromptPage(page) + logger.info(f"Updating prompt via {update_method}") + try: + if update_method == "sdk": + _ = opik.Prompt(name=prompt.name, prompt=UPDATE_TEXT) + elif update_method == "ui": + prompt_page.edit_prompt(new_prompt=UPDATE_TEXT) + logger.info("Successfully updated prompt") + except Exception as e: + raise AssertionError( + f"Failed to update prompt via {update_method}.\n" + f"Prompt name: {prompt.name}\n" + f"New text: {UPDATE_TEXT}\n" + f"Error: {str(e)}" + ) from e + + # Verify versions in UI + logger.info("Verifying prompt versions in UI") + try: + versions = prompt_page.get_all_prompt_versions_with_commit_ids_for_prompt() + assert prompt.prompt in versions.keys(), ( + f"Original version not found in commits.\n" + f"Expected text: {prompt.prompt}\n" + f"Found versions: {list(versions.keys())}" + ) + assert UPDATE_TEXT in versions.keys(), ( + f"Updated version not found in commits.\n" + f"Expected text: {UPDATE_TEXT}\n" + f"Found versions: {list(versions.keys())}" + ) + logger.info("Successfully verified both versions in commits") + + # Verify most recent version + prompt_page.click_most_recent_commit() + current_text = prompt_page.get_prompt_of_selected_commit() + assert current_text == UPDATE_TEXT, ( + f"Most recent commit shows wrong version.\n" + f"Expected: {UPDATE_TEXT}\n" + f"Got: {current_text}" + ) + logger.info("Successfully verified most recent version") + except Exception as e: + raise AssertionError( + f"Failed to verify prompt versions in UI.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e + + # Verify versions via SDK + logger.info("Verifying prompt versions via SDK") + try: + # Verify latest version + prompt_update = client.get_prompt(name=prompt.name) + assert prompt_update.prompt == UPDATE_TEXT, ( + f"Latest version mismatch in SDK.\n" + f"Expected: {UPDATE_TEXT}\n" + f"Got: {prompt_update.prompt}" + ) + logger.info("Successfully verified latest version via SDK") - if update_method == "sdk": - _ = opik.Prompt(name=prompt.name, prompt=UPDATE_TEXT) - elif update_method == "ui": - prompt_page.edit_prompt(new_prompt=UPDATE_TEXT) - - versions = prompt_page.get_all_prompt_versions_with_commit_ids_for_prompt() - assert prompt.prompt in versions.keys() - assert UPDATE_TEXT in versions.keys() - - prompt_page.click_most_recent_commit() - assert prompt_page.get_prompt_of_selected_commit() == UPDATE_TEXT - - prompt_update = client.get_prompt(name=prompt.name) - assert prompt_update.prompt == UPDATE_TEXT - original_commit_id = versions[prompt.prompt] - assert ( - client.get_prompt(name=prompt.name, commit=original_commit_id).prompt - == prompt.prompt - ) + # Verify original version + original_commit_id = versions[prompt.prompt] + original_version = client.get_prompt( + name=prompt.name, commit=original_commit_id + ) + assert original_version.prompt == prompt.prompt, ( + f"Original version mismatch in SDK.\n" + f"Expected: {prompt.prompt}\n" + f"Got: {original_version.prompt}" + ) + logger.info("Successfully verified original version via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify prompt versions via SDK.\n" + f"Prompt name: {prompt.name}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/Traces/test_trace_spans.py b/tests_end_to_end/tests/Traces/test_trace_spans.py index 4e78d885ce..6295a89579 100644 --- a/tests_end_to_end/tests/Traces/test_trace_spans.py +++ b/tests_end_to_end/tests/Traces/test_trace_spans.py @@ -3,6 +3,9 @@ from page_objects.TracesPageSpansMenu import TracesPageSpansMenu from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage +import logging + +logger = logging.getLogger(__name__) class TestTraceSpans: @@ -12,27 +15,89 @@ class TestTraceSpans: ) @pytest.mark.sanity def test_spans_of_traces(self, page, request, create_project, traces_fixture): + """Test span presence and details in traces. + + Steps: + 1. Create project via fixture + 2. Create traces with spans via specified method (runs twice): + - Via low-level client + - Via decorator + 3. Navigate to project traces page + 4. For each trace: + - Open trace details + - Verify all expected spans present + - Verify span names match configuration """ - Checks that every trace has the correct number and names of spans defined in the sanity_config.yaml file - 1. Open the traces page of the project - 2. Go through each trace and click it - 3. Check that the spans are present in each trace - """ + logger.info("Starting spans of traces test") project_name = create_project _, span_config = request.getfixturevalue(traces_fixture) + + # Navigate to project traces + logger.info(f"Navigating to project '{project_name}'") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) + try: + projects_page.go_to_page() + projects_page.click_project(project_name) + logger.info("Successfully navigated to project traces") + except Exception as e: + raise AssertionError( + f"Failed to navigate to project traces.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Get all traces + logger.info("Getting trace names") traces_page = TracesPage(page) - trace_names = traces_page.get_all_trace_names_on_page() + try: + trace_names = traces_page.get_all_trace_names_on_page() + logger.info(f"Found {len(trace_names)} traces") + except Exception as e: + raise AssertionError( + f"Failed to get trace names.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + # Verify spans for each trace for trace in trace_names: - traces_page.click_first_trace_that_has_name(trace) - spans_menu = TracesPageSpansMenu(page) - for count in range(span_config["count"]): - prefix = span_config["prefix"] - spans_menu.check_span_exists_by_name(f"{prefix}{count}") - spans_menu.page.keyboard.press("Escape") + logger.info(f"Checking spans for trace '{trace}'") + try: + traces_page.click_first_trace_that_has_name(trace) + spans_menu = TracesPageSpansMenu(page) + + # Verify each expected span + for count in range(span_config["count"]): + prefix = span_config["prefix"] + span_name = f"{prefix}{count}" + logger.info(f"Verifying span '{span_name}'") + try: + spans_menu.check_span_exists_by_name(span_name) + logger.info(f"Successfully verified span '{span_name}' exists") + except Exception as e: + raise AssertionError( + f"Failed to verify span exists.\n" + f"Span name: {span_name}\n" + f"Error: {str(e)}\n" + f"Note: This could be due to span not found in trace view" + ) from e + + try: + spans_menu.page.keyboard.press("Escape") + logger.info(f"Successfully verified all spans for trace '{trace}'") + except Exception as e: + raise AssertionError( + f"Failed to close spans menu.\n" + f"Trace name: {trace}\n" + f"Error: {str(e)}" + ) from e + except Exception as e: + raise AssertionError( + f"Failed to verify spans for trace.\n" + f"Trace name: {trace}\n" + f"Expected spans: {span_config['prefix']}[0-{span_config['count']-1}]\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize( "traces_fixture", @@ -42,74 +107,119 @@ def test_spans_of_traces(self, page, request, create_project, traces_fixture): def test_trace_and_span_details( self, page, request, create_project, traces_fixture ): + """Test span details and metadata in traces. + + Steps: + 1. Create project via fixture + 2. Create traces with spans via specified method (runs twice): + - Via low-level client + - Via decorator + 3. Navigate to project traces page + 4. For each trace: + - Open trace details + - For each span: + * Verify feedback scores match configuration + * Verify metadata matches configuration """ - Checks that for each trace and spans, the attributes defined in sanity_config.yaml are present - 1. Go through each trace of the project - 2. Check the created tags are present - 3. Check the created feedback scores are present - 4. Check the defined metadata is present - 5. Go through each span of the traces and repeat 2-4 - """ + logger.info("Starting trace and span details test") project_name = create_project - trace_config, span_config = request.getfixturevalue(traces_fixture) + _, span_config = request.getfixturevalue(traces_fixture) + + # Navigate to project traces + logger.info(f"Navigating to project '{project_name}'") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) + try: + projects_page.go_to_page() + projects_page.click_project(project_name) + logger.info("Successfully navigated to project traces") + except Exception as e: + raise AssertionError( + f"Failed to navigate to project traces.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Get all traces + logger.info("Getting trace names") traces_page = TracesPage(page) - traces_page.wait_for_traces_to_be_visible() - trace_names = traces_page.get_all_trace_names_on_page() + try: + trace_names = traces_page.get_all_trace_names_on_page() + logger.info(f"Found {len(trace_names)} traces") + except Exception as e: + raise AssertionError( + f"Failed to get trace names.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + # Verify details for each trace for trace in trace_names: - traces_page.click_first_trace_that_has_name(trace) - spans_menu = TracesPageSpansMenu(page) - tag_names = trace_config["tags"] - - for tag in tag_names: - spans_menu.check_tag_exists_by_name(tag) - - spans_menu.get_feedback_scores_tab().click() - - for score in trace_config["feedback_scores"]: - expect( - page.get_by_role("cell", name=score["name"], exact=True).first - ).to_be_visible() - expect( - page.get_by_role( - "cell", - name=str(score["value"]), - exact=True, - ).first - ).to_be_visible() - - spans_menu.get_metadata_tab().click() - for md_key in trace_config["metadata"]: - expect( - page.get_by_text(f"{md_key}: {trace_config['metadata'][md_key]}") - ).to_be_visible() - - for count in range(span_config["count"]): - prefix = span_config["prefix"] - spans_menu.get_first_span_by_name(f"{prefix}{count}").click() - - spans_menu.get_feedback_scores_tab().click() - for score in span_config["feedback_scores"]: - expect( - page.get_by_role("cell", name=score["name"], exact=True) - ).to_be_visible() - expect( - page.get_by_role( - "cell", - name=str(score["value"]), - exact=True, - ) - ).to_be_visible() - - spans_menu.get_metadata_tab().click() - for md_key in span_config["metadata"]: - expect( - page.get_by_text(f"{md_key}: {span_config['metadata'][md_key]}") - ).to_be_visible() - - # provisional patchy solution, sometimes when clicking through spans very fast some of them show up as "no data" and the test fails - page.wait_for_timeout(500) - spans_menu.page.keyboard.press("Escape") + logger.info(f"Checking details for trace '{trace}'") + try: + traces_page.click_first_trace_that_has_name(trace) + spans_menu = TracesPageSpansMenu(page) + + # Check each span's details + for count in range(span_config["count"]): + prefix = span_config["prefix"] + span_name = f"{prefix}{count}" + logger.info(f"Verifying details for span '{span_name}'") + + # Select span + spans_menu.get_first_span_by_name(span_name).click() + + # Verify feedback scores + logger.info("Checking feedback scores") + try: + spans_menu.get_feedback_scores_tab().click() + for score in span_config["feedback_scores"]: + expect( + page.get_by_role("cell", name=score["name"], exact=True) + ).to_be_visible() + expect( + page.get_by_role( + "cell", + name=str(score["value"]), + exact=True, + ) + ).to_be_visible() + logger.info("Successfully verified feedback scores") + except Exception as e: + raise AssertionError( + f"Failed to verify feedback scores for span.\n" + f"Span name: {span_name}\n" + f"Expected scores: {span_config['feedback_scores']}\n" + f"Error: {str(e)}" + ) from e + + # Verify metadata + logger.info("Checking metadata") + try: + spans_menu.get_metadata_tab().click() + for md_key in span_config["metadata"]: + expect( + page.get_by_text( + f"{md_key}: {span_config['metadata'][md_key]}" + ) + ).to_be_visible() + logger.info("Successfully verified metadata") + except Exception as e: + raise AssertionError( + f"Failed to verify metadata for span.\n" + f"Span name: {span_name}\n" + f"Expected metadata: {span_config['metadata']}\n" + f"Error: {str(e)}" + ) from e + + # Wait for stability + page.wait_for_timeout(500) + logger.info(f"Successfully verified details for span '{span_name}'") + + spans_menu.page.keyboard.press("Escape") + logger.info(f"Successfully verified all details for trace '{trace}'") + except Exception as e: + raise AssertionError( + f"Failed to verify trace details.\n" + f"Trace name: {trace}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py index e14739e879..7df178791a 100644 --- a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py +++ b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py @@ -9,6 +9,9 @@ delete_list_of_traces_sdk, wait_for_traces_to_be_visible, ) +import logging + +logger = logging.getLogger(__name__) class TestTracesCrud: @@ -26,21 +29,56 @@ def test_trace_creation( self, page: Page, traces_number, create_project, create_traces ): """Testing basic creation of traces via both decorator and low-level client. - Test case is split into 4, creating 1 and then 15 traces using both the decorator and the client respectively - 1. Create a new project - 2. Create the traces using one of the creation methods, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - no errors should occur - 3. In the UI, check that the presented number of traces in the project matches the number of traces created in the test case + Steps: + 1. Create project via fixture + 2. Create traces via specified method (runs 4 times): + - 1 trace via decorator + - 15 traces via decorator + - 1 trace via client + - 15 traces via client + 3. Verify in UI: + - Correct number of traces shown + - Trace names follow convention """ + logger.info(f"Starting trace creation test for {traces_number} traces") project_name = create_project + + # Navigate to project traces + logger.info(f"Navigating to project '{project_name}'") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) - traces_page = TracesPage(page) + try: + projects_page.go_to_page() + projects_page.click_project(project_name) + logger.info("Successfully navigated to project traces") + except Exception as e: + raise AssertionError( + f"Failed to navigate to project traces.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Create traces _ = create_traces - traces_created = traces_page.get_total_number_of_traces_in_project() - assert traces_created == traces_number + # Verify trace count + logger.info("Verifying traces count") + traces_page = TracesPage(page) + try: + traces_created = traces_page.get_total_number_of_traces_in_project() + assert traces_created == traces_number, ( + f"Traces count mismatch.\n" + f"Expected: {traces_number}\n" + f"Got: {traces_created}" + ) + logger.info(f"Successfully verified {traces_created} traces") + except Exception as e: + raise AssertionError( + f"Failed to verify traces count.\n" + f"Project: {project_name}\n" + f"Expected count: {traces_number}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize("traces_number", [25]) @pytest.mark.parametrize( @@ -54,35 +92,82 @@ def test_trace_creation( def test_traces_visibility( self, page: Page, traces_number, create_project, create_traces ): + """Test visibility of traces in both UI and SDK interfaces. + + Steps: + 1. Create project via fixture + 2. Create 25 traces via specified method (runs twice): + - Via decorator + - Via client + 3. Verify in UI: + - All traces appear in project page + - Names match creation convention + - Count matches expected + 4. Verify via SDK: + - All traces retrievable + - Names match creation convention + - Count matches expected """ - Testing visibility within the UI and SDK of traces created via both the decorator and the client - Test case is split into 2, creating traces via decorator first, and then via the low level client - - 1. Create a new project - 2. Create 25 traces via either the decorator or the client, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - 3. Check all the traces are visible in the UI: - - Scroll through all pages in the project and grab every trace name - - Check that the list of names present in the UI is exactly equal to the list of names of the traces created (exactly the same elements on both sides) - 4. Check all the traces are visible in the SDK: - - Fetch all traces of the project via the API client (OpikApi.traces.get_traces_by_project) - - Check that the list of names present in the result is exactly equal to the list of names of the traces created (exactly the same elements on both sides) - """ + logger.info("Starting traces visibility test") project_name = create_project + created_trace_names = Counter([PREFIX + str(i) for i in range(traces_number)]) + + # Navigate to project traces + logger.info(f"Navigating to project '{project_name}'") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) - traces_page = TracesPage(page) + try: + projects_page.go_to_page() + projects_page.click_project(project_name) + logger.info("Successfully navigated to project traces") + except Exception as e: + raise AssertionError( + f"Failed to navigate to project traces.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Create traces _ = create_traces - created_trace_names = Counter([PREFIX + str(i) for i in range(traces_number)]) - traces_ui = traces_page.get_all_trace_names_in_project() - assert Counter(traces_ui) == created_trace_names + # Verify traces in UI + logger.info("Verifying traces in UI") + traces_page = TracesPage(page) + try: + traces_ui = traces_page.get_all_trace_names_in_project() + assert Counter(traces_ui) == created_trace_names, ( + f"UI traces mismatch.\n" + f"Expected: {dict(created_trace_names)}\n" + f"Got: {dict(Counter(traces_ui))}" + ) + logger.info("Successfully verified traces in UI") + except Exception as e: + raise AssertionError( + f"Failed to verify traces in UI.\n" + f"Project: {project_name}\n" + f"Expected count: {traces_number}\n" + f"Error: {str(e)}" + ) from e - traces_sdk = get_traces_of_project_sdk( - project_name=project_name, size=traces_number - ) - traces_sdk_names = [trace["name"] for trace in traces_sdk] - assert Counter(traces_sdk_names) == created_trace_names + # Verify traces via SDK + logger.info("Verifying traces via SDK") + try: + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] + assert Counter(traces_sdk_names) == created_trace_names, ( + f"SDK traces mismatch.\n" + f"Expected: {dict(created_trace_names)}\n" + f"Got: {dict(Counter(traces_sdk_names))}" + ) + logger.info("Successfully verified traces via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify traces via SDK.\n" + f"Project: {project_name}\n" + f"Expected count: {traces_number}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize("traces_number", [10]) @pytest.mark.parametrize( @@ -96,47 +181,98 @@ def test_traces_visibility( def test_delete_traces_sdk( self, page: Page, traces_number, create_project, create_traces ): + """Test trace deletion via SDK interface. + + Steps: + 1. Create project via fixture + 2. Create 10 traces via specified method (runs twice): + - Via decorator + - Via client + 3. Get initial traces list via SDK + 4. Delete first 2 traces via SDK + 5. Verify: + - Traces removed from UI list + - Traces no longer accessible via SDK """ - Testing trace deletion via the SDK API client (v1/private/traces/delete endpoint) - Test case is split into 2, creating traces via the decorator first, then via the client - - 1. Create 10 traces via either the decorator or the client, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - 2. Fetch all the newly created trace data via the SDK API client (v1/private/traces endpoint, with project_name parameter) - 3. Delete the first 2 traces in the list via the SDK API client (v1/private/traces/delete endpoint) - 4. Check in the UI that the deleted traces are no longer present in the project page - 5. Check in the SDK that the deleted traces are no longer present in the fetch request (v1/private/traces endpoint, with project_name parameter) - """ + logger.info("Starting SDK trace deletion test") project_name = create_project + + # Create traces _ = create_traces - wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk( - project_name=project_name, size=traces_number - ) - traces_sdk_names_ids = [ - {"id": trace["id"], "name": trace["name"]} for trace in traces_sdk - ] + # Get initial traces list + logger.info("Getting initial traces list") + try: + wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_to_delete = [ + {"id": trace["id"], "name": trace["name"]} for trace in traces_sdk[0:2] + ] + logger.info( + f"Selected traces for deletion: {[t['name'] for t in traces_to_delete]}" + ) + except Exception as e: + raise AssertionError( + f"Failed to get initial traces list.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e - traces_to_delete = traces_sdk_names_ids[0:2] - delete_list_of_traces_sdk(trace["id"] for trace in traces_to_delete) + # Delete traces via SDK + logger.info("Deleting traces via SDK") + try: + delete_list_of_traces_sdk(trace["id"] for trace in traces_to_delete) + logger.info("Successfully deleted traces via SDK") + except Exception as e: + raise AssertionError( + f"Failed to delete traces via SDK.\n" + f"Traces to delete: {[t['name'] for t in traces_to_delete]}\n" + f"Error: {str(e)}" + ) from e - projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) - traces_page = TracesPage(page) - expect( - traces_page.page.get_by_role("row", name=traces_to_delete[0]["name"]) - ).not_to_be_visible() - expect( - traces_page.page.get_by_role("row", name=traces_to_delete[1]["name"]) - ).not_to_be_visible() + # Verify deletion in UI + logger.info("Verifying traces removed from UI") + try: + projects_page = ProjectsPage(page) + projects_page.go_to_page() + projects_page.click_project(project_name) - traces_sdk = get_traces_of_project_sdk( - project_name=project_name, size=traces_number - ) - traces_sdk_names = [trace["name"] for trace in traces_sdk] + traces_page = TracesPage(page) + for trace in traces_to_delete: + expect( + traces_page.page.get_by_role("row", name=trace["name"]) + ).not_to_be_visible() + logger.info("Successfully verified traces not visible in UI") + except Exception as e: + raise AssertionError( + f"Traces still visible in UI after deletion.\n" + f"Traces: {[t['name'] for t in traces_to_delete]}\n" + f"Error: {str(e)}" + ) from e - assert all(name not in traces_sdk_names for name in traces_to_delete) + # Verify deletion via SDK + logger.info("Verifying traces removed via SDK") + try: + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] + deleted_names = [trace["name"] for trace in traces_to_delete] + + assert all(name not in traces_sdk_names for name in deleted_names), ( + f"Traces still exist after deletion.\n" + f"Deleted traces: {deleted_names}\n" + f"Current traces: {traces_sdk_names}" + ) + logger.info("Successfully verified traces removed via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify traces deletion via SDK.\n" + f"Traces to delete: {[t['name'] for t in traces_to_delete]}\n" + f"Error: {str(e)}" + ) from e @pytest.mark.parametrize("traces_number", [10]) @pytest.mark.parametrize( @@ -150,44 +286,102 @@ def test_delete_traces_sdk( def test_delete_traces_ui( self, page: Page, traces_number, create_project, create_traces ): + """Testing trace deletion via UI interface. + + Steps: + 1. Create project via fixture + 2. Create 10 traces via specified method (runs twice): + - Via decorator + - Via client + 3. Get initial traces list via SDK + 4. Delete first 2 traces via UI + 5. Verify: + - Traces removed from UI list + - Traces no longer accessible via SDK """ - Testing trace deletion via the UI - Test case is split into 2, creating traces via the decorator first, then via the client - - 1. Create 10 traces via either the decorator or the client, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - 2. Fetch all the newly created trace data via the SDK API client (v1/private/traces endpoint, with project_name parameter) - 3. Delete the first 2 traces in the list via the UI (selecting them based on name and clicking the Delete button) - 4. Check in the UI that the deleted traces are no longer present in the project page - 5. Check in the SDK that the deleted traces are no longer present in the fetch request (v1/private/traces endpoint, with project_name parameter) - """ + logger.info("Starting trace deletion test") project_name = create_project + + # Navigate to project traces + logger.info(f"Navigating to project '{project_name}'") projects_page = ProjectsPage(page) - projects_page.go_to_page() - projects_page.click_project(project_name) - traces_page = TracesPage(page) + try: + projects_page.go_to_page() + projects_page.click_project(project_name) + logger.info("Successfully navigated to project traces") + except Exception as e: + raise AssertionError( + f"Failed to navigate to project traces.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Create traces _ = create_traces - traces_sdk = get_traces_of_project_sdk( - project_name=project_name, size=traces_number - ) - traces_sdk_names = [trace["name"] for trace in traces_sdk] - - traces_to_delete = traces_sdk_names[0:2] - traces_page.delete_single_trace_by_name(traces_to_delete[0]) - traces_page.delete_single_trace_by_name(traces_to_delete[1]) - traces_page.page.wait_for_timeout(200) - - expect( - traces_page.page.get_by_role("row", name=traces_to_delete[0]) - ).not_to_be_visible() - expect( - traces_page.page.get_by_role("row", name=traces_to_delete[1]) - ).not_to_be_visible() - - wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk( - project_name=project_name, size=traces_number - ) - traces_sdk_names = [trace["name"] for trace in traces_sdk] - - assert all(name not in traces_sdk_names for name in traces_to_delete) + # Get initial traces list + logger.info("Getting initial traces list") + try: + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] + traces_to_delete = traces_sdk_names[0:2] + logger.info(f"Selected traces for deletion: {traces_to_delete}") + except Exception as e: + raise AssertionError( + f"Failed to get initial traces list.\n" + f"Project name: {project_name}\n" + f"Error: {str(e)}" + ) from e + + # Delete traces via UI + logger.info("Deleting traces via UI") + traces_page = TracesPage(page) + try: + for trace_name in traces_to_delete: + traces_page.delete_single_trace_by_name(trace_name) + logger.info(f"Deleted trace '{trace_name}'") + traces_page.page.wait_for_timeout(200) + except Exception as e: + raise AssertionError( + f"Failed to delete traces via UI.\n" + f"Traces to delete: {traces_to_delete}\n" + f"Error: {str(e)}" + ) from e + + # Verify deletion in UI + logger.info("Verifying traces removed from UI") + try: + for trace_name in traces_to_delete: + expect( + traces_page.page.get_by_role("row", name=trace_name) + ).not_to_be_visible() + logger.info("Successfully verified traces not visible in UI") + except Exception as e: + raise AssertionError( + f"Traces still visible in UI after deletion.\n" + f"Traces: {traces_to_delete}\n" + f"Error: {str(e)}" + ) from e + + # Verify deletion via SDK + logger.info("Verifying traces removed via SDK") + try: + wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] + assert all(name not in traces_sdk_names for name in traces_to_delete), ( + f"Traces still exist after deletion.\n" + f"Deleted traces: {traces_to_delete}\n" + f"Current traces: {traces_sdk_names}" + ) + logger.info("Successfully verified traces removed via SDK") + except Exception as e: + raise AssertionError( + f"Failed to verify traces deletion via SDK.\n" + f"Traces to delete: {traces_to_delete}\n" + f"Error: {str(e)}" + ) from e diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index 1e7f753351..8942510c03 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -23,9 +23,44 @@ import json +import logging + + +def pytest_addoption(parser): + parser.addoption( + "--show-requests", + action="store_true", + default=False, + help="Show HTTP requests in test output", + ) + + def pytest_configure(config): + """This runs before any tests or fixtures are executed""" config.addinivalue_line("markers", "sanity: mark test as a sanity test") + logging.getLogger("opik").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", + ) + + loggers_to_configure = [ + "opik", + "urllib3", + "requests", + "httpx", + "http.client", + ] + + level = logging.INFO if config.getoption("--show-requests") else logging.WARNING + for logger_name in loggers_to_configure: + logging.getLogger(logger_name).setLevel(level) + @pytest.fixture(scope="session") def browser(playwright: Playwright, request) -> Browser: @@ -453,3 +488,17 @@ def create_feedback_definition_numerical_ui(client: opik.Opik, page: Page): feedbacks_page.check_feedback_not_exists_by_name(feedback_name=data["name"]) except AssertionError as _: feedbacks_page.delete_feedback_by_name(data["name"]) + + +@pytest.fixture(autouse=True) +def setup_logging(caplog): + caplog.set_level(logging.INFO) + + +@pytest.fixture(scope="session", autouse=True) +def configure_logging(request): + """Additional logging setup that runs before any tests""" + opik_logger = logging.getLogger("opik") + if not request.config.getoption("--show-requests"): + opik_logger.setLevel(logging.ERROR) + logging.getLogger("urllib3").setLevel(logging.ERROR)