From 01bdd5abf8fff68106e8ab9b372c8f75a8711dd9 Mon Sep 17 00:00:00 2001 From: Gabriel Atkin Date: Fri, 21 Nov 2025 15:05:51 -0500 Subject: [PATCH] Fix column checks and default dashboard creation --- fiddler_utils/schema.py | 15 +- misc-utils/copying_a_dashboard.ipynb | 296 +++++++++++++++++++++++---- 2 files changed, 273 insertions(+), 38 deletions(-) diff --git a/fiddler_utils/schema.py b/fiddler_utils/schema.py index 586deeb78..f7110f77a 100644 --- a/fiddler_utils/schema.py +++ b/fiddler_utils/schema.py @@ -419,7 +419,20 @@ def validate_columns( ``` """ model_columns = SchemaValidator.get_column_names(model) - missing_columns = [col for col in columns if col not in model_columns] + + missing_columns = [] + + # For each column, check if it appears as a substring in any model column name + # This is because Fiddler appends prefixes and suffixes to custom feature column names + missing_columns = [] + for col in columns: + found = False + for model_col in model_columns: + if col == model_col or col in model_col or model_col in col: + found = True + break + if not found: + missing_columns.append(col) is_valid = len(missing_columns) == 0 diff --git a/misc-utils/copying_a_dashboard.ipynb b/misc-utils/copying_a_dashboard.ipynb index 23c79325d..2cb13dc31 100644 --- a/misc-utils/copying_a_dashboard.ipynb +++ b/misc-utils/copying_a_dashboard.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "install", "metadata": {}, "outputs": [], @@ -39,10 +39,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "imports", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fiddler client version: 3.10.0\n", + "fiddler_utils: Successfully imported\n" + ] + } + ], "source": [ "import fiddler as fdl\n", "from fiddler_utils import (\n", @@ -68,23 +77,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "config-cell", "metadata": {}, "outputs": [], "source": [ "# Source Fiddler Instance\n", - "SOURCE_URL = \"\" # e.g., 'https://source.fiddler.ai'\n", - "SOURCE_TOKEN = \"\"\n", - "SOURCE_PROJECT_NAME = \"\"\n", - "SOURCE_MODEL_NAME = \"\"\n", + "SOURCE_URL = \"https://tech-exercise.cloud.fiddler.ai\" # e.g., 'https://source.fiddler.ai'\n", + "SOURCE_TOKEN = \"4hsiemgkUi4i2TI18THfFxlctAIrH2yDGxEHpBfBXpk\"\n", + "SOURCE_PROJECT_NAME = \"dashboard_templates_001\"\n", + "SOURCE_MODEL_NAME = \"fiddler_rag_llm_chatbot\"\n", "SOURCE_MODEL_VERSION = \"\" # Optional, leave empty for unversioned models\n", "\n", "# Target Fiddler Instance (can be same as source)\n", - "TARGET_URL = \"\" # e.g., 'https://target.fiddler.ai'\n", - "TARGET_TOKEN = \"\"\n", - "TARGET_PROJECT_NAME = \"\"\n", - "TARGET_MODEL_NAME = \"\"\n", + "TARGET_URL = \"https://tech-exercise.cloud.fiddler.ai\" # e.g., 'https://target.fiddler.ai'\n", + "TARGET_TOKEN = \"4hsiemgkUi4i2TI18THfFxlctAIrH2yDGxEHpBfBXpk\"\n", + "TARGET_PROJECT_NAME = \"my_test_use_case_009\"\n", + "TARGET_MODEL_NAME = \"fiddler_rag_llm_chatbot\"\n", "TARGET_MODEL_VERSION = \"\" # Optional, leave empty for unversioned models\n", "\n", "# Asset Selection (empty lists = export all)\n", @@ -93,7 +102,7 @@ "\n", "\n", "# Chart Export Options (for bonus section)\n", - "SOURCE_DASHBOARD_ID = \"\" # Dashboard UUID to export charts from (leave empty to skip)\n", + "SOURCE_DASHBOARD_ID = \"b2920aa0-b073-44f6-ac30-17d89c1ae703\" # Dashboard UUID to export charts from (leave empty to skip)\n", "CHART_IDS_TO_EXPORT = [] # e.g., ['uuid1', 'uuid2'] or [] for none" ] }, @@ -107,10 +116,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "setup-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ Connection manager configured\n", + "✓ Asset managers initialized\n" + ] + } + ], "source": [ "# Setup connection manager for handling multiple instances\n", "conn_mgr = ConnectionManager(log_level=\"WARNING\")\n", @@ -135,10 +153,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "get-models-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source model: fiddler_rag_llm_chatbot (ID: 2cc87e3b-d94e-4e4f-9853-59c43755dc95)\n", + "Target model: fiddler_rag_llm_chatbot (ID: 999330d6-3220-4616-b86c-044b1495ba61)\n" + ] + } + ], "source": [ "# Connect to source and get model\n", "with conn_mgr.use(\"source\"):\n", @@ -173,10 +200,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "schema-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source model has 21 columns\n", + "Target model has 21 columns\n", + "\n", + "============================================================\n", + "SCHEMA COMPARISON\n", + "============================================================\n", + "\n", + "Common columns: 21\n", + "\n", + "✅ No missing columns - all source columns exist in target\n", + "\n", + "✅ All common columns have matching data types\n", + "\n", + "✅ Schema compatibility: True\n", + "============================================================\n" + ] + } + ], "source": [ "# Get schema information from both models\n", "with conn_mgr.use(\"source\"):\n", @@ -240,10 +289,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "export-segments-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "EXPORTING SEGMENTS\n", + "============================================================\n", + "\n", + "✓ Exported 1 segment(s)\n", + "\n", + " Segment: Non-null Responses\n", + " Definition: is_not_null(\"response\")\n", + " Referenced columns: {'response'}\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"EXPORTING SEGMENTS\")\n", @@ -276,10 +342,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "import-segments-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "IMPORTING SEGMENTS\n", + "============================================================\n", + "\n", + "Results:\n", + " ✅ Successfully imported: 1\n", + " 🔄 Skipped (existing): 0\n", + " ⊘ Skipped (invalid): 0\n", + " ❌ Failed: 0\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"IMPORTING SEGMENTS\")\n", @@ -320,10 +403,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "export-metrics-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "EXPORTING CUSTOM METRICS\n", + "============================================================\n", + "\n", + "✓ Exported 1 custom metric(s)\n", + "\n", + " Metric: Total Non-Null Responses\n", + " Definition: sum(if(is_not_null(\"response\"), 1, 0))\n", + " Referenced columns: {'response'}\n", + " Type: Aggregation\n", + " Functions: is_not_null, if, sum\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"EXPORTING CUSTOM METRICS\")\n", @@ -363,10 +465,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "import-metrics-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "IMPORTING CUSTOM METRICS\n", + "============================================================\n", + "\n", + "Results:\n", + " ✅ Successfully imported: 1\n", + " 🔄 Skipped (existing): 0\n", + " ⊘ Skipped (invalid): 0\n", + " ❌ Failed: 0\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"IMPORTING CUSTOM METRICS\")\n", @@ -405,10 +524,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "summary-cell", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "EXPORT/IMPORT SUMMARY\n", + "============================================================\n", + "\n", + "Segments:\n", + " Total exported: 1\n", + " Successfully imported: 1\n", + " Skipped (existing): 0\n", + " Skipped (invalid): 0\n", + " Failed: 0\n", + "\n", + "Custom Metrics:\n", + " Total exported: 1\n", + " Successfully imported: 1\n", + " Skipped (existing): 0\n", + " Skipped (invalid): 0\n", + " Failed: 0\n", + "\n", + "============================================================\n", + "OVERALL: 2/2 assets successfully imported\n", + "============================================================\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"EXPORT/IMPORT SUMMARY\")\n", @@ -468,10 +616,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "charts-init", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "INITIALIZING CHART MANAGERS\n", + "============================================================\n", + "\n", + "✓ Source ChartManager initialized\n", + "✓ Target ChartManager initialized\n" + ] + } + ], "source": [ "from fiddler_utils import ChartManager\n", "\n", @@ -490,10 +652,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "charts-export", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "EXPORTING CHARTS\n", + "============================================================\n", + "\n", + "Exporting charts from dashboard: b2920aa0-b073-44f6-ac30-17d89c1ae703\n", + "\n", + "✓ Exported 6 chart(s)\n", + "\n", + "1. Embedding fiddler_rag_llm_chatbot (v1)\n", + " Type: EMBEDDING\n", + "\n", + "2. Traffic fiddler_rag_llm_chatbot (v1)\n", + " Type: MONITORING\n", + " Dependencies: custom_metric=47abbab9-398d-469a-84ba-106fcbfbb5c2, segment={'id': 'c845459f-9c83-4dbd-9036-f0d78a5ca308'}\n", + "\n", + "3. DI Violations fiddler_rag_llm_chatbot (v1)\n", + " Type: MONITORING\n", + "\n", + "4. Fast Trust Models Faithfulness Frequency Chart fiddler_rag_llm_chatbot (v1)\n", + " Type: MONITORING\n", + "\n", + "5. PII Frequency Chart fiddler_rag_llm_chatbot (v1)\n", + " Type: MONITORING\n", + "\n", + "6. Sentiment Frequency Chart fiddler_rag_llm_chatbot (v1)\n", + " Type: MONITORING\n", + "\n" + ] + } + ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"EXPORTING CHARTS\")\n", @@ -553,10 +750,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "charts-import", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "IMPORTING CHARTS\n", + "============================================================\n", + "\n", + "Results:\n", + " ✅ Successfully imported: 6\n", + " ❌ Failed: 0\n", + "\n", + "✓ Successfully imported 6 chart(s) to target model\n" + ] + } + ], "source": [ "if not exported_charts:\n", " print(\"\\n⊘ No charts to import. Skipping import.\")\n", @@ -603,10 +817,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "3b81646a-e6f8-4fab-ac06-fbebee909295", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ✅ Successfully created new default dashboard\n" + ] + } + ], "source": [ "client = RequestClient(\n", " TARGET_URL,\n",