diff --git a/examples/dashboards/Asset Utilization/Asset Utilization.json b/examples/dashboards/Asset Utilization/Asset Utilization.json new file mode 100644 index 0000000..2501e09 --- /dev/null +++ b/examples/dashboards/Asset Utilization/Asset Utilization.json @@ -0,0 +1,904 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Example Data --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 925006, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [ + "AAU" + ], + "targetBlank": true, + "title": "AAU Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#c6dfc9", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "filter": "(Workspace = \"$workspace\" && Location = \"$system\")", + "groupBy": [], + "minionIds": [ + "$system" + ], + "queryKind": "Metadata", + "refId": "A", + "type": "List Assets", + "workspace": "$workspace" + } + ], + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Number of Instruments", + "mode": "reduceRow", + "reduce": { + "reducer": "count" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#c6dfc9", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Past Calibration" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Calibrated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 4, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "filter": "(Workspace = \"$workspace\" && Location = \"$system\")", + "groupBy": [], + "minionIds": [ + "$system" + ], + "queryKind": "Metadata", + "refId": "A", + "type": "List Assets", + "workspace": "$workspace" + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": true, + "source": "calibration status" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Field": false + }, + "indexByName": {}, + "renameByName": { + "OK": "Calibrated", + "PAST_RECOMMENDED_DUE_DATE": "Past Calibration" + } + } + }, + { + "id": "reduce", + "options": { + "includeTimeField": false, + "labelsToFields": false, + "mode": "reduceFields", + "reducers": [ + "count" + ] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Dashboard --" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A" + } + ], + "title": "Average Utilization of Selected Assets", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "mean" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#c9c9c9" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Calibration Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "OK": { + "color": "super-light-green", + "index": 1, + "text": "Calibrated" + }, + "PAST_RECOMMENDED_DUE_DATE": { + "color": "super-light-red", + "index": 0, + "text": "Past Calibration" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 321 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Name" + } + ] + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "filter": "(Workspace = \"$workspace\" && Location = \"$system\")", + "groupBy": [], + "minionIds": [ + "$system" + ], + "queryKind": "Metadata", + "refId": "A", + "type": "List Assets", + "workspace": "" + }, + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "hide": false, + "path": "$asset.Utilization.Daily.All.All", + "properties": true, + "refId": "B", + "type": "Current", + "workspace": "$workspace" + } + ], + "title": "Instruments", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "asset type": true, + "asset type A": true, + "bus type": true, + "bus type A": true, + "calibration status A": true, + "discovery type A": true, + "displayName": true, + "id": false, + "is NI asset": true, + "is NI asset A": false, + "is system controller": true, + "is system controller A": true, + "keywords A": true, + "last updated timestamp": true, + "last updated timestamp A": true, + "location A": false, + "minionId": true, + "minionId A": true, + "model number A": true, + "name 2": true, + "name 3": true, + "parent name": true, + "parent name A": true, + "part number A": true, + "properties A": true, + "self calibration A": true, + "supports external calibration A": true, + "supports reset A": true, + "supports self calibration A": true, + "supports self test A": true, + "system name": true, + "vendor number A": true, + "visa resource name A": true, + "workspace": true, + "workspace 2": true + }, + "includeByName": {}, + "indexByName": { + "asset type A": 11, + "asset_type": 34, + "bus type A": 16, + "calibration due date A": 29, + "calibration status A": 32, + "discovery type A": 24, + "displayName": 35, + "firmware version A": 12, + "id": 0, + "is NI asset A": 17, + "is system controller A": 30, + "keywords A": 18, + "last updated timestamp A": 15, + "location A": 20, + "minionId A": 21, + "minion_id": 1, + "model name A": 4, + "model number A": 5, + "name 2": 33, + "name 3": 36, + "name A": 2, + "parent name A": 22, + "part number A": 14, + "properties A": 19, + "self calibration A": 27, + "serial number A": 3, + "supports external calibration A": 28, + "supports reset A": 26, + "supports self calibration A": 23, + "supports self test A": 25, + "units": 9, + "updated": 10, + "value": 8, + "vendor name A": 6, + "vendor number A": 7, + "visa resource name A": 13, + "workspace 2": 37, + "workspace A": 31 + }, + "renameByName": { + "asset_type": "Asset type", + "bus type": "", + "calibration due date": "Calibration Due Date", + "calibration due date A": "Calibration due date", + "calibration status": "Calibration Status", + "id": "ID", + "is NI asset A": "Is NI asset", + "is system controller": "", + "is system controller A": "", + "location A": "Location", + "minion_id": "Minion ID", + "model name": "Model Name", + "model name A": "Model name", + "name": "Name", + "name A": "Name", + "parent name A": "Parent name", + "serial number": "Serial Number", + "serial number A": "Serial number", + "units": "Unit", + "updated": "Updated", + "value": "Last Utilization", + "vendor name A": "Vendor name", + "workspace": "Location", + "workspace A": "Workspace" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "$asset.Utilization.Daily.All.All", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "$workspace" + } + ], + "title": "Asset Utilization", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "$asset.Utilization.Daily.All.Configuration", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "$workspace" + } + ], + "title": "Utilization (Configuration)", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 41, + "tags": [ + "TI Demo", + "AAU" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Default", + "value": "28bb747b-e192-4ba9-8aad-3f367adbe302" + }, + "datasource": { + "type": "ni-slworkspace-datasource", + "uid": "ni-slworkspace-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Labs", + "multi": false, + "name": "workspace", + "options": [], + "query": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "chotea-ro", + "value": "Precision_5570--SN-GPP8WT3--MAC-14-75-5B-DC-5F-AA" + }, + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "System", + "multi": false, + "name": "system", + "options": [], + "query": { + "workspace": "$workspace" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "child1 (child)" + ], + "value": [ + "Assets.child.child.child" + ] + }, + "datasource": { + "type": "ni-slasset-datasource", + "uid": "ni-slasset-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Asset", + "multi": true, + "name": "asset", + "options": [], + "query": { + "filter": "(Workspace = \"$workspace\" && Location = \"$system\")", + "minionId": "$system", + "queryReturnType": "Asset Tag Path", + "take": 1000 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "AAU_Asset Utilization", + "uid": "fbab1941-fce3-4a2c-ad38-4882389598a1", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/examples/dashboards/Asset Utilization/Attachments/asset-utilization.png b/examples/dashboards/Asset Utilization/Attachments/asset-utilization.png new file mode 100644 index 0000000..f9d93c6 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/asset-utilization.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/create-routine.png b/examples/dashboards/Asset Utilization/Attachments/create-routine.png new file mode 100644 index 0000000..f0dbcaf Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/create-routine.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/import-dashboard.png b/examples/dashboards/Asset Utilization/Attachments/import-dashboard.png new file mode 100644 index 0000000..422c987 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/import-dashboard.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/import-notebook.png b/examples/dashboards/Asset Utilization/Attachments/import-notebook.png new file mode 100644 index 0000000..2524fe9 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/import-notebook.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/input-parameters.png b/examples/dashboards/Asset Utilization/Attachments/input-parameters.png new file mode 100644 index 0000000..c366e51 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/input-parameters.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/lab-utilization.png b/examples/dashboards/Asset Utilization/Attachments/lab-utilization.png new file mode 100644 index 0000000..c6792c5 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/lab-utilization.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/publish-notebook.png b/examples/dashboards/Asset Utilization/Attachments/publish-notebook.png new file mode 100644 index 0000000..93bf2d1 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/publish-notebook.png differ diff --git a/examples/dashboards/Asset Utilization/Attachments/system-utilization.png b/examples/dashboards/Asset Utilization/Attachments/system-utilization.png new file mode 100644 index 0000000..e7e34f2 Binary files /dev/null and b/examples/dashboards/Asset Utilization/Attachments/system-utilization.png differ diff --git a/examples/dashboards/Asset Utilization/Delete Multiple Tags.ipynb b/examples/dashboards/Asset Utilization/Delete Multiple Tags.ipynb new file mode 100644 index 0000000..287aa90 --- /dev/null +++ b/examples/dashboards/Asset Utilization/Delete Multiple Tags.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "1f236472-b649-4d5e-acdf-cbc32247ca7d", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "import asyncio\n", + "from urllib3.util.retry import Retry\n", + "from requests.adapters import HTTPAdapter\n", + "from IPython.display import clear_output\n", + "from systemlink.clients.nitag.api.tags_api import TagsApi" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fa71dd56-d7cd-413d-85af-c8186998dc94", + "metadata": {}, + "outputs": [], + "source": [ + "api_key = os.getenv(\"SYSTEMLINK_API_KEY\")\n", + "http_url = os.getenv('SYSTEMLINK_HTTP_URI')\n", + "os.environ['SKYLINE_USER_SERVICES_URL'] = http_url\n", + "create_tag_selection_endpoint = f\"{http_url}/nitag/v2/selections\"\n", + "delete_tags_in_selection = f\"{http_url}/nitag/v2/update-tags\"\n", + "auth_api_key_endpoint = f\"{http_url}/niauth/v1/auth\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "db09aeb9-158f-433d-896c-877dec3b9680", + "metadata": {}, + "outputs": [], + "source": [ + "session = requests.Session()\n", + "session.headers.update({\n", + " \"Content-Type\": \"application/json\",\n", + " 'x-ni-api-key': api_key,\n", + "})\n", + " \n", + "retry_strategy = Retry(\n", + " total=3, # total retry attempts\n", + " status_forcelist=[500, 502, 503, 504], # which errors to retry\n", + " allowed_methods=[\"HEAD\", \"GET\", \"OPTIONS\", \"POST\", \"PUT\", \"DELETE\"], # methods to retry\n", + " backoff_factor=2 # wait 1s, 2s, 4s ... between retries\n", + ")\n", + "\n", + "# Mount the retry strategy to http and https\n", + "adapter = HTTPAdapter(max_retries=retry_strategy)\n", + "session.mount(\"http://\", adapter)\n", + "session.mount(\"https://\", adapter)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "8aabd583-91b1-4283-8ae9-5a3cebbf47e7", + "metadata": {}, + "source": [ + "## Get Workspace IDs" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a249864e-0ef3-4c8a-8352-2c0d6c39be2e", + "metadata": {}, + "outputs": [], + "source": [ + "def get_workspace_ids():\n", + " response = session.get(url=auth_api_key_endpoint)\n", + " response.raise_for_status()\n", + " data = response.json()\n", + " \n", + " workspaces = data[\"workspaces\"]\n", + "\n", + " enabled = [workspace['id'] for workspace in workspaces if workspace[\"enabled\"] == True]\n", + " return enabled\n", + " return [workspace['id'] for workspace in workspaces]" + ] + }, + { + "cell_type": "markdown", + "id": "d3425e0b-1c0c-43a9-af58-a351bd2e5097", + "metadata": {}, + "source": [ + "## Create selection for faster delete tags" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "aa2aaeef-f11c-4015-a943-f5fc64297aa9", + "metadata": {}, + "outputs": [], + "source": [ + "def create_selection(workspace_id):\n", + " try:\n", + " payload = {\n", + " \"searchPaths\": [\n", + " \"*Utilization*\"\n", + " ],\n", + " \"inactivityTimeout\": 900,\n", + " \"workspace\": workspace_id\n", + " }\n", + " resp = session.post(url=create_tag_selection_endpoint, json=payload)\n", + " resp.raise_for_status()\n", + " data = resp.json()\n", + " \n", + " return data['id']\n", + " except Exception as e:\n", + " print('workspace_id', workspace_id)\n", + " print(e)\n", + " return False" + ] + }, + { + "cell_type": "markdown", + "id": "55cd0356-3d7d-4caf-bcf1-5a0e115e19fa", + "metadata": {}, + "source": [ + "## Delete tags in selection" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "74670919-f66c-48dc-bd7d-14feb78d333b", + "metadata": {}, + "outputs": [], + "source": [ + "def delete_selection(selection_id):\n", + " try:\n", + " resp = session.delete(url=f\"{http_url}/nitag/v2/selections/{selection_id}/tags\")\n", + " resp.raise_for_status()\n", + " if resp.status_code == 204:\n", + " return True\n", + " except Exception as e:\n", + " print('selection_id', selection_id)\n", + " print(e)\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d3384b22-e7d9-4886-be68-b6eed56fe2d0", + "metadata": {}, + "outputs": [], + "source": [ + "workspace_ids = get_workspace_ids()\n", + "\n", + "for ws_id in workspace_ids:\n", + " selection_id = create_selection(ws_id)\n", + " if not selection_id:\n", + " continue\n", + " deleted = delete_selection(selection_id)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/dashboards/Asset Utilization/Lab utilization.json b/examples/dashboards/Asset Utilization/Lab utilization.json new file mode 100644 index 0000000..6577393 --- /dev/null +++ b/examples/dashboards/Asset Utilization/Lab utilization.json @@ -0,0 +1,612 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Example Data --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 938696, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [ + "AAU" + ], + "targetBlank": true, + "title": "AAU Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": false, + "type": "ni-slworkspace-datasource", + "uid": "ni-slworkspace-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Number of Labs", + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#c6dfc9" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slworkspace-datasource", + "uid": "ni-slworkspace-datasource" + }, + "init": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [], + "reducer": "count" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 5, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 10, + "refId": "A" + } + ], + "title": "Average Utilization of Labs", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [], + "reducer": "mean" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Workspace.cb289bd9-3f26-4b83-838c-76129c0126c3.Utilization.Daily.Peak.All" + }, + "properties": [ + { + "id": "displayName", + "value": "Semiconductor Lab" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://am-psa.ni.corp.natinst.com/grafana/d/d344ba9e-1327-476c-bb36-3a6c0711845f/system-utilization?orgId=1&var-workspace=cb289bd9-3f26-4b83-838c-76129c0126c3&var-system=Precision_5550--SN-DKQQ453--MAC-9C-29-76-0A-68-15&from=1699087265518&to=1706863265518" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace.e73fcd94-649b-4d0a-b164-bf647a5d0946.Utilization.Daily.Peak.All" + }, + "properties": [ + { + "id": "displayName", + "value": "ADG Lab" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://am-psa.ni.corp.natinst.com/grafana/d/d344ba9e-1327-476c-bb36-3a6c0711845f/system-utilization?orgId=1&var-workspace=e73fcd94-649b-4d0a-b164-bf647a5d0946&var-system=System_Product_Name--SN-System_Serial_Number--MAC-50-EB-F6-B5-9D-5A&from=1699087233262&to=1706863233262" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace.f473ade8-58e3-48de-a206-4fb88b2c8c88.Utilization.Daily.Peak.All" + }, + "properties": [ + { + "id": "displayName", + "value": "OPC" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://am-psa.ni.corp.natinst.com/grafana/d/d344ba9e-1327-476c-bb36-3a6c0711845f/system-utilization?orgId=1&var-workspace=f473ade8-58e3-48de-a206-4fb88b2c8c88&var-system=NI_PXIe-8880--SN-031B5023--MAC-00-80-2F-28-41-C7&from=1699087296059&to=1706863296059" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 10, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 57, + "minVizHeight": 6, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "Workspace.*.Utilization.Daily.All.All", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "" + } + ], + "title": "Utilization by Lab", + "type": "bargauge" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 7, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Open Dashboard", + "url": "/d/d344ba9e-1327-476c-bb36-3a6c0711845f/aau-system-utilization?var-workspace=${__data.fields.workspace}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "Workspace.$workspace.Utilization.Daily.All.All", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "" + } + ], + "title": "Lab Utilization", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 7, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "ffffff", + "url": "https://am-psa.ni.corp.natinst.com/grafana/d/d344ba9e-1327-476c-bb36-3a6c0711845f/system-utilization?var-workspace=${__field.name}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "Workspace.$workspace.Utilization.Daily.All.Configuration", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "" + } + ], + "title": "Utilization (Configuration)", + "type": "timeseries" + } + ], + "schemaVersion": 41, + "tags": [ + "TI Demo", + "AAU" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "Default", + "Demo", + "PA_Yerevan" + ], + "value": [ + "2300760d-38c4-48a1-9acb-800260812337", + "e2897cf7-6332-433c-8284-65b7b628f3f6", + "b562b4cb-0b4d-4f73-b78f-e3351f869e0d" + ] + }, + "datasource": { + "type": "ni-slworkspace-datasource", + "uid": "ni-slworkspace-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Labs", + "multi": true, + "name": "workspace", + "options": [], + "query": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "AAU_Lab Utilization", + "uid": "a1c56c8e-4b92-4286-a070-914ab6522fb8", + "version": 30, + "weekStart": "" +} diff --git a/examples/dashboards/Asset Utilization/README.md b/examples/dashboards/Asset Utilization/README.md new file mode 100644 index 0000000..5d3c48f --- /dev/null +++ b/examples/dashboards/Asset Utilization/README.md @@ -0,0 +1,105 @@ +# Lab/System/Asset Utilization Monitoring Example + +## Overview: + +This example project provides all required resources to configure and execute a Lab/System/Asset Utilization workflow +example. It includes a Jupyter Notebook and instructions for implementing a scheduled routine for data processing and +three Grafana dashboards, each designed to visualize utilization metrics at distinct hierarchical levels. + +## Solution Overview: + +This example leverages three Grafana dashboards that utilize built-in data sources. System-level and asset-level +utilization data is collected at the SystemLink client layer, automatically transmitted to SystemLink, and made +accessible through standard APIs. The example project includes a preconfigured Jupyter Notebook that processes raw +utilization history, generates initial tags, and subsequently appends new utilization values upon each execution. This +approach ensures that every system and asset is associated with utilization-specific tags, enabling straightforward +visualization through Grafana dashboards using out-of-the-box data sources. + +After importing the dashboard into SystemLink, the following predefined dashboards become available. Each dashboard +implements a hierarchical drill-down approach and includes dropdown selectors for Labs, Systems, and Assets. The +dashboards present high-level utilization metrics along with two time-series charts that display utilization trends for +the selected timeframe across two default categories: _Test_ and _Configuration_. + +1. Lab Utilization Dashboard + - Lab-level utilization is derived from the aggregate utilization of all systems grouped within a single workspace. + To enable tracking at the Lab level, systems must be organized into workspaces and mapped to their corresponding + physical Lab. + ![Lab Utilization Dashboard Screenshot](Attachments/lab-utilization.png) +2. System Utilization + - When the System Utilization dashboard is opened, users can select the desired Lab to locate a specific system for + utilization analysis. These selectors are positioned in the upper section of the dashboard. + - The drill-down view provides detailed insights, including the total number of systems within the selected Lab or + workspace, their connection status, and system-level details presented in a tabular format. + - All dashboards share a consistent UI layout and include two time-series charts that display utilization metrics + for each predefined utilization category. + ![System Utilization Dashboard Screenshot](Attachments/system-utilization.png) +3. Asset Utilization + - The Asset Utilization dashboard provides functionality like the System Utilization dashboard but extends + visibility to asset-level details within the selected system. This includes additional information such as + calibration status. The dashboard presents high-level metrics along with two time-series charts that visualize + utilization trends for the selected assets across predefined categories. + ![Asset Utilization Dashboard Screenshot](Attachments/asset-utilization.png) + +## Step-by-step installation Instructions: + +Solution installation and configuration information is provided with the below step-by-step instructions. + +**Publishing the Notebook** + +1. To import the Jupyter notebook into your SystemLink Enterprise open **Automation >> Scripts** from the SLE main menu, + click the **Upload** **Files** button and select the _Utilization to Tags SLE.ipynb_ notebook. + ![Import Notebook](Attachments/import-notebook.png) +2. Notebook has the following input parameters shown below. + ![Input Parameters](Attachments/input-parameters.png) +3. Right-click the notebook file and select **Publish to SystemLink** from the list. +4. In the **Publish Notebook** window select the workspace where you want the notebook to be available. + ![Publish Notebook](Attachments/publish-notebook.png) +5. From the **Interface** drop-down select **Periodic Execution**. +6. Click **Publish to SystemLink** button. + +_For more information about publishing a notebook, see the documentation available +on [ni.com](https://www.ni.com/docs/en-US/bundle/systemlink-enterprise/page/publishing-a-jupyter-notebook.html)_ + +After publishing the notebook to SLE, a confirmation popup will appear indicating the operation was successful. You can +then proceed to configure the routine for scheduled execution. + +**Setting Up a Routine** + +1. Open the SystemLink menu and navigate to **Automation >> Routines**. +2. On the Routines page, click **Create routine** in the upper-left corner of the window. +3. In the Create routine window under **General** section, provide the following details: + - Routine name and description + - Ensure **Routine State** is enabled + ![Create Routine](Attachments/create-routine.png) +4. In **Automation configuration** section: + - From the Event dropdown, select **at a specific data and time**. + - Set the **Start date and time**. This determines when the notebook will run daily to update the tags. + - Leave the **Repeat** field set to **Daily**. + - In the **Automation** field leave **Execute a notebook** selected. + - To select the notebook that should be executed in selected time every day, from the **Notebook** drop-down select + the notebook you published earlier. + - Click **Create**. Your routine will now appear in the table along with other routines. +5. Go to **Automation >> Execution** page to monitor the status and the execution history of your notebook. +6. After the notebook runs successfully at the scheduled time, tags will be generated for each system and asset. + Navigate to **Systems Management >> Systems,** select a system and open the **Tags** tab for viewing tags associated + with the system. You will see a new section of tags called Utilization. These tags will be updated every day with the + single data point reporting daily utilization. +7. Once all the steps are complete, you can import the dashboard to start viewing utilization data along with other + system and asset-level information. +8. If you need to start over and delete existing tags, use **Delete Multiple Tags** notebook provided in the same + location. This will delete all tags so you can restart the process. + +**Importing the Dashboard** + +1. First locate three `.json` files in the folder to be imported. +2. From SLE main menu, go to **Overview >> Dashboards**. +3. Click **New** in the upper-right corner and select **Import.** +4. In the Import Dashboard window click **Upload dashboard JSON file** and select one of the three .json files to import + the first dashboard. +5. Change the name of the Dashboard if needed. + ![Import Dashboard](Attachments/import-dashboard.png) +6. Select the folder where you want to store the imported dashboard. +7. Modify the UID to ensure uniqueness. +8. Click **Import.** The newly imported dashboard will appear immediately, pre-configured and ready for use. +9. Repeat the import process for the other two `.json` dashboard files until all three dashboards are successfully + imported. \ No newline at end of file diff --git a/examples/dashboards/Asset Utilization/System Utilization.json b/examples/dashboards/Asset Utilization/System Utilization.json new file mode 100644 index 0000000..c55f48a --- /dev/null +++ b/examples/dashboards/Asset Utilization/System Utilization.json @@ -0,0 +1,885 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Example Data --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 925016, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [ + "AAU" + ], + "targetBlank": true, + "title": "AAU Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Number of Systems", + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#c6dfc9" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "queryKind": "Metadata", + "refId": "A", + "systemName": "", + "workspace": "$workspace" + } + ], + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [], + "reducer": "count" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#c6dfc9", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#c6dfc9" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "VIRTUAL" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DISCONNECTED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "APPROVED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 9, + "x": 5, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "queryKind": "Metadata", + "refId": "A", + "systemName": "", + "workspace": "$workspace" + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": true, + "source": "connection status" + } + }, + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "reduceFields", + "reducers": [ + "count" + ] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A" + } + ], + "title": "Average Utilization of Selected Systems", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "mean" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#b2b2b2" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "CONNECTED": { + "color": "light-green", + "index": 1 + }, + "DISCONNECTED": { + "color": "light-red", + "index": 0 + }, + "VIRTUAL": { + "color": "light-blue", + "index": 2 + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System controller ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 206 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.5", + "targets": [ + { + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "hide": false, + "queryKind": "Metadata", + "refId": "A", + "systemName": "", + "workspace": "$workspace" + }, + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "hide": false, + "path": "$all_systems.Utilization.Daily.All.All", + "properties": true, + "refId": "B", + "type": "Current", + "workspace": "" + } + ], + "title": "Systems", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(minion_id)", + "renamePattern": "id" + } + }, + { + "id": "joinByField", + "options": { + "byField": "id", + "mode": "outerTabular" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "displayName": true, + "model": true, + "name 1": true, + "name 2": true, + "vendor": true, + "workspace 2": true + }, + "includeByName": {}, + "indexByName": { + "alias": 2, + "asset_identifier": 1, + "connection status": 6, + "displayName": 15, + "id": 0, + "ip address": 12, + "locked status": 7, + "model": 9, + "name 1": 14, + "name 2": 16, + "operating system": 11, + "system start time": 8, + "units": 4, + "updated": 5, + "value": 3, + "vendor": 10, + "workspace 1": 13, + "workspace 2": 17 + }, + "renameByName": { + "alias": "Alias", + "asset_identifier": "Controller ID", + "connection status": "Connection status", + "id": "ID", + "ip address": "IP address", + "locked status": "Locked status", + "model": "Model", + "name 1": "Name", + "operating system": "Operation system", + "system start time": "System start time", + "units": "Unit", + "updated": "Updated", + "value": "Last Utilization", + "vendor": "Vendor", + "workspace 1": "Workspace" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "$system.Utilization.Daily.All.All", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "$workspace" + } + ], + "title": "System Utilization", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "ni-sltag-datasource", + "uid": "ni-sltag-datasource" + }, + "path": "$system.Utilization.Daily.All.Configuration", + "properties": false, + "refId": "A", + "type": "History", + "workspace": "$workspace" + } + ], + "title": "Utilization (Configuration)", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 41, + "tags": [ + "TI Demo", + "AAU" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Default", + "value": "28bb747b-e192-4ba9-8aad-3f367adbe302" + }, + "datasource": { + "type": "ni-slworkspace-datasource", + "uid": "ni-slworkspace-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Labs", + "multi": false, + "name": "workspace", + "options": [], + "query": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "DESKTOP-3DA64PT", + "hotea-ro", + "inrd-vivinkri", + "inrd-vvenkate", + "NI-cRIO-9036-01C02439", + "NI-cRIO-9045-01E82ED0", + "NI_cRIO-9043--SN-01F08629--MAC-00-80-2F-30-D6-62", + "NI_cRIO-9053--SN-01DA9B1E--MAC-00-80-2F-22-D9-F8", + "OPCUA-9629-01F18A4A", + "Precision_5550--SN-DC2M2F3--MAC-54-14-F3-68-C5-6D", + "PXIE-8840-3", + "RFTraining-px02", + "robert-vm-1", + "rsctest-9039", + "rsctest-9047", + "Test", + "VirtualSystem", + "Vortex-0207DAEC", + "WinDev2407Eval" + ], + "value": [ + "Virtual_Machine--SN-2993-1784-5081-9965-9140-8588-32--MAC-00-15-5D-64-5E-01", + "Precision_5820_Tower_X-Series--SN-B6K1TB3--MAC-A4-BB-6D-DF-8F-67", + "Precision_7680--SN-3F3NW64--MAC-BC-38-98-50-98-2B", + "Precision_7680--SN-8J5PW64--MAC-C0-47-0E-1B-C1-CD", + "NI_cRIO-9036--SN-01C02439--MAC-00-80-2F-16-26-C9", + "NI_cRIO-9045--SN-01E82ED0--MAC-00-80-2F-28-56-49", + "NI_cRIO-9043--SN-01F08629--MAC-00-80-2F-30-D6-62", + "NI_cRIO-9053--SN-01DA9B1E--MAC-00-80-2F-22-D9-F8", + "NI_sbRIO-9629--SN-01F18A4A--MAC-00-80-2F-31-13-B3", + "Precision_5550--SN-DC2M2F3--MAC-54-14-F3-68-C5-6D", + "NI_PXIe-8840--MAC-00-80-2F-19-7F-CF", + "NI_PXIe-8135_Embedded_Controller--MAC-00-80-2F-25-F6-4F", + "Virtual_Machine--SN-2075-2089-8898-1175-6956-4020-78--MAC-00-15-5D-01-90-00", + "NI_cRIO-9039_-Sync---SN-01B909D1--MAC-00-80-2F-25-A8-AA", + "NI_cRIO-9047--SN-01CEE362--MAC-00-80-2F-18-55-19", + "b681d9f5-106d-479d-b416-709dc6ef7178", + "94398f69-432f-4483-a112-eeb6e5a9ef0e", + "NI_sbRIO-9629--SN-0207DAEC--MAC-00-80-2F-34-D8-B7", + "Virtual_Machine--SN-9230-9055-5750-2790-3532-1355-66--MAC-00-15-5D-13-0C-0E" + ] + }, + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Systems", + "multi": true, + "name": "system", + "options": [], + "query": { + "workspace": "$workspace" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "ni-slsystem-datasource", + "uid": "ni-slsystem-datasource" + }, + "definition": "", + "hide": 2, + "includeAll": true, + "label": "All systems", + "multi": true, + "name": "all_systems", + "options": [], + "query": { + "workspace": "$workspace" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "AAU_System Utilization", + "uid": "d344ba9e-1327-476c-bb36-3a6c0711845a", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/examples/dashboards/Asset Utilization/Utilization to Tags SLE.ipynb b/examples/dashboards/Asset Utilization/Utilization to Tags SLE.ipynb new file mode 100644 index 0000000..c227a5f --- /dev/null +++ b/examples/dashboards/Asset Utilization/Utilization to Tags SLE.ipynb @@ -0,0 +1,1425 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b4cf5bd9-a675-4ee1-b0b1-2acd86494798", + "metadata": {}, + "source": [ + "\n", + "# Asset Utilization\n", + "\n", + "This Jupyter notebook transform raw data of utilization into time series data:\n", + "\n", + "## Purpose\n", + "Easier access to utilization data\n", + "\n", + "## Outputs\n", + "- Utilization data will be available in the Tag service.\n" + ] + }, + { + "cell_type": "markdown", + "id": "20a4b9f4-a7a6-46cf-95e9-f3c2cd95bebb", + "metadata": {}, + "source": [ + "## Imports and Setup\n", + "This section imports all the required libraries and environment variables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cfaf51b-67fa-46ea-a3e6-0f8679f04ef9", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99d3013a-2f3e-485d-af52-b9d52118d42f", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "import requests\n", + "import copy\n", + "import asyncio\n", + "import json\n", + "import random\n", + "import string\n", + "import requests\n", + "import scrapbook as sb\n", + "from copy import deepcopy\n", + "from datetime import datetime, timedelta, timezone\n", + "from collections import defaultdict\n", + "from typing import Any, Callable, Dict, List, Tuple\n", + "from requests.adapters import HTTPAdapter\n", + "from urllib3.util.retry import Retry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6173e426-d49e-4416-bac8-c3627e267d6f", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from systemlink.clients.niapm.api.assets_api import AssetsApi\n", + "from systemlink.clients.niapm.api.utilization_api import UtilizationApi\n", + "from systemlink.clients.niapm.api.policy_api import PolicyApi\n", + "from systemlink.clients.niapm.models.query_assets_request import QueryAssetsRequest\n", + "from systemlink.clients.niapm.models.query_asset_utilizations_request import QueryAssetUtilizationsRequest\n", + "from systemlink.clients.niapm.models.utilization_time_interval_model import UtilizationTimeIntervalModel\n", + "\n", + "from systemlink.clients.nisysmgmt.api.systems_api import SystemsApi\n", + "from systemlink.clients.nisysmgmt.models.query_systems_request import QuerySystemsRequest\n", + "\n", + "from systemlink.clients.nitag.api.tags_api import TagsApi\n", + "from systemlink.clients.nitag.models.tag import Tag\n", + "from systemlink.clients.nitag.models.tag_update import TagUpdate\n", + "from systemlink.clients.nitag.models.tag_value import TagValue\n", + "from systemlink.clients.nitag.models.tag_type import TagType\n", + "from systemlink.clients.nitag.models.tag_list_and_merge_flag import TagListAndMergeFlag\n", + "from systemlink.clients.nitag.models.timestamped_tag_value import TimestampedTagValue\n", + "\n", + "from systemlink.httputilizationalgorithm import TimeIntervalGenerator\n", + "from systemlink.messagebus.workspace_translator import translate_ids_to_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fabc91c8-2d32-4ce6-9be3-991366f6a457", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "api_key = os.getenv(\"SYSTEMLINK_API_KEY\")\n", + "http_url = os.getenv('SYSTEMLINK_HTTP_URI')\n", + "os.environ['SKYLINE_USER_SERVICES_URL'] = http_url" + ] + }, + { + "cell_type": "markdown", + "id": "8df0097e-ec9a-43bd-ba30-ce2a18e8d855", + "metadata": {}, + "source": [ + "# Session configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99c40d7a-15f4-4417-8941-3e9697fec63e", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "session = requests.Session()\n", + "session.headers.update({\n", + " \"Content-Type\": \"application/json\",\n", + " 'x-ni-api-key': api_key,\n", + "})\n", + " \n", + "retry_strategy = Retry(\n", + " total=3, # total retry attempts\n", + " status_forcelist=[500, 502, 503, 504], # which errors to retry\n", + " allowed_methods=[\"HEAD\", \"GET\", \"OPTIONS\", \"POST\", \"PUT\", \"DELETE\"], # methods to retry\n", + " backoff_factor=2 # wait 1s, 2s, 4s ... between retries\n", + ")\n", + "\n", + "# Mount the retry strategy to http and https\n", + "adapter = HTTPAdapter(max_retries=retry_strategy)\n", + "session.mount(\"http://\", adapter)\n", + "session.mount(\"https://\", adapter)" + ] + }, + { + "cell_type": "markdown", + "id": "df7abdac-b7b0-4243-922d-29213c4011c9", + "metadata": {}, + "source": [ + "# Input parameters\n", + "\n", + "### 1. `target_date`\n", + "- **Expected Value:** Date/time for the day to calculate utilization, or empty string to use the default.\n", + "- **Format:** ISO-8601 string (e.g., '2025-11-12T23:59:59').\n", + "- **Impact:** Determines which day’s utilization is computed. If \"\", utilization is calculated for yesterday\n", + "\n", + "### 2. `target_workspaces`\n", + "- **Expected Value:** List of workspace IDs to generate utilization for.\n", + "- **Format:** Array of workspace ID values (e.g., [\"ws_123\", \"ws_456\"]).\n", + "- **Impact:** Limits utilization generation to the specified workspaces. If empty ([]), utilization is generated for all workspaces.\n", + "\n", + "### 3. `tag_chunk_size`\n", + "- **Expected Value:** Number of tag updates to include in a single request/query batch.\n", + "- **Format:** Positive integer.\n", + "- **Impact:** Increasing the values reduces the number of requests, keep in mind that the tag service can process up to ~1000 tag updates per second.\n", + "\n", + "### 4. `delay_between_tag_batches`\n", + "- **Expected Value:** Delay to wait between consecutive tag update requests/batches.\n", + "- **Format:** Positive float.\n", + "- **Impact:** Throttles request rate to keep tag update throughput within the Tag service limit ~1000 tag updates per second.\n", + "\n", + "### 5. `debug_tag_prefix`\n", + "- **Expected Value:** String prefix used for debug\n", + "- **Format:** String.\n", + "- **Impact:** After debug tags can be identified and deleted separately from production tags.\n", + "\n", + "### 6. `policy_start_time`\n", + "- **Expected Value:** Time of day (UTC) that defines when the workday starts for the utilization/work-time policy.\n", + "- **Format:** Time-only string in 24-hour UTC format, HH:MM (e.g., \"05:00\").\n", + "- **Impact:** The utilization percentage calculation based on working hours policy.\n", + "\n", + "### 7. `policy_end_time`\n", + "- **Expected Value:** Time of day (UTC) that defines when the workday ends for the utilization/work-time policy.\n", + "- **Format:** Time-only string in 24-hour UTC format, HH:MM (e.g., \"05:00\").\n", + "- **Impact:** The utilization percentage calculation based on working hours policy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f313581c-473c-4a9f-b663-02ccd9d93336", + "metadata": { + "editable": true, + "papermill": { + "parameters": { + "debug_tag_prefix": "", + "delay_between_tag_batches": 0.9, + "policy_end_time": "05:00", + "policy_start_time": "05:00", + "tag_chunk_size": 900, + "target_date": "2025-09-16T00:00:00", + "target_workspaces": [] + } + }, + "slideshow": { + "slide_type": "" + }, + "systemlink": { + "interfaces": [], + "outputs": [], + "parameters": [ + { + "display_name": "Target date", + "id": "target_date", + "type": "string" + }, + { + "display_name": "Target workspaces", + "id": "target_workspaces", + "type": "string[]" + }, + { + "display_name": "Tag chunk size", + "id": "tag_chunk_size", + "type": "number" + }, + { + "display_name": "Delay between tag batches", + "id": "delay_between_tag_batches", + "type": "number" + }, + { + "display_name": "Debug tag prefix", + "id": "debug_tag_prefix", + "type": "string" + }, + { + "display_name": "Policy start time", + "id": "policy_start_time", + "type": "string" + }, + { + "display_name": "Policy end time", + "id": "policy_end_time", + "type": "string" + } + ], + "version": 2 + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "target_date = \"\"\n", + "target_workspaces = []\n", + "tag_chunk_size = 900\n", + "delay_between_tag_batches = 0.9\n", + "debug_tag_prefix = \"\"\n", + "policy_start_time=\"05:00\"\n", + "policy_end_time=\"05:00\"" + ] + }, + { + "cell_type": "markdown", + "id": "41dde11b-147c-4b9d-8715-521a514ecca7", + "metadata": {}, + "source": [ + "# Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a4a5aff-f1c5-4e8b-818d-9cb04d7c867c", + "metadata": {}, + "outputs": [], + "source": [ + "class JobConfig:\n", + " MAX_TAG_UPDATES_PER_SECOND = 1000\n", + "\n", + " def __init__(\n", + " self,\n", + " target_date=\"\",\n", + " target_workspaces=[],\n", + " tag_chunk_size=500,\n", + " delay_between_tag_batches=1,\n", + " debug_tag_prefix=\"\",\n", + " policy_start_time=\"05:00\",\n", + " policy_end_time=\"05:00\",\n", + " ):\n", + " \n", + " self.tag_chunk_size = tag_chunk_size\n", + " self.delay_between_tag_batches = delay_between_tag_batches\n", + " self.policy_start_time = policy_start_time\n", + " self.policy_end_time = policy_end_time\n", + "\n", + " self.start_date, self.end_date = self._compute_date_range(target_date)\n", + " self.tag_middle_segment = f\"Utilization{debug_tag_prefix}\"\n", + "\n", + " self._validate_tag_service_capacity()\n", + "\n", + " self.target_workspaces = self.check_target_workspaces(target_workspaces)\n", + "\n", + " self.utilization_intervals, self.utilization_for_date = self.get_utilization_day()\n", + " \n", + " self.interval_group_function = lambda interval: interval.start.astype(datetime).strftime(\"%Y-%m-%d\")\n", + "\n", + " self.property_not_available_string = 'Not Available'\n", + " self.group_by_value = 'asset_identifier'\n", + " \n", + " def _compute_date_range(self, date_str: str):\n", + " if date_str == \"\":\n", + " yesterday = datetime.utcnow() - timedelta(days=1)\n", + " start_date = yesterday.strftime(\"%Y-%m-%dT00:00:00\")\n", + " end_date = yesterday.strftime(\"%Y-%m-%dT23:59:59\")\n", + " return start_date, end_date\n", + "\n", + " start_date = date_str[:-1]\n", + " end_date = date_str[:-9] + \"23:59:59\"\n", + " return start_date, end_date\n", + "\n", + " def _validate_tag_service_capacity(self):\n", + " \"\"\"\n", + " Validate tag update rate is within service capacity\n", + " \"\"\"\n", + " if self.delay_between_tag_batches <= 0:\n", + " raise ValueError(\n", + " f\"delay_between_tag_batches must be > 0, got {self.delay_between_tag_batches}.\"\n", + " )\n", + "\n", + " rate = self.tag_chunk_size / self.delay_between_tag_batches\n", + " if rate > self.MAX_TAG_UPDATES_PER_SECOND:\n", + " raise ValueError(\n", + " f\"Max allowed is {self.MAX_TAG_UPDATES_PER_SECOND} tags/sec. \"\n", + " )\n", + " \n", + " def check_target_workspaces(self, target_workspaces):\n", + " \"\"\"\n", + " All provided workspaces should be authorized+enabled\n", + " \"\"\"\n", + " auth_api_key_endpoint = f\"{http_url}/niauth/v1/auth\"\n", + " response = session.get(url=auth_api_key_endpoint)\n", + " response.raise_for_status()\n", + " data = response.json()\n", + " \n", + " workspaces = data[\"workspaces\"]\n", + "\n", + " authorized_and_enabled_workspaces = [ws for ws in workspaces if ws.get(\"enabled\") is True]\n", + " if len(target_workspaces) == 0:\n", + " return authorized_and_enabled_workspaces\n", + "\n", + " # Validate that every target id is authorized+enabled\n", + " missing = []\n", + " for target_id in target_workspaces:\n", + " found = False\n", + " for ws in authorized_and_enabled_workspaces:\n", + " if ws.get(\"id\") == target_id:\n", + " found = True\n", + " break\n", + " if not found:\n", + " missing.append(target_id)\n", + " \n", + " if len(missing) > 0:\n", + " raise ValueError(\n", + " \"target_workspaces contains workspace ids that are not authorized and enabled: \"\n", + " + \", \".join(missing)\n", + " )\n", + "\n", + " # Filter, preserving original order from authorized_and_enabled_workspaces\n", + " filtered = []\n", + " for ws in authorized_and_enabled_workspaces:\n", + " ws_id = ws.get(\"id\")\n", + " for target_id in target_workspaces:\n", + " if ws_id == target_id:\n", + " filtered.append(ws)\n", + " break\n", + " \n", + " return filtered\n", + "\n", + " def get_target_workspace_ids(self):\n", + " return [ws['id'] for ws in self.target_workspaces]\n", + "\n", + " def get_utilization_day(self):\n", + " group_by = 'DAY'\n", + " time_interval_generator = TimeIntervalGenerator(self.start_date,\n", + " self.end_date,\n", + " group_by,\n", + " None,\n", + " policy_start_time,\n", + " policy_end_time)\n", + " intervals = time_interval_generator.get_intervals()\n", + " \n", + " dates = [str(interval.start).split('T')[0] for interval in intervals]\n", + " \n", + " return intervals, dates[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c9e324-bacf-482f-af25-171c63753f73", + "metadata": {}, + "outputs": [], + "source": [ + "cfg = JobConfig(\n", + " target_date=target_date,\n", + " target_workspaces=target_workspaces,\n", + " tag_chunk_size=tag_chunk_size,\n", + " delay_between_tag_batches=delay_between_tag_batches,\n", + " debug_tag_prefix=debug_tag_prefix,\n", + " policy_start_time=policy_start_time,\n", + " policy_end_time=policy_end_time,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cc691db-9b65-4fe9-85fb-70a2b151d022", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def adjust_datetime(date_time_str, whp_start_time):\n", + " date_time = datetime.strptime(date_time_str, \"%Y-%m-%d\")\n", + " year = date_time.year\n", + " month = date_time.month\n", + " day = date_time.day\n", + " start_time = datetime.strptime(whp_start_time, '%H:%M').time()\n", + " adjusted_timezone = timezone(timedelta(hours=0))\n", + " target_datetime = datetime(year, month, day, start_time.hour, start_time.minute, start_time.second, tzinfo=adjusted_timezone)\n", + " iso_utc_datetime = target_datetime.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')\n", + " return iso_utc_datetime" + ] + }, + { + "cell_type": "markdown", + "id": "8f9b13d8-84c6-4410-9408-2415f914e6cf", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Implementation of compute_utilization function (do not touch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7624277e-22e1-4cb2-b78a-5c702034bebd", + "metadata": { + "editable": true, + "jupyter": { + "source_hidden": true + }, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#Just delete this part when SDK will be ready\n", + "\n", + "from systemlink.httputilizationalgorithm import date_time_helper, dataframe_utilization_algorithm\n", + "from systemlink.clients.niapm import ApiClient\n", + "import pandas as pd\n", + "\n", + "async def compute_utilization( # pylint: disable=too-many-arguments\n", + " utilization_filter: str,\n", + " asset_filter: str,\n", + " intervals: List[date_time_helper.Interval],\n", + " grouping_criteria: str,\n", + " null_group_value: Any,\n", + " startTime: str = None,\n", + " endTime: str = None,\n", + " hash_interval: Callable[[date_time_helper.Interval], Any] = lambda interval: interval.start.day) -> Tuple[Dict[Tuple[Any, Any], float], List[str]]: # pylint: disable=line-too-long\n", + " \"\"\"\n", + " Gets utilization data from the Asset Management service\n", + " and computed the utilization percentages for the passed in criteria.\n", + " \n", + " :param utilization_filter: The filter to be used when querying the utilization history\n", + " :type utilization_filter: str\n", + " :param asset_filter: The filter that determines what assets should be taken\n", + " into consideration when computing the utilization.\n", + " :type asset_filter: str\n", + " :param intervals: The intervals used to compute the utilization.\n", + " :type intervals: List[date_time_helper.Interval]\n", + " :param grouping_criteria: The name of a column from the input dataframe.\n", + " :type grouping_criteria: str\n", + " :param null_group_value: The value used to fill null like values for the\n", + " column used for grouping.\n", + " :type null_group_value: Any\n", + " :param hash_interval: Function that determines the time granularity.\n", + " Takes in a date_time_helper.Interval\n", + " E.g: If the results should be grouped by months, this function\n", + " should return the same value for any interval in the same month.\n", + " :type hash_interval: Callable[[date_time_helper.Interval], Any])\n", + " :return: A tuple with the computed utilization and the list of unused asset identifiers.\n", + " :rtype: Tuple[Dict[Tuple[Any, Any], float], List[str]]\n", + " \"\"\"\n", + " heartbeat_interval_minutes = 10\n", + " \n", + " raw_utilization = []\n", + " async for chunk in query_raw_utilization(utilization_filter, asset_filter, startTime, endTime):\n", + " raw_utilization.extend(chunk)\n", + " \n", + " asset_ids = await _query_asset_ids(asset_filter)\n", + " if not raw_utilization:\n", + " return {}, asset_ids\n", + " \n", + " if pd.__version__ >= \"2.0.0\":\n", + " utilization_df = pd.DataFrame([u.to_dict() for u in raw_utilization], dtype=(str)) # pylint: disable=line-too-long\n", + " for x in ['start_timestamp', 'end_timestamp', 'heartbeat_timestamp']:\n", + " utilization_df[x] = pd.to_datetime(utilization_df[x], utc=True, format=\"ISO8601\").dt.tz_localize(None)\n", + " else:\n", + " utilization_df = pd.DataFrame(\n", + " [u.to_dict() for u in raw_utilization], dtype=(\"datetime64[ns]\")\n", + " ) # pylint: disable=line-too-long\n", + " \n", + " utilization_df[grouping_criteria].fillna(null_group_value, inplace=True)\n", + " \n", + " computed_utilization, used_asset_ids = dataframe_utilization_algorithm.compute(\n", + " utilization_df,\n", + " intervals,\n", + " grouping_criteria,\n", + " hash_interval,\n", + " heartbeat_interval_minutes)\n", + " \n", + " unused_asset_ids = list(set(asset_ids) - set(used_asset_ids))\n", + " \n", + " return computed_utilization, unused_asset_ids\n", + " \n", + " \n", + "async def _query_asset_ids(asset_filter: str) -> List[str]:\n", + " async with ApiClient() as api_client:\n", + " assets_api = AssetsApi(api_client=api_client)\n", + " query_assets_request = QueryAssetsRequest(filter=asset_filter, take=-1)\n", + " query_assets_response = await assets_api.query_assets(query_assets=query_assets_request)\n", + " return [asset.id for asset in query_assets_response.assets]\n", + " \n", + " \n", + "async def query_raw_utilization(\n", + " utilization_filter: str,\n", + " asset_filter: str,\n", + " startTime: str,\n", + " endTime: str,\n", + " take: int = 10000\n", + ") -> List[object]:\n", + " \"\"\"\n", + " Queries raw utilization data.\n", + " \n", + " :param utilization_filter: The filter to be used when querying the utilization history\n", + " :type utilization_filter: str\n", + " :param asset_filter: The filter that determines what assets should be taken\n", + " into consideration when computing the utilization.\n", + " :type asset_filter: str\n", + " :param startTime: The start time for the query in ISO 8601 format.\n", + " :type startTime: str\n", + " :param endTime: The end time for the query in ISO 8601 format.\n", + " :type endTime: str\n", + " :param take: The maximum number of entries to get in one request. Maximum is 1000.\n", + " :type take: int = 1000\n", + " :return: A generator yielding chunks of utilization entries.\n", + " :rtype: List[systemlink.assetmgmtclient.messages.UtilizationHistoryModel]\n", + " \"\"\"\n", + " returned_assets_count = take\n", + " async with ApiClient() as api_client:\n", + " utilization_api = UtilizationApi(api_client=api_client)\n", + " \n", + " request = {\n", + " \"utilizationFilter\": utilization_filter,\n", + " \"assetFilter\": asset_filter,\n", + " \"continuationToken\": \"\",\n", + " \"orderBy\": \"START_TIMESTAMP\",\n", + " \"startTime\": startTime,\n", + " \"endTime\": endTime,\n", + " \"take\": take,\n", + " }\n", + " \n", + " while returned_assets_count == take:\n", + " query_utilization_response = await utilization_api.query_asset_utilizations(\n", + " query_body=request\n", + " ) # pylint: disable=line-too-long\n", + " utilizations = query_utilization_response.asset_utilizations\n", + " if not utilizations:\n", + " return\n", + " yield utilizations\n", + " returned_assets_count = len(utilizations)\n", + " request['continuationToken'] = query_utilization_response.continuation_token" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd99b014-213e-45b8-b548-f183526b9939", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def backfill_unused_assets_utilization(grouped_utilization, unused_asset_identifiers, date):\n", + " filled = dict(grouped_utilization)\n", + " all_asset_ids = set(asset_id for asset_id, _ in grouped_utilization.keys())\n", + " all_asset_ids.update(unused_asset_identifiers)\n", + " for asset_id in all_asset_ids:\n", + " key = (asset_id, date)\n", + " if key not in filled:\n", + " filled[key] = {\n", + " 'percentage': 0.0,\n", + " 'grouping_value': asset_id,\n", + " 'interval_hash': date\n", + " }\n", + "\n", + " return filled" + ] + }, + { + "cell_type": "markdown", + "id": "64b0558f-05ff-4238-aa82-c484bc42ad0f", + "metadata": {}, + "source": [ + "## Ensure tags exists" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88e6191a-ca1c-4ffb-a372-8ac8fe53655c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "async def ensure_tags_exists(tags_metadata=[]):\n", + " update_tags_endpoint = f\"{http_url}/nitag/v2/update-tags\"\n", + " tag_template = {\n", + " \"metadata\": {\n", + " \"properties\": {\n", + " \"nitagRetention\": \"PERMANENT\",\n", + " \"units\": \"%\"\n", + " }\n", + " }\n", + " }\n", + " tags = []\n", + " for metadata in tags_metadata:\n", + " merged_properties = {\n", + " **tag_template[\"metadata\"][\"properties\"],\n", + " **metadata.get(\"properties\", {})\n", + " }\n", + " new_tag = {\n", + " \"path\": metadata[\"path\"],\n", + " \"type\": \"DOUBLE\",\n", + " \"collectAggregates\": True,\n", + " \"properties\": merged_properties,\n", + " \"workspace\": metadata[\"properties\"][\"workspace\"],\n", + " }\n", + " tags.append(new_tag)\n", + " chunked_tags = [tags[i:i + cfg.tag_chunk_size] for i in range(0, len(tags), cfg.tag_chunk_size)]\n", + " payload_template = {\"tags\": [], \"merge\": True}\n", + " for chunk in chunked_tags:\n", + " payload = deepcopy(payload_template)\n", + " payload[\"tags\"].extend(chunk)\n", + " try:\n", + " response = session.post(url=update_tags_endpoint, json=payload)\n", + " response.raise_for_status()\n", + " except requests.exceptions.RequestException as e:\n", + " print(\"Request failed:\", repr(e))\n", + " # Sometimes requests keeps a response on the exception:\n", + " resp = getattr(e, \"response\", None)\n", + " if resp is not None:\n", + " print(\"final status:\", resp.status_code)\n", + " print(\"final body:\", resp.text[:2000])\n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "cc45187d-fd1b-4e28-9d75-012b700963f6", + "metadata": {}, + "source": [ + "## Functions to generate filter string" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18d63a6f-783e-4b49-9389-568d043712f6", + "metadata": {}, + "outputs": [], + "source": [ + "def create_asset_filter(assets, batch_size=500):\n", + " return [\n", + " \"(\" + \" OR \".join(f'AssetIdentifier = \"{asset['id']}\"' for asset in assets[i:i+batch_size]) + \")\"\n", + " for i in range(0, len(assets), batch_size)\n", + " ]\n", + "\n", + "def create_minion_filter(systems, workspaces=[]):\n", + " minion_ids = [f'Location.MinionId = \"{minion[\"id\"]}\"' for minion in systems]\n", + " filter_str = f'({\" OR \".join(minion_ids)}) AND IsSystemController = True'\n", + " if len(workspaces) == 0:\n", + " return filter_str\n", + " else:\n", + " workspace_ids = [f'workspace = \\\"{workspace}\\\"' for workspace in workspaces]\n", + " return f\"{filter_str} AND ({\" OR \".join(workspace_ids)})\"" + ] + }, + { + "cell_type": "markdown", + "id": "d4c1a1da-5169-45eb-8704-293e74e237c9", + "metadata": {}, + "source": [ + "## Fetch paginated data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "621fb38d-1d65-4b65-8de2-e2a905d9baf7", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "async def fetch_all_assets(query_request, skip=0, take=-1):\n", + " assets_api = AssetsApi()\n", + " query_request.skip = skip\n", + " query_request.take = take\n", + " \n", + " response = await assets_api.query_assets(query_assets=query_request)\n", + " \n", + " await assets_api.api_client.close()\n", + " return response.to_dict()['assets']\n", + " # return getattr(response, 'assets')\n", + " \n", + "async def fetch_all_systems(query_request, skip=0, take=1000):\n", + " system_api = SystemsApi()\n", + " items = []\n", + " query_request.skip = skip\n", + " query_request.take = take\n", + " \n", + " response = await system_api.get_systems_by_query(query=query_request)\n", + " items.extend(getattr(response, 'data'))\n", + " \n", + " \n", + " while response.count == take:\n", + " skip += take\n", + " query_request.skip = skip\n", + " response = await system_api.get_systems_by_query(query=query_request)\n", + " items.extend(getattr(response, 'data'))\n", + " \n", + " await system_api.api_client.close()\n", + " return items" + ] + }, + { + "cell_type": "markdown", + "id": "731b6103-b836-4326-bebe-4ae1c046e3a5", + "metadata": {}, + "source": [ + "## Aggregate different categories utilization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33d18c22-aaca-4292-b1b5-4a2ea32648de", + "metadata": {}, + "outputs": [], + "source": [ + "def aggregateCategoryUtilization(input_data):\n", + " # Dictionary to accumulate objects by assetIdentifier\n", + " grouped_data = defaultdict(list)\n", + " \n", + " # Separate objects with and without \"category\" field\n", + " non_category_items = [item for item in input_data if \"category\" not in item]\n", + " category_items = [item for item in input_data if \"category\" in item]\n", + " \n", + " # Group category items by assetIdentifier\n", + " for item in category_items:\n", + " grouped_data[item['assetIdentifier']].append(item)\n", + " \n", + " # Consolidate duplicate entries\n", + " consolidated_items = []\n", + " for asset_id, items in grouped_data.items():\n", + " if len(items) > 1:\n", + " # Calculate mean percentage for duplicates\n", + " mean_percentage = sum(item['percentage'] for item in items) / len(items)\n", + " # Create a new consolidated object\n", + " consolidated_item = items[0].copy()\n", + " consolidated_item['percentage'] = mean_percentage\n", + " del consolidated_item['category']\n", + " consolidated_items.append(consolidated_item)\n", + " else:\n", + " # No duplicates, keep the single item as is\n", + " consolidated_items.append(items[0])\n", + " \n", + " # Combine non-category items and consolidated category items\n", + " result = non_category_items + consolidated_items\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bebcf8ca-7063-4514-a2df-4547395db60f", + "metadata": {}, + "outputs": [], + "source": [ + "async def patch_virtual_system_utilization(tags_metadata, date_str):\n", + " tags_api = TagsApi()\n", + " await ensure_tags_exists(tags_metadata=tags_metadata)\n", + " \n", + " timestamp_adjusted_by_timezone = adjust_datetime(\n", + " date_time_str=date_str,\n", + " whp_start_time=cfg.policy_start_time\n", + " )\n", + " tag_value = TagValue(value=str(0), type=TagType.DOUBLE)\n", + " timestamped_tag_value = TimestampedTagValue(timestamp=timestamp_adjusted_by_timezone, value=tag_value)\n", + "\n", + " controlled_systems_tag_updates = []\n", + " for metadata in tags_metadata:\n", + " tag_update = TagUpdate(\n", + " path= metadata['path'], \n", + " updates=[timestamped_tag_value], \n", + " workspace=metadata['properties']['workspace']\n", + " )\n", + " controlled_systems_tag_updates.append(tag_update)\n", + " chunked_tag_updates = [controlled_systems_tag_updates[i:i + cfg.tag_chunk_size] for i in range(0, len(controlled_systems_tag_updates), cfg.tag_chunk_size)]\n", + " for tag_updates in chunked_tag_updates:\n", + " await tags_api.update_tag_current_values(updates=tag_updates)\n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " await tags_api.api_client.close()" + ] + }, + { + "cell_type": "markdown", + "id": "07e1ad83-7660-4be9-8779-451a995cbd8d", + "metadata": {}, + "source": [ + "## Is valid tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "708f501f-214c-42b1-a27f-ada134391885", + "metadata": {}, + "outputs": [], + "source": [ + "def is_valid_tag_name(name):\n", + " pattern = r'^[A-Za-z0-9\\-\\.\\s\\/\\\\()_\\+\\[\\]{}\\?~&\\*,\"\\'`=‑\\:]+$'\n", + " is_valid = bool(re.match(pattern, name))\n", + " return is_valid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b172e04d-d600-482b-8839-5a8912c6f824", + "metadata": {}, + "outputs": [], + "source": [ + "def get_utilization_settings(utilization: str):\n", + " u = (utilization or \"\").strip().lower()\n", + "\n", + " if u == \"configuration\":\n", + " return 'Category == \"Configuration\" or Category == \"configuration\"', \"Configuration\"\n", + " if u == \"test\":\n", + " return 'Category == \"Test\" or Category == \"test\"', \"Test\"\n", + "\n", + " return \"\", \"All\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "484f68c9-c055-40e2-886e-560d24622699", + "metadata": {}, + "outputs": [], + "source": [ + "def get_vendor_model_serial_number_settings(asset: Dict):\n", + " vendor = asset['vendor_name']\n", + " if vendor is None or vendor == \"\":\n", + " vendor = str(asset['vendor_number'])\n", + " model = asset['model_name']\n", + " if model is None or model == \"\":\n", + " model = str(asset['model_number'])\n", + " serial_number = str(asset['serial_number'])\n", + "\n", + " return vendor, model, serial_number" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd3dde69-cdae-4934-971e-fae8af562f62", + "metadata": {}, + "outputs": [], + "source": [ + "def group_systems_by_workspace(systems):\n", + " grouped = defaultdict(list)\n", + " for system in systems:\n", + " workspace_id = system.get(\"workspace\")\n", + " grouped[workspace_id].append(system)\n", + " return dict(grouped)" + ] + }, + { + "cell_type": "markdown", + "id": "c5ef7b22-37e1-4605-8296-8aae77aed708", + "metadata": {}, + "source": [ + "## Build workspace_system_map" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffb30f1f-5eaa-4798-9aab-97c7f6e48c3a", + "metadata": {}, + "outputs": [], + "source": [ + "def split_and_enrich_for_workspace(ws_systems, ws_controllers):\n", + " system_id_to_alias = {}\n", + " for system in ws_systems:\n", + " system_id = system.get(\"id\", system.get(\"system_id\"))\n", + " if system_id is not None:\n", + " system_id_to_alias[system_id] = system.get(\"alias\")\n", + "\n", + " controller_minion_ids = set()\n", + " for c in ws_controllers:\n", + " minion_id = c[\"location\"].get(\"minion_id\")\n", + " if minion_id is not None:\n", + " controller_minion_ids.add(minion_id)\n", + "\n", + " controllers_with_alias = []\n", + " for c in ws_controllers:\n", + " c2 = deepcopy(c)\n", + " minion_id = c2[\"location\"].get(\"minion_id\")\n", + " if minion_id in system_id_to_alias:\n", + " c2[\"alias\"] = system_id_to_alias[minion_id]\n", + " controllers_with_alias.append(c2)\n", + "\n", + " virtual_systems = []\n", + " for system in ws_systems:\n", + " system_id = system.get(\"id\", system.get(\"system_id\"))\n", + " if system_id not in controller_minion_ids:\n", + " virtual_systems.append(deepcopy(system))\n", + "\n", + " return controllers_with_alias, virtual_systems\n", + "\n", + "def build_workspace_system_map(systems, system_controllers):\n", + " systems_by_ws = defaultdict(list)\n", + " controllers_by_ws = defaultdict(list)\n", + "\n", + " for system in systems:\n", + " ws = system.get(\"workspace\")\n", + " if ws is not None:\n", + " systems_by_ws[ws].append(system)\n", + "\n", + " for controller in system_controllers:\n", + " ws = controller.get(\"workspace\")\n", + " if ws is not None:\n", + " controllers_by_ws[ws].append(controller)\n", + "\n", + " _workspace_system_map = {}\n", + "\n", + " all_workspaces = set(systems_by_ws.keys()) | set(controllers_by_ws.keys())\n", + " for ws in all_workspaces:\n", + " controllers_with_alias, virtual_systems = split_and_enrich_for_workspace(\n", + " systems_by_ws.get(ws, []),\n", + " controllers_by_ws.get(ws, []),\n", + " )\n", + " _workspace_system_map[ws] = {\n", + " \"controllers_with_alias\": controllers_with_alias,\n", + " \"virtual_systems\": virtual_systems,\n", + " }\n", + "\n", + " return _workspace_system_map" + ] + }, + { + "cell_type": "markdown", + "id": "e3986169-d68a-465a-9849-dfc48e6818bd", + "metadata": {}, + "source": [ + "## Asset Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8182e5c-cf36-4f8a-887d-54c57800acf7", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "async def update_asset_utilization_tags(utilization=None, assets=[]):\n", + " utilization_filter, category_prefix_for_tag = get_utilization_settings(utilization)\n", + " \n", + " tags_api = TagsApi()\n", + " if assets:\n", + " asset_id_to_asset_map = {}\n", + " for asset in assets:\n", + " asset_id_to_asset_map[asset['id']] = asset\n", + " \n", + " # ensure that tags are created for all assets\n", + " asset_tags = []\n", + " for asset in assets:\n", + " location = asset['location']['minion_id'] if asset['location']['minion_id'] else asset['location']['physical_location']\n", + " properties = {'name': str(asset['name']),\n", + " 'id': str(asset['id']),\n", + " # 'minion_id': str(asset['location']['minion_id']),\n", + " 'asset_type': str(asset['asset_type']),\n", + " 'workspace': str(asset['workspace']),\n", + " 'displayName': f\"{asset['name']} - Utilization Daily Peak\"}\n", + " \n", + " vendor, model, serial_number = get_vendor_model_serial_number_settings(asset)\n", + " \n", + " if not is_valid_tag_name(model) or not is_valid_tag_name(asset['serial_number']) or not is_valid_tag_name(vendor):\n", + " print (f\"Invalid asset {asset['id']} {asset['serial_number']} {model} {vendor}\")\n", + " continue \n", + " tag_path = f\"Assets.{vendor}.{model}.{serial_number}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\"\n", + " asset_tags.append({'path': tag_path, 'properties': properties})\n", + " await ensure_tags_exists(asset_tags)\n", + " \n", + " \n", + " asset_filters_list = create_asset_filter(assets)\n", + " tag_updates = []\n", + " for asset_filter in asset_filters_list:\n", + " grouped_utilization, unused_asset_identifiers = await compute_utilization(\n", + " utilization_filter,\n", + " asset_filter,\n", + " cfg.utilization_intervals,\n", + " cfg.group_by_value,\n", + " cfg.property_not_available_string,\n", + " cfg.start_date,\n", + " cfg.end_date,\n", + " cfg.interval_group_function)\n", + " filled_utilization = backfill_unused_assets_utilization(\n", + " grouped_utilization=grouped_utilization, \n", + " unused_asset_identifiers=unused_asset_identifiers,\n", + " date=cfg.utilization_for_date\n", + " )\n", + " for utilization in filled_utilization:\n", + " asset_id = utilization[0]\n", + " date_str = utilization[1]\n", + " timestamp_adjusted_by_timezone = adjust_datetime(\n", + " date_time_str=date_str, \n", + " whp_start_time=cfg.policy_start_time\n", + " )\n", + " asset = asset_id_to_asset_map[asset_id]\n", + " \n", + " vendor, model, serial_number = get_vendor_model_serial_number_settings(asset)\n", + " \n", + " path= f\"Assets.{vendor}.{model}.{serial_number}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\"\n", + " tag_value = TagValue(value=str(filled_utilization[utilization]['percentage']), type=TagType.DOUBLE)\n", + " timestamped_tag_value = TimestampedTagValue(timestamp=timestamp_adjusted_by_timezone, value=tag_value)\n", + " tag_update = TagUpdate(path= path,\n", + " updates=[timestamped_tag_value], \n", + " workspace=asset['workspace'])\n", + " tag_updates.append(tag_update)\n", + " chunked_tag_updates = [tag_updates[i:i + cfg.tag_chunk_size] for i in range(0, len(tag_updates), cfg.tag_chunk_size)]\n", + " for tag_updates in chunked_tag_updates:\n", + " await tags_api.update_tag_current_values(updates=tag_updates)\n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " await tags_api.api_client.close()" + ] + }, + { + "cell_type": "markdown", + "id": "8d855292-00d2-43e3-911b-ffbc05f9d323", + "metadata": {}, + "source": [ + "## System and Workspace Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86371429-64b1-4d67-acc4-5152f84034a8", + "metadata": {}, + "outputs": [], + "source": [ + "async def update_system_and_workspace_utilization_tags(utilization=None, controllers_by_workspace_id={}):\n", + " utilization_filter, category_prefix_for_tag = get_utilization_settings(utilization)\n", + "\n", + " tags_api = TagsApi()\n", + " workspace_tags = []\n", + " workspace_tag_updates = []\n", + " for ws_id in controllers_by_workspace_id:\n", + " \n", + " number_of_uncontrolled_systems_in_workspace = 0\n", + " workspace_utilization = []\n", + " utilization_of_systems_in_ws = []\n", + " sys_controllers = controllers_by_workspace_id[ws_id][\"controllers_with_alias\"]\n", + " virtual_systems = controllers_by_workspace_id[ws_id][\"virtual_systems\"]\n", + "\n", + " if not len(sys_controllers):\n", + " workspace_utilization.append(0)\n", + " system_tags_metadata = []\n", + " virtual_system_tags_metadata = []\n", + " # system controllers\n", + " for sys_controller in sys_controllers:\n", + " alias = sys_controller.get(\"alias\", \"\")\n", + " minion_id = sys_controller['location'].get(\"minion_id\", \"\")\n", + " properties = {\n", + " \"name\": alias,\n", + " \"asset_identifier\": sys_controller[\"id\"],\n", + " \"minion_id\": minion_id,\n", + " \"workspace\": str(sys_controller[\"workspace\"]),\n", + " \"displayName\": f\"{alias} - Utilization Daily All\"\n", + " }\n", + " tag_path = f\"{minion_id}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\"\n", + " # system_tags_metadata.append({\"path\": tag_path, \"properties\": properties})\n", + "\n", + " # virtual system\n", + " number_of_uncontrolled_systems_in_workspace = len(virtual_systems)\n", + " for virtual_system in virtual_systems:\n", + " minion_id = virtual_system.get(\"id\", \"\")\n", + " v_system_alias = virtual_system.get(\"alias\", \"\")\n", + " properties = {\n", + " \"name\": v_system_alias,\n", + " \"asset_identifier\": \"\",\n", + " \"minion_id\": minion_id,\n", + " \"workspace\": virtual_system[\"workspace\"],\n", + " \"displayName\": f\"{v_system_alias} - Utilization Daily All\"\n", + " }\n", + "\n", + " tag_path = f\"{minion_id}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\"\n", + " virtual_system_tags_metadata.append({\"path\": tag_path, \"properties\": properties})\n", + "\n", + " await patch_virtual_system_utilization(tags_metadata=virtual_system_tags_metadata, date_str=cfg.utilization_for_date)\n", + " await ensure_tags_exists(tags_metadata=system_tags_metadata)\n", + "\n", + " sys_controllers_filters_list = create_asset_filter(sys_controllers)\n", + " sys_controller_id_to_sys_controller_map = {}\n", + " for sys_controller in sys_controllers:\n", + " sys_controller_id_to_sys_controller_map[sys_controller['id']] = sys_controller\n", + "\n", + " tag_updates = []\n", + " for sys_controller_filter in sys_controllers_filters_list:\n", + " grouped_utilization, unused_asset_identifiers = await compute_utilization(\n", + " utilization_filter,\n", + " sys_controller_filter,\n", + " cfg.utilization_intervals,\n", + " cfg.group_by_value,\n", + " cfg.property_not_available_string,\n", + " cfg.start_date,\n", + " cfg.end_date,\n", + " cfg.interval_group_function)\n", + " filled_utilization = backfill_unused_assets_utilization(\n", + " grouped_utilization,\n", + " unused_asset_identifiers,\n", + " cfg.utilization_for_date\n", + " )\n", + " for utilization in filled_utilization:\n", + " system_id = utilization[0]\n", + " date_str = utilization[1]\n", + " timestamp_adjusted_by_timezone = adjust_datetime(\n", + " date_time_str=date_str,\n", + " whp_start_time=cfg.policy_start_time\n", + " )\n", + " sys_controller = sys_controller_id_to_sys_controller_map[system_id]\n", + " tag_value = TagValue(value=str(filled_utilization[utilization]['percentage']), type=TagType.DOUBLE)\n", + " timestamped_tag_value = TimestampedTagValue(timestamp=timestamp_adjusted_by_timezone, value=tag_value)\n", + " path = f\"{sys_controller['location']['minion_id']}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\"\n", + "\n", + " tag_update = TagUpdate(path=path,\n", + " updates=[timestamped_tag_value],\n", + " workspace=sys_controller['workspace'])\n", + " tag_updates.append(tag_update)\n", + " workspace_utilization.append(filled_utilization[utilization]['percentage'])\n", + " workspace_utilization.extend([0] * number_of_uncontrolled_systems_in_workspace)\n", + " chunked_tag_updates = [tag_updates[i:i + cfg.tag_chunk_size] for i in range(0, len(tag_updates), cfg.tag_chunk_size)]\n", + " for tag_updates in chunked_tag_updates:\n", + " await tags_api.update_tag_current_values(updates=tag_updates)\n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " workspace_tag = {\n", + " 'path': f\"Workspace.{ws_id}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\",\n", + " 'properties': {\n", + " 'workspace': ws_id,\n", + " 'displayName': f\"{translate_ids_to_names([ws_id], api_key)[0]} - Utilization Daily All\"\n", + " }}\n", + " workspace_tags.append(workspace_tag)\n", + " timestamp_adjusted_by_timezone = adjust_datetime(\n", + " date_time_str=cfg.utilization_for_date,\n", + " whp_start_time=cfg.policy_start_time\n", + " )\n", + " ws_utilization_precentage = sum(workspace_utilization) / len(workspace_utilization)\n", + " tag_value = TagValue(value=str(ws_utilization_precentage), type=TagType.DOUBLE)\n", + " timestamped_tag_value = TimestampedTagValue(timestamp=timestamp_adjusted_by_timezone, value=tag_value)\n", + " tag_update = TagUpdate(path=f\"Workspace.{ws_id}.{cfg.tag_middle_segment}.Daily.All.{category_prefix_for_tag}\",\n", + " updates=[timestamped_tag_value], workspace=ws_id)\n", + " workspace_tag_updates.append(tag_update)\n", + " \n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " await ensure_tags_exists(workspace_tags)\n", + " \n", + " chunked_workspace_tags = [workspace_tag_updates[i:i + cfg.tag_chunk_size] for i in range(0, len(workspace_tag_updates), cfg.tag_chunk_size)]\n", + " for workspace_tag_updates in chunked_workspace_tags:\n", + " await tags_api.update_tag_current_values(updates=workspace_tag_updates)\n", + " await asyncio.sleep(cfg.delay_between_tag_batches)\n", + " await tags_api.api_client.close()" + ] + }, + { + "cell_type": "markdown", + "id": "2a7b37d4-05b6-46f9-b75b-1a615d69d9d9", + "metadata": {}, + "source": [ + "## Get assets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a79b2e7-b4bd-4920-a88e-633d2e2a6826", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "assets_filter = \" OR \".join(f'workspace = \"{wid}\"' for wid in cfg.get_target_workspace_ids())\n", + "query_assets_request = QueryAssetsRequest(\n", + " filter=assets_filter\n", + ")\n", + "all_assets = await fetch_all_assets(\n", + " query_request=query_assets_request,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c84e1e4a-339d-4480-b865-60afc9996e88", + "metadata": {}, + "source": [ + "## Push asset utilization to tags" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62a45f8a-4d48-45c6-88fc-02b42a177de2", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "await update_asset_utilization_tags(assets=all_assets)\n", + "await update_asset_utilization_tags(assets=all_assets, utilization=\"configuration\")" + ] + }, + { + "cell_type": "markdown", + "id": "754a7337-6543-4e58-b39b-75d495c06270", + "metadata": {}, + "source": [ + "### Get systems" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "648e1b64-b2fe-4549-bb0b-29ee5d5f9849", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "systems_filter = \" OR \".join(f'workspace = \"{wid}\"' for wid in cfg.get_target_workspace_ids())\n", + "\n", + "query_systems_request = QuerySystemsRequest(\n", + " projection='new(id,alias,workspace)',\n", + " filter=systems_filter\n", + ")\n", + "\n", + "all_systems = await fetch_all_systems(\n", + " query_request=query_systems_request,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "eecf3703-0f72-40d9-a7a2-65fd5f036512", + "metadata": {}, + "source": [ + "### Get system controllers(assets)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30fd4f8a-acc4-42f0-b511-6424e75c54e7", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "system_controller_filter_string = create_minion_filter(all_systems, workspaces=cfg.get_target_workspace_ids())\n", + "\n", + "query_system_controller_request = QueryAssetsRequest(skip=0, take=1000, filter=system_controller_filter_string)\n", + "all_system_controllers = await fetch_all_assets(query_system_controller_request)" + ] + }, + { + "cell_type": "markdown", + "id": "d1dee0be-03a8-4273-b7cc-1452810e48e6", + "metadata": {}, + "source": [ + "### Prepare workspace id to controllers map" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "545f4029-f6ea-4e0b-bf71-b06f84705da7", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "workspace_system_map = build_workspace_system_map(all_systems, all_system_controllers)" + ] + }, + { + "cell_type": "markdown", + "id": "6604c734-d065-4f40-8e6a-7f81ab088dd6", + "metadata": {}, + "source": [ + "### Push workspace and systems utilization to tags" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e08aa8a-5e4d-4d0c-a557-e7f6dca22ccb", + "metadata": {}, + "outputs": [], + "source": [ + "await update_system_and_workspace_utilization_tags(controllers_by_workspace_id=workspace_system_map)\n", + "await update_system_and_workspace_utilization_tags(controllers_by_workspace_id=workspace_system_map, utilization='configuration')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}