Skip to content

Commit 0aecb45

Browse files
mmguerommguero
committed
for cisagov#401, create an API for exporting dashboards
1 parent 4a77400 commit 0aecb45

File tree

4 files changed

+180
-28
lines changed

4 files changed

+180
-28
lines changed

‎api/project/__init__.py

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1062,6 +1062,96 @@ def ready():
10621062
)
10631063

10641064

1065+
@app.route(
1066+
f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/dashboard-export/<dashid>",
1067+
methods=['GET', 'POST'],
1068+
)
1069+
def dashboard_export(dashid):
1070+
"""Uses the opensearch dashboards API to export a dashboard. Also handles the _REPLACER strings
1071+
as described in "Adding new visualizations and dashboards" at
1072+
https://idaholab.github.io/Malcolm/docs/contributing-dashboards.html#DashboardsNewViz
1073+
1074+
Parameters
1075+
----------
1076+
dashid : string
1077+
the ID of the dashboard to export
1078+
request : Request
1079+
Uses 'replace' from requests arguments, true (default) or false; indicates whether or not to do
1080+
MALCOLM_NETWORK_INDEX_PATTERN_REPLACER, MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER,
1081+
MALCOLM_OTHER_INDEX_PATTERN_REPLACER
1082+
1083+
Returns
1084+
-------
1085+
content
1086+
The JSON of the exported dashboard
1087+
"""
1088+
1089+
args = get_request_arguments(request)
1090+
try:
1091+
# call the API to get the dashboard JSON
1092+
response = requests.get(
1093+
f"{dashboardsUrl}/api/{'kibana' if (databaseMode == malcolm_utils.DatabaseMode.ElasticsearchRemote) else 'opensearch-dashboards'}/dashboards/export",
1094+
params={
1095+
'dashboard': dashid,
1096+
},
1097+
auth=opensearchReqHttpAuth,
1098+
verify=opensearchSslVerify,
1099+
)
1100+
response.raise_for_status()
1101+
1102+
if doReplacers := malcolm_utils.str2bool(args.get('replace', 'true')):
1103+
# replace references to index pattern names with the _REPLACER strings, which will allow other Malcolm
1104+
# instances that use different index pattern names to import them and substitute their own names
1105+
replacements = {
1106+
app.config['MALCOLM_NETWORK_INDEX_PATTERN']: 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER',
1107+
app.config['MALCOLM_NETWORK_INDEX_TIME_FIELD']: 'MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER',
1108+
app.config['MALCOLM_OTHER_INDEX_PATTERN']: 'MALCOLM_OTHER_INDEX_PATTERN_REPLACER',
1109+
}
1110+
pattern = re.compile('|'.join(re.escape(key) for key in replacements))
1111+
responseText = pattern.sub(lambda match: replacements[match.group(0)], response.text)
1112+
else:
1113+
# ... or just return it as-is
1114+
responseText = response.text
1115+
1116+
# remove index pattern definition from exported dashboard as they get created programatically
1117+
# on Malcolm startup and we don't want them to come in with imported dashboards
1118+
if responseParsed := malcolm_utils.LoadStrIfJson(responseText):
1119+
if 'objects' in responseParsed and isinstance(responseParsed['objects'], list):
1120+
responseParsed['objects'] = [
1121+
o
1122+
for o in responseParsed['objects']
1123+
if not (
1124+
(o.get("type") == "index-pattern")
1125+
and (
1126+
o.get("id")
1127+
in [
1128+
(
1129+
"MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
1130+
if doReplacers
1131+
else app.config['MALCOLM_NETWORK_INDEX_PATTERN']
1132+
),
1133+
(
1134+
"MALCOLM_OTHER_INDEX_PATTERN_REPLACER"
1135+
if doReplacers
1136+
else app.config['MALCOLM_OTHER_INDEX_PATTERN']
1137+
),
1138+
]
1139+
)
1140+
)
1141+
]
1142+
return jsonify(responseParsed)
1143+
1144+
else:
1145+
# what we got back from the API wasn't valid JSON, so sad
1146+
return jsonify(error=f'Could not process export response for {dashid}')
1147+
1148+
except Exception as e:
1149+
errStr = f"{type(e).__name__}: {str(e)} exporting OpenSearch Dashboard {dashid}"
1150+
if debugApi:
1151+
print(errStr)
1152+
return jsonify(error=errStr)
1153+
1154+
10651155
@app.route(
10661156
f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/ingest-stats",
10671157
methods=['GET'],

‎docs/api-dashboard-export.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Dashboard Export
2+
3+
`GET` or `POST` - /mapi/dashboard-export/`<dashid>`
4+
5+
Uses the [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/) or [Elastic Kibana](https://www.elastic.co/guide/en/kibana/current/dashboard-api-export.html) API to export the JSON document representing a dashboard (identified by `dashid`). If the query parameter `replace` is not set to `false`, this API will also perform some modifications on the dashboard as described in the [**Adding new visualizations and dashboards**](contributing-dashboards.md#DashboardsNewViz) section of the [contributor guide](contributing-guide.md#Contributing).
6+
7+
Parameters:
8+
9+
* `dashid` (URL parameter) - the [ID of the dashboard]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/dashboards/dashboards/) to be exported (e.g., `0ad3d7c2-3441-485e-9dfe-dbb22e84e576`)
10+
* `replace` (query parameter) - whether or not to perform the index pattern name replacements as described above (default: `true`)
11+
12+
Example (response truncated for brevity's sake):
13+
14+
```
15+
/mapi/dashboard-export/0ad3d7c2-3441-485e-9dfe-dbb22e84e576
16+
```
17+
18+
```json
19+
{
20+
21+
"version": "1.3.1",
22+
"objects": [
23+
{
24+
"id": "0ad3d7c2-3441-485e-9dfe-dbb22e84e576",
25+
"type": "dashboard",
26+
"namespaces": [
27+
"default"
28+
],
29+
"updated_at": "2024-04-29T15:49:16.000Z",
30+
"version": "WzEzNjIsMV0=",
31+
"attributes": {
32+
"title": "Overview"
33+
…
34+
}
35+
```

‎docs/api.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# <a name="API"></a>API
22

3+
* [Dashboard Export](api-dashboard-export.md)
34
* [Document Ingest Statistics](api-ingest-stats.md)
45
* [Document Lookup](api-document-lookup.md)
56
* [Event Logging](api-event-logging.md)

‎docs/contributing-dashboards.md

Lines changed: 54 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,62 @@
44

55
## <a name="DashboardsNewViz"></a>Adding new visualizations and dashboards
66

7-
Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported using the following steps:
7+
Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported either of two ways.
8+
9+
The easier (and preferred) method is to use the [dashboard export API](api-dashboard-export.md), as it handles the replacers (described below in the more complicated method) automatically.:
810

911
1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`)
10-
1. Export the dashboard with that ID and save it in the `./dashboards./dashboards/` directory with the following command:
11-
```
12-
export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
13-
docker compose exec dashboards curl -XGET \
14-
"http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \
15-
./dashboards/dashboards/$DASHID.json
16-
```
17-
1. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it):
18-
```
19-
,
20-
{
21-
"id": "arkime_sessions3-*",
22-
"type": "index-pattern",
23-
"namespaces": [
24-
"default"
25-
],
26-
"updated_at": "2021-12-13T18:21:42.973Z",
27-
"version": "Wzk3MSwxXQ==",
28-
…
29-
"references": [],
30-
"migrationVersion": {
31-
"index-pattern": "7.6.0"
32-
}
33-
}
34-
```
35-
1. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313).
36-
1. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
12+
13+
2. Using a web browser, enter the URL **https://localhost/mapi/dashboard-export/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx**, replacing `localhost` with the IP address or hostname of your Malcolm instance and the placeholder dashboard ID with the ID you identified in the previous step. Save the raw JSON document returned as `./dashboards/dashboards/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` (using the actual ID) under your Malcolm directory.
14+
15+
**OR**
16+
17+
2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command:
18+
19+
```
20+
export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
21+
docker compose exec api curl -sSL -XGET "http://localhost:5000/mapi/dashboard-export/$DASHID" > \
22+
./dashboards/dashboards/$DASHID.json
23+
```
24+
25+
3. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
26+
27+
28+
The manual, more complicated way, consists of the following steps:
29+
30+
1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`)
31+
32+
2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command:
33+
34+
```
35+
export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \
36+
docker compose exec dashboards curl -XGET \
37+
"http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \
38+
./dashboards/dashboards/$DASHID.json
39+
```
40+
41+
3. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it):
42+
43+
```
44+
,
45+
{
46+
"id": "arkime_sessions3-*",
47+
"type": "index-pattern",
48+
"namespaces": [
49+
"default"
50+
],
51+
"updated_at": "2021-12-13T18:21:42.973Z",
52+
"version": "Wzk3MSwxXQ==",
53+
…
54+
"references": [],
55+
"migrationVersion": {
56+
"index-pattern": "7.6.0"
57+
}
58+
}
59+
```
60+
61+
4. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313).
62+
5. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up.
3763

3864
## <a name="DashboardsPlugins"></a>OpenSearch Dashboards plugins
3965

0 commit comments

Comments
 (0)