add kibana dashboard transfer

Author: yonghua-sun

.vitepress/siderbars/getOscompatibleDemo.js

@@ -19,6 +19,7 @@ function getSidebar() {
     {
       text: 'ES兼容',
       items: [
+        { text: 'Kibana Dashboard迁移', link: '/oscompatibledemo/kibana_dashboard_transfer' },
         { text: '最佳实践', link: '/oscompatibledemo/es_best_practice' },
       ],
     },

src/oscompatibledemo/kibana_dashboard_transfer.md

@@ -0,0 +1,119 @@
+# 概要
+
+本文介绍在数据迁移到SLS后，如何将原来Kibana中的dashboard迁移到SLS对接的Kibana中
+
+# 操作思路
+
+![](/img/oscompatibledemo/dashboard10.png)
+
+
+* 从旧的Kibana导出dashboard的export.ndjson文件
+* 新的Kibana中准备好index pattern
+* 把export.ndjson中的旧的index pattern id替换成新的Kibana index patter Id
+* 完成替换后，把export.ndjson导入到新的kibana
+
+
+# Dashboard迁移演示
+
+## 在ES中准备数据和Dashboard, 并导出Dashboard配置
+向es1 写入数据
+
+```
+POST people/_bulk
+{ "index": {} }
+{ "name": "Alice", "age": 30 }
+{ "index": {} }
+{ "name": "Bob", "age": 25 }
+{ "index": {} }
+{ "name": "Charlie", "age": 35 }
+```
+
+在kibana1上基于people创建两个dashboard
+
+![](/img/oscompatibledemo/dashboard0.png)
+
+其中一个dashboard示例
+
+![](/img/oscompatibledemo/dashboard1.png)
+
+保存好Dashboard后，进入Kibana `Stack Management->Saved objects` 来导出
+
+![](/img/oscompatibledemo/dashboard2.png)
+
+勾选想要导出的dashboard，注意 不要勾选 `Include related objects` 
+
+导出后dashboard的内容存在 `export.ndjson` 中
+
+
+## 在SLS对接的Kibana中复用Dashboard
+
+在sls中准备好一份相同的数据，确保字段一致
+
+![](/img/oscompatibledemo/dashboard3.png)
+
+新部署一套Kibana并对接SLS，参考操作链接 -> [SLS对接Kibana](https://help.aliyun.com/zh/sls/user-guide/use-kibana-to-access-the-elasticsearch-compatible-api-of-log-service)
+
+注意，这里用的es/kbiana版本和之前的es/kibana版本保持一致
+
+使用 `Docker Compose`或 `Helm Chart` 方案对接SLS后,会自动在Kibana中创建好相应的Index Pattern
+
+![](/img/oscompatibledemo/dashboard4.png)
+
+
+### 执行迁移
+
+* 查看老的es中的Index Pattern的ID 
+
+准备好 kibana_config_1.json，然后使用  [ptn_list.py 点击下载](ptn_list.py) 来list Index Pattern
+```
+{
+    "url" : "http://xxx:5601",
+    "user" : "elastic",
+    "password" : "",
+    "space" :  "default"
+}
+```
+执行
+```
+➜  /tmp python ptn_list.py kibana_config_1.json
+f06fc2b0-b82d-11ef-88c6-15adf26175c7    people
+```
+
+f06fc2b0-b82d-11ef-88c6-15adf26175c7 这个就是旧的people这个index pattern的id
+
+在Kibana导出的dashboard配置文件 export.ndjson中可以看到对这个Index Pattern Id的引用
+![](/img/oscompatibledemo/dashboard5.png)
+
+
+* 在SLS对接的Kibana中找到新的Index Pattern的Id
+
+同样使用 [ptn_list.py 点击下载](ptn_list.py) 来list 新的Kibana中Index Pattern的ID
+
+```
+# 准备好kibana_config_2.json
+
+➜  /tmp python ptn_list.py kibana_config_2.json
+ef710470-b83a-11ef-bb2b-ad198b7b763d	etl.people
+```
+
+
+通过sed命令批量修改 `export.ndjson` 中的ID替换
+
+```
+sed -i 's/f06fc2b0-b82d-11ef-88c6-15adf26175c7/ef710470-b83a-11ef-bb2b-ad198b7b763d/' export.ndjson
+```
+
+* 进入新的Kibana `Stack Management->Saved objects` 来将export.ndjson导入
+
+![](/img/oscompatibledemo/dashboard6.png)
+
+点击Import
+
+![](/img/oscompatibledemo/dashboard7.png)
+
+出现下面界面表示导入成功，如有报错的情况需要看报错信息，再看如何处理
+
+![](/img/oscompatibledemo/dashboard8.png)
+
+打开新的dashboard是否符合预期
+![](/img/oscompatibledemo/dashboard9.png)

src/oscompatibledemo/ptn_delete.py

@@ -0,0 +1,77 @@
+import sys
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+from datetime import datetime
+
+kibanaEndpoint= ''
+kibanaUser = ''
+kibanaPassword = ''
+kibanaSpace = ''
+
+def print_with_time(message):
+    current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    print(f"[{current_time}] {message}")
+
+def deleteKibanaIndexPattern(space, indexPatternId):
+    uri = f"{kibanaEndpoint}/api/saved_objects/index-pattern/{indexPatternId}"
+
+    if space != "default":
+        uri = f"{kibanaEndpoint}/s/{space}/api/saved_objects/index-pattern/{indexPatternId}"
+    headers = {
+        "kbn-xsrf": "true",
+    }
+    ret = requests.delete(uri,
+                auth=kibanaAuth,
+                headers=headers)
+    if ret.status_code != 200:
+        print_with_time(f"Failed to delete index patterns. " + 
+            f"Status code: {ret.status_code}, body: {ret.content}")
+    else:
+        print_with_time(f"delete indexPattern {indexPatternId} @ {space} success")
+
+def print_help():
+    print('''
+Usage: python %s kibana_config.json index_pattern_ids.txt
+
+kibana_config.json exmplae:
+{
+    "url" : "http://192.168.1.1:5601",
+    "user" : "elastic",
+    "password" : "thePassword",
+    "space" :  "default"
+}
+
+index_pattern_ids.txt example content:
+aa77bf10-9c17-11ef-8665-41f4399ec183	etl-dev.mytest
+5a264770-a0a9-11ef-8279-7f5e6cac2855	etl-spe.es_test3
+d07fc4c0-a0d4-11ef-8279-7f5e6cac2855	etl-spe.es_test5
+
+you can use  kibana_index_pattern_list.py to get index_pattern_ids.txt file
+''' % sys.argv[0])
+indexPatternIds = []
+try:
+    configFile = sys.argv[1]
+    indexPatternIdFile = sys.argv[2]
+    kbnConfig = json.loads(open(configFile, 'r').read())
+
+    kibanaEndpoint=kbnConfig['url']
+    kibanaUser = kbnConfig['user']
+    kibanaPassword = kbnConfig['password']
+    kibanaSpace = kbnConfig['space']
+
+    for i in open(indexPatternIdFile,'r').read().split('\n'):
+        if i.strip()!="":
+            id = i.split('\t')[0].split(' ')[0]
+            indexPatternIds.append(id)
+except Exception as ex:
+    print_help()
+    print(str(ex))
+    sys.exit(1)
+
+
+kibanaAuth = HTTPBasicAuth(kibanaUser, kibanaPassword)
+
+for indexPatternId in indexPatternIds:
+    #print_with_time("start to delete index pattern %s" % indexPatternId)
+    deleteKibanaIndexPattern(kibanaSpace, indexPatternId)

src/oscompatibledemo/ptn_list.py

@@ -0,0 +1,93 @@
+import sys
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+from datetime import datetime
+
+kibanaEndpoint= ''
+kibanaUser = ''
+kibanaPassword = ''
+kibanaSpace = ''
+
+def print_with_time(message):
+    current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    print(f"[{current_time}] {message}")
+    
+def getKibanaExsitIndexPatterns(space):
+    page = 1
+    per_page = 100  # 每页获取100条记录
+    existing_patterns = {}
+    headers = {
+        "kbn-xsrf": "true",
+        "Content-Type": "application/json"
+    }
+    while True:
+        uri = f"{kibanaEndpoint}/api/saved_objects/_find"
+        if space != "default":
+            uri = f"{kibanaEndpoint}/s/{space}/api/saved_objects/_find"
+        response = requests.get(
+            uri,
+            auth=kibanaAuth,
+            headers=headers,
+            params={
+                "type": "index-pattern",
+                "page": page,
+                "per_page": per_page
+            }
+        )
+
+        if response.status_code == 200:
+            response_data = response.json()
+            saved_objects = response_data.get("saved_objects", [])
+
+            if not saved_objects:
+                break  # 如果当前页没有获取到新的记录，停止翻页
+
+            for pattern in saved_objects:
+                existing_patterns[pattern["attributes"]["title"]] = pattern["id"]
+            # existing_patterns.extend([pattern["attributes"]["title"] 
+                                      # for pattern in saved_objects])
+
+            # 检查是否还有更多的记录
+            total = response_data.get("total", 0)
+            if len(existing_patterns) >= total:
+                break
+
+            page += 1  # 下一页
+        else:
+            print_with_time(f"Failed to get existing index patterns. Status code: {response.status_code}")
+            return []
+
+    return existing_patterns
+
+def print_help():
+    print('''
+Usage: python %s kibana_config.json
+
+kibana_config.json exmplae:
+{
+    "url" : "http://192.168.1.1:5601",
+    "user" : "elastic",
+    "password" : "thePassword",
+    "space" :  "default"
+}
+
+''' % sys.argv[0])
+try:
+    configFile = sys.argv[1]
+    kbnConfig = json.loads(open(configFile, 'r').read())
+    
+    kibanaEndpoint=kbnConfig['url']
+    kibanaUser = kbnConfig['user']
+    kibanaPassword = kbnConfig['password']
+    kibanaSpace = kbnConfig['space']
+except Exception as ex:
+    print_help()
+    print(str(ex))
+    sys.exit(1)
+
+kibanaAuth = HTTPBasicAuth(kibanaUser, kibanaPassword)
+existsPatterns = getKibanaExsitIndexPatterns(kibanaSpace)
+
+for name, id in existsPatterns.items():
+    print("%s\t%s" %(id, name))