diff --git a/service/exceptionmonitor/helper/monitor/database_monitor.go b/service/exceptionmonitor/helper/monitor/database_monitor.go index baa8a188ba5a..09eaf2a37a01 100644 --- a/service/exceptionmonitor/helper/monitor/database_monitor.go +++ b/service/exceptionmonitor/helper/monitor/database_monitor.go @@ -174,7 +174,7 @@ func handleClusterException(notificationInfo *api.Info) { //api.DatabaseNamespaceMap[notificationInfo.DatabaseClusterUID] = notificationInfo.Namespace + "-" + notificationInfo.DatabaseClusterName //api.ExceptionDatabaseMap[notificationInfo.DatabaseClusterUID] = true //notificationInfo.DatabaseClusterUID, databaseClusterName, namespace, databaseType, status - if err := processClusterException(notificationInfo); err != nil { + if err := ProcessClusterException(notificationInfo); err != nil { log.Printf("Failed to process cluster %s exception in ns %s: %v", notificationInfo.DatabaseClusterName, notificationInfo.Namespace, err) } } else if _, ok := api.DatabaseNotificationInfoMap[notificationInfo.DatabaseClusterUID]; ok { @@ -183,7 +183,7 @@ func handleClusterException(notificationInfo *api.Info) { } } -func processClusterException(notificationInfo *api.Info) error { +func ProcessClusterException(notificationInfo *api.Info) error { debt, debtLevel, _ := checkDebt(notificationInfo.Namespace) notificationInfo.DebtLevel = debtLevel if debt { @@ -191,7 +191,7 @@ func processClusterException(notificationInfo *api.Info) error { databaseEvents, send := getDatabaseClusterEvents(notificationInfo) if send { //namespace, databaseClusterName, databaseType - maxUsage, err := checkPerformance(notificationInfo, "disk") + maxUsage, err := CheckPerformance(notificationInfo, "disk") if err != nil { return err } diff --git a/service/exceptionmonitor/helper/monitor/database_performance_monitor.go b/service/exceptionmonitor/helper/monitor/database_performance_monitor.go index d4aa24079a93..17903fca2587 100644 --- a/service/exceptionmonitor/helper/monitor/database_performance_monitor.go +++ b/service/exceptionmonitor/helper/monitor/database_performance_monitor.go @@ -97,7 +97,7 @@ func handleCPUMemMonitor(notificationInfo *api.Info) { } func handleDiskMonitor(notificationInfo *api.Info) { - if maxUsage, err := checkPerformance(notificationInfo, "disk"); err == nil { + if maxUsage, err := CheckPerformance(notificationInfo, "disk"); err == nil { processUsage(maxUsage, api.DatabaseDiskMonitorThreshold, api.DiskChinese, notificationInfo) } else { log.Printf("Failed to monitor Disk: %v", err) @@ -182,7 +182,7 @@ func processRecovery(notificationInfo *api.Info) { } func CPUMemMonitor(notificationInfo *api.Info, checkType string) (float64, error) { - return checkPerformance(notificationInfo, checkType) + return CheckPerformance(notificationInfo, checkType) } func NumberToChinese(num int) string { diff --git a/service/exceptionmonitor/helper/monitor/performance.go b/service/exceptionmonitor/helper/monitor/performance.go index a004066ec697..9c86d4bfa00e 100644 --- a/service/exceptionmonitor/helper/monitor/performance.go +++ b/service/exceptionmonitor/helper/monitor/performance.go @@ -16,7 +16,7 @@ import ( "github.com/labring/sealos/service/exceptionmonitor/api" ) -func checkPerformance(notificationInfo *api.Info, checkType string) (float64, error) { +func CheckPerformance(notificationInfo *api.Info, checkType string) (float64, error) { params := url.Values{} params.Add("namespace", notificationInfo.Namespace) params.Add("app", notificationInfo.DatabaseClusterName) diff --git a/service/exceptionmonitor/server/server.go b/service/exceptionmonitor/server/server.go index 59ff28d74f57..b12ccbde5eef 100644 --- a/service/exceptionmonitor/server/server.go +++ b/service/exceptionmonitor/server/server.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/labring/sealos/service/exceptionmonitor/api" - + "github.com/labring/sealos/service/exceptionmonitor/helper/monitor" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -62,7 +62,7 @@ func handleListDatabases(rw http.ResponseWriter, req *http.Request) { result := make([]DatabaseInfo, 0, len(clusters.Items)) for _, c := range clusters.Items { - info := collectDatabaseInfo(ctx, ns, c) + info := collectDatabaseInfo(ctx, c) result = append(result, info) } @@ -76,30 +76,43 @@ func handleListDatabases(rw http.ResponseWriter, req *http.Request) { } } -func collectDatabaseInfo(ctx context.Context, namespace string, cluster metav1unstructured.Unstructured) DatabaseInfo { - name := cluster.GetName() +func collectDatabaseInfo(ctx context.Context, cluster metav1unstructured.Unstructured) DatabaseInfo { status, _, _ := metav1unstructured.NestedString(cluster.Object, "status", "phase") + databaseClusterName, databaseType, namespace := cluster.GetName(), cluster.GetLabels()[api.DatabaseTypeLabel], cluster.GetNamespace() - // Gather related events as details - events, err := api.ClientSet.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{ - FieldSelector: fmt.Sprintf("involvedObject.name=%s", name), - }) - details := "" - if err == nil { - var lines []string - for _, e := range events.Items { - lines = append(lines, fmt.Sprintf("%s - %s", e.Reason, e.Message)) - } - details = strings.Join(lines, "\n") + notificationInfo := api.Info{} + notificationInfo.DatabaseClusterName = databaseClusterName + notificationInfo.Namespace = namespace + notificationInfo.DatabaseType = databaseType + maxUsage, err := monitor.CheckPerformance(¬ificationInfo, "disk") + if err != nil { + fmt.Println(err) } - reason := deriveReason(status, details) + // Gather related events as details + //events, err := api.ClientSet.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{ + // FieldSelector: fmt.Sprintf("involvedObject.name=%s", databaseClusterName), + //}) + if maxUsage > api.DatabaseExceptionMonitorThreshold { + notificationInfo.Reason = "disk is full" + } + //fmt.Println(events) + //details := "" + //if err == nil { + // var lines []string + // for _, e := range events.Items { + // lines = append(lines, fmt.Sprintf("%s - %s", e.Reason, e.Message)) + // } + // details = strings.Join(lines, "\n") + //} + // + //reason := deriveReason(status, details) return DatabaseInfo{ - Name: name, + Name: databaseClusterName, Status: status, - Reason: reason, - Details: details, + Reason: notificationInfo.Reason, + Details: notificationInfo.Events, } }