-
Notifications
You must be signed in to change notification settings - Fork 166
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Kubevirt: cdiupload retry, thin metrics
- Implement retries of CDI upload to pvc to handle intermittent timeouts to k8s api. - Switch kubevirt memory metric as "_total" suffix was removed from "kubevirt_vmi_memory_domain_bytes" - Create DiskMetric with names for sdX and pvc name to help match them in system debug. - Fix AppDiskMetric thin allocation reporting by moving csihandler GetVolumeDetails to reading allocated space from lh volume object and Populate() filling in FileLocation. - Use thin allocation check when dir has prefix of types.VolumeEncryptedDirName or types.VolumeClearDirName to handle subdirs like kubevirt /persist/vault/volumes/replicas/... - waitForPVCUploadComplete needs to use caller's context to allow quicker exit if volume create is cancelled Signed-off-by: Andrew Durbin <andrewd@zededa.com>
- Loading branch information
1 parent
15b6b30
commit 15144fd
Showing
13 changed files
with
302 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| // Copyright (c) 2025 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build kubevirt | ||
|
|
||
| package volumemgr | ||
|
|
||
| import ( | ||
| "github.com/lf-edge/eve/pkg/pillar/kubeapi" | ||
| "github.com/lf-edge/eve/pkg/pillar/types" | ||
| ) | ||
|
|
||
| // createOrUpdatePvcDiskMetrics creates or updates metrics for all kubevirt PVCs | ||
| // PVC mknod will match one of existing sdX devices, create copies for convenience | ||
| func createOrUpdatePvcDiskMetrics(ctx *volumemgrContext) { | ||
| log.Functionf("createOrUpdatePvcDiskMetrics") | ||
| var diskMetricList []*types.DiskMetric | ||
|
|
||
| sdMajMinToNameMap, _, err := kubeapi.SCSIGetMajMinMaps() | ||
| if err != nil { | ||
| log.Errorf("Failed to get SCSI device maps: %v", err) | ||
| return | ||
| } | ||
| _, pvNameToMajMin, err := kubeapi.LonghornGetMajorMinorMaps() | ||
| if err != nil { | ||
| log.Errorf("Failed to get Longhorn device maps: %v", err) | ||
| return | ||
| } | ||
| _, pvcToPvMap, err := kubeapi.PvPvcMaps() | ||
| if err != nil { | ||
| log.Errorf("Failed to get PVC-PV maps: %v", err) | ||
| return | ||
| } | ||
|
|
||
| kubeapi.CleanupDetachedDiskMetrics(ctx.pubDiskMetric, pvcToPvMap) | ||
|
|
||
| for pvcName, pvName := range pvcToPvMap { | ||
| // pv-name will be of format "pvc-<uuid>" | ||
| // pvc-name will be of format "<uuid>-pvc-0" | ||
| // pvc-name uuid prefix will show in VolumeStatus | ||
| // full pvc-name will be in VolumeStatus.FileLocation | ||
|
|
||
| if pvName == "" { | ||
| continue | ||
| } | ||
|
|
||
| pvMajMinStr, ok := pvNameToMajMin[pvName] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| sdName, ok := sdMajMinToNameMap[pvMajMinStr] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| var metric *types.DiskMetric | ||
| metric = lookupDiskMetric(ctx, sdName) | ||
| if metric == nil { | ||
| continue | ||
| } | ||
|
|
||
| pvcMetric := lookupDiskMetric(ctx, sdName+"-"+pvcName) | ||
| if pvcMetric == nil { | ||
| pvcMetric = &types.DiskMetric{DiskPath: sdName + "-" + pvcName, IsDir: false} | ||
| } | ||
| pvcMetric.ReadBytes = metric.ReadBytes | ||
| pvcMetric.WriteBytes = metric.WriteBytes | ||
| pvcMetric.ReadCount = metric.ReadCount | ||
| pvcMetric.WriteCount = metric.WriteCount | ||
| pvcMetric.TotalBytes = metric.TotalBytes | ||
| pvcMetric.UsedBytes = metric.UsedBytes | ||
| pvcMetric.FreeBytes = metric.FreeBytes | ||
| pvcMetric.IsDir = false | ||
| diskMetricList = append(diskMetricList, pvcMetric) | ||
| } | ||
| publishDiskMetrics(ctx, diskMetricList...) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| // Copyright (c) 2025 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build !kubevirt | ||
|
|
||
| package volumemgr | ||
|
|
||
| // createOrUpdatePvcDiskMetrics has no work in non kubevirt builds | ||
| func createOrUpdatePvcDiskMetrics(*volumemgrContext) { | ||
| return | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| // Copyright (c) 2025 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build kubevirt | ||
|
|
||
| package kubeapi | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/lf-edge/eve/pkg/pillar/base" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| // waitForPVCUploadComplete: Loop until PVC upload annotations show upload complete | ||
| // At this time the only caller of this is kubeapi.RolloutDiskToPVC() which runs in the | ||
| // volumecreate worker context. This does currently wait up to 5 minutes | ||
| // but does not need to bump a watchdog as the worker does not have one. | ||
| func waitForPVCUploadComplete(ctx context.Context, pvcName string, log *base.LogObject) error { | ||
| clientset, err := GetClientSet() | ||
| if err != nil { | ||
| log.Errorf("waitForPVCUploadComplete failed to get clientset err %v", err) | ||
| return err | ||
| } | ||
|
|
||
| i := 100 | ||
| for i > 0 { | ||
| i-- | ||
|
|
||
| select { | ||
| case <-ctx.Done(): | ||
| return fmt.Errorf("waitForPVCUploadComplete: context done") | ||
| default: | ||
| time.Sleep(5 * time.Second) | ||
| } | ||
|
|
||
| pvc, err := clientset.CoreV1().PersistentVolumeClaims(EVEKubeNameSpace). | ||
| Get(context.Background(), pvcName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| log.Errorf("waitForPVCUploadComplete failed to get pvc info err %v", err) | ||
| continue | ||
| } | ||
| if cdiUploadIsComplete(log, pvc) { | ||
| return nil | ||
| } | ||
| } | ||
|
|
||
| return fmt.Errorf("waitForPVCUploadComplete: time expired") | ||
| } | ||
|
|
||
| func cdiUploadIsComplete(log *base.LogObject, pvc *corev1.PersistentVolumeClaim) bool { | ||
| annotationKey := "cdi.kubevirt.io/storage.pod.phase" | ||
| annotationExpectedVal := "Succeeded" | ||
| foundVal, ok := pvc.Annotations[annotationKey] | ||
| if !ok { | ||
| log.Errorf("pvc %s annotation %s is missing", pvc.Name, annotationKey) | ||
| return false | ||
| } | ||
| if foundVal != annotationExpectedVal { | ||
| log.Warnf("pvc %s annotation %s is %s, waiting for %s", pvc.Name, annotationKey, foundVal, annotationExpectedVal) | ||
| return false | ||
| } | ||
|
|
||
| annotationKey = "cdi.kubevirt.io/storage.condition.running.message" | ||
| annotationExpectedVal = "Upload Complete" | ||
| foundVal, ok = pvc.Annotations[annotationKey] | ||
| if !ok { | ||
| log.Errorf("pvc %s annotation %s is missing", pvc.Name, annotationKey) | ||
| return false | ||
| } | ||
| if foundVal != annotationExpectedVal { | ||
| log.Warnf("pvc %s annotation %s is %s, waiting for %s", pvc.Name, annotationKey, foundVal, annotationExpectedVal) | ||
| return false | ||
| } | ||
| return true | ||
| } |
Oops, something went wrong.