-
Notifications
You must be signed in to change notification settings - Fork 166
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Kubevirt: cdiupload retry, thin metrics
- Implement retries of CDI upload to pvc to handle intermittent timeouts to k8s api. - Switch kubevirt memory metric as "_total" suffix was removed from "kubevirt_vmi_memory_domain_bytes" - Create DiskMetric with names for sdX and pvc name to help match them in system debug. - Fix AppDiskMetric thin allocation reporting by moving csihandler GetVolumeDetails to reading allocated space from lh volume object and Populate() filling in FileLocation. - syscall.Stat.Blocks are always reported as 512 size. (libc struct stat st_blocks) Don't use stat.Blksize which can return 4k and overinflate sizes. - Use thin allocation check when dir has prefix of types.VolumeEncryptedDirName or types.VolumeClearDirName to handle subdirs like kubevirt /persist/vault/volumes/replicas/... Signed-off-by: Andrew Durbin <andrewd@zededa.com>
- Loading branch information
1 parent
abf4f4f
commit 43665e9
Showing
13 changed files
with
277 additions
and
92 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| // Copyright (c) 2024 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build kubevirt | ||
|
|
||
| package volumemgr | ||
|
|
||
| import ( | ||
| "github.com/lf-edge/eve/pkg/pillar/kubeapi" | ||
| "github.com/lf-edge/eve/pkg/pillar/types" | ||
| ) | ||
|
|
||
| // createOrUpdatePvcDiskMetrics creates or updates metrics for all kubevirt PVCs | ||
| // PVC mknod will match one of existing sdX devices, create copies for convenience | ||
| func createOrUpdatePvcDiskMetrics(ctx *volumemgrContext) { | ||
| log.Functionf("createOrUpdatePvcDiskMetrics") | ||
| var diskMetricList []*types.DiskMetric | ||
|
|
||
| sdMajMinToNameMap, _, _ := kubeapi.SCSIGetMajMinMaps() | ||
| _, pvNameToMajMin, _ := kubeapi.LonghornGetMajorMinorMaps() | ||
| _, pvcToPvMap, _ := kubeapi.PvPvcMaps() | ||
|
|
||
| kubeapi.CleanupUnmountedDiskMetrics(ctx.pubDiskMetric, pvcToPvMap) | ||
|
|
||
| for pvcName, pvName := range pvcToPvMap { | ||
| // pv-name will be of format "pvc-<uuid>" | ||
| // pvc-name will be of format "<uuid>-pvc-0" | ||
| // pvc-name uuid prefix will show in VolumeStatus | ||
| // full pvc-name will be in VolumeStatus.FileLocation | ||
|
|
||
| if pvName == "" { | ||
| continue | ||
| } | ||
|
|
||
| pvMajMinStr, ok := pvNameToMajMin[pvName] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| sdName, ok := sdMajMinToNameMap[pvMajMinStr] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| var metric *types.DiskMetric | ||
| metric = lookupDiskMetric(ctx, sdName) | ||
| if metric == nil { | ||
| continue | ||
| } | ||
|
|
||
| pvcMetric := lookupDiskMetric(ctx, sdName+"-"+pvcName) | ||
| if pvcMetric == nil { | ||
| pvcMetric = &types.DiskMetric{DiskPath: sdName + "-" + pvcName, IsDir: false} | ||
| } | ||
| pvcMetric.ReadBytes = metric.ReadBytes | ||
| pvcMetric.WriteBytes = metric.WriteBytes | ||
| pvcMetric.ReadCount = metric.ReadCount | ||
| pvcMetric.WriteCount = metric.WriteCount | ||
| pvcMetric.TotalBytes = metric.TotalBytes | ||
| pvcMetric.UsedBytes = metric.UsedBytes | ||
| pvcMetric.FreeBytes = metric.FreeBytes | ||
| pvcMetric.IsDir = false | ||
| diskMetricList = append(diskMetricList, pvcMetric) | ||
| } | ||
| publishDiskMetrics(ctx, diskMetricList...) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| // Copyright (c) 2025 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build !kubevirt | ||
|
|
||
| package volumemgr | ||
|
|
||
| // createOrUpdatePvcDiskMetrics has no work in non kubevirt builds | ||
| func createOrUpdatePvcDiskMetrics(*volumemgrContext) { | ||
| return | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright (c) 2025 Zededa, Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build kubevirt | ||
|
|
||
| package kubeapi | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/lf-edge/eve/pkg/pillar/base" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| // waitForPVCUploadComplete: Loop until PVC upload annotations show upload complete | ||
| func waitForPVCUploadComplete(pvcName string, log *base.LogObject) error { | ||
| clientset, err := GetClientSet() | ||
| if err != nil { | ||
| log.Errorf("waitForPVCUploadComplete failed to get clientset err %v", err) | ||
| return err | ||
| } | ||
|
|
||
| i := 100 | ||
| for i > 0 { | ||
| i-- | ||
| time.Sleep(5 * time.Second) | ||
| pvc, err := clientset.CoreV1().PersistentVolumeClaims(EVEKubeNameSpace). | ||
| Get(context.Background(), pvcName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| log.Errorf("waitForPVCUploadComplete failed to get pvc info err %v", err) | ||
| continue | ||
| } | ||
| if cdiUploadIsComplete(log, pvc) { | ||
| return nil | ||
| } | ||
| } | ||
|
|
||
| return fmt.Errorf("waitForPVCUploadComplete: time expired") | ||
| } | ||
|
|
||
| func cdiUploadIsComplete(log *base.LogObject, pvc *corev1.PersistentVolumeClaim) bool { | ||
| annotationKey := "cdi.kubevirt.io/storage.pod.phase" | ||
| annotationExpectedVal := "Succeeded" | ||
| foundVal, ok := pvc.Annotations[annotationKey] | ||
| if !ok { | ||
| log.Errorf("pvc %s annotation %s is missing", pvc.Name, annotationKey) | ||
| return false | ||
| } | ||
| if foundVal != annotationExpectedVal { | ||
| log.Warnf("pvc %s annotation %s is %s, waiting for %s", pvc.Name, annotationKey, foundVal, annotationExpectedVal) | ||
| return false | ||
| } | ||
|
|
||
| annotationKey = "cdi.kubevirt.io/storage.condition.running.message" | ||
| annotationExpectedVal = "Upload Complete" | ||
| foundVal, ok = pvc.Annotations[annotationKey] | ||
| if !ok { | ||
| log.Errorf("pvc %s annotation %s is missing", pvc.Name, annotationKey) | ||
| return false | ||
| } | ||
| if foundVal != annotationExpectedVal { | ||
| log.Warnf("pvc %s annotation %s is %s, waiting for %s", pvc.Name, annotationKey, foundVal, annotationExpectedVal) | ||
| return false | ||
| } | ||
| return true | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.