Skip to content

Commit

Permalink
fix(node): add the node.Status.Condition ModulesLoaded
Browse files Browse the repository at this point in the history 
Check if the module `dm_crypt` is enabled as the first module.

ref: longhorn/longhorn 9153

Signed-off-by: James Lu <james.lu@suse.com>
  • Loading branch information
@mantissahz
mantissahz committed Sep 3, 2024
1 parent 8a98944 commit 84a4de8
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 29 deletions.
109 changes: 80 additions & 29 deletions controller/node_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ const (

unknownDiskID = "UNKNOWN_DISKID"

kernelConfigFilePathPrefix = "/boot/config-"

snapshotChangeEventQueueMax = 1048576
)

Expand Down Expand Up @@ -1015,6 +1017,7 @@ func (nc *NodeController) environmentCheck(kubeNode *corev1.Node, node *longhorn
namespaces := []lhtypes.Namespace{lhtypes.NamespaceMnt, lhtypes.NamespaceNet}
nc.syncPackagesInstalled(kubeNode, node, namespaces)
nc.syncMultipathd(node, namespaces)
nc.checkModulesLoaded(kubeNode, node, namespaces)
nc.syncNFSClientVersion(kubeNode, node, namespaces)
}

Expand Down Expand Up @@ -1120,6 +1123,51 @@ func (nc *NodeController) syncMultipathd(node *longhorn.Node, namespaces []lhtyp
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, "", "")
}

func (nc *NodeController) checkModulesLoaded(kubeNode *corev1.Node, node *longhorn.Node, namespaces []lhtypes.Namespace) {
checkingModules := []string{
"dm_crypt",
}
nsexec, err := lhns.NewNamespaceExecutor(lhtypes.ProcessNone, lhtypes.HostProcDirectory, namespaces)
if err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonNamespaceExecutorErr),
fmt.Sprintf("Failed to get namespace executor: %v", err.Error()))
return
}

kernelConfigPath := kernelConfigFilePathPrefix + kubeNode.Status.NodeInfo.KernelVersion
args := []string{kernelConfigPath}
if _, err := nsexec.Execute(nil, "ls", args, lhtypes.ExecuteDefaultTimeout); err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonKernelConfigIsNotFound),
fmt.Sprintf("Unable to find %v for checking %v: %v", kernelConfigPath, checkingModules, err.Error()))
return
}

notLoadedModules := []string{}
for _, module := range checkingModules {
moduleEnabled, err := nc.checkKernelModuleEabled(nsexec, kernelConfigPath, "CONFIG_"+strings.ToUpper(module), module)
if err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonNamespaceExecutorErr),
fmt.Sprintf("Failed to check kernel module %v: %v", module, err.Error()))
return
}
if !moduleEnabled {
notLoadedModules = append(notLoadedModules, module)
}
}

if len(notLoadedModules) != 0 {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonModulesNotLoaded),
fmt.Sprintf("Modules %v are not loaded on node %v", notLoadedModules, node.Name))
return
}

node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusTrue, "", "")
}

func (nc *NodeController) syncNFSClientVersion(kubeNode *corev1.Node, node *longhorn.Node, namespaces []lhtypes.Namespace) {
kernelVersion := kubeNode.Status.NodeInfo.KernelVersion
nfsClientVersions := []string{"CONFIG_NFS_V4_2", "CONFIG_NFS_V4_1", "CONFIG_NFS_V4"}
Expand All @@ -1132,7 +1180,7 @@ func (nc *NodeController) syncNFSClientVersion(kubeNode *corev1.Node, node *long
return
}

kernelConfigPath := "/boot/config-" + kernelVersion
kernelConfigPath := kernelConfigFilePathPrefix + kernelVersion
args := []string{kernelConfigPath}
if _, err := nsexec.Execute(nil, "ls", args, lhtypes.ExecuteDefaultTimeout); err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse,
Expand All @@ -1142,38 +1190,16 @@ func (nc *NodeController) syncNFSClientVersion(kubeNode *corev1.Node, node *long
}

for _, ver := range nfsClientVersions {
args := []string{ver + "=", kernelConfigPath}
result, err := nsexec.Execute(nil, "grep", args, lhtypes.ExecuteDefaultTimeout)
moduleEnabled, err := nc.checkKernelModuleEabled(nsexec, kernelConfigPath, ver, "nfs")
if err != nil {
nc.logger.WithError(err).Debugf("Failed to find kernel config %v on node %v", ver, node.Name)
continue
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonNamespaceExecutorErr),
fmt.Sprintf("Failed to check kernel module %v: %v", ver, err.Error()))
return
}
enabled := strings.TrimSpace(strings.Split(result, "=")[1])
switch enabled {
case "y":
if moduleEnabled {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusTrue, "", "")
return
case "m":
kmodResult, err := lhexec.NewExecutor().Execute(nil, "kmod", []string{"list"}, lhtypes.ExecuteDefaultTimeout)
if err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonNFSClientIsNotFound),
fmt.Sprintf("Failed to execute command `kmod`: %v", err.Error()))
return
}
res, err := lhexec.NewExecutor().ExecuteWithStdinPipe("grep", []string{"nfs"}, kmodResult, lhtypes.ExecuteDefaultTimeout)
if err != nil {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse,
string(longhorn.NodeConditionReasonNFSClientIsNotFound),
fmt.Sprintf("Failed to execute command `grep`: %v", err.Error()))
return
}
if res != "" {
node.Status.Conditions = types.SetCondition(node.Status.Conditions, longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusTrue, "", "")
return
}
default:
nc.logger.Debugf("Unknown kernel config value for %v: %v", ver, enabled)
}
}

Expand All @@ -1182,6 +1208,31 @@ func (nc *NodeController) syncNFSClientVersion(kubeNode *corev1.Node, node *long
fmt.Sprintf("NFS clients %v not found. At least one should be enabled", nfsClientVersions))
}

func (nc *NodeController) checkKernelModuleEabled(nsexec *lhns.Executor, kernelConfigPath, module, kmodName string) (bool, error) {
args := []string{module + "=", kernelConfigPath}
result, err := nsexec.Execute(nil, "grep", args, lhtypes.ExecuteDefaultTimeout)
if err != nil {
nc.logger.WithError(err).Debugf("Failed to find kernel config %v on node %v", module, nc.controllerID)
return false, errors.Wrapf(err, "Failed to find kernel config %v on node %v", module, nc.controllerID)
}
enabled := strings.TrimSpace(strings.Split(result, "=")[1])
switch enabled {
case "y":
return true, nil
case "m":
kmodResult, err := lhexec.NewExecutor().Execute(nil, "kmod", []string{"list"}, lhtypes.ExecuteDefaultTimeout)
if err != nil {
return false, errors.Wrap(err, "Failed to execute command `kmod`")
}
if strings.Contains(kmodResult, kmodName) {
return true, nil
}
default:
nc.logger.Debugf("Unknown kernel config value for %v: %v", module, enabled)
}
return false, nil
}

func (nc *NodeController) getImTypeDataEngines(node *longhorn.Node) map[longhorn.InstanceManagerType][]longhorn.DataEngineType {
log := getLoggerForNode(nc.logger, node)

Expand Down
10 changes: 10 additions & 0 deletions controller/node_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ func (s *NodeControllerSuite) TestManagerPodUp(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
},
Expand Down Expand Up @@ -273,6 +274,7 @@ func (s *NodeControllerSuite) TestManagerPodDown(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNoMountPropagationSupport),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
},
Expand Down Expand Up @@ -360,6 +362,7 @@ func (s *NodeControllerSuite) TestKubeNodeDown(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
},
Expand Down Expand Up @@ -447,6 +450,7 @@ func (s *NodeControllerSuite) TestKubeNodePressure(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
},
Expand Down Expand Up @@ -569,6 +573,7 @@ func (s *NodeControllerSuite) TestUpdateDiskStatus(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{
Expand Down Expand Up @@ -720,6 +725,7 @@ func (s *NodeControllerSuite) TestCleanDiskStatus(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{
Expand Down Expand Up @@ -877,6 +883,7 @@ func (s *NodeControllerSuite) TestDisableDiskOnFilesystemChange(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{
Expand Down Expand Up @@ -1005,6 +1012,7 @@ func (s *NodeControllerSuite) TestCreateDefaultInstanceManager(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{
Expand Down Expand Up @@ -1150,6 +1158,7 @@ func (s *NodeControllerSuite) TestCleanupRedundantInstanceManagers(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{
Expand Down Expand Up @@ -1265,6 +1274,7 @@ func (s *NodeControllerSuite) TestCleanupAllInstanceManagers(c *C) {
newNodeCondition(longhorn.NodeConditionTypeMountPropagation, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonUnknownOS),
newNodeCondition(longhorn.NodeConditionTypeMultipathd, longhorn.ConditionStatusTrue, ""),
newNodeCondition(longhorn.NodeConditionTypeModulesLoaded, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonNamespaceExecutorErr),
newNodeCondition(longhorn.NodeConditionTypeNFSClientInstalled, longhorn.ConditionStatusFalse, longhorn.NodeConditionReasonKernelConfigIsNotFound),
},
DiskStatus: map[string]*longhorn.DiskStatus{},
Expand Down
2 changes: 2 additions & 0 deletions k8s/pkg/apis/longhorn/v1beta2/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const (
NodeConditionTypeReady = "Ready"
NodeConditionTypeMountPropagation = "MountPropagation"
NodeConditionTypeMultipathd = "Multipathd"
NodeConditionTypeModulesLoaded = "ModulesLoaded"
NodeConditionTypeRequiredPackages = "RequiredPackages"
NodeConditionTypeNFSClientInstalled = "NFSClientInstalled"
NodeConditionTypeSchedulable = "Schedulable"
Expand All @@ -22,6 +23,7 @@ const (
NodeConditionReasonMultipathdIsRunning = "MultipathdIsRunning"
NodeConditionReasonUnknownOS = "UnknownOS"
NodeConditionReasonNamespaceExecutorErr = "NamespaceExecutorErr"
NodeConditionReasonModulesNotLoaded = "ModulesNotLoaded"
NodeConditionReasonPackagesNotInstalled = "PackagesNotInstalled"
NodeConditionReasonKernelConfigIsNotFound = "KernelConfigIsNotFound"
NodeConditionReasonNFSClientIsNotFound = "NFSClientIsNotFound"
Expand Down

0 comments on commit 84a4de8

Please sign in to comment.