Skip to content

Commit

Permalink
Merge pull request #115 from timdengyun/bug_fix
Browse files Browse the repository at this point in the history
Handle pod status StartTime access when it's not initilized
  • Loading branch information
timdengyun authored Mar 25, 2021
2 parents fa9480b + ef3cdc1 commit fced14c
Showing 1 changed file with 12 additions and 10 deletions.
22 changes: 12 additions & 10 deletions pkg/controller/statusmanager/pod_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func SetNodeCondition(conditions *[]corev1.NodeCondition, newCondition corev1.No
// CheckExistingAgentPods is for a case: nsx-node-agent becomes unhealthy -> NetworkUnavailable=True -> operator off ->
// nsx-node-agent becomes healthy and keeps running -> operator up -> operator cannot receive nsx-node-agent event to
// set NetworkUnavailable=False. So a full sync at the start time is necessary.
func (status *StatusManager) CheckExistingAgentPods (firstBoot *bool, sharedInfo *sharedinfo.SharedInfo) error {
func (status *StatusManager) CheckExistingAgentPods(firstBoot *bool, sharedInfo *sharedinfo.SharedInfo) error {
status.Lock()
defer status.Unlock()

Expand Down Expand Up @@ -213,7 +213,6 @@ func (status *StatusManager) updateNodeStatus(nodeName string, ready bool, reaso
}
}


// Get the pod status from API server
func (status *StatusManager) SetNodeConditionFromPod(podName types.NamespacedName, sharedInfo *sharedinfo.SharedInfo, pod *corev1.Pod) {
status.Lock()
Expand Down Expand Up @@ -247,13 +246,16 @@ func (status *StatusManager) setNodeConditionFromPod(podName types.NamespacedNam
// received the outdated pot event after the new pod running. For this case
// we should ignore the outdated event, otherwise, the operator may mistakenly
// set the NetworkUnavailbe status.
podStartedAt := pod.Status.StartTime.Time
nodeLastStartedAt := sharedInfo.LastNodeAgentStartTime[nodeName]
if podStartedAt.Before(nodeLastStartedAt) {
log.Info("Pod %s started at %v on node %s is outdated, there's new pod started at %v", podStartedAt, podName, nodeName, nodeLastStartedAt)
return
} else {
sharedInfo.LastNodeAgentStartTime[nodeName] = podStartedAt
// StartTime field will not be set until pod is initilized in pending state
if (pod.Status.StartTime != nil) {
podStartedAt := pod.Status.StartTime.Time
nodeLastStartedAt := sharedInfo.LastNodeAgentStartTime[nodeName]
if podStartedAt.Before(nodeLastStartedAt) {
log.Info(fmt.Sprintf("Pod %s started at %v on node %s is outdated, there's new pod started at %v", podName, podStartedAt, nodeName, nodeLastStartedAt))
return
} else {
sharedInfo.LastNodeAgentStartTime[nodeName] = podStartedAt
}
}

// when pod is pending, its ContainerStatuses is empty
Expand Down Expand Up @@ -533,4 +535,4 @@ func (status *StatusManager) CombineConditions(conditions *[]configv1.ClusterOpe
}
}
return changed, messages
}
}

0 comments on commit fced14c

Please sign in to comment.