Merge pull request #57533 from mtaufen/kc-lkg-timer

Automatic merge from submit-queue (batch tested with PRs 57533, 57524). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. periodically check whether assigned kubelet config should become last-known-good Fixes #57808 Previously, the last-known-good was only updated on Kubelet restart. This has been on my todo list for a while, good to finally have a PR up. Previously we could have this scenario, which is fixed by this PR: - lkg is set to local - we set config A - config A passes trial period, but nothing caused Kubelet to restart - we set config B, which turns out to be invalid - Kubelet will fall back to local, because lkg was never updated ```release-note NONE ```

Merge pull request #57533 from mtaufen/kc-lkg-timer
adeef3ea · Kubernetes Submit Queue · GitHub · 46eabb7d · fe17387f · adeef3ea
Unverified Commit adeef3ea authored Jan 03, 2018 by Kubernetes Submit Queue Committed by GitHub Jan 03, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 18 additions and 9 deletions

controller.go pkg/kubelet/kubeletconfig/controller.go +18 -9

No files found.
--- a/pkg/kubelet/kubeletconfig/controller.go
+++ b/pkg/kubelet/kubeletconfig/controller.go
@@ -141,15 +141,11 @@ func (cc *Controller) Bootstrap() (*kubeletconfig.KubeletConfiguration, error) {
 			cc.configOK.Set(status.CurLocalMessage, reason, apiv1.ConditionTrue)
 		}

-		// when the trial period is over, the assigned config becomes the last-known-good
-		if trial, err := cc.inTrial(assigned.ConfigTrialDuration.Duration); err != nil {
-			utillog.Errorf("failed to check trial period for assigned config, error: %v", err)
-		} else if !trial {
-			utillog.Infof("assigned config passed trial period, will set as last-known-good")
-			if err := cc.graduateAssignedToLastKnownGood(); err != nil {
-				utillog.Errorf("failed to set last-known-good to assigned config, error: %v", err)
-			}
-		}
+		// update the last-known-good config if necessary, and start a timer that
+		// periodically checks whether the last-known good needs to be updated
+		// we only do this when the assigned config loads and passes validation
+		// wait.Forever will call the func once before starting the timer
+		go wait.Forever(func() { cc.checkTrial(assigned.ConfigTrialDuration.Duration) }, 10*time.Second)

 		return assigned, nil
 	} // Assert: the assigned config failed to load, parse, or validate
@@ -319,6 +315,19 @@ func (cc *Controller) initializeDynamicConfigDir() error {
 	return cc.checkpointStore.Initialize()
 }

+// checkTrial checks whether the trial duration has passed, and updates the last-known-good config if necessary
+func (cc *Controller) checkTrial(duration time.Duration) {
+	// when the trial period is over, the assigned config becomes the last-known-good
+	if trial, err := cc.inTrial(duration); err != nil {
+		utillog.Errorf("failed to check trial period for assigned config, error: %v", err)
+	} else if !trial {
+		utillog.Infof("assigned config passed trial period, will set as last-known-good")
+		if err := cc.graduateAssignedToLastKnownGood(); err != nil {
+			utillog.Errorf("failed to set last-known-good to assigned config, error: %v", err)
+		}
+	}
+}
+
 // inTrial returns true if the time elapsed since the last modification of the current config does not exceed `trialDur`, false otherwise
 func (cc *Controller) inTrial(trialDur time.Duration) (bool, error) {
 	now := time.Now()