Report KCM as unhealthy if leader election is wedged.

cmd/cloud-controller-manager/app/BUILD

@@ -19,6 +19,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",

cmd/cloud-controller-manager/app/controllermanager.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
 	apiserverflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/leaderelection"
@@ -131,16 +132,24 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error
 		klog.Errorf("unable to register configz: %c", err)
 	}
 
+	// Setup any healthz checks we will want to use.
+	var checks []healthz.HealthzChecker
+	var electionChecker *leaderelection.HealthzAdaptor
+	if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
+		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
+		checks = append(checks, electionChecker)
+	}
+
 	// Start the controller manager HTTP server
 	if c.SecureServing != nil {
-		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging)
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
 		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
 		if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
 			return err
 		}
 	}
 	if c.InsecureServing != nil {
-		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging)
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
 		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
 		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
 		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
@@ -192,6 +201,8 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error
 				klog.Fatalf("leaderelection lost")
 			},
 		},
+		WatchDog: electionChecker,
+		Name:     "cloud-controller-manager",
 	})
 	panic("unreachable")
 }

cmd/controller-manager/app/serve.go

@@ -17,11 +17,10 @@ limitations under the License.
 package app
 
 import (
+	"github.com/prometheus/client_golang/prometheus"
 	"net/http"
 	goruntime "runtime"
 
-	"github.com/prometheus/client_golang/prometheus"
-
 	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -53,9 +52,9 @@ func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.Aut
 }
 
 // NewBaseHandler takes in CompletedConfig and returns a handler.
-func NewBaseHandler(c *apiserverconfig.DebuggingConfiguration) *mux.PathRecorderMux {
+func NewBaseHandler(c *apiserverconfig.DebuggingConfiguration, checks ...healthz.HealthzChecker) *mux.PathRecorderMux {
 	mux := mux.NewPathRecorderMux("controller-manager")
-	healthz.InstallHandler(mux)
+	healthz.InstallHandler(mux, checks...)
 	if c.EnableProfiling {
 		routes.Profiling{}.Install(mux)
 		if c.EnableContentionProfiling {

cmd/kube-controller-manager/app/BUILD

@@ -113,6 +113,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",

cmd/kube-controller-manager/app/controllermanager.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/mux"
 	apiserverflag "k8s.io/apiserver/pkg/util/flag"
 	cacheddiscovery "k8s.io/client-go/discovery/cached"
@@ -150,18 +151,26 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		klog.Errorf("unable to register configz: %c", err)
 	}
 
+	// Setup any healthz checks we will want to use.
+	var checks []healthz.HealthzChecker
+	var electionChecker *leaderelection.HealthzAdaptor
+	if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
+		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
+		checks = append(checks, electionChecker)
+	}
+
 	// Start the controller manager HTTP server
 	// unsecuredMux is the handler for these controller *after* authn/authz filters have been applied
 	var unsecuredMux *mux.PathRecorderMux
 	if c.SecureServing != nil {
-		unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging)
+		unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
 		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
 		if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
 			return err
 		}
 	}
 	if c.InsecureServing != nil {
-		unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging)
+		unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
 		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
 		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
 		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
@@ -240,6 +249,8 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 				klog.Fatalf("leaderelection lost")
 			},
 		},
+		WatchDog: electionChecker,
+		Name:     "kube-controller-manager",
 	})
 	panic("unreachable")
 }

staging/src/k8s.io/client-go/tools/leaderelection/BUILD

@@ -8,12 +8,16 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["leaderelection.go"],
+    srcs = [
+        "healthzadaptor.go",
+        "leaderelection.go",
+    ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection",
     importpath = "k8s.io/client-go/tools/leaderelection",
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
@@ -23,13 +27,17 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["leaderelection_test.go"],
+    srcs = [
+        "healthzadaptor_test.go",
+        "leaderelection_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1/fake:go_default_library",
         "//staging/src/k8s.io/client-go/testing:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",

staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor.go

+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaderelection
+
+import (
+	"net/http"
+	"sync"
+	"time"
+)
+
+// HealthzAdaptor associates the /healthz endpoint with the LeaderElection object.
+// It helps deal with the /healthz endpoint being set up prior to the LeaderElection.
+// This contains the code needed to act as an adaptor between the leader
+// election code the health check code. It allows us to provide health
+// status about the leader election. Most specifically about if the leader
+// has failed to renew without exiting the process. In that case we should
+// report not healthy and rely on the kubelet to take down the process.
+type HealthzAdaptor struct {
+	pointerLock sync.Mutex
+	le          *LeaderElector
+	timeout     time.Duration
+}
+
+// Name returns the name of the health check we are implementing.
+func (l *HealthzAdaptor) Name() string {
+	return "leaderElection"
+}
+
+// Check is called by the healthz endpoint handler.
+// It fails (returns an error) if we own the lease but had not been able to renew it.
+func (l *HealthzAdaptor) Check(req *http.Request) error {
+	l.pointerLock.Lock()
+	defer l.pointerLock.Unlock()
+	if l.le == nil {
+		return nil
+	}
+	return l.le.Check(l.timeout)
+}
+
+// SetLeaderElection ties a leader election object to a HealthzAdaptor
+func (l *HealthzAdaptor) SetLeaderElection(le *LeaderElector) {
+	l.pointerLock.Lock()
+	defer l.pointerLock.Unlock()
+	l.le = le
+}
+
+// NewLeaderHealthzAdaptor creates a basic healthz adaptor to monitor a leader election.
+// timeout determines the time beyond the lease expiry to be allowed for timeout.
+// checks within the timeout period after the lease expires will still return healthy.
+func NewLeaderHealthzAdaptor(timeout time.Duration) *HealthzAdaptor {
+	result := &HealthzAdaptor{
+		timeout: timeout,
+	}
+	return result
+}

staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor_test.go

+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaderelection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/clock"
+	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
+	"net/http"
+)
+
+type fakeLock struct {
+	identity string
+}
+
+// Get is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) Get() (ler *rl.LeaderElectionRecord, err error) {
+	return nil, nil
+}
+
+// Create is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) Create(ler rl.LeaderElectionRecord) error {
+	return nil
+}
+
+// Update is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) Update(ler rl.LeaderElectionRecord) error {
+	return nil
+}
+
+// RecordEvent is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) RecordEvent(string) {}
+
+// Identity is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) Identity() string {
+	return fl.identity
+}
+
+// Describe is a dummy to allow us to have a fakeLock for testing.
+func (fl *fakeLock) Describe() string {
+	return "Dummy implementation of lock for testing"
+}
+
+// TestLeaderElectionHealthChecker tests that the healthcheck for leader election handles its edge cases.
+func TestLeaderElectionHealthChecker(t *testing.T) {
+	current := time.Now()
+	req := &http.Request{}
+
+	tests := []struct {
+		description    string
+		expected       error
+		adaptorTimeout time.Duration
+		elector        *LeaderElector
+	}{
+		{
+			description:    "call check before leader elector initialized",
+			expected:       nil,
+			adaptorTimeout: time.Second * 20,
+			elector:        nil,
+		},
+		{
+			description:    "call check when the the lease is far expired",
+			expected:       fmt.Errorf("failed election to renew leadership on lease %s", "foo"),
+			adaptorTimeout: time.Second * 20,
+			elector: &LeaderElector{
+				config: LeaderElectionConfig{
+					Lock:          &fakeLock{identity: "healthTest"},
+					LeaseDuration: time.Minute,
+					Name:          "foo",
+				},
+				observedRecord: rl.LeaderElectionRecord{
+					HolderIdentity: "healthTest",
+				},
+				observedTime: current,
+				clock:        clock.NewFakeClock(current.Add(time.Hour)),
+			},
+		},
+		{
+			description:    "call check when the the lease is far expired but held by another server",
+			expected:       nil,
+			adaptorTimeout: time.Second * 20,
+			elector: &LeaderElector{
+				config: LeaderElectionConfig{
+					Lock:          &fakeLock{identity: "healthTest"},
+					LeaseDuration: time.Minute,
+					Name:          "foo",
+				},
+				observedRecord: rl.LeaderElectionRecord{
+					HolderIdentity: "otherServer",
+				},
+				observedTime: current,
+				clock:        clock.NewFakeClock(current.Add(time.Hour)),
+			},
+		},
+		{
+			description:    "call check when the the lease is not expired",
+			expected:       nil,
+			adaptorTimeout: time.Second * 20,
+			elector: &LeaderElector{
+				config: LeaderElectionConfig{
+					Lock:          &fakeLock{identity: "healthTest"},
+					LeaseDuration: time.Minute,
+					Name:          "foo",
+				},
+				observedRecord: rl.LeaderElectionRecord{
+					HolderIdentity: "healthTest",
+				},
+				observedTime: current,
+				clock:        clock.NewFakeClock(current),
+			},
+		},
+		{
+			description:    "call check when the the lease is expired but inside the timeout",
+			expected:       nil,
+			adaptorTimeout: time.Second * 20,
+			elector: &LeaderElector{
+				config: LeaderElectionConfig{
+					Lock:          &fakeLock{identity: "healthTest"},
+					LeaseDuration: time.Minute,
+					Name:          "foo",
+				},
+				observedRecord: rl.LeaderElectionRecord{
+					HolderIdentity: "healthTest",
+				},
+				observedTime: current,
+				clock:        clock.NewFakeClock(current.Add(time.Minute).Add(time.Second)),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		adaptor := NewLeaderHealthzAdaptor(test.adaptorTimeout)
+		if adaptor.le != nil {
+			t.Errorf("[%s] leaderChecker started with a LeaderElector %v", test.description, adaptor.le)
+		}
+		if test.elector != nil {
+			test.elector.config.WatchDog = adaptor
+			adaptor.SetLeaderElection(test.elector)
+			if adaptor.le == nil {
+				t.Errorf("[%s] adaptor failed to set the LeaderElector", test.description)
+			}
+		}
+		err := adaptor.Check(req)
+		if test.expected == nil {
+			if err == nil {
+				continue
+			}
+			t.Errorf("[%s] called check, expected no error but received \"%v\"", test.description, err)
+		} else {
+			if err == nil {
+				t.Errorf("[%s] called check and failed to received the expected error \"%v\"", test.description, test.expected)
+			}
+			if err.Error() != test.expected.Error() {
+				t.Errorf("[%s] called check, expected %v, received %v", test.description, test.expected, err)
+			}
+		}
+	}
+}

staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go

@@ -56,6 +56,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -90,6 +91,7 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
 	}
 	return &LeaderElector{
 		config: lec,
+		clock:  clock.RealClock{},
 	}, nil
 }
 
@@ -111,6 +113,13 @@ type LeaderElectionConfig struct {
 	// Callbacks are callbacks that are triggered during certain lifecycle
 	// events of the LeaderElector
 	Callbacks LeaderCallbacks
+
+	// WatchDog is the associated health checker
+	// WatchDog may be null if its not needed/configured.
+	WatchDog *HealthzAdaptor
+
+	// Name is the name of the resource lock for debugging
+	Name string
 }
 
 // LeaderCallbacks are callbacks that are triggered during certain
@@ -139,6 +148,12 @@ type LeaderElector struct {
 	// value observedRecord.HolderIdentity if the transition has
 	// not yet been reported.
 	reportedLeader string
+
+	// clock is wrapper around time to allow for less flaky testing
+	clock clock.Clock
+
+	// name is the name of the resource lock for debugging
+	name string
 }
 
 // Run starts the leader election loop
@@ -163,6 +178,9 @@ func RunOrDie(ctx context.Context, lec LeaderElectionConfig) {
 	if err != nil {
 		panic(err)
 	}
+	if lec.WatchDog != nil {
+		lec.WatchDog.SetLeaderElection(le)
+	}
 	le.Run(ctx)
 }
 
@@ -257,14 +275,14 @@ func (le *LeaderElector) tryAcquireOrRenew() bool {
 			return false
 		}
 		le.observedRecord = leaderElectionRecord
-		le.observedTime = time.Now()
+		le.observedTime = le.clock.Now()
 		return true
 	}
 
 	// 2. Record obtained, check the Identity & Time
 	if !reflect.DeepEqual(le.observedRecord, *oldLeaderElectionRecord) {
 		le.observedRecord = *oldLeaderElectionRecord
-		le.observedTime = time.Now()
+		le.observedTime = le.clock.Now()
 	}
 	if le.observedTime.Add(le.config.LeaseDuration).After(now.Time) &&
 		!le.IsLeader() {
@@ -287,7 +305,7 @@ func (le *LeaderElector) tryAcquireOrRenew() bool {
 		return false
 	}
 	le.observedRecord = leaderElectionRecord
-	le.observedTime = time.Now()
+	le.observedTime = le.clock.Now()
 	return true
 }
 
@@ -300,3 +318,19 @@ func (le *LeaderElector) maybeReportTransition() {
 		go le.config.Callbacks.OnNewLeader(le.reportedLeader)
 	}
 }
+
+// Check will determine if the current lease is expired by more than timeout.
+func (le *LeaderElector) Check(maxTolerableExpiredLease time.Duration) error {
+	if !le.IsLeader() {
+		// Currently not concerned with the case that we are hot standby
+		return nil
+	}
+	// If we are more than timeout seconds after the lease duration that is past the timeout
+	// on the lease renew. Time to start reporting ourselves as unhealthy. We should have
+	// died but conditions like deadlock can prevent this. (See #70819)
+	if le.clock.Since(le.observedTime) > le.config.LeaseDuration+maxTolerableExpiredLease {
+		return fmt.Errorf("failed election to renew leadership on lease %s", le.config.Name)
+	}
+
+	return nil
+}

staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/clock"
 	fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake"
 	core "k8s.io/client-go/testing"
 	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -257,6 +258,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) {
 			config:         lec,
 			observedRecord: test.observedRecord,
 			observedTime:   test.observedTime,
+			clock:          clock.RealClock{},
 		}
 
 		if test.expectSuccess != le.tryAcquireOrRenew() {