consolidate node deletion logic between node lifecycle and cloud node controller

cmd/cloud-controller-manager/app/controllermanager.go

@@ -226,12 +226,23 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru
 	nodeController := cloudcontrollers.NewCloudNodeController(
 		c.SharedInformers.Core().V1().Nodes(),
 		client("cloud-node-controller"), cloud,
-		c.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
 		c.ComponentConfig.NodeStatusUpdateFrequency.Duration)
 
-	nodeController.Run(stop)
+	go nodeController.Run(stop)
 	time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
 
+	cloudNodeLifecycleController, err := cloudcontrollers.NewCloudNodeLifecycleController(
+		c.SharedInformers.Core().V1().Nodes(),
+		client("cloud-node-lifecycle-controller"), cloud,
+		c.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
+	)
+	if err != nil {
+		klog.Errorf("failed to start cloud node lifecycle controller: %s", err)
+	} else {
+		go cloudNodeLifecycleController.Run(stop)
+		time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
+	}
+
 	// Start the PersistentVolumeLabelController
 	pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud)
 	go pvlController.Run(5, stop)

cmd/kube-controller-manager/app/BUILD

@@ -46,6 +46,7 @@ go_library(
         "//pkg/controller/certificates/cleaner:go_default_library",
         "//pkg/controller/certificates/rootcacertpublisher:go_default_library",
         "//pkg/controller/certificates/signer:go_default_library",
+        "//pkg/controller/cloud:go_default_library",
         "//pkg/controller/clusterroleaggregation:go_default_library",
         "//pkg/controller/cronjob:go_default_library",
         "//pkg/controller/daemon:go_default_library",

cmd/kube-controller-manager/app/controllermanager.go

@@ -383,13 +383,13 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
 	controllers["bootstrapsigner"] = startBootstrapSignerController
 	controllers["tokencleaner"] = startTokenCleanerController
 	controllers["nodeipam"] = startNodeIpamController
+	controllers["nodelifecycle"] = startNodeLifecycleController
 	if loopMode == IncludeCloudLoops {
 		controllers["service"] = startServiceController
 		controllers["route"] = startRouteController
+		controllers["cloudnodelifecycle"] = startCloudNodeLifecycleController
 		// TODO: volume controller into the IncludeCloudLoops only set.
-		// TODO: Separate cluster in cloud check from node lifecycle controller.
 	}
-	controllers["nodelifecycle"] = startNodeLifecycleController
 	controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
 	controllers["attachdetach"] = startAttachDetachController
 	controllers["persistentvolume-expander"] = startVolumeExpandController

cmd/kube-controller-manager/app/core.go

@@ -38,6 +38,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	"k8s.io/kubernetes/pkg/controller"
+	cloudcontroller "k8s.io/kubernetes/pkg/controller/cloud"
 	endpointcontroller "k8s.io/kubernetes/pkg/controller/endpoint"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace"
@@ -125,7 +126,6 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.InformerFactory.Core().V1().Nodes(),
 		ctx.InformerFactory.Extensions().V1beta1().DaemonSets(),
-		ctx.Cloud,
 		ctx.ClientBuilder.ClientOrDie("node-controller"),
 		ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
 		ctx.ComponentConfig.NodeLifecycleController.NodeStartupGracePeriod.Duration,
@@ -146,6 +146,24 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
 	return nil, true, nil
 }
 
+func startCloudNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, error) {
+	cloudNodeLifecycleController, err := cloudcontroller.NewCloudNodeLifecycleController(
+		ctx.InformerFactory.Core().V1().Nodes(),
+		ctx.ClientBuilder.ClientOrDie("cloud-node-lifecycle-controller"),
+		ctx.Cloud,
+		ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
+	)
+	if err != nil {
+		// the controller manager should continue to run if the "Instances" interface is not
+		// supported, though it's unlikely for a cloud provider to not support it
+		klog.Errorf("failed to start cloud node lifecycle controller: %v", err)
+		return nil, false, nil
+	}
+
+	go cloudNodeLifecycleController.Run(ctx.Stop)
+	return nil, true, nil
+}
+
 func startRouteController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes {
 		klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes)

pkg/controller/BUILD

@@ -55,7 +55,6 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/install:go_default_library",
         "//pkg/apis/core/validation:go_default_library",
-        "//pkg/scheduler/api:go_default_library",
         "//pkg/serviceaccount:go_default_library",
         "//pkg/util/hash:go_default_library",
         "//pkg/util/taints:go_default_library",

pkg/controller/cloud/BUILD

@@ -10,6 +10,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "node_controller.go",
+        "node_lifecycle_controller.go",
         "pvlcontroller.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/controller/cloud",
@@ -17,7 +18,6 @@ go_library(
         "//pkg/api/v1/node:go_default_library",
         "//pkg/apis/core/v1/helper:go_default_library",
         "//pkg/controller:go_default_library",
-        "//pkg/controller/util/node:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/scheduler/api:go_default_library",
@@ -26,6 +26,7 @@ go_library(
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
@@ -52,6 +53,7 @@ go_test(
     srcs = [
         "main_test.go",
         "node_controller_test.go",
+        "node_lifecycle_controller_test.go",
         "pvlcontroller_test.go",
     ],
     embed = [":go_default_library"],
@@ -68,10 +70,10 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//staging/src/k8s.io/client-go/testing:go_default_library",

pkg/controller/cloud/node_controller.go

@@ -37,9 +37,6 @@ import (
 	"k8s.io/client-go/tools/record"
 	clientretry "k8s.io/client-go/util/retry"
 	cloudprovider "k8s.io/cloud-provider"
-	nodeutilv1 "k8s.io/kubernetes/pkg/api/v1/node"
-	"k8s.io/kubernetes/pkg/controller"
-	nodectrlutil "k8s.io/kubernetes/pkg/controller/util/node"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
@@ -58,11 +55,6 @@ type CloudNodeController struct {
 
 	cloud cloudprovider.Interface
 
-	// Value controlling NodeController monitoring period, i.e. how often does NodeController
-	// check node status posted from kubelet. This value should be lower than nodeMonitorGracePeriod
-	// set in controller-manager
-	nodeMonitorPeriod time.Duration
-
 	nodeStatusUpdateFrequency time.Duration
 }
 
@@ -79,7 +71,6 @@ func NewCloudNodeController(
 	nodeInformer coreinformers.NodeInformer,
 	kubeClient clientset.Interface,
 	cloud cloudprovider.Interface,
-	nodeMonitorPeriod time.Duration,
 	nodeStatusUpdateFrequency time.Duration) *CloudNodeController {
 
 	eventBroadcaster := record.NewBroadcaster()
@@ -97,7 +88,6 @@ func NewCloudNodeController(
 		kubeClient:                kubeClient,
 		recorder:                  recorder,
 		cloud:                     cloud,
-		nodeMonitorPeriod:         nodeMonitorPeriod,
 		nodeStatusUpdateFrequency: nodeStatusUpdateFrequency,
 	}
 
@@ -111,8 +101,9 @@ func NewCloudNodeController(
 	return cnc
 }
 
-// This controller deletes a node if kubelet is not reporting
-// and the node is gone from the cloud provider.
+// This controller updates newly registered nodes with information
+// from the cloud provider. This call is blocking so should be called
+// via a goroutine
 func (cnc *CloudNodeController) Run(stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 
@@ -121,10 +112,7 @@ func (cnc *CloudNodeController) Run(stopCh <-chan struct{}) {
 	// very infrequently. DO NOT MODIFY this to perform frequent operations.
 
 	// Start a loop to periodically update the node addresses obtained from the cloud
-	go wait.Until(cnc.UpdateNodeStatus, cnc.nodeStatusUpdateFrequency, stopCh)
-
-	// Start a loop to periodically check if any nodes have been deleted from cloudprovider
-	go wait.Until(cnc.MonitorNode, cnc.nodeMonitorPeriod, stopCh)
+	wait.Until(cnc.UpdateNodeStatus, cnc.nodeStatusUpdateFrequency, stopCh)
 }
 
 // UpdateNodeStatus updates the node status, such as node addresses
@@ -210,108 +198,6 @@ func (cnc *CloudNodeController) updateNodeAddress(node *v1.Node, instances cloud
 	}
 }
 
-// Monitor node queries the cloudprovider for non-ready nodes and deletes them
-// if they cannot be found in the cloud provider
-func (cnc *CloudNodeController) MonitorNode() {
-	instances, ok := cnc.cloud.Instances()
-	if !ok {
-		utilruntime.HandleError(fmt.Errorf("failed to get instances from cloud provider"))
-		return
-	}
-
-	nodes, err := cnc.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{ResourceVersion: "0"})
-	if err != nil {
-		klog.Errorf("Error monitoring node status: %v", err)
-		return
-	}
-
-	for i := range nodes.Items {
-		var currentReadyCondition *v1.NodeCondition
-		node := &nodes.Items[i]
-		// Try to get the current node status
-		// If node status is empty, then kubelet has not posted ready status yet. In this case, process next node
-		for rep := 0; rep < nodeStatusUpdateRetry; rep++ {
-			_, currentReadyCondition = nodeutilv1.GetNodeCondition(&node.Status, v1.NodeReady)
-			if currentReadyCondition != nil {
-				break
-			}
-			name := node.Name
-			node, err = cnc.kubeClient.CoreV1().Nodes().Get(name, metav1.GetOptions{})
-			if err != nil {
-				klog.Errorf("Failed while getting a Node to retry updating NodeStatus. Probably Node %s was deleted.", name)
-				break
-			}
-			time.Sleep(retrySleepTime)
-		}
-		if currentReadyCondition == nil {
-			klog.Errorf("Update status of Node %v from CloudNodeController exceeds retry count or the Node was deleted.", node.Name)
-			continue
-		}
-		// If the known node status says that Node is NotReady, then check if the node has been removed
-		// from the cloud provider. If node cannot be found in cloudprovider, then delete the node immediately
-		if currentReadyCondition != nil {
-			if currentReadyCondition.Status != v1.ConditionTrue {
-				// we need to check this first to get taint working in similar in all cloudproviders
-				// current problem is that shutdown nodes are not working in similar way ie. all cloudproviders
-				// does not delete node from kubernetes cluster when instance it is shutdown see issue #46442
-				shutdown, err := nodectrlutil.ShutdownInCloudProvider(context.TODO(), cnc.cloud, node)
-				if err != nil {
-					klog.Errorf("Error checking if node %s is shutdown: %v", node.Name, err)
-				}
-
-				if shutdown && err == nil {
-					// if node is shutdown add shutdown taint
-					err = controller.AddOrUpdateTaintOnNode(cnc.kubeClient, node.Name, controller.ShutdownTaint)
-					if err != nil {
-						klog.Errorf("Error patching node taints: %v", err)
-					}
-					// Continue checking the remaining nodes since the current one is shutdown.
-					continue
-				}
-
-				// Check with the cloud provider to see if the node still exists. If it
-				// doesn't, delete the node immediately.
-				exists, err := ensureNodeExistsByProviderID(instances, node)
-				if err != nil {
-					klog.Errorf("Error checking if node %s exists: %v", node.Name, err)
-					continue
-				}
-
-				if exists {
-					// Continue checking the remaining nodes since the current one is fine.
-					continue
-				}
-
-				klog.V(2).Infof("Deleting node since it is no longer present in cloud provider: %s", node.Name)
-
-				ref := &v1.ObjectReference{
-					Kind:      "Node",
-					Name:      node.Name,
-					UID:       types.UID(node.UID),
-					Namespace: "",
-				}
-				klog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name)
-
-				cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode")
-
-				go func(nodeName string) {
-					defer utilruntime.HandleCrash()
-					if err := cnc.kubeClient.CoreV1().Nodes().Delete(nodeName, nil); err != nil {
-						klog.Errorf("unable to delete node %q: %v", nodeName, err)
-					}
-				}(node.Name)
-
-			} else {
-				// if taint exist remove taint
-				err = controller.RemoveTaintOffNode(cnc.kubeClient, node.Name, node, controller.ShutdownTaint)
-				if err != nil {
-					klog.Errorf("Error patching node taints: %v", err)
-				}
-			}
-		}
-	}
-}
-
 func (cnc *CloudNodeController) UpdateCloudNode(_, newObj interface{}) {
 	if _, ok := newObj.(*v1.Node); !ok {
 		utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", newObj))

pkg/controller/cloud/node_lifecycle_controller.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	coreinformers "k8s.io/client-go/informers/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	v1lister "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/record"
+	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/klog"
+	nodeutilv1 "k8s.io/kubernetes/pkg/api/v1/node"
+	"k8s.io/kubernetes/pkg/controller"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
+)
+
+const (
+	deleteNodeEvent = "DeletingNode"
+)
+
+var ShutdownTaint = &v1.Taint{
+	Key:    schedulerapi.TaintNodeShutdown,
+	Effect: v1.TaintEffectNoSchedule,
+}
+
+// CloudNodeLifecycleController is responsible for deleting/updating kubernetes
+// nodes that have been deleted/shutdown on the cloud provider
+type CloudNodeLifecycleController struct {
+	kubeClient clientset.Interface
+	nodeLister v1lister.NodeLister
+	recorder   record.EventRecorder
+
+	cloud cloudprovider.Interface
+
+	// Value controlling NodeController monitoring period, i.e. how often does NodeController
+	// check node status posted from kubelet. This value should be lower than nodeMonitorGracePeriod
+	// set in controller-manager
+	nodeMonitorPeriod time.Duration
+}
+
+func NewCloudNodeLifecycleController(
+	nodeInformer coreinformers.NodeInformer,
+	kubeClient clientset.Interface,
+	cloud cloudprovider.Interface,
+	nodeMonitorPeriod time.Duration) (*CloudNodeLifecycleController, error) {
+
+	eventBroadcaster := record.NewBroadcaster()
+	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-lifecycle-controller"})
+	eventBroadcaster.StartLogging(klog.Infof)
+
+	klog.Info("Sending events to api server")
+	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
+
+	if kubeClient == nil {
+		return nil, errors.New("kubernetes client is nil")
+	}
+
+	if cloud == nil {
+		return nil, errors.New("no cloud provider provided")
+	}
+
+	if _, ok := cloud.Instances(); !ok {
+		return nil, errors.New("cloud provider does not support instances")
+	}
+
+	c := &CloudNodeLifecycleController{
+		kubeClient:        kubeClient,
+		nodeLister:        nodeInformer.Lister(),
+		recorder:          recorder,
+		cloud:             cloud,
+		nodeMonitorPeriod: nodeMonitorPeriod,
+	}
+
+	return c, nil
+}
+
+// Run starts the main loop for this controller. Run is blocking so should
+// be called via a goroutine
+func (c *CloudNodeLifecycleController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+
+	// The following loops run communicate with the APIServer with a worst case complexity
+	// of O(num_nodes) per cycle. These functions are justified here because these events fire
+	// very infrequently. DO NOT MODIFY this to perform frequent operations.
+
+	// Start a loop to periodically check if any nodes have been
+	// deleted or shutdown from the cloudprovider
+	wait.Until(c.MonitorNodes, c.nodeMonitorPeriod, stopCh)
+}
+
+// MonitorNodes checks to see if nodes in the cluster have been deleted
+// or shutdown. If deleeted, it deletes the node resource. If shutdown it
+// applies a shutdown taint to the node
+func (c *CloudNodeLifecycleController) MonitorNodes() {
+	instances, ok := c.cloud.Instances()
+	if !ok {
+		utilruntime.HandleError(fmt.Errorf("failed to get instances from cloud provider"))
+		return
+	}
+
+	nodes, err := c.nodeLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("error listing nodes from cache: %s", err)
+		return
+	}
+
+	for _, node := range nodes {
+		// Default NodeReady status to v1.ConditionUnknown
+		status := v1.ConditionUnknown
+		if _, c := nodeutilv1.GetNodeCondition(&node.Status, v1.NodeReady); c != nil {
+			status = c.Status
+		}
+
+		if status == v1.ConditionTrue {
+			// if taint exist remove taint
+			err = controller.RemoveTaintOffNode(c.kubeClient, node.Name, node, ShutdownTaint)
+			if err != nil {
+				klog.Errorf("error patching node taints: %v", err)
+			}
+			continue
+		}
+
+		// node condition is unknown so we should not try to update it
+		if status == v1.ConditionUnknown {
+			continue
+		}
+
+		// status for NodeReady should be false at this point, this check
+		// is here as a fail safe in the case a new node condition is added
+		// without consideration of this controller
+		if status != v1.ConditionFalse {
+			continue
+		}
+
+		// we need to check this first to get taint working in similar in all cloudproviders
+		// current problem is that shutdown nodes are not working in similar way ie. all cloudproviders
+		// does not delete node from kubernetes cluster when instance it is shutdown see issue #46442
+		shutdown, err := shutdownInCloudProvider(context.TODO(), c.cloud, node)
+		if err != nil {
+			klog.Errorf("error checking if node %s is shutdown: %v", node.Name, err)
+		}
+
+		if shutdown && err == nil {
+			// if node is shutdown add shutdown taint
+			err = controller.AddOrUpdateTaintOnNode(c.kubeClient, node.Name, ShutdownTaint)
+			if err != nil {
+				klog.Errorf("failed to apply shutdown taint to node %s, it may have been deleted.", node.Name)
+			}
+			// Continue checking the remaining nodes since the current one is shutdown.
+			continue
+		}
+
+		// At this point the node has NotReady status, we need to check if the node has been removed
+		// from the cloud provider. If node cannot be found in cloudprovider, then delete the node
+		exists, err := ensureNodeExistsByProviderID(instances, node)
+		if err != nil {
+			klog.Errorf("error checking if node %s exists: %v", node.Name, err)
+			continue
+		}
+
+		if exists {
+			// Continue checking the remaining nodes since the current one is fine.
+			continue
+		}
+
+		klog.V(2).Infof("deleting node since it is no longer present in cloud provider: %s", node.Name)
+
+		ref := &v1.ObjectReference{
+			Kind:      "Node",
+			Name:      node.Name,
+			UID:       types.UID(node.UID),
+			Namespace: "",
+		}
+
+		c.recorder.Eventf(ref, v1.EventTypeNormal,
+			fmt.Sprintf("Deleting node %v because it does not exist in the cloud provider", node.Name),
+			"Node %s event: %s", node.Name, deleteNodeEvent)
+
+		if err := c.kubeClient.CoreV1().Nodes().Delete(node.Name, nil); err != nil {
+			klog.Errorf("unable to delete node %q: %v", node.Name, err)
+		}
+	}
+}
+
+// shutdownInCloudProvider returns true if the node is shutdown on the cloud provider
+func shutdownInCloudProvider(ctx context.Context, cloud cloudprovider.Interface, node *v1.Node) (bool, error) {
+	instances, ok := cloud.Instances()
+	if !ok {
+		return false, errors.New("cloud provider does not support instances")
+	}
+
+	shutdown, err := instances.InstanceShutdownByProviderID(ctx, node.Spec.ProviderID)
+	if err == cloudprovider.NotImplemented {
+		return false, nil
+	}
+
+	return shutdown, err
+}

pkg/controller/cloud/pvlcontroller.go

@@ -25,7 +25,6 @@ import (
 	"k8s.io/klog"
 
 	"k8s.io/api/core/v1"
-
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -35,18 +34,16 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	"k8s.io/kubernetes/pkg/features"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
-	volumeutil "k8s.io/kubernetes/pkg/volume/util"
-
 	"k8s.io/client-go/kubernetes"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
-
 	cloudprovider "k8s.io/cloud-provider"
+	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/features"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 )
 
 const initializerName = "pvlabel.kubernetes.io"

pkg/controller/controller_utils.go

@@ -47,7 +47,6 @@ import (
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	"k8s.io/kubernetes/pkg/apis/core/validation"
-	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	hashutil "k8s.io/kubernetes/pkg/util/hash"
 	taintutils "k8s.io/kubernetes/pkg/util/taints"
 
@@ -89,11 +88,6 @@ var UpdateTaintBackoff = wait.Backoff{
 	Jitter:   1.0,
 }
 
-var ShutdownTaint = &v1.Taint{
-	Key:    schedulerapi.TaintNodeShutdown,
-	Effect: v1.TaintEffectNoSchedule,
-}
-
 var (
 	KeyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
 )

pkg/controller/nodelifecycle/BUILD

@@ -25,7 +25,6 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
@@ -42,7 +41,6 @@ go_library(
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
-        "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],
@@ -70,7 +68,6 @@ go_test(
     srcs = ["node_lifecycle_controller_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/cloudprovider/providers/fake:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/nodelifecycle/scheduler:go_default_library",
         "//pkg/controller/testutil:go_default_library",
@@ -86,9 +83,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
@@ -98,7 +93,6 @@ go_test(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//staging/src/k8s.io/client-go/testing:go_default_library",
-        "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )

pkg/controller/nodelifecycle/node_lifecycle_controller.go

@@ -22,7 +22,6 @@ limitations under the License.
 package nodelifecycle
 
 import (
-	"context"
 	"fmt"
 	"hash/fnv"
 	"io"
@@ -37,7 +36,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -54,7 +52,6 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/client-go/util/workqueue"
-	cloudprovider "k8s.io/cloud-provider"
 	v1node "k8s.io/kubernetes/pkg/api/v1/node"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/controller/nodelifecycle/scheduler"
@@ -150,7 +147,6 @@ type Controller struct {
 	taintManager *scheduler.NoExecuteTaintManager
 
 	podInformerSynced cache.InformerSynced
-	cloud             cloudprovider.Interface
 	kubeClient        clientset.Interface
 
 	// This timestamp is to be used instead of LastProbeTime stored in Condition. We do this
@@ -179,12 +175,10 @@ type Controller struct {
 	daemonSetStore          extensionslisters.DaemonSetLister
 	daemonSetInformerSynced cache.InformerSynced
 
-	leaseLister                 coordlisters.LeaseLister
-	leaseInformerSynced         cache.InformerSynced
-	nodeLister                  corelisters.NodeLister
-	nodeInformerSynced          cache.InformerSynced
-	nodeExistsInCloudProvider   func(types.NodeName) (bool, error)
-	nodeShutdownInCloudProvider func(context.Context, *v1.Node) (bool, error)
+	leaseLister         coordlisters.LeaseLister
+	leaseInformerSynced cache.InformerSynced
+	nodeLister          corelisters.NodeLister
+	nodeInformerSynced  cache.InformerSynced
 
 	recorder record.EventRecorder
 
@@ -247,7 +241,6 @@ func NewNodeLifecycleController(
 	podInformer coreinformers.PodInformer,
 	nodeInformer coreinformers.NodeInformer,
 	daemonSetInformer extensionsinformers.DaemonSetInformer,
-	cloud cloudprovider.Interface,
 	kubeClient clientset.Interface,
 	nodeMonitorPeriod time.Duration,
 	nodeStartupGracePeriod time.Duration,
@@ -280,17 +273,10 @@ func NewNodeLifecycleController(
 	}
 
 	nc := &Controller{
-		cloud:         cloud,
-		kubeClient:    kubeClient,
-		now:           metav1.Now,
-		knownNodeSet:  make(map[string]*v1.Node),
-		nodeHealthMap: make(map[string]*nodeHealthData),
-		nodeExistsInCloudProvider: func(nodeName types.NodeName) (bool, error) {
-			return nodeutil.ExistsInCloudProvider(cloud, nodeName)
-		},
-		nodeShutdownInCloudProvider: func(ctx context.Context, node *v1.Node) (bool, error) {
-			return nodeutil.ShutdownInCloudProvider(ctx, cloud, node)
-		},
+		kubeClient:                  kubeClient,
+		now:                         metav1.Now,
+		knownNodeSet:                make(map[string]*v1.Node),
+		nodeHealthMap:               make(map[string]*nodeHealthData),
 		recorder:                    recorder,
 		nodeMonitorPeriod:           nodeMonitorPeriod,
 		nodeStartupGracePeriod:      nodeStartupGracePeriod,
@@ -779,11 +765,6 @@ func (nc *Controller) monitorNodeHealth() error {
 						klog.V(2).Infof("Node %s is ready again, cancelled pod eviction", node.Name)
 					}
 				}
-				// remove shutdown taint this is needed always depending do we use taintbased or not
-				err := nc.markNodeAsNotShutdown(node)
-				if err != nil {
-					klog.Errorf("Failed to remove taints from node %v. Will retry in next iteration.", node.Name)
-				}
 			}
 
 			// Report node event.
@@ -793,42 +774,6 @@ func (nc *Controller) monitorNodeHealth() error {
 					utilruntime.HandleError(fmt.Errorf("Unable to mark all pods NotReady on node %v: %v", node.Name, err))
 				}
 			}
-
-			// Check with the cloud provider to see if the node still exists. If it
-			// doesn't, delete the node immediately.
-			if currentReadyCondition.Status != v1.ConditionTrue && nc.cloud != nil {
-				// check is node shutdowned, if yes do not deleted it. Instead add taint
-				shutdown, err := nc.nodeShutdownInCloudProvider(context.TODO(), node)
-				if err != nil {
-					klog.Errorf("Error determining if node %v shutdown in cloud: %v", node.Name, err)
-				}
-				// node shutdown
-				if shutdown && err == nil {
-					err = controller.AddOrUpdateTaintOnNode(nc.kubeClient, node.Name, controller.ShutdownTaint)
-					if err != nil {
-						klog.Errorf("Error patching node taints: %v", err)
-					}
-					continue
-				}
-				exists, err := nc.nodeExistsInCloudProvider(types.NodeName(node.Name))
-				if err != nil {
-					klog.Errorf("Error determining if node %v exists in cloud: %v", node.Name, err)
-					continue
-				}
-				if !exists {
-					klog.V(2).Infof("Deleting node (no longer present in cloud provider): %s", node.Name)
-					nodeutil.RecordNodeEvent(nc.recorder, node.Name, string(node.UID), v1.EventTypeNormal, "DeletingNode", fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name))
-					go func(nodeName string) {
-						defer utilruntime.HandleCrash()
-						// Kubelet is not reporting and Cloud Provider says node
-						// is gone. Delete it without worrying about grace
-						// periods.
-						if err := nodeutil.ForcefullyDeleteNode(nc.kubeClient, nodeName); err != nil {
-							klog.Errorf("Unable to forcefully delete node %q: %v", nodeName, err)
-						}
-					}(node.Name)
-				}
-			}
 		}
 	}
 	nc.handleDisruption(zoneToNodeConditions, nodes)
@@ -1268,17 +1213,6 @@ func (nc *Controller) markNodeAsReachable(node *v1.Node) (bool, error) {
 	return nc.zoneNoExecuteTainter[utilnode.GetZoneKey(node)].Remove(node.Name), nil
 }
 
-func (nc *Controller) markNodeAsNotShutdown(node *v1.Node) error {
-	nc.evictorLock.Lock()
-	defer nc.evictorLock.Unlock()
-	err := controller.RemoveTaintOffNode(nc.kubeClient, node.Name, node, controller.ShutdownTaint)
-	if err != nil {
-		klog.Errorf("Failed to remove taint from node %v: %v", node.Name, err)
-		return err
-	}
-	return nil
-}
-
 // ComputeZoneState returns a slice of NodeReadyConditions for all Nodes in a given zone.
 // The zone is considered:
 // - fullyDisrupted if there're no Ready Nodes,

pkg/controller/util/node/BUILD

@@ -21,7 +21,6 @@ go_library(
         "//staging/src/k8s.io/client-go/listers/extensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
-        "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],
 )

pkg/controller/util/node/controller_utils.go

@@ -17,8 +17,6 @@ limitations under the License.
 package node
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"strings"
 
@@ -34,7 +32,6 @@ import (
 	"k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	extensionslisters "k8s.io/client-go/listers/extensions/v1beta1"
-	cloudprovider "k8s.io/cloud-provider"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
@@ -43,12 +40,6 @@ import (
 	"k8s.io/klog"
 )
 
-var (
-	// ErrCloudInstance occurs when the cloud provider does not support
-	// the Instances API.
-	ErrCloudInstance = errors.New("cloud provider doesn't support instances")
-)
-
 // DeletePods will delete all pods from master running on given node,
 // and return true if any pods were deleted, or were found pending
 // deletion.
@@ -125,15 +116,6 @@ func SetPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeNa
 	return updatedPod, nil
 }
 
-// ForcefullyDeleteNode deletes the node immediately. The pods on the
-// node are cleaned up by the podGC.
-func ForcefullyDeleteNode(kubeClient clientset.Interface, nodeName string) error {
-	if err := kubeClient.CoreV1().Nodes().Delete(nodeName, nil); err != nil {
-		return fmt.Errorf("unable to delete node %q: %v", nodeName, err)
-	}
-	return nil
-}
-
 // MarkAllPodsNotReady updates ready status of all pods running on
 // given node from master return true if success
 func MarkAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error {
@@ -171,36 +153,6 @@ func MarkAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error {
 	return fmt.Errorf("%v", strings.Join(errMsg, "; "))
 }
 
-// ExistsInCloudProvider returns true if the node exists in the
-// cloud provider.
-func ExistsInCloudProvider(cloud cloudprovider.Interface, nodeName types.NodeName) (bool, error) {
-	instances, ok := cloud.Instances()
-	if !ok {
-		return false, fmt.Errorf("%v", ErrCloudInstance)
-	}
-	if _, err := instances.InstanceID(context.TODO(), nodeName); err != nil {
-		if err == cloudprovider.InstanceNotFound {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
-}
-
-// ShutdownInCloudProvider returns true if the node is shutdowned in
-// cloud provider.
-func ShutdownInCloudProvider(ctx context.Context, cloud cloudprovider.Interface, node *v1.Node) (bool, error) {
-	instances, ok := cloud.Instances()
-	if !ok {
-		return false, fmt.Errorf("%v", ErrCloudInstance)
-	}
-	shutdown, err := instances.InstanceShutdownByProviderID(ctx, node.Spec.ProviderID)
-	if err == cloudprovider.NotImplemented {
-		return false, nil
-	}
-	return shutdown, err
-}
-
 // RecordNodeEvent records a event related to a node.
 func RecordNodeEvent(recorder record.EventRecorder, nodeName, nodeUID, eventtype, reason, event string) {
 	ref := &v1.ObjectReference{

test/integration/scheduler/taint_test.go

@@ -99,7 +99,6 @@ func TestTaintNodeByCondition(t *testing.T) {
 		informers.Core().V1().Pods(),
 		informers.Core().V1().Nodes(),
 		informers.Extensions().V1beta1().DaemonSets(),
-		nil, // CloudProvider
 		cs,
 		time.Hour,   // Node monitor grace period
 		time.Second, // Node startup grace period