Merge remote-tracking branch 'upstream/master' into test-cmd-what

cluster/gce/util.sh

@@ -1345,6 +1345,11 @@ EOF
 ADDON_MANAGER_LEADER_ELECTION: $(yaml-quote ${ADDON_MANAGER_LEADER_ELECTION})
 EOF
     fi
+    if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
+      cat >>$file <<EOF
+API_SERVER_TEST_LOG_LEVEL: $(yaml-quote ${API_SERVER_TEST_LOG_LEVEL})
+EOF
+    fi
 
   else
     # Node-only env vars.

cluster/log-dump/log-dump.sh

@@ -51,6 +51,7 @@ readonly kern_logfile="kern.log"
 readonly initd_logfiles="docker/log"
 readonly supervisord_logfiles="kubelet.log supervisor/supervisord.log supervisor/kubelet-stdout.log supervisor/kubelet-stderr.log supervisor/docker-stdout.log supervisor/docker-stderr.log"
 readonly systemd_services="kubelet kubelet-monitor kube-container-runtime-monitor ${LOG_DUMP_SYSTEMD_SERVICES:-docker}"
+readonly dump_systemd_journal="${LOG_DUMP_SYSTEMD_JOURNAL:-false}"
 
 # Limit the number of concurrent node connections so that we don't run out of
 # file descriptors for large clusters.
@@ -164,6 +165,10 @@ function save-logs() {
         for svc in "${services[@]}"; do
             log-dump-ssh "${node_name}" "sudo journalctl --output=cat -u ${svc}.service" > "${dir}/${svc}.log" || true
         done
+
+        if [[ "$dump_systemd_journal" == "true" ]]; then
+          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise" > "${dir}/systemd.log" || true
+        fi
     else
         files="${kern_logfile} ${files} ${initd_logfiles} ${supervisord_logfiles}"
     fi

cmd/kube-apiserver/app/options/options.go

@@ -212,11 +212,6 @@ func (s *ServerRunOptions) Flags() (fss apiserverflag.NamedFlagSets) {
 	fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile,
 		"Path to a cert file for the certificate authority.")
 
-	// TODO: delete this flag in 1.13
-	repair := false
-	fs.BoolVar(&repair, "repair-malformed-updates", false, "deprecated")
-	fs.MarkDeprecated("repair-malformed-updates", "This flag will be removed in a future version")
-
 	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
 		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
 		"when it must call out during a request. This includes proxying requests to a user "+

cmd/kube-controller-manager/app/core.go

@@ -36,6 +36,7 @@ import (
 	cacheddiscovery "k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	"k8s.io/kubernetes/pkg/controller"
 	cloudcontroller "k8s.io/kubernetes/pkg/controller/cloud"
@@ -331,6 +332,10 @@ func startNamespaceController(ctx ControllerContext) (http.Handler, bool, error)
 	nsKubeconfig.QPS *= 20
 	nsKubeconfig.Burst *= 100
 	namespaceKubeClient := clientset.NewForConfigOrDie(nsKubeconfig)
+	return startModifiedNamespaceController(ctx, namespaceKubeClient, nsKubeconfig)
+}
+
+func startModifiedNamespaceController(ctx ControllerContext, namespaceKubeClient clientset.Interface, nsKubeconfig *restclient.Config) (http.Handler, bool, error) {
 
 	dynamicClient, err := dynamic.NewForConfig(nsKubeconfig)
 	if err != nil {

cmd/kube-controller-manager/app/core_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"net/http"
 	"testing"
 	"time"
 
@@ -34,8 +35,11 @@ type TestClientBuilder struct {
 	clientset clientset.Interface
 }
 
-func (TestClientBuilder) Config(name string) (*restclient.Config, error)  { return nil, nil }
-func (TestClientBuilder) ConfigOrDie(name string) *restclient.Config      { return nil }
+func (TestClientBuilder) Config(name string) (*restclient.Config, error) { return nil, nil }
+func (TestClientBuilder) ConfigOrDie(name string) *restclient.Config {
+	return &restclient.Config{}
+}
+
 func (TestClientBuilder) Client(name string) (clientset.Interface, error) { return nil, nil }
 func (m TestClientBuilder) ClientOrDie(name string) clientset.Interface {
 	return m.clientset
@@ -62,6 +66,10 @@ func (c *FakeClientSet) Discovery() discovery.DiscoveryInterface {
 	return c.DiscoveryObj
 }
 
+func (c *FakeClientSet) GetPossibleResources() []*metav1.APIResourceList {
+	return c.DiscoveryObj.PossibleResources
+}
+
 // Create a fake Clientset with its Discovery method overridden.
 func NewFakeClientset(fakeDiscovery FakeDiscoveryWithError) *FakeClientSet {
 	cs := &FakeClientSet{}
@@ -69,7 +77,30 @@ func NewFakeClientset(fakeDiscovery FakeDiscoveryWithError) *FakeClientSet {
 	return cs
 }
 
-func TestStartResourceQuotaController_DiscoveryError(t *testing.T) {
+func possibleDiscoveryResource() []*metav1.APIResourceList {
+	return []*metav1.APIResourceList{
+		{
+			GroupVersion: "create/v1",
+			APIResources: []metav1.APIResource{
+				{
+					Name:       "jobs",
+					Verbs:      []string{"create", "list", "watch", "delete"},
+					ShortNames: []string{"jz"},
+					Categories: []string{"all"},
+				},
+			},
+		},
+	}
+}
+
+type controllerInitFunc func(ControllerContext) (http.Handler, bool, error)
+
+func TestController_DiscoveryError(t *testing.T) {
+	controllerInitFuncMap := map[string]controllerInitFunc{
+		"ResourceQuotaController":    startResourceQuotaController,
+		"GarbageCollectorController": startGarbageCollectorController,
+	}
+
 	tcs := map[string]struct {
 		discoveryError    error
 		expectedErr       bool
@@ -77,25 +108,13 @@ func TestStartResourceQuotaController_DiscoveryError(t *testing.T) {
 	}{
 		"No Discovery Error": {
 			discoveryError:    nil,
-			possibleResources: nil,
+			possibleResources: possibleDiscoveryResource(),
 			expectedErr:       false,
 		},
 		"Discovery Calls Partially Failed": {
-			discoveryError: new(discovery.ErrGroupDiscoveryFailed),
-			possibleResources: []*metav1.APIResourceList{
-				{
-					GroupVersion: "create/v1",
-					APIResources: []metav1.APIResource{
-						{
-							Name:       "jobs",
-							Verbs:      []string{"create", "list", "watch", "delete"},
-							ShortNames: []string{"jz"},
-							Categories: []string{"all"},
-						},
-					},
-				},
-			},
-			expectedErr: false,
+			discoveryError:    new(discovery.ErrGroupDiscoveryFailed),
+			possibleResources: possibleDiscoveryResource(),
+			expectedErr:       false,
 		},
 	}
 	for name, test := range tcs {
@@ -107,9 +126,16 @@ func TestStartResourceQuotaController_DiscoveryError(t *testing.T) {
 			InformerFactory:  informers.NewSharedInformerFactoryWithOptions(testClientset, time.Duration(1)),
 			InformersStarted: make(chan struct{}),
 		}
-		_, _, err := startResourceQuotaController(ctx)
+		for funcName, controllerInit := range controllerInitFuncMap {
+			_, _, err := controllerInit(ctx)
+			if test.expectedErr != (err != nil) {
+				t.Errorf("%v test failed for use case: %v", funcName, name)
+			}
+		}
+		_, _, err := startModifiedNamespaceController(
+			ctx, testClientset, testClientBuilder.ConfigOrDie("namespace-controller"))
 		if test.expectedErr != (err != nil) {
-			t.Errorf("test failed for use case: %v", name)
+			t.Errorf("Namespace Controller test failed for use case: %v", name)
 		}
 	}
 }

cmd/kubeadm/.import-restrictions

@@ -42,7 +42,8 @@
 				"k8s.io/utils/exec",
 				"k8s.io/utils/integer",
 				"k8s.io/utils/path",
-				"k8s.io/utils/pointer"
+				"k8s.io/utils/pointer",
+				"k8s.io/utils/net"
 			]
 		},
 		{
@@ -82,7 +83,6 @@
 				"k8s.io/kubernetes/pkg/util/initsystem",
 				"k8s.io/kubernetes/pkg/util/ipvs",
 				"k8s.io/kubernetes/pkg/util/metrics",
-				"k8s.io/kubernetes/pkg/util/net/sets",
 				"k8s.io/kubernetes/pkg/util/node",
 				"k8s.io/kubernetes/pkg/util/normalizer",
 				"k8s.io/kubernetes/pkg/util/parsers",

cmd/kubeadm/app/cmd/BUILD

@@ -28,6 +28,8 @@ go_library(
         "//cmd/kubeadm/app/cmd/alpha:go_default_library",
         "//cmd/kubeadm/app/cmd/options:go_default_library",
         "//cmd/kubeadm/app/cmd/phases:go_default_library",
+        "//cmd/kubeadm/app/cmd/phases/init:go_default_library",
+        "//cmd/kubeadm/app/cmd/phases/join:go_default_library",
         "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
         "//cmd/kubeadm/app/cmd/upgrade:go_default_library",
         "//cmd/kubeadm/app/cmd/util:go_default_library",
@@ -38,7 +40,6 @@ go_library(
         "//cmd/kubeadm/app/images:go_default_library",
         "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
-        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",

cmd/kubeadm/app/cmd/alpha/BUILD

@@ -7,7 +7,6 @@ go_library(
         "certs.go",
         "kubeconfig.go",
         "kubelet.go",
-        "preflight.go",
         "selfhosting.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha",

cmd/kubeadm/app/cmd/alpha/alpha.go

@@ -33,7 +33,6 @@ func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command {
 	cmd.AddCommand(newCmdCertsUtility())
 	cmd.AddCommand(newCmdKubeletUtility())
 	cmd.AddCommand(newCmdKubeConfigUtility(out))
-	cmd.AddCommand(newCmdPreFlightUtility())
 	cmd.AddCommand(NewCmdSelfhosting(in))
 
 	// TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed.

cmd/kubeadm/app/cmd/alpha/preflight.go

-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package alpha
-
-import (
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/cobra"
-	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
-	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
-	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
-	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
-	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
-	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
-	"k8s.io/kubernetes/pkg/util/normalizer"
-	utilsexec "k8s.io/utils/exec"
-)
-
-var (
-	nodePreflightLongDesc = normalizer.LongDesc(`
-		Run node pre-flight checks, functionally equivalent to what implemented by kubeadm join.
-		` + cmdutil.AlphaDisclaimer)
-
-	nodePreflightExample = normalizer.Examples(`
-		# Run node pre-flight checks.
-		kubeadm alpha preflight node
-	`)
-
-	errorMissingConfigFlag = errors.New("the --config flag is mandatory")
-)
-
-// newCmdPreFlightUtility calls cobra.Command for preflight checks
-func newCmdPreFlightUtility() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "preflight",
-		Short: "Commands related to pre-flight checks",
-		Long:  cmdutil.MacroCommandLongDescription,
-	}
-
-	cmd.AddCommand(newCmdPreFlightNode())
-
-	return cmd
-}
-
-// newCmdPreFlightNode calls cobra.Command for node preflight checks
-func newCmdPreFlightNode() *cobra.Command {
-	var cfgPath string
-	var ignorePreflightErrors []string
-
-	cmd := &cobra.Command{
-		Use:     "node",
-		Short:   "Run node pre-flight checks",
-		Long:    nodePreflightLongDesc,
-		Example: nodePreflightExample,
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(cfgPath) == 0 {
-				kubeadmutil.CheckErr(errorMissingConfigFlag)
-			}
-			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors)
-			kubeadmutil.CheckErr(err)
-
-			cfg := &kubeadmapiv1beta1.JoinConfiguration{}
-			kubeadmscheme.Scheme.Default(cfg)
-
-			internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
-			kubeadmutil.CheckErr(err)
-
-			if internalcfg.ControlPlane != nil {
-				err = configutil.VerifyAPIServerBindAddress(internalcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress)
-				kubeadmutil.CheckErr(err)
-			}
-
-			fmt.Println("[preflight] running pre-flight checks")
-
-			err = preflight.RunJoinNodeChecks(utilsexec.New(), internalcfg, ignorePreflightErrorsSet)
-			kubeadmutil.CheckErr(err)
-
-			fmt.Println("[preflight] pre-flight checks passed")
-		},
-	}
-
-	options.AddConfigFlag(cmd.PersistentFlags(), &cfgPath)
-	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)
-
-	return cmd
-}

cmd/kubeadm/app/cmd/alpha/selfhosting.go

@@ -141,7 +141,7 @@ func getSelfhostingSubCommand(in io.Reader) *cobra.Command {
 	// Add flags to the command
 	// flags bound to the configuration object
 	cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`)
-	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental")
+	options.AddConfigFlag(cmd.Flags(), &cfgPath)
 
 	cmd.Flags().BoolVarP(
 		&certsInSecrets, "store-certs-in-secrets", "s",

cmd/kubeadm/app/cmd/config.go

@@ -329,7 +329,7 @@ func NewCmdConfigUploadFromFile(out io.Writer, kubeConfigFile *string) *cobra.Co
 			kubeadmutil.CheckErr(err)
 		},
 	}
-	cmd.Flags().StringVar(&cfgPath, "config", "", "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental.")
+	options.AddConfigFlag(cmd.Flags(), &cfgPath)
 	return cmd
 }
 

cmd/kubeadm/app/cmd/init.go

@@ -36,7 +36,7 @@ import (
 	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
-	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
+	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -168,7 +168,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	})
 
 	// initialize the workflow runner with the list of phases
-	initRunner.AppendPhase(phases.NewPreflightMasterPhase())
+	initRunner.AppendPhase(phases.NewPreflightPhase())
 	initRunner.AppendPhase(phases.NewKubeletStartPhase())
 	initRunner.AppendPhase(phases.NewCertsPhase())
 	initRunner.AppendPhase(phases.NewKubeConfigPhase())
@@ -238,10 +238,7 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.InitConfig
 
 // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset
 func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipTokenPrint, dryRun *bool, ignorePreflightErrors *[]string) {
-	flagSet.StringVar(
-		cfgPath, options.CfgPath, *cfgPath,
-		"Path to kubeadm config file. WARNING: Usage of a configuration file is experimental.",
-	)
+	options.AddConfigFlag(flagSet, cfgPath)
 	flagSet.StringSliceVar(
 		ignorePreflightErrors, options.IgnorePreflightErrors, *ignorePreflightErrors,
 		"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",

cmd/kubeadm/app/cmd/join.go

@@ -29,6 +29,7 @@ import (
 	flag "github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog"
@@ -37,15 +38,12 @@ import (
 	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
-	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
+	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
-	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
-	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
-	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane"
 	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
@@ -161,6 +159,7 @@ type joinData struct {
 	cfg                   *kubeadmapi.JoinConfiguration
 	initCfg               *kubeadmapi.InitConfiguration
 	tlsBootstrapCfg       *clientcmdapi.Config
+	clientSets            map[string]*clientset.Clientset
 	ignorePreflightErrors sets.String
 	outputWriter          io.Writer
 }
@@ -199,7 +198,9 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 	addJoinConfigFlags(cmd.Flags(), joinOptions.externalcfg)
 	addJoinOtherFlags(cmd.Flags(), &joinOptions.cfgPath, &joinOptions.ignorePreflightErrors, &joinOptions.controlPlane, &joinOptions.token)
 
-	joinRunner.AppendPhase(phases.NewPreflightJoinPhase())
+	joinRunner.AppendPhase(phases.NewPreflightPhase())
+	joinRunner.AppendPhase(phases.NewControlPlanePreparePhase())
+	joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
 
 	// sets the data builder function, that will be used by the runner
 	// both when running the entire workflow or single phases
@@ -373,6 +374,7 @@ func newJoinData(cmd *cobra.Command, args []string, options *joinOptions, out io
 
 	return &joinData{
 		cfg:                   cfg,
+		clientSets:            map[string]*clientset.Clientset{},
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		outputWriter:          out,
 	}, nil
@@ -408,6 +410,18 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) {
 	return initCfg, err
 }
 
+func (j *joinData) ClientSetFromFile(path string) (*clientset.Clientset, error) {
+	if client, ok := j.clientSets[path]; ok {
+		return client, nil
+	}
+	client, err := kubeconfigutil.ClientSetFromFile(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "[join] couldn't create Kubernetes client")
+	}
+	j.clientSets[path] = client
+	return client, nil
+}
+
 // IgnorePreflightErrors returns the list of preflight errors to ignore.
 func (j *joinData) IgnorePreflightErrors() sets.String {
 	return j.ignorePreflightErrors
@@ -432,13 +446,6 @@ func (j *joinData) Run() error {
 	if err != nil {
 		return err
 	}
-	if j.cfg.ControlPlane != nil {
-		// Prepares the node for hosting a new control plane instance by writing necessary
-		// kubeconfig files, and static pod manifests
-		if err := j.PrepareForHostingControlPlane(initCfg); err != nil {
-			return err
-		}
-	}
 
 	// Executes the kubelet TLS bootstrap process, that completes with the node
 	// joining the cluster with a dedicates set of credentials as required by
@@ -478,47 +485,6 @@ func (j *joinData) Run() error {
 	return nil
 }
 
-// PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance
-func (j *joinData) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
-
-	// Generate missing certificates (if any)
-	if err := certsphase.CreatePKIAssets(initConfiguration); err != nil {
-		return err
-	}
-
-	// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
-	// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
-	// following steps of the join --experimental-control plane workflow
-	if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil {
-		return errors.Wrap(err, "error generating kubeconfig files")
-	}
-
-	// Creates static pod manifests file for the control plane components to be deployed on this node
-	// Static pods will be created and managed by the kubelet as soon as it starts
-	if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil {
-		return errors.Wrap(err, "error creating static pod manifest files for the control plane components")
-	}
-
-	// in case of local etcd
-	if initConfiguration.Etcd.External == nil {
-		// Checks that the etcd cluster is healthy
-		// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
-		//     for connecting to etcd already in place
-		kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
-
-		client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
-		if err != nil {
-			return errors.Wrap(err, "couldn't create Kubernetes client")
-		}
-
-		if err := etcdphase.CheckLocalEtcdClusterStatus(client, &initConfiguration.ClusterConfiguration); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // BootstrapKubelet executes the kubelet TLS bootstrap process.
 // This process is executed by the kubelet and completes with the node joining the cluster
 // with a dedicates set of credentials as required by the node authorizer

cmd/kubeadm/app/cmd/phases/BUILD

@@ -2,77 +2,22 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = [
-        "addons.go",
-        "bootstraptoken.go",
-        "certs.go",
-        "controlplane.go",
-        "etcd.go",
-        "kubeconfig.go",
-        "kubelet.go",
-        "markcontrolplane.go",
-        "preflight.go",
-        "uploadconfig.go",
-        "util.go",
-        "waitcontrolplane.go",
-    ],
+    srcs = ["util.go"],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases",
     visibility = ["//visibility:public"],
     deps = [
-        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
-        "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library",
-        "//cmd/kubeadm/app/cmd/options:go_default_library",
-        "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
-        "//cmd/kubeadm/app/cmd/util:go_default_library",
-        "//cmd/kubeadm/app/constants:go_default_library",
-        "//cmd/kubeadm/app/phases/addons/dns:go_default_library",
-        "//cmd/kubeadm/app/phases/addons/proxy:go_default_library",
-        "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
-        "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
-        "//cmd/kubeadm/app/phases/certs:go_default_library",
-        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
-        "//cmd/kubeadm/app/phases/etcd:go_default_library",
-        "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
-        "//cmd/kubeadm/app/phases/kubelet:go_default_library",
-        "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library",
-        "//cmd/kubeadm/app/phases/patchnode:go_default_library",
-        "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
-        "//cmd/kubeadm/app/preflight:go_default_library",
-        "//cmd/kubeadm/app/util:go_default_library",
-        "//cmd/kubeadm/app/util/apiclient:go_default_library",
-        "//cmd/kubeadm/app/util/dryrun:go_default_library",
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
-        "//pkg/util/normalizer:go_default_library",
         "//pkg/version:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/github.com/lithammer/dedent:go_default_library",
-        "//vendor/github.com/pkg/errors:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/klog:go_default_library",
-        "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = [
-        "certs_test.go",
-        "util_test.go",
-    ],
+    srcs = ["util_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library",
-        "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
-        "//cmd/kubeadm/app/phases/certs:go_default_library",
-        "//cmd/kubeadm/app/util/certs:go_default_library",
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
-        "//cmd/kubeadm/test:go_default_library",
         "//pkg/version:go_default_library",
-        "//vendor/github.com/spf13/cobra:go_default_library",
     ],
 )
 
@@ -87,6 +32,8 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//cmd/kubeadm/app/cmd/phases/init:all-srcs",
+        "//cmd/kubeadm/app/cmd/phases/join:all-srcs",
         "//cmd/kubeadm/app/cmd/phases/workflow:all-srcs",
     ],
     tags = ["automanaged"],

cmd/kubeadm/app/cmd/phases/init/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "addons.go",
+        "bootstraptoken.go",
+        "certs.go",
+        "controlplane.go",
+        "etcd.go",
+        "kubeconfig.go",
+        "kubelet.go",
+        "markcontrolplane.go",
+        "preflight.go",
+        "uploadconfig.go",
+        "waitcontrolplane.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
+        "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library",
+        "//cmd/kubeadm/app/cmd/options:go_default_library",
+        "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
+        "//cmd/kubeadm/app/cmd/util:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/addons/dns:go_default_library",
+        "//cmd/kubeadm/app/phases/addons/proxy:go_default_library",
+        "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
+        "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
+        "//cmd/kubeadm/app/phases/etcd:go_default_library",
+        "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/phases/kubelet:go_default_library",
+        "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library",
+        "//cmd/kubeadm/app/phases/patchnode:go_default_library",
+        "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
+        "//cmd/kubeadm/app/preflight:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/app/util/dryrun:go_default_library",
+        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
+        "//pkg/util/normalizer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["certs_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/util/certs:go_default_library",
+        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
+        "//cmd/kubeadm/test:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/kubeadm/app/cmd/phases/controlplane.go → cmd/kubeadm/app/cmd/phases/init/controlplane.go

@@ -32,7 +32,7 @@ import (
 
 var (
 	controlPlaneExample = normalizer.Examples(`
-		# Generates all static Pod manifest files for control plane components, 
+		# Generates all static Pod manifest files for control plane components,
 		# functionally equivalent to what is generated by kubeadm init.
 		kubeadm init phase control-plane all
 
@@ -83,20 +83,20 @@ func NewControlPlanePhase() workflow.Phase {
 				Example:        controlPlaneExample,
 				RunAllSiblings: true,
 			},
-			newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer),
-			newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager),
-			newControlPlaneSubPhase(kubeadmconstants.KubeScheduler),
+			newControlPlaneSubphase(kubeadmconstants.KubeAPIServer),
+			newControlPlaneSubphase(kubeadmconstants.KubeControllerManager),
+			newControlPlaneSubphase(kubeadmconstants.KubeScheduler),
 		},
 		Run: runControlPlanePhase,
 	}
 	return phase
 }
 
-func newControlPlaneSubPhase(component string) workflow.Phase {
+func newControlPlaneSubphase(component string) workflow.Phase {
 	phase := workflow.Phase{
 		Name:         controlPlanePhaseProperties[component].name,
 		Short:        controlPlanePhaseProperties[component].short,
-		Run:          runControlPlaneSubPhase(component),
+		Run:          runControlPlaneSubphase(component),
 		InheritFlags: getControlPlanePhaseFlags(component),
 	}
 	return phase
@@ -142,7 +142,7 @@ func runControlPlanePhase(c workflow.RunData) error {
 	return nil
 }
 
-func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
+func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
 	return func(c workflow.RunData) error {
 		data, ok := c.(controlPlaneData)
 		if !ok {

cmd/kubeadm/app/cmd/phases/init/preflight.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+	utilsexec "k8s.io/utils/exec"
+)
+
+var (
+	preflightExample = normalizer.Examples(`
+		# Run pre-flight checks for kubeadm init using a config file.
+		kubeadm init phase preflight --config kubeadm-config.yml
+		`)
+)
+
+// preflightData defines the behavior that a runtime data struct passed to the Preflight phase
+// should have. Please note that we are using an interface in order to make this phase reusable in different workflows
+// (and thus with different runtime data struct, all of them requested to be compliant to this interface)
+type preflightData interface {
+	Cfg() *kubeadmapi.InitConfiguration
+	DryRun() bool
+	IgnorePreflightErrors() sets.String
+}
+
+// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new control-plane node.
+func NewPreflightPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:    "preflight",
+		Short:   "Run pre-flight checks",
+		Long:    "Run pre-flight checks for kubeadm init.",
+		Example: preflightExample,
+		Run:     runPreflight,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.IgnorePreflightErrors,
+		},
+	}
+}
+
+// runPreflight executes preflight checks logic.
+func runPreflight(c workflow.RunData) error {
+	data, ok := c.(preflightData)
+	if !ok {
+		return errors.New("preflight phase invoked with an invalid data struct")
+	}
+
+	fmt.Println("[preflight] Running pre-flight checks")
+	if err := preflight.RunInitMasterChecks(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
+		return err
+	}
+
+	if !data.DryRun() {
+		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
+		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
+		fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
+		if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
+			return err
+		}
+	} else {
+		fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
+	}
+
+	return nil
+}

cmd/kubeadm/app/cmd/phases/join/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "checketcd.go",
+        "controlplane.go",
+        "preflight.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/options:go_default_library",
+        "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library",
+        "//cmd/kubeadm/app/cmd/util:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
+        "//cmd/kubeadm/app/phases/etcd:go_default_library",
+        "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/preflight:go_default_library",
+        "//pkg/util/normalizer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/kubeadm/app/cmd/phases/join/checketcd.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
+)
+
+type checkEtcdData interface {
+	Cfg() *kubeadmapi.JoinConfiguration
+	ClientSetFromFile(string) (*clientset.Clientset, error)
+	InitCfg() (*kubeadmapi.InitConfiguration, error)
+}
+
+// NewCheckEtcdPhase is a hidden phase that runs after the control-plane-prepare and
+// before the bootstrap-kubelet phase that ensures etcd is healthy
+func NewCheckEtcdPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:   "check-etcd",
+		Run:    runCheckEtcdPhase,
+		Hidden: true,
+	}
+}
+
+func runCheckEtcdPhase(c workflow.RunData) error {
+	data, ok := c.(checkEtcdData)
+	if !ok {
+		return errors.New("check-etcd phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	if cfg.Etcd.External != nil {
+		fmt.Println("[check-etcd] Skipping etcd check in external mode")
+		return nil
+	}
+
+	fmt.Println("[check-etcd] Checking that the etcd cluster is healthy")
+
+	// Checks that the etcd cluster is healthy
+	// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
+	//     for connecting to etcd already in place
+	client, err := data.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
+	if err != nil {
+		return err
+	}
+
+	return etcdphase.CheckLocalEtcdClusterStatus(client, &cfg.ClusterConfiguration)
+}

cmd/kubeadm/app/cmd/phases/join/controlplane.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
+	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
+)
+
+type controlPlanePrepareData interface {
+	Cfg() *kubeadmapi.JoinConfiguration
+	ClientSetFromFile(string) (*clientset.Clientset, error)
+	InitCfg() (*kubeadmapi.InitConfiguration, error)
+}
+
+// NewControlPlanePreparePhase creates a kubeadm workflow phase that implements the preparation of the node to serve a control plane
+func NewControlPlanePreparePhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "control-plane-prepare",
+		Short: "Prepares the machine for serving a control plane.",
+		Long:  cmdutil.MacroCommandLongDescription,
+		Phases: []workflow.Phase{
+			{
+				Name:           "all",
+				Short:          "Prepares the machine for serving a control plane.",
+				InheritFlags:   getControlPlanePreparePhaseFlags(),
+				RunAllSiblings: true,
+			},
+			newControlPlanePrepareCertsSubphase(),
+			newControlPlanePrepareKubeconfigSubphase(),
+			newControlPlanePrepareManifestsSubphases(),
+		},
+	}
+}
+
+func getControlPlanePreparePhaseFlags() []string {
+	return []string{
+		options.APIServerAdvertiseAddress,
+		options.APIServerBindPort,
+		options.CfgPath,
+		options.ControlPlane,
+		options.NodeName,
+		options.TokenDiscovery,
+		options.TokenDiscoveryCAHash,
+		options.TokenDiscoverySkipCAHash,
+	}
+}
+
+func newControlPlanePrepareCertsSubphase() workflow.Phase {
+	return workflow.Phase{
+		Name:         "certs",
+		Short:        "Generates the certificates for the new control plane components",
+		Run:          runControlPlanePrepareCertsPhaseLocal,
+		InheritFlags: getControlPlanePreparePhaseFlags(), //NB. eventually in future we would like to break down this in sub phases for each cert or add the --csr option
+	}
+}
+
+func newControlPlanePrepareKubeconfigSubphase() workflow.Phase {
+	return workflow.Phase{
+		Name:         "kubeconfig",
+		Short:        "Generates the kubeconfig for the new control plane components",
+		Run:          runControlPlanePrepareKubeconfigPhaseLocal,
+		InheritFlags: getControlPlanePreparePhaseFlags(), //NB. eventually in future we would like to break down this in sub phases for each kubeconfig
+	}
+}
+
+func newControlPlanePrepareManifestsSubphases() workflow.Phase {
+	return workflow.Phase{
+		Name:         "manifests",
+		Short:        "Generates the manifests for the new control plane components",
+		Run:          runControlPlaneSubphase,
+		InheritFlags: getControlPlanePreparePhaseFlags(), //NB. eventually in future we would like to break down this in sub phases for each component
+	}
+}
+
+func runControlPlaneSubphase(c workflow.RunData) error {
+	data, ok := c.(controlPlanePrepareData)
+	if !ok {
+		return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	// Generate missing certificates (if any)
+	return controlplane.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), cfg)
+}
+
+func runControlPlanePrepareCertsPhaseLocal(c workflow.RunData) error {
+	data, ok := c.(controlPlanePrepareData)
+	if !ok {
+		return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	// Generate missing certificates (if any)
+	return certsphase.CreatePKIAssets(cfg)
+}
+
+func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error {
+	data, ok := c.(controlPlanePrepareData)
+	if !ok {
+		return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("[control-plane-prepare] Generating kubeconfig files")
+
+	// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
+	// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
+	// following steps of the join --experimental-control plane workflow
+	if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, cfg); err != nil {
+		return errors.Wrap(err, "error generating kubeconfig files")
+	}
+
+	return nil
+}
+
+func runControlPlanePrepareJoinSubphase(component string) func(c workflow.RunData) error {
+	return func(c workflow.RunData) error {
+		data, ok := c.(controlPlanePrepareData)
+		if !ok {
+			return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+		}
+
+		// Skip if this is not a control plane
+		if data.Cfg().ControlPlane == nil {
+			return nil
+		}
+
+		cfg, err := data.InitCfg()
+		if err != nil {
+			return err
+		}
+
+		// Creates static pod manifests file for the control plane components to be deployed on this node
+		// Static pods will be created and managed by the kubelet as soon as it starts
+		fmt.Printf("[control-plane-prepare] Creating static Pod manifest for %q\n", component)
+		return controlplane.CreateStaticPodFiles(kubeadmconstants.GetStaticPodDirectory(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
+	}
+}

cmd/kubeadm/app/cmd/phases/preflight.go → cmd/kubeadm/app/cmd/phases/join/preflight.go

@@ -35,11 +35,7 @@ import (
 )
 
 var (
-	initPreflightExample = normalizer.Examples(`
-		# Run master pre-flight checks using a config file.
-		kubeadm init phase preflight --config kubeadm-config.yml
-		`)
-	joinPreflightExample = normalizer.Examples(`
+	preflightExample = normalizer.Examples(`
 		# Run join pre-flight checks using a config file.
 		kubeadm join phase preflight --config kubeadm-config.yml
 		`)
@@ -56,71 +52,20 @@ var (
 		`)))
 )
 
-// preflightMasterData defines the behavior that a runtime data struct passed to the PreflightMaster master phase
-// should have. Please note that we are using an interface in order to make this phase reusable in different workflows
-// (and thus with different runtime data struct, all of them requested to be compliant to this interface)
-type preflightMasterData interface {
-	Cfg() *kubeadmapi.InitConfiguration
-	DryRun() bool
-	IgnorePreflightErrors() sets.String
-}
-
-type preflightJoinData interface {
+type preflightData interface {
 	Cfg() *kubeadmapi.JoinConfiguration
 	InitCfg() (*kubeadmapi.InitConfiguration, error)
 	IgnorePreflightErrors() sets.String
 }
 
-// NewPreflightMasterPhase creates a kubeadm workflow phase that implements preflight checks for a new master node.
-func NewPreflightMasterPhase() workflow.Phase {
-	return workflow.Phase{
-		Name:    "preflight",
-		Short:   "Run master pre-flight checks",
-		Long:    "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.",
-		Example: initPreflightExample,
-		Run:     runPreflightMaster,
-		InheritFlags: []string{
-			options.CfgPath,
-			options.IgnorePreflightErrors,
-		},
-	}
-}
-
-// TODO(dmaiocchi):  rename all instances of master to controlPlane in this file.
-// runPreflightMaster executes preflight checks logic.
-func runPreflightMaster(c workflow.RunData) error {
-	data, ok := c.(preflightMasterData)
-	if !ok {
-		return errors.New("preflight phase invoked with an invalid data struct")
-	}
-
-	fmt.Println("[preflight] Running pre-flight checks")
-	if err := preflight.RunInitMasterChecks(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
-		return err
-	}
-
-	if !data.DryRun() {
-		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
-		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
-		fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
-		if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
-			return err
-		}
-	} else {
-		fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
-	}
-
-	return nil
-}
-
-// NewPreflightJoinPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
-func NewPreflightJoinPhase() workflow.Phase {
+// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
+func NewPreflightPhase() workflow.Phase {
 	return workflow.Phase{
 		Name:    "preflight",
 		Short:   "Run join pre-flight checks",
-		Long:    "Run join pre-flight checks, functionally equivalent to what is implemented by kubeadm join.",
-		Example: joinPreflightExample,
-		Run:     runPreflightJoin,
+		Long:    "Run pre-flight checks for kubeadm join.",
+		Example: preflightExample,
+		Run:     runPreflight,
 		InheritFlags: []string{
 			options.CfgPath,
 			options.IgnorePreflightErrors,
@@ -139,12 +84,14 @@ func NewPreflightJoinPhase() workflow.Phase {
 	}
 }
 
-// runPreflightJoin executes preflight checks logic.
-func runPreflightJoin(c workflow.RunData) error {
-	j, ok := c.(preflightJoinData)
+// runPreflight executes preflight checks logic.
+func runPreflight(c workflow.RunData) error {
+	j, ok := c.(preflightData)
 	if !ok {
 		return errors.New("preflight phase invoked with an invalid data struct")
 	}
+	fmt.Println("[preflight] Running pre-flight checks")
+
 	// Start with general checks
 	klog.V(1).Infoln("[preflight] Running general checks")
 	if err := preflight.RunJoinNodeChecks(utilsexec.New(), j.Cfg(), j.IgnorePreflightErrors()); err != nil {
@@ -182,7 +129,9 @@ func runPreflightJoin(c workflow.RunData) error {
 			return err
 		}
 
-		fmt.Println("[preflight] Pulling control-plane images")
+		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
+		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
+		fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
 		if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, j.IgnorePreflightErrors()); err != nil {
 			return err
 		}

cmd/kubeadm/app/cmd/token.go

@@ -129,8 +129,8 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command {
 			kubeadmutil.CheckErr(err)
 		},
 	}
-	createCmd.Flags().StringVar(&cfgPath,
-		"config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+
+	options.AddConfigFlag(createCmd.Flags(), &cfgPath)
 	createCmd.Flags().BoolVar(&printJoinCommand,
 		"print-join-command", false, "Instead of printing only the token, print the full 'kubeadm join' flag needed to join the cluster using the token.")
 	bto.AddTTLFlagWithName(createCmd.Flags(), "ttl")

cmd/kubelet/app/BUILD

@@ -122,7 +122,6 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
-        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",
@@ -136,6 +135,7 @@ go_library(
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
         "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
+        "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
         "//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/kubelet/app/server.go

@@ -48,7 +48,6 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/flag"
-	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -95,6 +94,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/rlimit"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/version/verflag"
+	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	"k8s.io/utils/exec"
 	"k8s.io/utils/nsenter"
 )
@@ -545,12 +545,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 	switch {
 	case standaloneMode:
 		kubeDeps.KubeClient = nil
-		kubeDeps.DynamicKubeClient = nil
 		kubeDeps.EventClient = nil
 		kubeDeps.HeartbeatClient = nil
 		klog.Warningf("standalone mode, no API client")
 
-	case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil, kubeDeps.DynamicKubeClient == nil:
+	case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil:
 		clientConfig, closeAllConns, err := buildKubeletClientConfig(s, nodeName)
 		if err != nil {
 			return err
@@ -562,11 +561,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 			return fmt.Errorf("failed to initialize kubelet client: %v", err)
 		}
 
-		kubeDeps.DynamicKubeClient, err = dynamic.NewForConfig(clientConfig)
-		if err != nil {
-			return fmt.Errorf("failed to initialize kubelet dynamic client: %v", err)
-		}
-
 		// make a separate client for events
 		eventClientConfig := *clientConfig
 		eventClientConfig.QPS = float32(s.EventRecordQPS)
@@ -593,12 +587,17 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 		}
 
 		// CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803
-		csiClientConfig := restclient.CopyConfig(clientConfig)
-		csiClientConfig.ContentType = "application/json"
-		kubeDeps.CSIClient, err = csiclientset.NewForConfig(csiClientConfig)
+		crdClientConfig := restclient.CopyConfig(clientConfig)
+		crdClientConfig.ContentType = "application/json"
+		kubeDeps.CSIClient, err = csiclientset.NewForConfig(crdClientConfig)
 		if err != nil {
 			return fmt.Errorf("failed to initialize kubelet storage client: %v", err)
 		}
+
+		kubeDeps.NodeAPIClient, err = nodeapiclientset.NewForConfig(crdClientConfig)
+		if err != nil {
+			return fmt.Errorf("failed to initialize kubelet node-api client: %v", err)
+		}
 	}
 
 	// If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops

hack/verify-generated-files-remake.sh

@@ -21,6 +21,21 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+kube::util::ensure_clean_working_dir
+
+_tmpdir="$(kube::realpath $(mktemp -d -t verify-generated-files.XXXXXX))"
+kube::util::trap_add "rm -rf ${_tmpdir}" EXIT
+
+_tmp_gopath="${_tmpdir}/go"
+_tmp_kuberoot="${_tmp_gopath}/src/k8s.io/kubernetes"
+mkdir -p "${_tmp_kuberoot}/.."
+cp -a "${KUBE_ROOT}" "${_tmp_kuberoot}/.."
+
+cd "${_tmp_kuberoot}"
+
+# clean out anything from the temp dir that's not checked in
+git clean -ffxd
+
 # $1 = filename pattern as in "zz_generated.$1.go"
 function find_genfiles() {
     find .                         \

pkg/api/pod/util_test.go

@@ -176,6 +176,11 @@ func collectResourcePaths(t *testing.T, resourcename string, path *field.Path, n
 	case reflect.Ptr:
 		resourcePaths.Insert(collectResourcePaths(t, resourcename, path, name, tp.Elem()).List()...)
 	case reflect.Struct:
+		// ObjectMeta is generic and therefore should never have a field with a specific resource's name;
+		// it contains cycles so it's easiest to just skip it.
+		if name == "ObjectMeta" {
+			break
+		}
 		for i := 0; i < tp.NumField(); i++ {
 			field := tp.Field(i)
 			resourcePaths.Insert(collectResourcePaths(t, resourcename, path.Child(field.Name), field.Name, field.Type).List()...)

pkg/api/service/BUILD

@@ -12,7 +12,7 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/api/service",
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//pkg/util/net/sets:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 
@@ -22,7 +22,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//pkg/util/net/sets:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 

pkg/api/service/util.go

@@ -18,17 +18,18 @@ package service
 
 import (
 	"fmt"
-	api "k8s.io/kubernetes/pkg/apis/core"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
 	"strings"
+
+	api "k8s.io/kubernetes/pkg/apis/core"
+	utilnet "k8s.io/utils/net"
 )
 
 const (
 	defaultLoadBalancerSourceRanges = "0.0.0.0/0"
 )
 
-// IsAllowAll checks whether the netsets.IPNet allows traffic from 0.0.0.0/0
-func IsAllowAll(ipnets netsets.IPNet) bool {
+// IsAllowAll checks whether the utilnet.IPNet allows traffic from 0.0.0.0/0
+func IsAllowAll(ipnets utilnet.IPNetSet) bool {
 	for _, s := range ipnets.StringSlice() {
 		if s == "0.0.0.0/0" {
 			return true
@@ -40,13 +41,13 @@ func IsAllowAll(ipnets netsets.IPNet) bool {
 // GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service.
 // If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service,
 // extracting the source ranges to allow, and if not present returns a default (allow-all) value.
-func GetLoadBalancerSourceRanges(service *api.Service) (netsets.IPNet, error) {
-	var ipnets netsets.IPNet
+func GetLoadBalancerSourceRanges(service *api.Service) (utilnet.IPNetSet, error) {
+	var ipnets utilnet.IPNetSet
 	var err error
 	// if SourceRange field is specified, ignore sourceRange annotation
 	if len(service.Spec.LoadBalancerSourceRanges) > 0 {
 		specs := service.Spec.LoadBalancerSourceRanges
-		ipnets, err = netsets.ParseIPNets(specs...)
+		ipnets, err = utilnet.ParseIPNets(specs...)
 
 		if err != nil {
 			return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err)
@@ -58,7 +59,7 @@ func GetLoadBalancerSourceRanges(service *api.Service) (netsets.IPNet, error) {
 			val = defaultLoadBalancerSourceRanges
 		}
 		specs := strings.Split(val, ",")
-		ipnets, err = netsets.ParseIPNets(specs...)
+		ipnets, err = utilnet.ParseIPNets(specs...)
 		if err != nil {
 			return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", api.AnnotationLoadBalancerSourceRangesKey, val)
 		}

pkg/api/service/util_test.go

@@ -21,7 +21,7 @@ import (
 	"testing"
 
 	api "k8s.io/kubernetes/pkg/apis/core"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	utilnet "k8s.io/utils/net"
 )
 
 func TestGetLoadBalancerSourceRanges(t *testing.T) {
@@ -48,7 +48,7 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) {
 	checkError("10.0.0.1/32, ")
 	checkError("10.0.0.1")
 
-	checkOK := func(v string) netsets.IPNet {
+	checkOK := func(v string) utilnet.IPNetSet {
 		annotations := make(map[string]string)
 		annotations[api.AnnotationLoadBalancerSourceRangesKey] = v
 		svc := api.Service{}
@@ -112,7 +112,7 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) {
 
 func TestAllowAll(t *testing.T) {
 	checkAllowAll := func(allowAll bool, cidrs ...string) {
-		ipnets, err := netsets.ParseIPNets(cidrs...)
+		ipnets, err := utilnet.ParseIPNets(cidrs...)
 		if err != nil {
 			t.Errorf("Unexpected error parsing cidrs: %v", cidrs)
 		}

pkg/api/testing/serialization_test.go

@@ -26,7 +26,6 @@ import (
 	"testing"
 
 	jsoniter "github.com/json-iterator/go"
-
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
@@ -162,9 +161,10 @@ var nonRoundTrippableTypes = sets.NewString(
 	"DeleteOptions",
 	"CreateOptions",
 	"UpdateOptions",
+	"PatchOptions",
 )
 
-var commonKinds = []string{"Status", "ListOptions", "DeleteOptions", "ExportOptions", "GetOptions", "CreateOptions", "UpdateOptions"}
+var commonKinds = []string{"Status", "ListOptions", "DeleteOptions", "ExportOptions", "GetOptions", "CreateOptions", "UpdateOptions", "PatchOptions"}
 
 // TestCommonKindsRegistered verifies that all group/versions registered with
 // the testapi package have the common kinds.

pkg/api/v1/persistentvolume/util_test.go

@@ -18,9 +18,8 @@ package persistentvolume
 
 import (
 	"reflect"
-	"testing"
-
 	"strings"
+	"testing"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -248,6 +247,11 @@ func collectSecretPaths(t *testing.T, path *field.Path, name string, tp reflect.
 	case reflect.Ptr:
 		secretPaths.Insert(collectSecretPaths(t, path, name, tp.Elem()).List()...)
 	case reflect.Struct:
+		// ObjectMeta should not have any field with the word "secret" in it;
+		// it contains cycles so it's easiest to just skip it.
+		if name == "ObjectMeta" {
+			break
+		}
 		for i := 0; i < tp.NumField(); i++ {
 			field := tp.Field(i)
 			secretPaths.Insert(collectSecretPaths(t, path.Child(field.Name), field.Name, field.Type).List()...)

pkg/api/v1/pod/util_test.go

@@ -340,6 +340,11 @@ func collectResourcePaths(t *testing.T, resourcename string, path *field.Path, n
 	case reflect.Ptr:
 		resourcePaths.Insert(collectResourcePaths(t, resourcename, path, name, tp.Elem()).List()...)
 	case reflect.Struct:
+		// ObjectMeta is generic and therefore should never have a field with a specific resource's name;
+		// it contains cycles so it's easiest to just skip it.
+		if name == "ObjectMeta" {
+			break
+		}
 		for i := 0; i < tp.NumField(); i++ {
 			field := tp.Field(i)
 			resourcePaths.Insert(collectResourcePaths(t, resourcename, path.Child(field.Name), field.Name, field.Type).List()...)

pkg/api/v1/service/BUILD

@@ -11,8 +11,8 @@ go_library(
     srcs = ["util.go"],
     importpath = "k8s.io/kubernetes/pkg/api/v1/service",
     deps = [
-        "//pkg/util/net/sets:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 
@@ -21,8 +21,8 @@ go_test(
     srcs = ["util_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/util/net/sets:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 

pkg/api/v1/service/util.go

@@ -21,15 +21,15 @@ import (
 	"strings"
 
 	"k8s.io/api/core/v1"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	utilnet "k8s.io/utils/net"
 )
 
 const (
 	defaultLoadBalancerSourceRanges = "0.0.0.0/0"
 )
 
-// IsAllowAll checks whether the netsets.IPNet allows traffic from 0.0.0.0/0
-func IsAllowAll(ipnets netsets.IPNet) bool {
+// IsAllowAll checks whether the utilnet.IPNet allows traffic from 0.0.0.0/0
+func IsAllowAll(ipnets utilnet.IPNetSet) bool {
 	for _, s := range ipnets.StringSlice() {
 		if s == "0.0.0.0/0" {
 			return true
@@ -41,13 +41,13 @@ func IsAllowAll(ipnets netsets.IPNet) bool {
 // GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service.
 // If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service,
 // extracting the source ranges to allow, and if not present returns a default (allow-all) value.
-func GetLoadBalancerSourceRanges(service *v1.Service) (netsets.IPNet, error) {
-	var ipnets netsets.IPNet
+func GetLoadBalancerSourceRanges(service *v1.Service) (utilnet.IPNetSet, error) {
+	var ipnets utilnet.IPNetSet
 	var err error
 	// if SourceRange field is specified, ignore sourceRange annotation
 	if len(service.Spec.LoadBalancerSourceRanges) > 0 {
 		specs := service.Spec.LoadBalancerSourceRanges
-		ipnets, err = netsets.ParseIPNets(specs...)
+		ipnets, err = utilnet.ParseIPNets(specs...)
 
 		if err != nil {
 			return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err)
@@ -59,7 +59,7 @@ func GetLoadBalancerSourceRanges(service *v1.Service) (netsets.IPNet, error) {
 			val = defaultLoadBalancerSourceRanges
 		}
 		specs := strings.Split(val, ",")
-		ipnets, err = netsets.ParseIPNets(specs...)
+		ipnets, err = utilnet.ParseIPNets(specs...)
 		if err != nil {
 			return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", v1.AnnotationLoadBalancerSourceRangesKey, val)
 		}

pkg/api/v1/service/util_test.go

@@ -21,7 +21,7 @@ import (
 	"testing"
 
 	"k8s.io/api/core/v1"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	utilnet "k8s.io/utils/net"
 )
 
 func TestGetLoadBalancerSourceRanges(t *testing.T) {
@@ -48,7 +48,7 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) {
 	checkError("10.0.0.1/32, ")
 	checkError("10.0.0.1")
 
-	checkOK := func(v string) netsets.IPNet {
+	checkOK := func(v string) utilnet.IPNetSet {
 		annotations := make(map[string]string)
 		annotations[v1.AnnotationLoadBalancerSourceRangesKey] = v
 		svc := v1.Service{}
@@ -112,7 +112,7 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) {
 
 func TestAllowAll(t *testing.T) {
 	checkAllowAll := func(allowAll bool, cidrs ...string) {
-		ipnets, err := netsets.ParseIPNets(cidrs...)
+		ipnets, err := utilnet.ParseIPNets(cidrs...)
 		if err != nil {
 			t.Errorf("Unexpected error parsing cidrs: %v", cidrs)
 		}

pkg/apis/authorization/validation/validation.go

@@ -56,7 +56,9 @@ func ValidateSelfSubjectRulesReview(review *authorizationapi.SelfSubjectRulesRev
 
 func ValidateSubjectAccessReview(sar *authorizationapi.SubjectAccessReview) field.ErrorList {
 	allErrs := ValidateSubjectAccessReviewSpec(sar.Spec, field.NewPath("spec"))
-	if !apiequality.Semantic.DeepEqual(metav1.ObjectMeta{}, sar.ObjectMeta) {
+	objectMetaShallowCopy := sar.ObjectMeta
+	objectMetaShallowCopy.ManagedFields = nil
+	if !apiequality.Semantic.DeepEqual(metav1.ObjectMeta{}, objectMetaShallowCopy) {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("metadata"), sar.ObjectMeta, `must be empty`))
 	}
 	return allErrs
@@ -64,7 +66,9 @@ func ValidateSubjectAccessReview(sar *authorizationapi.SubjectAccessReview) fiel
 
 func ValidateSelfSubjectAccessReview(sar *authorizationapi.SelfSubjectAccessReview) field.ErrorList {
 	allErrs := ValidateSelfSubjectAccessReviewSpec(sar.Spec, field.NewPath("spec"))
-	if !apiequality.Semantic.DeepEqual(metav1.ObjectMeta{}, sar.ObjectMeta) {
+	objectMetaShallowCopy := sar.ObjectMeta
+	objectMetaShallowCopy.ManagedFields = nil
+	if !apiequality.Semantic.DeepEqual(metav1.ObjectMeta{}, objectMetaShallowCopy) {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("metadata"), sar.ObjectMeta, `must be empty`))
 	}
 	return allErrs
@@ -75,6 +79,7 @@ func ValidateLocalSubjectAccessReview(sar *authorizationapi.LocalSubjectAccessRe
 
 	objectMetaShallowCopy := sar.ObjectMeta
 	objectMetaShallowCopy.Namespace = ""
+	objectMetaShallowCopy.ManagedFields = nil
 	if !apiequality.Semantic.DeepEqual(metav1.ObjectMeta{}, objectMetaShallowCopy) {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("metadata"), sar.ObjectMeta, `must be empty except for namespace`))
 	}

pkg/apis/core/types.go

@@ -918,6 +918,11 @@ type QuobyteVolumeSource struct {
 	// Default is no group
 	// +optional
 	Group string
+
+	// Tenant owning the given Quobyte volume in the Backend
+	// Used with dynamically provisioned Quobyte volumes, value is set by the plugin
+	// +optional
+	Tenant string
 }
 
 // Represents a Glusterfs mount that lasts the lifetime of a pod.

pkg/apis/core/v1/zz_generated.conversion.go

@@ -5989,6 +5989,7 @@ func autoConvert_v1_QuobyteVolumeSource_To_core_QuobyteVolumeSource(in *v1.Quoby
 	out.ReadOnly = in.ReadOnly
 	out.User = in.User
 	out.Group = in.Group
+	out.Tenant = in.Tenant
 	return nil
 }
 
@@ -6003,6 +6004,7 @@ func autoConvert_core_QuobyteVolumeSource_To_v1_QuobyteVolumeSource(in *core.Quo
 	out.ReadOnly = in.ReadOnly
 	out.User = in.User
 	out.Group = in.Group
+	out.Tenant = in.Tenant
 	return nil
 }
 

pkg/apis/core/validation/validation.go

@@ -415,9 +415,6 @@ func validateVolumeSource(source *core.VolumeSource, fldPath *field.Path, volNam
 		if source.EmptyDir.SizeLimit != nil && source.EmptyDir.SizeLimit.Cmp(resource.Quantity{}) < 0 {
 			allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("sizeLimit"), "SizeLimit field must be a valid resource quantity"))
 		}
-		if !utilfeature.DefaultFeatureGate.Enabled(features.HugePages) && source.EmptyDir.Medium == core.StorageMediumHugePages {
-			allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("medium"), "HugePages medium is disabled by feature-gate for EmptyDir volumes"))
-		}
 	}
 	if source.HostPath != nil {
 		if numVolumes > 0 {
@@ -873,6 +870,10 @@ func validateQuobyteVolumeSource(quobyte *core.QuobyteVolumeSource, fldPath *fie
 	allErrs := field.ErrorList{}
 	if len(quobyte.Registry) == 0 {
 		allErrs = append(allErrs, field.Required(fldPath.Child("registry"), "must be a host:port pair or multiple pairs separated by commas"))
+	} else if len(quobyte.Tenant) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("tenant"), "must be a UUID provided by the configuration and may not be omitted "))
+	} else if len(quobyte.Tenant) >= 65 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("tenant"), "must be a UUID and may not exceed a length of 64 characters"))
 	} else {
 		for _, hostPortPair := range strings.Split(quobyte.Registry, ",") {
 			if _, _, err := net.SplitHostPort(hostPortPair); err != nil {
@@ -2925,19 +2926,17 @@ func ValidatePod(pod *core.Pod) field.ErrorList {
 	allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.Containers, specPath.Child("containers"))...)
 	allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.InitContainers, specPath.Child("initContainers"))...)
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) {
-		hugePageResources := sets.NewString()
-		for i := range pod.Spec.Containers {
-			resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i])
-			for resourceStr := range resourceSet {
-				if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) {
-					hugePageResources.Insert(resourceStr)
-				}
+	hugePageResources := sets.NewString()
+	for i := range pod.Spec.Containers {
+		resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i])
+		for resourceStr := range resourceSet {
+			if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) {
+				hugePageResources.Insert(resourceStr)
 			}
 		}
-		if len(hugePageResources) > 1 {
-			allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec"))
-		}
+	}
+	if len(hugePageResources) > 1 {
+		allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec"))
 	}
 
 	return allErrs

pkg/apis/core/validation/validation_test.go

@@ -3379,6 +3379,7 @@ func TestValidateVolumes(t *testing.T) {
 						ReadOnly: false,
 						User:     "root",
 						Group:    "root",
+						Tenant:   "ThisIsSomeTenantUUID",
 					},
 				},
 			},
@@ -3390,6 +3391,7 @@ func TestValidateVolumes(t *testing.T) {
 				VolumeSource: core.VolumeSource{
 					Quobyte: &core.QuobyteVolumeSource{
 						Volume: "/test",
+						Tenant: "ThisIsSomeTenantUUID",
 					},
 				},
 			},
@@ -3406,6 +3408,7 @@ func TestValidateVolumes(t *testing.T) {
 					Quobyte: &core.QuobyteVolumeSource{
 						Registry: "registry7861",
 						Volume:   "/test",
+						Tenant:   "ThisIsSomeTenantUUID",
 					},
 				},
 			},
@@ -3422,6 +3425,7 @@ func TestValidateVolumes(t *testing.T) {
 					Quobyte: &core.QuobyteVolumeSource{
 						Registry: "registry:7861,reg2",
 						Volume:   "/test",
+						Tenant:   "ThisIsSomeTenantUUID",
 					},
 				},
 			},
@@ -3437,6 +3441,7 @@ func TestValidateVolumes(t *testing.T) {
 				VolumeSource: core.VolumeSource{
 					Quobyte: &core.QuobyteVolumeSource{
 						Registry: "registry:7861",
+						Tenant:   "ThisIsSomeTenantUUID",
 					},
 				},
 			},
@@ -3445,6 +3450,40 @@ func TestValidateVolumes(t *testing.T) {
 				field: "quobyte.volume",
 			}},
 		},
+		{
+			name: "empty tenant quobyte",
+			vol: core.Volume{
+				Name: "quobyte",
+				VolumeSource: core.VolumeSource{
+					Quobyte: &core.QuobyteVolumeSource{
+						Registry: "registry:7861,reg2",
+						Volume:   "/test",
+						Tenant:   "",
+					},
+				},
+			},
+			errs: []verr{{
+				etype: field.ErrorTypeRequired,
+				field: "quobyte.tenant",
+			}},
+		},
+		{
+			name: "too long tenant quobyte",
+			vol: core.Volume{
+				Name: "quobyte",
+				VolumeSource: core.VolumeSource{
+					Quobyte: &core.QuobyteVolumeSource{
+						Registry: "registry:7861,reg2",
+						Volume:   "/test",
+						Tenant:   "this is too long to be a valid uuid so this test has to fail on the maximum length validation of the tenant.",
+					},
+				},
+			},
+			errs: []verr{{
+				etype: field.ErrorTypeRequired,
+				field: "quobyte.tenant",
+			}},
+		},
 		// AzureDisk
 		{
 			name: "valid AzureDisk",
@@ -3659,24 +3698,17 @@ func TestValidateVolumes(t *testing.T) {
 		t.Errorf("expected error type %v, got %v", field.ErrorTypeDuplicate, errs[0].Type)
 	}
 
-	// Validate HugePages medium type for EmptyDir when HugePages feature is enabled/disabled
+	// Validate HugePages medium type for EmptyDir
 	hugePagesCase := core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{Medium: core.StorageMediumHugePages}}
 
 	// Enable HugePages
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
 	if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "working"); len(errs) != 0 {
 		t.Errorf("Unexpected error when HugePages feature is enabled.")
 	}
 
-	// Disable feature HugePages
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
-	if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "failing"); len(errs) == 0 {
-		t.Errorf("Expected error when HugePages feature is disabled got nothing.")
-	}
-
 }
 
-func TestAlphaHugePagesIsolation(t *testing.T) {
+func TestHugePagesIsolation(t *testing.T) {
 	successCases := []core.Pod{
 		{ // Basic fields.
 			ObjectMeta: metav1.ObjectMeta{Name: "123", Namespace: "ns"},
@@ -3774,8 +3806,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) {
 			},
 		},
 	}
-	// Enable feature HugePages
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
 	for i := range successCases {
 		pod := &successCases[i]
 		if errs := ValidatePod(pod); len(errs) != 0 {
@@ -3788,15 +3818,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) {
 			t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name)
 		}
 	}
-	// Disable feature HugePages
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
-	// Disable feature HugePages and ensure all success cases fail
-	for i := range successCases {
-		pod := &successCases[i]
-		if errs := ValidatePod(pod); len(errs) == 0 {
-			t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name)
-		}
-	}
 }
 
 func TestPVCVolumeMode(t *testing.T) {
@@ -6686,6 +6707,33 @@ func TestValidatePod(t *testing.T) {
 				DNSPolicy:     core.DNSClusterFirst,
 			},
 		},
+		{ // valid serviceaccount token projected volume with serviceaccount name specified
+			ObjectMeta: metav1.ObjectMeta{Name: "valid-extended", Namespace: "ns"},
+			Spec: core.PodSpec{
+				ServiceAccountName: "some-service-account",
+				Containers:         []core.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
+				RestartPolicy:      core.RestartPolicyAlways,
+				DNSPolicy:          core.DNSClusterFirst,
+				Volumes: []core.Volume{
+					{
+						Name: "projected-volume",
+						VolumeSource: core.VolumeSource{
+							Projected: &core.ProjectedVolumeSource{
+								Sources: []core.VolumeProjection{
+									{
+										ServiceAccountToken: &core.ServiceAccountTokenProjection{
+											Audience:          "foo-audience",
+											ExpirationSeconds: 6000,
+											Path:              "foo-path",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, pod := range successCases {
 		if errs := ValidatePod(&pod); len(errs) != 0 {

pkg/cloudprovider/providers/.import-restrictions

 {
 	"Rules": [
-	  {
-		"SelectorRegexp": "k8s[.]io/kubernetes",
-		"AllowedPrefixes": [
-		    "k8s.io/kubernetes/pkg/api/legacyscheme",
-		    "k8s.io/kubernetes/pkg/api/service",
-		    "k8s.io/kubernetes/pkg/api/v1/service",
-		    "k8s.io/kubernetes/pkg/apis/core",
-		    "k8s.io/kubernetes/pkg/cloudprovider",
-		    "k8s.io/kubernetes/pkg/credentialprovider",
-		    "k8s.io/kubernetes/pkg/features",
-		    "k8s.io/kubernetes/pkg/kubelet/apis",
-		    "k8s.io/kubernetes/pkg/master/ports",
-		    "k8s.io/kubernetes/pkg/util/mount",
-		    "k8s.io/kubernetes/pkg/util/file",
-		    "k8s.io/kubernetes/pkg/util/net/sets",
-		    "k8s.io/kubernetes/pkg/util/resizefs",
-		    "k8s.io/kubernetes/pkg/util/strings",
-		    "k8s.io/kubernetes/pkg/version",
-		    "k8s.io/kubernetes/pkg/volume"
-		],
-		"ForbiddenPrefixes": [
-		]
-	  }
+		{
+			"SelectorRegexp": "k8s[.]io/utils",
+			"AllowedPrefixes": [
+				"k8s.io/utils/net",
+				"k8s.io/utils/nsenter",
+				"k8s.io/utils/io",
+				"k8s.io/utils/strings",
+				"k8s.io/utils/exec",
+				"k8s.io/utils/path"
+			]
+		},
+		{
+			"SelectorRegexp": "k8s[.]io/kubernetes",
+			"AllowedPrefixes": [
+				"k8s.io/kubernetes/pkg/api/legacyscheme",
+				"k8s.io/kubernetes/pkg/apis/core",
+				"k8s.io/kubernetes/pkg/cloudprovider",
+				"k8s.io/kubernetes/pkg/credentialprovider",
+				"k8s.io/kubernetes/pkg/features",
+				"k8s.io/kubernetes/pkg/kubelet/apis",
+				"k8s.io/kubernetes/pkg/master/ports",
+				"k8s.io/kubernetes/pkg/util/mount",
+				"k8s.io/kubernetes/pkg/util/resizefs",
+				"k8s.io/kubernetes/pkg/version",
+				"k8s.io/kubernetes/pkg/volume"
+			],
+			"ForbiddenPrefixes": []
+		}
 	]
 }

pkg/cloudprovider/providers/aws/BUILD

@@ -27,7 +27,6 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/aws",
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/credentialprovider/aws:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/volume:go_default_library",
@@ -45,6 +44,7 @@ go_library(
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/credentials:go_default_library",

pkg/cloudprovider/providers/aws/aws.go

@@ -59,7 +59,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	cloudprovider "k8s.io/cloud-provider"
 	nodehelpers "k8s.io/cloud-provider/node/helpers"
-	"k8s.io/kubernetes/pkg/api/v1/service"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
@@ -3434,7 +3434,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS
 		return nil, err
 	}
 
-	sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService)
 	if err != nil {
 		return nil, err
 	}
@@ -3450,7 +3450,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS
 
 	if isNLB(annotations) {
 
-		if path, healthCheckNodePort := service.GetServiceHealthCheckPathPort(apiService); path != "" {
+		if path, healthCheckNodePort := servicehelpers.GetServiceHealthCheckPathPort(apiService); path != "" {
 			for i := range v2Mappings {
 				v2Mappings[i].HealthCheckPort = int64(healthCheckNodePort)
 				v2Mappings[i].HealthCheckPath = path
@@ -3708,7 +3708,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS
 		}
 	}
 
-	if path, healthCheckNodePort := service.GetServiceHealthCheckPathPort(apiService); path != "" {
+	if path, healthCheckNodePort := servicehelpers.GetServiceHealthCheckPathPort(apiService); path != "" {
 		klog.V(4).Infof("service %v (%v) needs health checks on :%d%s)", apiService.Name, loadBalancerName, healthCheckNodePort, path)
 		err = c.ensureLoadBalancerHealthCheck(loadBalancer, "HTTP", healthCheckNodePort, path, annotations)
 		if err != nil {

pkg/cloudprovider/providers/azure/BUILD

@@ -36,7 +36,6 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/azure",
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/cloudprovider/providers/azure/auth:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/volume:go_default_library",
@@ -58,6 +57,7 @@ go_library(
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library",
@@ -93,7 +93,6 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/cloudprovider/providers/azure/auth:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -101,6 +100,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library",

pkg/cloudprovider/providers/azure/azure_loadbalancer.go

@@ -27,8 +27,8 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	cloudprovider "k8s.io/cloud-provider"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	"k8s.io/klog"
-	serviceapi "k8s.io/kubernetes/pkg/api/v1/service"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -888,8 +888,8 @@ func (az *Cloud) reconcileLoadBalancerRule(
 			return expectedProbes, expectedRules, err
 		}
 
-		if serviceapi.NeedsHealthCheck(service) {
-			podPresencePath, podPresencePort := serviceapi.GetServiceHealthCheckPathPort(service)
+		if servicehelpers.NeedsHealthCheck(service) {
+			podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
 
 			expectedProbes = append(expectedProbes, network.Probe{
 				Name: &lbRuleName,
@@ -983,7 +983,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service,
 		destinationIPAddress = "*"
 	}
 
-	sourceRanges, err := serviceapi.GetLoadBalancerSourceRanges(service)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(service)
 	if err != nil {
 		return nil, err
 	}
@@ -992,7 +992,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service,
 		return nil, err
 	}
 	var sourceAddressPrefixes []string
-	if (sourceRanges == nil || serviceapi.IsAllowAll(sourceRanges)) && len(serviceTags) == 0 {
+	if (sourceRanges == nil || servicehelpers.IsAllowAll(sourceRanges)) && len(serviceTags) == 0 {
 		if !requiresInternalLoadBalancer(service) {
 			sourceAddressPrefixes = []string{"Internet"}
 		}

pkg/cloudprovider/providers/azure/azure_test.go

@@ -30,7 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
-	serviceapi "k8s.io/kubernetes/pkg/api/v1/service"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 
@@ -1231,8 +1231,8 @@ func validateLoadBalancer(t *testing.T, loadBalancer *network.LoadBalancer, serv
 
 			expectedProbeCount++
 			foundProbe := false
-			if serviceapi.NeedsHealthCheck(&svc) {
-				path, port := serviceapi.GetServiceHealthCheckPathPort(&svc)
+			if servicehelpers.NeedsHealthCheck(&svc) {
+				path, port := servicehelpers.GetServiceHealthCheckPathPort(&svc)
 				for _, actualProbe := range *loadBalancer.Probes {
 					if strings.EqualFold(*actualProbe.Name, wantedRuleName) &&
 						*actualProbe.Port == port &&

pkg/cloudprovider/providers/gce/BUILD

@@ -46,9 +46,7 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/gce",
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
-        "//pkg/util/net/sets:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -73,6 +71,7 @@ go_library(
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/cloud-provider/features:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/cloud.google.com/go/compute/metadata:go_default_library",
         "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud:go_default_library",
         "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/filter:go_default_library",
@@ -89,6 +88,7 @@ go_library(
         "//vendor/google.golang.org/api/tpu/v1:go_default_library",
         "//vendor/gopkg.in/gcfg.v1:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 
@@ -109,15 +109,14 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
-        "//pkg/util/net/sets:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud:go_default_library",
         "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta:go_default_library",
         "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock:go_default_library",
@@ -128,6 +127,7 @@ go_test(
         "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 

pkg/cloudprovider/providers/gce/gce_loadbalancer.go

@@ -24,16 +24,16 @@ import (
 	"sort"
 	"strings"
 
+	"k8s.io/api/core/v1"
 	"k8s.io/klog"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
-	"k8s.io/api/core/v1"
 	cloudprovider "k8s.io/cloud-provider"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	utilnet "k8s.io/utils/net"
 )
 
 type cidrs struct {
-	ipn   netsets.IPNet
+	ipn   utilnet.IPNetSet
 	isSet bool
 }
 
@@ -44,7 +44,7 @@ var (
 func init() {
 	var err error
 	// LB L7 proxies and all L3/4/7 health checkers have client addresses within these known CIDRs.
-	lbSrcRngsFlag.ipn, err = netsets.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"}...)
+	lbSrcRngsFlag.ipn, err = utilnet.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"}...)
 	if err != nil {
 		panic("Incorrect default GCE L7 source ranges")
 	}
@@ -64,7 +64,7 @@ func (c *cidrs) Set(value string) error {
 	// On first Set(), clear the original defaults
 	if !c.isSet {
 		c.isSet = true
-		c.ipn = make(netsets.IPNet)
+		c.ipn = make(utilnet.IPNetSet)
 	} else {
 		return fmt.Errorf("GCE LB CIDRs have already been set")
 	}

pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -28,8 +28,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
-	apiservice "k8s.io/kubernetes/pkg/api/v1/service"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
+	utilnet "k8s.io/utils/net"
 
 	computealpha "google.golang.org/api/compute/v0.alpha"
 	compute "google.golang.org/api/compute/v1"
@@ -162,7 +162,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 	// is because the forwarding rule is used as the indicator that the load
 	// balancer is fully created - it's what getLoadBalancer checks for.
 	// Check if user specified the allow source range
-	sourceRanges, err := apiservice.GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService)
 	if err != nil {
 		return nil, err
 	}
@@ -206,7 +206,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 	if err != nil && !isHTTPErrorCode(err, http.StatusNotFound) {
 		return nil, fmt.Errorf("error checking HTTP health check for load balancer (%s): %v", lbRefStr, err)
 	}
-	if path, healthCheckNodePort := apiservice.GetServiceHealthCheckPathPort(apiService); path != "" {
+	if path, healthCheckNodePort := servicehelpers.GetServiceHealthCheckPathPort(apiService); path != "" {
 		klog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs local traffic health checks on: %d%s.", lbRefStr, healthCheckNodePort, path)
 		if hcLocalTrafficExisting == nil {
 			// This logic exists to detect a transition for non-OnlyLocal to OnlyLocal service
@@ -292,7 +292,7 @@ func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string,
 	lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName)
 
 	var hcNames []string
-	if path, _ := apiservice.GetServiceHealthCheckPathPort(service); path != "" {
+	if path, _ := servicehelpers.GetServiceHealthCheckPathPort(service); path != "" {
 		hcToDelete, err := g.GetHTTPHealthCheck(loadBalancerName)
 		if err != nil && !isHTTPErrorCode(err, http.StatusNotFound) {
 			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Failed to retrieve health check:%v.", lbRefStr, err)
@@ -819,7 +819,7 @@ func translateAffinityType(affinityType v1.ServiceAffinity) string {
 	}
 }
 
-func (g *Cloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string, ports []v1.ServicePort, sourceRanges netsets.IPNet) (exists bool, needsUpdate bool, err error) {
+func (g *Cloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string, ports []v1.ServicePort, sourceRanges utilnet.IPNetSet) (exists bool, needsUpdate bool, err error) {
 	fw, err := g.GetFirewall(MakeFirewallName(name))
 	if err != nil {
 		if isHTTPErrorCode(err, http.StatusNotFound) {
@@ -843,7 +843,7 @@ func (g *Cloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string,
 	}
 	// The service controller already verified that the protocol matches on all ports, no need to check.
 
-	actualSourceRanges, err := netsets.ParseIPNets(fw.SourceRanges...)
+	actualSourceRanges, err := utilnet.ParseIPNets(fw.SourceRanges...)
 	if err != nil {
 		// This really shouldn't happen... GCE has returned something unexpected
 		klog.Warningf("Error parsing firewall SourceRanges: %v", fw.SourceRanges)
@@ -934,7 +934,7 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio
 	return nil
 }
 
-func (g *Cloud) createFirewall(svc *v1.Service, name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
+func (g *Cloud) createFirewall(svc *v1.Service, name, region, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
 	firewall, err := g.firewallObject(name, region, desc, sourceRanges, ports, hosts)
 	if err != nil {
 		return err
@@ -952,7 +952,7 @@ func (g *Cloud) createFirewall(svc *v1.Service, name, region, desc string, sourc
 	return nil
 }
 
-func (g *Cloud) updateFirewall(svc *v1.Service, name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
+func (g *Cloud) updateFirewall(svc *v1.Service, name, region, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
 	firewall, err := g.firewallObject(name, region, desc, sourceRanges, ports, hosts)
 	if err != nil {
 		return err
@@ -971,7 +971,7 @@ func (g *Cloud) updateFirewall(svc *v1.Service, name, region, desc string, sourc
 	return nil
 }
 
-func (g *Cloud) firewallObject(name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) (*compute.Firewall, error) {
+func (g *Cloud) firewallObject(name, region, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) (*compute.Firewall, error) {
 	allowedPorts := make([]string, len(ports))
 	for ix := range ports {
 		allowedPorts[ix] = strconv.Itoa(int(ports[ix].Port))

pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go

@@ -34,7 +34,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
-	netsets "k8s.io/kubernetes/pkg/util/net/sets"
+	utilnet "k8s.io/utils/net"
 )
 
 func TestEnsureStaticIP(t *testing.T) {
@@ -620,10 +620,10 @@ func TestFirewallNeedsUpdate(t *testing.T) {
 	ipAddr := status.Ingress[0].IP
 	lbName := gce.GetLoadBalancerName(context.TODO(), "", svc)
 
-	ipnet, err := netsets.ParseIPNets("0.0.0.0/0")
+	ipnet, err := utilnet.ParseIPNets("0.0.0.0/0")
 	require.NoError(t, err)
 
-	wrongIpnet, err := netsets.ParseIPNets("1.0.0.0/10")
+	wrongIpnet, err := utilnet.ParseIPNets("1.0.0.0/10")
 	require.NoError(t, err)
 
 	fw, err := gce.GetFirewall(MakeFirewallName(lbName))
@@ -633,7 +633,7 @@ func TestFirewallNeedsUpdate(t *testing.T) {
 		lbName       string
 		ipAddr       string
 		ports        []v1.ServicePort
-		ipnet        netsets.IPNet
+		ipnet        utilnet.IPNetSet
 		fwIPProtocol string
 		getHook      func(context.Context, *meta.Key, *cloud.MockFirewalls) (bool, *ga.Firewall, error)
 		sourceRange  string
@@ -864,7 +864,7 @@ func TestCreateAndUpdateFirewallSucceedsOnXPN(t *testing.T) {
 	hostNames := nodeNames(nodes)
 	hosts, err := gce.getInstancesByNames(hostNames)
 	require.NoError(t, err)
-	ipnet, err := netsets.ParseIPNets("10.0.0.0/20")
+	ipnet, err := utilnet.ParseIPNets("10.0.0.0/20")
 	require.NoError(t, err)
 	gce.createFirewall(
 		svc,

pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go

@@ -27,8 +27,8 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	"k8s.io/klog"
-	v1_service "k8s.io/kubernetes/pkg/api/v1/service"
 )
 
 const (
@@ -69,12 +69,12 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v
 
 	// Ensure health check exists before creating the backend service. The health check is shared
 	// if externalTrafficPolicy=Cluster.
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
+	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(svc)
 	hcName := makeHealthCheckName(loadBalancerName, clusterID, sharedHealthCheck)
 	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()
 	if !sharedHealthCheck {
 		// Service requires a special health check, retrieve the OnlyLocal port & path
-		hcPath, hcPort = v1_service.GetServiceHealthCheckPathPort(svc)
+		hcPath, hcPort = servicehelpers.GetServiceHealthCheckPathPort(svc)
 	}
 	hc, err := g.ensureInternalHealthCheck(hcName, nm, sharedHealthCheck, hcPath, hcPort)
 	if err != nil {
@@ -224,7 +224,7 @@ func (g *Cloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string,
 	_, protocol := getPortsAndProtocol(svc.Spec.Ports)
 	scheme := cloud.SchemeInternal
 	sharedBackend := shareBackendService(svc)
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
+	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(svc)
 
 	g.sharedResourceLock.Lock()
 	defer g.sharedResourceLock.Unlock()
@@ -367,7 +367,7 @@ func (g *Cloud) ensureInternalFirewalls(loadBalancerName, ipAddress, clusterID s
 	// First firewall is for ingress traffic
 	fwDesc := makeFirewallDescription(nm.String(), ipAddress)
 	ports, protocol := getPortsAndProtocol(svc.Spec.Ports)
-	sourceRanges, err := v1_service.GetLoadBalancerSourceRanges(svc)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(svc)
 	if err != nil {
 		return err
 	}
@@ -581,7 +581,7 @@ func (g *Cloud) ensureInternalBackendServiceGroups(name string, igLinks []string
 }
 
 func shareBackendService(svc *v1.Service) bool {
-	return GetLoadBalancerAnnotationBackendShare(svc) && !v1_service.RequestsOnlyLocalTraffic(svc)
+	return GetLoadBalancerAnnotationBackendShare(svc) && !servicehelpers.RequestsOnlyLocalTraffic(svc)
 }
 
 func backendsFromGroupLinks(igLinks []string) (backends []*compute.Backend) {

pkg/cloudprovider/providers/gce/gce_loadbalancer_internal_test.go

@@ -31,7 +31,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
-	v1_service "k8s.io/kubernetes/pkg/api/v1/service"
+	servicehelper "k8s.io/cloud-provider/service/helpers"
 )
 
 func createInternalLoadBalancer(gce *Cloud, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodeNames []string, clusterName, clusterID, zoneName string) (*v1.LoadBalancerStatus, error) {
@@ -169,7 +169,7 @@ func TestEnsureInternalLoadBalancerWithExistingResources(t *testing.T) {
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 	lbName := gce.GetLoadBalancerName(context.TODO(), "", svc)
 
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
+	sharedHealthCheck := !servicehelper.RequestsOnlyLocalTraffic(svc)
 	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
 	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()
 	existingHC := newInternalLBHealthCheck(hcName, nm, sharedHealthCheck, hcPath, hcPort)
@@ -224,7 +224,7 @@ func TestEnsureInternalLoadBalancerClearPreviousResources(t *testing.T) {
 	}
 	gce.CreateFirewall(existingFirewall)
 
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
+	sharedHealthCheck := !servicehelper.RequestsOnlyLocalTraffic(svc)
 	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
 	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
@@ -278,7 +278,7 @@ func TestEnsureInternalLoadBalancerHealthCheckConfigurable(t *testing.T) {
 	svc := fakeLoadbalancerService(string(LBTypeInternal))
 	lbName := gce.GetLoadBalancerName(context.TODO(), "", svc)
 
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
+	sharedHealthCheck := !servicehelper.RequestsOnlyLocalTraffic(svc)
 	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
 	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

pkg/cloudprovider/providers/gce/gce_loadbalancer_utils_test.go

@@ -35,7 +35,7 @@ import (
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
-	v1_service "k8s.io/kubernetes/pkg/api/v1/service"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 )
 
@@ -211,7 +211,7 @@ func assertInternalLbResources(t *testing.T, gce *Cloud, apiService *v1.Service,
 	}
 
 	// Check that HealthCheck is created
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(apiService)
+	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(apiService)
 	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
 	healthcheck, err := gce.GetHealthCheck(hcName)
 	require.NoError(t, err)
@@ -243,7 +243,7 @@ func assertInternalLbResources(t *testing.T, gce *Cloud, apiService *v1.Service,
 
 func assertInternalLbResourcesDeleted(t *testing.T, gce *Cloud, apiService *v1.Service, vals TestClusterValues, firewallsDeleted bool) {
 	lbName := gce.GetLoadBalancerName(context.TODO(), "", apiService)
-	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(apiService)
+	sharedHealthCheck := !servicehelpers.RequestsOnlyLocalTraffic(apiService)
 	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)
 
 	// ensureExternalLoadBalancer and ensureInternalLoadBalancer both create

pkg/cloudprovider/providers/openstack/BUILD

@@ -20,7 +20,6 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack",
     deps = [
-        "//pkg/api/v1/service:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/volume:go_default_library",
@@ -34,6 +33,7 @@ go_library(
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library",

pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go

@@ -44,7 +44,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	cloudprovider "k8s.io/cloud-provider"
-	"k8s.io/kubernetes/pkg/api/v1/service"
+	servicehelpers "k8s.io/cloud-provider/service/helpers"
 )
 
 // Note: when creating a new Loadbalancer (VM), it can take some time before it is ready for use,
@@ -722,12 +722,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string
 		}
 	}
 
-	sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
 	}
 
-	if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups {
+	if !servicehelpers.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups {
 		return nil, fmt.Errorf("source range restrictions are not supported for openstack load balancers without managing security groups")
 	}
 
@@ -1030,7 +1030,7 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *v1.Ser
 	}
 
 	// get service source ranges
-	sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService)
 	if err != nil {
 		return fmt.Errorf("failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
 	}

pkg/controller/.import-restrictions

@@ -307,7 +307,6 @@
         "k8s.io/kubernetes/pkg/scheduler/api",
         "k8s.io/kubernetes/pkg/scheduler/util",
         "k8s.io/kubernetes/pkg/security/apparmor",
-        "k8s.io/kubernetes/pkg/util/net/sets",
         "k8s.io/kubernetes/pkg/util/parsers",
         "k8s.io/kubernetes/pkg/fieldpath",
         "k8s.io/kubernetes/pkg/scheduler/volumebinder",
@@ -342,7 +341,8 @@
         "k8s.io/utils/path",
         "k8s.io/utils/pointer",
         "k8s.io/utils/exec",
-        "k8s.io/utils/strings"
+        "k8s.io/utils/strings",
+        "k8s.io/utils/net"
       ]
     },
     {

pkg/controller/certificates/signer/cfssl_signer.go

@@ -59,6 +59,9 @@ type cfsslSigner struct {
 	sigAlgo             x509.SignatureAlgorithm
 	client              clientset.Interface
 	certificateDuration time.Duration
+
+	// nowFn returns the current time.  We have here for unit testing
+	nowFn func() time.Time
 }
 
 func newCFSSLSigner(caFile, caKeyFile string, client clientset.Interface, certificateDuration time.Duration) (*cfsslSigner, error) {
@@ -92,6 +95,7 @@ func newCFSSLSigner(caFile, caKeyFile string, client clientset.Interface, certif
 		sigAlgo:             signer.DefaultSigAlgo(priv),
 		client:              client,
 		certificateDuration: certificateDuration,
+		nowFn:               time.Now,
 	}, nil
 }
 
@@ -115,11 +119,21 @@ func (s *cfsslSigner) sign(csr *capi.CertificateSigningRequest) (*capi.Certifica
 	for _, usage := range csr.Spec.Usages {
 		usages = append(usages, string(usage))
 	}
+
+	certExpiryDuration := s.certificateDuration
+	durationUntilExpiry := s.ca.NotAfter.Sub(s.nowFn())
+	if durationUntilExpiry <= 0 {
+		return nil, fmt.Errorf("the signer has expired: %v", s.ca.NotAfter)
+	}
+	if durationUntilExpiry < certExpiryDuration {
+		certExpiryDuration = durationUntilExpiry
+	}
+
 	policy := &config.Signing{
 		Default: &config.SigningProfile{
 			Usage:        usages,
-			Expiry:       s.certificateDuration,
-			ExpiryString: s.certificateDuration.String(),
+			Expiry:       certExpiryDuration,
+			ExpiryString: certExpiryDuration.String(),
 		},
 	}
 	cfs, err := local.NewSigner(s.priv, s.ca, s.sigAlgo, policy)

pkg/controller/certificates/signer/cfssl_signer_test.go

@@ -20,6 +20,7 @@ import (
 	"crypto/x509"
 	"io/ioutil"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -28,10 +29,16 @@ import (
 )
 
 func TestSigner(t *testing.T) {
+	testNow := time.Now()
+	testNowFn := func() time.Time {
+		return testNow
+	}
+
 	s, err := newCFSSLSigner("./testdata/ca.crt", "./testdata/ca.key", nil, 1*time.Hour)
 	if err != nil {
 		t.Fatalf("failed to create signer: %v", err)
 	}
+	s.nowFn = testNowFn
 
 	csrb, err := ioutil.ReadFile("./testdata/kubelet.csr")
 	if err != nil {
@@ -81,4 +88,108 @@ func TestSigner(t *testing.T) {
 	if !reflect.DeepEqual(crt.ExtKeyUsage, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}) {
 		t.Errorf("bad extended key usage")
 	}
+
+	expectedTime := testNow.Add(1 * time.Hour)
+	// there is some jitter that we need to tolerate
+	diff := expectedTime.Sub(crt.NotAfter)
+	if diff > 10*time.Minute || diff < -10*time.Minute {
+		t.Fatal(crt.NotAfter)
+	}
+}
+
+func TestSignerExpired(t *testing.T) {
+	hundredYearsFromNowFn := func() time.Time {
+		return time.Now().Add(24 * time.Hour * 365 * 100)
+	}
+	s, err := newCFSSLSigner("./testdata/ca.crt", "./testdata/ca.key", nil, 1*time.Hour)
+	if err != nil {
+		t.Fatalf("failed to create signer: %v", err)
+	}
+	s.nowFn = hundredYearsFromNowFn
+
+	csrb, err := ioutil.ReadFile("./testdata/kubelet.csr")
+	if err != nil {
+		t.Fatalf("failed to read CSR: %v", err)
+	}
+
+	csr := &capi.CertificateSigningRequest{
+		Spec: capi.CertificateSigningRequestSpec{
+			Request: []byte(csrb),
+			Usages: []capi.KeyUsage{
+				capi.UsageSigning,
+				capi.UsageKeyEncipherment,
+				capi.UsageServerAuth,
+				capi.UsageClientAuth,
+			},
+		},
+	}
+
+	_, err = s.sign(csr)
+	if err == nil {
+		t.Fatal("missing error")
+	}
+	if !strings.HasPrefix(err.Error(), "the signer has expired") {
+		t.Fatal(err)
+	}
+}
+
+func TestDurationLongerThanExpiry(t *testing.T) {
+	testNow := time.Now()
+	testNowFn := func() time.Time {
+		return testNow
+	}
+
+	hundredYears := 24 * time.Hour * 365 * 100
+	s, err := newCFSSLSigner("./testdata/ca.crt", "./testdata/ca.key", nil, hundredYears)
+	if err != nil {
+		t.Fatalf("failed to create signer: %v", err)
+	}
+	s.nowFn = testNowFn
+
+	csrb, err := ioutil.ReadFile("./testdata/kubelet.csr")
+	if err != nil {
+		t.Fatalf("failed to read CSR: %v", err)
+	}
+
+	csr := &capi.CertificateSigningRequest{
+		Spec: capi.CertificateSigningRequestSpec{
+			Request: []byte(csrb),
+			Usages: []capi.KeyUsage{
+				capi.UsageSigning,
+				capi.UsageKeyEncipherment,
+				capi.UsageServerAuth,
+				capi.UsageClientAuth,
+			},
+		},
+	}
+
+	_, err = s.sign(csr)
+	if err != nil {
+		t.Fatalf("failed to sign CSR: %v", err)
+	}
+
+	// now we just need to verify that the expiry is based on the signing cert
+	certData := csr.Status.Certificate
+	if len(certData) == 0 {
+		t.Fatalf("expected a certificate after signing")
+	}
+
+	certs, err := cert.ParseCertsPEM(certData)
+	if err != nil {
+		t.Fatalf("failed to parse certificate: %v", err)
+	}
+	if len(certs) != 1 {
+		t.Fatalf("expected one certificate")
+	}
+
+	crt := certs[0]
+	expected, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", "2044-05-09 00:20:11 +0000 UTC")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// there is some jitter that we need to tolerate
+	diff := expected.Sub(crt.NotAfter)
+	if diff > 10*time.Minute || diff < -10*time.Minute {
+		t.Fatal(crt.NotAfter)
+	}
 }

pkg/controller/garbagecollector/operations.go

@@ -19,8 +19,6 @@ package garbagecollector
 import (
 	"fmt"
 
-	"k8s.io/klog"
-
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
+	"k8s.io/klog"
 )
 
 // cluster scoped resources don't have namespaces.  Default to the item's namespace, but clear it for cluster scoped resources
@@ -81,7 +80,7 @@ func (gc *GarbageCollector) patchObject(item objectReference, patch []byte, pt t
 	if err != nil {
 		return nil, err
 	}
-	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Patch(item.Name, pt, patch, metav1.UpdateOptions{})
+	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Patch(item.Name, pt, patch, metav1.PatchOptions{})
 }
 
 // TODO: Using Patch when strategicmerge supports deleting an entry from a

pkg/controller/podautoscaler/horizontal.go

@@ -410,7 +410,7 @@ func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas in
 			return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
 		}
 		metricNameProposal := fmt.Sprintf("%s resource", metricSpec.Resource.Name)
-		status = &autoscalingv2.MetricStatus{
+		*status = autoscalingv2.MetricStatus{
 			Type: autoscalingv2.ResourceMetricSourceType,
 			Resource: &autoscalingv2.ResourceMetricStatus{
 				Name: metricSpec.Resource.Name,

pkg/credentialprovider/gcp/metadata.go

@@ -173,7 +173,7 @@ func (g *dockerConfigUrlKeyProvider) Provide() credentialprovider.DockerConfig {
 	return credentialprovider.DockerConfig{}
 }
 
-// runcWithBackoff runs input function `f` with an exponential backoff.
+// runWithBackoff runs input function `f` with an exponential backoff.
 // Note that this method can block indefinitely.
 func runWithBackoff(f func() ([]byte, error)) []byte {
 	var backoff = 100 * time.Millisecond

pkg/features/kube_features.go

@@ -163,6 +163,7 @@ const (
 
 	// owner: @derekwaynecarr
 	// beta: v1.10
+	// GA: v1.14
 	//
 	// Enable pods to consume pre-allocated huge pages of varying page sizes
 	HugePages utilfeature.Feature = "HugePages"
@@ -424,7 +425,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	RotateKubeletClientCertificate:              {Default: true, PreRelease: utilfeature.Beta},
 	PersistentLocalVolumes:                      {Default: true, PreRelease: utilfeature.Beta},
 	LocalStorageCapacityIsolation:               {Default: true, PreRelease: utilfeature.Beta},
-	HugePages:                                   {Default: true, PreRelease: utilfeature.Beta},
+	HugePages:                                   {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16
 	Sysctls:                                     {Default: true, PreRelease: utilfeature.Beta},
 	DebugContainers:                             {Default: false, PreRelease: utilfeature.Alpha},
 	PodShareProcessNamespace:                    {Default: true, PreRelease: utilfeature.Beta},
@@ -484,6 +485,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	genericfeatures.APIResponseCompression:  {Default: false, PreRelease: utilfeature.Alpha},
 	genericfeatures.APIListChunking:         {Default: true, PreRelease: utilfeature.Beta},
 	genericfeatures.DryRun:                  {Default: true, PreRelease: utilfeature.Beta},
+	genericfeatures.ServerSideApply:         {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from apiextensions-apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubectl/.import-restrictions

@@ -132,7 +132,6 @@
                 "k8s.io/kubernetes/pkg/util/labels",
                 "k8s.io/kubernetes/pkg/util/metrics",
                 "k8s.io/kubernetes/pkg/util/mount",
-                "k8s.io/kubernetes/pkg/util/net/sets",
                 "k8s.io/kubernetes/pkg/util/node",
                 "k8s.io/kubernetes/pkg/util/parsers",
                 "k8s.io/kubernetes/pkg/util/slice",
@@ -144,7 +143,8 @@
                 "k8s.io/utils/nsenter",
                 "k8s.io/utils/io",
                 "k8s.io/utils/path",
-                "k8s.io/utils/pointer"
+                "k8s.io/utils/pointer",
+                "k8s.io/utils/net"
 		],
 		"ForbiddenPrefixes": []
 	}]

pkg/kubectl/cmd/apply/apply.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"strings"
 	"time"
 
@@ -65,16 +66,18 @@ type ApplyOptions struct {
 	DeleteFlags   *delete.DeleteFlags
 	DeleteOptions *delete.DeleteOptions
 
-	Selector       string
-	DryRun         bool
-	ServerDryRun   bool
-	Prune          bool
-	PruneResources []pruneResource
-	cmdBaseName    string
-	All            bool
-	Overwrite      bool
-	OpenAPIPatch   bool
-	PruneWhitelist []string
+	ServerSideApply bool
+	ForceConflicts  bool
+	Selector        string
+	DryRun          bool
+	ServerDryRun    bool
+	Prune           bool
+	PruneResources  []pruneResource
+	cmdBaseName     string
+	All             bool
+	Overwrite       bool
+	OpenAPIPatch    bool
+	PruneWhitelist  []string
 
 	Validator       validation.Schema
 	Builder         *resource.Builder
@@ -178,6 +181,7 @@ func NewCmdApply(baseName string, f cmdutil.Factory, ioStreams genericclioptions
 	cmd.Flags().BoolVar(&o.ServerDryRun, "server-dry-run", o.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.")
 	cmd.Flags().Bool("dry-run", false, "If true, only print the object that would be sent, without sending it. Warning: --dry-run cannot accurately output the result of merging the local manifest and the server-side data. Use --server-dry-run to get the merged result instead.")
 	cmdutil.AddIncludeUninitializedFlag(cmd)
+	cmdutil.AddServerSideApplyFlags(cmd)
 
 	// apply subcommands
 	cmd.AddCommand(NewCmdApplyViewLastApplied(f, ioStreams))
@@ -188,8 +192,18 @@ func NewCmdApply(baseName string, f cmdutil.Factory, ioStreams genericclioptions
 }
 
 func (o *ApplyOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) error {
+	o.ServerSideApply = cmdutil.GetServerSideApplyFlag(cmd)
+	o.ForceConflicts = cmdutil.GetForceConflictsFlag(cmd)
 	o.DryRun = cmdutil.GetDryRunFlag(cmd)
 
+	if o.ForceConflicts && !o.ServerSideApply {
+		return fmt.Errorf("--force-conflicts only works with --server-side")
+	}
+
+	if o.DryRun && o.ServerSideApply {
+		return fmt.Errorf("--dry-run doesn't work with --server-side")
+	}
+
 	if o.DryRun && o.ServerDryRun {
 		return fmt.Errorf("--dry-run and --server-dry-run can't be used together")
 	}
@@ -293,6 +307,16 @@ func parsePruneResources(mapper meta.RESTMapper, gvks []string) ([]pruneResource
 	return pruneResources, nil
 }
 
+func isIncompatibleServerError(err error) bool {
+	// 415: Unsupported media type means we're talking to a server which doesn't
+	// support server-side apply.
+	if _, ok := err.(*errors.StatusError); !ok {
+		// Non-StatusError means the error isn't because the server is incompatible.
+		return false
+	}
+	return err.(*errors.StatusError).Status().Code == http.StatusUnsupportedMediaType
+}
+
 func (o *ApplyOptions) Run() error {
 	var openapiSchema openapi.Resources
 	if o.OpenAPIPatch {
@@ -356,6 +380,50 @@ func (o *ApplyOptions) Run() error {
 			klog.V(4).Infof("error recording current command: %v", err)
 		}
 
+		if o.ServerSideApply {
+			// Send the full object to be applied on the server side.
+			data, err := runtime.Encode(unstructured.UnstructuredJSONScheme, info.Object)
+			if err != nil {
+				return cmdutil.AddSourceToErr("serverside-apply", info.Source, err)
+			}
+			options := metav1.PatchOptions{
+				Force: &o.ForceConflicts,
+			}
+			if o.ServerDryRun {
+				options.DryRun = []string{metav1.DryRunAll}
+			}
+			obj, err := resource.NewHelper(info.Client, info.Mapping).Patch(
+				info.Namespace,
+				info.Name,
+				types.ApplyPatchType,
+				data,
+				&options,
+			)
+			if err == nil {
+				info.Refresh(obj, true)
+				metadata, err := meta.Accessor(info.Object)
+				if err != nil {
+					return err
+				}
+				visitedUids.Insert(string(metadata.GetUID()))
+				count++
+				if len(output) > 0 && !shortOutput {
+					objs = append(objs, info.Object)
+					return nil
+				}
+				printer, err := o.ToPrinter("serverside-applied")
+				if err != nil {
+					return err
+				}
+				return printer.PrintObj(info.Object, o.Out)
+			} else if !isIncompatibleServerError(err) {
+				return err
+			}
+			// If we're talking to a server which does not implement server-side apply,
+			// continue with the client side apply after this block.
+			klog.Warningf("serverside-apply incompatible server: %v", err)
+		}
+
 		// Get the modified configuration of the object. Embed the result
 		// as an annotation in the modified configuration, so that it will appear
 		// in the patch sent to the server.
@@ -840,7 +908,7 @@ func (p *Patcher) patchSimple(obj runtime.Object, modified []byte, source, names
 		}
 	}
 
-	options := metav1.UpdateOptions{}
+	options := metav1.PatchOptions{}
 	if p.ServerDryRun {
 		options.DryRun = []string{metav1.DryRunAll}
 	}

pkg/kubectl/cmd/attach/attach.go

@@ -76,7 +76,7 @@ type AttachOptions struct {
 	AttachFunc       func(*AttachOptions, *corev1.Container, bool, remotecommand.TerminalSizeQueue) func() error
 	Resources        []string
 	Builder          func() *resource.Builder
-	AttachablePodFn  polymorphichelpers.AttachableLogsForObjectFunc
+	AttachablePodFn  polymorphichelpers.AttachablePodForObjectFunc
 	restClientGetter genericclioptions.RESTClientGetter
 
 	Attach        RemoteAttach

pkg/kubectl/cmd/attach/attach_test.go

@@ -51,7 +51,7 @@ func (f *fakeRemoteAttach) Attach(method string, url *url.URL, config *restclien
 	return f.err
 }
 
-func fakeAttachablePodFn(pod *corev1.Pod) polymorphichelpers.AttachableLogsForObjectFunc {
+func fakeAttachablePodFn(pod *corev1.Pod) polymorphichelpers.AttachablePodForObjectFunc {
 	return func(getter genericclioptions.RESTClientGetter, obj runtime.Object, timeout time.Duration) (*corev1.Pod, error) {
 		return pod, nil
 	}

pkg/kubectl/cmd/auth/reconcile.go

@@ -59,7 +59,15 @@ var (
 	reconcileLong = templates.LongDesc(`
 		Reconciles rules for RBAC Role, RoleBinding, ClusterRole, and ClusterRole binding objects.
 
-		This is preferred to 'apply' for RBAC resources so that proper rule coverage checks are done.`)
+		Missing objects are created, and the containing namespace is created for namespaced objects, if required.
+		
+		Existing roles are updated to include the permissions in the input objects,
+		and remove extra permissions if --remove-extra-permissions is specified.
+
+		Existing bindings are updated to include the subjects in the input objects,
+		and remove extra subjects if --remove-extra-subjects is specified.
+
+		This is preferred to 'apply' for RBAC resources so that semantically-aware merging of rules and subjects is done.`)
 
 	reconcileExample = templates.Examples(`
 		# Reconcile rbac resources from a file

pkg/kubectl/cmd/diff/BUILD

@@ -18,8 +18,11 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library",
         "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library",
+        "//staging/src/k8s.io/client-go/discovery:go_default_library",
+        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/github.com/jonboulle/clockwork:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",

pkg/kubectl/cmd/diff/diff.go

@@ -30,8 +30,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/genericclioptions/resource"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/kubectl"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/apply"
@@ -67,21 +70,38 @@ const maxRetries = 4
 
 type DiffOptions struct {
 	FilenameOptions resource.FilenameOptions
+
+	ServerSideApply bool
+	ForceConflicts  bool
+
+	OpenAPISchema    openapi.Resources
+	DiscoveryClient  discovery.DiscoveryInterface
+	DynamicClient    dynamic.Interface
+	DryRunVerifier   *apply.DryRunVerifier
+	CmdNamespace     string
+	EnforceNamespace bool
+	Builder          *resource.Builder
+	Diff             *DiffProgram
 }
 
-func checkDiffArgs(cmd *cobra.Command, args []string) error {
+func validateArgs(cmd *cobra.Command, args []string) error {
 	if len(args) != 0 {
 		return cmdutil.UsageErrorf(cmd, "Unexpected args: %v", args)
 	}
 	return nil
 }
 
-func NewCmdDiff(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
-	var options DiffOptions
-	diff := DiffProgram{
-		Exec:      exec.New(),
-		IOStreams: streams,
+func NewDiffOptions(ioStreams genericclioptions.IOStreams) *DiffOptions {
+	return &DiffOptions{
+		Diff: &DiffProgram{
+			Exec:      exec.New(),
+			IOStreams: ioStreams,
+		},
 	}
+}
+
+func NewCmdDiff(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
+	options := NewDiffOptions(streams)
 	cmd := &cobra.Command{
 		Use:                   "diff -f FILENAME",
 		DisableFlagsInUseLine: true,
@@ -89,13 +109,15 @@ func NewCmdDiff(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C
 		Long:                  diffLong,
 		Example:               diffExample,
 		Run: func(cmd *cobra.Command, args []string) {
-			cmdutil.CheckErr(checkDiffArgs(cmd, args))
-			cmdutil.CheckErr(RunDiff(f, &diff, &options))
+			cmdutil.CheckErr(options.Complete(f, cmd))
+			cmdutil.CheckErr(validateArgs(cmd, args))
+			cmdutil.CheckErr(options.Run())
 		},
 	}
 
 	usage := "contains the configuration to diff"
 	cmdutil.AddFilenameOptionFlags(cmd, &options.FilenameOptions, usage)
+	cmdutil.AddServerSideApplyFlags(cmd)
 	cmd.MarkFlagRequired("filename")
 
 	return cmd
@@ -229,11 +251,13 @@ type Object interface {
 // InfoObject is an implementation of the Object interface. It gets all
 // the information from the Info object.
 type InfoObject struct {
-	LocalObj runtime.Object
-	Info     *resource.Info
-	Encoder  runtime.Encoder
-	OpenAPI  openapi.Resources
-	Force    bool
+	LocalObj        runtime.Object
+	Info            *resource.Info
+	Encoder         runtime.Encoder
+	OpenAPI         openapi.Resources
+	Force           bool
+	ServerSideApply bool
+	ForceConflicts  bool
 }
 
 var _ Object = &InfoObject{}
@@ -246,6 +270,24 @@ func (obj InfoObject) Live() runtime.Object {
 // Returns the "merged" object, as it would look like if applied or
 // created.
 func (obj InfoObject) Merged() (runtime.Object, error) {
+	if obj.ServerSideApply {
+		data, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj.LocalObj)
+		if err != nil {
+			return nil, err
+		}
+		options := metav1.PatchOptions{
+			Force:  &obj.ForceConflicts,
+			DryRun: []string{metav1.DryRunAll},
+		}
+		return resource.NewHelper(obj.Info.Client, obj.Info.Mapping).Patch(
+			obj.Info.Namespace,
+			obj.Info.Name,
+			types.ApplyPatchType,
+			data,
+			&options,
+		)
+	}
+
 	// Build the patcher, and then apply the patch with dry-run, unless the object doesn't exist, in which case we need to create it.
 	if obj.Live() == nil {
 		// Dry-run create if the object doesn't exist.
@@ -350,47 +392,62 @@ func isConflict(err error) bool {
 	return err != nil && errors.IsConflict(err)
 }
 
-// RunDiff uses the factory to parse file arguments, find the version to
-// diff, and find each Info object for each files, and runs against the
-// differ.
-func RunDiff(f cmdutil.Factory, diff *DiffProgram, options *DiffOptions) error {
-	schema, err := f.OpenAPISchema()
-	if err != nil {
-		return err
+func (o *DiffOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) error {
+	var err error
+
+	o.ServerSideApply = cmdutil.GetServerSideApplyFlag(cmd)
+	o.ForceConflicts = cmdutil.GetForceConflictsFlag(cmd)
+	if o.ForceConflicts && !o.ServerSideApply {
+		return fmt.Errorf("--force-conflicts only works with --server-side")
+	}
+
+	if !o.ServerSideApply {
+		o.OpenAPISchema, err = f.OpenAPISchema()
+		if err != nil {
+			return err
+		}
 	}
 
-	discovery, err := f.ToDiscoveryClient()
+	o.DiscoveryClient, err = f.ToDiscoveryClient()
 	if err != nil {
 		return err
 	}
 
-	dynamic, err := f.DynamicClient()
+	o.DynamicClient, err = f.DynamicClient()
 	if err != nil {
 		return err
 	}
 
-	dryRunVerifier := &apply.DryRunVerifier{
-		Finder:        cmdutil.NewCRDFinder(cmdutil.CRDFromDynamic(dynamic)),
-		OpenAPIGetter: discovery,
+	o.DryRunVerifier = &apply.DryRunVerifier{
+		Finder:        cmdutil.NewCRDFinder(cmdutil.CRDFromDynamic(o.DynamicClient)),
+		OpenAPIGetter: o.DiscoveryClient,
 	}
 
-	differ, err := NewDiffer("LIVE", "MERGED")
+	o.CmdNamespace, o.EnforceNamespace, err = f.ToRawKubeConfigLoader().Namespace()
 	if err != nil {
 		return err
 	}
-	defer differ.TearDown()
 
-	printer := Printer{}
+	o.Builder = f.NewBuilder()
+	return nil
+}
 
-	cmdNamespace, enforceNamespace, err := f.ToRawKubeConfigLoader().Namespace()
+// RunDiff uses the factory to parse file arguments, find the version to
+// diff, and find each Info object for each files, and runs against the
+// differ.
+func (o *DiffOptions) Run() error {
+	differ, err := NewDiffer("LIVE", "MERGED")
 	if err != nil {
 		return err
 	}
+	defer differ.TearDown()
+
+	printer := Printer{}
 
-	r := f.NewBuilder().
+	r := o.Builder.
 		Unstructured().
-		NamespaceParam(cmdNamespace).DefaultNamespace().
-		FilenameParam(enforceNamespace, &options.FilenameOptions).
+		NamespaceParam(o.CmdNamespace).DefaultNamespace().
+		FilenameParam(o.EnforceNamespace, &o.FilenameOptions).
 		Flatten().
 		Do()
 	if err := r.Err(); err != nil {
@@ -402,7 +459,7 @@ func RunDiff(f cmdutil.Factory, diff *DiffProgram, options *DiffOptions) error {
 			return err
 		}
 
-		if err := dryRunVerifier.HasSupport(info.Mapping.GroupVersionKind); err != nil {
+		if err := o.DryRunVerifier.HasSupport(info.Mapping.GroupVersionKind); err != nil {
 			return err
 		}
 
@@ -424,11 +481,13 @@ func RunDiff(f cmdutil.Factory, diff *DiffProgram, options *DiffOptions) error {
 				)
 			}
 			obj := InfoObject{
-				LocalObj: local,
-				Info:     info,
-				Encoder:  scheme.DefaultJSONEncoder(),
-				OpenAPI:  schema,
-				Force:    force,
+				LocalObj:        local,
+				Info:            info,
+				Encoder:         scheme.DefaultJSONEncoder(),
+				OpenAPI:         o.OpenAPISchema,
+				Force:           force,
+				ServerSideApply: o.ServerSideApply,
+				ForceConflicts:  o.ForceConflicts,
 			}
 
 			err = differ.Diff(obj, printer)
@@ -442,5 +501,5 @@ func RunDiff(f cmdutil.Factory, diff *DiffProgram, options *DiffOptions) error {
 		return err
 	}
 
-	return differ.Run(diff)
+	return differ.Run(o.Diff)
 }

pkg/kubectl/cmd/util/helpers.go

@@ -29,8 +29,6 @@ import (
 	"github.com/evanphx/json-patch"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
-
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,6 +42,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/scale"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
 	utilexec "k8s.io/utils/exec"
 )
 
@@ -400,6 +399,11 @@ func AddDryRunFlag(cmd *cobra.Command) {
 	cmd.Flags().Bool("dry-run", false, "If true, only print the object that would be sent, without sending it.")
 }
 
+func AddServerSideApplyFlags(cmd *cobra.Command) {
+	cmd.Flags().Bool("server-side", false, "If true, apply runs in the server instead of the client. This is an alpha feature and flag.")
+	cmd.Flags().Bool("force-conflicts", false, "If true, server-side apply will force the changes against conflicts. This is an alpha feature and flag.")
+}
+
 func AddIncludeUninitializedFlag(cmd *cobra.Command) {
 	cmd.Flags().Bool("include-uninitialized", false, `If true, the kubectl command applies to uninitialized objects. If explicitly set to false, this flag overrides other flags that make the kubectl commands apply to uninitialized objects, e.g., "--all". Objects with empty metadata.initializers are regarded as initialized.`)
 	cmd.Flags().MarkDeprecated("include-uninitialized", "The Initializers feature has been removed. This flag is now a no-op, and will be removed in v1.15")
@@ -473,6 +477,14 @@ func DumpReaderToFile(reader io.Reader, filename string) error {
 	return nil
 }
 
+func GetServerSideApplyFlag(cmd *cobra.Command) bool {
+	return GetFlagBool(cmd, "server-side")
+}
+
+func GetForceConflictsFlag(cmd *cobra.Command) bool {
+	return GetFlagBool(cmd, "force-conflicts")
+}
+
 func GetDryRunFlag(cmd *cobra.Command) bool {
 	return GetFlagBool(cmd, "dry-run")
 }

pkg/kubectl/cmd/wait/wait.go

@@ -90,6 +90,8 @@ func NewWaitFlags(restClientGetter genericclioptions.RESTClientGetter, streams g
 		PrintFlags:       genericclioptions.NewPrintFlags("condition met"),
 		ResourceBuilderFlags: genericclioptions.NewResourceBuilderFlags().
 			WithLabelSelector("").
+			WithFieldSelector("").
+			WithAll(false).
 			WithAllNamespaces(false).
 			WithAll(false).
 			WithLatest(),
@@ -105,11 +107,12 @@ func NewCmdWait(restClientGetter genericclioptions.RESTClientGetter, streams gen
 	flags := NewWaitFlags(restClientGetter, streams)
 
 	cmd := &cobra.Command{
-		Use:                   "wait resource.group/name [--for=delete|--for condition=available]",
+		Use:     "wait ([-f FILENAME] | resource.group/resource.name | resource.group [(-l label | --all)]) [--for=delete|--for condition=available]",
+		Short:   "Experimental: Wait for a specific condition on one or many resources.",
+		Long:    waitLong,
+		Example: waitExample,
+
 		DisableFlagsInUseLine: true,
-		Short:                 "Experimental: Wait for a specific condition on one or many resources.",
-		Long:                  waitLong,
-		Example:               waitExample,
 		Run: func(cmd *cobra.Command, args []string) {
 			o, err := flags.ToOptions(args)
 			cmdutil.CheckErr(err)

pkg/kubectl/cmd/wait/wait_test.go

@@ -311,6 +311,58 @@ func TestWaitForDeletion(t *testing.T) {
 			},
 		},
 		{
+			name: "handles watch delete multiple",
+			infos: []*resource.Info{
+				{
+					Mapping: &meta.RESTMapping{
+						Resource: schema.GroupVersionResource{Group: "group", Version: "version", Resource: "theresource-1"},
+					},
+					Name:      "name-foo-1",
+					Namespace: "ns-foo",
+				},
+				{
+					Mapping: &meta.RESTMapping{
+						Resource: schema.GroupVersionResource{Group: "group", Version: "version", Resource: "theresource-2"},
+					},
+					Name:      "name-foo-2",
+					Namespace: "ns-foo",
+				},
+			},
+			fakeClient: func() *dynamicfakeclient.FakeDynamicClient {
+				fakeClient := dynamicfakeclient.NewSimpleDynamicClient(scheme)
+				fakeClient.PrependReactor("get", "theresource-1", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, newUnstructured("group/version", "TheKind", "ns-foo", "name-foo-1"), nil
+				})
+				fakeClient.PrependReactor("get", "theresource-2", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, newUnstructured("group/version", "TheKind", "ns-foo", "name-foo-2"), nil
+				})
+				fakeClient.PrependWatchReactor("theresource-1", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
+					fakeWatch := watch.NewRaceFreeFake()
+					fakeWatch.Action(watch.Deleted, newUnstructured("group/version", "TheKind", "ns-foo", "name-foo-1"))
+					return true, fakeWatch, nil
+				})
+				fakeClient.PrependWatchReactor("theresource-2", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
+					fakeWatch := watch.NewRaceFreeFake()
+					fakeWatch.Action(watch.Deleted, newUnstructured("group/version", "TheKind", "ns-foo", "name-foo-2"))
+					return true, fakeWatch, nil
+				})
+				return fakeClient
+			},
+			timeout: 10 * time.Second,
+
+			validateActions: func(t *testing.T, actions []clienttesting.Action) {
+				if len(actions) != 2 {
+					t.Fatal(spew.Sdump(actions))
+				}
+				if !actions[0].Matches("list", "theresource-1") {
+					t.Error(spew.Sdump(actions))
+				}
+				if !actions[1].Matches("list", "theresource-2") {
+					t.Error(spew.Sdump(actions))
+				}
+			},
+		},
+		{
 			name: "ignores watch error",
 			infos: []*resource.Info{
 				{

pkg/kubectl/polymorphichelpers/interface.go

@@ -34,11 +34,11 @@ type LogsForObjectFunc func(restClientGetter genericclioptions.RESTClientGetter,
 // LogsForObjectFn gives a way to easily override the function for unit testing if needed.
 var LogsForObjectFn LogsForObjectFunc = logsForObject
 
-// AttachableLogsForObjectFunc is a function type that can tell you how to get the pod for which to attach a given object
-type AttachableLogsForObjectFunc func(restClientGetter genericclioptions.RESTClientGetter, object runtime.Object, timeout time.Duration) (*v1.Pod, error)
+// AttachablePodForObjectFunc is a function type that can tell you how to get the pod for which to attach a given object
+type AttachablePodForObjectFunc func(restClientGetter genericclioptions.RESTClientGetter, object runtime.Object, timeout time.Duration) (*v1.Pod, error)
 
 // AttachablePodForObjectFn gives a way to easily override the function for unit testing if needed.
-var AttachablePodForObjectFn AttachableLogsForObjectFunc = attachablePodForObject
+var AttachablePodForObjectFn AttachablePodForObjectFunc = attachablePodForObject
 
 // HistoryViewerFunc is a function type that can tell you how to view change history
 type HistoryViewerFunc func(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (kubectl.HistoryViewer, error)

pkg/kubelet/BUILD

@@ -131,7 +131,6 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
@@ -141,6 +140,7 @@ go_library(
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
+        "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
         "//third_party/forked/golang/expansion:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
         "//vendor/github.com/google/cadvisor/events:go_default_library",

pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go

@@ -135,7 +135,7 @@ func (*Empty) ProtoMessage()               {}
 func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{2} }
 
 // ListAndWatch returns a stream of List of Devices
-// Whenever a Device state change or a Device disapears, ListAndWatch
+// Whenever a Device state change or a Device disappears, ListAndWatch
 // returns the new list
 type ListAndWatchResponse struct {
 	Devices []*Device `protobuf:"bytes,1,rep,name=devices" json:"devices,omitempty"`
@@ -482,7 +482,7 @@ type DevicePluginClient interface {
 	// Manager
 	GetDevicePluginOptions(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*DevicePluginOptions, error)
 	// ListAndWatch returns a stream of List of Devices
-	// Whenever a Device state change or a Device disapears, ListAndWatch
+	// Whenever a Device state change or a Device disappears, ListAndWatch
 	// returns the new list
 	ListAndWatch(ctx context.Context, in *Empty, opts ...grpc.CallOption) (DevicePlugin_ListAndWatchClient, error)
 	// Allocate is called during container creation so that the Device
@@ -569,7 +569,7 @@ type DevicePluginServer interface {
 	// Manager
 	GetDevicePluginOptions(context.Context, *Empty) (*DevicePluginOptions, error)
 	// ListAndWatch returns a stream of List of Devices
-	// Whenever a Device state change or a Device disapears, ListAndWatch
+	// Whenever a Device state change or a Device disappears, ListAndWatch
 	// returns the new list
 	ListAndWatch(*Empty, DevicePlugin_ListAndWatchServer) error
 	// Allocate is called during container creation so that the Device

pkg/kubelet/apis/deviceplugin/v1beta1/api.proto

@@ -51,7 +51,7 @@ service DevicePlugin {
 	rpc GetDevicePluginOptions(Empty) returns (DevicePluginOptions) {}
 
 	// ListAndWatch returns a stream of List of Devices
-	// Whenever a Device state change or a Device disapears, ListAndWatch
+	// Whenever a Device state change or a Device disappears, ListAndWatch
 	// returns the new list
 	rpc ListAndWatch(Empty) returns (stream ListAndWatchResponse) {}
 
@@ -67,7 +67,7 @@ service DevicePlugin {
 }
 
 // ListAndWatch returns a stream of List of Devices
-// Whenever a Device state change or a Device disapears, ListAndWatch
+// Whenever a Device state change or a Device disappears, ListAndWatch
 // returns the new list
 message ListAndWatchResponse {
 	repeated Device devices = 1;

pkg/kubelet/cadvisor/BUILD

@@ -21,11 +21,9 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/kubelet/cadvisor",
     deps = [
         "//pkg/apis/core/v1/helper:go_default_library",
-        "//pkg/features:go_default_library",
         "//pkg/kubelet/types:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/github.com/google/cadvisor/events:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
@@ -51,11 +49,8 @@ go_test(
     embed = [":go_default_library"],
     deps = select({
         "@io_bazel_rules_go//go/platform:linux": [
-            "//pkg/features:go_default_library",
             "//staging/src/k8s.io/api/core/v1:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
             "//vendor/github.com/google/cadvisor/container/crio:go_default_library",
             "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
             "//vendor/github.com/stretchr/testify/assert:go_default_library",

pkg/kubelet/cadvisor/util.go

@@ -23,9 +23,7 @@ import (
 	cadvisorapi2 "github.com/google/cadvisor/info/v2"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	"k8s.io/kubernetes/pkg/features"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
@@ -46,13 +44,11 @@ func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList {
 	}
 
 	// if huge pages are enabled, we report them as a schedulable resource on the node
-	if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) {
-		for _, hugepagesInfo := range info.HugePages {
-			pageSizeBytes := int64(hugepagesInfo.PageSize * 1024)
-			hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages)
-			pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI)
-			c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI)
-		}
+	for _, hugepagesInfo := range info.HugePages {
+		pageSizeBytes := int64(hugepagesInfo.PageSize * 1024)
+		hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages)
+		pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI)
+		c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI)
 	}
 
 	return c

pkg/kubelet/cadvisor/util_test.go

@@ -27,9 +27,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	"k8s.io/kubernetes/pkg/features"
 )
 
 func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) {
@@ -49,36 +46,12 @@ func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) {
 		v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI),
 		"hugepages-5Ki":   *resource.NewQuantity(int64(51200), resource.BinarySI),
 	}
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
 	actual := CapacityFromMachineInfo(machineInfo)
 	if !reflect.DeepEqual(actual, expected) {
 		t.Errorf("when set hugepages true, got resource list %v, want %v", actual, expected)
 	}
 }
 
-func TestCapacityFromMachineInfoWithHugePagesDisable(t *testing.T) {
-	machineInfo := &info.MachineInfo{
-		NumCores:       2,
-		MemoryCapacity: 2048,
-		HugePages: []info.HugePagesInfo{
-			{
-				PageSize: 5,
-				NumPages: 10,
-			},
-		},
-	}
-
-	expected := v1.ResourceList{
-		v1.ResourceCPU:    *resource.NewMilliQuantity(int64(2000), resource.DecimalSI),
-		v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI),
-	}
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
-	actual := CapacityFromMachineInfo(machineInfo)
-	if !reflect.DeepEqual(actual, expected) {
-		t.Errorf("when set hugepages false, got resource list %v, want %v", actual, expected)
-	}
-}
-
 func TestCrioSocket(t *testing.T) {
 	assert.EqualValues(t, CrioSocket, crio.CrioSocket, "CrioSocket in this package must equal the one in github.com/google/cadvisor/container/crio/client.go")
 }

pkg/kubelet/cm/cgroup_manager_linux.go

@@ -327,9 +327,7 @@ func getSupportedSubsystems() map[subsystem]bool {
 		&cgroupfs.CpuGroup{}:    true,
 	}
 	// not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity.
-	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
-		supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
-	}
+	supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
 	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) {
 		supportedSubsystems[&cgroupfs.PidsGroup{}] = true
 	}
@@ -385,27 +383,25 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont
 		}
 	}
 	// if huge pages are enabled, we set them in libcontainer
-	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
-		// for each page size enumerated, set that value
-		pageSizes := sets.NewString()
-		for pageSize, limit := range resourceConfig.HugePageLimit {
-			sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList)
-			resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
-				Pagesize: sizeString,
-				Limit:    uint64(limit),
-			})
-			pageSizes.Insert(sizeString)
-		}
-		// for each page size omitted, limit to 0
-		for _, pageSize := range cgroupfs.HugePageSizes {
-			if pageSizes.Has(pageSize) {
-				continue
-			}
-			resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
-				Pagesize: pageSize,
-				Limit:    uint64(0),
-			})
+	// for each page size enumerated, set that value
+	pageSizes := sets.NewString()
+	for pageSize, limit := range resourceConfig.HugePageLimit {
+		sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList)
+		resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
+			Pagesize: sizeString,
+			Limit:    uint64(limit),
+		})
+		pageSizes.Insert(sizeString)
+	}
+	// for each page size omitted, limit to 0
+	for _, pageSize := range cgroupfs.HugePageSizes {
+		if pageSizes.Has(pageSize) {
+			continue
 		}
+		resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
+			Pagesize: pageSize,
+			Limit:    uint64(0),
+		})
 	}
 	return resources
 }

pkg/kubelet/cm/devicemanager/manager.go

@@ -612,7 +612,7 @@ func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, requi
 	devicesInUse := m.allocatedDevices[resource]
 	// Gets a list of available devices.
 	available := m.healthyDevices[resource].Difference(devicesInUse)
-	if int(available.Len()) < needed {
+	if available.Len() < needed {
 		return nil, fmt.Errorf("requested number of devices unavailable for %s. Requested: %d, Available: %d", resource, needed, available.Len())
 	}
 	allocated := available.UnsortedList()[:needed]

pkg/kubelet/cm/node_container_manager_linux.go

@@ -26,9 +26,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog"
-	kubefeatures "k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/events"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
@@ -159,9 +157,7 @@ func getCgroupConfig(rl v1.ResourceList) *ResourceConfig {
 		val := MilliCPUToShares(q.MilliValue())
 		rc.CpuShares = &val
 	}
-	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
-		rc.HugePageLimit = HugePageLimits(rl)
-	}
+	rc.HugePageLimit = HugePageLimits(rl)
 
 	return &rc
 }

pkg/kubelet/cm/qos_container_manager_linux.go

@@ -108,9 +108,7 @@ func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceLis
 		}
 
 		// for each enumerated huge page size, the qos tiers are unbounded
-		if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
-			m.setHugePagesUnbounded(containerConfig)
-		}
+		m.setHugePagesUnbounded(containerConfig)
 
 		// check if it exists
 		if !cm.Exists(containerName) {
@@ -290,10 +288,8 @@ func (m *qosContainerManagerImpl) UpdateCgroups() error {
 	}
 
 	// update the qos level cgroup settings for huge pages (ensure they remain unbounded)
-	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
-		if err := m.setHugePagesConfig(qosConfigs); err != nil {
-			return err
-		}
+	if err := m.setHugePagesConfig(qosConfigs); err != nil {
+		return err
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.QOSReserved) {

pkg/kubelet/dockershim/network/hostport/BUILD

@@ -19,12 +19,12 @@ go_library(
         "//pkg/proxy/iptables:go_default_library",
         "//pkg/util/conntrack:go_default_library",
         "//pkg/util/iptables:go_default_library",
-        "//pkg/util/net:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 

pkg/kubelet/dockershim/network/hostport/hostport_manager.go

@@ -31,8 +31,8 @@ import (
 	iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables"
 	"k8s.io/kubernetes/pkg/util/conntrack"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
-	utilnet "k8s.io/kubernetes/pkg/util/net"
 	"k8s.io/utils/exec"
+	utilnet "k8s.io/utils/net"
 )
 
 // HostPortManager is an interface for adding and removing hostport for a given pod sandbox.

pkg/kubelet/kubelet.go

@@ -43,7 +43,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
@@ -114,6 +113,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/oom"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/csi"
+	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	utilexec "k8s.io/utils/exec"
 	"k8s.io/utils/integer"
 )
@@ -249,7 +249,7 @@ type Dependencies struct {
 	OnHeartbeatFailure      func()
 	KubeClient              clientset.Interface
 	CSIClient               csiclientset.Interface
-	DynamicKubeClient       dynamic.Interface
+	NodeAPIClient           nodeapiclientset.Interface
 	Mounter                 mount.Interface
 	OOMAdjuster             *oom.OOMAdjuster
 	OSInterface             kubecontainer.OSInterface
@@ -658,8 +658,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	klet.runtimeService = runtimeService
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.DynamicKubeClient != nil {
-		klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.DynamicKubeClient)
+	if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.NodeAPIClient != nil {
+		klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.NodeAPIClient)
 	}
 
 	runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
@@ -1303,6 +1303,7 @@ func (kl *Kubelet) initializeModules() error {
 		collectors.NewVolumeStatsCollector(kl),
 		collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
 	)
+	metrics.SetNodeName(kl.nodeName)
 
 	// Setup filesystem directories.
 	if err := kl.setupDataDirs(); err != nil {
@@ -1426,7 +1427,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 
 	// Start syncing RuntimeClasses if enabled.
 	if kl.runtimeClassManager != nil {
-		go kl.runtimeClassManager.Run(wait.NeverStop)
+		kl.runtimeClassManager.Start(wait.NeverStop)
 	}
 
 	// Start the pod lifecycle event generator.