Merge remote-tracking branch 'origin/master' into release-1.14

build/kazel_generated.bzl

@@ -63,6 +63,8 @@ tags_values_pkgs = {"openapi-gen": {
         "staging/src/k8s.io/api/imagepolicy/v1alpha1",
         "staging/src/k8s.io/api/networking/v1",
         "staging/src/k8s.io/api/networking/v1beta1",
+        "staging/src/k8s.io/api/node/v1alpha1",
+        "staging/src/k8s.io/api/node/v1beta1",
         "staging/src/k8s.io/api/policy/v1beta1",
         "staging/src/k8s.io/api/rbac/v1",
         "staging/src/k8s.io/api/rbac/v1alpha1",
@@ -138,6 +140,8 @@ tags_pkgs_values = {"openapi-gen": {
     "staging/src/k8s.io/api/imagepolicy/v1alpha1": ["true"],
     "staging/src/k8s.io/api/networking/v1": ["true"],
     "staging/src/k8s.io/api/networking/v1beta1": ["true"],
+    "staging/src/k8s.io/api/node/v1alpha1": ["true"],
+    "staging/src/k8s.io/api/node/v1beta1": ["true"],
     "staging/src/k8s.io/api/policy/v1beta1": ["true"],
     "staging/src/k8s.io/api/rbac/v1": ["true"],
     "staging/src/k8s.io/api/rbac/v1alpha1": ["true"],

build/root/Makefile

@@ -282,8 +282,12 @@ endif
 define TEST_CMD_HELP_INFO
 # Build and run cmdline tests.
 #
+# Args:
+#   WHAT: List of tests to run, check test/cmd/legacy-script.sh for names.
+#     For example, WHAT=deployment will run run_deployment_tests function.
 # Example:
 #   make test-cmd
+#   make test-cmd WHAT="deployment impersonation"
 endef
 .PHONY: test-cmd
 ifeq ($(PRINT_HELP),y)

build/root/Makefile.generated_files

@@ -409,8 +409,10 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi
 OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
 OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
 OPENAPI_OUTPUT_PKG := pkg/generated/openapi
+CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi
 BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt
 REPORT_FILENAME := $(OUT_DIR)/violations.report
+IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report
 KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list
 # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
 # updated API violations to the known API violation exceptions list.
@@ -436,10 +438,11 @@ OPENAPI_DIRS := $(shell                                             \
 )
 
 OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
+CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
 
 # This rule is the user-friendly entrypoint for openapi generation.
 .PHONY: gen_openapi
-gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
+gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE)
 
 # For each dir in OPENAPI_DIRS, this establishes a dependency between the
 # output file and the input files that should trigger a rebuild.
@@ -469,6 +472,17 @@ $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME)
 	diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) ||                          \
 	(echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1)
 
+# TODO(roycaihw): move the automation to apiextensions-apiserver
+$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN)
+	./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
+	    --v $(KUBE_VERBOSE)                                                         \
+	    --logtostderr                                                               \
+	    -i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1"         \
+	    -p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG)                                \
+	    -O $(OPENAPI_BASENAME)                                                      \
+	    -h $(BOILERPLATE_FILENAME)                                                  \
+	    -r $(IGNORED_REPORT_FILENAME)                                               \
+	    "$$@"
 
 # How to build the generator tool.  The deps for this are defined in
 # the $(GO_PKGDEPS_FILE), above.

cluster/gce/gci/configure-helper.sh

@@ -773,6 +773,7 @@ function create-master-audit-policy {
       - group: "extensions"
       - group: "metrics.k8s.io"
       - group: "networking.k8s.io"
+      - group: "node.k8s.io"
       - group: "policy"
       - group: "rbac.authorization.k8s.io"
       - group: "scheduling.k8s.io"
@@ -1747,6 +1748,9 @@ function start-kube-apiserver {
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=" --feature-gates=${FEATURE_GATES}"
   fi
+  if [[ "${FEATURE_GATES:-}" =~ "RuntimeClass=true" ]]; then
+    params+=" --runtime-config=node.k8s.io/v1alpha1=true"
+  fi
   if [[ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]]; then
     params+=" --advertise-address=${MASTER_ADVERTISE_ADDRESS}"
     if [[ -n "${PROXY_SSH_USER:-}" ]]; then
@@ -2690,9 +2694,6 @@ EOF
       setup-addon-manifests "addons" "istio/noauth"
     fi
   fi
-  if [[ "${FEATURE_GATES:-}" =~ "RuntimeClass=true" ]]; then
-    setup-addon-manifests "addons" "runtimeclass"
-  fi
   if [[ -n "${EXTRA_ADDONS_URL:-}" ]]; then
     download-extra-addons
     setup-addon-manifests "addons" "gce-extras"

cluster/gce/util.sh

@@ -600,6 +600,7 @@ function write-linux-node-env {
 
 function write-windows-node-env {
   construct-windows-kubelet-flags
+  construct-windows-kubeproxy-flags
   build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml"
   build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml"
 }
@@ -871,6 +872,37 @@ function construct-windows-kubelet-flags {
   KUBELET_ARGS="${flags}"
 }
 
+function construct-windows-kubeproxy-flags {
+  local flags=""
+
+  # Use the same log level as the Kubelet during tests.
+  flags+=" ${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
+
+  # Windows uses kernelspace proxymode
+  flags+=" --proxy-mode=kernelspace"
+
+  # Configure kube-proxy to run as a windows service.
+  flags+=" --windows-service=true"
+
+  # TODO(mtaufen): Configure logging for kube-proxy running as a service.
+  # I haven't been able to figure out how to direct stdout/stderr into log
+  # files when configuring it to run via sc.exe, so we just manually
+  # override logging config here.
+  flags+=" --log-file=${WINDOWS_LOGS_DIR}\kube-proxy.log"
+
+  # klog sets this to true internally, so need to override to false
+  # so we actually log to the file
+  flags+=" --logtostderr=false"
+
+  # Configure flags with explicit empty string values. We can't escape
+  # double-quotes, because they still break sc.exe after expansion in the
+  # binPath parameter, and single-quotes get parsed as characters instead
+  # of string delimiters.
+  flags+=" --resource-container="
+
+  KUBEPROXY_ARGS="${flags}"
+}
+
 # $1: if 'true', we're rendering config for a master, else a node
 function build-kubelet-config {
   local master="$1"
@@ -1438,6 +1470,7 @@ CNI_CONFIG_DIR: $(yaml-quote ${WINDOWS_CNI_CONFIG_DIR})
 MANIFESTS_DIR: $(yaml-quote ${WINDOWS_MANIFESTS_DIR})
 PKI_DIR: $(yaml-quote ${WINDOWS_PKI_DIR})
 KUBELET_CONFIG_FILE: $(yaml-quote ${WINDOWS_KUBELET_CONFIG_FILE})
+KUBEPROXY_ARGS: $(yaml-quote ${KUBEPROXY_ARGS})
 KUBECONFIG_FILE: $(yaml-quote ${WINDOWS_KUBECONFIG_FILE})
 BOOTSTRAP_KUBECONFIG_FILE: $(yaml-quote ${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE})
 KUBEPROXY_KUBECONFIG_FILE: $(yaml-quote ${WINDOWS_KUBEPROXY_KUBECONFIG_FILE})

cluster/gce/windows/configure.ps1

@@ -100,6 +100,7 @@ try {
   Dump-DebugInfoToConsole
   Set-PrerequisiteOptions
   $kube_env = Fetch-KubeEnv
+  Disable-WindowsDefender
 
   if (Test-IsTestCluster $kube_env) {
     Log-Output 'Test cluster detected, installing OpenSSH.'
@@ -128,6 +129,9 @@ try {
   Log-Output 'Waiting 15 seconds for node to join cluster.'
   Start-Sleep 15
   Verify-WorkerServices
+
+  $config = New-FileRotationConfig
+  Schedule-LogRotation -Pattern '.*\.log$' -Path ${env:LOGS_DIR} -RepetitionInterval $(New-Timespan -Hour 1) -Config $config
 }
 catch {
   Write-Host 'Exception caught in script:'

cluster/gce/windows/k8s-node-setup.psm1

@@ -564,6 +564,7 @@ users:
 clusters:
 - name: local
   cluster:
+    server: https://APISERVER_ADDRESS
     certificate-authority-data: CA_CERT
 contexts:
 - context:
@@ -572,7 +573,8 @@ contexts:
   name: service-account-context
 current-context: service-account-context'.`
     replace('KUBEPROXY_TOKEN', ${kube_env}['KUBE_PROXY_TOKEN']).`
-    replace('CA_CERT', ${kube_env}['CA_CERT'])
+    replace('CA_CERT', ${kube_env}['CA_CERT']).`
+    replace('APISERVER_ADDRESS', ${kube_env}['KUBERNETES_MASTER_NAME'])
 
   Log-Output ("kubeproxy kubeconfig:`n" +
               "$(Get-Content -Raw ${env:KUBEPROXY_KUBECONFIG})")
@@ -934,7 +936,7 @@ function Configure-Kubelet {
 #
 # Required ${kube_env} keys:
 #   KUBELET_ARGS
-#   KUBERNETES_MASTER_NAME
+#   KUBEPROXY_ARGS
 #   CLUSTER_IP_RANGE
 function Start-WorkerServices {
   # Compute kubelet args
@@ -949,11 +951,7 @@ function Start-WorkerServices {
 
   # Compute kube-proxy args
   $kubeproxy_args_str = ${kube_env}['KUBEPROXY_ARGS']
-  Try {
-    $kubeproxy_args = $kubeproxy_args_str.Split(" ")
-  } Catch {
-    $kubeproxy_args = ""
-  }
+  $kubeproxy_args = $kubeproxy_args_str.Split(" ")
   Log-Output "kubeproxy_args from metadata: ${kubeproxy_args}"
 
   # kubeproxy is started on Linux nodes using
@@ -966,31 +964,9 @@ function Start-WorkerServices {
   #   --iptables-sync-period=1m --iptables-min-sync-period=10s
   #   --ipvs-sync-period=1m --ipvs-min-sync-period=10s
   # And also with various volumeMounts and "securityContext: privileged: true".
-  $apiserver_address = ${kube_env}['KUBERNETES_MASTER_NAME']
   $default_kubeproxy_args = @(`
-      "--v=4",
-      "--master=https://${apiserver_address}",
       "--kubeconfig=${env:KUBEPROXY_KUBECONFIG}",
-      "--proxy-mode=kernelspace",
-      "--cluster-cidr=$(${kube_env}['CLUSTER_IP_RANGE'])",
-
-      # Configure kube-proxy to run as a windows service.
-      "--windows-service=true",
-
-      # TODO(mtaufen): Configure logging for kube-proxy running as a service.
-      # I haven't been able to figure out how to direct stdout/stderr into log
-      # files when configuring it to run via sc.exe, so we just manually
-      # override logging config here.
-      "--log-file=${env:LOGS_DIR}\kube-proxy.log",
-      # klog sets this to true intenrally, so need to override to false
-      # so we actually log to the file
-      "--logtostderr=false",
-
-      # Configure flags with explicit empty string values. We can't escape
-      # double-quotes, because they still break sc.exe after expansion in the
-      # binPath parameter, and single-quotes get parsed as characters instead
-      # of string delimiters.
-      "--resource-container="
+      "--cluster-cidr=$(${kube_env}['CLUSTER_IP_RANGE'])"
   )
   $kubeproxy_args = ${default_kubeproxy_args} + ${kubeproxy_args}
   Log-Output "Final kubeproxy_args: ${kubeproxy_args}"

cluster/images/conformance/conformance-e2e.sh

@@ -25,6 +25,9 @@ while true; do
   if [[ "$STATUS" == "Succeeded" ]]; then
     echo "$timestamp Done."
     break
+  elif [[ "$STATUS" == "Failed" ]]; then
+    echo "$timestamp Failed."
+    break
   else
     sleep 5
   fi

cmd/kube-apiserver/app/aggregator.go

@@ -276,6 +276,8 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
 	{Group: "coordination.k8s.io", Version: "v1"}:               {group: 16500, version: 15},
 	{Group: "coordination.k8s.io", Version: "v1beta1"}:          {group: 16500, version: 9},
 	{Group: "auditregistration.k8s.io", Version: "v1alpha1"}:    {group: 16400, version: 1},
+	{Group: "node.k8s.io", Version: "v1alpha1"}:                 {group: 16300, version: 1},
+	{Group: "node.k8s.io", Version: "v1beta1"}:                  {group: 16300, version: 9},
 	// Append a new group to the end of the list if unsure.
 	// You can use min(existing group)-100 as the initial value for a group.
 	// Version can be set to 9 (to have space around) for a new group.

cmd/kube-controller-manager/app/core.go

@@ -223,6 +223,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
 			ctx.InformerFactory.Core().V1().Nodes(),
 			ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
 			ctx.InformerFactory.Core().V1().PersistentVolumes(),
+			ctx.InformerFactory.Storage().V1beta1().CSINodes(),
 			ctx.Cloud,
 			ProbeAttachableVolumePlugins(),
 			GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),

cmd/kubeadm/app/cmd/alpha/BUILD

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "alpha.go",
         "certs.go",
+        "kubeconfig.go",
         "kubelet.go",
         "selfhosting.go",
     ],
@@ -21,6 +22,7 @@ go_library(
         "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/phases/certs/renewal:go_default_library",
+        "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
@@ -52,7 +54,10 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = ["certs_test.go"],
+    srcs = [
+        "certs_test.go",
+        "kubeconfig_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//cmd/kubeadm/app/constants:go_default_library",
@@ -61,6 +66,8 @@ go_test(
         "//cmd/kubeadm/app/util/pkiutil:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//cmd/kubeadm/test/cmd:go_default_library",
+        "//cmd/kubeadm/test/kubeconfig:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
     ],
 )

cmd/kubeadm/app/cmd/alpha/alpha.go

@@ -32,6 +32,7 @@ func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command {
 
 	cmd.AddCommand(newCmdCertsUtility())
 	cmd.AddCommand(newCmdKubeletUtility())
+	cmd.AddCommand(newCmdKubeConfigUtility(out))
 	cmd.AddCommand(NewCmdSelfhosting(in))
 
 	// TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed.

cmd/kubeadm/app/cmd/alpha/kubeconfig.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package alpha
+
+import (
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
+	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	kubeconfigLongDesc = normalizer.LongDesc(`
+	Kubeconfig file utilities.
+	` + cmdutil.AlphaDisclaimer)
+
+	userKubeconfigLongDesc = normalizer.LongDesc(`
+	Outputs a kubeconfig file for an additional user.
+	` + cmdutil.AlphaDisclaimer)
+
+	userKubeconfigExample = normalizer.Examples(`
+	# Outputs a kubeconfig file for an additional user named foo
+	kubeadm alpha kubeconfig user --client-name=foo
+	`)
+)
+
+// newCmdKubeConfigUtility returns main command for kubeconfig phase
+func newCmdKubeConfigUtility(out io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "kubeconfig",
+		Short: "Kubeconfig file utilities",
+		Long:  kubeconfigLongDesc,
+	}
+
+	cmd.AddCommand(newCmdUserKubeConfig(out))
+	return cmd
+}
+
+// newCmdUserKubeConfig returns sub commands for kubeconfig phase
+func newCmdUserKubeConfig(out io.Writer) *cobra.Command {
+
+	cfg := &kubeadmapiv1beta1.InitConfiguration{}
+
+	// Default values for the cobra help text
+	kubeadmscheme.Scheme.Default(cfg)
+
+	var token, clientName string
+	var organizations []string
+
+	// Creates the UX Command
+	cmd := &cobra.Command{
+		Use:     "user",
+		Short:   "Outputs a kubeconfig file for an additional user",
+		Long:    userKubeconfigLongDesc,
+		Example: userKubeconfigExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			if clientName == "" {
+				kubeadmutil.CheckErr(errors.New("missing required argument --client-name"))
+			}
+
+			// This call returns the ready-to-use configuration based on the default cfg populated by flags
+			internalcfg, err := configutil.DefaultedInitConfiguration(cfg)
+			kubeadmutil.CheckErr(err)
+
+			// if the kubeconfig file for an additional user has to use a token, use it
+			if token != "" {
+				kubeadmutil.CheckErr(kubeconfigphase.WriteKubeConfigWithToken(out, internalcfg, clientName, token))
+				return
+			}
+
+			// Otherwise, write a kubeconfig file with a generate client cert
+			kubeadmutil.CheckErr(kubeconfigphase.WriteKubeConfigWithClientCert(out, internalcfg, clientName, organizations))
+		},
+	}
+
+	// Add flags to the command
+	cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, "The path where certificates are stored")
+	cmd.Flags().StringVar(&cfg.LocalAPIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.LocalAPIEndpoint.AdvertiseAddress, "The IP address the API server is accessible on")
+	cmd.Flags().Int32Var(&cfg.LocalAPIEndpoint.BindPort, "apiserver-bind-port", cfg.LocalAPIEndpoint.BindPort, "The port the API server is accessible on")
+	cmd.Flags().StringVar(&token, "token", token, "The token that should be used as the authentication mechanism for this kubeconfig, instead of client certificates")
+	cmd.Flags().StringVar(&clientName, "client-name", clientName, "The name of user. It will be used as the CN if client certificates are created")
+	cmd.Flags().StringSliceVar(&organizations, "org", organizations, "The orgnizations of the client certificate. It will be used as the O if client certificates are created")
+
+	return cmd
+}

cmd/kubeadm/app/cmd/alpha/kubeconfig_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package alpha
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+
+	"k8s.io/client-go/tools/clientcmd"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
+	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
+	kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig"
+)
+
+func TestKubeConfigSubCommandsThatWritesToOut(t *testing.T) {
+
+	// Temporary folders for the test case
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+	// Adds a pki folder with a ca cert to the temp folder
+	pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
+
+	// Retrieves ca cert for assertions
+	caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName)
+	if err != nil {
+		t.Fatalf("couldn't retrieve ca cert: %v", err)
+	}
+
+	commonFlags := []string{
+		"--apiserver-advertise-address=1.2.3.4",
+		"--apiserver-bind-port=1234",
+		"--client-name=myUser",
+		fmt.Sprintf("--cert-dir=%s", pkidir),
+	}
+
+	var tests = []struct {
+		name            string
+		command         string
+		withClientCert  bool
+		withToken       bool
+		additionalFlags []string
+	}{
+		{
+			name:           "user subCommand withClientCert",
+			command:        "user",
+			withClientCert: true,
+		},
+		{
+			name:            "user subCommand withToken",
+			withToken:       true,
+			command:         "user",
+			additionalFlags: []string{"--token=123456"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			buf := new(bytes.Buffer)
+
+			// Get subcommands working in the temporary directory
+			cmd := newCmdUserKubeConfig(buf)
+
+			// Execute the subcommand
+			allFlags := append(commonFlags, test.additionalFlags...)
+			cmd.SetArgs(allFlags)
+			if err := cmd.Execute(); err != nil {
+				t.Fatal("Could not execute subcommand")
+			}
+
+			// reads kubeconfig written to stdout
+			config, err := clientcmd.Load(buf.Bytes())
+			if err != nil {
+				t.Errorf("couldn't read kubeconfig file from buffer: %v", err)
+				return
+			}
+
+			// checks that CLI flags are properly propagated
+			kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert)
+
+			if test.withClientCert {
+				// checks that kubeconfig files have expected client cert
+				kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser")
+			}
+
+			if test.withToken {
+				// checks that kubeconfig files have expected token
+				kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "123456")
+			}
+		})
+	}
+}

cmd/kubeadm/app/cmd/reset.go

@@ -38,6 +38,7 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
+	uploadconfig "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
@@ -140,10 +141,15 @@ func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool,
 // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join".
 func (r *Reset) Run(out io.Writer, client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
 	var dirsToClean []string
-	// Only clear etcd data when using local etcd.
-	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
 
+	// Reset the ClusterStatus for a given control-plane node.
+	if isControlPlane() && cfg != nil {
+		uploadconfig.ResetClusterStatusForNode(cfg.NodeRegistration.Name, client)
+	}
+
+	// Only clear etcd data when using local etcd.
 	klog.V(1).Infoln("[reset] Checking for etcd config")
+	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
 	etcdDataDir, err := getEtcdDataDir(etcdManifestPath, cfg)
 	if err == nil {
 		dirsToClean = append(dirsToClean, etcdDataDir)
@@ -318,3 +324,13 @@ func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
 	// if this fails, try to detect it
 	return utilruntime.DetectCRISocket()
 }
+
+// isControlPlane checks if a node is a control-plane node by looking up
+// the kube-apiserver manifest file
+func isControlPlane() bool {
+	filepath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, kubeadmconstants.GetStaticPodDirectory())
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}

cmd/kubeadm/app/phases/uploadconfig/BUILD

@@ -20,6 +20,8 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -19,10 +19,12 @@ package uploadconfig
 import (
 	"fmt"
 
-	"k8s.io/api/core/v1"
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -37,6 +39,69 @@ const (
 	NodesKubeadmConfigClusterRoleName = "kubeadm:nodes-kubeadm-config"
 )
 
+// ResetClusterStatusForNode removes the APIEndpoint of a given control-plane node
+// from the ClusterStatus and updates the kubeadm ConfigMap
+func ResetClusterStatusForNode(nodeName string, client clientset.Interface) error {
+	fmt.Printf("[reset] Removing info for node %q from the ConfigMap %q in the %q Namespace\n",
+		nodeName, kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+
+	// Get the kubeadm ConfigMap
+	configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeadmConfigConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get config map")
+	}
+
+	// Handle missing ClusterConfiguration in the ConfigMap. Should only happen if someone manually
+	// interacted with the ConfigMap.
+	clusterConfigurationYaml, ok := configMap.Data[kubeadmconstants.ClusterConfigurationConfigMapKey]
+	if !ok {
+		return errors.Errorf("cannot find key %q in ConfigMap %q in the %q Namespace",
+			kubeadmconstants.ClusterConfigurationConfigMapKey, kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+	}
+
+	// Obtain the existing ClusterStatus object
+	clusterStatus, err := configutil.UnmarshalClusterStatus(configMap.Data)
+	if err != nil {
+		return err
+	}
+
+	// Handle a nil APIEndpoints map. Should only happen if someone manually
+	// interacted with the ConfigMap.
+	if clusterStatus.APIEndpoints == nil {
+		return errors.Errorf("APIEndpoints from ConfigMap %q in the %q Namespace is nil",
+			kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+	}
+
+	// Check for existence of the nodeName key in the list of APIEndpoints.
+	// Return early if it's missing.
+	apiEndpoint, ok := clusterStatus.APIEndpoints[nodeName]
+	if !ok {
+		klog.Warningf("No APIEndpoint registered for node %q", nodeName)
+		return nil
+	}
+
+	klog.V(2).Infof("Removing APIEndpoint %#v for node %q", apiEndpoint, nodeName)
+	delete(clusterStatus.APIEndpoints, nodeName)
+
+	// Marshal the ClusterStatus back into YAML
+	clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus)
+	if err != nil {
+		return err
+	}
+
+	// Update the ClusterStatus in the ConfigMap
+	return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kubeadmconstants.KubeadmConfigConfigMap,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Data: map[string]string{
+			kubeadmconstants.ClusterConfigurationConfigMapKey: clusterConfigurationYaml,
+			kubeadmconstants.ClusterStatusConfigMapKey:        string(clusterStatusYaml),
+		},
+	})
+}
+
 // UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
 func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
 	fmt.Printf("[upload-config] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)

cmd/kubeadm/app/util/config/cluster.go

@@ -167,7 +167,7 @@ func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) {
 // getAPIEndpoint returns the APIEndpoint for the current node
 func getAPIEndpoint(data map[string]string, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error {
 	// gets the ClusterStatus from kubeadm-config
-	clusterStatus, err := unmarshalClusterStatus(data)
+	clusterStatus, err := UnmarshalClusterStatus(data)
 	if err != nil {
 		return err
 	}
@@ -210,7 +210,7 @@ func GetClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, er
 		return nil, err
 	}
 
-	clusterStatus, err := unmarshalClusterStatus(configMap.Data)
+	clusterStatus, err := UnmarshalClusterStatus(configMap.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -218,7 +218,8 @@ func GetClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, er
 	return clusterStatus, nil
 }
 
-func unmarshalClusterStatus(data map[string]string) (*kubeadmapi.ClusterStatus, error) {
+// UnmarshalClusterStatus takes raw ConfigMap.Data and converts it to a ClusterStatus object
+func UnmarshalClusterStatus(data map[string]string) (*kubeadmapi.ClusterStatus, error) {
 	clusterStatusData, ok := data[constants.ClusterStatusConfigMapKey]
 	if !ok {
 		return nil, errors.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterStatusConfigMapKey)

cmd/kubelet/app/BUILD

@@ -137,7 +137,6 @@ go_library(
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
         "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
-        "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
         "//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/kubelet/app/server.go

@@ -95,7 +95,6 @@ import (
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/version/verflag"
 	"k8s.io/kubernetes/pkg/volume/util/subpath"
-	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	"k8s.io/utils/exec"
 	"k8s.io/utils/nsenter"
 )
@@ -592,15 +591,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 		if err != nil {
 			return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err)
 		}
-
-		// CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803
-		crdClientConfig := restclient.CopyConfig(clientConfig)
-		crdClientConfig.ContentType = "application/json"
-
-		kubeDeps.NodeAPIClient, err = nodeapiclientset.NewForConfig(crdClientConfig)
-		if err != nil {
-			return fmt.Errorf("failed to initialize kubelet node-api client: %v", err)
-		}
 	}
 
 	// If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops

hack/.golint_failures

@@ -50,6 +50,7 @@ pkg/apis/extensions/v1beta1
 pkg/apis/imagepolicy
 pkg/apis/networking
 pkg/apis/networking/v1
+pkg/apis/node/v1alpha1
 pkg/apis/policy
 pkg/apis/policy/v1beta1
 pkg/apis/policy/validation
@@ -279,6 +280,7 @@ pkg/registry/extensions/controller/storage
 pkg/registry/extensions/rest
 pkg/registry/networking/networkpolicy/storage
 pkg/registry/networking/rest
+pkg/registry/node/rest
 pkg/registry/policy/poddisruptionbudget/storage
 pkg/registry/policy/rest
 pkg/registry/rbac/clusterrole

hack/lib/init.sh

@@ -80,6 +80,8 @@ events.k8s.io/v1beta1 \
 imagepolicy.k8s.io/v1alpha1 \
 networking.k8s.io/v1 \
 networking.k8s.io/v1beta1 \
+node.k8s.io/v1alpha1 \
+node.k8s.io/v1beta1 \
 policy/v1beta1 \
 rbac.authorization.k8s.io/v1 \
 rbac.authorization.k8s.io/v1beta1 \

hack/update-bazel.sh

@@ -31,7 +31,7 @@ go install k8s.io/kubernetes/vendor/k8s.io/repo-infra/kazel
 
 # Find all of the staging repos.
 while IFS='' read -r repo; do staging_repos+=("${repo}"); done <\
-  <(cd "${KUBE_ROOT}/staging/src" && find k8s.io/ -mindepth 1 -maxdepth 1 -type d | LANG=C sort)
+  <(cd "${KUBE_ROOT}/staging/src" && find k8s.io -mindepth 1 -maxdepth 1 -type d | LANG=C sort)
 
 # Save the staging repos into a Starlark list that can be used by Bazel rules.
 (

hack/update-generated-protobuf-dockerized.sh

@@ -41,57 +41,8 @@ fi
 
 gotoprotobuf=$(kube::util::find-binary "go-to-protobuf")
 
-PACKAGES=(
-  k8s.io/apiserver/pkg/apis/example/v1
-  k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1
-  k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1
-  k8s.io/kube-aggregator/pkg/apis/apiregistration/v1
-  k8s.io/api/core/v1
-  k8s.io/api/policy/v1beta1
-  k8s.io/api/extensions/v1beta1
-  k8s.io/api/autoscaling/v1
-  k8s.io/api/authorization/v1
-  k8s.io/api/autoscaling/v2beta1
-  k8s.io/api/autoscaling/v2beta2
-  k8s.io/api/authorization/v1beta1
-  k8s.io/api/batch/v1
-  k8s.io/api/batch/v1beta1
-  k8s.io/api/batch/v2alpha1
-  k8s.io/api/apps/v1beta1
-  k8s.io/api/apps/v1beta2
-  k8s.io/api/apps/v1
-  k8s.io/api/authentication/v1
-  k8s.io/api/authentication/v1beta1
-  k8s.io/api/events/v1beta1
-  k8s.io/api/rbac/v1alpha1
-  k8s.io/api/rbac/v1beta1
-  k8s.io/api/rbac/v1
-  k8s.io/api/certificates/v1beta1
-  k8s.io/api/coordination/v1beta1
-  k8s.io/api/coordination/v1
-  k8s.io/api/imagepolicy/v1alpha1
-  k8s.io/api/scheduling/v1alpha1
-  k8s.io/api/scheduling/v1beta1
-  k8s.io/api/scheduling/v1
-  k8s.io/api/settings/v1alpha1
-  k8s.io/api/storage/v1alpha1
-  k8s.io/api/storage/v1beta1
-  k8s.io/api/storage/v1
-  k8s.io/api/admissionregistration/v1beta1
-  k8s.io/api/admission/v1beta1
-  k8s.io/api/auditregistration/v1alpha1
-  k8s.io/api/networking/v1beta1
-  k8s.io/api/networking/v1
-  k8s.io/metrics/pkg/apis/metrics/v1alpha1
-  k8s.io/metrics/pkg/apis/metrics/v1beta1
-  k8s.io/metrics/pkg/apis/custom_metrics/v1beta1
-  k8s.io/metrics/pkg/apis/custom_metrics/v1beta2
-  k8s.io/metrics/pkg/apis/external_metrics/v1beta1
-  k8s.io/apiserver/pkg/apis/audit/v1alpha1
-  k8s.io/apiserver/pkg/apis/audit/v1beta1
-  k8s.io/apiserver/pkg/apis/audit/v1
-  k8s.io/apiserver/pkg/apis/example2/v1
-)
+APIROOTS=( ${1} )
+shift
 
 # requires the 'proto' tag to build (will remove when ready)
 # searches for the protoc-gen-gogo extension in the output directory
@@ -101,6 +52,6 @@ PATH="${KUBE_ROOT}/_output/bin:${PATH}" \
   "${gotoprotobuf}" \
   --proto-import="${KUBE_ROOT}/vendor" \
   --proto-import="${KUBE_ROOT}/third_party/protobuf" \
-  --packages=$(IFS=, ; echo "${PACKAGES[*]}") \
+  --packages=$(IFS=, ; echo "${APIROOTS[*]}") \
   --go-header-file ${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt \
   "$@"

hack/update-generated-protobuf.sh

@@ -24,6 +24,12 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 # source tree.  This is managed in kube::build::copy_output in build/common.sh.
 # If the output set is changed update that function.
 
-"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "$@"
+APIROOTS=${APIROOTS:-$(git grep --files-with-matches -e '// +k8s:protobuf-gen=package' cmd pkg staging | \
+	xargs -n 1 dirname | \
+	sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' | \
+	sort | uniq
+)}
+
+"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@"
 
 # ex: ts=2 sw=2 et filetype=sh

hack/verify-generated-protobuf.sh

@@ -23,7 +23,8 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
-APIROOTS=${APIROOTS:-pkg/api pkg/apis pkg/watch staging/src/k8s.io/apimachinery/pkg/api staging/src/k8s.io/apimachinery/pkg/apis staging/src/k8s.io/apiserver/pkg staging/src/k8s.io/api staging/src/k8s.io/metrics/pkg/apis}
+APIROOTS=$(git grep --files-with-matches -e '// +k8s:protobuf-gen=package' cmd pkg staging | xargs -n 1 dirname | sort | uniq)
+
 _tmp="${KUBE_ROOT}/_tmp"
 
 cleanup() {

pkg/BUILD

@@ -44,6 +44,7 @@ filegroup(
         "//pkg/apis/extensions:all-srcs",
         "//pkg/apis/imagepolicy:all-srcs",
         "//pkg/apis/networking:all-srcs",
+        "//pkg/apis/node:all-srcs",
         "//pkg/apis/policy:all-srcs",
         "//pkg/apis/rbac:all-srcs",
         "//pkg/apis/scheduling:all-srcs",

pkg/api/pod/util.go

@@ -318,10 +318,6 @@ func dropDisabledFields(
 		podSpec.PriorityClassName = ""
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.PodReadinessGates) && !podReadinessGatesInUse(oldPodSpec) {
-		podSpec.ReadinessGates = nil
-	}
-
 	if !utilfeature.DefaultFeatureGate.Enabled(features.Sysctls) && !sysctlsInUse(oldPodSpec) {
 		if podSpec.SecurityContext != nil {
 			podSpec.SecurityContext.Sysctls = nil
@@ -534,17 +530,6 @@ func podPriorityInUse(podSpec *api.PodSpec) bool {
 	return false
 }
 
-// podReadinessGatesInUse returns true if the pod spec is non-nil and has ReadinessGates
-func podReadinessGatesInUse(podSpec *api.PodSpec) bool {
-	if podSpec == nil {
-		return false
-	}
-	if podSpec.ReadinessGates != nil {
-		return true
-	}
-	return false
-}
-
 func sysctlsInUse(podSpec *api.PodSpec) bool {
 	if podSpec == nil {
 		return false

pkg/api/pod/util_test.go

@@ -1101,102 +1101,6 @@ func TestDropAppArmor(t *testing.T) {
 	}
 }
 
-func TestDropReadinessGates(t *testing.T) {
-	podWithoutReadinessGates := func() *api.Pod {
-		return &api.Pod{
-			Spec: api.PodSpec{
-				ReadinessGates: nil,
-			},
-		}
-	}
-	podWithReadinessGates := func() *api.Pod {
-		return &api.Pod{
-			Spec: api.PodSpec{
-				ReadinessGates: []api.PodReadinessGate{
-					{
-						ConditionType: api.PodConditionType("example.com/condition1"),
-					},
-					{
-						ConditionType: api.PodConditionType("example.com/condition2"),
-					},
-				},
-			},
-		}
-	}
-
-	podInfo := []struct {
-		description          string
-		hasPodReadinessGates bool
-		pod                  func() *api.Pod
-	}{
-		{
-			description:          "has ReadinessGates",
-			hasPodReadinessGates: true,
-			pod:                  podWithReadinessGates,
-		},
-		{
-			description:          "does not have ReadinessGates",
-			hasPodReadinessGates: false,
-			pod:                  podWithoutReadinessGates,
-		},
-		{
-			description:          "is nil",
-			hasPodReadinessGates: false,
-			pod:                  func() *api.Pod { return nil },
-		},
-	}
-
-	for _, enabled := range []bool{true, false} {
-		for _, oldPodInfo := range podInfo {
-			for _, newPodInfo := range podInfo {
-				oldPodHasReadinessGates, oldPod := oldPodInfo.hasPodReadinessGates, oldPodInfo.pod()
-				newPodHasReadinessGates, newPod := newPodInfo.hasPodReadinessGates, newPodInfo.pod()
-				if newPod == nil {
-					continue
-				}
-
-				t.Run(fmt.Sprintf("featue enabled=%v, old pod %v, new pod %v", enabled, oldPodInfo.description, newPodInfo.description), func(t *testing.T) {
-					defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodReadinessGates, enabled)()
-
-					var oldPodSpec *api.PodSpec
-					if oldPod != nil {
-						oldPodSpec = &oldPod.Spec
-					}
-					dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil)
-
-					// old pod should never be changed
-					if !reflect.DeepEqual(oldPod, oldPodInfo.pod()) {
-						t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodInfo.pod()))
-					}
-
-					switch {
-					case enabled || oldPodHasReadinessGates:
-						// new pod should not be changed if the feature is enabled, or if the old pod had ReadinessGates
-						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
-							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
-						}
-					case newPodHasReadinessGates:
-						// new pod should be changed
-						if reflect.DeepEqual(newPod, newPodInfo.pod()) {
-							t.Errorf("new pod was not changed")
-						}
-						// new pod should not have ReadinessGates
-						if !reflect.DeepEqual(newPod, podWithoutReadinessGates()) {
-							t.Errorf("new pod had ReadinessGates: %v",
-								diff.ObjectReflectDiff(newPod, podWithoutReadinessGates()))
-						}
-					default:
-						// new pod should not need to be changed
-						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
-							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
-						}
-					}
-				})
-			}
-		}
-	}
-}
-
 func TestDropTokenRequestProjection(t *testing.T) {
 	podWithoutTRProjection := func() *api.Pod {
 		return &api.Pod{

pkg/api/testapi/BUILD

@@ -41,6 +41,8 @@ go_library(
         "//pkg/apis/imagepolicy/install:go_default_library",
         "//pkg/apis/networking:go_default_library",
         "//pkg/apis/networking/install:go_default_library",
+        "//pkg/apis/node:go_default_library",
+        "//pkg/apis/node/install:go_default_library",
         "//pkg/apis/policy:go_default_library",
         "//pkg/apis/policy/install:go_default_library",
         "//pkg/apis/rbac:go_default_library",

pkg/api/testapi/testapi.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/apis/imagepolicy"
 	"k8s.io/kubernetes/pkg/apis/networking"
+	"k8s.io/kubernetes/pkg/apis/node"
 	"k8s.io/kubernetes/pkg/apis/policy"
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	"k8s.io/kubernetes/pkg/apis/scheduling"
@@ -70,6 +71,7 @@ import (
 	_ "k8s.io/kubernetes/pkg/apis/extensions/install"
 	_ "k8s.io/kubernetes/pkg/apis/imagepolicy/install"
 	_ "k8s.io/kubernetes/pkg/apis/networking/install"
+	_ "k8s.io/kubernetes/pkg/apis/node/install"
 	_ "k8s.io/kubernetes/pkg/apis/policy/install"
 	_ "k8s.io/kubernetes/pkg/apis/rbac/install"
 	_ "k8s.io/kubernetes/pkg/apis/scheduling/install"
@@ -257,6 +259,12 @@ func init() {
 			externalGroupVersion: externalGroupVersion,
 		}
 	}
+	if _, ok := Groups[node.GroupName]; !ok {
+		externalGroupVersion := schema.GroupVersion{Group: node.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(node.GroupName)[0].Version}
+		Groups[node.GroupName] = TestGroup{
+			externalGroupVersion: externalGroupVersion,
+		}
+	}
 	if _, ok := Groups[events.GroupName]; !ok {
 		externalGroupVersion := schema.GroupVersion{Group: events.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(events.GroupName)[0].Version}
 		Groups[events.GroupName] = TestGroup{

pkg/apis/core/annotation_key_constants.go

@@ -101,4 +101,10 @@ const (
 	// This annotation will be used to compute the in-cluster network programming latency SLI, see
 	// https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_programming_latency.md
 	EndpointsLastChangeTriggerTime = "endpoints.kubernetes.io/last-change-trigger-time"
+
+	// MigratedPluginsAnnotationKey is the annotation key, set for CSINode objects, that is a comma-separated
+	// list of in-tree plugins that will be serviced by the CSI backend on the Node represented by CSINode.
+	// This annotation is used by the Attach Detach Controller to determine whether to use the in-tree or
+	// CSI Backend for a volume plugin on a specific node.
+	MigratedPluginsAnnotationKey = "storage.alpha.kubernetes.io/migrated-plugins"
 )

pkg/apis/node/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "register.go",
+        "types.go",
+        "zz_generated.deepcopy.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/apis/node",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/apis/node/install:all-srcs",
+        "//pkg/apis/node/v1alpha1:all-srcs",
+        "//pkg/apis/node/v1beta1:all-srcs",
+        "//pkg/apis/node/validation:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/apis/node/doc.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+// +groupName=node.k8s.io
+
+package node // import "k8s.io/kubernetes/pkg/apis/node"

pkg/apis/node/install/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["install.go"],
+    importpath = "k8s.io/kubernetes/pkg/apis/node/install",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/apis/node:go_default_library",
+        "//pkg/apis/node/v1alpha1:go_default_library",
+        "//pkg/apis/node/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/apis/node/install/install.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package install adds the node API group, making it available as
+// an option to all of the API encoding/decoding machinery.
+package install
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/apis/node"
+	"k8s.io/kubernetes/pkg/apis/node/v1alpha1"
+	"k8s.io/kubernetes/pkg/apis/node/v1beta1"
+)
+
+func init() {
+	Install(legacyscheme.Scheme)
+}
+
+// Install registers the API group and adds types to a scheme
+func Install(scheme *runtime.Scheme) {
+	utilruntime.Must(node.AddToScheme(scheme))
+	utilruntime.Must(v1alpha1.AddToScheme(scheme))
+	utilruntime.Must(v1beta1.AddToScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion))
+}

pkg/apis/node/register.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName is the group name use in this package
+const GroupName = "node.k8s.io"
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal}
+
+// Kind takes an unqualified kind and returns a Group qualified GroupKind
+func Kind(kind string) schema.GroupKind {
+	return SchemeGroupVersion.WithKind(kind).GroupKind()
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	// SchemeBuilder for node api registration.
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// AddToScheme for node api registration.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// Adds the list of known types to api.Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&RuntimeClass{},
+		&RuntimeClassList{},
+	)
+	return nil
+}

pkg/apis/node/types.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RuntimeClass defines a class of container runtime supported in the cluster.
+// The RuntimeClass is used to determine which container runtime is used to run
+// all containers in a pod. RuntimeClasses are (currently) manually defined by a
+// user or cluster provisioner, and referenced in the PodSpec. The Kubelet is
+// responsible for resolving the RuntimeClassName reference before running the
+// pod.  For more details, see
+// https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md
+type RuntimeClass struct {
+	metav1.TypeMeta
+	// +optional
+	metav1.ObjectMeta
+
+	// Handler specifies the underlying runtime and configuration that the CRI
+	// implementation will use to handle pods of this class. The possible values
+	// are specific to the node & CRI configuration.  It is assumed that all
+	// handlers are available on every node, and handlers of the same name are
+	// equivalent on every node.
+	// For example, a handler called "runc" might specify that the runc OCI
+	// runtime (using native Linux containers) will be used to run the containers
+	// in a pod.
+	// The Handler must conform to the DNS Label (RFC 1123) requirements, and is
+	// immutable.
+	Handler string
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RuntimeClassList is a list of RuntimeClass objects.
+type RuntimeClassList struct {
+	metav1.TypeMeta
+	// +optional
+	metav1.ListMeta
+
+	// Items is a list of schema objects.
+	Items []RuntimeClass
+}

pkg/apis/node/v1alpha1/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conversion.go",
+        "doc.go",
+        "register.go",
+        "zz_generated.conversion.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/apis/node/v1alpha1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/node:go_default_library",
+        "//staging/src/k8s.io/api/node/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["conversion_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/apis/node:go_default_library",
+        "//staging/src/k8s.io/api/node/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+    ],
+)

pkg/apis/node/v1alpha1/conversion.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1alpha1 "k8s.io/api/node/v1alpha1"
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	node "k8s.io/kubernetes/pkg/apis/node"
+)
+
+func addConversionFuncs(s *runtime.Scheme) error {
+	return s.AddConversionFuncs(
+		Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass,
+		Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass,
+	)
+}
+
+func Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass(in *v1alpha1.RuntimeClass, out *node.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	out.Handler = in.Spec.RuntimeHandler
+	return nil
+}
+
+func Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass(in *node.RuntimeClass, out *v1alpha1.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	out.Spec.RuntimeHandler = in.Handler
+	return nil
+}

pkg/apis/node/v1alpha1/conversion_test.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	v1alpha1 "k8s.io/api/node/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	node "k8s.io/kubernetes/pkg/apis/node"
+)
+
+func TestRuntimeClassConversion(t *testing.T) {
+	const (
+		name    = "puppy"
+		handler = "heidi"
+	)
+	internalRC := node.RuntimeClass{
+		ObjectMeta: metav1.ObjectMeta{Name: name},
+		Handler:    handler,
+	}
+	v1alpha1RC := v1alpha1.RuntimeClass{
+		ObjectMeta: metav1.ObjectMeta{Name: name},
+		Spec: v1alpha1.RuntimeClassSpec{
+			RuntimeHandler: handler,
+		},
+	}
+
+	convertedInternal := node.RuntimeClass{}
+	require.NoError(t,
+		Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass(&v1alpha1RC, &convertedInternal, nil))
+	assert.Equal(t, internalRC, convertedInternal)
+
+	convertedV1alpha1 := v1alpha1.RuntimeClass{}
+	require.NoError(t,
+		Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass(&internalRC, &convertedV1alpha1, nil))
+	assert.Equal(t, v1alpha1RC, convertedV1alpha1)
+}

pkg/apis/node/v1alpha1/doc.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:conversion-gen=k8s.io/kubernetes/pkg/apis/node
+// +k8s:conversion-gen-external-types=k8s.io/api/node/v1alpha1
+
+// +groupName=node.k8s.io
+
+package v1alpha1 // import "k8s.io/kubernetes/pkg/apis/node/v1alpha1"

pkg/apis/node/v1alpha1/register.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	nodev1alpha1 "k8s.io/api/node/v1alpha1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName for node API
+const GroupName = "node.k8s.io"
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	localSchemeBuilder = &nodev1alpha1.SchemeBuilder
+	// AddToScheme node API registration
+	AddToScheme = localSchemeBuilder.AddToScheme
+)
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	localSchemeBuilder.Register(addConversionFuncs)
+}

pkg/apis/node/v1alpha1/zz_generated.conversion.go

+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by conversion-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	v1alpha1 "k8s.io/api/node/v1alpha1"
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	node "k8s.io/kubernetes/pkg/apis/node"
+)
+
+func init() {
+	localSchemeBuilder.Register(RegisterConversions)
+}
+
+// RegisterConversions adds conversion functions to the given scheme.
+// Public to allow building arbitrary schemes.
+func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddGeneratedConversionFunc((*v1alpha1.RuntimeClass)(nil), (*node.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass(a.(*v1alpha1.RuntimeClass), b.(*node.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*node.RuntimeClass)(nil), (*v1alpha1.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass(a.(*node.RuntimeClass), b.(*v1alpha1.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*v1alpha1.RuntimeClassList)(nil), (*node.RuntimeClassList)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_RuntimeClassList_To_node_RuntimeClassList(a.(*v1alpha1.RuntimeClassList), b.(*node.RuntimeClassList), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*node.RuntimeClassList)(nil), (*v1alpha1.RuntimeClassList)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_node_RuntimeClassList_To_v1alpha1_RuntimeClassList(a.(*node.RuntimeClassList), b.(*v1alpha1.RuntimeClassList), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddConversionFunc((*node.RuntimeClass)(nil), (*v1alpha1.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass(a.(*node.RuntimeClass), b.(*v1alpha1.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddConversionFunc((*v1alpha1.RuntimeClass)(nil), (*node.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass(a.(*v1alpha1.RuntimeClass), b.(*node.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func autoConvert_v1alpha1_RuntimeClass_To_node_RuntimeClass(in *v1alpha1.RuntimeClass, out *node.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	// WARNING: in.Spec requires manual conversion: does not exist in peer-type
+	return nil
+}
+
+func autoConvert_node_RuntimeClass_To_v1alpha1_RuntimeClass(in *node.RuntimeClass, out *v1alpha1.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	// WARNING: in.Handler requires manual conversion: does not exist in peer-type
+	return nil
+}
+
+func autoConvert_v1alpha1_RuntimeClassList_To_node_RuntimeClassList(in *v1alpha1.RuntimeClassList, out *node.RuntimeClassList, s conversion.Scope) error {
+	out.ListMeta = in.ListMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]node.RuntimeClass, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha1_RuntimeClass_To_node_RuntimeClass(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Items = nil
+	}
+	return nil
+}
+
+// Convert_v1alpha1_RuntimeClassList_To_node_RuntimeClassList is an autogenerated conversion function.
+func Convert_v1alpha1_RuntimeClassList_To_node_RuntimeClassList(in *v1alpha1.RuntimeClassList, out *node.RuntimeClassList, s conversion.Scope) error {
+	return autoConvert_v1alpha1_RuntimeClassList_To_node_RuntimeClassList(in, out, s)
+}
+
+func autoConvert_node_RuntimeClassList_To_v1alpha1_RuntimeClassList(in *node.RuntimeClassList, out *v1alpha1.RuntimeClassList, s conversion.Scope) error {
+	out.ListMeta = in.ListMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]v1alpha1.RuntimeClass, len(*in))
+		for i := range *in {
+			if err := Convert_node_RuntimeClass_To_v1alpha1_RuntimeClass(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Items = nil
+	}
+	return nil
+}
+
+// Convert_node_RuntimeClassList_To_v1alpha1_RuntimeClassList is an autogenerated conversion function.
+func Convert_node_RuntimeClassList_To_v1alpha1_RuntimeClassList(in *node.RuntimeClassList, out *v1alpha1.RuntimeClassList, s conversion.Scope) error {
+	return autoConvert_node_RuntimeClassList_To_v1alpha1_RuntimeClassList(in, out, s)
+}

pkg/apis/node/v1beta1/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "register.go",
+        "zz_generated.conversion.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/apis/node/v1beta1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/node:go_default_library",
+        "//staging/src/k8s.io/api/node/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/apis/node/v1beta1/doc.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:conversion-gen=k8s.io/kubernetes/pkg/apis/node
+// +k8s:conversion-gen-external-types=k8s.io/api/node/v1beta1
+
+// +groupName=node.k8s.io
+
+package v1beta1 // import "k8s.io/kubernetes/pkg/apis/node/v1beta1"

pkg/apis/node/v1beta1/register.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	nodev1beta1 "k8s.io/api/node/v1beta1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName for node API
+const GroupName = "node.k8s.io"
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	localSchemeBuilder = &nodev1beta1.SchemeBuilder
+	// AddToScheme node API registration
+	AddToScheme = localSchemeBuilder.AddToScheme
+)

pkg/apis/node/v1beta1/zz_generated.conversion.go

+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by conversion-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	unsafe "unsafe"
+
+	v1beta1 "k8s.io/api/node/v1beta1"
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	node "k8s.io/kubernetes/pkg/apis/node"
+)
+
+func init() {
+	localSchemeBuilder.Register(RegisterConversions)
+}
+
+// RegisterConversions adds conversion functions to the given scheme.
+// Public to allow building arbitrary schemes.
+func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddGeneratedConversionFunc((*v1beta1.RuntimeClass)(nil), (*node.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1beta1_RuntimeClass_To_node_RuntimeClass(a.(*v1beta1.RuntimeClass), b.(*node.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*node.RuntimeClass)(nil), (*v1beta1.RuntimeClass)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_node_RuntimeClass_To_v1beta1_RuntimeClass(a.(*node.RuntimeClass), b.(*v1beta1.RuntimeClass), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*v1beta1.RuntimeClassList)(nil), (*node.RuntimeClassList)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1beta1_RuntimeClassList_To_node_RuntimeClassList(a.(*v1beta1.RuntimeClassList), b.(*node.RuntimeClassList), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*node.RuntimeClassList)(nil), (*v1beta1.RuntimeClassList)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_node_RuntimeClassList_To_v1beta1_RuntimeClassList(a.(*node.RuntimeClassList), b.(*v1beta1.RuntimeClassList), scope)
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func autoConvert_v1beta1_RuntimeClass_To_node_RuntimeClass(in *v1beta1.RuntimeClass, out *node.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	out.Handler = in.Handler
+	return nil
+}
+
+// Convert_v1beta1_RuntimeClass_To_node_RuntimeClass is an autogenerated conversion function.
+func Convert_v1beta1_RuntimeClass_To_node_RuntimeClass(in *v1beta1.RuntimeClass, out *node.RuntimeClass, s conversion.Scope) error {
+	return autoConvert_v1beta1_RuntimeClass_To_node_RuntimeClass(in, out, s)
+}
+
+func autoConvert_node_RuntimeClass_To_v1beta1_RuntimeClass(in *node.RuntimeClass, out *v1beta1.RuntimeClass, s conversion.Scope) error {
+	out.ObjectMeta = in.ObjectMeta
+	out.Handler = in.Handler
+	return nil
+}
+
+// Convert_node_RuntimeClass_To_v1beta1_RuntimeClass is an autogenerated conversion function.
+func Convert_node_RuntimeClass_To_v1beta1_RuntimeClass(in *node.RuntimeClass, out *v1beta1.RuntimeClass, s conversion.Scope) error {
+	return autoConvert_node_RuntimeClass_To_v1beta1_RuntimeClass(in, out, s)
+}
+
+func autoConvert_v1beta1_RuntimeClassList_To_node_RuntimeClassList(in *v1beta1.RuntimeClassList, out *node.RuntimeClassList, s conversion.Scope) error {
+	out.ListMeta = in.ListMeta
+	out.Items = *(*[]node.RuntimeClass)(unsafe.Pointer(&in.Items))
+	return nil
+}
+
+// Convert_v1beta1_RuntimeClassList_To_node_RuntimeClassList is an autogenerated conversion function.
+func Convert_v1beta1_RuntimeClassList_To_node_RuntimeClassList(in *v1beta1.RuntimeClassList, out *node.RuntimeClassList, s conversion.Scope) error {
+	return autoConvert_v1beta1_RuntimeClassList_To_node_RuntimeClassList(in, out, s)
+}
+
+func autoConvert_node_RuntimeClassList_To_v1beta1_RuntimeClassList(in *node.RuntimeClassList, out *v1beta1.RuntimeClassList, s conversion.Scope) error {
+	out.ListMeta = in.ListMeta
+	out.Items = *(*[]v1beta1.RuntimeClass)(unsafe.Pointer(&in.Items))
+	return nil
+}
+
+// Convert_node_RuntimeClassList_To_v1beta1_RuntimeClassList is an autogenerated conversion function.
+func Convert_node_RuntimeClassList_To_v1beta1_RuntimeClassList(in *node.RuntimeClassList, out *v1beta1.RuntimeClassList, s conversion.Scope) error {
+	return autoConvert_node_RuntimeClassList_To_v1beta1_RuntimeClassList(in, out, s)
+}

pkg/apis/node/validation/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["validation.go"],
+    importpath = "k8s.io/kubernetes/pkg/apis/node/validation",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/node:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["validation_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/apis/node:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/apis/node/validation/validation.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	apivalidation "k8s.io/apimachinery/pkg/api/validation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/kubernetes/pkg/apis/node"
+)
+
+// ValidateRuntimeClass validates the RuntimeClass
+func ValidateRuntimeClass(rc *node.RuntimeClass) field.ErrorList {
+	allErrs := apivalidation.ValidateObjectMeta(&rc.ObjectMeta, false, apivalidation.NameIsDNSSubdomain, field.NewPath("metadata"))
+
+	for _, msg := range apivalidation.NameIsDNSLabel(rc.Handler, false) {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("handler"), rc.Handler, msg))
+	}
+
+	return allErrs
+}
+
+// ValidateRuntimeClassUpdate validates an update to the object
+func ValidateRuntimeClassUpdate(new, old *node.RuntimeClass) field.ErrorList {
+	allErrs := apivalidation.ValidateObjectMetaUpdate(&new.ObjectMeta, &old.ObjectMeta, field.NewPath("metadata"))
+
+	allErrs = append(allErrs, apivalidation.ValidateImmutableField(new.Handler, old.Handler, field.NewPath("handler"))...)
+
+	return allErrs
+}

pkg/apis/node/validation/validation_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/apis/node"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateRuntimeClass(t *testing.T) {
+	tests := []struct {
+		name        string
+		rc          node.RuntimeClass
+		expectError bool
+	}{{
+		name:        "invalid name",
+		expectError: true,
+		rc: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{Name: "&!@#"},
+			Handler:    "foo",
+		},
+	}, {
+		name:        "invalid Handler name",
+		expectError: true,
+		rc: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+			Handler:    "&@#$",
+		},
+	}, {
+		name:        "invalid empty RuntimeClass",
+		expectError: true,
+		rc: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{Name: "empty"},
+		},
+	}, {
+		name:        "valid Handler",
+		expectError: false,
+		rc: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+			Handler:    "bar-baz",
+		},
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			errs := ValidateRuntimeClass(&test.rc)
+			if test.expectError {
+				assert.NotEmpty(t, errs)
+			} else {
+				assert.Empty(t, errs)
+			}
+		})
+	}
+}
+
+func TestValidateRuntimeUpdate(t *testing.T) {
+	old := node.RuntimeClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+		Handler:    "bar",
+	}
+	tests := []struct {
+		name        string
+		expectError bool
+		old, new    node.RuntimeClass
+	}{{
+		name: "valid metadata update",
+		old:  old,
+		new: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   "foo",
+				Labels: map[string]string{"foo": "bar"},
+			},
+			Handler: "bar",
+		},
+	}, {
+		name:        "invalid metadata update",
+		expectError: true,
+		old:         old,
+		new: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        "empty",
+				ClusterName: "somethingelse", // immutable
+			},
+			Handler: "bar",
+		},
+	}, {
+		name:        "invalid Handler update",
+		expectError: true,
+		old:         old,
+		new: node.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+			Handler:    "somethingelse",
+		},
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// So we don't need to write it in every test case...
+			test.old.ObjectMeta.ResourceVersion = "1"
+			test.new.ObjectMeta.ResourceVersion = "1"
+
+			errs := ValidateRuntimeClassUpdate(&test.new, &test.old)
+			if test.expectError {
+				assert.NotEmpty(t, errs)
+			} else {
+				assert.Empty(t, errs)
+			}
+		})
+	}
+}

pkg/apis/node/zz_generated.deepcopy.go

+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package node
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuntimeClass) DeepCopyInto(out *RuntimeClass) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeClass.
+func (in *RuntimeClass) DeepCopy() *RuntimeClass {
+	if in == nil {
+		return nil
+	}
+	out := new(RuntimeClass)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *RuntimeClass) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuntimeClassList) DeepCopyInto(out *RuntimeClassList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	out.ListMeta = in.ListMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]RuntimeClass, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeClassList.
+func (in *RuntimeClassList) DeepCopy() *RuntimeClassList {
+	if in == nil {
+		return nil
+	}
+	out := new(RuntimeClassList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *RuntimeClassList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}

pkg/client/clientset_generated/internalclientset/BUILD

@@ -26,6 +26,7 @@ go_library(
         "//pkg/client/clientset_generated/internalclientset/typed/events/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/networking/internalversion:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset/typed/node/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/policy/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion:go_default_library",
@@ -63,6 +64,7 @@ filegroup(
         "//pkg/client/clientset_generated/internalclientset/typed/events/internalversion:all-srcs",
         "//pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion:all-srcs",
         "//pkg/client/clientset_generated/internalclientset/typed/networking/internalversion:all-srcs",
+        "//pkg/client/clientset_generated/internalclientset/typed/node/internalversion:all-srcs",
         "//pkg/client/clientset_generated/internalclientset/typed/policy/internalversion:all-srcs",
         "//pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion:all-srcs",
         "//pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion:all-srcs",

pkg/client/clientset_generated/internalclientset/clientset.go

@@ -35,6 +35,7 @@ import (
 	eventsinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/events/internalversion"
 	extensionsinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion"
 	networkinginternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion"
+	nodeinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/node/internalversion"
 	policyinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion"
 	rbacinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion"
 	schedulinginternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion"
@@ -57,6 +58,7 @@ type Interface interface {
 	Events() eventsinternalversion.EventsInterface
 	Extensions() extensionsinternalversion.ExtensionsInterface
 	Networking() networkinginternalversion.NetworkingInterface
+	Node() nodeinternalversion.NodeInterface
 	Policy() policyinternalversion.PolicyInterface
 	Rbac() rbacinternalversion.RbacInterface
 	Scheduling() schedulinginternalversion.SchedulingInterface
@@ -81,6 +83,7 @@ type Clientset struct {
 	events                *eventsinternalversion.EventsClient
 	extensions            *extensionsinternalversion.ExtensionsClient
 	networking            *networkinginternalversion.NetworkingClient
+	node                  *nodeinternalversion.NodeClient
 	policy                *policyinternalversion.PolicyClient
 	rbac                  *rbacinternalversion.RbacClient
 	scheduling            *schedulinginternalversion.SchedulingClient
@@ -153,6 +156,11 @@ func (c *Clientset) Networking() networkinginternalversion.NetworkingInterface {
 	return c.networking
 }
 
+// Node retrieves the NodeClient
+func (c *Clientset) Node() nodeinternalversion.NodeInterface {
+	return c.node
+}
+
 // Policy retrieves the PolicyClient
 func (c *Clientset) Policy() policyinternalversion.PolicyInterface {
 	return c.policy
@@ -246,6 +254,10 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
 	if err != nil {
 		return nil, err
 	}
+	cs.node, err = nodeinternalversion.NewForConfig(&configShallowCopy)
+	if err != nil {
+		return nil, err
+	}
 	cs.policy, err = policyinternalversion.NewForConfig(&configShallowCopy)
 	if err != nil {
 		return nil, err
@@ -291,6 +303,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset {
 	cs.events = eventsinternalversion.NewForConfigOrDie(c)
 	cs.extensions = extensionsinternalversion.NewForConfigOrDie(c)
 	cs.networking = networkinginternalversion.NewForConfigOrDie(c)
+	cs.node = nodeinternalversion.NewForConfigOrDie(c)
 	cs.policy = policyinternalversion.NewForConfigOrDie(c)
 	cs.rbac = rbacinternalversion.NewForConfigOrDie(c)
 	cs.scheduling = schedulinginternalversion.NewForConfigOrDie(c)
@@ -317,6 +330,7 @@ func New(c rest.Interface) *Clientset {
 	cs.events = eventsinternalversion.New(c)
 	cs.extensions = extensionsinternalversion.New(c)
 	cs.networking = networkinginternalversion.New(c)
+	cs.node = nodeinternalversion.New(c)
 	cs.policy = policyinternalversion.New(c)
 	cs.rbac = rbacinternalversion.New(c)
 	cs.scheduling = schedulinginternalversion.New(c)

pkg/client/clientset_generated/internalclientset/fake/BUILD

@@ -27,6 +27,7 @@ go_library(
         "//pkg/apis/events:go_default_library",
         "//pkg/apis/extensions:go_default_library",
         "//pkg/apis/networking:go_default_library",
+        "//pkg/apis/node:go_default_library",
         "//pkg/apis/policy:go_default_library",
         "//pkg/apis/rbac:go_default_library",
         "//pkg/apis/scheduling:go_default_library",
@@ -59,6 +60,8 @@ go_library(
         "//pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/fake:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/networking/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/fake:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset/typed/node/internalversion:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/policy/internalversion:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/fake:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion:go_default_library",

pkg/client/clientset_generated/internalclientset/fake/clientset_generated.go

@@ -51,6 +51,8 @@ import (
 	fakeextensionsinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/fake"
 	networkinginternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion"
 	fakenetworkinginternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/fake"
+	nodeinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/node/internalversion"
+	fakenodeinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake"
 	policyinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion"
 	fakepolicyinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/fake"
 	rbacinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion"
@@ -170,6 +172,11 @@ func (c *Clientset) Networking() networkinginternalversion.NetworkingInterface {
 	return &fakenetworkinginternalversion.FakeNetworking{Fake: &c.Fake}
 }
 
+// Node retrieves the NodeClient
+func (c *Clientset) Node() nodeinternalversion.NodeInterface {
+	return &fakenodeinternalversion.FakeNode{Fake: &c.Fake}
+}
+
 // Policy retrieves the PolicyClient
 func (c *Clientset) Policy() policyinternalversion.PolicyInterface {
 	return &fakepolicyinternalversion.FakePolicy{Fake: &c.Fake}

pkg/client/clientset_generated/internalclientset/fake/register.go

@@ -37,6 +37,7 @@ import (
 	eventsinternalversion "k8s.io/kubernetes/pkg/apis/events"
 	extensionsinternalversion "k8s.io/kubernetes/pkg/apis/extensions"
 	networkinginternalversion "k8s.io/kubernetes/pkg/apis/networking"
+	nodeinternalversion "k8s.io/kubernetes/pkg/apis/node"
 	policyinternalversion "k8s.io/kubernetes/pkg/apis/policy"
 	rbacinternalversion "k8s.io/kubernetes/pkg/apis/rbac"
 	schedulinginternalversion "k8s.io/kubernetes/pkg/apis/scheduling"
@@ -61,6 +62,7 @@ var localSchemeBuilder = runtime.SchemeBuilder{
 	eventsinternalversion.AddToScheme,
 	extensionsinternalversion.AddToScheme,
 	networkinginternalversion.AddToScheme,
+	nodeinternalversion.AddToScheme,
 	policyinternalversion.AddToScheme,
 	rbacinternalversion.AddToScheme,
 	schedulinginternalversion.AddToScheme,

pkg/client/clientset_generated/internalclientset/scheme/BUILD

@@ -26,6 +26,7 @@ go_library(
         "//pkg/apis/events/install:go_default_library",
         "//pkg/apis/extensions/install:go_default_library",
         "//pkg/apis/networking/install:go_default_library",
+        "//pkg/apis/node/install:go_default_library",
         "//pkg/apis/policy/install:go_default_library",
         "//pkg/apis/rbac/install:go_default_library",
         "//pkg/apis/scheduling/install:go_default_library",

pkg/client/clientset_generated/internalclientset/scheme/register.go

@@ -36,6 +36,7 @@ import (
 	events "k8s.io/kubernetes/pkg/apis/events/install"
 	extensions "k8s.io/kubernetes/pkg/apis/extensions/install"
 	networking "k8s.io/kubernetes/pkg/apis/networking/install"
+	node "k8s.io/kubernetes/pkg/apis/node/install"
 	policy "k8s.io/kubernetes/pkg/apis/policy/install"
 	rbac "k8s.io/kubernetes/pkg/apis/rbac/install"
 	scheduling "k8s.io/kubernetes/pkg/apis/scheduling/install"
@@ -67,6 +68,7 @@ func Install(scheme *runtime.Scheme) {
 	events.Install(scheme)
 	extensions.Install(scheme)
 	networking.Install(scheme)
+	node.Install(scheme)
 	policy.Install(scheme)
 	rbac.Install(scheme)
 	scheduling.Install(scheme)

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "generated_expansion.go",
+        "node_client.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/node/internalversion",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/client/clientset_generated/internalclientset/scheme:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/doc.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package has the automatically generated typed clients.
+package internalversion

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "fake_node_client.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake/doc.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// Package fake has the automatically generated clients.
+package fake

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/fake/fake_node_client.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	rest "k8s.io/client-go/rest"
+	testing "k8s.io/client-go/testing"
+)
+
+type FakeNode struct {
+	*testing.Fake
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *FakeNode) RESTClient() rest.Interface {
+	var ret *rest.RESTClient
+	return ret
+}

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/generated_expansion.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package internalversion

pkg/client/clientset_generated/internalclientset/typed/node/internalversion/node_client.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package internalversion
+
+import (
+	rest "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/scheme"
+)
+
+type NodeInterface interface {
+	RESTClient() rest.Interface
+}
+
+// NodeClient is used to interact with features provided by the node.k8s.io group.
+type NodeClient struct {
+	restClient rest.Interface
+}
+
+// NewForConfig creates a new NodeClient for the given config.
+func NewForConfig(c *rest.Config) (*NodeClient, error) {
+	config := *c
+	if err := setConfigDefaults(&config); err != nil {
+		return nil, err
+	}
+	client, err := rest.RESTClientFor(&config)
+	if err != nil {
+		return nil, err
+	}
+	return &NodeClient{client}, nil
+}
+
+// NewForConfigOrDie creates a new NodeClient for the given config and
+// panics if there is an error in the config.
+func NewForConfigOrDie(c *rest.Config) *NodeClient {
+	client, err := NewForConfig(c)
+	if err != nil {
+		panic(err)
+	}
+	return client
+}
+
+// New creates a new NodeClient for the given RESTClient.
+func New(c rest.Interface) *NodeClient {
+	return &NodeClient{c}
+}
+
+func setConfigDefaults(config *rest.Config) error {
+	config.APIPath = "/apis"
+	if config.UserAgent == "" {
+		config.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+	if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("node.k8s.io")[0].Group {
+		gv := scheme.Scheme.PrioritizedVersionsForGroup("node.k8s.io")[0]
+		config.GroupVersion = &gv
+	}
+	config.NegotiatedSerializer = scheme.Codecs
+
+	if config.QPS == 0 {
+		config.QPS = 5
+	}
+	if config.Burst == 0 {
+		config.Burst = 10
+	}
+
+	return nil
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *NodeClient) RESTClient() rest.Interface {
+	if c == nil {
+		return nil
+	}
+	return c.restClient
+}

pkg/cloudprovider/providers/azure/azure_test.go

@@ -1676,7 +1676,8 @@ func validateEmptyConfig(t *testing.T, config string) {
 func TestGetZone(t *testing.T) {
 	cloud := &Cloud{
 		Config: Config{
-			Location: "eastus",
+			Location:            "eastus",
+			UseInstanceMetadata: true,
 		},
 	}
 	testcases := []struct {

pkg/cloudprovider/providers/azure/azure_zones.go

@@ -19,6 +19,7 @@ package azure
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
@@ -49,31 +50,39 @@ func (az *Cloud) GetZoneID(zoneLabel string) string {
 // GetZone returns the Zone containing the current availability zone and locality region that the program is running in.
 // If the node is not running with availability zones, then it will fall back to fault domain.
 func (az *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) {
-	metadata, err := az.metadata.GetMetadata()
-	if err != nil {
-		return cloudprovider.Zone{}, err
-	}
+	if az.UseInstanceMetadata {
+		metadata, err := az.metadata.GetMetadata()
+		if err != nil {
+			return cloudprovider.Zone{}, err
+		}
 
-	if metadata.Compute == nil {
-		return cloudprovider.Zone{}, fmt.Errorf("failure of getting compute information from instance metadata")
-	}
+		if metadata.Compute == nil {
+			return cloudprovider.Zone{}, fmt.Errorf("failure of getting compute information from instance metadata")
+		}
 
-	zone := ""
-	if metadata.Compute.Zone != "" {
-		zoneID, err := strconv.Atoi(metadata.Compute.Zone)
-		if err != nil {
-			return cloudprovider.Zone{}, fmt.Errorf("failed to parse zone ID %q: %v", metadata.Compute.Zone, err)
+		zone := ""
+		if metadata.Compute.Zone != "" {
+			zoneID, err := strconv.Atoi(metadata.Compute.Zone)
+			if err != nil {
+				return cloudprovider.Zone{}, fmt.Errorf("failed to parse zone ID %q: %v", metadata.Compute.Zone, err)
+			}
+			zone = az.makeZone(zoneID)
+		} else {
+			klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain")
+			zone = metadata.Compute.FaultDomain
 		}
-		zone = az.makeZone(zoneID)
-	} else {
-		klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain")
-		zone = metadata.Compute.FaultDomain
-	}
 
-	return cloudprovider.Zone{
-		FailureDomain: zone,
-		Region:        az.Location,
-	}, nil
+		return cloudprovider.Zone{
+			FailureDomain: zone,
+			Region:        az.Location,
+		}, nil
+	}
+	// if UseInstanceMetadata is false, get Zone name by calling ARM
+	hostname, err := os.Hostname()
+	if err != nil {
+		return cloudprovider.Zone{}, fmt.Errorf("failure getting hostname from kernel")
+	}
+	return az.vmSet.GetZoneByNodeName(strings.ToLower(hostname))
 }
 
 // GetZoneByProviderID implements Zones.GetZoneByProviderID

pkg/controller/controller_utils.go

@@ -88,6 +88,12 @@ var UpdateTaintBackoff = wait.Backoff{
 	Jitter:   1.0,
 }
 
+var UpdateLabelBackoff = wait.Backoff{
+	Steps:    5,
+	Duration: 100 * time.Millisecond,
+	Jitter:   1.0,
+}
+
 var (
 	KeyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
 )
@@ -1045,3 +1051,48 @@ func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) string {
 
 	return rand.SafeEncodeString(fmt.Sprint(podTemplateSpecHasher.Sum32()))
 }
+
+func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, nodeName string, labelsToUpdate map[string]string) error {
+	firstTry := true
+	return clientretry.RetryOnConflict(UpdateLabelBackoff, func() error {
+		var err error
+		var node *v1.Node
+		// First we try getting node from the API server cache, as it's cheaper. If it fails
+		// we get it from etcd to be sure to have fresh data.
+		if firstTry {
+			node, err = kubeClient.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{ResourceVersion: "0"})
+			firstTry = false
+		} else {
+			node, err = kubeClient.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		}
+		if err != nil {
+			return err
+		}
+
+		// Make a copy of the node and update the labels.
+		newNode := node.DeepCopy()
+		if newNode.Labels == nil {
+			newNode.Labels = make(map[string]string)
+		}
+		for key, value := range labelsToUpdate {
+			newNode.Labels[key] = value
+		}
+
+		oldData, err := json.Marshal(node)
+		if err != nil {
+			return fmt.Errorf("failed to marshal the existing node %#v: %v", node, err)
+		}
+		newData, err := json.Marshal(newNode)
+		if err != nil {
+			return fmt.Errorf("failed to marshal the new node %#v: %v", newNode, err)
+		}
+		patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
+		if err != nil {
+			return fmt.Errorf("failed to create a two-way merge patch: %v", err)
+		}
+		if _, err := kubeClient.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
+			return fmt.Errorf("failed to patch the node: %v", err)
+		}
+		return nil
+	})
+}

pkg/controller/nodelifecycle/BUILD

@@ -13,6 +13,7 @@ go_library(
         "//pkg/controller/nodelifecycle/scheduler:go_default_library",
         "//pkg/controller/util/node:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
         "//pkg/scheduler/api:go_default_library",
         "//pkg/util/metrics:go_default_library",
         "//pkg/util/node:go_default_library",
@@ -72,6 +73,7 @@ go_test(
         "//pkg/controller/testutil:go_default_library",
         "//pkg/controller/util/node:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
         "//pkg/scheduler/api:go_default_library",
         "//pkg/util/node:go_default_library",
         "//pkg/util/taints:go_default_library",

pkg/controller/nodelifecycle/node_lifecycle_controller.go

@@ -56,6 +56,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller/nodelifecycle/scheduler"
 	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
 	"k8s.io/kubernetes/pkg/features"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	"k8s.io/kubernetes/pkg/util/metrics"
 	utilnode "k8s.io/kubernetes/pkg/util/node"
@@ -130,6 +131,37 @@ const (
 	retrySleepTime = 20 * time.Millisecond
 )
 
+// labelReconcileInfo lists Node labels to reconcile, and how to reconcile them.
+// primaryKey and secondaryKey are keys of labels to reconcile.
+//   - If both keys exist, but their values don't match. Use the value from the
+//   primaryKey as the source of truth to reconcile.
+//   - If ensureSecondaryExists is true, and the secondaryKey does not
+//   exist, secondaryKey will be added with the value of the primaryKey.
+var labelReconcileInfo = []struct {
+	primaryKey            string
+	secondaryKey          string
+	ensureSecondaryExists bool
+}{
+	{
+		// Reconcile the beta and the stable OS label using the beta label as
+		// the source of truth.
+		// TODO(#73084): switch to using the stable label as the source of
+		// truth in v1.18.
+		primaryKey:            kubeletapis.LabelOS,
+		secondaryKey:          v1.LabelOSStable,
+		ensureSecondaryExists: true,
+	},
+	{
+		// Reconcile the beta and the stable arch label using the beta label as
+		// the source of truth.
+		// TODO(#73084): switch to using the stable label as the source of
+		// truth in v1.18.
+		primaryKey:            kubeletapis.LabelArch,
+		secondaryKey:          v1.LabelArchStable,
+		ensureSecondaryExists: true,
+	},
+}
+
 type nodeHealthData struct {
 	probeTimestamp           metav1.Time
 	readyTransitionTimestamp metav1.Time
@@ -355,18 +387,20 @@ func NewNodeLifecycleController(
 		})
 	}
 
+	klog.Infof("Controller will reconcile labels.")
+	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: nodeutil.CreateAddNodeHandler(func(node *v1.Node) error {
+			nc.nodeUpdateQueue.Add(node.Name)
+			return nil
+		}),
+		UpdateFunc: nodeutil.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
+			nc.nodeUpdateQueue.Add(newNode.Name)
+			return nil
+		}),
+	})
+
 	if nc.taintNodeByCondition {
 		klog.Infof("Controller will taint node by condition.")
-		nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-			AddFunc: nodeutil.CreateAddNodeHandler(func(node *v1.Node) error {
-				nc.nodeUpdateQueue.Add(node.Name)
-				return nil
-			}),
-			UpdateFunc: nodeutil.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
-				nc.nodeUpdateQueue.Add(newNode.Name)
-				return nil
-			}),
-		})
 	}
 
 	nc.leaseLister = leaseInformer.Lister()
@@ -401,18 +435,16 @@ func (nc *Controller) Run(stopCh <-chan struct{}) {
 		go nc.taintManager.Run(stopCh)
 	}
 
-	if nc.taintNodeByCondition {
-		// Close node update queue to cleanup go routine.
-		defer nc.nodeUpdateQueue.ShutDown()
-
-		// Start workers to update NoSchedule taint for nodes.
-		for i := 0; i < scheduler.UpdateWorkerSize; i++ {
-			// Thanks to "workqueue", each worker just need to get item from queue, because
-			// the item is flagged when got from queue: if new event come, the new item will
-			// be re-queued until "Done", so no more than one worker handle the same item and
-			// no event missed.
-			go wait.Until(nc.doNoScheduleTaintingPassWorker, time.Second, stopCh)
-		}
+	// Close node update queue to cleanup go routine.
+	defer nc.nodeUpdateQueue.ShutDown()
+
+	// Start workers to reconcile labels and/or update NoSchedule taint for nodes.
+	for i := 0; i < scheduler.UpdateWorkerSize; i++ {
+		// Thanks to "workqueue", each worker just need to get item from queue, because
+		// the item is flagged when got from queue: if new event come, the new item will
+		// be re-queued until "Done", so no more than one worker handle the same item and
+		// no event missed.
+		go wait.Until(nc.doNodeProcessingPassWorker, time.Second, stopCh)
 	}
 
 	if nc.useTaintBasedEvictions {
@@ -436,7 +468,7 @@ func (nc *Controller) Run(stopCh <-chan struct{}) {
 	<-stopCh
 }
 
-func (nc *Controller) doNoScheduleTaintingPassWorker() {
+func (nc *Controller) doNodeProcessingPassWorker() {
 	for {
 		obj, shutdown := nc.nodeUpdateQueue.Get()
 		// "nodeUpdateQueue" will be shutdown when "stopCh" closed;
@@ -445,10 +477,17 @@ func (nc *Controller) doNoScheduleTaintingPassWorker() {
 			return
 		}
 		nodeName := obj.(string)
-
-		if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
-			// TODO (k82cn): Add nodeName back to the queue.
-			klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
+		if nc.taintNodeByCondition {
+			if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
+				klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
+				// TODO(k82cn): Add nodeName back to the queue
+			}
+		}
+		// TODO: re-evaluate whether there are any labels that need to be
+		// reconcile in 1.19. Remove this function if it's no longer necessary.
+		if err := nc.reconcileNodeLabels(nodeName); err != nil {
+			klog.Errorf("Failed to reconcile labels for node <%s>, requeue it: %v", nodeName, err)
+			// TODO(yujuhong): Add nodeName back to the queue
 		}
 		nc.nodeUpdateQueue.Done(nodeName)
 	}
@@ -1191,6 +1230,53 @@ func (nc *Controller) ComputeZoneState(nodeReadyConditions []*v1.NodeCondition)
 	}
 }
 
+// reconcileNodeLabels reconciles node labels.
+func (nc *Controller) reconcileNodeLabels(nodeName string) error {
+	node, err := nc.nodeLister.Get(nodeName)
+	if err != nil {
+		// If node not found, just ignore it.
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	if node.Labels == nil {
+		// Nothing to reconcile.
+		return nil
+	}
+
+	labelsToUpdate := map[string]string{}
+	for _, r := range labelReconcileInfo {
+		primaryValue, primaryExists := node.Labels[r.primaryKey]
+		secondaryValue, secondaryExists := node.Labels[r.secondaryKey]
+
+		if !primaryExists {
+			// The primary label key does not exist. This should not happen
+			// within our supported version skew range, when no external
+			// components/factors modifying the node object. Ignore this case.
+			continue
+		}
+		if secondaryExists && primaryValue != secondaryValue {
+			// Secondary label exists, but not consistent with the primary
+			// label. Need to reconcile.
+			labelsToUpdate[r.secondaryKey] = primaryValue
+
+		} else if !secondaryExists && r.ensureSecondaryExists {
+			// Apply secondary label based on primary label.
+			labelsToUpdate[r.secondaryKey] = primaryValue
+		}
+	}
+
+	if len(labelsToUpdate) == 0 {
+		return nil
+	}
+	if !nodeutil.AddOrUpdateLabelsOnNode(nc.kubeClient, labelsToUpdate, node) {
+		return fmt.Errorf("failed update labels for node %+v", node)
+	}
+	return nil
+}
+
 func hash(val string, max int) int {
 	hasher := fnv.New32a()
 	io.WriteString(hasher, val)

pkg/controller/nodelifecycle/node_lifecycle_controller_test.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller/testutil"
 	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
 	"k8s.io/kubernetes/pkg/features"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	"k8s.io/kubernetes/pkg/util/node"
 	taintutils "k8s.io/kubernetes/pkg/util/taints"
@@ -2910,3 +2911,149 @@ func TestNodeEventGeneration(t *testing.T) {
 		}
 	}
 }
+
+func TestReconcileNodeLabels(t *testing.T) {
+	fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC)
+	evictionTimeout := 10 * time.Minute
+
+	fakeNodeHandler := &testutil.FakeNodeHandler{
+		Existing: []*v1.Node{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						v1.LabelZoneRegion:        "region1",
+						v1.LabelZoneFailureDomain: "zone1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Conditions: []v1.NodeCondition{
+						{
+							Type:               v1.NodeReady,
+							Status:             v1.ConditionTrue,
+							LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC),
+							LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+		},
+		Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}),
+	}
+
+	nodeController, _ := newNodeLifecycleControllerFromClient(
+		fakeNodeHandler,
+		evictionTimeout,
+		testRateLimiterQPS,
+		testRateLimiterQPS,
+		testLargeClusterThreshold,
+		testUnhealthyThreshold,
+		testNodeMonitorGracePeriod,
+		testNodeStartupGracePeriod,
+		testNodeMonitorPeriod,
+		true)
+	nodeController.now = func() metav1.Time { return fakeNow }
+	nodeController.recorder = testutil.NewFakeRecorder()
+
+	tests := []struct {
+		Name           string
+		Node           *v1.Node
+		ExpectedLabels map[string]string
+	}{
+		{
+			Name: "No-op if node has no labels",
+			Node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			ExpectedLabels: nil,
+		},
+		{
+			Name: "No-op if no target labels present",
+			Node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						v1.LabelZoneRegion: "region1",
+					},
+				},
+			},
+			ExpectedLabels: map[string]string{
+				v1.LabelZoneRegion: "region1",
+			},
+		},
+		{
+			Name: "Create OS/arch stable labels when they don't exist",
+			Node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						kubeletapis.LabelOS:   "linux",
+						kubeletapis.LabelArch: "amd64",
+					},
+				},
+			},
+			ExpectedLabels: map[string]string{
+				kubeletapis.LabelOS:   "linux",
+				kubeletapis.LabelArch: "amd64",
+				v1.LabelOSStable:      "linux",
+				v1.LabelArchStable:    "amd64",
+			},
+		},
+		{
+			Name: "Reconcile OS/arch stable labels to match beta labels",
+			Node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						kubeletapis.LabelOS:   "linux",
+						kubeletapis.LabelArch: "amd64",
+						v1.LabelOSStable:      "windows",
+						v1.LabelArchStable:    "arm",
+					},
+				},
+			},
+			ExpectedLabels: map[string]string{
+				kubeletapis.LabelOS:   "linux",
+				kubeletapis.LabelArch: "amd64",
+				v1.LabelOSStable:      "linux",
+				v1.LabelArchStable:    "amd64",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		fakeNodeHandler.Update(test.Node)
+		if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		nodeController.reconcileNodeLabels(test.Node.Name)
+		if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		node0, err := nodeController.nodeLister.Get("node0")
+		if err != nil {
+			t.Fatalf("Can't get current node0...")
+		}
+		if len(node0.Labels) != len(test.ExpectedLabels) {
+			t.Errorf("%s: Unexpected number of taints: expected %d, got %d",
+				test.Name, len(test.ExpectedLabels), len(node0.Labels))
+		}
+		for key, expectedValue := range test.ExpectedLabels {
+			actualValue, ok := node0.Labels[key]
+			if !ok {
+				t.Errorf("%s: Can't find label %v in %v", test.Name, key, node0.Labels)
+			}
+			if actualValue != expectedValue {
+				t.Errorf("%s: label %q: expected value %q, got value %q", test.Name, key, expectedValue, actualValue)
+			}
+
+		}
+	}
+}

pkg/controller/util/node/controller_utils.go

@@ -214,6 +214,23 @@ func SwapNodeControllerTaint(kubeClient clientset.Interface, taintsToAdd, taints
 	return true
 }
 
+// AddOrUpdateLabelsOnNode updates the labels on the node and returns true on
+// success and false on failure.
+func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, labelsToUpdate map[string]string, node *v1.Node) bool {
+	err := controller.AddOrUpdateLabelsOnNode(kubeClient, node.Name, labelsToUpdate)
+	if err != nil {
+		utilruntime.HandleError(
+			fmt.Errorf(
+				"unable to update labels %+v for Node %q: %v",
+				labelsToUpdate,
+				node.Name,
+				err))
+		return false
+	}
+	klog.V(4).Infof("Updated labels %+v to Node %v", labelsToUpdate, node.Name)
+	return true
+}
+
 // CreateAddNodeHandler creates an add node handler.
 func CreateAddNodeHandler(f func(node *v1.Node) error) func(obj interface{}) {
 	return func(originalObj interface{}) {

pkg/controller/volume/attachdetach/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/controller/volume/attachdetach/reconciler:go_default_library",
         "//pkg/controller/volume/attachdetach/statusupdater:go_default_library",
         "//pkg/controller/volume/attachdetach/util:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
@@ -31,11 +32,14 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/informers/storage/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",

pkg/controller/volume/attachdetach/attach_detach_controller.go

@@ -30,11 +30,14 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	coreinformers "k8s.io/client-go/informers/core/v1"
+	storageinformers "k8s.io/client-go/informers/storage/v1beta1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
+	storagelisters "k8s.io/client-go/listers/storage/v1beta1"
 	kcache "k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
@@ -47,6 +50,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/util"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
@@ -101,6 +105,7 @@ func NewAttachDetachController(
 	nodeInformer coreinformers.NodeInformer,
 	pvcInformer coreinformers.PersistentVolumeClaimInformer,
 	pvInformer coreinformers.PersistentVolumeInformer,
+	csiNodeInformer storageinformers.CSINodeInformer,
 	cloud cloudprovider.Interface,
 	plugins []volume.VolumePlugin,
 	prober volume.DynamicPluginProber,
@@ -136,6 +141,12 @@ func NewAttachDetachController(
 		pvcQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"),
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) &&
+		utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) {
+		adc.csiNodeLister = csiNodeInformer.Lister()
+		adc.csiNodeSynced = csiNodeInformer.Informer().HasSynced
+	}
+
 	if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil {
 		return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err)
 	}
@@ -257,6 +268,9 @@ type attachDetachController struct {
 	nodeLister  corelisters.NodeLister
 	nodesSynced kcache.InformerSynced
 
+	csiNodeLister storagelisters.CSINodeLister
+	csiNodeSynced kcache.InformerSynced
+
 	// cloud provider used by volume host
 	cloud cloudprovider.Interface
 
@@ -309,7 +323,12 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) {
 	klog.Infof("Starting attach detach controller")
 	defer klog.Infof("Shutting down attach detach controller")
 
-	if !controller.WaitForCacheSync("attach detach", stopCh, adc.podsSynced, adc.nodesSynced, adc.pvcsSynced, adc.pvsSynced) {
+	synced := []kcache.InformerSynced{adc.podsSynced, adc.nodesSynced, adc.pvcsSynced, adc.pvsSynced}
+	if adc.csiNodeSynced != nil {
+		synced = append(synced, adc.csiNodeSynced)
+	}
+
+	if !controller.WaitForCacheSync("attach detach", stopCh, synced...) {
 		return
 	}
 
@@ -643,6 +662,17 @@ func (adc *attachDetachController) processVolumesInUse(
 	}
 }
 
+var _ volume.VolumeHost = &attachDetachController{}
+var _ volume.AttachDetachVolumeHost = &attachDetachController{}
+
+func (adc *attachDetachController) CSINodeLister() storagelisters.CSINodeLister {
+	return adc.csiNodeLister
+}
+
+func (adc *attachDetachController) IsAttachDetachController() bool {
+	return true
+}
+
 // VolumeHost implementation
 // This is an unfortunate requirement of the current factoring of volume plugin
 // initializing code. It requires kubelet specific methods used by the mounting

pkg/controller/volume/attachdetach/attach_detach_controller_test.go

@@ -44,6 +44,7 @@ func Test_NewAttachDetachController_Positive(t *testing.T) {
 		informerFactory.Core().V1().Nodes(),
 		informerFactory.Core().V1().PersistentVolumeClaims(),
 		informerFactory.Core().V1().PersistentVolumes(),
+		informerFactory.Storage().V1beta1().CSINodes(),
 		nil, /* cloud */
 		nil, /* plugins */
 		nil, /* prober */
@@ -218,6 +219,7 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2
 		informerFactory.Core().V1().Nodes(),
 		informerFactory.Core().V1().PersistentVolumeClaims(),
 		informerFactory.Core().V1().PersistentVolumes(),
+		informerFactory.Storage().V1beta1().CSINodes(),
 		nil, /* cloud */
 		plugins,
 		prober,

pkg/features/kube_features.go

@@ -310,9 +310,9 @@ const (
 	BalanceAttachedNodeVolumes utilfeature.Feature = "BalanceAttachedNodeVolumes"
 
 	// owner @freehan
-	// beta: v1.11
+	// GA: v1.14
 	//
-	// Support Pod Ready++
+	// Allow user to specify additional conditions to be evaluated for Pod readiness.
 	PodReadinessGates utilfeature.Feature = "PodReadinessGates"
 
 	// owner: @kevtaylor
@@ -480,7 +480,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	CSIMigrationOpenStack:                       {Default: false, PreRelease: utilfeature.Alpha},
 	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
 	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha},
-	PodReadinessGates:                           {Default: true, PreRelease: utilfeature.Beta},
+	PodReadinessGates:                           {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16
 	VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha},
 	KubeletPluginsWatcher:                       {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16
 	ResourceQuotaScopeSelectors:                 {Default: true, PreRelease: utilfeature.Beta},
@@ -510,6 +510,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	apiextensionsfeatures.CustomResourceValidation:        {Default: true, PreRelease: utilfeature.Beta},
 	apiextensionsfeatures.CustomResourceSubresources:      {Default: true, PreRelease: utilfeature.Beta},
 	apiextensionsfeatures.CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha},
+	apiextensionsfeatures.CustomResourcePublishOpenAPI:    {Default: false, PreRelease: utilfeature.Alpha},
 
 	// features that enable backwards compatibility but are scheduled to be removed
 	// ...

pkg/kubectl/cmd/create/BUILD

@@ -7,6 +7,7 @@ go_library(
         "create_clusterrole.go",
         "create_clusterrolebinding.go",
         "create_configmap.go",
+        "create_cronjob.go",
         "create_deployment.go",
         "create_job.go",
         "create_namespace.go",
@@ -45,6 +46,7 @@ go_library(
         "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/batch/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/batch/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library",
         "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
@@ -58,6 +60,7 @@ go_test(
         "create_clusterrole_test.go",
         "create_clusterrolebinding_test.go",
         "create_configmap_test.go",
+        "create_cronjob_test.go",
         "create_deployment_test.go",
         "create_job_test.go",
         "create_namespace_test.go",

pkg/kubectl/cmd/create/create.go

@@ -146,6 +146,7 @@ func NewCmdCreate(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cob
 	cmd.AddCommand(NewCmdCreatePodDisruptionBudget(f, ioStreams))
 	cmd.AddCommand(NewCmdCreatePriorityClass(f, ioStreams))
 	cmd.AddCommand(NewCmdCreateJob(f, ioStreams))
+	cmd.AddCommand(NewCmdCreateCronJob(f, ioStreams))
 	return cmd
 }
 

pkg/kubectl/cmd/create/create_cronjob.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package create
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	batchv1 "k8s.io/api/batch/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/genericclioptions/resource"
+	batchv1beta1client "k8s.io/client-go/kubernetes/typed/batch/v1beta1"
+	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	"k8s.io/kubernetes/pkg/kubectl/scheme"
+	"k8s.io/kubernetes/pkg/kubectl/util/templates"
+)
+
+var (
+	cronjobLong = templates.LongDesc(`
+		Create a cronjob with the specified name.`)
+
+	cronjobExample = templates.Examples(`
+		# Create a cronjob
+		kubectl create cronjob my-job --image=busybox 
+
+		# Create a cronjob with command
+		kubectl create cronjob my-job --image=busybox -- date 
+
+		# Create a cronjob with schedule
+		kubectl create cronjob test-job --image=busybox --schedule="*/1 * * * *"`)
+)
+
+type CreateCronJobOptions struct {
+	PrintFlags *genericclioptions.PrintFlags
+
+	PrintObj func(obj runtime.Object) error
+
+	Name     string
+	Image    string
+	Schedule string
+	Command  []string
+	Restart  string
+
+	Namespace string
+	Client    batchv1beta1client.BatchV1beta1Interface
+	DryRun    bool
+	Builder   *resource.Builder
+	Cmd       *cobra.Command
+
+	genericclioptions.IOStreams
+}
+
+func NewCreateCronJobOptions(ioStreams genericclioptions.IOStreams) *CreateCronJobOptions {
+	return &CreateCronJobOptions{
+		PrintFlags: genericclioptions.NewPrintFlags("created").WithTypeSetter(scheme.Scheme),
+		IOStreams:  ioStreams,
+	}
+}
+
+// NewCmdCreateCronJob is a command to to create CronJobs.
+func NewCmdCreateCronJob(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command {
+	o := NewCreateCronJobOptions(ioStreams)
+	cmd := &cobra.Command{
+		Use:     "cronjob NAME --image=image --schedule='0/5 * * * ?' -- [COMMAND] [args...]",
+		Aliases: []string{"cj"},
+		Short:   cronjobLong,
+		Long:    cronjobLong,
+		Example: cronjobExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdutil.CheckErr(o.Complete(f, cmd, args))
+			cmdutil.CheckErr(o.Validate())
+			cmdutil.CheckErr(o.Run())
+		},
+	}
+
+	o.PrintFlags.AddFlags(cmd)
+
+	cmdutil.AddApplyAnnotationFlags(cmd)
+	cmdutil.AddValidateFlags(cmd)
+	cmdutil.AddDryRunFlag(cmd)
+	cmd.Flags().StringVar(&o.Image, "image", o.Image, "Image name to run.")
+	cmd.Flags().StringVar(&o.Schedule, "schedule", o.Schedule, "A schedule in the Cron format the job should be run with.")
+	cmd.Flags().StringVar(&o.Restart, "restart", o.Restart, "job's restart policy. supported values: OnFailure, Never")
+
+	return cmd
+}
+
+func (o *CreateCronJobOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error {
+	name, err := NameFromCommandArgs(cmd, args)
+	if err != nil {
+		return err
+	}
+	o.Name = name
+	if len(args) > 1 {
+		o.Command = args[1:]
+	}
+	if len(o.Restart) == 0 {
+		o.Restart = "OnFailure"
+	}
+
+	clientConfig, err := f.ToRESTConfig()
+	if err != nil {
+		return err
+	}
+	o.Client, err = batchv1beta1client.NewForConfig(clientConfig)
+	if err != nil {
+		return err
+	}
+
+	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
+	if err != nil {
+		return err
+	}
+	o.Builder = f.NewBuilder()
+	o.Cmd = cmd
+
+	o.DryRun = cmdutil.GetDryRunFlag(cmd)
+	if o.DryRun {
+		o.PrintFlags.Complete("%s (dry run)")
+	}
+	printer, err := o.PrintFlags.ToPrinter()
+	if err != nil {
+		return err
+	}
+	o.PrintObj = func(obj runtime.Object) error {
+		return printer.PrintObj(obj, o.Out)
+	}
+
+	return nil
+}
+
+func (o *CreateCronJobOptions) Validate() error {
+	if len(o.Image) == 0 {
+		return fmt.Errorf("--image must be specified")
+	}
+	if len(o.Schedule) == 0 {
+		return fmt.Errorf("--schedule must be specified")
+	}
+	return nil
+}
+
+func (o *CreateCronJobOptions) Run() error {
+	var cronjob *batchv1beta1.CronJob
+	cronjob = o.createCronJob()
+
+	if !o.DryRun {
+		var err error
+		cronjob, err = o.Client.CronJobs(o.Namespace).Create(cronjob)
+		if err != nil {
+			return fmt.Errorf("failed to create cronjob: %v", err)
+		}
+	}
+
+	return o.PrintObj(cronjob)
+}
+
+func (o *CreateCronJobOptions) createCronJob() *batchv1beta1.CronJob {
+	return &batchv1beta1.CronJob{
+		TypeMeta: metav1.TypeMeta{APIVersion: batchv1beta1.SchemeGroupVersion.String(), Kind: "CronJob"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: o.Name,
+		},
+		Spec: batchv1beta1.CronJobSpec{
+			Schedule: o.Schedule,
+			JobTemplate: batchv1beta1.JobTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: o.Name,
+				},
+				Spec: batchv1.JobSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name:    o.Name,
+									Image:   o.Image,
+									Command: o.Command,
+								},
+							},
+							RestartPolicy: corev1.RestartPolicy(o.Restart),
+						},
+					},
+				},
+			},
+		},
+	}
+}

pkg/kubectl/cmd/create/create_cronjob_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package create
+
+import (
+	"testing"
+
+	batchv1 "k8s.io/api/batch/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestCreateCronJob(t *testing.T) {
+	cronjobName := "test-job"
+	tests := map[string]struct {
+		image    string
+		command  []string
+		schedule string
+		restart  string
+		expected *batchv1beta1.CronJob
+	}{
+		"just image and OnFailure restart policy": {
+			image:    "busybox",
+			schedule: "0/5 * * * ?",
+			restart:  "OnFailure",
+			expected: &batchv1beta1.CronJob{
+				TypeMeta: metav1.TypeMeta{APIVersion: batchv1beta1.SchemeGroupVersion.String(), Kind: "CronJob"},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: cronjobName,
+				},
+				Spec: batchv1beta1.CronJobSpec{
+					Schedule: "0/5 * * * ?",
+					JobTemplate: batchv1beta1.JobTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: cronjobName,
+						},
+						Spec: batchv1.JobSpec{
+							Template: corev1.PodTemplateSpec{
+								Spec: corev1.PodSpec{
+									Containers: []corev1.Container{
+										{
+											Name:  cronjobName,
+											Image: "busybox",
+										},
+									},
+									RestartPolicy: corev1.RestartPolicyOnFailure,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"image, command , schedule and Never restart policy": {
+			image:    "busybox",
+			command:  []string{"date"},
+			schedule: "0/5 * * * ?",
+			restart:  "Never",
+			expected: &batchv1beta1.CronJob{
+				TypeMeta: metav1.TypeMeta{APIVersion: batchv1beta1.SchemeGroupVersion.String(), Kind: "CronJob"},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: cronjobName,
+				},
+				Spec: batchv1beta1.CronJobSpec{
+					Schedule: "0/5 * * * ?",
+					JobTemplate: batchv1beta1.JobTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: cronjobName,
+						},
+						Spec: batchv1.JobSpec{
+							Template: corev1.PodTemplateSpec{
+								Spec: corev1.PodSpec{
+									Containers: []corev1.Container{
+										{
+											Name:    cronjobName,
+											Image:   "busybox",
+											Command: []string{"date"},
+										},
+									},
+									RestartPolicy: corev1.RestartPolicyNever,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			o := &CreateCronJobOptions{
+				Name:     cronjobName,
+				Image:    tc.image,
+				Command:  tc.command,
+				Schedule: tc.schedule,
+				Restart:  tc.restart,
+			}
+			cronjob := o.createCronJob()
+			if !apiequality.Semantic.DeepEqual(cronjob, tc.expected) {
+				t.Errorf("expected:\n%#v\ngot:\n%#v", tc.expected, cronjob)
+			}
+		})
+	}
+}

pkg/kubectl/cmd/run/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
@@ -31,6 +32,7 @@ go_library(
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//staging/src/k8s.io/client-go/tools/watch:go_default_library",
         "//vendor/github.com/docker/distribution/reference:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",

pkg/kubectl/cmd/run/run.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/watch"
@@ -37,6 +38,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
 	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/kubernetes/pkg/kubectl"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/attach"
@@ -490,18 +492,41 @@ func (o *RunOptions) removeCreatedObjects(f cmdutil.Factory, createdObjects []*R
 
 // waitForPod watches the given pod until the exitCondition is true
 func waitForPod(podClient corev1client.PodsGetter, ns, name string, exitCondition watchtools.ConditionFunc) (*corev1.Pod, error) {
-	w, err := podClient.Pods(ns).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: name}))
-	if err != nil {
-		return nil, err
-	}
-
 	// TODO: expose the timeout
 	ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), 0*time.Second)
 	defer cancel()
+
+	preconditionFunc := func(store cache.Store) (bool, error) {
+		_, exists, err := store.Get(&metav1.ObjectMeta{Namespace: ns, Name: name})
+		if err != nil {
+			return true, err
+		}
+		if !exists {
+			// We need to make sure we see the object in the cache before we start waiting for events
+			// or we would be waiting for the timeout if such object didn't exist.
+			// (e.g. it was deleted before we started informers so they wouldn't even see the delete event)
+			return true, errors.NewNotFound(corev1.Resource("pods"), name)
+		}
+
+		return false, nil
+	}
+
+	fieldSelector := fields.OneTermEqualSelector("metadata.name", name).String()
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fieldSelector
+			return podClient.Pods(ns).List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			options.FieldSelector = fieldSelector
+			return podClient.Pods(ns).Watch(options)
+		},
+	}
+
 	intr := interrupt.New(nil, cancel)
 	var result *corev1.Pod
-	err = intr.Run(func() error {
-		ev, err := watchtools.UntilWithoutRetry(ctx, w, func(ev watch.Event) (bool, error) {
+	err := intr.Run(func() error {
+		ev, err := watchtools.UntilWithSync(ctx, lw, &corev1.Pod{}, preconditionFunc, func(ev watch.Event) (bool, error) {
 			return exitCondition(ev)
 		})
 		if ev != nil {
@@ -510,11 +535,6 @@ func waitForPod(podClient corev1client.PodsGetter, ns, name string, exitConditio
 		return err
 	})
 
-	// Fix generic not found error.
-	if err != nil && errors.IsNotFound(err) {
-		err = errors.NewNotFound(corev1.Resource("pods"), name)
-	}
-
 	return result, err
 }
 

pkg/kubelet/BUILD

@@ -140,7 +140,6 @@ go_library(
         "//staging/src/k8s.io/client-go/util/certificate:go_default_library",
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
-        "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
         "//third_party/forked/golang/expansion:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
         "//vendor/github.com/google/cadvisor/events:go_default_library",

pkg/kubelet/apis/BUILD

@@ -44,6 +44,7 @@ filegroup(
         "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
         "//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs",
         "//pkg/kubelet/apis/podresources:all-srcs",
+        "//pkg/kubelet/apis/resourcemetrics/v1alpha1:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],
     tags = ["automanaged"],

pkg/kubelet/apis/resourcemetrics/v1alpha1/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
+        "//pkg/kubelet/server/stats:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/kubelet/apis/resourcemetrics/v1alpha1/config.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"time"
+
+	summary "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	"k8s.io/kubernetes/pkg/kubelet/server/stats"
+)
+
+// Version is the string representation of the version of this configuration
+const Version = "v1alpha1"
+
+// Config is the v1alpha1 resource metrics definition
+func Config() stats.ResourceMetricsConfig {
+	return stats.ResourceMetricsConfig{
+		NodeMetrics: []stats.NodeResourceMetric{
+			{
+				Name:        "node_cpu_usage_seconds_total",
+				Description: "Cumulative cpu time consumed by the node in core-seconds",
+				ValueFn: func(s summary.NodeStats) (*float64, time.Time) {
+					if s.CPU == nil {
+						return nil, time.Time{}
+					}
+					v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second)
+					return &v, s.CPU.Time.Time
+				},
+			},
+			{
+				Name:        "node_memory_working_set_bytes",
+				Description: "Current working set of the node in bytes",
+				ValueFn: func(s summary.NodeStats) (*float64, time.Time) {
+					if s.Memory == nil {
+						return nil, time.Time{}
+					}
+					v := float64(*s.Memory.WorkingSetBytes)
+					return &v, s.Memory.Time.Time
+				},
+			},
+		},
+		ContainerMetrics: []stats.ContainerResourceMetric{
+			{
+				Name:        "container_cpu_usage_seconds_total",
+				Description: "Cumulative cpu time consumed by the container in core-seconds",
+				ValueFn: func(s summary.ContainerStats) (*float64, time.Time) {
+					if s.CPU == nil {
+						return nil, time.Time{}
+					}
+					v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second)
+					return &v, s.CPU.Time.Time
+				},
+			},
+			{
+				Name:        "container_memory_working_set_bytes",
+				Description: "Current working set of the container in bytes",
+				ValueFn: func(s summary.ContainerStats) (*float64, time.Time) {
+					if s.Memory == nil {
+						return nil, time.Time{}
+					}
+					v := float64(*s.Memory.WorkingSetBytes)
+					return &v, s.Memory.Time.Time
+				},
+			},
+		},
+	}
+}

pkg/kubelet/dockershim/docker_container.go

@@ -137,6 +137,9 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
 		},
 		HostConfig: &dockercontainer.HostConfig{
 			Binds: generateMountBindings(config.GetMounts()),
+			RestartPolicy: dockercontainer.RestartPolicy{
+				Name: "no",
+			},
 		},
 	}
 

pkg/kubelet/kubelet.go

@@ -113,7 +113,6 @@ import (
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/csi"
 	"k8s.io/kubernetes/pkg/volume/util/subpath"
-	nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
 	utilexec "k8s.io/utils/exec"
 	"k8s.io/utils/integer"
 )
@@ -248,7 +247,6 @@ type Dependencies struct {
 	HeartbeatClient         clientset.Interface
 	OnHeartbeatFailure      func()
 	KubeClient              clientset.Interface
-	NodeAPIClient           nodeapiclientset.Interface
 	Mounter                 mount.Interface
 	OOMAdjuster             *oom.OOMAdjuster
 	OSInterface             kubecontainer.OSInterface
@@ -658,8 +656,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	klet.runtimeService = runtimeService
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.NodeAPIClient != nil {
-		klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.NodeAPIClient)
+	if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
+		klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
 	}
 
 	runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
@@ -776,6 +774,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 
 	tokenManager := token.NewManager(kubeDeps.KubeClient)
 
+	// NewInitializedVolumePluginMgr intializes some storageErrors on the Kubelet runtimeState (in csi_plugin.go init)
+	// which affects node ready status. This function must be called before Kubelet is initialized so that the Node
+	// ReadyState is accurate with the storage state.
 	klet.volumePluginMgr, err =
 		NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber)
 	if err != nil {

pkg/kubelet/kubelet_node_status.go

@@ -544,7 +544,7 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
 		nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent),
 		nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent),
 		nodestatus.PIDPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderPIDPressure, kl.recordNodeStatusEvent),
-		nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
+		nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, kl.runtimeState.storageErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
 		nodestatus.VolumesInUse(kl.volumeManager.ReconcilerStatesHasBeenSynced, kl.volumeManager.GetVolumesInUse),
 		nodestatus.RemoveOutOfDiskCondition(),
 		// TODO(mtaufen): I decided not to move this setter for now, since all it does is send an event

pkg/kubelet/nodestatus/setters.go

@@ -440,6 +440,7 @@ func ReadyCondition(
 	nowFunc func() time.Time, // typically Kubelet.clock.Now
 	runtimeErrorsFunc func() error, // typically Kubelet.runtimeState.runtimeErrors
 	networkErrorsFunc func() error, // typically Kubelet.runtimeState.networkErrors
+	storageErrorsFunc func() error, // typically Kubelet.runtimeState.storageErrors
 	appArmorValidateHostFunc func() error, // typically Kubelet.appArmorValidator.ValidateHost, might be nil depending on whether there was an appArmorValidator
 	cmStatusFunc func() cm.Status, // typically Kubelet.containerManager.Status
 	recordEventFunc func(eventType, event string), // typically Kubelet.recordNodeStatusEvent
@@ -456,7 +457,7 @@ func ReadyCondition(
 			Message:           "kubelet is posting ready status",
 			LastHeartbeatTime: currentTime,
 		}
-		errs := []error{runtimeErrorsFunc(), networkErrorsFunc()}
+		errs := []error{runtimeErrorsFunc(), networkErrorsFunc(), storageErrorsFunc()}
 		requiredCapacities := []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, v1.ResourcePods}
 		if utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
 			requiredCapacities = append(requiredCapacities, v1.ResourceEphemeralStorage)

pkg/kubelet/nodestatus/setters_test.go

@@ -895,6 +895,7 @@ func TestReadyCondition(t *testing.T) {
 		node                     *v1.Node
 		runtimeErrors            error
 		networkErrors            error
+		storageErrors            error
 		appArmorValidateHostFunc func() error
 		cmStatus                 cm.Status
 		expectConditions         []v1.NodeCondition
@@ -930,6 +931,12 @@ func TestReadyCondition(t *testing.T) {
 			expectConditions: []v1.NodeCondition{*makeReadyCondition(true, "kubelet is posting ready status. WARNING: foo", now, now)},
 		},
 		{
+			desc:             "new, not ready: storage errors",
+			node:             withCapacity.DeepCopy(),
+			storageErrors:    errors.New("some storage error"),
+			expectConditions: []v1.NodeCondition{*makeReadyCondition(false, "some storage error", now, now)},
+		},
+		{
 			desc:             "new, not ready: runtime and network errors",
 			node:             withCapacity.DeepCopy(),
 			runtimeErrors:    errors.New("runtime"),
@@ -1003,6 +1010,9 @@ func TestReadyCondition(t *testing.T) {
 			networkErrorsFunc := func() error {
 				return tc.networkErrors
 			}
+			storageErrorsFunc := func() error {
+				return tc.storageErrors
+			}
 			cmStatusFunc := func() cm.Status {
 				return tc.cmStatus
 			}
@@ -1014,7 +1024,7 @@ func TestReadyCondition(t *testing.T) {
 				})
 			}
 			// construct setter
-			setter := ReadyCondition(nowFunc, runtimeErrorsFunc, networkErrorsFunc, tc.appArmorValidateHostFunc, cmStatusFunc, recordEventFunc)
+			setter := ReadyCondition(nowFunc, runtimeErrorsFunc, networkErrorsFunc, storageErrorsFunc, tc.appArmorValidateHostFunc, cmStatusFunc, recordEventFunc)
 			// call setter on node
 			if err := setter(tc.node); err != nil {
 				t.Fatalf("unexpected error: %v", err)

pkg/kubelet/runtime.go

@@ -30,6 +30,7 @@ type runtimeState struct {
 	lastBaseRuntimeSync      time.Time
 	baseRuntimeSyncThreshold time.Duration
 	networkError             error
+	storageError             error
 	cidr                     string
 	healthChecks             []*healthCheck
 }
@@ -61,6 +62,12 @@ func (s *runtimeState) setNetworkState(err error) {
 	s.networkError = err
 }
 
+func (s *runtimeState) setStorageState(err error) {
+	s.Lock()
+	defer s.Unlock()
+	s.storageError = err
+}
+
 func (s *runtimeState) setPodCIDR(cidr string) {
 	s.Lock()
 	defer s.Unlock()
@@ -101,6 +108,16 @@ func (s *runtimeState) networkErrors() error {
 	return utilerrors.NewAggregate(errs)
 }
 
+func (s *runtimeState) storageErrors() error {
+	s.RLock()
+	defer s.RUnlock()
+	errs := []error{}
+	if s.storageError != nil {
+		errs = append(errs, s.storageError)
+	}
+	return utilerrors.NewAggregate(errs)
+}
+
 func newRuntimeState(
 	runtimeSyncThreshold time.Duration,
 ) *runtimeState {

pkg/kubelet/runtimeclass/BUILD

@@ -7,10 +7,9 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
-        "//staging/src/k8s.io/node-api/pkg/client/informers/externalversions:go_default_library",
-        "//staging/src/k8s.io/node-api/pkg/client/listers/node/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/node/v1beta1:go_default_library",
     ],
 )
 

pkg/kubelet/runtimeclass/runtimeclass_manager.go

@@ -20,31 +20,23 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned"
-	nodeapiinformer "k8s.io/node-api/pkg/client/informers/externalversions"
-	nodev1alpha1 "k8s.io/node-api/pkg/client/listers/node/v1alpha1"
-)
-
-var (
-	runtimeClassGVR = schema.GroupVersionResource{
-		Group:    "node.k8s.io",
-		Version:  "v1alpha1",
-		Resource: "runtimeclasses",
-	}
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	nodev1beta1 "k8s.io/client-go/listers/node/v1beta1"
 )
 
 // Manager caches RuntimeClass API objects, and provides accessors to the Kubelet.
 type Manager struct {
-	informerFactory nodeapiinformer.SharedInformerFactory
-	lister          nodev1alpha1.RuntimeClassLister
+	informerFactory informers.SharedInformerFactory
+	lister          nodev1beta1.RuntimeClassLister
 }
 
 // NewManager returns a new RuntimeClass Manager. Run must be called before the manager can be used.
-func NewManager(client nodeapiclient.Interface) *Manager {
+func NewManager(client clientset.Interface) *Manager {
 	const resyncPeriod = 0
-	factory := nodeapiinformer.NewSharedInformerFactory(client, resyncPeriod)
-	lister := factory.Node().V1alpha1().RuntimeClasses().Lister()
+
+	factory := informers.NewSharedInformerFactory(client, resyncPeriod)
+	lister := factory.Node().V1beta1().RuntimeClasses().Lister()
 
 	return &Manager{
 		informerFactory: factory,
@@ -82,9 +74,5 @@ func (m *Manager) LookupRuntimeHandler(runtimeClassName *string) (string, error)
 		return "", fmt.Errorf("Failed to lookup RuntimeClass %s: %v", name, err)
 	}
 
-	handler := rc.Spec.RuntimeHandler
-	if handler == nil {
-		return "", nil
-	}
-	return *handler, nil
+	return rc.Handler, nil
 }

pkg/kubelet/runtimeclass/runtimeclass_manager_test.go

@@ -33,7 +33,6 @@ func TestLookupRuntimeHandler(t *testing.T) {
 		expected    string
 		expectError bool
 	}{
-		{rcn: nil, expected: ""},
 		{rcn: pointer.StringPtr(""), expected: ""},
 		{rcn: pointer.StringPtr(rctest.EmptyRuntimeClass), expected: ""},
 		{rcn: pointer.StringPtr(rctest.SandboxRuntimeClass), expected: "kata-containers"},