Wait for apiserver readyz instead of healthz

pkg/agent/run.go

@@ -37,7 +37,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/controller-manager/app"
 	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
 	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
 	utilsnet "k8s.io/utils/net"
@@ -128,7 +127,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		return err
 	}
 
-	app.WaitForAPIServer(coreClient, 30*time.Second)
+	util.WaitForAPIServerReady(coreClient, 30*time.Second)
 
 	if !nodeConfig.NoFlannel {
 		if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {

pkg/daemons/control/server.go

@@ -16,7 +16,7 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/control/deps"
 	"github.com/rancher/k3s/pkg/daemons/executor"
-	util2 "github.com/rancher/k3s/pkg/util"
+	"github.com/rancher/k3s/pkg/util"
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
@@ -26,9 +26,8 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	app2 "k8s.io/controller-manager/app"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
-	"k8s.io/kubernetes/pkg/proxy/util"
+	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
 
 	// for client metric registration
 	_ "k8s.io/component-base/metrics/prometheus/restclient"
@@ -47,7 +46,7 @@ func Server(ctx context.Context, cfg *config.Control) error {
 	}
 
 	cfg.Runtime.Tunnel = setupTunnel()
-	util.DisableProxyHostnameCheck = true
+	proxyutil.DisableProxyHostnameCheck = true
 
 	var auth authenticator.Request
 	var handler http.Handler
@@ -96,7 +95,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) erro
 		"kubeconfig":                       runtime.KubeConfigController,
 		"service-account-private-key-file": runtime.ServiceKey,
 		"allocate-node-cidrs":              "true",
-		"cluster-cidr":                     util2.JoinIPNets(cfg.ClusterIPRanges),
+		"cluster-cidr":                     util.JoinIPNets(cfg.ClusterIPRanges),
 		"root-ca-file":                     runtime.ServerCA,
 		"port":                             "10252",
 		"profiling":                        "false",
@@ -157,7 +156,7 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
 	argsMap["allow-privileged"] = "true"
 	argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
 	argsMap["service-account-signing-key-file"] = runtime.ServiceKey
-	argsMap["service-cluster-ip-range"] = util2.JoinIPNets(cfg.ServiceIPRanges)
+	argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges)
 	argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String()
 	argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
 	if cfg.AdvertiseIP != "" {
@@ -345,14 +344,14 @@ func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
 func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) error {
 	argsMap := map[string]string{
 		"profiling":                    "false",
-		"allocate-node-cidrs":          "true",                                // ccmOptions.KubeCloudShared.AllocateNodeCIDRs = true
-		"cloud-provider":               version.Program,                       // ccmOptions.KubeCloudShared.CloudProvider.Name = version.Program
-		"cluster-cidr":                 util2.JoinIPNets(cfg.ClusterIPRanges), // ccmOptions.KubeCloudShared.ClusterCIDR = util2.JoinIPNets(cfg.ClusterIPRanges)
-		"configure-cloud-routes":       "false",                               // ccmOptions.KubeCloudShared.ConfigureCloudRoutes = false
-		"kubeconfig":                   runtime.KubeConfigCloudController,     // ccmOptions.Kubeconfig = runtime.KubeConfigCloudController
-		"node-status-update-frequency": "1m0s",                                // ccmOptions.NodeStatusUpdateFrequency = metav1.Duration{Duration: 1 * time.Minute}
-		"bind-address":                 "127.0.0.1",                           // ccmOptions.SecureServing.BindAddress = localhostIP
-		"port":                         "0",                                   // ccmOptions.SecureServing.BindPort = 0
+		"allocate-node-cidrs":          "true",
+		"cloud-provider":               version.Program,
+		"cluster-cidr":                 util.JoinIPNets(cfg.ClusterIPRanges),
+		"configure-cloud-routes":       "false",
+		"kubeconfig":                   runtime.KubeConfigCloudController,
+		"node-status-update-frequency": "1m0s",
+		"bind-address":                 "127.0.0.1",
+		"port":                         "0",
 	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
@@ -453,7 +452,7 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu
 			select {
 			case <-ctx.Done():
 				return
-			case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }):
+			case err := <-promise(func() error { return util.WaitForAPIServerReady(k8sClient, 30*time.Second) }):
 				if err != nil {
 					logrus.Infof("Waiting for API server to become available")
 					continue

pkg/util/api.go

 package util
 
 import (
+	"context"
+	"fmt"
 	"net"
+	"net/http"
 	"strconv"
+	"time"
 
+	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 )
 
 func GetAddresses(endpoint *v1.Endpoints) []string {
@@ -26,3 +33,32 @@ func GetAddresses(endpoint *v1.Endpoints) []string {
 	}
 	return serverAddresses
 }
+
+// WaitForAPIServerReady waits for the API Server's /readyz endpoint to report "ok" with timeout.
+// This is cribbed from the Kubernetes controller-manager app, but checks the readyz endpoint instead of the deprecated healthz endpoint.
+func WaitForAPIServerReady(client clientset.Interface, timeout time.Duration) error {
+	var lastErr error
+
+	err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
+		healthStatus := 0
+		result := client.Discovery().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&healthStatus)
+		if result.Error() != nil {
+			lastErr = fmt.Errorf("failed to get apiserver /readyz status: %v", result.Error())
+			return false, nil
+		}
+		if healthStatus != http.StatusOK {
+			content, _ := result.Raw()
+			lastErr = fmt.Errorf("APIServer isn't ready: %v", string(content))
+			logrus.Warnf("APIServer isn't ready yet: %v. Waiting a little while.", string(content))
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("%v: %v", err, lastErr)
+	}
+
+	return nil
+}