Commit e2510617 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson

Ensure that apiserver ready channel checks re-dial every time

Closing idle connections isn't guaranteed to close out a pooled connection to a loadbalancer endpoint that has been removed. Instead, ensure that requests used to wait for the apiserver to become ready aren't reused. Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com>
parent e373d42a
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"context" "context"
"math/rand" "math/rand"
"net" "net"
"net/http"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
...@@ -33,6 +34,12 @@ import ( ...@@ -33,6 +34,12 @@ import (
var localhostIP = net.ParseIP("127.0.0.1") var localhostIP = net.ParseIP("127.0.0.1")
type roundTripFunc func(req *http.Request) (*http.Response, error)
func (w roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return w(req)
}
func Server(ctx context.Context, cfg *config.Control) error { func Server(ctx context.Context, cfg *config.Control) error {
rand.Seed(time.Now().UTC().UnixNano()) rand.Seed(time.Now().UTC().UnixNano())
...@@ -432,6 +439,16 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu ...@@ -432,6 +439,16 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu
return err return err
} }
// By default, idle connections to the apiserver are returned to a global pool
// between requests. Explicitly flag this client's request for closure so that
// we re-dial through the loadbalancer in case the endpoints have changed.
restConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
return roundTripFunc(func(req *http.Request) (*http.Response, error) {
req.Close = true
return rt.RoundTrip(req)
})
})
k8sClient, err := kubernetes.NewForConfig(restConfig) k8sClient, err := kubernetes.NewForConfig(restConfig)
if err != nil { if err != nil {
return err return err
......
...@@ -18,7 +18,6 @@ import ( ...@@ -18,7 +18,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
coregetter "k8s.io/client-go/kubernetes/typed/core/v1" coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
) )
...@@ -57,10 +56,6 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time ...@@ -57,10 +56,6 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time
err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
healthStatus := 0 healthStatus := 0
// Idle connections to the apiserver are returned to a global pool between requests. Explicitly
// close these idle connections so that we re-connect through the loadbalancer in case the endpoints
// have changed.
restClient.(*rest.RESTClient).Client.CloseIdleConnections()
result := restClient.Get().AbsPath("/readyz").Do(ctx).StatusCode(&healthStatus) result := restClient.Get().AbsPath("/readyz").Do(ctx).StatusCode(&healthStatus)
if rerr := result.Error(); rerr != nil { if rerr := result.Error(); rerr != nil {
if errors.Is(rerr, context.Canceled) { if errors.Is(rerr, context.Canceled) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment