Move error response generation code into util

pkg/daemons/control/tunnel.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/proxy"
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -22,10 +21,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/yl2chen/cidranger"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/client-go/kubernetes"
 )
@@ -173,29 +168,20 @@ func (t *TunnelServer) onChangePod(podName string, pod *v1.Pod) (*v1.Pod, error)
 func (t *TunnelServer) serveConnect(resp http.ResponseWriter, req *http.Request) {
 	bconn, err := t.dialBackend(req.Context(), req.Host)
 	if err != nil {
-		responsewriters.ErrorNegotiated(
-			newBadGateway(err.Error()),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(err, resp, req, http.StatusBadGateway)
 		return
 	}
 
 	hijacker, ok := resp.(http.Hijacker)
 	if !ok {
-		responsewriters.ErrorNegotiated(
-			apierrors.NewInternalError(errors.New("hijacking not supported")),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(errors.New("hijacking not supported"), resp, req, http.StatusInternalServerError)
 		return
 	}
 	resp.WriteHeader(http.StatusOK)
 
 	rconn, bufrw, err := hijacker.Hijack()
 	if err != nil {
-		responsewriters.ErrorNegotiated(
-			apierrors.NewInternalError(err),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(err, resp, req, http.StatusInternalServerError)
 		return
 	}
 
@@ -301,14 +287,3 @@ func (crw *connReadWriteCloser) Close() (err error) {
 	crw.once.Do(func() { err = crw.conn.Close() })
 	return
 }
-
-func newBadGateway(message string) *apierrors.StatusError {
-	return &apierrors.StatusError{
-		ErrStatus: metav1.Status{
-			Status:  metav1.StatusFailure,
-			Code:    http.StatusBadGateway,
-			Reason:  metav1.StatusReasonInternalError,
-			Message: message,
-		},
-	}
-}

pkg/server/auth.go

@@ -5,12 +5,9 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
@@ -29,23 +26,23 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw
 	switch {
 	case serverConfig == nil:
 		logrus.Errorf("Authenticate not initialized: serverConfig is nil")
-		unauthorized(rw, req)
+		util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized)
 		return
 	case serverConfig.Runtime.Authenticator == nil:
 		logrus.Errorf("Authenticate not initialized: serverConfig.Runtime.Authenticator is nil")
-		unauthorized(rw, req)
+		util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized)
 		return
 	}
 
 	resp, ok, err := serverConfig.Runtime.Authenticator.AuthenticateRequest(req)
 	if err != nil {
 		logrus.Errorf("Failed to authenticate request from %s: %v", req.RemoteAddr, err)
-		unauthorized(rw, req)
+		util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized)
 		return
 	}
 
 	if !ok || !hasRole(roles, resp.User.GetGroups()) {
-		forbidden(rw, req)
+		util.SendError(errors.New("forbidden"), rw, req, http.StatusForbidden)
 		return
 	}
 
@@ -61,27 +58,3 @@ func authMiddleware(serverConfig *config.Control, roles ...string) mux.Middlewar
 		})
 	}
 }
-
-func unauthorized(resp http.ResponseWriter, req *http.Request) {
-	responsewriters.ErrorNegotiated(
-		&apierrors.StatusError{ErrStatus: metav1.Status{
-			Status:  metav1.StatusFailure,
-			Code:    http.StatusUnauthorized,
-			Reason:  metav1.StatusReasonUnauthorized,
-			Message: "not authorized",
-		}},
-		scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-	)
-}
-
-func forbidden(resp http.ResponseWriter, req *http.Request) {
-	responsewriters.ErrorNegotiated(
-		&apierrors.StatusError{ErrStatus: metav1.Status{
-			Status:  metav1.StatusFailure,
-			Code:    http.StatusForbidden,
-			Reason:  metav1.StatusReasonForbidden,
-			Message: "forbidden",
-		}},
-		scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-	)
-}

pkg/server/router.go

@@ -19,7 +19,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
 	"github.com/k3s-io/k3s/pkg/nodepassword"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -28,14 +27,11 @@ import (
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
@@ -108,20 +104,14 @@ func apiserver(runtime *config.ControlRuntime) http.Handler {
 		if runtime != nil && runtime.APIServer != nil {
 			runtime.APIServer.ServeHTTP(resp, req)
 		} else {
-			responsewriters.ErrorNegotiated(
-				apierrors.NewServiceUnavailable("apiserver not ready"),
-				scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-			)
+			util.SendError(util.ErrNotReady, resp, req, http.StatusServiceUnavailable)
 		}
 	})
 }
 
 func apiserverDisabled() http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		responsewriters.ErrorNegotiated(
-			apierrors.NewServiceUnavailable("apiserver disabled"),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(errors.New("apiserver disabled"), resp, req, http.StatusServiceUnavailable)
 	})
 }
 
@@ -131,10 +121,7 @@ func bootstrapHandler(runtime *config.ControlRuntime) http.Handler {
 	}
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)
-		responsewriters.ErrorNegotiated(
-			apierrors.NewBadRequest("etcd disabled"),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(errors.New("etcd disabled"), resp, req, http.StatusBadRequest)
 	})
 }
 
@@ -145,7 +132,7 @@ func cacerts(serverCA string) http.Handler {
 			var err error
 			ca, err = os.ReadFile(serverCA)
 			if err != nil {
-				sendError(err, resp, req)
+				util.SendError(err, resp, req)
 				return
 			}
 		}
@@ -220,13 +207,13 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo
 
 		nodeName, errCode, err := auth(req)
 		if err != nil {
-			sendError(err, resp, req, errCode)
+			util.SendError(err, resp, req, errCode)
 			return
 		}
 
 		caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
 		if err != nil {
-			sendError(err, resp, req)
+			util.SendError(err, resp, req)
 			return
 		}
 
@@ -236,7 +223,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo
 			for _, v := range strings.Split(nodeIP, ",") {
 				ip := net.ParseIP(v)
 				if ip == nil {
-					sendError(fmt.Errorf("invalid node IP address %s", ip), resp, req)
+					util.SendError(fmt.Errorf("invalid node IP address %s", ip), resp, req)
 					return
 				}
 				ips = append(ips, ip)
@@ -252,7 +239,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo
 			},
 		}, key, caCerts[0], caKey)
 		if err != nil {
-			sendError(err, resp, req)
+			util.SendError(err, resp, req)
 			return
 		}
 
@@ -276,13 +263,13 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot
 
 		nodeName, errCode, err := auth(req)
 		if err != nil {
-			sendError(err, resp, req, errCode)
+			util.SendError(err, resp, req, errCode)
 			return
 		}
 
 		caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
 		if err != nil {
-			sendError(err, resp, req)
+			util.SendError(err, resp, req)
 			return
 		}
 
@@ -292,7 +279,7 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot
 			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
 		}, key, caCerts[0], caKey)
 		if err != nil {
-			sendError(err, resp, req)
+			util.SendError(err, resp, req)
 			return
 		}
 
@@ -402,21 +389,6 @@ func serveStatic(urlPrefix, staticDir string) http.Handler {
 	return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir)))
 }
 
-func sendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) {
-	var code int
-	if len(status) == 1 {
-		code = status[0]
-	}
-	if code == 0 || code == http.StatusOK {
-		code = http.StatusInternalServerError
-	}
-	logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err)
-	responsewriters.ErrorNegotiated(
-		apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true),
-		scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-	)
-}
-
 // nodePassBootstrapper returns a node name, or http error code and error
 type nodePassBootstrapper func(req *http.Request) (string, int, error)
 

pkg/util/apierrors.go

+package util
+
+import (
+	"net/http"
+
+	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+var ErrNotReady = errors.New("apiserver not ready")
+
+// SendError sends a properly formatted error response
+func SendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) {
+	var code int
+	if len(status) == 1 {
+		code = status[0]
+	}
+	if code == 0 || code == http.StatusOK {
+		code = http.StatusInternalServerError
+	}
+
+	// Don't log "apiserver not ready" errors, they are frequent during startup
+	if !errors.Is(err, ErrNotReady) {
+		logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err)
+	}
+
+	var serr *apierrors.StatusError
+	switch code {
+	case http.StatusBadRequest:
+		serr = apierrors.NewBadRequest(err.Error())
+	case http.StatusUnauthorized:
+		serr = apierrors.NewUnauthorized(err.Error())
+	case http.StatusForbidden:
+		serr = newForbidden(err)
+	case http.StatusInternalServerError:
+		serr = apierrors.NewInternalError(err)
+	case http.StatusBadGateway:
+		serr = newBadGateway(err)
+	case http.StatusServiceUnavailable:
+		serr = apierrors.NewServiceUnavailable(err.Error())
+	default:
+		serr = apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true)
+	}
+
+	responsewriters.ErrorNegotiated(serr, scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req)
+}
+
+func newForbidden(err error) *apierrors.StatusError {
+	return &apierrors.StatusError{
+		ErrStatus: metav1.Status{
+			Status:  metav1.StatusFailure,
+			Code:    http.StatusForbidden,
+			Reason:  metav1.StatusReasonForbidden,
+			Message: err.Error(),
+		}}
+}
+
+func newBadGateway(err error) *apierrors.StatusError {
+	return &apierrors.StatusError{
+		ErrStatus: metav1.Status{
+			Status:  metav1.StatusFailure,
+			Code:    http.StatusBadGateway,
+			Reason:  metav1.StatusReasonInternalError,
+			Message: err.Error(),
+		}}
+}