Skip to content

K
k3s
Commit d11adfd3 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson
Browse files
Options

Use distinct clients for supervisor, deploy, and helm controllers

Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com> (cherry picked from commit 8748813a)
parent 9e37a9d0
Hide whitespace changes
Inline Side-by-side
Showing with 71 additions and 38 deletions
pkg/server/context.go
View file @ d11adfd3
......@@ -2,26 +2,20 @@ package server
import (
"context"
"fmt"
"os"
"runtime"
helmcrd "github.com/k3s-io/helm-controller/pkg/crd"
"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
addoncrd "github.com/k3s-io/k3s/pkg/crd"
"github.com/k3s-io/k3s/pkg/deploy"
"github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/pkg/errors"
"github.com/rancher/wrangler/pkg/apply"
"github.com/rancher/wrangler/pkg/crd"
"github.com/rancher/wrangler/pkg/generated/controllers/apps"
"github.com/rancher/wrangler/pkg/generated/controllers/batch"
"github.com/rancher/wrangler/pkg/generated/controllers/core"
"github.com/rancher/wrangler/pkg/generated/controllers/rbac"
"github.com/rancher/wrangler/pkg/start"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
......@@ -35,7 +29,6 @@ type Context struct {
Auth *rbac.Factory
Core *core.Factory
K8s kubernetes.Interface
Apply apply.Apply
}
func (c *Context) Start(ctx context.Context) error {
......@@ -47,16 +40,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) {
if err != nil {
return nil, err
}
// Construct a custom user-agent string for the apply client used by the deploy controller
// so that we can track which node's deploy controller most recently modified a resource.
nodeName := os.Getenv("NODE_NAME")
managerName := deploy.ControllerName + "@" + nodeName
if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength {
logrus.Warn("Deploy controller node name is empty or too long, and will not be tracked via server side apply field management")
managerName = deploy.ControllerName
}
restConfig.UserAgent = fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit)
restConfig.UserAgent = util.GetUserAgent(version.Program + "-supervisor")
if err := crds(ctx, restConfig); err != nil {
return nil, errors.Wrap(err, "failed to register CRDs")
......@@ -74,7 +58,6 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) {
Apps: apps.NewFactoryFromConfigOrDie(restConfig),
Batch: batch.NewFactoryFromConfigOrDie(restConfig),
Core: core.NewFactoryFromConfigOrDie(restConfig),
Apply: apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup(),
}, nil
}
......
pkg/server/server.go
View file @ d11adfd3
......@@ -12,7 +12,7 @@ import (
"sync"
"time"
helm "github.com/k3s-io/helm-controller/pkg/controllers/chart"
helmchart "github.com/k3s-io/helm-controller/pkg/controllers/chart"
helmcommon "github.com/k3s-io/helm-controller/pkg/controllers/common"
"github.com/k3s-io/k3s/pkg/cli/cmds"
"github.com/k3s-io/k3s/pkg/clientaccess"
......@@ -28,12 +28,15 @@ import (
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/pkg/errors"
"github.com/rancher/wrangler/pkg/apply"
v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
"github.com/rancher/wrangler/pkg/leader"
"github.com/rancher/wrangler/pkg/resolvehome"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
const (
......@@ -205,26 +208,42 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
// apply SystemDefaultRegistry setting to Helm before starting controllers
if config.ControlConfig.SystemDefaultRegistry != "" {
helm.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helm.DefaultJobImage
helmchart.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helmchart.DefaultJobImage
}
if !config.ControlConfig.DisableHelmController {
helm.Register(ctx,
restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return err
}
restConfig.UserAgent = util.GetUserAgent(helmcommon.Name)
k8s, err := clientset.NewForConfig(restConfig)
if err != nil {
return err
}
apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup()
helm := sc.Helm.WithAgent(restConfig.UserAgent)
batch := sc.Batch.WithAgent(restConfig.UserAgent)
auth := sc.Auth.WithAgent(restConfig.UserAgent)
core := sc.Core.WithAgent(restConfig.UserAgent)
helmchart.Register(ctx,
metav1.NamespaceAll,
helmcommon.Name,
sc.K8s,
sc.Apply,
util.BuildControllerEventRecorder(sc.K8s, helmcommon.Name, metav1.NamespaceAll),
sc.Helm.Helm().V1().HelmChart(),
sc.Helm.Helm().V1().HelmChart().Cache(),
sc.Helm.Helm().V1().HelmChartConfig(),
sc.Helm.Helm().V1().HelmChartConfig().Cache(),
sc.Batch.Batch().V1().Job(),
sc.Batch.Batch().V1().Job().Cache(),
sc.Auth.Rbac().V1().ClusterRoleBinding(),
sc.Core.Core().V1().ServiceAccount(),
sc.Core.Core().V1().ConfigMap(),
sc.Core.Core().V1().Secret())
k8s,
apply,
util.BuildControllerEventRecorder(k8s, helmcommon.Name, metav1.NamespaceAll),
helm.V1().HelmChart(),
helm.V1().HelmChart().Cache(),
helm.V1().HelmChartConfig(),
helm.V1().HelmChartConfig().Cache(),
batch.V1().Job(),
batch.V1().Job().Cache(),
auth.V1().ClusterRoleBinding(),
core.V1().ServiceAccount(),
core.V1().ConfigMap(),
core.V1().Secret())
}
if config.ControlConfig.EncryptSecrets {
......@@ -274,10 +293,24 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
return err
}
restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return err
}
restConfig.UserAgent = util.GetUserAgent("deploy")
k8s, err := clientset.NewForConfig(restConfig)
if err != nil {
return err
}
apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup()
k3s := sc.K3s.WithAgent(restConfig.UserAgent)
return deploy.WatchFiles(ctx,
sc.K8s,
sc.Apply,
sc.K3s.K3s().V1().Addon(),
k8s,
apply,
k3s.V1().Addon(),
controlConfig.Disables,
dataDir)
}
......
pkg/util/client.go
View file @ d11adfd3
package util
import (
"fmt"
"os"
"runtime"
"strings"
"github.com/k3s-io/k3s/pkg/datadir"
"github.com/k3s-io/k3s/pkg/version"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
......@@ -30,6 +36,17 @@ func GetClientSet(file string) (clientset.Interface, error) {
return clientset.NewForConfig(restConfig)
}
// GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible.
func GetUserAgent(controllerName string) string {
nodeName := os.Getenv("NODE_NAME")
managerName := controllerName + "@" + nodeName
if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength {
logrus.Warnf("%s controller node name is empty or too long, and will not be tracked via server side apply field management", controllerName)
managerName = controllerName
}
return fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit)
}
// SplitStringSlice is a helper function to handle StringSliceFlag containing multiple values
// By default, StringSliceFlag only supports repeated values, not multiple values
// e.g. --foo="bar,car" --foo=baz will result in []string{"bar", "car". "baz"}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment