Skip to content

K
k3s
Commit 7fbd2901 authored by James DeFelice's avatar James DeFelice
Browse files
Options

moved cgroup-root detection to minion service; kube proxy now configured to run in mesos container

parent 66ff3c13
Hide whitespace changes
Inline Side-by-side
Showing with 46 additions and 42 deletions
contrib/mesos/pkg/executor/config/config.go
View file @ 7fbd2901
......@@ -26,5 +26,4 @@ const (
DefaultInfoSource = "kubernetes"
DefaultInfoName = "Kubelet-Executor"
DefaultSuicideTimeout = 20 * time.Minute
DefaultCgroupPrefix = "mesos"
)
contrib/mesos/pkg/executor/service/service.go
View file @ 7fbd2901
......@@ -17,13 +17,11 @@ limitations under the License.
package service
import (
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
......@@ -63,33 +61,12 @@ type KubeletExecutorServer struct {
SuicideTimeout time.Duration
ShutdownFD int
ShutdownFIFO string
cgroupRoot string
cgroupPrefix string
}
func findMesosCgroup(prefix string) string {
// derive our cgroup from MESOS_DIRECTORY environment
mesosDir := os.Getenv("MESOS_DIRECTORY")
if mesosDir == "" {
log.V(2).Infof("cannot derive executor's cgroup because MESOS_DIRECTORY is empty")
return ""
}
containerId := path.Base(mesosDir)
if containerId == "" {
log.V(2).Infof("cannot derive executor's cgroup from MESOS_DIRECTORY=%q", mesosDir)
return ""
}
trimmedPrefix := strings.Trim(prefix, "/")
cgroupRoot := fmt.Sprintf("/%s/%v", trimmedPrefix, containerId)
return cgroupRoot
}
func NewKubeletExecutorServer() *KubeletExecutorServer {
k := &KubeletExecutorServer{
KubeletServer: app.NewKubeletServer(),
SuicideTimeout: config.DefaultSuicideTimeout,
cgroupPrefix: config.DefaultCgroupPrefix,
}
if pwd, err := os.Getwd(); err != nil {
log.Warningf("failed to determine current directory: %v", err)
......@@ -107,7 +84,6 @@ func (s *KubeletExecutorServer) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.SuicideTimeout, "suicide-timeout", s.SuicideTimeout, "Self-terminate after this period of inactivity. Zero disables suicide watch.")
fs.IntVar(&s.ShutdownFD, "shutdown-fd", s.ShutdownFD, "File descriptor used to signal shutdown to external watchers, requires shutdown-fifo flag")
fs.StringVar(&s.ShutdownFIFO, "shutdown-fifo", s.ShutdownFIFO, "FIFO used to signal shutdown to external watchers, requires shutdown-fd flag")
fs.StringVar(&s.cgroupPrefix, "cgroup-prefix", s.cgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos")
}
// returns a Closer that should be closed to signal impending shutdown, but only if ShutdownFD
......@@ -131,21 +107,11 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
log.Info(err)
}
// derive the executor cgroup and use it as docker container cgroup root
mesosCgroup := findMesosCgroup(s.cgroupPrefix)
s.cgroupRoot = mesosCgroup
log.V(2).Infof("passing cgroup %q to the kubelet as cgroup root", s.CgroupRoot)
// empty string for the docker and system containers (= cgroup paths). This
// stops the kubelet taking any control over other system processes.
s.SystemContainer = ""
s.DockerDaemonContainer = ""
// We set kubelet container to its own cgroup below the executor cgroup.
// In contrast to the docker and system container, this has no other
// undesired side-effects.
s.ResourceContainer = mesosCgroup + "/kubelet"
// create apiserver client
var apiclient *client.Client
clientConfig, err := s.CreateAPIServerClientConfig()
......@@ -253,7 +219,7 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
Cloud: nil, // TODO(jdef) Cloud, specifying null here because we don't want all kubelets polling mesos-master; need to account for this in the cloudprovider impl
NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
ResourceContainer: s.ResourceContainer,
CgroupRoot: s.cgroupRoot,
CgroupRoot: s.CgroupRoot,
ContainerRuntime: s.ContainerRuntime,
Mounter: mounter,
DockerDaemonContainer: s.DockerDaemonContainer,
......
contrib/mesos/pkg/minion/config/config.go
View file @ 7fbd2901
......@@ -23,6 +23,8 @@ import (
const (
DefaultLogMaxBackups = 5 // how many backup to keep
DefaultLogMaxAgeInDays = 7 // after how many days to rotate at most
DefaultCgroupPrefix = "mesos"
)
// DefaultLogMaxSize returns the maximal log file size before rotation
......
contrib/mesos/pkg/minion/server.go
View file @ 7fbd2901
......@@ -22,6 +22,7 @@ import (
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"time"
......@@ -51,6 +52,8 @@ type MinionServer struct {
exit chan error // to signal fatal errors
pathOverride string // the PATH environment for the sub-processes
cgroupPrefix string // e.g. mesos
cgroupRoot string // e.g. /mesos/{container-id}, determined at runtime
logMaxSize resource.Quantity
logMaxBackups int
......@@ -69,6 +72,7 @@ func NewMinionServer() *MinionServer {
done: make(chan struct{}),
exit: make(chan error),
cgroupPrefix: config.DefaultCgroupPrefix,
logMaxSize: config.DefaultLogMaxSize(),
logMaxBackups: config.DefaultLogMaxBackups,
logMaxAgeInDays: config.DefaultLogMaxAgeInDays,
......@@ -102,6 +106,22 @@ func filterArgsByFlagSet(args []string, flags *pflag.FlagSet) ([]string, []strin
return matched, notMatched
}
func findMesosCgroup(prefix string) string {
// derive our cgroup from MESOS_DIRECTORY environment
mesosDir := os.Getenv("MESOS_DIRECTORY")
if mesosDir == "" {
log.V(2).Infof("cannot derive executor's cgroup because MESOS_DIRECTORY is empty")
return ""
}
containerId := path.Base(mesosDir)
if containerId == "" {
log.V(2).Infof("cannot derive executor's cgroup from MESOS_DIRECTORY=%q", mesosDir)
return ""
}
return path.Join("/", prefix, containerId)
}
func (ms *MinionServer) launchProxyServer() {
bindAddress := "0.0.0.0"
if !ms.proxyBindall {
......@@ -111,6 +131,7 @@ func (ms *MinionServer) launchProxyServer() {
fmt.Sprintf("--bind-address=%s", bindAddress),
fmt.Sprintf("--v=%d", ms.proxyLogV),
"--logtostderr=true",
"--resource-container=" + path.Join("/", ms.cgroupRoot, "kube-proxy"),
}
if ms.clientConfig.Host != "" {
......@@ -132,6 +153,11 @@ func (ms *MinionServer) launchExecutorServer() {
ms.AddExecutorFlags(executorFlags)
executorArgs, _ := filterArgsByFlagSet(allArgs, executorFlags)
executorArgs = append(executorArgs, "--resource-container="+path.Join("/", ms.cgroupRoot, "kubelet"))
if ms.cgroupRoot != "" {
executorArgs = append(executorArgs, "--cgroup-root="+ms.cgroupRoot)
}
// run executor and quit minion server when this exits cleanly
err := ms.launchHyperkubeServer(hyperkube.CommandExecutor, &executorArgs, "executor.log")
if err != nil {
......@@ -257,6 +283,17 @@ func (ms *MinionServer) Run(hks hyperkube.Interface, _ []string) error {
}
ms.clientConfig = clientConfig
// derive the executor cgroup and use it as:
// - pod container cgroup root (e.g. docker cgroup-parent)
// - parent of kubelet container
// - parent of kube-proxy container
ms.cgroupRoot = findMesosCgroup(ms.cgroupPrefix)
cgroupLogger := log.Infof
if ms.cgroupRoot == "" {
cgroupLogger = log.Warningf
}
cgroupLogger("using cgroup-root %q", ms.cgroupRoot)
// run subprocesses until ms.done is closed on return of this function
defer close(ms.done)
if ms.runProxy {
......@@ -275,6 +312,7 @@ func (ms *MinionServer) AddExecutorFlags(fs *pflag.FlagSet) {
func (ms *MinionServer) AddMinionFlags(fs *pflag.FlagSet) {
// general minion flags
fs.StringVar(&ms.cgroupPrefix, "mesos-cgroup-prefix", ms.cgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos")
fs.BoolVar(&ms.privateMountNS, "private-mountns", ms.privateMountNS, "Enter a private mount NS before spawning procs (linux only). Experimental, not yet compatible with k8s volumes.")
fs.StringVar(&ms.pathOverride, "path-override", ms.pathOverride, "Override the PATH in the environment of the sub-processes.")
......
contrib/mesos/pkg/scheduler/service/service.go
View file @ 7fbd2901
......@@ -95,13 +95,13 @@ type SchedulerServer struct {
MesosRole string
MesosAuthPrincipal string
MesosAuthSecretFile string
MesosCgroupPrefix string
Checkpoint bool
FailoverTimeout float64
ExecutorLogV int
ExecutorBindall bool
ExecutorSuicideTimeout time.Duration
ExecutorCgroupPrefix string
RunProxy bool
ProxyBindall bool
......@@ -163,7 +163,6 @@ func NewSchedulerServer() *SchedulerServer {
RunProxy: true,
ExecutorSuicideTimeout: execcfg.DefaultSuicideTimeout,
ExecutorCgroupPrefix: execcfg.DefaultCgroupPrefix,
DefaultContainerCPULimit: mresource.DefaultDefaultContainerCPULimit,
DefaultContainerMemLimit: mresource.DefaultDefaultContainerMemLimit,
......@@ -172,6 +171,7 @@ func NewSchedulerServer() *SchedulerServer {
MinionLogMaxAgeInDays: minioncfg.DefaultLogMaxAgeInDays,
MesosAuthProvider: sasl.ProviderName,
MesosCgroupPrefix: minioncfg.DefaultCgroupPrefix,
MesosMaster: defaultMesosMaster,
MesosUser: defaultMesosUser,
ReconcileInterval: defaultReconcileInterval,
......@@ -215,6 +215,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.MesosAuthSecretFile, "mesos-authentication-secret-file", s.MesosAuthSecretFile, "Mesos authentication secret file.")
fs.StringVar(&s.MesosAuthProvider, "mesos-authentication-provider", s.MesosAuthProvider, fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
fs.StringVar(&s.DockerCfgPath, "dockercfg-path", s.DockerCfgPath, "Path to a dockercfg file that will be used by the docker instance of the minions.")
fs.StringVar(&s.MesosCgroupPrefix, "mesos-cgroup-prefix", s.MesosCgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos")
fs.BoolVar(&s.Checkpoint, "checkpoint", s.Checkpoint, "Enable/disable checkpointing for the kubernetes-mesos framework.")
fs.Float64Var(&s.FailoverTimeout, "failover-timeout", s.FailoverTimeout, fmt.Sprintf("Framework failover timeout, in sec."))
fs.UintVar(&s.DriverPort, "driver-port", s.DriverPort, "Port that the Mesos scheduler driver process should listen on.")
......@@ -234,7 +235,6 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) {
fs.IntVar(&s.ExecutorLogV, "executor-logv", s.ExecutorLogV, "Logging verbosity of spawned minion and executor processes.")
fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.")
fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.")
fs.StringVar(&s.ExecutorCgroupPrefix, "executor-cgroup-prefix", s.ExecutorCgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos")
fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.")
fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.")
......@@ -364,7 +364,7 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E
ci.Arguments = append(ci.Arguments, "--address=0.0.0.0")
}
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cgroup-prefix=%v", s.ExecutorCgroupPrefix))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--mesos-cgroup-prefix=%v", s.MesosCgroupPrefix))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.KubeletCadvisorPort))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.KubeletSyncFrequency))
......
hack/verify-flags/known-flags.txt
View file @ 7fbd2901
......@@ -28,7 +28,6 @@ bind-pods-qps
cadvisor-port
cert-dir
certificate-authority
cgroup-prefix
cgroup-root
chaos-chance
cleanup-iptables
......@@ -74,7 +73,6 @@ etcd-server
etcd-servers
event-ttl
executor-bindall
executor-cgroup-prefix
executor-logv
executor-path
executor-suicide-timeout
......@@ -152,6 +150,7 @@ max-requests-inflight
mesos-authentication-principal
mesos-authentication-provider
mesos-authentication-secret-file
mesos-cgroup-prefix
mesos-master
mesos-role
mesos-user
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment