Merge pull request #14432 from mesosphere/sur-k8sm-492-launch-grace-period

contrib/mesos/pkg/executor/config/config.go

@@ -22,8 +22,9 @@ import (
 
 // default values to use when constructing mesos ExecutorInfo messages
 const (
-	DefaultInfoID         = "k8sm-executor"
-	DefaultInfoSource     = "kubernetes"
-	DefaultInfoName       = "Kubelet-Executor"
-	DefaultSuicideTimeout = 20 * time.Minute
+	DefaultInfoID            = "k8sm-executor"
+	DefaultInfoSource        = "kubernetes"
+	DefaultInfoName          = "Kubelet-Executor"
+	DefaultSuicideTimeout    = 20 * time.Minute
+	DefaultLaunchGracePeriod = 5 * time.Minute
 )

contrib/mesos/pkg/executor/executor.go

@@ -48,7 +48,6 @@ import (
 
 const (
 	containerPollTime = 1 * time.Second
-	launchGracePeriod = 5 * time.Minute
 	podRelistPeriod   = 5 * time.Minute
 )
 
@@ -124,6 +123,7 @@ type KubernetesExecutor struct {
 	staticPodsConfigPath string
 	initialRegComplete   chan struct{}
 	podController        *framework.Controller
+	launchGracePeriod    time.Duration
 }
 
 type Config struct {
@@ -139,6 +139,7 @@ type Config struct {
 	PodStatusFunc        func(KubeletInterface, *api.Pod) (*api.PodStatus, error)
 	StaticPodsConfigPath string
 	PodLW                cache.ListerWatcher
+	LaunchGracePeriod    time.Duration
 }
 
 func (k *KubernetesExecutor) isConnected() bool {
@@ -166,6 +167,7 @@ func New(config Config) *KubernetesExecutor {
 		podStatusFunc:        config.PodStatusFunc,
 		initialRegComplete:   make(chan struct{}),
 		staticPodsConfigPath: config.StaticPodsConfigPath,
+		launchGracePeriod:    config.LaunchGracePeriod,
 	}
 
 	// watch pods from the given pod ListWatch
@@ -599,7 +601,10 @@ func (k *KubernetesExecutor) launchTask(driver bindings.ExecutorDriver, taskId s
 func (k *KubernetesExecutor) _launchTask(driver bindings.ExecutorDriver, taskId, podFullName string, psf podStatusFunc) {
 
 	expired := make(chan struct{})
-	time.AfterFunc(launchGracePeriod, func() { close(expired) })
+
+	if k.launchGracePeriod > 0 {
+		time.AfterFunc(k.launchGracePeriod, func() { close(expired) })
+	}
 
 	getMarshalledInfo := func() (data []byte, cancel bool) {
 		// potentially long call..
@@ -638,7 +643,7 @@ waitForRunningPod:
 	for {
 		select {
 		case <-expired:
-			log.Warningf("Launch expired grace period of '%v'", launchGracePeriod)
+			log.Warningf("Launch expired grace period of '%v'", k.launchGracePeriod)
 			break waitForRunningPod
 		case <-time.After(containerPollTime):
 			if data, cancel := getMarshalledInfo(); cancel {

contrib/mesos/pkg/executor/service/service.go

@@ -58,13 +58,15 @@ const (
 
 type KubeletExecutorServer struct {
 	*app.KubeletServer
-	SuicideTimeout time.Duration
+	SuicideTimeout    time.Duration
+	LaunchGracePeriod time.Duration
 }
 
 func NewKubeletExecutorServer() *KubeletExecutorServer {
 	k := &KubeletExecutorServer{
-		KubeletServer:  app.NewKubeletServer(),
-		SuicideTimeout: config.DefaultSuicideTimeout,
+		KubeletServer:     app.NewKubeletServer(),
+		SuicideTimeout:    config.DefaultSuicideTimeout,
+		LaunchGracePeriod: config.DefaultLaunchGracePeriod,
 	}
 	if pwd, err := os.Getwd(); err != nil {
 		log.Warningf("failed to determine current directory: %v", err)
@@ -79,6 +81,7 @@ func NewKubeletExecutorServer() *KubeletExecutorServer {
 func (s *KubeletExecutorServer) AddFlags(fs *pflag.FlagSet) {
 	s.KubeletServer.AddFlags(fs)
 	fs.DurationVar(&s.SuicideTimeout, "suicide-timeout", s.SuicideTimeout, "Self-terminate after this period of inactivity. Zero disables suicide watch.")
+	fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.")
 }
 
 // Run runs the specified KubeletExecutorServer.
@@ -331,14 +334,15 @@ func (ks *KubeletExecutorServer) createAndInitKubelet(
 	kubeletFinished := make(chan struct{})
 	staticPodsConfigPath := filepath.Join(kc.RootDirectory, "static-pods")
 	exec := executor.New(executor.Config{
-		Kubelet:         klet,
-		Updates:         updates,
-		SourceName:      MESOS_CFG_SOURCE,
-		APIClient:       kc.KubeClient,
-		Docker:          kc.DockerClient,
-		SuicideTimeout:  ks.SuicideTimeout,
-		KubeletFinished: kubeletFinished,
-		ExitFunc:        os.Exit,
+		Kubelet:           klet,
+		Updates:           updates,
+		SourceName:        MESOS_CFG_SOURCE,
+		APIClient:         kc.KubeClient,
+		Docker:            kc.DockerClient,
+		SuicideTimeout:    ks.SuicideTimeout,
+		LaunchGracePeriod: ks.LaunchGracePeriod,
+		KubeletFinished:   kubeletFinished,
+		ExitFunc:          os.Exit,
 		PodStatusFunc: func(_ executor.KubeletInterface, pod *api.Pod) (*api.PodStatus, error) {
 			return klet.GetRuntime().GetPodStatus(pod)
 		},

contrib/mesos/pkg/scheduler/service/service.go

@@ -109,6 +109,7 @@ type SchedulerServer struct {
 	ExecutorLogV           int
 	ExecutorBindall        bool
 	ExecutorSuicideTimeout time.Duration
+	LaunchGracePeriod      time.Duration
 
 	RunProxy     bool
 	ProxyBindall bool
@@ -173,6 +174,7 @@ func NewSchedulerServer() *SchedulerServer {
 
 		RunProxy:                 true,
 		ExecutorSuicideTimeout:   execcfg.DefaultSuicideTimeout,
+		LaunchGracePeriod:        execcfg.DefaultLaunchGracePeriod,
 		DefaultContainerCPULimit: mresource.DefaultDefaultContainerCPULimit,
 		DefaultContainerMemLimit: mresource.DefaultDefaultContainerMemLimit,
 
@@ -255,6 +257,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.ExecutorLogV, "executor-logv", s.ExecutorLogV, "Logging verbosity of spawned minion and executor processes.")
 	fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.")
 	fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.")
+	fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.")
 
 	fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.")
 	fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.")
@@ -376,6 +379,7 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E
 	ci.Arguments = append(ci.Arguments, fmt.Sprintf("--v=%d", s.ExecutorLogV)) // this also applies to the minion
 	ci.Arguments = append(ci.Arguments, fmt.Sprintf("--allow-privileged=%t", s.AllowPrivileged))
 	ci.Arguments = append(ci.Arguments, fmt.Sprintf("--suicide-timeout=%v", s.ExecutorSuicideTimeout))
+	ci.Arguments = append(ci.Arguments, fmt.Sprintf("--mesos-launch-grace-period=%v", s.LaunchGracePeriod))
 
 	if s.ExecutorBindall {
 		//TODO(jdef) determine whether hostname-override is really needed for bindall because

hack/verify-flags/known-flags.txt

@@ -176,6 +176,7 @@ mesos-executor-mem
 mesos-master
 mesos-role
 mesos-user
+mesos-launch-grace-period
 minimum-container-ttl-duration
 minion-max-log-age
 minion-max-log-backups