Merge pull request #13036 from mesosphere/decouple-executor-and-kubelet

contrib/mesos/pkg/executor/mock_test.go

@@ -22,7 +22,9 @@ import (
 	"github.com/mesos/mesos-go/mesosproto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 type MockExecutorDriver struct {
@@ -64,16 +66,18 @@ func (m *MockExecutorDriver) SendFrameworkMessage(msg string) (mesosproto.Status
 	return args.Get(0).(mesosproto.Status), args.Error(1)
 }
 
-func NewTestKubernetesExecutor() *KubernetesExecutor {
+func NewTestKubernetesExecutor() (*KubernetesExecutor, chan kubetypes.PodUpdate) {
+	updates := make(chan kubetypes.PodUpdate, 1024)
 	return New(Config{
 		Docker:  dockertools.ConnectToDockerOrDie("fake://"),
-		Updates: make(chan interface{}, 1024),
-	})
+		Updates: updates,
+		PodLW:   &NewMockPodsListWatch(api.PodList{}).ListWatch,
+	}), updates
 }
 
 func TestExecutorNew(t *testing.T) {
 	mockDriver := &MockExecutorDriver{}
-	executor := NewTestKubernetesExecutor()
+	executor, _ := NewTestKubernetesExecutor()
 	executor.Init(mockDriver)
 
 	assert.Equal(t, executor.isDone(), false, "executor should not be in Done state on initialization")

contrib/mesos/pkg/executor/service/kubelet.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	log "github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/kubelet"
+	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/pkg/util"
+)
+
+// executorKubelet decorates the kubelet with a Run function that notifies the
+// executor by closing kubeletDone before entering blocking state.
+type executorKubelet struct {
+	*kubelet.Kubelet
+	kubeletDone  chan<- struct{} // closed once kubelet.Run() returns
+	executorDone <-chan struct{} // closed when executor terminates
+}
+
+// Run runs the main kubelet loop, closing the kubeletFinished chan when the
+// loop exits. Like the upstream Run, it will never return.
+func (kl *executorKubelet) Run(mergedUpdates <-chan kubetypes.PodUpdate) {
+	defer func() {
+		// When this Run function is called, we close it here.
+		// Otherwise, KubeletExecutorServer.runKubelet will.
+		close(kl.kubeletDone)
+		util.HandleCrash()
+		log.Infoln("kubelet run terminated") //TODO(jdef) turn down verbosity
+		// important: never return! this is in our contract
+		select {}
+	}()
+
+	// push merged updates into another, closable update channel which is closed
+	// when the executor shuts down.
+	closableUpdates := make(chan kubetypes.PodUpdate)
+	go func() {
+		// closing closableUpdates will cause our patched kubelet's syncLoop() to exit
+		defer close(closableUpdates)
+	pipeLoop:
+		for {
+			select {
+			case <-kl.executorDone:
+				break pipeLoop
+			default:
+				select {
+				case u := <-mergedUpdates:
+					select {
+					case closableUpdates <- u: // noop
+					case <-kl.executorDone:
+						break pipeLoop
+					}
+				case <-kl.executorDone:
+					break pipeLoop
+				}
+			}
+		}
+	}()
+
+	// we expect that Run() will complete after closableUpdates is closed and the
+	// kubelet's syncLoop() has finished processing its backlog, which hopefully
+	// will not take very long. Peeking into the future (current k8s master) it
+	// seems that the backlog has grown from 1 to 50 -- this may negatively impact
+	// us going forward, time will tell.
+	util.Until(func() { kl.Kubelet.Run(closableUpdates) }, 0, kl.executorDone)
+
+	//TODO(jdef) revisit this if/when executor failover lands
+	// Force kubelet to delete all pods.
+	kl.HandlePodDeletions(kl.GetPods())
+}

contrib/mesos/pkg/executor/suicide_test.go

@@ -67,7 +67,7 @@ func (t *suicideTracker) makeJumper(_ jumper) jumper {
 func TestSuicide_zeroTimeout(t *testing.T) {
 	defer glog.Flush()
 
-	k := New(Config{})
+	k, _ := NewTestKubernetesExecutor()
 	tracker := &suicideTracker{suicideWatcher: k.suicideWatch}
 	k.suicideWatch = tracker
 
@@ -92,9 +92,8 @@ func TestSuicide_zeroTimeout(t *testing.T) {
 func TestSuicide_WithTasks(t *testing.T) {
 	defer glog.Flush()
 
-	k := New(Config{
-		SuicideTimeout: 50 * time.Millisecond,
-	})
+	k, _ := NewTestKubernetesExecutor()
+	k.suicideTimeout = 50 * time.Millisecond
 
 	jumps := uint32(0)
 	tracker := &suicideTracker{suicideWatcher: k.suicideWatch, jumps: &jumps}