Skip to content

K
k3s
Commit b2013cb1 authored by James DeFelice's avatar James DeFelice
Browse files
Options

replace pod-observer and check-for-lost-pod polling with a single pod watcher and task registry

parent cfd046f7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 314 additions and 316 deletions
contrib/mesos/pkg/executor/apis.go 0 → 100644
View file @ b2013cb1
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package executor
import (
"k8s.io/kubernetes/contrib/mesos/pkg/node"
"k8s.io/kubernetes/pkg/api"
client "k8s.io/kubernetes/pkg/client/unversioned"
)
type kubeAPI interface {
killPod(ns, name string) error
}
type nodeAPI interface {
createOrUpdate(hostname string, slaveAttrLabels, annotations map[string]string) (*api.Node, error)
}
// clientAPIWrapper implements kubeAPI and node API, which serve to isolate external dependencies
// such that they're easier to mock in unit test.
type clientAPIWrapper struct {
client *client.Client
}
func (cw *clientAPIWrapper) killPod(ns, name string) error {
return cw.client.Pods(ns).Delete(name, api.NewDeleteOptions(0))
}
func (cw *clientAPIWrapper) createOrUpdate(hostname string, slaveAttrLabels, annotations map[string]string) (*api.Node, error) {
return node.CreateOrUpdate(cw.client, hostname, slaveAttrLabels, annotations)
}
contrib/mesos/pkg/executor/mock_test.go
View file @ b2013cb1
......@@ -22,11 +22,18 @@ import (
"github.com/mesos/mesos-go/mesosproto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)
type mockKubeAPI struct {
mock.Mock
}
func (m *mockKubeAPI) killPod(ns, name string) error {
args := m.Called(ns, name)
return args.Error(0)
}
type MockExecutorDriver struct {
mock.Mock
}
......@@ -66,18 +73,16 @@ func (m *MockExecutorDriver) SendFrameworkMessage(msg string) (mesosproto.Status
return args.Get(0).(mesosproto.Status), args.Error(1)
}
func NewTestKubernetesExecutor() (*Executor, chan kubetypes.PodUpdate) {
updates := make(chan kubetypes.PodUpdate, 1024)
func NewTestKubernetesExecutor() *Executor {
return New(Config{
Docker: dockertools.ConnectToDockerOrDie("fake://"),
Updates: updates,
PodLW: &NewMockPodsListWatch(api.PodList{}).ListWatch,
}), updates
Docker: dockertools.ConnectToDockerOrDie("fake://"),
Registry: newFakeRegistry(),
})
}
func TestExecutorNew(t *testing.T) {
mockDriver := &MockExecutorDriver{}
executor, _ := NewTestKubernetesExecutor()
executor := NewTestKubernetesExecutor()
executor.Init(mockDriver)
assert.Equal(t, executor.isDone(), false, "executor should not be in Done state on initialization")
......
contrib/mesos/pkg/executor/observer.go deleted 100644 → 0
View file @ cfd046f7
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package executor
import (
log "github.com/golang/glog"
"k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/client/cache"
"k8s.io/kubernetes/pkg/controller/framework"
)
// taskUpdateTx execute a task update transaction f for the task identified by
// taskId. if no such task exists then f is not invoked and an error is
// returned. if f is invoked then taskUpdateTx returns the bool result of f.
type taskUpdateTx func(taskId string, f func(*kuberTask, *api.Pod) bool) (changed bool, err error)
// podObserver receives callbacks for every pod state change on the apiserver and
// for each decides whether to execute a task update transaction.
type podObserver struct {
podController *framework.Controller
terminate <-chan struct{}
taskUpdateTx taskUpdateTx
}
func newPodObserver(podLW cache.ListerWatcher, taskUpdateTx taskUpdateTx, terminate <-chan struct{}) *podObserver {
// watch pods from the given pod ListWatch
if podLW == nil {
// fail early to make debugging easier
panic("cannot create executor with nil PodLW")
}
p := &podObserver{
terminate: terminate,
taskUpdateTx: taskUpdateTx,
}
_, p.podController = framework.NewInformer(podLW, &api.Pod{}, podRelistPeriod, &framework.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
pod := obj.(*api.Pod)
log.V(4).Infof("pod %s/%s created on apiserver", pod.Namespace, pod.Name)
p.handleChangedApiserverPod(pod)
},
UpdateFunc: func(oldObj, newObj interface{}) {
pod := newObj.(*api.Pod)
log.V(4).Infof("pod %s/%s updated on apiserver", pod.Namespace, pod.Name)
p.handleChangedApiserverPod(pod)
},
DeleteFunc: func(obj interface{}) {
pod := obj.(*api.Pod)
log.V(4).Infof("pod %s/%s deleted on apiserver", pod.Namespace, pod.Name)
},
})
return p
}
// run begins observing pod state changes; blocks until the terminate chan closes.
func (p *podObserver) run() {
p.podController.Run(p.terminate)
}
// handleChangedApiserverPod is invoked for pod add/update state changes and decides whether
// task updates are necessary. if so, a task update is executed via taskUpdateTx.
func (p *podObserver) handleChangedApiserverPod(pod *api.Pod) {
// Don't do anything for pods without task anotation which means:
// - "pre-scheduled" pods which have a NodeName set to this node without being scheduled already.
// - static/mirror pods: they'll never have a TaskID annotation, and we don't expect them to ever change.
// - all other pods that haven't passed through the launch-task-binding phase, which would set annotations.
taskId := pod.Annotations[meta.TaskIdKey]
if taskId == "" {
// There also could be a race between the overall launch-task process and this update, but here we
// will never be able to process such a stale update because the "update pod" that we're receiving
// in this func won't yet have a task ID annotation. It follows that we can safely drop such a stale
// update on the floor because we'll get another update later that, in addition to the changes that
// we're dropping now, will also include the changes from the binding process.
log.V(5).Infof("ignoring pod update for %s/%s because %s annotation is missing", pod.Namespace, pod.Name, meta.TaskIdKey)
return
}
_, err := p.taskUpdateTx(taskId, func(_ *kuberTask, relatedPod *api.Pod) (sendSnapshot bool) {
if relatedPod == nil {
// should never happen because:
// (a) the update pod record has already passed through the binding phase in launchTasks()
// (b) all remaining updates to executor.{pods,tasks} are sync'd in unison
log.Errorf("internal state error: pod not found for task %s", taskId)
return
}
// TODO(sttts): check whether we can and should send all "semantic" changes down to the kubelet
// see kubelet/config/config.go for semantic change detection
// check for updated labels/annotations: need to forward these for the downward API
sendSnapshot = sendSnapshot || updateMetaMap(&relatedPod.Labels, pod.Labels)
sendSnapshot = sendSnapshot || updateMetaMap(&relatedPod.Annotations, pod.Annotations)
// terminating pod?
if pod.Status.Phase == api.PodRunning {
timeModified := differentTime(relatedPod.DeletionTimestamp, pod.DeletionTimestamp)
graceModified := differentPeriod(relatedPod.DeletionGracePeriodSeconds, pod.DeletionGracePeriodSeconds)
if timeModified || graceModified {
log.Infof("pod %s/%s is terminating at %v with %vs grace period, telling kubelet",
pod.Namespace, pod.Name, *pod.DeletionTimestamp, *pod.DeletionGracePeriodSeconds)
// modify the pod in our registry instead of sending the new pod. The later
// would allow that other changes bleed into the kubelet. For now we are
// very conservative changing this behaviour.
relatedPod.DeletionTimestamp = pod.DeletionTimestamp
relatedPod.DeletionGracePeriodSeconds = pod.DeletionGracePeriodSeconds
sendSnapshot = true
}
}
return
})
if err != nil {
log.Errorf("failed to update pod %s/%s: %+v", pod.Namespace, pod.Name, err)
}
}
// updateMetaMap looks for differences between src and dest; if there are differences
// then dest is changed (possibly to point to src) and this func returns true.
func updateMetaMap(dest *map[string]string, src map[string]string) (changed bool) {
// check for things in dest that are missing in src
for k := range *dest {
if _, ok := src[k]; !ok {
changed = true
break
}
}
if !changed {
if len(*dest) == 0 {
if len(src) > 0 {
changed = true
goto finished
}
// no update needed
return
}
// check for things in src that are missing/different in dest
for k, v := range src {
if vv, ok := (*dest)[k]; !ok || vv != v {
changed = true
break
}
}
}
finished:
*dest = src
return
}
func differentTime(a, b *unversioned.Time) bool {
return (a == nil) != (b == nil) || (a != nil && b != nil && *a != *b)
}
func differentPeriod(a, b *int64) bool {
return (a == nil) != (b == nil) || (a != nil && b != nil && *a != *b)
}
contrib/mesos/pkg/executor/service/podsource.go
View file @ b2013cb1
......@@ -17,12 +17,12 @@ limitations under the License.
package service
import (
"fmt"
log "github.com/golang/glog"
"k8s.io/kubernetes/contrib/mesos/pkg/executor"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/client/cache"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
log "github.com/golang/glog"
)
const (
......@@ -32,97 +32,95 @@ const (
mesosSource = kubetypes.ApiserverSource
)
// sourceMesos merges pods from mesos, and mirror pods from the apiserver. why?
// (a) can't have two sources with the same name;
// (b) all sources, other than ApiserverSource are considered static/mirror
// sources, and;
// (c) kubelet wants to see mirror pods reflected in a non-static source.
//
// Mesos pods must appear to come from apiserver due to (b), while reflected
// static pods (mirror pods) must appear to come from apiserver due to (c).
//
// The only option I could think of was creating a source that merges the pod
// streams. I don't like it. But I could think of anything else, other than
// starting to hack up the kubelet's understanding of mirror/static pod
// sources (ouch!)
type sourceMesos struct {
sourceFinished chan struct{} // sourceFinished closes when mergeAndForward exits
out chan<- interface{} // out is the sink for merged pod snapshots
mirrorPods chan []*api.Pod // mirrorPods communicates snapshots of the current set of mirror pods
execUpdates <-chan kubetypes.PodUpdate // execUpdates receives snapshots of the current set of mesos pods
}
type (
podName struct {
namespace, name string
}
sourceMesos struct {
stop <-chan struct{}
out chan<- interface{} // never close this because pkg/util/config.mux doesn't handle that very well
registry executor.Registry
priorPodNames map[podName]string // map podName to taskID
}
)
// newSourceMesos creates a pod config source that merges pod updates from
// mesos (via execUpdates), and mirror pod updates from the apiserver (via
// podWatch) writing the merged update stream to the out chan. It is expected
// that execUpdates will only ever contain SET operations. The source takes
// ownership of the sourceFinished chan, closing it when the source terminates.
// Source termination happens when the execUpdates chan is closed and fully
// drained of updates.
func newSourceMesos(
sourceFinished chan struct{},
execUpdates <-chan kubetypes.PodUpdate,
stop <-chan struct{},
out chan<- interface{},
podWatch *cache.ListWatch,
registry executor.Registry,
) {
source := &sourceMesos{
sourceFinished: sourceFinished,
mirrorPods: make(chan []*api.Pod),
execUpdates: execUpdates,
out: out,
stop: stop,
out: out,
registry: registry,
priorPodNames: make(map[podName]string),
}
// reflect changes from the watch into a chan, filtered to include only mirror pods (have an ConfigMirrorAnnotationKey attr)
cache.NewReflector(podWatch, &api.Pod{}, cache.NewUndeltaStore(source.send, cache.MetaNamespaceKeyFunc), 0).RunUntil(sourceFinished)
go source.mergeAndForward()
// reflect changes from the watch into a chan, filtered to include only mirror pods
// (have an ConfigMirrorAnnotationKey attr)
cache.NewReflector(
podWatch,
&api.Pod{},
cache.NewUndeltaStore(source.send, cache.MetaNamespaceKeyFunc),
0,
).RunUntil(stop)
}
// send is an update callback invoked by NewUndeltaStore
func (source *sourceMesos) send(objs []interface{}) {
var mirrors []*api.Pod
var (
pods = make([]*api.Pod, 0, len(objs))
podNames = make(map[podName]string, len(objs))
)
for _, o := range objs {
p := o.(*api.Pod)
addPod := false
if _, ok := p.Annotations[kubetypes.ConfigMirrorAnnotationKey]; ok {
mirrors = append(mirrors, p)
// pass through all mirror pods
addPod = true
} else if rpod, err := source.registry.Update(p); err == nil {
// pod is bound to a task, and the update is compatible
// so we'll allow it through
addPod = true
p = rpod.Pod() // use the (possibly) updated pod spec!
podNames[podName{p.Namespace, p.Name}] = rpod.Task()
} else if rpod != nil {
// we were able to ID the pod but the update still failed...
log.Warningf("failed to update registry for task %v pod %v/%v: %v",
rpod.Task(), p.Namespace, p.Name, err)
} else {
// unrecognized pod, skip!
log.V(2).Infof("skipping pod %v/%v", p.Namespace, p.Name)
}
if addPod {
pods = append(pods, p)
}
}
select {
case <-source.sourceFinished:
case source.mirrorPods <- mirrors:
// detect when pods are deleted and notify the registry
for k, taskID := range source.priorPodNames {
if _, found := podNames[k]; !found {
source.registry.Remove(taskID)
}
}
}
func (source *sourceMesos) mergeAndForward() {
// execUpdates will be closed by the executor on shutdown
defer close(source.sourceFinished)
var (
mirrors = []*api.Pod{}
pods = []*api.Pod{}
)
eventLoop:
for {
source.priorPodNames = podNames
u := kubetypes.PodUpdate{
Op: kubetypes.SET,
Pods: pods,
Source: mesosSource,
}
select {
case <-source.stop:
default:
select {
case m := <-source.mirrorPods:
mirrors = m[:]
u := kubetypes.PodUpdate{
Op: kubetypes.SET,
Pods: append(m, pods...),
Source: mesosSource,
}
log.V(3).Infof("mirror update, sending snapshot of size %d", len(u.Pods))
source.out <- u
case u, ok := <-source.execUpdates:
if !ok {
break eventLoop
}
if u.Op != kubetypes.SET {
panic(fmt.Sprintf("unexpected Op type: %v", u.Op))
}
pods = u.Pods[:]
u.Pods = append(u.Pods, mirrors...)
u.Source = mesosSource
log.V(3).Infof("pods update, sending snapshot of size %d", len(u.Pods))
source.out <- u
case <-source.stop:
case source.out <- u:
}
}
log.V(2).Infoln("mesos pod source terminating normally")
log.V(2).Infof("sent %d pod updates", len(pods))
}
contrib/mesos/pkg/executor/service/service.go
View file @ b2013cb1
......@@ -46,11 +46,6 @@ type KubeletExecutorServer struct {
*options.KubeletServer
SuicideTimeout time.Duration
LaunchGracePeriod time.Duration
// TODO(sttts): remove necessity to access the kubelet from the executor
klet *kubelet.Kubelet // once set, immutable
kletReady chan struct{} // once closed, klet is guaranteed to be valid and concurrently readable
}
func NewKubeletExecutorServer() *KubeletExecutorServer {
......@@ -58,7 +53,6 @@ func NewKubeletExecutorServer() *KubeletExecutorServer {
KubeletServer: options.NewKubeletServer(),
SuicideTimeout: config.DefaultSuicideTimeout,
LaunchGracePeriod: config.DefaultLaunchGracePeriod,
kletReady: make(chan struct{}),
}
if pwd, err := os.Getwd(); err != nil {
log.Warningf("failed to determine current directory: %v", err)
......@@ -77,43 +71,20 @@ func (s *KubeletExecutorServer) AddFlags(fs *pflag.FlagSet) {
}
func (s *KubeletExecutorServer) runExecutor(
execUpdates chan<- kubetypes.PodUpdate,
nodeInfos chan<- executor.NodeInfo,
kubeletFinished <-chan struct{},
staticPodsConfigPath string,
apiclient *client.Client,
podLW *cache.ListWatch,
) error {
registry executor.Registry,
) (<-chan struct{}, error) {
exec := executor.New(executor.Config{
Updates: execUpdates,
APIClient: apiclient,
Docker: dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
SuicideTimeout: s.SuicideTimeout,
KubeletFinished: kubeletFinished,
ExitFunc: os.Exit,
PodStatusFunc: func(pod *api.Pod) (*api.PodStatus, error) {
select {
case <-s.kletReady:
default:
return nil, fmt.Errorf("PodStatucFunc called before kubelet is initialized")
}
status, err := s.klet.GetRuntime().GetAPIPodStatus(pod)
if err != nil {
return nil, err
}
status.Phase = kubelet.GetPhase(&pod.Spec, status.ContainerStatuses)
hostIP, err := s.klet.GetHostIP()
if err != nil {
log.Errorf("Cannot get host IP: %v", err)
} else {
status.HostIP = hostIP.String()
}
return status, nil
},
Registry: registry,
APIClient: apiclient,
Docker: dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
SuicideTimeout: s.SuicideTimeout,
KubeletFinished: kubeletFinished,
ExitFunc: os.Exit,
StaticPodsConfigPath: staticPodsConfigPath,
PodLW: podLW,
NodeInfos: nodeInfos,
})
......@@ -125,7 +96,7 @@ func (s *KubeletExecutorServer) runExecutor(
}
driver, err := bindings.NewMesosExecutorDriver(dconfig)
if err != nil {
return fmt.Errorf("failed to create executor driver: %v", err)
return nil, fmt.Errorf("failed to create executor driver: %v", err)
}
log.V(2).Infof("Initialize executor driver...")
exec.Init(driver)
......@@ -138,16 +109,17 @@ func (s *KubeletExecutorServer) runExecutor(
log.Info("executor Run completed")
}()
return nil
return exec.Done(), nil
}
func (s *KubeletExecutorServer) runKubelet(
execUpdates <-chan kubetypes.PodUpdate,
nodeInfos <-chan executor.NodeInfo,
kubeletDone chan<- struct{},
staticPodsConfigPath string,
apiclient *client.Client,
podLW *cache.ListWatch,
registry executor.Registry,
executorDone <-chan struct{},
) (err error) {
defer func() {
if err != nil {
......@@ -163,19 +135,15 @@ func (s *KubeletExecutorServer) runKubelet(
}
// apply Mesos specific settings
executorDone := make(chan struct{})
kcfg.Builder = func(kc *kubeletapp.KubeletConfig) (kubeletapp.KubeletBootstrap, *kconfig.PodConfig, error) {
k, pc, err := kubeletapp.CreateAndInitKubelet(kc)
if err != nil {
return k, pc, err
}
s.klet = k.(*kubelet.Kubelet)
close(s.kletReady) // intentionally crash if this is called more than once
// decorate kubelet such that it shuts down when the executor is
decorated := &executorKubelet{
Kubelet: s.klet,
Kubelet: k.(*kubelet.Kubelet),
kubeletDone: kubeletDone,
executorDone: executorDone,
}
......@@ -230,21 +198,20 @@ func (s *KubeletExecutorServer) runKubelet(
}
}()
// create main pod source, it will close executorDone when the executor updates stop flowing
newSourceMesos(executorDone, execUpdates, kcfg.PodConfig.Channel(mesosSource), podLW)
// create main pod source, it will stop generating events once executorDone is closed
newSourceMesos(executorDone, kcfg.PodConfig.Channel(mesosSource), podLW, registry)
// create static-pods directory file source
log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath)
fileSourceUpdates := kcfg.PodConfig.Channel(kubetypes.FileSource)
kconfig.NewSourceFile(staticPodsConfigPath, kcfg.HostnameOverride, kcfg.FileCheckFrequency, fileSourceUpdates)
// run the kubelet, until execUpdates is closed
// run the kubelet
// NOTE: because kcfg != nil holds, the upstream Run function will not
// initialize the cloud provider. We explicitly wouldn't want
// that because then every kubelet instance would query the master
// state.json which does not scale.
err = kubeletapp.Run(s.KubeletServer, kcfg)
return
}
......@@ -252,7 +219,6 @@ func (s *KubeletExecutorServer) runKubelet(
func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
// create shared channels
kubeletFinished := make(chan struct{})
execUpdates := make(chan kubetypes.PodUpdate, 1)
nodeInfos := make(chan executor.NodeInfo, 1)
// create static pods directory
......@@ -273,18 +239,22 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
return fmt.Errorf("cannot create API client: %v", err)
}
pw := cache.NewListWatchFromClient(apiclient, "pods", api.NamespaceAll,
fields.OneTermEqualSelector(client.PodHost, s.HostnameOverride),
var (
pw = cache.NewListWatchFromClient(apiclient, "pods", api.NamespaceAll,
fields.OneTermEqualSelector(client.PodHost, s.HostnameOverride),
)
reg = executor.NewRegistry(apiclient)
)
// start executor
err = s.runExecutor(execUpdates, nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw)
var executorDone <-chan struct{}
executorDone, err = s.runExecutor(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, reg)
if err != nil {
return err
}
// start kubelet, blocking
return s.runKubelet(execUpdates, nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw)
return s.runKubelet(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw, reg, executorDone)
}
func defaultBindingAddress() string {
......
contrib/mesos/pkg/executor/suicide_test.go
View file @ b2013cb1
......@@ -23,6 +23,7 @@ import (
"github.com/golang/glog"
bindings "github.com/mesos/mesos-go/executor"
"k8s.io/kubernetes/pkg/api"
)
type suicideTracker struct {
......@@ -67,7 +68,7 @@ func (t *suicideTracker) makeJumper(_ jumper) jumper {
func TestSuicide_zeroTimeout(t *testing.T) {
defer glog.Flush()
k, _ := NewTestKubernetesExecutor()
k := NewTestKubernetesExecutor()
tracker := &suicideTracker{suicideWatcher: k.suicideWatch}
k.suicideWatch = tracker
......@@ -92,14 +93,14 @@ func TestSuicide_zeroTimeout(t *testing.T) {
func TestSuicide_WithTasks(t *testing.T) {
defer glog.Flush()
k, _ := NewTestKubernetesExecutor()
k := NewTestKubernetesExecutor()
k.suicideTimeout = 50 * time.Millisecond
jumps := uint32(0)
tracker := &suicideTracker{suicideWatcher: k.suicideWatch, jumps: &jumps}
k.suicideWatch = tracker
k.tasks["foo"] = &kuberTask{} // prevent suicide attempts from succeeding
k.registry.bind("foo", &api.Pod{}) // prevent suicide attempts from succeeding
// call reset with a nil timer
glog.Infoln("Resetting suicide watch with 1 task")
......@@ -119,7 +120,7 @@ func TestSuicide_WithTasks(t *testing.T) {
t.Fatalf("initial suicide watch setup failed")
}
delete(k.tasks, "foo") // zero remaining tasks
k.registry.Remove("foo") // zero remaining tasks
k.suicideTimeout = 1500 * time.Millisecond
suicideStart := time.Now()
......@@ -142,7 +143,7 @@ func TestSuicide_WithTasks(t *testing.T) {
}
k.lock.Lock()
k.tasks["foo"] = &kuberTask{} // prevent suicide attempts from succeeding
k.registry.bind("foo", &api.Pod{}) // prevent suicide attempts from succeeding
k.lock.Unlock()
// reset the suicide watch, which should stop the existing timer
......@@ -164,7 +165,7 @@ func TestSuicide_WithTasks(t *testing.T) {
}
k.lock.Lock()
delete(k.tasks, "foo") // allow suicide attempts to schedule
k.registry.Remove("foo") // allow suicide attempts to schedule
k.lock.Unlock()
// reset the suicide watch, which should reset a stopped timer
......
contrib/mesos/pkg/executor/watcher.go 0 → 100644
View file @ b2013cb1
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package executor
import (
"sync"
"time"
log "github.com/golang/glog"
)
type (
// filter registration events, return false to abort further processing of the event
watchFilter func(pod *PodEvent) (accept bool)
watchExpiration struct {
// timeout closes when the handler has expired; it delivers at most one Time.
timeout <-chan time.Time
// onEvent is an optional callback that is invoked if/when the expired chan
// closes
onEvent func(taskID string)
}
watchHandler struct {
// prevent callbacks from being invoked simultaneously
sync.Mutex
// handle registration events, return true to indicate the handler should be
// de-registered upon completion. If pod is nil then the associated handler
// has expired.
onEvent func(pod *PodEvent) (done bool, err error)
// expiration is an optional configuration that indicates when a handler should
// be considered to have expired, and what action to take upon such
expiration watchExpiration
}
// watcher observes PodEvent events and conditionally executes handlers that
// have been associated with the taskID of the PodEvent.
watcher struct {
updates <-chan *PodEvent
rw sync.RWMutex
handlers map[string]watchHandler
filters []watchFilter
runOnce chan struct{}
}
)
func newWatcher(updates <-chan *PodEvent) *watcher {
return &watcher{
updates: updates,
handlers: make(map[string]watchHandler),
runOnce: make(chan struct{}),
}
}
func (pw *watcher) run() {
select {
case <-pw.runOnce:
log.Error("run() has already been invoked for this pod-watcher")
return
default:
close(pw.runOnce)
}
updateLoop:
for u := range pw.updates {
log.V(2).Info("filtering " + u.FormatShort())
for _, f := range pw.filters {
if !f(u) {
continue updateLoop
}
}
log.V(1).Info("handling " + u.FormatShort())
h, ok := func() (h watchHandler, ok bool) {
pw.rw.RLock()
defer pw.rw.RUnlock()
h, ok = pw.handlers[u.taskID]
return
}()
if ok {
log.V(1).Info("executing action for " + u.FormatShort())
done, err := func() (bool, error) {
h.Lock()
defer h.Unlock()
return h.onEvent(u)
}()
if err != nil {
log.Error(err)
}
if done {
// de-register handler upon successful completion of action
log.V(1).Info("de-registering handler for " + u.FormatShort())
func() {
pw.rw.Lock()
delete(pw.handlers, u.taskID)
pw.rw.Unlock()
}()
}
}
}
}
func (pw *watcher) addFilter(f watchFilter) {
select {
case <-pw.runOnce:
log.Errorf("failed to add filter because pod-watcher is already running")
default:
pw.filters = append(pw.filters, f)
}
}
// forTask associates a handler `h` with the given taskID.
func (pw *watcher) forTask(taskID string, h watchHandler) {
pw.rw.Lock()
pw.handlers[taskID] = h
pw.rw.Unlock()
if exp := h.expiration; exp.timeout != nil {
go func() {
<-exp.timeout
log.V(1).Infof("expiring handler for task %v", taskID)
// de-register handler upon expiration
pw.rw.Lock()
delete(pw.handlers, taskID)
pw.rw.Unlock()
if exp.onEvent != nil {
h.Lock()
defer h.Unlock()
exp.onEvent(taskID)
}
}()
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment