Commit 51b5ff59 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #32210 from deads2k/controller-01-cachewait-handle

Automatic merge from submit-queue update error handling for daemoncontroller Updates the DaemonSet controller to cleanly requeue with ratelimiting on errors, make use of the `utilruntime.HandleError` consistently, and wait for preconditions before doing work. @ncdc @liggitt @sttts My plan is to use this one as an example of how to handle requeuing, preconditions, and processing error handling. @foxish fyi related to https://github.com/kubernetes/kubernetes/issues/30629
parents 71af3312 38583182
...@@ -138,6 +138,7 @@ func newTestController() (*DaemonSetsController, *controller.FakePodControl) { ...@@ -138,6 +138,7 @@ func newTestController() (*DaemonSetsController, *controller.FakePodControl) {
clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: "", ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: "", ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
manager := NewDaemonSetsControllerFromClient(clientset, controller.NoResyncPeriodFunc, 0) manager := NewDaemonSetsControllerFromClient(clientset, controller.NoResyncPeriodFunc, 0)
manager.podStoreSynced = alwaysReady manager.podStoreSynced = alwaysReady
manager.nodeStoreSynced = alwaysReady
podControl := &controller.FakePodControl{} podControl := &controller.FakePodControl{}
manager.podControl = podControl manager.podControl = podControl
return manager, podControl return manager, podControl
...@@ -539,28 +540,6 @@ func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) { ...@@ -539,28 +540,6 @@ func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) {
syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0) syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0)
} }
func TestDSManagerNotReady(t *testing.T) {
manager, podControl := newTestController()
manager.podStoreSynced = func() bool { return false }
addNodes(manager.nodeStore.Store, 0, 1, nil)
// Simulates the ds reflector running before the pod reflector. We don't
// want to end up creating daemon pods in this case until the pod reflector
// has synced, so the ds manager should just requeue the ds.
ds := newDaemonSet("foo")
manager.dsStore.Add(ds)
dsKey := getKey(ds, t)
syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0)
queueDS, _ := manager.queue.Get()
if queueDS != dsKey {
t.Fatalf("Expected to find key %v in queue, found %v", dsKey, queueDS)
}
manager.podStoreSynced = alwaysReady
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0)
}
// Daemon with node affinity should launch pods on nodes matching affinity. // Daemon with node affinity should launch pods on nodes matching affinity.
func TestNodeAffinityDaemonLaunchesPods(t *testing.T) { func TestNodeAffinityDaemonLaunchesPods(t *testing.T) {
manager, podControl := newTestController() manager, podControl := newTestController()
......
...@@ -21,9 +21,12 @@ import ( ...@@ -21,9 +21,12 @@ import (
"sync" "sync"
"time" "time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/client/cache"
"k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/runtime"
utilruntime "k8s.io/kubernetes/pkg/util/runtime" utilruntime "k8s.io/kubernetes/pkg/util/runtime"
"k8s.io/kubernetes/pkg/util/wait"
) )
// if you use this, there is one behavior change compared to a standard Informer. // if you use this, there is one behavior change compared to a standard Informer.
...@@ -75,6 +78,34 @@ func NewSharedIndexInformer(lw cache.ListerWatcher, objType runtime.Object, resy ...@@ -75,6 +78,34 @@ func NewSharedIndexInformer(lw cache.ListerWatcher, objType runtime.Object, resy
return sharedIndexInformer return sharedIndexInformer
} }
// InformerSynced is a function that can be used to determine if an informer has synced. This is useful for determining if caches have synced.
type InformerSynced func() bool
// syncedPollPeriod controls how often you look at the status of your sync funcs
const syncedPollPeriod = 100 * time.Millisecond
// WaitForCacheSync waits for caches to populate. It returns true if it was successful, false
// if the contoller should shutdown
func WaitForCacheSync(stopCh <-chan struct{}, cacheSyncs ...InformerSynced) bool {
err := wait.PollUntil(syncedPollPeriod,
func() (bool, error) {
for _, syncFunc := range cacheSyncs {
if !syncFunc() {
return false, nil
}
}
return true, nil
},
stopCh)
if err != nil {
glog.V(2).Infof("stop requested")
return false
}
glog.V(4).Infof("caches populated")
return true
}
type sharedIndexInformer struct { type sharedIndexInformer struct {
indexer cache.Indexer indexer cache.Indexer
controller *Controller controller *Controller
......
...@@ -186,7 +186,12 @@ func pollImmediateInternal(wait WaitFunc, condition ConditionFunc) error { ...@@ -186,7 +186,12 @@ func pollImmediateInternal(wait WaitFunc, condition ConditionFunc) error {
func PollInfinite(interval time.Duration, condition ConditionFunc) error { func PollInfinite(interval time.Duration, condition ConditionFunc) error {
done := make(chan struct{}) done := make(chan struct{})
defer close(done) defer close(done)
return WaitFor(poller(interval, 0), condition, done) return PollUntil(interval, condition, done)
}
// PollUntil is like Poll, but it takes a stop change instead of total duration
func PollUntil(interval time.Duration, condition ConditionFunc, stopCh <-chan struct{}) error {
return WaitFor(poller(interval, 0), condition, stopCh)
} }
// WaitFunc creates a channel that receives an item every time a test // WaitFunc creates a channel that receives an item every time a test
......
...@@ -432,3 +432,33 @@ func TestWaitForWithDelay(t *testing.T) { ...@@ -432,3 +432,33 @@ func TestWaitForWithDelay(t *testing.T) {
t.Errorf("expected an ack of the done signal.") t.Errorf("expected an ack of the done signal.")
} }
} }
func TestPollUntil(t *testing.T) {
stopCh := make(chan struct{})
called := make(chan bool)
pollDone := make(chan struct{})
go func() {
PollUntil(time.Microsecond, ConditionFunc(func() (bool, error) {
called <- true
return false, nil
}), stopCh)
close(pollDone)
}()
// make sure we're called once
<-called
// this should trigger a "done"
close(stopCh)
go func() {
// release the condition func if needed
for {
<-called
}
}()
// make sure we finished the poll
<-pollDone
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment