Skip to content

K
k3s
Commit c4e5782a authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #32719 from liggitt/namespace-admission-wait

Automatic merge from submit-queue compensate for raft/cache delay in namespace admission When attempting to create an object in a just-created namespace, the admission plugin can reject the creation attempt for two reasons: 1. The cache of existing namespaces has not observed the creation watch event yet. There was already compensation to perform a live lookup if the namespace was not found in the cache. 2. The live lookup can be performed against a non-leader in a HA etcd setup, and can return NotFound if the non-leader etcd has not observed the creation. This PR adds a delay, then rechecks the cache before proceeding with the live lookup. This gives both the cache and the backing storage more time to observe the create, before failing the request.
parents 01dd125b 2b569eee
Show whitespace changes
Inline Side-by-side
Showing with 22 additions and 1 deletion
plugin/pkg/admission/namespace/lifecycle/admission.go
View file @ c4e5782a
...@@ -21,6 +21,7 @@ import ( ...@@ -21,6 +21,7 @@ import (
"io" "io"
"time" "time"
"github.com/golang/glog"
lru "github.com/hashicorp/golang-lru" lru "github.com/hashicorp/golang-lru"
"k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/client/cache"
...@@ -38,6 +39,12 @@ const ( ...@@ -38,6 +39,12 @@ const (
PluginName = "NamespaceLifecycle" PluginName = "NamespaceLifecycle"
// how long a namespace stays in the force live lookup cache before expiration. // how long a namespace stays in the force live lookup cache before expiration.
forceLiveLookupTTL = 30 * time.Second forceLiveLookupTTL = 30 * time.Second
// how long to wait for a missing namespace before re-checking the cache (and then doing a live lookup)
// this accomplishes two things:
// 1. It allows a watch-fed cache time to observe a namespace creation event
// 2. It allows time for a namespace creation to distribute to members of a storage cluster,
// so the live lookup has a better chance of succeeding even if it isn't performed against the leader.
missingNamespaceWait = 50 * time.Millisecond
) )
func init() { func init() {
...@@ -113,6 +120,19 @@ func (l *lifecycle) Admit(a admission.Attributes) error { ...@@ -113,6 +120,19 @@ func (l *lifecycle) Admit(a admission.Attributes) error {
return errors.NewInternalError(err) return errors.NewInternalError(err)
} }
if !exists && a.GetOperation() == admission.Create {
// give the cache time to observe the namespace before rejecting a create.
// this helps when creating a namespace and immediately creating objects within it.
time.Sleep(missingNamespaceWait)
namespaceObj, exists, err = l.namespaceInformer.GetStore().Get(key)
if err != nil {
return errors.NewInternalError(err)
}
if exists {
glog.V(4).Infof("found %s in cache after waiting", a.GetNamespace())
}
}
// forceLiveLookup if true will skip looking at local cache state and instead always make a live call to server. // forceLiveLookup if true will skip looking at local cache state and instead always make a live call to server.
forceLiveLookup := false forceLiveLookup := false
lruItemObj, ok := l.forceLiveLookupCache.Get(a.GetNamespace()) lruItemObj, ok := l.forceLiveLookupCache.Get(a.GetNamespace())
...@@ -123,7 +143,7 @@ func (l *lifecycle) Admit(a admission.Attributes) error { ...@@ -123,7 +143,7 @@ func (l *lifecycle) Admit(a admission.Attributes) error {
// refuse to operate on non-existent namespaces // refuse to operate on non-existent namespaces
if !exists || forceLiveLookup { if !exists || forceLiveLookup {
// in case of latency in our caches, make a call direct to storage to verify that it truly exists or not // as a last resort, make a call directly to storage
namespaceObj, err = l.client.Core().Namespaces().Get(a.GetNamespace()) namespaceObj, err = l.client.Core().Namespaces().Get(a.GetNamespace())
if err != nil { if err != nil {
if errors.IsNotFound(err) { if errors.IsNotFound(err) {
...@@ -131,6 +151,7 @@ func (l *lifecycle) Admit(a admission.Attributes) error { ...@@ -131,6 +151,7 @@ func (l *lifecycle) Admit(a admission.Attributes) error {
} }
return errors.NewInternalError(err) return errors.NewInternalError(err)
} }
glog.V(4).Infof("found %s via storage lookup", a.GetNamespace())
} }
// ensure that we're not trying to create objects in terminating namespaces // ensure that we're not trying to create objects in terminating namespaces
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment