Commit 0aae852a authored by Ryan McNamara's avatar Ryan McNamara

Stably sort controllerrevisions

Fixes https://github.com/kubernetes/kubernetes/issues/61998 There are times when multiple "equal" controllerrevisions are created with the same revision number. When this happens and this is the case for the largest revision number, the statefulset controller will periodically select one of the maximal controllerrevisions to be the target of the underlying statefulset. The selection happens here: https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/statefulset/stateful_set_control.go#L212. Prior to this change this selection was random as the sort was not stable, which caused the pods of a stable set to continually roll.
parent 7ac1f897
...@@ -113,7 +113,7 @@ func HashControllerRevision(revision *apps.ControllerRevision, probe *int32) str ...@@ -113,7 +113,7 @@ func HashControllerRevision(revision *apps.ControllerRevision, probe *int32) str
// SortControllerRevisions sorts revisions by their Revision. // SortControllerRevisions sorts revisions by their Revision.
func SortControllerRevisions(revisions []*apps.ControllerRevision) { func SortControllerRevisions(revisions []*apps.ControllerRevision) {
sort.Sort(byRevision(revisions)) sort.Stable(byRevision(revisions))
} }
// EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that // EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that
...@@ -162,7 +162,14 @@ func (br byRevision) Len() int { ...@@ -162,7 +162,14 @@ func (br byRevision) Len() int {
return len(br) return len(br)
} }
// Less breaks ties first by creation timestamp, then by name
func (br byRevision) Less(i, j int) bool { func (br byRevision) Less(i, j int) bool {
if br[i].Revision == br[j].Revision {
if br[j].CreationTimestamp.Equal(&br[i].CreationTimestamp) {
return br[i].Name < br[j].Name
}
return br[j].CreationTimestamp.After(br[i].CreationTimestamp.Time)
}
return br[i].Revision < br[j].Revision return br[i].Revision < br[j].Revision
} }
......
...@@ -40,6 +40,7 @@ import ( ...@@ -40,6 +40,7 @@ import (
"k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/strategicpatch"
core "k8s.io/client-go/testing" core "k8s.io/client-go/testing"
"time"
) )
func TestRealHistory_ListControllerRevisions(t *testing.T) { func TestRealHistory_ListControllerRevisions(t *testing.T) {
...@@ -1545,12 +1546,14 @@ func TestSortControllerRevisions(t *testing.T) { ...@@ -1545,12 +1546,14 @@ func TestSortControllerRevisions(t *testing.T) {
want []string want []string
} }
testFn := func(test *testcase, t *testing.T) { testFn := func(test *testcase, t *testing.T) {
SortControllerRevisions(test.revisions) t.Run(test.name, func(t *testing.T) {
for i := range test.revisions { SortControllerRevisions(test.revisions)
if test.revisions[i].Name != test.want[i] { for i := range test.revisions {
t.Errorf("%s: want %s at %d got %s", test.name, test.want[i], i, test.revisions[i].Name) if test.revisions[i].Name != test.want[i] {
t.Errorf("%s: want %s at %d got %s", test.name, test.want[i], i, test.revisions[i].Name)
}
} }
} })
} }
ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"})
ss1.Status.CollisionCount = new(int32) ss1.Status.CollisionCount = new(int32)
...@@ -1559,17 +1562,32 @@ func TestSortControllerRevisions(t *testing.T) { ...@@ -1559,17 +1562,32 @@ func TestSortControllerRevisions(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ss1Rev1.Namespace = ss1.Namespace ss1Rev1.Namespace = ss1.Namespace
ss1Rev2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) ss1Rev2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ss1Rev2.Namespace = ss1.Namespace ss1Rev2.Namespace = ss1.Namespace
ss1Rev3, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) ss1Rev3, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ss1Rev3.Namespace = ss1.Namespace
ss1Rev3Time2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount)
if err != nil {
t.Fatal(err)
}
ss1Rev3Time2.Namespace = ss1.Namespace
ss1Rev3Time2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)}
ss1Rev3Time2Name2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount)
if err != nil {
t.Fatal(err)
}
ss1Rev3Time2Name2.Namespace = ss1.Namespace
ss1Rev3Time2Name2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)}
tests := []testcase{ tests := []testcase{
{ {
...@@ -1588,6 +1606,11 @@ func TestSortControllerRevisions(t *testing.T) { ...@@ -1588,6 +1606,11 @@ func TestSortControllerRevisions(t *testing.T) {
want: []string{ss1Rev1.Name, ss1Rev2.Name, ss1Rev3.Name}, want: []string{ss1Rev1.Name, ss1Rev2.Name, ss1Rev3.Name},
}, },
{ {
name: "with ties",
revisions: []*apps.ControllerRevision{ss1Rev3, ss1Rev3Time2, ss1Rev2, ss1Rev1},
want: []string{ss1Rev1.Name, ss1Rev2.Name, ss1Rev3.Name, ss1Rev3Time2.Name, ss1Rev3Time2Name2.Name},
},
{
name: "empty", name: "empty",
revisions: nil, revisions: nil,
want: nil, want: nil,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment