Optimize etcd storage by compressing recurring events in to a single event

pkg/client/events.go

@@ -34,6 +34,7 @@ type EventNamespacer interface {
 // EventInterface has methods to work with Event resources
 type EventInterface interface {
 	Create(event *api.Event) (*api.Event, error)
+	Update(event *api.Event) (*api.Event, error)
 	List(label, field labels.Selector) (*api.EventList, error)
 	Get(name string) (*api.Event, error)
 	Watch(label, field labels.Selector, resourceVersion string) (watch.Interface, error)
@@ -73,6 +74,26 @@ func (e *events) Create(event *api.Event) (*api.Event, error) {
 	return result, err
 }
 
+// Update modifies an existing event. It returns the copy of the event that the server returns,
+// or an error. The namespace and key to update the event within is deduced from the event. The
+// namespace must either match this event client's namespace, or this event client must have been
+// created with the "" namespace. Update also requires the ResourceVersion to be set in the event
+// object.
+func (e *events) Update(event *api.Event) (*api.Event, error) {
+	if len(event.ResourceVersion) == 0 {
+		return nil, fmt.Errorf("invalid event update object, missing resource version: %#v", event)
+	}
+	result := &api.Event{}
+	err := e.client.Put().
+		Namespace(event.Namespace).
+		Resource("events").
+		Name(event.Name).
+		Body(event).
+		Do().
+		Into(result)
+	return result, err
+}
+
 // List returns a list of events matching the selectors.
 func (e *events) List(label, field labels.Selector) (*api.EventList, error) {
 	result := &api.EventList{}

pkg/client/fake_events.go

@@ -35,6 +35,12 @@ func (c *FakeEvents) Create(event *api.Event) (*api.Event, error) {
 	return &api.Event{}, nil
 }
 
+// Update replaces an existing event. Returns the copy of the event the server returns, or an error.
+func (c *FakeEvents) Update(event *api.Event) (*api.Event, error) {
+	c.Fake.Actions = append(c.Fake.Actions, FakeAction{Action: "update-event", Value: event.Name})
+	return &api.Event{}, nil
+}
+
 // List returns a list of events matching the selectors.
 func (c *FakeEvents) List(label, field labels.Selector) (*api.EventList, error) {
 	c.Fake.Actions = append(c.Fake.Actions, FakeAction{Action: "list-events"})

pkg/client/record/event.go

@@ -41,6 +41,7 @@ var sleepDuration = 10 * time.Second
 // pkg/client's REST client.
 type EventRecorder interface {
 	Create(event *api.Event) (*api.Event, error)
+	Update(event *api.Event) (*api.Event, error)
 }
 
 // StartRecording starts sending events to recorder. Call once while initializing
@@ -58,9 +59,18 @@ func StartRecording(recorder EventRecorder, source api.EventSource) watch.Interf
 		event = &eventCopy
 		event.Source = source
 
+		previousEvent := GetEvent(event)
+		updateExistingEvent := previousEvent.Count > 0
+		if updateExistingEvent {
+			event.Count = previousEvent.Count + 1
+			event.FirstTimestamp = previousEvent.FirstTimestamp
+			event.Name = previousEvent.Name
+			event.ResourceVersion = previousEvent.ResourceVersion
+		}
+
 		tries := 0
 		for {
-			if recordEvent(recorder, event) {
+			if recordEvent(recorder, event, updateExistingEvent) {
 				break
 			}
 			tries++
@@ -81,11 +91,21 @@ func StartRecording(recorder EventRecorder, source api.EventSource) watch.Interf
 
 // recordEvent attempts to write event to recorder. It returns true if the event
 // was successfully recorded or discarded, false if it should be retried.
-func recordEvent(recorder EventRecorder, event *api.Event) bool {
-	_, err := recorder.Create(event)
+// If updateExistingEvent is false, it creates a new event, otherwise it updates
+// existing event.
+func recordEvent(recorder EventRecorder, event *api.Event, updateExistingEvent bool) bool {
+	var newEvent *api.Event
+	var err error
+	if updateExistingEvent {
+		newEvent, err = recorder.Update(event)
+	} else {
+		newEvent, err = recorder.Create(event)
+	}
 	if err == nil {
+		AddOrUpdateEvent(newEvent)
 		return true
 	}
+
 	// If we can't contact the server, then hold everything while we keep trying.
 	// Otherwise, something about the event is malformed and we should abandon it.
 	giveUp := false

pkg/client/record/events_cache.go

+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package record
+
+import (
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"sync"
+)
+
+type History struct {
+	// The number of times the event has occured since first occurance.
+	Count int
+
+	// The time at which the event was first recorded.
+	FirstTimestamp util.Time
+
+	// The unique name of the first occurance of this event
+	Name string
+
+	// Resource version returned from previous interaction with server
+	ResourceVersion string
+}
+
+type historyMap struct {
+	sync.RWMutex
+	table map[string]History
+}
+
+var previousEvents = historyMap{table: make(map[string]History)}
+
+// AddOrUpdateEvent creates a new entry for the given event in the previous events hash table if the event
+// doesn't already exist, otherwise it updates the existing entry.
+func AddOrUpdateEvent(newEvent *api.Event) History {
+	key := getEventKey(newEvent)
+	previousEvents.Lock()
+	defer previousEvents.Unlock()
+	previousEvents.table[key] =
+		History{
+			Count:           newEvent.Count,
+			FirstTimestamp:  newEvent.FirstTimestamp,
+			Name:            newEvent.Name,
+			ResourceVersion: newEvent.ResourceVersion,
+		}
+	return previousEvents.table[key]
+}
+
+// GetEvent returns the entry corresponding to the given event, if one exists, otherwise a History object
+// with a count of 1 is returned.
+func GetEvent(event *api.Event) History {
+	key := getEventKey(event)
+	previousEvents.RLock()
+	defer previousEvents.RUnlock()
+	return previousEvents.table[key]
+}
+
+func getEventKey(event *api.Event) string {
+	return event.Source.Component +
+		event.Source.Host +
+		event.InvolvedObject.Kind +
+		event.InvolvedObject.Namespace +
+		event.InvolvedObject.Name +
+		string(event.InvolvedObject.UID) +
+		event.InvolvedObject.APIVersion +
+		event.Reason +
+		event.Message
+}

pkg/client/record/events_cache_test.go

+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package record
+
+import (
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"testing"
+)
+
+func TestAddOrUpdateEventNoExisting(t *testing.T) {
+	// Arrange
+	eventTime := util.Now()
+	event := api.Event{
+		Reason:  "my reasons are many",
+		Message: "my message is love",
+		InvolvedObject: api.ObjectReference{
+			Kind:       "BoundPod",
+			Name:       "awesome.name",
+			Namespace:  "betterNamespace",
+			UID:        "C934D34AFB20242",
+			APIVersion: "v1beta1",
+		},
+		Source: api.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node1",
+		},
+		Count:          1,
+		FirstTimestamp: eventTime,
+		LastTimestamp:  eventTime,
+	}
+
+	// Act
+	result := AddOrUpdateEvent(&event)
+
+	// Assert
+	compareEventWithHistoryEntry(&event, &result, t)
+}
+
+func TestAddOrUpdateEventExisting(t *testing.T) {
+	// Arrange
+	event1Time := util.Unix(2324, 2342)
+	event2Time := util.Now()
+	event1 := api.Event{
+		Reason:  "something happened",
+		Message: "can you believe it?",
+		ObjectMeta: api.ObjectMeta{
+			ResourceVersion: "rs1",
+		},
+		InvolvedObject: api.ObjectReference{
+			Kind:       "Scheduler",
+			Name:       "anOkName",
+			Namespace:  "someNamespace",
+			UID:        "C934D3234CD0242",
+			APIVersion: "v1beta2",
+		},
+		Source: api.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node2",
+		},
+		Count:          1,
+		FirstTimestamp: event1Time,
+		LastTimestamp:  event1Time,
+	}
+	event2 := api.Event{
+		Reason:  "something happened",
+		Message: "can you believe it?",
+		ObjectMeta: api.ObjectMeta{
+			ResourceVersion: "rs2",
+		},
+		InvolvedObject: api.ObjectReference{
+			Kind:       "Scheduler",
+			Name:       "anOkName",
+			Namespace:  "someNamespace",
+			UID:        "C934D3234CD0242",
+			APIVersion: "v1beta2",
+		},
+		Source: api.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node2",
+		},
+		Count:          3,
+		FirstTimestamp: event1Time,
+		LastTimestamp:  event2Time,
+	}
+
+	// Act
+	AddOrUpdateEvent(&event1)
+	result1 := AddOrUpdateEvent(&event2)
+	result2 := GetEvent(&event1)
+
+	// Assert
+	compareEventWithHistoryEntry(&event2, &result1, t)
+	compareEventWithHistoryEntry(&event2, &result2, t)
+}
+
+func TestGetEventNoExisting(t *testing.T) {
+	// Arrange
+	event := api.Event{
+		Reason:  "to be or not to be",
+		Message: "do I exist",
+		InvolvedObject: api.ObjectReference{
+			Kind:       "Controller",
+			Name:       "iAmAController",
+			Namespace:  "IHaveANamespace",
+			UID:        "9039D34AFBCDA42",
+			APIVersion: "v1beta3",
+		},
+		Source: api.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node3",
+		},
+		Count: 1,
+	}
+
+	// Act
+	existingEvent := GetEvent(&event)
+
+	// Assert
+	if existingEvent.Count != 0 {
+		t.Fatalf("There should be no existing instance of this event in the hash table.")
+	}
+}
+
+func TestGetEventExisting(t *testing.T) {
+	// Arrange
+	eventTime := util.Now()
+	event := api.Event{
+		Reason:  "do I exist",
+		Message: "I do, oh my",
+		InvolvedObject: api.ObjectReference{
+			Kind:       "BoundPod",
+			Name:       "clever.name.here",
+			Namespace:  "spaceOfName",
+			UID:        "D933D32AFB2A238",
+			APIVersion: "v1beta1",
+		},
+		Source: api.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node4",
+		},
+		Count:          1,
+		FirstTimestamp: eventTime,
+		LastTimestamp:  eventTime,
+	}
+	AddOrUpdateEvent(&event)
+
+	// Act
+	existingEvent := GetEvent(&event)
+
+	// Assert
+	compareEventWithHistoryEntry(&event, &existingEvent, t)
+}
+
+func compareEventWithHistoryEntry(expected *api.Event, actual *History, t *testing.T) {
+
+	if actual.Count != expected.Count {
+		t.Fatalf("There should be one existing instance of this event in the hash table.")
+	}
+
+	if !actual.FirstTimestamp.Equal(expected.FirstTimestamp.Time) {
+		t.Fatalf("Unexpected FirstTimestamp. Expected: <%v> Actual: <%v>", expected.FirstTimestamp, actual.FirstTimestamp)
+	}
+
+	if actual.Name != expected.Name {
+		t.Fatalf("Unexpected Name. Expected: <%v> Actual: <%v>", expected.Name, actual.Name)
+	}
+
+	if actual.ResourceVersion != expected.ResourceVersion {
+		t.Fatalf("Unexpected ResourceVersion. Expected: <%v> Actual: <%v>", expected.ResourceVersion, actual.ResourceVersion)
+	}
+
+}

pkg/kubectl/describe.go

@@ -318,10 +318,12 @@ func describeEvents(el *api.EventList, w io.Writer) {
 		return
 	}
 	sort.Sort(SortableEvents(el.Items))
-	fmt.Fprint(w, "Events:\nTime\tFrom\tSubobjectPath\tReason\tMessage\n")
+	fmt.Fprint(w, "Events:\nFirstSeen\tLastSeen\tCount\tFrom\tSubobjectPath\tReason\tMessage\n")
 	for _, e := range el.Items {
-		fmt.Fprintf(w, "%s\t%v\t%v\t%v\t%v\n",
+		fmt.Fprintf(w, "%s\t%s\t%d\t%v\t%v\t%v\t%v\n",
 			e.FirstTimestamp.Time.Format(time.RFC1123Z),
+			e.LastTimestamp.Time.Format(time.RFC1123Z),
+			e.Count,
 			e.Source,
 			e.InvolvedObject.FieldPath,
 			e.Reason,

pkg/kubectl/resource_printer.go

@@ -221,7 +221,7 @@ var serviceColumns = []string{"NAME", "LABELS", "SELECTOR", "IP", "PORT"}
 var endpointColumns = []string{"NAME", "ENDPOINTS"}
 var minionColumns = []string{"NAME", "LABELS", "STATUS"}
 var statusColumns = []string{"STATUS"}
-var eventColumns = []string{"TIME", "NAME", "KIND", "SUBOBJECT", "REASON", "SOURCE", "MESSAGE"}
+var eventColumns = []string{"FIRSTSEEN", "LASTSEEN", "COUNT", "NAME", "KIND", "SUBOBJECT", "REASON", "SOURCE", "MESSAGE"}
 var limitRangeColumns = []string{"NAME"}
 var resourceQuotaColumns = []string{"NAME"}
 var namespaceColumns = []string{"NAME", "LABELS"}
@@ -423,8 +423,10 @@ func printStatus(status *api.Status, w io.Writer) error {
 
 func printEvent(event *api.Event, w io.Writer) error {
 	_, err := fmt.Fprintf(
-		w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
+		w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
 		event.FirstTimestamp.Time.Format(time.RFC1123Z),
+		event.LastTimestamp.Time.Format(time.RFC1123Z),
+		event.Count,
 		event.InvolvedObject.Name,
 		event.InvolvedObject.Kind,
 		event.InvolvedObject.FieldPath,