Merge pull request #3083 from lavalamp/optimize

cmd/integration/integration.go

@@ -63,6 +63,7 @@ var (
 type fakeKubeletClient struct{}
 
 func (fakeKubeletClient) GetPodInfo(host, podNamespace, podID string) (api.PodContainerInfo, error) {
+	glog.V(3).Infof("Trying to get container info for %v/%v/%v", host, podNamespace, podID)
 	// This is a horrible hack to get around the fact that we can't provide
 	// different port numbers per kubelet...
 	var c client.PodInfoGetter

pkg/master/ip_cache.go

+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package master
+
+import (
+	"sync"
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+
+	"github.com/golang/glog"
+)
+
+type ipCacheEntry struct {
+	ip         string
+	lastUpdate time.Time
+}
+
+type ipCache struct {
+	clock         util.Clock
+	cloudProvider cloudprovider.Interface
+	cache         map[string]ipCacheEntry
+	lock          sync.Mutex
+	ttl           time.Duration
+}
+
+// NewIPCache makes a new ip caching layer, which will get IP addresses from cp,
+// and use clock for deciding when to re-get an IP address.
+// Thread-safe.
+//
+// TODO: when we switch to go1.4, this class would be a good candidate for something
+// that could be produced from a template and a type via `go generate`.
+func NewIPCache(cp cloudprovider.Interface, clock util.Clock, ttl time.Duration) *ipCache {
+	return &ipCache{
+		clock:         clock,
+		cloudProvider: cp,
+		cache:         map[string]ipCacheEntry{},
+		ttl:           ttl,
+	}
+}
+
+// GetInstanceIP returns the IP address of host, from the cache
+// if possible, otherwise it asks the cloud provider.
+func (c *ipCache) GetInstanceIP(host string) string {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	data, ok := c.cache[host]
+	now := c.clock.Now()
+
+	if !ok || now.Sub(data.lastUpdate) > c.ttl {
+		ip := getInstanceIPFromCloud(c.cloudProvider, host)
+		data = ipCacheEntry{
+			ip:         ip,
+			lastUpdate: now,
+		}
+		c.cache[host] = data
+	}
+	return data.ip
+}
+
+func getInstanceIPFromCloud(cloud cloudprovider.Interface, host string) string {
+	if cloud == nil {
+		return ""
+	}
+	instances, ok := cloud.Instances()
+	if instances == nil || !ok {
+		return ""
+	}
+	addr, err := instances.IPAddress(host)
+	if err != nil {
+		glog.Errorf("Error getting instance IP for %q: %v", host, err)
+		return ""
+	}
+	return addr.String()
+}

pkg/master/ip_cache_test.go

+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package master
+
+import (
+	"testing"
+	"time"
+
+	fake_cloud "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/fake"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+)
+
+func TestCacheExpire(t *testing.T) {
+	fakeCloud := &fake_cloud.FakeCloud{}
+	clock := &util.FakeClock{time.Now()}
+
+	c := NewIPCache(fakeCloud, clock, 60*time.Second)
+
+	_ = c.GetInstanceIP("foo")
+	// This call should hit the cache, so we expect no additional calls to the cloud
+	_ = c.GetInstanceIP("foo")
+	// Advance the clock, this call should miss the cache, so expect one more call.
+	clock.Time = clock.Time.Add(61 * time.Second)
+	_ = c.GetInstanceIP("foo")
+
+	if len(fakeCloud.Calls) != 2 || fakeCloud.Calls[1] != "ip-address" || fakeCloud.Calls[0] != "ip-address" {
+		t.Errorf("Unexpected calls: %+v", fakeCloud.Calls)
+	}
+}
+
+func TestCacheNotExpire(t *testing.T) {
+	fakeCloud := &fake_cloud.FakeCloud{}
+	clock := &util.FakeClock{time.Now()}
+
+	c := NewIPCache(fakeCloud, clock, 60*time.Second)
+
+	_ = c.GetInstanceIP("foo")
+	// This call should hit the cache, so we expect no additional calls to the cloud
+	clock.Time = clock.Time.Add(60 * time.Second)
+	_ = c.GetInstanceIP("foo")
+
+	if len(fakeCloud.Calls) != 1 || fakeCloud.Calls[0] != "ip-address" {
+		t.Errorf("Unexpected calls: %+v", fakeCloud.Calls)
+	}
+}

pkg/master/master.go

@@ -319,28 +319,24 @@ func makeMinionRegistry(c *Config) minion.Registry {
 
 // init initializes master.
 func (m *Master) init(c *Config) {
-	podCache := NewPodCache(c.KubeletClient, m.podRegistry)
-	go util.Forever(func() { podCache.UpdateAllContainers() }, time.Second*30)
-
 	var userContexts = handlers.NewUserRequestContext()
 	var authenticator = c.Authenticator
 
 	nodeRESTStorage := minion.NewREST(m.minionRegistry)
+	ipCache := NewIPCache(c.Cloud, util.RealClock{}, 30*time.Second)
+	podCache := NewPodCache(
+		ipCache,
+		c.KubeletClient,
+		RESTStorageToNodes(nodeRESTStorage).Nodes(),
+		m.podRegistry,
+	)
+	go util.Forever(func() { podCache.UpdateAllContainers() }, time.Second*30)
 
 	// TODO: Factor out the core API registration
 	m.storage = map[string]apiserver.RESTStorage{
 		"pods": pod.NewREST(&pod.RESTConfig{
-			CloudProvider: c.Cloud,
-			PodCache:      podCache,
-			PodInfoGetter: c.KubeletClient,
-			Registry:      m.podRegistry,
-			// Note: this allows the pod rest object to directly call
-			// the node rest object without going through the network &
-			// apiserver. This arrangement should be temporary, nodes
-			// shouldn't really need this at all. Once we add more auth in,
-			// we need to consider carefully if this sort of shortcut is a
-			// good idea.
-			Nodes: RESTStorageToNodes(nodeRESTStorage).Nodes(),
+			PodCache: podCache,
+			Registry: m.podRegistry,
 		}),
 		"replicationControllers": controller.NewREST(m.controllerRegistry, m.podRegistry),
 		"services":               service.NewREST(m.serviceRegistry, c.Cloud, m.minionRegistry, m.portalNet),

pkg/master/pod_cache.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry/pod"
@@ -27,69 +28,238 @@ import (
 	"github.com/golang/glog"
 )
 
+type IPGetter interface {
+	GetInstanceIP(host string) (ip string)
+}
+
 // PodCache contains both a cache of container information, as well as the mechanism for keeping
 // that cache up to date.
 type PodCache struct {
+	ipCache       IPGetter
 	containerInfo client.PodInfoGetter
 	pods          pod.Registry
-	// This is a map of pod id to a map of container name to the
-	podInfo map[string]api.PodContainerInfo
-	podLock sync.Mutex
+	// For confirming existance of a node
+	nodes client.NodeInterface
+
+	// lock protects access to all fields below
+	lock sync.Mutex
+	// cached pod statuses.
+	podStatus map[objKey]api.PodStatus
+	// nodes that we know exist. Cleared at the beginning of each
+	// UpdateAllPods call.
+	currentNodes map[objKey]bool
+}
+
+type objKey struct {
+	namespace, name string
 }
 
-// NewPodCache returns a new PodCache which watches container information registered in the given PodRegistry.
-func NewPodCache(info client.PodInfoGetter, pods pod.Registry) *PodCache {
+// NewPodCache returns a new PodCache which watches container information
+// registered in the given PodRegistry.
+// TODO(lavalamp): pods should be a client.PodInterface.
+func NewPodCache(ipCache IPGetter, info client.PodInfoGetter, nodes client.NodeInterface, pods pod.Registry) *PodCache {
 	return &PodCache{
+		ipCache:       ipCache,
 		containerInfo: info,
 		pods:          pods,
-		podInfo:       map[string]api.PodContainerInfo{},
+		nodes:         nodes,
+		currentNodes:  map[objKey]bool{},
+		podStatus:     map[objKey]api.PodStatus{},
 	}
 }
 
-// makePodCacheKey constructs a key for use in a map to address a pod with specified namespace and id
-func makePodCacheKey(podNamespace, podID string) string {
-	return podNamespace + "." + podID
+// GetPodStatus gets the stored pod status.
+func (p *PodCache) GetPodStatus(namespace, name string) (*api.PodStatus, error) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	value, ok := p.podStatus[objKey{namespace, name}]
+	if !ok {
+		return nil, client.ErrPodInfoNotAvailable
+	}
+	// Make a copy
+	return &value, nil
 }
 
-// GetPodInfo implements the PodInfoGetter.GetPodInfo.
-// The returned value should be treated as read-only.
-// TODO: Remove the host from this call, it's totally unnecessary.
-func (p *PodCache) GetPodInfo(host, podNamespace, podID string) (api.PodContainerInfo, error) {
-	p.podLock.Lock()
-	defer p.podLock.Unlock()
-	value, ok := p.podInfo[makePodCacheKey(podNamespace, podID)]
-	if !ok {
-		return api.PodContainerInfo{}, client.ErrPodInfoNotAvailable
+func (p *PodCache) nodeExistsInCache(name string) (exists, cacheHit bool) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	exists, cacheHit = p.currentNodes[objKey{"", name}]
+	return exists, cacheHit
+}
+
+// lock must *not* be held
+func (p *PodCache) nodeExists(name string) bool {
+	exists, cacheHit := p.nodeExistsInCache(name)
+	if cacheHit {
+		return exists
 	}
-	return value, nil
+	// TODO: suppose there's N concurrent requests for node "foo"; in that case
+	// it might be useful to block all of them and only look up "foo" once.
+	// (This code will make up to N lookups.) One way of doing that would be to
+	// have a pool of M mutexes and require that before looking up "foo" you must
+	// lock mutex hash("foo") % M.
+	_, err := p.nodes.Get(name)
+	exists = true
+	if err != nil {
+		exists = false
+		if !errors.IsNotFound(err) {
+			glog.Errorf("Unexpected error type verifying minion existence: %+v", err)
+		}
+	}
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	p.currentNodes[objKey{"", name}] = exists
+	return exists
+}
+
+// TODO: once Host gets moved to spec, this can take a podSpec + metadata instead of an
+// entire pod?
+func (p *PodCache) updatePodStatus(pod *api.Pod) error {
+	newStatus, err := p.computePodStatus(pod)
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	// Map accesses must be locked.
+	p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
+
+	return err
 }
 
-func (p *PodCache) updatePodInfo(host, podNamespace, podID string) error {
-	info, err := p.containerInfo.GetPodInfo(host, podNamespace, podID)
+// computePodStatus always returns a new status, even if it also returns a non-nil error.
+// TODO: once Host gets moved to spec, this can take a podSpec + metadata instead of an
+// entire pod?
+func (p *PodCache) computePodStatus(pod *api.Pod) (api.PodStatus, error) {
+	newStatus := pod.Status
+
+	if pod.Status.Host == "" {
+		// Not assigned.
+		newStatus.Phase = api.PodPending
+		return newStatus, nil
+	}
+
+	if !p.nodeExists(pod.Status.Host) {
+		// Assigned to non-existing node.
+		newStatus.Phase = api.PodFailed
+		return newStatus, nil
+	}
+
+	info, err := p.containerInfo.GetPodInfo(pod.Status.Host, pod.Namespace, pod.Name)
+	newStatus.HostIP = p.ipCache.GetInstanceIP(pod.Status.Host)
+
 	if err != nil {
-		return err
+		newStatus.Phase = api.PodUnknown
+	} else {
+		newStatus.Info = info.ContainerInfo
+		newStatus.Phase = getPhase(&pod.Spec, newStatus.Info)
+		if netContainerInfo, ok := newStatus.Info["net"]; ok {
+			if netContainerInfo.PodIP != "" {
+				newStatus.PodIP = netContainerInfo.PodIP
+			}
+		}
 	}
-	p.podLock.Lock()
-	defer p.podLock.Unlock()
-	p.podInfo[makePodCacheKey(podNamespace, podID)] = info
-	return nil
+	return newStatus, err
+}
+
+func (p *PodCache) resetNodeExistenceCache() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	p.currentNodes = map[objKey]bool{}
 }
 
-// UpdateAllContainers updates information about all containers.  Either called by Loop() below, or one-off.
+// UpdateAllContainers updates information about all containers.
+// Callers should let one call to UpdateAllContainers finish before
+// calling again, or risk having new info getting clobbered by delayed
+// old info.
 func (p *PodCache) UpdateAllContainers() {
+	p.resetNodeExistenceCache()
+
 	ctx := api.NewContext()
 	pods, err := p.pods.ListPods(ctx, labels.Everything())
 	if err != nil {
-		glog.Errorf("Error synchronizing container list: %v", err)
+		glog.Errorf("Error getting pod list: %v", err)
 		return
 	}
-	for _, pod := range pods.Items {
-		if pod.Status.Host == "" {
-			continue
+
+	// TODO: this algorithm is 1 goroutine & RPC per pod. With a little work,
+	// it should be possible to make it 1 per *node*, which will be important
+	// at very large scales. (To be clear, the goroutines shouldn't matter--
+	// it's the RPCs that need to be minimized.)
+	var wg sync.WaitGroup
+	for i := range pods.Items {
+		pod := &pods.Items[i]
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := p.updatePodStatus(pod)
+			if err != nil && err != client.ErrPodInfoNotAvailable {
+				glog.Errorf("Error getting info for pod %v/%v: %v", pod.Namespace, pod.Name, err)
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+// getPhase returns the phase of a pod given its container info.
+// TODO(dchen1107): push this all the way down into kubelet.
+func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
+	if info == nil {
+		return api.PodPending
+	}
+	running := 0
+	waiting := 0
+	stopped := 0
+	failed := 0
+	succeeded := 0
+	unknown := 0
+	for _, container := range spec.Containers {
+		if containerStatus, ok := info[container.Name]; ok {
+			if containerStatus.State.Running != nil {
+				running++
+			} else if containerStatus.State.Termination != nil {
+				stopped++
+				if containerStatus.State.Termination.ExitCode == 0 {
+					succeeded++
+				} else {
+					failed++
+				}
+			} else if containerStatus.State.Waiting != nil {
+				waiting++
+			} else {
+				unknown++
+			}
+		} else {
+			unknown++
+		}
+	}
+	switch {
+	case waiting > 0:
+		// One or more containers has not been started
+		return api.PodPending
+	case running > 0 && unknown == 0:
+		// All containers have been started, and at least
+		// one container is running
+		return api.PodRunning
+	case running == 0 && stopped > 0 && unknown == 0:
+		// All containers are terminated
+		if spec.RestartPolicy.Always != nil {
+			// All containers are in the process of restarting
+			return api.PodRunning
+		}
+		if stopped == succeeded {
+			// RestartPolicy is not Always, and all
+			// containers are terminated in success
+			return api.PodSucceeded
 		}
-		err := p.updatePodInfo(pod.Status.Host, pod.Namespace, pod.Name)
-		if err != nil && err != client.ErrPodInfoNotAvailable {
-			glog.Errorf("Error synchronizing container: %v", err)
+		if spec.RestartPolicy.Never != nil {
+			// RestartPolicy is Never, and all containers are
+			// terminated with at least one in failure
+			return api.PodFailed
 		}
+		// RestartPolicy is OnFailure, and at least one in failure
+		// and in the process of restarting
+		return api.PodRunning
+	default:
+		return api.PodPending
 	}
 }

pkg/registry/pod/rest.go

@@ -18,72 +18,37 @@ package pod
 
 import (
 	"fmt"
-	"sync"
-	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
-
-	"github.com/golang/glog"
 )
 
-type ipCacheEntry struct {
-	ip         string
-	lastUpdate time.Time
-}
-
-type ipCache map[string]ipCacheEntry
-
-type clock interface {
-	Now() time.Time
-}
-
-type realClock struct{}
-
-func (r realClock) Now() time.Time {
-	return time.Now()
+type PodStatusGetter interface {
+	GetPodStatus(namespace, name string) (*api.PodStatus, error)
 }
 
 // REST implements the RESTStorage interface in terms of a PodRegistry.
 type REST struct {
-	cloudProvider cloudprovider.Interface
-	mu            sync.Mutex
-	podCache      client.PodInfoGetter
-	podInfoGetter client.PodInfoGetter
-	podPollPeriod time.Duration
-	registry      Registry
-	nodes         client.NodeInterface
-	ipCache       ipCache
-	clock         clock
+	podCache PodStatusGetter
+	registry Registry
 }
 
 type RESTConfig struct {
-	CloudProvider cloudprovider.Interface
-	PodCache      client.PodInfoGetter
-	PodInfoGetter client.PodInfoGetter
-	Registry      Registry
-	Nodes         client.NodeInterface
+	PodCache PodStatusGetter
+	Registry Registry
 }
 
 // NewREST returns a new REST.
 func NewREST(config *RESTConfig) *REST {
 	return &REST{
-		cloudProvider: config.CloudProvider,
-		podCache:      config.PodCache,
-		podInfoGetter: config.PodInfoGetter,
-		podPollPeriod: time.Second * 10,
-		registry:      config.Registry,
-		nodes:         config.Nodes,
-		ipCache:       ipCache{},
-		clock:         realClock{},
+		podCache: config.PodCache,
+		registry: config.Registry,
 	}
 }
 
@@ -123,17 +88,17 @@ func (rs *REST) Get(ctx api.Context, id string) (runtime.Object, error) {
 	if pod == nil {
 		return pod, nil
 	}
-	if rs.podCache != nil || rs.podInfoGetter != nil {
-		rs.fillPodInfo(pod)
-		status, err := getPodStatus(pod, rs.nodes)
-		if err != nil {
-			return pod, err
+	host := pod.Status.Host
+	if status, err := rs.podCache.GetPodStatus(pod.Namespace, pod.Name); err != nil {
+		pod.Status = api.PodStatus{
+			Phase: api.PodUnknown,
 		}
-		pod.Status.Phase = status
-	}
-	if pod.Status.Host != "" {
-		pod.Status.HostIP = rs.getInstanceIP(pod.Status.Host)
+	} else {
+		pod.Status = *status
 	}
+	// Make sure not to hide a recent host with an old one from the cache.
+	// TODO: move host to spec
+	pod.Status.Host = host
 	return pod, err
 }
 
@@ -168,15 +133,18 @@ func (rs *REST) List(ctx api.Context, label, field labels.Selector) (runtime.Obj
 	if err == nil {
 		for i := range pods.Items {
 			pod := &pods.Items[i]
-			rs.fillPodInfo(pod)
-			status, err := getPodStatus(pod, rs.nodes)
-			if err != nil {
-				status = api.PodUnknown
-			}
-			pod.Status.Phase = status
-			if pod.Status.Host != "" {
-				pod.Status.HostIP = rs.getInstanceIP(pod.Status.Host)
+			host := pod.Status.Host
+			if status, err := rs.podCache.GetPodStatus(pod.Namespace, pod.Name); err != nil {
+				pod.Status = api.PodStatus{
+					Phase: api.PodUnknown,
+				}
+			} else {
+				pod.Status = *status
 			}
+			// Make sure not to hide a recent host with an old one from the cache.
+			// This is tested by the integration test.
+			// TODO: move host to spec
+			pod.Status.Host = host
 		}
 	}
 	return pods, err
@@ -207,148 +175,3 @@ func (rs *REST) Update(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE
 		return rs.registry.GetPod(ctx, pod.Name)
 	}), nil
 }
-
-func (rs *REST) fillPodInfo(pod *api.Pod) {
-	if pod.Status.Host == "" {
-		return
-	}
-	// Get cached info for the list currently.
-	// TODO: Optionally use fresh info
-	if rs.podCache != nil {
-		info, err := rs.podCache.GetPodInfo(pod.Status.Host, pod.Namespace, pod.Name)
-		if err != nil {
-			if err != client.ErrPodInfoNotAvailable {
-				glog.Errorf("Error getting container info from cache: %v", err)
-			}
-			if rs.podInfoGetter != nil {
-				info, err = rs.podInfoGetter.GetPodInfo(pod.Status.Host, pod.Namespace, pod.Name)
-			}
-			if err != nil {
-				if err != client.ErrPodInfoNotAvailable {
-					glog.Errorf("Error getting fresh container info: %v", err)
-				}
-				return
-			}
-		}
-		pod.Status.Info = info.ContainerInfo
-		netContainerInfo, ok := pod.Status.Info["net"]
-		if ok {
-			if netContainerInfo.PodIP != "" {
-				pod.Status.PodIP = netContainerInfo.PodIP
-			} else if netContainerInfo.State.Running != nil {
-				glog.Warningf("No network settings: %#v", netContainerInfo)
-			}
-		} else {
-			glog.Warningf("Couldn't find network container for %s in %v", pod.Name, info)
-		}
-	}
-}
-
-func (rs *REST) getInstanceIP(host string) string {
-	data, ok := rs.ipCache[host]
-	now := rs.clock.Now()
-
-	if !ok || now.Sub(data.lastUpdate) > (30*time.Second) {
-		ip := getInstanceIPFromCloud(rs.cloudProvider, host)
-		data = ipCacheEntry{
-			ip:         ip,
-			lastUpdate: now,
-		}
-		rs.ipCache[host] = data
-	}
-	return data.ip
-}
-
-func getInstanceIPFromCloud(cloud cloudprovider.Interface, host string) string {
-	if cloud == nil {
-		return ""
-	}
-	instances, ok := cloud.Instances()
-	if instances == nil || !ok {
-		return ""
-	}
-	addr, err := instances.IPAddress(host)
-	if err != nil {
-		glog.Errorf("Error getting instance IP for %q: %v", host, err)
-		return ""
-	}
-	return addr.String()
-}
-
-func getPodStatus(pod *api.Pod, nodes client.NodeInterface) (api.PodPhase, error) {
-	if pod.Status.Host == "" {
-		return api.PodPending, nil
-	}
-	if nodes != nil {
-		_, err := nodes.Get(pod.Status.Host)
-		if err != nil {
-			if errors.IsNotFound(err) {
-				return api.PodFailed, nil
-			}
-			glog.Errorf("Error getting pod info: %v", err)
-			return api.PodUnknown, nil
-		}
-	} else {
-		glog.Errorf("Unexpected missing minion interface, status may be in-accurate")
-	}
-	if pod.Status.Info == nil {
-		return api.PodPending, nil
-	}
-	// TODO(dchen1107): move the entire logic to kubelet?
-	running := 0
-	waiting := 0
-	stopped := 0
-	failed := 0
-	succeeded := 0
-	unknown := 0
-	for _, container := range pod.Spec.Containers {
-		if containerStatus, ok := pod.Status.Info[container.Name]; ok {
-			if containerStatus.State.Running != nil {
-				running++
-			} else if containerStatus.State.Termination != nil {
-				stopped++
-				if containerStatus.State.Termination.ExitCode == 0 {
-					succeeded++
-				} else {
-					failed++
-				}
-			} else if containerStatus.State.Waiting != nil {
-				waiting++
-			} else {
-				unknown++
-			}
-		} else {
-			unknown++
-		}
-	}
-	switch {
-	case waiting > 0:
-		// One or more containers has not been started
-		return api.PodPending, nil
-	case running > 0 && unknown == 0:
-		// All containers have been started, and at least
-		// one container is running
-		return api.PodRunning, nil
-	case running == 0 && stopped > 0 && unknown == 0:
-		// All containers are terminated
-		if pod.Spec.RestartPolicy.Always != nil {
-			// All containers are in the process of restarting
-			return api.PodRunning, nil
-		}
-		if stopped == succeeded {
-			// RestartPolicy is not Always, and all
-			// containers are terminated in success
-			return api.PodSucceeded, nil
-		}
-		if pod.Spec.RestartPolicy.Never != nil {
-			// RestartPolicy is Never, and all containers are
-			// terminated with at least one in failure
-			return api.PodFailed, nil
-		}
-		// RestartPolicy is OnFailure, and at least one in failure
-		// and in the process of restarting
-		return api.PodRunning, nil
-	default:
-		return api.PodPending, nil
-	}
-}

pkg/util/clock.go

+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"time"
+)
+
+// Clock allows for injecting fake or real clocks into code that
+// needs to do arbitrary things based on time.
+type Clock interface {
+	Now() time.Time
+}
+
+// RealClock really calls time.Now()
+type RealClock struct{}
+
+// Now returns the current time.
+func (r RealClock) Now() time.Time {
+	return time.Now()
+}
+
+// FakeClock implements Clock, but returns an arbitary time.
+type FakeClock struct {
+	Time time.Time
+}
+
+// Now returns f's time.
+func (f *FakeClock) Now() time.Time {
+	return f.Time
+}