Merge github.com:GoogleCloudPlatform/kubernetes

Godeps/Godeps.json

@@ -35,6 +35,10 @@
 			"Rev": "98c78185197025f935947caac56a7b6d022f89d2"
 		},
 		{
+			"ImportPath": "github.com/GoogleCloudPlatform/gcloud-golang/compute/metadata",
+			"Rev": "e34a32f9b0ecbc0784865fb2d47f3818c09521d4"
+		},
+		{
 			"ImportPath": "github.com/Sirupsen/logrus",
 			"Comment": "v0.6.2-10-g51fe59a",
 			"Rev": "51fe59aca108dc5680109e7b2051cbdcfa5a253c"
@@ -246,93 +250,93 @@
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/api",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/cache/memory",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/collector",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/container",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/events",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/fs",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/healthz",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/http",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/info/v1",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/info/v2",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/manager",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/metrics",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/pages",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/storage",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/summary",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/utils",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/validate",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/version",
-			"Comment": "0.15.1",
-			"Rev": "ec588def40e1bb59f28f5a293b279f6762d13d44"
+			"Comment": "0.16.0-51-g78419de",
+			"Rev": "78419de3ea9c2d23cb04ec9d63f8899de34ebd43"
 		},
 		{
 			"ImportPath": "github.com/google/go-github/github",
@@ -425,10 +429,10 @@
 			"ImportPath": "github.com/mitchellh/mapstructure",
 			"Rev": "740c764bc6149d3f1806231418adb9f52c11bcbf"
 		},
-                {
-                        "ImportPath": "github.com/mxk/go-flowrate/flowrate",
-                        "Rev": "cca7078d478f8520f85629ad7c68962d31ed7682"
-                },
+		{
+			"ImportPath": "github.com/mxk/go-flowrate/flowrate",
+			"Rev": "cca7078d478f8520f85629ad7c68962d31ed7682"
+		},
 		{
 			"ImportPath": "github.com/onsi/ginkgo",
 			"Comment": "v1.2.0-6-gd981d36",
@@ -574,6 +578,10 @@
 			"Rev": "d466437aa4adc35830964cffc5b5f262c63ddcb4"
 		},
 		{
+			"ImportPath": "gopkg.in/natefinch/lumberjack.v2/",
+			"Rev": "20b71e5b60d756d3d2f80def009790325acc2b23"
+		},
+		{
 			"ImportPath": "speter.net/go/exp/math/dec/inf",
 			"Rev": "42ca6cd68aa922bc3f32f1e056e61b65945d9ad7"
 		}

Godeps/_workspace/src/github.com/GoogleCloudPlatform/gcloud-golang/compute/metadata/metadata.go

+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package metadata provides access to Google Compute Engine (GCE)
+// metadata and API service accounts.
+//
+// This package is a wrapper around the GCE metadata service,
+// as documented at https://developers.google.com/compute/docs/metadata.
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/cloud/internal"
+)
+
+type cachedValue struct {
+	k    string
+	trim bool
+	mu   sync.Mutex
+	v    string
+}
+
+var (
+	projID  = &cachedValue{k: "project/project-id", trim: true}
+	projNum = &cachedValue{k: "project/numeric-project-id", trim: true}
+	instID  = &cachedValue{k: "instance/id", trim: true}
+)
+
+var metaClient = &http.Client{
+	Transport: &internal.Transport{
+		Base: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   750 * time.Millisecond,
+				KeepAlive: 30 * time.Second,
+			}).Dial,
+			ResponseHeaderTimeout: 750 * time.Millisecond,
+		},
+	},
+}
+
+// NotDefinedError is returned when requested metadata is not defined.
+//
+// The underlying string is the suffix after "/computeMetadata/v1/".
+//
+// This error is not returned if the value is defined to be the empty
+// string.
+type NotDefinedError string
+
+func (suffix NotDefinedError) Error() string {
+	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
+}
+
+// Get returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+func Get(suffix string) (string, error) {
+	// Using a fixed IP makes it very difficult to spoof the metadata service in
+	// a container, which is an important use-case for local testing of cloud
+	// deployments. To enable spoofing of the metadata service, the environment
+	// variable GCE_METADATA_HOST is first inspected to decide where metadata
+	// requests shall go.
+	host := os.Getenv("GCE_METADATA_HOST")
+	if host == "" {
+		// Using 169.254.169.254 instead of "metadata" here because Go
+		// binaries built with the "netgo" tag and without cgo won't
+		// know the search suffix for "metadata" is
+		// ".google.internal", and this IP address is documented as
+		// being stable anyway.
+		host = "169.254.169.254"
+	}
+	url := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Metadata-Flavor", "Google")
+	res, err := metaClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return "", NotDefinedError(suffix)
+	}
+	if res.StatusCode != 200 {
+		return "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
+	}
+	all, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(all), nil
+}
+
+func getTrimmed(suffix string) (s string, err error) {
+	s, err = Get(suffix)
+	s = strings.TrimSpace(s)
+	return
+}
+
+func (c *cachedValue) get() (v string, err error) {
+	defer c.mu.Unlock()
+	c.mu.Lock()
+	if c.v != "" {
+		return c.v, nil
+	}
+	if c.trim {
+		v, err = getTrimmed(c.k)
+	} else {
+		v, err = Get(c.k)
+	}
+	if err == nil {
+		c.v = v
+	}
+	return
+}
+
+var onGCE struct {
+	sync.Mutex
+	set bool
+	v   bool
+}
+
+// OnGCE reports whether this process is running on Google Compute Engine.
+func OnGCE() bool {
+	defer onGCE.Unlock()
+	onGCE.Lock()
+	if onGCE.set {
+		return onGCE.v
+	}
+	onGCE.set = true
+
+	// We use the DNS name of the metadata service here instead of the IP address
+	// because we expect that to fail faster in the not-on-GCE case.
+	res, err := metaClient.Get("http://metadata.google.internal")
+	if err != nil {
+		return false
+	}
+	onGCE.v = res.Header.Get("Metadata-Flavor") == "Google"
+	return onGCE.v
+}
+
+// ProjectID returns the current instance's project ID string.
+func ProjectID() (string, error) { return projID.get() }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func NumericProjectID() (string, error) { return projNum.get() }
+
+// InternalIP returns the instance's primary internal IP address.
+func InternalIP() (string, error) {
+	return getTrimmed("instance/network-interfaces/0/ip")
+}
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func ExternalIP() (string, error) {
+	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+}
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func Hostname() (string, error) {
+	return getTrimmed("instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTags() ([]string, error) {
+	var s []string
+	j, err := Get("instance/tags")
+	if err != nil {
+		return nil, err
+	}
+	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// InstanceID returns the current VM's numeric instance ID.
+func InstanceID() (string, error) {
+	return instID.get()
+}
+
+// InstanceName returns the current VM's instance ID string.
+func InstanceName() (string, error) {
+	host, err := Hostname()
+	if err != nil {
+		return "", err
+	}
+	return strings.Split(host, ".")[0], nil
+}
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func Zone() (string, error) {
+	zone, err := getTrimmed("instance/zone")
+	// zone is of the form "projects/<projNum>/zones/<zoneName>".
+	if err != nil {
+		return "", err
+	}
+	return zone[strings.LastIndex(zone, "/")+1:], nil
+}
+
+// InstanceAttributes returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }
+
+// ProjectAttributes returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func ProjectAttributes() ([]string, error) { return lines("project/attributes/") }
+
+func lines(suffix string) ([]string, error) {
+	j, err := Get(suffix)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.Split(strings.TrimSpace(j), "\n")
+	for i := range s {
+		s[i] = strings.TrimSpace(s[i])
+	}
+	return s, nil
+}
+
+// InstanceAttributeValue returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func InstanceAttributeValue(attr string) (string, error) {
+	return Get("instance/attributes/" + attr)
+}
+
+// ProjectAttributeValue returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func ProjectAttributeValue(attr string) (string, error) {
+	return Get("project/attributes/" + attr)
+}
+
+// Scopes returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func Scopes(serviceAccount string) ([]string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return lines("instance/service-accounts/" + serviceAccount + "/scopes")
+}

Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go

@@ -39,6 +39,7 @@ const (
 	attributesApi    = "attributes"
 	versionApi       = "version"
 	psApi            = "ps"
+	customMetricsApi = "appmetrics"
 )
 
 // Interface for a cAdvisor API version
@@ -305,7 +306,7 @@ func (self *version2_0) Version() string {
 }
 
 func (self *version2_0) SupportedRequestTypes() []string {
-	return []string{versionApi, attributesApi, eventsApi, machineApi, summaryApi, statsApi, specApi, storageApi, psApi}
+	return []string{versionApi, attributesApi, eventsApi, machineApi, summaryApi, statsApi, specApi, storageApi, psApi, customMetricsApi}
 }
 
 func (self *version2_0) HandleRequest(requestType string, request []string, m manager.Manager, w http.ResponseWriter, r *http.Request) error {
@@ -364,6 +365,32 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 			contStats[name] = convertStats(cont)
 		}
 		return writeResult(contStats, w)
+	case customMetricsApi:
+		containerName := getContainerName(request)
+		glog.V(4).Infof("Api - Custom Metrics: Looking for metrics for container %q, options %+v", containerName, opt)
+		conts, err := m.GetRequestedContainersInfo(containerName, opt)
+		if err != nil {
+			return err
+		}
+		specs, err := m.GetContainerSpec(containerName, opt)
+		if err != nil {
+			return err
+		}
+		contMetrics := make(map[string]map[string][]info.MetricVal, 0)
+		for _, cont := range conts {
+			metrics := map[string][]info.MetricVal{}
+			contStats := convertStats(cont)
+			spec := specs[cont.Name]
+			for _, contStat := range contStats {
+				for _, ms := range spec.CustomMetrics {
+					if contStat.HasCustomMetrics && !contStat.CustomMetrics[ms.Name].Timestamp.IsZero() {
+						metrics[ms.Name] = append(metrics[ms.Name], contStat.CustomMetrics[ms.Name])
+					}
+				}
+			}
+			contMetrics[containerName] = metrics
+		}
+		return writeResult(contMetrics, w)
 	case specApi:
 		containerName := getContainerName(request)
 		glog.V(4).Infof("Api - Spec for container %q, options %+v", containerName, opt)
@@ -412,12 +439,13 @@ func convertStats(cont *info.ContainerInfo) []v2.ContainerStats {
 	stats := []v2.ContainerStats{}
 	for _, val := range cont.Stats {
 		stat := v2.ContainerStats{
-			Timestamp:     val.Timestamp,
-			HasCpu:        cont.Spec.HasCpu,
-			HasMemory:     cont.Spec.HasMemory,
-			HasNetwork:    cont.Spec.HasNetwork,
-			HasFilesystem: cont.Spec.HasFilesystem,
-			HasDiskIo:     cont.Spec.HasDiskIo,
+			Timestamp:        val.Timestamp,
+			HasCpu:           cont.Spec.HasCpu,
+			HasMemory:        cont.Spec.HasMemory,
+			HasNetwork:       cont.Spec.HasNetwork,
+			HasFilesystem:    cont.Spec.HasFilesystem,
+			HasDiskIo:        cont.Spec.HasDiskIo,
+			HasCustomMetrics: cont.Spec.HasCustomMetrics,
 		}
 		if stat.HasCpu {
 			stat.Cpu = val.Cpu
@@ -434,6 +462,9 @@ func convertStats(cont *info.ContainerInfo) []v2.ContainerStats {
 		if stat.HasDiskIo {
 			stat.DiskIo = val.DiskIo
 		}
+		if stat.HasCustomMetrics {
+			stat.CustomMetrics = val.CustomMetrics
+		}
 		// TODO(rjnagal): Handle load stats.
 		stats = append(stats, stat)
 	}

Godeps/_workspace/src/github.com/google/cadvisor/collector/collector_manager.go

@@ -19,14 +19,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v1"
 )
 
-type collectorManager struct {
-	collectors []*collectorData
-}
+const metricLabelPrefix = "io.cadvisor.metric."
 
-var _ CollectorManager = &collectorManager{}
+type GenericCollectorManager struct {
+	Collectors         []*collectorData
+	NextCollectionTime time.Time
+}
 
 type collectorData struct {
 	collector          Collector
@@ -35,33 +36,54 @@ type collectorData struct {
 
 // Returns a new CollectorManager that is thread-compatible.
 func NewCollectorManager() (CollectorManager, error) {
-	return &collectorManager{
-		collectors: []*collectorData{},
+	return &GenericCollectorManager{
+		Collectors:         []*collectorData{},
+		NextCollectionTime: time.Now(),
 	}, nil
 }
 
-func (cm *collectorManager) RegisterCollector(collector Collector) error {
-	cm.collectors = append(cm.collectors, &collectorData{
+func GetCollectorConfigs(labels map[string]string) map[string]string {
+	configs := map[string]string{}
+	for k, v := range labels {
+		if strings.HasPrefix(k, metricLabelPrefix) {
+			name := strings.TrimPrefix(k, metricLabelPrefix)
+			configs[name] = v
+		}
+	}
+	return configs
+}
+
+func (cm *GenericCollectorManager) RegisterCollector(collector Collector) error {
+	cm.Collectors = append(cm.Collectors, &collectorData{
 		collector:          collector,
 		nextCollectionTime: time.Now(),
 	})
 	return nil
 }
 
-func (cm *collectorManager) Collect() (time.Time, []v2.Metric, error) {
+func (cm *GenericCollectorManager) GetSpec() ([]v1.MetricSpec, error) {
+	metricSpec := []v1.MetricSpec{}
+	for _, c := range cm.Collectors {
+		specs := c.collector.GetSpec()
+		metricSpec = append(metricSpec, specs...)
+	}
+
+	return metricSpec, nil
+}
+
+func (cm *GenericCollectorManager) Collect() (time.Time, map[string]v1.MetricVal, error) {
 	var errors []error
 
 	// Collect from all collectors that are ready.
 	var next time.Time
-	var metrics []v2.Metric
-	for _, c := range cm.collectors {
+	metrics := map[string]v1.MetricVal{}
+	for _, c := range cm.Collectors {
 		if c.nextCollectionTime.Before(time.Now()) {
-			nextCollection, newMetrics, err := c.collector.Collect()
+			var err error
+			c.nextCollectionTime, metrics, err = c.collector.Collect(metrics)
 			if err != nil {
 				errors = append(errors, err)
 			}
-			metrics = append(metrics, newMetrics...)
-			c.nextCollectionTime = nextCollection
 		}
 
 		// Keep track of the next collector that will be ready.
@@ -69,7 +91,7 @@ func (cm *collectorManager) Collect() (time.Time, []v2.Metric, error) {
 			next = c.nextCollectionTime
 		}
 	}
-
+	cm.NextCollectionTime = next
 	return next, metrics, compileErrors(errors)
 }
 

Godeps/_workspace/src/github.com/google/cadvisor/collector/collector_manager_test.go

@@ -18,7 +18,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v1"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,17 +28,21 @@ type fakeCollector struct {
 	collectedFrom      int
 }
 
-func (fc *fakeCollector) Collect() (time.Time, []v2.Metric, error) {
+func (fc *fakeCollector) Collect(metric map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
 	fc.collectedFrom++
-	return fc.nextCollectionTime, []v2.Metric{}, fc.err
+	return fc.nextCollectionTime, metric, fc.err
 }
 
 func (fc *fakeCollector) Name() string {
 	return "fake-collector"
 }
 
+func (fc *fakeCollector) GetSpec() []v1.MetricSpec {
+	return []v1.MetricSpec{}
+}
+
 func TestCollect(t *testing.T) {
-	cm := &collectorManager{}
+	cm := &GenericCollectorManager{}
 
 	firstTime := time.Now().Add(-time.Hour)
 	secondTime := time.Now().Add(time.Hour)

Godeps/_workspace/src/github.com/google/cadvisor/collector/config.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"github.com/google/cadvisor/info/v1"
+	"time"
+)
+
+type Config struct {
+	//the endpoint to hit to scrape metrics
+	Endpoint string `json:"endpoint"`
+
+	//holds information about different metrics that can be collected
+	MetricsConfig []MetricConfig `json:"metrics_config"`
+}
+
+// metricConfig holds information extracted from the config file about a metric
+type MetricConfig struct {
+	//the name of the metric
+	Name string `json:"name"`
+
+	//enum type for the metric type
+	MetricType v1.MetricType `json:"metric_type"`
+
+	// metric units to display on UI and in storage (eg: MB, cores)
+	// this is only used for display.
+	Units string `json:"units"`
+
+	//data type of the metric (eg: int, float)
+	DataType v1.DataType `json:"data_type"`
+
+	//the frequency at which the metric should be collected
+	PollingFrequency time.Duration `json:"polling_frequency"`
+
+	//the regular expression that can be used to extract the metric
+	Regex string `json:"regex"`
+}

Godeps/_workspace/src/github.com/google/cadvisor/collector/config/sample_config.json

+{
+	"endpoint" : "http://localhost:8000/nginx_status",
+	"metrics_config"  : [
+		{ "name" : "activeConnections",  
+		  "metric_type" : "gauge",
+		  "units" : "number of active connections",
+		  "data_type" : "int",
+		  "polling_frequency" : 10,
+		  "regex" : "Active connections: ([0-9]+)"			
+		},
+		{ "name" : "reading",
+		  "metric_type" : "gauge", 
+		  "units" : "number of reading connections",
+		  "data_type" : "int",
+		  "polling_frequency" : 10,
+		  "regex" : "Reading: ([0-9]+) .*"	
+		},
+		{ "name" : "writing",
+                  "metric_type" : "gauge",
+                  "data_type" : "int",
+		  "units" : "number of writing connections",
+                  "polling_frequency" : 10,
+                  "regex" : ".*Writing: ([0-9]+).*"
+                },
+		{ "name" : "waiting",
+                  "metric_type" : "gauge",
+                  "units" : "number of waiting connections",
+		  "data_type" : "int",
+                  "polling_frequency" : 10,
+                  "regex" : ".*Waiting: ([0-9]+)"
+                }
+	] 	
+	
+}

Godeps/_workspace/src/github.com/google/cadvisor/collector/fakes.go

@@ -17,7 +17,7 @@ package collector
 import (
 	"time"
 
-	"github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v1"
 )
 
 type FakeCollectorManager struct {
@@ -27,7 +27,11 @@ func (fkm *FakeCollectorManager) RegisterCollector(collector Collector) error {
 	return nil
 }
 
-func (fkm *FakeCollectorManager) Collect() (time.Time, []v2.Metric, error) {
+func (fkm *FakeCollectorManager) GetSpec() ([]v1.MetricSpec, error) {
+	return []v1.MetricSpec{}, nil
+}
+
+func (fkm *FakeCollectorManager) Collect(metric map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
 	var zero time.Time
-	return zero, []v2.Metric{}, nil
+	return zero, metric, nil
 }

Godeps/_workspace/src/github.com/google/cadvisor/collector/generic_collector.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/cadvisor/info/v1"
+)
+
+type GenericCollector struct {
+	//name of the collector
+	name string
+
+	//holds information extracted from the config file for a collector
+	configFile Config
+
+	//holds information necessary to extract metrics
+	info *collectorInfo
+}
+
+type collectorInfo struct {
+	//minimum polling frequency among all metrics
+	minPollingFrequency time.Duration
+
+	//regular expresssions for all metrics
+	regexps []*regexp.Regexp
+}
+
+//Returns a new collector using the information extracted from the configfile
+func NewCollector(collectorName string, configFile []byte) (*GenericCollector, error) {
+	var configInJSON Config
+	err := json.Unmarshal(configFile, &configInJSON)
+	if err != nil {
+		return nil, err
+	}
+
+	//TODO : Add checks for validity of config file (eg : Accurate JSON fields)
+
+	if len(configInJSON.MetricsConfig) == 0 {
+		return nil, fmt.Errorf("No metrics provided in config")
+	}
+
+	minPollFrequency := time.Duration(0)
+	regexprs := make([]*regexp.Regexp, len(configInJSON.MetricsConfig))
+
+	for ind, metricConfig := range configInJSON.MetricsConfig {
+		// Find the minimum specified polling frequency in metric config.
+		if metricConfig.PollingFrequency != 0 {
+			if minPollFrequency == 0 || metricConfig.PollingFrequency < minPollFrequency {
+				minPollFrequency = metricConfig.PollingFrequency
+			}
+		}
+
+		regexprs[ind], err = regexp.Compile(metricConfig.Regex)
+		if err != nil {
+			return nil, fmt.Errorf("Invalid regexp %v for metric %v", metricConfig.Regex, metricConfig.Name)
+		}
+	}
+
+	// Minimum supported polling frequency is 1s.
+	minSupportedFrequency := 1 * time.Second
+	if minPollFrequency < minSupportedFrequency {
+		minPollFrequency = minSupportedFrequency
+	}
+
+	return &GenericCollector{
+		name:       collectorName,
+		configFile: configInJSON,
+		info: &collectorInfo{
+			minPollingFrequency: minPollFrequency,
+			regexps:             regexprs},
+	}, nil
+}
+
+//Returns name of the collector
+func (collector *GenericCollector) Name() string {
+	return collector.name
+}
+
+func (collector *GenericCollector) configToSpec(config MetricConfig) v1.MetricSpec {
+	return v1.MetricSpec{
+		Name:   config.Name,
+		Type:   config.MetricType,
+		Format: config.DataType,
+		Units:  config.Units,
+	}
+}
+
+func (collector *GenericCollector) GetSpec() []v1.MetricSpec {
+	specs := []v1.MetricSpec{}
+	for _, metricConfig := range collector.configFile.MetricsConfig {
+		spec := collector.configToSpec(metricConfig)
+		specs = append(specs, spec)
+	}
+	return specs
+}
+
+//Returns collected metrics and the next collection time of the collector
+func (collector *GenericCollector) Collect(metrics map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
+	currentTime := time.Now()
+	nextCollectionTime := currentTime.Add(time.Duration(collector.info.minPollingFrequency))
+
+	uri := collector.configFile.Endpoint
+	response, err := http.Get(uri)
+	if err != nil {
+		return nextCollectionTime, nil, err
+	}
+
+	defer response.Body.Close()
+
+	pageContent, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nextCollectionTime, nil, err
+	}
+
+	var errorSlice []error
+	for ind, metricConfig := range collector.configFile.MetricsConfig {
+		matchString := collector.info.regexps[ind].FindStringSubmatch(string(pageContent))
+		if matchString != nil {
+			if metricConfig.DataType == v1.FloatType {
+				regVal, err := strconv.ParseFloat(strings.TrimSpace(matchString[1]), 64)
+				if err != nil {
+					errorSlice = append(errorSlice, err)
+				}
+				metrics[metricConfig.Name] = v1.MetricVal{
+					FloatValue: regVal, Timestamp: currentTime,
+				}
+			} else if metricConfig.DataType == v1.IntType {
+				regVal, err := strconv.ParseInt(strings.TrimSpace(matchString[1]), 10, 64)
+				if err != nil {
+					errorSlice = append(errorSlice, err)
+				}
+				metrics[metricConfig.Name] = v1.MetricVal{
+					IntValue: regVal, Timestamp: currentTime,
+				}
+
+			} else {
+				errorSlice = append(errorSlice, fmt.Errorf("Unexpected value of 'data_type' for metric '%v' in config ", metricConfig.Name))
+			}
+		} else {
+			errorSlice = append(errorSlice, fmt.Errorf("No match found for regexp: %v for metric '%v' in config", metricConfig.Regex, metricConfig.Name))
+		}
+	}
+	return nextCollectionTime, metrics, compileErrors(errorSlice)
+}

Godeps/_workspace/src/github.com/google/cadvisor/collector/generic_collector_test.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/google/cadvisor/info/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEmptyConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	emptyConfig := `
+        {
+                "endpoint" : "http://localhost:8000/nginx_status",
+                "metrics_config"  : [
+                ]
+        }
+        `
+
+	//Create a temporary config file 'temp.json' with invalid json format
+	assert.NoError(ioutil.WriteFile("temp.json", []byte(emptyConfig), 0777))
+
+	configFile, err := ioutil.ReadFile("temp.json")
+	assert.NoError(err)
+
+	_, err = NewCollector("tempCollector", configFile)
+	assert.Error(err)
+
+	assert.NoError(os.Remove("temp.json"))
+}
+
+func TestConfigWithErrors(t *testing.T) {
+	assert := assert.New(t)
+
+	//Syntax error: Missed '"' after activeConnections
+	invalid := `
+	{
+		"endpoint" : "http://localhost:8000/nginx_status",
+		"metrics_config"  : [
+			{
+				 "name" : "activeConnections,  
+		  		 "metric_type" : "gauge",
+		 	 	 "data_type" : "int",
+		  		 "polling_frequency" : 10,
+		    		 "regex" : "Active connections: ([0-9]+)"			
+			}
+		]
+	}
+	`
+
+	//Create a temporary config file 'temp.json' with invalid json format
+	assert.NoError(ioutil.WriteFile("temp.json", []byte(invalid), 0777))
+	configFile, err := ioutil.ReadFile("temp.json")
+	assert.NoError(err)
+
+	_, err = NewCollector("tempCollector", configFile)
+	assert.Error(err)
+
+	assert.NoError(os.Remove("temp.json"))
+}
+
+func TestConfigWithRegexErrors(t *testing.T) {
+	assert := assert.New(t)
+
+	//Error: Missed operand for '+' in activeConnections regex
+	invalid := `
+        {
+                "endpoint" : "host:port/nginx_status",
+                "metrics_config"  : [
+                        {
+                                 "name" : "activeConnections",
+                                 "metric_type" : "gauge",
+                                 "data_type" : "int",
+                                 "polling_frequency" : 10,
+                                 "regex" : "Active connections: (+)"
+                        },
+                        {
+                                 "name" : "reading",
+                                 "metric_type" : "gauge",
+                                 "data_type" : "int",
+                                 "polling_frequency" : 10,
+                                 "regex" : "Reading: ([0-9]+) .*"
+                        }
+                ]
+        }
+        `
+
+	//Create a temporary config file 'temp.json'
+	assert.NoError(ioutil.WriteFile("temp.json", []byte(invalid), 0777))
+
+	configFile, err := ioutil.ReadFile("temp.json")
+	assert.NoError(err)
+
+	_, err = NewCollector("tempCollector", configFile)
+	assert.Error(err)
+
+	assert.NoError(os.Remove("temp.json"))
+}
+
+func TestConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	//Create an nginx collector using the config file 'sample_config.json'
+	configFile, err := ioutil.ReadFile("config/sample_config.json")
+	assert.NoError(err)
+
+	collector, err := NewCollector("nginx", configFile)
+	assert.NoError(err)
+	assert.Equal(collector.name, "nginx")
+	assert.Equal(collector.configFile.Endpoint, "http://localhost:8000/nginx_status")
+	assert.Equal(collector.configFile.MetricsConfig[0].Name, "activeConnections")
+}
+
+func TestMetricCollection(t *testing.T) {
+	assert := assert.New(t)
+
+	//Collect nginx metrics from a fake nginx endpoint
+	configFile, err := ioutil.ReadFile("config/sample_config.json")
+	assert.NoError(err)
+
+	fakeCollector, err := NewCollector("nginx", configFile)
+	assert.NoError(err)
+
+	tempServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Active connections: 3\nserver accepts handled requests")
+		fmt.Fprintln(w, "5 5 32\nReading: 0 Writing: 1 Waiting: 2")
+	}))
+	defer tempServer.Close()
+	fakeCollector.configFile.Endpoint = tempServer.URL
+
+	metrics := map[string]v1.MetricVal{}
+	_, metrics, errMetric := fakeCollector.Collect(metrics)
+	assert.NoError(errMetric)
+	metricNames := []string{"activeConnections", "reading", "writing", "waiting"}
+	// activeConnections = 3
+	assert.Equal(metrics[metricNames[0]].IntValue, 3)
+	assert.Equal(metrics[metricNames[0]].FloatValue, 0)
+	// reading = 0
+	assert.Equal(metrics[metricNames[1]].IntValue, 0)
+	assert.Equal(metrics[metricNames[1]].FloatValue, 0)
+	// writing = 1
+	assert.Equal(metrics[metricNames[2]].IntValue, 1)
+	assert.Equal(metrics[metricNames[2]].FloatValue, 0)
+	// waiting = 2
+	assert.Equal(metrics[metricNames[3]].IntValue, 2)
+	assert.Equal(metrics[metricNames[3]].FloatValue, 0)
+}

Godeps/_workspace/src/github.com/google/cadvisor/collector/types.go

@@ -15,7 +15,7 @@
 package collector
 
 import (
-	"github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v1"
 	"time"
 )
 
@@ -27,7 +27,10 @@ type Collector interface {
 	// Returns the next time this collector should be collected from.
 	// Next collection time is always returned, even when an error occurs.
 	// A collection time of zero means no more collection.
-	Collect() (time.Time, []v2.Metric, error)
+	Collect(map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error)
+
+	// Return spec for all metrics associated with this collector
+	GetSpec() []v1.MetricSpec
 
 	// Name of this collector.
 	Name() string
@@ -42,5 +45,8 @@ type CollectorManager interface {
 	// at which a collector will be ready to collect from.
 	// Next collection time is always returned, even when an error occurs.
 	// A collection time of zero means no more collection.
-	Collect() (time.Time, []v2.Metric, error)
+	Collect() (time.Time, map[string]v1.MetricVal, error)
+
+	// Get metric spec from all registered collectors.
+	GetSpec() ([]v1.MetricSpec, error)
 }

Godeps/_workspace/src/github.com/google/cadvisor/container/container.go

@@ -73,6 +73,9 @@ type ContainerHandler interface {
 	// Returns absolute cgroup path for the requested resource.
 	GetCgroupPath(resource string) (string, error)
 
+	// Returns container labels, if available.
+	GetContainerLabels() map[string]string
+
 	// Returns whether the container still exists.
 	Exists() bool
 }

Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go

@@ -167,7 +167,7 @@ func libcontainerConfigToContainerSpec(config *libcontainerConfigs.Config, mi *i
 	}
 	spec.Cpu.Mask = utils.FixCpuMask(config.Cgroups.CpusetCpus, mi.NumCores)
 
-	spec.HasNetwork = true
+	spec.HasNetwork = len(config.Networks) > 0
 	spec.HasDiskIo = true
 
 	return spec
@@ -276,7 +276,7 @@ func (self *dockerContainerHandler) GetStats() (*info.ContainerStats, error) {
 }
 
 func convertInterfaceStats(stats *info.InterfaceStats) {
-	net := stats
+	net := *stats
 
 	// Ingress for host veth is from the container.
 	// Hence tx_bytes stat on the host veth is actually number of bytes received by the container.
@@ -332,6 +332,10 @@ func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([]
 	return nil, nil
 }
 
+func (self *dockerContainerHandler) GetContainerLabels() map[string]string {
+	return self.labels
+}
+
 func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
 	return containerLibcontainer.GetProcesses(self.cgroupManager)
 }

Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go

@@ -93,7 +93,7 @@ func GetStats(cgroupManager cgroups.Manager, networkInterfaces []string) (*info.
 		}
 		stats.Network.Interfaces[i] = interfaceStats
 	}
-	// For backwards compatability.
+	// For backwards compatibility.
 	if len(networkInterfaces) > 0 {
 		stats.Network.InterfaceStats = stats.Network.Interfaces[0]
 	}
@@ -233,7 +233,7 @@ func toContainerStats3(libcontainerStats *libcontainer.Stats, ret *info.Containe
 		}
 	}
 
-	// Add to base struct for backwards compatability.
+	// Add to base struct for backwards compatibility.
 	if len(ret.Network.Interfaces) > 0 {
 		ret.Network.InterfaceStats = ret.Network.Interfaces[0]
 	}

Godeps/_workspace/src/github.com/google/cadvisor/container/mock.go

@@ -95,6 +95,11 @@ func (self *MockContainerHandler) GetCgroupPath(path string) (string, error) {
 	return args.Get(0).(string), args.Error(1)
 }
 
+func (self *MockContainerHandler) GetContainerLabels() map[string]string {
+	args := self.Called()
+	return args.Get(0).(map[string]string)
+}
+
 type FactoryForMockContainerHandler struct {
 	Name                        string
 	PrepareContainerHandlerFunc func(name string, handler *MockContainerHandler)

Godeps/_workspace/src/github.com/google/cadvisor/container/raw/handler.go

@@ -33,6 +33,7 @@ import (
 	"github.com/google/cadvisor/fs"
 	info "github.com/google/cadvisor/info/v1"
 	"github.com/google/cadvisor/utils"
+	"github.com/google/cadvisor/utils/machine"
 	"golang.org/x/exp/inotify"
 )
 
@@ -210,13 +211,33 @@ func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) {
 		}
 	}
 
-	// Memory.
-	memoryRoot, ok := self.cgroupPaths["memory"]
-	if ok {
-		if utils.FileExists(memoryRoot) {
+	// Memory
+	if self.name == "/" {
+		// Get memory and swap limits of the running machine
+		memLimit, err := machine.GetMachineMemoryCapacity()
+		if err != nil {
+			glog.Warningf("failed to obtain memory limit for machine container")
+			spec.HasMemory = false
+		} else {
+			spec.Memory.Limit = uint64(memLimit)
+			// Spec is marked to have memory only if the memory limit is set
 			spec.HasMemory = true
-			spec.Memory.Limit = readInt64(memoryRoot, "memory.limit_in_bytes")
-			spec.Memory.SwapLimit = readInt64(memoryRoot, "memory.memsw.limit_in_bytes")
+		}
+
+		swapLimit, err := machine.GetMachineSwapCapacity()
+		if err != nil {
+			glog.Warningf("failed to obtain swap limit for machine container")
+		} else {
+			spec.Memory.SwapLimit = uint64(swapLimit)
+		}
+	} else {
+		memoryRoot, ok := self.cgroupPaths["memory"]
+		if ok {
+			if utils.FileExists(memoryRoot) {
+				spec.HasMemory = true
+				spec.Memory.Limit = readInt64(memoryRoot, "memory.limit_in_bytes")
+				spec.Memory.SwapLimit = readInt64(memoryRoot, "memory.memsw.limit_in_bytes")
+			}
 		}
 	}
 
@@ -335,6 +356,10 @@ func (self *rawContainerHandler) GetCgroupPath(resource string) (string, error)
 	return path, nil
 }
 
+func (self *rawContainerHandler) GetContainerLabels() map[string]string {
+	return map[string]string{}
+}
+
 // Lists all directories under "path" and outputs the results as children of "parent".
 func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error {
 	// Ignore if this hierarchy does not exist.

Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go

@@ -58,6 +58,9 @@ type ContainerSpec struct {
 
 	// HasDiskIo when true, indicates that DiskIo stats will be available.
 	HasDiskIo bool `json:"has_diskio"`
+
+	HasCustomMetrics bool         `json:"has_custom_metrics"`
+	CustomMetrics    []MetricSpec `json:"custom_metrics,omitempty"`
 }
 
 // Container reference contains enough information to uniquely identify a container
@@ -190,6 +193,9 @@ func (self *ContainerSpec) Eq(b *ContainerSpec) bool {
 	if self.HasDiskIo != b.HasDiskIo {
 		return false
 	}
+	if self.HasCustomMetrics != b.HasCustomMetrics {
+		return false
+	}
 	return true
 }
 
@@ -419,6 +425,9 @@ type ContainerStats struct {
 
 	// Task load stats
 	TaskStats LoadStats `json:"task_stats,omitempty"`
+
+	//Custom metrics from all collectors
+	CustomMetrics map[string]MetricVal `json:"custom_metrics,omitempty"`
 }
 
 func timeEq(t1, t2 time.Time, tolerance time.Duration) bool {

Godeps/_workspace/src/github.com/google/cadvisor/info/v1/machine.go

@@ -112,6 +112,22 @@ type NetInfo struct {
 	Mtu int64 `json:"mtu"`
 }
 
+type CloudProvider string
+
+const (
+	GCE            CloudProvider = "GCE"
+	AWS                          = "AWS"
+	Baremetal                    = "Baremetal"
+	UnkownProvider               = "Unknown"
+)
+
+type InstanceType string
+
+const (
+	NoInstance      InstanceType = "None"
+	UnknownInstance              = "Unknown"
+)
+
 type MachineInfo struct {
 	// The number of cores in this machine.
 	NumCores int `json:"num_cores"`
@@ -143,6 +159,12 @@ type MachineInfo struct {
 	// Machine Topology
 	// Describes cpu/memory layout and hierarchy.
 	Topology []Node `json:"topology"`
+
+	// Cloud provider the machine belongs to.
+	CloudProvider CloudProvider `json:"cloud_provider"`
+
+	// Type of cloud instance (e.g. GCE standard) the machine is.
+	InstanceType InstanceType `json:"instance_type"`
 }
 
 type VersionInfo struct {

Godeps/_workspace/src/github.com/google/cadvisor/info/v2/metric.go → Godeps/_workspace/src/github.com/google/cadvisor/info/v1/metric.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package v2
+package v1
 
 import (
 	"time"
@@ -32,38 +32,35 @@ const (
 	MetricDelta = "delta"
 )
 
-// An exported metric.
-type Metric struct {
+// DataType for metric being exported.
+type DataType string
+
+const (
+	IntType   DataType = "int"
+	FloatType          = "float"
+)
+
+// Spec for custom metric.
+type MetricSpec struct {
 	// The name of the metric.
 	Name string `json:"name"`
 
 	// Type of the metric.
 	Type MetricType `json:"type"`
 
-	// Metadata associated with this metric.
-	Labels map[string]string
-
-	// Value of the metric. Only one of these values will be
-	// available according to the output type of the metric.
-	// If no values are available, there are no data points.
-	IntPoints   []IntPoint   `json:"int_points,omitempty"`
-	FloatPoints []FloatPoint `json:"float_points,omitempty"`
-}
-
-// An integer metric data point.
-type IntPoint struct {
-	// Time at which the metric was queried
-	Timestamp time.Time `json:"timestamp"`
+	// Data Type for the stats.
+	Format DataType `json:"format"`
 
-	// The value of the metric at this point.
-	Value int64 `json:"value"`
+	// Display Units for the stats.
+	Units string `json:"units"`
 }
 
-// A float metric data point.
-type FloatPoint struct {
+// An exported metric.
+type MetricVal struct {
 	// Time at which the metric was queried
 	Timestamp time.Time `json:"timestamp"`
 
 	// The value of the metric at this point.
-	Value float64 `json:"value"`
+	IntValue   int64   `json:"int_value,omitempty"`
+	FloatValue float64 `json:"float_value,omitempty"`
 }

Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go

@@ -73,6 +73,9 @@ type ContainerSpec struct {
 	HasMemory bool       `json:"has_memory"`
 	Memory    MemorySpec `json:"memory,omitempty"`
 
+	HasCustomMetrics bool            `json:"has_custom_metrics"`
+	CustomMetrics    []v1.MetricSpec `json:"custom_metrics,omitempty"`
+
 	// Following resources have no associated spec, but are being isolated.
 	HasNetwork    bool `json:"has_network"`
 	HasFilesystem bool `json:"has_filesystem"`
@@ -100,6 +103,9 @@ type ContainerStats struct {
 	// Task load statistics
 	HasLoad bool         `json:"has_load"`
 	Load    v1.LoadStats `json:"load_stats,omitempty"`
+	// Custom Metrics
+	HasCustomMetrics bool                    `json:"has_custom_metrics"`
+	CustomMetrics    map[string]v1.MetricVal `json:"custom_metrics,omitempty"`
 }
 
 type Percentiles struct {
@@ -110,8 +116,12 @@ type Percentiles struct {
 	Mean uint64 `json:"mean"`
 	// Max seen over the collected sample.
 	Max uint64 `json:"max"`
+	// 50th percentile over the collected sample.
+	Fifty uint64 `json:"fifty"`
 	// 90th percentile over the collected sample.
 	Ninety uint64 `json:"ninety"`
+	// 95th percentile over the collected sample.
+	NinetyFive uint64 `json:"ninetyfive"`
 }
 
 type Usage struct {

Godeps/_workspace/src/github.com/google/cadvisor/info/v2/machine.go

@@ -59,6 +59,12 @@ type Attributes struct {
 	// Machine Topology
 	// Describes cpu/memory layout and hierarchy.
 	Topology []v1.Node `json:"topology"`
+
+	// Cloud provider the machine belongs to
+	CloudProvider v1.CloudProvider `json:"cloud_provider"`
+
+	// Type of cloud instance (e.g. GCE standard) the machine is.
+	InstanceType v1.InstanceType `json:"instance_type"`
 }
 
 func GetAttributes(mi *v1.MachineInfo, vi *v1.VersionInfo) Attributes {
@@ -76,5 +82,7 @@ func GetAttributes(mi *v1.MachineInfo, vi *v1.VersionInfo) Attributes {
 		DiskMap:            mi.DiskMap,
 		NetworkDevices:     mi.NetworkDevices,
 		Topology:           mi.Topology,
+		CloudProvider:      mi.CloudProvider,
+		InstanceType:       mi.InstanceType,
 	}
 }

Godeps/_workspace/src/github.com/google/cadvisor/manager/container_test.go

@@ -41,7 +41,7 @@ func setupContainerData(t *testing.T, spec info.ContainerSpec) (*containerData,
 		nil,
 	)
 	memoryCache := memory.New(60, nil)
-	ret, err := newContainerData(containerName, memoryCache, mockHandler, nil, false, &collector.FakeCollectorManager{})
+	ret, err := newContainerData(containerName, memoryCache, mockHandler, nil, false, &collector.GenericCollectorManager{}, 60*time.Second, true)
 	if err != nil {
 		t.Fatal(err)
 	}

Godeps/_workspace/src/github.com/google/cadvisor/manager/machine.go

@@ -17,10 +17,7 @@ package manager
 import (
 	"bytes"
 	"flag"
-	"fmt"
 	"io/ioutil"
-	"regexp"
-	"strconv"
 	"strings"
 	"syscall"
 
@@ -29,193 +26,16 @@ import (
 	"github.com/google/cadvisor/container/docker"
 	"github.com/google/cadvisor/fs"
 	info "github.com/google/cadvisor/info/v1"
-	"github.com/google/cadvisor/utils"
+	"github.com/google/cadvisor/utils/cloudinfo"
+	"github.com/google/cadvisor/utils/machine"
 	"github.com/google/cadvisor/utils/sysfs"
 	"github.com/google/cadvisor/utils/sysinfo"
 	version "github.com/google/cadvisor/version"
 )
 
-var cpuRegExp = regexp.MustCompile("processor\\t*: +([0-9]+)")
-var coreRegExp = regexp.MustCompile("core id\\t*: +([0-9]+)")
-var nodeRegExp = regexp.MustCompile("physical id\\t*: +([0-9]+)")
-var CpuClockSpeedMHz = regexp.MustCompile("cpu MHz\\t*: +([0-9]+.[0-9]+)")
-var memoryCapacityRegexp = regexp.MustCompile("MemTotal: *([0-9]+) kB")
-
 var machineIdFilePath = flag.String("machine_id_file", "/etc/machine-id,/var/lib/dbus/machine-id", "Comma-separated list of files to check for machine-id. Use the first one that exists.")
 var bootIdFilePath = flag.String("boot_id_file", "/proc/sys/kernel/random/boot_id", "Comma-separated list of files to check for boot-id. Use the first one that exists.")
 
-func getClockSpeed(procInfo []byte) (uint64, error) {
-	// First look through sys to find a max supported cpu frequency.
-	const maxFreqFile = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"
-	if utils.FileExists(maxFreqFile) {
-		val, err := ioutil.ReadFile(maxFreqFile)
-		if err != nil {
-			return 0, err
-		}
-		var maxFreq uint64
-		n, err := fmt.Sscanf(string(val), "%d", &maxFreq)
-		if err != nil || n != 1 {
-			return 0, fmt.Errorf("could not parse frequency %q", val)
-		}
-		return maxFreq, nil
-	}
-	// Fall back to /proc/cpuinfo
-	matches := CpuClockSpeedMHz.FindSubmatch(procInfo)
-	if len(matches) != 2 {
-		//Check if we are running on Power systems which have a different format
-		CpuClockSpeedMHz, _ = regexp.Compile("clock\\t*: +([0-9]+.[0-9]+)MHz")
-		matches = CpuClockSpeedMHz.FindSubmatch(procInfo)
-		if len(matches) != 2 {
-			return 0, fmt.Errorf("could not detect clock speed from output: %q", string(procInfo))
-		}
-	}
-	speed, err := strconv.ParseFloat(string(matches[1]), 64)
-	if err != nil {
-		return 0, err
-	}
-	// Convert to kHz
-	return uint64(speed * 1000), nil
-}
-
-func getMemoryCapacity(b []byte) (int64, error) {
-	matches := memoryCapacityRegexp.FindSubmatch(b)
-	if len(matches) != 2 {
-		return -1, fmt.Errorf("failed to find memory capacity in output: %q", string(b))
-	}
-	m, err := strconv.ParseInt(string(matches[1]), 10, 64)
-	if err != nil {
-		return -1, err
-	}
-
-	// Convert to bytes.
-	return m * 1024, err
-}
-
-func extractValue(s string, r *regexp.Regexp) (bool, int, error) {
-	matches := r.FindSubmatch([]byte(s))
-	if len(matches) == 2 {
-		val, err := strconv.ParseInt(string(matches[1]), 10, 32)
-		if err != nil {
-			return true, -1, err
-		}
-		return true, int(val), nil
-	}
-	return false, -1, nil
-}
-
-func findNode(nodes []info.Node, id int) (bool, int) {
-	for i, n := range nodes {
-		if n.Id == id {
-			return true, i
-		}
-	}
-	return false, -1
-}
-
-func addNode(nodes *[]info.Node, id int) (int, error) {
-	var idx int
-	if id == -1 {
-		// Some VMs don't fill topology data. Export single package.
-		id = 0
-	}
-
-	ok, idx := findNode(*nodes, id)
-	if !ok {
-		// New node
-		node := info.Node{Id: id}
-		// Add per-node memory information.
-		meminfo := fmt.Sprintf("/sys/devices/system/node/node%d/meminfo", id)
-		out, err := ioutil.ReadFile(meminfo)
-		// Ignore if per-node info is not available.
-		if err == nil {
-			m, err := getMemoryCapacity(out)
-			if err != nil {
-				return -1, err
-			}
-			node.Memory = uint64(m)
-		}
-		*nodes = append(*nodes, node)
-		idx = len(*nodes) - 1
-	}
-	return idx, nil
-}
-
-func getTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) {
-	nodes := []info.Node{}
-	numCores := 0
-	lastThread := -1
-	lastCore := -1
-	lastNode := -1
-	for _, line := range strings.Split(cpuinfo, "\n") {
-		ok, val, err := extractValue(line, cpuRegExp)
-		if err != nil {
-			return nil, -1, fmt.Errorf("could not parse cpu info from %q: %v", line, err)
-		}
-		if ok {
-			thread := val
-			numCores++
-			if lastThread != -1 {
-				// New cpu section. Save last one.
-				nodeIdx, err := addNode(&nodes, lastNode)
-				if err != nil {
-					return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err)
-				}
-				nodes[nodeIdx].AddThread(lastThread, lastCore)
-				lastCore = -1
-				lastNode = -1
-			}
-			lastThread = thread
-		}
-		ok, val, err = extractValue(line, coreRegExp)
-		if err != nil {
-			return nil, -1, fmt.Errorf("could not parse core info from %q: %v", line, err)
-		}
-		if ok {
-			lastCore = val
-		}
-		ok, val, err = extractValue(line, nodeRegExp)
-		if err != nil {
-			return nil, -1, fmt.Errorf("could not parse node info from %q: %v", line, err)
-		}
-		if ok {
-			lastNode = val
-		}
-	}
-	nodeIdx, err := addNode(&nodes, lastNode)
-	if err != nil {
-		return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err)
-	}
-	nodes[nodeIdx].AddThread(lastThread, lastCore)
-	if numCores < 1 {
-		return nil, numCores, fmt.Errorf("could not detect any cores")
-	}
-	for idx, node := range nodes {
-		caches, err := sysinfo.GetCacheInfo(sysFs, node.Cores[0].Threads[0])
-		if err != nil {
-			glog.Errorf("failed to get cache information for node %d: %v", node.Id, err)
-			continue
-		}
-		numThreadsPerCore := len(node.Cores[0].Threads)
-		numThreadsPerNode := len(node.Cores) * numThreadsPerCore
-		for _, cache := range caches {
-			c := info.Cache{
-				Size:  cache.Size,
-				Level: cache.Level,
-				Type:  cache.Type,
-			}
-			if cache.Cpus == numThreadsPerNode && cache.Level > 2 {
-				// Add a node-level cache.
-				nodes[idx].AddNodeCache(c)
-			} else if cache.Cpus == numThreadsPerCore {
-				// Add to each core.
-				nodes[idx].AddPerCoreCache(c)
-			}
-			// Ignore unknown caches.
-		}
-	}
-	return nodes, numCores, nil
-}
-
 func getInfoFromFiles(filePaths string) string {
 	if len(filePaths) == 0 {
 		return ""
@@ -232,18 +52,12 @@ func getInfoFromFiles(filePaths string) string {
 
 func getMachineInfo(sysFs sysfs.SysFs, fsInfo fs.FsInfo) (*info.MachineInfo, error) {
 	cpuinfo, err := ioutil.ReadFile("/proc/cpuinfo")
-	clockSpeed, err := getClockSpeed(cpuinfo)
-	if err != nil {
-		return nil, err
-	}
-
-	// Get the amount of usable memory from /proc/meminfo.
-	out, err := ioutil.ReadFile("/proc/meminfo")
+	clockSpeed, err := machine.GetClockSpeed(cpuinfo)
 	if err != nil {
 		return nil, err
 	}
 
-	memoryCapacity, err := getMemoryCapacity(out)
+	memoryCapacity, err := machine.GetMachineMemoryCapacity()
 	if err != nil {
 		return nil, err
 	}
@@ -263,7 +77,7 @@ func getMachineInfo(sysFs sysfs.SysFs, fsInfo fs.FsInfo) (*info.MachineInfo, err
 		glog.Errorf("Failed to get network devices: %v", err)
 	}
 
-	topology, numCores, err := getTopology(sysFs, string(cpuinfo))
+	topology, numCores, err := machine.GetTopology(sysFs, string(cpuinfo))
 	if err != nil {
 		glog.Errorf("Failed to get topology information: %v", err)
 	}
@@ -273,6 +87,10 @@ func getMachineInfo(sysFs sysfs.SysFs, fsInfo fs.FsInfo) (*info.MachineInfo, err
 		glog.Errorf("Failed to get system UUID: %v", err)
 	}
 
+	realCloudInfo := cloudinfo.NewRealCloudInfo()
+	cloudProvider := realCloudInfo.GetCloudProvider()
+	instanceType := realCloudInfo.GetInstanceType()
+
 	machineInfo := &info.MachineInfo{
 		NumCores:       numCores,
 		CpuFrequency:   clockSpeed,
@@ -283,6 +101,8 @@ func getMachineInfo(sysFs sysfs.SysFs, fsInfo fs.FsInfo) (*info.MachineInfo, err
 		MachineID:      getInfoFromFiles(*machineIdFilePath),
 		SystemUUID:     systemUUID,
 		BootID:         getInfoFromFiles(*bootIdFilePath),
+		CloudProvider:  cloudProvider,
+		InstanceType:   instanceType,
 	}
 
 	for _, fs := range filesystems {

Godeps/_workspace/src/github.com/google/cadvisor/manager/manager.go

@@ -114,7 +114,7 @@ type Manager interface {
 }
 
 // New takes a memory storage and returns a new manager.
-func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs) (Manager, error) {
+func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool) (Manager, error) {
 	if memoryCache == nil {
 		return nil, fmt.Errorf("manager requires memory storage")
 	}
@@ -139,13 +139,15 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs) (Manager, error)
 		inHostNamespace = true
 	}
 	newManager := &manager{
-		containers:        make(map[namespacedContainerName]*containerData),
-		quitChannels:      make([]chan error, 0, 2),
-		memoryCache:       memoryCache,
-		fsInfo:            fsInfo,
-		cadvisorContainer: selfContainer,
-		inHostNamespace:   inHostNamespace,
-		startupTime:       time.Now(),
+		containers:               make(map[namespacedContainerName]*containerData),
+		quitChannels:             make([]chan error, 0, 2),
+		memoryCache:              memoryCache,
+		fsInfo:                   fsInfo,
+		cadvisorContainer:        selfContainer,
+		inHostNamespace:          inHostNamespace,
+		startupTime:              time.Now(),
+		maxHousekeepingInterval:  maxHousekeepingInterval,
+		allowDynamicHousekeeping: allowDynamicHousekeeping,
 	}
 
 	machineInfo, err := getMachineInfo(sysfs, fsInfo)
@@ -176,19 +178,21 @@ type namespacedContainerName struct {
 }
 
 type manager struct {
-	containers             map[namespacedContainerName]*containerData
-	containersLock         sync.RWMutex
-	memoryCache            *memory.InMemoryCache
-	fsInfo                 fs.FsInfo
-	machineInfo            info.MachineInfo
-	versionInfo            info.VersionInfo
-	quitChannels           []chan error
-	cadvisorContainer      string
-	inHostNamespace        bool
-	dockerContainersRegexp *regexp.Regexp
-	loadReader             cpuload.CpuLoadReader
-	eventHandler           events.EventManager
-	startupTime            time.Time
+	containers               map[namespacedContainerName]*containerData
+	containersLock           sync.RWMutex
+	memoryCache              *memory.InMemoryCache
+	fsInfo                   fs.FsInfo
+	machineInfo              info.MachineInfo
+	versionInfo              info.VersionInfo
+	quitChannels             []chan error
+	cadvisorContainer        string
+	inHostNamespace          bool
+	dockerContainersRegexp   *regexp.Regexp
+	loadReader               cpuload.CpuLoadReader
+	eventHandler             events.EventManager
+	startupTime              time.Time
+	maxHousekeepingInterval  time.Duration
+	allowDynamicHousekeeping bool
 }
 
 // Start the container manager.
@@ -371,12 +375,13 @@ func (self *manager) GetContainerSpec(containerName string, options v2.RequestOp
 func (self *manager) getV2Spec(cinfo *containerInfo) v2.ContainerSpec {
 	specV1 := self.getAdjustedSpec(cinfo)
 	specV2 := v2.ContainerSpec{
-		CreationTime:  specV1.CreationTime,
-		HasCpu:        specV1.HasCpu,
-		HasMemory:     specV1.HasMemory,
-		HasFilesystem: specV1.HasFilesystem,
-		HasNetwork:    specV1.HasNetwork,
-		HasDiskIo:     specV1.HasDiskIo,
+		CreationTime:     specV1.CreationTime,
+		HasCpu:           specV1.HasCpu,
+		HasMemory:        specV1.HasMemory,
+		HasFilesystem:    specV1.HasFilesystem,
+		HasNetwork:       specV1.HasNetwork,
+		HasDiskIo:        specV1.HasDiskIo,
+		HasCustomMetrics: specV1.HasCustomMetrics,
 	}
 	if specV1.HasCpu {
 		specV2.Cpu.Limit = specV1.Cpu.Limit
@@ -388,6 +393,9 @@ func (self *manager) getV2Spec(cinfo *containerInfo) v2.ContainerSpec {
 		specV2.Memory.Reservation = specV1.Memory.Reservation
 		specV2.Memory.SwapLimit = specV1.Memory.SwapLimit
 	}
+	if specV1.HasCustomMetrics {
+		specV2.CustomMetrics = specV1.CustomMetrics
+	}
 	specV2.Aliases = cinfo.Aliases
 	specV2.Namespace = cinfo.Namespace
 	return specV2
@@ -689,6 +697,28 @@ func (m *manager) GetProcessList(containerName string, options v2.RequestOptions
 	return ps, nil
 }
 
+func (m *manager) registerCollectors(collectorConfigs map[string]string, cont *containerData) error {
+	for k, v := range collectorConfigs {
+		configFile, err := cont.ReadFile(v, m.inHostNamespace)
+		if err != nil {
+			return fmt.Errorf("failed to read config file %q for config %q, container %q: %v", k, v, cont.info.Name, err)
+		}
+		glog.V(3).Infof("Got config from %q: %q", v, configFile)
+
+		newCollector, err := collector.NewCollector(k, configFile)
+		if err != nil {
+			glog.Infof("failed to create collector for container %q, config %q: %v", cont.info.Name, k, err)
+			return err
+		}
+		err = cont.collectorManager.RegisterCollector(newCollector)
+		if err != nil {
+			glog.Infof("failed to register collector for container %q, config %q: %v", cont.info.Name, k, err)
+			return err
+		}
+	}
+	return nil
+}
+
 // Create a container.
 func (m *manager) createContainer(containerName string) error {
 	handler, accept, err := container.NewContainerHandler(containerName)
@@ -700,14 +730,23 @@ func (m *manager) createContainer(containerName string) error {
 		glog.V(4).Infof("ignoring container %q", containerName)
 		return nil
 	}
-	// TODO(vmarmol): Register collectors.
 	collectorManager, err := collector.NewCollectorManager()
 	if err != nil {
 		return err
 	}
+
 	logUsage := *logCadvisorUsage && containerName == m.cadvisorContainer
-	cont, err := newContainerData(containerName, m.memoryCache, handler, m.loadReader, logUsage, collectorManager)
+	cont, err := newContainerData(containerName, m.memoryCache, handler, m.loadReader, logUsage, collectorManager, m.maxHousekeepingInterval, m.allowDynamicHousekeeping)
+	if err != nil {
+		return err
+	}
+
+	// Add collectors
+	labels := handler.GetContainerLabels()
+	collectorConfigs := collector.GetCollectorConfigs(labels)
+	err = m.registerCollectors(collectorConfigs, cont)
 	if err != nil {
+		glog.Infof("failed to register collectors for %q: %v", containerName, err)
 		return err
 	}
 

Godeps/_workspace/src/github.com/google/cadvisor/manager/manager_test.go

@@ -53,7 +53,7 @@ func createManagerAndAddContainers(
 			spec,
 			nil,
 		).Once()
-		cont, err := newContainerData(name, memoryCache, mockHandler, nil, false, &collector.FakeCollectorManager{})
+		cont, err := newContainerData(name, memoryCache, mockHandler, nil, false, &collector.GenericCollectorManager{}, 60*time.Second, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -205,7 +205,7 @@ func TestDockerContainersInfo(t *testing.T) {
 }
 
 func TestNewNilManager(t *testing.T) {
-	_, err := New(nil, nil)
+	_, err := New(nil, nil, 60*time.Second, true)
 	if err == nil {
 		t.Fatalf("Expected nil manager to return error")
 	}

Godeps/_workspace/src/github.com/google/cadvisor/pages/containers.go

@@ -18,7 +18,6 @@ package pages
 import (
 	"fmt"
 	"html/template"
-	"math"
 	"net/http"
 	"net/url"
 	"path"
@@ -149,15 +148,19 @@ func toMegabytes(bytes uint64) float64 {
 	return float64(bytes) / (1 << 20)
 }
 
+// Size after which we consider memory to be "unlimited". This is not
+// MaxInt64 due to rounding by the kernel.
+const maxMemorySize = uint64(1 << 62)
+
 func printSize(bytes uint64) string {
-	if bytes >= math.MaxInt64 {
+	if bytes >= maxMemorySize {
 		return "unlimited"
 	}
 	return ByteSize(bytes).Size()
 }
 
 func printUnit(bytes uint64) string {
-	if bytes >= math.MaxInt64 {
+	if bytes >= maxMemorySize {
 		return ""
 	}
 	return ByteSize(bytes).Unit()
@@ -229,7 +232,7 @@ func serveContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) e
 
 	data := &pageData{
 		DisplayName:        displayName,
-		ContainerName:      cont.Name,
+		ContainerName:      escapeContainerName(cont.Name),
 		ParentContainers:   parentContainers,
 		Subcontainers:      subcontainerLinks,
 		Spec:               cont.Spec,

Godeps/_workspace/src/github.com/google/cadvisor/pages/docker.go

@@ -130,7 +130,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error
 		}
 		data = &pageData{
 			DisplayName:        displayName,
-			ContainerName:      cont.Name,
+			ContainerName:      escapeContainerName(cont.Name),
 			ParentContainers:   parentContainers,
 			Spec:               cont.Spec,
 			Stats:              cont.Stats,

Godeps/_workspace/src/github.com/google/cadvisor/pages/pages.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"html/template"
 	"net/http"
+	"net/url"
 	"strings"
 
 	auth "github.com/abbot/go-http-auth"
@@ -159,3 +160,12 @@ func getContainerDisplayName(cont info.ContainerReference) string {
 
 	return displayName
 }
+
+// Escape the non-path characters on a container name.
+func escapeContainerName(containerName string) string {
+	parts := strings.Split(containerName, "/")
+	for i := range parts {
+		parts[i] = url.QueryEscape(parts[i])
+	}
+	return strings.Join(parts, "/")
+}

Godeps/_workspace/src/github.com/google/cadvisor/storage/redis/redis.go

@@ -44,8 +44,8 @@ func (self *redisStorage) defaultReadyToFlush() bool {
 	return time.Since(self.lastWrite) >= self.bufferDuration
 }
 
-//We must add some defaut params (for example: MachineName,ContainerName...)because containerStats do not include them
-func (self *redisStorage) containerStatsAndDefautValues(ref info.ContainerReference, stats *info.ContainerStats) *detailSpec {
+//We must add some default params (for example: MachineName,ContainerName...)because containerStats do not include them
+func (self *redisStorage) containerStatsAndDefaultValues(ref info.ContainerReference, stats *info.ContainerStats) *detailSpec {
 	timestamp := stats.Timestamp.UnixNano() / 1E3
 	var containerName string
 	if len(ref.Aliases) > 0 {
@@ -72,8 +72,8 @@ func (self *redisStorage) AddStats(ref info.ContainerReference, stats *info.Cont
 		// AddStats will be invoked simultaneously from multiple threads and only one of them will perform a write.
 		self.lock.Lock()
 		defer self.lock.Unlock()
-		// Add some defaut params based on containerStats
-		detail := self.containerStatsAndDefautValues(ref, stats)
+		// Add some default params based on containerStats
+		detail := self.containerStatsAndDefaultValues(ref, stats)
 		//To json
 		b, _ := json.Marshal(detail)
 		if self.readyToFlush() {

Godeps/_workspace/src/github.com/google/cadvisor/storage/statsd/client.go → Godeps/_workspace/src/github.com/google/cadvisor/storage/statsd/client/client.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package statsd
+package client
 
 import (
 	"fmt"
@@ -22,8 +22,9 @@ import (
 )
 
 type Client struct {
-	HostPort string
-	conn     net.Conn
+	HostPort  string
+	Namespace string
+	conn      net.Conn
 }
 
 func (self *Client) Open() error {
@@ -36,38 +37,28 @@ func (self *Client) Open() error {
 	return nil
 }
 
-func (self *Client) Close() {
+func (self *Client) Close() error {
 	self.conn.Close()
-}
-
-func (self *Client) UpdateGauge(name, value string) error {
-	stats := make(map[string]string)
-	val := fmt.Sprintf("%s|g", value)
-	stats[name] = val
-	if err := self.send(stats); err != nil {
-		return err
-	}
+	self.conn = nil
 	return nil
 }
 
 // Simple send to statsd daemon without sampling.
-func (self *Client) send(data map[string]string) error {
-	for k, v := range data {
-		formatted := fmt.Sprintf("%s:%s", k, v)
-		_, err := fmt.Fprintf(self.conn, formatted)
-		if err != nil {
-			glog.V(3).Infof("failed to send data %q: %v", formatted, err)
-			// return on first error.
-			return err
-		}
+func (self *Client) Send(namespace, containerName, key string, value uint64) error {
+	// only send counter value
+	formatted := fmt.Sprintf("%s.%s.%s:%d|g", namespace, containerName, key, value)
+	_, err := fmt.Fprintf(self.conn, formatted)
+	if err != nil {
+		glog.V(3).Infof("failed to send data %q: %v", formatted, err)
+		return err
 	}
 	return nil
 }
 
 func New(hostPort string) (*Client, error) {
-	client := Client{HostPort: hostPort}
-	if err := client.Open(); err != nil {
+	Client := Client{HostPort: hostPort}
+	if err := Client.Open(); err != nil {
 		return nil, err
 	}
-	return &client, nil
+	return &Client, nil
 }

Godeps/_workspace/src/github.com/google/cadvisor/storage/statsd/statsd.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package statsd
+
+import (
+	info "github.com/google/cadvisor/info/v1"
+	client "github.com/google/cadvisor/storage/statsd/client"
+)
+
+type statsdStorage struct {
+	client    *client.Client
+	Namespace string
+}
+
+const (
+	colCpuCumulativeUsage string = "cpu_cumulative_usage"
+	// Memory Usage
+	colMemoryUsage string = "memory_usage"
+	// Working set size
+	colMemoryWorkingSet string = "memory_working_set"
+	// Cumulative count of bytes received.
+	colRxBytes string = "rx_bytes"
+	// Cumulative count of receive errors encountered.
+	colRxErrors string = "rx_errors"
+	// Cumulative count of bytes transmitted.
+	colTxBytes string = "tx_bytes"
+	// Cumulative count of transmit errors encountered.
+	colTxErrors string = "tx_errors"
+	// Filesystem summary
+	colFsSummary = "fs_summary"
+	// Filesystem limit.
+	colFsLimit = "fs_limit"
+	// Filesystem usage.
+	colFsUsage = "fs_usage"
+)
+
+func (self *statsdStorage) containerStatsToValues(
+	stats *info.ContainerStats,
+) (series map[string]uint64) {
+	series = make(map[string]uint64)
+
+	// Cumulative Cpu Usage
+	series[colCpuCumulativeUsage] = stats.Cpu.Usage.Total
+
+	// Memory Usage
+	series[colMemoryUsage] = stats.Memory.Usage
+
+	// Working set size
+	series[colMemoryWorkingSet] = stats.Memory.WorkingSet
+
+	// Network stats.
+	series[colRxBytes] = stats.Network.RxBytes
+	series[colRxErrors] = stats.Network.RxErrors
+	series[colTxBytes] = stats.Network.TxBytes
+	series[colTxErrors] = stats.Network.TxErrors
+
+	return series
+}
+
+func (self *statsdStorage) containerFsStatsToValues(
+	series *map[string]uint64,
+	stats *info.ContainerStats,
+) {
+	for _, fsStat := range stats.Filesystem {
+		// Summary stats.
+		(*series)[colFsSummary+"."+colFsLimit] += fsStat.Limit
+		(*series)[colFsSummary+"."+colFsUsage] += fsStat.Usage
+
+		// Per device stats.
+		(*series)[fsStat.Device+"."+colFsLimit] = fsStat.Limit
+		(*series)[fsStat.Device+"."+colFsUsage] = fsStat.Usage
+	}
+}
+
+//Push the data into redis
+func (self *statsdStorage) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
+	if stats == nil {
+		return nil
+	}
+
+	var containerName string
+	if len(ref.Aliases) > 0 {
+		containerName = ref.Aliases[0]
+	} else {
+		containerName = ref.Name
+	}
+
+	series := self.containerStatsToValues(stats)
+	self.containerFsStatsToValues(&series, stats)
+	for key, value := range series {
+		err := self.client.Send(self.Namespace, containerName, key, value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (self *statsdStorage) Close() error {
+	self.client.Close()
+	self.client = nil
+	return nil
+}
+
+func New(namespace, hostPort string) (*statsdStorage, error) {
+	statsdClient, err := client.New(hostPort)
+	if err != nil {
+		return nil, err
+	}
+	statsdStorage := &statsdStorage{
+		client:    statsdClient,
+		Namespace: namespace,
+	}
+	return statsdStorage, nil
+}

Godeps/_workspace/src/github.com/google/cadvisor/summary/percentiles.go

@@ -28,20 +28,23 @@ const secondsToMilliSeconds = 1000
 const milliSecondsToNanoSeconds = 1000000
 const secondsToNanoSeconds = secondsToMilliSeconds * milliSecondsToNanoSeconds
 
-type uint64Slice []uint64
+type Uint64Slice []uint64
 
-func (a uint64Slice) Len() int           { return len(a) }
-func (a uint64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }
+func (a Uint64Slice) Len() int           { return len(a) }
+func (a Uint64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a Uint64Slice) Less(i, j int) bool { return a[i] < a[j] }
 
-// Get 90th percentile of the provided samples. Round to integer.
-func (self uint64Slice) Get90Percentile() uint64 {
+// Get percentile of the provided samples. Round to integer.
+func (self Uint64Slice) GetPercentile(d float64) uint64 {
+	if d < 0.0 || d > 1.0 {
+		return 0
+	}
 	count := self.Len()
 	if count == 0 {
 		return 0
 	}
 	sort.Sort(self)
-	n := float64(0.9 * (float64(count) + 1))
+	n := float64(d * (float64(count) + 1))
 	idx, frac := math.Modf(n)
 	index := int(idx)
 	percentile := float64(self[index-1])
@@ -71,7 +74,7 @@ func (self *mean) Add(value uint64) {
 
 type resource struct {
 	// list of samples being tracked.
-	samples uint64Slice
+	samples Uint64Slice
 	// average from existing samples.
 	mean mean
 	// maximum value seen so far in the added samples.
@@ -94,27 +97,31 @@ func (self *resource) Add(p info.Percentiles) {
 // Add a single sample. Internally, we convert it to a fake percentile sample.
 func (self *resource) AddSample(val uint64) {
 	sample := info.Percentiles{
-		Present: true,
-		Mean:    val,
-		Max:     val,
-		Ninety:  val,
+		Present:    true,
+		Mean:       val,
+		Max:        val,
+		Fifty:      val,
+		Ninety:     val,
+		NinetyFive: val,
 	}
 	self.Add(sample)
 }
 
 // Get max, average, and 90p from existing samples.
-func (self *resource) GetPercentile() info.Percentiles {
+func (self *resource) GetAllPercentiles() info.Percentiles {
 	p := info.Percentiles{}
 	p.Mean = uint64(self.mean.Mean)
 	p.Max = self.max
-	p.Ninety = self.samples.Get90Percentile()
+	p.Fifty = self.samples.GetPercentile(0.5)
+	p.Ninety = self.samples.GetPercentile(0.9)
+	p.NinetyFive = self.samples.GetPercentile(0.95)
 	p.Present = true
 	return p
 }
 
 func NewResource(size int) *resource {
 	return &resource{
-		samples: make(uint64Slice, 0, size),
+		samples: make(Uint64Slice, 0, size),
 		mean:    mean{count: 0, Mean: 0},
 	}
 }
@@ -128,8 +135,8 @@ func GetDerivedPercentiles(stats []*info.Usage) info.Usage {
 		memory.Add(stat.Memory)
 	}
 	usage := info.Usage{}
-	usage.Cpu = cpu.GetPercentile()
-	usage.Memory = memory.GetPercentile()
+	usage.Cpu = cpu.GetAllPercentiles()
+	usage.Memory = memory.GetAllPercentiles()
 	return usage
 }
 
@@ -183,7 +190,7 @@ func GetMinutePercentiles(stats []*secondSample) info.Usage {
 	percent := getPercentComplete(stats)
 	return info.Usage{
 		PercentComplete: percent,
-		Cpu:             cpu.GetPercentile(),
-		Memory:          memory.GetPercentile(),
+		Cpu:             cpu.GetAllPercentiles(),
+		Memory:          memory.GetAllPercentiles(),
 	}
 }

Godeps/_workspace/src/github.com/google/cadvisor/summary/percentiles_test.go

@@ -23,25 +23,29 @@ import (
 
 const Nanosecond = 1000000000
 
-func Test90Percentile(t *testing.T) {
+func assertPercentile(t *testing.T, s Uint64Slice, f float64, want uint64) {
+	if got := s.GetPercentile(f); got != want {
+		t.Errorf("GetPercentile(%f) is %d, should be %d.", f, got, want)
+	}
+}
+
+func TestPercentile(t *testing.T) {
 	N := 100
-	stats := make(uint64Slice, 0, N)
+	s := make(Uint64Slice, 0, N)
 	for i := N; i > 0; i-- {
-		stats = append(stats, uint64(i))
-	}
-	p := stats.Get90Percentile()
-	if p != 90 {
-		t.Errorf("90th percentile is %d, should be 90.", p)
+		s = append(s, uint64(i))
 	}
-	// 90p should be between 94 and 95. Promoted to 95.
+	assertPercentile(t, s, 0.2, 20)
+	assertPercentile(t, s, 0.7, 70)
+	assertPercentile(t, s, 0.9, 90)
 	N = 105
 	for i := 101; i <= N; i++ {
-		stats = append(stats, uint64(i))
-	}
-	p = stats.Get90Percentile()
-	if p != 95 {
-		t.Errorf("90th percentile is %d, should be 95.", p)
+		s = append(s, uint64(i))
 	}
+	// 90p should be between 94 and 95. Promoted to 95.
+	assertPercentile(t, s, 0.2, 21)
+	assertPercentile(t, s, 0.7, 74)
+	assertPercentile(t, s, 0.9, 95)
 }
 
 func TestMean(t *testing.T) {
@@ -74,19 +78,23 @@ func TestAggregates(t *testing.T) {
 	usage := GetMinutePercentiles(stats)
 	// Cpu mean, max, and 90p should all be 1000 ms/s.
 	cpuExpected := info.Percentiles{
-		Present: true,
-		Mean:    1000,
-		Max:     1000,
-		Ninety:  1000,
+		Present:    true,
+		Mean:       1000,
+		Max:        1000,
+		Fifty:      1000,
+		Ninety:     1000,
+		NinetyFive: 1000,
 	}
 	if usage.Cpu != cpuExpected {
 		t.Errorf("cpu stats are %+v. Expected %+v", usage.Cpu, cpuExpected)
 	}
 	memExpected := info.Percentiles{
-		Present: true,
-		Mean:    50 * 1024,
-		Max:     99 * 1024,
-		Ninety:  90 * 1024,
+		Present:    true,
+		Mean:       50 * 1024,
+		Max:        99 * 1024,
+		Fifty:      50 * 1024,
+		Ninety:     90 * 1024,
+		NinetyFive: 95 * 1024,
 	}
 	if usage.Memory != memExpected {
 		t.Errorf("memory stats are mean %+v. Expected %+v", usage.Memory, memExpected)
@@ -119,19 +127,23 @@ func TestSamplesCloseInTimeIgnored(t *testing.T) {
 	usage := GetMinutePercentiles(stats)
 	// Cpu mean, max, and 90p should all be 1000 ms/s. All high-value samples are discarded.
 	cpuExpected := info.Percentiles{
-		Present: true,
-		Mean:    1000,
-		Max:     1000,
-		Ninety:  1000,
+		Present:    true,
+		Mean:       1000,
+		Max:        1000,
+		Fifty:      1000,
+		Ninety:     1000,
+		NinetyFive: 1000,
 	}
 	if usage.Cpu != cpuExpected {
 		t.Errorf("cpu stats are %+v. Expected %+v", usage.Cpu, cpuExpected)
 	}
 	memExpected := info.Percentiles{
-		Present: true,
-		Mean:    50 * 1024,
-		Max:     99 * 1024,
-		Ninety:  90 * 1024,
+		Present:    true,
+		Mean:       50 * 1024,
+		Max:        99 * 1024,
+		Fifty:      50 * 1024,
+		Ninety:     90 * 1024,
+		NinetyFive: 95 * 1024,
 	}
 	if usage.Memory != memExpected {
 		t.Errorf("memory stats are mean %+v. Expected %+v", usage.Memory, memExpected)
@@ -146,35 +158,43 @@ func TestDerivedStats(t *testing.T) {
 		s := &info.Usage{
 			PercentComplete: 100,
 			Cpu: info.Percentiles{
-				Present: true,
-				Mean:    i * Nanosecond,
-				Max:     i * Nanosecond,
-				Ninety:  i * Nanosecond,
+				Present:    true,
+				Mean:       i * Nanosecond,
+				Max:        i * Nanosecond,
+				Fifty:      i * Nanosecond,
+				Ninety:     i * Nanosecond,
+				NinetyFive: i * Nanosecond,
 			},
 			Memory: info.Percentiles{
-				Present: true,
-				Mean:    i * 1024,
-				Max:     i * 1024,
-				Ninety:  i * 1024,
+				Present:    true,
+				Mean:       i * 1024,
+				Max:        i * 1024,
+				Fifty:      i * 1024,
+				Ninety:     i * 1024,
+				NinetyFive: i * 1024,
 			},
 		}
 		stats = append(stats, s)
 	}
 	usage := GetDerivedPercentiles(stats)
 	cpuExpected := info.Percentiles{
-		Present: true,
-		Mean:    50 * Nanosecond,
-		Max:     99 * Nanosecond,
-		Ninety:  90 * Nanosecond,
+		Present:    true,
+		Mean:       50 * Nanosecond,
+		Max:        99 * Nanosecond,
+		Fifty:      50 * Nanosecond,
+		Ninety:     90 * Nanosecond,
+		NinetyFive: 95 * Nanosecond,
 	}
 	if usage.Cpu != cpuExpected {
 		t.Errorf("cpu stats are %+v. Expected %+v", usage.Cpu, cpuExpected)
 	}
 	memExpected := info.Percentiles{
-		Present: true,
-		Mean:    50 * 1024,
-		Max:     99 * 1024,
-		Ninety:  90 * 1024,
+		Present:    true,
+		Mean:       50 * 1024,
+		Max:        99 * 1024,
+		Fifty:      50 * 1024,
+		Ninety:     90 * 1024,
+		NinetyFive: 95 * 1024,
 	}
 	if usage.Memory != memExpected {
 		t.Errorf("memory stats are mean %+v. Expected %+v", usage.Memory, memExpected)

Godeps/_workspace/src/github.com/google/cadvisor/utils/cloudinfo/cloudinfo.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Get information about the cloud provider (if any) cAdvisor is running on.
+
+package cloudinfo
+
+import (
+	info "github.com/google/cadvisor/info/v1"
+)
+
+type CloudInfo interface {
+	GetCloudProvider() info.CloudProvider
+	GetInstanceType() info.InstanceType
+}
+
+type realCloudInfo struct {
+	cloudProvider info.CloudProvider
+	instanceType  info.InstanceType
+}
+
+func NewRealCloudInfo() CloudInfo {
+	cloudProvider := detectCloudProvider()
+	instanceType := detectInstanceType(cloudProvider)
+	return &realCloudInfo{
+		cloudProvider: cloudProvider,
+		instanceType:  instanceType,
+	}
+}
+
+func (self *realCloudInfo) GetCloudProvider() info.CloudProvider {
+	return self.cloudProvider
+}
+
+func (self *realCloudInfo) GetInstanceType() info.InstanceType {
+	return self.instanceType
+}
+
+func detectCloudProvider() info.CloudProvider {
+	switch {
+	case onGCE():
+		return info.GCE
+	case onAWS():
+		return info.AWS
+	case onBaremetal():
+		return info.Baremetal
+	}
+	return info.UnkownProvider
+}
+
+func detectInstanceType(cloudProvider info.CloudProvider) info.InstanceType {
+	switch cloudProvider {
+	case info.GCE:
+		return getGceInstanceType()
+	case info.AWS:
+		return getAwsInstanceType()
+	case info.Baremetal:
+		return info.NoInstance
+	}
+	return info.UnknownInstance
+}
+
+//TODO: Implement method.
+func onAWS() bool {
+	return false
+}
+
+//TODO: Implement method.
+func getAwsInstanceType() info.InstanceType {
+	return info.UnknownInstance
+}
+
+//TODO: Implement method.
+func onBaremetal() bool {
+	return false
+}

Godeps/_workspace/src/github.com/google/cadvisor/utils/cloudinfo/gce.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudinfo
+
+import (
+	"strings"
+
+	"github.com/GoogleCloudPlatform/gcloud-golang/compute/metadata"
+	info "github.com/google/cadvisor/info/v1"
+)
+
+func onGCE() bool {
+	return metadata.OnGCE()
+}
+
+func getGceInstanceType() info.InstanceType {
+	machineType, err := metadata.Get("instance/machine-type")
+	if err != nil {
+		return info.UnknownInstance
+	}
+
+	responseParts := strings.Split(machineType, "/") // Extract the instance name from the machine type.
+	return info.InstanceType(responseParts[len(responseParts)-1])
+}

Godeps/_workspace/src/github.com/google/cadvisor/utils/fs/mockfs/mockfs.go

@@ -18,7 +18,7 @@
 package mockfs
 
 import (
-	gomock "code.google.com/p/gomock/gomock"
+	gomock "github.com/golang/mock/gomock"
 	fs "github.com/google/cadvisor/utils/fs"
 )
 

Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/machine.go

+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package machine
+
+import (
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/golang/glog"
+	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/utils"
+	"github.com/google/cadvisor/utils/sysfs"
+	"github.com/google/cadvisor/utils/sysinfo"
+)
+
+// The utils/machine package contains functions that extract machine-level specs.
+
+var cpuRegExp = regexp.MustCompile("processor\\t*: +([0-9]+)")
+var coreRegExp = regexp.MustCompile("core id\\t*: +([0-9]+)")
+var nodeRegExp = regexp.MustCompile("physical id\\t*: +([0-9]+)")
+var CpuClockSpeedMHz = regexp.MustCompile("cpu MHz\\t*: +([0-9]+.[0-9]+)")
+var memoryCapacityRegexp = regexp.MustCompile("MemTotal: *([0-9]+) kB")
+var swapCapacityRegexp = regexp.MustCompile("SwapTotal: *([0-9]+) kB")
+
+// GetClockSpeed returns the CPU clock speed, given a []byte formatted as the /proc/cpuinfo file.
+func GetClockSpeed(procInfo []byte) (uint64, error) {
+	// First look through sys to find a max supported cpu frequency.
+	const maxFreqFile = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"
+	if utils.FileExists(maxFreqFile) {
+		val, err := ioutil.ReadFile(maxFreqFile)
+		if err != nil {
+			return 0, err
+		}
+		var maxFreq uint64
+		n, err := fmt.Sscanf(string(val), "%d", &maxFreq)
+		if err != nil || n != 1 {
+			return 0, fmt.Errorf("could not parse frequency %q", val)
+		}
+		return maxFreq, nil
+	}
+	// Fall back to /proc/cpuinfo
+	matches := CpuClockSpeedMHz.FindSubmatch(procInfo)
+	if len(matches) != 2 {
+		//Check if we are running on Power systems which have a different format
+		CpuClockSpeedMHz, _ = regexp.Compile("clock\\t*: +([0-9]+.[0-9]+)MHz")
+		matches = CpuClockSpeedMHz.FindSubmatch(procInfo)
+		if len(matches) != 2 {
+			return 0, fmt.Errorf("could not detect clock speed from output: %q", string(procInfo))
+		}
+	}
+	speed, err := strconv.ParseFloat(string(matches[1]), 64)
+	if err != nil {
+		return 0, err
+	}
+	// Convert to kHz
+	return uint64(speed * 1000), nil
+}
+
+// GetMachineMemoryCapacity returns the machine's total memory from /proc/meminfo.
+// Returns the total memory capacity as an int64 (number of bytes).
+func GetMachineMemoryCapacity() (int64, error) {
+	out, err := ioutil.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0, err
+	}
+
+	memoryCapacity, err := parseCapacity(out, memoryCapacityRegexp)
+	if err != nil {
+		return 0, err
+	}
+	return memoryCapacity, err
+}
+
+// GetMachineSwapCapacity returns the machine's total swap from /proc/meminfo.
+// Returns the total swap capacity as an int64 (number of bytes).
+func GetMachineSwapCapacity() (int64, error) {
+	out, err := ioutil.ReadFile("/proc/meminfo")
+	if err != nil {
+		return 0, err
+	}
+
+	swapCapacity, err := parseCapacity(out, swapCapacityRegexp)
+	if err != nil {
+		return 0, err
+	}
+	return swapCapacity, err
+}
+
+// parseCapacity matches a Regexp in a []byte, returning the resulting value in bytes.
+// Assumes that the value matched by the Regexp is in KB.
+func parseCapacity(b []byte, r *regexp.Regexp) (int64, error) {
+	matches := r.FindSubmatch(b)
+	if len(matches) != 2 {
+		return -1, fmt.Errorf("failed to match regexp in output: %q", string(b))
+	}
+	m, err := strconv.ParseInt(string(matches[1]), 10, 64)
+	if err != nil {
+		return -1, err
+	}
+
+	// Convert to bytes.
+	return m * 1024, err
+}
+
+func GetTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) {
+	nodes := []info.Node{}
+	numCores := 0
+	lastThread := -1
+	lastCore := -1
+	lastNode := -1
+	for _, line := range strings.Split(cpuinfo, "\n") {
+		ok, val, err := extractValue(line, cpuRegExp)
+		if err != nil {
+			return nil, -1, fmt.Errorf("could not parse cpu info from %q: %v", line, err)
+		}
+		if ok {
+			thread := val
+			numCores++
+			if lastThread != -1 {
+				// New cpu section. Save last one.
+				nodeIdx, err := addNode(&nodes, lastNode)
+				if err != nil {
+					return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err)
+				}
+				nodes[nodeIdx].AddThread(lastThread, lastCore)
+				lastCore = -1
+				lastNode = -1
+			}
+			lastThread = thread
+		}
+		ok, val, err = extractValue(line, coreRegExp)
+		if err != nil {
+			return nil, -1, fmt.Errorf("could not parse core info from %q: %v", line, err)
+		}
+		if ok {
+			lastCore = val
+		}
+		ok, val, err = extractValue(line, nodeRegExp)
+		if err != nil {
+			return nil, -1, fmt.Errorf("could not parse node info from %q: %v", line, err)
+		}
+		if ok {
+			lastNode = val
+		}
+	}
+	nodeIdx, err := addNode(&nodes, lastNode)
+	if err != nil {
+		return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err)
+	}
+	nodes[nodeIdx].AddThread(lastThread, lastCore)
+	if numCores < 1 {
+		return nil, numCores, fmt.Errorf("could not detect any cores")
+	}
+	for idx, node := range nodes {
+		caches, err := sysinfo.GetCacheInfo(sysFs, node.Cores[0].Threads[0])
+		if err != nil {
+			glog.Errorf("failed to get cache information for node %d: %v", node.Id, err)
+			continue
+		}
+		numThreadsPerCore := len(node.Cores[0].Threads)
+		numThreadsPerNode := len(node.Cores) * numThreadsPerCore
+		for _, cache := range caches {
+			c := info.Cache{
+				Size:  cache.Size,
+				Level: cache.Level,
+				Type:  cache.Type,
+			}
+			if cache.Cpus == numThreadsPerNode && cache.Level > 2 {
+				// Add a node-level cache.
+				nodes[idx].AddNodeCache(c)
+			} else if cache.Cpus == numThreadsPerCore {
+				// Add to each core.
+				nodes[idx].AddPerCoreCache(c)
+			}
+			// Ignore unknown caches.
+		}
+	}
+	return nodes, numCores, nil
+}
+
+func extractValue(s string, r *regexp.Regexp) (bool, int, error) {
+	matches := r.FindSubmatch([]byte(s))
+	if len(matches) == 2 {
+		val, err := strconv.ParseInt(string(matches[1]), 10, 32)
+		if err != nil {
+			return true, -1, err
+		}
+		return true, int(val), nil
+	}
+	return false, -1, nil
+}
+
+func findNode(nodes []info.Node, id int) (bool, int) {
+	for i, n := range nodes {
+		if n.Id == id {
+			return true, i
+		}
+	}
+	return false, -1
+}
+
+func addNode(nodes *[]info.Node, id int) (int, error) {
+	var idx int
+	if id == -1 {
+		// Some VMs don't fill topology data. Export single package.
+		id = 0
+	}
+
+	ok, idx := findNode(*nodes, id)
+	if !ok {
+		// New node
+		node := info.Node{Id: id}
+		// Add per-node memory information.
+		meminfo := fmt.Sprintf("/sys/devices/system/node/node%d/meminfo", id)
+		out, err := ioutil.ReadFile(meminfo)
+		// Ignore if per-node info is not available.
+		if err == nil {
+			m, err := parseCapacity(out, memoryCapacityRegexp)
+			if err != nil {
+				return -1, err
+			}
+			node.Memory = uint64(m)
+		}
+		*nodes = append(*nodes, node)
+		idx = len(*nodes) - 1
+	}
+	return idx, nil
+}

Godeps/_workspace/src/github.com/google/cadvisor/manager/topology_test.go → Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/topology_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package manager
+package machine
 
 import (
 	"io/ioutil"
@@ -38,7 +38,7 @@ func TestTopology(t *testing.T) {
 		Cpus:  2,
 	}
 	sysFs.SetCacheInfo(c)
-	topology, numCores, err := getTopology(sysFs, string(testcpuinfo))
+	topology, numCores, err := GetTopology(sysFs, string(testcpuinfo))
 	if err != nil {
 		t.Errorf("failed to get topology for sample cpuinfo %s", string(testcpuinfo))
 	}
@@ -84,7 +84,7 @@ func TestTopologyWithSimpleCpuinfo(t *testing.T) {
 		Cpus:  1,
 	}
 	sysFs.SetCacheInfo(c)
-	topology, numCores, err := getTopology(sysFs, "processor\t: 0\n")
+	topology, numCores, err := GetTopology(sysFs, "processor\t: 0\n")
 	if err != nil {
 		t.Errorf("Expected cpuinfo with no topology data to succeed.")
 	}
@@ -110,7 +110,7 @@ func TestTopologyWithSimpleCpuinfo(t *testing.T) {
 }
 
 func TestTopologyEmptyCpuinfo(t *testing.T) {
-	_, _, err := getTopology(&fakesysfs.FakeSysFs{}, "")
+	_, _, err := GetTopology(&fakesysfs.FakeSysFs{}, "")
 	if err == nil {
 		t.Errorf("Expected empty cpuinfo to fail.")
 	}

Godeps/_workspace/src/github.com/google/cadvisor/version/version.go

@@ -15,4 +15,4 @@
 package version
 
 // Version of cAdvisor.
-const VERSION = "0.15.1"
+const VERSION = "0.16.0"

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/.gitignore

+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2014 Nate Finch 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/README.md

+# lumberjack  [![GoDoc](https://godoc.org/gopkg.in/natefinch/lumberjack.v2?status.png)](https://godoc.org/gopkg.in/natefinch/lumberjack.v2) [![Build Status](https://drone.io/github.com/natefinch/lumberjack/status.png)](https://drone.io/github.com/natefinch/lumberjack/latest) [![Build status](https://ci.appveyor.com/api/projects/status/00gchpxtg4gkrt5d)](https://ci.appveyor.com/project/natefinch/lumberjack) [![Coverage Status](https://coveralls.io/repos/natefinch/lumberjack/badge.svg?branch=v2.0)](https://coveralls.io/r/natefinch/lumberjack?branch=v2.0)
+
+### Lumberjack is a Go package for writing logs to rolling files.
+
+Package lumberjack provides a rolling logger.
+
+Note that this is v2.0 of lumberjack, and should be imported using gopkg.in
+thusly:
+
+    import "gopkg.in/natefinch/lumberjack.v2"
+
+The package name remains simply lumberjack, and the code resides at
+https://github.com/natefinch/lumberjack under the v2.0 branch.
+
+Lumberjack is intended to be one part of a logging infrastructure.
+It is not an all-in-one solution, but instead is a pluggable
+component at the bottom of the logging stack that simply controls the files
+to which logs are written.
+
+Lumberjack plays well with any logging package that can write to an
+io.Writer, including the standard library's log package.
+
+Lumberjack assumes that only one process is writing to the output files.
+Using the same lumberjack configuration from multiple processes on the same
+machine will result in improper behavior.
+
+
+**Example**
+
+To use lumberjack with the standard library's log package, just pass it into the SetOutput function when your application starts.
+
+Code:
+
+```go
+log.SetOutput(&lumberjack.Logger{
+    Filename:   "/var/log/myapp/foo.log",
+    MaxSize:    500, // megabytes
+    MaxBackups: 3,
+    MaxAge:     28, //days
+})
+```
+
+
+
+## type Logger
+``` go
+type Logger struct {
+    // Filename is the file to write logs to.  Backup log files will be retained
+    // in the same directory.  It uses <processname>-lumberjack.log in
+    // os.TempDir() if empty.
+    Filename string `json:"filename" yaml:"filename"`
+
+    // MaxSize is the maximum size in megabytes of the log file before it gets
+    // rotated. It defaults to 100 megabytes.
+    MaxSize int `json:"maxsize" yaml:"maxsize"`
+
+    // MaxAge is the maximum number of days to retain old log files based on the
+    // timestamp encoded in their filename.  Note that a day is defined as 24
+    // hours and may not exactly correspond to calendar days due to daylight
+    // savings, leap seconds, etc. The default is not to remove old log files
+    // based on age.
+    MaxAge int `json:"maxage" yaml:"maxage"`
+
+    // MaxBackups is the maximum number of old log files to retain.  The default
+    // is to retain all old log files (though MaxAge may still cause them to get
+    // deleted.)
+    MaxBackups int `json:"maxbackups" yaml:"maxbackups"`
+
+    // LocalTime determines if the time used for formatting the timestamps in
+    // backup files is the computer's local time.  The default is to use UTC
+    // time.
+    LocalTime bool `json:"localtime" yaml:"localtime"`
+    // contains filtered or unexported fields
+}
+```
+Logger is an io.WriteCloser that writes to the specified filename.
+
+Logger opens or creates the logfile on first Write.  If the file exists and
+is less than MaxSize megabytes, lumberjack will open and append to that file.
+If the file exists and its size is >= MaxSize megabytes, the file is renamed
+by putting the current time in a timestamp in the name immediately before the
+file's extension (or the end of the filename if there's no extension). A new
+log file is then created using original filename.
+
+Whenever a write would cause the current log file exceed MaxSize megabytes,
+the current file is closed, renamed, and a new log file created with the
+original name. Thus, the filename you give Logger is always the "current" log
+file.
+
+### Cleaning Up Old Log Files
+Whenever a new logfile gets created, old log files may be deleted.  The most
+recent files according to the encoded timestamp will be retained, up to a
+number equal to MaxBackups (or all of them if MaxBackups is 0).  Any files
+with an encoded timestamp older than MaxAge days are deleted, regardless of
+MaxBackups.  Note that the time encoded in the timestamp is the rotation
+time, which may differ from the last time that file was written to.
+
+If MaxBackups and MaxAge are both 0, no old log files will be deleted.
+
+
+
+
+
+
+
+
+
+
+
+### func (\*Logger) Close
+``` go
+func (l *Logger) Close() error
+```
+Close implements io.Closer, and closes the current logfile.
+
+
+
+### func (\*Logger) Rotate
+``` go
+func (l *Logger) Rotate() error
+```
+Rotate causes Logger to close the existing log file and immediately create a
+new one.  This is a helper function for applications that want to initiate
+rotations outside of the normal rotation rules, such as in response to
+SIGHUP.  After rotating, this initiates a cleanup of old log files according
+to the normal rules.
+
+**Example**
+
+Example of how to rotate in response to SIGHUP.
+
+Code:
+
+```go
+l := &lumberjack.Logger{}
+log.SetOutput(l)
+c := make(chan os.Signal, 1)
+signal.Notify(c, syscall.SIGHUP)
+
+go func() {
+    for {
+        <-c
+        l.Rotate()
+    }
+}()
+```
+
+### func (\*Logger) Write
+``` go
+func (l *Logger) Write(p []byte) (n int, err error)
+```
+Write implements io.Writer.  If a write would cause the log file to be larger
+than MaxSize, the file is closed, renamed to include a timestamp of the
+current time, and a new log file is created using the original log file name.
+If the length of the write is greater than MaxSize, an error is returned.
+
+
+
+
+
+
+
+
+
+- - -
+Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md)

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/chown.go

+// +build !linux
+
+package lumberjack
+
+import (
+	"os"
+)
+
+func chown(_ string, _ os.FileInfo) error {
+	return nil
+}

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/chown_linux.go

+package lumberjack
+
+import (
+	"os"
+	"syscall"
+)
+
+// os_Chown is a var so we can mock it out during tests.
+var os_Chown = os.Chown
+
+func chown(name string, info os.FileInfo) error {
+	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode())
+	if err != nil {
+		return err
+	}
+	f.Close()
+	stat := info.Sys().(*syscall.Stat_t)
+	return os_Chown(name, int(stat.Uid), int(stat.Gid))
+}

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/example_test.go

+package lumberjack_test
+
+import (
+	"log"
+
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+// To use lumberjack with the standard library's log package, just pass it into
+// the SetOutput function when your application starts.
+func Example() {
+	log.SetOutput(&lumberjack.Logger{
+		Filename:   "/var/log/myapp/foo.log",
+		MaxSize:    500, // megabytes
+		MaxBackups: 3,
+		MaxAge:     28, // days
+	})
+}

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/linux_test.go

+// +build linux
+
+package lumberjack
+
+import (
+	"os"
+	"syscall"
+	"testing"
+)
+
+func TestMaintainMode(t *testing.T) {
+	currentTime = fakeTime
+	dir := makeTempDir("TestMaintainMode", t)
+	defer os.RemoveAll(dir)
+
+	filename := logFile(dir)
+
+	mode := os.FileMode(0770)
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, mode)
+	isNil(err, t)
+	f.Close()
+
+	l := &Logger{
+		Filename:   filename,
+		MaxBackups: 1,
+		MaxSize:    100, // megabytes
+	}
+	defer l.Close()
+	b := []byte("boo!")
+	n, err := l.Write(b)
+	isNil(err, t)
+	equals(len(b), n, t)
+
+	newFakeTime()
+
+	err = l.Rotate()
+	isNil(err, t)
+
+	filename2 := backupFile(dir)
+	info, err := os.Stat(filename)
+	isNil(err, t)
+	info2, err := os.Stat(filename2)
+	isNil(err, t)
+	equals(mode, info.Mode(), t)
+	equals(mode, info2.Mode(), t)
+}
+
+func TestMaintainOwner(t *testing.T) {
+	fakeC := fakeChown{}
+	os_Chown = fakeC.Set
+	os_Stat = fakeStat
+	defer func() {
+		os_Chown = os.Chown
+		os_Stat = os.Stat
+	}()
+	currentTime = fakeTime
+	dir := makeTempDir("TestMaintainOwner", t)
+	defer os.RemoveAll(dir)
+
+	filename := logFile(dir)
+
+	l := &Logger{
+		Filename:   filename,
+		MaxBackups: 1,
+		MaxSize:    100, // megabytes
+	}
+	defer l.Close()
+	b := []byte("boo!")
+	n, err := l.Write(b)
+	isNil(err, t)
+	equals(len(b), n, t)
+
+	newFakeTime()
+
+	err = l.Rotate()
+	isNil(err, t)
+
+	equals(555, fakeC.uid, t)
+	equals(666, fakeC.gid, t)
+}
+
+type fakeChown struct {
+	name string
+	uid  int
+	gid  int
+}
+
+func (f *fakeChown) Set(name string, uid, gid int) error {
+	f.name = name
+	f.uid = uid
+	f.gid = gid
+	return nil
+}
+
+func fakeStat(name string) (os.FileInfo, error) {
+	info, err := os.Stat(name)
+	if err != nil {
+		return info, err
+	}
+	stat := info.Sys().(*syscall.Stat_t)
+	stat.Uid = 555
+	stat.Gid = 666
+	return info, nil
+}

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/rotate_test.go

+// +build linux
+
+package lumberjack_test
+
+import (
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/natefinch/lumberjack"
+)
+
+// Example of how to rotate in response to SIGHUP.
+func ExampleLogger_Rotate() {
+	l := &lumberjack.Logger{}
+	log.SetOutput(l)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP)
+
+	go func() {
+		for {
+			<-c
+			l.Rotate()
+		}
+	}()
+}

Godeps/_workspace/src/gopkg.in/natefinch/lumberjack.v2/testing_test.go

+package lumberjack
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+)
+
+// assert will log the given message if condition is false.
+func assert(condition bool, t testing.TB, msg string, v ...interface{}) {
+	assertUp(condition, t, 1, msg, v...)
+}
+
+// assertUp is like assert, but used inside helper functions, to ensure that
+// the file and line number reported by failures corresponds to one or more
+// levels up the stack.
+func assertUp(condition bool, t testing.TB, caller int, msg string, v ...interface{}) {
+	if !condition {
+		_, file, line, _ := runtime.Caller(caller + 1)
+		v = append([]interface{}{filepath.Base(file), line}, v...)
+		fmt.Printf("%s:%d: "+msg+"\n", v...)
+		t.FailNow()
+	}
+}
+
+// equals tests that the two values are equal according to reflect.DeepEqual.
+func equals(exp, act interface{}, t testing.TB) {
+	equalsUp(exp, act, t, 1)
+}
+
+// equalsUp is like equals, but used inside helper functions, to ensure that the
+// file and line number reported by failures corresponds to one or more levels
+// up the stack.
+func equalsUp(exp, act interface{}, t testing.TB, caller int) {
+	if !reflect.DeepEqual(exp, act) {
+		_, file, line, _ := runtime.Caller(caller + 1)
+		fmt.Printf("%s:%d: exp: %v (%T), got: %v (%T)\n",
+			filepath.Base(file), line, exp, exp, act, act)
+		t.FailNow()
+	}
+}
+
+// isNil reports a failure if the given value is not nil.  Note that values
+// which cannot be nil will always fail this check.
+func isNil(obtained interface{}, t testing.TB) {
+	isNilUp(obtained, t, 1)
+}
+
+// isNilUp is like isNil, but used inside helper functions, to ensure that the
+// file and line number reported by failures corresponds to one or more levels
+// up the stack.
+func isNilUp(obtained interface{}, t testing.TB, caller int) {
+	if !_isNil(obtained) {
+		_, file, line, _ := runtime.Caller(caller + 1)
+		fmt.Printf("%s:%d: expected nil, got: %v\n", filepath.Base(file), line, obtained)
+		t.FailNow()
+	}
+}
+
+// notNil reports a failure if the given value is nil.
+func notNil(obtained interface{}, t testing.TB) {
+	notNilUp(obtained, t, 1)
+}
+
+// notNilUp is like notNil, but used inside helper functions, to ensure that the
+// file and line number reported by failures corresponds to one or more levels
+// up the stack.
+func notNilUp(obtained interface{}, t testing.TB, caller int) {
+	if _isNil(obtained) {
+		_, file, line, _ := runtime.Caller(caller + 1)
+		fmt.Printf("%s:%d: expected non-nil, got: %v\n", filepath.Base(file), line, obtained)
+		t.FailNow()
+	}
+}
+
+// _isNil is a helper function for isNil and notNil, and should not be used
+// directly.
+func _isNil(obtained interface{}) bool {
+	if obtained == nil {
+		return true
+	}
+
+	switch v := reflect.ValueOf(obtained); v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+		return v.IsNil()
+	}
+
+	return false
+}

api/swagger-spec/v1.json

@@ -5638,6 +5638,152 @@
     ]
    },
    {
+    "path": "/api/v1/namespaces/{namespace}/pods/{name}/attach",
+    "description": "API at /api/v1 version v1",
+    "operations": [
+     {
+      "type": "string",
+      "method": "GET",
+      "summary": "connect GET requests to attach of Pod",
+      "nickname": "connectGetNamespacedPodAttach",
+      "parameters": [
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stdin",
+        "description": "redirect the standard input stream of the pod for this call; defaults to false",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stdout",
+        "description": "redirect the standard output stream of the pod for this call; defaults to true",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stderr",
+        "description": "redirect the standard error stream of the pod for this call; defaults to true",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "tty",
+        "description": "allocate a terminal for this attach call; defaults to false",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "query",
+        "name": "container",
+        "description": "the container in which to execute the command. Defaults to only container if there is only one container in the pod.",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "path",
+        "name": "namespace",
+        "description": "object name and auth scope, such as for teams and projects",
+        "required": true,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "path",
+        "name": "name",
+        "description": "name of the Pod",
+        "required": true,
+        "allowMultiple": false
+       }
+      ],
+      "produces": [
+       "*/*"
+      ],
+      "consumes": [
+       "*/*"
+      ]
+     },
+     {
+      "type": "string",
+      "method": "POST",
+      "summary": "connect POST requests to attach of Pod",
+      "nickname": "connectPostNamespacedPodAttach",
+      "parameters": [
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stdin",
+        "description": "redirect the standard input stream of the pod for this call; defaults to false",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stdout",
+        "description": "redirect the standard output stream of the pod for this call; defaults to true",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "stderr",
+        "description": "redirect the standard error stream of the pod for this call; defaults to true",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "boolean",
+        "paramType": "query",
+        "name": "tty",
+        "description": "allocate a terminal for this attach call; defaults to false",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "query",
+        "name": "container",
+        "description": "the container in which to execute the command. Defaults to only container if there is only one container in the pod.",
+        "required": false,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "path",
+        "name": "namespace",
+        "description": "object name and auth scope, such as for teams and projects",
+        "required": true,
+        "allowMultiple": false
+       },
+       {
+        "type": "string",
+        "paramType": "path",
+        "name": "name",
+        "description": "name of the Pod",
+        "required": true,
+        "allowMultiple": false
+       }
+      ],
+      "produces": [
+       "*/*"
+      ],
+      "consumes": [
+       "*/*"
+      ]
+     }
+    ]
+   },
+   {
     "path": "/api/v1/namespaces/{namespace}/pods/{name}/binding",
     "description": "API at /api/v1 version v1",
     "operations": [

build/README.md

@@ -5,7 +5,8 @@ To build Kubernetes you need to have access to a Docker installation through eit
 ## Requirements
 
 1. Be running Docker.  2 options supported/tested:
-  1. **Mac OS X** The best way to go is to use `boot2docker`.  See instructions [here](https://docs.docker.com/installation/mac/).
+  1. **Mac OS X** The best way to go is to use `boot2docker`.  See instructions [here](https://docs.docker.com/installation/mac/).  
+  **Note**: You will want to set the boot2docker vm to have at least 3GB of initial memory or building will likely fail. (See: [#11852]( https://github.com/GoogleCloudPlatform/kubernetes/issues/11852))
   2. **Linux with local Docker**  Install Docker according to the [instructions](https://docs.docker.com/installation/#installation) for your OS.  The scripts here assume that they are using a local Docker server and that they can "reach around" docker and grab results directly from the file system.
 2. Have python installed.  Pretty much it is installed everywhere at this point so you can probably ignore this.
 3. *Optional* For uploading your release to Google Cloud Storage, have the [Google Cloud SDK](https://developers.google.com/cloud/sdk/) installed and configured.

build/mark-new-version.sh

@@ -52,6 +52,7 @@ fi
 
 release_branch="release-${VERSION_MAJOR}.${VERSION_MINOR}"
 current_branch=$(git rev-parse --abbrev-ref HEAD)
+head_commit=$(git rev-parse --short HEAD)
 
 if [[ "${VERSION_PATCH}" != "0" ]]; then
   # sorry, no going back in time, pull latest from upstream
@@ -93,15 +94,13 @@ echo "+++ Running ./versionize-docs"
 ${KUBE_ROOT}/build/versionize-docs.sh ${NEW_VERSION}
 git commit -am "Versioning docs and examples for ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}"
 
-dochash=$(git log -n1 --format=%H)
-
 VERSION_FILE="${KUBE_ROOT}/pkg/version/base.go"
 
 GIT_MINOR="${VERSION_MINOR}.${VERSION_PATCH}"
 echo "+++ Updating to ${NEW_VERSION}"
 $SED -ri -e "s/gitMajor\s+string = \"[^\"]*\"/gitMajor string = \"${VERSION_MAJOR}\"/" "${VERSION_FILE}"
 $SED -ri -e "s/gitMinor\s+string = \"[^\"]*\"/gitMinor string = \"${GIT_MINOR}\"/" "${VERSION_FILE}"
-$SED -ri -e "s/gitVersion\s+string = \"[^\"]*\"/gitVersion string = \"$NEW_VERSION\"/" "${VERSION_FILE}"
+$SED -ri -e "s/gitVersion\s+string = \"[^\"]*\"/gitVersion string = \"$NEW_VERSION-${release_branch}+\$Format:%h\$\"/" "${VERSION_FILE}"
 gofmt -s -w "${VERSION_FILE}"
 
 echo "+++ Committing version change"
@@ -110,35 +109,30 @@ git commit -m "Kubernetes version $NEW_VERSION"
 
 echo "+++ Tagging version"
 git tag -a -m "Kubernetes version $NEW_VERSION" "${NEW_VERSION}"
+newtag=$(git rev-parse --short HEAD)
 
-echo "+++ Updating to ${NEW_VERSION}-dev"
-$SED -ri -e "s/gitMajor\s+string = \"[^\"]*\"/gitMajor string = \"${VERSION_MAJOR}\"/" "${VERSION_FILE}"
-$SED -ri -e "s/gitMinor\s+string = \"[^\"]*\"/gitMinor string = \"${GIT_MINOR}\+\"/" "${VERSION_FILE}"
-$SED -ri -e "s/gitVersion\s+string = \"[^\"]*\"/gitVersion string = \"$NEW_VERSION-dev\"/" "${VERSION_FILE}"
-gofmt -s -w "${VERSION_FILE}"
-
-echo "+++ Committing version change"
-git add "${VERSION_FILE}"
-git commit -m "Kubernetes version ${NEW_VERSION}-dev"
+if [[ "${VERSION_PATCH}" == "0" ]]; then
+  declare -r alpha_ver="v${VERSION_MAJOR}.$((${VERSION_MINOR}+1)).0-alpha.0"
+  git tag -a -m "Kubernetes pre-release branch ${alpha-ver}" "${alpha_ver}" "${head_commit}"
+fi
 
 echo ""
 echo "Success you must now:"
 echo ""
 echo "- Push the tag:"
 echo "   git push ${push_url} v${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}"
-echo "   - Please note you are pushing the tag live BEFORE your PRs."
-echo "       You need this so the builds pick up the right tag info (and so your reviewers can see it)."
-echo "       If something goes wrong further down please fix the tag!"
-echo "       Either delete this tag and give up, fix the tag before your next PR,"
-echo "       or find someone who can help solve the tag problem!"
-echo ""
 
 if [[ "${VERSION_PATCH}" == "0" ]]; then
-  echo "- Send branch: ${current_branch} as a PR to ${push_url}/master"
-  echo "   For major/minor releases, this gets the branch tag merged and changes the version numbers."
+  echo "- Push the alpha tag:"
+  echo "   git push ${push_url} ${alpha_ver}"
   echo "- Push the new release branch:"
   echo "   git push ${push_url} ${current_branch}:${release_branch}"
+  echo "- DO NOTHING TO MASTER. You were done with master when you pushed the alpha tag."
 else
   echo "- Send branch: ${current_branch} as a PR to ${release_branch} <-- NOTE THIS"
-  echo "  Get someone to review and merge that PR"
+  echo "- In the contents of the PR, include the PRs in the release:"
+  echo "    hack/cherry_pick_list.sh ${current_branch}^1"
+  echo "  This helps cross-link PRs to patch releases they're part of in GitHub."
+  echo "- Have someone review the PR. This is a mechanical review to ensure it contains"
+  echo "  the ${NEW_VERSION} commit, which was tagged at ${newtag}."
 fi

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -33,3 +33,11 @@ spec:
             - --sink=gcl
             - --poll_duration=2m
             - --stats_resolution=1m
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs
+              readOnly: true
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs"

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -33,3 +33,11 @@ spec:
             - --sink=influxdb:http://monitoring-influxdb:8086
             - --poll_duration=2m
             - --stats_resolution=1m
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs
+              readOnly: true
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs"

cluster/addons/dns/kube2sky/kube2sky.go

@@ -37,7 +37,7 @@ import (
 	kclientcmd "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
 	kframework "github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
 	kSelector "github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
-	tools "github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
+	etcdstorage "github.com/GoogleCloudPlatform/kubernetes/pkg/storage/etcd"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 	etcd "github.com/coreos/go-etcd/etcd"
@@ -354,7 +354,7 @@ func newEtcdClient(etcdServer string) (*etcd.Client, error) {
 		err    error
 	)
 	for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
-		if _, err = tools.GetEtcdVersion(etcdServer); err == nil {
+		if _, err = etcdstorage.GetEtcdVersion(etcdServer); err == nil {
 			break
 		}
 		if attempt == maxConnectAttempts {

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

 .PHONY:	build push
 
 IMAGE = fluentd-elasticsearch
-TAG = 1.6
+TAG = 1.7
 
 build:	
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -94,20 +94,13 @@
   tag docker
 </source>
 
-<match kubernetes.**>
-   type elasticsearch
-   log_level info
-   include_tag_key true
-   host elasticsearch-logging
-   port 9200
-   logstash_format true
-   flush_interval 5s
-   # Never wait longer than 5 minutes between retries.
-   max_retry_wait 300
-   # Disable the limit on the number of retries (retry forever).
-   disable_retry_limit
-</match>
->>>>>>> Move things into a 'kube-system' namespace.
+<source>
+  type tail
+  format none
+  path /varlog/etcd.log
+  pos_file /varlog/es-etcd.log.pos
+  tag etcd
+</source>
 
 <source>
   type tail

cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile

@@ -15,7 +15,7 @@
 
 .PHONY:	kbuild kpush
 
-TAG = 1.9
+TAG = 1.10
 
 # Rules for building the test image for deployment to Dockerhub with user kubernetes.
 

cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf

@@ -82,6 +82,14 @@
 <source>
   type tail
   format none
+  path /varlog/etcd.log
+  pos_file /varlog/gcp-etcd.log.pos
+  tag etcd
+</source>
+
+<source>
+  type tail
+  format none
   path /varlog/kubelet.log
   pos_file /varlog/gcp-kubelet.log.pos
   tag kubelet

cluster/saltbase/README.md

 # SaltStack configuration
 
 This is the root of the SaltStack configuration for Kubernetes. A high
-level overview for the Kubernetes SaltStack configuration can be found [in the docs tree.](../../docs/salt.md)
+level overview for the Kubernetes SaltStack configuration can be found [in the docs tree.](../../docs/admin/salt.md)
 
 This SaltStack configuration currently applies to default
 configurations for Debian-on-GCE, Fedora-on-Vagrant, Ubuntu-on-AWS and

cluster/saltbase/salt/fluentd-es/fluentd-es.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-elasticsearch
-    image: gcr.io/google_containers/fluentd-elasticsearch:1.6
+    image: gcr.io/google_containers/fluentd-elasticsearch:1.7
     resources:
       limits:
         cpu: 100m

cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-cloud-logging
-    image: gcr.io/google_containers/fluentd-gcp:1.9
+    image: gcr.io/google_containers/fluentd-gcp:1.10
     resources:
       limits:
         cpu: 100m

cluster/saltbase/salt/kubelet/default

@@ -41,6 +41,12 @@
 {% endif -%}
 
 {% set config = "--config=/etc/kubernetes/manifests" -%}
+
+{% set manifest_url = "" -%}
+{% if grains['roles'][0] == 'kubernetes-master' and grains.cloud in ['gce'] -%}
+  {% set manifest_url = "--manifest-url=http://metadata.google.internal/computeMetadata/v1/instance/attributes/google-container-manifest --manifest-url-header=Metadata-Flavor:Google" -%}
+{% endif -%}
+
 {% set hostname_override = "" -%}
 {% if grains.hostname_override is defined -%}
   {% set hostname_override = " --hostname_override=" + grains.hostname_override -%}
@@ -84,4 +90,4 @@
   {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %}
 {% endif %} 
 
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}}"

cmd/integration/integration.go

@@ -42,6 +42,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/nodecontroller"
 	replicationControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/controller/replication"
+	explatest "github.com/GoogleCloudPlatform/kubernetes/pkg/expapi/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
@@ -132,10 +133,14 @@ func startComponents(firstManifestURL, secondManifestURL, apiVersion string) (st
 
 	cl := client.NewOrDie(&client.Config{Host: apiServer.URL, Version: apiVersion})
 
-	etcdStorage, err := master.NewEtcdStorage(etcdClient, "", etcdtest.PathPrefix())
+	etcdStorage, err := master.NewEtcdStorage(etcdClient, latest.InterfacesFor, latest.Version, etcdtest.PathPrefix())
 	if err != nil {
 		glog.Fatalf("Unable to get etcd storage: %v", err)
 	}
+	expEtcdStorage, err := master.NewEtcdStorage(etcdClient, explatest.InterfacesFor, explatest.Version, etcdtest.PathPrefix())
+	if err != nil {
+		glog.Fatalf("Unable to get etcd storage for experimental: %v", err)
+	}
 
 	// Master
 	host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://"))
@@ -155,11 +160,13 @@ func startComponents(firstManifestURL, secondManifestURL, apiVersion string) (st
 	// Create a master and install handlers into mux.
 	m := master.New(&master.Config{
 		DatabaseStorage:       etcdStorage,
+		ExpDatabaseStorage:    expEtcdStorage,
 		KubeletClient:         fakeKubeletClient{},
 		EnableCoreControllers: true,
 		EnableLogsSupport:     false,
 		EnableProfiling:       true,
 		APIPrefix:             "/api",
+		ExpAPIPrefix:          "/experimental",
 		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
 		AdmissionControl:      admit.NewAlwaysAdmit(),
 		ReadWritePort:         portNumber,

cmd/kube-apiserver/app/server.go

@@ -32,12 +32,16 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/meta"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
+	explatest "github.com/GoogleCloudPlatform/kubernetes/pkg/expapi/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/storage"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	forked "github.com/GoogleCloudPlatform/kubernetes/third_party/forked/coreos/go-etcd/etcd"
@@ -70,7 +74,9 @@ type APIServer struct {
 	TLSPrivateKeyFile          string
 	CertDirectory              string
 	APIPrefix                  string
+	ExpAPIPrefix               string
 	StorageVersion             string
+	ExpStorageVersion          string
 	CloudProvider              string
 	CloudConfigFile            string
 	EventTTL                   time.Duration
@@ -114,6 +120,7 @@ func NewAPIServer() *APIServer {
 		APIRate:                10.0,
 		APIBurst:               200,
 		APIPrefix:              "/api",
+		ExpAPIPrefix:           "/experimental",
 		EventTTL:               1 * time.Hour,
 		AuthorizationMode:      "AlwaysAllow",
 		AdmissionControl:       "AlwaysAdmit",
@@ -171,6 +178,7 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+
 		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
 	fs.StringVar(&s.APIPrefix, "api-prefix", s.APIPrefix, "The prefix for API requests on the server. Default '/api'.")
+	fs.StringVar(&s.ExpAPIPrefix, "experimental-prefix", s.ExpAPIPrefix, "The prefix for experimental API requests on the server. Default '/experimental'.")
 	fs.StringVar(&s.StorageVersion, "storage-version", s.StorageVersion, "The version to store resources with. Defaults to server preferred")
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
 	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
@@ -216,7 +224,7 @@ func (s *APIServer) verifyClusterIPFlags() {
 	}
 }
 
-func newEtcd(etcdConfigFile string, etcdServerList util.StringList, storageVersion string, pathPrefix string) (etcdStorage tools.StorageInterface, err error) {
+func newEtcd(etcdConfigFile string, etcdServerList util.StringList, interfacesFunc meta.VersionInterfacesFunc, defaultVersion, storageVersion, pathPrefix string) (etcdStorage storage.Interface, err error) {
 	var client tools.EtcdClient
 	if etcdConfigFile != "" {
 		client, err = etcd.NewClientFromFile(etcdConfigFile)
@@ -236,7 +244,10 @@ func newEtcd(etcdConfigFile string, etcdServerList util.StringList, storageVersi
 		client = etcdClient
 	}
 
-	return master.NewEtcdStorage(client, storageVersion, pathPrefix)
+	if storageVersion == "" {
+		storageVersion = defaultVersion
+	}
+	return master.NewEtcdStorage(client, interfacesFunc, storageVersion, pathPrefix)
 }
 
 // Run runs the specified APIServer.  This should never exit.
@@ -291,6 +302,10 @@ func (s *APIServer) Run(_ []string) error {
 	disableV1 := disableAllAPIs
 	disableV1 = !s.getRuntimeConfigValue("api/v1", !disableV1)
 
+	// "experimental/v1={true|false} allows users to enable/disable the experimental API.
+	// This takes preference over api/all, if specified.
+	enableExp := s.getRuntimeConfigValue("experimental/v1", false)
+
 	// TODO: expose same flags as client.BindClientConfigFlags but for a server
 	clientConfig := &client.Config{
 		Host:    net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
@@ -301,10 +316,14 @@ func (s *APIServer) Run(_ []string) error {
 		glog.Fatalf("Invalid server address: %v", err)
 	}
 
-	etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, s.StorageVersion, s.EtcdPathPrefix)
+	etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, latest.InterfacesFor, latest.Version, s.StorageVersion, s.EtcdPathPrefix)
 	if err != nil {
 		glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
 	}
+	expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, explatest.InterfacesFor, explatest.Version, s.ExpStorageVersion, s.EtcdPathPrefix)
+	if err != nil {
+		glog.Fatalf("Invalid experimental storage version or misconfigured etcd: %v", err)
+	}
 
 	n := net.IPNet(s.ServiceClusterIPRange)
 
@@ -359,7 +378,9 @@ func (s *APIServer) Run(_ []string) error {
 		}
 	}
 	config := &master.Config{
-		DatabaseStorage:        etcdStorage,
+		DatabaseStorage:    etcdStorage,
+		ExpDatabaseStorage: expEtcdStorage,
+
 		EventTTL:               s.EventTTL,
 		KubeletClient:          kubeletClient,
 		ServiceClusterIPRange:  &n,
@@ -370,6 +391,7 @@ func (s *APIServer) Run(_ []string) error {
 		EnableProfiling:        s.EnableProfiling,
 		EnableIndex:            true,
 		APIPrefix:              s.APIPrefix,
+		ExpAPIPrefix:           s.ExpAPIPrefix,
 		CorsAllowedOriginList:  s.CorsAllowedOriginList,
 		ReadWritePort:          s.SecurePort,
 		PublicAddress:          net.IP(s.AdvertiseAddress),
@@ -378,6 +400,7 @@ func (s *APIServer) Run(_ []string) error {
 		Authorizer:             authorizer,
 		AdmissionControl:       admissionController,
 		DisableV1:              disableV1,
+		EnableExp:              enableExp,
 		MasterServiceNamespace: s.MasterServiceNamespace,
 		ClusterName:            s.ClusterName,
 		ExternalHost:           s.ExternalHost,

cmd/kubelet/app/server.go

@@ -319,7 +319,7 @@ func (s *KubeletServer) Run(_ []string) error {
 	mounter := mount.New()
 	if s.Containerized {
 		glog.V(2).Info("Running kubelet in containerized mode (experimental)")
-		mounter = &mount.NsenterMounter{}
+		mounter = mount.NewNsenterMounter()
 	}
 
 	var dockerExecHandler dockertools.ExecHandler

cmd/kubernetes/kubernetes.go

@@ -30,17 +30,20 @@ import (
 
 	kubeletapp "github.com/GoogleCloudPlatform/kubernetes/cmd/kubelet/app"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/nodecontroller"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/servicecontroller"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/replication"
+	explatest "github.com/GoogleCloudPlatform/kubernetes/pkg/expapi/latest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
 	kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/service"
+	etcdstorage "github.com/GoogleCloudPlatform/kubernetes/pkg/storage/etcd"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
@@ -78,14 +81,19 @@ func (h *delegateHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 func runApiServer(etcdClient tools.EtcdClient, addr net.IP, port int, masterServiceNamespace string) {
 	handler := delegateHandler{}
 
-	etcdStorage, err := master.NewEtcdStorage(etcdClient, "", master.DefaultEtcdPathPrefix)
+	etcdStorage, err := master.NewEtcdStorage(etcdClient, latest.InterfacesFor, latest.Version, master.DefaultEtcdPathPrefix)
 	if err != nil {
 		glog.Fatalf("Unable to get etcd storage: %v", err)
 	}
+	expEtcdStorage, err := master.NewEtcdStorage(etcdClient, explatest.InterfacesFor, explatest.Version, master.DefaultEtcdPathPrefix)
+	if err != nil {
+		glog.Fatalf("Unable to get etcd storage for experimental: %v", err)
+	}
 
 	// Create a master and install handlers into mux.
 	m := master.New(&master.Config{
-		DatabaseStorage: etcdStorage,
+		DatabaseStorage:    etcdStorage,
+		ExpDatabaseStorage: expEtcdStorage,
 		KubeletClient: &client.HTTPKubeletClient{
 			Client: http.DefaultClient,
 			Config: &client.KubeletConfig{Port: 10250},
@@ -95,6 +103,7 @@ func runApiServer(etcdClient tools.EtcdClient, addr net.IP, port int, masterServ
 		EnableSwaggerSupport:  true,
 		EnableProfiling:       *enableProfiling,
 		APIPrefix:             "/api",
+		ExpAPIPrefix:          "/experimental",
 		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
 
 		ReadWritePort:          port,
@@ -167,7 +176,7 @@ func main() {
 	defer util.FlushLogs()
 
 	glog.Infof("Creating etcd client pointing to %v", *etcdServer)
-	etcdClient, err := tools.NewEtcdClientStartServerIfNecessary(*etcdServer)
+	etcdClient, err := etcdstorage.NewEtcdClientStartServerIfNecessary(*etcdServer)
 	if err != nil {
 		glog.Fatalf("Failed to connect to etcd: %v", err)
 	}

cmd/mungedocs/README.md

+# Documentation Mungers
+
+Basically this is like lint/gofmt for md docs.
+
+It basically does the following:
+- iterate over all files in the given doc root.
+- for each file split it into a slice (mungeLines) of lines (mungeLine)
+- a mungeline has metadata about each line typically determined by a 'fast' regex.
+  - metadata contains things like 'is inside a preformmatted block'
+  - contains a markdown header
+  - has a link to another file
+  - etc..
+  - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one.
+- each munger is then called in turn
+  - they are given the mungeLines
+  - they create an entirely new set of mungeLines with their modifications
+  - the new set is returned
+- the new set is then fed into the next munger.
+- in the end we might commit the end mungeLines to the file or not (--verify)
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]()

cmd/mungedocs/analytics.go

@@ -17,43 +17,42 @@ limitations under the License.
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"os"
-	"regexp"
+	"strings"
 )
 
-var (
-	beginMungeExp = regexp.QuoteMeta(beginMungeTag("GENERATED_ANALYTICS"))
-	endMungeExp   = regexp.QuoteMeta(endMungeTag("GENERATED_ANALYTICS"))
-	analyticsExp  = regexp.QuoteMeta("[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/") +
-		"[^?]*" +
-		regexp.QuoteMeta("?pixel)]()")
+const analyticsMungeTag = "GENERATED_ANALYTICS"
+const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/"
 
-	// Matches the analytics blurb, with or without the munge headers.
-	analyticsRE = regexp.MustCompile(`[\n]*` + analyticsExp + `[\n]?` +
-		`|` + `[\n]*` + beginMungeExp + `[^<]*` + endMungeExp)
-)
-
-// This adds the analytics link to every .md file.
-func checkAnalytics(fileName string, fileBytes []byte) (output []byte, err error) {
-	fileName = makeRepoRelative(fileName)
-	desired := fmt.Sprintf(`
+func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) {
+	var out mungeLines
+	fileName, err := makeRepoRelative(fileName, fileName)
+	if err != nil {
+		return mlines, err
+	}
 
+	link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName)
+	insertLines := getMungeLines(link)
+	mlines, err = removeMacroBlock(analyticsMungeTag, mlines)
+	if err != nil {
+		return mlines, err
+	}
 
-`+beginMungeTag("GENERATED_ANALYTICS")+`
-[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/%s?pixel)]()
-`+endMungeTag("GENERATED_ANALYTICS")+`
-`, fileName)
-	if !analyticsRE.MatchString(desired) {
-		fmt.Printf("%q does not match %q", analyticsRE.String(), desired)
-		os.Exit(1)
+	// Remove floating analytics links not surrounded by the munge tags.
+	for _, mline := range mlines {
+		if mline.preformatted || mline.header || mline.beginTag || mline.endTag {
+			out = append(out, mline)
+			continue
+		}
+		if strings.HasPrefix(mline.data, analyticsLinePrefix) {
+			continue
+		}
+		out = append(out, mline)
+	}
+	out = appendMacroBlock(out, analyticsMungeTag)
+	out, err = updateMacroBlock(out, analyticsMungeTag, insertLines)
+	if err != nil {
+		return mlines, err
 	}
-	//output = replaceNonPreformattedRegexp(fileBytes, analyticsRE, func(in []byte) []byte {
-	output = analyticsRE.ReplaceAllFunc(fileBytes, func(in []byte) []byte {
-		return []byte{}
-	})
-	output = bytes.TrimRight(output, "\n")
-	output = append(output, []byte(desired)...)
-	return output, nil
+	return out, nil
 }

cmd/mungedocs/analytics_test.go

@@ -23,67 +23,71 @@ import (
 )
 
 func TestAnalytics(t *testing.T) {
+	b := beginMungeTag("GENERATED_ANALYTICS")
+	e := endMungeTag("GENERATED_ANALYTICS")
 	var cases = []struct {
-		in  string
-		out string
+		in       string
+		expected string
 	}{
 		{
 			"aoeu",
-			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+			"aoeu" + "\n" + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 		{
 			"aoeu" + "\n" + "\n" + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()",
 			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 		{
 			"aoeu" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n",
-			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				e + "\n",
+			"aoeu" + "\n" + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 		{
 			"aoeu" + "\n" + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n",
-			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				e + "\n",
+			"aoeu" + "\n" + "\n" + "\n" + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 		{
 			"prefix" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") +
+				e +
 				"\n" + "suffix",
-			"prefix" + "\n" + "suffix" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+			"prefix" + "\n" + "suffix" + "\n" + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 		{
 			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n",
+				e + "\n",
 			"aoeu" + "\n" + "\n" + "\n" +
-				beginMungeTag("GENERATED_ANALYTICS") + "\n" +
+				b + "\n" +
 				"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
-				endMungeTag("GENERATED_ANALYTICS") + "\n"},
+				e + "\n"},
 	}
-	for _, c := range cases {
-		out, err := checkAnalytics("path/to/file-name.md", []byte(c.in))
+	for i, c := range cases {
+		in := getMungeLines(c.in)
+		expected := getMungeLines(c.expected)
+		out, err := updateAnalytics("path/to/file-name.md", in)
 		assert.NoError(t, err)
-		if string(out) != c.out {
-			t.Errorf("Expected \n\n%v\n\n but got \n\n%v\n\n", c.out, string(out))
+		if !expected.Equal(out) {
+			t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String())
 		}
 	}
 }

cmd/mungedocs/example_syncer.go

@@ -17,15 +17,17 @@ limitations under the License.
 package main
 
 import (
-	"bytes"
 	"fmt"
 	"io/ioutil"
-	"path"
 	"regexp"
 	"strings"
 )
 
-const exampleMungeTag = "EXAMPLE"
+const exampleToken = "EXAMPLE"
+
+const exampleLineStart = "<!-- BEGIN MUNGE: EXAMPLE"
+
+var exampleMungeTagRE = regexp.MustCompile(beginMungeTag(fmt.Sprintf("%s %s", exampleToken, `(([^ ])*.(yaml|json))`)))
 
 // syncExamples updates all examples in markdown file.
 //
@@ -43,75 +45,70 @@ const exampleMungeTag = "EXAMPLE"
 //
 // [Download example](../../examples/guestbook/frontend-controller.yaml)
 // <!-- END MUNGE: EXAMPLE -->
-func syncExamples(filePath string, markdown []byte) ([]byte, error) {
-	// find the example syncer begin tag
-	header := beginMungeTag(fmt.Sprintf("%s %s", exampleMungeTag, `(([^ ])*.(yaml|json))`))
-	exampleLinkRE := regexp.MustCompile(header)
-	lines := splitLines(markdown)
-	updatedMarkdown, err := updateExampleMacroBlock(filePath, lines, exampleLinkRE, endMungeTag(exampleMungeTag))
-	if err != nil {
-		return updatedMarkdown, err
+func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) {
+	var err error
+	type exampleTag struct {
+		token    string
+		linkText string
+		fileType string
+	}
+	exampleTags := []exampleTag{}
+
+	// collect all example Tags
+	for _, mline := range mlines {
+		if mline.preformatted || !mline.beginTag {
+			continue
+		}
+		line := mline.data
+		if !strings.HasPrefix(line, exampleLineStart) {
+			continue
+		}
+		match := exampleMungeTagRE.FindStringSubmatch(line)
+		if len(match) < 4 {
+			err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line)
+			return mlines, err
+		}
+		tag := exampleTag{
+			token:    exampleToken + " " + match[1],
+			linkText: match[1],
+			fileType: match[3],
+		}
+		exampleTags = append(exampleTags, tag)
+	}
+	// update all example Tags
+	for _, tag := range exampleTags {
+		example, err := exampleContent(filePath, tag.linkText, tag.fileType)
+		if err != nil {
+			return mlines, err
+		}
+		mlines, err = updateMacroBlock(mlines, tag.token, example)
+		if err != nil {
+			return mlines, err
+		}
 	}
-	return updatedMarkdown, nil
+	return mlines, nil
 }
 
 // exampleContent retrieves the content of the file at linkPath
-func exampleContent(filePath, linkPath, fileType string) (content string, err error) {
-	realRoot := path.Join(*rootDir, *repoRoot) + "/"
-	path := path.Join(realRoot, path.Dir(filePath), linkPath)
-	dat, err := ioutil.ReadFile(path)
+func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) {
+	repoRel, err := makeRepoRelative(linkPath, filePath)
 	if err != nil {
-		return content, err
+		return nil, err
 	}
-	// remove leading and trailing spaces and newlines
-	trimmedFileContent := strings.TrimSpace(string(dat))
-	content = fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s)", fileType, trimmedFileContent, linkPath)
-	return
-}
 
-// updateExampleMacroBlock sync the yaml/json example between begin tag and end tag
-func updateExampleMacroBlock(filePath string, lines []string, beginMarkExp *regexp.Regexp, endMark string) ([]byte, error) {
-	var buffer bytes.Buffer
-	betweenBeginAndEnd := false
-	for _, line := range lines {
-		trimmedLine := strings.Trim(line, " \n")
-		if beginMarkExp.Match([]byte(trimmedLine)) {
-			if betweenBeginAndEnd {
-				return nil, fmt.Errorf("found second begin mark while updating macro blocks")
-			}
-			betweenBeginAndEnd = true
-			buffer.WriteString(line)
-			buffer.WriteString("\n")
-			match := beginMarkExp.FindStringSubmatch(line)
-			if len(match) < 4 {
-				return nil, fmt.Errorf("failed to parse the link in example header")
-			}
-			// match[0] is the entire expression; [1] is the link text and [3] is the file type (yaml or json).
-			linkText := match[1]
-			fileType := match[3]
-			example, err := exampleContent(filePath, linkText, fileType)
-			if err != nil {
-				return nil, err
-			}
-			buffer.WriteString(example)
-		} else if trimmedLine == endMark {
-			if !betweenBeginAndEnd {
-				return nil, fmt.Errorf("found end mark without being mark while updating macro blocks")
-			}
-			// Extra newline avoids github markdown bug where comment ends up on same line as last bullet.
-			buffer.WriteString("\n")
-			buffer.WriteString(line)
-			buffer.WriteString("\n")
-			betweenBeginAndEnd = false
-		} else {
-			if !betweenBeginAndEnd {
-				buffer.WriteString(line)
-				buffer.WriteString("\n")
-			}
-		}
+	fileRel, err := makeFileRelative(linkPath, filePath)
+	if err != nil {
+		return nil, err
 	}
-	if betweenBeginAndEnd {
-		return nil, fmt.Errorf("never found closing end mark while updating macro blocks")
+
+	dat, err := ioutil.ReadFile(repoRel)
+	if err != nil {
+		return nil, err
 	}
-	return buffer.Bytes(), nil
+
+	// remove leading and trailing spaces and newlines
+	trimmedFileContent := strings.TrimSpace(string(dat))
+	content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s)", fileType, trimmedFileContent, fileRel)
+	out := getMungeLines(content)
+	return out, nil
 }

cmd/mungedocs/example_syncer_test.go

@@ -35,24 +35,27 @@ spec:
     - containerPort: 80
 `
 	var cases = []struct {
-		in  string
-		out string
+		in       string
+		expected string
 	}{
 		{"", ""},
 		{
-			"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE -->\n",
-			"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE -->\n",
+			"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE testdata/pod.yaml -->\n",
+			"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE testdata/pod.yaml -->\n",
 		},
 		{
-			"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE -->\n",
-			"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE -->\n",
+			"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n",
+			"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n",
 		},
 	}
+	repoRoot = ""
 	for _, c := range cases {
-		actual, err := syncExamples("mungedocs/filename.md", []byte(c.in))
+		in := getMungeLines(c.in)
+		expected := getMungeLines(c.expected)
+		actual, err := syncExamples("filename.md", in)
 		assert.NoError(t, err)
-		if c.out != string(actual) {
-			t.Errorf("Expected example \n'%v' but got \n'%v'", c.out, string(actual))
+		if !expected.Equal(actual) {
+			t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String())
 		}
 	}
 }

cmd/mungedocs/headers.go

@@ -19,53 +19,56 @@ package main
 import (
 	"fmt"
 	"regexp"
-	"strings"
 )
 
 var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`)
-var whitespaceRegex = regexp.MustCompile(`^\s*$`)
 
-func fixHeaderLines(fileBytes []byte) []byte {
-	lines := splitLines(fileBytes)
-	out := []string{}
-	for i := range lines {
-		matches := headerRegex.FindStringSubmatch(lines[i])
-		if matches == nil {
-			out = append(out, lines[i])
-			continue
-		}
-		if i > 0 && !whitespaceRegex.Match([]byte(out[len(out)-1])) {
-			out = append(out, "")
-		}
-		out = append(out, fmt.Sprintf("%s %s", matches[1], matches[2]))
-		if i+1 < len(lines) && !whitespaceRegex.Match([]byte(lines[i+1])) {
-			out = append(out, "")
+func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines {
+	var out mungeLines
+
+	mline := mlines[linenum]
+	line := mlines[linenum].data
+
+	matches := headerRegex.FindStringSubmatch(line)
+	if matches == nil {
+		out = append(out, mline)
+		return out
+	}
+
+	// There must be a blank line before the # (unless first line in file)
+	if linenum != 0 {
+		newlen := len(newlines)
+		if newlines[newlen-1].data != "" {
+			out = append(out, blankMungeLine)
 		}
 	}
-	final := strings.Join(out, "\n")
-	// Preserve the end of the file.
-	if len(fileBytes) > 0 && fileBytes[len(fileBytes)-1] == '\n' {
-		final += "\n"
+
+	// There must be a space AFTER the ##'s
+	newline := fmt.Sprintf("%s %s", matches[1], matches[2])
+	newmline := newMungeLine(newline)
+	out = append(out, newmline)
+
+	// The next line needs to be a blank line (unless last line in file)
+	if len(mlines) > linenum+1 && mlines[linenum+1].data != "" {
+		out = append(out, blankMungeLine)
 	}
-	return []byte(final)
+	return out
 }
 
 // Header lines need whitespace around them and after the #s.
-func checkHeaderLines(filePath string, fileBytes []byte) ([]byte, error) {
-	fbs := splitByPreformatted(fileBytes)
-	fbs = append([]fileBlock{{false, []byte{}}}, fbs...)
-	fbs = append(fbs, fileBlock{false, []byte{}})
-
-	for i := range fbs {
-		block := &fbs[i]
-		if block.preformatted {
+func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) {
+	var out mungeLines
+	for i, mline := range mlines {
+		if mline.preformatted {
+			out = append(out, mline)
 			continue
 		}
-		block.data = fixHeaderLines(block.data)
-	}
-	output := []byte{}
-	for _, block := range fbs {
-		output = append(output, block.data...)
+		if !mline.header {
+			out = append(out, mline)
+			continue
+		}
+		newLines := fixHeaderLine(mlines, out, i)
+		out = append(out, newLines...)
 	}
-	return output, nil
+	return out, nil
 }

cmd/mungedocs/headers_test.go

@@ -24,8 +24,8 @@ import (
 
 func TestHeaderLines(t *testing.T) {
 	var cases = []struct {
-		in  string
-		out string
+		in       string
+		expected string
 	}{
 		{"", ""},
 		{
@@ -62,10 +62,12 @@ func TestHeaderLines(t *testing.T) {
 		},
 	}
 	for i, c := range cases {
-		actual, err := checkHeaderLines("filename.md", []byte(c.in))
+		in := getMungeLines(c.in)
+		expected := getMungeLines(c.expected)
+		actual, err := updateHeaderLines("filename.md", in)
 		assert.NoError(t, err)
-		if string(actual) != c.out {
-			t.Errorf("case[%d]: expected %q got %q", i, c.out, string(actual))
+		if !actual.Equal(expected) {
+			t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String())
 		}
 	}
 }

cmd/mungedocs/kubectl_dash_f.go

@@ -25,29 +25,25 @@ import (
 
 // Looks for lines that have kubectl commands with -f flags and files that
 // don't exist.
-func checkKubectlFileTargets(file string, markdown []byte) ([]byte, error) {
-	inside := false
-	lines := splitLines(markdown)
-	errors := []string{}
-	for i := range lines {
-		if strings.HasPrefix(lines[i], "```") {
-			inside = !inside
+func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) {
+	var errors []string
+	for i, mline := range mlines {
+		if !mline.preformatted {
+			continue
 		}
-		if inside {
-			if err := lookForKubectl(lines, i); err != nil {
-				errors = append(errors, err.Error())
-			}
+		if err := lookForKubectl(mline.data, i); err != nil {
+			errors = append(errors, err.Error())
 		}
 	}
 	err := error(nil)
 	if len(errors) != 0 {
 		err = fmt.Errorf("%s", strings.Join(errors, "\n"))
 	}
-	return markdown, err
+	return mlines, err
 }
 
-func lookForKubectl(lines []string, lineNum int) error {
-	fields := strings.Fields(lines[lineNum])
+func lookForKubectl(line string, lineNum int) error {
+	fields := strings.Fields(line)
 	for i := range fields {
 		if fields[i] == "kubectl" {
 			return gotKubectl(lineNum, fields, i)
@@ -56,26 +52,26 @@ func lookForKubectl(lines []string, lineNum int) error {
 	return nil
 }
 
-func gotKubectl(line int, fields []string, fieldNum int) error {
+func gotKubectl(lineNum int, fields []string, fieldNum int) error {
 	for i := fieldNum + 1; i < len(fields); i++ {
 		switch fields[i] {
 		case "create", "update", "replace", "delete":
-			return gotCommand(line, fields, i)
+			return gotCommand(lineNum, fields, i)
 		}
 	}
 	return nil
 }
 
-func gotCommand(line int, fields []string, fieldNum int) error {
+func gotCommand(lineNum int, fields []string, fieldNum int) error {
 	for i := fieldNum + 1; i < len(fields); i++ {
 		if strings.HasPrefix(fields[i], "-f") {
-			return gotDashF(line, fields, i)
+			return gotDashF(lineNum, fields, i)
 		}
 	}
 	return nil
 }
 
-func gotDashF(line int, fields []string, fieldNum int) error {
+func gotDashF(lineNum int, fields []string, fieldNum int) error {
 	target := ""
 	if fields[fieldNum] == "-f" {
 		if fieldNum+1 == len(fields) {
@@ -112,9 +108,9 @@ func gotDashF(line int, fields []string, fieldNum int) error {
 	}
 
 	// If we got here we expect the file to exist.
-	_, err := os.Stat(path.Join(*rootDir, *repoRoot, target))
+	_, err := os.Stat(path.Join(repoRoot, target))
 	if os.IsNotExist(err) {
-		return fmt.Errorf("%d: target file %q does not exist", line, target)
+		return fmt.Errorf("%d: target file %q does not exist", lineNum, target)
 	}
 	return err
 }

cmd/mungedocs/kubectl_dash_f_test.go

@@ -130,9 +130,9 @@ func TestKubectlDashF(t *testing.T) {
 		},
 	}
 	for i, c := range cases {
-		*rootDir = ""
-		*repoRoot = ""
-		_, err := checkKubectlFileTargets("filename.md", []byte(c.in))
+		repoRoot = ""
+		in := getMungeLines(c.in)
+		_, err := updateKubectlFileTargets("filename.md", in)
 		if err != nil && c.ok {
 			t.Errorf("case[%d]: expected success, got %v", i, err)
 		}