Add an integration test that checks for the metrics we expect to be exported

test/integration/client_test.go

@@ -21,8 +21,6 @@ package integration
 import (
 	"fmt"
 	"log"
-	"net/http"
-	"net/http/httptest"
 	"reflect"
 	"runtime"
 	"sync"
@@ -31,47 +29,19 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/version"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
-	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
 )
 
 func init() {
 	requireEtcd()
 }
 
-func RunAMaster(t *testing.T) (*master.Master, *httptest.Server) {
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version())
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	var m *master.Master
-	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		m.Handler.ServeHTTP(w, req)
-	}))
-
-	m = master.New(&master.Config{
-		EtcdHelper:        helper,
-		KubeletClient:     client.FakeKubeletClient{},
-		EnableLogsSupport: false,
-		EnableProfiling:   true,
-		EnableUISupport:   false,
-		APIPrefix:         "/api",
-		Authorizer:        apiserver.NewAlwaysAllowAuthorizer(),
-		AdmissionControl:  admit.NewAlwaysAdmit(),
-	})
-
-	return m, s
-}
-
 func TestClient(t *testing.T) {
-	_, s := RunAMaster(t)
+	_, s := runAMaster(t)
 	defer s.Close()
 
 	ns := api.NamespaceDefault
@@ -149,7 +119,7 @@ func TestMultiWatch(t *testing.T) {
 
 	deleteAllEtcdKeys()
 	defer deleteAllEtcdKeys()
-	_, s := RunAMaster(t)
+	_, s := runAMaster(t)
 	defer s.Close()
 
 	ns := api.NamespaceDefault

test/integration/metrics_test.go

+// +build integration,!no-etcd
+
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+	"github.com/golang/glog"
+	"github.com/golang/protobuf/proto"
+	prometheuspb "github.com/prometheus/client_model/go"
+)
+
+const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text"
+
+func init() {
+	requireEtcd()
+}
+
+func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) {
+	req, err := http.NewRequest("GET", s.URL+"/metrics", nil)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to create http request: %v", err)
+	}
+	// Ask the prometheus exporter for its text protocol buffer format, since it's
+	// much easier to parse than its plain-text format. Don't use the serialized
+	// proto representation since it uses a non-standard varint delimiter between
+	// metric families.
+	req.Header.Add("Accept", scrapeRequestHeader)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to contact metrics endpoint of master: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("Non-200 response trying to scrape metrics from master: %v", resp)
+	}
+
+	// Each line in the response body should contain all the data for a single metric.
+	var metrics []*prometheuspb.MetricFamily
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		var metric prometheuspb.MetricFamily
+		if err := proto.UnmarshalText(scanner.Text(), &metric); err != nil {
+			return nil, fmt.Errorf("Failed to unmarshal line of metrics response: %v", err)
+		}
+		glog.Infof("Got metric %q", metric.GetName())
+		metrics = append(metrics, &metric)
+	}
+	return metrics, nil
+}
+
+func checkForExpectedMetrics(t *testing.T, metrics []*prometheuspb.MetricFamily, expectedMetrics []string) {
+	foundMetrics := make(map[string]bool)
+	for _, metric := range metrics {
+		foundMetrics[metric.GetName()] = true
+	}
+	for _, expected := range expectedMetrics {
+		if _, found := foundMetrics[expected]; !found {
+			t.Errorf("Master metrics did not include expected metric %q", expected)
+		}
+	}
+}
+
+func TestMasterProcessMetrics(t *testing.T) {
+	_, s := runAMaster(t)
+	defer s.Close()
+
+	metrics, err := scrapeMetrics(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkForExpectedMetrics(t, metrics, []string{
+		"process_start_time_seconds",
+		"process_cpu_seconds_total",
+		"process_goroutines",
+		"process_open_fds",
+		"process_resident_memory_bytes",
+	})
+}
+
+func TestApiserverMetrics(t *testing.T) {
+	_, s := runAMaster(t)
+	defer s.Close()
+
+	// Make a request to the apiserver to ensure there's at least one data point
+	// for the metrics we're expecting -- otherwise, they won't be exported.
+	client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()})
+	if _, err := client.Pods(api.NamespaceDefault).List(labels.Everything()); err != nil {
+		t.Fatalf("unexpected error getting pods: %v", err)
+	}
+
+	metrics, err := scrapeMetrics(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkForExpectedMetrics(t, metrics, []string{
+		"apiserver_request_count",
+		"apiserver_request_latencies",
+	})
+}

test/integration/utils.go

@@ -21,6 +21,15 @@ package integration
 import (
 	"fmt"
 	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
+	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
 
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/golang/glog"
@@ -55,3 +64,27 @@ func deleteAllEtcdKeys() {
 	}
 
 }
+
+func runAMaster(t *testing.T) (*master.Master, *httptest.Server) {
+	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	m := master.New(&master.Config{
+		EtcdHelper:        helper,
+		KubeletClient:     client.FakeKubeletClient{},
+		EnableLogsSupport: false,
+		EnableProfiling:   true,
+		EnableUISupport:   false,
+		APIPrefix:         "/api",
+		Authorizer:        apiserver.NewAlwaysAllowAuthorizer(),
+		AdmissionControl:  admit.NewAlwaysAdmit(),
+	})
+
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		m.Handler.ServeHTTP(w, req)
+	}))
+
+	return m, s
+}