Merge pull request #7639 from jayunit100/e2e-k8bps

examples/k8petstore/k8petstore.sh

@@ -16,16 +16,30 @@
 
 echo "WRITING KUBE FILES , will overwrite the jsons, then testing pods. is kube clean ready to go?"
 
+
+#Args below can be overriden when calling from cmd line.
+#Just send all the args in order.
 #for dev/test you can use:
 #kubectl=$GOPATH/src/github.com/GoogleCloudPlatform/kubernetes/cluster/kubectl.sh"
 kubectl="kubectl"
 VERSION="r.2.8.19"
 PUBLIC_IP="10.1.4.89" # ip which we use to access the Web server.
-SECONDS=1000          # number of seconds to measure throughput.
 FE="1"                # amount of Web server  
 LG="1"                # amount of load generators
 SLAVE="1"             # amount of redis slaves 
+TEST_SECONDS="1000"   # 0 = Dont run tests, if > 0,  run tests for n seconds.
+NS="k8petstore"       # namespace
+
+kubectl="${1:-$kubectl}"
+VERSION="${2:-$VERSION}"
+PUBLIC_IP="${3:-$PUBLIC_IP}"
+FE="${4:-$FE}"  
+LG="${5:-$LG}"
+SLAVE="${6:-$SLAVE}" 
+TEST_SECONDS="${7:-$TEST}"
+NS="${8:-$NS}" 
 
+echo "Running w/ args: kubectl $kubectl version $VERSION ip $PUBLIC_IP sec $_SECONDS fe $FE lg $LG slave $SLAVE test $TEST NAMESPACE $NS"
 function create { 
 
 cat << EOF > fe-rc.json
@@ -188,53 +202,74 @@ cat << EOF > slave-rc.json
   "labels": {"name": "redisslave"}
 }
 EOF
-$kubectl create -f rm.json --api-version=v1beta1
-$kubectl create -f rm-s.json --api-version=v1beta1
+$kubectl create -f rm.json --api-version=v1beta1 --namespace=$NS
+$kubectl create -f rm-s.json --api-version=v1beta1 --namespace=$NS
 sleep 3 # precaution to prevent fe from spinning up too soon.
-$kubectl create -f slave-rc.json --api-version=v1beta1
-$kubectl create -f rs-s.json --api-version=v1beta1
+$kubectl create -f slave-rc.json --api-version=v1beta1 --namespace=$NS
+$kubectl create -f rs-s.json --api-version=v1beta1 --namespace=$NS
 sleep 3 # see above comment.
-$kubectl create -f fe-rc.json --api-version=v1beta1 
-$kubectl create -f fe-s.json --api-version=v1beta1
-$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1
+$kubectl create -f fe-rc.json --api-version=v1beta1 --namespace=$NS
+$kubectl create -f fe-s.json --api-version=v1beta1 --namespace=$NS
+$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1 --namespace=$NS
 }
 
-function test { 
+function pollfor {
 	pass_http=0
 
 	### Test HTTP Server comes up.
 	for i in `seq 1 150`;
 	do
 	    ### Just testing that the front end comes up.  Not sure how to test total entries etc... (yet)
-	    echo "Trying curl ... $i . expect a few failures while pulling images... " 
+	    echo "Trying curl ... $PUBLIC_IP:3000 , attempt $i . expect a few failures while pulling images... " 
 	    curl "$PUBLIC_IP:3000" > result
 	    cat result
 	    cat result | grep -q "k8-bps"
 	    if [ $? -eq 0 ]; then
-		echo "TEST PASSED after $i tries !"
-		i=1000
-		break
+    		echo "TEST PASSED after $i tries !"
+	    	i=1000
+		    break
 	    else	
 		echo "the above RESULT didn't contain target string for trial $i"
 	    fi
-	    sleep 5
+	    sleep 3
 	done
 
 	if [ $i -eq 1000 ]; then
-	   pass_http=-1
+	   pass_http=1
 	fi
+    
+}
 
+function tests {
 	pass_load=0 
 
 	### Print statistics of db size, every second, until $SECONDS are up.
-	for i in `seq 1 $SECONDS`;
-	do
-	    echo "curl : $i"
-            curl "$PUBLIC_IP:3000/llen" >> result
+	for i in `seq 1 $TEST_SECONDS`;
+	 do
+	    echo "curl : $PUBLIC_IP:3000 , $i of $TEST_SECONDS"
+        curr_cnt="`curl "$PUBLIC_IP:3000/llen"`" 
+        ### Write CSV File of # of trials / total transcations.
+        echo "$i $curr_cnt" >> result
+        echo "total transactions so far : $curr_cnt"
 	    sleep 1
 	done
 }
 
 create
 
-test
+pollfor
+
+if [[ $pass_http -eq 1 ]]; then 
+    echo "Passed..."
+else
+    exit 2
+fi
+
+if [[ $TEST_SECONDS -eq 0 ]]; then
+    echo "skipping tests, TEST_SECONDS value was 0"
+else
+    echo "running polling tests now for $TEST_SECONDS"
+    tests
+fi
+
+exit 0

test/e2e/soak_k8petstore.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	root0                    = absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), "..")))
+	err       error          = nil
+	namespace *api.Namespace = nil
+	ns        string         = ""
+)
+
+//This must run on a machine which is in the kube-proxy ring.  Otherwise, the publicIP binding will fail.
+//The IP Below ~ letter->number cipher for k8petstore (most likely it won't be bound by any other process)
+var ip = "165.201.92.15"
+
+// i.e. after 50 trials, expect 3000 transactions... minimum we settle for is 500.
+var minionCount int
+
+// readTransactions reads # of transactions from the k8petstore web server endpoint.
+// for more details see the source of the k8petstore web server.
+func readTransactions(c *client.Client) int {
+	resp, err := http.Get("http://165.201.92.15:3000/llen")
+	if err != nil {
+		Expect(err).NotTo(HaveOccurred())
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	Expect(err).NotTo(HaveOccurred())
+	totalTrans, err := strconv.Atoi(string(body))
+	Expect(err).NotTo(HaveOccurred())
+	return totalTrans
+}
+
+// runK8petstore runs the k8petstore application, bound to external ip "ip", and polls it to assert that "min_expected"
+// transactions are acquired in a maximum of "max_seconds".
+func runK8petstore(ip string, restServers int, loadGenerators int, c *client.Client, minExpected int, maxSeconds int64) {
+	k8bps := filepath.Join(root0, "examples/k8petstore/k8petstore.sh")
+
+	//Get the count of minions.  We'll use this to decide expected throughput and loadgen/REST server count.
+	cmd := exec.Command(
+		k8bps,
+		"kubectl", //for dev, replace w/ "cluster/kubectl.sh"
+		"r.2.8.19",
+		ip,
+		strconv.Itoa(restServers),
+		strconv.Itoa(loadGenerators),
+		"1", "0", ns) // 1= # slave, 0 = don't bother running the embedded test.
+
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	Logf("Starting k8petstore application....")
+	//Run the k8petstore app, and log / fail if it returns any errors.
+	//This should return quickly, assuming containers are downloaded.
+	if err = cmd.Start(); err != nil {
+		log.Fatal(err)
+	}
+	//Make sure exit code != 0
+	if err = cmd.Wait(); err != nil {
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				log.Printf("Exit Status: %d", status.ExitStatus())
+			}
+		}
+	}
+	Expect(err).NotTo(HaveOccurred())
+	Logf("... Done starting k8petstore successfully.... Now we will poll it...")
+
+	//By now, the original k8petstore app has run and spun up a k8petstore w/ load generators.
+	//Lets poll until we reach min_expected transactions
+	totalTransactions := 0
+	Logf("Start polling, timeout is %v seconds", maxSeconds)
+
+	timeout := time.After(time.Duration(maxSeconds) * time.Second)
+	tick := time.Tick(2 * time.Second)
+
+T:
+	for {
+		select {
+		case <-timeout:
+			Logf("Timeout %v reached. Breaking!", tick)
+			break T
+		case <-tick:
+			totalTransactions = readTransactions(c)
+			Expect(err).NotTo(HaveOccurred())
+			if totalTransactions > minExpected {
+				break T
+			}
+			Logf("%v == %v total petstore transactions stored into redis. ==", time.Now(), totalTransactions)
+		}
+	}
+
+	//Finally! We should have exceeded the min_expected num of transactions.
+	//If this fails, but there are transactions being created, we may need to recalibrate
+	//the min_expected value - or else - your cluster is broken/slow !
+	Ω(totalTransactions).Should(BeNumerically(">", minExpected))
+}
+
+var _ = Describe("k8bps", func() {
+	BeforeEach(func() {
+		By("Creating a kubernetes client")
+		c, err = loadClient()
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Building a namespace api object")
+		namespace, err = createTestingNS("k8petstore-soak", c)
+		ns = namespace.Name
+		Expect(err).NotTo(HaveOccurred())
+
+		//Now get the # of minions and calibrate the test params to them...
+		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
+		Expect(err).NotTo(HaveOccurred())
+		minionCount = len(minions.Items)
+
+		//simple calibration.  TODO, make more ambitious (i.e. 10x density style load generators)...
+		//load_generators = minionCount
+		//rest_servers = minionCount
+
+	})
+
+	AfterEach(func() {
+		By(fmt.Sprintf("Destroying namespace for this test %v", namespace.Name))
+		if err := c.Namespaces().Delete(namespace.Name); err != nil {
+			Failf("Couldn't delete ns %s", err)
+		}
+	})
+
+	//On a single node cluster, we expect about 10 transactions every second on average.
+	//admittedly, this is a very rough estimate given that the ETL is done in batches...
+	It(fmt.Sprintf("k8petstore-FUNCTIONAL : Should quickly acquire 500 petstore transactions."), func() {
+
+		//max number of trial before k8petstore.sh dies.
+		var loadGenerators int = minionCount
+		var restServers int = minionCount
+
+		fmt.Printf("load generators / rest generators [ %v  /  %v ] ", loadGenerators, restServers)
+
+		//At least 500 transactions should be acquired within 30 seconds after startup.
+		runK8petstore(ip, restServers, loadGenerators, c, 500, 30)
+
+	})
+
+	//This test takes a few minutes... for simple CI setups testing pure functionality, filter it out.
+	It(fmt.Sprintf("k8petstore-SCALE : Should support acquiring up to 5000 petstore transactions per minion"), func() {
+
+		//We double the load generators, to increase the load.  Maybe parameterize this later,
+		//the more generators -> the more transactions and the more bursty CPU used per minion.
+		var loadGenerators int = minionCount * 2
+		var restServers int = minionCount
+		//var min_expected int = trials * 10 * minionCount // more minions --> higher expected # of transactions.
+
+		fmt.Printf("load generators / rest generators [ %v  /  %v ] ", loadGenerators, restServers)
+
+		//5000*M transactions in 6 minutes.  That gives 6 minutes of cold-start time.
+		runK8petstore(ip, restServers, loadGenerators, c, 5000*minionCount, 60*6)
+	})
+})