Merge pull request #30577 from chrislovecnm/cassandra-petset

examples/examples_test.go

@@ -29,6 +29,8 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/api/validation"
+	"k8s.io/kubernetes/pkg/apis/apps"
+	appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation"
 	"k8s.io/kubernetes/pkg/apis/batch"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	expvalidation "k8s.io/kubernetes/pkg/apis/extensions/validation"
@@ -128,6 +130,11 @@ func validateObject(obj runtime.Object) (errors field.ErrorList) {
 			t.Namespace = api.NamespaceDefault
 		}
 		errors = expvalidation.ValidateDaemonSet(t)
+	case *apps.PetSet:
+		if t.Namespace == "" {
+			t.Namespace = api.NamespaceDefault
+		}
+		errors = appsvalidation.ValidatePetSet(t)
 	default:
 		errors = field.ErrorList{}
 		errors = append(errors, field.InternalError(field.NewPath(""), fmt.Errorf("no validation defined for %#v", obj)))
@@ -214,6 +221,7 @@ func TestExampleObjectSchemas(t *testing.T) {
 			"cassandra-daemonset":  &extensions.DaemonSet{},
 			"cassandra-controller": &api.ReplicationController{},
 			"cassandra-service":    &api.Service{},
+			"cassandra-petset":     &apps.PetSet{},
 		},
 		"../examples/cluster-dns": {
 			"dns-backend-rc":      &api.ReplicationController{},

examples/storage/cassandra/cassandra-controller.yaml

@@ -28,6 +28,8 @@ spec:
               value: 512M
             - name: HEAP_NEWSIZE
               value: 100M
+            - name: CASSANDRA_SEED_PROVIDER
+              value: "io.k8s.cassandra.KubernetesSeedProvider"
             - name: POD_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -36,7 +38,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
-          image: gcr.io/google-samples/cassandra:v9
+          image: gcr.io/google-samples/cassandra:v11
           name: cassandra
           ports:
             - containerPort: 7000
@@ -47,9 +49,6 @@ spec:
               name: jmx
             - containerPort: 9042
               name: cql
-            # If you need it it is going away in C* 4.0
-            #- containerPort: 9160
-            #  name: thrift
           volumeMounts:
             - mountPath: /cassandra_data
               name: data

examples/storage/cassandra/cassandra-daemonset.yaml

@@ -21,6 +21,8 @@ spec:
               value: 512M
             - name: HEAP_NEWSIZE
               value: 100M
+            - name: CASSANDRA_SEED_PROVIDER
+              value: "io.k8s.cassandra.KubernetesSeedProvider"
             - name: POD_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -29,7 +31,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
-          image: gcr.io/google-samples/cassandra:v9
+          image: gcr.io/google-samples/cassandra:v11
           name: cassandra
           ports:
             - containerPort: 7000

examples/storage/cassandra/cassandra-petset.yaml

+apiVersion: "apps/v1alpha1"
+kind: PetSet
+metadata:
+  name: cassandra
+spec:
+  serviceName: cassandra
+  replicas: 3
+  template:
+    metadata:
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+      labels:
+        app: cassandra
+    spec:
+      containers:
+      - name: cassandra
+        image: gcr.io/google-samples/cassandra:v11
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 7000
+          name: intra-node
+        - containerPort: 7001
+          name: tls-intra-node
+        - containerPort: 7199
+          name: jmx
+        - containerPort: 9042
+          name: cql
+        resources:
+          limits:
+            cpu: "1"
+            memory: 1Gi
+          requests:
+           cpu: "1"
+           memory: 1Gi
+        securityContext:
+          capabilities:
+            add:
+              - IPC_LOCK
+        env:
+          - name: MAX_HEAP_SIZE
+            value: 512M
+          - name: HEAP_NEWSIZE
+            value: 100M
+          - name: CASSANDRA_SEEDS
+            value: "cassandra-0.cassandra.default.svc.cluster.local"
+          - name: CASSANDRA_CLUSTER_NAME
+            value: "K8Demo"
+          - name: CASSANDRA_DC
+            value: "DC1-K8Demo"
+          - name: CASSANDRA_RACK
+            value: "Rack1-K8Demo"
+          - name: CASSANDRA_AUTO_BOOTSTRAP
+            value: "false"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - /ready-probe.sh
+          initialDelaySeconds: 15
+          timeoutSeconds: 5
+        # These volume mounts are persistent. They are like inline claims,
+        # but not exactly because the names need to match exactly one of
+        # the pet volumes.
+        volumeMounts:
+        - name: cassandra-data
+          mountPath: /cassandra_data
+  # These are converted to volume claims by the controller
+  # and mounted at the paths mentioned above.
+  # do not use these in production until ssd GCEPersistentDisk or other ssd pd
+  volumeClaimTemplates:
+  - metadata:
+      name: cassandra-data
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 1Gi

examples/storage/cassandra/cassandra-service.yaml

@@ -5,6 +5,7 @@ metadata:
     app: cassandra
   name: cassandra
 spec:
+  clusterIP: None
   ports:
     - port: 9042
   selector:

examples/storage/cassandra/image/Dockerfile

@@ -14,28 +14,36 @@
 
 FROM google/debian:jessie
 
-COPY cassandra.list /etc/apt/sources.list.d/cassandra.list
-COPY run.sh /run.sh
-COPY cassandra.yaml /cassandra.yaml
-COPY logback.xml /logback.xml
-COPY kubernetes-cassandra.jar /kubernetes-cassandra.jar
+ADD files /
 
-RUN gpg --keyserver pgp.mit.edu --recv-keys F758CE318D77295D && \
-  gpg --export --armor F758CE318D77295D | apt-key add - && \
-  gpg --keyserver pgp.mit.edu --recv-keys 2B5C1B00 && \
-  gpg --export --armor 2B5C1B00 | apt-key add - && \
-  gpg --keyserver pgp.mit.edu --recv-keys 0353B12C && \
-  gpg --export --armor 0353B12C | apt-key add - && \
-  apt-get update && \
-  apt-get -qq -y install procps cassandra openjdk-8-jre-headless && \
-  chmod a+rx /run.sh && \
-  mkdir -p /cassandra_data/data && \
-  mv /logback.xml /etc/cassandra/ && \
-  mv /cassandra.yaml /etc/cassandra/ && \
-  chown -R cassandra: /etc/cassandra /cassandra_data /run.sh \
-    /kubernetes-cassandra.jar && \
-  chmod o+w -R /etc/cassandra /cassandra_data && \
-  rm -rf \
+ENV DI_VERSION="1.1.1" DI_SHA="dec8167091671df0dd3748a8938102479db5fffc"
+
+RUN mv /java.list /etc/apt/sources.list.d/java.list \
+  && apt-get update \
+  && apt-get -qq -y install --no-install-recommends procps openjdk-8-jre-headless libjemalloc1 curl \
+    localepurge \
+  && curl -L https://github.com/Yelp/dumb-init/releases/download/v${DI_VERSION}/dumb-init_${DI_VERSION}_amd64 > /sbin/dumb-init \
+  && echo "$DI_SHA  /sbin/dumb-init" | sha1sum -c - \
+  && mv /cassandra.list /etc/apt/sources.list.d/cassandra.list \
+  && gpg --keyserver pgp.mit.edu --recv-keys F758CE318D77295D \
+  && gpg --export --armor F758CE318D77295D | apt-key add - \
+  && gpg --keyserver pgp.mit.edu --recv-keys 2B5C1B00 \
+  && gpg --export --armor 2B5C1B00 | apt-key add - \
+  && gpg --keyserver pgp.mit.edu --recv-keys 0353B12C \
+  && gpg --export --armor 0353B12C | apt-key add - \
+  && apt-get update \
+  && apt-get -qq -y install --no-install-recommends curl cassandra localepurge \
+  && chmod a+rx /run.sh /sbin/dumb-init /ready-probe.sh \
+  && mkdir -p /cassandra_data/data \
+  && mv /logback.xml /cassandra.yaml /etc/cassandra/ \
+
+  # Not able to run as cassandra until https://github.com/kubernetes/kubernetes/issues/2630 is resolved
+  # && chown -R cassandra: /etc/cassandra /cassandra_data /run.sh /kubernetes-cassandra.jar \
+  # && chmod o+w -R /etc/cassandra /cassandra_data \
+
+  && apt-get -y purge curl localepurge \
+  && apt-get clean \
+  && rm -rf \
         doc \
         man \
         info \
@@ -54,7 +62,7 @@ RUN gpg --keyserver pgp.mit.edu --recv-keys F758CE318D77295D && \
         /tmp/*
 
 
-VOLUME ["/cassandra_data/data"]
+VOLUME ["/cassandra_data"]
 
 # 7000: intra-node communication
 # 7001: TLS intra-node communication
@@ -63,6 +71,8 @@ VOLUME ["/cassandra_data/data"]
 # 9160: thrift service not included cause it is going away
 EXPOSE 7000 7001 7199 9042
 
-USER cassandra
+# Not able to do this until https://github.com/kubernetes/kubernetes/issues/2630 is resolved
+# if you are using attached storage
+# USER cassandra
 
-CMD /run.sh
+CMD ["/sbin/dumb-init", "/bin/bash", "/run.sh"]

examples/storage/cassandra/image/Makefile

@@ -14,19 +14,21 @@
 
 # build the cassandra image.
 
-VERSION=v9
+VERSION=v11
+PROJECT_ID=google_samples
+PROJECT=gcr.io/${PROJECT_ID}
 
 all: build
 
 kubernetes-cassandra.jar: ../java/* ../java/src/main/java/io/k8s/cassandra/*.java
-	cd ../java && mvn package
-	mv ../java/target/kubernetes-cassandra*.jar kubernetes-cassandra.jar
+	cd ../java && mvn clean && mvn package
+	mv ../java/target/kubernetes-cassandra*.jar files/kubernetes-cassandra.jar
 	cd ../java && mvn clean
 
 build: kubernetes-cassandra.jar
-	docker build -t gcr.io/google_samples/cassandra:${VERSION} .
+	docker build -t ${PROJECT}/cassandra:${VERSION} .
 
 push: build
-	gcloud docker push gcr.io/google_samples/cassandra:${VERSION}
+	gcloud docker push ${PROJECT}/cassandra:${VERSION}
 
 .PHONY: all build push

examples/storage/cassandra/image/files/cassandra.list

+deb http://www.apache.org/dist/cassandra/debian 37x main
+deb-src http://www.apache.org/dist/cassandra/debian 37x main

examples/storage/cassandra/image/cassandra.list → examples/storage/cassandra/image/files/java.list

-deb http://www.apache.org/dist/cassandra/debian 35x main
-deb-src http://www.apache.org/dist/cassandra/debian 35x main
-
 # for jre8
 deb http://http.debian.net/debian jessie-backports main

examples/storage/cassandra/image/files/ready-probe.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ $(nodetool status | grep $POD_IP) == *"UN"* ]]; then
+  if [[ $DEBUG ]]; then
+    echo "Not Up";
+  fi
+  exit 0;
+else
+  if [[ $DEBUG ]]; then
+    echo "UN";
+  fi
+  exit 1;
+fi

examples/storage/cassandra/image/files/run.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+CONF_DIR=/etc/cassandra
+CFG=$CONF_DIR/cassandra.yaml
+
+# we are doing PetSet or just setting our seeds
+if [ -z "$CASSANDRA_SEEDS" ]; then
+  HOSTNAME=$(hostname -f)
+fi
+
+# The following vars relate to there counter parts in $CFG
+# for instance rpc_address
+CASSANDRA_RPC_ADDRESS="${CASSANDRA_RPC_ADDRESS:-0.0.0.0}"
+CASSANDRA_NUM_TOKENS="${CASSANDRA_NUM_TOKENS:-32}"
+CASSANDRA_CLUSTER_NAME="${CASSANDRA_CLUSTER_NAME:='Test Cluster'}"
+CASSANDRA_LISTEN_ADDRESS=${POD_IP:-$HOSTNAME}
+CASSANDRA_BROADCAST_ADDRESS=${POD_IP:-$HOSTNAME}
+CASSANDRA_BROADCAST_RPC_ADDRESS=${POD_IP:-$HOSTNAME}
+CASSANDRA_DISK_OPTIMIZATION_STRATEGY="${CASSANDRA_DISK_OPTIMIZATION_STRATEGY:-ssd}"
+CASSANDRA_MIGRATION_WAIT="${CASSANDRA_MIGRATION_WAIT:-1}"
+CASSANDRA_ENDPOINT_SNITCH="${CASSANDRA_ENDPOINT_SNITCH:-SimpleSnitch}"
+CASSANDRA_DC="${CASSANDRA_DC}"
+CASSANDRA_RACK="${CASSANDRA_RACK}"
+CASSANDRA_RING_DELAY="${CASSANDRA_RING_DELAY:-30000}"
+CASSANDRA_AUTO_BOOTSTRAP="${CASSANDRA_AUTO_BOOTSTRAP:-true}"
+CASSANDRA_SEEDS="${CASSANDRA_SEEDS:false}"
+CASSANDRA_SEED_PROVIDER="${CASSANDRA_SEED_PROVIDER:-org.apache.cassandra.locator.SimpleSeedProvider}"
+CASSANDRA_AUTO_BOOTSTRAP="${CASSANDRA_AUTO_BOOTSTRAP:false}"
+
+# Turn off JMX auth
+CASSANDRA_OPEN_JMX="${CASSANDRA_OPEN_JMX:-false}"
+# send GC to STDOUT
+CASSANDRA_GC_STDOUT="${CASSANDRA_GC_STDOUT:-false}"
+
+# if DC and RACK are set, use GossipingPropertyFileSnitch
+if [[ $CASSANDRA_DC && $CASSANDRA_RACK ]]; then
+  echo "dc=$CASSANDRA_DC" > $CONF_DIR/cassandra-rackdc.properties
+  echo "rack=$CASSANDRA_RACK" >> $CONF_DIR/cassandra-rackdc.properties
+  CASSANDRA_ENDPOINT_SNITCH="GossipingPropertyFileSnitch"
+fi
+
+# TODO what else needs to be modified
+for yaml in \
+  broadcast_address \
+  broadcast_rpc_address \
+  cluster_name \
+  listen_address \
+  num_tokens \
+  rpc_address \
+  disk_optimization_strategy \
+  endpoint_snitch \
+; do
+  var="CASSANDRA_${yaml^^}"
+  val="${!var}"
+  if [ "$val" ]; then
+    sed -ri 's/^(# )?('"$yaml"':).*/\2 '"$val"'/' "$CFG"
+  fi
+done
+
+echo "auto_bootstrap: ${CASSANDRA_AUTO_BOOTSTRAP}" >> $CFG
+
+# set the seed to itself.  This is only for the first pod, otherwise
+# it will be able to get seeds from the seed provider
+if [[ $CASSANDRA_SEEDS == 'false' ]]; then
+  sed -ri 's/- seeds:.*/- seeds: "'"$POD_IP"'"/' $CFG
+else # if we have seeds set them.  Probably PetSet
+  sed -ri 's/- seeds:.*/- seeds: "'"$CASSANDRA_SEEDS"'"/' $CFG
+fi
+
+sed -ri 's/- class_name: SEED_PROVIDER/- class_name: '"$CASSANDRA_SEED_PROVIDER"'/' $CFG
+
+# send gc to stdout
+if [[ $CASSANDRA_GC_STDOUT == 'true' ]]; then
+  sed -ri 's/ -Xloggc:\/var\/log\/cassandra\/gc\.log//' $CONF_DIR/cassandra-env.sh
+fi
+
+# enable RMI and JMX to work on one port
+echo "JVM_OPTS=\"\$JVM_OPTS -Djava.rmi.server.hostname=$POD_IP\"" >> $CONF_DIR/cassandra-env.sh
+
+# getting WARNING messages with Migration Service
+echo "-Dcassandra.migration_task_wait_in_seconds=${CASSANDRA_MIGRATION_WAIT}" >> $CONF_DIR/jvm.options
+echo "-Dcassandra.ring_delay_ms=${CASSANDRA_RING_DELAY}" >> $CONF_DIR/jvm.options
+
+
+if [[ $CASSANDRA_OPEN_JMX == 'true' ]]; then
+  export LOCAL_JMX=no
+  sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.authenticate=true/ -Dcom\.sun\.management\.jmxremote\.authenticate=false/' $CONF_DIR/cassandra-env.sh
+  sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.password\.file=\/etc\/cassandra\/jmxremote\.password//' $CONF_DIR/cassandra-env.sh
+fi
+
+echo Starting Cassandra on ${CASSANDRA_LISTEN_ADDRESS}
+echo CASSANDRA_RPC_ADDRESS ${CASSANDRA_RPC_ADDRESS}
+echo CASSANDRA_NUM_TOKENS ${CASSANDRA_NUM_TOKENS}
+echo CASSANDRA_CLUSTER_NAME ${CASSANDRA_CLUSTER_NAME}
+echo CASSANDRA_LISTEN_ADDRESS ${CASSANDRA_LISTEN_ADDRESS}
+echo CASSANDRA_BROADCAST_ADDRESS ${CASSANDRA_BROADCAST_ADDRESS}
+echo CASSANDRA_BROADCAST_RPC_ADDRESS ${CASSANDRA_BROADCAST_RPC_ADDRESS}
+echo CASSANDRA_DISK_OPTIMIZATION_STRATEGY ${CASSANDRA_DISK_OPTIMIZATION_STRATEGY}
+echo CASSANDRA_MIGRATION_WAIT ${CASSANDRA_MIGRATION_WAIT}
+echo CASSANDRA_ENDPOINT_SNITCH ${CASSANDRA_ENDPOINT_SNITCH}
+echo CASSANDRA_DC ${CASSANDRA_DC}
+echo CASSANDRA_RACK ${CASSANDRA_RACK}
+echo CASSANDRA_RING_DELAY ${CASSANDRA_RING_DELAY}
+echo CASSANDRA_AUTO_BOOTSTRAP ${CASSANDRA_AUTO_BOOTSTRAP}
+echo CASSANDRA_SEEDS ${CASSANDRA_SEEDS}
+echo CASSANDRA_SEED_PROVIDER ${CASSANDRA_SEED_PROVIDER}
+echo CASSANDRA_AUTO_BOOTSTRAP ${CASSANDRA_AUTO_BOOTSTRAP}
+
+export CLASSPATH=/kubernetes-cassandra.jar
+cassandra -R -f

examples/storage/cassandra/image/run.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-CFG=/etc/cassandra/cassandra.yaml
-CASSANDRA_RPC_ADDRESS="${CASSANDRA_RPC_ADDRESS:-0.0.0.0}"
-CASSANDRA_NUM_TOKENS="${CASSANDRA_NUM_TOKENS:-32}"
-CASSANDRA_CLUSTER_NAME="${CASSANDRA_CLUSTER_NAME:='Test Cluster'}"
-CASSANDRA_LISTEN_ADDRESS=${POD_IP}
-CASSANDRA_BROADCAST_ADDRESS=${POD_IP}
-CASSANDRA_BROADCAST_RPC_ADDRESS=${POD_IP}
-
-# TODO what else needs to be modified
-
-for yaml in \
-  broadcast_address \
-	broadcast_rpc_address \
-	cluster_name \
-	listen_address \
-	num_tokens \
-	rpc_address \
-; do
-  var="CASSANDRA_${yaml^^}"
-	val="${!var}"
-	if [ "$val" ]; then
-		sed -ri 's/^(# )?('"$yaml"':).*/\2 '"$val"'/' "$CFG"
-	fi
-done
-
-# Eventual do snitch $DC && $RACK?
-#if [[ $SNITCH ]]; then
-#  sed -i -e "s/endpoint_snitch: SimpleSnitch/endpoint_snitch: $SNITCH/" $CONFIG/cassandra.yaml
-#fi
-#if [[ $DC && $RACK ]]; then
-#  echo "dc=$DC" > $CONFIG/cassandra-rackdc.properties
-#  echo "rack=$RACK" >> $CONFIG/cassandra-rackdc.properties
-#fi
-
-#
-# see if this is needed
-#echo "JVM_OPTS=\"\$JVM_OPTS -Djava.rmi.server.hostname=$IP\"" >> $CASSANDRA_CONFIG/cassandra-env.sh
-#
-
-# FIXME create README for these args
-echo "Starting Cassandra on $POD_IP"
-echo CASSANDRA_RPC_ADDRESS ${CASSANDRA_RPC_ADDRESS}
-echo CASSANDRA_NUM_TOKENS ${CASSANDRA_NUM_TOKENS}
-echo CASSANDRA_CLUSTER_NAME ${CASSANDRA_CLUSTER_NAME}
-echo CASSANDRA_LISTEN_ADDRESS ${POD_IP}
-echo CASSANDRA_BROADCAST_ADDRESS ${POD_IP}
-echo CASSANDRA_BROADCAST_RPC_ADDRESS ${POD_IP}
-
-export CLASSPATH=/kubernetes-cassandra.jar
-cassandra -f

examples/storage/cassandra/java/pom.xml

@@ -17,7 +17,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>io.k8s.cassandra</groupId>
     <artifactId>kubernetes-cassandra</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.2</version>
     <build>
         <plugins>
             <plugin>
@@ -33,6 +33,7 @@
 
     <properties>
         <logback.version>1.1.3</logback.version>
+        <cassandra.version>3.7</cassandra.version>
     </properties>
 
     <dependencies>
@@ -85,7 +86,7 @@
         <dependency>
             <groupId>org.apache.cassandra</groupId>
             <artifactId>cassandra-all</artifactId>
-            <version>3.5</version>
+            <version>${cassandra.version}</version>
             <scope>provided</scope>
         </dependency>
 

hack/verify-flags/exceptions.txt

@@ -44,6 +44,7 @@ cluster/saltbase/salt/etcd/etcd.manifest:        "value": "{{ storage_backend }}
 cluster/saltbase/salt/etcd/etcd.manifest:{% set storage_backend = pillar.get('storage_backend', 'etcd2') -%}
 cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %}
 cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
+cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
 cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider  + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%}
 cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = params + " " + feature_gates -%}
 cluster/saltbase/salt/kube-proxy/kube-proxy.manifest:  {% set api_servers_with_port = api_servers + ":6443" -%}
@@ -69,7 +70,7 @@ cluster/vagrant/provision-utils.sh:  node_ip: '$(echo "$MASTER_IP" | sed -e "s/'
 cluster/vagrant/provision-utils.sh:  runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")'
 cluster/vsphere/templates/salt-minion.sh:  hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}')
 examples/cluster-dns/images/frontend/client.py:  service_address = socket.gethostbyname(hostname)
-examples/storage/cassandra/image/run.sh:	cluster_name \
+examples/storage/cassandra/image/files/run.sh:  cluster_name \
 examples/storage/vitess/env.sh:    node_ip=$(get_node_ip)
 federation/cluster/common.sh:  local cert_dir="${kube_temp}/easy-rsa-master/easyrsa3"
 federation/deploy/config.json.sample:      "cloud_provider": "gce",

test/e2e/examples.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -28,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -225,19 +227,92 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 			Expect(err).NotTo(HaveOccurred())
 			forEachPod("app", "cassandra", func(pod api.Pod) {
 				framework.Logf("Verifying pod %v ", pod.Name)
-				_, err = framework.LookForStringInLog(ns, pod.Name, "cassandra", "Listening for thrift clients", serverStartTimeout)
-				Expect(err).NotTo(HaveOccurred())
-				_, err = framework.LookForStringInLog(ns, pod.Name, "cassandra", "Handshaking version", serverStartTimeout)
+				// TODO how do we do this better?  Ready Probe?
+				_, err = framework.LookForStringInLog(ns, pod.Name, "cassandra", "Starting listening for CQL clients", serverStartTimeout)
 				Expect(err).NotTo(HaveOccurred())
 			})
 
 			By("Finding each node in the nodetool status lines")
 			forEachPod("app", "cassandra", func(pod api.Pod) {
 				output := framework.RunKubectlOrDie("exec", pod.Name, nsFlag, "--", "nodetool", "status")
-				if !strings.Contains(output, pod.Status.PodIP) {
-					framework.Failf("Pod ip %s not found in nodetool status", pod.Status.PodIP)
+				matched, _ := regexp.MatchString("UN.*"+pod.Status.PodIP, output)
+				if matched != true {
+					framework.Failf("Cassandra pod ip %s is not reporting Up and Normal 'UN' via nodetool status", pod.Status.PodIP)
+				}
+			})
+		})
+	})
+
+	framework.KubeDescribe("CassandraPetSet", func() {
+		It("should create petset", func() {
+			mkpath := func(file string) string {
+				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/cassandra", file)
+			}
+			serviceYaml := mkpath("cassandra-service.yaml")
+			nsFlag := fmt.Sprintf("--namespace=%v", ns)
+
+			// have to change dns prefix because of the dynamic namespace
+			input, err := ioutil.ReadFile(mkpath("cassandra-petset.yaml"))
+			Expect(err).NotTo(HaveOccurred())
+
+			output := strings.Replace(string(input), "cassandra-0.cassandra.default.svc.cluster.local", "cassandra-0.cassandra."+ns+".svc.cluster.local", -1)
+
+			petSetYaml := "/tmp/cassandra-petset.yaml"
+
+			err = ioutil.WriteFile(petSetYaml, []byte(output), 0644)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Starting the cassandra service")
+			framework.RunKubectlOrDie("create", "-f", serviceYaml, nsFlag)
+			framework.Logf("wait for service")
+			err = framework.WaitForService(c, ns, "cassandra", true, framework.Poll, framework.ServiceRespondingTimeout)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Create an PetSet with n nodes in it.  Each node will then be verified.
+			By("Creating a Cassandra PetSet")
+
+			framework.RunKubectlOrDie("create", "-f", petSetYaml, nsFlag)
+
+			petsetPoll := 30 * time.Second
+			petsetTimeout := 10 * time.Minute
+			// TODO - parse this number out of the yaml
+			numPets := 3
+			label := labels.SelectorFromSet(labels.Set(map[string]string{"app": "cassandra"}))
+			err = wait.PollImmediate(petsetPoll, petsetTimeout,
+				func() (bool, error) {
+					podList, err := c.Pods(ns).List(api.ListOptions{LabelSelector: label})
+					if err != nil {
+						return false, fmt.Errorf("Unable to get list of pods in petset %s", label)
+					}
+					ExpectNoError(err)
+					if len(podList.Items) < numPets {
+						framework.Logf("Found %d pets, waiting for %d", len(podList.Items), numPets)
+						return false, nil
+					}
+					if len(podList.Items) > numPets {
+						return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numPets, len(podList.Items))
+					}
+					for _, p := range podList.Items {
+						isReady := api.IsPodReady(&p)
+						if p.Status.Phase != api.PodRunning || !isReady {
+							framework.Logf("Waiting for pod %v to enter %v - Ready=True, currently %v - Ready=%v", p.Name, api.PodRunning, p.Status.Phase, isReady)
+							return false, nil
+						}
+					}
+					return true, nil
+				})
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Finding each node in the nodetool status lines")
+			forEachPod("app", "cassandra", func(pod api.Pod) {
+				output := framework.RunKubectlOrDie("exec", pod.Name, nsFlag, "--", "nodetool", "status")
+				matched, _ := regexp.MatchString("UN.*"+pod.Status.PodIP, output)
+				if matched != true {
+					framework.Failf("Cassandra pod ip %s is not reporting Up and Normal 'UN' via nodetool status", pod.Status.PodIP)
 				}
 			})
+			// using out of petset e2e as deleting pvc is a pain
+			deleteAllPetSets(c, ns)
 		})
 	})