Merge pull request #9816 from erictune/conformance

hack/conformance-test.sh

@@ -14,26 +14,97 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# The conformance test is provided to let users run an e2e test
-# against an already-setup cluster for which there is no automated
-# setup, teardown, and other cluster/... scripts.
+# The conformance test checks whether a kubernetes cluster supports
+# a minimum set of features to be called "Kubernetes".  It is similar
+# to `hack/e2e-test.sh` but it differs in that:
+#  - hack/e2e-test.sh is intended to test a cluster with binaries built at HEAD,
+#    while this conformance test does not care what version the binaries are.
+#    - this means the user needs to setup a cluster first.
+#    - this means the user does not need to write any cluster/... scripts.  Custom
+#      clusters can be tested.
+#  - hack/e2e-test.sh is intended to run e2e tests built at HEAD, while
+#    this conformance test is intended to be run e2e tests built at a particular
+#    version.  This ensures that all conformance testees run the same set of tests,
+#    regardless of when they test for conformance.
+#  - it excludes certain e2e tests:
+#    - tests that are specific to certain cloud providers
+#    - tests of optional features, such as volume types.
+#    - tests of performance, scale, or reliability
+#    - known flaky tests.
+
+# The conformance test should be run from a github repository at
+# commit TBDCOMMITNUMBER.  Otherwise, it may not include the right
+# set of tests.
+# e.g.:
+#   cd /new/directory
+#   git clone git://github.com/GoogleCloudPlatform/kubernetes.git
+#   cd kubernetes
+#   git checkout TBDCOMMITNUMBER.
+# The working tree will be in a "detached HEAD" state.
+#
+# When run as described above, the conformance test tests whether a cluster is
+# supports key features for Kubernetes version 1.0.
 #
-# The user must export these environment variables:
-# KUBE_MASTER_IP to the ip address of the master.
-# AUTH_CONFIG to the argument of the "--auth_config=" flag.
-# If certs required, set CERT_DIR.
-# 
-# Example to test against a local vagrant cluster:
-# declare -x AUTH_CONFIG="$HOME/.kubernetes_vagrant_auth"
-# declare -x KUBE_MASTER_IP=10.245.1.2
-# hack/conformance-test.sh
-if [[ -z "KUBE_MASTER_IP" ]]; then
-  echo "Must set KUBE_MASTER_IP before running conformance test."
-  exit 1
-fi
-if [[ -z "AUTH_CONFIG" ]]; then
-  echo "Must set AUTH_CONFIG before running conformance test."
-  exit 1
-fi
+# TODO: when preparing to release a new major or minor version of Kubernetes,
+# then update above commit number, reevaluate the set of e2e tests,
+# update documentation at docs/getting-started-guides/README.md to have
+# a new column for conformance at that new version, and notify
+# community.
+
+# Instructions:
+#  - Setup a Kubernetes cluster with $NUM_MINIONS nodes (defined below).
+#  - Provide a Kubeconfig file whose current context is set to the
+#    cluster to be tested, and with suitable auth setting.
+#  - Specify the location of that kubeconfig with, e.g.:
+#    declare -x KUBECONFIG="$HOME/.kube/config"
+#  - Specify the location of the master with, e.g.:
+#    declare -x KUBE_MASTER_IP="1.2.3.4"
+#  - Make sure only essential pods are running and there are no failed/pending pods.
+#  - Make binaries needed by e2e, e.g.:
+#      make clean
+#      make quick-release
+#  - Run the test and capture output:
+#      hack/conformance-test.sh 2>&1 | tee conformance.$(date +%FT%T%z).log
+
+: ${KUBECONFIG:?"Must set KUBECONFIG before running conformance test."}
+: ${KUBE_MASTER_IP:?"Must set KUBE_MASTER_IP before running conformance test."}
+echo "Conformance test using ${KUBECONFIG} against master at ${KUBE_MASTER_IP}"
+echo -n "Conformance test run date:"
+date
+echo -n "Conformance test SHA:"
+HEAD_SHA=$(git rev-parse HEAD)
+echo $HEAD_SHA
+echo "Conformance test version tag(s):"
+git show-ref | grep $HEAD_SHA | grep refs/tags
+echo
+echo "Conformance test checking conformance with Kubernetes version 1.0"
+
+# It runs a whitelist of tests.  This whitelist was assembled at commit
+# b70b7084c93d4ce80b7463f48c23d5ac04edb2b1 starting from this list of tests:
+#   grep -h 'It(\|Describe(' -R test
+#
+# List of test name patterns not included and why not included:
+#  Cadvisor: impl detail how stats gotten from containers.
+#  MasterCerts: GKE/GCE specific
+#  Density: performance
+#  Cluster level logging...: optional feature
+#  Etcd failure: reliability
+#  Load Capacity: performance
+#  Monitoring: optional feature.
+#  Namespaces.*seconds: performance.
+#  Pod disks: uses GCE specific feature.
+#  Reboot: node management
+#  Nodes: node management.
+#  Restart: node management.
+#  Scale: performance
+#  Services.*load balancer: not all cloud providers have a load balancer.
+#  Services.*NodePort: flaky
+#  Shell: replies on optional ssh access to nodes.
+#  SSH: optional feature.
+#  Volumes: contained only skipped tests.
+export CONFORMANCE_TEST_SKIP_REGEX="Cadvisor|MasterCerts|Density|Cluster\slevel\slogging.*|Etcd\sfailure.*|Load\sCapacity|Monitoring|Namespaces.*seconds|Pod\sdisks|Reboot|Restart|Nodes|Scale|Services.*load\sbalancer|Services.*NodePort|Shell|SSH|Volumes"
+
+declare -x KUBERNETES_CONFORMANCE_TEST="1"
+declare -x NUM_MINIONS=4
 hack/ginkgo-e2e.sh
 exit $?

hack/ginkgo-e2e.sh

@@ -42,7 +42,13 @@ export KUBECTL KUBE_CONFIG_FILE
 source "${KUBE_ROOT}/cluster/kube-env.sh"
 
 # ---- Do cloud-provider-specific setup
-if [[ -z "${AUTH_CONFIG:-}" ]];  then
+if [[ -n "${KUBERNETES_CONFORMANCE_TEST:-}" ]]; then
+    echo "Conformance test: not doing test setup."
+  KUBERNETES_PROVIDER=""
+  auth_config=(
+    "--kubeconfig=${KUBECONFIG}"
+  )
+else
     echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"."
 
     source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
@@ -51,23 +57,9 @@ if [[ -z "${AUTH_CONFIG:-}" ]];  then
 
     detect-master >/dev/null
 
-    if [[ "${KUBERNETES_PROVIDER}" == "conformance_test" ]]; then
-      auth_config=(
-        "--auth-config=${KUBERNETES_CONFORMANCE_TEST_AUTH_CONFIG:-}"
-        "--cert-dir=${KUBERNETES_CONFORMANCE_TEST_CERT_DIR:-}"
-      )
-    else
-      auth_config=(
-      "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
-      )
-    fi
-else
-  echo "Conformance Test.  No cloud-provider-specific preparation."
-  KUBERNETES_PROVIDER=""
-  auth_config=(
-    "--auth-config=${AUTH_CONFIG:-}"
-    "--cert-dir=${CERT_DIR:-}"
-  )
+    auth_config=(
+    "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
+    )
 fi
 
 if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
@@ -81,6 +73,9 @@ if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
 fi
 
 ginkgo_args=()
+if [[ -n ${CONFORMANCE_TEST_SKIP_REGEX:-} ]]; then
+  ginkgo_args+=("--skip=\"${CONFORMANCE_TEST_SKIP_REGEX}\"")
+fi
 if [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then
   ginkgo_args+=("-p")
 fi

test/e2e/e2e_test.go

@@ -103,9 +103,8 @@ func TestE2E(t *testing.T) {
 	util.InitLogs()
 	defer util.FlushLogs()
 
-	// TODO: possibly clean up or refactor this functionality.
 	if testContext.Provider == "" {
-		glog.Fatal("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")
+		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")
 	}
 
 	if testContext.Provider == "aws" {

test/e2e/util.go

@@ -183,9 +183,6 @@ func Failf(format string, a ...interface{}) {
 }
 
 func providerIs(providers ...string) bool {
-	if testContext.Provider == "" {
-		Fail("testContext.Provider is not defined")
-	}
 	for _, provider := range providers {
 		if strings.ToLower(provider) == strings.ToLower(testContext.Provider) {
 			return true