Add a flag to allow non-fatal errors in validate-cluster

cluster/kube-up.sh

@@ -25,6 +25,7 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+EXIT_ON_WEAK_ERROR="${EXIT_ON_WEAK_ERROR:-true}"
 
 if [ -f "${KUBE_ROOT}/cluster/env.sh" ]; then
     source "${KUBE_ROOT}/cluster/env.sh"
@@ -47,7 +48,18 @@ echo "... calling kube-up" >&2
 kube-up
 
 echo "... calling validate-cluster" >&2
-validate-cluster
+if [[ "${EXIT_ON_WEAK_ERROR}" == "true" ]]; then
+	validate-cluster
+else
+	if ! validate-cluster; then
+		local validate_result="$?"
+		if [[ "${validate_result}" == "1" ]]; then
+			exit 1
+		elif [[ "${validate_result}" == "2" ]]; then
+			echo "...ignoring non-fatal errors in validate-cluster" >&2
+		fi
+	fi
+fi
 
 echo -e "Done, listing cluster services:\n" >&2
 "${KUBE_ROOT}/cluster/kubectl.sh" cluster-info

hack/jenkins/e2e.sh

@@ -528,6 +528,7 @@ case ${JOB_NAME} in
     NODE_DISK_SIZE="50GB"
     NUM_NODES="1000"
     ALLOWED_NOTREADY_NODES="2"
+    EXIT_ON_WEAK_ERROR="false"
     # Reduce logs verbosity
     TEST_CLUSTER_LOG_LEVEL="--v=1"
     # Increase resync period to simulate production
@@ -928,6 +929,7 @@ export KUBE_OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-}
 export GCE_SERVICE_ACCOUNT=$(gcloud auth list 2> /dev/null | grep active | cut -f3 -d' ')
 export FAIL_ON_GCP_RESOURCE_LEAK="${FAIL_ON_GCP_RESOURCE_LEAK:-false}"
 export ALLOWED_NOTREADY_NODES=${ALLOWED_NOTREADY_NODES:-}
+export EXIT_ON_WEAK_ERROR=${EXIT_ON_WEAK_ERROR:-}
 
 # GKE variables
 export CLUSTER_NAME=${E2E_CLUSTER_NAME}