Add CRI Validation test. The test run non-flaky, non-serial test against

hack/verify-flags/known-flags.txt

@@ -445,6 +445,7 @@ run-proxy
 run-services-mode
 runtime-cgroups
 runtime-config
+runtime-integration-type
 runtime-request-timeout
 save-config
 scheduler-config

test/e2e/framework/test_context.go

@@ -94,6 +94,8 @@ type NodeTestContextType struct {
 	ManifestPath string
 	// PrepullImages indicates whether node e2e framework should prepull images.
 	PrepullImages bool
+	// RuntimeIntegrationType indicates how runtime is integrated with Kubelet. This is mainly used for CRI validation test.
+	RuntimeIntegrationType string
 }
 
 type CloudConfig struct {
@@ -186,6 +188,7 @@ func RegisterNodeFlags() {
 	flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,imagefs.available<10%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
 	flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
 	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+	flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.")
 }
 
 // Enable viper configuration management of flags.

test/e2e_node/e2e_node_suite_test.go

@@ -55,10 +55,19 @@ func init() {
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	// Mark the run-services-mode flag as hidden to prevent user from using it.
 	pflag.CommandLine.MarkHidden("run-services-mode")
+	// It's weird that if I directly use pflag in TestContext, it will report error.
+	// It seems that someone is using flag.Parse() after init() and TestMain().
+	// TODO(random-liu): Find who is using flag.Parse() and cause errors and move the following logic
+	// into TestContext.
+	pflag.CommandLine.MarkHidden("runtime-integration-type")
 }
 
-func TestE2eNode(t *testing.T) {
+func TestMain(m *testing.M) {
 	pflag.Parse()
+	os.Exit(m.Run())
+}
+
+func TestE2eNode(t *testing.T) {
 	if *runServicesMode {
 		// If run-services-mode is specified, only run services in current process.
 		services.RunE2EServices()

test/e2e_node/jenkins/cri_validation/jenkins-validation.properties

+GCI_IMAGE_PROJECT=container-vm-image-staging
+GCI_IMAGE_FAMILY=gci-canary-test
+GCI_IMAGE=$(gcloud compute images describe-from-family ${GCI_IMAGE_FAMILY} --project=${GCI_IMAGE_PROJECT} --format="value(name)")
+DOCKER_VERSION=v1.11.2
+GCI_CLOUD_INIT=test/e2e_node/jenkins/gci-init.yaml
+
+GCE_HOSTS=
+GCE_IMAGES=${GCI_IMAGE}
+GCE_IMAGE_PROJECT=${GCI_IMAGE_PROJECT}
+GCE_ZONE=us-central1-f
+GCE_PROJECT=k8s-jkns-ci-node-e2e
+# user-data is the GCI cloud init config file.
+# gci-docker-version specifies docker version in GCI image.
+GCE_INSTANCE_METADATA="user-data<${GCI_CLOUD_INIT},gci-docker-version=${DOCKER_VERSION}"
+CLEANUP=true
+GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"'
+SETUP_NODE=false
+TEST_ARGS='--runtime-integration-type=cri"

test/e2e_node/services/services.go

@@ -93,6 +93,7 @@ func (e *E2EServices) Start() error {
 		"--manifest-path", framework.TestContext.ManifestPath,
 		"--eviction-hard", framework.TestContext.EvictionHard,
 		"--feature-gates", framework.TestContext.FeatureGates,
+		"--runtime-integration-type", framework.TestContext.RuntimeIntegrationType,
 		"--logtostderr",
 		"--vmodule=*=4",
 	)
@@ -381,12 +382,16 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		"--serialize-image-pulls", "false",
 		"--config", framework.TestContext.ManifestPath,
 		"--file-check-frequency", "10s", // Check file frequently so tests won't wait too long
-		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
 		"--pod-cidr=10.180.0.0/24", // Assign a fixed CIDR to the node because there is no node controller.
 		"--eviction-hard", framework.TestContext.EvictionHard,
 		"--eviction-pressure-transition-period", "30s",
 		"--feature-gates", framework.TestContext.FeatureGates,
+		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
 	)
+	if framework.TestContext.RuntimeIntegrationType != "" {
+		cmdArgs = append(cmdArgs, "--experimental-runtime-integration-type",
+			framework.TestContext.RuntimeIntegrationType) // Whether to use experimental cri integration.
+	}
 	if framework.TestContext.CgroupsPerQOS {
 		// TODO: enable this when the flag is stable and available in kubelet.
 		// cmdArgs = append(cmdArgs,