Extends GPUDevicePlugin e2e test to exercise device plugin restarts.

test/e2e/scheduling/nvidia-gpus.go

@@ -240,6 +240,26 @@ var _ = SIGDescribe("[Feature:GPU]", func() {
 var _ = SIGDescribe("[Feature:GPUDevicePlugin]", func() {
 	f := framework.NewDefaultFramework("device-plugin-gpus")
 	It("run Nvidia GPU Device Plugin tests on Container Optimized OS only", func() {
+		// 1. Verifies GPU resource is successfully advertised on the nodes
+		// and we can run pods using GPUs.
+		By("Starting device plugin daemonset and running GPU pods")
+		testNvidiaGPUsOnCOS(f)
+
+		// 2. Verifies that when the device plugin DaemonSet is removed, resource capacity drops to zero.
+		By("Deleting device plugin daemonset")
+		ds := dsFromManifest(dsYamlUrl)
+		falseVar := false
+		err := f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Delete(ds.Name, &metav1.DeleteOptions{OrphanDependents: &falseVar})
+		framework.ExpectNoError(err, "failed to delete daemonset")
+		framework.Logf("Successfully deleted device plugin daemonset. Wait for resource to be removed.")
+		// Wait for Nvidia GPUs to be not available on nodes
+		Eventually(func() bool {
+			return !areGPUsAvailableOnAllSchedulableNodes(f)
+		}, 5*time.Minute, time.Second).Should(BeTrue())
+
+		// 3. Restarts the device plugin DaemonSet. Verifies GPU resource is successfully advertised
+		// on the nodes and we can run pods using GPUs.
+		By("Restarting device plugin daemonset and running GPU pods")
 		testNvidiaGPUsOnCOS(f)
 	})
 })