node e2e tests - retry fkaky failures caused by gcloud authentication

test/e2e_node/gcloud/gcloud.go

@@ -136,6 +136,16 @@ func (gc *gCloudClientImpl) Run(
 	go func() {
 		// Create the tmp directory
 		out, err := gc.Command("mkdir", "-p", tDir)
+
+		// Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts
+		// If there was an error about credentials, retry making the directory 6 times to see if it can be resolved
+		// This is to help debug if the credential issues are persistent for a given host on a given run, or transient
+		// And if downstream gcloud commands are also impacted
+		for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ {
+			glog.Errorf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out)
+			time.Sleep(5 * time.Second)
+			out, err = gc.Command("mkdir", "-p", tDir)
+		}
 		if err != nil {
 			glog.Errorf("mkdir failed %v %s", err, out)
 			h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}