run-gcloud-compute-with-retries in kubemark handles resource already exists case

test/kubemark/common.sh

@@ -49,8 +49,17 @@ export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
 # arguments:
 # $@: all stuff that goes after 'gcloud compute '
 function run-gcloud-compute-with-retries {
+  echo "" > /tmp/gcloud_retries
   for attempt in $(seq 1 ${RETRIES}); do
-    if ! gcloud compute $@; then
+    if ! gcloud compute $@ &> /tmp/gcloud_retries; then
+      if [[ $(grep -c "already exists" /tmp/gcloud_retries) -gt 0 ]]; then
+        if [[ "${attempt}" == 1 ]]; then
+          echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
+          exit 1
+        fi
+        echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
+        return 0
+      fi
       echo -e "${color_yellow}Attempt $(($attempt+1)) failed to $1 $2 $3. Retrying.${color_norm}" >& 2
       sleep $(($attempt * 5))
     else

test/kubemark/start-kubemark.sh

@@ -144,7 +144,7 @@ done
 
 password=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
 
-gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
+run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
   --command="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \
     sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \
     sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \
@@ -156,7 +156,7 @@ gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
     sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
     sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\""
 
-gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \
+run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" \
   "${SERVER_BINARY_TAR}" \
   "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \
   "${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \