AWS kube-up: add retries around delete-security-group

If we deleted an ELB, we often fail to delete the security group, because deleting the ELB is invisibly asynchronous. Add a retry loop around delete-security-group to work around this. Fix #21147

AWS kube-up: add retries around delete-security-group
6678a3f0 · Justin Santa Barbara · d7a87c22 · 6678a3f0
Commit 6678a3f0 authored Mar 10, 2016 by Justin Santa Barbara
Hide whitespace changes
Inline Side-by-side

Showing with 19 additions and 2 deletions

util.sh cluster/aws/util.sh +19 -2

No files found.
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@@ -781,6 +781,24 @@ function release-elastic-ip {
  fi
 }

+# Deletes a security group
+# usage: delete_security_group <sgid>
+function delete_security_group {
+  local -r sg_id=${1}
+
+  echo "Deleting security group: ${sg_id}"
+
+  # We retry in case there's a dependent resource - typically an ELB
+  n=0
+  until [ $n -ge 20 ]; do
+    $AWS_CMD delete-security-group --group-id ${sg_id} > $LOG && return
+    n=$[$n+1]
+    sleep 3
+  done
+  echo "Unable to delete security group: ${sg_id}"
+  exit 1
+}
+
 function ssh-key-setup {
  if [[ ! -f "$AWS_SSH_KEY" ]]; then
    ssh-keygen -f "$AWS_SSH_KEY" -N ''
@@ -1372,8 +1390,7 @@ function kube-down {
        continue
      fi

-      echo "Deleting security group: ${sg_id}"
-      $AWS_CMD delete-security-group --group-id ${sg_id} > $LOG
+      delete_security_group ${sg_id}
    done

    subnet_ids=$($AWS_CMD describe-subnets \