Merge pull request #21164 from justinsb/aws_reboot_with_ephemeral

cluster/aws/templates/format-disks.sh

@@ -18,6 +18,12 @@
 
 function ensure-local-disks() {
 
+# Skip if already mounted (a reboot)
+if ( grep "/mnt/ephemeral" /proc/mounts ); then
+  echo "Found /mnt/ephemeral in /proc/mounts; skipping local disk initialization"
+  return
+fi
+
 block_devices=()
 
 ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
@@ -61,10 +67,14 @@ else
   # Remove any existing mounts
   for block_device in ${block_devices}; do
     echo "Unmounting ${block_device}"
-    /bin/umount ${block_device}
+    /bin/umount ${block_device} || echo "Ignoring failure umounting ${block_device}"
     sed -i -e "\|^${block_device}|d" /etc/fstab
   done
 
+  # Remove any existing /mnt/ephemeral entry in /etc/fstab
+  sed -i -e "\|/mnt/ephemeral|d" /etc/fstab
+
+  # Mount the storage
   if [[ ${docker_storage} == "btrfs" ]]; then
     apt-get-install btrfs-tools
 
@@ -75,7 +85,7 @@ else
       echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
       mkfs.btrfs -f --data raid0 ${block_devices[@]}
     fi
-    echo "${block_devices[0]}  /mnt/ephemeral  btrfs  noatime  0 0" >> /etc/fstab
+    echo "${block_devices[0]}  /mnt/ephemeral  btrfs  noatime,nofail  0 0" >> /etc/fstab
     mkdir -p /mnt/ephemeral
     mount /mnt/ephemeral
 
@@ -89,7 +99,7 @@ else
     fi
 
     mkfs -t ext4 ${block_devices[0]}
-    echo "${block_devices[0]}  /mnt/ephemeral  ext4     noatime  0 0" >> /etc/fstab
+    echo "${block_devices[0]}  /mnt/ephemeral  ext4     noatime,nofail  0 0" >> /etc/fstab
     mkdir -p /mnt/ephemeral
     mount /mnt/ephemeral
 
@@ -129,7 +139,7 @@ else
       lvcreate -l 100%FREE -n kubernetes vg-ephemeral
       mkfs -t ext4 /dev/vg-ephemeral/kubernetes
       mkdir -p /mnt/ephemeral/kubernetes
-      echo "/dev/vg-ephemeral/kubernetes  /mnt/ephemeral/kubernetes  ext4  noatime  0 0" >> /etc/fstab
+      echo "/dev/vg-ephemeral/kubernetes  /mnt/ephemeral/kubernetes  ext4  noatime,nofail  0 0" >> /etc/fstab
       mount /mnt/ephemeral/kubernetes
 
       move_kubelet="/mnt/ephemeral/kubernetes"
@@ -146,7 +156,7 @@ else
       lvcreate -l 100%VG -n ephemeral vg-ephemeral
       mkfs -t ext4 /dev/vg-ephemeral/ephemeral
       mkdir -p /mnt/ephemeral
-      echo "/dev/vg-ephemeral/ephemeral  /mnt/ephemeral  ext4  noatime  0 0" >> /etc/fstab
+      echo "/dev/vg-ephemeral/ephemeral  /mnt/ephemeral  ext4  noatime,nofail 0 0" >> /etc/fstab
       mount /mnt/ephemeral
 
       mkdir -p /mnt/ephemeral/kubernetes
@@ -179,11 +189,16 @@ fi
 
 if [[ -n "${move_docker}" ]]; then
   # Move docker to e.g. /mnt
+  # but only if it is a directory, not a symlink left over from a previous run
   if [[ -d /var/lib/docker ]]; then
     mv /var/lib/docker ${move_docker}/
   fi
   mkdir -p ${move_docker}/docker
-  ln -s ${move_docker}/docker /var/lib/docker
+  # If /var/lib/docker doesn't exist (it will exist if it is already a symlink),
+  # then symlink it to the ephemeral docker area
+  if [[ ! -e /var/lib/docker ]]; then
+    ln -s ${move_docker}/docker /var/lib/docker
+  fi
   DOCKER_ROOT="${move_docker}/docker"
   DOCKER_OPTS="${DOCKER_OPTS:-} -g ${DOCKER_ROOT}"
 fi
@@ -191,11 +206,15 @@ fi
 if [[ -n "${move_kubelet}" ]]; then
   # Move /var/lib/kubelet to e.g. /mnt
   # (the backing for empty-dir volumes can use a lot of space!)
+  # (As with /var/lib/docker, only if it is a directory; skip if symlink)
   if [[ -d /var/lib/kubelet ]]; then
     mv /var/lib/kubelet ${move_kubelet}/
   fi
   mkdir -p ${move_kubelet}/kubelet
-  ln -s ${move_kubelet}/kubelet /var/lib/kubelet
+  # Create symlink for /var/lib/kubelet, unless it is already a symlink
+  if [[ ! -e /var/lib/kubelet ]]; then
+    ln -s ${move_kubelet}/kubelet /var/lib/kubelet
+  fi
   KUBELET_ROOT="${move_kubelet}/kubelet"
 fi