Retry downloads, respect URL list, validate tar hash

cluster/gce/win1803/common.psm1

@@ -18,6 +18,9 @@
   and scripts for configuring Windows nodes.
 #>
 
+# Disable progress bar to increase download speed.
+$ProgressPreference = 'SilentlyContinue'
+
 # REDO_STEPS affects the behavior of a node that is rebooted after initial
 # bringup. When true, on a reboot the scripts will redo steps that were
 # determined to have already been completed once (e.g. to overwrite
@@ -86,5 +89,62 @@ function Get-InstanceMetadataValue {
   }
 }
 
+function Validate-SHA1 {
+  param(
+    [parameter(Mandatory=$true)] [string]$Hash,
+    [parameter(Mandatory=$true)] [string]$Path
+  )
+  $actual = Get-FileHash -Path $Path -Algorithm SHA1
+  # Note: Powershell string comparisons are case-insensitive by default, and this
+  # is important here because Linux shell scripts produce lowercase hashes but
+  # Powershell Get-FileHash produces uppercase hashes. This must be case-insensitive
+  # to work.
+  if ($actual.Hash -ne $Hash) {
+    Log-Output "$Path corrupted, sha1 $actual doesn't match expected $Hash"
+    Throw ("$Path corrupted, sha1 $actual doesn't match expected $Hash")
+  }
+}
+
+# Attempts to download the file from URLs, trying each URL until it succeeds.
+# It will loop through the URLs list forever until it has a success.
+# If successful, it will write the file to OutFile. You can optionally provide a SHA1 Hash
+# argument, in which case it will attempt to validate the downloaded file against the hash.
+function MustDownload-File {
+  param (
+    [parameter(Mandatory=$false)] [string]$Hash,
+    [parameter(Mandatory=$true)] [string]$OutFile,
+    [parameter(Mandatory=$true)] [System.Collections.Generic.List[String]]$URLs
+  )
+
+  While($true) {
+    ForEach($url in $URLs) {
+      # Attempt to download the file
+      Try {
+        # TODO(mtaufen): When we finally get a Windows version that has Powershell 6
+        # installed we can set `-MaximumRetryCount 6 -RetryIntervalSec 10` to make this even more robust.
+        Invoke-WebRequest $url -OutFile $OutFile -TimeoutSec 300
+      } Catch {
+        $message = $_.Exception.ToString()
+        Log-Output "Failed to download file from $url. Will retry. Error: $message"
+        continue
+      }
+      # Attempt to validate the hash
+      if ($Hash -ne $null) {
+        Try {
+            Validate-SHA1 -Hash $Hash -Path $OutFile
+        } Catch {
+            $message = $_.Exception.ToString()
+            Log-Output "Hash validation of $url failed. Will retry. Error: $message"
+            continue
+        }
+        Log-Output "Downloaded $url (SHA1 = $Hash)"
+        return
+      }
+      Log-Output "Downloaded $url"
+      return
+    }
+  }
+}
+
 # Export all public functions:
 Export-ModuleMember -Function *-*

cluster/gce/win1803/k8s-node-setup.psm1

@@ -254,9 +254,8 @@ function Download-HelperScripts {
   if (-not (ShouldWrite-File ${env:K8S_DIR}\hns.psm1)) {
     return
   }
-  Invoke-WebRequest `
-      https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1 `
-      -OutFile ${env:K8S_DIR}\hns.psm1
+  MustDownload-File -OutFile ${env:K8S_DIR}\hns.psm1 `
+    -URLs "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1"
 }
 
 # Takes the Windows version string from the cluster bash scripts (e.g.
@@ -318,14 +317,13 @@ function DownloadAndInstall-KubernetesBinaries {
   $tmp_dir = 'C:\k8s_tmp'
   New-Item -Force -ItemType 'directory' $tmp_dir | Out-Null
 
-  $uri = ${kube_env}['NODE_BINARY_TAR_URL']
-  $filename = Split-Path -leaf $uri
-
-  # Disable progress bar to increase download speed.
-  $ProgressPreference = 'SilentlyContinue'
-  Invoke-WebRequest $uri -OutFile ${tmp_dir}\${filename}
-
-  # TODO(yujuhong): Verify hash of the tarball.
+  $urls = ${kube_env}['NODE_BINARY_TAR_URL'].Split(",")
+  $filename = Split-Path -leaf $urls[0]
+  $hash = $null
+  if ($kube_env.ContainsKey('NODE_BINARY_TAR_HASH')) {
+    $hash = ${kube_env}['NODE_BINARY_TAR_HASH']
+  }
+  MustDownload-File -Hash $hash -OutFile ${tmp_dir}\${filename} -URLs $urls
 
   # Change the directory to the parent directory of ${env:K8S_DIR} and untar.
   # This (over-)writes ${dest_dir}/kubernetes/node/bin/*.exe files.
@@ -791,9 +789,8 @@ function Configure-HostNetworkingService {
 function Configure-CniNetworking {
   if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or
       (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) {
-    Invoke-WebRequest `
-        https://github.com/yujuhong/gce-k8s-windows-testing/raw/master/windows-cni-plugins.zip `
-        -OutFile ${env:CNI_DIR}\windows-cni-plugins.zip
+    MustDownload-File -OutFile ${env:CNI_DIR}\windows-cni-plugins.zip `
+      -URLs "https://github.com/yujuhong/gce-k8s-windows-testing/raw/master/windows-cni-plugins.zip"
     rm ${env:CNI_DIR}\*.exe
     Expand-Archive ${env:CNI_DIR}\windows-cni-plugins.zip ${env:CNI_DIR}
     mv ${env:CNI_DIR}\bin\*.exe ${env:CNI_DIR}\