Merge pull request #31367 from Amey-D/gci-kubelet

cluster/gce/gci/configure-helper.sh

@@ -436,13 +436,26 @@ function load-docker-images {
   fi
 }
 
-# A kubelet systemd service is built in GCI image, but by default it is not started
-# when an instance is up. To start kubelet, the command line flags should be written
-# to /etc/default/kubelet in the format "KUBELET_OPTS=<flags>", and then start kubelet
-# using systemctl. This function assembles the command line and start the kubelet
-# systemd service.
+# This function assembles the kubelet systemd service file and starts it
+# using systemctl.
 function start-kubelet {
   echo "Start kubelet"
+  local kubelet_bin="${KUBE_HOME}/bin/kubelet"
+  local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
+  local -r builtin_kubelet="/usr/bin/kubelet"
+  if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
+    # Determine which binary to use on test clusters. We use the built-in
+    # version only if the downloaded version is the same as the built-in
+    # version. This allows GCI to run some of the e2e tests to qualify the
+    # built-in kubelet.
+    if [[ -x "${builtin_kubelet}" ]]; then
+      local -r builtin_version="$("${builtin_kubelet}"  --version=true | cut -f2 -d " ")"
+      if [[ "${builtin_version}" == "${version}" ]]; then
+        kubelet_bin="${builtin_kubelet}"
+      fi
+    fi
+  fi
+  echo "Using kubelet binary at ${kubelet_bin}"
   local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
   flags+=" --allow-privileged=true"
   flags+=" --babysit-daemons=true"
@@ -506,7 +519,26 @@ function start-kubelet {
   if [[ -n "${FEATURE_GATES:-}" ]]; then
      flags+=" --feature-gates=${FEATURE_GATES}"
   fi
-  echo "KUBELET_OPTS=\"${flags}\"" > /etc/default/kubelet
+
+  local -r kubelet_env_file="/etc/default/kubelet"
+  echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
+
+  # Write the systemd service file for kubelet.
+  cat <<EOF >/etc/systemd/system/kubelet.service
+[Unit]
+Description=Kubernetes kubelet
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+Restart=always
+RestartSec=10
+EnvironmentFile=${kubelet_env_file}
+ExecStart=${kubelet_bin} \$KUBELET_OPTS
+
+[Install]
+WantedBy=multi-user.target
+EOF
 
   # Delete docker0 to avoid interference
   iptables -t nat -F || true
@@ -1070,7 +1102,7 @@ function start-rescheduler {
 
 function reset-motd {
   # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
-  local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
+  local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
   # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
   # or the git hash that's in the build info.
   local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"

cluster/gce/gci/configure.sh

@@ -128,32 +128,19 @@ function install-kube-binary-config {
     cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}"
   fi
   local -r kube_bin="${KUBE_HOME}/bin"
-  # If the built-in binary version is different from the expected version, we use
-  # the downloaded binary. The simplest implementation is to always use the downloaded
-  # binary without checking the version. But we have another version guardian in GKE.
-  # So, we compare the versions to ensure this run-time binary replacement is only
-  # applied for OSS kubernetes.
-  cp "${src_dir}/kubelet" "${kube_bin}"
-  local -r builtin_version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
-  local -r required_version="$(/home/kubernetes/bin/kubelet --version=true | cut -f2 -d " ")"
-  if [[ "${TEST_CLUSTER:-}" == "true" ]] || \
-     [[ "${builtin_version}" != "${required_version}" ]]; then
-    cp "${src_dir}/kubectl" "${kube_bin}"
-    chmod 755 "${kube_bin}/kubelet"
-    chmod 755 "${kube_bin}/kubectl"
-    mount --bind "${kube_bin}/kubelet" /usr/bin/kubelet
-    mount --bind "${kube_bin}/kubectl" /usr/bin/kubectl
-  else
-    rm -f "${kube_bin}/kubelet"
-  fi
+  mv "${src_dir}/kubelet" "${kube_bin}"
+  mv "${src_dir}/kubectl" "${kube_bin}"
+
   if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \
      [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
     #TODO(andyzheng0831): We should make the cni version number as a k8s env variable.
     local -r cni_tar="cni-8a936732094c0941e1543ef5d292a1f4fffa1ac5.tar.gz"
     download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}"
-    tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite
-    mv "${kube_bin}/bin"/* "${kube_bin}"
-    rmdir "${kube_bin}/bin"
+    local -r cni_dir="${KUBE_HOME}/cni"
+    mkdir -p "${cni_dir}"
+    tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}" --overwrite
+    mv "${cni_dir}/bin"/* "${kube_bin}"
+    rmdir "${cni_dir}/bin"
     rm -f "${KUBE_HOME}/${cni_tar}"
   fi
 
@@ -184,8 +171,7 @@ function install-kube-binary-config {
   fi
   cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
   cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
-  chmod 544 "${KUBE_HOME}/bin/configure-helper.sh"
-  chmod 544 "${KUBE_HOME}/bin/health-monitor.sh"
+  chmod -R 755 "${kube_bin}"
 
   # Clean up.
   rm -rf "${KUBE_HOME}/kubernetes"

cluster/gce/gci/master.yaml

@@ -33,6 +33,7 @@ write_files:
       [Service]
       Type=oneshot
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]
@@ -51,6 +52,7 @@ write_files:
       RestartSec=10
       RemainAfterExit=yes
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
       ExecStart=/home/kubernetes/bin/health-monitor.sh docker
 
       [Install]
@@ -69,6 +71,7 @@ write_files:
       RestartSec=10
       RemainAfterExit=yes
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
       ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
 
       [Install]

cluster/gce/gci/node.yaml

@@ -33,6 +33,7 @@ write_files:
       [Service]
       Type=oneshot
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]
@@ -51,6 +52,7 @@ write_files:
       RestartSec=10
       RemainAfterExit=yes
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
       ExecStart=/home/kubernetes/bin/health-monitor.sh docker
 
       [Install]
@@ -69,6 +71,7 @@ write_files:
       RestartSec=10
       RemainAfterExit=yes
       RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
       ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
 
       [Install]