Merge pull request #18150 from gmarek/grab-profiles

hack/grab-profiles.sh

@@ -18,17 +18,48 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+function grab_profiles_from_component {
+  local requested_profiles=$1
+  local mem_pprof_flags=$2
+  local binary=$3
+  local tunnel_port=$4
+  local path=$5
+  local output_prefix=$6
+  local timestamp=$7
+
+  echo "binary: $binary"
+
+  for profile in ${requested_profiles}; do
+    case ${profile} in
+      cpu)
+        go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/profile" > "${output_prefix}-${profile}-profile-${timestamp}.pdf"
+        ;;
+      mem)
+        # There are different kinds of memory profiles that are available that
+        # had to be grabbed separately: --inuse-space, --inuse-objects,
+        # --alloc-space, --alloc-objects. We need to iterate over all requested
+        # kinds.
+        for flag in ${mem_pprof_flags}; do
+          go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/heap" > "${output_prefix}-${profile}-${flag}-profile-${timestamp}.pdf"
+        done
+        ;;
+    esac
+  done
+}
+
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 server_addr=""
+kubelet_addreses=""
+kubelet_binary=""
 requested_profiles=""
 mem_pprof_flags=""
 profile_components=""
 output_dir="."
 tunnel_port="${tunnel_port:-1234}"
 
-args=$(getopt -o s:mho: -l server:,master,heapster,output:,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu -- "$@")
+args=$(getopt -o s:mho:k: -l server:,master,heapster,output:,kubelet:,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu,kubelet-binary: -- "$@")
 if [[ $? -ne 0 ]]; then
   >&2 echo "Error in getopt"
   exit 1
@@ -37,6 +68,7 @@ fi
 HEAPSTER_VERSION="v0.18.2"
 MASTER_PPROF_PATH="debug/pprof"
 HEAPSTER_PPROF_PATH="api/v1/proxy/namespaces/kube-system/services/monitoring-heapster/debug/pprof"
+KUBELET_PPROF_PATH_PREFIX="api/v1/proxy/nodes"
 
 eval set -- "${args}"
 
@@ -68,6 +100,25 @@ while true; do
       output_dir=$1
       shift
       ;;
+    -k|--kubelet)
+      shift
+      profile_components="kubelet ${profile_components}"
+      if [ -z "$1" ]; then
+        >&2 echo "empty argumet to --kubelet flag"
+        exit 1
+      fi
+      kubelet_addreses="$1 $kubelet_addreses"
+      shift
+      ;;
+    --kubelet-binary)
+      shift
+      if [ -z "$1" ]; then
+        >&2 echo "empty argumet to --kubelet-binary flag"
+        exit 1
+      fi
+      kubelet_binary=$1
+      shift
+      ;;
     --inuse-space)
       shift
       requested_profiles="mem ${requested_profiles}"
@@ -139,6 +190,8 @@ kube::util::trap_add 'kill $SSH_PID' EXIT
 kube::util::trap_add 'kill $SSH_PID' SIGTERM
 
 requested_profiles=$(echo ${requested_profiles} | xargs -n1 | sort -u | xargs)
+profile_components=$(echo ${profile_components} | xargs -n1 | sort -u | xargs)
+kubelet_addreses=$(echo ${kubelet_addreses} | xargs -n1 | sort -u | xargs)
 echo "requested profiles: ${requested_profiles}"
 echo "flags for heap profile: ${mem_pprof_flags}"
 
@@ -159,19 +212,21 @@ for component in ${profile_components}; do
       binary=heapster
       path=${HEAPSTER_PPROF_PATH}
       ;;
+    kubelet)
+      path="${KUBELET_PPROF_PATH_PREFIX}"
+      if [[ -z "${kubelet_binary}" ]]; then
+        binary="${KUBE_ROOT}/_output/local/bin/linux/amd64/kubelet"
+      else
+        binary=${kubelet_binary}
+      fi
+      ;;
   esac
 
-  for profile in ${requested_profiles}; do
-    case ${profile} in
-      cpu)
-        go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/profile" > "${output_dir}/${component}-${profile}-profile-${timestamp}.pdf"
-        ;;
-      mem)
-        for flag in ${mem_pprof_flags}; do
-          go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/heap" > "${output_dir}/${component}-${profile}-${flag}-profile-${timestamp}.pdf"
-        done
-        ;;
-    esac
-  done
+  if [[ "${component}" == "kubelet" ]]; then
+    for node in $(echo ${kubelet_addreses} | sed 's/[,;]/\n/g'); do
+      grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}/${node}/debug/pprof" "${output_dir}/${component}" "${timestamp}"
+    done    
+  else 
+    grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}" "${output_dir}/${component}" "${timestamp}"
+  fi
 done
-