Skip to content

K
k3s
Commit e3210c6c authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #51533 from shyamjvs/master-creation-retry

Automatic merge from submit-queue Retry master instance creation in case of retriable error (with sleep) To help with our 5k-node CI tests failing to startup the cluster. And also towards the greater goal - https://github.com/kubernetes/kubernetes/issues/43140 cc @kubernetes/sig-scalability-misc @kubernetes/sig-cluster-lifecycle-misc
parents 0596891e 2595f664
Hide whitespace changes
Inline Side-by-side
Showing with 58 additions and 26 deletions
cluster/gce/container-linux/master-helper.sh
View file @ e3210c6c
...@@ -66,6 +66,7 @@ function replicate-master-instance() { ...@@ -66,6 +66,7 @@ function replicate-master-instance() {
function create-master-instance-internal() { function create-master-instance-internal() {
local gcloud="gcloud" local gcloud="gcloud"
local retries=5
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
gcloud="gcloud beta" gcloud="gcloud beta"
fi fi
...@@ -93,19 +94,34 @@ function create-master-instance-internal() { ...@@ -93,19 +94,34 @@ function create-master-instance-internal() {
disk="${disk},boot=no" disk="${disk},boot=no"
disk="${disk},auto-delete=no" disk="${disk},auto-delete=no"
${gcloud} compute instances create "${master_name}" \ for attempt in $(seq 1 ${retries}); do
--project "${PROJECT}" \ if result=$(${gcloud} compute instances create "${master_name}" \
--zone "${ZONE}" \ --project "${PROJECT}" \
--machine-type "${MASTER_SIZE}" \ --zone "${ZONE}" \
--image-project="${MASTER_IMAGE_PROJECT}" \ --machine-type "${MASTER_SIZE}" \
--image "${MASTER_IMAGE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \
--tags "${MASTER_TAG}" \ --image "${MASTER_IMAGE}" \
--scopes "storage-ro,compute-rw,monitoring,logging-write" \ --tags "${MASTER_TAG}" \
--metadata-from-file "${metadata}" \ --scopes "storage-ro,compute-rw,monitoring,logging-write" \
--disk "${disk}" \ --metadata-from-file "${metadata}" \
--boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \ --disk "${disk}" \
${preemptible_master} \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
${network} ${preemptible_master} \
${network} 2>&1); then
echo "${result}" >&2
return 0
else
echo "${result}" >&2
if [[ ! "${result}" =~ "try again later" ]]; then
echo "Failed to create master instance due to non-retryable error" >&2
return 1
fi
sleep 10
fi
done
echo "Failed to create master instance despite ${retries} attempts" >&2
return 1
} }
function get-metadata() { function get-metadata() {
......
cluster/gce/gci/master-helper.sh
View file @ e3210c6c
...@@ -75,6 +75,7 @@ function replicate-master-instance() { ...@@ -75,6 +75,7 @@ function replicate-master-instance() {
function create-master-instance-internal() { function create-master-instance-internal() {
local gcloud="gcloud" local gcloud="gcloud"
local retries=5
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
gcloud="gcloud beta" gcloud="gcloud beta"
fi fi
...@@ -106,19 +107,34 @@ function create-master-instance-internal() { ...@@ -106,19 +107,34 @@ function create-master-instance-internal() {
disk="${disk},boot=no" disk="${disk},boot=no"
disk="${disk},auto-delete=no" disk="${disk},auto-delete=no"
${gcloud} compute instances create "${master_name}" \ for attempt in $(seq 1 ${retries}); do
--project "${PROJECT}" \ if result=$(${gcloud} compute instances create "${master_name}" \
--zone "${ZONE}" \ --project "${PROJECT}" \
--machine-type "${MASTER_SIZE}" \ --zone "${ZONE}" \
--image-project="${MASTER_IMAGE_PROJECT}" \ --machine-type "${MASTER_SIZE}" \
--image "${MASTER_IMAGE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \
--tags "${MASTER_TAG}" \ --image "${MASTER_IMAGE}" \
--scopes "storage-ro,compute-rw,monitoring,logging-write" \ --tags "${MASTER_TAG}" \
--metadata-from-file "${metadata}" \ --scopes "storage-ro,compute-rw,monitoring,logging-write" \
--disk "${disk}" \ --metadata-from-file "${metadata}" \
--boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ --disk "${disk}" \
${preemptible_master} \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
${network} ${preemptible_master} \
${network} 2>&1); then
echo "${result}" >&2
return 0
else
echo "${result}" >&2
if [[ ! "${result}" =~ "try again later" ]]; then
echo "Failed to create master instance due to non-retryable error" >&2
return 1
fi
sleep 10
fi
done
echo "Failed to create master instance despite ${retries} attempts" >&2
return 1
} }
function get-metadata() { function get-metadata() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment