Switch from Google Buckets to AWS S3 Buckets (#6497)

* Add python pip pakacge to install aws cli Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> * Upload build artifacts to aws s3 instead of gcp bucket Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> * Upload logs to aws s3 instead of google buckets Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> * Replace gcloud auth with aws credentials for artifact uploading to buckets Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> * Replace usage of google bucket with aws s3 buckets Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> Signed-off-by: Venkata Krishna Rohit Sakala <rohitsakala@gmail.com> (cherry picked from commit 4e2e91e0) Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

Switch from Google Buckets to AWS S3 Buckets (#6497)
f942d07e · Sakala Venkata Krishna Rohit · Brad Davidson · 357f6292 · f942d07e · f942d07e
Commit f942d07e authored Nov 28, 2022 by Sakala Venkata Krishna Rohit Committed by Brad Davidson Nov 28, 2022
8 changed files
--- a/.drone.yml
+++ b/.drone.yml
@@ -9,12 +9,14 @@ platform:
 steps:
 - name: build
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth, unprivileged_github_token ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader, unprivileged_github_token ]
  environment:
    GITHUB_TOKEN:
      from_secret: unprivileged_github_token
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper ci
  - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags
@@ -88,11 +90,13 @@ steps:

 - name: test
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
    ENABLE_REGISTRY: 'true'
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper -f Dockerfile.test.dapper
  volumes:
@@ -132,10 +136,12 @@ platform:
 steps:
 - name: build
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper ci
  - echo "${DRONE_TAG}-arm64" | sed -e 's/+/-/g' >.tags
@@ -184,11 +190,13 @@ steps:

 - name: test
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
    ENABLE_REGISTRY: 'true'
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper -f Dockerfile.test.dapper
  volumes:
@@ -211,10 +219,12 @@ platform:
 steps:
 - name: build
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper ci
  - echo "${DRONE_TAG}-arm" | sed -e 's/+/-/g' >.tags
@@ -263,11 +273,13 @@ steps:

 - name: test
  image: rancher/dapper:v0.5.0
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
    ENABLE_REGISTRY: 'true'
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper -f Dockerfile.test.dapper
  volumes:
@@ -304,10 +316,12 @@ steps:

 - name: build
  image: rancher/dapper:v0.5.8
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper ci
  - echo "${DRONE_TAG}-s390x" | sed -e 's/+/-/g' >.tags
@@ -359,11 +373,13 @@ steps:

 - name: test
  image: rancher/dapper:v0.5.8
-  secrets: [ gcloud_auth ]
+  secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ]
  environment:
    ENABLE_REGISTRY: 'true'
-    GCLOUD_AUTH:
-      from_secret: gcloud_auth
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader
+    AWS_ACCESS_KEY_ID:
+      from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
  commands:
  - dapper -f Dockerfile.test.dapper
  volumes:

--- a/Dockerfile.dapper
+++ b/Dockerfile.dapper
@@ -9,7 +9,7 @@ ENV https_proxy=$https_proxy
 ENV no_proxy=$no_proxy

 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
-    zlib-dev tar zip squashfs-tools npm coreutils python3 openssl-dev libffi-dev libseccomp libseccomp-dev \
+    zlib-dev tar zip squashfs-tools npm coreutils python3 py3-pip openssl-dev libffi-dev libseccomp libseccomp-dev \
    libseccomp-static make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static \
    zstd pigz alpine-sdk binutils-gold btrfs-progs-dev btrfs-progs-static gawk \
    && \
@@ -17,6 +17,8 @@ RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget c
    apk -U --no-cache add mingw-w64-gcc; \
    fi

+RUN python3 -m pip install awscli
+
 RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
    wget https://github.com/aquasecurity/trivy/releases/download/v0.25.3/trivy_0.25.3_Linux-ARM64.tar.gz && \
    tar -zxvf trivy_0.25.3_Linux-ARM64.tar.gz                                                            && \
@@ -45,7 +47,7 @@ ARG SELINUX=true
 ENV SELINUX $SELINUX

 ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/k3s-io/k3s/.cache -v trivy-cache:/root/.cache/trivy
-ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE SKIP_AIRGAP GCLOUD_AUTH GITHUB_TOKEN GOLANG DEBUG
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE SKIP_AIRGAP AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID GITHUB_TOKEN GOLANG DEBUG
 ENV DAPPER_SOURCE /go/src/github.com/k3s-io/k3s/
 ENV DAPPER_OUTPUT ./bin ./dist ./build/out ./build/static ./pkg/static ./pkg/deploy


--- a/Dockerfile.test.dapper
+++ b/Dockerfile.test.dapper
@@ -19,7 +19,7 @@ RUN OS=linux; \
 ENV TEST_CLEANUP true

 ENV DAPPER_RUN_ARGS --privileged --network host -v /tmp:/tmp
-ENV DAPPER_ENV REPO TAG DRONE_TAG DRONE_BUILD_EVENT IMAGE_NAME GCLOUD_AUTH SONOBUOY_VERSION ENABLE_REGISTRY
+ENV DAPPER_ENV REPO TAG DRONE_TAG DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID SONOBUOY_VERSION ENABLE_REGISTRY
 ENV DAPPER_SOURCE /go/src/github.com/k3s-io/k3s/
 ENV DAPPER_OUTPUT ./dist
 ENV DAPPER_DOCKER_SOCKET true

--- a/install.sh
+++ b/install.sh
@@ -92,7 +92,7 @@ set -o noglob
 #     Defaults to 'stable'.

 GITHUB_URL=https://github.com/k3s-io/k3s/releases
-STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds
+STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com
 DOWNLOADER=

 # --- helper functions for logs ---

--- a/package/rpm/install.sh
+++ b/package/rpm/install.sh
@@ -94,7 +94,7 @@ if [ "$INSTALL_K3S_DEBUG" = "true" ]; then
 fi

 GITHUB_URL=https://github.com/k3s-io/k3s/releases
-STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds
+STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com
 DOWNLOADER=

 # --- helper functions for logs ---

--- a/scripts/build-upload
+++ b/scripts/build-upload
 #!/bin/bash

-[ -n "$GCLOUD_AUTH" ] || {
+# Check for AWS Credentials
+[ -n "$AWS_SECRET_ACCESS_KEY" ] || {
  exit 0
 }
+[ -n "$AWS_ACCESS_KEY_ID" ] || {
+  exit 0
+}
+
 [ -x "$1" ] || {
  echo "First argument should be an executable" >&2
  exit 1
@@ -23,35 +28,10 @@ cleanup() {
 }
 trap cleanup EXIT INT

-GCLOUD_JSON=${TMPDIR}/.gcloud.json
-[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
-[ -s "${GCLOUD_JSON}" ] || {
-  echo "gcloud auth not defined" >&2
-  exit 1
-}
-
-BOTO_CONF=${TMPDIR}/.boto
-[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
-[Credentials]
-gs_service_key_file = ${GCLOUD_JSON}
-[Boto]
-https_validate_certificates = True
-[GSUtil]
-content_language = en
-default_api_version = 2
-default_project_id = rancher-dev
-END
-
 BUILD_NAME=$(basename $1)-$2
 (cd $(dirname $1) && sha256sum $(basename $1)) >${TMPDIR}/${BUILD_NAME}.sha256sum
 cp $1 ${TMPDIR}/${BUILD_NAME}

-[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
-
-HOME=${TMPDIR}
-PATH=${PATH}:${HOME}/gsutil
-
-gsutil cp ${TMPDIR}/${BUILD_NAME}* gs://k3s-ci-builds || exit 1
-
+aws s3 cp ${TMPDIR}/${BUILD_NAME}* s3://k3s-ci-builds || exit 1
 echo "Build uploaded" >&2
-echo "https://storage.googleapis.com/k3s-ci-builds/${BUILD_NAME}"
+echo "https://k3s-ci-builds.s3.amazonaws.com/${BUILD_NAME}"
--- a/scripts/log-upload
+++ b/scripts/log-upload
 #!/bin/bash

+# Check for AWS Credentials
+[ -n "$AWS_SECRET_ACCESS_KEY" ] || {
+  exit 0
+}
+[ -n "$AWS_ACCESS_KEY_ID" ] || {
+  exit 0
+}
+
 [ -d "$1" ] || {
  echo "First argument should be a directory" >&2
  exit 1
@@ -18,33 +26,10 @@ cleanup() {
 }
 trap cleanup EXIT INT

-GCLOUD_JSON=${TMPDIR}/.gcloud.json
-[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
-[ -s "${GCLOUD_JSON}" ] || {
-  echo "gcloud auth not defined" >&2
-  exit 1
-}
-
-BOTO_CONF=${TMPDIR}/.boto
-[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
-[Credentials]
-gs_service_key_file = ${GCLOUD_JSON}
-[Boto]
-https_validate_certificates = True
-[GSUtil]
-content_language = en
-default_api_version = 2
-default_project_id = rancher-dev
-END
-
-[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
-
-HOME=${TMPDIR}
-PATH=$PATH:${HOME}/gsutil
-
 LOG_TGZ=k3s-log-$(date +%s)-$("${GO}" env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz

 tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
-gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1
+aws s3 cp ${TMPDIR}/${LOG_TGZ} s3://k3s-ci-logs || exit 1
 echo "Logs uploaded" >&2
-echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}"
+echo "https://k3s-ci-logs.s3.amazonaws.com/${LOG_TGZ}"
+
--- a/tests/e2e/scripts/latest_commit.sh
+++ b/tests/e2e/scripts/latest_commit.sh
@@ -3,17 +3,17 @@
 iterations=0
 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=$1" | jq -r '.[] | .sha'  &> $2
 # The VMs take time on startup to hit googleapis.com, wait loop until we can
-while ! curl -s --fail https://storage.googleapis.com/k3s-ci-builds > /dev/null; do
+while ! curl -s --fail https://k3s-ci-builds.s3.amazonaws.com > /dev/null; do
    ((iterations++))
    if [ "$iterations" -ge 30 ]; then
-        echo "Unable to hit googleapis.com/k3s-ci-builds"
+        echo "Unable to hit https://k3s-ci-builds.s3.amazonaws.com"
        exit 1
    fi
    sleep 1
 done

 iterations=0
-curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum
+curl -s --fail https://k3s-ci-builds.s3.amazonaws.com/k3s-$(head -n 1 $2).sha256sum
 while [ $? -ne 0 ]; do
    ((iterations++))
    if [ "$iterations" -ge 6 ]; then
@@ -22,5 +22,5 @@ while [ $? -ne 0 ]; do
    fi
    sed -i 1d "$2"
    sleep 1
-    curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum
+    curl -s --fail https://k3s-ci-builds.s3.amazonaws.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum
 done
\ No newline at end of file