Update hack scripts to use go mod

.gitignore

@@ -130,5 +130,5 @@ zz_generated_*_test.go
 /bazel-*
 *.pyc
 
-# generated by verify-godeps.sh
+# generated by verify-vendor.sh
 vendordiff.patch

build/root/Makefile

@@ -111,7 +111,7 @@ define VERIFY_HELP_INFO
 # Runs all the presubmission verifications.
 #
 # Args:
-#   BRANCH: Branch to be passed to verify-godeps.sh script.
+#   BRANCH: Branch to be passed to verify-vendor.sh script.
 #   WHAT: List of checks to run
 #
 # Example:

hack/.shellcheck_failures

@@ -53,12 +53,11 @@
 ./hack/update-generated-kms-dockerized.sh
 ./hack/update-generated-protobuf-dockerized.sh
 ./hack/update-generated-runtime-dockerized.sh
-./hack/update-godep-licenses.sh
 ./hack/update-gofmt.sh
 ./hack/update-openapi-spec.sh
-./hack/update-staging-godeps-dockerized.sh
-./hack/update-staging-godeps.sh
 ./hack/update-translations.sh
+./hack/update-vendor-licenses.sh
+./hack/update-vendor.sh
 ./hack/update-workspace-mirror.sh
 ./hack/verify-api-groups.sh
 ./hack/verify-boilerplate.sh

hack/godep-restore.sh

-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::log::status "Restoring kubernetes godeps"
-
-if kube::util::godep_restored 2>&1; then
-    kube::log::status "Dependencies appear to be current - skipping download"
-    exit 0
-fi
-
-kube::util::ensure_godep_version
-
-kube::log::status "Downloading dependencies - this might take a while"
-GOPATH="${GOPATH}:${KUBE_ROOT}/staging" ${KUBE_GODEP:?} restore "$@"
-kube::log::status "Done"

hack/godep-save.sh

-#!/usr/bin/env bash
-
-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::log::status "Ensuring prereqs"
-kube::util::ensure_single_dir_gopath
-kube::util::ensure_no_staging_repos_in_gopath
-
-kube::util::ensure_godep_version
-
-BACKUP=_tmp/godep-save.$RANDOM
-mkdir -p "${BACKUP}"
-
-function kube::godep_save::cleanup() {
-    if [[ -d "${BACKUP}/vendor" ]]; then
-        kube::log::error "${BACKUP}/vendor exists, restoring it"
-        rm -rf vendor
-        mv "${BACKUP}/vendor" vendor
-    fi
-    if [[ -d "${BACKUP}/Godeps" ]]; then
-        kube::log::error "${BACKUP}/Godeps exists, restoring it"
-        rm -rf Godeps
-        mv "${BACKUP}/Godeps" Godeps
-    fi
-}
-kube::util::trap_add kube::godep_save::cleanup EXIT
-
-# Clear old state, but save it in case of error
-if [[ -d vendor ]]; then
-    mv vendor "${BACKUP}/vendor"
-fi
-if [[ -d Godeps ]]; then
-    mv Godeps "${BACKUP}/Godeps"
-fi
-
-## Workaround "httplex" was dropped from golang.org/x/net repo and the code
-## was moved to the "golang.org/x/net/http/httpguts" directory, we do not use
-## this directly, however many packages we vendor are still using the older
-## golang.org/x/net and we need to keep this until all those dependencies
-## are switched to newer golang.org/x/net.
-IGNORED_PACKAGES=(
-  "golang.org/x/net/lex/httplex"
-)
-REQUIRED_BINS=(
-  "golang.org/x/net/internal/nettest"
-  "golang.org/x/net/internal/socks"
-  "golang.org/x/net/internal/sockstest"
-)
-
-# Some things we want in godeps aren't code dependencies, so ./...
-# won't pick them up.
-REQUIRED_BINS+=(
-  "github.com/bazelbuild/bazel-gazelle/cmd/gazelle"
-  "github.com/bazelbuild/buildtools/buildozer"
-  "github.com/cespare/prettybench"
-  "github.com/client9/misspell/cmd/misspell"
-  "github.com/cloudflare/cfssl/cmd/cfssl"
-  "github.com/cloudflare/cfssl/cmd/cfssljson"
-  "github.com/jstemmer/go-junit-report"
-  "github.com/jteeuwen/go-bindata/go-bindata"
-  "github.com/onsi/ginkgo/ginkgo"
-  "golang.org/x/lint/golint"
-  "k8s.io/kube-openapi/cmd/openapi-gen"
-  "k8s.io/repo-infra/kazel"
-  "./..."
-)
-
-kube::log::status "Running godep save - this might take a while"
-# This uses $(pwd) rather than ${KUBE_ROOT} because KUBE_ROOT will be
-# realpath'ed, and godep barfs ("... is not using a known version control
-# system") on our staging dirs.
-GOPATH="${GOPATH}:$(pwd)/staging" "${KUBE_GODEP:?}" save -i "$(IFS=,; echo "${IGNORED_PACKAGES[*]}")" "${REQUIRED_BINS[@]}"
-
-# create a symlink in vendor directory pointing to the staging client. This
-# let other packages use the staging client as if it were vendored.
-for repo in staging/src/k8s.io/*; do
-  repo="${repo#staging/src/k8s.io/}"
-  if [ ! -e "vendor/k8s.io/${repo}" ]; then
-    ln -s "../../staging/src/k8s.io/${repo}" "vendor/k8s.io/${repo}"
-  fi
-done
-
-# Workaround broken symlink in docker repo because godep copies the link, but
-# not the target
-rm -rf vendor/github.com/docker/docker/project/
-
-kube::log::status "Updating BUILD files"
-# Assume that anything imported through godep doesn't need Bazel to build.
-# Prune out any Bazel build files, since these can break the build due to
-# missing dependencies that aren't included by godep.
-find vendor/ -type f \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) \
-  -exec rm -f {} \;
-hack/update-bazel.sh >/dev/null
-
-kube::log::status "Updating LICENSES file"
-hack/update-godep-licenses.sh >/dev/null
-
-kube::log::status "Creating OWNERS file"
-rm -f "Godeps/OWNERS" "vendor/OWNERS"
-cat <<__EOF__ > "Godeps/OWNERS"
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-- dep-approvers
-__EOF__
-cp "Godeps/OWNERS" "vendor/OWNERS"
-
-# Clean up
-rm -rf "${BACKUP}"

hack/jenkins/verify.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Sets up the environment (e.g. installing godep and etcd if necessary)
+# Sets up the environment (e.g. installing etcd if necessary)
 # and then runs all of the verification checks.
 
 set -o errexit

hack/lib/golang.sh

@@ -437,7 +437,7 @@ kube::golang::setup_env() {
   # Unset GOBIN in case it already exists in the current session.
   unset GOBIN
 
-  # This seems to matter to some tools (godep, ginkgo...)
+  # This seems to matter to some tools
   export GO15VENDOREXPERIMENT=1
 }
 

hack/lib/init.sh

@@ -22,6 +22,12 @@ set -o pipefail
 # https://github.com/kubernetes/kubernetes/issues/52255
 unset CDPATH
 
+# Until all GOPATH references are removed from all build scripts as well,
+# explicitly reset to auto mode to avoid picking up user-set GO111MODULE preferences.
+# As individual scripts (like hack/update-vendor.sh) make use of go modules,
+# they can explicitly set GO111MODULE=on
+export GO111MODULE=auto
+
 # The root of the build/dist directory
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../.." && pwd -P)"
 

hack/lib/util.sh

@@ -330,64 +330,6 @@ kube::util::git_upstream_remote_name() {
     head -n 1 | awk '{print $1}'
 }
 
-# Ensures the current directory is a git tree for doing things like restoring or
-# validating godeps
-kube::util::create-fake-git-tree() {
-  local -r target_dir=${1:-$(pwd)}
-
-  pushd "${target_dir}" >/dev/null || return 1
-    git init >/dev/null
-    git config --local user.email "nobody@k8s.io"
-    git config --local user.name "$0"
-    git add . >/dev/null
-    git commit -q -m "Snapshot" >/dev/null
-    if (( ${KUBE_VERBOSE:-5} >= 6 )); then
-      kube::log::status "${target_dir} is now a git tree."
-    fi
-  popd >/dev/null || return 1
-}
-
-# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist
-# and are checked out to the referenced rev.
-kube::util::godep_restored() {
-  local -r godeps_json=${1:-Godeps/Godeps.json}
-  local -r gopath=${2:-${GOPATH%:*}}
-
-  kube::util::require-jq
-
-  local root
-  local old_rev=""
-  while read -r path rev; do
-    rev="${rev//[\'\"]}" # remove quotes which are around revs sometimes
-
-    if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then
-      # avoid checking the same git/hg root again
-      continue
-    fi
-
-    root="${path}"
-    while [ "${root}" != "." ] && [ ! -d "${gopath}/src/${root}/.git" ] && [ ! -d "${gopath}/src/${root}/.hg" ]; do
-      root=$(dirname "${root}")
-    done
-    if [ "${root}" == "." ]; then
-      echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2
-      return 1
-    fi
-    local head
-    if [ -d "${gopath}/src/${root}/.git" ]; then
-      head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)"
-    else
-      head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')"
-    fi
-    if [ "${head}" != "${rev}" ]; then
-      echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2
-      return 1
-    fi
-    old_rev="${rev}"
-  done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}")
-  return 0
-}
-
 # Exits script if working directory is dirty. If it's run interactively in the terminal
 # the user can commit changes in a second terminal. This script will wait.
 kube::util::ensure_clean_working_dir() {
@@ -404,35 +346,6 @@ kube::util::ensure_clean_working_dir() {
   done 1>&2
 }
 
-# Ensure that none of the staging repos is checked out in the GOPATH because this
-# easily confused godep.
-kube::util::ensure_no_staging_repos_in_gopath() {
-  kube::util::ensure_single_dir_gopath
-  local error=0
-  for repo_file in "${KUBE_ROOT}"/staging/src/k8s.io/*; do
-    if [[ ! -d "${repo_file}" ]]; then
-      # not a directory or there were no files
-      continue;
-    fi
-    repo="$(basename "${repo_file}")"
-    if [ -e "${GOPATH}/src/k8s.io/${repo}" ]; then
-      echo "k8s.io/${repo} exists in GOPATH. Remove before running godep-save.sh." 1>&2
-      error=1
-    fi
-  done
-  if [ "${error}" = "1" ]; then
-    exit 1
-  fi
-}
-
-# Checks that the GOPATH is simple, i.e. consists only of one directory, not multiple.
-kube::util::ensure_single_dir_gopath() {
-  if [[ "${GOPATH}" == *:* ]]; then
-    echo "GOPATH must consist of a single directory." 1>&2
-    exit 1
-  fi
-}
-
 # Find the base commit using:
 # $PULL_BASE_SHA if set (from Prow)
 # current ref from the remote upstream branch

hack/make-rules/update.sh

@@ -47,8 +47,6 @@ if ! ${ALL} ; then
 	echo "Running in short-circuit mode; run with FORCE_ALL=true to force all scripts to run."
 fi
 
-"${KUBE_ROOT}/hack/godep-restore.sh" ${V}
-
 BASH_TARGETS="
 	update-generated-protobuf
 	update-codegen
@@ -57,7 +55,6 @@ BASH_TARGETS="
 	update-generated-docs
 	update-generated-swagger-docs
 	update-openapi-spec
-	update-staging-godeps
 	update-bazel"
 
 for t in ${BASH_TARGETS}; do

hack/make-rules/verify.sh

@@ -45,12 +45,13 @@ if [[ ${EXCLUDE_TYPECHECK:-} =~ ^[yY]$ ]]; then
 fi
 
 
-# Exclude godep checks in certain cases, if they're running in a separate job.
+# Exclude vendor checks in certain cases, if they're running in a separate job.
 if [[ ${EXCLUDE_GODEP:-} =~ ^[yY]$ ]]; then
   EXCLUDED_PATTERNS+=(
     "verify-godeps.sh"             # runs in separate godeps job
-    "verify-staging-godeps.sh"     # runs in separate godeps job
-    "verify-godep-licenses.sh"     # runs in separate godeps job
+    "verify-godeps-licenses.sh"    # runs in separate godeps job
+    "verify-vendor.sh"             # runs in separate godeps job
+    "verify-vendor-licenses.sh"    # runs in separate godeps job
     )
 fi
 
@@ -63,12 +64,12 @@ fi
 
 # Only run whitelisted fast checks in quick mode.
 # These run in <10s each on enisoc's workstation, assuming that
-# `make` and `hack/godep-restore.sh` had already been run.
+# `make` had already been run.
 QUICK_PATTERNS+=(
   "verify-api-groups.sh"
   "verify-bazel.sh"
   "verify-boilerplate.sh"
-  "verify-godep-licenses.sh"
+  "verify-vendor-licenses.sh"
   "verify-gofmt.sh"
   "verify-imports.sh"
   "verify-pkg-names.sh"

hack/pin-dependency.sh

+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Usage:
+#   hack/pin-dependency.sh $MODULE $SHA-OR-TAG
+#
+# Example:
+#   hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7
+
+# Explicitly opt into go modules, even though we're inside a GOPATH directory
+export GO111MODULE=on
+# Explicitly clear GOPATH, to ensure nothing this script calls makes use of that path info
+export GOPATH=
+# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+export GOFLAGS=
+# Detect problematic GOPROXY settings that prevent lookup of dependencies
+if [[ "${GOPROXY:-}" == "off" ]]; then
+  kube::log::error "Cannot run hack/pin-dependency.sh with \$GOPROXY=off"
+  exit 1
+fi
+
+kube::golang::verify_go_version
+kube::util::require-jq
+
+dep="${1:-}"
+sha="${2:-}"
+if [[ -z "${dep}" || -z "${sha}" ]]; then
+  echo "Usage:"
+  echo "  hack/pin-dependency.sh \$MODULE \$SHA-OR-TAG"
+  echo ""
+  echo "Example:"
+  echo "  hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7"
+  echo ""
+  exit 1
+fi
+
+# Add the require directive
+echo "Running: go get ${dep}@${sha}"
+go get "${dep}@${sha}"
+
+# Find the resolved version
+rev=$(go mod edit -json | jq -r '.Require[] | select(.Path == "github.com/docker/docker") | .Version')
+echo "Resolved to ${dep}@${rev}"
+
+# Add the replace directive
+echo "Running: go mod edit -replace ${dep}=${dep}@${rev}"
+go mod edit -replace "${dep}=${dep}@${rev}"
+
+echo ""
+echo "Run hack/update-vendor.sh to rebuild the vendor directory"

hack/update-bazel.sh

@@ -25,9 +25,11 @@ export GOBIN="${KUBE_OUTPUT_BINPATH}"
 PATH="${GOBIN}:${PATH}"
 
 # Install tools we need, but only from vendor/...
-go install k8s.io/kubernetes/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle
-go install k8s.io/kubernetes/vendor/github.com/bazelbuild/buildtools/buildozer
-go install k8s.io/kubernetes/vendor/k8s.io/repo-infra/kazel
+pushd "${KUBE_ROOT}/vendor"
+  go install ./github.com/bazelbuild/bazel-gazelle/cmd/gazelle
+  go install ./github.com/bazelbuild/buildtools/buildozer
+  go install ./k8s.io/repo-infra/kazel
+popd
 
 # Find all of the staging repos.
 while IFS='' read -r repo; do staging_repos+=("${repo}"); done <\

hack/update-staging-godeps-dockerized.sh

-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# updates the godeps.json file in the staging folders to allow clean vendoring
-# based on kubernetes levels.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-# Sets default values
-DRY_RUN=false
-FAIL_ON_DIFF=false
-GODEP_OPTS=""
-
-# Controls verbosity of the script output and logging.
-KUBE_VERBOSE="${KUBE_VERBOSE:-5}"
-if (( ${KUBE_VERBOSE} >= 6 )); then
-  GODEP_OPTS+=("-v")
-fi
-
-while getopts ":df" opt; do
-  case ${opt} in
-    d) # do not godep-restore into a temporary directory, but use the existing GOPATH
-      DRY_RUN=true
-      ;;
-    f) # fail if something in the Godeps.json files changed
-      FAIL_ON_DIFF=true
-      ;;
-    \?)
-      echo "Invalid option: -${OPTARG}" >&2
-      exit 1
-      ;;
-  esac
-done
-
-# Confirm this is running inside a docker container, as this will modify the git tree (unsafe to run outside of container)
-kube::util::ensure_dockerized
-kube::golang::setup_env
-# Ensure we have a simple gopath so that we can modify it, and that no staging repos have made their way in
-kube::util::ensure_single_dir_gopath
-kube::util::ensure_no_staging_repos_in_gopath
-# Confirm we have the right godep version installed
-kube::util::ensure_godep_version
-# Create a fake git repo the root of the repo to prevent godeps from complaining
-kube::util::create-fake-git-tree "${KUBE_ROOT}"
-
-"${KUBE_ROOT}/hack/godep-restore.sh"
-
-kube::util::ensure-temp-dir
-TMP_GOPATH="${KUBE_TEMP}/go"
-
-function updateGodepManifest() {
-  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
-    kube::log::status "Updating godeps for k8s.io/${repo}"
-    rm -rf Godeps # remove the current Godeps.json so we always rebuild it
-    GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" ${KUBE_GODEP:?} save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/  /'
-
-    # Rewriting Godeps.json to cross-out commits that don't really exist because we haven't pushed the prereqs yet
-    local repo
-    for repo in $(ls -1 ${KUBE_ROOT}/staging/src/k8s.io); do
-      # remove staging prefix
-      jq '.Deps |= map(.ImportPath |= ltrimstr("k8s.io/kubernetes/staging/src/"))' Godeps/Godeps.json |
-
-        # x-out staging repo revisions. They will only be known when the publisher bot has created the final export.
-        # We keep the staging dependencies in here though to give the publisher bot a way to detect when the staging
-        # dependencies changed. If they have changed, the bot will run a complete godep restore+save. If they didn't
-        # it will avoid that step, which takes quite some time.
-        jq '.Deps |= map((select(.ImportPath | (startswith("k8s.io/'${repo}'/") or . == "k8s.io/'${repo}'")) | .Rev |= "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") // .)' |
-
-        # remove comments
-        jq 'del(.Deps[].Comment)' |
-
-        # format with tabs
-        unexpand --first-only --tabs=2 > Godeps/Godeps.json.out
-
-      mv Godeps/Godeps.json.out Godeps/Godeps.json
-    done
-
-    # commit so that following repos do not see this repo as dirty
-    git add vendor >/dev/null
-    git commit -a -m "Updated Godeps.json" >/dev/null
-  popd >/dev/null
-}
-
-function diffGodepManifest() {
-  local ret=0
-  diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" || ret=$?
-  if [[ "${ret}" != "0" && "${FAIL_ON_DIFF}" == true ]]; then
-    exit ${ret}
-  fi
-}
-
-# move into staging and save the dependencies for everything in order
-mkdir -p "${TMP_GOPATH}/src/k8s.io"
-for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
-  cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
-
-  # Create a fake git tree for the staging repo to prevent godeps from complaining
-  kube::util::create-fake-git-tree "${TMP_GOPATH}/src/k8s.io/${repo}"
-
-  updateGodepManifest
-  diffGodepManifest
-
-  if [ "${DRY_RUN}" != true ]; then
-    cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json"
-    cp "${KUBE_ROOT}/Godeps/OWNERS" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/OWNERS"
-    # Assume Godeps.json is not updated, as the working tree needs to be clean
-    # Without this, the script would pause after each staging repo to prompt the
-    # user to commit all changes (difficult inside a container). It's safe to
-    # ignore this file, as we know it's going to be changing, and we don't copy
-    # the git tree back out from the container.
-    git update-index --assume-unchanged "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json"
-  fi
-done

hack/update-staging-godeps.sh

-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-# Ensure you have a clean working directory before starting
-kube::util::ensure_clean_working_dir
-
-# NOTE: All output from this script needs to be copied back to the calling
-# source tree.  This is managed in kube::build::copy_output in build/common.sh.
-# If the output set is changed update that function.
-
-${KUBE_ROOT}/build/run.sh hack/update-staging-godeps-dockerized.sh "$@"
-
-# ex: ts=2 sw=2 et filetype=sh

hack/update-godep-licenses.sh → hack/update-vendor-licenses.sh

@@ -21,9 +21,9 @@
 #
 #    --create-missing will write the files that only exist upstream, locally.
 #    This option is mostly used for testing as we cannot check-in any of the
-#    additionally created files into the godep auto-generated tree.
+#    additionally created files into the vendor auto-generated tree.
 #
-#    Run every time a license file is added/modified within /Godeps to
+#    Run every time a license file is added/modified within /vendor to
 #    update /Godeps/LICENSES
 
 set -o errexit
@@ -129,6 +129,8 @@ process_content () {
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+export GO111MODULE=on
+
 # Check bash version
 if ((${BASH_VERSINFO[0]}<4)); then
   echo
@@ -147,7 +149,7 @@ fi
 LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}"
 cd "${LICENSE_ROOT}"
 
-GODEPS_LICENSE_FILE="Godeps/LICENSES"
+VENDOR_LICENSE_FILE="Godeps/LICENSES"
 TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$"
 DEPS_DIR="vendor"
 declare -Ag CONTENT
@@ -163,8 +165,17 @@ echo "= LICENSE $(cat "${LICENSE_ROOT}/LICENSE" | md5sum | awk '{print $1}')"
 echo "================================================================================"
 ) > ${TMP_LICENSE_FILE}
 
-# Loop through every package in Godeps.json
-for PACKAGE in $(jq -r ".Deps[].ImportPath" < Godeps/Godeps.json | sort -f); do
+# Loop through every vendored package
+for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
+  if [[ -e "staging/src/${PACKAGE}" ]]; then
+    echo "$PACKAGE is a staging package, skipping" > /dev/stderr
+    continue
+  fi
+  if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then
+    echo "$PACKAGE doesn't exist in vendor, skipping" > /dev/stderr
+    continue
+  fi
+
   process_content "${PACKAGE}" LICENSE
   process_content "${PACKAGE}" COPYRIGHT
   process_content "${PACKAGE}" COPYING
@@ -204,4 +215,4 @@ __EOF__
   echo
 done >> ${TMP_LICENSE_FILE}
 
-cat ${TMP_LICENSE_FILE} > ${GODEPS_LICENSE_FILE}
+cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE}

hack/verify-godep-licenses.sh

@@ -19,37 +19,5 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
-if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
-  ! kube::util::has_changes "${branch}" 'Godeps/' && \
-  ! kube::util::has_changes "${branch}" 'vendor/'; then
-  exit 0
-fi
-
-# create a nice clean place to put our new godeps
-# must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount
-# to work with docker-machine on macs
-mkdir -p "${KUBE_ROOT}/_tmp"
-_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-godep-licenses.XXXXXX")"
-#echo "Created workspace: ${_tmpdir}"
-function cleanup {
-  #echo "Removing workspace: ${_tmpdir}"
-  rm -rf "${_tmpdir}"
-}
-trap cleanup EXIT
-
-cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
-ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}"
-ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}"
-
-# Update Godep Licenses
-LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-godep-licenses.sh"
-
-# Compare Godep Licenses
-if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then
-  echo "Your godep licenses file is out of date. Run hack/update-godep-licenses.sh and commit the results." >&2
-  echo "${_out}" >&2
-  exit 1
-fi
+echo "Deprecated, use verify-vendor-licenses.sh instead"
+"${KUBE_ROOT}/hack/verify-vendor-licenses.sh" "$@"

hack/verify-godeps.sh

@@ -19,109 +19,5 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
-if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
-  ! kube::util::has_changes "${branch}" 'Godeps/' && \
-  ! kube::util::has_changes "${branch}" 'vendor/' && \
-  ! kube::util::has_changes "${branch}" 'hack/lib/' && \
-  ! kube::util::has_changes "${branch}" 'hack/.*godep'; then
-  exit 0
-fi
-
-# Ensure we have the right godep version available
-kube::util::ensure_godep_version
-
-if [[ -z ${TMP_GOPATH:-} ]]; then
-  # Create a nice clean place to put our new godeps
-  _tmpdir="$(kube::realpath "$(mktemp -d -t gopath.XXXXXX)")"
-else
-  # reuse what we might have saved previously
-  _tmpdir="${TMP_GOPATH}"
-fi
-
-if [[ -z ${KEEP_TMP:-} ]]; then
-    KEEP_TMP=false
-fi
-function cleanup {
-  if [ "${KEEP_TMP}" == "true" ]; then
-    echo "Leaving ${_tmpdir} for you to examine or copy. Please delete it manually when finished. (rm -rf ${_tmpdir})"
-  else
-    echo "Removing ${_tmpdir}"
-    rm -rf "${_tmpdir}"
-  fi
-}
-trap cleanup EXIT
-
-# Copy the contents of the kube directory into the nice clean place
-_kubetmp="${_tmpdir}/src/k8s.io"
-mkdir -p "${_kubetmp}"
-# should create ${_kubectmp}/kubernetes
-git archive --format=tar --prefix=kubernetes/ "$(git write-tree)" | (cd "${_kubetmp}" && tar xf -)
-_kubetmp="${_kubetmp}/kubernetes"
-
-# Do all our work in the new GOPATH
-export GOPATH="${_tmpdir}"
-export PATH="${GOPATH}/bin:${PATH}"
-
-pushd "${_kubetmp}" > /dev/null 2>&1
-  # Restore the Godeps into our temp directory
-  hack/godep-restore.sh
-
-  # Destroy deps in the copy of the kube tree
-  rm -rf ./Godeps ./vendor
-
-  # For some reason the kube tree needs to be a git repo for the godep tool to
-  # run. Doesn't make sense.
-  git init > /dev/null 2>&1
-
-  # Recreate the Godeps using the nice clean set we just downloaded
-  hack/godep-save.sh
-popd > /dev/null 2>&1
-
-ret=0
-
-pushd "${KUBE_ROOT}" > /dev/null 2>&1
-  # Test for diffs
-  if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' Godeps/Godeps.json "${_kubetmp}/Godeps/Godeps.json")"; then
-    echo "Your Godeps.json is different:" >&2
-    echo "${_out}" >&2
-    echo "Godeps Verify failed." >&2
-    echo "${_out}" > godepdiff.patch
-    echo "If you're seeing this locally, run the below command to fix your Godeps.json:" >&2
-    echo "patch -p0 < godepdiff.patch" >&2
-    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2
-    echo "(The patch file should also be exported as a build artifact if run through CI)" >&2
-    KEEP_TMP=true
-    if [[ -f godepdiff.patch && -d "${ARTIFACTS:-}" ]]; then
-      echo "Copying patch to artifacts.."
-      cp godepdiff.patch "${ARTIFACTS:-}/"
-    fi
-    ret=1
-  fi
-
-  if ! _out="$(diff -Naupr -x "BUILD" -x "AUTHORS*" -x "CONTRIBUTORS*" vendor "${_kubetmp}/vendor")"; then
-    echo "Your vendored results are different:" >&2
-    echo "${_out}" >&2
-    echo "Godeps Verify failed." >&2
-    echo "${_out}" > vendordiff.patch
-    echo "If you're seeing this locally, run the below command to fix your directories:" >&2
-    echo "patch -p0 < vendordiff.patch" >&2
-    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2
-    echo "(The patch file should also be exported as a build artifact if run through CI)" >&2
-    KEEP_TMP=true
-    if [[ -f vendordiff.patch && -d "${ARTIFACTS:-}" ]]; then
-      echo "Copying patch to artifacts.."
-      cp vendordiff.patch "${ARTIFACTS:-}/"
-    fi
-    ret=1
-  fi
-popd > /dev/null 2>&1
-
-if [[ ${ret} -gt 0 ]]; then
-  exit ${ret}
-fi
-
-echo "Godeps Verified."
-# ex: ts=2 sw=2 et filetype=sh
+echo "Deprecated, use verify-vendor.sh instead"
+"${KUBE_ROOT}/hack/verify-vendor.sh" "$@"

hack/verify-no-vendor-cycles.sh

@@ -20,6 +20,8 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 
+export GO111MODULE=auto
+
 staging_repos=($(ls "${KUBE_ROOT}/staging/src/k8s.io/"))
 staging_repos_pattern=$(IFS="|"; echo "${staging_repos[*]}")
 

hack/verify-publishing-bot.py

@@ -23,18 +23,23 @@ import json
 import yaml
 
 
-def get_godeps_dependencies(godeps):
+def get_godeps_dependencies(rootdir, components):
     all_dependencies = {}
-    for arg in godeps:
-        with open(arg) as f:
-            data = json.load(f)
-            dependencies = []
-            for dep in data["Deps"]:
-                if dep['Rev'].startswith('xxxx') and dep['ImportPath'].startswith("k8s.io"):
-                    package = "/".join(dep['ImportPath'].split('/')[0:2])
-                    if package not in dependencies:
-                        dependencies.append(package.split('/')[1])
-            all_dependencies[(data["ImportPath"].split('/')[1])] = dependencies
+    for component in components:
+        with open(os.path.join(rootdir, component, "go.mod")) as f:
+            print(component + " dependencies")
+            all_dependencies[component] = []
+            lines = list(set(f))
+            lines.sort()
+            for line in lines:
+                for dep in components:
+                    if dep == component:
+                        continue
+                    if ("k8s.io/" + dep + " v0.0.0") not in line:
+                        continue
+                    print("\t"+dep)
+                    if dep not in all_dependencies[component]:
+                        all_dependencies[component].append(dep)
     return all_dependencies
 
 
@@ -48,12 +53,12 @@ def main():
     rootdir = os.path.dirname(__file__) + "/../"
     rootdir = os.path.abspath(rootdir)
 
-    godeps = []
-    for root, dirnames, filenames in os.walk(rootdir + '/staging/'):
-        for filename in fnmatch.filter(filenames, 'Godeps.json'):
-            godeps.append(os.path.join(root, filename))
+    components = []
+    for component in os.listdir(rootdir + '/staging/src/k8s.io/'):
+        components.append(component)
+    components.sort()
 
-    godep_dependencies = get_godeps_dependencies(godeps)
+    godep_dependencies = get_godeps_dependencies(rootdir + '/staging/src/k8s.io/', components)
     rules_dependencies = get_rules_dependencies(rootdir + "/staging/publishing/rules.yaml")
 
     processed_repos = []

hack/verify-staging-godeps.sh → hack/verify-vendor-licenses.sh

 #!/usr/bin/env bash
 
-# Copyright 2017 The Kubernetes Authors.
+# Copyright 2015 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,13 +23,36 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
 if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
-  ! kube::util::has_changes "${branch}" 'staging/' && \
-  ! kube::util::has_changes "${branch}" 'build/' && \
   ! kube::util::has_changes "${branch}" 'Godeps/' && \
-  ! kube::util::has_changes "${branch}" 'vendor/' && \
-  ! kube::util::has_changes "${branch}" 'hack/lib/' && \
-  ! kube::util::has_changes "${branch}" 'hack/.*godep'; then
+  ! kube::util::has_changes "${branch}" 'go.mod' && \
+  ! kube::util::has_changes "${branch}" 'go.sum' && \
+  ! kube::util::has_changes "${branch}" 'vendor/'; then
   exit 0
 fi
 
-KUBE_VERBOSE="${KUBE_VERBOSE:-3}" KUBE_RUN_COPY_OUTPUT=N "${KUBE_ROOT}/hack/update-staging-godeps.sh" -d -f "$@"
+# create a nice clean place to put our new licenses
+# must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount
+# to work with docker-machine on macs
+mkdir -p "${KUBE_ROOT}/_tmp"
+_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-vendor-licenses.XXXXXX")"
+#echo "Created workspace: ${_tmpdir}"
+function cleanup {
+  #echo "Removing workspace: ${_tmpdir}"
+  rm -rf "${_tmpdir}"
+}
+trap cleanup EXIT
+
+cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
+ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}"
+ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}"
+ln -s "${KUBE_ROOT}/staging" "${_tmpdir}"
+
+# Update vendor Licenses
+LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"
+
+# Compare vendor Licenses
+if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then
+  echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2
+  echo "${_out}" >&2
+  exit 1
+fi

hack/verify-vendor.sh

+#!/usr/bin/env bash
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
+if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
+  ! kube::util::has_changes "${branch}" 'Godeps/' && \
+  ! kube::util::has_changes "${branch}" 'go.mod' && \
+  ! kube::util::has_changes "${branch}" 'go.sum' && \
+  ! kube::util::has_changes "${branch}" 'vendor/' && \
+  ! kube::util::has_changes "${branch}" 'staging/' && \
+  ! kube::util::has_changes "${branch}" 'hack/lib/' && \
+  ! kube::util::has_changes "${branch}" 'hack/.*vendor'; then
+  exit 0
+fi
+
+if [[ -z ${TMP_GOPATH:-} ]]; then
+  # Create a nice clean place to put our new vendor
+  _tmpdir="$(kube::realpath "$(mktemp -d -t verifyvendor.XXXXXX)")"
+else
+  # reuse what we might have saved previously
+  _tmpdir="${TMP_GOPATH}"
+fi
+
+if [[ -z ${KEEP_TMP:-} ]]; then
+    KEEP_TMP=false
+fi
+function cleanup {
+  # make go module dirs writeable
+  chmod -R +w "${_tmpdir}"
+  if [ "${KEEP_TMP}" == "true" ]; then
+    echo "Leaving ${_tmpdir} for you to examine or copy. Please delete it manually when finished. (rm -rf ${_tmpdir})"
+  else
+    echo "Removing ${_tmpdir}"
+    rm -rf "${_tmpdir}"
+  fi
+}
+trap cleanup EXIT
+
+# Copy the contents of the kube directory into the nice clean place (which is NOT shaped like a GOPATH)
+_kubetmp="${_tmpdir}"
+mkdir -p "${_kubetmp}"
+# should create ${_kubectmp}/kubernetes
+git archive --format=tar --prefix=kubernetes/ "$(git write-tree)" | (cd "${_kubetmp}" && tar xf -)
+_kubetmp="${_kubetmp}/kubernetes"
+
+# Do all our work with an unset GOPATH
+export GOPATH=
+
+pushd "${_kubetmp}" > /dev/null 2>&1
+  # Destroy deps in the copy of the kube tree
+  rm -rf ./Godeps/LICENSES ./vendor
+
+  # Recreate the vendor tree using the nice clean set we just downloaded
+  hack/update-vendor.sh
+popd > /dev/null 2>&1
+
+ret=0
+
+pushd "${KUBE_ROOT}" > /dev/null 2>&1
+  # Test for diffs
+  if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' go.mod "${_kubetmp}/go.mod")"; then
+    echo "Your go.mod file is different:" >&2
+    echo "${_out}" >&2
+    echo "Vendor Verify failed." >&2
+    echo "If you're seeing this locally, run the below command to fix your go.mod:" >&2
+    echo "hack/update-vendor.sh" >&2
+    ret=1
+  fi
+
+  if ! _out="$(diff -Naupr -x "BUILD" -x "AUTHORS*" -x "CONTRIBUTORS*" vendor "${_kubetmp}/vendor")"; then
+    echo "Your vendored results are different:" >&2
+    echo "${_out}" >&2
+    echo "Vendor Verify failed." >&2
+    echo "${_out}" > vendordiff.patch
+    echo "If you're seeing this locally, run the below command to fix your directories:" >&2
+    echo "hack/update-vendor.sh" >&2
+    ret=1
+  fi
+popd > /dev/null 2>&1
+
+if [[ ${ret} -gt 0 ]]; then
+  exit ${ret}
+fi
+
+echo "Vendor Verified."
+# ex: ts=2 sw=2 et filetype=sh