Skip to content

K
k3s
Commit dd9219f3 authored by Dr. Stefan Schimanski's avatar Dr. Stefan Schimanski
Browse files
Options

update-staging-{client-go,godeps}.sh: no godep-restore, pin godep, check workdir

parent 6ba69ed9
Show whitespace changes
Inline Side-by-side
Showing with 175 additions and 177 deletions
hack/lib/util.sh
View file @ dd9219f3
...@@ -411,6 +411,76 @@ kube::util::git_upstream_remote_name() { ...@@ -411,6 +411,76 @@ kube::util::git_upstream_remote_name() {
head -n 1 | awk '{print $1}' head -n 1 | awk '{print $1}'
} }
# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist
# and are checked out to the referenced rev.
kube::util::godep_restored() {
local -r godeps_json=${1:-Godeps/Godeps.json}
local -r gopath=${2:-${GOPATH%:*}}
if ! which jq &>/dev/null; then
echo "jq not found. Please install." 1>&2
return 1
fi
local root
local old_rev=""
while read path rev; do
rev=$(echo "${rev}" | sed "s/['\"]//g") # remove quotes which are around revs sometimes
if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then
# avoid checking the same git/hg root again
continue
fi
root="${path}"
while [ "${root}" != "." -a ! -d "${gopath}/src/${root}/.git" -a ! -d "${gopath}/src/${root}/.hg" ]; do
root=$(dirname "${root}")
done
if [ "${root}" == "." ]; then
echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2
return 1
fi
local head
if [ -d "${gopath}/src/${root}/.git" ]; then
head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)"
else
head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')"
fi
if [ "${head}" != "${rev}" ]; then
echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2
return 1
fi
old_rev="${rev}"
done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}")
return 0
}
# Exits script if working directory is dirty.
kube::util::ensure_clean_working_dir() {
if ! git diff --exit-code; then
echo -e "\nUnexpected dirty working directory."
exit 1
fi
}
# Ensure that the given godep version is installed and in the path
kube::util::ensure_godep_version() {
if [[ "$(godep version)" == *"godep ${1}"* ]]; then
return
fi
kube::util::ensure-temp-dir
mkdir -p "${KUBE_TEMP}/go/src"
GOPATH="${KUBE_TEMP}/go" go get -u github.com/tools/godep 2>/dev/null
pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null
git checkout "${1:-v74}"
GOPATH="${KUBE_TEMP}/go" go install .
popd >/dev/null
PATH="${KUBE_TEMP}/go/bin:${PATH}"
hash -r # force bash to clear PATH cache
godep version
}
# Checks whether there are any files matching pattern $2 changed between the # Checks whether there are any files matching pattern $2 changed between the
# current branch and upstream branch named by $1. # current branch and upstream branch named by $1.
# Returns 1 (false) if there are no changes, 0 (true) if there are changes # Returns 1 (false) if there are no changes, 0 (true) if there are changes
......
hack/update-staging-client-go.sh
View file @ dd9219f3
...@@ -19,4 +19,18 @@ set -o nounset ...@@ -19,4 +19,18 @@ set -o nounset
set -o pipefail set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
"${KUBE_ROOT}"/staging/copy.sh source "${KUBE_ROOT}/hack/lib/util.sh"
kube::util::ensure_clean_working_dir
kube::util::ensure_godep_version v74
cd ${KUBE_ROOT}
echo "Checking whether godeps are restored"
if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then
echo -e '\nRun 'godep restore' to download dependencies.' 1>&2
exit 1
fi
echo "Running staging/copy.sh"
staging/copy.sh -u "$@" 2>&1 | sed 's/^/ /'
hack/update-staging-godeps.sh
View file @ dd9219f3
...@@ -23,102 +23,84 @@ set -o errexit ...@@ -23,102 +23,84 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
V=""
DRY_RUN=false
FAIL_ON_DIFF=false
while getopts ":vdf" opt; do
case $opt in
v) # increase verbosity
V="-v"
;;
d) # do not godep-restore into a temporary directory, but use the existing GOPATH
DRY_RUN=true
;;
f) # fail if something in the Godeps.json files changed
FAIL_ON_DIFF=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
esac
done
readonly V IN_PLACE
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh" source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
kube::golang::setup_env echo "Checking whether godeps are restored"
if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then
ORIGINAL_GOPATH="${GOPATH}" echo -e '\nRun 'godep restore' to download dependencies.' 1>&2
exit 1
godepBinDir=${TMPDIR:-/tmp/}/kube-godep-bin
mkdir -p "${godepBinDir}"
godep=${godepBinDir}/bin/godep
pushd "${godepBinDir}" 2>&1 > /dev/null
# Build the godep tool
GOPATH="${godepBinDir}"
rm -rf *
go get -u github.com/tools/godep 2>/dev/null
export GODEP="${GOPATH}/bin/godep"
pin-godep() {
pushd "${GOPATH}/src/github.com/tools/godep" > /dev/null
git checkout "$1"
"${GODEP}" go install
popd > /dev/null
}
# Use to following if we ever need to pin godep to a specific version again
pin-godep 'v74'
"${godep}" version
popd 2>&1 > /dev/null
# keep the godep restore path reasonably stable to avoid unnecessary restores
godepRestoreDir=${TMPDIR:-/tmp/}/kube-godep-restore
TARGET_DIR=${TARGET_DIR:-${KUBE_ROOT}/staging}
echo "working in ${TARGET_DIR}"
SKIP_RESTORE=${SKIP_RESTORE:-}
if [ "${SKIP_RESTORE}" != "true" ]; then
echo "starting godep restore"
mkdir -p "${godepRestoreDir}"
# add the vendor folder so that we don't redownload things during restore
GOPATH="${godepRestoreDir}:${KUBE_ROOT}/staging:${ORIGINAL_GOPATH}"
# restore from kubernetes godeps to ensure we get the correct levels
# you get errors about the staging repos not using a known version control system
${godep} restore > ${godepRestoreDir}/godep-restore.log
echo "finished godep restore"
fi fi
echo "forcing fake godep folders to match the current state of master in tmp" kube::util::ensure_clean_working_dir
for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do kube::util::ensure-temp-dir
echo " creating ${stagingRepo}" kube::util::ensure_godep_version v74
rm -rf ${godepRestoreDir}/src/k8s.io/${stagingRepo}
cp -a ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo} ${godepRestoreDir}/src/k8s.io TMP_GOPATH="${KUBE_TEMP}/go"
# we need a git commit in the godep folder, otherwise godep won't save
pushd ${godepRestoreDir}/src/k8s.io/${stagingRepo}
git init > /dev/null
# we need this so later commands work, but nothing should ever actually include these commits
# these are local only, not global
git config user.email "you@example.com"
git config user.name "Your Name"
git add . > /dev/null
git commit -qm "fake commit"
popd
done
function updateGodepManifest() { function updateGodepManifest() {
local repo=${1} local repo="${1}"
pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
echo "starting ${repo} save" echo "Updating godeps for k8s.io/${repo}"
mkdir -p ${TARGET_DIR}/src/k8s.io rm -rf Godeps # remove the current Godeps.json so we always rebuild it
# if target_dir isn't the same as source dir, you need copy GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/ /'
test "${KUBE_ROOT}/staging" = "${TARGET_DIR}" || cp -a ${KUBE_ROOT}/staging/src/${repo} ${TARGET_DIR}/src/k8s.io
# remove the current Godeps.json so we always rebuild it echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet"
rm -rf ${TARGET_DIR}/src/${repo}/Godeps go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --client-go-import-path="k8s.io/${repo}"
GOPATH="${godepRestoreDir}:${TARGET_DIR}"
pushd ${TARGET_DIR}/src/${repo} # commit so that following repos do not see this repo as dirty
${godep} save ./... git add vendor >/dev/null
git commit -a -m "Updated Godeps.json" >/dev/null
# now remove all the go files. We'll re-run a restore, go get, godep save cycle in the sync scripts popd >/dev/null
# to get the commits for other staging k8s.io repos anyway, so we don't need the added files
rm -rf vendor
echo "rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet"
GOPATH="${ORIGINAL_GOPATH}"
go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TARGET_DIR}/src/${repo}/Godeps/Godeps.json" --client-go-import-path="${repo}"
popd
echo "finished ${repo} save"
} }
# move into staging and save the dependencies for everything in order # move into staging and save the dependencies for everything in order
for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do mkdir -p "${TMP_GOPATH}/src/k8s.io"
for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
# we have to skip client-go because it does unusual manipulation of its godeps # we have to skip client-go because it does unusual manipulation of its godeps
if [ "${stagingRepo}" == "client-go" ]; then if [ "${repo}" == "client-go" ]; then
continue continue
fi fi
updateGodepManifest "k8s.io/${stagingRepo}" cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
git init >/dev/null
git config --local user.email "nobody@k8s.io"
git config --local user.name "$0"
git add . >/dev/null
git commit -q -m "Snapshot" >/dev/null
popd >/dev/null
updateGodepManifest "${repo}"
if [ "${FAIL_ON_DIFF}" == true ]; then
diff --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json"
fi
if [ "${DRY_RUN}" != true ]; then
cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps"
fi
done done
hack/verify-staging-client-go.sh
View file @ dd9219f3
...@@ -18,65 +18,12 @@ set -o errexit ...@@ -18,65 +18,12 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
V=""
while getopts ":v" opt; do
case $opt in
v) # increase verbosity
V="-v"
;;
\?)
echo "Invalid option: -$OPTARG" >&2
;;
esac
done
readonly V
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
cd ${KUBE_ROOT} cd ${KUBE_ROOT}
if ! git diff --exit-code; then
echo "ERROR: $0 cannot be run on a dirty working directory."
exit 1
fi
# Smoke test client-go examples # Smoke test client-go examples
go install ./staging/src/k8s.io/client-go/examples/... echo "Smoke testing client-go examples"
go install ./staging/src/k8s.io/client-go/examples/... 2>&1 | sed 's/^/ /'
# Create a temporary GOPATH for apimachinery and client-go, copy the current HEAD into each and turn them
# into a git repo. Then we can run copy.sh with this additional GOPATH. The Godeps.json will
# have invalid git SHA1s, but it's good enough as a smoke test of copy.sh.
if [ "${USE_TEMP_DIR:-1}" = 1 ]; then
TEMP_STAGING_GOPATH=$(mktemp -d -t verify-staging-client-go.XXXXX)
echo "Creating the temporary staging GOPATH directory: ${TEMP_STAGING_GOPATH}"
cleanup() {
if [ "${KEEP_TEMP_DIR:-0}" != 1 ]; then
rm -rf "${TEMP_STAGING_GOPATH}"
fi
}
trap cleanup EXIT SIGINT
mkdir -p "${TEMP_STAGING_GOPATH}/src/k8s.io"
ln -s "${PWD}" "${TEMP_STAGING_GOPATH}/src/k8s.io"
else
TEMP_STAGING_GOPATH="${GOPATH}"
fi
for PACKAGE in apimachinery client-go; do
PACKAGE_PATH="${TEMP_STAGING_GOPATH}/src/k8s.io/${PACKAGE}"
echo "Creating a temporary ${PACKAGE} repo with a snapshot of HEAD"
rm -rf "${PACKAGE_PATH}"
mkdir -p "${PACKAGE_PATH}"
git archive --format=tar "HEAD:staging/src/k8s.io/${PACKAGE}" | tar -xf - -C "${PACKAGE_PATH}"
pushd "${PACKAGE_PATH}" >/dev/null
git init >/dev/null
git add *
git -c user.email="nobody@k8s.io" -c user.name="verify-staging-client-go.sh" commit -q -m "Snapshot"
popd >/dev/null
done
echo "Running godep restore"
pushd "${TEMP_STAGING_GOPATH}/src/k8s.io/kubernetes" >/dev/null
export GOPATH="${TEMP_STAGING_GOPATH}"
godep restore ${V} 2>&1 | sed 's/^/ /'
echo "Testing staging/copy.sh" # Run update-staging-client-go.sh in dry-run mode, copy nothing into the staging dir
staging/copy.sh -d 2>&1 | sed 's/^/ /' hack/update-staging-client-go.sh -d "$@"
popd
hack/verify-staging-godeps.sh
View file @ dd9219f3
...@@ -14,36 +14,9 @@ ...@@ -14,36 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# TODO this does not address client-go, since it takes a different approach to vendoring
# TODO client-go should probably be made consistent
set -o errexit set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh" ${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"
TARGET_DIR=$(mktemp -d "${TMPDIR:-/tmp/}$(basename 0).XXXXXXXXXXXX")
# Register function to be called on EXIT to remove folder.
function cleanup {
SKIP_CLEANUP=${SKIP_CLEANUP:-}
if [ "${SKIP_CLEANUP}" != "true" ]; then
rm -rf "${TARGET_DIR}"
fi
}
trap cleanup EXIT
TARGET_DIR=${TARGET_DIR} ${KUBE_ROOT}/hack/update-staging-godeps.sh
# check each staging repo to make sure its Godeps.json is correct
for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
# we have to skip client-go because it does unusual manipulation of its godeps
if [ "${stagingRepo}" == "client-go" ]; then
continue
fi
diff --ignore-matching-lines='^\s*\"Comment\"' ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo}/Godeps/Godeps.json ${TARGET_DIR}/src/k8s.io/${stagingRepo}/Godeps/Godeps.json
done
staging/copy.sh
View file @ dd9219f3
...@@ -18,9 +18,15 @@ set -o errexit ...@@ -18,9 +18,15 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
FAIL_ON_CHANGES=false FAIL_ON_CHANGES=false
DRY_RUN=false DRY_RUN=false
while getopts ":fd" opt; do RUN_FROM_UPDATE_SCRIPT=false
while getopts ":fdu" opt; do
case $opt in case $opt in
f) f)
FAIL_ON_CHANGES=true FAIL_ON_CHANGES=true
...@@ -28,25 +34,31 @@ while getopts ":fd" opt; do ...@@ -28,25 +34,31 @@ while getopts ":fd" opt; do
d) d)
DRY_RUN=true DRY_RUN=true
;; ;;
u)
RUN_FROM_UPDATE_SCRIPT=true
;;
\?) \?)
echo "Invalid option: -$OPTARG" >&2 echo "Invalid option: -$OPTARG" >&2
exit 1
;; ;;
esac esac
done done
readonly FAIL_ON_CHANGES DRY_RUN readonly FAIL_ON_CHANGES DRY_RUN
echo "**PLEASE** run \"godep restore\" before running this script" if [ "${RUN_FROM_UPDATE_SCRIPT}" != true ]; then
echo "Do not run this script directly, but via hack/update-staging-client-go.sh."
exit 1
fi
# PREREQUISITES: run `godep restore` in the main repo before calling this script. # PREREQUISITES: run `godep restore` in the main repo before calling this script.
CLIENTSET="clientset" CLIENTSET="clientset"
MAIN_REPO_FROM_SRC="k8s.io/kubernetes" MAIN_REPO_FROM_SRC="k8s.io/kubernetes"
MAIN_REPO="${GOPATH%:*}/src/${MAIN_REPO_FROM_SRC}" MAIN_REPO="$(cd "${KUBE_ROOT}"; pwd)" # absolute path
CLIENT_REPO_FROM_SRC="k8s.io/client-go" CLIENT_REPO_FROM_SRC="k8s.io/client-go"
CLIENT_REPO_TEMP_FROM_SRC="k8s.io/_tmp" CLIENT_REPO_TEMP_FROM_SRC="k8s.io/_tmp"
CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}" CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}"
CLIENT_REPO_TEMP="${MAIN_REPO}/staging/src/${CLIENT_REPO_TEMP_FROM_SRC}" CLIENT_REPO_TEMP="${MAIN_REPO}/staging/src/${CLIENT_REPO_TEMP_FROM_SRC}"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if LANG=C sed --help 2>&1 | grep -q GNU; then if LANG=C sed --help 2>&1 | grep -q GNU; then
SED="sed" SED="sed"
elif which gsed &>/dev/null; then elif which gsed &>/dev/null; then
...@@ -127,7 +139,7 @@ rm -rf "${CLIENT_REPO_TEMP}"/vendor/k8s.io/kubernetes ...@@ -127,7 +139,7 @@ rm -rf "${CLIENT_REPO_TEMP}"/vendor/k8s.io/kubernetes
mv "${CLIENT_REPO_TEMP}"/vendor "${CLIENT_REPO_TEMP}"/_vendor mv "${CLIENT_REPO_TEMP}"/vendor "${CLIENT_REPO_TEMP}"/_vendor
echo "rewriting Godeps.json" echo "rewriting Godeps.json"
go run "${DIR}/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}" go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}"
echo "rewriting imports" echo "rewriting imports"
grep -Rl "\"${MAIN_REPO_FROM_SRC}" "${CLIENT_REPO_TEMP}" | \ grep -Rl "\"${MAIN_REPO_FROM_SRC}" "${CLIENT_REPO_TEMP}" | \
...@@ -206,12 +218,12 @@ rm -rf "${CLIENT_REPO_TEMP}/staging" ...@@ -206,12 +218,12 @@ rm -rf "${CLIENT_REPO_TEMP}/staging"
if [ "${FAIL_ON_CHANGES}" = true ]; then if [ "${FAIL_ON_CHANGES}" = true ]; then
echo "running FAIL_ON_CHANGES" echo "running FAIL_ON_CHANGES"
ret=0 ret=0
if diff -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then if diff --ignore-matching-lines='^\s*\"Comment\"' -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then
echo "${CLIENT_REPO} up to date." echo "${CLIENT_REPO} up to date."
cleanup cleanup
exit 0 exit 0
else else
echo "${CLIENT_REPO} is out of date. Please run hack/update-client-go.sh" echo "${CLIENT_REPO} is out of date. Please run hack/update-staging-client-go.sh"
cleanup cleanup
exit 1 exit 1
fi fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment