Commit 654c522e authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #52477 from jamiehannaford/kubernetes-anywhere

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.. Support kubernetes-anywhere provider **What this PR does / why we need it**: Implements a new `kubernetes-anywhere` provider to allow upgrade testing in the e2e binary. This is the final step to allow https://github.com/kubernetes/test-infra/pull/4495 and https://github.com/kubernetes/kubernetes-anywhere/pull/450. **Which issue this PR fixes**: https://github.com/kubernetes/kubeadm/issues/311 **Special notes for your reviewer**: Some questions I had - Does the `--provider` flag specified [here](https://github.com/jamiehannaford/test-infra/blob/dbbf6261e0b44fc81944ed5c3b7152005cc0f1ff/jobs/config.json#L8587) get sent to the flag defined [here](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/framework/test_context.go#L219)? Or should I add another `--provider` flag inside `--upgrade_args` like this: `--upgrade_args=... --provider=kubernetes-anywhere`? - Is it necessary to add waiting logic after the `make` command, or will it implicitly handle that by itself? Some other points: - I chose `sed` to manipulate the current kubernetes-anywhere `.config` rather than duplicating another [`anywhere.go`](https://github.com/kubernetes/test-infra/blob/master/kubetest/anywhere.go). One suggestion was to use `jq` but since the config on disk is not serialized to JSON yet, I'm not sure how that'd work. - Since I don't have a GCE/GKE account or vCenter, I can't actually verify the e2e binary works. I've managed to build it, but if somebody could quickly run a smoke test, I'd appreciate it. This is my first poke around test-infra and e2e, so there might be some plumbing missing /cc @jessicaochen @luxas @pipejakob @roberthbailey
parents 1a15b3db 69f5feb2
...@@ -20,6 +20,7 @@ import ( ...@@ -20,6 +20,7 @@ import (
"fmt" "fmt"
"os" "os"
"path" "path"
"path/filepath"
"strings" "strings"
"time" "time"
...@@ -46,6 +47,8 @@ func MasterUpgrade(v string) error { ...@@ -46,6 +47,8 @@ func MasterUpgrade(v string) error {
return masterUpgradeGCE(v, false) return masterUpgradeGCE(v, false)
case "gke": case "gke":
return masterUpgradeGKE(v) return masterUpgradeGKE(v)
case "kubernetes-anywhere":
return masterUpgradeKubernetesAnywhere(v)
default: default:
return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider) return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
} }
...@@ -103,6 +106,46 @@ func masterUpgradeGKE(v string) error { ...@@ -103,6 +106,46 @@ func masterUpgradeGKE(v string) error {
return nil return nil
} }
func masterUpgradeKubernetesAnywhere(v string) error {
Logf("Upgrading master to %q", v)
kaPath := TestContext.KubernetesAnywherePath
originalConfigPath := filepath.Join(kaPath, ".config")
backupConfigPath := filepath.Join(kaPath, ".config.bak")
updatedConfigPath := filepath.Join(kaPath, fmt.Sprintf(".config-%s", v))
// backup .config to .config.bak
if err := os.Rename(originalConfigPath, backupConfigPath); err != nil {
return err
}
defer func() {
// revert .config.bak to .config
if err := os.Rename(backupConfigPath, originalConfigPath); err != nil {
Logf("Could not rename %s back to %s", backupConfigPath, originalConfigPath)
}
}()
// modify config with specified k8s version
if _, _, err := RunCmd("sed",
fmt.Sprintf(`s/kubernetes_version=.*$/kubernetes_version=%s/`, v),
backupConfigPath, ">", originalConfigPath); err != nil {
return err
}
// invoke ka upgrade
if _, _, err := RunCmd("make", "-C", TestContext.KubernetesAnywherePath,
"WAIT_FOR_KUBECONFIG=y", "upgrade-master"); err != nil {
return err
}
// move .config to .config.<version>
if err := os.Rename(originalConfigPath, updatedConfigPath); err != nil {
return err
}
return nil
}
func NodeUpgrade(f *Framework, v string, img string) error { func NodeUpgrade(f *Framework, v string, img string) error {
// Perform the upgrade. // Perform the upgrade.
var err error var err error
......
...@@ -106,6 +106,9 @@ type TestContextType struct { ...@@ -106,6 +106,9 @@ type TestContextType struct {
// Whether configuration for accessing federation member clusters should be sourced from the host cluster // Whether configuration for accessing federation member clusters should be sourced from the host cluster
FederationConfigFromCluster bool FederationConfigFromCluster bool
// Indicates what path the kubernetes-anywhere is installed on
KubernetesAnywherePath string
// Viper-only parameters. These will in time replace all flags. // Viper-only parameters. These will in time replace all flags.
// Example: Create a file 'e2e.json' with the following: // Example: Create a file 'e2e.json' with the following:
...@@ -201,6 +204,7 @@ func RegisterCommonFlags() { ...@@ -201,6 +204,7 @@ func RegisterCommonFlags() {
flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "", "The container runtime endpoint of cluster VM instances.") flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "", "The container runtime endpoint of cluster VM instances.")
flag.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.") flag.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.")
flag.StringVar(&TestContext.DockershimCheckpointDir, "dockershim-checkpoint-dir", "/var/lib/dockershim/sandbox", "The directory for dockershim to store sandbox checkpoints.") flag.StringVar(&TestContext.DockershimCheckpointDir, "dockershim-checkpoint-dir", "/var/lib/dockershim/sandbox", "The directory for dockershim to store sandbox checkpoints.")
flag.StringVar(&TestContext.KubernetesAnywherePath, "kubernetes-anywhere-path", "/workspace/kubernetes-anywhere", "Which directory kubernetes-anywhere is installed to.")
} }
// Register flags specific to the cluster e2e test suite. // Register flags specific to the cluster e2e test suite.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment