Skip to content

K
k3s
Commit 6ee2b7bc authored by Jeff Lowdermilk's avatar Jeff Lowdermilk
Browse files
Options

Fix some gce-only tests to run on gke as well

DaemonRestart kubelet test, and Services apiserver restart test.
parent 6a234a2c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 92 deletions
test/e2e/daemon_restart.go
View file @ 6ee2b7bc
......@@ -199,8 +199,7 @@ var _ = framework.KubeDescribe("DaemonRestart [Disruptive]", func() {
BeforeEach(func() {
// These tests require SSH
// TODO(11834): Enable this test in GKE once experimental API there is switched on
framework.SkipUnlessProviderIs("gce", "aws")
framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
ns = f.Namespace.Name
// All the restart tests need an rc and a watch on pods of the rc.
......@@ -252,6 +251,8 @@ var _ = framework.KubeDescribe("DaemonRestart [Disruptive]", func() {
It("Controller Manager should not create/delete replicas across restart", func() {
// Requires master ssh access.
framework.SkipUnlessProviderIs("gce", "aws")
restarter := NewRestartConfig(
framework.GetMasterHost(), "kube-controller", ports.ControllerManagerPort, restartPollInterval, restartTimeout)
restarter.restart()
......@@ -281,6 +282,8 @@ var _ = framework.KubeDescribe("DaemonRestart [Disruptive]", func() {
It("Scheduler should continue assigning pods to nodes across restart", func() {
// Requires master ssh access.
framework.SkipUnlessProviderIs("gce", "aws")
restarter := NewRestartConfig(
framework.GetMasterHost(), "kube-scheduler", ports.SchedulerPort, restartPollInterval, restartTimeout)
......
test/e2e/framework/util.go
View file @ 6ee2b7bc
......@@ -134,6 +134,22 @@ const (
// When these values are updated, also update cmd/kubelet/app/options/options.go
currentPodInfraContainerImageName = "gcr.io/google_containers/pause"
currentPodInfraContainerImageVersion = "3.0"
// How long each node is given during a process that restarts all nodes
// before the test is considered failed. (Note that the total time to
// restart all nodes will be this number times the number of nodes.)
RestartPerNodeTimeout = 5 * time.Minute
// How often to Poll the statues of a restart.
RestartPoll = 20 * time.Second
// How long a node is allowed to become "Ready" after it is restarted before
// the test is considered failed.
RestartNodeReadyAgainTimeout = 5 * time.Minute
// How long a pod is allowed to become "running" and "ready" after a node
// restart before test is considered failed.
RestartPodReadyAgainTimeout = 5 * time.Minute
)
// Label allocated to the image puller static pod that runs on each node
......@@ -3505,13 +3521,29 @@ func RestartKubeProxy(host string) error {
return nil
}
func RestartApiserver() error {
func RestartApiserver(c *client.Client) error {
// TODO: Make it work for all providers.
if !ProviderIs("gce", "gke", "aws") {
return fmt.Errorf("unsupported provider: %s", TestContext.Provider)
}
if ProviderIs("gce", "aws") {
return sshRestartMaster()
}
// GKE doesn't allow ssh accesss, so use a same-version master
// upgrade to teardown/recreate master.
v, err := c.ServerVersion()
if err != nil {
return err
}
return masterUpgradeGKE(v.GitVersion[1:]) // strip leading 'v'
}
func sshRestartMaster() error {
if !ProviderIs("gce", "aws") {
return fmt.Errorf("unsupported provider: %s", TestContext.Provider)
}
var command string
if ProviderIs("gce", "gke") {
if ProviderIs("gce") {
command = "sudo docker ps | grep /kube-apiserver | cut -d ' ' -f 1 | xargs sudo docker kill"
} else {
command = "sudo /etc/init.d/kube-apiserver restart"
......@@ -4127,3 +4159,41 @@ func GetPodsInNamespace(c *client.Client, ns string, ignoreLabels map[string]str
}
return filtered, nil
}
// RunCmd runs cmd using args and returns its stdout and stderr. It also outputs
// cmd's stdout and stderr to their respective OS streams.
func RunCmd(command string, args ...string) (string, string, error) {
Logf("Running %s %v", command, args)
var bout, berr bytes.Buffer
cmd := exec.Command(command, args...)
// We also output to the OS stdout/stderr to aid in debugging in case cmd
// hangs and never returns before the test gets killed.
//
// This creates some ugly output because gcloud doesn't always provide
// newlines.
cmd.Stdout = io.MultiWriter(os.Stdout, &bout)
cmd.Stderr = io.MultiWriter(os.Stderr, &berr)
err := cmd.Run()
stdout, stderr := bout.String(), berr.String()
if err != nil {
return "", "", fmt.Errorf("error running %s %v; got error %v, stdout %q, stderr %q",
command, args, err, stdout, stderr)
}
return stdout, stderr, nil
}
// retryCmd runs cmd using args and retries it for up to SingleCallTimeout if
// it returns an error. It returns stdout and stderr.
func retryCmd(command string, args ...string) (string, string, error) {
var err error
stdout, stderr := "", ""
wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
stdout, stderr, err = RunCmd(command, args...)
if err != nil {
Logf("Got %v", err)
return false, nil
}
return true, nil
})
return stdout, stderr, err
}
test/e2e/restart.go
View file @ 6ee2b7bc
......@@ -21,7 +21,6 @@ import (
"time"
"k8s.io/kubernetes/pkg/api"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/wait"
......@@ -31,24 +30,6 @@ import (
. "github.com/onsi/gomega"
)
const (
// How long each node is given during a process that restarts all nodes
// before the test is considered failed. (Note that the total time to
// restart all nodes will be this number times the number of nodes.)
restartPerNodeTimeout = 5 * time.Minute
// How often to framework.Poll the statues of a restart.
restartPoll = 20 * time.Second
// How long a node is allowed to become "Ready" after it is restarted before
// the test is considered failed.
restartNodeReadyAgainTimeout = 5 * time.Minute
// How long a pod is allowed to become "running" and "ready" after a node
// restart before test is considered failed.
restartPodReadyAgainTimeout = 5 * time.Minute
)
var _ = framework.KubeDescribe("Restart [Disruptive]", func() {
f := framework.NewDefaultFramework("restart")
var ps *framework.PodStore
......@@ -71,7 +52,7 @@ var _ = framework.KubeDescribe("Restart [Disruptive]", func() {
nn := framework.TestContext.CloudConfig.NumNodes
By("ensuring all nodes are ready")
nodeNamesBefore, err := checkNodesReady(f.Client, framework.NodeReadyInitialTimeout, nn)
nodeNamesBefore, err := framework.CheckNodesReady(f.Client, framework.NodeReadyInitialTimeout, nn)
Expect(err).NotTo(HaveOccurred())
framework.Logf("Got the following nodes before restart: %v", nodeNamesBefore)
......@@ -87,11 +68,11 @@ var _ = framework.KubeDescribe("Restart [Disruptive]", func() {
}
By("restarting all of the nodes")
err = restartNodes(framework.TestContext.Provider, restartPerNodeTimeout)
err = restartNodes(framework.TestContext.Provider, framework.RestartPerNodeTimeout)
Expect(err).NotTo(HaveOccurred())
By("ensuring all nodes are ready after the restart")
nodeNamesAfter, err := checkNodesReady(f.Client, restartNodeReadyAgainTimeout, nn)
nodeNamesAfter, err := framework.CheckNodesReady(f.Client, framework.RestartNodeReadyAgainTimeout, nn)
Expect(err).NotTo(HaveOccurred())
framework.Logf("Got the following nodes after restart: %v", nodeNamesAfter)
......@@ -108,10 +89,10 @@ var _ = framework.KubeDescribe("Restart [Disruptive]", func() {
// across node restarts.
By("ensuring the same number of pods are running and ready after restart")
podCheckStart := time.Now()
podNamesAfter, err := waitForNPods(ps, len(podNamesBefore), restartPodReadyAgainTimeout)
podNamesAfter, err := waitForNPods(ps, len(podNamesBefore), framework.RestartPodReadyAgainTimeout)
Expect(err).NotTo(HaveOccurred())
remaining := restartPodReadyAgainTimeout - time.Since(podCheckStart)
if !framework.CheckPodsRunningReadyOrSucceeded(f.Client, ns, podNamesAfter, remaining) {
remaining := framework.RestartPodReadyAgainTimeout - time.Since(podCheckStart)
if !framework.CheckPodsRunningReady(f.Client, ns, podNamesAfter, remaining) {
framework.Failf("At least one pod wasn't running and ready after the restart.")
}
})
......@@ -144,64 +125,6 @@ func waitForNPods(ps *framework.PodStore, expect int, timeout time.Duration) ([]
return podNames, nil
}
// checkNodesReady waits up to nt for expect nodes accessed by c to be ready,
// returning an error if this doesn't happen in time. It returns the names of
// nodes it finds.
func checkNodesReady(c *client.Client, nt time.Duration, expect int) ([]string, error) {
// First, keep getting all of the nodes until we get the number we expect.
var nodeList *api.NodeList
var errLast error
start := time.Now()
found := wait.Poll(framework.Poll, nt, func() (bool, error) {
// A rolling-update (GCE/GKE implementation of restart) can complete before the apiserver
// knows about all of the nodes. Thus, we retry the list nodes call
// until we get the expected number of nodes.
nodeList, errLast = c.Nodes().List(api.ListOptions{
FieldSelector: fields.Set{"spec.unschedulable": "false"}.AsSelector()})
if errLast != nil {
return false, nil
}
if len(nodeList.Items) != expect {
errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)",
expect, len(nodeList.Items), time.Since(start))
framework.Logf("%v", errLast)
return false, nil
}
return true, nil
}) == nil
nodeNames := make([]string, len(nodeList.Items))
for i, n := range nodeList.Items {
nodeNames[i] = n.ObjectMeta.Name
}
if !found {
return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v",
expect, nt, errLast)
}
framework.Logf("Successfully found %d nodes", expect)
// Next, ensure in parallel that all the nodes are ready. We subtract the
// time we spent waiting above.
timeout := nt - time.Since(start)
result := make(chan bool, len(nodeList.Items))
for _, n := range nodeNames {
n := n
go func() { result <- framework.WaitForNodeToBeReady(c, n, timeout) }()
}
failed := false
// TODO(mbforbes): Change to `for range` syntax once we support only Go
// >= 1.4.
for i := range nodeList.Items {
_ = i
if !<-result {
failed = true
}
}
if failed {
return nodeNames, fmt.Errorf("at least one node failed to be ready")
}
return nodeNames, nil
}
// restartNodes uses provider to do a restart of all nodes in the cluster,
// allowing up to nt per node.
func restartNodes(provider string, nt time.Duration) error {
......@@ -235,9 +158,9 @@ func restartNodes(provider string, nt time.Duration) error {
// done
func migRollingUpdateSelf(nt time.Duration) error {
By("getting the name of the template for the managed instance group")
tmpl, err := migTemplate()
tmpl, err := framework.MigTemplate()
if err != nil {
return fmt.Errorf("couldn't get MIG template name: %v", err)
}
return migRollingUpdate(tmpl, nt)
return framework.MigRollingUpdate(tmpl, nt)
}
test/e2e/service.go
View file @ 6ee2b7bc
......@@ -331,8 +331,7 @@ var _ = framework.KubeDescribe("Services", func() {
It("should work after restarting apiserver [Disruptive]", func() {
// TODO: use the ServiceTestJig here
// TODO: framework.RestartApiserver doesn't work in GKE - fix it and reenable this test.
framework.SkipUnlessProviderIs("gce")
framework.SkipUnlessProviderIs("gce", "gke")
ns := f.Namespace.Name
numPods, servicePort := 3, 80
......@@ -351,7 +350,7 @@ var _ = framework.KubeDescribe("Services", func() {
framework.ExpectNoError(verifyServeHostnameServiceUp(c, ns, host, podNames1, svc1IP, servicePort))
// Restart apiserver
if err := framework.RestartApiserver(); err != nil {
if err := framework.RestartApiserver(c); err != nil {
framework.Failf("error restarting apiserver: %v", err)
}
if err := framework.WaitForApiserverUp(c); err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment