kubeadm upgrade node

cmd/kubeadm/app/cmd/options/constant.go

@@ -59,6 +59,9 @@ const (
 	// KubernetesVersion flag sets the Kubernetes version for the control plane.
 	KubernetesVersion = "kubernetes-version"
 
+	// KubeletVersion flag sets the version for the kubelet config.
+	KubeletVersion = "kubelet-version"
+
 	// NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal".
 	NetworkingDNSDomain = "service-dns-domain"
 

cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+)
+
+// NewControlPlane creates a kubeadm workflow phase that implements handling of control-plane upgrade.
+func NewControlPlane() workflow.Phase {
+	phase := workflow.Phase{
+		Name:  "control-plane",
+		Short: "Upgrade the control plane instance deployed on this node, if any",
+		Run:   runControlPlane(),
+		InheritFlags: []string{
+			options.DryRun,
+			options.KubeconfigPath,
+		},
+	}
+	return phase
+}
+
+func runControlPlane() func(c workflow.RunData) error {
+	return func(c workflow.RunData) error {
+		data, ok := c.(Data)
+		if !ok {
+			return errors.New("control-plane phase invoked with an invalid data struct")
+		}
+
+		// if this is not a control-plande node, this phase should not be executed
+		if !data.IsControlPlaneNode() {
+			fmt.Printf("[upgrade] Skipping phase. Not a control plane node")
+		}
+
+		// otherwise, retrieve all the info required for control plane upgrade
+		cfg := data.Cfg()
+		client := data.Client()
+		dryRun := data.DryRun()
+		etcdUpgrade := data.EtcdUpgrade()
+		renewCerts := data.RenewCerts()
+
+		// Upgrade the control plane and etcd if installed on this node
+		fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion)
+		if dryRun {
+			return upgrade.DryRunStaticPodUpgrade(cfg)
+		}
+
+		waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout)
+
+		if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts); err != nil {
+			return errors.Wrap(err, "couldn't complete the static pod upgrade")
+		}
+
+		fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
+
+		return nil
+	}
+}

cmd/kubeadm/app/cmd/phases/upgrade/node/data.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+// Data is the interface to use for kubeadm upgrade node phases.
+// The "nodeData" type from "cmd/upgrade/node.go" must satisfy this interface.
+type Data interface {
+	EtcdUpgrade() bool
+	RenewCerts() bool
+	DryRun() bool
+	KubeletVersion() string
+	Cfg() *kubeadmapi.InitConfiguration
+	IsControlPlaneNode() bool
+	Client() clientset.Interface
+}

cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	kubeletConfigLongDesc = normalizer.LongDesc(`
+		Download the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster,
+		where X is the minor version of the kubelet. kubeadm uses the KuberneteVersion field in the kubeadm-config
+		ConfigMap to determine what the _desired_ kubelet version is, but the user can override this by using the
+		--kubelet-version parameter.
+		`)
+)
+
+// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade.
+func NewKubeletConfigPhase() workflow.Phase {
+	phase := workflow.Phase{
+		Name:  "kubelet-config",
+		Short: "Upgrade the kubelet configuration for this node",
+		Long:  kubeletConfigLongDesc,
+		Run:   runKubeletConfigPhase(),
+		InheritFlags: []string{
+			options.DryRun,
+			options.KubeconfigPath,
+			options.KubeletVersion,
+		},
+	}
+	return phase
+}
+
+func runKubeletConfigPhase() func(c workflow.RunData) error {
+	return func(c workflow.RunData) error {
+		data, ok := c.(Data)
+		if !ok {
+			return errors.New("kubelet-config phase invoked with an invalid data struct")
+		}
+
+		// otherwise, retrieve all the info required for kubelet config upgrade
+		cfg := data.Cfg()
+		client := data.Client()
+		dryRun := data.DryRun()
+
+		// Set up the kubelet directory to use. If dry-running, this will return a fake directory
+		kubeletDir, err := upgrade.GetKubeletDir(dryRun)
+		if err != nil {
+			return err
+		}
+
+		// Gets the target kubelet version.
+		// by default kubelet version is expected to be equal to ClusterConfiguration.KubernetesVersion, but
+		// users can specify a different kubelet version (this is a legacy of the original implementation
+		// of `kubeam upgrade node config` which we are preserving in order to don't break GA contract)
+		kubeletVersionStr := cfg.ClusterConfiguration.KubernetesVersion
+		if data.KubeletVersion() != "" && data.KubeletVersion() != kubeletVersionStr {
+			kubeletVersionStr = data.KubeletVersion()
+			fmt.Printf("[upgrade] Using kubelet config version %s, while kubernetes-version is %s\n", kubeletVersionStr, cfg.ClusterConfiguration.KubernetesVersion)
+		}
+
+		// Parse the desired kubelet version
+		kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
+		if err != nil {
+			return err
+		}
+
+		// TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered
+		if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil {
+			return err
+		}
+
+		// If we're dry-running, print the generated manifests
+		if dryRun {
+			if err := printFilesIfDryRunning(dryRun, kubeletDir); err != nil {
+				return errors.Wrap(err, "error printing files on dryrun")
+			}
+			return nil
+		}
+
+		fmt.Println("[upgrade] The configuration for this node was successfully updated!")
+		fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.")
+		return nil
+	}
+}
+
+// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup
+func printFilesIfDryRunning(dryRun bool, kubeletDir string) error {
+	if !dryRun {
+		return nil
+	}
+
+	// Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory
+	fileToPrint := dryrunutil.FileToPrint{
+		RealPath:  filepath.Join(kubeletDir, constants.KubeletConfigurationFileName),
+		PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName),
+	}
+	return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout)
+}

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -18,7 +18,6 @@ package upgrade
 
 import (
 	"fmt"
-	"os"
 	"time"
 
 	"github.com/pkg/errors"
@@ -31,13 +30,10 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
-	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
-	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
 )
 
 const (
@@ -230,50 +226,9 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w
 	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion)
 
 	if flags.dryRun {
-		return DryRunStaticPodUpgrade(internalcfg)
+		return upgrade.DryRunStaticPodUpgrade(internalcfg)
 	}
 
 	// Don't save etcd backup directory if etcd is HA, as this could cause corruption
-	return PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts)
-}
-
-// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
-func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (upgrade.StaticPodPathManager, error) {
-	isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd)
-	return upgrade.NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd)
-}
-
-// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
-func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error {
-	pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade)
-	if err != nil {
-		return err
-	}
-
-	// The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing
-	return upgrade.StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil)
-}
-
-// DryRunStaticPodUpgrade fakes an upgrade of the control plane
-func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error {
-
-	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(dryRunManifestDir)
-
-	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil {
-		return err
-	}
-
-	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
-	files := []dryrunutil.FileToPrint{}
-	for _, component := range constants.ControlPlaneComponents {
-		realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir)
-		outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
-		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
-	}
-
-	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
+	return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts)
 }

cmd/kubeadm/app/cmd/upgrade/apply_test.go

@@ -17,11 +17,7 @@ limitations under the License.
 package upgrade
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
-
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 )
 
 func TestSessionIsInteractive(t *testing.T) {
@@ -65,89 +61,3 @@ func TestSessionIsInteractive(t *testing.T) {
 		})
 	}
 }
-
-func TestGetPathManagerForUpgrade(t *testing.T) {
-
-	haEtcd := &kubeadmapi.InitConfiguration{
-		ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-			Etcd: kubeadmapi.Etcd{
-				External: &kubeadmapi.ExternalEtcd{
-					Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"},
-				},
-			},
-		},
-	}
-
-	noHAEtcd := &kubeadmapi.InitConfiguration{}
-
-	tests := []struct {
-		name             string
-		cfg              *kubeadmapi.InitConfiguration
-		etcdUpgrade      bool
-		shouldDeleteEtcd bool
-	}{
-		{
-			name:             "ha etcd but no etcd upgrade",
-			cfg:              haEtcd,
-			etcdUpgrade:      false,
-			shouldDeleteEtcd: true,
-		},
-		{
-			name:             "non-ha etcd with etcd upgrade",
-			cfg:              noHAEtcd,
-			etcdUpgrade:      true,
-			shouldDeleteEtcd: false,
-		},
-		{
-			name:             "ha etcd and etcd upgrade",
-			cfg:              haEtcd,
-			etcdUpgrade:      true,
-			shouldDeleteEtcd: true,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			// Use a temporary directory
-			tmpdir, err := ioutil.TempDir("", "TestGetPathManagerForUpgrade")
-			if err != nil {
-				t.Fatalf("unexpected error making temporary directory: %v", err)
-			}
-			defer func() {
-				os.RemoveAll(tmpdir)
-			}()
-
-			pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade)
-			if err != nil {
-				t.Fatalf("unexpected error creating path manager: %v", err)
-			}
-
-			if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
-				t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
-			}
-
-			if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
-				t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err)
-			}
-
-			if err := pathmgr.CleanupDirs(); err != nil {
-				t.Fatalf("unexpected error cleaning up directories: %v", err)
-			}
-
-			if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
-				t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
-			}
-
-			if test.shouldDeleteEtcd {
-				if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) {
-					t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err)
-				}
-			} else {
-				if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
-					t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir())
-				}
-			}
-		})
-	}
-
-}

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -31,10 +31,12 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 )
@@ -583,3 +585,44 @@ func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string,
 
 	return nil
 }
+
+// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
+func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) {
+	isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd)
+	return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd)
+}
+
+// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
+func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error {
+	pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade)
+	if err != nil {
+		return err
+	}
+
+	// The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing
+	return StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil)
+}
+
+// DryRunStaticPodUpgrade fakes an upgrade of the control plane
+func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error {
+
+	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dryRunManifestDir)
+
+	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil {
+		return err
+	}
+
+	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
+	files := []dryrunutil.FileToPrint{}
+	for _, component := range constants.ControlPlaneComponents {
+		realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir)
+		outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
+		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
+	}
+
+	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
+}

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -909,3 +909,89 @@ func getEmbeddedCerts(tmpDir, kubeConfig string) ([]*x509.Certificate, error) {
 
 	return certutil.ParseCertsPEM(authInfo.ClientCertificateData)
 }
+
+func TestGetPathManagerForUpgrade(t *testing.T) {
+
+	haEtcd := &kubeadmapi.InitConfiguration{
+		ClusterConfiguration: kubeadmapi.ClusterConfiguration{
+			Etcd: kubeadmapi.Etcd{
+				External: &kubeadmapi.ExternalEtcd{
+					Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"},
+				},
+			},
+		},
+	}
+
+	noHAEtcd := &kubeadmapi.InitConfiguration{}
+
+	tests := []struct {
+		name             string
+		cfg              *kubeadmapi.InitConfiguration
+		etcdUpgrade      bool
+		shouldDeleteEtcd bool
+	}{
+		{
+			name:             "ha etcd but no etcd upgrade",
+			cfg:              haEtcd,
+			etcdUpgrade:      false,
+			shouldDeleteEtcd: true,
+		},
+		{
+			name:             "non-ha etcd with etcd upgrade",
+			cfg:              noHAEtcd,
+			etcdUpgrade:      true,
+			shouldDeleteEtcd: false,
+		},
+		{
+			name:             "ha etcd and etcd upgrade",
+			cfg:              haEtcd,
+			etcdUpgrade:      true,
+			shouldDeleteEtcd: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Use a temporary directory
+			tmpdir, err := ioutil.TempDir("", "TestGetPathManagerForUpgrade")
+			if err != nil {
+				t.Fatalf("unexpected error making temporary directory: %v", err)
+			}
+			defer func() {
+				os.RemoveAll(tmpdir)
+			}()
+
+			pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade)
+			if err != nil {
+				t.Fatalf("unexpected error creating path manager: %v", err)
+			}
+
+			if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
+				t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
+			}
+
+			if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
+				t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err)
+			}
+
+			if err := pathmgr.CleanupDirs(); err != nil {
+				t.Fatalf("unexpected error cleaning up directories: %v", err)
+			}
+
+			if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
+				t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
+			}
+
+			if test.shouldDeleteEtcd {
+				if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) {
+					t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err)
+				}
+			} else {
+				if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
+					t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir())
+				}
+			}
+		})
+	}
+
+}