Merge pull request #77188 from SataQiu/fix-kubeadm-20190428

cmd/kubeadm/app/cmd/init.go

@@ -334,7 +334,7 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 	// if dry running creates a temporary folder for saving kubeadm generated files
 	dryRunDir := ""
 	if options.dryRun {
-		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm("kubeadm-init-dryrun"); err != nil {
+		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-init-dryrun"); err != nil {
 			return nil, errors.Wrap(err, "couldn't create a temporary directory")
 		}
 	}

cmd/kubeadm/app/cmd/upgrade/BUILD

@@ -58,7 +58,6 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
-        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/upgrade:go_default_library",
     ],
 )

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -238,14 +238,14 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w
 }
 
 // GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
-func GetPathManagerForUpgrade(internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (upgrade.StaticPodPathManager, error) {
+func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (upgrade.StaticPodPathManager, error) {
 	isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd)
-	return upgrade.NewKubeStaticPodPathManagerUsingTempDirs(constants.GetStaticPodDirectory(), true, etcdUpgrade && !isHAEtcd)
+	return upgrade.NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd)
 }
 
 // PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
 func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error {
-	pathManager, err := GetPathManagerForUpgrade(internalcfg, etcdUpgrade)
+	pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade)
 	if err != nil {
 		return err
 	}
@@ -257,7 +257,7 @@ func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter
 // DryRunStaticPodUpgrade fakes an upgrade of the control plane
 func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error {
 
-	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("kubeadm-upgrade-dryrun")
+	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/cmd/upgrade/apply_test.go

@@ -22,7 +22,6 @@ import (
 	"testing"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 func TestSessionIsInteractive(t *testing.T) {
@@ -114,14 +113,11 @@ func TestGetPathManagerForUpgrade(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error making temporary directory: %v", err)
 			}
-			oldK8sDir := constants.KubernetesDir
-			constants.KubernetesDir = tmpdir
 			defer func() {
-				constants.KubernetesDir = oldK8sDir
 				os.RemoveAll(tmpdir)
 			}()
 
-			pathmgr, err := GetPathManagerForUpgrade(test.cfg, test.etcdUpgrade)
+			pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade)
 			if err != nil {
 				t.Fatalf("unexpected error creating path manager: %v", err)
 			}

cmd/kubeadm/app/constants/constants.go

@@ -33,11 +33,9 @@ import (
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 )
 
-// KubernetesDir is the directory Kubernetes owns for storing various configuration files
-// This semi-constant MUST NOT be modified during runtime. It's a variable solely for use in unit testing.
-var KubernetesDir = "/etc/kubernetes"
-
 const (
+	// KubernetesDir is the directory Kubernetes owns for storing various configuration files
+	KubernetesDir = "/etc/kubernetes"
 	// ManifestsSubDirName defines directory name to store manifests
 	ManifestsSubDirName = "manifests"
 	// TempDirForKubeadm defines temporary directory for kubeadm
@@ -448,8 +446,12 @@ func AddSelfHostedPrefix(componentName string) string {
 }
 
 // CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous)
-func CreateTempDirForKubeadm(dirName string) (string, error) {
+func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) {
 	tempDir := path.Join(KubernetesDir, TempDirForKubeadm)
+	if len(kubernetesDir) != 0 {
+		tempDir = path.Join(kubernetesDir, TempDirForKubeadm)
+	}
+
 	// creates target folder if not already exists
 	if err := os.MkdirAll(tempDir, 0700); err != nil {
 		return "", errors.Wrapf(err, "failed to create directory %q", tempDir)
@@ -463,8 +465,12 @@ func CreateTempDirForKubeadm(dirName string) (string, error) {
 }
 
 // CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date
-func CreateTimestampDirForKubeadm(dirName string) (string, error) {
+func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) {
 	tempDir := path.Join(KubernetesDir, TempDirForKubeadm)
+	if len(kubernetesDir) != 0 {
+		tempDir = path.Join(kubernetesDir, TempDirForKubeadm)
+	}
+
 	// creates target folder if not already exists
 	if err := os.MkdirAll(tempDir, 0700); err != nil {
 		return "", errors.Wrapf(err, "failed to create directory %q", tempDir)

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -180,7 +180,7 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon
 // GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
 func GetKubeletDir(dryRun bool) (string, error) {
 	if dryRun {
-		return kubeadmconstants.CreateTempDirForKubeadm("kubeadm-upgrade-dryrun")
+		return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
 	}
 	return kubeadmconstants.KubeletRunDirectory, nil
 }

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -19,6 +19,7 @@ package upgrade
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -47,6 +48,8 @@ const (
 type StaticPodPathManager interface {
 	// MoveFile should move a file from oldPath to newPath
 	MoveFile(oldPath, newPath string) error
+	// KubernetesDir is the directory Kubernetes owns for storing various configuration files
+	KubernetesDir() string
 	// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
 	RealManifestPath(component string) string
 	// RealManifestDir should point to the static pod manifest directory used by the kubelet
@@ -67,6 +70,7 @@ type StaticPodPathManager interface {
 
 // KubeStaticPodPathManager is a real implementation of StaticPodPathManager that is used when upgrading a static pod cluster
 type KubeStaticPodPathManager struct {
+	kubernetesDir     string
 	realManifestDir   string
 	tempManifestDir   string
 	backupManifestDir string
@@ -77,9 +81,10 @@ type KubeStaticPodPathManager struct {
 }
 
 // NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager
-func NewKubeStaticPodPathManager(realDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager {
+func NewKubeStaticPodPathManager(kubernetesDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager {
 	return &KubeStaticPodPathManager{
-		realManifestDir:   realDir,
+		kubernetesDir:     kubernetesDir,
+		realManifestDir:   filepath.Join(kubernetesDir, constants.ManifestsSubDirName),
 		tempManifestDir:   tempDir,
 		backupManifestDir: backupDir,
 		backupEtcdDir:     backupEtcdDir,
@@ -89,21 +94,22 @@ func NewKubeStaticPodPathManager(realDir, tempDir, backupDir, backupEtcdDir stri
 }
 
 // NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it
-func NewKubeStaticPodPathManagerUsingTempDirs(realManifestDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) {
-	upgradedManifestsDir, err := constants.CreateTempDirForKubeadm("kubeadm-upgraded-manifests")
+func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) {
+
+	upgradedManifestsDir, err := constants.CreateTempDirForKubeadm(kubernetesDir, "kubeadm-upgraded-manifests")
 	if err != nil {
 		return nil, err
 	}
-	backupManifestsDir, err := constants.CreateTimestampDirForKubeadm("kubeadm-backup-manifests")
+	backupManifestsDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-manifests")
 	if err != nil {
 		return nil, err
 	}
-	backupEtcdDir, err := constants.CreateTimestampDirForKubeadm("kubeadm-backup-etcd")
+	backupEtcdDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-etcd")
 	if err != nil {
 		return nil, err
 	}
 
-	return NewKubeStaticPodPathManager(realManifestDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil
+	return NewKubeStaticPodPathManager(kubernetesDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil
 }
 
 // MoveFile should move a file from oldPath to newPath
@@ -111,6 +117,11 @@ func (spm *KubeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
 	return os.Rename(oldPath, newPath)
 }
 
+// KubernetesDir should point to the directory Kubernetes owns for storing various configuration files
+func (spm *KubeStaticPodPathManager) KubernetesDir() string {
+	return spm.kubernetesDir
+}
+
 // RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
 func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string {
 	return constants.GetStaticPodFilepath(component, spm.realManifestDir)
@@ -202,7 +213,7 @@ func upgradeComponent(component string, renewCerts bool, waiter apiclient.Waiter
 	// if certificate renewal should be performed
 	if renewCerts {
 		// renew all the certificates used by the current component
-		if err := renewCertsByComponent(cfg, constants.KubernetesDir, component); err != nil {
+		if err := renewCertsByComponent(cfg, pathMgr.KubernetesDir(), component); err != nil {
 			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to renew certificates for component %q", component), pathMgr, recoverEtcd)
 		}
 	}
@@ -452,7 +463,7 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
 
 	if renewCerts {
 		// renew the certificate embedded in the admin.conf file
-		err := renewEmbeddedCertsByName(cfg, constants.KubernetesDir, constants.AdminKubeConfigFileName)
+		err := renewEmbeddedCertsByName(cfg, pathMgr.KubernetesDir(), constants.AdminKubeConfigFileName)
 		if err != nil {
 			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.AdminKubeConfigFileName), pathMgr, false)
 		}

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -468,7 +468,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				t.Fatalf("couldn't run NewFakeStaticPodPathManager: %v", err)
 			}
 			defer os.RemoveAll(pathMgr.(*fakeStaticPodPathManager).KubernetesDir())
-			constants.KubernetesDir = pathMgr.(*fakeStaticPodPathManager).KubernetesDir()
+			tmpKubernetesDir := pathMgr.(*fakeStaticPodPathManager).KubernetesDir()
 
 			tempCertsDir, err := ioutil.TempDir("", "kubeadm-certs")
 			if err != nil {
@@ -505,7 +505,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				if rt.skipKubeConfig == kubeConfig {
 					continue
 				}
-				if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, constants.KubernetesDir, oldcfg); err != nil {
+				if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpKubernetesDir, oldcfg); err != nil {
 					t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
 				}
 			}
@@ -639,7 +639,7 @@ func TestCleanupDirs(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			realManifestDir, cleanup := getTempDir(t, "realManifestDir")
+			realKubernetesDir, cleanup := getTempDir(t, "realKubernetesDir")
 			defer cleanup()
 
 			tempManifestDir, cleanup := getTempDir(t, "tempManifestDir")
@@ -651,7 +651,7 @@ func TestCleanupDirs(t *testing.T) {
 			backupEtcdDir, cleanup := getTempDir(t, "backupEtcdDir")
 			defer cleanup()
 
-			mgr := NewKubeStaticPodPathManager(realManifestDir, tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd)
+			mgr := NewKubeStaticPodPathManager(realKubernetesDir, tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd)
 			err := mgr.CleanupDirs()
 			if err != nil {
 				t.Errorf("unexpected error cleaning up: %v", err)