Update kubeadm config for setting kube-proxy bind address

cmd/kubeadm/app/apis/kubeadm/BUILD

@@ -17,6 +17,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm",
     deps = [
         "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",

cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD

@@ -12,6 +12,7 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
         "//pkg/util/pointer:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -25,6 +25,7 @@ import (
 	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
@@ -76,6 +77,41 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 				},
 			}
 			kubeletconfigv1alpha1.SetDefaults_KubeletConfiguration(obj.KubeletConfiguration.BaseConfig)
+			obj.KubeProxy = kubeadm.KubeProxy{
+				Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
+					FeatureGates:       "foo",
+					BindAddress:        "foo",
+					HealthzBindAddress: "foo:10256",
+					MetricsBindAddress: "foo:",
+					EnableProfiling:    bool(true),
+					ClusterCIDR:        "foo",
+					HostnameOverride:   "foo",
+					ClientConnection: kubeproxyconfigv1alpha1.ClientConnectionConfiguration{
+						KubeConfigFile:     "foo",
+						AcceptContentTypes: "foo",
+						ContentType:        "foo",
+						QPS:                float32(5),
+						Burst:              10,
+					},
+					IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{
+						SyncPeriod: metav1.Duration{Duration: 1},
+					},
+					IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{
+						MasqueradeBit: utilpointer.Int32Ptr(0),
+						SyncPeriod:    metav1.Duration{Duration: 1},
+					},
+					OOMScoreAdj:       utilpointer.Int32Ptr(0),
+					ResourceContainer: "foo",
+					UDPIdleTimeout:    metav1.Duration{Duration: 1},
+					Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
+						MaxPerCore: utilpointer.Int32Ptr(2),
+						Min:        utilpointer.Int32Ptr(1),
+						TCPEstablishedTimeout: &metav1.Duration{Duration: 5},
+						TCPCloseWaitTimeout:   &metav1.Duration{Duration: 5},
+					},
+					ConfigSyncPeriod: metav1.Duration{Duration: 1},
+				},
+			}
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -19,6 +19,7 @@ package kubeadm
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -29,6 +30,7 @@ type MasterConfiguration struct {
 	metav1.TypeMeta
 
 	API                  API
+	KubeProxy            KubeProxy
 	Etcd                 Etcd
 	KubeletConfiguration KubeletConfiguration
 	Networking           Networking
@@ -173,3 +175,8 @@ type HostPathMount struct {
 	HostPath  string
 	MountPath string
 }
+
+// KubeProxy contains elements describing the proxy configuration
+type KubeProxy struct {
+	Config *kubeproxyconfigv1alpha1.KubeProxyConfiguration
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD

@@ -26,7 +26,10 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
         "//pkg/util/pointer:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -23,7 +23,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
@@ -57,6 +60,10 @@ const (
 	DefaultEtcdCertDir = "/etc/kubernetes/pki/etcd"
 	// DefaultEtcdClusterServiceName is the default name of the service backing the etcd cluster
 	DefaultEtcdClusterServiceName = "etcd-cluster"
+	// DefaultProxyBindAddressv4 is the default bind address when the advertise address is v4
+	DefaultProxyBindAddressv4 = "0.0.0.0"
+	// DefaultProxyBindAddressv6 is the default bind address when the advertise address is v6
+	DefaultProxyBindAddressv6 = "::"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -105,6 +112,24 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 
 	SetDefaultsEtcdSelfHosted(obj)
 	SetDefaults_KubeletConfiguration(obj)
+	SetDefaults_ProxyConfiguration(obj)
+}
+
+// SetDefaults_ProxyConfiguration assigns default values for the Proxy
+func SetDefaults_ProxyConfiguration(obj *MasterConfiguration) {
+
+	if obj.KubeProxy.Config == nil {
+		obj.KubeProxy.Config = &kubeproxyconfigv1alpha1.KubeProxyConfiguration{}
+	}
+	if obj.KubeProxy.Config.ClusterCIDR == "" && obj.Networking.PodSubnet != "" {
+		obj.KubeProxy.Config.ClusterCIDR = obj.Networking.PodSubnet
+	}
+
+	if features.Enabled(obj.FeatureGates, features.SupportIPVSProxyMode) {
+		obj.KubeProxy.Config.FeatureGates = "true"
+		obj.KubeProxy.Config.Mode = "ipvs"
+	}
+	kubeproxyscheme.Scheme.Default(obj.KubeProxy.Config)
 }
 
 // SetDefaults_NodeConfiguration assigns default values to a regular node

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -29,6 +30,7 @@ type MasterConfiguration struct {
 	metav1.TypeMeta `json:",inline"`
 
 	API                  API                  `json:"api"`
+	KubeProxy            KubeProxy            `json:"kubeProxy"`
 	Etcd                 Etcd                 `json:"etcd"`
 	KubeletConfiguration KubeletConfiguration `json:"kubeletConfiguration"`
 	Networking           Networking           `json:"networking"`
@@ -155,3 +157,8 @@ type HostPathMount struct {
 	HostPath  string `json:"hostPath"`
 	MountPath string `json:"mountPath"`
 }
+
+// KubeProxy contains elements describing the proxy configuration
+type KubeProxy struct {
+	Config *kubeproxyconfigv1alpha1.KubeProxyConfiguration `json:"config,omitempty"`
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go

@@ -26,6 +26,7 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeletconfig_v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfig_v1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 	unsafe "unsafe"
 )
 
@@ -43,6 +44,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kubeadm_Etcd_To_v1alpha1_Etcd,
 		Convert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount,
 		Convert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount,
+		Convert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy,
+		Convert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy,
 		Convert_v1alpha1_KubeletConfiguration_To_kubeadm_KubeletConfiguration,
 		Convert_kubeadm_KubeletConfiguration_To_v1alpha1_KubeletConfiguration,
 		Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration,
@@ -138,6 +141,26 @@ func Convert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in *kubeadm.HostPat
 	return autoConvert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in, out, s)
 }
 
+func autoConvert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy(in *KubeProxy, out *kubeadm.KubeProxy, s conversion.Scope) error {
+	out.Config = (*kubeproxyconfig_v1alpha1.KubeProxyConfiguration)(unsafe.Pointer(in.Config))
+	return nil
+}
+
+// Convert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy is an autogenerated conversion function.
+func Convert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy(in *KubeProxy, out *kubeadm.KubeProxy, s conversion.Scope) error {
+	return autoConvert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy(in, out, s)
+}
+
+func autoConvert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy(in *kubeadm.KubeProxy, out *KubeProxy, s conversion.Scope) error {
+	out.Config = (*kubeproxyconfig_v1alpha1.KubeProxyConfiguration)(unsafe.Pointer(in.Config))
+	return nil
+}
+
+// Convert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy is an autogenerated conversion function.
+func Convert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy(in *kubeadm.KubeProxy, out *KubeProxy, s conversion.Scope) error {
+	return autoConvert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy(in, out, s)
+}
+
 func autoConvert_v1alpha1_KubeletConfiguration_To_kubeadm_KubeletConfiguration(in *KubeletConfiguration, out *kubeadm.KubeletConfiguration, s conversion.Scope) error {
 	out.BaseConfig = (*kubeletconfig_v1alpha1.KubeletConfiguration)(unsafe.Pointer(in.BaseConfig))
 	return nil
@@ -162,6 +185,9 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in
 	if err := Convert_v1alpha1_API_To_kubeadm_API(&in.API, &out.API, s); err != nil {
 		return err
 	}
+	if err := Convert_v1alpha1_KubeProxy_To_kubeadm_KubeProxy(&in.KubeProxy, &out.KubeProxy, s); err != nil {
+		return err
+	}
 	if err := Convert_v1alpha1_Etcd_To_kubeadm_Etcd(&in.Etcd, &out.Etcd, s); err != nil {
 		return err
 	}
@@ -200,6 +226,9 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in
 	if err := Convert_kubeadm_API_To_v1alpha1_API(&in.API, &out.API, s); err != nil {
 		return err
 	}
+	if err := Convert_kubeadm_KubeProxy_To_v1alpha1_KubeProxy(&in.KubeProxy, &out.KubeProxy, s); err != nil {
+		return err
+	}
 	if err := Convert_kubeadm_Etcd_To_v1alpha1_Etcd(&in.Etcd, &out.Etcd, s); err != nil {
 		return err
 	}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go

@@ -24,6 +24,7 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	kubeletconfig_v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfig_v1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -96,6 +97,31 @@ func (in *HostPathMount) DeepCopy() *HostPathMount {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxy) DeepCopyInto(out *KubeProxy) {
+	*out = *in
+	if in.Config != nil {
+		in, out := &in.Config, &out.Config
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(kubeproxyconfig_v1alpha1.KubeProxyConfiguration)
+			(*in).DeepCopyInto(*out)
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxy.
+func (in *KubeProxy) DeepCopy() *KubeProxy {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 	*out = *in
 	if in.BaseConfig != nil {
@@ -125,6 +151,7 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	out.API = in.API
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
 	in.Etcd.DeepCopyInto(&out.Etcd)
 	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
 	out.Networking = in.Networking

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.defaults.go

@@ -23,6 +23,7 @@ package v1alpha1
 import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	kubeletconfig_v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeproxyconfig_v1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 // RegisterDefaults adds defaulters functions to the given scheme.
@@ -36,6 +37,9 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
 
 func SetObjectDefaults_MasterConfiguration(in *MasterConfiguration) {
 	SetDefaults_MasterConfiguration(in)
+	if in.KubeProxy.Config != nil {
+		kubeproxyconfig_v1alpha1.SetDefaults_KubeProxyConfiguration(in.KubeProxy.Config)
+	}
 	if in.KubeletConfiguration.BaseConfig != nil {
 		kubeletconfig_v1alpha1.SetDefaults_KubeletConfiguration(in.KubeletConfiguration.BaseConfig)
 	}

cmd/kubeadm/app/apis/kubeadm/validation/BUILD

@@ -13,7 +13,10 @@ go_test(
     library = ":go_default_library",
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
+        "//pkg/util/pointer:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
     ],
 )
@@ -30,6 +33,9 @@ go_library(
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/apis/core/validation:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/validation:go_default_library",
         "//pkg/registry/core/service/ipallocator:go_default_library",
         "//pkg/util/node:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -35,6 +35,9 @@ import (
 	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
+	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
+	proxyvalidation "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/validation"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/util/node"
 )
@@ -70,9 +73,24 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
 	allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...)
+	//allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...)
 	return allErrs
 }
 
+// ValidateProxy validates proxy configuration and collects all encountered errors
+func ValidateProxy(c *kubeadm.MasterConfiguration, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	// Convert to the internal version
+	internalcfg := &kubeproxyconfig.KubeProxyConfiguration{}
+	err := kubeproxyscheme.Scheme.Convert(c.KubeProxy.Config, internalcfg, nil)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, "KubeProxy.Config", err.Error()))
+		return allErrs
+	}
+	return proxyvalidation.Validate(internalcfg)
+}
+
 // ValidateNodeConfiguration validates node configuration and collects all encountered errors
 func ValidateNodeConfiguration(c *kubeadm.NodeConfiguration) field.ErrorList {
 	allErrs := field.ErrorList{}

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -18,11 +18,15 @@ package validation
 
 import (
 	"testing"
+	"time"
 
 	"github.com/spf13/pflag"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
+	"k8s.io/kubernetes/pkg/util/pointer"
 )
 
 func TestValidateTokenDiscovery(t *testing.T) {
@@ -331,6 +335,17 @@ func TestValidateMasterConfiguration(t *testing.T) {
 					AdvertiseAddress: "1.2.3.4",
 					BindPort:         6443,
 				},
+				KubeProxy: kubeadm.KubeProxy{
+					Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
+						Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
+							Max:        pointer.Int32Ptr(2),
+							MaxPerCore: pointer.Int32Ptr(1),
+							Min:        pointer.Int32Ptr(1),
+							TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
+							TCPCloseWaitTimeout:   &metav1.Duration{Duration: 5 * time.Second},
+						},
+					},
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "10.96.0.1/12",

cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go

@@ -23,7 +23,8 @@ package kubeadm
 import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
-	v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeletconfig_v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	v1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -96,6 +97,31 @@ func (in *HostPathMount) DeepCopy() *HostPathMount {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxy) DeepCopyInto(out *KubeProxy) {
+	*out = *in
+	if in.Config != nil {
+		in, out := &in.Config, &out.Config
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(v1alpha1.KubeProxyConfiguration)
+			(*in).DeepCopyInto(*out)
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxy.
+func (in *KubeProxy) DeepCopy() *KubeProxy {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 	*out = *in
 	if in.BaseConfig != nil {
@@ -103,7 +129,7 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 		if *in == nil {
 			*out = nil
 		} else {
-			*out = new(v1alpha1.KubeletConfiguration)
+			*out = new(kubeletconfig_v1alpha1.KubeletConfiguration)
 			(*in).DeepCopyInto(*out)
 		}
 	}
@@ -125,6 +151,7 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	out.API = in.API
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
 	in.Etcd.DeepCopyInto(&out.Etcd)
 	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
 	out.Networking = in.Networking

cmd/kubeadm/app/cmd/upgrade/common_test.go

@@ -51,6 +51,7 @@ func TestPrintConfiguration(t *testing.T) {
 	  image: ""
 	  keyFile: ""
 	imageRepository: ""
+	kubeProxy: {}
 	kubeletConfiguration:
 	  baseConfig: null
 	kubernetesVersion: v1.7.1
@@ -84,6 +85,7 @@ func TestPrintConfiguration(t *testing.T) {
 	  image: ""
 	  keyFile: ""
 	imageRepository: ""
+	kubeProxy: {}
 	kubeletConfiguration:
 	  baseConfig: null
 	kubernetesVersion: v1.7.1
@@ -127,6 +129,7 @@ func TestPrintConfiguration(t *testing.T) {
 	    etcdVersion: v0.1.0
 	    operatorVersion: v0.1.0
 	imageRepository: ""
+	kubeProxy: {}
 	kubeletConfiguration:
 	  baseConfig: null
 	kubernetesVersion: v1.7.1

cmd/kubeadm/app/constants/constants.go

@@ -194,7 +194,6 @@ const (
 )
 
 var (
-
 	// MasterTaint is the taint to apply on the PodSpec for being able to run that Pod on the master
 	MasterTaint = v1.Taint{
 		Key:    LabelNodeRoleMaster,
@@ -224,6 +223,9 @@ var (
 	// MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports
 	MinimumKubeletVersion = version.MustParseSemantic("v1.8.0")
 
+	// MinimumKubeProxyComponentConfigVersion specifies the minimum version for the kubeProxyComponent
+	MinimumKubeProxyComponentConfigVersion = version.MustParseSemantic("v1.9.0-alpha.3")
+
 	// SupportedEtcdVersion lists officially supported etcd versions with corresponding kubernetes releases
 	SupportedEtcdVersion = map[uint8]string{
 		8: "3.0.17",

cmd/kubeadm/app/phases/addons/proxy/BUILD

@@ -12,9 +12,14 @@ go_test(
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy",
     library = ":go_default_library",
     deps = [
+        "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/config:go_default_library",
         "//pkg/apis/core:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
+        "//pkg/util/pointer:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//vendor/k8s.io/client-go/testing:go_default_library",
@@ -35,6 +40,9 @@ go_library(
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//plugin/pkg/scheduler/algorithm:go_default_library",
         "//vendor/k8s.io/api/apps/v1beta2:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

cmd/kubeadm/app/phases/addons/proxy/manifests.go

@@ -17,8 +17,8 @@ limitations under the License.
 package proxy
 
 const (
-	// KubeProxyConfigMap is the proxy ConfigMap manifest
-	KubeProxyConfigMap = `
+	// KubeProxyConfigMap18 is the proxy ConfigMap manifest for Kubernetes version 1.8
+	KubeProxyConfigMap18 = `
 kind: ConfigMap
 apiVersion: v1
 metadata:
@@ -48,8 +48,40 @@ data:
         tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
 `
 
-	// KubeProxyDaemonSet is the proxy DaemonSet manifest
-	KubeProxyDaemonSet = `
+	// KubeProxyConfigMap19 is the proxy ConfigMap manifest for Kubernetes 1.9 and above
+	KubeProxyConfigMap19 = `
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+  labels:
+    app: kube-proxy
+data:
+  kubeconfig.conf: |-
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - cluster:
+        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        server: {{ .MasterEndpoint }}
+      name: default
+    contexts:
+    - context:
+        cluster: default
+        namespace: default
+        user: default
+      name: default
+    current-context: default
+    users:
+    - name: default
+      user:
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  config.conf: |-
+{{ .ProxyConfig}}
+`
+	// KubeProxyDaemonSet18 is the proxy DaemonSet manifest for Kubernetes version 1.8
+	KubeProxyDaemonSet18 = `
 apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
@@ -108,4 +140,63 @@ spec:
         hostPath:
           path: /lib/modules
 `
+
+	// KubeProxyDaemonSet19 is the proxy DaemonSet manifest for Kubernetes 1.9 and above
+	KubeProxyDaemonSet19 = `
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-proxy
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-proxy
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-proxy
+    spec:
+      containers:
+      - name: kube-proxy
+        image: {{ if .ImageOverride }}{{ .ImageOverride }}{{ else }}{{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }}{{ end }}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /usr/local/bin/kube-proxy
+        - --config=/var/lib/kube-proxy/config.conf
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kube-proxy
+          name: kube-proxy
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+          readOnly: false
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+      hostNetwork: true
+      serviceAccountName: kube-proxy
+      tolerations:
+      - key: {{ .MasterTaintKey }}
+        effect: NoSchedule
+      - key: {{ .CloudTaintKey }}
+        value: "true"
+        effect: NoSchedule
+      volumes:
+      - name: kube-proxy
+        configMap:
+          name: kube-proxy
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+`
 )

cmd/kubeadm/app/phases/addons/proxy/proxy.go

@@ -32,6 +32,9 @@ import (
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
+	"k8s.io/kubernetes/pkg/util/version"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
 )
 
@@ -56,27 +59,65 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
 		return err
 	}
 
-	proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{
-		// Fetch this value from the kubeconfig file
-		MasterEndpoint: masterEndpoint})
+	proxyBytes, err := kubeadmutil.MarshalToYamlForCodecsWithShift(cfg.KubeProxy.Config, kubeproxyconfigv1alpha1.SchemeGroupVersion,
+		kubeproxyconfigscheme.Codecs)
 	if err != nil {
-		return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
+		return fmt.Errorf("error when marshaling: %v", err)
 	}
-
-	proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ ImageRepository, Arch, Version, ImageOverride, ExtraParams, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{
-		ImageRepository: cfg.GetControlPlaneImageRepository(),
-		Arch:            runtime.GOARCH,
-		Version:         kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
-		ImageOverride:   cfg.UnifiedControlPlaneImage,
-		ExtraParams:     getParams(cfg.FeatureGates),
-		ClusterCIDR:     getClusterCIDR(cfg.Networking.PodSubnet),
-		MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster,
-		CloudTaintKey:   algorithm.TaintExternalCloudProvider,
-	})
+	// Parse the given kubernetes version
+	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
 	if err != nil {
-		return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
+		return fmt.Errorf("couldn't parse kubernetes version %q: %v", cfg.KubernetesVersion, err)
+	}
+	var proxyConfigMapBytes, proxyDaemonSetBytes []byte
+	if k8sVersion.AtLeast(kubeadmconstants.MinimumKubeProxyComponentConfigVersion) {
+		proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19,
+			struct {
+				MasterEndpoint string
+				ProxyConfig    string
+			}{
+				MasterEndpoint: masterEndpoint,
+				ProxyConfig:    proxyBytes,
+			})
+		if err != nil {
+			return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
+		}
+		proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride, ExtraParams, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{
+			ImageRepository: cfg.GetControlPlaneImageRepository(),
+			Arch:            runtime.GOARCH,
+			Version:         kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
+			ImageOverride:   cfg.UnifiedControlPlaneImage,
+			MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster,
+			CloudTaintKey:   algorithm.TaintExternalCloudProvider,
+		})
+		if err != nil {
+			return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
+		}
+	} else {
+		proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap18,
+			struct {
+				MasterEndpoint string
+			}{
+				MasterEndpoint: masterEndpoint,
+			})
+		if err != nil {
+			return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
+		}
+
+		proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet18, struct{ ImageRepository, Arch, Version, ImageOverride, ExtraParams, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{
+			ImageRepository: cfg.GetControlPlaneImageRepository(),
+			Arch:            runtime.GOARCH,
+			Version:         kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
+			ImageOverride:   cfg.UnifiedControlPlaneImage,
+			ExtraParams:     getParams(cfg.FeatureGates),
+			ClusterCIDR:     getClusterCIDR(cfg.Networking.PodSubnet),
+			MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster,
+			CloudTaintKey:   algorithm.TaintExternalCloudProvider,
+		})
+		if err != nil {
+			return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
+		}
 	}
-
 	if err := createKubeProxyAddon(proxyConfigMapBytes, proxyDaemonSetBytes, client); err != nil {
 		return err
 	}

cmd/kubeadm/app/phases/addons/proxy/proxy_test.go

@@ -17,14 +17,21 @@ limitations under the License.
 package proxy
 
 import (
+	"strings"
 	"testing"
+	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientsetfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
+	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
+	"k8s.io/kubernetes/pkg/util/pointer"
 )
 
 func TestCreateServiceAccount(t *testing.T) {
@@ -107,14 +114,26 @@ func TestCompileManifests(t *testing.T) {
 		expected bool
 	}{
 		{
-			manifest: KubeProxyConfigMap,
-			data: struct{ MasterEndpoint string }{
+			manifest: KubeProxyConfigMap18,
+			data: struct {
+				MasterEndpoint, ProxyConfig string
+			}{
 				MasterEndpoint: "foo",
 			},
 			expected: true,
 		},
 		{
-			manifest: KubeProxyDaemonSet,
+			manifest: KubeProxyConfigMap19,
+			data: struct {
+				MasterEndpoint, ProxyConfig string
+			}{
+				MasterEndpoint: "foo",
+				ProxyConfig:    "  bindAddress: 0.0.0.0\n  clusterCIDR: 192.168.1.1\n  enableProfiling: false",
+			},
+			expected: true,
+		},
+		{
+			manifest: KubeProxyDaemonSet18,
 			data: struct{ ImageRepository, Arch, Version, ImageOverride, ExtraParams, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{
 				ImageRepository: "foo",
 				Arch:            "foo",
@@ -127,15 +146,149 @@ func TestCompileManifests(t *testing.T) {
 			},
 			expected: true,
 		},
+		{
+			manifest: KubeProxyDaemonSet19,
+			data: struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey, CloudTaintKey string }{
+				ImageRepository: "foo",
+				Arch:            "foo",
+				Version:         "foo",
+				ImageOverride:   "foo",
+				MasterTaintKey:  "foo",
+				CloudTaintKey:   "foo",
+			},
+			expected: true,
+		},
 	}
 	for _, rt := range tests {
 		_, actual := kubeadmutil.ParseTemplate(rt.manifest, rt.data)
 		if (actual == nil) != rt.expected {
 			t.Errorf(
-				"failed CompileManifests:\n\texpected: %t\n\t  actual: %t",
+				"failed to compile %s manifest:\n\texpected: %t\n\t  actual: %t",
+				rt.manifest,
 				rt.expected,
 				(actual == nil),
 			)
 		}
 	}
 }
+
+func TestEnsureProxyAddon(t *testing.T) {
+	type SimulatedError int
+	const (
+		NoError SimulatedError = iota
+		ServiceAccountError
+		InvalidMasterEndpoint
+		IPv6SetBindAddress
+	)
+
+	var testCases = []struct {
+		name           string
+		simError       SimulatedError
+		expErrString   string
+		expBindAddr    string
+		expClusterCIDR string
+	}{
+		{
+			name:           "Successful proxy addon",
+			simError:       NoError,
+			expErrString:   "",
+			expBindAddr:    "0.0.0.0",
+			expClusterCIDR: "5.6.7.8/24",
+		}, {
+			name:           "Simulated service account error",
+			simError:       ServiceAccountError,
+			expErrString:   "error when creating kube-proxy service account",
+			expBindAddr:    "0.0.0.0",
+			expClusterCIDR: "5.6.7.8/24",
+		}, {
+			name:           "IPv6 AdvertiseAddress address",
+			simError:       IPv6SetBindAddress,
+			expErrString:   "",
+			expBindAddr:    "::",
+			expClusterCIDR: "2001:101::/96",
+		},
+	}
+
+	for _, tc := range testCases {
+
+		// Create a fake client and set up default test configuration
+		client := clientsetfake.NewSimpleClientset()
+
+		masterConfig := &kubeadmapiext.MasterConfiguration{
+			API: kubeadmapiext.API{
+				AdvertiseAddress: "1.2.3.4",
+				BindPort:         1234,
+			},
+			KubeProxy: kubeadmapiext.KubeProxy{
+				Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
+					BindAddress:        "",
+					HealthzBindAddress: "0.0.0.0:10256",
+					MetricsBindAddress: "127.0.0.1:10249",
+					Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
+						Max:        pointer.Int32Ptr(2),
+						MaxPerCore: pointer.Int32Ptr(1),
+						Min:        pointer.Int32Ptr(1),
+						TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
+						TCPCloseWaitTimeout:   &metav1.Duration{Duration: 5 * time.Second},
+					},
+				},
+			},
+			Networking: kubeadmapiext.Networking{
+				PodSubnet: "5.6.7.8/24",
+			},
+			ImageRepository:          "someRepo",
+			KubernetesVersion:        "v1.9.0",
+			UnifiedControlPlaneImage: "someImage",
+		}
+
+		// Simulate an error if neccessary
+		switch tc.simError {
+		case ServiceAccountError:
+			client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) {
+				return true, nil, apierrors.NewUnauthorized("")
+			})
+		case InvalidMasterEndpoint:
+			masterConfig.API.AdvertiseAddress = "1.2.3"
+		case IPv6SetBindAddress:
+			masterConfig.API.AdvertiseAddress = "1:2::3:4"
+			masterConfig.Networking.PodSubnet = "2001:101::/96"
+		}
+
+		kubeadmapiext.SetDefaults_MasterConfiguration(masterConfig)
+		intMaster, err := cmdutil.ConfigFileAndDefaultsToInternalConfig("", masterConfig)
+		if err != nil {
+			t.Errorf(" test failed to convert v1alpha1 to internal version")
+			break
+		}
+		err = EnsureProxyAddon(intMaster, client)
+
+		// Compare actual to expected errors
+		actErr := "No error"
+		if err != nil {
+			actErr = err.Error()
+		}
+		expErr := "No error"
+		if tc.expErrString != "" {
+			expErr = tc.expErrString
+		}
+		if !strings.Contains(actErr, expErr) {
+			t.Errorf(
+				"%s test failed, expected: %s, got: %s",
+				tc.name,
+				expErr,
+				actErr)
+		}
+		if intMaster.KubeProxy.Config.BindAddress != tc.expBindAddr {
+			t.Errorf("%s test failed, expected: %s, got: %s",
+				tc.name,
+				tc.expBindAddr,
+				intMaster.KubeProxy.Config.BindAddress)
+		}
+		if intMaster.KubeProxy.Config.ClusterCIDR != tc.expClusterCIDR {
+			t.Errorf("%s test failed, expected: %s, got: %s",
+				tc.name,
+				tc.expClusterCIDR,
+				intMaster.KubeProxy.Config.ClusterCIDR)
+		}
+	}
+}

cmd/kubeadm/app/util/config/masterconfig.go

@@ -44,7 +44,12 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 		return err
 	}
 	cfg.API.AdvertiseAddress = ip.String()
-
+	ip = net.ParseIP(cfg.API.AdvertiseAddress)
+	if ip.To4() != nil {
+		cfg.KubeProxy.Config.BindAddress = kubeadmapiext.DefaultProxyBindAddressv4
+	} else {
+		cfg.KubeProxy.Config.BindAddress = kubeadmapiext.DefaultProxyBindAddressv6
+	}
 	// Resolve possible version labels and validate version string
 	err = NormalizeKubernetesVersion(cfg)
 	if err != nil {

cmd/kubeadm/app/util/marshal.go

@@ -18,6 +18,7 @@ package util
 
 import (
 	"fmt"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -41,3 +42,18 @@ func MarshalToYamlForCodecs(obj runtime.Object, gv schema.GroupVersion, codecs s
 	encoder := codecs.EncoderForVersion(info.Serializer, gv)
 	return runtime.Encode(encoder, obj)
 }
+
+// MarshalToYamlForCodecsWithShift adds spaces in front of each line so the indents line up
+// correctly in the manifest
+func MarshalToYamlForCodecsWithShift(obj runtime.Object, gv schema.GroupVersion, codecs serializer.CodecFactory) (string, error) {
+	serial, err := MarshalToYamlForCodecs(obj, gv, codecs)
+	if err != nil {
+		return "", err
+	}
+	lines := strings.Split(string(serial), "\n")
+	var newSerial string
+	for _, line := range lines {
+		newSerial = newSerial + "    " + line + "\n"
+	}
+	return newSerial, err
+}