Skip to content

K
k3s
Commit a9f5940c authored by Darren Shepherd's avatar Darren Shepherd
Browse files
Options

Delete cloud provider

parent d4f4602f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 4588 deletions
cmd/controller-manager/app/options/globalflags.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"github.com/spf13/pflag"
"k8s.io/apiserver/pkg/util/globalflag"
// ensure libs have a chance to globally register their flags
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
)
// AddCustomGlobalFlags explicitly registers flags that internal packages register
// against the global flagsets from "flag". We do this in order to prevent
// unwanted flags from leaking into the *-controller-manager's flagset.
func AddCustomGlobalFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
// adds flags from k8s.io/kubernetes/pkg/cloudprovider/providers
globalflag.Register(fs, "cloud-provider-gce-lb-src-cidrs")
}
cmd/kube-apiserver/app/options/globalflags.go
View file @ a9f5940c
......@@ -23,7 +23,6 @@ import (
// ensure libs have a chance to globally register their flags
_ "k8s.io/apiserver/pkg/admission"
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
)
// AddCustomGlobalFlags explicitly registers flags that internal packages register
......@@ -32,9 +31,6 @@ import (
func AddCustomGlobalFlags(fs *pflag.FlagSet) {
// Lookup flags in global flag set and re-register the values with our flagset.
// Adds flags from k8s.io/kubernetes/pkg/cloudprovider/providers.
globalflag.Register(fs, "cloud-provider-gce-lb-src-cidrs")
// Adds flags from k8s.io/apiserver/pkg/admission.
globalflag.Register(fs, "default-not-ready-toleration-seconds")
globalflag.Register(fs, "default-unreachable-toleration-seconds")
......
cmd/kube-apiserver/app/server.go
View file @ a9f5940c
......@@ -52,7 +52,6 @@ import (
clientgoinformers "k8s.io/client-go/informers"
clientgoclientset "k8s.io/client-go/kubernetes"
certutil "k8s.io/client-go/util/cert"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
......@@ -223,15 +222,6 @@ func CreateNodeDialer(s completedServerRunOptions) (tunneler.Tunneler, *http.Tra
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSHKey tunneler.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile)
if err != nil {
return nil, nil, fmt.Errorf("cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSHKey = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
return nil, nil, fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified")
}
......
cmd/kube-controller-manager/app/cloudproviders.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"fmt"
"k8s.io/klog"
"k8s.io/client-go/informers"
cloudprovider "k8s.io/cloud-provider"
)
// createCloudProvider helps consolidate what is needed for cloud providers, we explicitly list the things
// that the cloud providers need as parameters, so we can control
func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string, cloudConfigFile string,
allowUntaggedCloud bool, sharedInformers informers.SharedInformerFactory) (cloudprovider.Interface, ControllerLoopMode, error) {
var cloud cloudprovider.Interface
var loopMode ControllerLoopMode
var err error
if cloudprovider.IsExternal(cloudProvider) {
loopMode = ExternalLoops
if externalCloudVolumePlugin == "" {
// externalCloudVolumePlugin is temporary until we split all cloud providers out.
// So we just tell the caller that we need to run ExternalLoops without any cloud provider.
return nil, loopMode, nil
}
cloud, err = cloudprovider.InitCloudProvider(externalCloudVolumePlugin, cloudConfigFile)
} else {
loopMode = IncludeCloudLoops
cloud, err = cloudprovider.InitCloudProvider(cloudProvider, cloudConfigFile)
}
if err != nil {
return nil, loopMode, fmt.Errorf("cloud provider could not be initialized: %v", err)
}
if cloud != nil && cloud.HasClusterID() == false {
if allowUntaggedCloud == true {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
return nil, loopMode, fmt.Errorf("no ClusterID Found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(sharedInformers)
}
return cloud, loopMode, err
}
cmd/kube-controller-manager/app/controllermanager.go
View file @ a9f5940c
......@@ -48,10 +48,8 @@ import (
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
certutil "k8s.io/client-go/util/cert"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
"k8s.io/kubernetes/pkg/controller"
......@@ -71,13 +69,6 @@ const (
ConfigzName = "kubecontrollermanager.config.k8s.io"
)
type ControllerLoopMode int
const (
IncludeCloudLoops ControllerLoopMode = iota
ExternalLoops
)
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
s, err := options.NewKubeControllerManagerOptions()
......@@ -116,7 +107,6 @@ controller, and serviceaccounts controller.`,
namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
cmoptions.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
......@@ -209,7 +199,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
}
saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController
if err := StartControllers(controllerContext, saTokenControllerInitFunc, NewControllerInitializers(controllerContext.LoopMode), unsecuredMux); err != nil {
if err := StartControllers(controllerContext, saTokenControllerInitFunc, NewControllerInitializers(), unsecuredMux); err != nil {
klog.Fatalf("error starting controllers: %v", err)
}
......@@ -278,15 +268,6 @@ type ControllerContext struct {
// AvailableResources is a map listing currently available resources
AvailableResources map[schema.GroupVersionResource]bool
// Cloud is the cloud provider interface for the controllers to use.
// It must be initialized and ready to use.
Cloud cloudprovider.Interface
// Control for which control loops to be run
// IncludeCloudLoops is for a kube-controller-manager running all loops
// ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager
LoopMode ControllerLoopMode
// Stop is the stop channel
Stop <-chan struct{}
......@@ -335,7 +316,7 @@ func IsControllerEnabled(name string, disabledByDefaultControllers sets.String,
type InitFunc func(ctx ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)
func KnownControllers() []string {
ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops))
ret := sets.StringKeySet(NewControllerInitializers())
// add "special" controllers that aren't initialized normally. These controllers cannot be initialized
// using a normal function. The only known special case is the SA token controller which *must* be started
......@@ -359,7 +340,7 @@ const (
// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
func NewControllerInitializers() map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["endpoint"] = startEndpointController
controllers["replicationcontroller"] = startReplicationController
......@@ -381,11 +362,7 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
controllers["csrcleaner"] = startCSRCleanerController
controllers["ttl"] = startTTLController
controllers["nodeipam"] = startNodeIpamController
if loopMode == IncludeCloudLoops {
controllers["service"] = startServiceController
// TODO: volume controller into the IncludeCloudLoops only set.
// TODO: Separate cluster in cloud check from node lifecycle controller.
}
controllers["service"] = startServiceController
controllers["nodelifecycle"] = startNodeLifecycleController
controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
controllers["attachdetach"] = startAttachDetachController
......@@ -452,20 +429,12 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
return ControllerContext{}, err
}
cloud, loopMode, err := createCloudProvider(s.ComponentConfig.KubeCloudShared.CloudProvider.Name, s.ComponentConfig.KubeCloudShared.ExternalCloudVolumePlugin,
s.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers)
if err != nil {
return ControllerContext{}, err
}
ctx := ControllerContext{
ClientBuilder: clientBuilder,
InformerFactory: sharedInformers,
ComponentConfig: s.ComponentConfig,
RESTMapper: restMapper,
AvailableResources: availableResources,
Cloud: cloud,
LoopMode: loopMode,
Stop: stop,
InformersStarted: make(chan struct{}),
ResyncPeriod: ResyncPeriod(s),
......@@ -480,12 +449,6 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co
return err
}
// Initialize the cloud provider with a reference to the clientBuilder only after token controller
// has started in case the cloud provider uses the client builder.
if ctx.Cloud != nil {
ctx.Cloud.Initialize(ctx.ClientBuilder, ctx.Stop)
}
for controllerName, initFn := range controllers {
if !ctx.IsControllerEnabled(controllerName) {
klog.Warningf("%q is disabled", controllerName)
......
cmd/kube-controller-manager/app/core.go
View file @ a9f5940c
......@@ -63,7 +63,6 @@ import (
func startServiceController(ctx ControllerContext) (http.Handler, bool, error) {
serviceController, err := servicecontroller.New(
ctx.Cloud,
ctx.ClientBuilder.ClientOrDie("service-controller"),
ctx.InformerFactory.Core().V1().Services(),
ctx.InformerFactory.Core().V1().Nodes(),
......@@ -103,7 +102,6 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error)
nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
ctx.InformerFactory.Core().V1().Nodes(),
ctx.Cloud,
ctx.ClientBuilder.ClientOrDie("node-controller"),
clusterCIDR,
serviceCIDR,
......@@ -122,7 +120,6 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
ctx.InformerFactory.Core().V1().Pods(),
ctx.InformerFactory.Core().V1().Nodes(),
ctx.InformerFactory.Extensions().V1beta1().DaemonSets(),
ctx.Cloud,
ctx.ClientBuilder.ClientOrDie("node-controller"),
ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
ctx.ComponentConfig.NodeLifecycleController.NodeStartupGracePeriod.Duration,
......@@ -147,8 +144,7 @@ func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler,
params := persistentvolumecontroller.ControllerParameters{
KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"),
SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration,
VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
Cloud: ctx.Cloud,
VolumePlugins: ProbeControllerVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
ClusterName: ctx.ComponentConfig.KubeCloudShared.ClusterName,
VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(),
ClaimInformer: ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
......@@ -181,7 +177,6 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
ctx.InformerFactory.Core().V1().Nodes(),
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ctx.InformerFactory.Core().V1().PersistentVolumes(),
ctx.Cloud,
ProbeAttachableVolumePlugins(),
GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync,
......@@ -201,7 +196,6 @@ func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, err
ctx.ClientBuilder.ClientOrDie("expand-controller"),
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ctx.InformerFactory.Core().V1().PersistentVolumes(),
ctx.Cloud,
ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration))
if expandControllerErr != nil {
......
cmd/kube-controller-manager/app/plugins.go
View file @ a9f5940c
......@@ -25,33 +25,23 @@ import (
"k8s.io/klog"
// Cloud providers
cloudprovider "k8s.io/cloud-provider"
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
// Volume plugins
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/awsebs"
"k8s.io/kubernetes/pkg/volume/azure_dd"
"k8s.io/kubernetes/pkg/volume/azure_file"
"k8s.io/kubernetes/pkg/volume/cinder"
"k8s.io/kubernetes/pkg/volume/csi"
"k8s.io/kubernetes/pkg/volume/fc"
"k8s.io/kubernetes/pkg/volume/flexvolume"
"k8s.io/kubernetes/pkg/volume/flocker"
"k8s.io/kubernetes/pkg/volume/gcepd"
"k8s.io/kubernetes/pkg/volume/glusterfs"
"k8s.io/kubernetes/pkg/volume/host_path"
"k8s.io/kubernetes/pkg/volume/iscsi"
"k8s.io/kubernetes/pkg/volume/local"
"k8s.io/kubernetes/pkg/volume/nfs"
"k8s.io/kubernetes/pkg/volume/photon_pd"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/quobyte"
"k8s.io/kubernetes/pkg/volume/rbd"
"k8s.io/kubernetes/pkg/volume/scaleio"
"k8s.io/kubernetes/pkg/volume/storageos"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
utilfeature "k8s.io/apiserver/pkg/util/feature"
kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
......@@ -66,13 +56,7 @@ import (
func ProbeAttachableVolumePlugins() []volume.VolumePlugin {
allPlugins := []volume.VolumePlugin{}
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
......@@ -95,16 +79,9 @@ func GetDynamicPluginProber(config kubectrlmgrconfig.VolumeConfiguration) volume
func ProbeExpandableVolumePlugins(config kubectrlmgrconfig.VolumeConfiguration) []volume.VolumePlugin {
allPlugins := []volume.VolumePlugin{}
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
......@@ -114,7 +91,7 @@ func ProbeExpandableVolumePlugins(config kubectrlmgrconfig.VolumeConfiguration)
// ProbeControllerVolumePlugins collects all persistent volume plugins into an
// easy to use list. Only volume plugins that implement any of
// provisioner/recycler/deleter interface should be returned.
func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config kubectrlmgrconfig.VolumeConfiguration) []volume.VolumePlugin {
func ProbeControllerVolumePlugins(config kubectrlmgrconfig.VolumeConfiguration) []volume.VolumePlugin {
allPlugins := []volume.VolumePlugin{}
// The list of plugins to probe is decided by this binary, not
......@@ -150,7 +127,6 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config kubectrl
// add rbd provisioner
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
......@@ -158,13 +134,6 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config kubectrl
allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
return allPlugins
}
......
cmd/kubelet/app/options/globalflags.go
View file @ a9f5940c
......@@ -30,8 +30,6 @@ import (
// ensure libs have a chance to globally register their flags
_ "k8s.io/klog"
_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
)
// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
......@@ -40,7 +38,6 @@ import (
func AddGlobalFlags(fs *pflag.FlagSet) {
addGlogFlags(fs)
addCadvisorFlags(fs)
addCredentialProviderFlags(fs)
verflag.AddFlags(fs)
logs.AddFlags(fs)
}
......@@ -62,35 +59,12 @@ func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) {
}
}
// pflagRegister adds a flag to local that targets the Value associated with the Flag named globalName in global
func pflagRegister(global, local *pflag.FlagSet, globalName string) {
if f := global.Lookup(globalName); f != nil {
f.Name = normalize(f.Name)
local.AddFlag(f)
} else {
panic(fmt.Sprintf("failed to find flag in global flagset (pflag): %s", globalName))
}
}
// registerDeprecated registers the flag with register, and then marks it deprecated
func registerDeprecated(global *flag.FlagSet, local *pflag.FlagSet, globalName, deprecated string) {
register(global, local, globalName)
local.Lookup(normalize(globalName)).Deprecated = deprecated
}
// addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
func addCredentialProviderFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
global := pflag.CommandLine
local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
// TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key.
// We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this.
pflagRegister(global, local, "azure-container-registry-config")
fs.AddFlagSet(local)
}
// addGlogFlags adds flags from k8s.io/klog
func addGlogFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
......
cmd/kubelet/app/options/options.go
View file @ a9f5940c
......@@ -91,14 +91,6 @@ type KubeletFlags struct {
// are provided, this flag will be ignored.
CertDirectory string
// cloudProvider is the provider for cloud services.
// +optional
CloudProvider string
// cloudConfigFile is the path to the cloud provider configuration file.
// +optional
CloudConfigFile string
// rootDirectory is the directory path to place kubelet files (volume
// mounts,etc).
RootDirectory string
......@@ -395,9 +387,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. Specify empty string for running with no cloud provider. If set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. The DynamicKubeletConfig feature gate must be enabled to pass this flag; this gate currently defaults to true because the feature is beta.")
......
cmd/kubelet/app/plugins.go
View file @ a9f5940c
......@@ -18,19 +18,10 @@ package app
// This file exists to force the desired plugin implementations to be linked.
import (
// Credential providers
_ "k8s.io/kubernetes/pkg/credentialprovider/aws"
_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
_ "k8s.io/kubernetes/pkg/credentialprovider/rancher"
"k8s.io/utils/exec"
// Volume plugins
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/awsebs"
"k8s.io/kubernetes/pkg/volume/azure_dd"
"k8s.io/kubernetes/pkg/volume/azure_file"
"k8s.io/kubernetes/pkg/volume/cephfs"
"k8s.io/kubernetes/pkg/volume/cinder"
"k8s.io/kubernetes/pkg/volume/configmap"
"k8s.io/kubernetes/pkg/volume/csi"
"k8s.io/kubernetes/pkg/volume/downwardapi"
......@@ -38,14 +29,12 @@ import (
"k8s.io/kubernetes/pkg/volume/fc"
"k8s.io/kubernetes/pkg/volume/flexvolume"
"k8s.io/kubernetes/pkg/volume/flocker"
"k8s.io/kubernetes/pkg/volume/gcepd"
"k8s.io/kubernetes/pkg/volume/git_repo"
"k8s.io/kubernetes/pkg/volume/glusterfs"
"k8s.io/kubernetes/pkg/volume/host_path"
"k8s.io/kubernetes/pkg/volume/iscsi"
"k8s.io/kubernetes/pkg/volume/local"
"k8s.io/kubernetes/pkg/volume/nfs"
"k8s.io/kubernetes/pkg/volume/photon_pd"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/projected"
"k8s.io/kubernetes/pkg/volume/quobyte"
......@@ -53,9 +42,6 @@ import (
"k8s.io/kubernetes/pkg/volume/scaleio"
"k8s.io/kubernetes/pkg/volume/secret"
"k8s.io/kubernetes/pkg/volume/storageos"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
// Cloud providers
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
// features check
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
......@@ -71,9 +57,7 @@ func ProbeVolumePlugins() []volume.VolumePlugin {
//
// Kubelet does not currently need to configure volume plugins.
// If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, host_path.ProbeVolumePlugins(volume.VolumeConfig{})...)
allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
......@@ -81,17 +65,12 @@ func ProbeVolumePlugins() []volume.VolumePlugin {
allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
......
cmd/kubelet/app/server.go
View file @ a9f5940c
......@@ -18,7 +18,6 @@ limitations under the License.
package app
import (
"context"
"crypto/tls"
"errors"
"fmt"
......@@ -56,7 +55,6 @@ import (
"k8s.io/client-go/tools/record"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
"k8s.io/kubernetes/cmd/kubelet/app/options"
......@@ -383,7 +381,6 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
return &kubelet.Dependencies{
Auth: nil, // default does not enforce auth[nz]
CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here
Cloud: nil, // cloud provider might start background processes
ContainerManager: nil,
DockerClientConfig: dockerClientConfig,
KubeClient: nil,
......@@ -513,26 +510,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
}
}
if kubeDeps.Cloud == nil {
if !cloudprovider.IsExternal(s.CloudProvider) {
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
return err
}
if cloud == nil {
klog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
} else {
klog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
}
kubeDeps.Cloud = cloud
}
}
hostName, err := nodeutil.GetHostname(s.HostnameOverride)
if err != nil {
return err
}
nodeName, err := getNodeName(kubeDeps.Cloud, hostName)
nodeName, err := getNodeName(hostName)
if err != nil {
return err
}
......@@ -766,24 +748,8 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
// getNodeName returns the node name according to the cloud provider
// if cloud provider is specified. Otherwise, returns the hostname of the node.
func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
if cloud == nil {
return types.NodeName(hostname), nil
}
instances, ok := cloud.Instances()
if !ok {
return "", fmt.Errorf("failed to get instances from cloud provider")
}
nodeName, err := instances.CurrentNodeName(context.TODO(), hostname)
if err != nil {
return "", fmt.Errorf("error fetching current node name from cloud provider: %v", err)
}
klog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
return nodeName, nil
func getNodeName(hostname string) (types.NodeName, error) {
return types.NodeName(hostname), nil
}
// InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
......@@ -896,7 +862,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
return err
}
// Query the cloud provider for our node name, default to hostname if kubeDeps.Cloud == nil
nodeName, err := getNodeName(kubeDeps.Cloud, hostname)
nodeName, err := getNodeName(hostname)
if err != nil {
return err
}
......@@ -945,7 +911,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
kubeServer.HostnameOverride,
kubeServer.NodeIP,
kubeServer.ProviderID,
kubeServer.CloudProvider,
kubeServer.CertDirectory,
kubeServer.RootDirectory,
kubeServer.RegisterNode,
......@@ -1018,7 +983,6 @@ func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostnameOverride string,
nodeIP string,
providerID string,
cloudProvider string,
certDirectory string,
rootDirectory string,
registerNode bool,
......@@ -1052,7 +1016,6 @@ func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostnameOverride,
nodeIP,
providerID,
cloudProvider,
certDirectory,
rootDirectory,
registerNode,
......
pkg/cloudprovider/BUILD deleted 100644 → 0
View file @ d4f4602f
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["doc.go"],
importpath = "k8s.io/kubernetes/pkg/cloudprovider",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/cloudprovider/providers:all-srcs",
],
tags = ["automanaged"],
)
pkg/cloudprovider/OWNERS deleted 100644 → 0
View file @ d4f4602f
approvers:
- mikedanese
- dims
- wlan0
- andrewsykim
reviewers:
- thockin
- lavalamp
- smarterclayton
- wojtek-t
- deads2k
- brendandburns
- derekwaynecarr
- caesarxuchao
- vishh
- mikedanese
- liggitt
- gmarek
- erictune
- davidopp
- pmorie
- sttts
- quinton-hoole
- dchen1107
- saad-ali
- zmerlynn
- luxas
- justinsb
- roberthbailey
- eparis
- jlowdermilk
- piosz
- jsafrane
- dims
- krousey
- rootfs
- jszczepkowski
- markturansky
- girishkalele
- jdef
- freehan
- jingxu97
- wlan0
- cheftako
- andrewsykim
labels:
- sig/cloud-provider
pkg/cloudprovider/README.md deleted 100644 → 0
View file @ d4f4602f
##### Deprecation Notice: cloud providers in this directory are deprecated and will be removed in favor of external (a.k.a out-of-tree) providers. Existing providers in this directory (a.k.a in-tree providers) should only make small incremental changes as needed and avoid large refactors or new features. New providers seeking to support Kubernetes should follow the out-of-tree model as specified in the [Running Kubernetes Cloud Controller Manager docs](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/). For more information on the future of Kubernetes cloud providers see [KEP-0002](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0002-cloud-controller-manager.md) and [KEP-0013](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0013-build-deploy-ccm.md).
Cloud Providers in this directory will continue to be actively developed or maintained and supported at their current level of support as a longer-term solution evolves.
## Overview:
The mechanism for supporting cloud providers is currently in transition: the original method of implementing cloud provider-specific functionality within the main kubernetes tree (here) is no longer advised; however, the proposed solution is still in development.
#### Guidance for potential cloud providers:
* Support for cloud providers is currently in a state of flux. Background information on motivation and the proposal for improving is in the github [proposal](https://git.k8s.io/community/contributors/design-proposals/cloud-provider/cloud-provider-refactoring.md).
* In support of this plan, a new cloud-controller-manager binary was added in 1.6. This was the first of several steps (see the proposal for more information).
* Attempts to contribute new cloud providers or (to a lesser extent) persistent volumes to the core repo will likely meet with some pushback from reviewers/approvers.
* It is understood that this is an unfortunate situation in which 'the old way is no longer supported but the new way is not ready yet', but the initial path is unsustainable, and contributors are encouraged to participate in the implementation of the proposed long-term solution, as there is risk that PRs for new cloud providers here will not be approved.
* Though the fully productized support envisioned in the proposal is still 2 - 3 releases out, the foundational work is underway, and a motivated cloud provider could accomplish the work in a forward-looking way. Contributors are encouraged to assist with the implementation of the design outlined in the proposal.
#### Some additional context on status / direction:
* 1.6 added a new cloud-controller-manager binary that may be used for testing the new out-of-core cloudprovider flow.
* Setting cloud-provider=external allows for creation of a separate controller-manager binary
* 1.7 adds [extensible admission control](https://git.k8s.io/community/contributors/design-proposals/api-machinery/admission_control_extension.md), further enabling topology customization.
pkg/cloudprovider/doc.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package cloudprovider supplies interfaces and implementations for cloud service providers.
// To use: import cloudprovider "k8s.io/cloud-provider"
package cloudprovider
pkg/cloudprovider/providers/BUILD deleted 100644 → 0
View file @ d4f4602f
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["providers.go"],
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers",
deps = [
"//pkg/cloudprovider/providers/aws:go_default_library",
"//pkg/cloudprovider/providers/azure:go_default_library",
"//pkg/cloudprovider/providers/cloudstack:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/cloudprovider/providers/openstack:go_default_library",
"//pkg/cloudprovider/providers/ovirt:go_default_library",
"//pkg/cloudprovider/providers/photon:go_default_library",
"//pkg/cloudprovider/providers/vsphere:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/cloudprovider/providers/aws:all-srcs",
"//pkg/cloudprovider/providers/azure:all-srcs",
"//pkg/cloudprovider/providers/cloudstack:all-srcs",
"//pkg/cloudprovider/providers/fake:all-srcs",
"//pkg/cloudprovider/providers/gce:all-srcs",
"//pkg/cloudprovider/providers/openstack:all-srcs",
"//pkg/cloudprovider/providers/ovirt:all-srcs",
"//pkg/cloudprovider/providers/photon:all-srcs",
"//pkg/cloudprovider/providers/vsphere:all-srcs",
],
tags = ["automanaged"],
)
pkg/cloudprovider/providers/aws/BUILD deleted 100644 → 0
View file @ d4f4602f
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"aws.go",
"aws_fakes.go",
"aws_instancegroups.go",
"aws_loadbalancer.go",
"aws_metrics.go",
"aws_routes.go",
"aws_utils.go",
"device_allocator.go",
"instances.go",
"log_handler.go",
"regions.go",
"retry_handler.go",
"sets_ippermissions.go",
"tags.go",
"volumes.go",
],
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/aws",
deps = [
"//pkg/api/v1/service:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/credentialprovider/aws:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/credentials:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/autoscaling:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/kms:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"aws_loadbalancer_test.go",
"aws_test.go",
"device_allocator_test.go",
"instances_test.go",
"regions_test.go",
"retry_handler_test.go",
"tags_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/mock:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
pkg/cloudprovider/providers/aws/OWNERS deleted 100644 → 0
View file @ d4f4602f
approvers:
- justinsb
- zmerlynn
- gnufied
- jsafrane
reviewers:
- gnufied
- jsafrane
- justinsb
- zmerlynn
- chrislovecnm
- nckturner
- micahhausler
pkg/cloudprovider/providers/aws/aws.go deleted 100644 → 0
View file @ d4f4602f
This source diff could not be displayed because it is too large. You can view the blob instead.
pkg/cloudprovider/providers/aws/aws_instancegroups.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"k8s.io/klog"
)
// AWSCloud implements InstanceGroups
var _ InstanceGroups = &Cloud{}
// ResizeInstanceGroup sets the size of the specificed instancegroup Exported
// so it can be used by the e2e tests, which don't want to instantiate a full
// cloudprovider.
func ResizeInstanceGroup(asg ASG, instanceGroupName string, size int) error {
request := &autoscaling.UpdateAutoScalingGroupInput{
AutoScalingGroupName: aws.String(instanceGroupName),
MinSize: aws.Int64(int64(size)),
MaxSize: aws.Int64(int64(size)),
}
if _, err := asg.UpdateAutoScalingGroup(request); err != nil {
return fmt.Errorf("error resizing AWS autoscaling group: %q", err)
}
return nil
}
// ResizeInstanceGroup implements InstanceGroups.ResizeInstanceGroup
// Set the size to the fixed size
func (c *Cloud) ResizeInstanceGroup(instanceGroupName string, size int) error {
return ResizeInstanceGroup(c.asg, instanceGroupName, size)
}
// DescribeInstanceGroup gets info about the specified instancegroup
// Exported so it can be used by the e2e tests,
// which don't want to instantiate a full cloudprovider.
func DescribeInstanceGroup(asg ASG, instanceGroupName string) (InstanceGroupInfo, error) {
request := &autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{aws.String(instanceGroupName)},
}
response, err := asg.DescribeAutoScalingGroups(request)
if err != nil {
return nil, fmt.Errorf("error listing AWS autoscaling group (%s): %q", instanceGroupName, err)
}
if len(response.AutoScalingGroups) == 0 {
return nil, nil
}
if len(response.AutoScalingGroups) > 1 {
klog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName)
}
group := response.AutoScalingGroups[0]
return &awsInstanceGroup{group: group}, nil
}
// DescribeInstanceGroup implements InstanceGroups.DescribeInstanceGroup
// Queries the cloud provider for information about the specified instance group
func (c *Cloud) DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) {
return DescribeInstanceGroup(c.asg, instanceGroupName)
}
// awsInstanceGroup implements InstanceGroupInfo
var _ InstanceGroupInfo = &awsInstanceGroup{}
type awsInstanceGroup struct {
group *autoscaling.Group
}
// Implement InstanceGroupInfo.CurrentSize
// The number of instances currently running under control of this group
func (g *awsInstanceGroup) CurrentSize() (int, error) {
return len(g.group.Instances), nil
}
pkg/cloudprovider/providers/aws/aws_loadbalancer_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
)
func TestElbProtocolsAreEqual(t *testing.T) {
grid := []struct {
L *string
R *string
Expected bool
}{
{
L: aws.String("http"),
R: aws.String("http"),
Expected: true,
},
{
L: aws.String("HTTP"),
R: aws.String("http"),
Expected: true,
},
{
L: aws.String("HTTP"),
R: aws.String("TCP"),
Expected: false,
},
{
L: aws.String(""),
R: aws.String("TCP"),
Expected: false,
},
{
L: aws.String(""),
R: aws.String(""),
Expected: true,
},
{
L: nil,
R: aws.String(""),
Expected: false,
},
{
L: aws.String(""),
R: nil,
Expected: false,
},
{
L: nil,
R: nil,
Expected: true,
},
}
for _, g := range grid {
actual := elbProtocolsAreEqual(g.L, g.R)
if actual != g.Expected {
t.Errorf("unexpected result from protocolsEquals(%v, %v)", g.L, g.R)
}
}
}
func TestAWSARNEquals(t *testing.T) {
grid := []struct {
L *string
R *string
Expected bool
}{
{
L: aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
R: aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
Expected: true,
},
{
L: aws.String("ARN:AWS:ACM:US-EAST-1:123456789012:CERTIFICATE/12345678-1234-1234-1234-123456789012"),
R: aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
Expected: true,
},
{
L: aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
R: aws.String(""),
Expected: false,
},
{
L: aws.String(""),
R: aws.String(""),
Expected: true,
},
{
L: nil,
R: aws.String(""),
Expected: false,
},
{
L: aws.String(""),
R: nil,
Expected: false,
},
{
L: nil,
R: nil,
Expected: true,
},
}
for _, g := range grid {
actual := awsArnEquals(g.L, g.R)
if actual != g.Expected {
t.Errorf("unexpected result from awsArnEquals(%v, %v)", g.L, g.R)
}
}
}
func TestIsNLB(t *testing.T) {
tests := []struct {
name string
annotations map[string]string
want bool
}{
{
"NLB annotation provided",
map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "nlb"},
true,
},
{
"NLB annotation has invalid value",
map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "elb"},
false,
},
{
"NLB annotation absent",
map[string]string{},
false,
},
}
for _, test := range tests {
t.Logf("Running test case %s", test.name)
got := isNLB(test.annotations)
if got != test.want {
t.Errorf("Incorrect value for isNLB() case %s. Got %t, expected %t.", test.name, got, test.want)
}
}
}
func TestSecurityGroupFiltering(t *testing.T) {
grid := []struct {
in []*ec2.SecurityGroup
name string
expected int
description string
}{
{
in: []*ec2.SecurityGroup{
{
IpPermissions: []*ec2.IpPermission{
{
IpRanges: []*ec2.IpRange{
{
Description: aws.String("an unmanaged"),
},
},
},
},
},
},
name: "unmanaged",
expected: 0,
description: "An environment without managed LBs should have %d, but found %d SecurityGroups",
},
{
in: []*ec2.SecurityGroup{
{
IpPermissions: []*ec2.IpPermission{
{
IpRanges: []*ec2.IpRange{
{
Description: aws.String("an unmanaged"),
},
{
Description: aws.String(fmt.Sprintf("%s=%s", NLBClientRuleDescription, "managedlb")),
},
{
Description: aws.String(fmt.Sprintf("%s=%s", NLBHealthCheckRuleDescription, "managedlb")),
},
},
},
},
},
},
name: "managedlb",
expected: 1,
description: "Found %d, but should have %d Security Groups",
},
}
for _, g := range grid {
actual := len(filterForIPRangeDescription(g.in, g.name))
if actual != g.expected {
t.Errorf(g.description, actual, g.expected)
}
}
}
pkg/cloudprovider/providers/aws/aws_metrics.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import "github.com/prometheus/client_golang/prometheus"
var (
awsAPIMetric = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "cloudprovider_aws_api_request_duration_seconds",
Help: "Latency of AWS API calls",
},
[]string{"request"})
awsAPIErrorMetric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cloudprovider_aws_api_request_errors",
Help: "AWS API errors",
},
[]string{"request"})
awsAPIThrottlesMetric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cloudprovider_aws_api_throttled_requests_total",
Help: "AWS API throttled requests",
},
[]string{"operation_name"})
)
func recordAWSMetric(actionName string, timeTaken float64, err error) {
if err != nil {
awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
} else {
awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
}
}
func recordAWSThrottlesMetric(operation string) {
awsAPIThrottlesMetric.With(prometheus.Labels{"operation_name": operation}).Inc()
}
func registerMetrics() {
prometheus.MustRegister(awsAPIMetric)
prometheus.MustRegister(awsAPIErrorMetric)
prometheus.MustRegister(awsAPIThrottlesMetric)
}
pkg/cloudprovider/providers/aws/aws_routes.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog"
cloudprovider "k8s.io/cloud-provider"
)
func (c *Cloud) findRouteTable(clusterName string) (*ec2.RouteTable, error) {
// This should be unnecessary (we already filter on TagNameKubernetesCluster,
// and something is broken if cluster name doesn't match, but anyway...
// TODO: All clouds should be cluster-aware by default
var tables []*ec2.RouteTable
if c.cfg.Global.RouteTableID != "" {
request := &ec2.DescribeRouteTablesInput{Filters: []*ec2.Filter{newEc2Filter("route-table-id", c.cfg.Global.RouteTableID)}}
response, err := c.ec2.DescribeRouteTables(request)
if err != nil {
return nil, err
}
tables = response
} else {
request := &ec2.DescribeRouteTablesInput{Filters: c.tagging.addFilters(nil)}
response, err := c.ec2.DescribeRouteTables(request)
if err != nil {
return nil, err
}
for _, table := range response {
if c.tagging.hasClusterTag(table.Tags) {
tables = append(tables, table)
}
}
}
if len(tables) == 0 {
return nil, fmt.Errorf("unable to find route table for AWS cluster: %s", clusterName)
}
if len(tables) != 1 {
return nil, fmt.Errorf("found multiple matching AWS route tables for AWS cluster: %s", clusterName)
}
return tables[0], nil
}
// ListRoutes implements Routes.ListRoutes
// List all routes that match the filter
func (c *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
table, err := c.findRouteTable(clusterName)
if err != nil {
return nil, err
}
var routes []*cloudprovider.Route
var instanceIDs []*string
for _, r := range table.Routes {
instanceID := aws.StringValue(r.InstanceId)
if instanceID == "" {
continue
}
instanceIDs = append(instanceIDs, &instanceID)
}
instances, err := c.getInstancesByIDs(instanceIDs)
if err != nil {
return nil, err
}
for _, r := range table.Routes {
destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
if destinationCIDR == "" {
continue
}
route := &cloudprovider.Route{
Name: clusterName + "-" + destinationCIDR,
DestinationCIDR: destinationCIDR,
}
// Capture blackhole routes
if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
route.Blackhole = true
routes = append(routes, route)
continue
}
// Capture instance routes
instanceID := aws.StringValue(r.InstanceId)
if instanceID != "" {
instance, found := instances[instanceID]
if found {
route.TargetNode = mapInstanceToNodeName(instance)
routes = append(routes, route)
} else {
klog.Warningf("unable to find instance ID %s in the list of instances being routed to", instanceID)
}
}
}
return routes, nil
}
// Sets the instance attribute "source-dest-check" to the specified value
func (c *Cloud) configureInstanceSourceDestCheck(instanceID string, sourceDestCheck bool) error {
request := &ec2.ModifyInstanceAttributeInput{}
request.InstanceId = aws.String(instanceID)
request.SourceDestCheck = &ec2.AttributeBooleanValue{Value: aws.Bool(sourceDestCheck)}
_, err := c.ec2.ModifyInstanceAttribute(request)
if err != nil {
return fmt.Errorf("error configuring source-dest-check on instance %s: %q", instanceID, err)
}
return nil
}
// CreateRoute implements Routes.CreateRoute
// Create the described route
func (c *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error {
instance, err := c.getInstanceByNodeName(route.TargetNode)
if err != nil {
return err
}
// In addition to configuring the route itself, we also need to configure the instance to accept that traffic
// On AWS, this requires turning source-dest checks off
err = c.configureInstanceSourceDestCheck(aws.StringValue(instance.InstanceId), false)
if err != nil {
return err
}
table, err := c.findRouteTable(clusterName)
if err != nil {
return err
}
var deleteRoute *ec2.Route
for _, r := range table.Routes {
destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
if destinationCIDR != route.DestinationCIDR {
continue
}
if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
deleteRoute = r
}
}
if deleteRoute != nil {
klog.Infof("deleting blackholed route: %s", aws.StringValue(deleteRoute.DestinationCidrBlock))
request := &ec2.DeleteRouteInput{}
request.DestinationCidrBlock = deleteRoute.DestinationCidrBlock
request.RouteTableId = table.RouteTableId
_, err = c.ec2.DeleteRoute(request)
if err != nil {
return fmt.Errorf("error deleting blackholed AWS route (%s): %q", aws.StringValue(deleteRoute.DestinationCidrBlock), err)
}
}
request := &ec2.CreateRouteInput{}
// TODO: use ClientToken for idempotency?
request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
request.InstanceId = instance.InstanceId
request.RouteTableId = table.RouteTableId
_, err = c.ec2.CreateRoute(request)
if err != nil {
return fmt.Errorf("error creating AWS route (%s): %q", route.DestinationCIDR, err)
}
return nil
}
// DeleteRoute implements Routes.DeleteRoute
// Delete the specified route
func (c *Cloud) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error {
table, err := c.findRouteTable(clusterName)
if err != nil {
return err
}
request := &ec2.DeleteRouteInput{}
request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
request.RouteTableId = table.RouteTableId
_, err = c.ec2.DeleteRoute(request)
if err != nil {
return fmt.Errorf("error deleting AWS route (%s): %q", route.DestinationCIDR, err)
}
return nil
}
pkg/cloudprovider/providers/aws/aws_utils.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"github.com/aws/aws-sdk-go/aws"
"k8s.io/apimachinery/pkg/util/sets"
)
func stringSetToPointers(in sets.String) []*string {
if in == nil {
return nil
}
out := make([]*string, 0, len(in))
for k := range in {
out = append(out, aws.String(k))
}
return out
}
func stringSetFromPointers(in []*string) sets.String {
if in == nil {
return nil
}
out := sets.NewString()
for i := range in {
out.Insert(aws.StringValue(in[i]))
}
return out
}
pkg/cloudprovider/providers/aws/device_allocator.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"sort"
"sync"
)
// ExistingDevices is a map of assigned devices. Presence of a key with a device
// name in the map means that the device is allocated. Value is irrelevant and
// can be used for anything that DeviceAllocator user wants.
// Only the relevant part of device name should be in the map, e.g. "ba" for
// "/dev/xvdba".
type ExistingDevices map[mountDevice]EBSVolumeID
// DeviceAllocator finds available device name, taking into account already
// assigned device names from ExistingDevices map. It tries to find the next
// device name to the previously assigned one (from previous DeviceAllocator
// call), so all available device names are used eventually and it minimizes
// device name reuse.
//
// All these allocations are in-memory, nothing is written to / read from
// /dev directory.
//
// On AWS, we should assign new (not yet used) device names to attached volumes.
// If we reuse a previously used name, we may get the volume "attaching" forever,
// see https://aws.amazon.com/premiumsupport/knowledge-center/ebs-stuck-attaching/.
type DeviceAllocator interface {
// GetNext returns a free device name or error when there is no free device
// name. Only the device suffix is returned, e.g. "ba" for "/dev/xvdba".
// It's up to the called to add appropriate "/dev/sd" or "/dev/xvd" prefix.
GetNext(existingDevices ExistingDevices) (mountDevice, error)
// Deprioritize the device so as it can't be used immediately again
Deprioritize(mountDevice)
// Lock the deviceAllocator
Lock()
// Unlock the deviceAllocator
Unlock()
}
type deviceAllocator struct {
possibleDevices map[mountDevice]int
counter int
deviceLock sync.Mutex
}
var _ DeviceAllocator = &deviceAllocator{}
type devicePair struct {
deviceName mountDevice
deviceIndex int
}
type devicePairList []devicePair
func (p devicePairList) Len() int { return len(p) }
func (p devicePairList) Less(i, j int) bool { return p[i].deviceIndex < p[j].deviceIndex }
func (p devicePairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// NewDeviceAllocator allocates device names according to scheme ba..bz, ca..cz
// it moves along the ring and always picks next device until device list is
// exhausted.
func NewDeviceAllocator() DeviceAllocator {
possibleDevices := make(map[mountDevice]int)
for _, firstChar := range []rune{'b', 'c'} {
for i := 'a'; i <= 'z'; i++ {
dev := mountDevice([]rune{firstChar, i})
possibleDevices[dev] = 0
}
}
return &deviceAllocator{
possibleDevices: possibleDevices,
counter: 0,
}
}
// GetNext gets next available device from the pool, this function assumes that caller
// holds the necessary lock on deviceAllocator
func (d *deviceAllocator) GetNext(existingDevices ExistingDevices) (mountDevice, error) {
for _, devicePair := range d.sortByCount() {
if _, found := existingDevices[devicePair.deviceName]; !found {
return devicePair.deviceName, nil
}
}
return "", fmt.Errorf("no devices are available")
}
func (d *deviceAllocator) sortByCount() devicePairList {
dpl := make(devicePairList, 0)
for deviceName, deviceIndex := range d.possibleDevices {
dpl = append(dpl, devicePair{deviceName, deviceIndex})
}
sort.Sort(dpl)
return dpl
}
func (d *deviceAllocator) Lock() {
d.deviceLock.Lock()
}
func (d *deviceAllocator) Unlock() {
d.deviceLock.Unlock()
}
// Deprioritize the device so as it can't be used immediately again
func (d *deviceAllocator) Deprioritize(chosen mountDevice) {
d.deviceLock.Lock()
defer d.deviceLock.Unlock()
if _, ok := d.possibleDevices[chosen]; ok {
d.counter++
d.possibleDevices[chosen] = d.counter
}
}
pkg/cloudprovider/providers/aws/device_allocator_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import "testing"
func TestDeviceAllocator(t *testing.T) {
tests := []struct {
name string
existingDevices ExistingDevices
deviceMap map[mountDevice]int
expectedOutput mountDevice
}{
{
"empty device list with wrap",
ExistingDevices{},
generateUnsortedDeviceList(),
"bd", // next to 'zz' is the first one, 'ba'
},
}
for _, test := range tests {
allocator := NewDeviceAllocator().(*deviceAllocator)
for k, v := range test.deviceMap {
allocator.possibleDevices[k] = v
}
got, err := allocator.GetNext(test.existingDevices)
if err != nil {
t.Errorf("text %q: unexpected error: %v", test.name, err)
}
if got != test.expectedOutput {
t.Errorf("text %q: expected %q, got %q", test.name, test.expectedOutput, got)
}
}
}
func generateUnsortedDeviceList() map[mountDevice]int {
possibleDevices := make(map[mountDevice]int)
for _, firstChar := range []rune{'b', 'c'} {
for i := 'a'; i <= 'z'; i++ {
dev := mountDevice([]rune{firstChar, i})
possibleDevices[dev] = 3
}
}
possibleDevices["bd"] = 0
return possibleDevices
}
func TestDeviceAllocatorError(t *testing.T) {
allocator := NewDeviceAllocator().(*deviceAllocator)
existingDevices := ExistingDevices{}
// make all devices used
var first, second byte
for first = 'b'; first <= 'c'; first++ {
for second = 'a'; second <= 'z'; second++ {
device := [2]byte{first, second}
existingDevices[mountDevice(device[:])] = "used"
}
}
device, err := allocator.GetNext(existingDevices)
if err == nil {
t.Errorf("expected error, got device %q", device)
}
}
pkg/cloudprovider/providers/aws/instances.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"net/url"
"regexp"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog"
"k8s.io/api/core/v1"
)
// awsInstanceRegMatch represents Regex Match for AWS instance.
var awsInstanceRegMatch = regexp.MustCompile("^i-[^/]*$")
// awsInstanceID represents the ID of the instance in the AWS API, e.g. i-12345678
// The "traditional" format is "i-12345678"
// A new longer format is also being introduced: "i-12345678abcdef01"
// We should not assume anything about the length or format, though it seems
// reasonable to assume that instances will continue to start with "i-".
type awsInstanceID string
func (i awsInstanceID) awsString() *string {
return aws.String(string(i))
}
// kubernetesInstanceID represents the id for an instance in the kubernetes API;
// the following form
// * aws:///<zone>/<awsInstanceId>
// * aws:////<awsInstanceId>
// * <awsInstanceId>
type kubernetesInstanceID string
// mapToAWSInstanceID extracts the awsInstanceID from the kubernetesInstanceID
func (name kubernetesInstanceID) mapToAWSInstanceID() (awsInstanceID, error) {
s := string(name)
if !strings.HasPrefix(s, "aws://") {
// Assume a bare aws volume id (vol-1234...)
// Build a URL with an empty host (AZ)
s = "aws://" + "/" + "/" + s
}
url, err := url.Parse(s)
if err != nil {
return "", fmt.Errorf("Invalid instance name (%s): %v", name, err)
}
if url.Scheme != "aws" {
return "", fmt.Errorf("Invalid scheme for AWS instance (%s)", name)
}
awsID := ""
tokens := strings.Split(strings.Trim(url.Path, "/"), "/")
if len(tokens) == 1 {
// instanceId
awsID = tokens[0]
} else if len(tokens) == 2 {
// az/instanceId
awsID = tokens[1]
}
// We sanity check the resulting volume; the two known formats are
// i-12345678 and i-12345678abcdef01
if awsID == "" || !awsInstanceRegMatch.MatchString(awsID) {
return "", fmt.Errorf("Invalid format for AWS instance (%s)", name)
}
return awsInstanceID(awsID), nil
}
// mapToAWSInstanceID extracts the awsInstanceIDs from the Nodes, returning an error if a Node cannot be mapped
func mapToAWSInstanceIDs(nodes []*v1.Node) ([]awsInstanceID, error) {
var instanceIDs []awsInstanceID
for _, node := range nodes {
if node.Spec.ProviderID == "" {
return nil, fmt.Errorf("node %q did not have ProviderID set", node.Name)
}
instanceID, err := kubernetesInstanceID(node.Spec.ProviderID).mapToAWSInstanceID()
if err != nil {
return nil, fmt.Errorf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
}
instanceIDs = append(instanceIDs, instanceID)
}
return instanceIDs, nil
}
// mapToAWSInstanceIDsTolerant extracts the awsInstanceIDs from the Nodes, skipping Nodes that cannot be mapped
func mapToAWSInstanceIDsTolerant(nodes []*v1.Node) []awsInstanceID {
var instanceIDs []awsInstanceID
for _, node := range nodes {
if node.Spec.ProviderID == "" {
klog.Warningf("node %q did not have ProviderID set", node.Name)
continue
}
instanceID, err := kubernetesInstanceID(node.Spec.ProviderID).mapToAWSInstanceID()
if err != nil {
klog.Warningf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
continue
}
instanceIDs = append(instanceIDs, instanceID)
}
return instanceIDs
}
// Gets the full information about this instance from the EC2 API
func describeInstance(ec2Client EC2, instanceID awsInstanceID) (*ec2.Instance, error) {
request := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instanceID.awsString()},
}
instances, err := ec2Client.DescribeInstances(request)
if err != nil {
return nil, err
}
if len(instances) == 0 {
return nil, fmt.Errorf("no instances found for instance: %s", instanceID)
}
if len(instances) > 1 {
return nil, fmt.Errorf("multiple instances found for instance: %s", instanceID)
}
return instances[0], nil
}
// instanceCache manages the cache of DescribeInstances
type instanceCache struct {
// TODO: Get rid of this field, send all calls through the instanceCache
cloud *Cloud
mutex sync.Mutex
snapshot *allInstancesSnapshot
}
// Gets the full information about these instance from the EC2 API
func (c *instanceCache) describeAllInstancesUncached() (*allInstancesSnapshot, error) {
now := time.Now()
klog.V(4).Infof("EC2 DescribeInstances - fetching all instances")
filters := []*ec2.Filter{}
instances, err := c.cloud.describeInstances(filters)
if err != nil {
return nil, err
}
m := make(map[awsInstanceID]*ec2.Instance)
for _, i := range instances {
id := awsInstanceID(aws.StringValue(i.InstanceId))
m[id] = i
}
snapshot := &allInstancesSnapshot{now, m}
c.mutex.Lock()
defer c.mutex.Unlock()
if c.snapshot != nil && snapshot.olderThan(c.snapshot) {
// If this happens a lot, we could run this function in a mutex and only return one result
klog.Infof("Not caching concurrent AWS DescribeInstances results")
} else {
c.snapshot = snapshot
}
return snapshot, nil
}
// cacheCriteria holds criteria that must hold to use a cached snapshot
type cacheCriteria struct {
// MaxAge indicates the maximum age of a cached snapshot we can accept.
// If set to 0 (i.e. unset), cached values will not time out because of age.
MaxAge time.Duration
// HasInstances is a list of awsInstanceIDs that must be in a cached snapshot for it to be considered valid.
// If an instance is not found in the cached snapshot, the snapshot be ignored and we will re-fetch.
HasInstances []awsInstanceID
}
// describeAllInstancesCached returns all instances, using cached results if applicable
func (c *instanceCache) describeAllInstancesCached(criteria cacheCriteria) (*allInstancesSnapshot, error) {
var err error
snapshot := c.getSnapshot()
if snapshot != nil && !snapshot.MeetsCriteria(criteria) {
snapshot = nil
}
if snapshot == nil {
snapshot, err = c.describeAllInstancesUncached()
if err != nil {
return nil, err
}
} else {
klog.V(6).Infof("EC2 DescribeInstances - using cached results")
}
return snapshot, nil
}
// getSnapshot returns a snapshot if one exists
func (c *instanceCache) getSnapshot() *allInstancesSnapshot {
c.mutex.Lock()
defer c.mutex.Unlock()
return c.snapshot
}
// olderThan is a simple helper to encapsulate timestamp comparison
func (s *allInstancesSnapshot) olderThan(other *allInstancesSnapshot) bool {
// After() is technically broken by time changes until we have monotonic time
return other.timestamp.After(s.timestamp)
}
// MeetsCriteria returns true if the snapshot meets the criteria in cacheCriteria
func (s *allInstancesSnapshot) MeetsCriteria(criteria cacheCriteria) bool {
if criteria.MaxAge > 0 {
// Sub() is technically broken by time changes until we have monotonic time
now := time.Now()
if now.Sub(s.timestamp) > criteria.MaxAge {
klog.V(6).Infof("instanceCache snapshot cannot be used as is older than MaxAge=%s", criteria.MaxAge)
return false
}
}
if len(criteria.HasInstances) != 0 {
for _, id := range criteria.HasInstances {
if nil == s.instances[id] {
klog.V(6).Infof("instanceCache snapshot cannot be used as does not contain instance %s", id)
return false
}
}
}
return true
}
// allInstancesSnapshot holds the results from querying for all instances,
// along with the timestamp for cache-invalidation purposes
type allInstancesSnapshot struct {
timestamp time.Time
instances map[awsInstanceID]*ec2.Instance
}
// FindInstances returns the instances corresponding to the specified ids. If an id is not found, it is ignored.
func (s *allInstancesSnapshot) FindInstances(ids []awsInstanceID) map[awsInstanceID]*ec2.Instance {
m := make(map[awsInstanceID]*ec2.Instance)
for _, id := range ids {
instance := s.instances[id]
if instance != nil {
m[id] = instance
}
}
return m
}
pkg/cloudprovider/providers/aws/instances_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
)
func TestMapToAWSInstanceIDs(t *testing.T) {
tests := []struct {
Kubernetes kubernetesInstanceID
Aws awsInstanceID
ExpectError bool
}{
{
Kubernetes: "aws:///us-east-1a/i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "aws:////i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "aws:///us-east-1a/i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "aws:////i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "vol-123456789",
ExpectError: true,
},
{
Kubernetes: "aws:///us-east-1a/vol-12345678abcdef01",
ExpectError: true,
},
{
Kubernetes: "aws://accountid/us-east-1a/vol-12345678abcdef01",
ExpectError: true,
},
{
Kubernetes: "aws:///us-east-1a/vol-12345678abcdef01/suffix",
ExpectError: true,
},
{
Kubernetes: "",
ExpectError: true,
},
}
for _, test := range tests {
awsID, err := test.Kubernetes.mapToAWSInstanceID()
if err != nil {
if !test.ExpectError {
t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
}
} else {
if test.ExpectError {
t.Errorf("expected error parsing %s", test.Kubernetes)
} else if test.Aws != awsID {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsID)
}
}
}
for _, test := range tests {
node := &v1.Node{}
node.Spec.ProviderID = string(test.Kubernetes)
awsInstanceIds, err := mapToAWSInstanceIDs([]*v1.Node{node})
if err != nil {
if !test.ExpectError {
t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
}
} else {
if test.ExpectError {
t.Errorf("expected error parsing %s", test.Kubernetes)
} else if len(awsInstanceIds) != 1 {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
} else if awsInstanceIds[0] != test.Aws {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
}
}
awsInstanceIds = mapToAWSInstanceIDsTolerant([]*v1.Node{node})
if test.ExpectError {
if len(awsInstanceIds) != 0 {
t.Errorf("unexpected results parsing %s: %s", test.Kubernetes, awsInstanceIds)
}
} else {
if len(awsInstanceIds) != 1 {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
} else if awsInstanceIds[0] != test.Aws {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
}
}
}
}
func TestSnapshotMeetsCriteria(t *testing.T) {
snapshot := &allInstancesSnapshot{timestamp: time.Now().Add(-3601 * time.Second)}
if !snapshot.MeetsCriteria(cacheCriteria{}) {
t.Errorf("Snapshot should always meet empty criteria")
}
if snapshot.MeetsCriteria(cacheCriteria{MaxAge: time.Hour}) {
t.Errorf("Snapshot did not honor MaxAge")
}
if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678")}}) {
t.Errorf("Snapshot did not honor HasInstances with missing instances")
}
snapshot.instances = make(map[awsInstanceID]*ec2.Instance)
snapshot.instances[awsInstanceID("i-12345678")] = &ec2.Instance{}
if !snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678")}}) {
t.Errorf("Snapshot did not honor HasInstances with matching instances")
}
if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-00000000")}}) {
t.Errorf("Snapshot did not honor HasInstances with partially matching instances")
}
}
func TestOlderThan(t *testing.T) {
t1 := time.Now()
t2 := t1.Add(time.Second)
s1 := &allInstancesSnapshot{timestamp: t1}
s2 := &allInstancesSnapshot{timestamp: t2}
assert.True(t, s1.olderThan(s2), "s1 should be olderThan s2")
assert.False(t, s2.olderThan(s1), "s2 not should be olderThan s1")
assert.False(t, s1.olderThan(s1), "s1 not should be olderThan itself")
}
func TestSnapshotFindInstances(t *testing.T) {
snapshot := &allInstancesSnapshot{}
snapshot.instances = make(map[awsInstanceID]*ec2.Instance)
{
id := awsInstanceID("i-12345678")
snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
}
{
id := awsInstanceID("i-23456789")
snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
}
instances := snapshot.FindInstances([]awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-23456789"), awsInstanceID("i-00000000")})
if len(instances) != 2 {
t.Errorf("findInstances returned %d results, expected 2", len(instances))
}
for _, id := range []awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-23456789")} {
i := instances[id]
if i == nil {
t.Errorf("findInstances did not return %s", id)
continue
}
if aws.StringValue(i.InstanceId) != string(id) {
t.Errorf("findInstances did not return expected instanceId for %s", id)
}
if i != snapshot.instances[id] {
t.Errorf("findInstances did not return expected instance (reference equality) for %s", id)
}
}
}
pkg/cloudprovider/providers/aws/log_handler.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"github.com/aws/aws-sdk-go/aws/request"
"k8s.io/klog"
)
// Handler for aws-sdk-go that logs all requests
func awsHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS request: %s %s", service, name)
}
func awsSendHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS API Send: %s %s %v %v", service, name, req.Operation, req.Params)
}
func awsValidateResponseHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS API ValidateResponse: %s %s %v %v %s", service, name, req.Operation, req.Params, req.HTTPResponse.Status)
}
func awsServiceAndName(req *request.Request) (string, string) {
service := req.ClientInfo.ServiceName
name := "?"
if req.Operation != nil {
name = req.Operation.Name
}
return service, name
}
pkg/cloudprovider/providers/aws/regions.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"sync"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/util/sets"
awscredentialprovider "k8s.io/kubernetes/pkg/credentialprovider/aws"
)
// wellKnownRegions is the complete list of regions known to the AWS cloudprovider
// and credentialprovider.
var wellKnownRegions = [...]string{
// from `aws ec2 describe-regions --region us-east-1 --query Regions[].RegionName | sort`
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-southeast-1",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2",
// these are not registered in many / most accounts
"cn-north-1",
"cn-northwest-1",
"us-gov-west-1",
}
// awsRegionsMutex protects awsRegions
var awsRegionsMutex sync.Mutex
// awsRegions is a set of recognized regions
var awsRegions sets.String
// recognizeRegion is called for each AWS region we know about.
// It currently registers a credential provider for that region.
// There are two paths to discovering a region:
// * we hard-code some well-known regions
// * if a region is discovered from instance metadata, we add that
func recognizeRegion(region string) {
awsRegionsMutex.Lock()
defer awsRegionsMutex.Unlock()
if awsRegions == nil {
awsRegions = sets.NewString()
}
if awsRegions.Has(region) {
klog.V(6).Infof("found AWS region %q again - ignoring", region)
return
}
klog.V(4).Infof("found AWS region %q", region)
awscredentialprovider.RegisterCredentialsProvider(region)
awsRegions.Insert(region)
}
// recognizeWellKnownRegions calls RecognizeRegion on each WellKnownRegion
func recognizeWellKnownRegions() {
for _, region := range wellKnownRegions {
recognizeRegion(region)
}
}
// isRegionValid checks if the region is in the set of known regions
func isRegionValid(region string) bool {
awsRegionsMutex.Lock()
defer awsRegionsMutex.Unlock()
return awsRegions.Has(region)
}
pkg/cloudprovider/providers/aws/regions_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
)
// TestRegions does basic checking of region verification / addition
func TestRegions(t *testing.T) {
recognizeWellKnownRegions()
tests := []struct {
Add string
Lookup string
ExpectIsRegion bool
}{
{
Lookup: "us-east-1",
ExpectIsRegion: true,
},
{
Lookup: "us-east-1a",
ExpectIsRegion: false,
},
{
Add: "us-test-1",
Lookup: "us-east-1",
ExpectIsRegion: true,
},
{
Lookup: "us-test-1",
ExpectIsRegion: true,
},
{
Add: "us-test-1",
Lookup: "us-test-1",
ExpectIsRegion: true,
},
}
for _, test := range tests {
if test.Add != "" {
recognizeRegion(test.Add)
}
if test.Lookup != "" {
if isRegionValid(test.Lookup) != test.ExpectIsRegion {
t.Fatalf("region valid mismatch: %q", test.Lookup)
}
}
}
}
// TestRecognizesNewRegion verifies that we see a region from metadata, we recognize it as valid
func TestRecognizesNewRegion(t *testing.T) {
region := "us-testrecognizesnewregion-1"
if isRegionValid(region) {
t.Fatalf("region already valid: %q", region)
}
awsServices := NewFakeAWSServices(TestClusterID).WithAz(region + "a")
_, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("error building AWS cloud: %v", err)
}
if !isRegionValid(region) {
t.Fatalf("newly discovered region not valid: %q", region)
}
}
pkg/cloudprovider/providers/aws/retry_handler.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"math"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"k8s.io/klog"
)
const (
decayIntervalSeconds = 20
decayFraction = 0.8
maxDelay = 60 * time.Second
)
// CrossRequestRetryDelay inserts delays before AWS calls, when we are observing RequestLimitExceeded errors
// Note that we share a CrossRequestRetryDelay across multiple AWS requests; this is a process-wide back-off,
// whereas the aws-sdk-go implements a per-request exponential backoff/retry
type CrossRequestRetryDelay struct {
backoff Backoff
}
// NewCrossRequestRetryDelay creates a new CrossRequestRetryDelay
func NewCrossRequestRetryDelay() *CrossRequestRetryDelay {
c := &CrossRequestRetryDelay{}
c.backoff.init(decayIntervalSeconds, decayFraction, maxDelay)
return c
}
// BeforeSign is added to the Sign chain; called before each request
func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
now := time.Now()
delay := c.backoff.ComputeDelayForRequest(now)
if delay > 0 {
klog.Warningf("Inserting delay before AWS request (%s) to avoid RequestLimitExceeded: %s",
describeRequest(r), delay.String())
if sleepFn := r.Config.SleepDelay; sleepFn != nil {
// Support SleepDelay for backwards compatibility
sleepFn(delay)
} else if err := aws.SleepWithContext(r.Context(), delay); err != nil {
r.Error = awserr.New(request.CanceledErrorCode, "request context canceled", err)
r.Retryable = aws.Bool(false)
return
}
// Avoid clock skew problems
r.Time = now
}
}
// Return the operation name, for use in log messages and metrics
func operationName(r *request.Request) string {
name := "?"
if r.Operation != nil {
name = r.Operation.Name
}
return name
}
// Return a user-friendly string describing the request, for use in log messages
func describeRequest(r *request.Request) string {
service := r.ClientInfo.ServiceName
return service + "::" + operationName(r)
}
// AfterRetry is added to the AfterRetry chain; called after any error
func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
if r.Error == nil {
return
}
awsError, ok := r.Error.(awserr.Error)
if !ok {
return
}
if awsError.Code() == "RequestLimitExceeded" {
c.backoff.ReportError()
recordAWSThrottlesMetric(operationName(r))
klog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
describeRequest(r))
}
}
// Backoff manages a backoff that varies based on the recently observed failures
type Backoff struct {
decayIntervalSeconds int64
decayFraction float64
maxDelay time.Duration
mutex sync.Mutex
// We count all requests & the number of requests which hit a
// RequestLimit. We only really care about 'recent' requests, so we
// decay the counts exponentially to bias towards recent values.
countErrorsRequestLimit float32
countRequests float32
lastDecay int64
}
func (b *Backoff) init(decayIntervalSeconds int, decayFraction float64, maxDelay time.Duration) {
b.lastDecay = time.Now().Unix()
// Bias so that if the first request hits the limit we don't immediately apply the full delay
b.countRequests = 4
b.decayIntervalSeconds = int64(decayIntervalSeconds)
b.decayFraction = decayFraction
b.maxDelay = maxDelay
}
// ComputeDelayForRequest computes the delay required for a request, also
// updates internal state to count this request
func (b *Backoff) ComputeDelayForRequest(now time.Time) time.Duration {
b.mutex.Lock()
defer b.mutex.Unlock()
// Apply exponential decay to the counters
timeDeltaSeconds := now.Unix() - b.lastDecay
if timeDeltaSeconds > b.decayIntervalSeconds {
intervals := float64(timeDeltaSeconds) / float64(b.decayIntervalSeconds)
decay := float32(math.Pow(b.decayFraction, intervals))
b.countErrorsRequestLimit *= decay
b.countRequests *= decay
b.lastDecay = now.Unix()
}
// Count this request
b.countRequests += 1.0
// Compute the failure rate
errorFraction := float32(0.0)
if b.countRequests > 0.5 {
// Avoid tiny residuals & rounding errors
errorFraction = b.countErrorsRequestLimit / b.countRequests
}
// Ignore a low fraction of errors
// This also allows them to time-out
if errorFraction < 0.1 {
return time.Duration(0)
}
// Delay by the max delay multiplied by the recent error rate
// (i.e. we apply a linear delay function)
// TODO: This is pretty arbitrary
delay := time.Nanosecond * time.Duration(float32(b.maxDelay.Nanoseconds())*errorFraction)
// Round down to the nearest second for sanity
return time.Second * time.Duration(int(delay.Seconds()))
}
// ReportError is called when we observe a throttling error
func (b *Backoff) ReportError() {
b.mutex.Lock()
defer b.mutex.Unlock()
b.countErrorsRequestLimit += 1.0
}
pkg/cloudprovider/providers/aws/retry_handler_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"time"
)
// There follows a group of tests for the backoff logic. There's nothing
// particularly special about the values chosen: if we tweak the values in the
// backoff logic then we might well have to update the tests. However the key
// behavioural elements should remain (e.g. no errors => no backoff), and these
// are each tested by one of the tests below.
// Test that we don't apply any delays when there are no errors
func TestBackoffNoErrors(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
for i := 0; i < 100; i++ {
d := b.ComputeDelayForRequest(now)
if d.Nanoseconds() != 0 {
t.Fatalf("unexpected delay during no-error case")
}
now = now.Add(time.Second)
}
}
// Test that we always apply a delay when there are errors, and also that we
// don't "flap" - that our own delay doesn't cause us to oscillate between
// delay and no-delay.
func TestBackoffAllErrors(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
// Warm up
for i := 0; i < 10; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 100; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if d.Seconds() < 5 {
t.Fatalf("unexpected short-delay during all-error case: %v", d)
}
t.Logf("delay @%d %v", i, d)
now = now.Add(d)
}
}
// Test that we do come close to our max delay, when we see all errors at 1
// second intervals (this simulates multiple concurrent requests, because we
// don't wait for delay in between requests)
func TestBackoffHitsMax(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if float32(d.Nanoseconds()) < (float32(maxDelay.Nanoseconds()) * 0.95) {
t.Fatalf("expected delay to be >= 95 percent of max delay, was %v", d)
}
t.Logf("delay @%d %v", i, d)
now = now.Add(time.Second)
}
}
// Test that after a phase of errors, we eventually stop applying a delay once there are
// no more errors.
func TestBackoffRecovers(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
// Phase of all-errors
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if d.Seconds() < 5 {
t.Fatalf("unexpected short-delay during all-error phase: %v", d)
}
t.Logf("error phase delay @%d %v", i, d)
now = now.Add(time.Second)
}
// Phase of no errors
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
now = now.Add(3 * time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
if d.Seconds() != 0 {
t.Fatalf("unexpected delay during error recovery phase: %v", d)
}
t.Logf("no-error phase delay @%d %v", i, d)
now = now.Add(time.Second)
}
}
pkg/cloudprovider/providers/aws/sets_ippermissions.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/service/ec2"
)
// IPPermissionSet maps IP strings of strings to EC2 IpPermissions
type IPPermissionSet map[string]*ec2.IpPermission
// NewIPPermissionSet creates a new IPPermissionSet
func NewIPPermissionSet(items ...*ec2.IpPermission) IPPermissionSet {
s := make(IPPermissionSet)
s.Insert(items...)
return s
}
// Ungroup splits permissions out into individual permissions
// EC2 will combine permissions with the same port but different SourceRanges together, for example
// We ungroup them so we can process them
func (s IPPermissionSet) Ungroup() IPPermissionSet {
l := []*ec2.IpPermission{}
for _, p := range s.List() {
if len(p.IpRanges) <= 1 {
l = append(l, p)
continue
}
for _, ipRange := range p.IpRanges {
c := &ec2.IpPermission{}
*c = *p
c.IpRanges = []*ec2.IpRange{ipRange}
l = append(l, c)
}
}
l2 := []*ec2.IpPermission{}
for _, p := range l {
if len(p.UserIdGroupPairs) <= 1 {
l2 = append(l2, p)
continue
}
for _, u := range p.UserIdGroupPairs {
c := &ec2.IpPermission{}
*c = *p
c.UserIdGroupPairs = []*ec2.UserIdGroupPair{u}
l2 = append(l, c)
}
}
l3 := []*ec2.IpPermission{}
for _, p := range l2 {
if len(p.PrefixListIds) <= 1 {
l3 = append(l3, p)
continue
}
for _, v := range p.PrefixListIds {
c := &ec2.IpPermission{}
*c = *p
c.PrefixListIds = []*ec2.PrefixListId{v}
l3 = append(l3, c)
}
}
return NewIPPermissionSet(l3...)
}
// Insert adds items to the set.
func (s IPPermissionSet) Insert(items ...*ec2.IpPermission) {
for _, p := range items {
k := keyForIPPermission(p)
s[k] = p
}
}
// List returns the contents as a slice. Order is not defined.
func (s IPPermissionSet) List() []*ec2.IpPermission {
res := make([]*ec2.IpPermission, 0, len(s))
for _, v := range s {
res = append(res, v)
}
return res
}
// IsSuperset returns true if and only if s is a superset of s2.
func (s IPPermissionSet) IsSuperset(s2 IPPermissionSet) bool {
for k := range s2 {
_, found := s[k]
if !found {
return false
}
}
return true
}
// Equal returns true if and only if s is equal (as a set) to s2.
// Two sets are equal if their membership is identical.
// (In practice, this means same elements, order doesn't matter)
func (s IPPermissionSet) Equal(s2 IPPermissionSet) bool {
return len(s) == len(s2) && s.IsSuperset(s2)
}
// Difference returns a set of objects that are not in s2
// For example:
// s1 = {a1, a2, a3}
// s2 = {a1, a2, a4, a5}
// s1.Difference(s2) = {a3}
// s2.Difference(s1) = {a4, a5}
func (s IPPermissionSet) Difference(s2 IPPermissionSet) IPPermissionSet {
result := NewIPPermissionSet()
for k, v := range s {
_, found := s2[k]
if !found {
result[k] = v
}
}
return result
}
// Len returns the size of the set.
func (s IPPermissionSet) Len() int {
return len(s)
}
func keyForIPPermission(p *ec2.IpPermission) string {
v, err := json.Marshal(p)
if err != nil {
panic(fmt.Sprintf("error building JSON representation of ec2.IpPermission: %v", err))
}
return string(v)
}
pkg/cloudprovider/providers/aws/tags.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/util/wait"
)
// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"
// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ. The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"
// ResourceLifecycle is the cluster lifecycle state used in tagging
type ResourceLifecycle string
const (
// ResourceLifecycleOwned is the value we use when tagging resources to indicate
// that the resource is considered owned and managed by the cluster,
// and in particular that the lifecycle is tied to the lifecycle of the cluster.
ResourceLifecycleOwned = "owned"
// ResourceLifecycleShared is the value we use when tagging resources to indicate
// that the resource is shared between multiple clusters, and should not be destroyed
// if the cluster is destroyed.
ResourceLifecycleShared = "shared"
)
type awsTagging struct {
// ClusterID is our cluster identifier: we tag AWS resources with this value,
// and thus we can run two independent clusters in the same VPC or subnets.
// This gives us similar functionality to GCE projects.
ClusterID string
// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
usesLegacyTags bool
}
func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
if legacyClusterID != "" {
if clusterID != "" && legacyClusterID != clusterID {
return fmt.Errorf("ClusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
}
t.usesLegacyTags = true
clusterID = legacyClusterID
}
t.ClusterID = clusterID
if clusterID != "" {
klog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
} else {
return fmt.Errorf("AWS cloud failed to find ClusterID")
}
return nil
}
// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
legacyClusterID, newClusterID, err := findClusterIDs(tags)
if err != nil {
return err
}
if legacyClusterID == "" && newClusterID == "" {
klog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
}
return t.init(legacyClusterID, newClusterID)
}
// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
legacyClusterID := ""
newClusterID := ""
for _, tag := range tags {
tagKey := aws.StringValue(tag.Key)
if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
if newClusterID != "" {
return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
}
newClusterID = id
}
if tagKey == TagNameKubernetesClusterLegacy {
id := aws.StringValue(tag.Value)
if legacyClusterID != "" {
return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
}
legacyClusterID = id
}
}
return legacyClusterID, newClusterID, nil
}
func (t *awsTagging) clusterTagKey() string {
return TagNameKubernetesClusterPrefix + t.ClusterID
}
func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
// if the clusterID is not configured -- we consider all instances.
if len(t.ClusterID) == 0 {
return true
}
clusterTagKey := t.clusterTagKey()
for _, tag := range tags {
tagKey := aws.StringValue(tag.Key)
// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
// Note that we want to continue traversing tag list if we see a legacy tag with value != ClusterID
if (tagKey == TagNameKubernetesClusterLegacy) && (aws.StringValue(tag.Value) == t.ClusterID) {
return true
}
if tagKey == clusterTagKey {
return true
}
}
return false
}
// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags. If it has a different cluster's tags, that is an error.
func (t *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
actualTagMap := make(map[string]string)
for _, tag := range observedTags {
actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
}
expectedTags := t.buildTags(lifecycle, additionalTags)
addTags := make(map[string]string)
for k, expected := range expectedTags {
actual := actualTagMap[k]
if actual == expected {
continue
}
if actual == "" {
klog.Warningf("Resource %q was missing expected cluster tag %q. Will add (with value %q)", resourceID, k, expected)
addTags[k] = expected
} else {
return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
}
}
if len(addTags) == 0 {
return nil
}
if err := t.createTags(client, resourceID, lifecycle, addTags); err != nil {
return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
}
return nil
}
// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
tags := t.buildTags(lifecycle, additionalTags)
if tags == nil || len(tags) == 0 {
return nil
}
var awsTags []*ec2.Tag
for k, v := range tags {
tag := &ec2.Tag{
Key: aws.String(k),
Value: aws.String(v),
}
awsTags = append(awsTags, tag)
}
backoff := wait.Backoff{
Duration: createTagInitialDelay,
Factor: createTagFactor,
Steps: createTagSteps,
}
request := &ec2.CreateTagsInput{}
request.Resources = []*string{&resourceID}
request.Tags = awsTags
var lastErr error
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
_, err := client.CreateTags(request)
if err == nil {
return true, nil
}
// We could check that the error is retryable, but the error code changes based on what we are tagging
// SecurityGroup: InvalidGroup.NotFound
klog.V(2).Infof("Failed to create tags; will retry. Error was %q", err)
lastErr = err
return false, nil
})
if err == wait.ErrWaitTimeout {
// return real CreateTags error instead of timeout
err = lastErr
}
return err
}
// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
// if there are no clusterID configured - no filtering by special tag names
// should be applied to revert to legacy behaviour.
if len(t.ClusterID) == 0 {
if len(filters) == 0 {
// We can't pass a zero-length Filters to AWS (it's an error)
// So if we end up with no filters; just return nil
return nil
}
return filters
}
// For 1.6, we always recognize the legacy tag, for the 1.5 -> 1.6 upgrade
// There are no "or" filters by key, so we look for both the legacy and new key, and then we have to post-filter
f := newEc2Filter("tag-key", TagNameKubernetesClusterLegacy, t.clusterTagKey())
// We can't pass a zero-length Filters to AWS (it's an error)
// So if we end up with no filters; we need to return nil
filters = append(filters, f)
return filters
}
func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
tags := make(map[string]string)
for k, v := range additionalTags {
tags[k] = v
}
// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
// strings
if len(t.ClusterID) == 0 {
return tags
}
// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
if t.usesLegacyTags {
tags[TagNameKubernetesClusterLegacy] = t.ClusterID
}
tags[t.clusterTagKey()] = string(lifecycle)
return tags
}
func (t *awsTagging) clusterID() string {
return t.ClusterID
}
pkg/cloudprovider/providers/aws/tags_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
)
func TestFilterTags(t *testing.T) {
awsServices := NewFakeAWSServices(TestClusterID)
c, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("Error building aws cloud: %v", err)
return
}
if c.tagging.ClusterID != TestClusterID {
t.Errorf("unexpected ClusterID: %v", c.tagging.ClusterID)
}
}
func TestFindClusterID(t *testing.T) {
grid := []struct {
Tags map[string]string
ExpectedNew string
ExpectedLegacy string
ExpectError bool
}{
{
Tags: map[string]string{},
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
},
ExpectedLegacy: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "owned",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "shared",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
TagNameKubernetesClusterPrefix + "a": "",
},
ExpectedLegacy: "a",
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "",
TagNameKubernetesClusterPrefix + "b": "",
},
ExpectError: true,
},
}
for _, g := range grid {
var ec2Tags []*ec2.Tag
for k, v := range g.Tags {
ec2Tags = append(ec2Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
}
actualLegacy, actualNew, err := findClusterIDs(ec2Tags)
if g.ExpectError {
if err == nil {
t.Errorf("expected error for tags %v", g.Tags)
continue
}
} else {
if err != nil {
t.Errorf("unexpected error for tags %v: %v", g.Tags, err)
continue
}
if g.ExpectedNew != actualNew {
t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedNew, actualNew)
continue
}
if g.ExpectedLegacy != actualLegacy {
t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedLegacy, actualLegacy)
continue
}
}
}
}
func TestHasClusterTag(t *testing.T) {
awsServices := NewFakeAWSServices(TestClusterID)
c, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("Error building aws cloud: %v", err)
return
}
grid := []struct {
Tags map[string]string
Expected bool
}{
{
Tags: map[string]string{},
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: TestClusterID,
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
},
Expected: false,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "owned",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
TagNameKubernetesClusterPrefix + TestClusterID: "shared",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "shared",
TagNameKubernetesClusterPrefix + "b": "shared",
},
Expected: true,
},
}
for _, g := range grid {
var ec2Tags []*ec2.Tag
for k, v := range g.Tags {
ec2Tags = append(ec2Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
}
result := c.tagging.hasClusterTag(ec2Tags)
if result != g.Expected {
t.Errorf("Unexpected result for tags %v: %t", g.Tags, result)
}
}
}
pkg/cloudprovider/providers/aws/volumes.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"net/url"
"regexp"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/types"
)
// awsVolumeRegMatch represents Regex Match for AWS volume.
var awsVolumeRegMatch = regexp.MustCompile("^vol-[^/]*$")
// EBSVolumeID represents the ID of the volume in the AWS API, e.g.
// vol-12345678 The "traditional" format is "vol-12345678" A new longer format
// is also being introduced: "vol-12345678abcdef01" We should not assume
// anything about the length or format, though it seems reasonable to assume
// that volumes will continue to start with "vol-".
type EBSVolumeID string
func (i EBSVolumeID) awsString() *string {
return aws.String(string(i))
}
// KubernetesVolumeID represents the id for a volume in the kubernetes API;
// a few forms are recognized:
// * aws://<zone>/<awsVolumeId>
// * aws:///<awsVolumeId>
// * <awsVolumeId>
type KubernetesVolumeID string
// DiskInfo returns aws disk information in easy to use manner
type diskInfo struct {
ec2Instance *ec2.Instance
nodeName types.NodeName
volumeState string
attachmentState string
hasAttachment bool
disk *awsDisk
}
// MapToAWSVolumeID extracts the EBSVolumeID from the KubernetesVolumeID
func (name KubernetesVolumeID) MapToAWSVolumeID() (EBSVolumeID, error) {
// name looks like aws://availability-zone/awsVolumeId
// The original idea of the URL-style name was to put the AZ into the
// host, so we could find the AZ immediately from the name without
// querying the API. But it turns out we don't actually need it for
// multi-AZ clusters, as we put the AZ into the labels on the PV instead.
// However, if in future we want to support multi-AZ cluster
// volume-awareness without using PersistentVolumes, we likely will
// want the AZ in the host.
s := string(name)
if !strings.HasPrefix(s, "aws://") {
// Assume a bare aws volume id (vol-1234...)
// Build a URL with an empty host (AZ)
s = "aws://" + "" + "/" + s
}
url, err := url.Parse(s)
if err != nil {
// TODO: Maybe we should pass a URL into the Volume functions
return "", fmt.Errorf("Invalid disk name (%s): %v", name, err)
}
if url.Scheme != "aws" {
return "", fmt.Errorf("Invalid scheme for AWS volume (%s)", name)
}
awsID := url.Path
awsID = strings.Trim(awsID, "/")
// We sanity check the resulting volume; the two known formats are
// vol-12345678 and vol-12345678abcdef01
if !awsVolumeRegMatch.MatchString(awsID) {
return "", fmt.Errorf("Invalid format for AWS volume (%s)", name)
}
return EBSVolumeID(awsID), nil
}
// GetAWSVolumeID converts a Kubernetes volume ID to an AWS volume ID
func GetAWSVolumeID(kubeVolumeID string) (string, error) {
kid := KubernetesVolumeID(kubeVolumeID)
awsID, err := kid.MapToAWSVolumeID()
return string(awsID), err
}
func (c *Cloud) checkIfAttachedToNode(diskName KubernetesVolumeID, nodeName types.NodeName) (*diskInfo, bool, error) {
disk, err := newAWSDisk(c, diskName)
if err != nil {
return nil, true, err
}
awsDiskInfo := &diskInfo{
disk: disk,
}
info, err := disk.describeVolume()
if err != nil {
klog.Warningf("Error describing volume %s with %v", diskName, err)
awsDiskInfo.volumeState = "unknown"
return awsDiskInfo, false, err
}
awsDiskInfo.volumeState = aws.StringValue(info.State)
if len(info.Attachments) > 0 {
attachment := info.Attachments[0]
awsDiskInfo.attachmentState = aws.StringValue(attachment.State)
instanceID := aws.StringValue(attachment.InstanceId)
instanceInfo, err := c.getInstanceByID(instanceID)
// This should never happen but if it does it could mean there was a race and instance
// has been deleted
if err != nil {
fetchErr := fmt.Errorf("Error fetching instance %s for volume %s", instanceID, diskName)
klog.Warning(fetchErr)
return awsDiskInfo, false, fetchErr
}
awsDiskInfo.ec2Instance = instanceInfo
awsDiskInfo.nodeName = mapInstanceToNodeName(instanceInfo)
awsDiskInfo.hasAttachment = true
if awsDiskInfo.nodeName == nodeName {
return awsDiskInfo, true, nil
}
}
return awsDiskInfo, false, nil
}
pkg/cloudprovider/providers/azure/BUILD deleted 100644 → 0
View file @ d4f4602f
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"azure.go",
"azure_backoff.go",
"azure_blobDiskController.go",
"azure_cache.go",
"azure_client.go",
"azure_controller_common.go",
"azure_controller_standard.go",
"azure_controller_vmss.go",
"azure_fakes.go",
"azure_file.go",
"azure_instance_metadata.go",
"azure_instances.go",
"azure_loadbalancer.go",
"azure_managedDiskController.go",
"azure_metrics.go",
"azure_routes.go",
"azure_standard.go",
"azure_storage.go",
"azure_storageaccount.go",
"azure_vmsets.go",
"azure_vmss.go",
"azure_vmss_cache.go",
"azure_wrap.go",
"azure_zones.go",
],
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/azure",
deps = [
"//pkg/api/v1/service:go_default_library",
"//pkg/cloudprovider/providers/azure/auth:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/version:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"azure_backoff_test.go",
"azure_cache_test.go",
"azure_instances_test.go",
"azure_loadbalancer_test.go",
"azure_metrics_test.go",
"azure_routes_test.go",
"azure_standard_test.go",
"azure_storage_test.go",
"azure_storageaccount_test.go",
"azure_test.go",
"azure_vmss_cache_test.go",
"azure_vmss_test.go",
"azure_wrap_test.go",
"azure_zones_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/api/v1/service:go_default_library",
"//pkg/cloudprovider/providers/azure/auth:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/cloudprovider/providers/azure/auth:all-srcs",
],
tags = ["automanaged"],
)
pkg/cloudprovider/providers/azure/OWNERS deleted 100644 → 0
View file @ d4f4602f
approvers:
- andyzhangx
- brendandburns
- feiskyer
- karataliu
- khenidak
reviewers:
- andyzhangx
- brendandburns
- feiskyer
- justaugustus
- karataliu
- khenidak
pkg/cloudprovider/providers/azure/auth/BUILD deleted 100644 → 0
View file @ d4f4602f
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["azure_auth.go"],
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
"//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
"//vendor/golang.org/x/crypto/pkcs12:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
pkg/cloudprovider/providers/azure/auth/azure_auth.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"io/ioutil"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/Azure/go-autorest/autorest/azure"
"golang.org/x/crypto/pkcs12"
"k8s.io/klog"
)
// AzureAuthConfig holds auth related part of cloud config
type AzureAuthConfig struct {
// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
Cloud string `json:"cloud" yaml:"cloud"`
// The AAD Tenant ID for the Subscription that the cluster is deployed in
TenantID string `json:"tenantId" yaml:"tenantId"`
// The ClientID for an AAD application with RBAC access to talk to Azure RM APIs
AADClientID string `json:"aadClientId" yaml:"aadClientId"`
// The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs
AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
// The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs
AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
// The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs
AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
// Use managed service identity for the virtual machine to access Azure ARM APIs
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
// UserAssignedIdentityID contains the Client ID of the user assigned MSI which is assigned to the underlying VMs. If empty the user assigned identity is not used.
// More details of the user assigned identity can be found at: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview
// For the user assigned identity specified here to be used, the UseManagedIdentityExtension has to be set to true.
UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
// The ID of the Azure Subscription that the cluster is deployed in
SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
}
// GetServicePrincipalToken creates a new service principal token based on the configuration
func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
if config.UseManagedIdentityExtension {
klog.V(2).Infoln("azure: using managed identity extension to retrieve access token")
msiEndpoint, err := adal.GetMSIVMEndpoint()
if err != nil {
return nil, fmt.Errorf("Getting the managed service identity endpoint: %v", err)
}
if len(config.UserAssignedIdentityID) > 0 {
klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token")
return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint,
env.ServiceManagementEndpoint,
config.UserAssignedIdentityID)
}
klog.V(4).Info("azure: using System Assigned MSI to retrieve access token")
return adal.NewServicePrincipalTokenFromMSI(
msiEndpoint,
env.ServiceManagementEndpoint)
}
oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID)
if err != nil {
return nil, fmt.Errorf("creating the OAuth config: %v", err)
}
if len(config.AADClientSecret) > 0 {
klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token")
return adal.NewServicePrincipalToken(
*oauthConfig,
config.AADClientID,
config.AADClientSecret,
env.ServiceManagementEndpoint)
}
if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token")
certData, err := ioutil.ReadFile(config.AADClientCertPath)
if err != nil {
return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err)
}
certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword)
if err != nil {
return nil, fmt.Errorf("decoding the client certificate: %v", err)
}
return adal.NewServicePrincipalTokenFromCertificate(
*oauthConfig,
config.AADClientID,
certificate,
privateKey,
env.ServiceManagementEndpoint)
}
return nil, fmt.Errorf("No credentials provided for AAD application %s", config.AADClientID)
}
// ParseAzureEnvironment returns azure environment by name
func ParseAzureEnvironment(cloudName string) (*azure.Environment, error) {
var env azure.Environment
var err error
if cloudName == "" {
env = azure.PublicCloud
} else {
env, err = azure.EnvironmentFromName(cloudName)
}
return &env, err
}
// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and
// the private RSA key
func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
privateKey, certificate, err := pkcs12.Decode(pkcs, password)
if err != nil {
return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err)
}
rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey)
if !isRsaKey {
return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key")
}
return certificate, rsaPrivateKey, nil
}
pkg/cloudprovider/providers/azure/azure_backoff_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"net/http"
"testing"
)
func TestShouldRetryHTTPRequest(t *testing.T) {
tests := []struct {
code int
err error
expected bool
}{
{
code: http.StatusBadRequest,
expected: true,
},
{
code: http.StatusInternalServerError,
expected: true,
},
{
code: http.StatusOK,
err: fmt.Errorf("some error"),
expected: true,
},
{
code: http.StatusOK,
expected: false,
},
{
code: 399,
expected: false,
},
}
for _, test := range tests {
resp := &http.Response{
StatusCode: test.code,
}
res := shouldRetryHTTPRequest(resp, test.err)
if res != test.expected {
t.Errorf("expected: %v, saw: %v", test.expected, res)
}
}
}
func TestIsSuccessResponse(t *testing.T) {
tests := []struct {
code int
expected bool
}{
{
code: http.StatusNotFound,
expected: false,
},
{
code: http.StatusInternalServerError,
expected: false,
},
{
code: http.StatusOK,
expected: true,
},
}
for _, test := range tests {
resp := http.Response{
StatusCode: test.code,
}
res := isSuccessHTTPResponse(resp)
if res != test.expected {
t.Errorf("expected: %v, saw: %v", test.expected, res)
}
}
}
func TestProcessRetryResponse(t *testing.T) {
az := &Cloud{}
tests := []struct {
code int
err error
stop bool
}{
{
code: http.StatusBadRequest,
stop: false,
},
{
code: http.StatusInternalServerError,
stop: false,
},
{
code: http.StatusSeeOther,
err: fmt.Errorf("some error"),
stop: false,
},
{
code: http.StatusSeeOther,
stop: true,
},
{
code: http.StatusOK,
stop: true,
},
{
code: 399,
stop: true,
},
}
for _, test := range tests {
resp := &http.Response{
StatusCode: test.code,
}
res, err := az.processHTTPRetryResponse(nil, "", resp, test.err)
if res != test.stop {
t.Errorf("expected: %v, saw: %v", test.stop, res)
}
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
}
pkg/cloudprovider/providers/azure/azure_cache.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"sync"
"time"
"k8s.io/client-go/tools/cache"
)
// getFunc defines a getter function for timedCache.
type getFunc func(key string) (interface{}, error)
// cacheEntry is the internal structure stores inside TTLStore.
type cacheEntry struct {
key string
data interface{}
// The lock to ensure not updating same entry simultaneously.
lock sync.Mutex
}
// cacheKeyFunc defines the key function required in TTLStore.
func cacheKeyFunc(obj interface{}) (string, error) {
return obj.(*cacheEntry).key, nil
}
// timedCache is a cache with TTL.
type timedCache struct {
store cache.Store
lock sync.Mutex
getter getFunc
}
// newTimedcache creates a new timedCache.
func newTimedcache(ttl time.Duration, getter getFunc) (*timedCache, error) {
if getter == nil {
return nil, fmt.Errorf("getter is not provided")
}
return &timedCache{
getter: getter,
store: cache.NewTTLStore(cacheKeyFunc, ttl),
}, nil
}
// getInternal returns cacheEntry by key. If the key is not cached yet,
// it returns a cacheEntry with nil data.
func (t *timedCache) getInternal(key string) (*cacheEntry, error) {
entry, exists, err := t.store.GetByKey(key)
if err != nil {
return nil, err
}
if exists {
return entry.(*cacheEntry), nil
}
t.lock.Lock()
defer t.lock.Unlock()
entry, exists, err = t.store.GetByKey(key)
if err != nil {
return nil, err
}
if exists {
return entry.(*cacheEntry), nil
}
// Still not found, add new entry with nil data.
// Note the data will be filled later by getter.
newEntry := &cacheEntry{
key: key,
data: nil,
}
t.store.Add(newEntry)
return newEntry, nil
}
// Get returns the requested item by key.
func (t *timedCache) Get(key string) (interface{}, error) {
entry, err := t.getInternal(key)
if err != nil {
return nil, err
}
// Data is still not cached yet, cache it by getter.
if entry.data == nil {
entry.lock.Lock()
defer entry.lock.Unlock()
if entry.data == nil {
data, err := t.getter(key)
if err != nil {
return nil, err
}
entry.data = data
}
}
return entry.data, nil
}
// Delete removes an item from the cache.
func (t *timedCache) Delete(key string) error {
return t.store.Delete(&cacheEntry{
key: key,
})
}
pkg/cloudprovider/providers/azure/azure_cache_test.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var (
fakeCacheTTL = 2 * time.Second
)
type fakeDataObj struct{}
type fakeDataSource struct {
called int
data map[string]*fakeDataObj
lock sync.Mutex
}
func (fake *fakeDataSource) get(key string) (interface{}, error) {
fake.lock.Lock()
defer fake.lock.Unlock()
fake.called = fake.called + 1
if v, ok := fake.data[key]; ok {
return v, nil
}
return nil, nil
}
func (fake *fakeDataSource) set(data map[string]*fakeDataObj) {
fake.lock.Lock()
defer fake.lock.Unlock()
fake.data = data
fake.called = 0
}
func newFakeCache(t *testing.T) (*fakeDataSource, *timedCache) {
dataSource := &fakeDataSource{
data: make(map[string]*fakeDataObj),
}
getter := dataSource.get
cache, err := newTimedcache(fakeCacheTTL, getter)
assert.NoError(t, err)
return dataSource, cache
}
func TestCacheGet(t *testing.T) {
val := &fakeDataObj{}
cases := []struct {
name string
data map[string]*fakeDataObj
key string
expected interface{}
}{
{
name: "cache should return nil for empty data source",
key: "key1",
expected: nil,
},
{
name: "cache should return nil for non exist key",
data: map[string]*fakeDataObj{"key2": val},
key: "key1",
expected: nil,
},
{
name: "cache should return data for existing key",
data: map[string]*fakeDataObj{"key1": val},
key: "key1",
expected: val,
},
}
for _, c := range cases {
dataSource, cache := newFakeCache(t)
dataSource.set(c.data)
val, err := cache.Get(c.key)
assert.NoError(t, err, c.name)
assert.Equal(t, c.expected, val, c.name)
}
}
func TestCacheGetError(t *testing.T) {
getError := fmt.Errorf("getError")
getter := func(key string) (interface{}, error) {
return nil, getError
}
cache, err := newTimedcache(fakeCacheTTL, getter)
assert.NoError(t, err)
val, err := cache.Get("key")
assert.Error(t, err)
assert.Equal(t, getError, err)
assert.Nil(t, val)
}
func TestCacheDelete(t *testing.T) {
key := "key1"
val := &fakeDataObj{}
data := map[string]*fakeDataObj{
key: val,
}
dataSource, cache := newFakeCache(t)
dataSource.set(data)
v, err := cache.Get(key)
assert.NoError(t, err)
assert.Equal(t, val, v, "cache should get correct data")
dataSource.set(nil)
cache.Delete(key)
v, err = cache.Get(key)
assert.NoError(t, err)
assert.Equal(t, 1, dataSource.called)
assert.Equal(t, nil, v, "cache should get nil after data is removed")
}
func TestCacheExpired(t *testing.T) {
key := "key1"
val := &fakeDataObj{}
data := map[string]*fakeDataObj{
key: val,
}
dataSource, cache := newFakeCache(t)
dataSource.set(data)
v, err := cache.Get(key)
assert.NoError(t, err)
assert.Equal(t, 1, dataSource.called)
assert.Equal(t, val, v, "cache should get correct data")
time.Sleep(fakeCacheTTL)
v, err = cache.Get(key)
assert.NoError(t, err)
assert.Equal(t, 2, dataSource.called)
assert.Equal(t, val, v, "cache should get correct data even after expired")
}
pkg/cloudprovider/providers/azure/azure_controller_common.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"time"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/types"
kwait "k8s.io/apimachinery/pkg/util/wait"
cloudprovider "k8s.io/cloud-provider"
)
const (
storageAccountNameTemplate = "pvc%s"
// for limits check https://docs.microsoft.com/en-us/azure/azure-subscription-service-limits#storage-limits
maxStorageAccounts = 100 // max # is 200 (250 with special request). this allows 100 for everything else including stand alone disks
maxDisksPerStorageAccounts = 60
storageAccountUtilizationBeforeGrowing = 0.5
maxLUN = 64 // max number of LUNs per VM
errLeaseFailed = "AcquireDiskLeaseFailed"
errLeaseIDMissing = "LeaseIdMissing"
errContainerNotFound = "ContainerNotFound"
errDiskBlobNotFound = "DiskBlobNotFound"
)
var defaultBackOff = kwait.Backoff{
Steps: 20,
Duration: 2 * time.Second,
Factor: 1.5,
Jitter: 0.0,
}
type controllerCommon struct {
subscriptionID string
location string
storageEndpointSuffix string
resourceGroup string
cloud *Cloud
}
// getNodeVMSet gets the VMSet interface based on config.VMType and the real virtual machine type.
func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName) (VMSet, error) {
// 1. vmType is standard, return cloud.vmSet directly.
if c.cloud.VMType == vmTypeStandard {
return c.cloud.vmSet, nil
}
// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
ss, ok := c.cloud.vmSet.(*scaleSet)
if !ok {
return nil, fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
}
// 3. If the node is managed by availability set, then return ss.availabilitySet.
managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
if err != nil {
return nil, err
}
if managedByAS {
// vm is managed by availability set.
return ss.availabilitySet, nil
}
// 4. Node is managed by vmss
return ss, nil
}
// AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI, and lun.
func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
vmset, err := c.getNodeVMSet(nodeName)
if err != nil {
return err
}
return vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
}
// DetachDiskByName detaches a vhd from host. The vhd can be identified by diskName or diskURI.
func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
vmset, err := c.getNodeVMSet(nodeName)
if err != nil {
return err
}
return vmset.DetachDiskByName(diskName, diskURI, nodeName)
}
// getNodeDataDisks invokes vmSet interfaces to get data disks for the node.
func (c *controllerCommon) getNodeDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) {
vmset, err := c.getNodeVMSet(nodeName)
if err != nil {
return nil, err
}
return vmset.GetDataDisks(nodeName)
}
// GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI.
func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
disks, err := c.getNodeDataDisks(nodeName)
if err != nil {
klog.Errorf("error of getting data disks for node %q: %v", nodeName, err)
return -1, err
}
for _, disk := range disks {
if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
(disk.ManagedDisk != nil && *disk.ManagedDisk.ID == diskURI) {
// found the disk
klog.V(2).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
return *disk.Lun, nil
}
}
return -1, fmt.Errorf("Cannot find Lun for disk %s", diskName)
}
// GetNextDiskLun searches all vhd attachment on the host and find unused lun. Return -1 if all luns are used.
func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
disks, err := c.getNodeDataDisks(nodeName)
if err != nil {
klog.Errorf("error of getting data disks for node %q: %v", nodeName, err)
return -1, err
}
used := make([]bool, maxLUN)
for _, disk := range disks {
if disk.Lun != nil {
used[*disk.Lun] = true
}
}
for k, v := range used {
if !v {
return int32(k), nil
}
}
return -1, fmt.Errorf("all luns are used")
}
// DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName.
func (c *controllerCommon) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
attached := make(map[string]bool)
for _, diskName := range diskNames {
attached[diskName] = false
}
disks, err := c.getNodeDataDisks(nodeName)
if err != nil {
if err == cloudprovider.InstanceNotFound {
// if host doesn't exist, no need to detach
klog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.",
nodeName, diskNames)
return attached, nil
}
return attached, err
}
for _, disk := range disks {
for _, diskName := range diskNames {
if disk.Name != nil && diskName != "" && *disk.Name == diskName {
attached[diskName] = true
}
}
}
return attached, nil
}
pkg/cloudprovider/providers/azure/azure_controller_standard.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/types"
)
// AttachDisk attaches a vhd to vm
// the vhd must exist, can be identified by diskName, diskURI, and lun.
func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
vm, err := as.getVirtualMachine(nodeName)
if err != nil {
return err
}
vmName := mapNodeNameToVMName(nodeName)
nodeResourceGroup, err := as.GetNodeResourceGroup(vmName)
if err != nil {
return err
}
disks := *vm.StorageProfile.DataDisks
if isManagedDisk {
disks = append(disks,
compute.DataDisk{
Name: &diskName,
Lun: &lun,
Caching: cachingMode,
CreateOption: "attach",
ManagedDisk: &compute.ManagedDiskParameters{
ID: &diskURI,
},
})
} else {
disks = append(disks,
compute.DataDisk{
Name: &diskName,
Vhd: &compute.VirtualHardDisk{
URI: &diskURI,
},
Lun: &lun,
Caching: cachingMode,
CreateOption: "attach",
})
}
newVM := compute.VirtualMachine{
Location: vm.Location,
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: vm.HardwareProfile,
StorageProfile: &compute.StorageProfile{
DataDisks: &disks,
},
},
}
klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, vmName, diskName, diskURI)
ctx, cancel := getContextWithCancel()
defer cancel()
// Invalidate the cache right after updating
defer as.cloud.vmCache.Delete(vmName)
_, err = as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM)
if err != nil {
klog.Errorf("azureDisk - attach disk(%s, %s) failed, err: %v", diskName, diskURI, err)
detail := err.Error()
if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
klog.V(2).Infof("azureDisk - err %v, try detach disk(%s, %s)", err, diskName, diskURI)
as.DetachDiskByName(diskName, diskURI, nodeName)
}
} else {
klog.V(2).Infof("azureDisk - attach disk(%s, %s) succeeded", diskName, diskURI)
}
return err
}
// DetachDiskByName detaches a vhd from host
// the vhd can be identified by diskName or diskURI
func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
vm, err := as.getVirtualMachine(nodeName)
if err != nil {
// if host doesn't exist, no need to detach
klog.Warningf("azureDisk - cannot find node %s, skip detaching disk(%s, %s)", nodeName, diskName, diskURI)
return nil
}
vmName := mapNodeNameToVMName(nodeName)
nodeResourceGroup, err := as.GetNodeResourceGroup(vmName)
if err != nil {
return err
}
disks := *vm.StorageProfile.DataDisks
bFoundDisk := false
for i, disk := range disks {
if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
// found the disk
klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
disks = append(disks[:i], disks[i+1:]...)
bFoundDisk = true
break
}
}
if !bFoundDisk {
return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
}
newVM := compute.VirtualMachine{
Location: vm.Location,
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: vm.HardwareProfile,
StorageProfile: &compute.StorageProfile{
DataDisks: &disks,
},
},
}
klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s, %s)", nodeResourceGroup, vmName, diskName, diskURI)
ctx, cancel := getContextWithCancel()
defer cancel()
// Invalidate the cache right after updating
defer as.cloud.vmCache.Delete(vmName)
resp, err := as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM)
if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err)
retryErr := as.CreateOrUpdateVMWithRetry(nodeResourceGroup, vmName, newVM)
if retryErr != nil {
err = retryErr
klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err)
}
}
if err != nil {
klog.Errorf("azureDisk - detach disk(%s, %s) failed, err: %v", diskName, diskURI, err)
} else {
klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI)
}
return err
}
// GetDataDisks gets a list of data disks attached to the node.
func (as *availabilitySet) GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) {
vm, err := as.getVirtualMachine(nodeName)
if err != nil {
return nil, err
}
if vm.StorageProfile.DataDisks == nil {
return nil, nil
}
return *vm.StorageProfile.DataDisks, nil
}
pkg/cloudprovider/providers/azure/azure_controller_vmss.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/types"
)
// AttachDisk attaches a vhd to vm
// the vhd must exist, can be identified by diskName, diskURI, and lun.
func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
vmName := mapNodeNameToVMName(nodeName)
ssName, instanceID, vm, err := ss.getVmssVM(vmName)
if err != nil {
return err
}
nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
if err != nil {
return err
}
disks := []compute.DataDisk{}
if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
disks = *vm.StorageProfile.DataDisks
}
if isManagedDisk {
disks = append(disks,
compute.DataDisk{
Name: &diskName,
Lun: &lun,
Caching: compute.CachingTypes(cachingMode),
CreateOption: "attach",
ManagedDisk: &compute.ManagedDiskParameters{
ID: &diskURI,
},
})
} else {
disks = append(disks,
compute.DataDisk{
Name: &diskName,
Vhd: &compute.VirtualHardDisk{
URI: &diskURI,
},
Lun: &lun,
Caching: compute.CachingTypes(cachingMode),
CreateOption: "attach",
})
}
newVM := compute.VirtualMachineScaleSetVM{
Sku: vm.Sku,
Location: vm.Location,
VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
HardwareProfile: vm.HardwareProfile,
StorageProfile: &compute.StorageProfile{
OsDisk: vm.StorageProfile.OsDisk,
DataDisks: &disks,
},
},
}
ctx, cancel := getContextWithCancel()
defer cancel()
// Invalidate the cache right after updating
key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
defer ss.vmssVMCache.Delete(key)
klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI)
_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM)
if err != nil {
detail := err.Error()
if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
klog.Infof("azureDisk - err %s, try detach disk(%s, %s)", detail, diskName, diskURI)
ss.DetachDiskByName(diskName, diskURI, nodeName)
}
} else {
klog.V(2).Infof("azureDisk - attach disk(%s, %s) succeeded", diskName, diskURI)
}
return err
}
// DetachDiskByName detaches a vhd from host
// the vhd can be identified by diskName or diskURI
func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
vmName := mapNodeNameToVMName(nodeName)
ssName, instanceID, vm, err := ss.getVmssVM(vmName)
if err != nil {
return err
}
nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
if err != nil {
return err
}
disks := []compute.DataDisk{}
if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
disks = *vm.StorageProfile.DataDisks
}
bFoundDisk := false
for i, disk := range disks {
if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
// found the disk
klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
disks = append(disks[:i], disks[i+1:]...)
bFoundDisk = true
break
}
}
if !bFoundDisk {
return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
}
newVM := compute.VirtualMachineScaleSetVM{
Sku: vm.Sku,
Location: vm.Location,
VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
HardwareProfile: vm.HardwareProfile,
StorageProfile: &compute.StorageProfile{
OsDisk: vm.StorageProfile.OsDisk,
DataDisks: &disks,
},
},
}
ctx, cancel := getContextWithCancel()
defer cancel()
// Invalidate the cache right after updating
key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
defer ss.vmssVMCache.Delete(key)
klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI)
resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM)
if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err)
retryErr := ss.UpdateVmssVMWithRetry(ctx, nodeResourceGroup, ssName, instanceID, newVM)
if retryErr != nil {
err = retryErr
klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err)
}
}
if err != nil {
klog.Errorf("azureDisk - detach disk(%s, %s) from %s failed, err: %v", diskName, diskURI, nodeName, err)
} else {
klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI)
}
return err
}
// GetDataDisks gets a list of data disks attached to the node.
func (ss *scaleSet) GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) {
_, _, vm, err := ss.getVmssVM(string(nodeName))
if err != nil {
return nil, err
}
if vm.StorageProfile == nil || vm.StorageProfile.DataDisks == nil {
return nil, nil
}
return *vm.StorageProfile.DataDisks, nil
}
pkg/cloudprovider/providers/azure/azure_file.go deleted 100644 → 0
View file @ d4f4602f
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
azs "github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
"k8s.io/klog"
)
const (
useHTTPS = true
)
// FileClient is the interface for creating file shares, interface for test
// injection.
type FileClient interface {
createFileShare(accountName, accountKey, name string, sizeGiB int) error
deleteFileShare(accountName, accountKey, name string) error
resizeFileShare(accountName, accountKey, name string, sizeGiB int) error
}
// create file share
func (az *Cloud) createFileShare(accountName, accountKey, name string, sizeGiB int) error {
return az.FileClient.createFileShare(accountName, accountKey, name, sizeGiB)
}
func (az *Cloud) deleteFileShare(accountName, accountKey, name string) error {
return az.FileClient.deleteFileShare(accountName, accountKey, name)
}
func (az *Cloud) resizeFileShare(accountName, accountKey, name string, sizeGiB int) error {
return az.FileClient.resizeFileShare(accountName, accountKey, name, sizeGiB)
}
type azureFileClient struct {
env azure.Environment
}
func (f *azureFileClient) createFileShare(accountName, accountKey, name string, sizeGiB int) error {
fileClient, err := f.getFileSvcClient(accountName, accountKey)
if err != nil {
return err
}
share := fileClient.GetShareReference(name)
share.Properties.Quota = sizeGiB
if err = share.Create(nil); err != nil {
return fmt.Errorf("failed to create file share, err: %v", err)
}
return nil
}
// delete a file share
func (f *azureFileClient) deleteFileShare(accountName, accountKey, name string) error {
fileClient, err := f.getFileSvcClient(accountName, accountKey)
if err != nil {
return err
}
return fileClient.GetShareReference(name).Delete(nil)
}
func (f *azureFileClient) resizeFileShare(accountName, accountKey, name string, sizeGiB int) error {
fileClient, err := f.getFileSvcClient(accountName, accountKey)
if err != nil {
return err
}
share := fileClient.GetShareReference(name)
if share.Properties.Quota >= sizeGiB {
klog.Warningf("file share size(%dGi) is already greater or equal than requested size(%dGi), accountName: %s, shareName: %s",
share.Properties.Quota, sizeGiB, accountName, name)
return nil
}
share.Properties.Quota = sizeGiB
if err = share.SetProperties(nil); err != nil {
return fmt.Errorf("failed to set quota on file share %s, err: %v", name, err)
}
klog.V(4).Infof("resize file share completed, accountName: %s, shareName: %s, sizeGiB: %d", accountName, name, sizeGiB)
return nil
}
func (f *azureFileClient) getFileSvcClient(accountName, accountKey string) (*azs.FileServiceClient, error) {
fileClient, err := azs.NewClient(accountName, accountKey, f.env.StorageEndpointSuffix, azs.DefaultAPIVersion, useHTTPS)
if err != nil {
return nil, fmt.Errorf("error creating azure client: %v", err)
}
fc := fileClient.GetFileService()
return &fc, nil
}
pkg/cloudprovider/providers/gce/cloud/gen.go deleted 100644 → 0
View file @ d4f4602f
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment