Commit f270cb1b authored by Jan Safranek's avatar Jan Safranek

Allow 39 atached EBS devices.

AWS has soft support limit for 40 attached EBS devices. Assuming there is just one root device, use the rest for persistent volumes. The devices will have name /dev/xvdba - /dev/xvdcm, leaving /dev/sda - /dev/sdz to the system. Also, add better error handling and propagate error "Too many EBS volumes attached to node XYZ" to a pod.
parent 70a303d3
...@@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30 ...@@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30
// need hardcoded defaults. // need hardcoded defaults.
const DefaultVolumeType = "gp2" const DefaultVolumeType = "gp2"
// Amazon recommends having no more that 40 volumes attached to an instance,
// and at least one of those is for the system root volume.
// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits
const DefaultMaxEBSVolumes = 39
// Used to call aws_credentials.Init() just once // Used to call aws_credentials.Init() just once
var once sync.Once var once sync.Once
...@@ -902,10 +907,17 @@ type mountDevice string ...@@ -902,10 +907,17 @@ type mountDevice string
// TODO: Also return number of mounts allowed? // TODO: Also return number of mounts allowed?
func (self *awsInstanceType) getEBSMountDevices() []mountDevice { func (self *awsInstanceType) getEBSMountDevices() []mountDevice {
// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
// We will generate "ba", "bb", "bc"..."bz", "ca", ..., up to DefaultMaxEBSVolumes
devices := []mountDevice{} devices := []mountDevice{}
for c := 'f'; c <= 'p'; c++ { count := 0
devices = append(devices, mountDevice(fmt.Sprintf("%c", c))) for first := 'b'; count < DefaultMaxEBSVolumes; first++ {
for second := 'a'; count < DefaultMaxEBSVolumes && second <= 'z'; second++ {
device := mountDevice(fmt.Sprintf("%c%c", first, second))
devices = append(devices, device)
count++
}
} }
return devices return devices
} }
...@@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned ...@@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
if strings.HasPrefix(name, "/dev/xvd") { if strings.HasPrefix(name, "/dev/xvd") {
name = name[8:] name = name[8:]
} }
if len(name) != 1 { if len(name) < 1 || len(name) > 2 {
glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName))
} }
deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId) deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId)
...@@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned ...@@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
if chosen == "" { if chosen == "" {
glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid) glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid)
return "", false, nil return "", false, fmt.Errorf("Too many EBS volumes attached to node %s.", self.nodeName)
} }
self.attaching[chosen] = volumeID self.attaching[chosen] = volumeID
......
...@@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin ...@@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. // Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) { func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) {
var awsCloud *aws.AWSCloud var awsCloud *aws.AWSCloud
var attachError error
for numRetries := 0; numRetries < maxRetries; numRetries++ { for numRetries := 0; numRetries < maxRetries; numRetries++ {
var err error var err error
if awsCloud == nil { if awsCloud == nil {
...@@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin ...@@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries) glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries)
} }
devicePath, err := awsCloud.AttachDisk(b.volumeID, "", b.readOnly) var devicePath string
if err != nil { devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly)
glog.Errorf("Error attaching PD %q: %v", b.volumeID, err) if attachError != nil {
glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError)
time.Sleep(errorSleepDuration) time.Sleep(errorSleepDuration)
continue continue
} }
...@@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin ...@@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
} }
} }
if attachError != nil {
return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError)
}
return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID) return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID)
} }
......
...@@ -21,6 +21,7 @@ import ( ...@@ -21,6 +21,7 @@ import (
"os" "os"
"strconv" "strconv"
"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
"k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler"
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
...@@ -31,10 +32,6 @@ import ( ...@@ -31,10 +32,6 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
) )
// Amazon recommends having no more that 40 volumes attached to an instance,
// and at least one of those is for the system root volume.
const DefaultMaxEBSVolumes = 39
// GCE instances can have up to 16 PD volumes attached. // GCE instances can have up to 16 PD volumes attached.
const DefaultMaxGCEPDVolumes = 16 const DefaultMaxGCEPDVolumes = 16
...@@ -117,7 +114,7 @@ func defaultPredicates() sets.String { ...@@ -117,7 +114,7 @@ func defaultPredicates() sets.String {
"MaxEBSVolumeCount", "MaxEBSVolumeCount",
func(args factory.PluginFactoryArgs) algorithm.FitPredicate { func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
// TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
maxVols := getMaxVols(DefaultMaxEBSVolumes) maxVols := getMaxVols(aws.DefaultMaxEBSVolumes)
return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo) return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
}, },
), ),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment