Unverified Commit 37b29297 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #67432 from lichuqiang/topo_provision_beta

Automatic merge from submit-queue (batch tested with PRs 67745, 67432, 67569, 67825, 67943). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Move volume dynamic provisioning scheduling to beta **What this PR does / why we need it**: * Combine feature gate VolumeScheduling and DynamicProvisioningScheduling into one * Add allowedTopologies description in kubectl **Special notes for your reviewer**: Wait until related e2e and downside plugins are ready. /hold **Release note**: ```release-note Move volume dynamic provisioning scheduling to beta (ACTION REQUIRED: The DynamicProvisioningScheduling alpha feature gate has been removed. The VolumeScheduling beta feature gate is still required for this feature) ```
parents 720781e6 eefd337b
......@@ -88492,7 +88492,7 @@
"type": "boolean"
},
"allowedTopologies": {
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.",
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.",
"type": "array",
"items": {
"$ref": "#/definitions/io.k8s.api.core.v1.TopologySelectorTerm"
......@@ -88533,7 +88533,7 @@
"type": "string"
},
"volumeBindingMode": {
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.",
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.",
"type": "string"
}
},
......@@ -88730,7 +88730,7 @@
"type": "boolean"
},
"allowedTopologies": {
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.",
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.",
"type": "array",
"items": {
"$ref": "#/definitions/io.k8s.api.core.v1.TopologySelectorTerm"
......@@ -88771,7 +88771,7 @@
"type": "string"
},
"volumeBindingMode": {
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.",
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.",
"type": "string"
}
},
......@@ -830,14 +830,14 @@
},
"volumeBindingMode": {
"$ref": "v1.VolumeBindingMode",
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature."
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature."
},
"allowedTopologies": {
"type": "array",
"items": {
"$ref": "v1.TopologySelectorTerm"
},
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature."
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature."
}
}
},
......
......@@ -1537,14 +1537,14 @@
},
"volumeBindingMode": {
"$ref": "v1beta1.VolumeBindingMode",
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature."
"description": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature."
},
"allowedTopologies": {
"type": "array",
"items": {
"$ref": "v1.TopologySelectorTerm"
},
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature."
"description": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature."
}
}
},
......
......@@ -1046,14 +1046,14 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">volumeBindingMode</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1_volumebindingmode">v1.VolumeBindingMode</a></p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">allowedTopologies</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1_topologyselectorterm">v1.TopologySelectorTerm</a> array</p></td>
<td class="tableblock halign-left valign-top"></td>
......
......@@ -1283,14 +1283,14 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">volumeBindingMode</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1beta1_volumebindingmode">v1beta1.VolumeBindingMode</a></p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">allowedTopologies</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1_topologyselectorterm">v1.TopologySelectorTerm</a> array</p></td>
<td class="tableblock halign-left valign-top"></td>
......
......@@ -3153,7 +3153,7 @@ func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field
exprMap := make(map[string]sets.String)
exprPath := fldPath.Child("matchLabelExpressions")
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
// Allow empty MatchLabelExpressions, in case this field becomes optional in the future.
for i, req := range term.MatchLabelExpressions {
......@@ -3168,7 +3168,7 @@ func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field
exprMap[req.Key] = valueSet
}
} else if len(term.MatchLabelExpressions) != 0 {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling"))
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate VolumeScheduling"))
}
return exprMap, allErrs
......
......@@ -68,16 +68,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable
// the VolumeScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
AllowedTopologies []api.TopologySelectorTerm
}
......
......@@ -26,8 +26,6 @@ import (
func DropDisabledAlphaFields(class *storage.StorageClass) {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
class.VolumeBindingMode = nil
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
class.AllowedTopologies = nil
}
}
......@@ -39,8 +39,8 @@ func TestDropAlphaFields(t *testing.T) {
}
// Test that field gets dropped when feature gate is not set
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,DynamicProvisioningScheduling=false"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err)
}
class := &storage.StorageClass{
VolumeBindingMode: &bindingMode,
......@@ -59,8 +59,8 @@ func TestDropAlphaFields(t *testing.T) {
VolumeBindingMode: &bindingMode,
AllowedTopologies: allowedTopologies,
}
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true,DynamicProvisioningScheduling=true"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err)
}
DropDisabledAlphaFields(class)
if class.VolumeBindingMode != &bindingMode {
......@@ -70,7 +70,7 @@ func TestDropAlphaFields(t *testing.T) {
t.Errorf("AllowedTopologies field got unexpectantly modified: %+v", class.AllowedTopologies)
}
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,DynamicProvisioningScheduling=false"); err != nil {
t.Fatalf("Failed to disable feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil {
t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err)
}
}
......@@ -250,8 +250,8 @@ func validateAllowedTopologies(topologies []api.TopologySelectorTerm, fldPath *f
return allErrs
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling"))
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate VolumeScheduling"))
}
rawTopologies := make([]map[string]sets.String, len(topologies))
......
......@@ -827,73 +827,67 @@ func TestValidateAllowedTopologies(t *testing.T) {
cases := map[string]bindingTest{
"no topology": {
class: makeClass(nil, nil),
class: makeClass(&waitingMode, nil),
shouldSucceed: true,
},
"valid topology": {
class: makeClass(nil, validTopology),
class: makeClass(&waitingMode, validTopology),
shouldSucceed: true,
},
"topology invalid key": {
class: makeClass(nil, topologyInvalidKey),
class: makeClass(&waitingMode, topologyInvalidKey),
shouldSucceed: false,
},
"topology lack of values": {
class: makeClass(nil, topologyLackOfValues),
class: makeClass(&waitingMode, topologyLackOfValues),
shouldSucceed: false,
},
"duplicate TopologySelectorRequirement values": {
class: makeClass(nil, topologyDupValues),
class: makeClass(&waitingMode, topologyDupValues),
shouldSucceed: false,
},
"multiple TopologySelectorRequirement values": {
class: makeClass(nil, topologyMultiValues),
class: makeClass(&waitingMode, topologyMultiValues),
shouldSucceed: true,
},
"empty MatchLabelExpressions": {
class: makeClass(nil, topologyEmptyMatchLabelExpressions),
class: makeClass(&waitingMode, topologyEmptyMatchLabelExpressions),
shouldSucceed: false,
},
"duplicate MatchLabelExpression keys": {
class: makeClass(nil, topologyDupKeys),
class: makeClass(&waitingMode, topologyDupKeys),
shouldSucceed: false,
},
"duplicate MatchLabelExpression keys but across separate terms": {
class: makeClass(nil, topologyMultiTerm),
class: makeClass(&waitingMode, topologyMultiTerm),
shouldSucceed: true,
},
"duplicate AllowedTopologies terms - identical": {
class: makeClass(nil, topologyDupTermsIdentical),
class: makeClass(&waitingMode, topologyDupTermsIdentical),
shouldSucceed: false,
},
"two AllowedTopologies terms, with a pair of the same MatchLabelExpressions and a pair of different ones": {
class: makeClass(nil, topologyExprsOneSameOneDiff),
class: makeClass(&waitingMode, topologyExprsOneSameOneDiff),
shouldSucceed: true,
},
"two AllowedTopologies terms, with a pair of the same Values and a pair of different ones": {
class: makeClass(nil, topologyValuesOneSameOneDiff),
class: makeClass(&waitingMode, topologyValuesOneSameOneDiff),
shouldSucceed: true,
},
"duplicate AllowedTopologies terms - different MatchLabelExpressions order": {
class: makeClass(nil, topologyDupTermsDiffExprOrder),
class: makeClass(&waitingMode, topologyDupTermsDiffExprOrder),
shouldSucceed: false,
},
"duplicate AllowedTopologies terms - different TopologySelectorRequirement values order": {
class: makeClass(nil, topologyDupTermsDiffValueOrder),
class: makeClass(&waitingMode, topologyDupTermsDiffValueOrder),
shouldSucceed: false,
},
}
// Disable VolumeScheduling so nil VolumeBindingMode doesn't fail to validate.
err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false")
if err != nil {
t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err)
}
// TODO: remove when feature gate not required
err = utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true")
err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true")
if err != nil {
t.Fatalf("Failed to enable feature gate for DynamicProvisioningScheduling: %v", err)
t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err)
}
for testName, testCase := range cases {
......@@ -906,9 +900,9 @@ func TestValidateAllowedTopologies(t *testing.T) {
}
}
err = utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=false")
err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false")
if err != nil {
t.Fatalf("Failed to disable feature gate for DynamicProvisioningScheduling: %v", err)
t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err)
}
for testName, testCase := range cases {
......
......@@ -140,7 +140,7 @@ const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
// This annotation is added to a PVC that has been triggered by scheduler to
// be dynamically provisioned. Its value is the name of the selected node.
const annSelectedNode = "volume.alpha.kubernetes.io/selected-node"
const annSelectedNode = "volume.kubernetes.io/selected-node"
// If the provisioner name in a storage class is set to "kubernetes.io/no-provisioner",
// then dynamic provisioning is not supported by the storage.
......@@ -290,14 +290,12 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV
return false, nil
}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
// When feature DynamicProvisioningScheduling enabled,
// Scheduler signal to the PV controller to start dynamic
// provisioning by setting the "annSelectedNode" annotation
// in the PVC
if _, ok := claim.Annotations[annSelectedNode]; ok {
return false, nil
}
// When feature VolumeScheduling enabled,
// Scheduler signal to the PV controller to start dynamic
// provisioning by setting the "annSelectedNode" annotation
// in the PVC
if _, ok := claim.Annotations[annSelectedNode]; ok {
return false, nil
}
className := v1helper.GetPersistentVolumeClaimClass(claim)
......@@ -1477,25 +1475,22 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis
}
var selectedNode *v1.Node = nil
var allowedTopologies []v1.TopologySelectorTerm = nil
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
if nodeName, ok := claim.Annotations[annSelectedNode]; ok {
selectedNode, err = ctrl.NodeLister.Get(nodeName)
if err != nil {
strerr := fmt.Sprintf("Failed to get target node: %v", err)
glog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err)
ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr)
return pluginName, err
}
if nodeName, ok := claim.Annotations[annSelectedNode]; ok {
selectedNode, err = ctrl.NodeLister.Get(nodeName)
if err != nil {
strerr := fmt.Sprintf("Failed to get target node: %v", err)
glog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err)
ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr)
return pluginName, err
}
allowedTopologies = storageClass.AllowedTopologies
}
allowedTopologies := storageClass.AllowedTopologies
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
volume, err = provisioner.Provision(selectedNode, allowedTopologies)
opComplete(&err)
if err != nil {
// Other places of failure have nothing to do with DynamicProvisioningScheduling,
// Other places of failure have nothing to do with VolumeScheduling,
// so just let controller retry in the next sync. We'll only call func
// rescheduleProvisioning here when the underlying provisioning actually failed.
ctrl.rescheduleProvisioning(claim)
......
......@@ -237,12 +237,21 @@ func addVolumeAnnotation(volume *v1.PersistentVolume, annName, annValue string)
return volume
}
func makePVCClass(scName *string) *v1.PersistentVolumeClaim {
return &v1.PersistentVolumeClaim{
func makePVCClass(scName *string, hasSelectNodeAnno bool) *v1.PersistentVolumeClaim {
claim := &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{},
},
Spec: v1.PersistentVolumeClaimSpec{
StorageClassName: scName,
},
}
if hasSelectNodeAnno {
claim.Annotations[annSelectedNode] = "node-name"
}
return claim
}
func makeStorageClass(scName string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass {
......@@ -271,26 +280,30 @@ func TestDelayBinding(t *testing.T) {
shouldFail bool
}{
"nil-class": {
pvc: makePVCClass(nil),
pvc: makePVCClass(nil, false),
shouldDelay: false,
},
"class-not-found": {
pvc: makePVCClass(&classNotHere),
pvc: makePVCClass(&classNotHere, false),
shouldDelay: false,
},
"no-mode-class": {
pvc: makePVCClass(&classNoMode),
pvc: makePVCClass(&classNoMode, false),
shouldDelay: false,
shouldFail: true,
},
"immediate-mode-class": {
pvc: makePVCClass(&classImmediateMode),
pvc: makePVCClass(&classImmediateMode, false),
shouldDelay: false,
},
"wait-mode-class": {
pvc: makePVCClass(&classWaitMode),
pvc: makePVCClass(&classWaitMode, false),
shouldDelay: true,
},
"wait-mode-class-with-selectedNode": {
pvc: makePVCClass(&classWaitMode, true),
shouldDelay: false,
},
}
classes := []*storagev1.StorageClass{
......@@ -314,7 +327,7 @@ func TestDelayBinding(t *testing.T) {
// When volumeScheduling feature gate is disabled, should always be delayed
name := "volumeScheduling-feature-disabled"
shouldDelay, err := ctrl.shouldDelayBinding(makePVCClass(&classWaitMode))
shouldDelay, err := ctrl.shouldDelayBinding(makePVCClass(&classWaitMode, false))
if err != nil {
t.Errorf("Test %q returned error: %v", name, err)
}
......@@ -338,43 +351,4 @@ func TestDelayBinding(t *testing.T) {
t.Errorf("Test %q returned unexpected %v", name, test.shouldDelay)
}
}
// When dynamicProvisioningScheduling feature gate is disabled, should be delayed,
// even if the pvc has selectedNode annotation.
provisionedClaim := makePVCClass(&classWaitMode)
provisionedClaim.Annotations = map[string]string{annSelectedNode: "node-name"}
name = "dynamicProvisioningScheduling-feature-disabled"
shouldDelay, err = ctrl.shouldDelayBinding(provisionedClaim)
if err != nil {
t.Errorf("Test %q returned error: %v", name, err)
}
if !shouldDelay {
t.Errorf("Test %q returned false, expected true", name)
}
// Enable DynamicProvisioningScheduling feature gate
utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true")
defer utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=false")
// When the pvc does not have selectedNode annotation, should be delayed,
// even if dynamicProvisioningScheduling feature gate is enabled.
name = "dynamicProvisioningScheduling-feature-enabled, selectedNode-annotation-not-set"
shouldDelay, err = ctrl.shouldDelayBinding(makePVCClass(&classWaitMode))
if err != nil {
t.Errorf("Test %q returned error: %v", name, err)
}
if !shouldDelay {
t.Errorf("Test %q returned false, expected true", name)
}
// Should not be delayed when dynamicProvisioningScheduling feature gate is enabled,
// and the pvc has selectedNode annotation.
name = "dynamicProvisioningScheduling-feature-enabled, selectedNode-annotation-set"
shouldDelay, err = ctrl.shouldDelayBinding(provisionedClaim)
if err != nil {
t.Errorf("Test %q returned error: %v", name, err)
}
if shouldDelay {
t.Errorf("Test %q returned true, expected false", name)
}
}
......@@ -456,7 +456,7 @@ func TestAssumeUpdatePVCCache(t *testing.T) {
// Assume PVC
newPVC := pvc.DeepCopy()
newPVC.Annotations["volume.alpha.kubernetes.io/selected-node"] = "test-node"
newPVC.Annotations[annSelectedNode] = "test-node"
if err := cache.Assume(newPVC); err != nil {
t.Fatalf("failed to assume PVC: %v", err)
}
......
......@@ -25,12 +25,10 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilfeature "k8s.io/apiserver/pkg/util/feature"
coreinformers "k8s.io/client-go/informers/core/v1"
storageinformers "k8s.io/client-go/informers/storage/v1"
clientset "k8s.io/client-go/kubernetes"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
)
......@@ -169,13 +167,11 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, node *v1.Node) (unboundVolume
return false, false, err
}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
// Try to provision for unbound volumes
if !unboundVolumesSatisfied {
unboundVolumesSatisfied, err = b.checkVolumeProvisions(pod, claimsToProvision, node)
if err != nil {
return false, false, err
}
// Try to provision for unbound volumes
if !unboundVolumesSatisfied {
unboundVolumesSatisfied, err = b.checkVolumeProvisions(pod, claimsToProvision, node)
if err != nil {
return false, false, err
}
}
}
......
......@@ -46,10 +46,10 @@ var (
badPVC = makeBadPVC()
immediateUnboundPVC = makeTestPVC("immediate-unbound-pvc", "1G", pvcUnbound, "", "1", &immediateClass)
immediateBoundPVC = makeTestPVC("immediate-bound-pvc", "1G", pvcBound, "pv-bound-immediate", "1", &immediateClass)
provisionedPVC = makeTestPVC("provisioned-pvc", "1Gi", pvcUnbound, "", "1", &waitClass)
provisionedPVC2 = makeTestPVC("provisioned-pvc2", "1Gi", pvcUnbound, "", "1", &waitClass)
provisionedPVCHigherVersion = makeTestPVC("provisioned-pvc2", "1Gi", pvcUnbound, "", "2", &waitClass)
noProvisionerPVC = makeTestPVC("no-provisioner-pvc", "1Gi", pvcUnbound, "", "1", &provisionNotSupportClass)
provisionedPVC = makeTestPVC("provisioned-pvc", "1Gi", pvcUnbound, "", "1", &waitClassWithProvisioner)
provisionedPVC2 = makeTestPVC("provisioned-pvc2", "1Gi", pvcUnbound, "", "1", &waitClassWithProvisioner)
provisionedPVCHigherVersion = makeTestPVC("provisioned-pvc2", "1Gi", pvcUnbound, "", "2", &waitClassWithProvisioner)
noProvisionerPVC = makeTestPVC("no-provisioner-pvc", "1Gi", pvcUnbound, "", "1", &waitClass)
topoMismatchPVC = makeTestPVC("topo-mismatch-pvc", "1Gi", pvcUnbound, "", "1", &topoMismatchClass)
pvNoNode = makeTestPV("pv-no-node", "", "1G", "1", nil, waitClass)
......@@ -74,7 +74,7 @@ var (
waitClass = "waitClass"
immediateClass = "immediateClass"
provisionNotSupportClass = "provisionNotSupportedClass"
waitClassWithProvisioner = "waitClassWithProvisioner"
topoMismatchClass = "topoMismatchClass"
nodeLabelKey = "nodeKey"
......@@ -110,7 +110,7 @@ func newTestBinder(t *testing.T) *testEnv {
classes := []*storagev1.StorageClass{
{
ObjectMeta: metav1.ObjectMeta{
Name: waitClass,
Name: waitClassWithProvisioner,
},
VolumeBindingMode: &waitMode,
Provisioner: "test-provisioner",
......@@ -133,7 +133,7 @@ func newTestBinder(t *testing.T) *testEnv {
},
{
ObjectMeta: metav1.ObjectMeta{
Name: provisionNotSupportClass,
Name: waitClass,
},
VolumeBindingMode: &waitMode,
Provisioner: "kubernetes.io/no-provisioner",
......@@ -776,9 +776,9 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) {
},
}
// Set VolumeScheduling and DynamicProvisioningScheduling feature gate
utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true,DynamicProvisioningScheduling=true")
defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,DynamicProvisioningScheduling=false")
// Set VolumeScheduling feature gate
utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true")
defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false")
testNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
......
......@@ -312,12 +312,6 @@ const (
// Support Pod Ready++
PodReadinessGates utilfeature.Feature = "PodReadinessGates"
// owner: @lichuqiang
// alpha: v1.11
//
// Extend the default scheduler to be aware of volume topology and handle PV provisioning
DynamicProvisioningScheduling utilfeature.Feature = "DynamicProvisioningScheduling"
// owner: @kevtaylor
// alpha: v1.11
//
......@@ -421,7 +415,6 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
RunAsGroup: {Default: false, PreRelease: utilfeature.Alpha},
VolumeSubpath: {Default: true, PreRelease: utilfeature.GA},
BalanceAttachedNodeVolumes: {Default: false, PreRelease: utilfeature.Alpha},
DynamicProvisioningScheduling: {Default: false, PreRelease: utilfeature.Alpha},
PodReadinessGates: {Default: true, PreRelease: utilfeature.Beta},
VolumeSubpathEnvExpansion: {Default: false, PreRelease: utilfeature.Alpha},
KubeletPluginsWatcher: {Default: false, PreRelease: utilfeature.Alpha},
......
......@@ -3474,6 +3474,9 @@ func describeStorageClass(sc *storage.StorageClass, events *api.EventList) (stri
if sc.VolumeBindingMode != nil {
w.Write(LEVEL_0, "VolumeBindingMode:\t%s\n", *sc.VolumeBindingMode)
}
if sc.AllowedTopologies != nil {
printAllowedTopologies(w, sc.AllowedTopologies)
}
if events != nil {
DescribeEvents(events, w)
}
......@@ -3482,6 +3485,38 @@ func describeStorageClass(sc *storage.StorageClass, events *api.EventList) (stri
})
}
func printAllowedTopologies(w PrefixWriter, topologies []api.TopologySelectorTerm) {
w.Write(LEVEL_0, "AllowedTopologies:\t")
if len(topologies) == 0 {
w.WriteLine("<none>")
return
}
w.WriteLine("")
for i, term := range topologies {
printTopologySelectorTermsMultilineWithIndent(w, LEVEL_1, fmt.Sprintf("Term %d", i), "\t", term.MatchLabelExpressions)
}
}
func printTopologySelectorTermsMultilineWithIndent(w PrefixWriter, indentLevel int, title, innerIndent string, reqs []api.TopologySelectorLabelRequirement) {
w.Write(indentLevel, "%s:%s", title, innerIndent)
if len(reqs) == 0 {
w.WriteLine("<none>")
return
}
for i, req := range reqs {
if i != 0 {
w.Write(indentLevel, "%s", innerIndent)
}
exprStr := fmt.Sprintf("%s %s", req.Key, "in")
if len(req.Values) > 0 {
exprStr = fmt.Sprintf("%s [%s]", exprStr, strings.Join(req.Values, ", "))
}
w.Write(LEVEL_0, "%s\n", exprStr)
}
}
type PodDisruptionBudgetDescriber struct {
clientset.Interface
}
......
......@@ -1433,6 +1433,32 @@ func TestDescribeStorageClass(t *testing.T) {
},
ReclaimPolicy: &reclaimPolicy,
VolumeBindingMode: &bindingMode,
AllowedTopologies: []api.TopologySelectorTerm{
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "failure-domain.beta.kubernetes.io/zone",
Values: []string{"zone1"},
},
{
Key: "kubernetes.io/hostname",
Values: []string{"node1"},
},
},
},
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "failure-domain.beta.kubernetes.io/zone",
Values: []string{"zone2"},
},
{
Key: "kubernetes.io/hostname",
Values: []string{"node2"},
},
},
},
},
})
s := StorageClassDescriber{f}
out, err := s.Describe("", "foo", printers.DescriberSettings{ShowEvents: true})
......@@ -1446,7 +1472,13 @@ func TestDescribeStorageClass(t *testing.T) {
!strings.Contains(out, "value1") ||
!strings.Contains(out, "value2") ||
!strings.Contains(out, "Retain") ||
!strings.Contains(out, "bindingmode") {
!strings.Contains(out, "bindingmode") ||
!strings.Contains(out, "failure-domain.beta.kubernetes.io/zone") ||
!strings.Contains(out, "zone1") ||
!strings.Contains(out, "kubernetes.io/hostname") ||
!strings.Contains(out, "node1") ||
!strings.Contains(out, "zone2") ||
!strings.Contains(out, "node2") {
t.Errorf("unexpected out: %s", out)
}
}
......
......@@ -345,7 +345,7 @@ func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zone
var zoneFromNode string
// pick one zone from node if present
if node != nil {
// DynamicProvisioningScheduling implicit since node is not nil
// VolumeScheduling implicit since node is not nil
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones")
}
......@@ -373,7 +373,7 @@ func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zone
}
if allowedZones.Len() > 0 {
// DynamicProvisioningScheduling implicit since allowedZones present
// VolumeScheduling implicit since allowedZones present
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified")
}
......
......@@ -487,16 +487,13 @@ func ClusterRoles() []rbacv1.ClusterRole {
}
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
rules := []rbacv1.PolicyRule{
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
rules = append(rules, rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie())
}
roles = append(roles, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "system:volume-scheduler"},
Rules: rules,
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(),
},
})
}
......
......@@ -1116,6 +1116,16 @@ items:
- get
- list
- watch
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- get
- list
- patch
- update
- watch
- aggregationRule:
clusterRoleSelectors:
- matchLabels:
......
......@@ -65,16 +65,14 @@ message StorageClass {
// VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable
// the VolumeScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
optional string volumeBindingMode = 7;
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
repeated k8s.io.api.core.v1.TopologySelectorTerm allowedTopologies = 8;
}
......
......@@ -62,16 +62,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable
// the VolumeScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
}
......
......@@ -35,8 +35,8 @@ var map_StorageClass = map[string]string{
"reclaimPolicy": "Dynamically provisioned PersistentVolumes of this storage class are created with this reclaimPolicy. Defaults to Delete.",
"mountOptions": "Dynamically provisioned PersistentVolumes of this storage class are created with these mountOptions, e.g. [\"ro\", \"soft\"]. Not validated - mount of the PVs will simply fail if one is invalid.",
"allowVolumeExpansion": "AllowVolumeExpansion shows whether the storage class allow volume expand",
"volumeBindingMode": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.",
"allowedTopologies": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.",
"volumeBindingMode": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.",
"allowedTopologies": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.",
}
func (StorageClass) SwaggerDoc() map[string]string {
......
......@@ -65,16 +65,14 @@ message StorageClass {
// VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable
// the VolumeScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
optional string volumeBindingMode = 7;
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
repeated k8s.io.api.core.v1.TopologySelectorTerm allowedTopologies = 8;
}
......
......@@ -62,16 +62,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable
// the VolumeScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// This field is only honored by servers that enable the VolumeScheduling feature.
// +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
}
......
......@@ -35,8 +35,8 @@ var map_StorageClass = map[string]string{
"reclaimPolicy": "Dynamically provisioned PersistentVolumes of this storage class are created with this reclaimPolicy. Defaults to Delete.",
"mountOptions": "Dynamically provisioned PersistentVolumes of this storage class are created with these mountOptions, e.g. [\"ro\", \"soft\"]. Not validated - mount of the PVs will simply fail if one is invalid.",
"allowVolumeExpansion": "AllowVolumeExpansion shows whether the storage class allow volume expand",
"volumeBindingMode": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is alpha-level and is only honored by servers that enable the VolumeScheduling feature.",
"allowedTopologies": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is alpha-level and is only honored by servers that enable the DynamicProvisioningScheduling feature.",
"volumeBindingMode": "VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.",
"allowedTopologies": "Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.",
}
func (StorageClass) SwaggerDoc() map[string]string {
......
......@@ -264,9 +264,8 @@ func TestVolumeBinding(t *testing.T) {
// TestVolumeBindingRescheduling tests scheduler will retry scheduling when needed.
func TestVolumeBindingRescheduling(t *testing.T) {
features := map[string]bool{
"VolumeScheduling": true,
"PersistentLocalVolumes": true,
"DynamicProvisioningScheduling": true,
"VolumeScheduling": true,
"PersistentLocalVolumes": true,
}
config := setupCluster(t, "volume-scheduling", 2, features, 0)
defer config.teardown()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment