Merge pull request #63232 from lichuqiang/provision_plumbing

pkg/controller/volume/persistentvolume/pv_controller.go

@@ -135,6 +135,14 @@ const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
 // a volume for this PVC.
 const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
 
+// This annotation is added to a PVC that has been triggered by scheduler to
+// be dynamically provisioned. Its value is the name of the selected node.
+const annSelectedNode = "volume.alpha.kubernetes.io/selected-node"
+
+// If the provisioner name in a storage class is set to "kubernetes.io/no-provisioner",
+// then dynamic provisioning is not supported by the storage.
+const notSupportedProvisioner = "kubernetes.io/no-provisioner"
+
 // CloudVolumeCreatedForClaimNamespaceTag is a name of a tag attached to a real volume in cloud (e.g. AWS EBS or GCE PD)
 // with namespace of a persistent volume claim used to create this volume.
 const CloudVolumeCreatedForClaimNamespaceTag = "kubernetes.io/created-for/pvc/namespace"
@@ -277,6 +285,16 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV
 		return false, nil
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
+		// When feature DynamicProvisioningScheduling enabled,
+		// Scheduler signal to the PV controller to start dynamic
+		// provisioning by setting the "annSelectedNode" annotation
+		// in the PVC
+		if _, ok := claim.Annotations[annSelectedNode]; ok {
+			return false, nil
+		}
+	}
+
 	className := v1helper.GetPersistentVolumeClaimClass(claim)
 	if className == "" {
 		return false, nil
@@ -291,8 +309,6 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV
 		return false, fmt.Errorf("VolumeBindingMode not set for StorageClass %q", className)
 	}
 
-	// TODO: add check to handle dynamic provisioning later
-
 	return *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer, nil
 }
 
@@ -320,7 +336,6 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol
 			// OBSERVATION: pvc is "Pending", will retry
 			switch {
 			case delayBinding:
-				// TODO: Skip dynamic provisioning for now
 				ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, events.WaitForFirstConsumer, "waiting for first consumer to be created before binding")
 			case v1helper.GetPersistentVolumeClaimClass(claim) != "":
 				if err = ctrl.provisionClaim(claim); err != nil {
@@ -1420,9 +1435,16 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis
 	}
 
 	opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
+	// TODO: modify the Provision() interface to pass in the allowed topology information
+	// of the provisioned volume.
 	volume, err = provisioner.Provision()
 	opComplete(&err)
 	if err != nil {
+		// Other places of failure has nothing to do with DynamicProvisioningScheduling,
+		// so just let controller retry in the next sync. We'll only call func
+		// rescheduleProvisioning here when the underlying provisioning actually failed.
+		ctrl.rescheduleProvisioning(claim)
+
 		strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err)
 		glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)
 		ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr)
@@ -1513,6 +1535,29 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis
 	}
 }
 
+// rescheduleProvisioning signal back to the scheduler to retry dynamic provisioning
+// by removing the annSelectedNode annotation
+func (ctrl *PersistentVolumeController) rescheduleProvisioning(claim *v1.PersistentVolumeClaim) {
+	if _, ok := claim.Annotations[annSelectedNode]; !ok {
+		// Provisioning not triggered by the scheduler, skip
+		return
+	}
+
+	// The claim from method args can be pointing to watcher cache. We must not
+	// modify these, therefore create a copy.
+	newClaim := claim.DeepCopy()
+	delete(newClaim.Annotations, annSelectedNode)
+	// Try to update the PVC object
+	if _, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(newClaim.Namespace).Update(newClaim); err != nil {
+		glog.V(4).Infof("Failed to delete annotation 'annSelectedNode' for PersistentVolumeClaim %q: %v", claimToClaimKey(newClaim), err)
+		return
+	}
+	if _, err := ctrl.storeClaimUpdate(newClaim); err != nil {
+		// We will get an "claim updated" event soon, this is not a big error
+		glog.V(4).Infof("Updating PersistentVolumeClaim %q: cannot update internal cache: %v", claimToClaimKey(newClaim), err)
+	}
+}
+
 // getProvisionedVolumeNameForClaim returns PV.Name for the provisioned volume.
 // The name must be unique.
 func (ctrl *PersistentVolumeController) getProvisionedVolumeNameForClaim(claim *v1.PersistentVolumeClaim) string {

pkg/controller/volume/persistentvolume/pv_controller_test.go

@@ -312,8 +312,8 @@ func TestDelayBinding(t *testing.T) {
 		}
 	}
 
-	// When feature gate is disabled, should always be delayed
-	name := "feature-disabled"
+	// When volumeScheduling feature gate is disabled, should always be delayed
+	name := "volumeScheduling-feature-disabled"
 	shouldDelay, err := ctrl.shouldDelayBinding(makePVCClass(&classWaitMode))
 	if err != nil {
 		t.Errorf("Test %q returned error: %v", name, err)
@@ -322,7 +322,7 @@ func TestDelayBinding(t *testing.T) {
 		t.Errorf("Test %q returned true, expected false", name)
 	}
 
-	// Enable feature gate
+	// Enable volumeScheduling feature gate
 	utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true")
 	defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false")
 
@@ -338,4 +338,43 @@ func TestDelayBinding(t *testing.T) {
 			t.Errorf("Test %q returned unexpected %v", name, test.shouldDelay)
 		}
 	}
+
+	// When dynamicProvisioningScheduling feature gate is disabled, should be delayed,
+	// even if the pvc has selectedNode annotation.
+	provisionedClaim := makePVCClass(&classWaitMode)
+	provisionedClaim.Annotations = map[string]string{annSelectedNode: "node-name"}
+	name = "dynamicProvisioningScheduling-feature-disabled"
+	shouldDelay, err = ctrl.shouldDelayBinding(provisionedClaim)
+	if err != nil {
+		t.Errorf("Test %q returned error: %v", name, err)
+	}
+	if !shouldDelay {
+		t.Errorf("Test %q returned false, expected true", name)
+	}
+
+	// Enable DynamicProvisioningScheduling feature gate
+	utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true")
+	defer utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=false")
+
+	// When the pvc does not have selectedNode annotation, should be delayed,
+	// even if dynamicProvisioningScheduling feature gate is enabled.
+	name = "dynamicProvisioningScheduling-feature-enabled, selectedNode-annotation-not-set"
+	shouldDelay, err = ctrl.shouldDelayBinding(makePVCClass(&classWaitMode))
+	if err != nil {
+		t.Errorf("Test %q returned error: %v", name, err)
+	}
+	if !shouldDelay {
+		t.Errorf("Test %q returned false, expected true", name)
+	}
+
+	// Should not be delayed when dynamicProvisioningScheduling feature gate is enabled,
+	// and the pvc has selectedNode annotation.
+	name = "dynamicProvisioningScheduling-feature-enabled, selectedNode-annotation-set"
+	shouldDelay, err = ctrl.shouldDelayBinding(provisionedClaim)
+	if err != nil {
+		t.Errorf("Test %q returned error: %v", name, err)
+	}
+	if shouldDelay {
+		t.Errorf("Test %q returned true, expected false", name)
+	}
 }

pkg/controller/volume/persistentvolume/scheduler_assume_cache.go

@@ -371,3 +371,34 @@ func (c *pvAssumeCache) ListPVs(storageClassName string) []*v1.PersistentVolume
 	}
 	return pvs
 }
+
+// PVCAssumeCache is a AssumeCache for PersistentVolumeClaim objects
+type PVCAssumeCache interface {
+	AssumeCache
+
+	// GetPVC returns the PVC from the cache with the same
+	// namespace and the same name of the specified pod.
+	// pvcKey is the result of MetaNamespaceKeyFunc on PVC obj
+	GetPVC(pvcKey string) (*v1.PersistentVolumeClaim, error)
+}
+
+type pvcAssumeCache struct {
+	*assumeCache
+}
+
+func NewPVCAssumeCache(informer cache.SharedIndexInformer) PVCAssumeCache {
+	return &pvcAssumeCache{assumeCache: NewAssumeCache(informer, "v1.PersistentVolumeClaim", "namespace", cache.MetaNamespaceIndexFunc)}
+}
+
+func (c *pvcAssumeCache) GetPVC(pvcKey string) (*v1.PersistentVolumeClaim, error) {
+	obj, err := c.Get(pvcKey)
+	if err != nil {
+		return nil, err
+	}
+
+	pvc, ok := obj.(*v1.PersistentVolumeClaim)
+	if !ok {
+		return nil, &errWrongType{"v1.PersistentVolumeClaim", obj}
+	}
+	return pvc, nil
+}

pkg/controller/volume/persistentvolume/scheduler_assume_cache_test.go

@@ -36,6 +36,33 @@ func makePV(name, version, storageClass string) *v1.PersistentVolume {
 	}
 }
 
+func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) {
+	pvList := cache.ListPVs(storageClassName)
+	if len(pvList) != len(expectedPVs) {
+		t.Errorf("ListPVs() returned %v PVs, expected %v", len(pvList), len(expectedPVs))
+	}
+	for _, pv := range pvList {
+		expectedPV, ok := expectedPVs[pv.Name]
+		if !ok {
+			t.Errorf("ListPVs() returned unexpected PV %q", pv.Name)
+		}
+		if expectedPV != pv {
+			t.Errorf("ListPVs() returned PV %p, expected %p", pv, expectedPV)
+		}
+	}
+}
+
+func verifyPV(cache PVAssumeCache, name string, expectedPV *v1.PersistentVolume) error {
+	pv, err := cache.GetPV(name)
+	if err != nil {
+		return err
+	}
+	if pv != expectedPV {
+		return fmt.Errorf("GetPV() returned %p, expected %p", pv, expectedPV)
+	}
+	return nil
+}
+
 func TestAssumePV(t *testing.T) {
 	scenarios := map[string]struct {
 		oldPV         *v1.PersistentVolume
@@ -276,29 +303,170 @@ func TestAssumeUpdatePVCache(t *testing.T) {
 	}
 }
 
-func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) {
-	pvList := cache.ListPVs(storageClassName)
-	if len(pvList) != len(expectedPVs) {
-		t.Errorf("ListPVs() returned %v PVs, expected %v", len(pvList), len(expectedPVs))
+func makeClaim(name, version, namespace string) *v1.PersistentVolumeClaim {
+	return &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			Namespace:       namespace,
+			ResourceVersion: version,
+			Annotations:     map[string]string{},
+		},
 	}
-	for _, pv := range pvList {
-		expectedPV, ok := expectedPVs[pv.Name]
+}
+
+func verifyPVC(cache PVCAssumeCache, pvcKey string, expectedPVC *v1.PersistentVolumeClaim) error {
+	pvc, err := cache.GetPVC(pvcKey)
+	if err != nil {
+		return err
+	}
+	if pvc != expectedPVC {
+		return fmt.Errorf("GetPVC() returned %p, expected %p", pvc, expectedPVC)
+	}
+	return nil
+}
+
+func TestAssumePVC(t *testing.T) {
+	scenarios := map[string]struct {
+		oldPVC        *v1.PersistentVolumeClaim
+		newPVC        *v1.PersistentVolumeClaim
+		shouldSucceed bool
+	}{
+		"success-same-version": {
+			oldPVC:        makeClaim("pvc1", "5", "ns1"),
+			newPVC:        makeClaim("pvc1", "5", "ns1"),
+			shouldSucceed: true,
+		},
+		"success-new-higher-version": {
+			oldPVC:        makeClaim("pvc1", "5", "ns1"),
+			newPVC:        makeClaim("pvc1", "6", "ns1"),
+			shouldSucceed: true,
+		},
+		"fail-old-not-found": {
+			oldPVC:        makeClaim("pvc2", "5", "ns1"),
+			newPVC:        makeClaim("pvc1", "5", "ns1"),
+			shouldSucceed: false,
+		},
+		"fail-new-lower-version": {
+			oldPVC:        makeClaim("pvc1", "5", "ns1"),
+			newPVC:        makeClaim("pvc1", "4", "ns1"),
+			shouldSucceed: false,
+		},
+		"fail-new-bad-version": {
+			oldPVC:        makeClaim("pvc1", "5", "ns1"),
+			newPVC:        makeClaim("pvc1", "a", "ns1"),
+			shouldSucceed: false,
+		},
+		"fail-old-bad-version": {
+			oldPVC:        makeClaim("pvc1", "a", "ns1"),
+			newPVC:        makeClaim("pvc1", "5", "ns1"),
+			shouldSucceed: false,
+		},
+	}
+
+	for name, scenario := range scenarios {
+		cache := NewPVCAssumeCache(nil)
+		internal_cache, ok := cache.(*pvcAssumeCache)
 		if !ok {
-			t.Errorf("ListPVs() returned unexpected PV %q", pv.Name)
+			t.Fatalf("Failed to get internal cache")
 		}
-		if expectedPV != pv {
-			t.Errorf("ListPVs() returned PV %p, expected %p", pv, expectedPV)
+
+		// Add oldPVC to cache
+		internal_cache.add(scenario.oldPVC)
+		if err := verifyPVC(cache, getPVCName(scenario.oldPVC), scenario.oldPVC); err != nil {
+			t.Errorf("Failed to GetPVC() after initial update: %v", err)
+			continue
+		}
+
+		// Assume newPVC
+		err := cache.Assume(scenario.newPVC)
+		if scenario.shouldSucceed && err != nil {
+			t.Errorf("Test %q failed: Assume() returned error %v", name, err)
+		}
+		if !scenario.shouldSucceed && err == nil {
+			t.Errorf("Test %q failed: Assume() returned success but expected error", name)
+		}
+
+		// Check that GetPVC returns correct PVC
+		expectedPV := scenario.newPVC
+		if !scenario.shouldSucceed {
+			expectedPV = scenario.oldPVC
+		}
+		if err := verifyPVC(cache, getPVCName(scenario.oldPVC), expectedPV); err != nil {
+			t.Errorf("Failed to GetPVC() after initial update: %v", err)
 		}
 	}
 }
 
-func verifyPV(cache PVAssumeCache, name string, expectedPV *v1.PersistentVolume) error {
-	pv, err := cache.GetPV(name)
-	if err != nil {
-		return err
+func TestRestorePVC(t *testing.T) {
+	cache := NewPVCAssumeCache(nil)
+	internal_cache, ok := cache.(*pvcAssumeCache)
+	if !ok {
+		t.Fatalf("Failed to get internal cache")
 	}
-	if pv != expectedPV {
-		return fmt.Errorf("GetPV() returned %p, expected %p", pv, expectedPV)
+
+	oldPVC := makeClaim("pvc1", "5", "ns1")
+	newPVC := makeClaim("pvc1", "5", "ns1")
+
+	// Restore PVC that doesn't exist
+	cache.Restore("nothing")
+
+	// Add oldPVC to cache
+	internal_cache.add(oldPVC)
+	if err := verifyPVC(cache, getPVCName(oldPVC), oldPVC); err != nil {
+		t.Fatalf("Failed to GetPVC() after initial update: %v", err)
+	}
+
+	// Restore PVC
+	cache.Restore(getPVCName(oldPVC))
+	if err := verifyPVC(cache, getPVCName(oldPVC), oldPVC); err != nil {
+		t.Fatalf("Failed to GetPVC() after iniital restore: %v", err)
+	}
+
+	// Assume newPVC
+	if err := cache.Assume(newPVC); err != nil {
+		t.Fatalf("Assume() returned error %v", err)
+	}
+	if err := verifyPVC(cache, getPVCName(oldPVC), newPVC); err != nil {
+		t.Fatalf("Failed to GetPVC() after Assume: %v", err)
+	}
+
+	// Restore PVC
+	cache.Restore(getPVCName(oldPVC))
+	if err := verifyPVC(cache, getPVCName(oldPVC), oldPVC); err != nil {
+		t.Fatalf("Failed to GetPVC() after restore: %v", err)
+	}
+}
+
+func TestAssumeUpdatePVCCache(t *testing.T) {
+	cache := NewPVCAssumeCache(nil)
+	internal_cache, ok := cache.(*pvcAssumeCache)
+	if !ok {
+		t.Fatalf("Failed to get internal cache")
+	}
+
+	pvcName := "test-pvc0"
+	pvcNamespace := "test-ns"
+
+	// Add a PVC
+	pvc := makeClaim(pvcName, "1", pvcNamespace)
+	internal_cache.add(pvc)
+	if err := verifyPVC(cache, getPVCName(pvc), pvc); err != nil {
+		t.Fatalf("failed to get PVC: %v", err)
+	}
+
+	// Assume PVC
+	newPVC := pvc.DeepCopy()
+	newPVC.Annotations["volume.alpha.kubernetes.io/selected-node"] = "test-node"
+	if err := cache.Assume(newPVC); err != nil {
+		t.Fatalf("failed to assume PVC: %v", err)
+	}
+	if err := verifyPVC(cache, getPVCName(pvc), newPVC); err != nil {
+		t.Fatalf("failed to get PVC after assume: %v", err)
+	}
+
+	// Add old PVC
+	internal_cache.add(pvc)
+	if err := verifyPVC(cache, getPVCName(pvc), newPVC); err != nil {
+		t.Fatalf("failed to get PVC after old PVC added: %v", err)
 	}
-	return nil
 }

pkg/controller/volume/persistentvolume/scheduler_binder_cache.go

@@ -30,27 +30,41 @@ type PodBindingCache interface {
 	// pod and node.
 	UpdateBindings(pod *v1.Pod, node string, bindings []*bindingInfo)
 
-	// DeleteBindings will remove all cached bindings for the given pod.
-	DeleteBindings(pod *v1.Pod)
-
 	// GetBindings will return the cached bindings for the given pod and node.
 	GetBindings(pod *v1.Pod, node string) []*bindingInfo
+
+	// UpdateProvisionedPVCs will update the cache with the given provisioning decisions
+	// for the pod and node.
+	UpdateProvisionedPVCs(pod *v1.Pod, node string, provisionings []*v1.PersistentVolumeClaim)
+
+	// GetProvisionedPVCs will return the cached provisioning decisions for the given pod and node.
+	GetProvisionedPVCs(pod *v1.Pod, node string) []*v1.PersistentVolumeClaim
+
+	// DeleteBindings will remove all cached bindings and provisionings for the given pod.
+	// TODO: separate the func if it is needed to delete bindings/provisionings individually
+	DeleteBindings(pod *v1.Pod)
 }
 
 type podBindingCache struct {
 	mutex sync.Mutex
 
 	// Key = pod name
-	// Value = nodeBindings
-	bindings map[string]nodeBindings
+	// Value = nodeDecisions
+	bindingDecisions map[string]nodeDecisions
 }
 
 // Key = nodeName
-// Value = array of bindingInfo
-type nodeBindings map[string][]*bindingInfo
+// Value = bindings & provisioned PVCs of the node
+type nodeDecisions map[string]nodeDecision
+
+// A decision includes bindingInfo and provisioned PVCs of the node
+type nodeDecision struct {
+	bindings      []*bindingInfo
+	provisionings []*v1.PersistentVolumeClaim
+}
 
 func NewPodBindingCache() PodBindingCache {
-	return &podBindingCache{bindings: map[string]nodeBindings{}}
+	return &podBindingCache{bindingDecisions: map[string]nodeDecisions{}}
 }
 
 func (c *podBindingCache) DeleteBindings(pod *v1.Pod) {
@@ -58,7 +72,7 @@ func (c *podBindingCache) DeleteBindings(pod *v1.Pod) {
 	defer c.mutex.Unlock()
 
 	podName := getPodName(pod)
-	delete(c.bindings, podName)
+	delete(c.bindingDecisions, podName)
 }
 
 func (c *podBindingCache) UpdateBindings(pod *v1.Pod, node string, bindings []*bindingInfo) {
@@ -66,12 +80,20 @@ func (c *podBindingCache) UpdateBindings(pod *v1.Pod, node string, bindings []*b
 	defer c.mutex.Unlock()
 
 	podName := getPodName(pod)
-	nodeBinding, ok := c.bindings[podName]
+	decisions, ok := c.bindingDecisions[podName]
+	if !ok {
+		decisions = nodeDecisions{}
+		c.bindingDecisions[podName] = decisions
+	}
+	decision, ok := decisions[node]
 	if !ok {
-		nodeBinding = nodeBindings{}
-		c.bindings[podName] = nodeBinding
+		decision = nodeDecision{
+			bindings: bindings,
+		}
+	} else {
+		decision.bindings = bindings
 	}
-	nodeBinding[node] = bindings
+	decisions[node] = decision
 }
 
 func (c *podBindingCache) GetBindings(pod *v1.Pod, node string) []*bindingInfo {
@@ -79,9 +101,50 @@ func (c *podBindingCache) GetBindings(pod *v1.Pod, node string) []*bindingInfo {
 	defer c.mutex.Unlock()
 
 	podName := getPodName(pod)
-	nodeBindings, ok := c.bindings[podName]
+	decisions, ok := c.bindingDecisions[podName]
+	if !ok {
+		return nil
+	}
+	decision, ok := decisions[node]
+	if !ok {
+		return nil
+	}
+	return decision.bindings
+}
+
+func (c *podBindingCache) UpdateProvisionedPVCs(pod *v1.Pod, node string, pvcs []*v1.PersistentVolumeClaim) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	podName := getPodName(pod)
+	decisions, ok := c.bindingDecisions[podName]
+	if !ok {
+		decisions = nodeDecisions{}
+		c.bindingDecisions[podName] = decisions
+	}
+	decision, ok := decisions[node]
+	if !ok {
+		decision = nodeDecision{
+			provisionings: pvcs,
+		}
+	} else {
+		decision.provisionings = pvcs
+	}
+	decisions[node] = decision
+}
+
+func (c *podBindingCache) GetProvisionedPVCs(pod *v1.Pod, node string) []*v1.PersistentVolumeClaim {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	podName := getPodName(pod)
+	decisions, ok := c.bindingDecisions[podName]
+	if !ok {
+		return nil
+	}
+	decision, ok := decisions[node]
 	if !ok {
 		return nil
 	}
-	return nodeBindings[node]
+	return decision.provisionings
 }

pkg/controller/volume/persistentvolume/scheduler_binder_cache_test.go

@@ -26,32 +26,37 @@ import (
 
 func TestUpdateGetBindings(t *testing.T) {
 	scenarios := map[string]struct {
-		updateBindings []*bindingInfo
-		updatePod      string
-		updateNode     string
-
-		getBindings []*bindingInfo
-		getPod      string
-		getNode     string
+		updateBindings      []*bindingInfo
+		updateProvisionings []*v1.PersistentVolumeClaim
+		updatePod           string
+		updateNode          string
+
+		getBindings      []*bindingInfo
+		getProvisionings []*v1.PersistentVolumeClaim
+		getPod           string
+		getNode          string
 	}{
 		"no-pod": {
 			getPod:  "pod1",
 			getNode: "node1",
 		},
 		"no-node": {
-			updatePod:      "pod1",
-			updateNode:     "node1",
-			updateBindings: []*bindingInfo{},
-			getPod:         "pod1",
-			getNode:        "node2",
+			updatePod:           "pod1",
+			updateNode:          "node1",
+			updateBindings:      []*bindingInfo{},
+			updateProvisionings: []*v1.PersistentVolumeClaim{},
+			getPod:              "pod1",
+			getNode:             "node2",
 		},
 		"binding-exists": {
-			updatePod:      "pod1",
-			updateNode:     "node1",
-			updateBindings: []*bindingInfo{{pvc: &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pvc1"}}}},
-			getPod:         "pod1",
-			getNode:        "node1",
-			getBindings:    []*bindingInfo{{pvc: &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pvc1"}}}},
+			updatePod:           "pod1",
+			updateNode:          "node1",
+			updateBindings:      []*bindingInfo{{pvc: &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pvc1"}}}},
+			updateProvisionings: []*v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "pvc2"}}},
+			getPod:              "pod1",
+			getNode:             "node1",
+			getBindings:         []*bindingInfo{{pvc: &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pvc1"}}}},
+			getProvisionings:    []*v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "pvc2"}}},
 		},
 	}
 
@@ -61,6 +66,7 @@ func TestUpdateGetBindings(t *testing.T) {
 		// Perform updates
 		updatePod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: scenario.updatePod, Namespace: "ns"}}
 		cache.UpdateBindings(updatePod, scenario.updateNode, scenario.updateBindings)
+		cache.UpdateProvisionedPVCs(updatePod, scenario.updateNode, scenario.updateProvisionings)
 
 		// Verify updated bindings
 		bindings := cache.GetBindings(updatePod, scenario.updateNode)
@@ -68,45 +74,71 @@ func TestUpdateGetBindings(t *testing.T) {
 			t.Errorf("Test %v failed: returned bindings after update different. Got %+v, expected %+v", name, bindings, scenario.updateBindings)
 		}
 
+		// Verify updated provisionings
+		provisionings := cache.GetProvisionedPVCs(updatePod, scenario.updateNode)
+		if !reflect.DeepEqual(provisionings, scenario.updateProvisionings) {
+			t.Errorf("Test %v failed: returned provisionings after update different. Got %+v, expected %+v", name, provisionings, scenario.updateProvisionings)
+		}
+
 		// Get bindings
 		getPod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: scenario.getPod, Namespace: "ns"}}
 		bindings = cache.GetBindings(getPod, scenario.getNode)
 		if !reflect.DeepEqual(bindings, scenario.getBindings) {
 			t.Errorf("Test %v failed: unexpected bindings returned. Got %+v, expected %+v", name, bindings, scenario.updateBindings)
 		}
+
+		// Get provisionings
+		provisionings = cache.GetProvisionedPVCs(getPod, scenario.getNode)
+		if !reflect.DeepEqual(provisionings, scenario.getProvisionings) {
+			t.Errorf("Test %v failed: unexpected bindings returned. Got %+v, expected %+v", name, provisionings, scenario.getProvisionings)
+		}
 	}
 }
 
 func TestDeleteBindings(t *testing.T) {
 	initialBindings := []*bindingInfo{{pvc: &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pvc1"}}}}
+	initialProvisionings := []*v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "pvc2"}}}
 	cache := NewPodBindingCache()
 
 	pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod1", Namespace: "ns"}}
 
-	// Get nil bindings
+	// Get nil bindings and provisionings
 	bindings := cache.GetBindings(pod, "node1")
 	if bindings != nil {
 		t.Errorf("Test failed: expected initial nil bindings, got %+v", bindings)
 	}
+	provisionings := cache.GetProvisionedPVCs(pod, "node1")
+	if provisionings != nil {
+		t.Errorf("Test failed: expected initial nil provisionings, got %+v", provisionings)
+	}
 
 	// Delete nothing
 	cache.DeleteBindings(pod)
 
 	// Perform updates
 	cache.UpdateBindings(pod, "node1", initialBindings)
+	cache.UpdateProvisionedPVCs(pod, "node1", initialProvisionings)
 
-	// Get bindings
+	// Get bindings and provisionings
 	bindings = cache.GetBindings(pod, "node1")
 	if !reflect.DeepEqual(bindings, initialBindings) {
 		t.Errorf("Test failed: expected bindings %+v, got %+v", initialBindings, bindings)
 	}
+	provisionings = cache.GetProvisionedPVCs(pod, "node1")
+	if !reflect.DeepEqual(provisionings, initialProvisionings) {
+		t.Errorf("Test failed: expected provisionings %+v, got %+v", initialProvisionings, provisionings)
+	}
 
 	// Delete
 	cache.DeleteBindings(pod)
 
-	// Get bindings
+	// Get bindings and provisionings
 	bindings = cache.GetBindings(pod, "node1")
 	if bindings != nil {
 		t.Errorf("Test failed: expected nil bindings, got %+v", bindings)
 	}
+	provisionings = cache.GetProvisionedPVCs(pod, "node1")
+	if provisionings != nil {
+		t.Errorf("Test failed: expected nil provisionings, got %+v", provisionings)
+	}
 }

pkg/features/kube_features.go

@@ -280,6 +280,12 @@ const (
 	// A node which has closer cpu,memory utilization and volume count is favoured by scheduler
 	// while making decisions.
 	BalanceAttachedNodeVolumes utilfeature.Feature = "BalanceAttachedNodeVolumes"
+
+	// owner: @lichuqiang
+	// alpha: v1.11
+	//
+	// Extend the default scheduler to be aware of volume topology and handle PV provisioning
+	DynamicProvisioningScheduling utilfeature.Feature = "DynamicProvisioningScheduling"
 )
 
 func init() {
@@ -328,6 +334,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	RunAsGroup:                                  {Default: false, PreRelease: utilfeature.Alpha},
 	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
 	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha},
+	DynamicProvisioningScheduling:               {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -506,12 +506,16 @@ func ClusterRoles() []rbacv1.ClusterRole {
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+		rules := []rbacv1.PolicyRule{
+			rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
+			rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
+			rules = append(rules, rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie())
+		}
 		roles = append(roles, rbacv1.ClusterRole{
 			ObjectMeta: metav1.ObjectMeta{Name: "system:volume-scheduler"},
-			Rules: []rbacv1.PolicyRule{
-				rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
-				rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
-			},
+			Rules:      rules,
 		})
 	}