Skip to content

K
k3s
Commit 76dfee04 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #38615 from jsafrane/worker-thread

Automatic merge from submit-queue (batch tested with PRs 39150, 38615) Add work queues to PV controller PV controller should not use Controller.Requeue, as as it is not available in shared informers. We need to implement our own work queues instead, where we can enqueue volumes/claims as we want.
parents cfc3c4b9 0fd5f202
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 44 additions and 22 deletions
cmd/kube-controller-manager/app/controllermanager.go
View file @ 76dfee04
......@@ -405,7 +405,7 @@ func StartControllers(controllers map[string]InitFunc, s *options.CMServer, root
EnableDynamicProvisioning: s.VolumeConfiguration.EnableDynamicProvisioning,
}
volumeController := persistentvolumecontroller.NewController(params)
volumeController.Run(stop)
go volumeController.Run(stop)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
attachDetachController, attachDetachControllerErr :=
......
pkg/controller/volume/persistentvolume/BUILD
View file @ 76dfee04
......@@ -30,12 +30,15 @@ go_library(
"//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library",
"//pkg/client/record:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/labels:go_default_library",
"//pkg/runtime:go_default_library",
"//pkg/types:go_default_library",
"//pkg/util/goroutinemap:go_default_library",
"//pkg/util/io:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/wait:go_default_library",
"//pkg/util/workqueue:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/watch:go_default_library",
"//vendor:github.com/golang/glog",
......
pkg/controller/volume/persistentvolume/pv_controller.go
View file @ 76dfee04
......@@ -32,6 +32,7 @@ import (
"k8s.io/kubernetes/pkg/client/record"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/util/goroutinemap"
"k8s.io/kubernetes/pkg/util/workqueue"
vol "k8s.io/kubernetes/pkg/volume"
"github.com/golang/glog"
......@@ -146,8 +147,10 @@ const createProvisionedPVInterval = 10 * time.Second
// changes.
type PersistentVolumeController struct {
volumeController *cache.Controller
volumeInformer cache.Indexer
volumeSource cache.ListerWatcher
claimController *cache.Controller
claimInformer cache.Store
claimSource cache.ListerWatcher
classReflector *cache.Reflector
classSource cache.ListerWatcher
......@@ -163,10 +166,34 @@ type PersistentVolumeController struct {
// must be cloned before any modification. These caches get updated both by
// "xxx added/updated/deleted" events from etcd and by the controller when
// it saves newer version to etcd.
// Why local cache: binding a volume to a claim generates 4 events, roughly
// in this order (depends on goroutine ordering):
// - volume.Spec update
// - volume.Status update
// - claim.Spec update
// - claim.Status update
// With these caches, the controller can check that it has already saved
// volume.Status and claim.Spec+Status and does not need to do anything
// when e.g. volume.Spec update event arrives before all the other events.
// Without this cache, it would see the old version of volume.Status and
// claim in the informers (it has not been updated from API server events
// yet) and it would try to fix these objects to be bound together.
// Any write to API server would fail with version conflict - these objects
// have been already written.
volumes persistentVolumeOrderedIndex
claims cache.Store
classes cache.Store
// Work queues of claims and volumes to process. Every queue should have
// exactly one worker thread, especially syncClaim() is not reentrant.
// Two syncClaims could bind two different claims to the same volume or one
// claim to two volumes. The controller would recover from this (due to
// version errors in API server and other checks in this controller),
// however overall speed of multi-worker controller would be lower than if
// it runs single thread only.
claimQueue *workqueue.Type
volumeQueue *workqueue.Type
// Map of scheduled/running operations.
runningOperations goroutinemap.GoRoutineMap
......@@ -463,19 +490,11 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume)
// In both cases, the volume is Bound and the claim is Pending.
// Next syncClaim will fix it. To speed it up, we enqueue the claim
// into the controller, which results in syncClaim to be called
// shortly (and in the right goroutine).
// shortly (and in the right worker goroutine).
// This speeds up binding of provisioned volumes - provisioner saves
// only the new PV and it expects that next syncClaim will bind the
// claim to it.
clone, err := api.Scheme.DeepCopy(claim)
if err != nil {
return fmt.Errorf("error cloning claim %q: %v", claimToClaimKey(claim), err)
}
glog.V(5).Infof("requeueing claim %q for faster syncClaim", claimToClaimKey(claim))
err = ctrl.claimController.Requeue(clone)
if err != nil {
return fmt.Errorf("error enqueing claim %q for faster sync: %v", claimToClaimKey(claim), err)
}
ctrl.claimQueue.Add(claimToClaimKey(claim))
return nil
} else if claim.Spec.VolumeName == volume.Name {
// Volume is bound to a claim properly, update status if necessary
......
pkg/controller/volume/persistentvolume/pv_controller_test.go
View file @ 76dfee04
......@@ -176,7 +176,7 @@ func TestControllerSync(t *testing.T) {
// Start the controller
stopCh := make(chan struct{})
ctrl.Run(stopCh)
go ctrl.Run(stopCh)
// Wait for the controller to pass initial sync and fill its caches.
for !ctrl.volumeController.HasSynced() ||
......
test/integration/volume/persistent_volumes_test.go
View file @ 76dfee04
......@@ -120,7 +120,7 @@ func TestPersistentVolumeRecycler(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
ctrl.Run(stopCh)
go ctrl.Run(stopCh)
defer close(stopCh)
// This PV will be claimed, released, and recycled.
......@@ -174,7 +174,7 @@ func TestPersistentVolumeDeleter(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
ctrl.Run(stopCh)
go ctrl.Run(stopCh)
defer close(stopCh)
// This PV will be claimed, released, and deleted.
......@@ -233,7 +233,7 @@ func TestPersistentVolumeBindRace(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
ctrl.Run(stopCh)
go ctrl.Run(stopCh)
defer close(stopCh)
pv := createPV("fake-pv-race", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
......@@ -304,7 +304,7 @@ func TestPersistentVolumeClaimLabelSelector(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
controller.Run(stopCh)
go controller.Run(stopCh)
defer close(stopCh)
var (
......@@ -384,7 +384,7 @@ func TestPersistentVolumeClaimLabelSelectorMatchExpressions(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
controller.Run(stopCh)
go controller.Run(stopCh)
defer close(stopCh)
var (
......@@ -483,7 +483,7 @@ func TestPersistentVolumeMultiPVs(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
controller.Run(stopCh)
go controller.Run(stopCh)
defer close(stopCh)
maxPVs := getObjectCount()
......@@ -572,7 +572,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
controllerStopCh := make(chan struct{})
binder.Run(controllerStopCh)
go binder.Run(controllerStopCh)
defer close(controllerStopCh)
objCount := getObjectCount()
......@@ -785,7 +785,7 @@ func TestPersistentVolumeControllerStartup(t *testing.T) {
// Start the controller when all PVs and PVCs are already saved in etcd
stopCh := make(chan struct{})
binder.Run(stopCh)
go binder.Run(stopCh)
defer close(stopCh)
// wait for at least two sync periods for changes. No volume should be
......@@ -872,7 +872,7 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) {
testClient.Storage().StorageClasses().Create(&storageClass)
stopCh := make(chan struct{})
binder.Run(stopCh)
go binder.Run(stopCh)
defer close(stopCh)
objCount := getObjectCount()
......@@ -957,7 +957,7 @@ func TestPersistentVolumeMultiPVsDiffAccessModes(t *testing.T) {
defer testClient.Core().PersistentVolumes().DeleteCollection(nil, v1.ListOptions{})
stopCh := make(chan struct{})
controller.Run(stopCh)
go controller.Run(stopCh)
defer close(stopCh)
// This PV will be claimed, released, and deleted
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment