Unverified Commit 4da3bdc4 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #64621 from RenaudWasTaken/pluginwatcher

Automatic merge from submit-queue (batch tested with PRs 68087, 68256, 64621, 68299, 68296). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Change plugin watcher registration mechanism **Which issue(s) this PR fixes**: #64637 **Notes For Reviewers**: The current API the plugin watcher exposes to kubelet is the following: ```golang type RegisterCallbackFn func(pluginName string, endpoint string, versions []string, socketPath string) (error, chan bool) ``` The callback channel is here to signal the plugin watcher consumer when the plugin watcher API has notified the plugin of it's successful registration. In other words the current lifecycle of a plugin is the following: ``` (pluginwatcher) GetInfo -> (pluginwatcher) NotifyRegistrationStatus -> (deviceplugin) ListWatch ``` Rather than ``` (pluginwatcher) GetInfo (race) -> (pluginwatcher) NotifyRegistrationStatus (race) -> (deviceplugin) ListWatch ``` This PR changes the callback/channel mechanism to a more explicit, interfaced based contract (and more maintainable than a function to which we add more channels for more lifecycle events). This PR also introduces three new states: {Init, Register, DeRegister} ```golang // PluginHandler is an interface a client of the pluginwatcher API needs to implement in // order to consume plugins // The PluginHandler follows the simple following state machine: // // +--------------------------------------+ // | ReRegistration | // | Socket created with same plugin name | // | | // | | // Socket Created v + Socket Deleted // +------------------> Validate +----------> Init +---------> Register +------------------> DeRegister // + + + // | | | // | Error | Error | // | | | // v v v // Out Out Out // // The pluginwatcher module follows strictly and sequentially this state machine for each *plugin name*. // e.g: If you are Registering a plugin foo, you cannot get a DeRegister call for plugin foo // until the Register("foo") call returns. Nor will you get a Validate("foo", "Different endpoint", ...) // call until the Register("foo") call returns. // // ReRegistration: Socket created with same plugin name, usually for a plugin update // e.g: plugin with name foo registers at foo.com/foo-1.9.7 later a plugin with name foo // registers at foo.com/foo-1.9.9 // // DeRegistration: When ReRegistration happens only the deletion of the new socket will trigger a DeRegister call type PluginHandler interface { // Validate returns an error if the information provided by // the potential plugin is erroneous (unsupported version, ...) ValidatePlugin(pluginName string, endpoint string, versions []string) error // Init starts the plugin (e.g: contact the gRPC client, gets plugin // specific information, ...) but if another plugin with the same name // exists does not switch to the newer one. // Any error encountered here can still be Notified to the plugin. InitPlugin(pluginName string, endpoint string) error // Register is called once the pluginwatcher has notified the plugin // of its successful registration. // Errors at this point can no longer be bubbled up to the plugin RegisterPlugin(pluginName, endpoint string) // DeRegister is called once the pluginwatcher observes that the socket has // been deleted. DeRegisterPlugin(pluginName string) } ``` ```release-note NONE ``` /sig node /area hw-accelerators /cc @jiayingz @vikaschoudhary16 @vishh @vladimirvivien @sbezverk @figo (ccing the main reviewers of the original PR, feel free to cc more people)
parents 5878b287 8dd1d27c
...@@ -95,7 +95,11 @@ type ContainerManager interface { ...@@ -95,7 +95,11 @@ type ContainerManager interface {
// GetPodCgroupRoot returns the cgroup which contains all pods. // GetPodCgroupRoot returns the cgroup which contains all pods.
GetPodCgroupRoot() string GetPodCgroupRoot() string
GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn
// GetPluginRegistrationHandler returns a plugin registration handler
// The pluginwatcher's Handlers allow to have a single module for handling
// registration.
GetPluginRegistrationHandler() pluginwatcher.PluginHandler
} }
type NodeConfig struct { type NodeConfig struct {
......
...@@ -605,8 +605,8 @@ func (cm *containerManagerImpl) Start(node *v1.Node, ...@@ -605,8 +605,8 @@ func (cm *containerManagerImpl) Start(node *v1.Node,
return nil return nil
} }
func (cm *containerManagerImpl) GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn { func (cm *containerManagerImpl) GetPluginRegistrationHandler() pluginwatcher.PluginHandler {
return cm.deviceManager.GetWatcherCallback() return cm.deviceManager.GetWatcherHandler()
} }
// TODO: move the GetResources logic to PodContainerManager. // TODO: move the GetResources logic to PodContainerManager.
......
...@@ -77,10 +77,8 @@ func (cm *containerManagerStub) GetCapacity() v1.ResourceList { ...@@ -77,10 +77,8 @@ func (cm *containerManagerStub) GetCapacity() v1.ResourceList {
return c return c
} }
func (cm *containerManagerStub) GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn { func (cm *containerManagerStub) GetPluginRegistrationHandler() pluginwatcher.PluginHandler {
return func(name string, endpoint string, versions []string, sockPath string) (chan bool, error) { return nil
return nil, nil
}
} }
func (cm *containerManagerStub) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) { func (cm *containerManagerStub) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) {
......
...@@ -57,9 +57,7 @@ func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) ...@@ -57,9 +57,7 @@ func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string)
return nil, nil, []string{} return nil, nil, []string{}
} }
// GetWatcherCallback returns plugin watcher callback // GetWatcherHandler returns plugin watcher interface
func (h *ManagerStub) GetWatcherCallback() pluginwatcher.RegisterCallbackFn { func (h *ManagerStub) GetWatcherHandler() pluginwatcher.PluginHandler {
return func(name string, endpoint string, versions []string, sockPath string) (chan bool, error) { return nil
return nil, nil
}
} }
...@@ -249,9 +249,10 @@ func setupDevicePlugin(t *testing.T, devs []*pluginapi.Device, pluginSocketName ...@@ -249,9 +249,10 @@ func setupDevicePlugin(t *testing.T, devs []*pluginapi.Device, pluginSocketName
func setupPluginWatcher(pluginSocketName string, m Manager) *pluginwatcher.Watcher { func setupPluginWatcher(pluginSocketName string, m Manager) *pluginwatcher.Watcher {
w := pluginwatcher.NewWatcher(filepath.Dir(pluginSocketName)) w := pluginwatcher.NewWatcher(filepath.Dir(pluginSocketName))
w.AddHandler(watcherapi.DevicePlugin, m.GetWatcherCallback()) w.AddHandler(watcherapi.DevicePlugin, m.GetWatcherHandler())
w.Start() w.Start()
return &w
return w
} }
func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback, socketName string, pluginSocketName string) (Manager, <-chan interface{}, *Stub) { func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback, socketName string, pluginSocketName string) (Manager, <-chan interface{}, *Stub) {
...@@ -295,7 +296,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) { ...@@ -295,7 +296,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) {
// Expects capacity for resource1 to be 2. // Expects capacity for resource1 to be 2.
resourceName1 := "domain1.com/resource1" resourceName1 := "domain1.com/resource1"
e1 := &endpointImpl{} e1 := &endpointImpl{}
testManager.endpoints[resourceName1] = e1 testManager.endpoints[resourceName1] = endpointInfo{e: e1, opts: nil}
callback(resourceName1, devs) callback(resourceName1, devs)
capacity, allocatable, removedResources := testManager.GetCapacity() capacity, allocatable, removedResources := testManager.GetCapacity()
resource1Capacity, ok := capacity[v1.ResourceName(resourceName1)] resource1Capacity, ok := capacity[v1.ResourceName(resourceName1)]
...@@ -345,7 +346,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) { ...@@ -345,7 +346,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) {
// Tests adding another resource. // Tests adding another resource.
resourceName2 := "resource2" resourceName2 := "resource2"
e2 := &endpointImpl{} e2 := &endpointImpl{}
testManager.endpoints[resourceName2] = e2 testManager.endpoints[resourceName2] = endpointInfo{e: e2, opts: nil}
callback(resourceName2, devs) callback(resourceName2, devs)
capacity, allocatable, removedResources = testManager.GetCapacity() capacity, allocatable, removedResources = testManager.GetCapacity()
as.Equal(2, len(capacity)) as.Equal(2, len(capacity))
...@@ -456,7 +457,7 @@ func TestCheckpoint(t *testing.T) { ...@@ -456,7 +457,7 @@ func TestCheckpoint(t *testing.T) {
ckm, err := checkpointmanager.NewCheckpointManager(tmpDir) ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
as.Nil(err) as.Nil(err)
testManager := &ManagerImpl{ testManager := &ManagerImpl{
endpoints: make(map[string]endpoint), endpoints: make(map[string]endpointInfo),
healthyDevices: make(map[string]sets.String), healthyDevices: make(map[string]sets.String),
unhealthyDevices: make(map[string]sets.String), unhealthyDevices: make(map[string]sets.String),
allocatedDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String),
...@@ -577,7 +578,7 @@ func makePod(limits v1.ResourceList) *v1.Pod { ...@@ -577,7 +578,7 @@ func makePod(limits v1.ResourceList) *v1.Pod {
} }
} }
func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource, opts map[string]*pluginapi.DevicePluginOptions) (*ManagerImpl, error) { func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource) (*ManagerImpl, error) {
monitorCallback := func(resourceName string, devices []pluginapi.Device) {} monitorCallback := func(resourceName string, devices []pluginapi.Device) {}
ckm, err := checkpointmanager.NewCheckpointManager(tmpDir) ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
if err != nil { if err != nil {
...@@ -589,25 +590,27 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso ...@@ -589,25 +590,27 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso
healthyDevices: make(map[string]sets.String), healthyDevices: make(map[string]sets.String),
unhealthyDevices: make(map[string]sets.String), unhealthyDevices: make(map[string]sets.String),
allocatedDevices: make(map[string]sets.String), allocatedDevices: make(map[string]sets.String),
endpoints: make(map[string]endpoint), endpoints: make(map[string]endpointInfo),
pluginOpts: opts,
podDevices: make(podDevices), podDevices: make(podDevices),
activePods: activePods, activePods: activePods,
sourcesReady: &sourcesReadyStub{}, sourcesReady: &sourcesReadyStub{},
checkpointManager: ckm, checkpointManager: ckm,
} }
for _, res := range testRes { for _, res := range testRes {
testManager.healthyDevices[res.resourceName] = sets.NewString() testManager.healthyDevices[res.resourceName] = sets.NewString()
for _, dev := range res.devs { for _, dev := range res.devs {
testManager.healthyDevices[res.resourceName].Insert(dev) testManager.healthyDevices[res.resourceName].Insert(dev)
} }
if res.resourceName == "domain1.com/resource1" { if res.resourceName == "domain1.com/resource1" {
testManager.endpoints[res.resourceName] = &MockEndpoint{ testManager.endpoints[res.resourceName] = endpointInfo{
allocateFunc: allocateStubFunc(), e: &MockEndpoint{allocateFunc: allocateStubFunc()},
opts: nil,
} }
} }
if res.resourceName == "domain2.com/resource2" { if res.resourceName == "domain2.com/resource2" {
testManager.endpoints[res.resourceName] = &MockEndpoint{ testManager.endpoints[res.resourceName] = endpointInfo{
e: &MockEndpoint{
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) { allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
resp := new(pluginapi.ContainerAllocateResponse) resp := new(pluginapi.ContainerAllocateResponse)
resp.Envs = make(map[string]string) resp.Envs = make(map[string]string)
...@@ -624,6 +627,8 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso ...@@ -624,6 +627,8 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso
resps.ContainerResponses = append(resps.ContainerResponses, resp) resps.ContainerResponses = append(resps.ContainerResponses, resp)
return resps, nil return resps, nil
}, },
},
opts: nil,
} }
} }
} }
...@@ -669,10 +674,7 @@ func TestPodContainerDeviceAllocation(t *testing.T) { ...@@ -669,10 +674,7 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
as.Nil(err) as.Nil(err)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
nodeInfo := getTestNodeInfo(v1.ResourceList{}) nodeInfo := getTestNodeInfo(v1.ResourceList{})
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions) testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources)
pluginOpts[res1.resourceName] = nil
pluginOpts[res2.resourceName] = nil
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
as.Nil(err) as.Nil(err)
testPods := []*v1.Pod{ testPods := []*v1.Pod{
...@@ -767,10 +769,8 @@ func TestInitContainerDeviceAllocation(t *testing.T) { ...@@ -767,10 +769,8 @@ func TestInitContainerDeviceAllocation(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "checkpoint") tmpDir, err := ioutil.TempDir("", "checkpoint")
as.Nil(err) as.Nil(err)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
pluginOpts[res1.resourceName] = nil testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources)
pluginOpts[res2.resourceName] = nil
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
as.Nil(err) as.Nil(err)
podWithPluginResourcesInInitContainers := &v1.Pod{ podWithPluginResourcesInInitContainers := &v1.Pod{
...@@ -904,18 +904,18 @@ func TestDevicePreStartContainer(t *testing.T) { ...@@ -904,18 +904,18 @@ func TestDevicePreStartContainer(t *testing.T) {
as.Nil(err) as.Nil(err)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
nodeInfo := getTestNodeInfo(v1.ResourceList{}) nodeInfo := getTestNodeInfo(v1.ResourceList{})
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
pluginOpts[res1.resourceName] = &pluginapi.DevicePluginOptions{PreStartRequired: true}
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, []TestResource{res1}, pluginOpts) testManager, err := getTestManager(tmpDir, podsStub.getActivePods, []TestResource{res1})
as.Nil(err) as.Nil(err)
ch := make(chan []string, 1) ch := make(chan []string, 1)
testManager.endpoints[res1.resourceName] = &MockEndpoint{ testManager.endpoints[res1.resourceName] = endpointInfo{
e: &MockEndpoint{
initChan: ch, initChan: ch,
allocateFunc: allocateStubFunc(), allocateFunc: allocateStubFunc(),
},
opts: &pluginapi.DevicePluginOptions{PreStartRequired: true},
} }
pod := makePod(v1.ResourceList{ pod := makePod(v1.ResourceList{
v1.ResourceName(res1.resourceName): res1.resourceQuantity}) v1.ResourceName(res1.resourceName): res1.resourceQuantity})
activePods := []*v1.Pod{} activePods := []*v1.Pod{}
......
...@@ -53,7 +53,7 @@ type Manager interface { ...@@ -53,7 +53,7 @@ type Manager interface {
// GetCapacity returns the amount of available device plugin resource capacity, resource allocatable // GetCapacity returns the amount of available device plugin resource capacity, resource allocatable
// and inactive device plugin resources previously registered on the node. // and inactive device plugin resources previously registered on the node.
GetCapacity() (v1.ResourceList, v1.ResourceList, []string) GetCapacity() (v1.ResourceList, v1.ResourceList, []string)
GetWatcherCallback() watcher.RegisterCallbackFn GetWatcherHandler() watcher.PluginHandler
} }
// DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices. // DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices.
......
...@@ -1194,7 +1194,7 @@ type Kubelet struct { ...@@ -1194,7 +1194,7 @@ type Kubelet struct {
// pluginwatcher is a utility for Kubelet to register different types of node-level plugins // pluginwatcher is a utility for Kubelet to register different types of node-level plugins
// such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the // such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the
// directory returned by kubelet.getPluginsDir() // directory returned by kubelet.getPluginsDir()
pluginWatcher pluginwatcher.Watcher pluginWatcher *pluginwatcher.Watcher
// This flag sets a maximum number of images to report in the node status. // This flag sets a maximum number of images to report in the node status.
nodeStatusMaxImages int32 nodeStatusMaxImages int32
...@@ -1365,9 +1365,9 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { ...@@ -1365,9 +1365,9 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
kl.containerLogManager.Start() kl.containerLogManager.Start()
if kl.enablePluginsWatcher { if kl.enablePluginsWatcher {
// Adding Registration Callback function for CSI Driver // Adding Registration Callback function for CSI Driver
kl.pluginWatcher.AddHandler("CSIPlugin", csi.RegistrationCallback) kl.pluginWatcher.AddHandler("CSIPlugin", pluginwatcher.PluginHandler(csi.PluginHandler))
// Adding Registration Callback function for Device Manager // Adding Registration Callback function for Device Manager
kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandlerCallback()) kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandler())
// Start the plugin watcher // Start the plugin watcher
glog.V(4).Infof("starting watcher") glog.V(4).Infof("starting watcher")
if err := kl.pluginWatcher.Start(); err != nil { if err := kl.pluginWatcher.Start(); err != nil {
......
package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library( go_library(
name = "go_default_library", name = "go_default_library",
...@@ -12,8 +6,10 @@ go_library( ...@@ -12,8 +6,10 @@ go_library(
"example_handler.go", "example_handler.go",
"example_plugin.go", "example_plugin.go",
"plugin_watcher.go", "plugin_watcher.go",
"types.go",
], ],
importpath = "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher", importpath = "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher",
visibility = ["//visibility:public"],
deps = [ deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library", "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library",
...@@ -27,6 +23,16 @@ go_library( ...@@ -27,6 +23,16 @@ go_library(
], ],
) )
go_test(
name = "go_default_test",
srcs = ["plugin_watcher_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
filegroup( filegroup(
name = "package-srcs", name = "package-srcs",
srcs = glob(["**"]), srcs = glob(["**"]),
...@@ -44,14 +50,3 @@ filegroup( ...@@ -44,14 +50,3 @@ filegroup(
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
go_test(
name = "go_default_test",
srcs = ["plugin_watcher_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
...@@ -23,6 +23,7 @@ import ( ...@@ -23,6 +23,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/golang/glog"
"golang.org/x/net/context" "golang.org/x/net/context"
v1beta1 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1" v1beta1 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1"
...@@ -30,41 +31,61 @@ import ( ...@@ -30,41 +31,61 @@ import (
) )
type exampleHandler struct { type exampleHandler struct {
registeredPlugins map[string]struct{} SupportedVersions []string
mutex sync.Mutex ExpectedNames map[string]int
chanForHandlerAckErrors chan error // for testing
eventChans map[string]chan examplePluginEvent // map[pluginName]eventChan
m sync.Mutex
count int
} }
type examplePluginEvent int
const (
exampleEventValidate examplePluginEvent = 0
exampleEventRegister examplePluginEvent = 1
exampleEventDeRegister examplePluginEvent = 2
exampleEventError examplePluginEvent = 3
)
// NewExampleHandler provide a example handler // NewExampleHandler provide a example handler
func NewExampleHandler() *exampleHandler { func NewExampleHandler(supportedVersions []string) *exampleHandler {
return &exampleHandler{ return &exampleHandler{
chanForHandlerAckErrors: make(chan error), SupportedVersions: supportedVersions,
registeredPlugins: make(map[string]struct{}), ExpectedNames: make(map[string]int),
eventChans: make(map[string]chan examplePluginEvent),
} }
} }
func (h *exampleHandler) Cleanup() error { func (p *exampleHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
h.mutex.Lock() p.SendEvent(pluginName, exampleEventValidate)
defer h.mutex.Unlock()
h.registeredPlugins = make(map[string]struct{})
return nil
}
func (h *exampleHandler) Handler(pluginName string, endpoint string, versions []string, sockPath string) (chan bool, error) { n, ok := p.DecreasePluginCount(pluginName)
if !ok && n > 0 {
return fmt.Errorf("pluginName('%s') wasn't expected (count is %d)", pluginName, n)
}
// check for supported versions if !reflect.DeepEqual(versions, p.SupportedVersions) {
if !reflect.DeepEqual([]string{"v1beta1", "v1beta2"}, versions) { return fmt.Errorf("versions('%v') != supported versions('%v')", versions, p.SupportedVersions)
return nil, fmt.Errorf("not the supported versions: %s", versions)
} }
// this handler expects non-empty endpoint as an example // this handler expects non-empty endpoint as an example
if len(endpoint) == 0 { if len(endpoint) == 0 {
return nil, errors.New("expecting non empty endpoint") return errors.New("expecting non empty endpoint")
} }
_, conn, err := dial(sockPath) return nil
}
func (p *exampleHandler) RegisterPlugin(pluginName, endpoint string) error {
p.SendEvent(pluginName, exampleEventRegister)
// Verifies the grpcServer is ready to serve services.
_, conn, err := dial(endpoint, time.Second)
if err != nil { if err != nil {
return nil, err return fmt.Errorf("Failed dialing endpoint (%s): %v", endpoint, err)
} }
defer conn.Close() defer conn.Close()
...@@ -73,33 +94,54 @@ func (h *exampleHandler) Handler(pluginName string, endpoint string, versions [] ...@@ -73,33 +94,54 @@ func (h *exampleHandler) Handler(pluginName string, endpoint string, versions []
v1beta2Client := v1beta2.NewExampleClient(conn) v1beta2Client := v1beta2.NewExampleClient(conn)
// Tests v1beta1 GetExampleInfo // Tests v1beta1 GetExampleInfo
if _, err = v1beta1Client.GetExampleInfo(context.Background(), &v1beta1.ExampleRequest{}); err != nil { _, err = v1beta1Client.GetExampleInfo(context.Background(), &v1beta1.ExampleRequest{})
return nil, err if err != nil {
return fmt.Errorf("Failed GetExampleInfo for v1beta2Client(%s): %v", endpoint, err)
} }
// Tests v1beta2 GetExampleInfo // Tests v1beta1 GetExampleInfo
if _, err = v1beta2Client.GetExampleInfo(context.Background(), &v1beta2.ExampleRequest{}); err != nil { _, err = v1beta2Client.GetExampleInfo(context.Background(), &v1beta2.ExampleRequest{})
return nil, err if err != nil {
return fmt.Errorf("Failed GetExampleInfo for v1beta2Client(%s): %v", endpoint, err)
} }
// handle registered plugin return nil
h.mutex.Lock() }
if _, exist := h.registeredPlugins[pluginName]; exist {
h.mutex.Unlock() func (p *exampleHandler) DeRegisterPlugin(pluginName string) {
return nil, fmt.Errorf("plugin %s already registered", pluginName) p.SendEvent(pluginName, exampleEventDeRegister)
}
func (p *exampleHandler) EventChan(pluginName string) chan examplePluginEvent {
return p.eventChans[pluginName]
}
func (p *exampleHandler) SendEvent(pluginName string, event examplePluginEvent) {
glog.V(2).Infof("Sending %v for plugin %s over chan %v", event, pluginName, p.eventChans[pluginName])
p.eventChans[pluginName] <- event
}
func (p *exampleHandler) AddPluginName(pluginName string) {
p.m.Lock()
defer p.m.Unlock()
v, ok := p.ExpectedNames[pluginName]
if !ok {
p.eventChans[pluginName] = make(chan examplePluginEvent)
v = 1
} }
h.registeredPlugins[pluginName] = struct{}{}
h.mutex.Unlock() p.ExpectedNames[pluginName] = v
}
chanForAckOfNotification := make(chan bool)
go func() { func (p *exampleHandler) DecreasePluginCount(pluginName string) (old int, ok bool) {
select { p.m.Lock()
case <-chanForAckOfNotification: defer p.m.Unlock()
// TODO: handle the negative scenario
close(chanForAckOfNotification) v, ok := p.ExpectedNames[pluginName]
case <-time.After(time.Second): if !ok {
h.chanForHandlerAckErrors <- errors.New("Timed out while waiting for notification ack") v = -1
} }
}()
return chanForAckOfNotification, nil return v, ok
} }
...@@ -18,7 +18,9 @@ package pluginwatcher ...@@ -18,7 +18,9 @@ package pluginwatcher
import ( import (
"errors" "errors"
"fmt"
"net" "net"
"os"
"sync" "sync"
"time" "time"
...@@ -39,6 +41,7 @@ type examplePlugin struct { ...@@ -39,6 +41,7 @@ type examplePlugin struct {
endpoint string // for testing endpoint string // for testing
pluginName string pluginName string
pluginType string pluginType string
versions []string
} }
type pluginServiceV1Beta1 struct { type pluginServiceV1Beta1 struct {
...@@ -73,12 +76,13 @@ func NewExamplePlugin() *examplePlugin { ...@@ -73,12 +76,13 @@ func NewExamplePlugin() *examplePlugin {
} }
// NewTestExamplePlugin returns an initialized examplePlugin instance for testing // NewTestExamplePlugin returns an initialized examplePlugin instance for testing
func NewTestExamplePlugin(pluginName string, pluginType string, endpoint string) *examplePlugin { func NewTestExamplePlugin(pluginName string, pluginType string, endpoint string, advertisedVersions ...string) *examplePlugin {
return &examplePlugin{ return &examplePlugin{
pluginName: pluginName, pluginName: pluginName,
pluginType: pluginType, pluginType: pluginType,
registrationStatus: make(chan registerapi.RegistrationStatus),
endpoint: endpoint, endpoint: endpoint,
versions: advertisedVersions,
registrationStatus: make(chan registerapi.RegistrationStatus),
} }
} }
...@@ -88,36 +92,48 @@ func (e *examplePlugin) GetInfo(ctx context.Context, req *registerapi.InfoReques ...@@ -88,36 +92,48 @@ func (e *examplePlugin) GetInfo(ctx context.Context, req *registerapi.InfoReques
Type: e.pluginType, Type: e.pluginType,
Name: e.pluginName, Name: e.pluginName,
Endpoint: e.endpoint, Endpoint: e.endpoint,
SupportedVersions: []string{"v1beta1", "v1beta2"}, SupportedVersions: e.versions,
}, nil }, nil
} }
func (e *examplePlugin) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) { func (e *examplePlugin) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) {
glog.Errorf("Registration is: %v\n", status)
if e.registrationStatus != nil { if e.registrationStatus != nil {
e.registrationStatus <- *status e.registrationStatus <- *status
} }
if !status.PluginRegistered {
glog.Errorf("Registration failed: %s\n", status.Error)
}
return &registerapi.RegistrationStatusResponse{}, nil return &registerapi.RegistrationStatusResponse{}, nil
} }
// Serve starts example plugin grpc server // Serve starts a pluginwatcher server and one or more of the plugin services
func (e *examplePlugin) Serve(socketPath string) error { func (e *examplePlugin) Serve(services ...string) error {
glog.Infof("starting example server at: %s\n", socketPath) glog.Infof("starting example server at: %s\n", e.endpoint)
lis, err := net.Listen("unix", socketPath) lis, err := net.Listen("unix", e.endpoint)
if err != nil { if err != nil {
return err return err
} }
glog.Infof("example server started at: %s\n", socketPath)
glog.Infof("example server started at: %s\n", e.endpoint)
e.grpcServer = grpc.NewServer() e.grpcServer = grpc.NewServer()
// Registers kubelet plugin watcher api. // Registers kubelet plugin watcher api.
registerapi.RegisterRegistrationServer(e.grpcServer, e) registerapi.RegisterRegistrationServer(e.grpcServer, e)
// Registers services for both v1beta1 and v1beta2 versions.
for _, service := range services {
switch service {
case "v1beta1":
v1beta1 := &pluginServiceV1Beta1{server: e} v1beta1 := &pluginServiceV1Beta1{server: e}
v1beta1.RegisterService() v1beta1.RegisterService()
break
case "v1beta2":
v1beta2 := &pluginServiceV1Beta2{server: e} v1beta2 := &pluginServiceV1Beta2{server: e}
v1beta2.RegisterService() v1beta2.RegisterService()
break
default:
return fmt.Errorf("Unsupported service: '%s'", service)
}
}
// Starts service // Starts service
e.wg.Add(1) e.wg.Add(1)
...@@ -128,22 +144,30 @@ func (e *examplePlugin) Serve(socketPath string) error { ...@@ -128,22 +144,30 @@ func (e *examplePlugin) Serve(socketPath string) error {
glog.Errorf("example server stopped serving: %v", err) glog.Errorf("example server stopped serving: %v", err)
} }
}() }()
return nil return nil
} }
func (e *examplePlugin) Stop() error { func (e *examplePlugin) Stop() error {
glog.Infof("Stopping example server\n") glog.Infof("Stopping example server at: %s\n", e.endpoint)
e.grpcServer.Stop() e.grpcServer.Stop()
c := make(chan struct{}) c := make(chan struct{})
go func() { go func() {
defer close(c) defer close(c)
e.wg.Wait() e.wg.Wait()
}() }()
select { select {
case <-c: case <-c:
return nil break
case <-time.After(time.Second): case <-time.After(time.Second):
glog.Errorf("Timed out on waiting for stop completion")
return errors.New("Timed out on waiting for stop completion") return errors.New("Timed out on waiting for stop completion")
} }
if err := os.Remove(e.endpoint); err != nil && !os.IsNotExist(err) {
return err
}
return nil
} }
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pluginwatcher
// PluginHandler is an interface a client of the pluginwatcher API needs to implement in
// order to consume plugins
// The PluginHandler follows the simple following state machine:
//
// +--------------------------------------+
// | ReRegistration |
// | Socket created with same plugin name |
// | |
// | |
// Socket Created v + Socket Deleted
// +------------------> Validate +---------------------------> Register +------------------> DeRegister
// + + +
// | | |
// | Error | Error |
// | | |
// v v v
// Out Out Out
//
// The pluginwatcher module follows strictly and sequentially this state machine for each *plugin name*.
// e.g: If you are Registering a plugin foo, you cannot get a DeRegister call for plugin foo
// until the Register("foo") call returns. Nor will you get a Validate("foo", "Different endpoint", ...)
// call until the Register("foo") call returns.
//
// ReRegistration: Socket created with same plugin name, usually for a plugin update
// e.g: plugin with name foo registers at foo.com/foo-1.9.7 later a plugin with name foo
// registers at foo.com/foo-1.9.9
//
// DeRegistration: When ReRegistration happens only the deletion of the new socket will trigger a DeRegister call
type PluginHandler interface {
// Validate returns an error if the information provided by
// the potential plugin is erroneous (unsupported version, ...)
ValidatePlugin(pluginName string, endpoint string, versions []string) error
// RegisterPlugin is called so that the plugin can be register by any
// plugin consumer
// Error encountered here can still be Notified to the plugin.
RegisterPlugin(pluginName, endpoint string) error
// DeRegister is called once the pluginwatcher observes that the socket has
// been deleted.
DeRegisterPlugin(pluginName string)
}
...@@ -28,6 +28,7 @@ import ( ...@@ -28,6 +28,7 @@ import (
"context" "context"
"github.com/golang/glog" "github.com/golang/glog"
api "k8s.io/api/core/v1" api "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1"
...@@ -89,6 +90,10 @@ type csiDriversStore struct { ...@@ -89,6 +90,10 @@ type csiDriversStore struct {
sync.RWMutex sync.RWMutex
} }
// RegistrationHandler is the handler which is fed to the pluginwatcher API.
type RegistrationHandler struct {
}
// TODO (verult) consider using a struct instead of global variables // TODO (verult) consider using a struct instead of global variables
// csiDrivers map keep track of all registered CSI drivers on the node and their // csiDrivers map keep track of all registered CSI drivers on the node and their
// corresponding sockets // corresponding sockets
...@@ -96,21 +101,28 @@ var csiDrivers csiDriversStore ...@@ -96,21 +101,28 @@ var csiDrivers csiDriversStore
var nodeUpdater nodeupdater.Interface var nodeUpdater nodeupdater.Interface
// RegistrationCallback is called by kubelet's plugin watcher upon detection // PluginHandler is the plugin registration handler interface passed to the
// pluginwatcher module in kubelet
var PluginHandler = &RegistrationHandler{}
// ValidatePlugin is called by kubelet's plugin watcher upon detection
// of a new registration socket opened by CSI Driver registrar side car. // of a new registration socket opened by CSI Driver registrar side car.
func RegistrationCallback(pluginName string, endpoint string, versions []string, socketPath string) (chan bool, error) { func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
glog.Infof(log("Trying to register a new plugin with name: %s endpoint: %s versions: %s",
pluginName, endpoint, strings.Join(versions, ",")))
glog.Infof(log("Callback from kubelet with plugin name: %s endpoint: %s versions: %s socket path: %s", return nil
pluginName, endpoint, strings.Join(versions, ","), socketPath)) }
if endpoint == "" { // RegisterPlugin is called when a plugin can be registered
endpoint = socketPath func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string) error {
} glog.Infof(log("Register new plugin with name: %s at endpoint: %s", pluginName, endpoint))
// Storing endpoint of newly registered CSI driver into the map, where CSI driver name will be the key // Storing endpoint of newly registered CSI driver into the map, where CSI driver name will be the key
// all other CSI components will be able to get the actual socket of CSI drivers by its name. // all other CSI components will be able to get the actual socket of CSI drivers by its name.
csiDrivers.Lock() csiDrivers.Lock()
defer csiDrivers.Unlock() defer csiDrivers.Unlock()
csiDrivers.driversMap[pluginName] = csiDriver{driverName: pluginName, driverEndpoint: endpoint} csiDrivers.driversMap[pluginName] = csiDriver{driverName: pluginName, driverEndpoint: endpoint}
// Get node info from the driver. // Get node info from the driver.
...@@ -118,22 +130,27 @@ func RegistrationCallback(pluginName string, endpoint string, versions []string, ...@@ -118,22 +130,27 @@ func RegistrationCallback(pluginName string, endpoint string, versions []string,
// TODO (verult) retry with exponential backoff, possibly added in csi client library. // TODO (verult) retry with exponential backoff, possibly added in csi client library.
ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) ctx, cancel := context.WithTimeout(context.Background(), csiTimeout)
defer cancel() defer cancel()
driverNodeID, maxVolumePerNode, _, err := csi.NodeGetInfo(ctx) driverNodeID, maxVolumePerNode, _, err := csi.NodeGetInfo(ctx)
if err != nil { if err != nil {
return nil, fmt.Errorf("error during CSI NodeGetInfo() call: %v", err) return fmt.Errorf("error during CSI NodeGetInfo() call: %v", err)
} }
// Calling nodeLabelManager to update annotations and labels for newly registered CSI driver // Calling nodeLabelManager to update annotations and labels for newly registered CSI driver
err = nodeUpdater.AddLabelsAndLimits(pluginName, driverNodeID, maxVolumePerNode) err = nodeUpdater.AddLabelsAndLimits(pluginName, driverNodeID, maxVolumePerNode)
if err != nil { if err != nil {
// Unregister the driver and return error // Unregister the driver and return error
csiDrivers.Lock()
defer csiDrivers.Unlock()
delete(csiDrivers.driversMap, pluginName) delete(csiDrivers.driversMap, pluginName)
return nil, err return fmt.Errorf("error while adding CSI labels: %v", err)
} }
return nil, nil return nil
}
// DeRegisterPlugin is called when a plugin removed it's socket, signaling
// it is no longer available
// TODO: Handle DeRegistration
func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) {
} }
func (p *csiPlugin) Init(host volume.VolumeHost) error { func (p *csiPlugin) Init(host volume.VolumeHost) error {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment