Unverified Commit 4da3bdc4 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #64621 from RenaudWasTaken/pluginwatcher

Automatic merge from submit-queue (batch tested with PRs 68087, 68256, 64621, 68299, 68296). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Change plugin watcher registration mechanism **Which issue(s) this PR fixes**: #64637 **Notes For Reviewers**: The current API the plugin watcher exposes to kubelet is the following: ```golang type RegisterCallbackFn func(pluginName string, endpoint string, versions []string, socketPath string) (error, chan bool) ``` The callback channel is here to signal the plugin watcher consumer when the plugin watcher API has notified the plugin of it's successful registration. In other words the current lifecycle of a plugin is the following: ``` (pluginwatcher) GetInfo -> (pluginwatcher) NotifyRegistrationStatus -> (deviceplugin) ListWatch ``` Rather than ``` (pluginwatcher) GetInfo (race) -> (pluginwatcher) NotifyRegistrationStatus (race) -> (deviceplugin) ListWatch ``` This PR changes the callback/channel mechanism to a more explicit, interfaced based contract (and more maintainable than a function to which we add more channels for more lifecycle events). This PR also introduces three new states: {Init, Register, DeRegister} ```golang // PluginHandler is an interface a client of the pluginwatcher API needs to implement in // order to consume plugins // The PluginHandler follows the simple following state machine: // // +--------------------------------------+ // | ReRegistration | // | Socket created with same plugin name | // | | // | | // Socket Created v + Socket Deleted // +------------------> Validate +----------> Init +---------> Register +------------------> DeRegister // + + + // | | | // | Error | Error | // | | | // v v v // Out Out Out // // The pluginwatcher module follows strictly and sequentially this state machine for each *plugin name*. // e.g: If you are Registering a plugin foo, you cannot get a DeRegister call for plugin foo // until the Register("foo") call returns. Nor will you get a Validate("foo", "Different endpoint", ...) // call until the Register("foo") call returns. // // ReRegistration: Socket created with same plugin name, usually for a plugin update // e.g: plugin with name foo registers at foo.com/foo-1.9.7 later a plugin with name foo // registers at foo.com/foo-1.9.9 // // DeRegistration: When ReRegistration happens only the deletion of the new socket will trigger a DeRegister call type PluginHandler interface { // Validate returns an error if the information provided by // the potential plugin is erroneous (unsupported version, ...) ValidatePlugin(pluginName string, endpoint string, versions []string) error // Init starts the plugin (e.g: contact the gRPC client, gets plugin // specific information, ...) but if another plugin with the same name // exists does not switch to the newer one. // Any error encountered here can still be Notified to the plugin. InitPlugin(pluginName string, endpoint string) error // Register is called once the pluginwatcher has notified the plugin // of its successful registration. // Errors at this point can no longer be bubbled up to the plugin RegisterPlugin(pluginName, endpoint string) // DeRegister is called once the pluginwatcher observes that the socket has // been deleted. DeRegisterPlugin(pluginName string) } ``` ```release-note NONE ``` /sig node /area hw-accelerators /cc @jiayingz @vikaschoudhary16 @vishh @vladimirvivien @sbezverk @figo (ccing the main reviewers of the original PR, feel free to cc more people)
parents 5878b287 8dd1d27c
......@@ -95,7 +95,11 @@ type ContainerManager interface {
// GetPodCgroupRoot returns the cgroup which contains all pods.
GetPodCgroupRoot() string
GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn
// GetPluginRegistrationHandler returns a plugin registration handler
// The pluginwatcher's Handlers allow to have a single module for handling
// registration.
GetPluginRegistrationHandler() pluginwatcher.PluginHandler
}
type NodeConfig struct {
......
......@@ -605,8 +605,8 @@ func (cm *containerManagerImpl) Start(node *v1.Node,
return nil
}
func (cm *containerManagerImpl) GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn {
return cm.deviceManager.GetWatcherCallback()
func (cm *containerManagerImpl) GetPluginRegistrationHandler() pluginwatcher.PluginHandler {
return cm.deviceManager.GetWatcherHandler()
}
// TODO: move the GetResources logic to PodContainerManager.
......
......@@ -77,10 +77,8 @@ func (cm *containerManagerStub) GetCapacity() v1.ResourceList {
return c
}
func (cm *containerManagerStub) GetPluginRegistrationHandlerCallback() pluginwatcher.RegisterCallbackFn {
return func(name string, endpoint string, versions []string, sockPath string) (chan bool, error) {
return nil, nil
}
func (cm *containerManagerStub) GetPluginRegistrationHandler() pluginwatcher.PluginHandler {
return nil
}
func (cm *containerManagerStub) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) {
......
......@@ -57,9 +57,7 @@ func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string)
return nil, nil, []string{}
}
// GetWatcherCallback returns plugin watcher callback
func (h *ManagerStub) GetWatcherCallback() pluginwatcher.RegisterCallbackFn {
return func(name string, endpoint string, versions []string, sockPath string) (chan bool, error) {
return nil, nil
}
// GetWatcherHandler returns plugin watcher interface
func (h *ManagerStub) GetWatcherHandler() pluginwatcher.PluginHandler {
return nil
}
......@@ -249,9 +249,10 @@ func setupDevicePlugin(t *testing.T, devs []*pluginapi.Device, pluginSocketName
func setupPluginWatcher(pluginSocketName string, m Manager) *pluginwatcher.Watcher {
w := pluginwatcher.NewWatcher(filepath.Dir(pluginSocketName))
w.AddHandler(watcherapi.DevicePlugin, m.GetWatcherCallback())
w.AddHandler(watcherapi.DevicePlugin, m.GetWatcherHandler())
w.Start()
return &w
return w
}
func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback, socketName string, pluginSocketName string) (Manager, <-chan interface{}, *Stub) {
......@@ -295,7 +296,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) {
// Expects capacity for resource1 to be 2.
resourceName1 := "domain1.com/resource1"
e1 := &endpointImpl{}
testManager.endpoints[resourceName1] = e1
testManager.endpoints[resourceName1] = endpointInfo{e: e1, opts: nil}
callback(resourceName1, devs)
capacity, allocatable, removedResources := testManager.GetCapacity()
resource1Capacity, ok := capacity[v1.ResourceName(resourceName1)]
......@@ -345,7 +346,7 @@ func TestUpdateCapacityAllocatable(t *testing.T) {
// Tests adding another resource.
resourceName2 := "resource2"
e2 := &endpointImpl{}
testManager.endpoints[resourceName2] = e2
testManager.endpoints[resourceName2] = endpointInfo{e: e2, opts: nil}
callback(resourceName2, devs)
capacity, allocatable, removedResources = testManager.GetCapacity()
as.Equal(2, len(capacity))
......@@ -456,7 +457,7 @@ func TestCheckpoint(t *testing.T) {
ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
as.Nil(err)
testManager := &ManagerImpl{
endpoints: make(map[string]endpoint),
endpoints: make(map[string]endpointInfo),
healthyDevices: make(map[string]sets.String),
unhealthyDevices: make(map[string]sets.String),
allocatedDevices: make(map[string]sets.String),
......@@ -577,7 +578,7 @@ func makePod(limits v1.ResourceList) *v1.Pod {
}
}
func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource, opts map[string]*pluginapi.DevicePluginOptions) (*ManagerImpl, error) {
func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource) (*ManagerImpl, error) {
monitorCallback := func(resourceName string, devices []pluginapi.Device) {}
ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
if err != nil {
......@@ -589,41 +590,45 @@ func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestReso
healthyDevices: make(map[string]sets.String),
unhealthyDevices: make(map[string]sets.String),
allocatedDevices: make(map[string]sets.String),
endpoints: make(map[string]endpoint),
pluginOpts: opts,
endpoints: make(map[string]endpointInfo),
podDevices: make(podDevices),
activePods: activePods,
sourcesReady: &sourcesReadyStub{},
checkpointManager: ckm,
}
for _, res := range testRes {
testManager.healthyDevices[res.resourceName] = sets.NewString()
for _, dev := range res.devs {
testManager.healthyDevices[res.resourceName].Insert(dev)
}
if res.resourceName == "domain1.com/resource1" {
testManager.endpoints[res.resourceName] = &MockEndpoint{
allocateFunc: allocateStubFunc(),
testManager.endpoints[res.resourceName] = endpointInfo{
e: &MockEndpoint{allocateFunc: allocateStubFunc()},
opts: nil,
}
}
if res.resourceName == "domain2.com/resource2" {
testManager.endpoints[res.resourceName] = &MockEndpoint{
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
resp := new(pluginapi.ContainerAllocateResponse)
resp.Envs = make(map[string]string)
for _, dev := range devs {
switch dev {
case "dev3":
resp.Envs["key2"] = "val2"
case "dev4":
resp.Envs["key2"] = "val3"
testManager.endpoints[res.resourceName] = endpointInfo{
e: &MockEndpoint{
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
resp := new(pluginapi.ContainerAllocateResponse)
resp.Envs = make(map[string]string)
for _, dev := range devs {
switch dev {
case "dev3":
resp.Envs["key2"] = "val2"
case "dev4":
resp.Envs["key2"] = "val3"
}
}
}
resps := new(pluginapi.AllocateResponse)
resps.ContainerResponses = append(resps.ContainerResponses, resp)
return resps, nil
resps := new(pluginapi.AllocateResponse)
resps.ContainerResponses = append(resps.ContainerResponses, resp)
return resps, nil
},
},
opts: nil,
}
}
}
......@@ -669,10 +674,7 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
as.Nil(err)
defer os.RemoveAll(tmpDir)
nodeInfo := getTestNodeInfo(v1.ResourceList{})
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
pluginOpts[res1.resourceName] = nil
pluginOpts[res2.resourceName] = nil
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources)
as.Nil(err)
testPods := []*v1.Pod{
......@@ -767,10 +769,8 @@ func TestInitContainerDeviceAllocation(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "checkpoint")
as.Nil(err)
defer os.RemoveAll(tmpDir)
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
pluginOpts[res1.resourceName] = nil
pluginOpts[res2.resourceName] = nil
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources)
as.Nil(err)
podWithPluginResourcesInInitContainers := &v1.Pod{
......@@ -904,18 +904,18 @@ func TestDevicePreStartContainer(t *testing.T) {
as.Nil(err)
defer os.RemoveAll(tmpDir)
nodeInfo := getTestNodeInfo(v1.ResourceList{})
pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
pluginOpts[res1.resourceName] = &pluginapi.DevicePluginOptions{PreStartRequired: true}
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, []TestResource{res1}, pluginOpts)
testManager, err := getTestManager(tmpDir, podsStub.getActivePods, []TestResource{res1})
as.Nil(err)
ch := make(chan []string, 1)
testManager.endpoints[res1.resourceName] = &MockEndpoint{
initChan: ch,
allocateFunc: allocateStubFunc(),
testManager.endpoints[res1.resourceName] = endpointInfo{
e: &MockEndpoint{
initChan: ch,
allocateFunc: allocateStubFunc(),
},
opts: &pluginapi.DevicePluginOptions{PreStartRequired: true},
}
pod := makePod(v1.ResourceList{
v1.ResourceName(res1.resourceName): res1.resourceQuantity})
activePods := []*v1.Pod{}
......
......@@ -53,7 +53,7 @@ type Manager interface {
// GetCapacity returns the amount of available device plugin resource capacity, resource allocatable
// and inactive device plugin resources previously registered on the node.
GetCapacity() (v1.ResourceList, v1.ResourceList, []string)
GetWatcherCallback() watcher.RegisterCallbackFn
GetWatcherHandler() watcher.PluginHandler
}
// DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices.
......
......@@ -1194,7 +1194,7 @@ type Kubelet struct {
// pluginwatcher is a utility for Kubelet to register different types of node-level plugins
// such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the
// directory returned by kubelet.getPluginsDir()
pluginWatcher pluginwatcher.Watcher
pluginWatcher *pluginwatcher.Watcher
// This flag sets a maximum number of images to report in the node status.
nodeStatusMaxImages int32
......@@ -1365,9 +1365,9 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
kl.containerLogManager.Start()
if kl.enablePluginsWatcher {
// Adding Registration Callback function for CSI Driver
kl.pluginWatcher.AddHandler("CSIPlugin", csi.RegistrationCallback)
kl.pluginWatcher.AddHandler("CSIPlugin", pluginwatcher.PluginHandler(csi.PluginHandler))
// Adding Registration Callback function for Device Manager
kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandlerCallback())
kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandler())
// Start the plugin watcher
glog.V(4).Infof("starting watcher")
if err := kl.pluginWatcher.Start(); err != nil {
......
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
......@@ -12,8 +6,10 @@ go_library(
"example_handler.go",
"example_plugin.go",
"plugin_watcher.go",
"types.go",
],
importpath = "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher",
visibility = ["//visibility:public"],
deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library",
......@@ -27,6 +23,16 @@ go_library(
],
)
go_test(
name = "go_default_test",
srcs = ["plugin_watcher_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
......@@ -44,14 +50,3 @@ filegroup(
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["plugin_watcher_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
......@@ -23,6 +23,7 @@ import (
"sync"
"time"
"github.com/golang/glog"
"golang.org/x/net/context"
v1beta1 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1"
......@@ -30,41 +31,61 @@ import (
)
type exampleHandler struct {
registeredPlugins map[string]struct{}
mutex sync.Mutex
chanForHandlerAckErrors chan error // for testing
SupportedVersions []string
ExpectedNames map[string]int
eventChans map[string]chan examplePluginEvent // map[pluginName]eventChan
m sync.Mutex
count int
}
type examplePluginEvent int
const (
exampleEventValidate examplePluginEvent = 0
exampleEventRegister examplePluginEvent = 1
exampleEventDeRegister examplePluginEvent = 2
exampleEventError examplePluginEvent = 3
)
// NewExampleHandler provide a example handler
func NewExampleHandler() *exampleHandler {
func NewExampleHandler(supportedVersions []string) *exampleHandler {
return &exampleHandler{
chanForHandlerAckErrors: make(chan error),
registeredPlugins: make(map[string]struct{}),
SupportedVersions: supportedVersions,
ExpectedNames: make(map[string]int),
eventChans: make(map[string]chan examplePluginEvent),
}
}
func (h *exampleHandler) Cleanup() error {
h.mutex.Lock()
defer h.mutex.Unlock()
h.registeredPlugins = make(map[string]struct{})
return nil
}
func (p *exampleHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
p.SendEvent(pluginName, exampleEventValidate)
func (h *exampleHandler) Handler(pluginName string, endpoint string, versions []string, sockPath string) (chan bool, error) {
n, ok := p.DecreasePluginCount(pluginName)
if !ok && n > 0 {
return fmt.Errorf("pluginName('%s') wasn't expected (count is %d)", pluginName, n)
}
// check for supported versions
if !reflect.DeepEqual([]string{"v1beta1", "v1beta2"}, versions) {
return nil, fmt.Errorf("not the supported versions: %s", versions)
if !reflect.DeepEqual(versions, p.SupportedVersions) {
return fmt.Errorf("versions('%v') != supported versions('%v')", versions, p.SupportedVersions)
}
// this handler expects non-empty endpoint as an example
if len(endpoint) == 0 {
return nil, errors.New("expecting non empty endpoint")
return errors.New("expecting non empty endpoint")
}
_, conn, err := dial(sockPath)
return nil
}
func (p *exampleHandler) RegisterPlugin(pluginName, endpoint string) error {
p.SendEvent(pluginName, exampleEventRegister)
// Verifies the grpcServer is ready to serve services.
_, conn, err := dial(endpoint, time.Second)
if err != nil {
return nil, err
return fmt.Errorf("Failed dialing endpoint (%s): %v", endpoint, err)
}
defer conn.Close()
......@@ -73,33 +94,54 @@ func (h *exampleHandler) Handler(pluginName string, endpoint string, versions []
v1beta2Client := v1beta2.NewExampleClient(conn)
// Tests v1beta1 GetExampleInfo
if _, err = v1beta1Client.GetExampleInfo(context.Background(), &v1beta1.ExampleRequest{}); err != nil {
return nil, err
_, err = v1beta1Client.GetExampleInfo(context.Background(), &v1beta1.ExampleRequest{})
if err != nil {
return fmt.Errorf("Failed GetExampleInfo for v1beta2Client(%s): %v", endpoint, err)
}
// Tests v1beta1 GetExampleInfo
_, err = v1beta2Client.GetExampleInfo(context.Background(), &v1beta2.ExampleRequest{})
if err != nil {
return fmt.Errorf("Failed GetExampleInfo for v1beta2Client(%s): %v", endpoint, err)
}
// Tests v1beta2 GetExampleInfo
if _, err = v1beta2Client.GetExampleInfo(context.Background(), &v1beta2.ExampleRequest{}); err != nil {
return nil, err
return nil
}
func (p *exampleHandler) DeRegisterPlugin(pluginName string) {
p.SendEvent(pluginName, exampleEventDeRegister)
}
func (p *exampleHandler) EventChan(pluginName string) chan examplePluginEvent {
return p.eventChans[pluginName]
}
func (p *exampleHandler) SendEvent(pluginName string, event examplePluginEvent) {
glog.V(2).Infof("Sending %v for plugin %s over chan %v", event, pluginName, p.eventChans[pluginName])
p.eventChans[pluginName] <- event
}
func (p *exampleHandler) AddPluginName(pluginName string) {
p.m.Lock()
defer p.m.Unlock()
v, ok := p.ExpectedNames[pluginName]
if !ok {
p.eventChans[pluginName] = make(chan examplePluginEvent)
v = 1
}
// handle registered plugin
h.mutex.Lock()
if _, exist := h.registeredPlugins[pluginName]; exist {
h.mutex.Unlock()
return nil, fmt.Errorf("plugin %s already registered", pluginName)
p.ExpectedNames[pluginName] = v
}
func (p *exampleHandler) DecreasePluginCount(pluginName string) (old int, ok bool) {
p.m.Lock()
defer p.m.Unlock()
v, ok := p.ExpectedNames[pluginName]
if !ok {
v = -1
}
h.registeredPlugins[pluginName] = struct{}{}
h.mutex.Unlock()
chanForAckOfNotification := make(chan bool)
go func() {
select {
case <-chanForAckOfNotification:
// TODO: handle the negative scenario
close(chanForAckOfNotification)
case <-time.After(time.Second):
h.chanForHandlerAckErrors <- errors.New("Timed out while waiting for notification ack")
}
}()
return chanForAckOfNotification, nil
return v, ok
}
......@@ -18,7 +18,9 @@ package pluginwatcher
import (
"errors"
"fmt"
"net"
"os"
"sync"
"time"
......@@ -39,6 +41,7 @@ type examplePlugin struct {
endpoint string // for testing
pluginName string
pluginType string
versions []string
}
type pluginServiceV1Beta1 struct {
......@@ -73,12 +76,13 @@ func NewExamplePlugin() *examplePlugin {
}
// NewTestExamplePlugin returns an initialized examplePlugin instance for testing
func NewTestExamplePlugin(pluginName string, pluginType string, endpoint string) *examplePlugin {
func NewTestExamplePlugin(pluginName string, pluginType string, endpoint string, advertisedVersions ...string) *examplePlugin {
return &examplePlugin{
pluginName: pluginName,
pluginType: pluginType,
registrationStatus: make(chan registerapi.RegistrationStatus),
endpoint: endpoint,
versions: advertisedVersions,
registrationStatus: make(chan registerapi.RegistrationStatus),
}
}
......@@ -88,36 +92,48 @@ func (e *examplePlugin) GetInfo(ctx context.Context, req *registerapi.InfoReques
Type: e.pluginType,
Name: e.pluginName,
Endpoint: e.endpoint,
SupportedVersions: []string{"v1beta1", "v1beta2"},
SupportedVersions: e.versions,
}, nil
}
func (e *examplePlugin) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) {
glog.Errorf("Registration is: %v\n", status)
if e.registrationStatus != nil {
e.registrationStatus <- *status
}
if !status.PluginRegistered {
glog.Errorf("Registration failed: %s\n", status.Error)
}
return &registerapi.RegistrationStatusResponse{}, nil
}
// Serve starts example plugin grpc server
func (e *examplePlugin) Serve(socketPath string) error {
glog.Infof("starting example server at: %s\n", socketPath)
lis, err := net.Listen("unix", socketPath)
// Serve starts a pluginwatcher server and one or more of the plugin services
func (e *examplePlugin) Serve(services ...string) error {
glog.Infof("starting example server at: %s\n", e.endpoint)
lis, err := net.Listen("unix", e.endpoint)
if err != nil {
return err
}
glog.Infof("example server started at: %s\n", socketPath)
glog.Infof("example server started at: %s\n", e.endpoint)
e.grpcServer = grpc.NewServer()
// Registers kubelet plugin watcher api.
registerapi.RegisterRegistrationServer(e.grpcServer, e)
// Registers services for both v1beta1 and v1beta2 versions.
v1beta1 := &pluginServiceV1Beta1{server: e}
v1beta1.RegisterService()
v1beta2 := &pluginServiceV1Beta2{server: e}
v1beta2.RegisterService()
for _, service := range services {
switch service {
case "v1beta1":
v1beta1 := &pluginServiceV1Beta1{server: e}
v1beta1.RegisterService()
break
case "v1beta2":
v1beta2 := &pluginServiceV1Beta2{server: e}
v1beta2.RegisterService()
break
default:
return fmt.Errorf("Unsupported service: '%s'", service)
}
}
// Starts service
e.wg.Add(1)
......@@ -128,22 +144,30 @@ func (e *examplePlugin) Serve(socketPath string) error {
glog.Errorf("example server stopped serving: %v", err)
}
}()
return nil
}
func (e *examplePlugin) Stop() error {
glog.Infof("Stopping example server\n")
glog.Infof("Stopping example server at: %s\n", e.endpoint)
e.grpcServer.Stop()
c := make(chan struct{})
go func() {
defer close(c)
e.wg.Wait()
}()
select {
case <-c:
return nil
break
case <-time.After(time.Second):
glog.Errorf("Timed out on waiting for stop completion")
return errors.New("Timed out on waiting for stop completion")
}
if err := os.Remove(e.endpoint); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pluginwatcher
// PluginHandler is an interface a client of the pluginwatcher API needs to implement in
// order to consume plugins
// The PluginHandler follows the simple following state machine:
//
// +--------------------------------------+
// | ReRegistration |
// | Socket created with same plugin name |
// | |
// | |
// Socket Created v + Socket Deleted
// +------------------> Validate +---------------------------> Register +------------------> DeRegister
// + + +
// | | |
// | Error | Error |
// | | |
// v v v
// Out Out Out
//
// The pluginwatcher module follows strictly and sequentially this state machine for each *plugin name*.
// e.g: If you are Registering a plugin foo, you cannot get a DeRegister call for plugin foo
// until the Register("foo") call returns. Nor will you get a Validate("foo", "Different endpoint", ...)
// call until the Register("foo") call returns.
//
// ReRegistration: Socket created with same plugin name, usually for a plugin update
// e.g: plugin with name foo registers at foo.com/foo-1.9.7 later a plugin with name foo
// registers at foo.com/foo-1.9.9
//
// DeRegistration: When ReRegistration happens only the deletion of the new socket will trigger a DeRegister call
type PluginHandler interface {
// Validate returns an error if the information provided by
// the potential plugin is erroneous (unsupported version, ...)
ValidatePlugin(pluginName string, endpoint string, versions []string) error
// RegisterPlugin is called so that the plugin can be register by any
// plugin consumer
// Error encountered here can still be Notified to the plugin.
RegisterPlugin(pluginName, endpoint string) error
// DeRegister is called once the pluginwatcher observes that the socket has
// been deleted.
DeRegisterPlugin(pluginName string)
}
......@@ -28,6 +28,7 @@ import (
"context"
"github.com/golang/glog"
api "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
......@@ -89,6 +90,10 @@ type csiDriversStore struct {
sync.RWMutex
}
// RegistrationHandler is the handler which is fed to the pluginwatcher API.
type RegistrationHandler struct {
}
// TODO (verult) consider using a struct instead of global variables
// csiDrivers map keep track of all registered CSI drivers on the node and their
// corresponding sockets
......@@ -96,21 +101,28 @@ var csiDrivers csiDriversStore
var nodeUpdater nodeupdater.Interface
// RegistrationCallback is called by kubelet's plugin watcher upon detection
// PluginHandler is the plugin registration handler interface passed to the
// pluginwatcher module in kubelet
var PluginHandler = &RegistrationHandler{}
// ValidatePlugin is called by kubelet's plugin watcher upon detection
// of a new registration socket opened by CSI Driver registrar side car.
func RegistrationCallback(pluginName string, endpoint string, versions []string, socketPath string) (chan bool, error) {
func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
glog.Infof(log("Trying to register a new plugin with name: %s endpoint: %s versions: %s",
pluginName, endpoint, strings.Join(versions, ",")))
glog.Infof(log("Callback from kubelet with plugin name: %s endpoint: %s versions: %s socket path: %s",
pluginName, endpoint, strings.Join(versions, ","), socketPath))
return nil
}
if endpoint == "" {
endpoint = socketPath
}
// RegisterPlugin is called when a plugin can be registered
func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string) error {
glog.Infof(log("Register new plugin with name: %s at endpoint: %s", pluginName, endpoint))
// Storing endpoint of newly registered CSI driver into the map, where CSI driver name will be the key
// all other CSI components will be able to get the actual socket of CSI drivers by its name.
csiDrivers.Lock()
defer csiDrivers.Unlock()
csiDrivers.driversMap[pluginName] = csiDriver{driverName: pluginName, driverEndpoint: endpoint}
// Get node info from the driver.
......@@ -118,22 +130,27 @@ func RegistrationCallback(pluginName string, endpoint string, versions []string,
// TODO (verult) retry with exponential backoff, possibly added in csi client library.
ctx, cancel := context.WithTimeout(context.Background(), csiTimeout)
defer cancel()
driverNodeID, maxVolumePerNode, _, err := csi.NodeGetInfo(ctx)
if err != nil {
return nil, fmt.Errorf("error during CSI NodeGetInfo() call: %v", err)
return fmt.Errorf("error during CSI NodeGetInfo() call: %v", err)
}
// Calling nodeLabelManager to update annotations and labels for newly registered CSI driver
err = nodeUpdater.AddLabelsAndLimits(pluginName, driverNodeID, maxVolumePerNode)
if err != nil {
// Unregister the driver and return error
csiDrivers.Lock()
defer csiDrivers.Unlock()
delete(csiDrivers.driversMap, pluginName)
return nil, err
return fmt.Errorf("error while adding CSI labels: %v", err)
}
return nil, nil
return nil
}
// DeRegisterPlugin is called when a plugin removed it's socket, signaling
// it is no longer available
// TODO: Handle DeRegistration
func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) {
}
func (p *csiPlugin) Init(host volume.VolumeHost) error {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment