Merge pull request #55185 from Cynerva/gkk/scheduler-extra-args

cluster/juju/layers/kubernetes-master/config.yaml

@@ -46,6 +46,24 @@ options:
         runtime-config=batch/v2alpha1=true profiling=true
       will result in kube-apiserver being run with the following options:
         --runtime-config=batch/v2alpha1=true --profiling=true
+  controller-manager-extra-args:
+    type: string
+    default: ""
+    description: |
+      Space separated list of flags and key=value pairs that will be passed as arguments to
+      kube-controller-manager. For example a value like this:
+        runtime-config=batch/v2alpha1=true profiling=true
+      will result in kube-controller-manager being run with the following options:
+        --runtime-config=batch/v2alpha1=true --profiling=true
+  scheduler-extra-args:
+    type: string
+    default: ""
+    description: |
+      Space separated list of flags and key=value pairs that will be passed as arguments to
+      kube-scheduler. For example a value like this:
+        runtime-config=batch/v2alpha1=true profiling=true
+      will result in kube-scheduler being run with the following options:
+        --runtime-config=batch/v2alpha1=true --profiling=true
   authorization-mode:
     type: string
     default: "AlwaysAllow"

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -772,6 +772,18 @@ def on_config_api_extra_args_change():
     configure_apiserver()
 
 
+@when('config.changed.controller-manager-extra-args')
+@when('kubernetes-master.components.started')
+def on_config_controller_manager_extra_args_change():
+    configure_controller_manager()
+
+
+@when('config.changed.scheduler-extra-args')
+@when('kubernetes-master.components.started')
+def on_config_scheduler_extra_args_change():
+    configure_scheduler()
+
+
 @when('kube-control.gpu.available')
 @when('kubernetes-master.components.started')
 @when_not('kubernetes-master.gpu.enabled')
@@ -975,27 +987,27 @@ def handle_etcd_relation(reldata):
     api_opts.add('etcd-servers', connection_string, strict=True)
 
 
-def get_config_args():
+def get_config_args(key):
     db = unitdata.kv()
-    old_config_args = db.get('api-extra-args', [])
+    old_config_args = db.get(key, [])
     # We have to convert them to tuples becuase we use sets
     old_config_args = [tuple(i) for i in old_config_args]
     new_config_args = []
     new_config_arg_names = []
-    for arg in hookenv.config().get('api-extra-args', '').split():
+    for arg in hookenv.config().get(key, '').split():
         new_config_arg_names.append(arg.split('=', 1)[0])
         if len(arg.split('=', 1)) == 1:  # handle flags ie. --profiling
             new_config_args.append(tuple([arg, 'true']))
         else:
             new_config_args.append(tuple(arg.split('=', 1)))
 
-    hookenv.log('Handling "api-extra-args" option.')
+    hookenv.log('Handling "%s" option.' % key)
     hookenv.log('Old arguments: {}'.format(old_config_args))
     hookenv.log('New arguments: {}'.format(new_config_args))
     if set(new_config_args) == set(old_config_args):
         return (new_config_args, [])
     # Store new args
-    db.set('api-extra-args', new_config_args)
+    db.set(key, new_config_args)
     to_add = set(new_config_args)
     to_remove = set(old_config_args) - set(new_config_args)
     # Extract option names only
@@ -1003,23 +1015,40 @@ def get_config_args():
     return (to_add, to_remove)
 
 
-def configure_apiserver():
-    # TODO: investigate if it's possible to use config file to store args
-    # https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/315
+def configure_kubernetes_service(service, base_args, extra_args_key):
     # Handle api-extra-args config option
-    to_add, to_remove = get_config_args()
+    to_add, to_remove = get_config_args(extra_args_key)
 
-    api_opts = FlagManager('kube-apiserver')
+    flag_manager = FlagManager(service)
 
     # Remove arguments that are no longer provided as config option
     # this allows them to be reverted to charm defaults
     for arg in to_remove:
         hookenv.log('Removing option: {}'.format(arg))
-        api_opts.destroy(arg)
-        # We need to "unset" options by settig their value to "null" string
-        cmd = ['snap', 'set', 'kube-apiserver', '{}=null'.format(arg)]
+        flag_manager.destroy(arg)
+        # We need to "unset" options by setting their value to "null" string
+        cmd = ['snap', 'set', service, '{}=null'.format(arg)]
         check_call(cmd)
 
+    # Add base arguments
+    for k, v in base_args.items():
+        flag_manager.add(k, v, strict=True)
+
+    # Add operator-provided arguments, this allows operators
+    # to override defaults
+    for arg in to_add:
+        hookenv.log('Adding option: {} {}'.format(arg[0], arg[1]))
+        # Make sure old value is gone
+        flag_manager.destroy(arg[0])
+        flag_manager.add(arg[0], arg[1], strict=True)
+
+    cmd = ['snap', 'set', service] + flag_manager.to_s().split(' ')
+    check_call(cmd)
+
+
+def configure_apiserver():
+    api_opts = {}
+
     # Get the tls paths from the layer data.
     layer_options = layer.options('tls-client')
     ca_cert_path = layer_options.get('ca_certificate_path')
@@ -1029,25 +1058,25 @@ def configure_apiserver():
     server_key_path = layer_options.get('server_key_path')
 
     if is_privileged():
-        api_opts.add('allow-privileged', 'true', strict=True)
+        api_opts['allow-privileged'] = 'true'
         set_state('kubernetes-master.privileged')
     else:
-        api_opts.add('allow-privileged', 'false', strict=True)
+        api_opts['allow-privileged'] = 'false'
         remove_state('kubernetes-master.privileged')
 
     # Handle static options for now
-    api_opts.add('service-cluster-ip-range', service_cidr())
-    api_opts.add('min-request-timeout', '300')
-    api_opts.add('v', '4')
-    api_opts.add('tls-cert-file', server_cert_path)
-    api_opts.add('tls-private-key-file', server_key_path)
-    api_opts.add('kubelet-certificate-authority', ca_cert_path)
-    api_opts.add('kubelet-client-certificate', client_cert_path)
-    api_opts.add('kubelet-client-key', client_key_path)
-    api_opts.add('logtostderr', 'true')
-    api_opts.add('insecure-bind-address', '127.0.0.1')
-    api_opts.add('insecure-port', '8080')
-    api_opts.add('storage-backend', 'etcd2')  # FIXME: add etcd3 support
+    api_opts['service-cluster-ip-range'] = service_cidr()
+    api_opts['min-request-timeout'] = '300'
+    api_opts['v'] = '4'
+    api_opts['tls-cert-file'] = server_cert_path
+    api_opts['tls-private-key-file'] = server_key_path
+    api_opts['kubelet-certificate-authority'] = ca_cert_path
+    api_opts['kubelet-client-certificate'] = client_cert_path
+    api_opts['kubelet-client-key'] = client_key_path
+    api_opts['logtostderr'] = 'true'
+    api_opts['insecure-bind-address'] = '127.0.0.1'
+    api_opts['insecure-port'] = '8080'
+    api_opts['storage-backend'] = 'etcd2'  # FIXME: add etcd3 support
 
     admission_control = [
         'Initializers',
@@ -1062,7 +1091,7 @@ def configure_apiserver():
     if 'Node' in auth_mode:
         admission_control.append('NodeRestriction')
 
-    api_opts.add('authorization-mode', auth_mode, strict=True)
+    api_opts['authorization-mode'] = auth_mode
 
     if get_version('kube-apiserver') < (1, 6):
         hookenv.log('Removing DefaultTolerationSeconds from admission-control')
@@ -1070,52 +1099,43 @@ def configure_apiserver():
     if get_version('kube-apiserver') < (1, 7):
         hookenv.log('Removing Initializers from admission-control')
         admission_control.remove('Initializers')
-    api_opts.add('admission-control', ','.join(admission_control), strict=True)
+    api_opts['admission-control'] = ','.join(admission_control)
 
-    # Add operator-provided arguments, this allows operators
-    # to override defaults
-    for arg in to_add:
-        hookenv.log('Adding option: {} {}'.format(arg[0], arg[1]))
-        # Make sure old value is gone
-        api_opts.destroy(arg[0])
-        api_opts.add(arg[0], arg[1])
+    configure_kubernetes_service('kube-apiserver', api_opts, 'api-extra-args')
 
-    cmd = ['snap', 'set', 'kube-apiserver'] + api_opts.to_s().split(' ')
-    check_call(cmd)
     set_state('kube-apiserver.do-restart')
 
 
 def configure_controller_manager():
-    controller_opts = FlagManager('kube-controller-manager')
+    controller_opts = {}
 
     # Get the tls paths from the layer data.
     layer_options = layer.options('tls-client')
     ca_cert_path = layer_options.get('ca_certificate_path')
 
     # Default to 3 minute resync. TODO: Make this configureable?
-    controller_opts.add('min-resync-period', '3m')
-    controller_opts.add('v', '2')
-    controller_opts.add('root-ca-file', ca_cert_path)
-    controller_opts.add('logtostderr', 'true')
-    controller_opts.add('master', 'http://127.0.0.1:8080')
-
-    cmd = (
-        ['snap', 'set', 'kube-controller-manager'] +
-        controller_opts.to_s().split(' ')
-    )
-    check_call(cmd)
+    controller_opts['min-resync-period'] = '3m'
+    controller_opts['v'] = '2'
+    controller_opts['root-ca-file'] = ca_cert_path
+    controller_opts['logtostderr'] = 'true'
+    controller_opts['master'] = 'http://127.0.0.1:8080'
+
+    configure_kubernetes_service('kube-controller-manager', controller_opts,
+                                 'controller-manager-extra-args')
+
     set_state('kube-controller-manager.do-restart')
 
 
 def configure_scheduler():
-    scheduler_opts = FlagManager('kube-scheduler')
+    scheduler_opts = {}
 
-    scheduler_opts.add('v', '2')
-    scheduler_opts.add('logtostderr', 'true')
-    scheduler_opts.add('master', 'http://127.0.0.1:8080')
+    scheduler_opts['v'] = '2'
+    scheduler_opts['logtostderr'] = 'true'
+    scheduler_opts['master'] = 'http://127.0.0.1:8080'
+
+    configure_kubernetes_service('kube-scheduler', scheduler_opts,
+                                 'scheduler-extra-args')
 
-    cmd = ['snap', 'set', 'kube-scheduler'] + scheduler_opts.to_s().split(' ')
-    check_call(cmd)
     set_state('kube-scheduler.do-restart')