Merge pull request #44500 from Cynerva/gkk/cdk-1.6-support

cluster/juju/.gitignore

+builds
+deps

cluster/juju/layers/kubeapi-load-balancer/layer.yaml

 repo: https://github.com/kubernetes/kubernetes.git
 includes:
+  - 'layer:metrics'
   - 'layer:nagios'
   - 'layer:nginx'
   - 'layer:tls-client'

cluster/juju/layers/kubeapi-load-balancer/metrics.yaml

+metrics:
+  juju-units: {}

cluster/juju/layers/kubernetes-e2e/README.md

@@ -73,7 +73,7 @@ a deployed cluster. The following example will skip the `Flaky`, `Slow`, and
 `Feature` labeled tests:
 
 ```shell
-juju run-action kubernetes-e2e/0 skip='\[(Flaky|Slow|Feature:.*)\]'
+juju run-action kubernetes-e2e/0 test skip='\[(Flaky|Slow|Feature:.*)\]'
 ```
 
 > Note: the escaping of the regex due to how bash handles brackets.

cluster/juju/layers/kubernetes-e2e/actions/test

@@ -45,3 +45,7 @@ tar -czf $ACTION_LOG_TGZ ${JUJU_ACTION_UUID}.log
 
 action-set log="$ACTION_LOG_TGZ"
 action-set junit="$ACTION_JUNIT_TGZ"
+
+if tail ${JUJU_ACTION_UUID}.log | grep -q "Test Suite Failed"; then
+  action-fail "Failure detected in the logs"
+fi

cluster/juju/layers/kubernetes-master/actions/create-rbd-pv

@@ -28,6 +28,9 @@ import os
 import sys
 
 
+os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin')
+
+
 def main():
     ''' Control logic to enlist Ceph RBD volumes as PersistentVolumes in
     Kubernetes. This will invoke the validation steps, and only execute if

cluster/juju/layers/kubernetes-master/config.yaml

@@ -21,3 +21,8 @@ options:
       privileged mode. If "auto", kube-apiserver will not run in privileged
       mode by default, but will switch to privileged mode if gpu hardware is
       detected on a worker node.
+  channel:
+    type: string
+    default: "stable"
+    description: |
+      Snap channel to install Kubernetes master services from

cluster/juju/layers/kubernetes-master/debug-scripts/kubectl

 #!/bin/sh
 set -ux
 
+export PATH=$PATH:/snap/bin
+
 alias kubectl="kubectl --kubeconfig=/home/ubuntu/config"
 
 kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info

cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services

@@ -2,12 +2,8 @@
 set -ux
 
 for service in kube-apiserver kube-controller-manager kube-scheduler; do
-  systemctl status $service > $DEBUG_SCRIPT_DIR/$service-systemctl-status
-  journalctl -u $service > $DEBUG_SCRIPT_DIR/$service-journal
+  systemctl status snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-systemctl-status
+  journalctl -u snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-journal
 done
 
-mkdir -p $DEBUG_SCRIPT_DIR/etc-default
-cp -v /etc/default/kube* $DEBUG_SCRIPT_DIR/etc-default
-
-mkdir -p $DEBUG_SCRIPT_DIR/lib-systemd-system
-cp -v /lib/systemd/system/kube* $DEBUG_SCRIPT_DIR/lib-systemd-system
+# FIXME: grab snap config or something

cluster/juju/layers/kubernetes-master/layer.yaml

 repo: https://github.com/kubernetes/kubernetes.git
 includes:
   - 'layer:basic'
+  - 'layer:snap'
   - 'layer:tls-client'
   - 'layer:leadership'
   - 'layer:debug'
+  - 'layer:metrics'
   - 'layer:nagios'
   - 'interface:ceph-admin'
   - 'interface:etcd'
@@ -17,10 +19,8 @@ options:
     packages:
       - socat
   tls-client:
-    ca_certificate_path: '/srv/kubernetes/ca.crt'
-    server_certificate_path: '/srv/kubernetes/server.crt'
-    server_key_path: '/srv/kubernetes/server.key'
-    client_certificate_path: '/srv/kubernetes/client.crt'
-    client_key_path: '/srv/kubernetes/client.key'
-tactics:
-  - 'tactics.update_addons.UpdateAddonsTactic'
+    ca_certificate_path: '/root/cdk/ca.crt'
+    server_certificate_path: '/root/cdk/server.crt'
+    server_key_path: '/root/cdk/server.key'
+    client_certificate_path: '/root/cdk/client.crt'
+    client_key_path: '/root/cdk/client.key'

cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py

@@ -17,10 +17,6 @@
 import re
 import subprocess
 
-from charmhelpers.core import unitdata
-
-BIN_VERSIONS = 'bin_versions'
-
 
 def get_version(bin_name):
     """Get the version of an installed Kubernetes binary.
@@ -34,31 +30,6 @@ def get_version(bin_name):
         (1, 6, 0)
 
     """
-    db = unitdata.kv()
-    bin_versions = db.get(BIN_VERSIONS, {})
-
-    cached_version = bin_versions.get(bin_name)
-    if cached_version:
-        return tuple(cached_version)
-
-    version = _get_bin_version(bin_name)
-    bin_versions[bin_name] = list(version)
-    db.set(BIN_VERSIONS, bin_versions)
-    return version
-
-
-def reset_versions():
-    """Reset the cache of bin versions.
-
-    """
-    db = unitdata.kv()
-    db.unset(BIN_VERSIONS)
-
-
-def _get_bin_version(bin_name):
-    """Get a binary version by calling it with --version and parsing output.
-
-    """
     cmd = '{} --version'.format(bin_name).split()
     version_string = subprocess.check_output(cmd).decode('utf-8')
     return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])

cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/flagmanager.py

@@ -118,6 +118,13 @@ class FlagManager:
         """
         return self.data.get(key, default)
 
+    def destroy_all(self):
+        '''
+        Destructively removes all data from the FlagManager.
+        '''
+        self.data.clear()
+        self.__save()
+
     def to_s(self):
         '''
         Render the flags to a single string, prepared for the Docker

cluster/juju/layers/kubernetes-master/metadata.yaml

@@ -37,7 +37,23 @@ requires:
   ceph-storage:
     interface: ceph-admin
 resources:
-  kubernetes:
+  kubectl:
     type: file
-    filename: kubernetes.tar.gz
-    description: "A tarball packaged release of the kubernetes bins."
+    filename: kubectl.snap
+    description: kubectl snap
+  kube-apiserver:
+    type: file
+    filename: kube-apiserver.snap
+    description: kube-apiserver snap
+  kube-controller-manager:
+    type: file
+    filename: kube-controller-manager.snap
+    description: kube-controller-manager snap
+  kube-scheduler:
+    type: file
+    filename: kube-scheduler.snap
+    description: kube-scheduler snap
+  cdk-addons:
+    type: file
+    filename: cdk-addons.snap
+    description: CDK addons snap

cluster/juju/layers/kubernetes-master/metrics.yaml

+metrics:
+  juju-units: {}
+  pods:
+    type: gauge
+    description: number of pods
+    command: /snap/bin/kubectl get po --all-namespaces | tail -n+2 | wc -l
+  services:
+    type: gauge
+    description: number of services
+    command: /snap/bin/kubectl get svc --all-namespaces | tail -n+2 | wc -l
+  replicasets:
+    type: gauge
+    description: number of replicasets
+    command: /snap/bin/kubectl get rs --all-namespaces | tail -n+2 | wc -l
+  replicationcontrollers:
+    type: gauge
+    description: number of replicationcontrollers
+    command: /snap/bin/kubectl get rc --all-namespaces | tail -n+2 | wc -l
+  nodes:
+    type: gauge
+    description: number of kubernetes nodes
+    command: /snap/bin/kubectl get nodes | tail -n+2 | wc -l
+  persistentvolume:
+    type: gauge
+    description: number of pv
+    command: /snap/bin/kubectl get pv --all-namespaces | tail -n+2 | wc -l
+  persistentvolumeclaims:
+    type: gauge
+    description: number of claims
+    command: /snap/bin/kubectl get pvc --all-namespaces | tail -n+2 | wc -l
+  serviceaccounts:
+    type: gauge
+    description: number of sa
+    command: /snap/bin/kubectl get sa --all-namespaces | tail -n+2 | wc -l

cluster/juju/layers/kubernetes-master/tactics/__init__.py

-#!/usr/bin/env python
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-

cluster/juju/layers/kubernetes-master/tactics/update_addons.py

-#!/usr/bin/env python
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import os
-import shutil
-import subprocess
-import tempfile
-import logging
-from contextlib import contextmanager
-
-import charmtools.utils
-from charmtools.build.tactics import Tactic
-
-
-description = """
-Update addon manifests for the charm.
-
-This will clone the kubernetes repo and place the addons in
-<charm>/templates/addons.
-
-Can be run with no arguments and from any folder.
-"""
-
-log = logging.getLogger(__name__)
-
-
-def clean_addon_dir(addon_dir):
-    """ Remove and recreate the addons folder """
-    log.debug("Cleaning " + addon_dir)
-    shutil.rmtree(addon_dir, ignore_errors=True)
-    os.makedirs(addon_dir)
-
-
-def run_with_logging(command):
-    """ Run a command with controlled logging """
-    log.debug("Running: %s" % command)
-    process = subprocess.Popen(command, stderr=subprocess.PIPE)
-    stderr = process.communicate()[1].rstrip()
-    process.wait()
-    if process.returncode != 0:
-        log.error(stderr)
-        raise Exception("%s: exit code %d" % (command, process.returncode))
-    log.debug(stderr)
-
-
-@contextmanager
-def kubernetes_repo():
-    """ Yield a kubernetes repo to copy addons from.
-
-    If KUBE_VERSION is set, this will clone the local repo and checkout the
-    corresponding branch. Otherwise, the local branch will be used. """
-    repo = os.path.abspath("../../../..")
-    if "KUBE_VERSION" in os.environ:
-        branch = os.environ["KUBE_VERSION"]
-        log.info("Cloning %s with branch %s" % (repo, branch))
-        path = tempfile.mkdtemp(prefix="kubernetes")
-        try:
-            cmd = ["git", "clone", repo, path, "-b", branch]
-            run_with_logging(cmd)
-            yield path
-        finally:
-            shutil.rmtree(path)
-    else:
-        log.info("Using local repo " + repo)
-        yield repo
-
-
-def add_addon(repo, source, dest):
-    """ Add an addon manifest from the given repo and source.
-
-    Any occurrences of 'amd64' are replaced with '{{ arch }}' so the charm can
-    fill it in during deployment. """
-    source = os.path.join(repo, "cluster/addons", source)
-    if os.path.isdir(dest):
-        dest = os.path.join(dest, os.path.basename(source))
-    log.debug("Copying: %s -> %s" % (source, dest))
-    with open(source, "r") as f:
-        content = f.read()
-    content = content.replace("amd64", "{{ arch }}")
-    with open(dest, "w") as f:
-        f.write(content)
-
-
-def update_addons(dest):
-    """ Update addons. This will clean the addons folder and add new manifests
-    from upstream. """
-    with kubernetes_repo() as repo:
-        log.info("Copying addons to charm")
-        clean_addon_dir(dest)
-        add_addon(repo, "dashboard/dashboard-controller.yaml", dest)
-        add_addon(repo, "dashboard/dashboard-service.yaml", dest)
-        try:
-            add_addon(repo, "dns/kubedns-sa.yaml",
-                      dest + "/kubedns-sa.yaml")
-            add_addon(repo, "dns/kubedns-cm.yaml",
-                      dest + "/kubedns-cm.yaml")
-            add_addon(repo, "dns/kubedns-controller.yaml.in",
-                      dest + "/kubedns-controller.yaml")
-            add_addon(repo, "dns/kubedns-svc.yaml.in",
-                      dest + "/kubedns-svc.yaml")
-        except IOError as e:
-            # fall back to the older filenames
-            log.debug(e)
-            add_addon(repo, "dns/skydns-rc.yaml.in",
-                      dest + "/kubedns-controller.yaml")
-            add_addon(repo, "dns/skydns-svc.yaml.in",
-                      dest + "/kubedns-svc.yaml")
-        influxdb = "cluster-monitoring/influxdb"
-        add_addon(repo, influxdb + "/grafana-service.yaml", dest)
-        add_addon(repo, influxdb + "/heapster-controller.yaml", dest)
-        add_addon(repo, influxdb + "/heapster-service.yaml", dest)
-        add_addon(repo, influxdb + "/influxdb-grafana-controller.yaml", dest)
-        add_addon(repo, influxdb + "/influxdb-service.yaml", dest)
-
-# Entry points
-
-
-class UpdateAddonsTactic(Tactic):
-    """ This tactic is used by charm-tools to dynamically populate the
-    template/addons folder at `charm build` time. """
-
-    @classmethod
-    def trigger(cls, entity, target=None, layer=None, next_config=None):
-        """ Determines which files the tactic should apply to. We only want
-        this tactic to trigger once, so let's use the templates/ folder
-        """
-        relpath = entity.relpath(layer.directory) if layer else entity
-        return relpath == "templates"
-
-    @property
-    def dest(self):
-        """ The destination we are writing to. This isn't a Tactic thing,
-        it's just a helper for UpdateAddonsTactic """
-        return self.target / "templates" / "addons"
-
-    def __call__(self):
-        """ When the tactic is called, update addons and put them directly in
-        our build destination """
-        update_addons(self.dest)
-
-    def sign(self):
-        """ Return signatures for the charm build manifest. We need to do this
-        because the addon template files were added dynamically """
-        sigs = {}
-        for file in os.listdir(self.dest):
-            path = self.dest / file
-            relpath = path.relpath(self.target.directory)
-            sigs[relpath] = (
-                self.current.url,
-                "dynamic",
-                charmtools.utils.sign(path)
-            )
-        return sigs
-
-
-def parse_args():
-    """ Parse args. This is solely done for the usage output with -h """
-    parser = argparse.ArgumentParser(description=description)
-    parser.parse_args()
-
-
-def main():
-    """ Update addons into the layer's templates/addons folder """
-    parse_args()
-    os.chdir(os.path.join(os.path.dirname(__file__), ".."))
-    dest = "templates/addons"
-    update_addons(dest)
-
-
-if __name__ == "__main__":
-    main()

cluster/juju/layers/kubernetes-master/templates/kube-apiserver.defaults

-###
-# kubernetes system config
-#
-# The following values are used to configure the kube-apiserver
-#
-
-# The address on the local server to listen to.
-KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
-
-# The port on the local server to listen on.
-KUBE_API_PORT="--insecure-port=8080"
-
-# default admission control policies
-KUBE_ADMISSION_CONTROL=""
-
-# Add your own!
-KUBE_API_ARGS="{{ kube_apiserver_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-apiserver.service

-[Unit]
-Description=Kubernetes API Server
-Documentation=http://kubernetes.io/docs/admin/kube-apiserver/
-After=network.target
-
-[Service]
-EnvironmentFile=-/etc/default/kube-defaults
-EnvironmentFile=-/etc/default/kube-apiserver
-ExecStart=/usr/local/bin/kube-apiserver \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_API_ADDRESS \
-	    $KUBE_API_PORT \
-	    $KUBE_ALLOW_PRIV \
-	    $KUBE_ADMISSION_CONTROL \
-	    $KUBE_API_ARGS
-Restart=on-failure
-Type=notify
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cluster/juju/layers/kubernetes-master/templates/kube-controller-manager.defaults

-
-###
-# The following values are used to configure the kubernetes controller-manager
-
-# defaults from config and apiserver should be adequate
-
-# Add your own!
-KUBE_CONTROLLER_MANAGER_ARGS="{{ kube_controller_manager_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-controller-manager.service

-
-[Unit]
-Description=Kubernetes Controller Manager
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-
-[Service]
-EnvironmentFile=-/etc/default/kube-defaults
-EnvironmentFile=-/etc/default/kube-controller-manager
-ExecStart=/usr/local/bin/kube-controller-manager \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_CONTROLLER_MANAGER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cluster/juju/layers/kubernetes-master/templates/kube-defaults.defaults

-###
-# kubernetes system config
-#
-# The following values are used to configure various aspects of all
-# kubernetes services, including
-#
-#   kube-apiserver.service
-#   kube-controller-manager.service
-#   kube-scheduler.service
-#   kubelet.service
-#   kube-proxy.service
-# logging to stderr means we get it in the systemd journal
-KUBE_LOGTOSTDERR="--logtostderr=true"
-
-# journal message level, 0 is debug
-KUBE_LOG_LEVEL="--v=0"
-
-# Should this cluster be allowed to run privileged docker containers
-KUBE_ALLOW_PRIV="{{ kube_allow_priv }}"
-
-# How the controller-manager, scheduler, and proxy find the apiserver
-KUBE_MASTER="--master=http://127.0.0.1:8080"

cluster/juju/layers/kubernetes-master/templates/kube-scheduler.defaults

-###
-# kubernetes scheduler config
-
-# default config should be adequate
-
-# Add your own!
-KUBE_SCHEDULER_ARGS="{{ kube_scheduler_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-scheduler.service

-[Unit]
-Description=Kubernetes Scheduler Plugin
-Documentation=http://kubernetes.io/docs/admin/multiple-schedulers/
-
-[Service]
-EnvironmentFile=-/etc/default/kube-defaults
-EnvironmentFile=-/etc/default/kube-scheduler
-ExecStart=/usr/local/bin/kube-scheduler \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_SCHEDULER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cluster/juju/layers/kubernetes-worker/README.md

@@ -41,6 +41,27 @@ a unit for maintenance.
 
 Resuming the workload will [uncordon](http://kubernetes.io/docs/user-guide/kubectl/kubectl_uncordon/) a paused unit. Workloads will automatically migrate unless otherwise directed via their application declaration.
 
+## Private registry
+
+With the "registry" action that is part for the kubernetes-worker charm, you can very easily create a private docker registry, with authentication, and available over TLS. Please note that the registry deployed with the action is not HA, and uses storage tied to the kubernetes node where the pod is running. So if the registry pod changes is migrated from one node to another for whatever reason, you will need to re-publish the images.
+
+### Example usage
+
+Create the relevant authentication files. Let's say you want user `userA` to authenticate with the password `passwordA`. Then you'll do :
+
+    echo "userA:passwordA" > htpasswd-plain
+    htpasswd -c -b -B htpasswd userA passwordA
+
+(the `htpasswd` program comes with the `apache2-utils` package)
+
+Supposing your registry will be reachable at `myregistry.company.com`, and that you already have your TLS key in the `registry.key` file, and your TLS certificate (with `myregistry.company.com` as Common Name) in the `registry.crt` file, you would then run :
+
+    juju run-action kubernetes-worker/0 registry domain=myregistry.company.com htpasswd="$(base64 -w0 htpasswd)" htpasswd-plain="$(base64 -w0 htpasswd-plain)" tlscert="$(base64 -w0 registry.crt)" tlskey="$(base64 -w0 registry.key)" ingress=true
+
+If you then decide that you want do delete the registry, just run :
+
+    juju run-action kubernetes-worker/0 registry delete=true ingress=true
+
 ## Known Limitations
 
 Kubernetes workers currently only support 'phaux' HA scenarios. Even when configured with an HA cluster string, they will only ever contact the first unit in the cluster map. To enable a proper HA story, kubernetes-worker units are encouraged to proxy through a [kubeapi-load-balancer](https://jujucharms.com/kubeapi-load-balancer)
@@ -48,5 +69,4 @@ application. This enables a HA deployment without the need to
 re-render configuration and disrupt the worker services.
 
 External access to pods must be performed through a [Kubernetes
-Ingress Resource](http://kubernetes.io/docs/user-guide/ingress/). More
-information
+Ingress Resource](http://kubernetes.io/docs/user-guide/ingress/).

cluster/juju/layers/kubernetes-worker/actions.yaml

@@ -14,4 +14,32 @@ microbot:
       delete:
         type: boolean
         default: False
-        description: Removes a microbots deployment, service, and ingress if True.
+        description: Remove a microbots deployment, service, and ingress if True.
+upgrade:
+    description: Upgrade the kubernetes snaps
+registry:
+     description: Create a private Docker registry
+     params:
+      htpasswd:
+        type: string
+        description: base64 encoded htpasswd file used for authentication.
+      htpasswd-plain:
+        type: string
+        description: base64 encoded plaintext version of the htpasswd file, needed by docker daemons to authenticate to the registry.
+      tlscert:
+        type: string
+        description: base64 encoded TLS certificate for the registry. Common Name must match the domain name of the registry.
+      tlskey:
+        type: string
+        description: base64 encoded TLS key for the registry.
+      domain:
+        type: string
+        description: The domain name for the registry. Must match the Common Name of the certificate.
+      ingress:
+        type: boolean
+        default: false
+        description: Create an Ingress resource for the registry (or delete resource object if "delete" is True)
+      delete:
+        type: boolean
+        default: false
+        description: Remove a registry replication controller, service, and ingress if True.

cluster/juju/layers/kubernetes-worker/actions/microbot

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 
 from charmhelpers.core.hookenv import action_get
@@ -22,6 +23,7 @@ from charmhelpers.core.hookenv import unit_public_ip
 from charms.templating.jinja2 import render
 from subprocess import call
 
+os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin')
 
 context = {}
 context['replicas'] = action_get('replicas')
@@ -32,7 +34,7 @@ if not context['replicas']:
     context['replicas'] = 3
 
 # Declare a kubectl template when invoking kubectl
-kubectl = ['kubectl', '--kubeconfig=/srv/kubernetes/config']
+kubectl = ['kubectl', '--kubeconfig=/root/cdk/kubeconfig']
 
 # Remove deployment if requested
 if context['delete']:
@@ -56,11 +58,11 @@ if context['delete']:
 
 # Creation request
 
-render('microbot-example.yaml', '/etc/kubernetes/addons/microbot.yaml',
+render('microbot-example.yaml', '/root/cdk/addons/microbot.yaml',
        context)
 
 create_command = kubectl + ['create', '-f',
-                            '/etc/kubernetes/addons/microbot.yaml']
+                            '/root/cdk/addons/microbot.yaml']
 
 create_response = call(create_command)
 

cluster/juju/layers/kubernetes-worker/actions/pause

@@ -2,6 +2,8 @@
 
 set -ex
 
-kubectl --kubeconfig=/srv/kubernetes/config cordon $(hostname)
-kubectl --kubeconfig=/srv/kubernetes/config drain $(hostname) --force
+export PATH=$PATH:/snap/bin
+
+kubectl --kubeconfig=/root/cdk/kubeconfig cordon $(hostname)
+kubectl --kubeconfig=/root/cdk/kubeconfig drain $(hostname) --force
 status-set 'waiting' 'Kubernetes unit paused'

cluster/juju/layers/kubernetes-worker/actions/registry

+#!/usr/bin/python3
+#
+# For a usage examples, see README.md
+#
+# TODO
+#
+# - make the action idempotent (i.e. if you run it multiple times, the first
+# run will create/delete the registry, and the reset will be a no-op and won't
+# error out)
+#
+# - take only a plain authentication file, and create the encrypted version in
+# the action
+#
+# - validate the parameters (make sure tlscert is a certificate, that tlskey is a
+# proper key, etc)
+#
+# - when https://bugs.launchpad.net/juju/+bug/1661015 is fixed, handle the
+# base64 encoding the parameters in the action itself
+
+import os
+import sys
+
+from base64 import b64encode
+
+from charmhelpers.core.hookenv import action_get
+from charmhelpers.core.hookenv import action_set
+from charms.templating.jinja2 import render
+from subprocess import call
+
+os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin')
+
+deletion = action_get('delete')
+
+context = {}
+
+# These config options must be defined in the case of a creation
+param_error = False
+for param in ('tlscert', 'tlskey', 'domain', 'htpasswd', 'htpasswd-plain'):
+    value = action_get(param)
+    if not value and not deletion:
+        key = "registry-create-parameter-{}".format(param)
+        error = "failure, parameter {} is required".format(param)
+        action_set({key: error})
+        param_error = True
+
+    context[param] = value
+
+# Create the dockercfg template variable
+dockercfg = '{"%s:443": {"auth": "%s", "email": "root@localhost"}}' % \
+            (context['domain'], context['htpasswd-plain'])
+context['dockercfg'] = b64encode(dockercfg.encode()).decode('ASCII')
+
+if param_error:
+    sys.exit(0)
+
+# This one is either true or false, no need to check if it has a "good" value.
+context['ingress'] = action_get('ingress')
+
+# Declare a kubectl template when invoking kubectl
+kubectl = ['kubectl', '--kubeconfig=/root/cdk/kubeconfig']
+
+# Remove deployment if requested
+if deletion:
+    resources = ['svc/kube-registry', 'rc/kube-registry-v0', 'secrets/registry-tls-data',
+                 'secrets/registry-auth-data', 'secrets/registry-access']
+
+    if action_get('ingress'):
+        resources.append('ing/registry-ing')
+
+    delete_command = kubectl + ['delete', '--ignore-not-found=true'] + resources
+    delete_response = call(delete_command)
+    if delete_response == 0:
+        action_set({'registry-delete': 'success'})
+    else:
+        action_set({'registry-delete': 'failure'})
+
+    sys.exit(0)
+
+# Creation request
+render('registry.yaml', '/root/cdk/addons/registry.yaml',
+       context)
+
+create_command = kubectl + ['create', '-f',
+                            '/root/cdk/addons/registry.yaml']
+
+create_response = call(create_command)
+
+if create_response == 0:
+    action_set({'registry-create': 'success'})
+
+    # Create a ConfigMap if it doesn't exist yet, else patch it.
+    # A ConfigMap is needed to change the default value for nginx' client_max_body_size.
+    # The default is 1MB, and this is the maximum size of images that can be
+    # pushed on the registry. 1MB images aren't useful, so we bump this value to 1024MB.
+    cm_name = 'nginx-load-balancer-conf'
+    check_cm_command = kubectl + ['get', 'cm', cm_name]
+    check_cm_response = call(check_cm_command)
+
+    if check_cm_response == 0:
+        # There is an existing ConfigMap, patch it
+        patch = '{"data":{"max-body-size":"1024m"}}'
+        patch_cm_command = kubectl + ['patch', 'cm', cm_name, '-p', patch]
+        patch_cm_response = call(patch_cm_command)
+
+        if patch_cm_response == 0:
+            action_set({'configmap-patch': 'success'})
+        else:
+            action_set({'configmap-patch': 'failure'})
+
+    else:
+        # No existing ConfigMap, create it
+        render('registry-configmap.yaml', '/root/cdk/addons/registry-configmap.yaml',
+               context)
+        create_cm_command = kubectl + ['create', '-f', '/root/cdk/addons/registry-configmap.yaml']
+        create_cm_response = call(create_cm_command)
+
+        if create_cm_response == 0:
+            action_set({'configmap-create': 'success'})
+        else:
+            action_set({'configmap-create': 'failure'})
+
+    # Patch the "default" serviceaccount with an imagePullSecret.
+    # This will allow the docker daemons to authenticate to our private
+    # registry automatically
+    patch = '{"imagePullSecrets":[{"name":"registry-access"}]}'
+    patch_sa_command = kubectl + ['patch', 'sa', 'default', '-p', patch]
+    patch_sa_response = call(patch_sa_command)
+
+    if patch_sa_response == 0:
+        action_set({'serviceaccount-patch': 'success'})
+    else:
+        action_set({'serviceaccount-patch': 'failure'})
+
+
+else:
+    action_set({'registry-create': 'failure'})

cluster/juju/layers/kubernetes-worker/actions/resume

@@ -2,5 +2,7 @@
 
 set -ex
 
-kubectl --kubeconfig=/srv/kubernetes/config uncordon $(hostname)
+export PATH=$PATH:/snap/bin
+
+kubectl --kubeconfig=/root/cdk/kubeconfig uncordon $(hostname)
 status-set 'active' 'Kubernetes unit resumed'

cluster/juju/layers/kubernetes-worker/actions/upgrade

+#!/bin/sh
+set -eux
+
+charms.reactive set_state kubernetes-worker.snaps.upgrade-specified
+exec hooks/config-changed

cluster/juju/layers/kubernetes-worker/config.yaml

@@ -20,3 +20,14 @@ options:
       mode by default. If "false", kubelet will never run in privileged mode.
       If "auto", kubelet will not run in privileged mode by default, but will
       switch to privileged mode if gpu hardware is detected.
+  channel:
+    type: string
+    default: "stable"
+    description: |
+      Snap channel to install Kubernetes worker services from
+  require-manual-upgrade:
+    type: boolean
+    default: true
+    description: |
+      When true, worker services will not be upgraded until the user triggers
+      it manually by running the upgrade action.

cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl

 #!/bin/sh
 set -ux
 
-alias kubectl="kubectl --kubeconfig=/srv/kubernetes/config"
+export PATH=$PATH:/snap/bin
+
+alias kubectl="kubectl --kubeconfig=/root/cdk/kubeconfig"
 
 kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info
 kubectl cluster-info dump > $DEBUG_SCRIPT_DIR/cluster-info-dump

cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services

@@ -2,12 +2,8 @@
 set -ux
 
 for service in kubelet kube-proxy; do
-  systemctl status $service > $DEBUG_SCRIPT_DIR/$service-systemctl-status
-  journalctl -u $service > $DEBUG_SCRIPT_DIR/$service-journal
+  systemctl status snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-systemctl-status
+  journalctl -u snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-journal
 done
 
-mkdir -p $DEBUG_SCRIPT_DIR/etc-default
-cp -v /etc/default/kube* $DEBUG_SCRIPT_DIR/etc-default
-
-mkdir -p $DEBUG_SCRIPT_DIR/lib-systemd-system
-cp -v /lib/systemd/system/kube* $DEBUG_SCRIPT_DIR/lib-systemd-system
+# FIXME: get the snap config or something

cluster/juju/layers/kubernetes-worker/layer.yaml

@@ -2,7 +2,9 @@ repo: https://github.com/kubernetes/kubernetes.git
 includes:
   - 'layer:basic'
   - 'layer:debug'
+  - 'layer:snap'
   - 'layer:docker'
+  - 'layer:metrics'
   - 'layer:nagios'
   - 'layer:tls-client'
   - 'layer:nvidia-cuda'
@@ -17,8 +19,8 @@ options:
       - 'ceph-common'
       - 'socat'
   tls-client:
-    ca_certificate_path: '/srv/kubernetes/ca.crt'
-    server_certificate_path: '/srv/kubernetes/server.crt'
-    server_key_path: '/srv/kubernetes/server.key'
-    client_certificate_path: '/srv/kubernetes/client.crt'
-    client_key_path: '/srv/kubernetes/client.key'
+    ca_certificate_path: '/root/cdk/ca.crt'
+    server_certificate_path: '/root/cdk/server.crt'
+    server_key_path: '/root/cdk/server.key'
+    client_certificate_path: '/root/cdk/client.crt'
+    client_key_path: '/root/cdk/client.key'

cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/common.py

@@ -17,10 +17,6 @@
 import re
 import subprocess
 
-from charmhelpers.core import unitdata
-
-BIN_VERSIONS = 'bin_versions'
-
 
 def get_version(bin_name):
     """Get the version of an installed Kubernetes binary.
@@ -34,31 +30,6 @@ def get_version(bin_name):
         (1, 6, 0)
 
     """
-    db = unitdata.kv()
-    bin_versions = db.get(BIN_VERSIONS, {})
-
-    cached_version = bin_versions.get(bin_name)
-    if cached_version:
-        return tuple(cached_version)
-
-    version = _get_bin_version(bin_name)
-    bin_versions[bin_name] = list(version)
-    db.set(BIN_VERSIONS, bin_versions)
-    return version
-
-
-def reset_versions():
-    """Reset the cache of bin versions.
-
-    """
-    db = unitdata.kv()
-    db.unset(BIN_VERSIONS)
-
-
-def _get_bin_version(bin_name):
-    """Get a binary version by calling it with --version and parsing output.
-
-    """
     cmd = '{} --version'.format(bin_name).split()
     version_string = subprocess.check_output(cmd).decode('utf-8')
     return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])

cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/flagmanager.py

@@ -118,6 +118,13 @@ class FlagManager:
         """
         return self.data.get(key, default)
 
+    def destroy_all(self):
+        '''
+        Destructively removes all data from the FlagManager.
+        '''
+        self.data.clear()
+        self.__save()
+
     def to_s(self):
         '''
         Render the flags to a single string, prepared for the Docker

cluster/juju/layers/kubernetes-worker/metadata.yaml

@@ -29,7 +29,19 @@ provides:
     interface: kubernetes-cni
     scope: container
 resources:
-  kubernetes:
+  cni:
+    type: file
+    filename: cni.tgz
+    description: CNI plugins
+  kubectl:
+    type: file
+    filename: kubectl.snap
+    description: kubectl snap
+  kubelet:
+    type: file
+    filename: kubelet.snap
+    description: kubelet snap
+  kube-proxy:
     type: file
-    filename: kubernetes.tar.gz
-    description: "An archive of kubernetes binaries for the worker."
+    filename: kube-proxy.snap
+    description: kube-proxy snap

cluster/juju/layers/kubernetes-worker/metrics.yaml

+metrics:
+  juju-units: {}

cluster/juju/layers/kubernetes-worker/registry-configmap.yaml

+apiVersion: v1
+data:
+  body-size: 1024m
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf

cluster/juju/layers/kubernetes-worker/templates/ingress-replication-controller.yaml

 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+---
+apiVersion: v1
 kind: ReplicationController
 metadata:
   name: nginx-ingress-controller
@@ -45,3 +50,4 @@ spec:
         args:
         - /nginx-ingress-controller
         - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+        - --nginx-configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf

cluster/juju/layers/kubernetes-worker/templates/kube-default

-###
-# kubernetes system config
-#
-# The following values are used to configure various aspects of all
-# kubernetes services, including
-#
-#   kube-apiserver.service
-#   kube-controller-manager.service
-#   kube-scheduler.service
-#   kubelet.service
-#   kube-proxy.service
-# logging to stderr means we get it in the systemd journal
-KUBE_LOGTOSTDERR="--logtostderr=true"
-
-# journal message level, 0 is debug
-KUBE_LOG_LEVEL="--v=0"
-
-# Should this cluster be allowed to run privileged docker containers
-KUBE_ALLOW_PRIV="{{ kube_allow_priv }}"
-
-# How the controller-manager, scheduler, and proxy find the apiserver
-KUBE_MASTER="{{ kube_api_endpoint }}"

cluster/juju/layers/kubernetes-worker/templates/kube-proxy.defaults

-KUBE_PROXY_ARGS="{{ kube_proxy_opts }}"

cluster/juju/layers/kubernetes-worker/templates/kube-proxy.service

-
-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=http://kubernetes.io/docs/admin/kube-proxy/
-After=network.target
-
-[Service]
-EnvironmentFile=-/etc/default/kube-default
-EnvironmentFile=-/etc/default/kube-proxy
-ExecStart=/usr/local/bin/kube-proxy \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_PROXY_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cluster/juju/layers/kubernetes-worker/templates/kubelet.defaults

-# kubernetes kubelet (node) config
-
-# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
-KUBELET_ADDRESS="--address=0.0.0.0"
-
-# The port for the info server to serve on
-KUBELET_PORT="--port=10250"
-
-# You may leave this blank to use the actual hostname. If you override this
-# reachability problems become your own issue.
-# KUBELET_HOSTNAME="--hostname-override={{ JUJU_UNIT_NAME }}"
-
-# Add your own!
-KUBELET_ARGS="{{ kubelet_opts }}"

cluster/juju/layers/kubernetes-worker/templates/kubelet.service

-[Unit]
-Description=Kubernetes Kubelet Server
-Documentation=http://kubernetes.io/docs/admin/kubelet/
-After=docker.service
-Requires=docker.service
-
-[Service]
-WorkingDirectory=/var/lib/kubelet
-EnvironmentFile=-/etc/default/kube-default
-EnvironmentFile=-/etc/default/kubelet
-ExecStart=/usr/local/bin/kubelet \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBELET_ADDRESS \
-	    $KUBELET_PORT \
-	    $KUBELET_HOSTNAME \
-	    $KUBE_ALLOW_PRIV \
-	    $KUBELET_ARGS
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target

cluster/juju/layers/kubernetes-worker/templates/registry.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-tls-data
+type: Opaque
+data:
+  tls.crt: {{ tlscert }}
+  tls.key: {{ tlskey }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-auth-data
+type: Opaque
+data:
+  htpasswd: {{ htpasswd }}
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kube-registry-v0
+  labels:
+    k8s-app: kube-registry
+    version: v0
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: kube-registry
+    version: v0
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-registry
+        version: v0
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: registry
+        image: registry:2
+        resources:
+          # keep request = limit to keep this container in guaranteed class
+          limits:
+            cpu: 100m
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: :5000
+        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+          value: /var/lib/registry
+        - name: REGISTRY_AUTH_HTPASSWD_REALM
+          value: basic_realm
+        - name: REGISTRY_AUTH_HTPASSWD_PATH
+          value: /auth/htpasswd
+        volumeMounts:
+        - name: image-store
+          mountPath: /var/lib/registry
+        - name: auth-dir
+          mountPath: /auth
+        ports:
+        - containerPort: 5000
+          name: registry
+          protocol: TCP
+      volumes:
+      - name: image-store
+        hostPath:
+          path: /srv/registry
+      - name: auth-dir
+        secret:
+          secretName: registry-auth-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-registry
+  labels:
+    k8s-app: kube-registry
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "KubeRegistry"
+spec:
+  selector:
+    k8s-app: kube-registry
+  type: LoadBalancer
+  ports:
+  - name: registry
+    port: 5000
+    protocol: TCP
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-access
+data:
+  .dockercfg: {{ dockercfg }}
+type: kubernetes.io/dockercfg
+{%- if ingress %}
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: registry-ing
+spec:
+  tls:
+  - hosts:
+    - {{ domain }}
+    secretName: registry-tls-data
+  rules:
+  - host: {{ domain }}
+    http:
+      paths:
+      - backend:
+          serviceName: kube-registry
+          servicePort: 5000
+        path: /
+{% endif %}

hack/verify-flags/exceptions.txt

@@ -47,7 +47,7 @@ cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:def send_clu
 cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:def service_cidr():
 cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:    context.update({'kube_api_endpoint': ','.join(api_servers),
 cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:    ca_cert_path = layer_options.get('ca_certificate_path')
-cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:def render_init_scripts(api_servers):
+cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:def configure_worker_services(api_servers, dns, cluster_cidr):
 cluster/lib/logging.sh:      local source_file=${BASH_SOURCE[$frame_no]}
 cluster/lib/logging.sh:    local source_file=${BASH_SOURCE[$stack_skip]}
 cluster/log-dump.sh:    local -r node_name="${1}"