Update vendor

vendor.conf

@@ -9,9 +9,9 @@ package=github.com/opencontainers/runc/libcontainer/nsenter
 package=github.com/opencontainers/runc/libcontainer/specconv
 package=github.com/opencontainers/runc/contrib/cmd/recvtty
 
-k8s.io/kubernetes                   v1.13.1-k3s2                              https://github.com/ibuildthecloud/k3s.git          transitive=true,staging=true
+k8s.io/kubernetes                   v1.13.2-k3s2                              https://github.com/ibuildthecloud/k3s.git          transitive=true,staging=true
 
-github.com/rancher/norman           29915f8336c0a242560a9fef1d11bbaf04660915  https://github.com/ibuildthecloud/norman.git
+github.com/rancher/norman           628eb6b32906125ddd77e23f3ad3c9bdb6e93664  https://github.com/ibuildthecloud/norman.git
 github.com/coreos/flannel           3d7cff78e2ca4cade87c6c7d44adf27fe3de2709  https://github.com/ibuildthecloud/flannel.git
 github.com/natefinch/lumberjack     aee4629129445bbdfb69aa565537dcfa16544311
 github.com/gorilla/mux              v1.6.2

vendor/github.com/ibuildthecloud/kvsql/clientv3/driver/generic.go

@@ -15,6 +15,9 @@ import (
 )
 
 type Generic struct {
+	// revision must be first to ensure that this is properly aligned for atomic.LoadInt64
+	revision        int64
+
 	db *sql.DB
 
 	CleanupSQL      string
@@ -27,7 +30,6 @@ type Generic struct {
 	GetRevisionSQL  string
 	ToDeleteSQL     string
 	DeleteOldSQL    string
-	revision        int64
 
 	changes     chan *KeyValue
 	broadcaster broadcast.Broadcaster

vendor/github.com/ibuildthecloud/kvsql/clientv3/driver/sqlite/sqlite.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"database/sql"
+	"os"
 	"strings"
 
 	"github.com/ibuildthecloud/kvsql/clientv3/driver"
@@ -63,7 +64,8 @@ func NewSQLite() *driver.Generic {
 
 func Open(dataSourceName string) (*sql.DB, error) {
 	if dataSourceName == "" {
-		dataSourceName = "./state.db?_journal=WAL&cache=shared"
+		os.MkdirAll("./db", 700)
+		dataSourceName = "./db/state.db?_journal=WAL&cache=shared"
 	}
 	db, err := sql.Open("sqlite3", dataSourceName)
 	if err != nil {

vendor/github.com/rancher/norman/build.go

@@ -185,6 +185,8 @@ func (c *Config) defaults(ctx context.Context, r *Runtime, opts Options) (contex
 		}
 	}
 
+	r.LocalConfig = c.Config
+
 	if c.ClientGetter == nil {
 		cg, err := proxy.NewClientGetterFromConfig(*c.Config)
 		if err != nil {

vendor/github.com/rancher/norman/pkg/clientaccess/clientaccess.go

@@ -122,7 +122,7 @@ func ParseAndValidateToken(server, token string) (*Info, error) {
 		return nil, err
 	}
 
-	if len(cacerts) > 0 {
+	if len(cacerts) > 0 && len(parsedToken.caHash) > 0 {
 		if ok, hash, newHash := validateCACerts(cacerts, parsedToken.caHash); !ok {
 			return nil, fmt.Errorf("token does not match the server %s != %s", hash, newHash)
 		}

vendor/github.com/rancher/norman/pkg/objectset/desiredset.go

+package objectset
+
+import (
+	"github.com/rancher/norman/objectclient"
+	"github.com/rancher/norman/pkg/objectset/injectors"
+	"github.com/rancher/norman/types"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+)
+
+type DesiredSet struct {
+	discoveredClients map[schema.GroupVersionKind]*objectclient.ObjectClient
+	discovery         discovery.DiscoveryInterface
+	restConfig        rest.Config
+	remove            bool
+	setID             string
+	objs              *ObjectSet
+	codeVersion       string
+	clients           map[schema.GroupVersionKind]Client
+	owner             runtime.Object
+	injectors         []injectors.ConfigInjector
+	errs              []error
+}
+
+func (o *DesiredSet) AddDiscoveredClient(gvk schema.GroupVersionKind, client *objectclient.ObjectClient) {
+	if o.discoveredClients == nil {
+		o.discoveredClients = map[schema.GroupVersionKind]*objectclient.ObjectClient{}
+	}
+	o.discoveredClients[gvk] = client
+}
+
+func (o *DesiredSet) DiscoveredClients() map[schema.GroupVersionKind]*objectclient.ObjectClient {
+	return o.discoveredClients
+}
+
+func (o *DesiredSet) AddInjector(inj injectors.ConfigInjector) {
+	o.injectors = append(o.injectors, inj)
+}
+
+func (o *DesiredSet) err(err error) error {
+	o.errs = append(o.errs, err)
+	return o.Err()
+}
+
+func (o *DesiredSet) Err() error {
+	return types.NewErrors(append(o.objs.errs, o.errs...)...)
+}

vendor/github.com/rancher/norman/pkg/objectset/desiredset_apply.go

+package objectset
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/pkg/errors"
+	errors2 "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/client-go/util/flowcontrol"
+)
+
+const (
+	LabelID        = "objectset.rio.cattle.io/id"
+	LabelGVK       = "objectset.rio.cattle.io/owner-gvk"
+	LabelName      = "objectset.rio.cattle.io/owner-name"
+	LabelNamespace = "objectset.rio.cattle.io/owner-namespace"
+	LabelHash      = "objectset.rio.cattle.io/hash"
+)
+
+var (
+	hashOrder = []string{
+		LabelID,
+		LabelGVK,
+		LabelName,
+		LabelNamespace,
+	}
+	rls     = map[string]flowcontrol.RateLimiter{}
+	rlsLock sync.Mutex
+)
+
+func (o *DesiredSet) getRateLimit(inputID string) flowcontrol.RateLimiter {
+	var rl flowcontrol.RateLimiter
+
+	rlsLock.Lock()
+	defer rlsLock.Unlock()
+	if o.remove {
+		delete(rls, inputID)
+	} else {
+		rl = rls[inputID]
+		if rl == nil {
+			rl = flowcontrol.NewTokenBucketRateLimiter(4.0/60.0, 10)
+			rls[inputID] = rl
+		}
+	}
+
+	return rl
+}
+
+func (o *DesiredSet) Apply() error {
+	if err := o.Err(); err != nil {
+		return err
+	}
+
+	labelSet, annotationSet, err := o.getLabelsAndAnnotations()
+	if err != nil {
+		return o.err(err)
+	}
+
+	rl := o.getRateLimit(labelSet[LabelHash])
+	if rl != nil && !rl.TryAccept() {
+		return errors2.NewConflict(schema.GroupResource{}, o.setID, errors.New("delaying object set"))
+	}
+
+	inputID := o.inputID(labelSet[LabelHash])
+
+	objList, err := o.injectLabelsAndAnnotations(labelSet, annotationSet)
+	if err != nil {
+		return o.err(err)
+	}
+
+	objList, err = o.runInjectors(objList)
+	if err != nil {
+		return o.err(err)
+	}
+
+	objs := o.collect(objList)
+
+	debugID := o.debugID()
+	req, err := labels.NewRequirement(LabelHash, selection.Equals, []string{labelSet[LabelHash]})
+	if err != nil {
+		return o.err(err)
+	}
+
+	for _, gvk := range o.gvkOrder() {
+		o.process(inputID, debugID, labels.NewSelector().Add(*req), gvk, objs[gvk])
+	}
+
+	return o.Err()
+}
+
+func (o *DesiredSet) gvkOrder() []schema.GroupVersionKind {
+	seen := map[schema.GroupVersionKind]bool{}
+	var gvkOrder []schema.GroupVersionKind
+
+	for _, obj := range o.objs.order {
+		if seen[obj.GetObjectKind().GroupVersionKind()] {
+			continue
+		}
+		seen[obj.GetObjectKind().GroupVersionKind()] = true
+		gvkOrder = append(gvkOrder, obj.GetObjectKind().GroupVersionKind())
+	}
+
+	var rest []schema.GroupVersionKind
+
+	for gvk := range o.clients {
+		if seen[gvk] {
+			continue
+		}
+
+		seen[gvk] = true
+		rest = append(rest, gvk)
+	}
+
+	sort.Slice(rest, func(i, j int) bool {
+		return rest[i].String() < rest[j].String()
+	})
+
+	return append(gvkOrder, rest...)
+}
+
+func (o *DesiredSet) inputID(labelHash string) string {
+	sort.Slice(o.objs.inputs, func(i, j int) bool {
+		left, lErr := meta.Accessor(o.objs.inputs[i])
+		right, rErr := meta.Accessor(o.objs.inputs[j])
+		if lErr != nil || rErr != nil {
+			return true
+		}
+
+		lKey := o.objs.inputs[i].GetObjectKind().GroupVersionKind().String() + "/" + newObjectKey(left).String()
+		rKey := o.objs.inputs[j].GetObjectKind().GroupVersionKind().String() + "/" + newObjectKey(right).String()
+		return lKey < rKey
+	})
+
+	dig := sha1.New()
+	dig.Write([]byte(o.codeVersion))
+	dig.Write([]byte(labelHash))
+
+	inputs := o.objs.inputs
+	if o.owner != nil {
+		inputs = append([]runtime.Object{o.owner}, o.objs.inputs...)
+	}
+
+	for _, obj := range inputs {
+		metadata, err := meta.Accessor(obj)
+		if err != nil {
+			dig.Write([]byte(obj.GetObjectKind().GroupVersionKind().String()))
+			continue
+		}
+
+		key := newObjectKey(metadata)
+		dig.Write([]byte(key.String()))
+		dig.Write([]byte(metadata.GetResourceVersion()))
+	}
+
+	return hex.EncodeToString(dig.Sum(nil))
+}
+
+func (o *DesiredSet) debugID() string {
+	if o.owner == nil {
+		return o.setID
+	}
+	metadata, err := meta.Accessor(o.owner)
+	if err != nil {
+		return o.setID
+	}
+
+	return fmt.Sprintf("%s %s", o.setID, objectKey{
+		namespace: metadata.GetNamespace(),
+		name:      metadata.GetName(),
+	})
+}
+
+func (o *DesiredSet) collect(objList []runtime.Object) objectCollection {
+	result := objectCollection{}
+	for _, obj := range objList {
+		result.add(obj)
+	}
+	return result
+}
+
+func (o *DesiredSet) runInjectors(objList []runtime.Object) ([]runtime.Object, error) {
+	var err error
+
+	for _, inj := range o.injectors {
+		if inj == nil {
+			continue
+		}
+
+		objList, err = inj(objList)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return objList, nil
+}
+
+func (o *DesiredSet) getLabelsAndAnnotations() (map[string]string, map[string]string, error) {
+	annotations := map[string]string{
+		LabelID: o.setID,
+	}
+
+	if o.owner != nil {
+		annotations[LabelGVK] = o.owner.GetObjectKind().GroupVersionKind().String()
+		metadata, err := meta.Accessor(o.owner)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get metadata for %s", o.owner.GetObjectKind().GroupVersionKind())
+		}
+		annotations[LabelName] = metadata.GetName()
+		annotations[LabelNamespace] = metadata.GetNamespace()
+	}
+
+	labels := map[string]string{
+		LabelHash: objectSetHash(annotations),
+	}
+
+	return labels, annotations, nil
+}
+
+func (o *DesiredSet) injectLabelsAndAnnotations(labels, annotations map[string]string) ([]runtime.Object, error) {
+	var result []runtime.Object
+
+	for _, objMap := range o.objs.objects {
+		for key, obj := range objMap {
+			obj = obj.DeepCopyObject()
+			meta, err := meta.Accessor(obj)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get metadata for %s", key)
+			}
+
+			setLabels(meta, labels)
+			setAnnotations(meta, annotations)
+
+			result = append(result, obj)
+		}
+	}
+
+	return result, nil
+}
+
+func setAnnotations(meta metav1.Object, annotations map[string]string) {
+	objAnn := meta.GetAnnotations()
+	if objAnn == nil {
+		objAnn = map[string]string{}
+	}
+	delete(objAnn, LabelInputID)
+	delete(objAnn, LabelApplied)
+	for k, v := range annotations {
+		objAnn[k] = v
+	}
+	meta.SetAnnotations(objAnn)
+}
+
+func setLabels(meta metav1.Object, labels map[string]string) {
+	objLabels := meta.GetLabels()
+	if objLabels == nil {
+		objLabels = map[string]string{}
+	}
+	for k, v := range labels {
+		objLabels[k] = v
+	}
+	meta.SetLabels(objLabels)
+}
+
+func objectSetHash(labels map[string]string) string {
+	dig := sha1.New()
+	for _, key := range hashOrder {
+		dig.Write([]byte(labels[key]))
+	}
+	return hex.EncodeToString(dig.Sum(nil))
+}

vendor/github.com/rancher/norman/pkg/objectset/desiredset_compare.go

+package objectset
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"io/ioutil"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman/objectclient"
+	"github.com/rancher/norman/types/convert"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/jsonmergepatch"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+const (
+	LabelApplied = "objectset.rio.cattle.io/applied"
+	LabelInputID = "objectset.rio.cattle.io/inputid"
+)
+
+var (
+	patchCache     = map[schema.GroupVersionKind]patchCacheEntry{}
+	patchCacheLock = sync.Mutex{}
+)
+
+type patchCacheEntry struct {
+	patchType types.PatchType
+	lookup    strategicpatch.LookupPatchMeta
+}
+
+func prepareObjectForCreate(inputID string, obj runtime.Object) (runtime.Object, error) {
+	serialized, err := json.Marshal(obj)
+	if err != nil {
+		return nil, err
+	}
+
+	obj = obj.DeepCopyObject()
+	meta, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, err
+	}
+	annotations := meta.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+
+	annotations[LabelInputID] = inputID
+	annotations[LabelApplied] = appliedToAnnotation(serialized)
+	meta.SetAnnotations(annotations)
+
+	return obj, nil
+}
+
+func originalAndModifiedForInputID(inputID string, oldMetadata v1.Object, newObject runtime.Object) ([]byte, []byte, error) {
+	original, err := getOriginal(inputID, oldMetadata)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	newObject, err = prepareObjectForCreate(inputID, newObject)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	modified, err := json.Marshal(newObject)
+
+	return original, modified, err
+}
+
+func onlyKeys(data map[string]interface{}, keys ...string) bool {
+	for i, key := range keys {
+		if len(data) > 1 {
+			return false
+		} else if len(data) == 0 {
+			return true
+		}
+
+		value, ok := data[key]
+		if !ok {
+			return false
+		}
+
+		if i == len(keys)-1 {
+			return true
+		}
+
+		data = convert.ToMapInterface(value)
+	}
+
+	return false
+}
+
+func sanitizePatch(patch []byte) ([]byte, error) {
+	mod := false
+	data := map[string]interface{}{}
+	err := json.Unmarshal(patch, &data)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, ok := data["kind"]; ok {
+		mod = true
+		delete(data, "kind")
+	}
+
+	if _, ok := data["apiVersion"]; ok {
+		mod = true
+		delete(data, "apiVersion")
+	}
+
+	if deleted := removeCreationTimestamp(data); deleted {
+		mod = true
+	}
+
+	if onlyKeys(data, "metadata", "annotations", LabelInputID) {
+		return []byte("{}"), nil
+	}
+
+	if !mod {
+		return patch, nil
+	}
+
+	return json.Marshal(data)
+}
+
+func applyPatch(client objectclient.GenericClient, debugID, inputID string, oldObject, newObject runtime.Object) (bool, error) {
+	gvk := client.GroupVersionKind()
+
+	oldMetadata, err := meta.Accessor(oldObject)
+	if err != nil {
+		return false, err
+	}
+
+	original, modified, err := originalAndModifiedForInputID(inputID, oldMetadata, newObject)
+	if err != nil {
+		return false, err
+	}
+
+	current, err := json.Marshal(oldObject)
+	if err != nil {
+		return false, err
+	}
+
+	patchType, patch, err := doPatch(gvk, original, modified, current)
+	if err != nil {
+		return false, errors.Wrap(err, "patch generation")
+	}
+
+	if string(patch) == "{}" {
+		return false, nil
+	}
+
+	patch, err = sanitizePatch(patch)
+	if err != nil {
+		return false, err
+	}
+
+	if string(patch) == "{}" {
+		return false, nil
+	}
+
+	logrus.Debugf("DesiredSet - Patch %s %s/%s for %s -- [%s, %s, %s, %s]", gvk, oldMetadata.GetNamespace(), oldMetadata.GetName(), debugID,
+		patch, original, modified, current)
+
+	logrus.Debugf("DesiredSet - Updated %s %s/%s for %s -- %s %s", gvk, oldMetadata.GetNamespace(), oldMetadata.GetName(), debugID, patchType, patch)
+	_, err = client.Patch(oldMetadata.GetName(), oldObject, patchType, patch)
+
+	return true, err
+}
+
+func (o *DesiredSet) compareObjects(client objectclient.GenericClient, debugID, inputID string, oldObject, newObject runtime.Object, force bool) error {
+	oldMetadata, err := meta.Accessor(oldObject)
+	if err != nil {
+		return err
+	}
+
+	gvk := client.GroupVersionKind()
+	if ran, err := applyPatch(client, debugID, inputID, oldObject, newObject); err != nil {
+		return err
+	} else if !ran {
+		logrus.Debugf("DesiredSet - No change(2) %s %s/%s for %s", gvk, oldMetadata.GetNamespace(), oldMetadata.GetName(), debugID)
+	}
+
+	return nil
+}
+
+func removeCreationTimestamp(data map[string]interface{}) bool {
+	metadata, ok := data["metadata"]
+	if !ok {
+		return false
+	}
+
+	data = convert.ToMapInterface(metadata)
+	if _, ok := data["creationTimestamp"]; ok {
+		delete(data, "creationTimestamp")
+		return true
+	}
+
+	return false
+}
+
+func getOriginal(inputID string, obj v1.Object) ([]byte, error) {
+	original := appliedFromAnnotation(obj.GetAnnotations()[LabelApplied])
+	if len(original) == 0 {
+		return []byte("{}"), nil
+	}
+
+	mapObj := &unstructured.Unstructured{}
+	err := json.Unmarshal(original, mapObj)
+	if err != nil {
+		return nil, err
+	}
+
+	removeCreationTimestamp(mapObj.Object)
+
+	objCopy, err := prepareObjectForCreate(inputID, mapObj)
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(objCopy)
+}
+
+func appliedFromAnnotation(str string) []byte {
+	if len(str) == 0 || str[0] == '{' {
+		return []byte(str)
+	}
+
+	b, err := base64.RawStdEncoding.DecodeString(str)
+	if err != nil {
+		return nil
+	}
+
+	r, err := gzip.NewReader(bytes.NewBuffer(b))
+	if err != nil {
+		return nil
+	}
+
+	b, err = ioutil.ReadAll(r)
+	if err != nil {
+		return nil
+	}
+
+	return b
+}
+
+func appliedToAnnotation(b []byte) string {
+	if len(b) < 1024 {
+		return string(b)
+	}
+	buf := &bytes.Buffer{}
+	w := gzip.NewWriter(buf)
+	if _, err := w.Write(b); err != nil {
+		return string(b)
+	}
+	if err := w.Close(); err != nil {
+		return string(b)
+	}
+	return base64.RawStdEncoding.EncodeToString(buf.Bytes())
+}
+
+// doPatch is adapted from "kubectl apply"
+func doPatch(gvk schema.GroupVersionKind, original, modified, current []byte) (types.PatchType, []byte, error) {
+	var patchType types.PatchType
+	var patch []byte
+	var lookupPatchMeta strategicpatch.LookupPatchMeta
+
+	patchType, lookupPatchMeta, err := getPatchStyle(gvk)
+	if err != nil {
+		return patchType, nil, err
+	}
+
+	if patchType == types.StrategicMergePatchType {
+		patch, err = strategicpatch.CreateThreeWayMergePatch(original, modified, current, lookupPatchMeta, true)
+	} else {
+		patch, err = jsonmergepatch.CreateThreeWayJSONMergePatch(original, modified, current)
+	}
+
+	if err != nil {
+		logrus.Errorf("Failed to calcuated patch: %v", err)
+	}
+
+	return patchType, patch, err
+}
+
+func getPatchStyle(gvk schema.GroupVersionKind) (types.PatchType, strategicpatch.LookupPatchMeta, error) {
+	var (
+		patchType       types.PatchType
+		lookupPatchMeta strategicpatch.LookupPatchMeta
+	)
+
+	patchCacheLock.Lock()
+	entry, ok := patchCache[gvk]
+	patchCacheLock.Unlock()
+
+	if ok {
+		return entry.patchType, entry.lookup, nil
+	}
+
+	versionedObject, err := scheme.Scheme.New(gvk)
+
+	if runtime.IsNotRegisteredError(err) {
+		patchType = types.MergePatchType
+	} else if err != nil {
+		return patchType, nil, err
+	} else {
+		patchType = types.StrategicMergePatchType
+		lookupPatchMeta, err = strategicpatch.NewPatchMetaFromStruct(versionedObject)
+		if err != nil {
+			return patchType, nil, err
+		}
+	}
+
+	patchCacheLock.Lock()
+	patchCache[gvk] = patchCacheEntry{
+		patchType: patchType,
+		lookup:    lookupPatchMeta,
+	}
+	patchCacheLock.Unlock()
+
+	return patchType, lookupPatchMeta, nil
+}

vendor/github.com/rancher/norman/pkg/objectset/desiredset_process.go

+package objectset
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman/controller"
+	"github.com/rancher/norman/objectclient"
+	"github.com/rancher/norman/objectclient/dynamic"
+	"github.com/rancher/norman/restwatch"
+	"github.com/rancher/norman/types"
+	"github.com/sirupsen/logrus"
+	errors2 "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+)
+
+var (
+	deletePolicy = v1.DeletePropagationBackground
+)
+
+func NewDiscoveredClient(gvk schema.GroupVersionKind, restConfig rest.Config, discovery discovery.DiscoveryInterface) (*objectclient.ObjectClient, error) {
+	resources, err := discovery.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
+	if err != nil {
+		return nil, err
+	}
+
+	for _, resource := range resources.APIResources {
+		if resource.Kind != gvk.Kind {
+			continue
+		}
+
+		if restConfig.NegotiatedSerializer == nil {
+			restConfig.NegotiatedSerializer = dynamic.NegotiatedSerializer
+		}
+
+		restClient, err := restwatch.UnversionedRESTClientFor(&restConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		objectClient := objectclient.NewObjectClient("", restClient, &resource, gvk, &objectclient.UnstructuredObjectFactory{})
+		return objectClient, nil
+	}
+
+	return nil, fmt.Errorf("failed to discover client for %s", gvk)
+}
+
+func (o *DesiredSet) getControllerAndObjectClient(debugID string, gvk schema.GroupVersionKind) (controller.GenericController, *objectclient.ObjectClient, error) {
+	client, ok := o.clients[gvk]
+	if !ok && o.discovery == nil {
+		return nil, nil, fmt.Errorf("failed to find client for %s for %s", gvk, debugID)
+	}
+
+	if client != nil {
+		return client.Generic(), client.ObjectClient(), nil
+	}
+
+	objectClient := o.discoveredClients[gvk]
+	if objectClient != nil {
+		return nil, objectClient, nil
+	}
+
+	objectClient, err := NewDiscoveredClient(gvk, o.restConfig, o.discovery)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to find client for %s for %s", gvk, debugID)
+	}
+
+	o.AddDiscoveredClient(gvk, objectClient)
+	return nil, objectClient, nil
+}
+
+func (o *DesiredSet) process(inputID, debugID string, set labels.Selector, gvk schema.GroupVersionKind, objs map[objectKey]runtime.Object) {
+	controller, objectClient, err := o.getControllerAndObjectClient(debugID, gvk)
+	if err != nil {
+		o.err(err)
+		return
+	}
+
+	existing, err := list(controller, objectClient, set)
+	if err != nil {
+		o.err(fmt.Errorf("failed to list %s for %s", gvk, debugID))
+		return
+	}
+
+	toCreate, toDelete, toUpdate := compareSets(existing, objs)
+	for _, k := range toCreate {
+		obj := objs[k]
+		obj, err := prepareObjectForCreate(inputID, obj)
+		if err != nil {
+			o.err(errors.Wrapf(err, "failed to prepare create %s %s for %s", k, gvk, debugID))
+			continue
+		}
+
+		_, err = objectClient.Create(obj)
+		if errors2.IsAlreadyExists(err) {
+			// Taking over an object that wasn't previously managed by us
+			existingObj, err := objectClient.GetNamespaced(k.namespace, k.name, v1.GetOptions{})
+			if err == nil {
+				toUpdate = append(toUpdate, k)
+				existing[k] = existingObj
+				continue
+			}
+		}
+		if err != nil {
+			o.err(errors.Wrapf(err, "failed to create %s %s for %s", k, gvk, debugID))
+			continue
+		}
+		logrus.Debugf("DesiredSet - Created %s %s for %s", gvk, k, debugID)
+	}
+
+	for _, k := range toUpdate {
+		err := o.compareObjects(objectClient, debugID, inputID, existing[k], objs[k], len(toCreate) > 0 || len(toDelete) > 0)
+		if err != nil {
+			o.err(errors.Wrapf(err, "failed to update %s %s for %s", k, gvk, debugID))
+			continue
+		}
+	}
+
+	for _, k := range toDelete {
+		err := objectClient.DeleteNamespaced(k.namespace, k.name, &v1.DeleteOptions{
+			PropagationPolicy: &deletePolicy,
+		})
+		if err != nil {
+			o.err(errors.Wrapf(err, "failed to delete %s %s for %s", k, gvk, debugID))
+			continue
+		}
+		logrus.Debugf("DesiredSet - Delete %s %s for %s", gvk, k, debugID)
+	}
+}
+
+func compareSets(existingSet, newSet map[objectKey]runtime.Object) (toCreate, toDelete, toUpdate []objectKey) {
+	for k := range newSet {
+		if _, ok := existingSet[k]; ok {
+			toUpdate = append(toUpdate, k)
+		} else {
+			toCreate = append(toCreate, k)
+		}
+	}
+
+	for k := range existingSet {
+		if _, ok := newSet[k]; !ok {
+			toDelete = append(toDelete, k)
+		}
+	}
+
+	sortObjectKeys(toCreate)
+	sortObjectKeys(toDelete)
+	sortObjectKeys(toUpdate)
+
+	return
+}
+
+func sortObjectKeys(keys []objectKey) {
+	sort.Slice(keys, func(i, j int) bool {
+		return keys[i].String() < keys[j].String()
+	})
+}
+
+func addObjectToMap(objs map[objectKey]runtime.Object, obj interface{}) error {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	objs[objectKey{
+		namespace: metadata.GetNamespace(),
+		name:      metadata.GetName(),
+	}] = obj.(runtime.Object)
+
+	return nil
+}
+
+func list(controller controller.GenericController, objectClient *objectclient.ObjectClient, selector labels.Selector) (map[objectKey]runtime.Object, error) {
+	var (
+		errs []error
+		objs = map[objectKey]runtime.Object{}
+	)
+
+	if controller == nil {
+		objList, err := objectClient.List(v1.ListOptions{
+			LabelSelector: selector.String(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		list, ok := objList.(*unstructured.UnstructuredList)
+		if !ok {
+			return nil, fmt.Errorf("invalid list type %T", objList)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		for _, obj := range list.Items {
+			if err := addObjectToMap(objs, obj); err != nil {
+				errs = append(errs, err)
+			}
+		}
+
+		return objs, nil
+	}
+
+	err := cache.ListAllByNamespace(controller.Informer().GetIndexer(), "", selector, func(obj interface{}) {
+		if err := addObjectToMap(objs, obj); err != nil {
+			errs = append(errs, err)
+		}
+	})
+	if err != nil {
+		errs = append(errs, err)
+	}
+
+	return objs, types.NewErrors(errs...)
+}

vendor/github.com/rancher/norman/pkg/objectset/injectors/marshal.go

+package injectors
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
+)
+
+func ToBytes(objects []runtime.Object) ([]byte, error) {
+	if len(objects) == 0 {
+		return nil, nil
+	}
+
+	buffer := &bytes.Buffer{}
+	for i, obj := range objects {
+		if i > 0 {
+			buffer.WriteString("\n---\n")
+		}
+
+		bytes, err := yaml.Marshal(obj)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to encode %s", obj.GetObjectKind().GroupVersionKind())
+		}
+		buffer.Write(bytes)
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func FromBytes(content []byte) ([]runtime.Object, error) {
+	var result []runtime.Object
+
+	reader := yamlDecoder.NewYAMLReader(bufio.NewReader(bytes.NewBuffer(content)))
+	for {
+		raw, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+
+		data := map[string]interface{}{}
+		if err := yaml.Unmarshal(raw, &data); err != nil {
+			return nil, err
+		}
+
+		result = append(result, &unstructured.Unstructured{Object: data})
+	}
+
+	return result, nil
+}

vendor/github.com/rancher/norman/pkg/objectset/injectors/registry.go

+package injectors
+
+import "k8s.io/apimachinery/pkg/runtime"
+
+var (
+	injectors = map[string]ConfigInjector{}
+	order     []string
+)
+
+type ConfigInjector func(config []runtime.Object) ([]runtime.Object, error)
+
+func Register(name string, injector ConfigInjector) {
+	if _, ok := injectors[name]; !ok {
+		order = append(order, name)
+	}
+	injectors[name] = injector
+}
+
+func Get(name string) ConfigInjector {
+	return injectors[name]
+}

vendor/github.com/rancher/norman/pkg/objectset/objectset.go

+package objectset
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/rancher/norman/types"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type objectKey struct {
+	name      string
+	namespace string
+}
+
+func newObjectKey(obj v1.Object) objectKey {
+	return objectKey{
+		namespace: obj.GetNamespace(),
+		name:      obj.GetName(),
+	}
+}
+
+func (o objectKey) String() string {
+	if o.namespace == "" {
+		return o.name
+	}
+	return fmt.Sprintf("%s/%s", o.namespace, o.name)
+}
+
+type objectCollection map[schema.GroupVersionKind]map[objectKey]runtime.Object
+
+func (o objectCollection) add(obj runtime.Object) error {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	objs := o[obj.GetObjectKind().GroupVersionKind()]
+	if objs == nil {
+		objs = map[objectKey]runtime.Object{}
+		o[obj.GetObjectKind().GroupVersionKind()] = objs
+	}
+
+	objs[objectKey{
+		namespace: metadata.GetNamespace(),
+		name:      metadata.GetName(),
+	}] = obj
+
+	return nil
+}
+
+type ObjectSet struct {
+	errs    []error
+	objects objectCollection
+	nsed    map[schema.GroupVersionKind]bool
+	inputs  []runtime.Object
+	order   []runtime.Object
+}
+
+func NewObjectSet() *ObjectSet {
+	return &ObjectSet{
+		nsed:    map[schema.GroupVersionKind]bool{},
+		objects: objectCollection{},
+	}
+}
+
+func (o *ObjectSet) AddInput(objs ...runtime.Object) *ObjectSet {
+	for _, obj := range objs {
+		if obj == nil || reflect.ValueOf(obj).IsNil() {
+			continue
+		}
+		o.inputs = append(o.inputs, obj)
+	}
+	return o
+}
+
+func (o *ObjectSet) Add(objs ...runtime.Object) *ObjectSet {
+	for _, obj := range objs {
+		o.add(obj)
+	}
+	return o
+}
+
+func (o *ObjectSet) add(obj runtime.Object) {
+	if obj == nil || reflect.ValueOf(obj).IsNil() {
+		return
+	}
+
+	gvk := obj.GetObjectKind().GroupVersionKind()
+
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		o.err(fmt.Errorf("failed to get metadata for %s", gvk))
+		return
+	}
+
+	name := metadata.GetName()
+	if name == "" {
+		o.err(fmt.Errorf("%s is missing name", gvk))
+		return
+	}
+
+	namespace := metadata.GetNamespace()
+	nsed, ok := o.nsed[gvk]
+	if ok && nsed != (namespace != "") {
+		o.err(fmt.Errorf("got %s objects that are both namespaced and not namespaced", gvk))
+		return
+	}
+	o.nsed[gvk] = namespace != ""
+
+	if err := o.objects.add(obj); err != nil {
+		o.err(fmt.Errorf("failed to get metadata for %s", gvk))
+		return
+	}
+
+	o.order = append(o.order, obj)
+}
+
+func (o *ObjectSet) err(err error) error {
+	o.errs = append(o.errs, err)
+	return o.Err()
+}
+
+func (o *ObjectSet) AddErr(err error) {
+	o.errs = append(o.errs, err)
+}
+
+func (o *ObjectSet) Err() error {
+	return types.NewErrors(o.errs...)
+}

vendor/github.com/rancher/norman/pkg/objectset/template.go

+package objectset
+
+import (
+	"github.com/rancher/norman/controller"
+	"github.com/rancher/norman/objectclient"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+)
+
+type Client interface {
+	Generic() controller.GenericController
+	ObjectClient() *objectclient.ObjectClient
+}
+
+type Processor struct {
+	setID         string
+	codeVersion   string
+	discovery     discovery.DiscoveryInterface
+	restConfig    rest.Config
+	allowSlowPath string
+	slowClient    rest.HTTPClient
+	clients       map[schema.GroupVersionKind]Client
+}
+
+func NewProcessor(setID string) *Processor {
+	return &Processor{
+		setID:   setID,
+		clients: map[schema.GroupVersionKind]Client{},
+	}
+}
+
+func (t *Processor) SetID() string {
+	return t.setID
+}
+
+func (t *Processor) CodeVersion(version string) *Processor {
+	t.codeVersion = version
+	return t
+}
+
+func (t *Processor) AllowDiscovery(discovery discovery.DiscoveryInterface, restConfig rest.Config) *Processor {
+	t.discovery = discovery
+	t.restConfig = restConfig
+	return t
+}
+
+func (t *Processor) Clients() map[schema.GroupVersionKind]Client {
+	return t.clients
+}
+
+func (t *Processor) Client(clients ...Client) *Processor {
+	// ensure cache is enabled
+	for _, client := range clients {
+		client.Generic()
+		t.clients[client.ObjectClient().GroupVersionKind()] = client
+	}
+	return t
+}
+
+func (t Processor) Remove(owner runtime.Object) error {
+	return t.NewDesiredSet(owner, nil).Apply()
+}
+
+func (t Processor) NewDesiredSet(owner runtime.Object, objs *ObjectSet) *DesiredSet {
+	remove := false
+	if objs == nil {
+		remove = true
+		objs = &ObjectSet{}
+	}
+	return &DesiredSet{
+		discovery:   t.discovery,
+		restConfig:  t.restConfig,
+		remove:      remove,
+		objs:        objs,
+		setID:       t.setID,
+		codeVersion: t.codeVersion,
+		clients:     t.clients,
+		owner:       owner,
+	}
+}

vendor/github.com/rancher/norman/types/mapper/embed.go

@@ -70,10 +70,11 @@ func (e *Embed) ModifySchema(schema *types.Schema, schemas *types.Schemas) error
 	}
 
 	deleteField := true
+outer:
 	for name, field := range embeddedSchema.ResourceFields {
 		for _, ignore := range e.Ignore {
 			if ignore == name {
-				continue
+				continue outer
 			}
 		}
 

vendor/k8s.io/client-go/pkg/version/base.go

@@ -3,8 +3,8 @@ package version
 var (
 	gitMajor = "1"
 	gitMinor = "12"
-	gitVersion   = "v1.13.1-k3s2"
-	gitCommit    = "2054ab40be790e5f209e480b18ab8a7015262796"
+	gitVersion   = "v1.13.2-k3s2"
+	gitCommit    = "149316edfe67e696f622608282a3fc4252ad92b3"
 	gitTreeState = "clean" 
-	buildDate = "2019-01-09T20:07+00:00Z"
+	buildDate = "2019-01-22T20:48+00:00Z"
 )

vendor/k8s.io/client-go/transport/token_source.go

@@ -47,14 +47,14 @@ func TokenSourceWrapTransport(ts oauth2.TokenSource) func(http.RoundTripper) htt
 func NewCachedFileTokenSource(path string) oauth2.TokenSource {
 	return &cachingTokenSource{
 		now:    time.Now,
-		leeway: 1 * time.Minute,
+		leeway: 10 * time.Second,
 		base: &fileTokenSource{
 			path: path,
-			// This period was picked because it is half of the minimum validity
-			// duration for a token provisioned by they TokenRequest API. This is
-			// unsophisticated and should induce rotation at a frequency that should
-			// work with the token volume source.
-			period: 5 * time.Minute,
+			// This period was picked because it is half of the duration between when the kubelet
+			// refreshes a projected service account token and when the original token expires.
+			// Default token lifetime is 10 minutes, and the kubelet starts refreshing at 80% of lifetime.
+			// This should induce re-reading at a frequency that works with the token volume source.
+			period: time.Minute,
 		},
 	}
 }

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/controllermanager.go

@@ -412,7 +412,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
 
 	// If apiserver is not running we should wait for some time and fail only then. This is particularly
 	// important when we start apiserver and controller manager at the same time.
-	if err := genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second); err != nil {
+	if err := genericcontrollermanager.WaitForAPIServer(versionedClient, 120*time.Second); err != nil {
 		return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err)
 	}
 

vendor/k8s.io/kubernetes/deps.sh

@@ -11,7 +11,7 @@ package=k8s.io/kubernetes
 package=k8s.io/kubernetes/cmd/hyperkube
 $(cat ./Godeps/Godeps.json | jq -r '(.Deps | .[] | "\(.ImportPath) \(.Comment) \(.Rev)\n")' | sed 's/null//' | awk '{print $1 " " $2}' | grep -Ev 'github.com/opencontainers/runc|bitbucket.org/ww/goautoneg|github.com/google/cadvisor' | sort -k2,1 | uniq -f1)
 bitbucket.org/ww/goautoneg       a547fc61f48d567d5b4ec6f8aee5573d8efce11d  https://github.com/rancher/goautoneg.git
-github.com/ibuildthecloud/kvsql  6bb3d252056655760ed8ca6557d6d5e607b361d2
+github.com/ibuildthecloud/kvsql  8dfe3deb0646c4817567e4a53ed1dea41ea5668f
 github.com/google/cadvisor       91dab6eb91496ed68acbef68b02b34b3392ca754  https://github.com/ibuildthecloud/cadvisor.git
 github.com/opencontainers/runc   96ec2177ae841256168fcf76954f7177af9446eb
 EOF

vendor/k8s.io/kubernetes/pkg/controller/podautoscaler/horizontal.go

@@ -207,14 +207,18 @@ func (a *HorizontalController) processNextWorkItem() bool {
 	}
 	defer a.queue.Done(key)
 
-	err := a.reconcileKey(key.(string))
-	if err == nil {
-		// don't "forget" here because we want to only process a given HPA once per resync interval
-		return true
+	deleted, err := a.reconcileKey(key.(string))
+	if err != nil {
+		utilruntime.HandleError(err)
+	}
+	// Add request processing HPA after resync interval just in case last resync didn't insert
+	// request into the queue. Request is not inserted into queue by resync if previous one wasn't processed yet.
+	// This happens quite often because requests from previous resync are removed from the queue at the same moment
+	// as next resync inserts new requests.
+	if !deleted {
+		a.queue.AddRateLimited(key)
 	}
 
-	a.queue.AddRateLimited(key)
-	utilruntime.HandleError(err)
 	return true
 }
 
@@ -298,20 +302,20 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori
 	return replicas, metric, statuses, timestamp, nil
 }
 
-func (a *HorizontalController) reconcileKey(key string) error {
+func (a *HorizontalController) reconcileKey(key string) (deleted bool, err error) {
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
-		return err
+		return true, err
 	}
 
 	hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
 	if errors.IsNotFound(err) {
 		klog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
 		delete(a.recommendations, key)
-		return nil
+		return true, nil
 	}
 
-	return a.reconcileAutoscaler(hpa, key)
+	return false, a.reconcileAutoscaler(hpa, key)
 }
 
 // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.

vendor/k8s.io/kubernetes/pkg/kubectl/generate/versioned/secret_for_docker_registry.go

@@ -17,6 +17,7 @@ limitations under the License.
 package versioned
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 
@@ -152,6 +153,7 @@ func handleDockerCfgJSONContent(username, password, email, server string) ([]byt
 		Username: username,
 		Password: password,
 		Email:    email,
+		Auth:     encodeDockerConfigFieldAuth(username, password),
 	}
 
 	dockerCfgJSON := DockerConfigJSON{
@@ -161,6 +163,11 @@ func handleDockerCfgJSONContent(username, password, email, server string) ([]byt
 	return json.Marshal(dockerCfgJSON)
 }
 
+func encodeDockerConfigFieldAuth(username, password string) string {
+	fieldValue := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(fieldValue))
+}
+
 // DockerConfigJSON represents a local docker auth config file
 // for pulling images.
 type DockerConfigJSON struct {
@@ -175,7 +182,8 @@ type DockerConfigJSON struct {
 type DockerConfig map[string]DockerConfigEntry
 
 type DockerConfigEntry struct {
-	Username string
-	Password string
-	Email    string
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	Email    string `json:"email,omitempty"`
+	Auth     string `json:"auth,omitempty"`
 }

vendor/k8s.io/kubernetes/pkg/kubelet/BUILD

@@ -165,6 +165,7 @@ go_test(
         "kubelet_pods_windows_test.go",
         "kubelet_resources_test.go",
         "kubelet_test.go",
+        "kubelet_volumes_linux_test.go",
         "kubelet_volumes_test.go",
         "main_test.go",
         "oom_watcher_test.go",

vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go

@@ -158,7 +158,7 @@ func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
 		return f, fmt.Errorf("%s - %v", localErr, err)
 	}
 
-	expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
+	expectedCgroups := sets.NewString("cpu", "cpuacct", "memory")
 	for _, mountPoint := range mountPoints {
 		if mountPoint.Type == cgroupMountType {
 			for _, opt := range mountPoint.Opts {

vendor/k8s.io/kubernetes/pkg/kubelet/config/defaults.go

@@ -19,6 +19,7 @@ package config
 const (
 	DefaultKubeletPodsDirName                = "pods"
 	DefaultKubeletVolumesDirName             = "volumes"
+	DefaultKubeletVolumeSubpathsDirName      = "volume-subpaths"
 	DefaultKubeletVolumeDevicesDirName       = "volumeDevices"
 	DefaultKubeletPluginsDirName             = "plugins"
 	DefaultKubeletPluginsRegistrationDirName = "plugins_registry"

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_getters.go

@@ -25,7 +25,7 @@ import (
 	cadvisorapiv1 "github.com/google/cadvisor/info/v1"
 	"k8s.io/klog"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
@@ -99,6 +99,13 @@ func (kl *Kubelet) getPodDir(podUID types.UID) string {
 	return filepath.Join(kl.getPodsDir(), string(podUID))
 }
 
+// getPodVolumesSubpathsDir returns the full path to the per-pod subpaths directory under
+// which subpath volumes are created for the specified pod.  This directory may not
+// exist if the pod does not exist or subpaths are not specified.
+func (kl *Kubelet) getPodVolumeSubpathsDir(podUID types.UID) string {
+	return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletVolumeSubpathsDirName)
+}
+
 // getPodVolumesDir returns the full path to the per-pod data directory under
 // which volumes are created for the specified pod.  This directory may not
 // exist if the pod does not exist.
@@ -310,6 +317,19 @@ func (kl *Kubelet) getMountedVolumePathListFromDisk(podUID types.UID) ([]string,
 	return mountedVolumes, nil
 }
 
+// podVolumesSubpathsDirExists returns true if the pod volume-subpaths directory for
+// a given pod exists
+func (kl *Kubelet) podVolumeSubpathsDirExists(podUID types.UID) (bool, error) {
+	podVolDir := kl.getPodVolumeSubpathsDir(podUID)
+
+	if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
+		return true, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
+	} else if !pathExists {
+		return false, nil
+	}
+	return true, nil
+}
+
 // GetVersionInfo returns information about the version of cAdvisor in use.
 func (kl *Kubelet) GetVersionInfo() (*cadvisorapiv1.VersionInfo, error) {
 	return kl.cadvisor.VersionInfo()

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_volumes.go

@@ -19,7 +19,7 @@ package kubelet
 import (
 	"fmt"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -114,6 +114,8 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon
 		}
 		// If volumes have not been unmounted/detached, do not delete directory.
 		// Doing so may result in corruption of data.
+		// TODO: getMountedVolumePathListFromDisk() call may be redundant with
+		// kl.getPodVolumePathListFromDisk(). Can this be cleaned up?
 		if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist {
 			klog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
 			continue
@@ -128,6 +130,18 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon
 			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk", uid))
 			continue
 		}
+
+		// If there are any volume-subpaths, do not cleanup directories
+		volumeSubpathExists, err := kl.podVolumeSubpathsDirExists(uid)
+		if err != nil {
+			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but error %v occurred during reading of volume-subpaths dir from disk", uid, err))
+			continue
+		}
+		if volumeSubpathExists {
+			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume subpaths are still present on disk", uid))
+			continue
+		}
+
 		klog.V(3).Infof("Orphaned pod %q found, removing", uid)
 		if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil {
 			klog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)

vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/util:go_default_library",
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/volume:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",

vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/summary.go

@@ -19,7 +19,11 @@ package stats
 import (
 	"fmt"
 
+	"k8s.io/klog"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
 type SummaryProvider interface {
@@ -32,6 +36,11 @@ type SummaryProvider interface {
 
 // summaryProviderImpl implements the SummaryProvider interface.
 type summaryProviderImpl struct {
+	// kubeletCreationTime is the time at which the summaryProvider was created.
+	kubeletCreationTime metav1.Time
+	// systemBootTime is the time at which the system was started
+	systemBootTime metav1.Time
+
 	provider StatsProvider
 }
 
@@ -40,7 +49,18 @@ var _ SummaryProvider = &summaryProviderImpl{}
 // NewSummaryProvider returns a SummaryProvider using the stats provided by the
 // specified statsProvider.
 func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider {
-	return &summaryProviderImpl{statsProvider}
+	kubeletCreationTime := metav1.Now()
+	bootTime, err := util.GetBootTime()
+	if err != nil {
+		// bootTime will be zero if we encounter an error getting the boot time.
+		klog.Warningf("Error getting system boot time.  Node metrics will have an incorrect start time: %v", err)
+	}
+
+	return &summaryProviderImpl{
+		kubeletCreationTime: kubeletCreationTime,
+		systemBootTime:      metav1.NewTime(bootTime),
+		provider:            statsProvider,
+	}
 }
 
 func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) {
@@ -77,7 +97,7 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
 		CPU:              rootStats.CPU,
 		Memory:           rootStats.Memory,
 		Network:          networkStats,
-		StartTime:        rootStats.StartTime,
+		StartTime:        sp.systemBootTime,
 		Fs:               rootFsStats,
 		Runtime:          &statsapi.RuntimeStats{ImageFs: imageFsStats},
 		Rlimit:           rlimit,

vendor/k8s.io/kubernetes/pkg/kubelet/server/stats/summary_sys_containers.go

@@ -21,6 +21,7 @@ package stats
 import (
 	"k8s.io/klog"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 )
@@ -29,11 +30,12 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig
 	systemContainers := map[string]struct {
 		name             string
 		forceStatsUpdate bool
+		startTime        metav1.Time
 	}{
-		statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
-		statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
-		statsapi.SystemContainerMisc:    {nodeConfig.SystemCgroupsName, false},
-		statsapi.SystemContainerPods:    {sp.provider.GetPodCgroupRoot(), updateStats},
+		statsapi.SystemContainerKubelet: {name: nodeConfig.KubeletCgroupsName, forceStatsUpdate: false, startTime: sp.kubeletCreationTime},
+		statsapi.SystemContainerRuntime: {name: nodeConfig.RuntimeCgroupsName, forceStatsUpdate: false},
+		statsapi.SystemContainerMisc:    {name: nodeConfig.SystemCgroupsName, forceStatsUpdate: false},
+		statsapi.SystemContainerPods:    {name: sp.provider.GetPodCgroupRoot(), forceStatsUpdate: updateStats},
 	}
 	for sys, cont := range systemContainers {
 		// skip if cgroup name is undefined (not all system containers are required)
@@ -48,6 +50,11 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig
 		// System containers don't have a filesystem associated with them.
 		s.Logs, s.Rootfs = nil, nil
 		s.Name = sys
+
+		// if we know the start time of a system container, use that instead of the start time provided by cAdvisor
+		if !cont.startTime.IsZero() {
+			s.StartTime = cont.startTime
+		}
 		stats = append(stats, *s)
 	}
 
@@ -58,11 +65,12 @@ func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig c
 	systemContainers := map[string]struct {
 		name             string
 		forceStatsUpdate bool
+		startTime        metav1.Time
 	}{
-		statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
-		statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
-		statsapi.SystemContainerMisc:    {nodeConfig.SystemCgroupsName, false},
-		statsapi.SystemContainerPods:    {sp.provider.GetPodCgroupRoot(), updateStats},
+		statsapi.SystemContainerKubelet: {name: nodeConfig.KubeletCgroupsName, forceStatsUpdate: false, startTime: sp.kubeletCreationTime},
+		statsapi.SystemContainerRuntime: {name: nodeConfig.RuntimeCgroupsName, forceStatsUpdate: false},
+		statsapi.SystemContainerMisc:    {name: nodeConfig.SystemCgroupsName, forceStatsUpdate: false},
+		statsapi.SystemContainerPods:    {name: sp.provider.GetPodCgroupRoot(), forceStatsUpdate: updateStats},
 	}
 	for sys, cont := range systemContainers {
 		// skip if cgroup name is undefined (not all system containers are required)
@@ -75,6 +83,11 @@ func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig c
 			continue
 		}
 		s.Name = sys
+
+		// if we know the start time of a system container, use that instead of the start time provided by cAdvisor
+		if !cont.startTime.IsZero() {
+			s.StartTime = cont.startTime
+		}
 		stats = append(stats, *s)
 	}
 

vendor/k8s.io/kubernetes/pkg/kubelet/util/BUILD

@@ -34,6 +34,8 @@ go_test(
 go_library(
     name = "go_default_library",
     srcs = [
+        "boottime_util_darwin.go",
+        "boottime_util_linux.go",
         "doc.go",
         "util.go",
         "util_unix.go",

vendor/k8s.io/kubernetes/pkg/kubelet/util/boottime_util_darwin.go

+// +build darwin
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// GetBootTime returns the time at which the machine was started, truncated to the nearest second
+func GetBootTime() (time.Time, error) {
+	output, err := unix.SysctlRaw("kern.boottime")
+	if err != nil {
+		return time.Time{}, err
+	}
+	var timeval syscall.Timeval
+	if len(output) != int(unsafe.Sizeof(timeval)) {
+		return time.Time{}, fmt.Errorf("unexpected output when calling syscall kern.bootime.  Expected len(output) to be %v, but got %v",
+			int(unsafe.Sizeof(timeval)), len(output))
+	}
+	timeval = *(*syscall.Timeval)(unsafe.Pointer(&output[0]))
+	sec, nsec := timeval.Unix()
+	return time.Unix(sec, nsec).Truncate(time.Second), nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/util/boottime_util_linux.go

+// +build freebsd linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// GetBootTime returns the time at which the machine was started, truncated to the nearest second
+func GetBootTime() (time.Time, error) {
+	currentTime := time.Now()
+	var info unix.Sysinfo_t
+	if err := unix.Sysinfo(&info); err != nil {
+		return time.Time{}, fmt.Errorf("error getting system uptime: %s", err)
+	}
+	return currentTime.Add(-time.Duration(info.Uptime) * time.Second).Truncate(time.Second), nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/util/util_unsupported.go

@@ -45,3 +45,8 @@ func UnlockPath(fileHandles []uintptr) {
 func LocalEndpoint(path, file string) string {
 	return ""
 }
+
+// GetBootTime empty implementation
+func GetBootTime() (time.Time, error) {
+	return time.Time{}, fmt.Errorf("GetBootTime is unsupported in this build")
+}

vendor/k8s.io/kubernetes/pkg/kubelet/util/util_windows.go

@@ -23,6 +23,7 @@ import (
 	"net"
 	"net/url"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/Microsoft/go-winio"
@@ -112,3 +113,15 @@ func LocalEndpoint(path, file string) string {
 	}
 	return u.String() + "//./pipe/" + file
 }
+
+var tickCount = syscall.NewLazyDLL("kernel32.dll").NewProc("GetTickCount64")
+
+// GetBootTime returns the time at which the machine was started, truncated to the nearest second
+func GetBootTime() (time.Time, error) {
+	currentTime := time.Now()
+	output, _, err := tickCount.Call()
+	if errno, ok := err.(syscall.Errno); !ok || errno != 0 {
+		return time.Time{}, err
+	}
+	return currentTime.Add(-time.Duration(output) * time.Millisecond).Truncate(time.Second), nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/cache/desired_state_of_world.go

@@ -203,11 +203,12 @@ func (dsw *desiredStateOfWorld) AddPodToVolume(
 
 	var volumeName v1.UniqueVolumeName
 
-	// The unique volume name used depends on whether the volume is attachable
+	// The unique volume name used depends on whether the volume is attachable/device-mountable
 	// or not.
 	attachable := dsw.isAttachableVolume(volumeSpec)
-	if attachable {
-		// For attachable volumes, use the unique volume name as reported by
+	deviceMountable := dsw.isDeviceMountableVolume(volumeSpec)
+	if attachable || deviceMountable {
+		// For attachable/device-mountable volumes, use the unique volume name as reported by
 		// the plugin.
 		volumeName, err =
 			util.GetUniqueVolumeNameFromSpec(volumePlugin, volumeSpec)
@@ -219,13 +220,11 @@ func (dsw *desiredStateOfWorld) AddPodToVolume(
 				err)
 		}
 	} else {
-		// For non-attachable volumes, generate a unique name based on the pod
+		// For non-attachable and non-device-mountable volumes, generate a unique name based on the pod
 		// namespace and name and the name of the volume within the pod.
-		volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec)
+		volumeName = util.GetUniqueVolumeNameFromSpecWithPod(podName, volumePlugin, volumeSpec)
 	}
 
-	deviceMountable := dsw.isDeviceMountableVolume(volumeSpec)
-
 	if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists {
 		dsw.volumesToMount[volumeName] = volumeToMount{
 			volumeName:              volumeName,

vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler/reconciler.go

@@ -439,6 +439,10 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
 	if err != nil {
 		return nil, err
 	}
+	deviceMountablePlugin, err := rc.volumePluginMgr.FindDeviceMountablePluginByName(volume.pluginName)
+	if err != nil {
+		return nil, err
+	}
 
 	// Create pod object
 	pod := &v1.Pod{
@@ -464,13 +468,13 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
 	}
 
 	var uniqueVolumeName v1.UniqueVolumeName
-	if attachablePlugin != nil {
+	if attachablePlugin != nil || deviceMountablePlugin != nil {
 		uniqueVolumeName, err = util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		uniqueVolumeName = util.GetUniqueVolumeNameForNonAttachableVolume(volume.podName, plugin, volumeSpec)
+		uniqueVolumeName = util.GetUniqueVolumeNameFromSpecWithPod(volume.podName, plugin, volumeSpec)
 	}
 	// Check existence of mount point for filesystem volume or symbolic link for block volume
 	isExist, checkErr := rc.operationExecutor.CheckVolumeExistenceOperation(volumeSpec, volume.mountPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin)

vendor/k8s.io/kubernetes/pkg/proxy/ipvs/graceful_termination.go

@@ -168,6 +168,7 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e
 			// For UDP, ActiveConn is always 0
 			// For TCP, InactiveConn are connections not in ESTABLISHED state
 			if rs.ActiveConn+rs.InactiveConn != 0 {
+				klog.Infof("Not deleting, RS %v: %v ActiveConn, %v InactiveConn", rsToDelete.String(), rs.ActiveConn, rs.InactiveConn)
 				return false, nil
 			}
 			klog.Infof("Deleting rs: %s", rsToDelete.String())

vendor/k8s.io/kubernetes/pkg/proxy/ipvs/proxier.go

@@ -162,6 +162,8 @@ const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
 const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
 const sysctlVSConnTrack = "net/ipv4/vs/conntrack"
 const sysctlConnReuse = "net/ipv4/vs/conn_reuse_mode"
+const sysctlExpireNoDestConn = "net/ipv4/vs/expire_nodest_conn"
+const sysctlExpireQuiescentTemplate = "net/ipv4/vs/expire_quiescent_template"
 const sysctlForward = "net/ipv4/ip_forward"
 const sysctlArpIgnore = "net/ipv4/conf/all/arp_ignore"
 const sysctlArpAnnounce = "net/ipv4/conf/all/arp_announce"
@@ -321,6 +323,20 @@ func NewProxier(ipt utiliptables.Interface,
 		}
 	}
 
+	// Set the expire_nodest_conn sysctl we need for
+	if val, _ := sysctl.GetSysctl(sysctlExpireNoDestConn); val != 1 {
+		if err := sysctl.SetSysctl(sysctlExpireNoDestConn, 1); err != nil {
+			return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlExpireNoDestConn, err)
+		}
+	}
+
+	// Set the expire_quiescent_template sysctl we need for
+	if val, _ := sysctl.GetSysctl(sysctlExpireQuiescentTemplate); val != 1 {
+		if err := sysctl.SetSysctl(sysctlExpireQuiescentTemplate, 1); err != nil {
+			return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlExpireQuiescentTemplate, err)
+		}
+	}
+
 	// Set the ip_forward sysctl we need for
 	if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 {
 		if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {
@@ -1190,7 +1206,15 @@ func (proxier *Proxier) syncProxyRules() {
 	}
 	proxier.portsMap = replacementPortsMap
 
-	// Clean up legacy IPVS services
+	// Get legacy bind address
+	// currentBindAddrs represents ip addresses bind to DefaultDummyDevice from the system
+	currentBindAddrs, err := proxier.netlinkHandle.ListBindAddress(DefaultDummyDevice)
+	if err != nil {
+		klog.Errorf("Failed to get bind address, err: %v", err)
+	}
+	legacyBindAddrs := proxier.getLegacyBindAddr(activeBindAddrs, currentBindAddrs)
+
+	// Clean up legacy IPVS services and unbind addresses
 	appliedSvcs, err := proxier.ipvs.GetVirtualServers()
 	if err == nil {
 		for _, appliedSvc := range appliedSvcs {
@@ -1199,15 +1223,7 @@ func (proxier *Proxier) syncProxyRules() {
 	} else {
 		klog.Errorf("Failed to get ipvs service, err: %v", err)
 	}
-	proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices)
-
-	// Clean up legacy bind address
-	// currentBindAddrs represents ip addresses bind to DefaultDummyDevice from the system
-	currentBindAddrs, err := proxier.netlinkHandle.ListBindAddress(DefaultDummyDevice)
-	if err != nil {
-		klog.Errorf("Failed to get bind address, err: %v", err)
-	}
-	proxier.cleanLegacyBindAddr(activeBindAddrs, currentBindAddrs)
+	proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices, legacyBindAddrs)
 
 	// Update healthz timestamp
 	if proxier.healthzServer != nil {
@@ -1605,29 +1621,38 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
 		klog.V(5).Infof("Using graceful delete to delete: %v", uniqueRS)
 		err = proxier.gracefuldeleteManager.GracefulDeleteRS(appliedVirtualServer, delDest)
 		if err != nil {
-			klog.Errorf("Failed to delete destination: %v, error: %v", delDest, err)
+			klog.Errorf("Failed to delete destination: %v, error: %v", uniqueRS, err)
 			continue
 		}
 	}
 	return nil
 }
 
-func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer) {
+func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer, legacyBindAddrs map[string]bool) {
 	for cs := range currentServices {
 		svc := currentServices[cs]
 		if _, ok := activeServices[cs]; !ok {
 			// This service was not processed in the latest sync loop so before deleting it,
-			// make sure it does not fall within an excluded CIDR range.
 			okayToDelete := true
 			rsList, _ := proxier.ipvs.GetRealServers(svc)
+
+			// If we still have real servers graceful termination is not done
+			if len(rsList) > 0 {
+				okayToDelete = false
+			}
+			// Applying graceful termination to all real servers
 			for _, rs := range rsList {
 				uniqueRS := GetUniqueRSName(svc, rs)
-				// if there are in terminating real server in this service, then handle it later
+				// If RS is already in the graceful termination list, no need to add it again
 				if proxier.gracefuldeleteManager.InTerminationList(uniqueRS) {
-					okayToDelete = false
-					break
+					continue
+				}
+				klog.V(5).Infof("Using graceful delete to delete: %v", uniqueRS)
+				if err := proxier.gracefuldeleteManager.GracefulDeleteRS(svc, rs); err != nil {
+					klog.Errorf("Failed to delete destination: %v, error: %v", uniqueRS, err)
 				}
 			}
+			// make sure it does not fall within an excluded CIDR range.
 			for _, excludedCIDR := range proxier.excludeCIDRs {
 				// Any validation of this CIDR already should have occurred.
 				_, n, _ := net.ParseCIDR(excludedCIDR)
@@ -1637,26 +1662,33 @@ func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, curre
 				}
 			}
 			if okayToDelete {
+				klog.V(4).Infof("Delete service %s", svc.String())
 				if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil {
-					klog.Errorf("Failed to delete service, error: %v", err)
+					klog.Errorf("Failed to delete service %s, error: %v", svc.String(), err)
+				}
+				addr := svc.Address.String()
+				if _, ok := legacyBindAddrs[addr]; ok {
+					klog.V(4).Infof("Unbinding address %s", addr)
+					if err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice); err != nil {
+						klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err)
+					} else {
+						// In case we delete a multi-port service, avoid trying to unbind multiple times
+						delete(legacyBindAddrs, addr)
+					}
 				}
 			}
 		}
 	}
 }
 
-func (proxier *Proxier) cleanLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) {
+func (proxier *Proxier) getLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) map[string]bool {
+	legacyAddrs := make(map[string]bool)
 	for _, addr := range currentBindAddrs {
 		if _, ok := activeBindAddrs[addr]; !ok {
-			// This address was not processed in the latest sync loop
-			klog.V(4).Infof("Unbind addr %s", addr)
-			err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice)
-			// Ignore no such address error when try to unbind address
-			if err != nil {
-				klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err)
-			}
+			legacyAddrs[addr] = true
 		}
 	}
+	return legacyAddrs
 }
 
 // Join all words with spaces, terminate with newline and write to buff.

vendor/k8s.io/kubernetes/pkg/registry/scheduling/rest/storage_scheduling.go

@@ -89,7 +89,7 @@ func AddSystemPriorityClasses() genericapiserver.PostStartHookFunc {
 					} else {
 						// Unable to get the priority class for reasons other than "not found".
 						klog.Warningf("unable to get PriorityClass %v: %v. Retrying...", pc.Name, err)
-						return false, err
+						return false, nil
 					}
 				}
 			}

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/BUILD

@@ -17,7 +17,6 @@ go_library(
     deps = [
         "//pkg/scheduler/api:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
-        "//pkg/scheduler/internal/cache:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/policy/v1beta1:go_default_library",

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/types.go

@@ -23,7 +23,6 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
-	schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
 )
 
 // NodeFieldSelectorKeys is a map that: the key are node field selector keys; the values are
@@ -92,6 +91,9 @@ type NodeLister interface {
 	List() ([]*v1.Node, error)
 }
 
+// PodFilter is a function to filter a pod. If pod passed return true else return false.
+type PodFilter func(*v1.Pod) bool
+
 // PodLister interface represents anything that can list pods for a scheduler.
 type PodLister interface {
 	// We explicitly return []*v1.Pod, instead of v1.PodList, to avoid
@@ -99,7 +101,7 @@ type PodLister interface {
 	List(labels.Selector) ([]*v1.Pod, error)
 	// This is similar to "List()", but the returned slice does not
 	// contain pods that don't pass `podFilter`.
-	FilteredList(podFilter schedulerinternalcache.PodFilter, selector labels.Selector) ([]*v1.Pod, error)
+	FilteredList(podFilter PodFilter, selector labels.Selector) ([]*v1.Pod, error)
 }
 
 // ServiceLister interface represents anything that can produce a list of services; the list is consumed by a scheduler.

vendor/k8s.io/kubernetes/pkg/scheduler/core/generic_scheduler.go

@@ -337,7 +337,7 @@ func (g *genericScheduler) processPreemptionWithExtenders(
 // worth the complexity, especially because we generally expect to have a very
 // small number of nominated pods per node.
 func (g *genericScheduler) getLowerPriorityNominatedPods(pod *v1.Pod, nodeName string) []*v1.Pod {
-	pods := g.schedulingQueue.WaitingPodsForNode(nodeName)
+	pods := g.schedulingQueue.NominatedPodsForNode(nodeName)
 
 	if len(pods) == 0 {
 		return nil
@@ -475,7 +475,7 @@ func addNominatedPods(pod *v1.Pod, meta algorithm.PredicateMetadata,
 		// This may happen only in tests.
 		return false, meta, nodeInfo
 	}
-	nominatedPods := queue.WaitingPodsForNode(nodeInfo.Node().Name)
+	nominatedPods := queue.NominatedPodsForNode(nodeInfo.Node().Name)
 	if nominatedPods == nil || len(nominatedPods) == 0 {
 		return false, meta, nodeInfo
 	}

vendor/k8s.io/kubernetes/pkg/scheduler/internal/cache/BUILD

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/features:go_default_library",
+        "//pkg/scheduler/algorithm:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/util/node:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",

vendor/k8s.io/kubernetes/pkg/scheduler/internal/cache/cache.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
 
 	"k8s.io/klog"
@@ -143,7 +144,7 @@ func (cache *schedulerCache) List(selector labels.Selector) ([]*v1.Pod, error) {
 	return cache.FilteredList(alwaysTrue, selector)
 }
 
-func (cache *schedulerCache) FilteredList(podFilter PodFilter, selector labels.Selector) ([]*v1.Pod, error) {
+func (cache *schedulerCache) FilteredList(podFilter algorithm.PodFilter, selector labels.Selector) ([]*v1.Pod, error) {
 	cache.mu.RLock()
 	defer cache.mu.RUnlock()
 	// podFilter is expected to return true for most or all of the pods. We

vendor/k8s.io/kubernetes/pkg/scheduler/internal/cache/interface.go

@@ -19,12 +19,10 @@ package cache
 import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
 )
 
-// PodFilter is a function to filter a pod. If pod passed return true else return false.
-type PodFilter func(*v1.Pod) bool
-
 // Cache collects pods' information and provides node-level aggregated information.
 // It's intended for generic scheduler to do efficient lookup.
 // Cache's operations are pod centric. It does incremental updates based on pod events.
@@ -106,7 +104,7 @@ type Cache interface {
 	List(labels.Selector) ([]*v1.Pod, error)
 
 	// FilteredList returns all cached pods that pass the filter.
-	FilteredList(filter PodFilter, selector labels.Selector) ([]*v1.Pod, error)
+	FilteredList(filter algorithm.PodFilter, selector labels.Selector) ([]*v1.Pod, error)
 
 	// Snapshot takes a snapshot on current cache
 	Snapshot() *Snapshot

vendor/k8s.io/kubernetes/pkg/scheduler/internal/queue/BUILD

@@ -12,6 +12,7 @@ go_library(
         "//pkg/scheduler/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

vendor/k8s.io/kubernetes/pkg/scheduler/scheduler.go

@@ -316,11 +316,19 @@ func (sched *Scheduler) preempt(preemptor *v1.Pod, scheduleErr error) (string, e
 	var nodeName = ""
 	if node != nil {
 		nodeName = node.Name
+		// Update the scheduling queue with the nominated pod information. Without
+		// this, there would be a race condition between the next scheduling cycle
+		// and the time the scheduler receives a Pod Update for the nominated pod.
+		sched.config.SchedulingQueue.UpdateNominatedPodForNode(preemptor, nodeName)
+
+		// Make a call to update nominated node name of the pod on the API server.
 		err = sched.config.PodPreemptor.SetNominatedNodeName(preemptor, nodeName)
 		if err != nil {
 			klog.Errorf("Error in preemption process. Cannot update pod %v/%v annotations: %v", preemptor.Namespace, preemptor.Name, err)
+			sched.config.SchedulingQueue.DeleteNominatedPodIfExists(preemptor)
 			return "", err
 		}
+
 		for _, victim := range victims {
 			if err := sched.config.PodPreemptor.DeletePod(victim); err != nil {
 				klog.Errorf("Error preempting pod %v/%v: %v", victim.Namespace, victim.Name, err)

vendor/k8s.io/kubernetes/pkg/util/ipvs/ipvs_linux.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"sync"
 	"syscall"
 
 	libipvs "github.com/docker/libnetwork/ipvs"
@@ -34,6 +35,7 @@ import (
 type runner struct {
 	exec       utilexec.Interface
 	ipvsHandle *libipvs.Handle
+	mu         sync.Mutex // Protect Netlink calls
 }
 
 // Protocol is the IPVS service protocol type
@@ -58,6 +60,8 @@ func (runner *runner) AddVirtualServer(vs *VirtualServer) error {
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.NewService(svc)
 }
 
@@ -67,6 +71,8 @@ func (runner *runner) UpdateVirtualServer(vs *VirtualServer) error {
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.UpdateService(svc)
 }
 
@@ -76,6 +82,8 @@ func (runner *runner) DeleteVirtualServer(vs *VirtualServer) error {
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.DelService(svc)
 }
 
@@ -85,7 +93,10 @@ func (runner *runner) GetVirtualServer(vs *VirtualServer) (*VirtualServer, error
 	if err != nil {
 		return nil, err
 	}
+	runner.mu.Lock()
 	ipvsSvc, err := runner.ipvsHandle.GetService(svc)
+	runner.mu.Unlock()
+
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +109,9 @@ func (runner *runner) GetVirtualServer(vs *VirtualServer) (*VirtualServer, error
 
 // GetVirtualServers is part of ipvs.Interface.
 func (runner *runner) GetVirtualServers() ([]*VirtualServer, error) {
+	runner.mu.Lock()
 	ipvsSvcs, err := runner.ipvsHandle.GetServices()
+	runner.mu.Unlock()
 	if err != nil {
 		return nil, err
 	}
@@ -115,6 +128,8 @@ func (runner *runner) GetVirtualServers() ([]*VirtualServer, error) {
 
 // Flush is part of ipvs.Interface. Currently we delete IPVS services one by one
 func (runner *runner) Flush() error {
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.Flush()
 }
 
@@ -128,6 +143,8 @@ func (runner *runner) AddRealServer(vs *VirtualServer, rs *RealServer) error {
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.NewDestination(svc, dst)
 }
 
@@ -141,6 +158,8 @@ func (runner *runner) DeleteRealServer(vs *VirtualServer, rs *RealServer) error
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.DelDestination(svc, dst)
 }
 
@@ -153,6 +172,8 @@ func (runner *runner) UpdateRealServer(vs *VirtualServer, rs *RealServer) error
 	if err != nil {
 		return err
 	}
+	runner.mu.Lock()
+	defer runner.mu.Unlock()
 	return runner.ipvsHandle.UpdateDestination(svc, dst)
 }
 
@@ -162,7 +183,9 @@ func (runner *runner) GetRealServers(vs *VirtualServer) ([]*RealServer, error) {
 	if err != nil {
 		return nil, err
 	}
+	runner.mu.Lock()
 	dsts, err := runner.ipvsHandle.GetDestinations(svc)
+	runner.mu.Unlock()
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/kubernetes/pkg/util/mount/BUILD

@@ -9,6 +9,7 @@ go_library(
         "exec_mount_unsupported.go",
         "fake.go",
         "mount.go",
+        "mount_helper.go",
         "mount_linux.go",
         "mount_unsupported.go",
         "mount_windows.go",
@@ -67,6 +68,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "exec_mount_test.go",
+        "mount_helper_test.go",
         "mount_linux_test.go",
         "mount_test.go",
         "mount_windows_test.go",

vendor/k8s.io/kubernetes/pkg/util/mount/fake.go

@@ -30,6 +30,8 @@ type FakeMounter struct {
 	MountPoints []MountPoint
 	Log         []FakeAction
 	Filesystem  map[string]FileType
+	// Error to return for a path when calling IsLikelyNotMountPoint
+	MountCheckErrors map[string]error
 	// Some tests run things in parallel, make sure the mounter does not produce
 	// any golang's DATA RACE warnings.
 	mutex sync.Mutex
@@ -119,6 +121,7 @@ func (f *FakeMounter) Unmount(target string) error {
 	}
 	f.MountPoints = newMountpoints
 	f.Log = append(f.Log, FakeAction{Action: FakeActionUnmount, Target: absTarget})
+	delete(f.MountCheckErrors, target)
 	return nil
 }
 
@@ -141,7 +144,12 @@ func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	f.mutex.Lock()
 	defer f.mutex.Unlock()
 
-	_, err := os.Stat(file)
+	err := f.MountCheckErrors[file]
+	if err != nil {
+		return false, err
+	}
+
+	_, err = os.Stat(file)
 	if err != nil {
 		return true, err
 	}

vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+
+	"k8s.io/klog"
+)
+
+// CleanupMountPoint unmounts the given path and
+// deletes the remaining directory if successful.
+// if extensiveMountPointCheck is true
+// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
+// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs.
+func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error {
+	// mounter.ExistsPath cannot be used because for containerized kubelet, we need to check
+	// the path in the kubelet container, not on the host.
+	pathExists, pathErr := PathExists(mountPath)
+	if !pathExists {
+		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
+		return nil
+	}
+	corruptedMnt := IsCorruptedMnt(pathErr)
+	if pathErr != nil && !corruptedMnt {
+		return fmt.Errorf("Error checking path: %v", pathErr)
+	}
+	return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt)
+}
+
+// doCleanupMountPoint unmounts the given path and
+// deletes the remaining directory if successful.
+// if extensiveMountPointCheck is true
+// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
+// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs.
+// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check
+// will be skipped
+func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error {
+	if !corruptedMnt {
+		var notMnt bool
+		var err error
+		if extensiveMountPointCheck {
+			notMnt, err = IsNotMountPoint(mounter, mountPath)
+		} else {
+			notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if notMnt {
+			klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
+			return os.Remove(mountPath)
+		}
+	}
+
+	// Unmount the mount path
+	klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
+	if err := mounter.Unmount(mountPath); err != nil {
+		return err
+	}
+
+	notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath)
+	if mntErr != nil {
+		return mntErr
+	}
+	if notMnt {
+		klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
+		return os.Remove(mountPath)
+	}
+	return fmt.Errorf("Failed to unmount path %v", mountPath)
+}
+
+// TODO: clean this up to use pkg/util/file/FileExists
+// PathExists returns true if the specified path exists.
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else if IsCorruptedMnt(err) {
+		return true, err
+	} else {
+		return false, err
+	}
+}
+
+// IsCorruptedMnt return true if err is about corrupted mount point
+func IsCorruptedMnt(err error) bool {
+	if err == nil {
+		return false
+	}
+	var underlyingError error
+	switch pe := err.(type) {
+	case nil:
+		return false
+	case *os.PathError:
+		underlyingError = pe.Err
+	case *os.LinkError:
+		underlyingError = pe.Err
+	case *os.SyscallError:
+		underlyingError = pe.Err
+	}
+
+	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO
+}

vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go

@@ -55,6 +55,7 @@ const (
 	fsckErrorsUncorrected = 4
 
 	// place for subpath mounts
+	// TODO: pass in directory using kubelet_getters instead
 	containerSubPathDirectoryName = "volume-subpaths"
 	// syscall.Openat flags used to traverse directories not following symlinks
 	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
@@ -890,15 +891,22 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error
 
 		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
 		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
-		subPaths, err := ioutil.ReadDir(fullContainerDirPath)
-		if err != nil {
-			return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err)
-		}
-		for _, subPath := range subPaths {
-			if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil {
+		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
+			if path == fullContainerDirPath {
+				// Skip top level directory
+				return nil
+			}
+
+			// pass through errors and let doCleanSubPath handle them
+			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
 				return err
 			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
 		}
+
 		// Whole container has been processed, remove its directory.
 		if err := os.Remove(fullContainerDirPath); err != nil {
 			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
@@ -925,22 +933,12 @@ func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string
 	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
 	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
 	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
-	notMnt, err := IsNotMountPoint(mounter, fullSubPath)
-	if err != nil {
-		return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err)
-	}
-	// Unmount it
-	if !notMnt {
-		if err = mounter.Unmount(fullSubPath); err != nil {
-			return fmt.Errorf("error unmounting %s: %s", fullSubPath, err)
-		}
-		klog.V(5).Infof("Unmounted %s", fullSubPath)
-	}
-	// Remove it *non*-recursively, just in case there were some hiccups.
-	if err = os.Remove(fullSubPath); err != nil {
-		return fmt.Errorf("error deleting %s: %s", fullSubPath, err)
+
+	if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
+		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
 	}
-	klog.V(5).Infof("Removed %s", fullSubPath)
+
+	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
 	return nil
 }
 

vendor/k8s.io/kubernetes/pkg/version/base.go

@@ -3,8 +3,8 @@ package version
 var (
 	gitMajor = "1"
 	gitMinor = "12"
-	gitVersion   = "v1.13.1-k3s2"
-	gitCommit    = "2054ab40be790e5f209e480b18ab8a7015262796"
+	gitVersion   = "v1.13.2-k3s2"
+	gitCommit    = "149316edfe67e696f622608282a3fc4252ad92b3"
 	gitTreeState = "clean" 
-	buildDate = "2019-01-09T20:07+00:00Z"
+	buildDate = "2019-01-22T20:48+00:00Z"
 )

vendor/k8s.io/kubernetes/pkg/volume/util/BUILD

@@ -63,7 +63,6 @@ go_test(
         "//pkg/apis/core/v1/helper:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
-        "//pkg/util/mount:go_default_library",
         "//pkg/util/slice:go_default_library",
         "//pkg/volume:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -73,8 +72,12 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
-        "//staging/src/k8s.io/client-go/util/testing:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//staging/src/k8s.io/client-go/util/testing:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(

vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor.go

@@ -725,7 +725,7 @@ func (oe *operationExecutor) MountVolume(
 	if fsVolume {
 		// Filesystem volume case
 		// Mount/remount a volume when a volume is attached
-		generatedOperations, err = oe.operationGenerator.GenerateMountVolumeFunc(
+		generatedOperations = oe.operationGenerator.GenerateMountVolumeFunc(
 			waitForAttachTimeout, volumeToMount, actualStateOfWorld, isRemount)
 
 	} else {

vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go

@@ -40,6 +40,10 @@ import (
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 )
 
+const (
+	unknownVolumePlugin string = "UnknownVolumePlugin"
+)
+
 var _ OperationGenerator = &operationGenerator{}
 
 type operationGenerator struct {
@@ -82,7 +86,7 @@ func NewOperationGenerator(kubeClient clientset.Interface,
 // OperationGenerator interface that extracts out the functions from operation_executor to make it dependency injectable
 type OperationGenerator interface {
 	// Generates the MountVolume function needed to perform the mount of a volume plugin
-	GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (volumetypes.GeneratedOperations, error)
+	GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) volumetypes.GeneratedOperations
 
 	// Generates the UnmountVolume function needed to perform the unmount of a volume plugin
 	GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error)
@@ -436,61 +440,61 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 	waitForAttachTimeout time.Duration,
 	volumeToMount VolumeToMount,
 	actualStateOfWorld ActualStateOfWorldMounterUpdater,
-	isRemount bool) (volumetypes.GeneratedOperations, error) {
+	isRemount bool) volumetypes.GeneratedOperations {
 	// Get mounter plugin
+	volumePluginName := unknownVolumePlugin
 	volumePlugin, err :=
 		og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec)
-	if err != nil || volumePlugin == nil {
-		return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("MountVolume.FindPluginBySpec failed", err)
+	if err == nil && volumePlugin != nil {
+		volumePluginName = volumePlugin.GetPluginName()
 	}
 
-	affinityErr := checkNodeAffinity(og, volumeToMount, volumePlugin)
-	if affinityErr != nil {
-		eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.NodeAffinity check failed", affinityErr)
-		og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMountVolume, eventErr.Error())
-		return volumetypes.GeneratedOperations{}, detailedErr
-	}
+	mountVolumeFunc := func() (error, error) {
+		if err != nil || volumePlugin == nil {
+			return volumeToMount.GenerateError("MountVolume.FindPluginBySpec failed", err)
+		}
 
-	volumeMounter, newMounterErr := volumePlugin.NewMounter(
-		volumeToMount.VolumeSpec,
-		volumeToMount.Pod,
-		volume.VolumeOptions{})
-	if newMounterErr != nil {
-		eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.NewMounter initialization failed", newMounterErr)
-		og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMountVolume, eventErr.Error())
-		return volumetypes.GeneratedOperations{}, detailedErr
-	}
+		affinityErr := checkNodeAffinity(og, volumeToMount, volumePlugin)
+		if affinityErr != nil {
+			return volumeToMount.GenerateError("MountVolume.NodeAffinity check failed", affinityErr)
+		}
 
-	mountCheckError := checkMountOptionSupport(og, volumeToMount, volumePlugin)
+		volumeMounter, newMounterErr := volumePlugin.NewMounter(
+			volumeToMount.VolumeSpec,
+			volumeToMount.Pod,
+			volume.VolumeOptions{})
+		if newMounterErr != nil {
+			return volumeToMount.GenerateError("MountVolume.NewMounter initialization failed", newMounterErr)
 
-	if mountCheckError != nil {
-		eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.MountOptionSupport check failed", mountCheckError)
-		og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.UnsupportedMountOption, eventErr.Error())
-		return volumetypes.GeneratedOperations{}, detailedErr
-	}
+		}
 
-	// Get attacher, if possible
-	attachableVolumePlugin, _ :=
-		og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec)
-	var volumeAttacher volume.Attacher
-	if attachableVolumePlugin != nil {
-		volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
-	}
+		mountCheckError := checkMountOptionSupport(og, volumeToMount, volumePlugin)
 
-	// get deviceMounter, if possible
-	deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec)
-	var volumeDeviceMounter volume.DeviceMounter
-	if deviceMountableVolumePlugin != nil {
-		volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter()
-	}
+		if mountCheckError != nil {
+			return volumeToMount.GenerateError("MountVolume.MountOptionSupport check failed", mountCheckError)
+		}
 
-	var fsGroup *int64
-	if volumeToMount.Pod.Spec.SecurityContext != nil &&
-		volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil {
-		fsGroup = volumeToMount.Pod.Spec.SecurityContext.FSGroup
-	}
+		// Get attacher, if possible
+		attachableVolumePlugin, _ :=
+			og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec)
+		var volumeAttacher volume.Attacher
+		if attachableVolumePlugin != nil {
+			volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
+		}
+
+		// get deviceMounter, if possible
+		deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec)
+		var volumeDeviceMounter volume.DeviceMounter
+		if deviceMountableVolumePlugin != nil {
+			volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter()
+		}
+
+		var fsGroup *int64
+		if volumeToMount.Pod.Spec.SecurityContext != nil &&
+			volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil {
+			fsGroup = volumeToMount.Pod.Spec.SecurityContext.FSGroup
+		}
 
-	mountVolumeFunc := func() (error, error) {
 		devicePath := volumeToMount.DevicePath
 		if volumeAttacher != nil {
 			// Wait for attachable volumes to finish attaching
@@ -536,7 +540,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 
 			// resizeFileSystem will resize the file system if user has requested a resize of
 			// underlying persistent volume and is allowed to do so.
-			resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePlugin.GetPluginName())
+			resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePluginName)
 
 			if resizeSimpleError != nil || resizeDetailedError != nil {
 				return resizeSimpleError, resizeDetailedError
@@ -593,8 +597,8 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 	return volumetypes.GeneratedOperations{
 		OperationFunc:     mountVolumeFunc,
 		EventRecorderFunc: eventRecorderFunc,
-		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "volume_mount"),
-	}, nil
+		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePluginName, volumeToMount.VolumeSpec), "volume_mount"),
+	}
 }
 
 func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) {

vendor/k8s.io/kubernetes/pkg/volume/util/util.go

@@ -23,9 +23,8 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"syscall"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -128,8 +127,9 @@ func SetReady(dir string) {
 
 // UnmountPath is a common unmount routine that unmounts the given path and
 // deletes the remaining directory if successful.
+// TODO: Remove this function and change callers to call mount pkg directly
 func UnmountPath(mountPath string, mounter mount.Interface) error {
-	return UnmountMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
+	return mount.CleanupMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
 }
 
 // UnmountMountPoint is a common unmount routine that unmounts the given path and
@@ -137,93 +137,21 @@ func UnmountPath(mountPath string, mounter mount.Interface) error {
 // if extensiveMountPointCheck is true
 // IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
 // IsNotMountPoint is more expensive but properly handles bind mounts.
+// TODO: Change callers to call mount pkg directly
 func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error {
-	pathExists, pathErr := PathExists(mountPath)
-	if !pathExists {
-		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
-		return nil
-	}
-	corruptedMnt := IsCorruptedMnt(pathErr)
-	if pathErr != nil && !corruptedMnt {
-		return fmt.Errorf("Error checking path: %v", pathErr)
-	}
-	return doUnmountMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt)
-}
-
-// doUnmountMountPoint is a common unmount routine that unmounts the given path and
-// deletes the remaining directory if successful.
-// if extensiveMountPointCheck is true
-// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
-// IsNotMountPoint is more expensive but properly handles bind mounts.
-// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, Take it as an argument for convenience of testing
-func doUnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool, corruptedMnt bool) error {
-	if !corruptedMnt {
-		var notMnt bool
-		var err error
-		if extensiveMountPointCheck {
-			notMnt, err = mount.IsNotMountPoint(mounter, mountPath)
-		} else {
-			notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
-		}
-
-		if err != nil {
-			return err
-		}
-
-		if notMnt {
-			klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
-			return os.Remove(mountPath)
-		}
-	}
-
-	// Unmount the mount path
-	klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
-	if err := mounter.Unmount(mountPath); err != nil {
-		return err
-	}
-	notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath)
-	if mntErr != nil {
-		return mntErr
-	}
-	if notMnt {
-		klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
-		return os.Remove(mountPath)
-	}
-	return fmt.Errorf("Failed to unmount path %v", mountPath)
+	return mount.CleanupMountPoint(mountPath, mounter, extensiveMountPointCheck)
 }
 
 // PathExists returns true if the specified path exists.
+// TODO: Change callers to call mount pkg directly
 func PathExists(path string) (bool, error) {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true, nil
-	} else if os.IsNotExist(err) {
-		return false, nil
-	} else if IsCorruptedMnt(err) {
-		return true, err
-	} else {
-		return false, err
-	}
+	return mount.PathExists(path)
 }
 
 // IsCorruptedMnt return true if err is about corrupted mount point
+// TODO: Change callers to call mount pkg directly
 func IsCorruptedMnt(err error) bool {
-	if err == nil {
-		return false
-	}
-	var underlyingError error
-	switch pe := err.(type) {
-	case nil:
-		return false
-	case *os.PathError:
-		underlyingError = pe.Err
-	case *os.LinkError:
-		underlyingError = pe.Err
-	case *os.SyscallError:
-		underlyingError = pe.Err
-	}
-
-	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO
+	return mount.IsCorruptedMnt(err)
 }
 
 // GetSecretForPod locates secret by name in the pod's namespace and returns secret map
@@ -825,9 +753,10 @@ func GetUniqueVolumeName(pluginName, volumeName string) v1.UniqueVolumeName {
 	return v1.UniqueVolumeName(fmt.Sprintf("%s/%s", pluginName, volumeName))
 }
 
-// GetUniqueVolumeNameForNonAttachableVolume returns the unique volume name
-// for a non-attachable volume.
-func GetUniqueVolumeNameForNonAttachableVolume(
+// GetUniqueVolumeNameFromSpecWithPod returns a unique volume name with pod
+// name included. This is useful to generate different names for different pods
+// on same volume.
+func GetUniqueVolumeNameFromSpecWithPod(
 	podName types.UniquePodName, volumePlugin volume.VolumePlugin, volumeSpec *volume.Spec) v1.UniqueVolumeName {
 	return v1.UniqueVolumeName(
 		fmt.Sprintf("%s/%v-%s", volumePlugin.GetPluginName(), podName, volumeSpec.Name()))

vendor/k8s.io/kubernetes/vendor.conf

@@ -67,8 +67,11 @@ github.com/google/certificate-transparency-go v1.0.21
 github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c
 github.com/google/uuid 0.2-15-g8c31c18f31ede9
 github.com/go-openapi/analysis v0.17.2
+github.com/go-openapi/jsonpointer v0.18.0
+github.com/go-openapi/loads v0.17.2
 github.com/go-openapi/strfmt v0.17.0
 github.com/go-openapi/swag v0.17.2
+github.com/go-openapi/validate v0.18.0
 github.com/go-ozzo/ozzo-validation v3.5.0
 github.com/gophercloud/gophercloud 781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d
 github.com/gorilla/websocket v1.2.0-9-g4201258b820c74
@@ -174,6 +177,7 @@ github.com/vmware/photon-controller-go-sdk PROMOTED-488
 github.com/xanzy/go-cloudstack v2.1.1-1-g1e2cbf647e57fa
 github.com/xiang90/probing 0.0.1
 golang.org/x/crypto de0752318171da717af4ce24d0a2e8626afaeb11
+golang.org/x/lint 8f45f776aaf18cebc8d65861cc70c33c60471952
 golang.org/x/net 0ed95abb35c445290478a5348a7b38bb154135fd
 golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4
 golang.org/x/sys 95c6576299259db960f6c5b9b69ea52422860fce
@@ -201,6 +205,6 @@ k8s.io/utils 66066c83e385e385ccc3c964b44fd7dcd413d0ed
 sigs.k8s.io/yaml v1.1.0
 vbom.ml/util db5cfe13f5cc80a4990d98e2e1b0707a4d1a5394
 bitbucket.org/ww/goautoneg       a547fc61f48d567d5b4ec6f8aee5573d8efce11d  https://github.com/rancher/goautoneg.git
-github.com/ibuildthecloud/kvsql  6bb3d252056655760ed8ca6557d6d5e607b361d2
+github.com/ibuildthecloud/kvsql  8dfe3deb0646c4817567e4a53ed1dea41ea5668f
 github.com/google/cadvisor       91dab6eb91496ed68acbef68b02b34b3392ca754  https://github.com/ibuildthecloud/cadvisor.git
 github.com/opencontainers/runc   96ec2177ae841256168fcf76954f7177af9446eb