Merge branch 'k3s-io:master' into issue-backport

Dockerfile.dapper

-ARG GOLANG=golang:1.16.4-alpine3.12
+ARG GOLANG=golang:1.16.6-alpine3.13
 FROM ${GOLANG}
 
 ARG http_proxy=$http_proxy
@@ -9,8 +9,9 @@ ENV https_proxy=$https_proxy
 ENV no_proxy=$no_proxy
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
-    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev make \
-    libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd gzip alpine-sdk binutils-gold
+    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev \
+    libseccomp-static make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static \
+    zstd gzip alpine-sdk binutils-gold
 RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
     wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
     tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz                                                            && \

Dockerfile.manifest

-ARG GOLANG=golang:1.16.4-alpine3.12
+ARG GOLANG=golang:1.16.6-alpine3.13
 FROM ${GOLANG}
 
 COPY --from=plugins/manifest:1.2.3 /bin/* /bin/

Dockerfile.test.dapper

-ARG GOLANG=golang:1.16.4-alpine3.12
+ARG GOLANG=golang:1.16.6-alpine3.13
 FROM ${GOLANG}
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl py-pip

Dockerfile.test.mod.dapper

-ARG GOLANG=golang:1.16.4-alpine3.12
+ARG GOLANG=golang:1.16.6-alpine3.13
 FROM ${GOLANG}
 
 RUN apk -U --no-cache add bash jq

channel.yaml

 # Example channels config
 channels:
 - name: stable
-  latest: v1.21.2+k3s1
+  latest: v1.21.3+k3s1
 - name: latest
   latestRegexp: .*
   excludeRegexp: ^[^+]+-

go.mod

@@ -3,18 +3,20 @@ module github.com/rancher/k3s
 go 1.16
 
 replace (
-	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.10
+	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.20
 	github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7
-	github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e
-	github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340
-	github.com/containerd/console => github.com/containerd/console v1.0.0
-	github.com/containerd/containerd => github.com/k3s-io/containerd v1.4.4-k3s2 // k3s-release/1.4
+	github.com/containerd/aufs => github.com/containerd/aufs v1.0.0
+	github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0
+	github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1
+	github.com/containerd/console => github.com/containerd/console v1.0.2
+	github.com/containerd/containerd => github.com/k3s-io/containerd v1.4.8-k3s1 // k3s-release/1.4
 	github.com/containerd/continuity => github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1
-	github.com/containerd/cri => github.com/k3s-io/cri v1.4.0-k3s.6 // k3s-release/1.4
-	github.com/containerd/fifo => github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b
-	github.com/containerd/go-runc => github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328
-	github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.1
-	github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.1
+	github.com/containerd/cri => github.com/k3s-io/cri v1.4.0-k3s.7 // k3s-release/1.4
+	github.com/containerd/fifo => github.com/containerd/fifo v1.0.0
+	github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0
+	github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2
+	github.com/containerd/zfs => github.com/containerd/zfs v1.0.0
 	github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
 	github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible
@@ -34,44 +36,46 @@ replace (
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884
 	google.golang.org/grpc => google.golang.org/grpc v1.27.1
 	gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2
-	k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.21.2-k3s1
-	k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.21.2-k3s1
-	k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.21.2-k3s1
-	k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.21.2-k3s1
-	k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.21.2-k3s1
-	k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.21.2-k3s1
-	k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.21.2-k3s1
-	k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.21.2-k3s1
-	k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.21.2-k3s1
-	k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.21.2-k3s1
-	k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.21.2-k3s1
-	k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.21.2-k3s1
-	k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.21.2-k3s1
-	k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.21.2-k3s1
-	k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.2-k3s1
-	k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.21.2-k3s1
-	k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.21.2-k3s1
-	k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.21.2-k3s1
-	k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.21.2-k3s1
-	k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.21.2-k3s1
-	k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.21.2-k3s1
-	k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.21.2-k3s1
-	k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.21.2-k3s1
-	k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.21.2-k3s1
-	k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.21.2-k3s1
-	k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.21.2-k3s1
-	k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.21.2-k3s1
-	k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.21.2-k3s1
+	k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.21.3-k3s1
+	k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.21.3-k3s1
+	k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.21.3-k3s1
+	k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.21.3-k3s1
+	k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.21.3-k3s1
+	k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.21.3-k3s1
+	k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.21.3-k3s1
+	k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.21.3-k3s1
+	k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.21.3-k3s1
+	k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.21.3-k3s1
+	k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.21.3-k3s1
+	k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.21.3-k3s1
+	k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.21.3-k3s1
+	k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.21.3-k3s1
+	k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.3-k3s1
+	k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.21.3-k3s1
+	k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.21.3-k3s1
+	k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.21.3-k3s1
+	k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.21.3-k3s1
+	k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.21.3-k3s1
+	k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.21.3-k3s1
+	k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.21.3-k3s1
+	k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.21.3-k3s1
+	k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.21.3-k3s1
+	k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.21.3-k3s1
+	k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.21.3-k3s1
+	k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.21.3-k3s1
+	k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.21.3-k3s1
 	mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7
 )
 
 require (
-	github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990
+	github.com/Microsoft/hcsshim v0.8.20
 	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
-	github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340
-	github.com/containerd/containerd v1.5.0-beta.4
+	github.com/containerd/cgroups v1.0.1
+	github.com/containerd/containerd v1.5.1
 	github.com/containerd/cri v1.11.1-0.20200820101445-b0cc07999aa5
 	github.com/containerd/fuse-overlayfs-snapshotter v1.0.2
+	github.com/containerd/go-cni v1.0.2 // indirect
+	github.com/containerd/imgcrypt v1.1.1 // indirect
 	github.com/coreos/go-iptables v0.4.5
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/docker/docker v20.10.5+incompatible
@@ -111,7 +115,7 @@ require (
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/urfave/cli v1.22.2
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea
-	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 	google.golang.org/grpc v1.37.0
@@ -128,7 +132,7 @@ require (
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.8.0
 	k8s.io/kubectl v0.21.2
-	k8s.io/kubernetes v1.21.2
+	k8s.io/kubernetes v1.21.3
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 	sigs.k8s.io/yaml v1.2.0
 )

pkg/cli/cmds/server.go

@@ -13,6 +13,15 @@ const (
 	defaultSnapshotIntervalHours = 12
 )
 
+type StartupHookArgs struct {
+	APIServerReady  <-chan struct{}
+	KubeConfigAdmin string
+	Skips           map[string]bool
+	Disables        map[string]bool
+}
+
+type StartupHook func(context.Context, *sync.WaitGroup, StartupHookArgs) error
+
 type Server struct {
 	ClusterCIDR          cli.StringSlice
 	AgentToken           string
@@ -64,7 +73,7 @@ type Server struct {
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
 	SystemDefaultRegistry    string
-	StartupHooks             []func(context.Context, *sync.WaitGroup, <-chan struct{}, string) error
+	StartupHooks             []StartupHook
 	EtcdSnapshotName         string
 	EtcdDisableSnapshots     bool
 	EtcdExposeMetrics        bool

pkg/cluster/storage.go

@@ -47,14 +47,14 @@ func (c *Cluster) save(ctx context.Context) error {
 		return err
 	}
 
-	_, _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken)
+	_, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
 	if err != nil {
 		return err
 	}
 
 	if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil {
 		if err.Error() == "key exists" {
-			logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore")
+			logrus.Warnln("bootstrap key already exists")
 			return nil
 		} else if strings.Contains(err.Error(), "not supported for learner") {
 			logrus.Debug("skipping bootstrap data save on learner")
@@ -97,16 +97,14 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	value, emptyKey, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken)
+	value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
 	if err != nil {
 		return err
 	}
 	if value == nil {
 		return nil
 	}
-	if emptyKey {
-		normalizedToken = ""
-	}
+
 	data, err := decrypt(normalizedToken, value.Data)
 	if err != nil {
 		return err
@@ -119,36 +117,38 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 // hashed with empty string and will check for any key that is hashed by different token than the one
 // passed to it, it will return error if it finds a key that is hashed with different token and will return
 // value if it finds the key hashed by passed token or empty string
-func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, token string) (*client.Value, bool, error) {
+func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) {
 	emptyStringKey := storageKey("")
-	tokenKey := storageKey(token)
-
+	tokenKey := storageKey(normalizedToken)
 	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 	if len(bootstrapList) == 0 {
 		c.saveBootstrap = true
-		return nil, false, nil
+		return nil, nil
 	}
 	if len(bootstrapList) > 1 {
-		return nil, false, errors.New("found multiple bootstrap keys in storage")
+		logrus.Warn("found multiple bootstrap keys in storage")
 	}
-	bootstrapKV := bootstrapList[0]
-	// checking for empty string bootstrap key
-	switch string(bootstrapKV.Key) {
-	case emptyStringKey:
-		logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token")
-		c.saveBootstrap = true
-		if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil {
-			return nil, false, err
+	// check for empty string key and for old token format with k10 prefix
+	if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+		return nil, err
+	}
+
+	// getting the list of bootstrap again after migrating the empty key
+	bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
+	if err != nil {
+		return nil, err
+	}
+	for _, bootstrapKV := range bootstrapList {
+		// ensure bootstrap is stored in the current token's key
+		if string(bootstrapKV.Key) == tokenKey {
+			return &bootstrapKV, nil
 		}
-		return &bootstrapKV, true, nil
-	case tokenKey:
-		return &bootstrapKV, false, nil
 	}
 
-	return nil, false, errors.New("bootstrap data already found and encrypted with different token")
+	return nil, errors.New("bootstrap data already found and encrypted with different token")
 }
 
 // readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
@@ -180,3 +180,54 @@ func normalizeToken(token string) (string, error) {
 	}
 	return password, nil
 }
+
+// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
+// hashed with empty string and keys that is hashed with old token format before normalizing
+// then migrate those and resave only with the normalized token
+func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+	oldTokenKey := storageKey(oldToken)
+	for _, bootstrapKV := range bootstrapList {
+		// checking for empty string bootstrap key
+		if string(bootstrapKV.Key) == emptyStringKey {
+			logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token")
+			if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil {
+				return err
+			}
+		} else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey {
+			logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+			if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue client.Value, oldToken, oldTokenKey, newToken, newTokenKey string) error {
+	// make sure that the process is non-destructive by decrypting/re-encrypting/storing the data before deleting the old key
+	data, err := decrypt(oldToken, keyValue.Data)
+	if err != nil {
+		return err
+	}
+	encryptedData, err := encrypt(newToken, data)
+	if err != nil {
+		return err
+	}
+	// saving the new encrypted data with the right token key
+	if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil {
+		if err.Error() == "key exists" {
+			logrus.Warn("bootstrap key exists")
+		} else if strings.Contains(err.Error(), "not supported for learner") {
+			logrus.Debug("skipping bootstrap data save on learner")
+			return nil
+		} else {
+			return err
+		}
+	}
+	logrus.Infof("created bootstrap key %s", newTokenKey)
+	// deleting the old key
+	if err := storageClient.Delete(ctx, oldTokenKey, keyValue.Modified); err != nil {
+		logrus.Warnf("failed to delete old bootstrap key %s", oldTokenKey)
+	}
+	return nil
+}

pkg/server/server.go

@@ -13,11 +13,10 @@ import (
 	"sync"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
-
 	"github.com/k3s-io/helm-controller/pkg/helm"
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/apiaddresses"
+	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/control"
@@ -34,6 +33,7 @@ import (
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
 )
@@ -62,22 +62,28 @@ func StartServer(ctx context.Context, config *Config) error {
 		return errors.Wrap(err, "starting kubernetes")
 	}
 
-	config.ControlConfig.Runtime.Handler = router(ctx, config)
+	wg := &sync.WaitGroup{}
+	wg.Add(len(config.StartupHooks))
 
-	if config.ControlConfig.DisableAPIServer {
-		go setETCDLabelsAndAnnotations(ctx, config)
-	} else {
-		go startOnAPIServerReady(ctx, config)
+	config.ControlConfig.Runtime.Handler = router(ctx, config)
+	shArgs := cmds.StartupHookArgs{
+		APIServerReady:  config.ControlConfig.Runtime.APIServerReady,
+		KubeConfigAdmin: config.ControlConfig.Runtime.KubeConfigAdmin,
+		Skips:           config.ControlConfig.Skips,
+		Disables:        config.ControlConfig.Disables,
 	}
-
-	config.StartupHooksWg = &sync.WaitGroup{}
-	config.StartupHooksWg.Add(len(config.StartupHooks))
 	for _, hook := range config.StartupHooks {
-		if err := hook(ctx, config.StartupHooksWg, config.ControlConfig.Runtime.APIServerReady, config.ControlConfig.Runtime.KubeConfigAdmin); err != nil {
+		if err := hook(ctx, wg, shArgs); err != nil {
 			return errors.Wrap(err, "startup hook")
 		}
 	}
 
+	if config.ControlConfig.DisableAPIServer {
+		go setETCDLabelsAndAnnotations(ctx, config)
+	} else {
+		go startOnAPIServerReady(ctx, wg, config)
+	}
+
 	ip := net2.ParseIP(config.ControlConfig.BindAddress)
 	if ip == nil {
 		hostIP, err := net.ChooseHostInterface()
@@ -95,18 +101,18 @@ func StartServer(ctx context.Context, config *Config) error {
 	return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
 }
 
-func startOnAPIServerReady(ctx context.Context, config *Config) {
+func startOnAPIServerReady(ctx context.Context, wg *sync.WaitGroup, config *Config) {
 	select {
 	case <-ctx.Done():
 		return
 	case <-config.ControlConfig.Runtime.APIServerReady:
-		if err := runControllers(ctx, config); err != nil {
+		if err := runControllers(ctx, wg, config); err != nil {
 			logrus.Fatalf("failed to start controllers: %v", err)
 		}
 	}
 }
 
-func runControllers(ctx context.Context, config *Config) error {
+func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) error {
 	controlConfig := &config.ControlConfig
 
 	sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
@@ -114,7 +120,7 @@ func runControllers(ctx context.Context, config *Config) error {
 		return err
 	}
 
-	config.StartupHooksWg.Wait()
+	wg.Wait()
 	if err := stageFiles(ctx, sc, controlConfig); err != nil {
 		return err
 	}

pkg/server/types.go

@@ -2,8 +2,8 @@ package server
 
 import (
 	"context"
-	"sync"
 
+	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/daemons/config"
 )
 
@@ -13,8 +13,7 @@ type Config struct {
 	ControlConfig     config.Control
 	Rootless          bool
 	SupervisorPort    int
-	StartupHooks      []func(context.Context, *sync.WaitGroup, <-chan struct{}, string) error
-	StartupHooksWg    *sync.WaitGroup
+	StartupHooks      []cmds.StartupHook
 	LeaderControllers CustomControllers
 	Controllers       CustomControllers
 }

vendor/github.com/Microsoft/go-winio/CODEOWNERS

+  * @microsoft/containerplat

vendor/github.com/Microsoft/go-winio/README.md

-# go-winio
+# go-winio [![Build Status](https://github.com/microsoft/go-winio/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/go-winio/actions/workflows/ci.yml)
 
 This repository contains utilities for efficiently performing Win32 IO operations in
 Go. Currently, this is focused on accessing named pipes and other file handles, and

vendor/github.com/Microsoft/go-winio/go.mod

@@ -3,7 +3,7 @@ module github.com/Microsoft/go-winio
 go 1.12
 
 require (
-	github.com/pkg/errors v0.8.1
-	github.com/sirupsen/logrus v1.4.1
-	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.7.0
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
 )

vendor/github.com/Microsoft/go-winio/go.sum

 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/Microsoft/go-winio/hvsock.go

+// +build windows
+
 package winio
 
 import (

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -429,10 +429,10 @@ type PipeConfig struct {
 	// when the pipe is in message mode.
 	MessageMode bool
 
-	// InputBufferSize specifies the size the input buffer, in bytes.
+	// InputBufferSize specifies the size of the input buffer, in bytes.
 	InputBufferSize int32
 
-	// OutputBufferSize specifies the size the input buffer, in bytes.
+	// OutputBufferSize specifies the size of the output buffer, in bytes.
 	OutputBufferSize int32
 }
 

vendor/github.com/Microsoft/go-winio/pkg/etw/eventdata.go

+// +build windows
+
 package etw
 
 import (

vendor/github.com/Microsoft/go-winio/pkg/etw/eventopt.go

+// +build windows
+
 package etw
 
 import (

vendor/github.com/Microsoft/go-winio/pkg/etw/fieldopt.go

+// +build windows
+
 package etw
 
 import (

vendor/github.com/Microsoft/go-winio/pkg/etw/newprovider.go

+// +build windows
 // +build amd64 arm64 386
 
 package etw
@@ -11,11 +12,20 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-// NewProviderWithID creates and registers a new ETW provider, allowing the
-// provider ID to be manually specified. This is most useful when there is an
-// existing provider ID that must be used to conform to existing diagnostic
-// infrastructure.
-func NewProviderWithID(name string, id guid.GUID, callback EnableCallback) (provider *Provider, err error) {
+// NewProviderWithOptions creates and registers a new ETW provider, allowing
+// the provider ID and Group to be manually specified. This is most useful when
+// there is an existing provider ID that must be used to conform to existing
+// diagnostic infrastructure.
+func NewProviderWithOptions(name string, options ...ProviderOpt) (provider *Provider, err error) {
+	var opts providerOpts
+	for _, opt := range options {
+		opt(&opts)
+	}
+
+	if opts.id == (guid.GUID{}) {
+		opts.id = providerIDFromName(name)
+	}
+
 	providerCallbackOnce.Do(func() {
 		globalProviderCallback = windows.NewCallback(providerCallbackAdapter)
 	})
@@ -26,17 +36,27 @@ func NewProviderWithID(name string, id guid.GUID, callback EnableCallback) (prov
 			providers.removeProvider(provider)
 		}
 	}(provider)
-	provider.ID = id
-	provider.callback = callback
+	provider.ID = opts.id
+	provider.callback = opts.callback
 
 	if err := eventRegister((*windows.GUID)(&provider.ID), globalProviderCallback, uintptr(provider.index), &provider.handle); err != nil {
 		return nil, err
 	}
 
+	trait := &bytes.Buffer{}
+	if opts.group != (guid.GUID{}) {
+		binary.Write(trait, binary.LittleEndian, uint16(0)) // Write empty size for buffer (update later)
+		binary.Write(trait, binary.LittleEndian, uint8(1))  // EtwProviderTraitTypeGroup
+		traitArray := opts.group.ToWindowsArray()           // Append group guid
+		trait.Write(traitArray[:])
+		binary.LittleEndian.PutUint16(trait.Bytes(), uint16(trait.Len())) // Update size
+	}
+
 	metadata := &bytes.Buffer{}
 	binary.Write(metadata, binary.LittleEndian, uint16(0)) // Write empty size for buffer (to update later)
 	metadata.WriteString(name)
 	metadata.WriteByte(0)                                                   // Null terminator for name
+	trait.WriteTo(metadata)                                                 // Add traits if applicable
 	binary.LittleEndian.PutUint16(metadata.Bytes(), uint16(metadata.Len())) // Update the size at the beginning of the buffer
 	provider.metadata = metadata.Bytes()
 

vendor/github.com/Microsoft/go-winio/pkg/etw/newprovider_unsupported.go

+// +build windows
 // +build arm
 
 package etw
 
-import (
-	"github.com/Microsoft/go-winio/pkg/guid"
-)
-
 // NewProviderWithID returns a nil provider on unsupported platforms.
-func NewProviderWithID(name string, id guid.GUID, callback EnableCallback) (provider *Provider, err error) {
+func NewProviderWithOptions(name string, options ...ProviderOpt) (provider *Provider, err error) {
 	return nil, nil
 }

vendor/github.com/Microsoft/go-winio/pkg/etw/provider.go

+// +build windows
+
 package etw
 
 import (
@@ -117,10 +119,50 @@ func providerIDFromName(name string) guid.GUID {
 	return guid.FromWindowsArray(a)
 }
 
+type providerOpts struct {
+	callback EnableCallback
+	id       guid.GUID
+	group    guid.GUID
+}
+
+// ProviderOpt allows the caller to specify provider options to
+// NewProviderWithOptions
+type ProviderOpt func(*providerOpts)
+
+// WithCallback is used to provide a callback option to NewProviderWithOptions
+func WithCallback(callback EnableCallback) ProviderOpt {
+	return func(opts *providerOpts) {
+		opts.callback = callback
+	}
+}
+
+// WithID is used to provide a provider ID option to NewProviderWithOptions
+func WithID(id guid.GUID) ProviderOpt {
+	return func(opts *providerOpts) {
+		opts.id = id
+	}
+}
+
+// WithGroup is used to provide a provider group option to
+// NewProviderWithOptions
+func WithGroup(group guid.GUID) ProviderOpt {
+	return func(opts *providerOpts) {
+		opts.group = group
+	}
+}
+
+// NewProviderWithID creates and registers a new ETW provider, allowing the
+// provider ID to be manually specified. This is most useful when there is an
+// existing provider ID that must be used to conform to existing diagnostic
+// infrastructure.
+func NewProviderWithID(name string, id guid.GUID, callback EnableCallback) (provider *Provider, err error) {
+	return NewProviderWithOptions(name, WithID(id), WithCallback(callback))
+}
+
 // NewProvider creates and registers a new ETW provider. The provider ID is
 // generated based on the provider name.
 func NewProvider(name string, callback EnableCallback) (provider *Provider, err error) {
-	return NewProviderWithID(name, providerIDFromName(name), callback)
+	return NewProviderWithOptions(name, WithCallback(callback))
 }
 
 // Close unregisters the provider.

vendor/github.com/Microsoft/go-winio/pkg/etw/providerglobal.go

+// +build windows
+
 package etw
 
 import (

vendor/github.com/Microsoft/go-winio/pkg/etw/wrapper_32.go

+// +build windows
 // +build 386 arm
 
 package etw

vendor/github.com/Microsoft/go-winio/pkg/etw/wrapper_64.go

+// +build windows
 // +build amd64 arm64
 
 package etw

vendor/github.com/Microsoft/go-winio/pkg/etw/zsyscall_windows.go

@@ -19,6 +19,7 @@ const (
 
 var (
 	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
 )
 
 // errnoErr returns common boxed Errno values, to prevent
@@ -26,7 +27,7 @@ var (
 func errnoErr(e syscall.Errno) error {
 	switch e {
 	case 0:
-		return nil
+		return errERROR_EINVAL
 	case errnoERROR_IO_PENDING:
 		return errERROR_IO_PENDING
 	}
@@ -40,9 +41,9 @@ var (
 	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
 
 	procEventRegister       = modadvapi32.NewProc("EventRegister")
+	procEventSetInformation = modadvapi32.NewProc("EventSetInformation")
 	procEventUnregister     = modadvapi32.NewProc("EventUnregister")
 	procEventWriteTransfer  = modadvapi32.NewProc("EventWriteTransfer")
-	procEventSetInformation = modadvapi32.NewProc("EventSetInformation")
 )
 
 func eventRegister(providerId *windows.GUID, callback uintptr, callbackContext uintptr, providerHandle *providerHandle) (win32err error) {
@@ -53,24 +54,24 @@ func eventRegister(providerId *windows.GUID, callback uintptr, callbackContext u
 	return
 }
 
-func eventUnregister_64(providerHandle providerHandle) (win32err error) {
-	r0, _, _ := syscall.Syscall(procEventUnregister.Addr(), 1, uintptr(providerHandle), 0, 0)
+func eventSetInformation_64(providerHandle providerHandle, class eventInfoClass, information uintptr, length uint32) (win32err error) {
+	r0, _, _ := syscall.Syscall6(procEventSetInformation.Addr(), 4, uintptr(providerHandle), uintptr(class), uintptr(information), uintptr(length), 0, 0)
 	if r0 != 0 {
 		win32err = syscall.Errno(r0)
 	}
 	return
 }
 
-func eventWriteTransfer_64(providerHandle providerHandle, descriptor *eventDescriptor, activityID *windows.GUID, relatedActivityID *windows.GUID, dataDescriptorCount uint32, dataDescriptors *eventDataDescriptor) (win32err error) {
-	r0, _, _ := syscall.Syscall6(procEventWriteTransfer.Addr(), 6, uintptr(providerHandle), uintptr(unsafe.Pointer(descriptor)), uintptr(unsafe.Pointer(activityID)), uintptr(unsafe.Pointer(relatedActivityID)), uintptr(dataDescriptorCount), uintptr(unsafe.Pointer(dataDescriptors)))
+func eventSetInformation_32(providerHandle_low uint32, providerHandle_high uint32, class eventInfoClass, information uintptr, length uint32) (win32err error) {
+	r0, _, _ := syscall.Syscall6(procEventSetInformation.Addr(), 5, uintptr(providerHandle_low), uintptr(providerHandle_high), uintptr(class), uintptr(information), uintptr(length), 0)
 	if r0 != 0 {
 		win32err = syscall.Errno(r0)
 	}
 	return
 }
 
-func eventSetInformation_64(providerHandle providerHandle, class eventInfoClass, information uintptr, length uint32) (win32err error) {
-	r0, _, _ := syscall.Syscall6(procEventSetInformation.Addr(), 4, uintptr(providerHandle), uintptr(class), uintptr(information), uintptr(length), 0, 0)
+func eventUnregister_64(providerHandle providerHandle) (win32err error) {
+	r0, _, _ := syscall.Syscall(procEventUnregister.Addr(), 1, uintptr(providerHandle), 0, 0)
 	if r0 != 0 {
 		win32err = syscall.Errno(r0)
 	}
@@ -85,16 +86,16 @@ func eventUnregister_32(providerHandle_low uint32, providerHandle_high uint32) (
 	return
 }
 
-func eventWriteTransfer_32(providerHandle_low uint32, providerHandle_high uint32, descriptor *eventDescriptor, activityID *windows.GUID, relatedActivityID *windows.GUID, dataDescriptorCount uint32, dataDescriptors *eventDataDescriptor) (win32err error) {
-	r0, _, _ := syscall.Syscall9(procEventWriteTransfer.Addr(), 7, uintptr(providerHandle_low), uintptr(providerHandle_high), uintptr(unsafe.Pointer(descriptor)), uintptr(unsafe.Pointer(activityID)), uintptr(unsafe.Pointer(relatedActivityID)), uintptr(dataDescriptorCount), uintptr(unsafe.Pointer(dataDescriptors)), 0, 0)
+func eventWriteTransfer_64(providerHandle providerHandle, descriptor *eventDescriptor, activityID *windows.GUID, relatedActivityID *windows.GUID, dataDescriptorCount uint32, dataDescriptors *eventDataDescriptor) (win32err error) {
+	r0, _, _ := syscall.Syscall6(procEventWriteTransfer.Addr(), 6, uintptr(providerHandle), uintptr(unsafe.Pointer(descriptor)), uintptr(unsafe.Pointer(activityID)), uintptr(unsafe.Pointer(relatedActivityID)), uintptr(dataDescriptorCount), uintptr(unsafe.Pointer(dataDescriptors)))
 	if r0 != 0 {
 		win32err = syscall.Errno(r0)
 	}
 	return
 }
 
-func eventSetInformation_32(providerHandle_low uint32, providerHandle_high uint32, class eventInfoClass, information uintptr, length uint32) (win32err error) {
-	r0, _, _ := syscall.Syscall6(procEventSetInformation.Addr(), 5, uintptr(providerHandle_low), uintptr(providerHandle_high), uintptr(class), uintptr(information), uintptr(length), 0)
+func eventWriteTransfer_32(providerHandle_low uint32, providerHandle_high uint32, descriptor *eventDescriptor, activityID *windows.GUID, relatedActivityID *windows.GUID, dataDescriptorCount uint32, dataDescriptors *eventDataDescriptor) (win32err error) {
+	r0, _, _ := syscall.Syscall9(procEventWriteTransfer.Addr(), 7, uintptr(providerHandle_low), uintptr(providerHandle_high), uintptr(unsafe.Pointer(descriptor)), uintptr(unsafe.Pointer(activityID)), uintptr(unsafe.Pointer(relatedActivityID)), uintptr(dataDescriptorCount), uintptr(unsafe.Pointer(dataDescriptors)), 0, 0)
 	if r0 != 0 {
 		win32err = syscall.Errno(r0)
 	}

vendor/github.com/Microsoft/go-winio/pkg/etwlogrus/hook.go

+// +build windows
+
 package etwlogrus
 
 import (

vendor/github.com/Microsoft/go-winio/pkg/guid/guid.go

+// +build windows
+
 // Package guid provides a GUID type. The backing structure for a GUID is
 // identical to that used by the golang.org/x/sys/windows GUID type.
 // There are two main binary encodings used for a GUID, the big-endian encoding,

vendor/github.com/Microsoft/go-winio/pkg/security/grantvmgroupaccess.go

+// +build windows
+
+package security
+
+import (
+	"os"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+)
+
+type (
+	accessMask          uint32
+	accessMode          uint32
+	desiredAccess       uint32
+	inheritMode         uint32
+	objectType          uint32
+	shareMode           uint32
+	securityInformation uint32
+	trusteeForm         uint32
+	trusteeType         uint32
+
+	explicitAccess struct {
+		accessPermissions accessMask
+		accessMode        accessMode
+		inheritance       inheritMode
+		trustee           trustee
+	}
+
+	trustee struct {
+		multipleTrustee          *trustee
+		multipleTrusteeOperation int32
+		trusteeForm              trusteeForm
+		trusteeType              trusteeType
+		name                     uintptr
+	}
+)
+
+const (
+	accessMaskDesiredPermission accessMask = 1 << 31 // GENERIC_READ
+
+	accessModeGrant accessMode = 1
+
+	desiredAccessReadControl desiredAccess = 0x20000
+	desiredAccessWriteDac    desiredAccess = 0x40000
+
+	gvmga = "GrantVmGroupAccess:"
+
+	inheritModeNoInheritance                  inheritMode = 0x0
+	inheritModeSubContainersAndObjectsInherit inheritMode = 0x3
+
+	objectTypeFileObject objectType = 0x1
+
+	securityInformationDACL securityInformation = 0x4
+
+	shareModeRead  shareMode = 0x1
+	shareModeWrite shareMode = 0x2
+
+	sidVmGroup = "S-1-5-83-0"
+
+	trusteeFormIsSid trusteeForm = 0
+
+	trusteeTypeWellKnownGroup trusteeType = 5
+)
+
+// GrantVMGroupAccess sets the DACL for a specified file or directory to
+// include Grant ACE entries for the VM Group SID. This is a golang re-
+// implementation of the same function in vmcompute, just not exported in
+// RS5. Which kind of sucks. Sucks a lot :/
+func GrantVmGroupAccess(name string) error {
+	// Stat (to determine if `name` is a directory).
+	s, err := os.Stat(name)
+	if err != nil {
+		return errors.Wrapf(err, "%s os.Stat %s", gvmga, name)
+	}
+
+	// Get a handle to the file/directory. Must defer Close on success.
+	fd, err := createFile(name, s.IsDir())
+	if err != nil {
+		return err // Already wrapped
+	}
+	defer syscall.CloseHandle(fd)
+
+	// Get the current DACL and Security Descriptor. Must defer LocalFree on success.
+	ot := objectTypeFileObject
+	si := securityInformationDACL
+	sd := uintptr(0)
+	origDACL := uintptr(0)
+	if err := getSecurityInfo(fd, uint32(ot), uint32(si), nil, nil, &origDACL, nil, &sd); err != nil {
+		return errors.Wrapf(err, "%s GetSecurityInfo %s", gvmga, name)
+	}
+	defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd)))
+
+	// Generate a new DACL which is the current DACL with the required ACEs added.
+	// Must defer LocalFree on success.
+	newDACL, err := generateDACLWithAcesAdded(name, s.IsDir(), origDACL)
+	if err != nil {
+		return err // Already wrapped
+	}
+	defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL)))
+
+	// And finally use SetSecurityInfo to apply the updated DACL.
+	if err := setSecurityInfo(fd, uint32(ot), uint32(si), uintptr(0), uintptr(0), newDACL, uintptr(0)); err != nil {
+		return errors.Wrapf(err, "%s SetSecurityInfo %s", gvmga, name)
+	}
+
+	return nil
+}
+
+// createFile is a helper function to call [Nt]CreateFile to get a handle to
+// the file or directory.
+func createFile(name string, isDir bool) (syscall.Handle, error) {
+	namep := syscall.StringToUTF16(name)
+	da := uint32(desiredAccessReadControl | desiredAccessWriteDac)
+	sm := uint32(shareModeRead | shareModeWrite)
+	fa := uint32(syscall.FILE_ATTRIBUTE_NORMAL)
+	if isDir {
+		fa = uint32(fa | syscall.FILE_FLAG_BACKUP_SEMANTICS)
+	}
+	fd, err := syscall.CreateFile(&namep[0], da, sm, nil, syscall.OPEN_EXISTING, fa, 0)
+	if err != nil {
+		return 0, errors.Wrapf(err, "%s syscall.CreateFile %s", gvmga, name)
+	}
+	return fd, nil
+}
+
+// generateDACLWithAcesAdded generates a new DACL with the two needed ACEs added.
+// The caller is responsible for LocalFree of the returned DACL on success.
+func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintptr, error) {
+	// Generate pointers to the SIDs based on the string SIDs
+	sid, err := syscall.StringToSid(sidVmGroup)
+	if err != nil {
+		return 0, errors.Wrapf(err, "%s syscall.StringToSid %s %s", gvmga, name, sidVmGroup)
+	}
+
+	inheritance := inheritModeNoInheritance
+	if isDir {
+		inheritance = inheritModeSubContainersAndObjectsInherit
+	}
+
+	eaArray := []explicitAccess{
+		explicitAccess{
+			accessPermissions: accessMaskDesiredPermission,
+			accessMode:        accessModeGrant,
+			inheritance:       inheritance,
+			trustee: trustee{
+				trusteeForm: trusteeFormIsSid,
+				trusteeType: trusteeTypeWellKnownGroup,
+				name:        uintptr(unsafe.Pointer(sid)),
+			},
+		},
+	}
+
+	modifiedDACL := uintptr(0)
+	if err := setEntriesInAcl(uintptr(uint32(1)), uintptr(unsafe.Pointer(&eaArray[0])), origDACL, &modifiedDACL); err != nil {
+		return 0, errors.Wrapf(err, "%s SetEntriesInAcl %s", gvmga, name)
+	}
+
+	return modifiedDACL, nil
+}

vendor/github.com/Microsoft/go-winio/pkg/security/syscall_windows.go

+package security
+
+//go:generate go run mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go
+
+//sys getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (err error) [failretval!=0] = advapi32.GetSecurityInfo
+//sys setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (err error) [failretval!=0] = advapi32.SetSecurityInfo
+//sys setEntriesInAcl(count uintptr, pListOfEEs uintptr, oldAcl uintptr, newAcl *uintptr) (err error) [failretval!=0] = advapi32.SetEntriesInAclW

vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go → vendor/github.com/Microsoft/go-winio/pkg/security/zsyscall_windows.go

-// Code generated mksyscall_windows.exe DO NOT EDIT
+// Code generated by 'go generate'; DO NOT EDIT.
 
-package hcs
+package security
 
 import (
 	"syscall"
@@ -19,6 +19,7 @@ const (
 
 var (
 	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
 )
 
 // errnoErr returns common boxed Errno values, to prevent
@@ -26,7 +27,7 @@ var (
 func errnoErr(e syscall.Errno) error {
 	switch e {
 	case 0:
-		return nil
+		return errERROR_EINVAL
 	case errnoERROR_IO_PENDING:
 		return errERROR_IO_PENDING
 	}
@@ -37,18 +38,33 @@ func errnoErr(e syscall.Errno) error {
 }
 
 var (
-	modcomputestorage = windows.NewLazySystemDLL("computestorage.dll")
+	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
 
-	procHcsFormatWritableLayerVhd = modcomputestorage.NewProc("HcsFormatWritableLayerVhd")
+	procGetSecurityInfo  = modadvapi32.NewProc("GetSecurityInfo")
+	procSetEntriesInAclW = modadvapi32.NewProc("SetEntriesInAclW")
+	procSetSecurityInfo  = modadvapi32.NewProc("SetSecurityInfo")
 )
 
-func hcsFormatWritableLayerVhd(handle uintptr) (hr error) {
-	r0, _, _ := syscall.Syscall(procHcsFormatWritableLayerVhd.Addr(), 1, uintptr(handle), 0, 0)
-	if int32(r0) < 0 {
-		if r0&0x1fff0000 == 0x00070000 {
-			r0 &= 0xffff
-		}
-		hr = syscall.Errno(r0)
+func getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall9(procGetSecurityInfo.Addr(), 8, uintptr(handle), uintptr(objectType), uintptr(si), uintptr(unsafe.Pointer(ppsidOwner)), uintptr(unsafe.Pointer(ppsidGroup)), uintptr(unsafe.Pointer(ppDacl)), uintptr(unsafe.Pointer(ppSacl)), uintptr(unsafe.Pointer(ppSecurityDescriptor)), 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func setEntriesInAcl(count uintptr, pListOfEEs uintptr, oldAcl uintptr, newAcl *uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall6(procSetEntriesInAclW.Addr(), 4, uintptr(count), uintptr(pListOfEEs), uintptr(oldAcl), uintptr(unsafe.Pointer(newAcl)), 0, 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall9(procSetSecurityInfo.Addr(), 7, uintptr(handle), uintptr(objectType), uintptr(si), uintptr(psidOwner), uintptr(psidGroup), uintptr(pDacl), uintptr(pSacl), 0, 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
 	}
 	return
 }

vendor/github.com/Microsoft/go-winio/privilege.go

@@ -28,8 +28,9 @@ const (
 
 	ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
 
-	SeBackupPrivilege  = "SeBackupPrivilege"
-	SeRestorePrivilege = "SeRestorePrivilege"
+	SeBackupPrivilege   = "SeBackupPrivilege"
+	SeRestorePrivilege  = "SeRestorePrivilege"
+	SeSecurityPrivilege = "SeSecurityPrivilege"
 )
 
 const (

vendor/github.com/Microsoft/go-winio/syscall.go

 package winio
 
-//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go
+//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go

vendor/github.com/Microsoft/go-winio/vhd/zvhd.go → vendor/github.com/Microsoft/go-winio/vhd/zvhd_windows.go

-// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
+// Code generated by 'go generate'; DO NOT EDIT.
 
 package vhd
 
@@ -19,6 +19,7 @@ const (
 
 var (
 	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
 )
 
 // errnoErr returns common boxed Errno values, to prevent
@@ -26,7 +27,7 @@ var (
 func errnoErr(e syscall.Errno) error {
 	switch e {
 	case 0:
-		return nil
+		return errERROR_EINVAL
 	case errnoERROR_IO_PENDING:
 		return errERROR_IO_PENDING
 	}
@@ -37,63 +38,69 @@ func errnoErr(e syscall.Errno) error {
 }
 
 var (
-	modVirtDisk = windows.NewLazySystemDLL("VirtDisk.dll")
+	modvirtdisk = windows.NewLazySystemDLL("virtdisk.dll")
 
-	procCreateVirtualDisk = modVirtDisk.NewProc("CreateVirtualDisk")
-	procOpenVirtualDisk   = modVirtDisk.NewProc("OpenVirtualDisk")
-	procDetachVirtualDisk = modVirtDisk.NewProc("DetachVirtualDisk")
+	procAttachVirtualDisk          = modvirtdisk.NewProc("AttachVirtualDisk")
+	procCreateVirtualDisk          = modvirtdisk.NewProc("CreateVirtualDisk")
+	procDetachVirtualDisk          = modvirtdisk.NewProc("DetachVirtualDisk")
+	procGetVirtualDiskPhysicalPath = modvirtdisk.NewProc("GetVirtualDiskPhysicalPath")
+	procOpenVirtualDisk            = modvirtdisk.NewProc("OpenVirtualDisk")
 )
 
-func createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
+func attachVirtualDisk(handle syscall.Handle, securityDescriptor *uintptr, attachVirtualDiskFlag uint32, providerSpecificFlags uint32, parameters *AttachVirtualDiskParameters, overlapped *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall6(procAttachVirtualDisk.Addr(), 6, uintptr(handle), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(attachVirtualDiskFlag), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(overlapped)))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func createVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (err error) {
 	var _p0 *uint16
 	_p0, err = syscall.UTF16PtrFromString(path)
 	if err != nil {
 		return
 	}
-	return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, flags, providerSpecificFlags, parameters, o, handle)
+	return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, createVirtualDiskFlags, providerSpecificFlags, parameters, overlapped, handle)
 }
 
-func _createVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
-	r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(flags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(handle)))
+func _createVirtualDisk(virtualStorageType *VirtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(createVirtualDiskFlags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(handle)))
 	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return
+func detachVirtualDisk(handle syscall.Handle, detachVirtualDiskFlags uint32, providerSpecificFlags uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(detachVirtualDiskFlags), uintptr(providerSpecificFlags))
+	if r1 != 0 {
+		err = errnoErr(e1)
 	}
-	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, flags, parameters, handle)
+	return
 }
 
-func _openVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
-	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(flags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
+func getVirtualDiskPhysicalPath(handle syscall.Handle, diskPathSizeInBytes *uint32, buffer *uint16) (err error) {
+	r1, _, e1 := syscall.Syscall(procGetVirtualDiskPhysicalPath.Addr(), 3, uintptr(handle), uintptr(unsafe.Pointer(diskPathSizeInBytes)), uintptr(unsafe.Pointer(buffer)))
 	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) {
-	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(flags), uintptr(providerSpecificFlags))
+func openVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *OpenVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, openVirtualDiskFlags, parameters, handle)
+}
+
+func _openVirtualDisk(virtualStorageType *VirtualStorageType, path *uint16, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *OpenVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(openVirtualDiskFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
 	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }

vendor/github.com/Microsoft/hcsshim/.gitattributes

+* text=auto eol=lf
\ No newline at end of file

vendor/github.com/Microsoft/hcsshim/.gitignore

 *.exe
+.idea
+.vscode

vendor/github.com/Microsoft/hcsshim/.gometalinter.json

-{
-    "Vendor": true,
-    "Deadline": "2m",
-    "Sort": [
-        "linter",
-        "severity",
-        "path",
-        "line"
-    ],
-    "Skip": [
-        "internal\\schema2"
-    ],
-    "EnableGC": true,
-    "Enable": [
-        "gofmt"
-    ]
-}
\ No newline at end of file

vendor/github.com/Microsoft/hcsshim/CODEOWNERS

-* @microsoft/containerplat
-
-/hcn/* @nagiesek
\ No newline at end of file
+* @microsoft/containerplat
\ No newline at end of file

vendor/github.com/Microsoft/hcsshim/Protobuild.toml

@@ -25,6 +25,7 @@ plugins = ["grpc", "fieldpath"]
   "gogoproto/gogo.proto" = "github.com/gogo/protobuf/gogoproto"
   "google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
   "google/protobuf/empty.proto" = "github.com/gogo/protobuf/types"
+  "google/protobuf/struct.proto" = "github.com/gogo/protobuf/types"
   "google/protobuf/descriptor.proto" = "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
   "google/protobuf/field_mask.proto" = "github.com/gogo/protobuf/types"
   "google/protobuf/timestamp.proto" = "github.com/gogo/protobuf/types"
@@ -35,20 +36,14 @@ plugins = ["grpc", "fieldpath"]
 prefixes = ["github.com/Microsoft/hcsshim/internal/shimdiag"]
 plugins = ["ttrpc"]
 
-# Lock down runhcs config
-
-[[descriptors]]
-prefix = "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
-target = "cmd/containerd-shim-runhcs-v1/options/next.pb.txt"
-ignore_files = [
-	"google/protobuf/descriptor.proto",
-	"gogoproto/gogo.proto"
-]
-
-[[descriptors]]
-prefix = "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
-target = "cmd/containerd-shim-runhcs-v1/stats/next.pb.txt"
-ignore_files = [
-	"google/protobuf/descriptor.proto",
-	"gogoproto/gogo.proto"
-]
\ No newline at end of file
+[[overrides]]
+prefixes = ["github.com/Microsoft/hcsshim/internal/computeagent"]
+plugins = ["ttrpc"]
+
+[[overrides]]
+prefixes = ["github.com/Microsoft/hcsshim/internal/ncproxyttrpc"]
+plugins = ["ttrpc"]
+
+[[overrides]]
+prefixes = ["github.com/Microsoft/hcsshim/internal/vmservice"]
+plugins = ["ttrpc"]
\ No newline at end of file

vendor/github.com/Microsoft/hcsshim/README.md

 # hcsshim
 
-[![Build status](https://ci.appveyor.com/api/projects/status/nbcw28mnkqml0loa/branch/master?svg=true)](https://ci.appveyor.com/project/WindowsVirtualization/hcsshim/branch/master)
+[![Build status](https://github.com/microsoft/hcsshim/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/microsoft/hcsshim/actions?query=branch%3Amaster)
 
 This package contains the Golang interface for using the Windows [Host Compute Service](https://techcommunity.microsoft.com/t5/containers/introducing-the-host-compute-service-hcs/ba-p/382332) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS).
 

vendor/github.com/Microsoft/hcsshim/appveyor.yml

-version: 0.1.{build}
-
-image: Visual Studio 2017
-
-clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim
-
-environment:
-  GOPATH: c:\gopath
-  PATH: "%GOPATH%\\bin;C:\\gometalinter-2.0.12-windows-amd64;%PATH%"
-
-stack: go 1.13.4
-
-build_script:
-  - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip
-  - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL
-  - gometalinter.exe --config .gometalinter.json ./...
-  - go build ./cmd/containerd-shim-runhcs-v1
-  - go build ./cmd/runhcs
-  - go build ./cmd/tar2ext4
-  - go build ./cmd/wclayer
-  - go build ./cmd/device-util
-  - go build ./internal/tools/grantvmgroupaccess 
-  - go build ./internal/tools/uvmboot
-  - go build ./internal/tools/zapdir
-  - go test -v ./... -tags admin
-  - cd test
-  - go test -v ./internal -tags admin
-  - go test -c ./containerd-shim-runhcs-v1/ -tags functional
-  - go test -c ./cri-containerd/ -tags functional
-  - go test -c ./functional/ -tags functional
-  - go test -c ./runhcs/ -tags functional
-
-artifacts:
-  - path: 'containerd-shim-runhcs-v1.exe'
-  - path: 'runhcs.exe'
-  - path: 'tar2ext4.exe'
-  - path: 'device-util.exe'
-  - path: 'wclayer.exe'
-  - path: 'grantvmgroupaccess.exe'  
-  - path: 'uvmboot.exe'
-  - path: 'zapdir.exe'
-  - path: './test/containerd-shim-runhcs-v1.test.exe'
-  - path: './test/cri-containerd.test.exe'
-  - path: './test/functional.test.exe'
-  - path: './test/runhcs.test.exe'

vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/next.pb.txt

-file {
-  name: "google/protobuf/timestamp.proto"
-  package: "google.protobuf"
-  message_type {
-    name: "Timestamp"
-    field {
-      name: "seconds"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-      json_name: "seconds"
-    }
-    field {
-      name: "nanos"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-      json_name: "nanos"
-    }
-  }
-  options {
-    java_package: "com.google.protobuf"
-    java_outer_classname: "TimestampProto"
-    java_multiple_files: true
-    go_package: "github.com/golang/protobuf/ptypes/timestamp"
-    cc_enable_arenas: true
-    objc_class_prefix: "GPB"
-    csharp_namespace: "Google.Protobuf.WellKnownTypes"
-  }
-  syntax: "proto3"
-}
-file {
-  name: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto"
-  package: "containerd.runhcs.v1"
-  dependency: "gogoproto/gogo.proto"
-  dependency: "google/protobuf/timestamp.proto"
-  message_type {
-    name: "Options"
-    field {
-      name: "debug"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-      json_name: "debug"
-    }
-    field {
-      name: "debug_type"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".containerd.runhcs.v1.Options.DebugType"
-      json_name: "debugType"
-    }
-    field {
-      name: "registry_root"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "registryRoot"
-    }
-    field {
-      name: "sandbox_image"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "sandboxImage"
-    }
-    field {
-      name: "sandbox_platform"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "sandboxPlatform"
-    }
-    field {
-      name: "sandbox_isolation"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".containerd.runhcs.v1.Options.SandboxIsolation"
-      json_name: "sandboxIsolation"
-    }
-    field {
-      name: "boot_files_root_path"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "bootFilesRootPath"
-    }
-    field {
-      name: "vm_processor_count"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-      json_name: "vmProcessorCount"
-    }
-    field {
-      name: "vm_memory_size_in_mb"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-      json_name: "vmMemorySizeInMb"
-    }
-    field {
-      name: "GPUVHDPath"
-      number: 10
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "GPUVHDPath"
-    }
-    enum_type {
-      name: "DebugType"
-      value {
-        name: "NPIPE"
-        number: 0
-      }
-      value {
-        name: "FILE"
-        number: 1
-      }
-      value {
-        name: "ETW"
-        number: 2
-      }
-    }
-    enum_type {
-      name: "SandboxIsolation"
-      value {
-        name: "PROCESS"
-        number: 0
-      }
-      value {
-        name: "HYPERVISOR"
-        number: 1
-      }
-    }
-  }
-  message_type {
-    name: "ProcessDetails"
-    field {
-      name: "image_name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "imageName"
-    }
-    field {
-      name: "created_at"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".google.protobuf.Timestamp"
-      options {
-        65001: 0
-        65010: 1
-      }
-      json_name: "createdAt"
-    }
-    field {
-      name: "kernel_time_100_ns"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT64
-      json_name: "kernelTime100Ns"
-    }
-    field {
-      name: "memory_commit_bytes"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT64
-      json_name: "memoryCommitBytes"
-    }
-    field {
-      name: "memory_working_set_private_bytes"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT64
-      json_name: "memoryWorkingSetPrivateBytes"
-    }
-    field {
-      name: "memory_working_set_shared_bytes"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT64
-      json_name: "memoryWorkingSetSharedBytes"
-    }
-    field {
-      name: "process_id"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT32
-      json_name: "processId"
-    }
-    field {
-      name: "user_time_100_ns"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_UINT64
-      json_name: "userTime100Ns"
-    }
-    field {
-      name: "exec_id"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      json_name: "execId"
-    }
-  }
-  options {
-    go_package: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options;options"
-  }
-  weak_dependency: 0
-  syntax: "proto3"
-}

vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto

@@ -62,7 +62,30 @@ message Options {
 
 	// GPUVHDPath is the path to the gpu vhd to add to the uvm
 	// when a container requests a gpu
-	string GPUVHDPath = 10; 
+	string GPUVHDPath = 10;
+
+	// scale_cpu_limits_to_sandbox indicates that container CPU limits should
+	// be adjusted to account for the difference in number of cores between the
+	// host and UVM.
+	bool scale_cpu_limits_to_sandbox = 11;
+
+	// default_container_scratch_size_in_gb is the default scratch size (sandbox.vhdx) 
+	// to be used for containers. Every container will get a sandbox of `size_in_gb` assigned
+	// instead of the default of 20GB. 
+	int32 default_container_scratch_size_in_gb = 12;
+
+	// default_vm_scratch_size_in_gb is the default scratch size (sandbox.vhdx) 
+	// to be used for the UVM. This only applies to WCOW as LCOW doesn't mount a scratch 
+	// specifically for the UVM.
+	int32 default_vm_scratch_size_in_gb = 13;
+
+	// share_scratch specifies if we'd like to reuse scratch space between multiple containers. 
+	// This currently only affects LCOW. The sandbox containers scratch space is re-used for all
+	// subsequent containers launched in the pod.  
+	bool share_scratch = 14;
+	//NCProxyAddr is the address of the network configuration proxy service. If omitted 
+	// the network is setup locally. 
+	string NCProxyAddr = 15; 
 }
 
 // ProcessDetails contains additional information about a process. This is the additional

vendor/github.com/Microsoft/hcsshim/computestorage/attach.go

+package computestorage
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// AttachLayerStorageFilter sets up the layer storage filter on a writable
+// container layer.
+//
+// `layerPath` is a path to a directory the writable layer is mounted. If the
+// path does not end in a `\` the platform will append it automatically.
+//
+// `layerData` is the parent read-only layer data.
+func AttachLayerStorageFilter(ctx context.Context, layerPath string, layerData LayerData) (err error) {
+	title := "hcsshim.AttachLayerStorageFilter"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+	)
+
+	bytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	err = hcsAttachLayerStorageFilter(layerPath, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to attach layer storage filter")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/destroy.go

+package computestorage
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// DestroyLayer deletes a container layer.
+//
+// `layerPath` is a path to a directory containing the layer to export.
+func DestroyLayer(ctx context.Context, layerPath string) (err error) {
+	title := "hcsshim.DestroyLayer"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(trace.StringAttribute("layerPath", layerPath))
+
+	err = hcsDestroyLayer(layerPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to destroy layer")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/detach.go

+package computestorage
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// DetachLayerStorageFilter detaches the layer storage filter on a writable container layer.
+//
+// `layerPath` is a path to a directory containing the layer to export.
+func DetachLayerStorageFilter(ctx context.Context, layerPath string) (err error) {
+	title := "hcsshim.DetachLayerStorageFilter"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(trace.StringAttribute("layerPath", layerPath))
+
+	err = hcsDetachLayerStorageFilter(layerPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to detach layer storage filter")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/export.go

+package computestorage
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// ExportLayer exports a container layer.
+//
+// `layerPath` is a path to a directory containing the layer to export.
+//
+// `exportFolderPath` is a pre-existing folder to export the layer to.
+//
+// `layerData` is the parent layer data.
+//
+// `options` are the export options applied to the exported layer.
+func ExportLayer(ctx context.Context, layerPath, exportFolderPath string, layerData LayerData, options ExportLayerOptions) (err error) {
+	title := "hcsshim.ExportLayer"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+		trace.StringAttribute("exportFolderPath", exportFolderPath),
+	)
+
+	ldbytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	obytes, err := json.Marshal(options)
+	if err != nil {
+		return err
+	}
+
+	err = hcsExportLayer(layerPath, exportFolderPath, string(ldbytes), string(obytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to export layer")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/format.go

+package computestorage
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+	"golang.org/x/sys/windows"
+)
+
+// FormatWritableLayerVhd formats a virtual disk for use as a writable container layer.
+//
+// If the VHD is not mounted it will be temporarily mounted.
+func FormatWritableLayerVhd(ctx context.Context, vhdHandle windows.Handle) (err error) {
+	title := "hcsshim.FormatWritableLayerVhd"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+
+	err = hcsFormatWritableLayerVhd(vhdHandle)
+	if err != nil {
+		return errors.Wrap(err, "failed to format writable layer vhd")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/helpers.go

+package computestorage
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/Microsoft/go-winio/pkg/security"
+	"github.com/Microsoft/go-winio/vhd"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
+
+const defaultVHDXBlockSizeInMB = 1
+
+// SetupContainerBaseLayer is a helper to setup a containers scratch. It
+// will create and format the vhdx's inside and the size is configurable with the sizeInGB
+// parameter.
+//
+// `layerPath` is the path to the base container layer on disk.
+//
+// `baseVhdPath` is the path to where the base vhdx for the base layer should be created.
+//
+// `diffVhdPath` is the path where the differencing disk for the base layer should be created.
+//
+// `sizeInGB` is the size in gigabytes to make the base vhdx.
+func SetupContainerBaseLayer(ctx context.Context, layerPath, baseVhdPath, diffVhdPath string, sizeInGB uint64) (err error) {
+	var (
+		hivesPath  = filepath.Join(layerPath, "Hives")
+		layoutPath = filepath.Join(layerPath, "Layout")
+	)
+
+	// We need to remove the hives directory and layout file as `SetupBaseOSLayer` fails if these files
+	// already exist. `SetupBaseOSLayer` will create these files internally. We also remove the base and
+	// differencing disks if they exist in case we're asking for a different size.
+	if _, err := os.Stat(hivesPath); err == nil {
+		if err := os.RemoveAll(hivesPath); err != nil {
+			return errors.Wrap(err, "failed to remove prexisting hives directory")
+		}
+	}
+	if _, err := os.Stat(layoutPath); err == nil {
+		if err := os.RemoveAll(layoutPath); err != nil {
+			return errors.Wrap(err, "failed to remove prexisting layout file")
+		}
+	}
+
+	if _, err := os.Stat(baseVhdPath); err == nil {
+		if err := os.RemoveAll(baseVhdPath); err != nil {
+			return errors.Wrap(err, "failed to remove base vhdx path")
+		}
+	}
+	if _, err := os.Stat(diffVhdPath); err == nil {
+		if err := os.RemoveAll(diffVhdPath); err != nil {
+			return errors.Wrap(err, "failed to remove differencing vhdx")
+		}
+	}
+
+	createParams := &vhd.CreateVirtualDiskParameters{
+		Version: 2,
+		Version2: vhd.CreateVersion2{
+			MaximumSize:      sizeInGB * 1024 * 1024 * 1024,
+			BlockSizeInBytes: defaultVHDXBlockSizeInMB * 1024 * 1024,
+		},
+	}
+	handle, err := vhd.CreateVirtualDisk(baseVhdPath, vhd.VirtualDiskAccessNone, vhd.CreateVirtualDiskFlagNone, createParams)
+	if err != nil {
+		return errors.Wrap(err, "failed to create vhdx")
+	}
+
+	defer func() {
+		if err != nil {
+			_ = syscall.CloseHandle(handle)
+			os.RemoveAll(baseVhdPath)
+			os.RemoveAll(diffVhdPath)
+		}
+	}()
+
+	if err = FormatWritableLayerVhd(ctx, windows.Handle(handle)); err != nil {
+		return err
+	}
+	// Base vhd handle must be closed before calling SetupBaseLayer in case of Container layer
+	if err = syscall.CloseHandle(handle); err != nil {
+		return errors.Wrap(err, "failed to close vhdx handle")
+	}
+
+	options := OsLayerOptions{
+		Type: OsLayerTypeContainer,
+	}
+
+	// SetupBaseOSLayer expects an empty vhd handle for a container layer and will
+	// error out otherwise.
+	if err = SetupBaseOSLayer(ctx, layerPath, 0, options); err != nil {
+		return err
+	}
+	// Create the differencing disk that will be what's copied for the final rw layer
+	// for a container.
+	if err = vhd.CreateDiffVhd(diffVhdPath, baseVhdPath, defaultVHDXBlockSizeInMB); err != nil {
+		return errors.Wrap(err, "failed to create differencing disk")
+	}
+
+	if err = security.GrantVmGroupAccess(baseVhdPath); err != nil {
+		return errors.Wrapf(err, "failed to grant vm group access to %s", baseVhdPath)
+	}
+	if err = security.GrantVmGroupAccess(diffVhdPath); err != nil {
+		return errors.Wrapf(err, "failed to grant vm group access to %s", diffVhdPath)
+	}
+	return nil
+}
+
+// SetupUtilityVMBaseLayer is a helper to setup a UVMs scratch space. It will create and format
+// the vhdx inside and the size is configurable by the sizeInGB parameter.
+//
+// `uvmPath` is the path to the UtilityVM filesystem.
+//
+// `baseVhdPath` is the path to where the base vhdx for the UVM should be created.
+//
+// `diffVhdPath` is the path where the differencing disk for the UVM should be created.
+//
+// `sizeInGB` specifies the size in gigabytes to make the base vhdx.
+func SetupUtilityVMBaseLayer(ctx context.Context, uvmPath, baseVhdPath, diffVhdPath string, sizeInGB uint64) (err error) {
+	// Remove the base and differencing disks if they exist in case we're asking for a different size.
+	if _, err := os.Stat(baseVhdPath); err == nil {
+		if err := os.RemoveAll(baseVhdPath); err != nil {
+			return errors.Wrap(err, "failed to remove base vhdx")
+		}
+	}
+	if _, err := os.Stat(diffVhdPath); err == nil {
+		if err := os.RemoveAll(diffVhdPath); err != nil {
+			return errors.Wrap(err, "failed to remove differencing vhdx")
+		}
+	}
+
+	// Just create the vhdx for utilityVM layer, no need to format it.
+	createParams := &vhd.CreateVirtualDiskParameters{
+		Version: 2,
+		Version2: vhd.CreateVersion2{
+			MaximumSize:      sizeInGB * 1024 * 1024 * 1024,
+			BlockSizeInBytes: defaultVHDXBlockSizeInMB * 1024 * 1024,
+		},
+	}
+	handle, err := vhd.CreateVirtualDisk(baseVhdPath, vhd.VirtualDiskAccessNone, vhd.CreateVirtualDiskFlagNone, createParams)
+	if err != nil {
+		return errors.Wrap(err, "failed to create vhdx")
+	}
+
+	defer func() {
+		if err != nil {
+			_ = syscall.CloseHandle(handle)
+			os.RemoveAll(baseVhdPath)
+			os.RemoveAll(diffVhdPath)
+		}
+	}()
+
+	// If it is a UtilityVM layer then the base vhdx must be attached when calling
+	// `SetupBaseOSLayer`
+	attachParams := &vhd.AttachVirtualDiskParameters{
+		Version: 2,
+	}
+	if err := vhd.AttachVirtualDisk(handle, vhd.AttachVirtualDiskFlagNone, attachParams); err != nil {
+		return errors.Wrapf(err, "failed to attach virtual disk")
+	}
+
+	options := OsLayerOptions{
+		Type: OsLayerTypeVM,
+	}
+	if err := SetupBaseOSLayer(ctx, uvmPath, windows.Handle(handle), options); err != nil {
+		return err
+	}
+
+	// Detach and close the handle after setting up the layer as we don't need the handle
+	// for anything else and we no longer need to be attached either.
+	if err = vhd.DetachVirtualDisk(handle); err != nil {
+		return errors.Wrap(err, "failed to detach vhdx")
+	}
+	if err = syscall.CloseHandle(handle); err != nil {
+		return errors.Wrap(err, "failed to close vhdx handle")
+	}
+
+	// Create the differencing disk that will be what's copied for the final rw layer
+	// for a container.
+	if err = vhd.CreateDiffVhd(diffVhdPath, baseVhdPath, defaultVHDXBlockSizeInMB); err != nil {
+		return errors.Wrap(err, "failed to create differencing disk")
+	}
+
+	if err := security.GrantVmGroupAccess(baseVhdPath); err != nil {
+		return errors.Wrapf(err, "failed to grant vm group access to %s", baseVhdPath)
+	}
+	if err := security.GrantVmGroupAccess(diffVhdPath); err != nil {
+		return errors.Wrapf(err, "failed to grant vm group access to %s", diffVhdPath)
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/import.go

+package computestorage
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// ImportLayer imports a container layer.
+//
+// `layerPath` is a path to a directory to import the layer to. If the directory
+// does not exist it will be automatically created.
+//
+// `sourceFolderpath` is a pre-existing folder that contains the layer to
+// import.
+//
+// `layerData` is the parent layer data.
+func ImportLayer(ctx context.Context, layerPath, sourceFolderPath string, layerData LayerData) (err error) {
+	title := "hcsshim.ImportLayer"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+		trace.StringAttribute("sourceFolderPath", sourceFolderPath),
+	)
+
+	bytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	err = hcsImportLayer(layerPath, sourceFolderPath, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to import layer")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/initialize.go

+package computestorage
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+// InitializeWritableLayer initializes a writable layer for a container.
+//
+// `layerPath` is a path to a directory the layer is mounted. If the
+// path does not end in a `\` the platform will append it automatically.
+//
+// `layerData` is the parent read-only layer data.
+func InitializeWritableLayer(ctx context.Context, layerPath string, layerData LayerData) (err error) {
+	title := "hcsshim.InitializeWritableLayer"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+	)
+
+	bytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	// Options are not used in the platform as of RS5
+	err = hcsInitializeWritableLayer(layerPath, string(bytes), "")
+	if err != nil {
+		return errors.Wrap(err, "failed to intitialize container layer")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/mount.go

+package computestorage
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/interop"
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+	"golang.org/x/sys/windows"
+)
+
+// GetLayerVhdMountPath returns the volume path for a virtual disk of a writable container layer.
+func GetLayerVhdMountPath(ctx context.Context, vhdHandle windows.Handle) (path string, err error) {
+	title := "hcsshim.GetLayerVhdMountPath"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+
+	var mountPath *uint16
+	err = hcsGetLayerVhdMountPath(vhdHandle, &mountPath)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get vhd mount path")
+	}
+	path = interop.ConvertAndFreeCoTaskMemString(mountPath)
+	return path, nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/setup.go

+package computestorage
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/Microsoft/hcsshim/osversion"
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+	"golang.org/x/sys/windows"
+)
+
+// SetupBaseOSLayer sets up a layer that contains a base OS for a container.
+//
+// `layerPath` is a path to a directory containing the layer.
+//
+// `vhdHandle` is an empty file handle of `options.Type == OsLayerTypeContainer`
+// or else it is a file handle to the 'SystemTemplateBase.vhdx' if `options.Type
+// == OsLayerTypeVm`.
+//
+// `options` are the options applied while processing the layer.
+func SetupBaseOSLayer(ctx context.Context, layerPath string, vhdHandle windows.Handle, options OsLayerOptions) (err error) {
+	title := "hcsshim.SetupBaseOSLayer"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+	)
+
+	bytes, err := json.Marshal(options)
+	if err != nil {
+		return err
+	}
+
+	err = hcsSetupBaseOSLayer(layerPath, vhdHandle, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to setup base OS layer")
+	}
+	return nil
+}
+
+// SetupBaseOSVolume sets up a volume that contains a base OS for a container.
+//
+// `layerPath` is a path to a directory containing the layer.
+//
+// `volumePath` is the path to the volume to be used for setup.
+//
+// `options` are the options applied while processing the layer.
+func SetupBaseOSVolume(ctx context.Context, layerPath, volumePath string, options OsLayerOptions) (err error) {
+	if osversion.Build() < 19645 {
+		return errors.New("SetupBaseOSVolume is not present on builds older than 19645")
+	}
+	title := "hcsshim.SetupBaseOSVolume"
+	ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("layerPath", layerPath),
+		trace.StringAttribute("volumePath", volumePath),
+	)
+
+	bytes, err := json.Marshal(options)
+	if err != nil {
+		return err
+	}
+
+	err = hcsSetupBaseOSVolume(layerPath, volumePath, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to setup base OS layer")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/computestorage/storage.go

+// Package computestorage is a wrapper around the HCS storage APIs. These are new storage APIs introduced
+// separate from the original graphdriver calls intended to give more freedom around creating
+// and managing container layers and scratch spaces.
+package computestorage
+
+import (
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+)
+
+//go:generate go run ../mksyscall_windows.go -output zsyscall_windows.go storage.go
+
+//sys hcsImportLayer(layerPath string, sourceFolderPath string, layerData string) (hr error) = computestorage.HcsImportLayer?
+//sys hcsExportLayer(layerPath string, exportFolderPath string, layerData string, options string) (hr error) = computestorage.HcsExportLayer?
+//sys hcsDestroyLayer(layerPath string) (hr error) = computestorage.HcsDestoryLayer?
+//sys hcsSetupBaseOSLayer(layerPath string, handle windows.Handle, options string) (hr error) = computestorage.HcsSetupBaseOSLayer?
+//sys hcsInitializeWritableLayer(writableLayerPath string, layerData string, options string) (hr error) = computestorage.HcsInitializeWritableLayer?
+//sys hcsAttachLayerStorageFilter(layerPath string, layerData string) (hr error) = computestorage.HcsAttachLayerStorageFilter?
+//sys hcsDetachLayerStorageFilter(layerPath string) (hr error) = computestorage.HcsDetachLayerStorageFilter?
+//sys hcsFormatWritableLayerVhd(handle windows.Handle) (hr error) = computestorage.HcsFormatWritableLayerVhd?
+//sys hcsGetLayerVhdMountPath(vhdHandle windows.Handle, mountPath **uint16) (hr error) = computestorage.HcsGetLayerVhdMountPath?
+//sys hcsSetupBaseOSVolume(layerPath string, volumePath string, options string) (hr error) = computestorage.HcsSetupBaseOSVolume?
+
+// LayerData is the data used to describe parent layer information.
+type LayerData struct {
+	SchemaVersion hcsschema.Version `json:"SchemaVersion,omitempty"`
+	Layers        []hcsschema.Layer `json:"Layers,omitempty"`
+}
+
+// ExportLayerOptions are the set of options that are used with the `computestorage.HcsExportLayer` syscall.
+type ExportLayerOptions struct {
+	IsWritableLayer bool `json:"IsWritableLayer,omitempty"`
+}
+
+// OsLayerType is the type of layer being operated on.
+type OsLayerType string
+
+const (
+	// OsLayerTypeContainer is a container layer.
+	OsLayerTypeContainer OsLayerType = "Container"
+	// OsLayerTypeVM is a virtual machine layer.
+	OsLayerTypeVM OsLayerType = "Vm"
+)
+
+// OsLayerOptions are the set of options that are used with the `SetupBaseOSLayer` and
+// `SetupBaseOSVolume` calls.
+type OsLayerOptions struct {
+	Type                       OsLayerType `json:"Type,omitempty"`
+	DisableCiCacheOptimization bool        `json:"DisableCiCacheOptimization,omitempty"`
+	SkipUpdateBcdForBoot       bool        `json:"SkipUpdateBcdForBoot,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/computestorage/zsyscall_windows.go

+// Code generated mksyscall_windows.exe DO NOT EDIT
+
+package computestorage
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return nil
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modcomputestorage = windows.NewLazySystemDLL("computestorage.dll")
+
+	procHcsImportLayer              = modcomputestorage.NewProc("HcsImportLayer")
+	procHcsExportLayer              = modcomputestorage.NewProc("HcsExportLayer")
+	procHcsDestoryLayer             = modcomputestorage.NewProc("HcsDestoryLayer")
+	procHcsSetupBaseOSLayer         = modcomputestorage.NewProc("HcsSetupBaseOSLayer")
+	procHcsInitializeWritableLayer  = modcomputestorage.NewProc("HcsInitializeWritableLayer")
+	procHcsAttachLayerStorageFilter = modcomputestorage.NewProc("HcsAttachLayerStorageFilter")
+	procHcsDetachLayerStorageFilter = modcomputestorage.NewProc("HcsDetachLayerStorageFilter")
+	procHcsFormatWritableLayerVhd   = modcomputestorage.NewProc("HcsFormatWritableLayerVhd")
+	procHcsGetLayerVhdMountPath     = modcomputestorage.NewProc("HcsGetLayerVhdMountPath")
+	procHcsSetupBaseOSVolume        = modcomputestorage.NewProc("HcsSetupBaseOSVolume")
+)
+
+func hcsImportLayer(layerPath string, sourceFolderPath string, layerData string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(sourceFolderPath)
+	if hr != nil {
+		return
+	}
+	var _p2 *uint16
+	_p2, hr = syscall.UTF16PtrFromString(layerData)
+	if hr != nil {
+		return
+	}
+	return _hcsImportLayer(_p0, _p1, _p2)
+}
+
+func _hcsImportLayer(layerPath *uint16, sourceFolderPath *uint16, layerData *uint16) (hr error) {
+	if hr = procHcsImportLayer.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsImportLayer.Addr(), 3, uintptr(unsafe.Pointer(layerPath)), uintptr(unsafe.Pointer(sourceFolderPath)), uintptr(unsafe.Pointer(layerData)))
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsExportLayer(layerPath string, exportFolderPath string, layerData string, options string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(exportFolderPath)
+	if hr != nil {
+		return
+	}
+	var _p2 *uint16
+	_p2, hr = syscall.UTF16PtrFromString(layerData)
+	if hr != nil {
+		return
+	}
+	var _p3 *uint16
+	_p3, hr = syscall.UTF16PtrFromString(options)
+	if hr != nil {
+		return
+	}
+	return _hcsExportLayer(_p0, _p1, _p2, _p3)
+}
+
+func _hcsExportLayer(layerPath *uint16, exportFolderPath *uint16, layerData *uint16, options *uint16) (hr error) {
+	if hr = procHcsExportLayer.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall6(procHcsExportLayer.Addr(), 4, uintptr(unsafe.Pointer(layerPath)), uintptr(unsafe.Pointer(exportFolderPath)), uintptr(unsafe.Pointer(layerData)), uintptr(unsafe.Pointer(options)), 0, 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsDestroyLayer(layerPath string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	return _hcsDestroyLayer(_p0)
+}
+
+func _hcsDestroyLayer(layerPath *uint16) (hr error) {
+	if hr = procHcsDestoryLayer.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsDestoryLayer.Addr(), 1, uintptr(unsafe.Pointer(layerPath)), 0, 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsSetupBaseOSLayer(layerPath string, handle windows.Handle, options string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(options)
+	if hr != nil {
+		return
+	}
+	return _hcsSetupBaseOSLayer(_p0, handle, _p1)
+}
+
+func _hcsSetupBaseOSLayer(layerPath *uint16, handle windows.Handle, options *uint16) (hr error) {
+	if hr = procHcsSetupBaseOSLayer.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsSetupBaseOSLayer.Addr(), 3, uintptr(unsafe.Pointer(layerPath)), uintptr(handle), uintptr(unsafe.Pointer(options)))
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsInitializeWritableLayer(writableLayerPath string, layerData string, options string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(writableLayerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(layerData)
+	if hr != nil {
+		return
+	}
+	var _p2 *uint16
+	_p2, hr = syscall.UTF16PtrFromString(options)
+	if hr != nil {
+		return
+	}
+	return _hcsInitializeWritableLayer(_p0, _p1, _p2)
+}
+
+func _hcsInitializeWritableLayer(writableLayerPath *uint16, layerData *uint16, options *uint16) (hr error) {
+	if hr = procHcsInitializeWritableLayer.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsInitializeWritableLayer.Addr(), 3, uintptr(unsafe.Pointer(writableLayerPath)), uintptr(unsafe.Pointer(layerData)), uintptr(unsafe.Pointer(options)))
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsAttachLayerStorageFilter(layerPath string, layerData string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(layerData)
+	if hr != nil {
+		return
+	}
+	return _hcsAttachLayerStorageFilter(_p0, _p1)
+}
+
+func _hcsAttachLayerStorageFilter(layerPath *uint16, layerData *uint16) (hr error) {
+	if hr = procHcsAttachLayerStorageFilter.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsAttachLayerStorageFilter.Addr(), 2, uintptr(unsafe.Pointer(layerPath)), uintptr(unsafe.Pointer(layerData)), 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsDetachLayerStorageFilter(layerPath string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	return _hcsDetachLayerStorageFilter(_p0)
+}
+
+func _hcsDetachLayerStorageFilter(layerPath *uint16) (hr error) {
+	if hr = procHcsDetachLayerStorageFilter.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsDetachLayerStorageFilter.Addr(), 1, uintptr(unsafe.Pointer(layerPath)), 0, 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsFormatWritableLayerVhd(handle windows.Handle) (hr error) {
+	if hr = procHcsFormatWritableLayerVhd.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsFormatWritableLayerVhd.Addr(), 1, uintptr(handle), 0, 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsGetLayerVhdMountPath(vhdHandle windows.Handle, mountPath **uint16) (hr error) {
+	if hr = procHcsGetLayerVhdMountPath.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsGetLayerVhdMountPath.Addr(), 2, uintptr(vhdHandle), uintptr(unsafe.Pointer(mountPath)), 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func hcsSetupBaseOSVolume(layerPath string, volumePath string, options string) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(layerPath)
+	if hr != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, hr = syscall.UTF16PtrFromString(volumePath)
+	if hr != nil {
+		return
+	}
+	var _p2 *uint16
+	_p2, hr = syscall.UTF16PtrFromString(options)
+	if hr != nil {
+		return
+	}
+	return _hcsSetupBaseOSVolume(_p0, _p1, _p2)
+}
+
+func _hcsSetupBaseOSVolume(layerPath *uint16, volumePath *uint16, options *uint16) (hr error) {
+	if hr = procHcsSetupBaseOSVolume.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsSetupBaseOSVolume.Addr(), 3, uintptr(unsafe.Pointer(layerPath)), uintptr(unsafe.Pointer(volumePath)), uintptr(unsafe.Pointer(options)))
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}

vendor/github.com/Microsoft/hcsshim/container.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/hcs/schema1"
 	"github.com/Microsoft/hcsshim/internal/mergemaps"
-	"github.com/Microsoft/hcsshim/internal/schema1"
 )
 
 // ContainerProperties holds the properties for a container and the processes running in that container

vendor/github.com/Microsoft/hcsshim/errors.go

@@ -59,7 +59,7 @@ var (
 	// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
 	ErrVmcomputeOperationInvalidState = hcs.ErrVmcomputeOperationInvalidState
 
-	// ErrProcNotFound is an error encountered when the the process cannot be found
+	// ErrProcNotFound is an error encountered when a procedure look up fails.
 	ErrProcNotFound = hcs.ErrProcNotFound
 
 	// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
@@ -83,7 +83,6 @@ type NetworkNotFoundError = hns.NetworkNotFoundError
 type ProcessError struct {
 	Process   *process
 	Operation string
-	ExtraInfo string
 	Err       error
 	Events    []hcs.ErrorEvent
 }
@@ -92,7 +91,6 @@ type ProcessError struct {
 type ContainerError struct {
 	Container *container
 	Operation string
-	ExtraInfo string
 	Err       error
 	Events    []hcs.ErrorEvent
 }
@@ -125,22 +123,9 @@ func (e *ContainerError) Error() string {
 		s += "\n" + ev.String()
 	}
 
-	if e.ExtraInfo != "" {
-		s += " extra info: " + e.ExtraInfo
-	}
-
 	return s
 }
 
-func makeContainerError(container *container, operation string, extraInfo string, err error) error {
-	// Don't double wrap errors
-	if _, ok := err.(*ContainerError); ok {
-		return err
-	}
-	containerError := &ContainerError{Container: container, Operation: operation, ExtraInfo: extraInfo, Err: err}
-	return containerError
-}
-
 func (e *ProcessError) Error() string {
 	if e == nil {
 		return "<nil>"
@@ -171,19 +156,10 @@ func (e *ProcessError) Error() string {
 	return s
 }
 
-func makeProcessError(process *process, operation string, extraInfo string, err error) error {
-	// Don't double wrap errors
-	if _, ok := err.(*ProcessError); ok {
-		return err
-	}
-	processError := &ProcessError{Process: process, Operation: operation, ExtraInfo: extraInfo, Err: err}
-	return processError
-}
-
 // IsNotExist checks if an error is caused by the Container or Process not existing.
 // Note: Currently, ErrElementNotFound can mean that a Process has either
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
-// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+// will currently return true when the error is ErrElementNotFound.
 func IsNotExist(err error) bool {
 	if _, ok := err.(EndpointNotFoundError); ok {
 		return true
@@ -216,7 +192,7 @@ func IsTimeout(err error) bool {
 // a Container or Process being already stopped.
 // Note: Currently, ErrElementNotFound can mean that a Process has either
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
-// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+// will currently return true when the error is ErrElementNotFound.
 func IsAlreadyStopped(err error) bool {
 	return hcs.IsAlreadyStopped(getInnerError(err))
 }
@@ -230,6 +206,18 @@ func IsNotSupported(err error) bool {
 	return hcs.IsNotSupported(getInnerError(err))
 }
 
+// IsOperationInvalidState returns true when err is caused by
+// `ErrVmcomputeOperationInvalidState`.
+func IsOperationInvalidState(err error) bool {
+	return hcs.IsOperationInvalidState(getInnerError(err))
+}
+
+// IsAccessIsDenied returns true when err is caused by
+// `ErrVmcomputeOperationAccessIsDenied`.
+func IsAccessIsDenied(err error) bool {
+	return hcs.IsAccessIsDenied(getInnerError(err))
+}
+
 func getInnerError(err error) error {
 	switch pe := err.(type) {
 	case nil:
@@ -244,7 +232,7 @@ func getInnerError(err error) error {
 
 func convertSystemError(err error, c *container) error {
 	if serr, ok := err.(*hcs.SystemError); ok {
-		return &ContainerError{Container: c, Operation: serr.Op, ExtraInfo: serr.Extra, Err: serr.Err, Events: serr.Events}
+		return &ContainerError{Container: c, Operation: serr.Op, Err: serr.Err, Events: serr.Events}
 	}
 	return err
 }

vendor/github.com/Microsoft/hcsshim/go.mod

@@ -3,33 +3,25 @@ module github.com/Microsoft/hcsshim
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab
-	github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59
-	github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1
-	github.com/containerd/containerd v1.3.2
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
-	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
-	github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de
-	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
-	github.com/gogo/protobuf v1.3.1
-	github.com/golang/protobuf v1.3.2 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 // indirect
-	github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f // indirect
-	github.com/opencontainers/runtime-spec v1.0.2
+	github.com/Microsoft/go-winio v0.4.17
+	github.com/containerd/cgroups v1.0.1
+	github.com/containerd/console v1.0.2
+	github.com/containerd/containerd v1.5.1
+	github.com/containerd/go-runc v1.0.0
+	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/typeurl v1.0.2
+	github.com/gogo/protobuf v1.3.2
+	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 // indirect
-	github.com/sirupsen/logrus v1.4.2
-	github.com/stretchr/testify v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.7.0
 	github.com/urfave/cli v1.22.2
-	go.opencensus.io v0.22.0
-	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20200120151820-655fe14d7479
-	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 // indirect
-	google.golang.org/grpc v1.23.1
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	gotest.tools v2.2.0+incompatible // indirect
+	go.opencensus.io v0.22.3
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
+	google.golang.org/grpc v1.33.2
+)
+
+replace (
+	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
+	google.golang.org/grpc => google.golang.org/grpc v1.27.1
 )

vendor/github.com/Microsoft/hcsshim/hcn/hcn.go

@@ -64,12 +64,6 @@ import (
 //sys hcnDeleteRoute(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteSdnRoute?
 //sys hcnCloseRoute(route hcnRoute) (hr error) = computenetwork.HcnCloseSdnRoute?
 
-// Service
-//sys hcnOpenService(service *hcnService, result **uint16) (hr error) = computenetwork.HcnOpenService?
-//sys hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) = computenetwork.HcnRegisterServiceCallback?
-//sys hcnUnregisterServiceCallback(callbackHandle hcnCallbackHandle) (hr error) = computenetwork.HcnUnregisterServiceCallback?
-//sys hcnCloseService(service hcnService) (hr error) = computenetwork.HcnCloseService?
-
 type _guid = guid.GUID
 
 type hcnNetwork syscall.Handle
@@ -77,8 +71,6 @@ type hcnEndpoint syscall.Handle
 type hcnNamespace syscall.Handle
 type hcnLoadBalancer syscall.Handle
 type hcnRoute syscall.Handle
-type hcnService syscall.Handle
-type hcnCallbackHandle syscall.Handle
 
 // SchemaVersion for HCN Objects/Queries.
 type SchemaVersion = Version // hcnglobals.go
@@ -101,6 +93,20 @@ type HostComputeQuery struct {
 	Filter        string                `json:",omitempty"`
 }
 
+type ExtraParams struct {
+	Resources        json.RawMessage `json:",omitempty"`
+	SharedContainers json.RawMessage `json:",omitempty"`
+	LayeredOn        string          `json:",omitempty"`
+	SwitchGuid       string          `json:",omitempty"`
+	UtilityVM        string          `json:",omitempty"`
+	VirtualMachine   string          `json:",omitempty"`
+}
+
+type Health struct {
+	Data  interface{} `json:",omitempty"`
+	Extra ExtraParams `json:",omitempty"`
+}
+
 // defaultQuery generates HCN Query.
 // Passed into get/enumerate calls to filter results.
 func defaultQuery() HostComputeQuery {
@@ -114,15 +120,6 @@ func defaultQuery() HostComputeQuery {
 	return query
 }
 
-func defaultQueryJson() string {
-	query := defaultQuery()
-	queryJson, err := json.Marshal(query)
-	if err != nil {
-		return ""
-	}
-	return string(queryJson)
-}
-
 // PlatformDoesNotSupportError happens when users are attempting to use a newer shim on an older OS
 func platformDoesNotSupportError(featureName string) error {
 	return fmt.Errorf("Platform does not support feature %s", featureName)
@@ -243,6 +240,15 @@ func VxlanPortSupported() error {
 	return platformDoesNotSupportError("VXLAN port configuration")
 }
 
+// TierAclPolicySupported returns an error if the HCN version does not support configuring the TierAcl.
+func TierAclPolicySupported() error {
+	supported := GetSupportedFeatures()
+	if supported.TierAcl {
+		return nil
+	}
+	return platformDoesNotSupportError("TierAcl")
+}
+
 // RequestType are the different operations performed to settings.
 // Used to update the settings of Endpoint/Namespace objects.
 type RequestType string

vendor/github.com/Microsoft/hcsshim/hcn/hcnendpoint.go

@@ -37,6 +37,7 @@ type HostComputeEndpoint struct {
 	Routes               []Route          `json:",omitempty"`
 	MacAddress           string           `json:",omitempty"`
 	Flags                EndpointFlags    `json:",omitempty"`
+	Health               Health           `json:",omitempty"`
 	SchemaVersion        SchemaVersion    `json:",omitempty"`
 }
 
@@ -58,6 +59,13 @@ type ModifyEndpointSettingRequest struct {
 	Settings     json.RawMessage      `json:",omitempty"`
 }
 
+// VmEndpointRequest creates a switch port with identifier `PortId`.
+type VmEndpointRequest struct {
+	PortId           guid.GUID `json:",omitempty"`
+	VirtualNicName   string    `json:",omitempty"`
+	VirtualMachineId guid.GUID `json:",omitempty"`
+}
+
 type PolicyEndpointRequest struct {
 	Policies []EndpointPolicy `json:",omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go

@@ -53,21 +53,29 @@ var (
 	}
 	// HNS 12.0 allows for session affinity for loadbalancing
 	SessionAffinityVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 12, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
-	// HNS 10.5 through 11 (not included) and 12.0+ supports Ipv6 dual stack.
+	// HNS 11.10+ supports Ipv6 dual stack.
 	IPv6DualStackVersion = VersionRanges{
-		VersionRange{MinVersion: Version{Major: 10, Minor: 5}, MaxVersion: Version{Major: 10, Minor: math.MaxInt32}},
-		VersionRange{MinVersion: Version{Major: 12, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 11, Minor: 10}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
 	}
 	// HNS 13.0 allows for Set Policy support
 	SetPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
 	// HNS 10.3 allows for VXLAN ports
 	VxlanPortVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 10, Minor: 3}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
 
-	//HNS 13.1 allows for L4Proxy Policy support
-	L4ProxyPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 1}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+	//HNS 9.5 through 10.0(not included), 10.5 through 11.0(not included), 11.11 through 12.0(not included), 12.1 through 13.0(not included), 13.1+ allows for Network L4Proxy Policy support
+	L4ProxyPolicyVersion = VersionRanges{
+		VersionRange{MinVersion: Version{Major: 9, Minor: 5}, MaxVersion: Version{Major: 9, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 10, Minor: 5}, MaxVersion: Version{Major: 10, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 11, Minor: 11}, MaxVersion: Version{Major: 11, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 12, Minor: 1}, MaxVersion: Version{Major: 12, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 13, Minor: 1}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
+	}
 
 	//HNS 13.2 allows for L4WfpProxy Policy support
 	L4WfpProxyPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+
+	//HNS 14.0 allows for TierAcl Policy support
+	TierAclPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 14, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
 )
 
 // GetGlobals returns the global properties of the HCN Service.

vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go

@@ -36,6 +36,7 @@ var (
 	LoadBalancerFlagsNone LoadBalancerFlags = 0
 	// LoadBalancerFlagsDSR enables Direct Server Return (DSR)
 	LoadBalancerFlagsDSR LoadBalancerFlags = 1
+	LoadBalancerFlagsIPv6 LoadBalancerFlags = 2
 )
 
 // LoadBalancerPortMappingFlags are special settings on a loadbalancer.
@@ -160,50 +161,6 @@ func createLoadBalancer(settings string) (*HostComputeLoadBalancer, error) {
 	return &outputLoadBalancer, nil
 }
 
-func modifyLoadBalancer(loadBalancerId string, settings string) (*HostComputeLoadBalancer, error) {
-	loadBalancerGuid, err := guid.FromString(loadBalancerId)
-	if err != nil {
-		return nil, errInvalidLoadBalancerID
-	}
-	// Open loadBalancer.
-	var (
-		loadBalancerHandle hcnLoadBalancer
-		resultBuffer       *uint16
-		propertiesBuffer   *uint16
-	)
-	hr := hcnOpenLoadBalancer(&loadBalancerGuid, &loadBalancerHandle, &resultBuffer)
-	if err := checkForErrors("hcnOpenLoadBalancer", hr, resultBuffer); err != nil {
-		return nil, err
-	}
-	// Modify loadBalancer.
-	hr = hcnModifyLoadBalancer(loadBalancerHandle, settings, &resultBuffer)
-	if err := checkForErrors("hcnModifyLoadBalancer", hr, resultBuffer); err != nil {
-		return nil, err
-	}
-	// Query loadBalancer.
-	hcnQuery := defaultQuery()
-	query, err := json.Marshal(hcnQuery)
-	if err != nil {
-		return nil, err
-	}
-	hr = hcnQueryLoadBalancerProperties(loadBalancerHandle, string(query), &propertiesBuffer, &resultBuffer)
-	if err := checkForErrors("hcnQueryLoadBalancerProperties", hr, resultBuffer); err != nil {
-		return nil, err
-	}
-	properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer)
-	// Close loadBalancer.
-	hr = hcnCloseLoadBalancer(loadBalancerHandle)
-	if err := checkForErrors("hcnCloseLoadBalancer", hr, nil); err != nil {
-		return nil, err
-	}
-	// Convert output to LoadBalancer
-	var outputLoadBalancer HostComputeLoadBalancer
-	if err := json.Unmarshal([]byte(properties), &outputLoadBalancer); err != nil {
-		return nil, err
-	}
-	return &outputLoadBalancer, nil
-}
-
 func deleteLoadBalancer(loadBalancerId string) error {
 	loadBalancerGuid, err := guid.FromString(loadBalancerId)
 	if err != nil {

vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go

@@ -378,7 +378,7 @@ func (namespace *HostComputeNamespace) Sync() error {
 		// The shim is likey gone. Simply ignore the sync as if it didn't exist.
 		if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ERROR_FILE_NOT_FOUND {
 			// Remove the reg key there is no point to try again
-			cfg.Remove()
+			_ = cfg.Remove()
 			return nil
 		}
 		f := map[string]interface{}{

vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go

@@ -9,21 +9,21 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// Route is assoicated with a subnet.
+// Route is associated with a subnet.
 type Route struct {
 	NextHop           string `json:",omitempty"`
 	DestinationPrefix string `json:",omitempty"`
 	Metric            uint16 `json:",omitempty"`
 }
 
-// Subnet is assoicated with a Ipam.
+// Subnet is associated with a Ipam.
 type Subnet struct {
 	IpAddressPrefix string            `json:",omitempty"`
 	Policies        []json.RawMessage `json:",omitempty"`
 	Routes          []Route           `json:",omitempty"`
 }
 
-// Ipam (Internet Protocol Addres Management) is assoicated with a network
+// Ipam (Internet Protocol Address Management) is associated with a network
 // and represents the address space(s) of a network.
 type Ipam struct {
 	Type    string   `json:",omitempty"` // Ex: Static, DHCP
@@ -36,12 +36,12 @@ type MacRange struct {
 	EndMacAddress   string `json:",omitempty"`
 }
 
-// MacPool is assoicated with a network and represents pool of MacRanges.
+// MacPool is associated with a network and represents pool of MacRanges.
 type MacPool struct {
 	Ranges []MacRange `json:",omitempty"`
 }
 
-// Dns (Domain Name System is associated with a network.
+// Dns (Domain Name System is associated with a network).
 type Dns struct {
 	Domain     string   `json:",omitempty"`
 	Search     []string `json:",omitempty"`
@@ -82,6 +82,7 @@ type HostComputeNetwork struct {
 	Dns           Dns             `json:",omitempty"`
 	Ipams         []Ipam          `json:",omitempty"`
 	Flags         NetworkFlags    `json:",omitempty"` // 0: None
+	Health        Health          `json:",omitempty"`
 	SchemaVersion SchemaVersion   `json:",omitempty"`
 }
 

vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go

@@ -19,9 +19,11 @@ const (
 	L4WFPPROXY    EndpointPolicyType = "L4WFPPROXY"
 	PortName      EndpointPolicyType = "PortName"
 	EncapOverhead EndpointPolicyType = "EncapOverhead"
+	IOV           EndpointPolicyType = "Iov"
 	// Endpoint and Network have InterfaceConstraint and ProviderAddress
 	NetworkProviderAddress     EndpointPolicyType = "ProviderAddress"
 	NetworkInterfaceConstraint EndpointPolicyType = "InterfaceConstraint"
+	TierAcl       EndpointPolicyType = "TierAcl"	
 )
 
 // EndpointPolicy is a collection of Policy settings for an Endpoint.
@@ -47,6 +49,7 @@ const (
 	HostRoute           NetworkPolicyType = "HostRoute"
 	SetPolicy           NetworkPolicyType = "SetPolicy"
 	NetworkL4Proxy      NetworkPolicyType = "L4Proxy"
+	LayerConstraint     NetworkPolicyType = "LayerConstraint"
 )
 
 // NetworkPolicy is a collection of Policy settings for a Network.
@@ -73,6 +76,12 @@ type SubnetPolicy struct {
 // NatFlags are flags for portmappings.
 type NatFlags uint32
 
+const (
+	NatFlagsNone NatFlags = iota
+	NatFlagsLocalRoutedVip
+	NatFlagsIPv6
+)
+
 /// Endpoint Policy objects
 
 // PortMappingPolicySetting defines Port Mapping (NAT)
@@ -98,6 +107,8 @@ const (
 	ActionTypeAllow ActionType = "Allow"
 	// Block traffic
 	ActionTypeBlock ActionType = "Block"
+	// Pass traffic
+	ActionTypePass ActionType = "Pass"
 
 	// In is traffic coming to the Endpoint
 	DirectionTypeIn DirectionType = "In"
@@ -133,6 +144,7 @@ type OutboundNatPolicySetting struct {
 	VirtualIP    string   `json:",omitempty"`
 	Exceptions   []string `json:",omitempty"`
 	Destinations []string `json:",omitempty"`
+	Flags        NatFlags `json:",omitempty"`
 }
 
 // SDNRoutePolicySetting sets SDN Route on an Endpoint.
@@ -152,11 +164,20 @@ type FiveTuple struct {
 	Priority        uint16 `json:",omitempty"`
 }
 
+// ProxyExceptions exempts traffic to IpAddresses and Ports
+type ProxyExceptions struct {
+	IpAddressExceptions []string `json:",omitempty"`
+	PortExceptions      []string `json:",omitempty"`
+}
+
 // L4WfpProxyPolicySetting sets Layer-4 Proxy on an endpoint.
 type L4WfpProxyPolicySetting struct {
-	Port        string    `json:",omitempty"`
-	FilterTuple FiveTuple `json:",omitempty"`
-	UserSID     string    `json:",omitempty"`
+	InboundProxyPort   string          `json:",omitempty"`
+	OutboundProxyPort  string          `json:",omitempty"`
+	FilterTuple        FiveTuple       `json:",omitempty"`
+	UserSID            string          `json:",omitempty"`
+	InboundExceptions  ProxyExceptions `json:",omitempty"`
+	OutboundExceptions ProxyExceptions `json:",omitempty"`
 }
 
 // PortnameEndpointPolicySetting sets the port name for an endpoint.
@@ -169,6 +190,13 @@ type EncapOverheadEndpointPolicySetting struct {
 	Overhead uint16 `json:",omitempty"`
 }
 
+// IovPolicySetting sets the Iov settings for an endpoint.
+type IovPolicySetting struct {
+	IovOffloadWeight    uint32 `json:",omitempty"`
+	QueuePairsRequested uint32 `json:",omitempty"`
+	InterruptModeration uint32 `json:",omitempty"`
+}
+
 /// Endpoint and Network Policy objects
 
 // ProviderAddressEndpointPolicySetting sets the PA for an endpoint.
@@ -214,6 +242,10 @@ type AutomaticDNSNetworkPolicySetting struct {
 	Enable bool `json:",omitempty"`
 }
 
+type LayerConstraintNetworkPolicySetting struct {
+	LayerId string `json:",omitempty"`
+}
+
 /// Subnet Policy objects
 
 // VlanPolicySetting isolates a subnet with VLAN tagging.
@@ -275,3 +307,23 @@ type L4ProxyPolicySetting struct {
 	Destination string
 	OutboundNAT bool `json:",omitempty"`
 }
+
+// TierAclRule represents an ACL within TierAclPolicySetting
+type TierAclRule struct {
+	Id                string        `json:",omitempty"`
+	Protocols         string        `json:",omitempty"`
+	TierAclRuleAction ActionType    `json:","`
+	LocalAddresses    string        `json:",omitempty"`
+	RemoteAddresses   string        `json:",omitempty"`
+	LocalPorts        string        `json:",omitempty"`
+	RemotePorts       string        `json:",omitempty"`
+	Priority          uint16        `json:",omitempty"`
+}
+
+// TierAclPolicySetting represents a Tier containing ACLs
+type TierAclPolicySetting struct {
+	Name            string         `json:","`
+	Direction       DirectionType  `json:","`
+	Order           uint16         `json:""`
+	TierAclRules    []TierAclRule  `json:",omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go

@@ -19,6 +19,7 @@ type SupportedFeatures struct {
 	VxlanPort                bool        `json:"VxlanPort"`
 	L4Proxy                  bool        `json:"L4Proxy"`    // network policy that applies VFP rules to all endpoints on the network to redirect traffic
 	L4WfpProxy               bool        `json:"L4WfpProxy"` // endpoint policy that applies WFP filters to redirect traffic to/from that endpoint
+	TierAcl                  bool        `json:"TierAcl"`
 }
 
 // AclFeatures are the supported ACL possibilities.
@@ -69,6 +70,7 @@ func GetSupportedFeatures() SupportedFeatures {
 	features.VxlanPort = isFeatureSupported(globals.Version, VxlanPortVersion)
 	features.L4Proxy = isFeatureSupported(globals.Version, L4ProxyPolicyVersion)
 	features.L4WfpProxy = isFeatureSupported(globals.Version, L4WfpProxyPolicyVersion)
+	features.TierAcl = isFeatureSupported(globals.Version, TierAclPolicyVersion)
 
 	return features
 }

vendor/github.com/Microsoft/hcsshim/hcn/zsyscall_windows.go

@@ -78,10 +78,6 @@ var (
 	procHcnQuerySdnRouteProperties     = modcomputenetwork.NewProc("HcnQuerySdnRouteProperties")
 	procHcnDeleteSdnRoute              = modcomputenetwork.NewProc("HcnDeleteSdnRoute")
 	procHcnCloseSdnRoute               = modcomputenetwork.NewProc("HcnCloseSdnRoute")
-	procHcnOpenService                 = modcomputenetwork.NewProc("HcnOpenService")
-	procHcnRegisterServiceCallback     = modcomputenetwork.NewProc("HcnRegisterServiceCallback")
-	procHcnUnregisterServiceCallback   = modcomputenetwork.NewProc("HcnUnregisterServiceCallback")
-	procHcnCloseService                = modcomputenetwork.NewProc("HcnCloseService")
 )
 
 func SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) {
@@ -797,59 +793,3 @@ func hcnCloseRoute(route hcnRoute) (hr error) {
 	}
 	return
 }
-
-func hcnOpenService(service *hcnService, result **uint16) (hr error) {
-	if hr = procHcnOpenService.Find(); hr != nil {
-		return
-	}
-	r0, _, _ := syscall.Syscall(procHcnOpenService.Addr(), 2, uintptr(unsafe.Pointer(service)), uintptr(unsafe.Pointer(result)), 0)
-	if int32(r0) < 0 {
-		if r0&0x1fff0000 == 0x00070000 {
-			r0 &= 0xffff
-		}
-		hr = syscall.Errno(r0)
-	}
-	return
-}
-
-func hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) {
-	if hr = procHcnRegisterServiceCallback.Find(); hr != nil {
-		return
-	}
-	r0, _, _ := syscall.Syscall6(procHcnRegisterServiceCallback.Addr(), 4, uintptr(service), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0)
-	if int32(r0) < 0 {
-		if r0&0x1fff0000 == 0x00070000 {
-			r0 &= 0xffff
-		}
-		hr = syscall.Errno(r0)
-	}
-	return
-}
-
-func hcnUnregisterServiceCallback(callbackHandle hcnCallbackHandle) (hr error) {
-	if hr = procHcnUnregisterServiceCallback.Find(); hr != nil {
-		return
-	}
-	r0, _, _ := syscall.Syscall(procHcnUnregisterServiceCallback.Addr(), 1, uintptr(callbackHandle), 0, 0)
-	if int32(r0) < 0 {
-		if r0&0x1fff0000 == 0x00070000 {
-			r0 &= 0xffff
-		}
-		hr = syscall.Errno(r0)
-	}
-	return
-}
-
-func hcnCloseService(service hcnService) (hr error) {
-	if hr = procHcnCloseService.Find(); hr != nil {
-		return
-	}
-	r0, _, _ := syscall.Syscall(procHcnCloseService.Addr(), 1, uintptr(service), 0, 0)
-	if int32(r0) < 0 {
-		if r0&0x1fff0000 == 0x00070000 {
-			r0 &= 0xffff
-		}
-		hr = syscall.Errno(r0)
-	}
-	return
-}

vendor/github.com/Microsoft/hcsshim/hnsendpoint.go

@@ -40,6 +40,9 @@ func HNSListEndpointRequest() ([]HNSEndpoint, error) {
 // HotAttachEndpoint makes a HCS Call to attach the endpoint to the container
 func HotAttachEndpoint(containerID string, endpointID string) error {
 	endpoint, err := GetHNSEndpointByID(endpointID)
+	if err != nil {
+		return err
+	}
 	isAttached, err := endpoint.IsAttached(containerID)
 	if isAttached {
 		return err
@@ -50,6 +53,9 @@ func HotAttachEndpoint(containerID string, endpointID string) error {
 // HotDetachEndpoint makes a HCS Call to detach the endpoint from the container
 func HotDetachEndpoint(containerID string, endpointID string) error {
 	endpoint, err := GetHNSEndpointByID(endpointID)
+	if err != nil {
+		return err
+	}
 	isAttached, err := endpoint.IsAttached(containerID)
 	if !isAttached {
 		return err

vendor/github.com/Microsoft/hcsshim/interface.go

@@ -4,7 +4,7 @@ import (
 	"io"
 	"time"
 
-	"github.com/Microsoft/hcsshim/internal/schema1"
+	"github.com/Microsoft/hcsshim/internal/hcs/schema1"
 )
 
 // ProcessConfig is used as both the input of Container.CreateProcess

vendor/github.com/Microsoft/hcsshim/internal/cow/cow.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"io"
 
-	"github.com/Microsoft/hcsshim/internal/schema1"
-	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+	"github.com/Microsoft/hcsshim/internal/hcs/schema1"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 )
 
 // Process is the interface for an OS process running in a container or utility VM.
@@ -17,6 +17,12 @@ type Process interface {
 	// CloseStdin causes the process's stdin handle to receive EOF/EPIPE/whatever
 	// is appropriate to indicate that no more data is available.
 	CloseStdin(ctx context.Context) error
+	// CloseStdout closes the stdout connection to the process. It is used to indicate
+	// that we are done receiving output on the shim side.
+	CloseStdout(ctx context.Context) error
+	// CloseStderr closes the stderr connection to the process. It is used to indicate
+	// that we are done receiving output on the shim side.
+	CloseStderr(ctx context.Context) error
 	// Pid returns the process ID.
 	Pid() int
 	// Stdio returns the stdio streams for a process. These may be nil if a stream
@@ -80,4 +86,6 @@ type Container interface {
 	// container to be terminated by some error condition (including calling
 	// Close).
 	Wait() error
+	// Modify sends a request to modify container resources
+	Modify(ctx context.Context, config interface{}) error
 }

vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go

@@ -13,7 +13,7 @@ import (
 
 var (
 	nextCallback    uintptr
-	callbackMap     = map[uintptr]*notifcationWatcherContext{}
+	callbackMap     = map[uintptr]*notificationWatcherContext{}
 	callbackMapLock = sync.RWMutex{}
 
 	notificationWatcherCallback = syscall.NewCallback(notificationWatcher)
@@ -87,7 +87,7 @@ func (hn hcsNotification) String() string {
 
 type notificationChannel chan error
 
-type notifcationWatcherContext struct {
+type notificationWatcherContext struct {
 	channels notificationChannels
 	handle   vmcompute.HcsCallback
 
@@ -106,6 +106,7 @@ func newSystemChannels() notificationChannels {
 		hcsNotificationSystemStartCompleted,
 		hcsNotificationSystemPauseCompleted,
 		hcsNotificationSystemResumeCompleted,
+		hcsNotificationSystemSaveCompleted,
 	} {
 		channels[notif] = make(notificationChannel, 1)
 	}

vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go

@@ -60,7 +60,7 @@ var (
 	// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
 	ErrVmcomputeOperationInvalidState = syscall.Errno(0xc0370105)
 
-	// ErrProcNotFound is an error encountered when the the process cannot be found
+	// ErrProcNotFound is an error encountered when a procedure look up fails.
 	ErrProcNotFound = syscall.Errno(0x7f)
 
 	// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
@@ -171,7 +171,6 @@ type SystemError struct {
 	ID     string
 	Op     string
 	Err    error
-	Extra  string
 	Events []ErrorEvent
 }
 
@@ -182,9 +181,6 @@ func (e *SystemError) Error() string {
 	for _, ev := range e.Events {
 		s += "\n" + ev.String()
 	}
-	if e.Extra != "" {
-		s += "\n(extra info: " + e.Extra + ")"
-	}
 	return s
 }
 
@@ -198,7 +194,7 @@ func (e *SystemError) Timeout() bool {
 	return ok && err.Timeout()
 }
 
-func makeSystemError(system *System, op string, extra string, err error, events []ErrorEvent) error {
+func makeSystemError(system *System, op string, err error, events []ErrorEvent) error {
 	// Don't double wrap errors
 	if _, ok := err.(*SystemError); ok {
 		return err
@@ -206,7 +202,6 @@ func makeSystemError(system *System, op string, extra string, err error, events
 	return &SystemError{
 		ID:     system.ID(),
 		Op:     op,
-		Extra:  extra,
 		Err:    err,
 		Events: events,
 	}
@@ -247,12 +242,11 @@ func makeProcessError(process *Process, op string, err error, events []ErrorEven
 // IsNotExist checks if an error is caused by the Container or Process not existing.
 // Note: Currently, ErrElementNotFound can mean that a Process has either
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
-// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+// will currently return true when the error is ErrElementNotFound.
 func IsNotExist(err error) bool {
 	err = getInnerError(err)
 	return err == ErrComputeSystemDoesNotExist ||
-		err == ErrElementNotFound ||
-		err == ErrProcNotFound
+		err == ErrElementNotFound
 }
 
 // IsAlreadyClosed checks if an error is caused by the Container or Process having been
@@ -283,12 +277,11 @@ func IsTimeout(err error) bool {
 // a Container or Process being already stopped.
 // Note: Currently, ErrElementNotFound can mean that a Process has either
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
-// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+// will currently return true when the error is ErrElementNotFound.
 func IsAlreadyStopped(err error) bool {
 	err = getInnerError(err)
 	return err == ErrVmcomputeAlreadyStopped ||
-		err == ErrElementNotFound ||
-		err == ErrProcNotFound
+		err == ErrElementNotFound
 }
 
 // IsNotSupported returns a boolean indicating whether the error is caused by
@@ -312,6 +305,13 @@ func IsOperationInvalidState(err error) bool {
 	return err == ErrVmcomputeOperationInvalidState
 }
 
+// IsAccessIsDenied returns true when err is caused by
+// `ErrVmcomputeOperationAccessIsDenied`.
+func IsAccessIsDenied(err error) bool {
+	err = getInnerError(err)
+	return err == ErrVmcomputeOperationAccessIsDenied
+}
+
 func getInnerError(err error) error {
 	switch pe := err.(type) {
 	case nil:
@@ -325,12 +325,3 @@ func getInnerError(err error) error {
 	}
 	return err
 }
-
-func getOperationLogResult(err error) (string, error) {
-	switch err {
-	case nil:
-		return "Success", nil
-	default:
-		return "Error", err
-	}
-}

vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go

@@ -64,11 +64,7 @@ type processStatus struct {
 	LastWaitResult int32
 }
 
-const (
-	stdIn  string = "StdIn"
-	stdOut string = "StdOut"
-	stdErr string = "StdErr"
-)
+const stdIn string = "StdIn"
 
 const (
 	modifyConsoleSize string = "ConsoleSize"
@@ -122,7 +118,7 @@ func (process *Process) Signal(ctx context.Context, options interface{}) (bool,
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
 
-	operation := "hcsshim::Process::Signal"
+	operation := "hcs::Process::Signal"
 
 	if process.handle == 0 {
 		return false, makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -147,7 +143,7 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
 
-	operation := "hcsshim::Process::Kill"
+	operation := "hcs::Process::Kill"
 
 	if process.handle == 0 {
 		return false, makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -168,7 +164,7 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 // This MUST be called exactly once per `process.handle` but `Wait` is safe to
 // call multiple times.
 func (process *Process) waitBackground() {
-	operation := "hcsshim::Process::waitBackground"
+	operation := "hcs::Process::waitBackground"
 	ctx, span := trace.StartSpan(context.Background(), operation)
 	defer span.End()
 	span.AddAttributes(
@@ -176,8 +172,10 @@ func (process *Process) waitBackground() {
 		trace.Int64Attribute("pid", int64(process.processID)))
 
 	var (
-		err      error
-		exitCode = -1
+		err            error
+		exitCode       = -1
+		propertiesJSON string
+		resultJSON     string
 	)
 
 	err = waitForNotification(ctx, process.callbackNumber, hcsNotificationProcessExited, nil)
@@ -190,15 +188,15 @@ func (process *Process) waitBackground() {
 
 		// Make sure we didnt race with Close() here
 		if process.handle != 0 {
-			propertiesJSON, resultJSON, err := vmcompute.HcsGetProcessProperties(ctx, process.handle)
+			propertiesJSON, resultJSON, err = vmcompute.HcsGetProcessProperties(ctx, process.handle)
 			events := processHcsResult(ctx, resultJSON)
 			if err != nil {
-				err = makeProcessError(process, operation, err, events)
+				err = makeProcessError(process, operation, err, events) //nolint:ineffassign
 			} else {
 				properties := &processStatus{}
 				err = json.Unmarshal([]byte(propertiesJSON), properties)
 				if err != nil {
-					err = makeProcessError(process, operation, err, nil)
+					err = makeProcessError(process, operation, err, nil) //nolint:ineffassign
 				} else {
 					if properties.LastWaitResult != 0 {
 						log.G(ctx).WithField("wait-result", properties.LastWaitResult).Warning("non-zero last wait result")
@@ -231,7 +229,7 @@ func (process *Process) ResizeConsole(ctx context.Context, width, height uint16)
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
 
-	operation := "hcsshim::Process::ResizeConsole"
+	operation := "hcs::Process::ResizeConsole"
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -269,7 +267,7 @@ func (process *Process) ExitCode() (int, error) {
 		}
 		return process.exitCode, nil
 	default:
-		return -1, makeProcessError(process, "hcsshim::Process::ExitCode", ErrInvalidProcessState, nil)
+		return -1, makeProcessError(process, "hcs::Process::ExitCode", ErrInvalidProcessState, nil)
 	}
 }
 
@@ -277,7 +275,7 @@ func (process *Process) ExitCode() (int, error) {
 // these pipes does not close the underlying pipes. Once returned, these pipes
 // are the responsibility of the caller to close.
 func (process *Process) StdioLegacy() (_ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, err error) {
-	operation := "hcsshim::Process::StdioLegacy"
+	operation := "hcs::Process::StdioLegacy"
 	ctx, span := trace.StartSpan(context.Background(), operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
@@ -329,7 +327,7 @@ func (process *Process) CloseStdin(ctx context.Context) error {
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
 
-	operation := "hcsshim::Process::CloseStdin"
+	operation := "hcs::Process::CloseStdin"
 
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
@@ -363,10 +361,59 @@ func (process *Process) CloseStdin(ctx context.Context) error {
 	return nil
 }
 
+func (process *Process) CloseStdout(ctx context.Context) (err error) {
+	ctx, span := trace.StartSpan(ctx, "hcs::Process::CloseStdout") //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("cid", process.SystemID()),
+		trace.Int64Attribute("pid", int64(process.processID)))
+
+	process.handleLock.Lock()
+	defer process.handleLock.Unlock()
+
+	if process.handle == 0 {
+		return nil
+	}
+
+	process.stdioLock.Lock()
+	defer process.stdioLock.Unlock()
+	if process.stdout != nil {
+		process.stdout.Close()
+		process.stdout = nil
+	}
+	return nil
+}
+
+func (process *Process) CloseStderr(ctx context.Context) (err error) {
+	ctx, span := trace.StartSpan(ctx, "hcs::Process::CloseStderr") //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("cid", process.SystemID()),
+		trace.Int64Attribute("pid", int64(process.processID)))
+
+	process.handleLock.Lock()
+	defer process.handleLock.Unlock()
+
+	if process.handle == 0 {
+		return nil
+	}
+
+	process.stdioLock.Lock()
+	defer process.stdioLock.Unlock()
+	if process.stderr != nil {
+		process.stderr.Close()
+		process.stderr = nil
+
+	}
+	return nil
+}
+
 // Close cleans up any state associated with the process but does not kill
 // or wait on it.
 func (process *Process) Close() (err error) {
-	operation := "hcsshim::Process::Close"
+	operation := "hcs::Process::Close"
 	ctx, span := trace.StartSpan(context.Background(), operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
@@ -416,7 +463,7 @@ func (process *Process) Close() (err error) {
 }
 
 func (process *Process) registerCallback(ctx context.Context) error {
-	callbackContext := &notifcationWatcherContext{
+	callbackContext := &notificationWatcherContext{
 		channels:  newProcessChannels(),
 		systemID:  process.SystemID(),
 		processID: process.processID,
@@ -468,7 +515,7 @@ func (process *Process) unregisterCallback(ctx context.Context) error {
 	delete(callbackMap, callbackNumber)
 	callbackMapLock.Unlock()
 
-	handle = 0
+	handle = 0 //nolint:ineffassign
 
 	return nil
 }

vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go → vendor/github.com/Microsoft/hcsshim/internal/hcs/schema1/schema1.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/Microsoft/go-winio/pkg/guid"
-	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 )
 
 // ProcessConfig is used as both the input of Container.CreateProcess
@@ -119,9 +119,9 @@ type PropertyType string
 
 const (
 	PropertyTypeStatistics        PropertyType = "Statistics"        // V1 and V2
-	PropertyTypeProcessList                    = "ProcessList"       // V1 and V2
-	PropertyTypeMappedVirtualDisk              = "MappedVirtualDisk" // Not supported in V2 schema call
-	PropertyTypeGuestConnection                = "GuestConnection"   // V1 and V2. Nil return from HCS before RS5
+	PropertyTypeProcessList       PropertyType = "ProcessList"       // V1 and V2
+	PropertyTypeMappedVirtualDisk PropertyType = "MappedVirtualDisk" // Not supported in V2 schema call
+	PropertyTypeGuestConnection   PropertyType = "GuestConnection"   // V1 and V2. Nil return from HCS before RS5
 )
 
 type PropertyQuery struct {
@@ -218,6 +218,7 @@ type GuestDefinedCapabilities struct {
 	SignalProcessSupported        bool `json:",omitempty"`
 	DumpStacksSupported           bool `json:",omitempty"`
 	DeleteContainerStateSupported bool `json:",omitempty"`
+	UpdateContainerSupported      bool `json:",omitempty"`
 }
 
 // GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// CPU groups allow Hyper-V administrators to better manage and allocate the host's CPU resources across guest virtual machines
+type CpuGroup struct {
+	Id string `json:"Id,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_affinity.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type CpuGroupAffinity struct {
+	LogicalProcessorCount int32   `json:"LogicalProcessorCount,omitempty"`
+	LogicalProcessors     []int32 `json:"LogicalProcessors,omitempty"`
+}