Basic windows agent that will join a cluster without CNI.

go.mod

@@ -3,7 +3,7 @@ module github.com/rancher/k3s
 go 1.16
 
 replace (
-	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.9
+	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.10
 	github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7
 	github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e
 	github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340
@@ -66,6 +66,7 @@ replace (
 )
 
 require (
+	github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990
 	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
 	github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340
 	github.com/containerd/containerd v1.5.0-beta.4

go.sum

@@ -61,10 +61,11 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YH
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc=
 github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/hcsshim v0.8.9 h1:VrfodqvztU8YSOvygU+DN1BGaSGxmrNfqOv5oOuX2Bk=
-github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
+github.com/Microsoft/hcsshim v0.8.10 h1:k5wTrpnVU2/xv8ZuzGkbXVd3js5zJ8RnumPo5RxiIxU=
+github.com/Microsoft/hcsshim v0.8.10/go.mod h1:g5uw8EV2mAlzqe94tfNBNdr89fnbD/n3HV0OhsddkmM=
 github.com/Microsoft/hcsshim/test v0.0.0-20200803203718-d80bc7196cb0 h1:+qqN+HZE55Zap8OE1BmcVStbX6JHVhqeWKPi0kzF1eA=
 github.com/Microsoft/hcsshim/test v0.0.0-20200803203718-d80bc7196cb0/go.mod h1:30A5igQ91GEmhYJF8TaRP79pMBOYynRsyOByfVV0dU4=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -940,7 +941,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/u-root/u-root v7.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
 github.com/ugorji/go v0.0.0-20170107133203-ded73eae5db7/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

pkg/agent/config/config.go

@@ -461,8 +461,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	if !envInfo.Debug {
 		nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
 	}
-	nodeConfig.Containerd.State = "/run/k3s/containerd"
-	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
+	applyContainerdStateAndAddress(nodeConfig)
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
 	nodeConfig.Certificate = servingCert
 

pkg/agent/config/config_linux.go

+// +build linux
+
+package config
+
+import (
+	"path/filepath"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+func applyContainerdStateAndAddress(nodeConfig *config.Node) {
+	nodeConfig.Containerd.State = "/run/k3s/containerd"
+	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
+}

pkg/agent/config/config_windows.go

+package config
+
+import (
+	"path/filepath"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+func applyContainerdStateAndAddress(nodeConfig *config.Node) {
+	nodeConfig.Containerd.State = filepath.Join(nodeConfig.Containerd.Root, "state")
+	nodeConfig.Containerd.Address = "npipe://///./pipe/containerd-containerd"
+}

pkg/agent/containerd/config_linux.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"io/ioutil"
 	"os"
+	"time"
 
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/pkg/errors"
@@ -17,8 +18,22 @@ import (
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
+func getContainerdArgs(cfg *config.Node) []string {
+	args := []string{
+		"containerd",
+		"-c", cfg.Containerd.Config,
+		"-a", cfg.Containerd.Address,
+		"--state", cfg.Containerd.State,
+		"--root", cfg.Containerd.Root,
+	}
+	return args
+}
+
 // setupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
@@ -71,3 +86,27 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 
 	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
 }
+
+// criConnection connects to a CRI socket at the given path.
+func CriConnection(ctx context.Context, address string) (*grpc.ClientConn, error) {
+	addr, dialer, err := util.GetAddressAndDialer("unix://" + address)
+	if err != nil {
+		return nil, err
+	}
+
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(3*time.Second), grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+	if err != nil {
+		return nil, err
+	}
+
+	c := runtimeapi.NewRuntimeServiceClient(conn)
+	_, err = c.Version(ctx, &runtimeapi.VersionRequest{
+		Version: "0.1.0",
+	})
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return conn, nil
+}

pkg/agent/containerd/config_windows.go

@@ -4,13 +4,86 @@ package containerd
 
 import (
 	"context"
+	"io/ioutil"
+	"os"
+	"time"
 
+	"github.com/rancher/k3s/pkg/agent/templates"
+	util2 "github.com/rancher/k3s/pkg/agent/util"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/wharfie/pkg/registries"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
+func getContainerdArgs(cfg *config.Node) []string {
+	args := []string{
+		"containerd",
+		"-c", cfg.Containerd.Config,
+	}
+	return args
+}
+
 // setupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
-	// TODO: Create windows config setup.
-	return nil
+	privRegistries, err := registries.GetPrivateRegistries(cfg.AgentConfig.PrivateRegistry)
+	if err != nil {
+		return err
+	}
+
+	if cfg.SELinux {
+		logrus.Warn("SELinux isn't supported on windows")
+	}
+
+	var containerdTemplate string
+
+	containerdConfig := templates.ContainerdConfig{
+		NodeConfig:            cfg,
+		DisableCgroup:         true,
+		IsRunningInUserNS:     false,
+		PrivateRegistryConfig: privRegistries.Registry(),
+	}
+
+	containerdTemplateBytes, err := ioutil.ReadFile(cfg.Containerd.Template)
+	if err == nil {
+		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
+		containerdTemplate = string(containerdTemplateBytes)
+	} else if os.IsNotExist(err) {
+		containerdTemplate = templates.ContainerdConfigTemplate
+	} else {
+		return err
+	}
+	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
+	if err != nil {
+		return err
+	}
+
+	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+}
+
+// criConnection connects to a CRI socket at the given path.
+func CriConnection(ctx context.Context, address string) (*grpc.ClientConn, error) {
+	addr, dialer, err := util.GetAddressAndDialer(address)
+	if err != nil {
+		return nil, err
+	}
+
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(3*time.Second), grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+	if err != nil {
+		return nil, err
+	}
+
+	c := runtimeapi.NewRuntimeServiceClient(conn)
+	_, err = c.Version(ctx, &runtimeapi.VersionRequest{
+		Version: "0.1.0",
+	})
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return conn, nil
 }

pkg/agent/containerd/containerd.go

@@ -32,7 +32,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
 const (
@@ -42,13 +41,7 @@ const (
 // Run configures and starts containerd as a child process. Once it is up, images are preloaded
 // or pulled from files found in the agent images directory.
 func Run(ctx context.Context, cfg *config.Node) error {
-	args := []string{
-		"containerd",
-		"-c", cfg.Containerd.Config,
-		"-a", cfg.Containerd.Address,
-		"--state", cfg.Containerd.State,
-		"--root", cfg.Containerd.Root,
-	}
+	args := getContainerdArgs(cfg)
 
 	if err := setupContainerdConfig(ctx, cfg); err != nil {
 		return err
@@ -116,30 +109,6 @@ func Run(ctx context.Context, cfg *config.Node) error {
 	return preloadImages(ctx, cfg)
 }
 
-// criConnection connects to a CRI socket at the given path.
-func CriConnection(ctx context.Context, address string) (*grpc.ClientConn, error) {
-	addr, dialer, err := util.GetAddressAndDialer("unix://" + address)
-	if err != nil {
-		return nil, err
-	}
-
-	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(3*time.Second), grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
-	if err != nil {
-		return nil, err
-	}
-
-	c := runtimeapi.NewRuntimeServiceClient(conn)
-	_, err = c.Version(ctx, &runtimeapi.VersionRequest{
-		Version: "0.1.0",
-	})
-	if err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	return conn, nil
-}
-
 // preloadImages reads the contents of the agent images directory, and attempts to
 // import into containerd any files found there. Supported compressed types are decompressed, and
 // any .txt files are processed as a list of images that should be pre-pulled from remote registries.

pkg/agent/run.go

@@ -3,7 +3,6 @@ package agent
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -43,35 +42,6 @@ import (
 	utilpointer "k8s.io/utils/pointer"
 )
 
-const (
-	dockershimSock = "unix:///var/run/dockershim.sock"
-	containerdSock = "unix:///run/k3s/containerd/containerd.sock"
-)
-
-// setupCriCtlConfig creates the crictl config file and populates it
-// with the given data from config.
-func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *daemonconfig.Node) error {
-	cre := nodeConfig.ContainerRuntimeEndpoint
-	if cre == "" {
-		switch {
-		case cfg.Docker:
-			cre = dockershimSock
-		default:
-			cre = containerdSock
-		}
-	}
-
-	agentConfDir := filepath.Join(cfg.DataDir, "agent", "etc")
-	if _, err := os.Stat(agentConfDir); os.IsNotExist(err) {
-		if err := os.MkdirAll(agentConfDir, 0700); err != nil {
-			return err
-		}
-	}
-
-	crp := "runtime-endpoint: " + cre + "\n"
-	return ioutil.WriteFile(agentConfDir+"/crictl.yaml", []byte(crp), 0600)
-}
-
 func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	nodeConfig := config.Get(ctx, cfg, proxy)
 

pkg/agent/run_linux.go

+// +build linux
+
+package agent
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+const (
+	dockershimSock = "unix:///var/run/dockershim.sock"
+	containerdSock = "unix:///run/k3s/containerd/containerd.sock"
+)
+
+// setupCriCtlConfig creates the crictl config file and populates it
+// with the given data from config.
+func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
+	cre := nodeConfig.ContainerRuntimeEndpoint
+	if cre == "" {
+		switch {
+		case cfg.Docker:
+			cre = dockershimSock
+		default:
+			cre = containerdSock
+		}
+	}
+
+	agentConfDir := filepath.Join(cfg.DataDir, "agent", "etc")
+	if _, err := os.Stat(agentConfDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(agentConfDir, 0700); err != nil {
+			return err
+		}
+	}
+
+	crp := "runtime-endpoint: " + cre + "\n"
+	return ioutil.WriteFile(agentConfDir+"/crictl.yaml", []byte(crp), 0600)
+}

pkg/agent/run_windows.go

+// +build windows
+
+package agent
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+const (
+	dockershimSock = "npipe:////./pipe/docker_engine"
+	containerdSock = "npipe:////./pipe/containerd-containerd"
+)
+
+// setupCriCtlConfig creates the crictl config file and populates it
+// with the given data from config.
+func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
+	cre := nodeConfig.ContainerRuntimeEndpoint
+	if cre == "" || strings.HasPrefix(cre, "npipe:") {
+		switch {
+		case cfg.Docker:
+			cre = dockershimSock
+		default:
+			cre = containerdSock
+		}
+	} else {
+		cre = containerdSock
+	}
+	agentConfDir := filepath.Join(cfg.DataDir, "agent", "etc")
+	if _, err := os.Stat(agentConfDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(agentConfDir, 0700); err != nil {
+			return err
+		}
+	}
+
+	crp := "runtime-endpoint: " + cre + "\n"
+	return ioutil.WriteFile(filepath.Join(agentConfDir, "crictl.yaml"), []byte(crp), 0600)
+}

pkg/agent/templates/templates.go

 package templates
 
 import (
-	"bytes"
-	"text/template"
-
 	"github.com/rancher/wharfie/pkg/registries"
 
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -15,81 +12,3 @@ type ContainerdConfig struct {
 	IsRunningInUserNS     bool
 	PrivateRegistryConfig *registries.Registry
 }
-
-const ContainerdConfigTemplate = `
-[plugins.opt]
-  path = "{{ .NodeConfig.Containerd.Opt }}"
-
-[plugins.cri]
-  stream_server_address = "127.0.0.1"
-  stream_server_port = "10010"
-  enable_selinux = {{ .NodeConfig.SELinux }}
-
-{{- if .DisableCgroup}}
-  disable_cgroup = true
-{{end}}
-{{- if .IsRunningInUserNS }}
-  disable_apparmor = true
-  restrict_oom_score_adj = true
-{{end}}
-
-{{- if .NodeConfig.AgentConfig.PauseImage }}
-  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{end}}
-
-{{- if .NodeConfig.AgentConfig.Snapshotter }}
-[plugins.cri.containerd]
-  disable_snapshot_annotations = true
-  snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
-{{end}}
-
-{{- if not .NodeConfig.NoFlannel }}
-[plugins.cri.cni]
-  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{end}}
-
-[plugins.cri.containerd.runtimes.runc]
-  runtime_type = "io.containerd.runc.v2"
-
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins.cri.registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
-
-{{range $k, $v := .PrivateRegistryConfig.Configs }}
-{{ if $v.Auth }}
-[plugins.cri.registry.configs."{{$k}}".auth]
-  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
-  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
-  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
-  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
-{{end}}
-{{ if $v.TLS }}
-[plugins.cri.registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
-{{end}}
-{{end}}
-`
-
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
-}

pkg/agent/templates/templates_linux.go

+// +build linux
+
+package templates
+
+import (
+	"bytes"
+	"text/template"
+)
+
+const ContainerdConfigTemplate = `
+[plugins.opt]
+  path = "{{ .NodeConfig.Containerd.Opt }}"
+
+[plugins.cri]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+  enable_selinux = {{ .NodeConfig.SELinux }}
+
+{{- if .DisableCgroup}}
+  disable_cgroup = true
+{{end}}
+{{- if .IsRunningInUserNS }}
+  disable_apparmor = true
+  restrict_oom_score_adj = true
+{{end}}
+
+{{- if .NodeConfig.AgentConfig.PauseImage }}
+  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{end}}
+
+{{- if .NodeConfig.AgentConfig.Snapshotter }}
+[plugins.cri.containerd]
+  disable_snapshot_annotations = true
+  snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
+{{end}}
+
+{{- if not .NodeConfig.NoFlannel }}
+[plugins.cri.cni]
+  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{end}}
+
+[plugins.cri.containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+{{ if .PrivateRegistryConfig }}
+{{ if .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors]{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors."{{$k}}"]
+  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+{{if $v.Rewrites}}
+  [plugins.cri.registry.mirrors."{{$k}}".rewrite]
+{{range $pattern, $replace := $v.Rewrites}}
+    "{{$pattern}}" = "{{$replace}}"
+{{end}}
+{{end}}
+{{end}}
+
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.cri.registry.configs."{{$k}}".auth]
+  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+{{end}}
+{{ if $v.TLS }}
+[plugins.cri.registry.configs."{{$k}}".tls]
+  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
+  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
+  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
+  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
+{{end}}
+{{end}}
+{{end}}
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}

pkg/agent/templates/templates_windows.go

+// +build windows
+
+package templates
+
+import (
+	"bytes"
+	"strings"
+	"text/template"
+)
+
+const ContainerdConfigTemplate = `
+version = 2
+root = "{{ replace .NodeConfig.Containerd.Root }}"
+state = "{{ replace .NodeConfig.Containerd.State }}"
+plugin_dir = ""
+disabled_plugins = []
+required_plugins = []
+oom_score = 0
+
+[grpc]
+  address = "{{ replace .NodeConfig.Containerd.Address }}"
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[ttrpc]
+  address = ""
+  uid = 0
+  gid = 0
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[plugins]
+  [plugins."io.containerd.gc.v1.scheduler"]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_tcp_service = true
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    stream_idle_timeout = "4h0m0s"
+    enable_selinux = false
+    selinux_category_range = 0
+    sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    disable_cgroup = false
+    disable_apparmor = false
+    restrict_oom_score_adj = false
+    max_concurrent_downloads = 3
+    disable_proc_mount = false
+    unset_seccomp_profile = ""
+    tolerate_missing_hugetlb_controller = false
+    disable_hugetlb_controller = false
+    ignore_image_defined_volumes = false
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      snapshotter = "windows"
+      default_runtime_name = "runhcs-wcow-process"
+      no_pivot = false
+      disable_snapshot_annotations = false
+      discard_unpacked_layers = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+        privileged_without_host_devices = false
+        base_runtime_spec = ""
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+        privileged_without_host_devices = false
+        base_runtime_spec = ""
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
+          runtime_type = "io.containerd.runhcs.v1"
+          runtime_engine = ""
+          runtime_root = ""
+          privileged_without_host_devices = false
+          base_runtime_spec = ""
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "{{ replace .NodeConfig.AgentConfig.CNIBinDir }}"
+      conf_dir = "{{ replace .NodeConfig.AgentConfig.CNIConfDir }}"
+      max_conf_num = 1
+      conf_template = ""
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = ""
+      {{range $k, $v := .PrivateRegistryConfig.Configs }}
+      [plugins."io.containerd.grpc.v1.cri".registry.auths]
+      {{ if $v.Auth }}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"]
+          {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+          {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+          {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+          {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+      {{end}}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]
+      {{ if $v.TLS }}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs.tls."{{$k}}"]
+          {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
+          {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
+          {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
+          {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
+      {{end}}
+      {{end}}
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      {{ if .PrivateRegistryConfig.Mirrors }}
+      {{range $k, $v := .PrivateRegistryConfig.Mirrors }}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
+	        endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+	    {{if $v.Rewrites}}
+	      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
+	      {{range $pattern, $replace := $v.Rewrites}}
+	        "{{$pattern}}" = "{{$replace}}"
+	      {{end}}
+	    {{end}}
+      {{end}}
+    {{end}}
+    [plugins."io.containerd.grpc.v1.cri".image_decryption]
+      key_model = ""
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "{{ replace .NodeConfig.Containerd.Opt }}"
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["windows/amd64", "linux/amd64"]
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["windows", "windows-lcow"]
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	funcs := template.FuncMap{
+		"replace": func(s string) string {
+			return strings.ReplaceAll(s, "\\", "\\\\")
+		},
+	}
+	t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}

pkg/cli/cmds/agent.go

@@ -103,13 +103,13 @@ var (
 		Name:        "pause-image",
 		Usage:       "(agent/runtime) Customized pause image for containerd or docker sandbox",
 		Destination: &AgentConfig.PauseImage,
-		Value:       "rancher/pause:3.1",
+		Value:       DefaultPauseImage,
 	}
 	SnapshotterFlag = cli.StringFlag{
 		Name:        "snapshotter",
 		Usage:       "(agent/runtime) Override default containerd snapshotter",
 		Destination: &AgentConfig.Snapshotter,
-		Value:       "overlayfs",
+		Value:       DefaultSnapshotter,
 	}
 	FlannelFlag = cli.BoolFlag{
 		Name:        "no-flannel",

pkg/cli/cmds/const_linux.go

+// +build linux
+
+package cmds
+
+const (
+	DefaultPauseImage  = "rancher/pause:3.1"
+	DefaultSnapshotter = "overlayfs"
+)

pkg/cli/cmds/const_windows.go

+// +build windows
+
+package cmds
+
+const (
+	DefaultPauseImage  = "mcr.microsoft.com/oss/kubernetes/pause:1.4.0"
+	DefaultSnapshotter = "native"
+)

pkg/daemons/agent/agent.go

@@ -3,24 +3,20 @@ package agent
 import (
 	"math/rand"
 	"os"
-	"path/filepath"
-	"strings"
 	"time"
 
-	"github.com/opencontainers/runc/libcontainer/system"
-	"github.com/rancher/k3s/pkg/cgroups"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/executor"
-	"github.com/rancher/k3s/pkg/util"
 	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
 	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
-	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 )
 
-const unixPrefix = "unix://"
+const (
+	unixPrefix    = "unix://"
+	windowsPrefix = "npipe:"
+)
 
 func Agent(config *config.Agent) error {
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -39,138 +35,14 @@ func Agent(config *config.Agent) error {
 }
 
 func startKubeProxy(cfg *config.Agent) error {
-	argsMap := map[string]string{
-		"proxy-mode":                        "iptables",
-		"healthz-bind-address":              "127.0.0.1",
-		"kubeconfig":                        cfg.KubeConfigKubeProxy,
-		"cluster-cidr":                      util.JoinIPNets(cfg.ClusterCIDRs),
-		"conntrack-max-per-core":            "0",
-		"conntrack-tcp-timeout-established": "0s",
-		"conntrack-tcp-timeout-close-wait":  "0s",
-	}
-	if cfg.NodeName != "" {
-		argsMap["hostname-override"] = cfg.NodeName
-	}
-
+	argsMap := kubeProxyArgs(cfg)
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
 	logrus.Infof("Running kube-proxy %s", config.ArgString(args))
 	return executor.KubeProxy(args)
 }
 
 func startKubelet(cfg *config.Agent) error {
-	argsMap := map[string]string{
-		"healthz-bind-address":     "127.0.0.1",
-		"read-only-port":           "0",
-		"cluster-domain":           cfg.ClusterDomain,
-		"kubeconfig":               cfg.KubeConfigKubelet,
-		"eviction-hard":            "imagefs.available<5%,nodefs.available<5%",
-		"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
-		"fail-swap-on":             "false",
-		//"cgroup-root": "/k3s",
-		"cgroup-driver":                "cgroupfs",
-		"authentication-token-webhook": "true",
-		"anonymous-auth":               "false",
-		"authorization-mode":           modes.ModeWebhook,
-	}
-	if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
-		argsMap["pod-manifest-path"] = cfg.PodManifests
-	}
-	if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
-		logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
-	}
-	if cfg.RootDir != "" {
-		argsMap["root-dir"] = cfg.RootDir
-		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
-		argsMap["seccomp-profile-root"] = filepath.Join(cfg.RootDir, "seccomp")
-	}
-	if cfg.CNIConfDir != "" {
-		argsMap["cni-conf-dir"] = cfg.CNIConfDir
-	}
-	if cfg.CNIBinDir != "" {
-		argsMap["cni-bin-dir"] = cfg.CNIBinDir
-	}
-	if cfg.CNIPlugin {
-		argsMap["network-plugin"] = "cni"
-	}
-	if len(cfg.ClusterDNS) > 0 {
-		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
-	}
-	if cfg.ResolvConf != "" {
-		argsMap["resolv-conf"] = cfg.ResolvConf
-	}
-	if cfg.RuntimeSocket != "" {
-		argsMap["container-runtime"] = "remote"
-		argsMap["containerd"] = cfg.RuntimeSocket
-		argsMap["serialize-image-pulls"] = "false"
-		if strings.HasPrefix(argsMap["container-runtime-endpoint"], unixPrefix) {
-			argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
-		} else {
-			argsMap["container-runtime-endpoint"] = unixPrefix + cfg.RuntimeSocket
-		}
-	} else if cfg.PauseImage != "" {
-		argsMap["pod-infra-container-image"] = cfg.PauseImage
-	}
-	if cfg.ListenAddress != "" {
-		argsMap["address"] = cfg.ListenAddress
-	}
-	if cfg.ClientCA != "" {
-		argsMap["anonymous-auth"] = "false"
-		argsMap["client-ca-file"] = cfg.ClientCA
-	}
-	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
-		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
-		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
-	}
-	if cfg.NodeName != "" {
-		argsMap["hostname-override"] = cfg.NodeName
-	}
-	defaultIP, err := net.ChooseHostInterface()
-	if err != nil || defaultIP.String() != cfg.NodeIP {
-		argsMap["node-ip"] = cfg.NodeIP
-	}
-	kubeletRoot, runtimeRoot, hasCFS, hasPIDs := cgroups.CheckCgroups()
-	if !hasCFS {
-		logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us")
-		argsMap["cpu-cfs-quota"] = "false"
-	}
-	if !hasPIDs {
-		logrus.Warn("Disabling pod PIDs limit feature due to missing cgroup pids support")
-		argsMap["cgroups-per-qos"] = "false"
-		argsMap["enforce-node-allocatable"] = ""
-		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "SupportPodPidsLimit=false")
-	}
-	if kubeletRoot != "" {
-		argsMap["kubelet-cgroups"] = kubeletRoot
-	}
-	if runtimeRoot != "" {
-		argsMap["runtime-cgroups"] = runtimeRoot
-	}
-	if system.RunningInUserNS() {
-		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "DevicePlugins=false")
-	}
-
-	argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
-	if len(cfg.NodeTaints) > 0 {
-		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
-	}
-	if !cfg.DisableCCM {
-		argsMap["cloud-provider"] = "external"
-	}
-
-	if ImageCredProvAvailable(cfg) {
-		logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
-		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "KubeletCredentialProviders=true")
-		argsMap["image-credential-provider-bin-dir"] = cfg.ImageCredProvBinDir
-		argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
-	}
-
-	if cfg.Rootless {
-		createRootlessConfig(argsMap, hasCFS, hasCFS)
-	}
-
-	if cfg.ProtectKernelDefaults {
-		argsMap["protect-kernel-defaults"] = "true"
-	}
+	argsMap := kubeletArgs(cfg)
 
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
 	logrus.Infof("Running kubelet %s", config.ArgString(args))

pkg/daemons/agent/agent_linux.go

@@ -3,8 +3,18 @@
 package agent
 
 import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/system"
+	"github.com/rancher/k3s/pkg/cgroups"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/util"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+	"k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 )
 
 func createRootlessConfig(argsMap map[string]string, hasCFS, hasPIDs bool) {
@@ -23,3 +33,140 @@ func createRootlessConfig(argsMap map[string]string, hasCFS, hasPIDs bool) {
 		argsMap["enforce-node-allocatable"] = ""
 	}
 }
+
+func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
+	if strings.HasPrefix(argsMap["container-runtime-endpoint"], unixPrefix) {
+		argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
+	} else {
+		argsMap["container-runtime-endpoint"] = unixPrefix + cfg.RuntimeSocket
+	}
+}
+
+func kubeProxyArgs(cfg *config.Agent) map[string]string {
+	argsMap := map[string]string{
+		"proxy-mode":                        "iptables",
+		"healthz-bind-address":              "127.0.0.1",
+		"kubeconfig":                        cfg.KubeConfigKubeProxy,
+		"cluster-cidr":                      util.JoinIPNets(cfg.ClusterCIDRs),
+		"conntrack-max-per-core":            "0",
+		"conntrack-tcp-timeout-established": "0s",
+		"conntrack-tcp-timeout-close-wait":  "0s",
+	}
+	if cfg.NodeName != "" {
+		argsMap["hostname-override"] = cfg.NodeName
+	}
+	return argsMap
+}
+
+func kubeletArgs(cfg *config.Agent) map[string]string {
+	argsMap := map[string]string{
+		"healthz-bind-address":     "127.0.0.1",
+		"read-only-port":           "0",
+		"cluster-domain":           cfg.ClusterDomain,
+		"kubeconfig":               cfg.KubeConfigKubelet,
+		"eviction-hard":            "imagefs.available<5%,nodefs.available<5%",
+		"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
+		"fail-swap-on":             "false",
+		//"cgroup-root": "/k3s",
+		"cgroup-driver":                "cgroupfs",
+		"authentication-token-webhook": "true",
+		"anonymous-auth":               "false",
+		"authorization-mode":           modes.ModeWebhook,
+	}
+	if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
+		argsMap["pod-manifest-path"] = cfg.PodManifests
+	}
+	if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
+		logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
+	}
+	if cfg.RootDir != "" {
+		argsMap["root-dir"] = cfg.RootDir
+		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
+		argsMap["seccomp-profile-root"] = filepath.Join(cfg.RootDir, "seccomp")
+	}
+	if cfg.CNIConfDir != "" {
+		argsMap["cni-conf-dir"] = cfg.CNIConfDir
+	}
+	if cfg.CNIBinDir != "" {
+		argsMap["cni-bin-dir"] = cfg.CNIBinDir
+	}
+	if cfg.CNIPlugin {
+		argsMap["network-plugin"] = "cni"
+	}
+	if len(cfg.ClusterDNS) > 0 {
+		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
+	}
+	if cfg.ResolvConf != "" {
+		argsMap["resolv-conf"] = cfg.ResolvConf
+	}
+	if cfg.RuntimeSocket != "" {
+		argsMap["container-runtime"] = "remote"
+		argsMap["containerd"] = cfg.RuntimeSocket
+		argsMap["serialize-image-pulls"] = "false"
+		checkRuntimeEndpoint(cfg, argsMap)
+	} else if cfg.PauseImage != "" {
+		argsMap["pod-infra-container-image"] = cfg.PauseImage
+	}
+	if cfg.ListenAddress != "" {
+		argsMap["address"] = cfg.ListenAddress
+	}
+	if cfg.ClientCA != "" {
+		argsMap["anonymous-auth"] = "false"
+		argsMap["client-ca-file"] = cfg.ClientCA
+	}
+	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
+		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
+		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
+	}
+	if cfg.NodeName != "" {
+		argsMap["hostname-override"] = cfg.NodeName
+	}
+	defaultIP, err := net.ChooseHostInterface()
+	if err != nil || defaultIP.String() != cfg.NodeIP {
+		argsMap["node-ip"] = cfg.NodeIP
+	}
+	kubeletRoot, runtimeRoot, hasCFS, hasPIDs := cgroups.CheckCgroups()
+	if !hasCFS {
+		logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us")
+		argsMap["cpu-cfs-quota"] = "false"
+	}
+	if !hasPIDs {
+		logrus.Warn("Disabling pod PIDs limit feature due to missing cgroup pids support")
+		argsMap["cgroups-per-qos"] = "false"
+		argsMap["enforce-node-allocatable"] = ""
+		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "SupportPodPidsLimit=false")
+	}
+	if kubeletRoot != "" {
+		argsMap["kubelet-cgroups"] = kubeletRoot
+	}
+	if runtimeRoot != "" {
+		argsMap["runtime-cgroups"] = runtimeRoot
+	}
+	if system.RunningInUserNS() {
+		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "DevicePlugins=false")
+	}
+
+	argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
+	if len(cfg.NodeTaints) > 0 {
+		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
+	}
+	if !cfg.DisableCCM {
+		argsMap["cloud-provider"] = "external"
+	}
+
+	if ImageCredProvAvailable(cfg) {
+		logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
+		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "KubeletCredentialProviders=true")
+		argsMap["image-credential-provider-bin-dir"] = cfg.ImageCredProvBinDir
+		argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
+	}
+
+	if cfg.Rootless {
+		createRootlessConfig(argsMap, hasCFS, hasCFS)
+	}
+
+	if cfg.ProtectKernelDefaults {
+		argsMap["protect-kernel-defaults"] = "true"
+	}
+	return argsMap
+}

pkg/daemons/agent/agent_windows.go

@@ -2,5 +2,141 @@
 
 package agent
 
-func createRootlessConfig(argsMap map[string]string, hasCfs, hasPIDs bool) {
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/util"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+)
+
+func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
+	if strings.HasPrefix(argsMap["container-runtime-endpoint"], windowsPrefix) {
+		argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
+	} else {
+		argsMap["container-runtime-endpoint"] = windowsPrefix + cfg.RuntimeSocket
+	}
+}
+
+func kubeProxyArgs(cfg *config.Agent) map[string]string {
+	argsMap := map[string]string{
+		"proxy-mode":           "kernelspace",
+		"healthz-bind-address": "127.0.0.1",
+		"kubeconfig":           cfg.KubeConfigKubeProxy,
+		"cluster-cidr":         util.JoinIPNets(cfg.ClusterCIDRs),
+	}
+	if cfg.NodeName != "" {
+		argsMap["hostname-override"] = cfg.NodeName
+	}
+
+	argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "WinOverlay=true")
+
+	if sourceVip := waitForManagementIp("vxlan0"); sourceVip != "" {
+		argsMap["source-vip"] = sourceVip
+	}
+
+	return argsMap
+}
+
+func kubeletArgs(cfg *config.Agent) map[string]string {
+	argsMap := map[string]string{
+		"healthz-bind-address":         "127.0.0.1",
+		"read-only-port":               "0",
+		"cluster-domain":               cfg.ClusterDomain,
+		"kubeconfig":                   cfg.KubeConfigKubelet,
+		"eviction-hard":                "imagefs.available<5%,nodefs.available<5%",
+		"eviction-minimum-reclaim":     "imagefs.available=10%,nodefs.available=10%",
+		"fail-swap-on":                 "false",
+		"authentication-token-webhook": "true",
+		"anonymous-auth":               "false",
+		"authorization-mode":           modes.ModeWebhook,
+	}
+	if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
+		argsMap["pod-manifest-path"] = cfg.PodManifests
+	}
+	if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
+		logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
+	}
+	if cfg.RootDir != "" {
+		argsMap["root-dir"] = cfg.RootDir
+		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
+	}
+	if cfg.CNIConfDir != "" {
+		argsMap["cni-conf-dir"] = cfg.CNIConfDir
+	}
+	if cfg.CNIBinDir != "" {
+		argsMap["cni-bin-dir"] = cfg.CNIBinDir
+	}
+	if cfg.CNIPlugin {
+		argsMap["network-plugin"] = "cni"
+	}
+	if len(cfg.ClusterDNS) > 0 {
+		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
+	}
+	if cfg.ResolvConf != "" {
+		argsMap["resolv-conf"] = cfg.ResolvConf
+	}
+	if cfg.RuntimeSocket != "" {
+		argsMap["container-runtime"] = "remote"
+		argsMap["serialize-image-pulls"] = "false"
+		checkRuntimeEndpoint(cfg, argsMap)
+	} else if cfg.PauseImage != "" {
+		argsMap["pod-infra-container-image"] = cfg.PauseImage
+	}
+	if cfg.ListenAddress != "" {
+		argsMap["address"] = cfg.ListenAddress
+	}
+	if cfg.ClientCA != "" {
+		argsMap["anonymous-auth"] = "false"
+		argsMap["client-ca-file"] = cfg.ClientCA
+	}
+	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
+		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
+		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
+	}
+	if cfg.NodeName != "" {
+		argsMap["hostname-override"] = cfg.NodeName
+	}
+	defaultIP, err := net.ChooseHostInterface()
+	if err != nil || defaultIP.String() != cfg.NodeIP {
+		argsMap["node-ip"] = cfg.NodeIP
+	}
+
+	argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
+	if len(cfg.NodeTaints) > 0 {
+		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
+	}
+	if !cfg.DisableCCM {
+		argsMap["cloud-provider"] = "external"
+	}
+
+	if ImageCredProvAvailable(cfg) {
+		logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
+		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "KubeletCredentialProviders=true")
+		argsMap["image-credential-provider-bin-dir"] = cfg.ImageCredProvBinDir
+		argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
+	}
+
+	if cfg.ProtectKernelDefaults {
+		argsMap["protect-kernel-defaults"] = "true"
+	}
+	return argsMap
+}
+
+func waitForManagementIp(networkName string) string {
+	for range time.Tick(time.Second * 5) {
+		network, err := hcsshim.GetHNSEndpointByName(networkName)
+		if err != nil {
+			logrus.WithError(err).Warning("can't find %s, retrying", networkName)
+			continue
+		}
+		return network.IPAddress.String()
+	}
+	return ""
 }

vendor/github.com/Microsoft/hcsshim/appveyor.yml

@@ -18,6 +18,7 @@ build_script:
   - go build ./cmd/runhcs
   - go build ./cmd/tar2ext4
   - go build ./cmd/wclayer
+  - go build ./cmd/device-util
   - go build ./internal/tools/grantvmgroupaccess 
   - go build ./internal/tools/uvmboot
   - go build ./internal/tools/zapdir
@@ -33,6 +34,7 @@ artifacts:
   - path: 'containerd-shim-runhcs-v1.exe'
   - path: 'runhcs.exe'
   - path: 'tar2ext4.exe'
+  - path: 'device-util.exe'
   - path: 'wclayer.exe'
   - path: 'grantvmgroupaccess.exe'  
   - path: 'uvmboot.exe'

vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/next.pb.txt

-file {	
-  name: "google/protobuf/timestamp.proto"	
-  package: "google.protobuf"	
-  message_type {	
-    name: "Timestamp"	
-    field {	
-      name: "seconds"	
-      number: 1	
-      label: LABEL_OPTIONAL	
-      type: TYPE_INT64	
-      json_name: "seconds"	
-    }	
-    field {	
-      name: "nanos"	
-      number: 2	
-      label: LABEL_OPTIONAL	
-      type: TYPE_INT32	
-      json_name: "nanos"	
-    }	
-  }	
-  options {	
-    java_package: "com.google.protobuf"	
-    java_outer_classname: "TimestampProto"	
-    java_multiple_files: true	
-    go_package: "github.com/golang/protobuf/ptypes/timestamp"	
-    cc_enable_arenas: true	
-    objc_class_prefix: "GPB"	
-    csharp_namespace: "Google.Protobuf.WellKnownTypes"	
-  }	
-  syntax: "proto3"	
-}	
-file {	
-  name: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto"	
-  package: "containerd.runhcs.v1"	
-  dependency: "gogoproto/gogo.proto"	
-  dependency: "google/protobuf/timestamp.proto"	
-  message_type {	
-    name: "Options"	
-    field {	
-      name: "debug"	
-      number: 1	
-      label: LABEL_OPTIONAL	
-      type: TYPE_BOOL	
-      json_name: "debug"	
-    }	
-    field {	
-      name: "debug_type"	
-      number: 2	
-      label: LABEL_OPTIONAL	
-      type: TYPE_ENUM	
-      type_name: ".containerd.runhcs.v1.Options.DebugType"	
-      json_name: "debugType"	
-    }	
-    field {	
-      name: "registry_root"	
-      number: 3	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "registryRoot"	
-    }	
-    field {	
-      name: "sandbox_image"	
-      number: 4	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "sandboxImage"	
-    }	
-    field {	
-      name: "sandbox_platform"	
-      number: 5	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "sandboxPlatform"	
-    }	
-    field {	
-      name: "sandbox_isolation"	
-      number: 6	
-      label: LABEL_OPTIONAL	
-      type: TYPE_ENUM	
-      type_name: ".containerd.runhcs.v1.Options.SandboxIsolation"	
-      json_name: "sandboxIsolation"	
-    }	
-    field {	
-      name: "boot_files_root_path"	
-      number: 7	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "bootFilesRootPath"	
-    }	
-    field {	
-      name: "vm_processor_count"	
-      number: 8	
-      label: LABEL_OPTIONAL	
-      type: TYPE_INT32	
-      json_name: "vmProcessorCount"	
-    }	
-    field {	
-      name: "vm_memory_size_in_mb"	
-      number: 9	
-      label: LABEL_OPTIONAL	
-      type: TYPE_INT32	
-      json_name: "vmMemorySizeInMb"	
-    }	
-    enum_type {	
-      name: "DebugType"	
-      value {	
-        name: "NPIPE"	
-        number: 0	
-      }	
-      value {	
-        name: "FILE"	
-        number: 1	
-      }	
-      value {	
-        name: "ETW"	
-        number: 2	
-      }	
-    }	
-    enum_type {	
-      name: "SandboxIsolation"	
-      value {	
-        name: "PROCESS"	
-        number: 0	
-      }	
-      value {	
-        name: "HYPERVISOR"	
-        number: 1	
-      }	
-    }	
-  }	
-  message_type {	
-    name: "ProcessDetails"	
-    field {	
-      name: "image_name"	
-      number: 1	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "imageName"	
-    }	
-    field {	
-      name: "created_at"	
-      number: 2	
-      label: LABEL_OPTIONAL	
-      type: TYPE_MESSAGE	
-      type_name: ".google.protobuf.Timestamp"	
-      options {	
-        65001: 0	
-        65010: 1	
-      }	
-      json_name: "createdAt"	
-    }	
-    field {	
-      name: "kernel_time_100_ns"	
-      number: 3	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT64	
-      json_name: "kernelTime100Ns"	
-    }	
-    field {	
-      name: "memory_commit_bytes"	
-      number: 4	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT64	
-      json_name: "memoryCommitBytes"	
-    }	
-    field {	
-      name: "memory_working_set_private_bytes"	
-      number: 5	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT64	
-      json_name: "memoryWorkingSetPrivateBytes"	
-    }	
-    field {	
-      name: "memory_working_set_shared_bytes"	
-      number: 6	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT64	
-      json_name: "memoryWorkingSetSharedBytes"	
-    }	
-    field {	
-      name: "process_id"	
-      number: 7	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT32	
-      json_name: "processId"	
-    }	
-    field {	
-      name: "user_time_100_ns"	
-      number: 8	
-      label: LABEL_OPTIONAL	
-      type: TYPE_UINT64	
-      json_name: "userTime100Ns"	
-    }	
-    field {	
-      name: "exec_id"	
-      number: 9	
-      label: LABEL_OPTIONAL	
-      type: TYPE_STRING	
-      json_name: "execId"	
-    }	
-  }	
-  options {	
-    go_package: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options;options"	
-  }	
-  weak_dependency: 0	
-  syntax: "proto3"	
-}
+file {
+  name: "google/protobuf/timestamp.proto"
+  package: "google.protobuf"
+  message_type {
+    name: "Timestamp"
+    field {
+      name: "seconds"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_INT64
+      json_name: "seconds"
+    }
+    field {
+      name: "nanos"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_INT32
+      json_name: "nanos"
+    }
+  }
+  options {
+    java_package: "com.google.protobuf"
+    java_outer_classname: "TimestampProto"
+    java_multiple_files: true
+    go_package: "github.com/golang/protobuf/ptypes/timestamp"
+    cc_enable_arenas: true
+    objc_class_prefix: "GPB"
+    csharp_namespace: "Google.Protobuf.WellKnownTypes"
+  }
+  syntax: "proto3"
+}
+file {
+  name: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options/runhcs.proto"
+  package: "containerd.runhcs.v1"
+  dependency: "gogoproto/gogo.proto"
+  dependency: "google/protobuf/timestamp.proto"
+  message_type {
+    name: "Options"
+    field {
+      name: "debug"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_BOOL
+      json_name: "debug"
+    }
+    field {
+      name: "debug_type"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_ENUM
+      type_name: ".containerd.runhcs.v1.Options.DebugType"
+      json_name: "debugType"
+    }
+    field {
+      name: "registry_root"
+      number: 3
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "registryRoot"
+    }
+    field {
+      name: "sandbox_image"
+      number: 4
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "sandboxImage"
+    }
+    field {
+      name: "sandbox_platform"
+      number: 5
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "sandboxPlatform"
+    }
+    field {
+      name: "sandbox_isolation"
+      number: 6
+      label: LABEL_OPTIONAL
+      type: TYPE_ENUM
+      type_name: ".containerd.runhcs.v1.Options.SandboxIsolation"
+      json_name: "sandboxIsolation"
+    }
+    field {
+      name: "boot_files_root_path"
+      number: 7
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "bootFilesRootPath"
+    }
+    field {
+      name: "vm_processor_count"
+      number: 8
+      label: LABEL_OPTIONAL
+      type: TYPE_INT32
+      json_name: "vmProcessorCount"
+    }
+    field {
+      name: "vm_memory_size_in_mb"
+      number: 9
+      label: LABEL_OPTIONAL
+      type: TYPE_INT32
+      json_name: "vmMemorySizeInMb"
+    }
+    field {
+      name: "GPUVHDPath"
+      number: 10
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "GPUVHDPath"
+    }
+    enum_type {
+      name: "DebugType"
+      value {
+        name: "NPIPE"
+        number: 0
+      }
+      value {
+        name: "FILE"
+        number: 1
+      }
+      value {
+        name: "ETW"
+        number: 2
+      }
+    }
+    enum_type {
+      name: "SandboxIsolation"
+      value {
+        name: "PROCESS"
+        number: 0
+      }
+      value {
+        name: "HYPERVISOR"
+        number: 1
+      }
+    }
+  }
+  message_type {
+    name: "ProcessDetails"
+    field {
+      name: "image_name"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "imageName"
+    }
+    field {
+      name: "created_at"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".google.protobuf.Timestamp"
+      options {
+        65001: 0
+        65010: 1
+      }
+      json_name: "createdAt"
+    }
+    field {
+      name: "kernel_time_100_ns"
+      number: 3
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "kernelTime100Ns"
+    }
+    field {
+      name: "memory_commit_bytes"
+      number: 4
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "memoryCommitBytes"
+    }
+    field {
+      name: "memory_working_set_private_bytes"
+      number: 5
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "memoryWorkingSetPrivateBytes"
+    }
+    field {
+      name: "memory_working_set_shared_bytes"
+      number: 6
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "memoryWorkingSetSharedBytes"
+    }
+    field {
+      name: "process_id"
+      number: 7
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT32
+      json_name: "processId"
+    }
+    field {
+      name: "user_time_100_ns"
+      number: 8
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "userTime100Ns"
+    }
+    field {
+      name: "exec_id"
+      number: 9
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "execId"
+    }
\ No newline at end of file
+  }
+  options {
+    go_package: "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options;options"
+  }
+  weak_dependency: 0
+  syntax: "proto3"
+}

vendor/github.com/Microsoft/hcsshim/go.mod

@@ -3,8 +3,8 @@ module github.com/Microsoft/hcsshim
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
-	github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f
+	github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab
+	github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59
 	github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1
 	github.com/containerd/containerd v1.3.2
 	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
@@ -17,16 +17,16 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 // indirect
 	github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f // indirect
-	github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700
-	github.com/pkg/errors v0.8.1
+	github.com/opencontainers/runtime-spec v1.0.2
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/stretchr/testify v1.4.0 // indirect
-	github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5
+	github.com/urfave/cli v1.22.2
 	go.opencensus.io v0.22.0
 	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
+	golang.org/x/sys v0.0.0-20200120151820-655fe14d7479
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 // indirect
 	google.golang.org/grpc v1.23.1
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

vendor/github.com/Microsoft/hcsshim/hcn/hcn.go

@@ -207,6 +207,42 @@ func IPv6DualStackSupported() error {
 	return platformDoesNotSupportError("IPv6 DualStack")
 }
 
+//L4proxySupported returns an error if the HCN verison does not support L4Proxy
+func L4proxyPolicySupported() error {
+	supported := GetSupportedFeatures()
+	if supported.L4Proxy {
+		return nil
+	}
+	return platformDoesNotSupportError("L4ProxyPolicy")
+}
+
+// L4WfpProxySupported returns an error if the HCN verison does not support L4WfpProxy
+func L4WfpProxyPolicySupported() error {
+	supported := GetSupportedFeatures()
+	if supported.L4WfpProxy {
+		return nil
+	}
+	return platformDoesNotSupportError("L4WfpProxyPolicy")
+}
+
+// SetPolicySupported returns an error if the HCN version does not support SetPolicy.
+func SetPolicySupported() error {
+	supported := GetSupportedFeatures()
+	if supported.SetPolicy {
+		return nil
+	}
+	return platformDoesNotSupportError("SetPolicy")
+}
+
+// VxlanPortSupported returns an error if the HCN version does not support configuring the VXLAN TCP port.
+func VxlanPortSupported() error {
+	supported := GetSupportedFeatures()
+	if supported.VxlanPort {
+		return nil
+	}
+	return platformDoesNotSupportError("VXLAN port configuration")
+}
+
 // RequestType are the different operations performed to settings.
 // Used to update the settings of Endpoint/Namespace objects.
 type RequestType string

vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go

@@ -37,8 +37,11 @@ var (
 	RemoteSubnetVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 9, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
 	// A Host Route policy allows for local container to local host communication Overlay networks
 	HostRouteVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 9, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
-	// HNS 10.2 allows for Direct Server Return for loadbalancing
-	DSRVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 10, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+	// HNS 9.3 through 10.0 (not included), and 10.2+ allows for Direct Server Return for loadbalancing
+	DSRVersion = VersionRanges{
+		VersionRange{MinVersion: Version{Major: 9, Minor: 3}, MaxVersion: Version{Major: 9, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 10, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
+	}
 	// HNS 9.3 through 10.0 (not included) and, 10.4+ provide support for configuring endpoints with /32 prefixes
 	Slash32EndpointPrefixesVersion = VersionRanges{
 		VersionRange{MinVersion: Version{Major: 9, Minor: 3}, MaxVersion: Version{Major: 9, Minor: math.MaxInt32}},
@@ -46,8 +49,7 @@ var (
 	}
 	// HNS 9.3 through 10.0 (not included) and, 10.4+ allow for HNS ACL Policies to support protocol 252 for VXLAN
 	AclSupportForProtocol252Version = VersionRanges{
-		VersionRange{MinVersion: Version{Major: 9, Minor: 3}, MaxVersion: Version{Major: 9, Minor: math.MaxInt32}},
-		VersionRange{MinVersion: Version{Major: 10, Minor: 4}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
+		VersionRange{MinVersion: Version{Major: 11, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
 	}
 	// HNS 12.0 allows for session affinity for loadbalancing
 	SessionAffinityVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 12, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
@@ -56,6 +58,16 @@ var (
 		VersionRange{MinVersion: Version{Major: 10, Minor: 5}, MaxVersion: Version{Major: 10, Minor: math.MaxInt32}},
 		VersionRange{MinVersion: Version{Major: 12, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}},
 	}
+	// HNS 13.0 allows for Set Policy support
+	SetPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+	// HNS 10.3 allows for VXLAN ports
+	VxlanPortVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 10, Minor: 3}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+
+	//HNS 13.1 allows for L4Proxy Policy support
+	L4ProxyPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 1}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
+
+	//HNS 13.2 allows for L4WfpProxy Policy support
+	L4WfpProxyPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 13, Minor: 2}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}}
 )
 
 // GetGlobals returns the global properties of the HCN Service.

vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go

@@ -43,7 +43,10 @@ const (
 	InterfaceConstraint NetworkPolicyType = "InterfaceConstraint"
 	ProviderAddress     NetworkPolicyType = "ProviderAddress"
 	RemoteSubnetRoute   NetworkPolicyType = "RemoteSubnetRoute"
+	VxlanPort           NetworkPolicyType = "VxlanPort"
 	HostRoute           NetworkPolicyType = "HostRoute"
+	SetPolicy           NetworkPolicyType = "SetPolicy"
+	NetworkL4Proxy      NetworkPolicyType = "L4Proxy"
 )
 
 // NetworkPolicy is a collection of Policy settings for a Network.
@@ -139,7 +142,7 @@ type SDNRoutePolicySetting struct {
 	NeedEncap         bool   `json:",omitempty"`
 }
 
-// FiveTuple is nested in L4ProxyPolicySetting for WFP support.
+// FiveTuple is nested in L4ProxyPolicySetting  for WFP support.
 type FiveTuple struct {
 	Protocols       string `json:",omitempty"`
 	LocalAddresses  string `json:",omitempty"`
@@ -230,3 +233,45 @@ type RemoteSubnetRoutePolicySetting struct {
 	ProviderAddress             string
 	DistributedRouterMacAddress string
 }
+
+// SetPolicyTypes associated with SetPolicy. Value is IPSET.
+type SetPolicyType string
+
+const (
+	SetPolicyTypeIpSet SetPolicyType = "IPSET"
+)
+
+// SetPolicySetting creates IPSets on network
+type SetPolicySetting struct {
+	Id     string
+	Name   string
+	Type   SetPolicyType
+	Values string
+}
+
+// VxlanPortPolicySetting allows configuring the VXLAN TCP port
+type VxlanPortPolicySetting struct {
+	Port uint16
+}
+
+// ProtocolType associated with L4ProxyPolicy
+type ProtocolType uint32
+
+const (
+	ProtocolTypeUnknown ProtocolType = 0
+	ProtocolTypeICMPv4  ProtocolType = 1
+	ProtocolTypeIGMP    ProtocolType = 2
+	ProtocolTypeTCP     ProtocolType = 6
+	ProtocolTypeUDP     ProtocolType = 17
+	ProtocolTypeICMPv6  ProtocolType = 58
+)
+
+//L4ProxyPolicySetting applies proxy policy on network/endpoint
+type L4ProxyPolicySetting struct {
+	IP          string       `json:",omitempty"`
+	Port        string       `json:",omitempty"`
+	Protocol    ProtocolType `json:",omitempty"`
+	Exceptions  []string     `json:",omitempty"`
+	Destination string
+	OutboundNAT bool `json:",omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go

@@ -15,6 +15,10 @@ type SupportedFeatures struct {
 	AclSupportForProtocol252 bool        `json:"AclSupportForProtocol252"`
 	SessionAffinity          bool        `json:"SessionAffinity"`
 	IPv6DualStack            bool        `json:"IPv6DualStack"`
+	SetPolicy                bool        `json:"SetPolicy"`
+	VxlanPort                bool        `json:"VxlanPort"`
+	L4Proxy                  bool        `json:"L4Proxy"`    // network policy that applies VFP rules to all endpoints on the network to redirect traffic
+	L4WfpProxy               bool        `json:"L4WfpProxy"` // endpoint policy that applies WFP filters to redirect traffic to/from that endpoint
 }
 
 // AclFeatures are the supported ACL possibilities.
@@ -61,6 +65,10 @@ func GetSupportedFeatures() SupportedFeatures {
 	features.AclSupportForProtocol252 = isFeatureSupported(globals.Version, AclSupportForProtocol252Version)
 	features.SessionAffinity = isFeatureSupported(globals.Version, SessionAffinityVersion)
 	features.IPv6DualStack = isFeatureSupported(globals.Version, IPv6DualStackVersion)
+	features.SetPolicy = isFeatureSupported(globals.Version, SetPolicyVersion)
+	features.VxlanPort = isFeatureSupported(globals.Version, VxlanPortVersion)
+	features.L4Proxy = isFeatureSupported(globals.Version, L4ProxyPolicyVersion)
+	features.L4WfpProxy = isFeatureSupported(globals.Version, L4WfpProxyPolicyVersion)
 
 	return features
 }

vendor/github.com/Microsoft/hcsshim/internal/hcs/service.go

+package hcs
+
+import (
+	"context"
+	"encoding/json"
+
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+	"github.com/Microsoft/hcsshim/internal/vmcompute"
+)
+
+// GetServiceProperties returns properties of the host compute service.
+func GetServiceProperties(ctx context.Context, q hcsschema.PropertyQuery) (*hcsschema.ServiceProperties, error) {
+	operation := "hcsshim::GetServiceProperties"
+
+	queryb, err := json.Marshal(q)
+	if err != nil {
+		return nil, err
+	}
+	propertiesJSON, resultJSON, err := vmcompute.HcsGetServiceProperties(ctx, string(queryb))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return nil, &HcsError{Op: operation, Err: err, Events: events}
+	}
+
+	if propertiesJSON == "" {
+		return nil, ErrUnexpectedValue
+	}
+	properties := &hcsschema.ServiceProperties{}
+	if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil {
+		return nil, err
+	}
+	return properties, nil
+}
+
+// ModifyServiceSettings modifies settings of the host compute service.
+func ModifyServiceSettings(ctx context.Context, settings hcsschema.ModificationRequest) error {
+	operation := "hcsshim::ModifyServiceSettings"
+
+	settingsJSON, err := json.Marshal(settings)
+	if err != nil {
+		return err
+	}
+	resultJSON, err := vmcompute.HcsModifyServiceSettings(ctx, string(settingsJSON))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return &HcsError{Op: operation, Err: err, Events: events}
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -28,8 +28,7 @@ type System struct {
 	waitBlock      chan struct{}
 	waitError      error
 	exitError      error
-
-	os, typ string
+	os, typ        string
 }
 
 func newSystem(id string) *System {

vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go

@@ -17,12 +17,15 @@ type HNSEndpoint struct {
 	Policies           []json.RawMessage `json:",omitempty"`
 	MacAddress         string            `json:",omitempty"`
 	IPAddress          net.IP            `json:",omitempty"`
+	IPv6Address        net.IP            `json:",omitempty"`
 	DNSSuffix          string            `json:",omitempty"`
 	DNSServerList      string            `json:",omitempty"`
 	GatewayAddress     string            `json:",omitempty"`
+	GatewayAddressV6   string            `json:",omitempty"`
 	EnableInternalDNS  bool              `json:",omitempty"`
 	DisableICC         bool              `json:",omitempty"`
 	PrefixLength       uint8             `json:",omitempty"`
+	IPv6PrefixLength   uint8             `json:",omitempty"`
 	IsRemoteEndpoint   bool              `json:",omitempty"`
 	EnableLowMetric    bool              `json:",omitempty"`
 	Namespace          *Namespace        `json:",omitempty"`

vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go

-// Code generated by 'go generate'; DO NOT EDIT.
-
-package safefile
-
-import (
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-var _ unsafe.Pointer
-
-// Do the interface allocations only once for common
-// Errno values.
-const (
-	errnoERROR_IO_PENDING = 997
-)
-
-var (
-	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
-)
-
-// errnoErr returns common boxed Errno values, to prevent
-// allocations at runtime.
-func errnoErr(e syscall.Errno) error {
-	switch e {
-	case 0:
-		return nil
-	case errnoERROR_IO_PENDING:
-		return errERROR_IO_PENDING
-	}
-	// TODO: add more here, after collecting data on the common
-	// error values see on Windows. (perhaps when running
-	// all.bat?)
-	return e
-}
-
-var (
-	modntdll    = windows.NewLazySystemDLL("ntdll.dll")
-	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
-
-	procNtCreateFile               = modntdll.NewProc("NtCreateFile")
-	procNtSetInformationFile       = modntdll.NewProc("NtSetInformationFile")
-	procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
-	procLocalAlloc                 = modkernel32.NewProc("LocalAlloc")
-	procLocalFree                  = modkernel32.NewProc("LocalFree")
-)
-
-func ntCreateFile(handle *uintptr, accessMask uint32, oa *objectAttributes, iosb *ioStatusBlock, allocationSize *uint64, fileAttributes uint32, shareAccess uint32, createDisposition uint32, createOptions uint32, eaBuffer *byte, eaLength uint32) (status uint32) {
-	r0, _, _ := syscall.Syscall12(procNtCreateFile.Addr(), 11, uintptr(unsafe.Pointer(handle)), uintptr(accessMask), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(allocationSize)), uintptr(fileAttributes), uintptr(shareAccess), uintptr(createDisposition), uintptr(createOptions), uintptr(unsafe.Pointer(eaBuffer)), uintptr(eaLength), 0)
-	status = uint32(r0)
-	return
-}
-
-func ntSetInformationFile(handle uintptr, iosb *ioStatusBlock, information uintptr, length uint32, class uint32) (status uint32) {
-	r0, _, _ := syscall.Syscall6(procNtSetInformationFile.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(information), uintptr(length), uintptr(class), 0)
-	status = uint32(r0)
-	return
-}
-
-func rtlNtStatusToDosError(status uint32) (winerr error) {
-	r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
-	if r0 != 0 {
-		winerr = syscall.Errno(r0)
-	}
-	return
-}
-
-func localAlloc(flags uint32, size int) (ptr uintptr) {
-	r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(flags), uintptr(size), 0)
-	ptr = uintptr(r0)
-	return
-}
-
-func localFree(ptr uintptr) {
-	syscall.Syscall(procLocalFree.Addr(), 1, uintptr(ptr), 0, 0)
-	return
-}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_add_instance_request.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardAddInstanceRequest struct {
+	Id             string `json:"Id,omitempty"`
+	CredentialSpec string `json:"CredentialSpec,omitempty"`
+	Transport      string `json:"Transport,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_hv_socket_service_config.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardHvSocketServiceConfig struct {
+	ServiceId     string                 `json:"ServiceId,omitempty"`
+	ServiceConfig *HvSocketServiceConfig `json:"ServiceConfig,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_instance.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardInstance struct {
+	Id              string                                         `json:"Id,omitempty"`
+	CredentialGuard *ContainerCredentialGuardState                 `json:"CredentialGuard,omitempty"`
+	HvSocketConfig  *ContainerCredentialGuardHvSocketServiceConfig `json:"HvSocketConfig,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_modify_operation.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardModifyOperation string
+
+const (
+	AddInstance    ContainerCredentialGuardModifyOperation = "AddInstance"
+	RemoveInstance ContainerCredentialGuardModifyOperation = "RemoveInstance"
+)

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_operation_request.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardOperationRequest struct {
+	Operation        ContainerCredentialGuardModifyOperation `json:"Operation,omitempty"`
+	OperationDetails interface{}                             `json:"OperationDetails,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_remove_instance_request.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardRemoveInstanceRequest struct {
+	Id string `json:"Id,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_system_info.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardSystemInfo struct {
+	Instances []ContainerCredentialGuardInstance `json:"Instances,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go

@@ -9,8 +9,19 @@
 
 package hcsschema
 
-type Device struct {
+type DeviceType string
+
+const (
+	ClassGUID      DeviceType = "ClassGuid"
+	DeviceInstance            = "DeviceInstance"
+	GPUMirror                 = "GpuMirror"
+)
 
-	//  The interface class guid of the device to assign to container.
+type Device struct {
+	//  The type of device to assign to the container.
+	Type DeviceType `json:"Type,omitempty"`
+	//  The interface class guid of the device interfaces to assign to the  container.  Only used when Type is ClassGuid.
 	InterfaceClassGuid string `json:"InterfaceClassGuid,omitempty"`
+	//  The location path of the device to assign to the container.  Only used when Type is DeviceInstance.
+	LocationPath string `json:"LocationPath,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_address.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+//  This class defines address settings applied to a VM
+//  by the GCS every time a VM starts or restores.
+type HvSocketAddress struct {
+	LocalAddress  string `json:"LocalAddress,omitempty"`
+	ParentAddress string `json:"ParentAddress,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/logical_processor.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type LogicalProcessor struct {
+	LpIndex     uint32 `json:"LpIndex,omitempty"`
+	NodeNumber  uint8  `json:"NodeNumber, omitempty"`
+	PackageId   uint32 `json:"PackageId, omitempty"`
+	CoreId      uint32 `json:"CoreId, omitempty"`
+	RootVpIndex int32  `json:"RootVpIndex, omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go

@@ -10,5 +10,5 @@
 package hcsschema
 
 type Memory struct {
-	SizeInMB int32 `json:"SizeInMB,omitempty"`
+	SizeInMB uint64 `json:"SizeInMB,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go

@@ -10,7 +10,7 @@
 package hcsschema
 
 type Memory2 struct {
-	SizeInMB int32 `json:"SizeInMB,omitempty"`
+	SizeInMB uint64 `json:"SizeInMB,omitempty"`
 
 	AllowOvercommit bool `json:"AllowOvercommit,omitempty"`
 

vendor/github.com/Microsoft/hcsshim/internal/schema2/modification_request.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ModificationRequest struct {
+	PropertyType PropertyType `json:"PropertyType,omitempty"`
+	Settings     interface{}  `json:"Settings,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_topology.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ProcessorTopology struct {
+	LogicalProcessorCount uint32             `json:"LogicalProcessorCount,omitempty"`
+	LogicalProcessors     []LogicalProcessor `json:"LogicalProcessors,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/schema2/property_type.go

@@ -18,6 +18,8 @@ const (
 	PTProcessList                 PropertyType = "ProcessList"
 	PTTerminateOnLastHandleClosed PropertyType = "TerminateOnLastHandleClosed"
 	PTSharedMemoryRegion          PropertyType = "SharedMemoryRegion"
+	PTContainerCredentialGuard    PropertyType = "ContainerCredentialGuard" // This field is not generated by swagger. This was added manually.
 	PTGuestConnection             PropertyType = "GuestConnection"
 	PTICHeartbeatStatus           PropertyType = "ICHeartbeatStatus"
+	PTProcessorTopology           PropertyType = "ProcessorTopology"
 )

vendor/github.com/Microsoft/hcsshim/internal/schema2/service_properties.go

+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+import "encoding/json"
+
+type ServiceProperties struct {
+	// Changed Properties field to []json.RawMessage from []interface{} to avoid having to
+	// remarshal sp.Properties[n] and unmarshal into the type(s) we want.
+	Properties []json.RawMessage `json:"Properties,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/vmcompute/vmcompute.go

@@ -26,6 +26,7 @@ import (
 //sys hcsResumeComputeSystem(computeSystem HcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem?
 //sys hcsGetComputeSystemProperties(computeSystem HcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties?
 //sys hcsModifyComputeSystem(computeSystem HcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem?
+//sys hcsModifyServiceSettings(settings string, result **uint16) (hr error) = vmcompute.HcsModifyServiceSettings?
 //sys hcsRegisterComputeSystemCallback(computeSystem HcsSystem, callback uintptr, context uintptr, callbackHandle *HcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback?
 //sys hcsUnregisterComputeSystemCallback(callbackHandle HcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback?
 
@@ -337,6 +338,27 @@ func HcsModifyComputeSystem(ctx gcontext.Context, computeSystem HcsSystem, confi
 	})
 }
 
+func HcsModifyServiceSettings(ctx gcontext.Context, settings string) (result string, hr error) {
+	ctx, span := trace.StartSpan(ctx, "HcsModifyServiceSettings")
+	defer span.End()
+	defer func() {
+		if result != "" {
+			span.AddAttributes(trace.StringAttribute("result", result))
+		}
+		oc.SetSpanStatus(span, hr)
+	}()
+	span.AddAttributes(trace.StringAttribute("settings", settings))
+
+	return result, execute(ctx, timeout.SyscallWatcher, func() error {
+		var resultp *uint16
+		err := hcsModifyServiceSettings(settings, &resultp)
+		if resultp != nil {
+			result = interop.ConvertAndFreeCoTaskMemString(resultp)
+		}
+		return err
+	})
+}
+
 func HcsRegisterComputeSystemCallback(ctx gcontext.Context, computeSystem HcsSystem, callback uintptr, context uintptr) (callbackHandle HcsCallback, hr error) {
 	ctx, span := trace.StartSpan(ctx, "HcsRegisterComputeSystemCallback")
 	defer span.End()

vendor/github.com/Microsoft/hcsshim/internal/vmcompute/zsyscall_windows.go

@@ -50,6 +50,7 @@ var (
 	procHcsResumeComputeSystem             = modvmcompute.NewProc("HcsResumeComputeSystem")
 	procHcsGetComputeSystemProperties      = modvmcompute.NewProc("HcsGetComputeSystemProperties")
 	procHcsModifyComputeSystem             = modvmcompute.NewProc("HcsModifyComputeSystem")
+	procHcsModifyServiceSettings           = modvmcompute.NewProc("HcsModifyServiceSettings")
 	procHcsRegisterComputeSystemCallback   = modvmcompute.NewProc("HcsRegisterComputeSystemCallback")
 	procHcsUnregisterComputeSystemCallback = modvmcompute.NewProc("HcsUnregisterComputeSystemCallback")
 	procHcsCreateProcess                   = modvmcompute.NewProc("HcsCreateProcess")
@@ -314,6 +315,29 @@ func _hcsModifyComputeSystem(computeSystem HcsSystem, configuration *uint16, res
 	return
 }
 
+func hcsModifyServiceSettings(settings string, result **uint16) (hr error) {
+	var _p0 *uint16
+	_p0, hr = syscall.UTF16PtrFromString(settings)
+	if hr != nil {
+		return
+	}
+	return _hcsModifyServiceSettings(_p0, result)
+}
+
+func _hcsModifyServiceSettings(settings *uint16, result **uint16) (hr error) {
+	if hr = procHcsModifyServiceSettings.Find(); hr != nil {
+		return
+	}
+	r0, _, _ := syscall.Syscall(procHcsModifyServiceSettings.Addr(), 2, uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result)), 0)
+	if int32(r0) < 0 {
+		if r0&0x1fff0000 == 0x00070000 {
+			r0 &= 0xffff
+		}
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
 func hcsRegisterComputeSystemCallback(computeSystem HcsSystem, callback uintptr, context uintptr, callbackHandle *HcsCallback) (hr error) {
 	if hr = procHcsRegisterComputeSystemCallback.Find(); hr != nil {
 		return

vendor/github.com/Microsoft/hcsshim/internal/wclayer/baselayer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/hcserror"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/safefile"
+	"github.com/Microsoft/hcsshim/internal/winapi"
 	"go.opencensus.io/trace"
 )
 
@@ -37,7 +38,7 @@ type dirInfo struct {
 func reapplyDirectoryTimes(root *os.File, dis []dirInfo) error {
 	for i := range dis {
 		di := &dis[len(dis)-i-1] // reverse order: process child directories first
-		f, err := safefile.OpenRelative(di.path, root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, safefile.FILE_OPEN, safefile.FILE_DIRECTORY_FILE)
+		f, err := safefile.OpenRelative(di.path, root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, winapi.FILE_OPEN, winapi.FILE_DIRECTORY_FILE|syscall.FILE_FLAG_OPEN_REPARSE_POINT)
 		if err != nil {
 			return err
 		}
@@ -47,6 +48,7 @@ func reapplyDirectoryTimes(root *os.File, dis []dirInfo) error {
 		if err != nil {
 			return err
 		}
+
 	}
 	return nil
 }
@@ -92,14 +94,12 @@ func (w *baseLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) (err e
 
 	extraFlags := uint32(0)
 	if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
-		extraFlags |= safefile.FILE_DIRECTORY_FILE
-		if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
-			w.dirInfo = append(w.dirInfo, dirInfo{name, *fileInfo})
-		}
+		extraFlags |= winapi.FILE_DIRECTORY_FILE
+		w.dirInfo = append(w.dirInfo, dirInfo{name, *fileInfo})
 	}
 
 	mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY)
-	f, err = safefile.OpenRelative(name, w.root, mode, syscall.FILE_SHARE_READ, safefile.FILE_CREATE, extraFlags)
+	f, err = safefile.OpenRelative(name, w.root, mode, syscall.FILE_SHARE_READ, winapi.FILE_CREATE, extraFlags)
 	if err != nil {
 		return hcserror.New(err, "Failed to safefile.OpenRelative", name)
 	}

vendor/github.com/Microsoft/hcsshim/internal/wclayer/createscratchlayer.go

@@ -10,9 +10,7 @@ import (
 )
 
 // CreateScratchLayer creates and populates new read-write layer for use by a container.
-// This requires both the id of the direct parent layer, as well as the full list
-// of paths to all parent layers up to the base (and including the direct parent
-// whose id was provided).
+// This requires the full list of paths to all parent layers up to the base
 func CreateScratchLayer(ctx context.Context, path string, parentLayerPaths []string) (err error) {
 	title := "hcsshim::CreateScratchLayer"
 	ctx, span := trace.StartSpan(ctx, title)

vendor/github.com/Microsoft/hcsshim/internal/wclayer/importlayer.go

@@ -93,6 +93,19 @@ func (r *legacyLayerWriterWrapper) Close() (err error) {
 			return err
 		}
 	}
+
+	// The reapplyDirectoryTimes must be called AFTER we are done with Tombstone
+	// deletion and hard link creation. This is because Tombstone deletion and hard link
+	// creation updates the directory last write timestamps so that will change the
+	// timestamps added by the `Add` call. Some container applications depend on the
+	// correctness of these timestamps and so we should change the timestamps back to
+	// the original value (i.e the value provided in the Add call) after this
+	// processing is done.
+	err = reapplyDirectoryTimes(r.destRoot, r.changedDi)
+	if err != nil {
+		return err
+	}
+
 	// Prepare the utility VM for use if one is present in the layer.
 	if r.HasUtilityVM {
 		err := safefile.EnsureNotReparsePointRelative("UtilityVM", r.destRoot)

vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go

@@ -15,6 +15,7 @@ import (
 	"github.com/Microsoft/go-winio"
 	"github.com/Microsoft/hcsshim/internal/longpath"
 	"github.com/Microsoft/hcsshim/internal/safefile"
+	"github.com/Microsoft/hcsshim/internal/winapi"
 )
 
 var errorIterationCanceled = errors.New("")
@@ -341,7 +342,7 @@ type legacyLayerWriter struct {
 	backupWriter    *winio.BackupFileWriter
 	Tombstones      []string
 	HasUtilityVM    bool
-	uvmDi           []dirInfo
+	changedDi       []dirInfo
 	addedFiles      map[string]bool
 	PendingLinks    []pendingLink
 	pendingDirs     []pendingDir
@@ -472,8 +473,8 @@ func copyFileWithMetadata(srcRoot, destRoot *os.File, subPath string, isDir bool
 		srcRoot,
 		syscall.GENERIC_READ|winio.ACCESS_SYSTEM_SECURITY,
 		syscall.FILE_SHARE_READ,
-		safefile.FILE_OPEN,
-		safefile.FILE_OPEN_REPARSE_POINT)
+		winapi.FILE_OPEN,
+		winapi.FILE_OPEN_REPARSE_POINT)
 	if err != nil {
 		return nil, err
 	}
@@ -488,14 +489,14 @@ func copyFileWithMetadata(srcRoot, destRoot *os.File, subPath string, isDir bool
 
 	extraFlags := uint32(0)
 	if isDir {
-		extraFlags |= safefile.FILE_DIRECTORY_FILE
+		extraFlags |= winapi.FILE_DIRECTORY_FILE
 	}
 	dest, err := safefile.OpenRelative(
 		subPath,
 		destRoot,
 		syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY,
 		syscall.FILE_SHARE_READ,
-		safefile.FILE_CREATE,
+		winapi.FILE_CREATE,
 		extraFlags)
 	if err != nil {
 		return nil, err
@@ -555,7 +556,7 @@ func cloneTree(srcRoot *os.File, destRoot *os.File, subPath string, mutatedFiles
 			if err != nil {
 				return err
 			}
-			if isDir && !isReparsePoint {
+			if isDir {
 				di = append(di, dirInfo{path: relPath, fileInfo: *fi})
 			}
 		} else {
@@ -583,6 +584,10 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		return w.initUtilityVM()
 	}
 
+	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
+		w.changedDi = append(w.changedDi, dirInfo{path: name, fileInfo: *fileInfo})
+	}
+
 	name = filepath.Clean(name)
 	if hasPathPrefix(name, utilityVMPath) {
 		if !w.HasUtilityVM {
@@ -591,7 +596,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		if !hasPathPrefix(name, utilityVMFilesPath) && name != utilityVMFilesPath {
 			return errors.New("invalid UtilityVM layer")
 		}
-		createDisposition := uint32(safefile.FILE_OPEN)
+		createDisposition := uint32(winapi.FILE_OPEN)
 		if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
 			st, err := safefile.LstatRelative(name, w.destRoot)
 			if err != nil && !os.IsNotExist(err) {
@@ -612,16 +617,13 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 					return err
 				}
 			}
-			if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
-				w.uvmDi = append(w.uvmDi, dirInfo{path: name, fileInfo: *fileInfo})
-			}
 		} else {
 			// Overwrite any existing hard link.
 			err := safefile.RemoveRelative(name, w.destRoot)
 			if err != nil && !os.IsNotExist(err) {
 				return err
 			}
-			createDisposition = safefile.FILE_CREATE
+			createDisposition = winapi.FILE_CREATE
 		}
 
 		f, err := safefile.OpenRelative(
@@ -630,7 +632,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 			syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY,
 			syscall.FILE_SHARE_READ,
 			createDisposition,
-			safefile.FILE_OPEN_REPARSE_POINT,
+			winapi.FILE_OPEN_REPARSE_POINT,
 		)
 		if err != nil {
 			return err
@@ -667,7 +669,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		w.currentIsDir = true
 	}
 
-	f, err := safefile.OpenRelative(fname, w.root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, safefile.FILE_CREATE, 0)
+	f, err := safefile.OpenRelative(fname, w.root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, winapi.FILE_CREATE, 0)
 	if err != nil {
 		return err
 	}
@@ -805,11 +807,5 @@ func (w *legacyLayerWriter) Close() error {
 			return err
 		}
 	}
-	if w.HasUtilityVM {
-		err := reapplyDirectoryTimes(w.destRoot, w.uvmDi)
-		if err != nil {
-			return err
-		}
-	}
 	return nil
 }

vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go

+// Package wclayer provides bindings to HCS's legacy layer management API and
+// provides a higher level interface around these calls for container layer
+// management.
 package wclayer
 
 import "github.com/Microsoft/go-winio/pkg/guid"

vendor/github.com/Microsoft/hcsshim/internal/winapi/devices.go

+package winapi
+
+import "github.com/Microsoft/go-winio/pkg/guid"
+
+//sys CMGetDeviceIDListSize(pulLen *uint32, pszFilter *byte, uFlags uint32) (hr error) = cfgmgr32.CM_Get_Device_ID_List_SizeA
+//sys CMGetDeviceIDList(pszFilter *byte, buffer *byte, bufferLen uint32, uFlags uint32) (hr error)= cfgmgr32.CM_Get_Device_ID_ListA
+//sys CMLocateDevNode(pdnDevInst *uint32, pDeviceID string, uFlags uint32) (hr error) = cfgmgr32.CM_Locate_DevNodeW
+//sys CMGetDevNodeProperty(dnDevInst uint32, propertyKey *DevPropKey, propertyType *uint32, propertyBuffer *uint16, propertyBufferSize *uint32, uFlags uint32) (hr error) = cfgmgr32.CM_Get_DevNode_PropertyW
+
+type DevPropKey struct {
+	Fmtid guid.GUID
+	Pid   uint32
+}

vendor/github.com/Microsoft/hcsshim/internal/winapi/errors.go

+package winapi
+
+import "syscall"
+
+//sys RtlNtStatusToDosError(status uint32) (winerr error) = ntdll.RtlNtStatusToDosError
+
+const (
+	STATUS_REPARSE_POINT_ENCOUNTERED               = 0xC000050B
+	ERROR_NO_MORE_ITEMS                            = 0x103
+	ERROR_MORE_DATA                  syscall.Errno = 234
+)
+
+func NTSuccess(status uint32) bool {
+	return status == 0
+}

vendor/github.com/Microsoft/hcsshim/internal/winapi/filesystem.go

+package winapi
+
+//sys NtCreateFile(handle *uintptr, accessMask uint32, oa *ObjectAttributes, iosb *IOStatusBlock, allocationSize *uint64, fileAttributes uint32, shareAccess uint32, createDisposition uint32, createOptions uint32, eaBuffer *byte, eaLength uint32) (status uint32) = ntdll.NtCreateFile
+//sys NtSetInformationFile(handle uintptr, iosb *IOStatusBlock, information uintptr, length uint32, class uint32) (status uint32) = ntdll.NtSetInformationFile
+
+//sys NtOpenDirectoryObject(handle *uintptr, accessMask uint32, oa *ObjectAttributes) (status uint32) = ntdll.NtOpenDirectoryObject
+//sys NtQueryDirectoryObject(handle uintptr, buffer *byte, length uint32, singleEntry bool, restartScan bool, context *uint32, returnLength *uint32)(status uint32) = ntdll.NtQueryDirectoryObject
+
+const (
+	FileLinkInformationClass          = 11
+	FileDispositionInformationExClass = 64
+
+	FILE_READ_ATTRIBUTES  = 0x0080
+	FILE_WRITE_ATTRIBUTES = 0x0100
+	DELETE                = 0x10000
+
+	FILE_OPEN   = 1
+	FILE_CREATE = 2
+
+	FILE_LIST_DIRECTORY          = 0x00000001
+	FILE_DIRECTORY_FILE          = 0x00000001
+	FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020
+	FILE_OPEN_FOR_BACKUP_INTENT  = 0x00004000
+	FILE_OPEN_REPARSE_POINT      = 0x00200000
+
+	FILE_DISPOSITION_DELETE = 0x00000001
+
+	OBJ_DONT_REPARSE = 0x1000
+
+	STATUS_MORE_ENTRIES    = 0x105
+	STATUS_NO_MORE_ENTRIES = 0x8000001a
+)
+
+type FileDispositionInformationEx struct {
+	Flags uintptr
+}
+
+type IOStatusBlock struct {
+	Status, Information uintptr
+}
+
+type ObjectAttributes struct {
+	Length             uintptr
+	RootDirectory      uintptr
+	ObjectName         uintptr
+	Attributes         uintptr
+	SecurityDescriptor uintptr
+	SecurityQoS        uintptr
+}
+
+type ObjectDirectoryInformation struct {
+	Name     UnicodeString
+	TypeName UnicodeString
+}
+
+type FileLinkInformation struct {
+	ReplaceIfExists bool
+	RootDirectory   uintptr
+	FileNameLength  uint32
+	FileName        [1]uint16
+}

vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go

+package winapi
+
+import (
+	"golang.org/x/sys/windows"
+)
+
+// Messages that can be received from an assigned io completion port.
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_associate_completion_port
+const (
+	JOB_OBJECT_MSG_END_OF_JOB_TIME       = 1
+	JOB_OBJECT_MSG_END_OF_PROCESS_TIME   = 2
+	JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT  = 3
+	JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO   = 4
+	JOB_OBJECT_MSG_NEW_PROCESS           = 6
+	JOB_OBJECT_MSG_EXIT_PROCESS          = 7
+	JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS = 8
+	JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT  = 9
+	JOB_OBJECT_MSG_JOB_MEMORY_LIMIT      = 10
+	JOB_OBJECT_MSG_NOTIFICATION_LIMIT    = 11
+)
+
+// IO limit flags
+//
+// https://docs.microsoft.com/en-us/windows/win32/api/jobapi2/ns-jobapi2-jobobject_io_rate_control_information
+const JOB_OBJECT_IO_RATE_CONTROL_ENABLE = 0x1
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_cpu_rate_control_information
+const (
+	JOB_OBJECT_CPU_RATE_CONTROL_ENABLE = 1 << iota
+	JOB_OBJECT_CPU_RATE_CONTROL_WEIGHT_BASED
+	JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP
+	JOB_OBJECT_CPU_RATE_CONTROL_NOTIFY
+	JOB_OBJECT_CPU_RATE_CONTROL_MIN_MAX_RATE
+)
+
+// JobObjectInformationClass values. Used for a call to QueryInformationJobObject
+//
+// https://docs.microsoft.com/en-us/windows/win32/api/jobapi2/nf-jobapi2-queryinformationjobobject
+const (
+	JobObjectBasicAccountingInformation      uint32 = 1
+	JobObjectBasicProcessIdList              uint32 = 3
+	JobObjectBasicAndIoAccountingInformation uint32 = 8
+	JobObjectLimitViolationInformation       uint32 = 13
+	JobObjectNotificationLimitInformation2   uint32 = 33
+)
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_basic_limit_information
+type JOBOBJECT_BASIC_LIMIT_INFORMATION struct {
+	PerProcessUserTimeLimit int64
+	PerJobUserTimeLimit     int64
+	LimitFlags              uint32
+	MinimumWorkingSetSize   uintptr
+	MaximumWorkingSetSize   uintptr
+	ActiveProcessLimit      uint32
+	Affinity                uintptr
+	PriorityClass           uint32
+	SchedulingClass         uint32
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_cpu_rate_control_information
+type JOBOBJECT_CPU_RATE_CONTROL_INFORMATION struct {
+	ControlFlags uint32
+	Rate         uint32
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/jobapi2/ns-jobapi2-jobobject_io_rate_control_information
+type JOBOBJECT_IO_RATE_CONTROL_INFORMATION struct {
+	MaxIops         int64
+	MaxBandwidth    int64
+	ReservationIops int64
+	BaseIOSize      uint32
+	VolumeName      string
+	ControlFlags    uint32
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_basic_process_id_list
+type JOBOBJECT_BASIC_PROCESS_ID_LIST struct {
+	NumberOfAssignedProcesses uint32
+	NumberOfProcessIdsInList  uint32
+	ProcessIdList             [1]uintptr
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_associate_completion_port
+type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct {
+	CompletionKey  uintptr
+	CompletionPort windows.Handle
+}
+
+// BOOL IsProcessInJob(
+// 		HANDLE ProcessHandle,
+// 		HANDLE JobHandle,
+// 		PBOOL  Result
+// );
+//
+//sys IsProcessInJob(procHandle windows.Handle, jobHandle windows.Handle, result *bool) (err error) = kernel32.IsProcessInJob
+
+// BOOL QueryInformationJobObject(
+//		HANDLE             hJob,
+//		JOBOBJECTINFOCLASS JobObjectInformationClass,
+//		LPVOID             lpJobObjectInformation,
+//		DWORD              cbJobObjectInformationLength,
+//		LPDWORD            lpReturnLength
+// );
+//
+//sys QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo uintptr, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) = kernel32.QueryInformationJobObject
+
+// HANDLE OpenJobObjectW(
+//		DWORD   dwDesiredAccess,
+//		BOOL    bInheritHandle,
+//		LPCWSTR lpName
+// );
+//
+//sys OpenJobObject(desiredAccess uint32, inheritHandle bool, lpName *uint16) (handle windows.Handle, err error) = kernel32.OpenJobObjectW
+
+// DWORD SetIoRateControlInformationJobObject(
+//		HANDLE                                hJob,
+//		JOBOBJECT_IO_RATE_CONTROL_INFORMATION *IoRateControlInfo
+// );
+//
+//sys SetIoRateControlInformationJobObject(jobHandle windows.Handle, ioRateControlInfo *JOBOBJECT_IO_RATE_CONTROL_INFORMATION) (ret uint32, err error) = kernel32.SetIoRateControlInformationJobObject

vendor/github.com/Microsoft/hcsshim/internal/winapi/logon.go

+package winapi
+
+// BOOL LogonUserA(
+// 	LPCWSTR  lpszUsername,
+// 	LPCWSTR  lpszDomain,
+// 	LPCWSTR  lpszPassword,
+// 	DWORD   dwLogonType,
+// 	DWORD   dwLogonProvider,
+// 	PHANDLE phToken
+// );
+//
+//sys LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *windows.Token) (err error) = advapi32.LogonUserW
+
+// Logon types
+const (
+	LOGON32_LOGON_INTERACTIVE       uint32 = 2
+	LOGON32_LOGON_NETWORK           uint32 = 3
+	LOGON32_LOGON_BATCH             uint32 = 4
+	LOGON32_LOGON_SERVICE           uint32 = 5
+	LOGON32_LOGON_UNLOCK            uint32 = 7
+	LOGON32_LOGON_NETWORK_CLEARTEXT uint32 = 8
+	LOGON32_LOGON_NEW_CREDENTIALS   uint32 = 9
+)
+
+// Logon providers
+const (
+	LOGON32_PROVIDER_DEFAULT uint32 = 0
+	LOGON32_PROVIDER_WINNT40 uint32 = 2
+	LOGON32_PROVIDER_WINNT50 uint32 = 3
+)

vendor/github.com/Microsoft/hcsshim/internal/winapi/memory.go

+package winapi
+
+// VOID RtlMoveMemory(
+// 	_Out_       VOID UNALIGNED *Destination,
+// 	_In_  const VOID UNALIGNED *Source,
+// 	_In_        SIZE_T         Length
+// );
+//sys RtlMoveMemory(destination *byte, source *byte, length uintptr) (err error) = kernel32.RtlMoveMemory
+
+//sys LocalAlloc(flags uint32, size int) (ptr uintptr) = kernel32.LocalAlloc
+//sys LocalFree(ptr uintptr) = kernel32.LocalFree

vendor/github.com/Microsoft/hcsshim/internal/winapi/path.go

+package winapi
+
+// DWORD SearchPathW(
+// 	LPCWSTR lpPath,
+// 	LPCWSTR lpFileName,
+// 	LPCWSTR lpExtension,
+// 	DWORD   nBufferLength,
+// 	LPWSTR  lpBuffer,
+// 	LPWSTR  *lpFilePart
+// );
+//sys SearchPath(lpPath *uint16, lpFileName *uint16, lpExtension *uint16, nBufferLength uint32, lpBuffer *uint16, lpFilePath **uint16) (size uint32, err error) = kernel32.SearchPathW

vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go

+package winapi
+
+const PROCESS_ALL_ACCESS uint32 = 2097151

vendor/github.com/Microsoft/hcsshim/internal/winapi/processor.go

+package winapi
+
+// Get count from all processor groups.
+// https://docs.microsoft.com/en-us/windows/win32/procthread/processor-groups
+const ALL_PROCESSOR_GROUPS = 0xFFFF
+
+//sys GetActiveProcessorCount(groupNumber uint16) (amount uint32) = kernel32.GetActiveProcessorCount

vendor/github.com/Microsoft/hcsshim/internal/winapi/utils.go

+package winapi
+
+import (
+	"errors"
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+type UnicodeString struct {
+	Length        uint16
+	MaximumLength uint16
+	Buffer        *uint16
+}
+
+//String converts a UnicodeString to a golang string
+func (uni UnicodeString) String() string {
+	p := (*[0xffff]uint16)(unsafe.Pointer(uni.Buffer))
+
+	// UnicodeString is not guaranteed to be null terminated, therefore
+	// use the UnicodeString's Length field
+	lengthInChars := uni.Length / 2
+	return syscall.UTF16ToString(p[:lengthInChars])
+}
+
+// NewUnicodeString allocates a new UnicodeString and copies `s` into
+// the buffer of the new UnicodeString.
+func NewUnicodeString(s string) (*UnicodeString, error) {
+	ws := utf16.Encode(([]rune)(s))
+	if len(ws) > 32767 {
+		return nil, syscall.ENAMETOOLONG
+	}
+
+	uni := &UnicodeString{
+		Length:        uint16(len(ws) * 2),
+		MaximumLength: uint16(len(ws) * 2),
+		Buffer:        &make([]uint16, len(ws))[0],
+	}
+	copy((*[32768]uint16)(unsafe.Pointer(uni.Buffer))[:], ws)
+	return uni, nil
+}
+
+// ConvertStringSetToSlice is a helper function used to convert the contents of
+// `buf` into a string slice. `buf` contains a set of null terminated strings
+// with an additional null at the end to indicate the end of the set.
+func ConvertStringSetToSlice(buf []byte) ([]string, error) {
+	var results []string
+	prev := 0
+	for i := range buf {
+		if buf[i] == 0 {
+			if prev == i {
+				// found two null characters in a row, return result
+				return results, nil
+			}
+			results = append(results, string(buf[prev:i]))
+			prev = i + 1
+		}
+	}
+	return nil, errors.New("string set malformed: missing null terminator at end of buffer")
+}

vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go

+// Package winapi contains various low-level bindings to Windows APIs. It can
+// be thought of as an extension to golang.org/x/sys/windows.
+package winapi
+
+//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go jobobject.go path.go logon.go memory.go processor.go devices.go filesystem.go errors.go

vendor/github.com/Microsoft/hcsshim/osversion/windowsbuilds.go

@@ -24,4 +24,12 @@ const (
 	// V19H1 (version 1903) corresponds to Windows Server 1903 (semi-annual
 	// channel).
 	V19H1 = 18362
+
+	// V19H2 (version 1909) corresponds to Windows Server 1909 (semi-annual
+	// channel).
+	V19H2 = 18363
+
+	// V20H1 (version 2004) corresponds to Windows Server 2004 (semi-annual
+	// channel).
+	V20H1 = 19041
 )

vendor/modules.txt

@@ -48,7 +48,8 @@ github.com/Microsoft/go-winio/pkg/etw
 github.com/Microsoft/go-winio/pkg/etwlogrus
 github.com/Microsoft/go-winio/pkg/guid
 github.com/Microsoft/go-winio/vhd
-# github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990 => github.com/Microsoft/hcsshim v0.8.9
+# github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990 => github.com/Microsoft/hcsshim v0.8.10
+## explicit
 github.com/Microsoft/hcsshim
 github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options
 github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats
@@ -72,6 +73,7 @@ github.com/Microsoft/hcsshim/internal/schema2
 github.com/Microsoft/hcsshim/internal/timeout
 github.com/Microsoft/hcsshim/internal/vmcompute
 github.com/Microsoft/hcsshim/internal/wclayer
+github.com/Microsoft/hcsshim/internal/winapi
 github.com/Microsoft/hcsshim/osversion
 # github.com/NYTimes/gziphandler v1.1.1
 github.com/NYTimes/gziphandler
@@ -3176,7 +3178,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 ## explicit
 sigs.k8s.io/yaml
-# github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.9
+# github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.10
 # github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7
 # github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e
 # github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340