Sandbox Support for Windows + CNI

pkg/kubelet/dockershim/helpers.go

@@ -223,14 +223,6 @@ func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separ
 	return fmtOpts, nil
 }
 
-func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
-	if c.State.Pid == 0 {
-		// Docker reports pid 0 for an exited container.
-		return "", fmt.Errorf("Cannot find network namespace for the terminated container %q", c.ID)
-	}
-	return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil
-}
-
 // dockerFilter wraps around dockerfilters.Args and provides methods to modify
 // the filter easily.
 type dockerFilter struct {

pkg/kubelet/dockershim/helpers_linux.go

@@ -133,3 +133,11 @@ func (ds *dockerService) updateCreateConfig(
 func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
 	return ""
 }
+
+func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
+	if c.State.Pid == 0 {
+		// Docker reports pid 0 for an exited container.
+		return "", fmt.Errorf("cannot find network namespace for the terminated container %q", c.ID)
+	}
+	return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil
+}

pkg/kubelet/dockershim/helpers_unsupported.go

@@ -19,6 +19,8 @@ limitations under the License.
 package dockershim
 
 import (
+	"fmt"
+
 	"github.com/blang/semver"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/golang/glog"
@@ -47,3 +49,7 @@ func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
 	glog.Warningf("determinePodIPBySandboxID is unsupported in this build")
 	return ""
 }
+
+func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
+	return "", fmt.Errorf("unsupported platform")
+}

pkg/kubelet/dockershim/helpers_windows.go

@@ -47,6 +47,9 @@ func (ds *dockerService) updateCreateConfig(
 	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
 	if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" {
 		createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode)
+	} else {
+		// Todo: Refactor this call in future for calling methods directly in security_context.go
+		modifyHostNetworkOptionForContainer(false, podSandboxID, createConfig.HostConfig)
 	}
 
 	return nil
@@ -71,14 +74,49 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
 		if err != nil {
 			continue
 		}
-		if containerIP := getContainerIP(r); containerIP != "" {
-			return containerIP
+
+		// Versions and feature support
+		// ============================
+		// Windows version == Windows Server, Version 1709,, Supports both sandbox and non-sandbox case
+		// Windows version == Windows Server 2016   Support only non-sandbox case
+		// Windows version < Windows Server 2016 is Not Supported
+
+		// Sandbox support in Windows mandates CNI Plugin.
+		// Presense of CONTAINER_NETWORK flag is considered as non-Sandbox cases here
+
+		// Todo: Add a kernel version check for more validation
+
+		if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
+			// Do not return any IP, so that we would continue and get the IP of the Sandbox
+			ds.getIP(sandboxID, r)
+		} else {
+			// On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network,
+			// with the shared container name as NetNS info,
+			// This is passed down to the platform to replicate some necessary information to the new container
+
+			//
+			// This place is chosen as a hack for now, since getContainerIP would end up calling CNI's addToNetwork
+			// That is why addToNetwork is required to be idempotent
+
+			// Instead of relying on this call, an explicit call to addToNetwork should be
+			// done immediately after ContainerCreation, in case of Windows only. TBD Issue # to handle this
+
+			if containerIP := getContainerIP(r); containerIP != "" {
+				return containerIP
+			}
 		}
 	}
 
 	return ""
 }
 
+func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
+	// Currently in windows there is no identifier exposed for network namespace
+	// Like docker, the referenced container id is used to figure out the network namespace id internally by the platform
+	// so returning the docker networkMode (which holds container:<ref containerid> for network namespace here
+	return string(c.HostConfig.NetworkMode), nil
+}
+
 func getContainerIP(container *dockertypes.ContainerJSON) string {
 	if container.NetworkSettings != nil {
 		for _, network := range container.NetworkSettings.Networks {

pkg/kubelet/network/cni/BUILD

@@ -8,7 +8,15 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["cni.go"],
+    srcs = [
+        "cni.go",
+        "cni_others.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "cni_windows.go",
+        ],
+        "//conditions:default": [],
+    }),
     importpath = "k8s.io/kubernetes/pkg/kubelet/network/cni",
     deps = [
         "//pkg/kubelet/apis/kubeletconfig:go_default_library",
@@ -18,7 +26,12 @@ go_library(
         "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 go_test(

pkg/kubelet/network/cni/cni.go

@@ -153,37 +153,12 @@ func vendorCNIDir(prefix, pluginType string) string {
 	return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType)
 }
 
-func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
-	loConfig, err := libcni.ConfListFromBytes([]byte(`{
-  "cniVersion": "0.2.0",
-  "name": "cni-loopback",
-  "plugins":[{
-    "type": "loopback"
-  }]
-}`))
-	if err != nil {
-		// The hardcoded config above should always be valid and unit tests will
-		// catch this
-		panic(err)
-	}
-	cninet := &libcni.CNIConfig{
-		Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir},
-	}
-	loNetwork := &cniNetwork{
-		name:          "lo",
-		NetworkConfig: loConfig,
-		CNIConfig:     cninet,
-	}
-
-	return loNetwork
-}
-
 func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
-	var err error
-	plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
+	err := plugin.platformInit()
 	if err != nil {
 		return err
 	}
+
 	plugin.host = host
 
 	plugin.syncNetworkConfig()
@@ -239,10 +214,12 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec
 		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}
 
-	_, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath)
-	if err != nil {
-		glog.Errorf("Error while adding to cni lo network: %s", err)
-		return err
+	// Windows doesn't have loNetwork. It comes only with Linux
+	if plugin.loNetwork != nil {
+		if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath); err != nil {
+			glog.Errorf("Error while adding to cni lo network: %s", err)
+			return err
+		}
 	}
 
 	_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
@@ -268,25 +245,6 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
 	return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
 }
 
-// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
-// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
-func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
-	netnsPath, err := plugin.host.GetNetNS(id.ID)
-	if err != nil {
-		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
-	}
-	if netnsPath == "" {
-		return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id)
-	}
-
-	ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &network.PodNetworkStatus{IP: ip}, nil
-}
-
 func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) {
 	rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
 	if err != nil {

pkg/kubelet/network/cni/cni_others.go

+// +build !windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"fmt"
+
+	"github.com/containernetworking/cni/libcni"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+	loConfig, err := libcni.ConfListFromBytes([]byte(`{
+  "cniVersion": "0.2.0",
+  "name": "cni-loopback",
+  "plugins":[{
+    "type": "loopback"
+  }]
+}`))
+	if err != nil {
+		// The hardcoded config above should always be valid and unit tests will
+		// catch this
+		panic(err)
+	}
+	cninet := &libcni.CNIConfig{
+		Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir},
+	}
+	loNetwork := &cniNetwork{
+		name:          "lo",
+		NetworkConfig: loConfig,
+		CNIConfig:     cninet,
+	}
+
+	return loNetwork
+}
+
+func (plugin *cniNetworkPlugin) platformInit() error {
+	var err error
+	plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
+// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
+func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+	}
+	if netnsPath == "" {
+		return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id)
+	}
+
+	ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &network.PodNetworkStatus{IP: ip}, nil
+}

pkg/kubelet/network/cni/cni_windows.go

+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"fmt"
+
+	cniTypes020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/golang/glog"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) platformInit() error {
+	return nil
+}
+
+// GetPodNetworkStatus : Assuming addToNetwork is idempotent, we can call this API as many times as required to get the IPAddress
+func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+	}
+
+	result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
+
+	glog.V(5).Infof("GetPodNetworkStatus result %+v", result)
+	if err != nil {
+		glog.Errorf("error while adding to cni network: %s", err)
+		return nil, err
+	}
+
+	// Parse the result and get the IPAddress
+	var result020 *cniTypes020.Result
+	result020, err = cniTypes020.GetResult(result)
+	if err != nil {
+		glog.Errorf("error while cni parsing result: %s", err)
+		return nil, err
+	}
+	return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil
+}