kubenet masqurade for outbound traffic

pkg/kubelet/dockertools/docker_test.go

@@ -651,7 +651,7 @@ func TestFindContainersByPod(t *testing.T) {
 		},
 	}
 	fakeClient := NewFakeDockerClient()
-	np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	// image back-off is set to nil, this test should not pull images
 	containerManager := NewFakeDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, &cadvisorapi.MachineInfo{}, options.GetDefaultPodInfraContainerImage(), 0, 0, "", &containertest.FakeOS{}, np, nil, nil, nil)
 	for i, test := range tests {

pkg/kubelet/dockertools/manager_test.go

@@ -107,7 +107,7 @@ func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerCli
 	}
 	fakeRecorder := &record.FakeRecorder{}
 	containerRefManager := kubecontainer.NewRefManager()
-	networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	dockerManager := NewFakeDockerManager(
 		fakeDocker,
 		fakeRecorder,

pkg/kubelet/kubelet.go

@@ -385,7 +385,7 @@ func NewMainKubelet(
 	}
 	glog.Infof("Hairpin mode set to %q", klet.hairpinMode)
 
-	if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode); err != nil {
+	if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode, klet.nonMasqueradeCIDR); err != nil {
 		return nil, err
 	} else {
 		klet.networkPlugin = plug

pkg/kubelet/kubelet_test.go

@@ -188,7 +188,7 @@ func newTestKubeletWithImageList(t *testing.T, imageList []kubecontainer.Image)
 	kubelet.nodeName = testKubeletHostname
 	kubelet.runtimeState = newRuntimeState(maxWaitForContainerRuntime)
 	kubelet.runtimeState.setNetworkState(nil)
-	kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kubelet.nonMasqueradeCIDR)
 	if tempDir, err := ioutil.TempDir("/tmp", "kubelet_test."); err != nil {
 		t.Fatalf("can't make a temp rootdir: %v", err)
 	} else {

pkg/kubelet/network/cni/cni.go

@@ -94,7 +94,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er
 	return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
 }
 
-func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
+func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	plugin.host = host
 	return nil
 }

pkg/kubelet/network/cni/cni_test.go

@@ -145,7 +145,7 @@ func newTestDockerManager() (*dockertools.DockerManager, *dockertools.FakeDocker
 	fakeDocker := dockertools.NewFakeDockerClient()
 	fakeRecorder := &record.FakeRecorder{}
 	containerRefManager := kubecontainer.NewRefManager()
-	networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	dockerManager := dockertools.NewFakeDockerManager(
 		fakeDocker,
 		fakeRecorder,
@@ -175,7 +175,7 @@ func TestCNIPlugin(t *testing.T) {
 	installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName)
 
 	np := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), testVendorCNIDirPrefix)
-	plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err != nil {
 		t.Fatalf("Failed to select the desired plugin: %v", err)
 	}

pkg/kubelet/network/exec/exec.go

@@ -105,7 +105,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
 	return execPlugins
 }
 
-func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
+func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	err := plugin.validate()
 	if err != nil {
 		return err

pkg/kubelet/network/exec/exec_test.go

@@ -135,7 +135,7 @@ func TestSelectPlugin(t *testing.T) {
 
 	installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
 
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err != nil {
 		t.Errorf("Failed to select the desired plugin: %v", err)
 	}
@@ -157,7 +157,7 @@ func TestSelectVendoredPlugin(t *testing.T) {
 	installPluginUnderTest(t, vendor, testPluginPath, pluginName, nil)
 
 	vendoredPluginName := fmt.Sprintf("%s/%s", vendor, pluginName)
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err != nil {
 		t.Errorf("Failed to select the desired plugin: %v", err)
 	}
@@ -178,7 +178,7 @@ func TestSelectWrongPlugin(t *testing.T) {
 	installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
 
 	wrongPlugin := "abcd"
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if plug != nil || err == nil {
 		t.Errorf("Expected to see an error. Wrong plugin selected.")
 	}
@@ -206,7 +206,7 @@ func TestPluginValidation(t *testing.T) {
 	}
 	f.Close()
 
-	_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err == nil {
 		// we expected an error here because validation would have failed
 		t.Errorf("Expected non-nil value.")
@@ -224,7 +224,7 @@ func TestPluginSetupHook(t *testing.T) {
 
 	installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
 
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 
 	err = plug.SetUpPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
 	if err != nil {
@@ -252,7 +252,7 @@ func TestPluginTearDownHook(t *testing.T) {
 
 	installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
 
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 
 	err = plug.TearDownPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
 	if err != nil {
@@ -280,7 +280,7 @@ func TestPluginStatusHook(t *testing.T) {
 
 	installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
 
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 
 	ip, err := plug.GetPodNetworkStatus("namespace", "name", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
 	if err != nil {
@@ -316,7 +316,7 @@ func TestPluginStatusHookIPv6(t *testing.T) {
 	}
 	installPluginUnderTest(t, "", testPluginPath, pluginName, execTemplate)
 
-	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err != nil {
 		t.Errorf("InitNetworkPlugin() failed: %v", err)
 	}

pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -79,7 +79,8 @@ type kubenetNetworkPlugin struct {
 	iptables        utiliptables.Interface
 	// vendorDir is passed by kubelet network-plugin-dir parameter.
 	// kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir.
-	vendorDir string
+	vendorDir         string
+	nonMasqueradeCIDR string
 }
 
 func NewPlugin(networkPluginDir string) network.NetworkPlugin {
@@ -88,16 +89,17 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin {
 	dbus := utildbus.New()
 	iptInterface := utiliptables.New(execer, dbus, protocol)
 	return &kubenetNetworkPlugin{
-		podIPs:      make(map[kubecontainer.ContainerID]string),
-		hostPortMap: make(map[hostport]closeable),
-		MTU:         1460, //TODO: don't hardcode this
-		execer:      utilexec.New(),
-		iptables:    iptInterface,
-		vendorDir:   networkPluginDir,
+		podIPs:            make(map[kubecontainer.ContainerID]string),
+		hostPortMap:       make(map[hostport]closeable),
+		MTU:               1460, //TODO: don't hardcode this
+		execer:            utilexec.New(),
+		iptables:          iptInterface,
+		vendorDir:         networkPluginDir,
+		nonMasqueradeCIDR: "10.0.0.0/8",
 	}
 }
 
-func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
+func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	plugin.host = host
 	plugin.hairpinMode = hairpinMode
 	plugin.cniConfig = &libcni.CNIConfig{
@@ -132,6 +134,23 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componen
 		return fmt.Errorf("Failed to generate loopback config: %v", err)
 	}
 
+	plugin.nonMasqueradeCIDR = nonMasqueradeCIDR
+	// Need to SNAT outbound traffic from cluster
+	if err = plugin.ensureMasqRule(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO: move thic logic into cni bridge plugin and remove this from kubenet
+func (plugin *kubenetNetworkPlugin) ensureMasqRule() error {
+	if _, err := plugin.iptables.EnsureRule(utiliptables.Append, utiliptables.TableNAT, utiliptables.ChainPostrouting,
+		"-m", "comment", "--comment", "kubenet: SNAT for outbound traffic from cluster",
+		"-m", "addrtype", "!", "--dst-type", "LOCAL",
+		"!", "-d", plugin.nonMasqueradeCIDR,
+		"-j", "MASQUERADE"); err != nil {
+		return fmt.Errorf("Failed to ensure that %s chain %s jumps to MASQUERADE: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, err)
+	}
 	return nil
 }
 
@@ -167,7 +186,7 @@ const NET_CONFIG_TEMPLATE = `{
   "mtu": %d,
   "addIf": "%s",
   "isGateway": true,
-  "ipMasq": true,
+  "ipMasq": false,
   "ipam": {
     "type": "host-local",
     "subnet": "%s",
@@ -354,6 +373,11 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k
 	}
 
 	plugin.syncHostportsRules()
+
+	// Need to SNAT outbound traffic from cluster
+	if err = plugin.ensureMasqRule(); err != nil {
+		glog.Errorf("Failed to ensure MASQ rule: %v", err)
+	}
 	return nil
 }
 
@@ -391,6 +415,11 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i
 	delete(plugin.podIPs, id)
 
 	plugin.syncHostportsRules()
+
+	// Need to SNAT outbound traffic from cluster
+	if err := plugin.ensureMasqRule(); err != nil {
+		glog.Errorf("Failed to ensure MASQ rule: %v", err)
+	}
 	return nil
 }
 

pkg/kubelet/network/kubenet/kubenet_unsupported.go

@@ -34,7 +34,7 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin {
 	return &kubenetNetworkPlugin{}
 }
 
-func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
+func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	return fmt.Errorf("Kubenet is not supported in this build")
 }
 

pkg/kubelet/network/mock_network/network_plugins.go

@@ -78,7 +78,7 @@ func (_mr *_MockNetworkPluginRecorder) GetPodNetworkStatus(arg0, arg1, arg2 inte
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetPodNetworkStatus", arg0, arg1, arg2)
 }
 
-func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode) error {
+func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	ret := _m.ctrl.Call(_m, "Init", _param0, _param1)
 	ret0, _ := ret[0].(error)
 	return ret0

pkg/kubelet/network/plugins.go

@@ -52,7 +52,7 @@ const (
 type NetworkPlugin interface {
 	// Init initializes the plugin.  This will be called exactly once
 	// before any other methods are called.
-	Init(host Host, hairpinMode componentconfig.HairpinMode) error
+	Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error
 
 	// Called on various events like:
 	// NET_PLUGIN_EVENT_POD_CIDR_CHANGE
@@ -105,11 +105,11 @@ type Host interface {
 }
 
 // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.
-func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode) (NetworkPlugin, error) {
+func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) (NetworkPlugin, error) {
 	if networkPluginName == "" {
 		// default to the no_op plugin
 		plug := &NoopNetworkPlugin{}
-		if err := plug.Init(host, hairpinMode); err != nil {
+		if err := plug.Init(host, hairpinMode, nonMasqueradeCIDR); err != nil {
 			return nil, err
 		}
 		return plug, nil
@@ -134,7 +134,7 @@ func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host H
 
 	chosenPlugin := pluginMap[networkPluginName]
 	if chosenPlugin != nil {
-		err := chosenPlugin.Init(host, hairpinMode)
+		err := chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR)
 		if err != nil {
 			allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err))
 		} else {
@@ -156,7 +156,7 @@ type NoopNetworkPlugin struct {
 
 const sysctlBridgeCallIptables = "net/bridge/bridge-nf-call-iptables"
 
-func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode) error {
+func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
 	// Set bridge-nf-call-iptables=1 to maintain compatibility with older
 	// kubernetes versions to ensure the iptables-based kube proxy functions
 	// correctly.  Other plugins are responsible for setting this correctly

pkg/kubelet/network/plugins_test.go

@@ -25,7 +25,7 @@ import (
 
 func TestSelectDefaultPlugin(t *testing.T) {
 	all_plugins := []NetworkPlugin{}
-	plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
 	if err != nil {
 		t.Fatalf("Unexpected error in selecting default plugin: %v", err)
 	}

pkg/kubelet/runonce_test.go

@@ -84,7 +84,7 @@ func TestRunOnce(t *testing.T) {
 	}
 	kb.containerManager = cm.NewStubContainerManager()
 
-	kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kb.nonMasqueradeCIDR)
 	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
 	volumeStatsAggPeriod := time.Second * 10
 	kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod, kb.containerRuntime)