openstack-heat: Drop flannel for cloud Routes API

cluster/openstack-heat/config-default.sh

@@ -38,6 +38,8 @@ EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public}
 LBAAS_VERSION=${LBAAS_VERSION:-}
 
 FIXED_NETWORK_CIDR=${FIXED_NETWORK_CIDR:-10.0.0.0/24}
+SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}
+CLUSTER_IP_RANGE=${CLUSTER_IP_RANGE:-10.244.0.0/16}
 
 SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-}
 

cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml

@@ -12,6 +12,7 @@ write_files:
     content: |
       grains:
         node_ip: $MASTER_IP
+        cbr-cidr: $MASTER_IP_RANGE
         publicAddressOverride: $MASTER_IP
         network_mode: openvswitch
         networkInterfaceName: eth0
@@ -21,7 +22,7 @@ write_files:
         roles:
           - $role
         runtime_config: ""
-        docker_opts: ""
+        docker_opts: "--bridge=cbr0 --iptables=false --ip-masq=false"
         master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master"
         keep_host_etcd: true
         kube_user: $KUBE_USER
@@ -37,8 +38,11 @@ write_files:
       lb-version=$LBAAS_VERSION
       subnet-id=$SUBNET_ID
       floating-network-id=$FLOATING_NETWORK_ID
+      [Route]
+      router-id=$router_id
   - path: /srv/salt-overlay/pillar/cluster-params.sls
     content: |
+      allocate_node_cidrs: "true"
       service_cluster_ip_range: 10.246.0.0/16
       cert_ip: 10.246.0.1
       enable_cluster_monitoring: influxdb
@@ -56,6 +60,7 @@ write_files:
       admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,DefaultStorageClass,ResourceQuota
       enable_cpu_cfs_quota: "true"
       network_provider: none
+      cluster_cidr: "$cluster_cidr"
       opencontrail_tag: R2.20
       opencontrail_kubernetes_tag: master
       opencontrail_public_subnet: 10.1.0.0/16

cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh

@@ -20,47 +20,4 @@ set -o pipefail
 
 . /etc/sysconfig/heat-params
 
-FLANNEL_ETCD_URL="http://${MASTER_IP}:4379"
-
-# Install etcd for flannel data
-if ! which etcd > /dev/null 2>&1; then
-  yum install -y etcd
-fi
-
-cat <<EOF > /etc/etcd/etcd.conf
-ETCD_NAME=flannel
-ETCD_DATA_DIR="/var/lib/etcd/flannel.etcd"
-ETCD_LISTEN_PEER_URLS="http://${MASTER_IP}:4380"
-ETCD_LISTEN_CLIENT_URLS="http://${MASTER_IP}:4379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${MASTER_IP}:4380"
-ETCD_INITIAL_CLUSTER="flannel=http://${MASTER_IP}:4380"
-ETCD_ADVERTISE_CLIENT_URLS="${FLANNEL_ETCD_URL}"
-EOF
-systemctl enable etcd
-systemctl restart etcd
-
-# Install flannel for overlay
-if ! which flanneld > /dev/null 2>&1; then
-  yum install -y flannel
-fi
-
-cat <<EOF > /etc/flannel-config.json
-{
-  "Network": "${CONTAINER_SUBNET}",
-  "SubnetLen": 24,
-  "Backend": {
-    "Type": "host-gw"
-  }
-}
-EOF
-
-etcdctl -C ${FLANNEL_ETCD_URL} set /coreos.com/network/config < /etc/flannel-config.json
-
-cat <<EOF > /etc/sysconfig/flanneld
-FLANNEL_ETCD="${FLANNEL_ETCD_URL}"
-FLANNEL_ETCD_KEY="/coreos.com/network"
-FLANNEL_OPTIONS="-iface=eth0 --ip-masq"
-EOF
-
-systemctl enable flanneld
-systemctl restart flanneld
+# nothing to do

cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh

@@ -18,24 +18,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-. /etc/sysconfig/heat-params
-
-FLANNEL_ETCD_URL="http://${MASTER_IP}:4379"
-
-# Install flannel for overlay
-if ! which flanneld >/dev/null 2>&1; then
-  yum install -y flannel
-fi
-
-cat <<EOF >/etc/sysconfig/flanneld
-FLANNEL_ETCD="${FLANNEL_ETCD_URL}"
-FLANNEL_ETCD_KEY="/coreos.com/network"
-FLANNEL_OPTIONS="-iface=eth0 --ip-masq"
-EOF
-
-systemctl enable flanneld
-systemctl restart flanneld
-
 # Kubernetes node shoud be able to resolve its hostname.
 # In some cloud providers, myhostname is not enabled by default.
 grep '^hosts:.*myhostname' /etc/nsswitch.conf || (

cluster/openstack-heat/kubernetes-heat/kubecluster.yaml

@@ -52,6 +52,23 @@ parameters:
     description: network range for fixed ip network
     default: 10.0.0.0/24
 
+  cluster_cidr:
+    type: string
+    description: network range for pod IPs
+    default: 10.244.0.0/16
+
+  service_cluster_cidr:
+    type: string
+    description: network range for service IPs
+    default: 10.10.0.0/16
+
+  master_pod_cidr:
+    type: string
+    description: >-
+      network range for master pod IPs (ignored, but must not conflict
+      with other subnets)
+    default: 10.245.1.0/24
+
   kubernetes_server_url:
     type: string
     description: URL of kubernetes server binary. Must be tar.gz.
@@ -301,6 +318,9 @@ resources:
             "$SUBNET_ID": {get_resource: fixed_subnet}
             "$FLOATING_NETWORK_ID": {get_attr: [kube_master_floating, floating_network_id]}
             "$role": "kubernetes-master"
+            "$router_id": {get_resource: extrouter}
+            "$cluster_cidr": {get_param: cluster_cidr}
+            "$MASTER_IP_RANGE": {get_param: master_pod_cidr}
 
   run_salt:
     type: OS::Heat::SoftwareConfig
@@ -390,6 +410,7 @@ resources:
           token_kube_proxy: {get_param: token_kube_proxy}
           fixed_network: {get_resource: fixed_network}
           fixed_subnet: {get_resource: fixed_subnet}
+          cluster_cidr: {get_param: cluster_cidr}
           kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
           external_network: {get_param: external_network}
           wait_condition_timeout: {get_param: wait_condition_timeout}

cluster/openstack-heat/kubernetes-heat/kubeminion.yaml

@@ -106,6 +106,9 @@ parameters:
   fixed_subnet:
     type: string
     description: Subnet from which to allocate fixed addresses.
+  cluster_cidr:
+    type: string
+    description: Subnet from which to allocate pod subnets.
   wait_condition_timeout:
     type: number
     description : >
@@ -215,6 +218,7 @@ resources:
             "$OS_REGION_NAME": {get_param: os_region_name}
             "$OS_TENANT_ID": {get_param: os_tenant_id}
             "$role": "kubernetes-pool"
+            "$cluster_cidr": {get_param: cluster_cidr}
 
   run_salt:
     type: OS::Heat::SoftwareConfig

cluster/saltbase/salt/docker/init.sls

@@ -18,6 +18,27 @@ bridge-utils:
     - mode: 644
     - makedirs: true
 
+{% if grains.cloud is defined and grains.cloud == 'openstack' %}
+
+cbr0:
+  # workaround https://github.com/saltstack/salt/issues/20570
+  kmod.present:
+    - name: bridge
+
+  network.managed:
+    - enabled: True
+    - type: bridge
+    - proto: none
+    - ports: none
+    - bridge: cbr0
+    - delay: 0
+    - bypassfirewall: True
+    - require_in:
+      - service: docker
+    - require:
+      - kmod: cbr0
+{% endif %}
+
 {% if (grains.os == 'Fedora' and grains.osrelease_info[0] >= 22) or (grains.os == 'CentOS' and grains.osrelease_info[0] >= 7) %}
 
 docker:
@@ -512,4 +533,3 @@ docker:
       - cmd: fix-service-docker
 {% endif %}
 {% endif %} # end grains.os_family != 'RedHat'
-