Merge remote-tracking branch 'rancher/master' into release-1.17

.dockerignore

 ./bin
 ./etc
-./build/data
-./build/data.tar.gz
 ./pkg/data/zz_generated_bindata.go
-./package/data.tar.gz
 ./.vagrant
 ./.cache
 ./.dapper
-./data-dir
-./dist
 ./.trash-cache

.drone.yml

@@ -59,6 +59,31 @@ steps:
     event:
     - tag
 
+- name: rpm-publish
+  image: centos:7
+  environment:
+    PRIVATE_KEY:
+      from_secret: private_key
+    PRIVATE_KEY_PASS_PHRASE:
+      from_secret: private_key_pass_phrase
+    AWS_S3_BUCKET:
+      from_secret: aws_s3_bucket
+    AWS_ACCESS_KEY_ID:
+      from_secret: aws_access_key_id
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: aws_secret_access_key
+  commands:
+  - scripts/provision/generic/centos7/yum-install-rpm-tools
+  - scripts/package-rpm
+  when:
+    instance:
+    - drone-publish.rancher.io
+    ref:
+    - refs/head/master
+    - refs/tags/*
+    event:
+    - tag
+
 - name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
@@ -154,6 +179,31 @@ steps:
     event:
     - tag
 
+- name: rpm-publish
+  image: centos:7
+  environment:
+    PRIVATE_KEY:
+      from_secret: private_key
+    PRIVATE_KEY_PASS_PHRASE:
+      from_secret: private_key_pass_phrase
+    AWS_S3_BUCKET:
+      from_secret: aws_s3_bucket
+    AWS_ACCESS_KEY_ID:
+      from_secret: aws_access_key_id
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: aws_secret_access_key
+  commands:
+  - scripts/provision/generic/centos7/yum-install-rpm-tools
+  - scripts/package-rpm
+  when:
+    instance:
+    - drone-publish.rancher.io
+    ref:
+    - refs/head/master
+    - refs/tags/*
+    event:
+    - tag
+
 - name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]

.gitignore

@@ -28,3 +28,4 @@ __pycache__
 /tests/.tox/
 /tests/.vscode
 /sonobuoy-output
+*.tmp

Dockerfile.dapper

 FROM golang:1.13.8-alpine3.10
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
-    python2 python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
-RUN pip3 install 'tox==3.6.0'
-RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static
+    python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
+
+RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static libselinux libselinux-dev
 RUN mkdir -p /go/src/golang.org/x && \
-    cd /go/src/golang.org/x && git clone https://github.com/golang/tools && \
-    git -C /go/src/golang.org/x/tools checkout -b current aa82965741a9fecd12b026fbb3d3c6ed3231b8f8 && \
-    go install golang.org/x/tools/cmd/goimports
+    cd /go/src/golang.org/x && git clone https://github.com/golang/tools && cd tools && \
+    git checkout -b current aa82965741a9fecd12b026fbb3d3c6ed3231b8f8 && \
+    go install golang.org/x/tools/cmd/goimports && cd
 RUN rm -rf /go/src /go/pkg
 
 ARG DAPPER_HOST_ARCH
@@ -17,6 +17,9 @@ RUN if [ "${ARCH}" == "amd64" ]; then \
         curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \
     fi
 
+ARG SELINUX=true
+ENV SELINUX $SELINUX
+
 ARG DQLITE=true
 ENV DQLITE $DQLITE
 COPY --from=rancher/dqlite-build:v1.3.1-r1 /dist/artifacts /usr/src/

Vagrantfile

-BOX = "generic/alpine310"
+OS = (ENV['OS'] || "alpine310")
+BOX_REPO = (ENV['BOX_REPO'] || "generic")
 HOME = File.dirname(__FILE__)
 PROJECT = File.basename(HOME)
-MOUNT_TYPE = ENV['MOUNT_TYPE'] || "nfs"
 NUM_NODES = (ENV['NUM_NODES'] || 0).to_i
 NODE_CPUS = (ENV['NODE_CPUS'] || 4).to_i
 NODE_MEMORY = (ENV['NODE_MEMORY'] || 8192).to_i
 NETWORK_PREFIX = ENV['NETWORK_PREFIX'] || "10.135.135"
-VAGRANT_PROVISION = ENV['VAGRANT_PROVISION'] || "./scripts/vagrant-provision"
+VAGRANT_PROVISION = ENV['VAGRANT_PROVISION'] || "./scripts/provision/vagrant"
+MOUNT_TYPE = ENV['MOUNT_TYPE'] || "nfs"
 
 # --- Rules for /etc/sudoers to avoid password entry configuring NFS:
 # %admin	ALL = (root) NOPASSWD: /usr/bin/sed -E -e * -ibak /etc/exports
@@ -14,61 +15,36 @@ VAGRANT_PROVISION = ENV['VAGRANT_PROVISION'] || "./scripts/vagrant-provision"
 # %admin	ALL = (root) NOPASSWD: /sbin/nfsd restart
 # --- May need to add terminal to System Preferences -> Security & Privacy -> Privacy -> Full Disk Access
 
-# --- Check for missing plugins
-required_plugins = %w( vagrant-alpine vagrant-timezone )
-plugin_installed = false
-required_plugins.each do |plugin|
-  unless Vagrant.has_plugin?(plugin)
-    system "vagrant plugin install #{plugin}"
-    plugin_installed = true
-  end
+def provision(vm, node_num)
+  node_os = (ENV["OS_#{node_num}"] || OS)
+  vm.box = (ENV["BOX_#{node_num}"] || ENV["BOX"] || "#{BOX_REPO}/#{node_os}")
+  vm.hostname = "#{PROJECT}-#{node_num}-#{node_os}"
+  vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}"
+  vm.provision "shell",
+      path: VAGRANT_PROVISION,
+      env: { 'HOME' => HOME, 'GOPATH' => ENV['GOPATH'], 'BOX' => vm.box }
 end
-# --- If new plugins installed, restart Vagrant process
-if plugin_installed === true
-  exec "vagrant #{ARGV.join' '}"
-end
-
-provision = <<SCRIPT
-# --- Use system gopath if available
-export GOPATH=#{ENV['GOPATH']}
-# --- Default to root user for vagrant ssh
-cat <<\\EOF >/etc/profile.d/root.sh
-[ $EUID -ne 0 ] && exec sudo -i
-EOF
-# --- Set home to current directory
-cat <<\\EOF >/etc/profile.d/home.sh
-export HOME="#{HOME}" && cd
-EOF
-. /etc/profile.d/home.sh
-# --- Run vagrant provision script if available
-if [ ! -x #{VAGRANT_PROVISION} ]; then
-  echo 'WARNING: Unable to execute provision script "#{VAGRANT_PROVISION}"'
-  exit
-fi
-echo "running '#{VAGRANT_PROVISION}'..." && \
-  #{VAGRANT_PROVISION} && \
-  echo "finished '#{VAGRANT_PROVISION}'!"
-SCRIPT
 
 Vagrant.configure("2") do |config|
+
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
     v.customize ["modifyvm", :id, "--audio", "none"]
   end
-
-  config.vm.box = BOX
-  config.vm.hostname = PROJECT
+  if Vagrant.has_plugin?("vagrant-timezone")
+    config.timezone.value = :host
+  end
   config.vm.synced_folder ".", HOME, type: MOUNT_TYPE
-  config.vm.provision "shell", inline: provision
-  config.timezone.value = :host
 
-  config.vm.network "private_network", ip: "#{NETWORK_PREFIX}.100" if NUM_NODES==0
-
-  (1..NUM_NODES).each do |i|
-    config.vm.define ".#{i}" do |node|
-      node.vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+i}"
-      node.vm.hostname = "#{PROJECT}-#{i}"
+  if NUM_NODES==0
+    provision(config.vm, 0)
+  else
+    (1..NUM_NODES).each do |i|
+      config.vm.define ".#{i}" do |node|
+        provision(node.vm, i)
+      end
     end
   end
+
 end

contrib/ansible/README.md

@@ -40,7 +40,7 @@ node
 Start provisioning of the cluster using the following command:
 
 ```
-ansible-playbook site.yml
+ansible-playbook site.yml -i inventory/hosts.ini
 ```
 
 ## Kubeconfig

contrib/ansible/group_vars/all.yml → contrib/ansible/inventory/group_vars/all.yml

@@ -2,3 +2,4 @@ k3s_version: v0.8.1
 ansible_user: debian
 systemd_dir: /etc/systemd/system
 master_ip: "{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}"
+extra_server_args: ""

contrib/ansible/roles/k3s/master/tasks/main.yml

@@ -46,22 +46,21 @@
 
 - name: Create directory .kube
   file:
-    path: /home/{{ ansible_user }}/.kube
+    path: ~{{ ansible_user }}/.kube
     state: directory
     owner: "{{ ansible_user }}"
 
 - name: Copy config file to user home directory
   copy:
     src: /etc/rancher/k3s/k3s.yaml
-    dest: /home/{{ ansible_user }}/.kube/config
+    dest: ~{{ ansible_user }}/.kube/config
     remote_src: yes
     owner: "{{ ansible_user }}"
 
 - name: Replace https://localhost:6443 by https://master-pi:6443
-  replace:
-    path: /home/{{ ansible_user }}/.kube/config
-    regexp: 'https://localhost:6443'
-    replace: 'https://{{master_ip}}:6443'
+  command: k3s kubectl config set-cluster default
+    --server=https://{{ master_ip }}:6443
+    --kubeconfig ~{{ ansible_user }}/.kube/config
 
 - name: Create kubectl symlink
   file:

contrib/ansible/roles/k3s/master/templates/k3s.service.j2

@@ -2,16 +2,19 @@
 Description=Lightweight Kubernetes
 Documentation=https://k3s.io
 After=network-online.target
+
 [Service]
+Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s server
+ExecStart=/usr/local/bin/k3s server {{ extra_server_args | default("") }}
 KillMode=process
 Delegate=yes
 LimitNOFILE=infinity
 LimitNPROC=infinity
 LimitCORE=infinity
 TasksMax=infinity
+TimeoutStartSec=infinity
 Restart=always
 RestartSec=5s
 

contrib/ansible/roles/k3s/node/templates/k3s.service.j2

@@ -2,7 +2,11 @@
 Description=Lightweight Kubernetes
 Documentation=https://k3s.io
 After=network-online.target
+
 [Service]
+Type=notify
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
 ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['master'][0]]['token'] }}
 KillMode=process
 Delegate=yes
@@ -10,6 +14,7 @@ LimitNOFILE=infinity
 LimitNPROC=infinity
 LimitCORE=infinity
 TasksMax=infinity
+TimeoutStartSec=infinity
 Restart=always
 RestartSec=5s
 

go.mod

@@ -7,9 +7,9 @@ replace (
 	github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20181101203652-af5082808c83
 	github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601
 	github.com/containerd/console => github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
-	github.com/containerd/containerd => github.com/rancher/containerd v1.3.3-k3s1
+	github.com/containerd/containerd => github.com/rancher/containerd v1.3.3-k3s2
 	github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02
-	github.com/containerd/cri => github.com/rancher/cri v1.3.0-k3s.3
+	github.com/containerd/cri => github.com/rancher/cri v1.3.0-k3s.4
 	github.com/containerd/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
 	github.com/containerd/go-runc => github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
 	github.com/containerd/typeurl => github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
@@ -82,7 +82,6 @@ require (
 	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
-	github.com/gofrs/flock v0.7.1 // indirect
 	github.com/gogo/googleapis v1.3.0 // indirect
 	github.com/google/tcpproxy v0.0.0-20180808230851-dfa16c61dad2
 	github.com/gorilla/mux v1.7.3
@@ -95,6 +94,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.13.0
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/opencontainers/runc v1.0.0-rc9
+	github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52
 	github.com/pkg/errors v0.8.1
 	github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect
 	github.com/rancher/dynamiclistener v0.2.0
@@ -103,11 +103,10 @@ require (
 	github.com/rancher/remotedialer v0.2.0
 	github.com/rancher/wrangler v0.4.0
 	github.com/rancher/wrangler-api v0.4.0
-	github.com/rootless-containers/rootlesskit v0.6.0
+	github.com/rootless-containers/rootlesskit v0.7.2
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/pflag v1.0.5
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
-	github.com/theckman/go-flock v0.7.1 // indirect
 	github.com/urfave/cli v1.22.2
 	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
 	golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a
@@ -124,6 +123,5 @@ require (
 	k8s.io/component-base v0.0.0
 	k8s.io/cri-api v0.0.0
 	k8s.io/klog v1.0.0
-	k8s.io/kubelet v0.0.0
 	k8s.io/kubernetes v1.16.0
 )

go.sum

@@ -489,7 +489,9 @@ github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/insomniacslk/dhcp v0.0.0-20190712084813-dc1a53400564/go.mod h1:CfMdguCK66I5DAUJgGKyNz8aB6vO5dZzkm9Xep6WGvw=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
+github.com/jamescun/tuntap v0.0.0-20190712092105-cb1fb277045c/go.mod h1:zzwpsgcYhzzIP5WyF8g9ivCv38cY9uAV9Gu0m3lThhE=
 github.com/jefferai/jsonx v1.0.0/go.mod h1:OGmqmi2tTeI/PS+qQfBDToLHHJIy/RMp24fPo8vFvoQ=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jetstack/cert-manager v0.7.2/go.mod h1:nbddmhjWxYGt04bxvwVGUSeLhZ2PCyNvd7MpXdq+yWY=
@@ -614,6 +616,7 @@ github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:F
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/vpnkit v0.3.1-0.20190720080441-7dd3dcce7d3d/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -705,10 +708,10 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:
 github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
 github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 h1:83l9gPhYtgxODlZKU0Odq4pQuDcMZEVgAh364+PV3OU=
 github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8/go.mod h1:4XHkfaUj+URzGO9sohoAgt2V9Y8nIW7fugpu0E6gShk=
-github.com/rancher/containerd v1.3.3-k3s1 h1:j8NGZdXKsZd2ne0XQg6OBfMJ/NkY/Qri6QhscGrJp2M=
-github.com/rancher/containerd v1.3.3-k3s1/go.mod h1:ZMfzmqce2Z+QSEqdHMfeJs1TZ/UeJ1aDrazjpQT4ehM=
-github.com/rancher/cri v1.3.0-k3s.3 h1:j/Sq2LMyg6gBn2MS1j5dEudpdL+UYVH7ubbewUCXkS0=
-github.com/rancher/cri v1.3.0-k3s.3/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY=
+github.com/rancher/containerd v1.3.3-k3s2 h1:RZr+TqFt7+YsrSYkyytlhW4HmneWeFNM7IymNOoGW6A=
+github.com/rancher/containerd v1.3.3-k3s2/go.mod h1:ZMfzmqce2Z+QSEqdHMfeJs1TZ/UeJ1aDrazjpQT4ehM=
+github.com/rancher/cri v1.3.0-k3s.4 h1:BXER8109dxgNw4qq8HHOCJ+3sHO+9AA1cwZTOLlqoTo=
+github.com/rancher/cri v1.3.0-k3s.4/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY=
 github.com/rancher/cri-tools v1.17.0-k3s1 h1:jfu97FowbraTDc7b6fxWtO+dq+DU2oW+ABBQSEFiRb0=
 github.com/rancher/cri-tools v1.17.0-k3s1/go.mod h1:bRTZttsvk+nCG8tSFs8D6UUx8CkMXR5TAsRLS0fXAqI=
 github.com/rancher/dynamiclistener v0.2.0 h1:KucYwJXVVGhZ/NndfMCeQoCafT/VN7kvqSGgmlX8Lxk=
@@ -778,8 +781,8 @@ github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfm
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rootless-containers/rootlesskit v0.6.0 h1:L7DxVAlaNhg4M/+i2GCl24kRkXO5q81C/lu4jlMz3bE=
-github.com/rootless-containers/rootlesskit v0.6.0/go.mod h1:HO7iU3+dH2N6yQL4DcUGQpzVZ7e7VYWNdtTAOR/P3FM=
+github.com/rootless-containers/rootlesskit v0.7.2 h1:gcWQ9/GN98ne1AqnoeOgQ8e6qpKd3BuB4ug+2h95Fr0=
+github.com/rootless-containers/rootlesskit v0.7.2/go.mod h1:r9YL5mKRIdnwcYk4G8E5CSc9MDeFtgYmhfE4CSvDGYA=
 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c h1:ht7N4d/B7Ezf58nvMNVF3OlvDlz9pp+WHVcRNS0nink=
 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -859,6 +862,7 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/u-root/u-root v5.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
@@ -966,6 +970,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1012,6 +1017,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

install.sh

@@ -26,6 +26,9 @@ set -e
 #     If set to 'skip' will not create symlinks, 'force' will overwrite,
 #     default will symlink if command does not exist in path.
 #
+#   - INSTALL_K3S_SKIP_ENABLE
+#     If set to true will not enable or start k3s service.
+#
 #   - INSTALL_K3S_SKIP_START
 #     If set to true will not start k3s service.
 #
@@ -166,11 +169,6 @@ setup_env() {
             ${invalid_chars}"
     fi
 
-    # --- set related files from system name ---
-    SERVICE_K3S=${SYSTEM_NAME}.service
-    UNINSTALL_K3S_SH=${SYSTEM_NAME}-uninstall.sh
-    KILLALL_K3S_SH=k3s-killall.sh
-
     # --- use sudo if we are not already root ---
     SUDO=sudo
     if [ $(id -u) -eq 0 ]; then
@@ -202,6 +200,11 @@ setup_env() {
         SYSTEMD_DIR=/etc/systemd/system
     fi
 
+    # --- set related files from system name ---
+    SERVICE_K3S=${SYSTEM_NAME}.service
+    UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh}
+    KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh}
+
     # --- use service or environment location depending on systemd/openrc ---
     if [ "${HAS_SYSTEMD}" = true ]; then
         FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S}
@@ -396,7 +399,7 @@ setup_binary() {
     $SUDO chown root:root ${TMP_BIN}
     $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s
 
-    if command -v getenforce > /dev/null 2>&1; then
+    if command -v getenforce >/dev/null 2>&1; then
         if [ "Disabled" != $(getenforce) ]; then
 	    info 'SELinux is enabled, setting permissions'
 	    if ! $SUDO semanage fcontext -l | grep "${BIN_DIR}/k3s" > /dev/null 2>&1; then
@@ -439,7 +442,7 @@ create_symlinks() {
 
     for cmd in kubectl crictl ctr; do
         if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then
-            which_cmd=$(which ${cmd} || true)
+            which_cmd=$(which ${cmd} 2>/dev/null || true)
             if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then
                 info "Creating ${BIN_DIR}/${cmd} symlink to k3s"
                 $SUDO ln -sf k3s ${BIN_DIR}/${cmd}
@@ -455,13 +458,13 @@ create_symlinks() {
 # --- create killall script ---
 create_killall() {
     [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return
-    info "Creating killall script ${BIN_DIR}/${KILLALL_K3S_SH}"
-    $SUDO tee ${BIN_DIR}/${KILLALL_K3S_SH} >/dev/null << \EOF
+    info "Creating killall script ${KILLALL_K3S_SH}"
+    $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF
 #!/bin/sh
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 
 for bin in /var/lib/rancher/k3s/data/**/bin/; do
-    [ -d $bin ] && export PATH=$bin:$PATH
+    [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux
 done
 
 set -x
@@ -499,7 +502,7 @@ killtree() {
 }
 
 getshims() {
-    lsof | sed -e 's/^[^0-9]*//g; s/  */\t/g' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 | sort -n -u
+    ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
 }
 
 killtree $({ set +x; } 2>/dev/null; getshims; set -x)
@@ -534,20 +537,20 @@ ip link delete flannel.1
 rm -rf /var/lib/cni/
 iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
 EOF
-    $SUDO chmod 755 ${BIN_DIR}/${KILLALL_K3S_SH}
-    $SUDO chown root:root ${BIN_DIR}/${KILLALL_K3S_SH}
+    $SUDO chmod 755 ${KILLALL_K3S_SH}
+    $SUDO chown root:root ${KILLALL_K3S_SH}
 }
 
 # --- create uninstall script ---
 create_uninstall() {
     [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return
-    info "Creating uninstall script ${BIN_DIR}/${UNINSTALL_K3S_SH}"
-    $SUDO tee ${BIN_DIR}/${UNINSTALL_K3S_SH} >/dev/null << EOF
+    info "Creating uninstall script ${UNINSTALL_K3S_SH}"
+    $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF
 #!/bin/sh
 set -x
 [ \$(id -u) -eq 0 ] || exec sudo \$0 \$@
 
-${BIN_DIR}/${KILLALL_K3S_SH}
+${KILLALL_K3S_SH}
 
 if which systemctl; then
     systemctl disable ${SYSTEM_NAME}
@@ -562,7 +565,7 @@ rm -f ${FILE_K3S_SERVICE}
 rm -f ${FILE_K3S_ENV}
 
 remove_uninstall() {
-    rm -f ${BIN_DIR}/${UNINSTALL_K3S_SH}
+    rm -f ${UNINSTALL_K3S_SH}
 }
 trap remove_uninstall EXIT
 
@@ -581,10 +584,10 @@ rm -rf /etc/rancher/k3s
 rm -rf /var/lib/rancher/k3s
 rm -rf /var/lib/kubelet
 rm -f ${BIN_DIR}/k3s
-rm -f ${BIN_DIR}/${KILLALL_K3S_SH}
+rm -f ${KILLALL_K3S_SH}
 EOF
-    $SUDO chmod 755 ${BIN_DIR}/${UNINSTALL_K3S_SH}
-    $SUDO chown root:root ${BIN_DIR}/${UNINSTALL_K3S_SH}
+    $SUDO chmod 755 ${UNINSTALL_K3S_SH}
+    $SUDO chown root:root ${UNINSTALL_K3S_SH}
 }
 
 # --- disable current service if loaded --
@@ -646,6 +649,7 @@ create_openrc_service_file() {
 
 depend() {
     after network-online
+    want cgroups
 }
 
 start_pre() {
@@ -717,6 +721,8 @@ openrc_start() {
 
 # --- startup systemd or openrc service ---
 service_enable_and_start() {
+    [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return
+
     [ "${HAS_SYSTEMD}" = true ] && systemd_enable
     [ "${HAS_OPENRC}" = true ] && openrc_enable
 

package/k3s.spec

+# vim: sw=4:ts=4:et
+
+%define install_path  /usr/bin
+%define util_path     %{_datadir}/k3s
+%define install_sh    %{util_path}/.install.sh
+%define uninstall_sh  %{util_path}/.uninstall.sh
+
+Name:    k3s
+Version: %{k3s_version}
+Release: %{k3s_release}%{?dist}
+Summary: Lightweight Kubernetes
+
+Group:   System Environment/Base		
+License: ASL 2.0
+URL:     http://k3s.io
+
+BuildRequires: systemd
+Requires(post): k3s-selinux >= %{k3s_policyver}
+
+%description
+The certified Kubernetes distribution built for IoT & Edge computing.
+
+%install
+install -d %{buildroot}%{install_path}
+install dist/artifacts/%{k3s_binary} %{buildroot}%{install_path}/k3s
+install -d %{buildroot}%{util_path}
+install install.sh %{buildroot}%{install_sh}
+
+%post
+# do not run install script on upgrade
+echo post-install args: $@
+if [ $1 == 1 ]; then
+    INSTALL_K3S_BIN_DIR=%{install_path} \
+    INSTALL_K3S_SKIP_DOWNLOAD=true \
+    INSTALL_K3S_SKIP_ENABLE=true \
+    UNINSTALL_K3S_SH=%{uninstall_sh} \
+        %{install_sh}
+fi
+%systemd_post k3s.service
+exit 0
+
+%postun
+echo post-uninstall args: $@
+# do not run uninstall script on upgrade
+if [ $1 == 0 ]; then
+    %{uninstall_sh}
+    rm -rf %{util_path}
+fi
+exit 0
+
+%files
+%{install_path}/k3s
+%{install_sh}
+
+%changelog
+* Mon Mar 2 2020 Erik Wilson <erik@rancher.com> 0.1-1
+- Initial version

pkg/agent/config/config.go

@@ -397,6 +397,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
+		DisableSELinux:           envInfo.DisableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
 		FlannelBackend:           controlConfig.FlannelBackend,
 	}
@@ -474,6 +475,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM
 	nodeConfig.AgentConfig.DisableNPC = controlConfig.DisableNPC
 	nodeConfig.AgentConfig.Rootless = envInfo.Rootless
+	nodeConfig.DisableSELinux = envInfo.DisableSELinux
 
 	return nodeConfig, nil
 }

pkg/agent/containerd/command.go

+// +build !windows
+
+package containerd
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func addDeathSig(cmd *exec.Cmd) {
+	// not supported in this OS
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: syscall.SIGKILL,
+	}
+}

pkg/agent/containerd/command_windows.go

+package containerd
+
+import "os/exec"
+
+func addDeathSig(_ *exec.Cmd) {
+	// not supported in this OS
+}

pkg/agent/containerd/containerd.go

@@ -8,13 +8,13 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"syscall"
 	"time"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/natefinch/lumberjack"
 	"github.com/opencontainers/runc/libcontainer/system"
+	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/agent/templates"
 	util2 "github.com/rancher/k3s/pkg/agent/util"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -66,9 +66,7 @@ func Run(ctx context.Context, cfg *config.Node) error {
 		cmd := exec.Command(args[0], args[1:]...)
 		cmd.Stdout = stdOut
 		cmd.Stderr = stdErr
-		cmd.SysProcAttr = &syscall.SysProcAttr{
-			Pdeathsig: syscall.SIGKILL,
-		}
+		addDeathSig(cmd)
 		if err := cmd.Run(); err != nil {
 			fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
 		}
@@ -170,6 +168,22 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		PrivateRegistryConfig: privRegistries,
 	}
 
+	selEnabled, selConfigured, err := selinuxStatus()
+	if err != nil {
+		return errors.Wrap(err, "failed to detect selinux")
+	}
+	if cfg.DisableSELinux {
+		containerdConfig.SELinuxEnabled = false
+		if selEnabled {
+			logrus.Warn("SELinux is enabled for system but has been disabled for containerd by override")
+		}
+	} else {
+		containerdConfig.SELinuxEnabled = selEnabled
+	}
+	if containerdConfig.SELinuxEnabled && !selConfigured {
+		logrus.Warnf("SELinux is enabled for k3s but process is not running in context '%s', k3s-selinux policy may need to be applied", SELinuxContextType)
+	}
+
 	containerdTemplateBytes, err := ioutil.ReadFile(cfg.Containerd.Template)
 	if err == nil {
 		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)

pkg/agent/containerd/selinux.go

+package containerd
+
+import (
+	"github.com/opencontainers/selinux/go-selinux"
+)
+
+const (
+	SELinuxContextType = "container_runtime_t"
+)
+
+func selinuxStatus() (bool, bool, error) {
+	if !selinux.GetEnabled() {
+		return false, false, nil
+	}
+
+	label, err := selinux.CurrentLabel()
+	if err != nil {
+		return true, false, err
+	}
+
+	ctx, err := selinux.NewContext(label)
+	if err != nil {
+		return true, false, err
+	}
+
+	return true, ctx["type"] == SELinuxContextType, nil
+}

pkg/agent/netpol/netpol_windows.go

+package netpol
+
+import (
+	"context"
+	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
+)
+
+func Run(ctx context.Context, nodeConfig *daemonconfig.Node) error {
+	panic("Netpol is not supported on windows ensure to pass --disable-network-policy")
+}

pkg/agent/netpol/network_policy.go

+// +build !windows
+
 package netpol
 
 import (

pkg/agent/netpol/network_policy_controller.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
 // - modified from https://github.com/cloudnativelabs/kube-router/blob/d6f9f31a7b/pkg/controllers/netpol/network_policy_controller.go
 
+// +build !windows
+
 package netpol
 
 import (

pkg/agent/netpol/utils.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
 // - modified from https://github.com/cloudnativelabs/kube-router/tree/d6f9f31a7b/pkg/utils
 
+// +build !windows
+
 package netpol
 
 import (

pkg/agent/run.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/nodeconfig"
 	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -71,7 +72,7 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 		}
 	}
 
-	if err := syncLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
+	if err := configureNode(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
 		return err
 	}
 
@@ -155,7 +156,7 @@ func validate() error {
 	return nil
 }
 
-func syncLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
+func configureNode(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
 	for {
 		node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{})
 		if err != nil {
@@ -171,8 +172,16 @@ func syncLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.N
 			newLabels, updateAddresses = updateAddressLabels(agentConfig, newLabels)
 		}
 
+		// inject node config
+		updateNode, err := nodeconfig.SetNodeConfigAnnotations(node)
+		if err != nil {
+			return err
+		}
 		if updateAddresses || updateMutables {
 			node.Labels = newLabels
+			updateNode = true
+		}
+		if updateNode {
 			if _, err := nodes.Update(node); err != nil {
 				logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
 				select {

pkg/agent/syssetup/setup.go

+// +build !windows
+
 package syssetup
 
 import (

pkg/agent/syssetup/setup_windows.go

+package syssetup
+
+func Configure() {}

pkg/agent/templates/templates.go

@@ -10,6 +10,7 @@ import (
 type ContainerdConfig struct {
 	NodeConfig            *config.Node
 	IsRunningInUserNS     bool
+	SELinuxEnabled        bool
 	PrivateRegistryConfig *Registry
 }
 
@@ -20,6 +21,7 @@ const ContainerdConfigTemplate = `
 [plugins.cri]
   stream_server_address = "127.0.0.1"
   stream_server_port = "10010"
+  enable_selinux = {{ .SELinuxEnabled }}
 
 {{- if .IsRunningInUserNS }}
   disable_cgroup = true

pkg/cli/cmds/agent.go

@@ -28,6 +28,7 @@ type Agent struct {
 	Rootless                 bool
 	RootlessAlreadyUnshared  bool
 	WithNodeID               bool
+	DisableSELinux           bool
 	AgentShared
 	ExtraKubeletArgs   cli.StringSlice
 	ExtraKubeProxyArgs cli.StringSlice
@@ -127,6 +128,12 @@ var (
 		Usage: "(agent/node) Registering and starting kubelet with set of labels",
 		Value: &AgentConfig.Labels,
 	}
+	DisableSELinuxFlag = cli.BoolFlag{
+		Name:        "disable-selinux",
+		Usage:       "(agent/node) Disable SELinux in containerd if currently enabled",
+		Hidden:      true,
+		Destination: &AgentConfig.DisableSELinux,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -169,6 +176,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			NodeLabels,
 			NodeTaints,
 			DockerFlag,
+			DisableSELinuxFlag,
 			CRIEndpointFlag,
 			PauseImageFlag,
 			PrivateRegistryFlag,

pkg/cli/cmds/server.go

@@ -42,6 +42,7 @@ type Server struct {
 	DisableNPC               bool
 	ClusterInit              bool
 	ClusterReset             bool
+	EncryptSecrets           bool
 }
 
 var ServerConfig Server
@@ -192,8 +193,8 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.DefaultLocalStoragePath,
 			},
 			cli.StringSliceFlag{
-				Name:  "no-deploy",
-				Usage: "(components) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
+				Name:  "disable",
+				Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
 			},
 			cli.BoolFlag{
 				Name:        "disable-scheduler",
@@ -215,6 +216,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			NodeLabels,
 			NodeTaints,
 			DockerFlag,
+			DisableSELinuxFlag,
 			CRIEndpointFlag,
 			PauseImageFlag,
 			PrivateRegistryFlag,
@@ -262,10 +264,19 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				EnvVar:      "K3S_CLUSTER_RESET",
 				Destination: &ServerConfig.ClusterReset,
 			},
+			cli.BoolFlag{
+				Name:        "secrets-encryption",
+				Usage:       "(experimental) Enable Secret encryption at rest",
+				Destination: &ServerConfig.EncryptSecrets,
+			},
 
 			// Hidden/Deprecated flags below
 
 			FlannelFlag,
+			cli.StringSliceFlag{
+				Name:  "no-deploy",
+				Usage: "(deprecated) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
+			},
 			cli.StringFlag{
 				Name:        "cluster-secret",
 				Usage:       "(deprecated) use --token",

pkg/cli/server/server.go

@@ -100,6 +100,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
 	serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
 	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
+	serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
 
 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
@@ -150,19 +151,21 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		serverConfig.ControlConfig.DefaultLocalStoragePath = cfg.DefaultLocalStoragePath
 	}
 
-	noDeploys := make([]string, 0)
+	serverConfig.ControlConfig.Skips = map[string]bool{}
 	for _, noDeploy := range app.StringSlice("no-deploy") {
-		for _, splitNoDeploy := range strings.Split(noDeploy, ",") {
-			noDeploys = append(noDeploys, splitNoDeploy)
+		for _, v := range strings.Split(noDeploy, ",") {
+			serverConfig.ControlConfig.Skips[v] = true
 		}
 	}
-
-	for _, noDeploy := range noDeploys {
-		if noDeploy == "servicelb" {
-			serverConfig.DisableServiceLB = true
-			continue
+	serverConfig.ControlConfig.Disables = map[string]bool{}
+	for _, disable := range app.StringSlice("disable") {
+		for _, v := range strings.Split(disable, ",") {
+			serverConfig.ControlConfig.Skips[v] = true
+			serverConfig.ControlConfig.Disables[v] = true
 		}
-		serverConfig.ControlConfig.Skips = append(serverConfig.ControlConfig.Skips, noDeploy)
+	}
+	if serverConfig.ControlConfig.Skips["servicelb"] {
+		serverConfig.DisableServiceLB = true
 	}
 
 	logrus.Info("Starting k3s ", app.App.Version)

pkg/daemons/config/types.go

@@ -25,6 +25,7 @@ type Node struct {
 	Docker                   bool
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
+	DisableSELinux           bool
 	FlannelBackend           string
 	FlannelConf              string
 	FlannelConfOverride      bool
@@ -96,7 +97,8 @@ type Control struct {
 	KubeConfigOutput         string
 	KubeConfigMode           string
 	DataDir                  string
-	Skips                    []string
+	Skips                    map[string]bool
+	Disables                 map[string]bool
 	Datastore                endpoint.Config
 	NoScheduler              bool
 	ExtraAPIArgs             []string
@@ -112,6 +114,7 @@ type Control struct {
 	DisableNPC               bool
 	ClusterInit              bool
 	ClusterReset             bool
+	EncryptSecrets           bool
 
 	BindAddress string
 	SANs        []string
@@ -129,6 +132,7 @@ type ControlRuntimeBootstrap struct {
 	RequestHeaderCA    string
 	RequestHeaderCAKey string
 	IPSECKey           string
+	EncryptionConfig   string
 }
 
 type ControlRuntime struct {

pkg/daemons/control/server.go

@@ -3,9 +3,11 @@ package control
 import (
 	"context"
 	"crypto"
+	cryptorand "crypto/rand"
 	"crypto/x509"
+	b64 "encoding/base64"
+	"encoding/json"
 	"fmt"
-	"html/template"
 	"io/ioutil"
 	"math/rand"
 	"net"
@@ -15,6 +17,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	// registering k3s cloud provider
@@ -30,6 +33,8 @@ import (
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -71,8 +76,9 @@ users:
 )
 
 const (
-	userTokenSize  = 16
+	userTokenSize  = 8
 	ipsecTokenSize = 48
+	aescbcKeySize  = 32
 )
 
 func Server(ctx context.Context, cfg *config.Control) error {
@@ -202,7 +208,9 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
 	argsMap["client-ca-file"] = runtime.ClientCA
 	argsMap["enable-admission-plugins"] = "NodeRestriction"
 	argsMap["anonymous-auth"] = "false"
-
+	if cfg.EncryptSecrets {
+		argsMap["encryption-provider-config"] = runtime.EncryptionConfig
+	}
 	args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
 
 	command := app.NewAPIServerCommand(ctx.Done())
@@ -309,6 +317,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
+	if config.EncryptSecrets {
+		runtime.EncryptionConfig = path.Join(config.DataDir, "cred", "encryption-config.json")
+	}
+
 	cluster := cluster.New(config)
 
 	if err := cluster.Join(ctx); err != nil {
@@ -331,6 +343,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
+	if err := genEncryptionConfig(config, runtime); err != nil {
+		return err
+	}
+
 	if err := readTokens(runtime); err != nil {
 		return err
 	}
@@ -862,3 +878,51 @@ func promise(f func() error) <-chan error {
 	}()
 	return c
 }
+
+func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
+	if !controlConfig.EncryptSecrets {
+		return nil
+	}
+	if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 {
+		return nil
+	}
+
+	aescbcKey := make([]byte, aescbcKeySize, aescbcKeySize)
+	_, err := cryptorand.Read(aescbcKey)
+	if err != nil {
+		return err
+	}
+	encodedKey := b64.StdEncoding.EncodeToString(aescbcKey)
+
+	encConfig := apiserverconfigv1.EncryptionConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "EncryptionConfiguration",
+			APIVersion: "apiserver.config.k8s.io/v1",
+		},
+		Resources: []apiserverconfigv1.ResourceConfiguration{
+			{
+				Resources: []string{"secrets"},
+				Providers: []apiserverconfigv1.ProviderConfiguration{
+					{
+						AESCBC: &apiserverconfigv1.AESConfiguration{
+							Keys: []apiserverconfigv1.Key{
+								{
+									Name:   "aescbckey",
+									Secret: encodedKey,
+								},
+							},
+						},
+					},
+					{
+						Identity: &apiserverconfigv1.IdentityConfiguration{},
+					},
+				},
+			},
+		},
+	}
+	jsonfile, err := json.Marshal(encConfig)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
+}

pkg/daemons/control/tunnel.go

@@ -21,14 +21,18 @@ func setupTunnel() http.Handler {
 }
 
 func setupProxyDialer(tunnelServer *remotedialer.Server) {
-	app.DefaultProxyDialerFn = utilnet.DialFunc(func(_ context.Context, network, address string) (net.Conn, error) {
+	app.DefaultProxyDialerFn = utilnet.DialFunc(func(ctx context.Context, network, address string) (net.Conn, error) {
 		_, port, _ := net.SplitHostPort(address)
 		addr := "127.0.0.1"
 		if port != "" {
 			addr += ":" + port
 		}
 		nodeName, _ := kv.Split(address, ":")
-		return tunnelServer.Dial(nodeName, 15*time.Second, "tcp", addr)
+		if tunnelServer.HasSession(nodeName) {
+			return tunnelServer.Dial(nodeName, 15*time.Second, "tcp", addr)
+		}
+		var d net.Dialer
+		return d.DialContext(ctx, network, address)
 	})
 }
 

pkg/deploy/controller.go

@@ -22,8 +22,10 @@ import (
 	"github.com/rancher/wrangler/pkg/objectset"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
 )
 
@@ -32,12 +34,13 @@ const (
 	startKey = "_start_"
 )
 
-func WatchFiles(ctx context.Context, apply apply.Apply, addons v1.AddonController, bases ...string) error {
+func WatchFiles(ctx context.Context, apply apply.Apply, addons v1.AddonController, disables map[string]bool, bases ...string) error {
 	w := &watcher{
 		apply:      apply,
 		addonCache: addons.Cache(),
 		addons:     addons,
 		bases:      bases,
+		disables:   disables,
 	}
 
 	addons.Enqueue("", startKey)
@@ -56,6 +59,7 @@ type watcher struct {
 	addonCache v1.AddonCache
 	addons     v1.AddonClient
 	bases      []string
+	disables   map[string]bool
 }
 
 func (w *watcher) start(ctx context.Context) {
@@ -111,6 +115,12 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 
 	var errs []error
 	for _, path := range keys {
+		if shouldDisableService(base, path, w.disables) {
+			if err := w.delete(path); err != nil {
+				errs = append(errs, errors2.Wrapf(err, "failed to delete %s", path))
+			}
+			continue
+		}
 		if skipFile(files[path].Name(), skips) {
 			continue
 		}
@@ -162,6 +172,39 @@ func (w *watcher) deploy(path string, compareChecksum bool) error {
 	return err
 }
 
+func (w *watcher) delete(path string) error {
+	name := name(path)
+	addon, err := w.addon(name)
+	if err != nil {
+		return err
+	}
+
+	// ensure that the addon is completely removed before deleting the objectSet,
+	// so return when err == nil, otherwise pods may get stuck terminating
+	if err := w.addons.Delete(addon.Namespace, addon.Name, &metav1.DeleteOptions{}); err == nil || !errors.IsNotFound(err) {
+		return err
+	}
+
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	objectSet, err := objectSet(content)
+	if err != nil {
+		return err
+	}
+	var gvk []schema.GroupVersionKind
+	for k := range objectSet.ObjectsByGVK() {
+		gvk = append(gvk, k)
+	}
+	// apply an empty set with owner & gvk data to delete
+	if err := w.apply.WithOwner(&addon).WithGVK(gvk...).Apply(nil); err != nil {
+		return err
+	}
+
+	return os.Remove(path)
+}
+
 func (w *watcher) addon(name string) (v12.Addon, error) {
 	addon, err := w.addonCache.Get(ns, name)
 	if errors.IsNotFound(err) {
@@ -269,3 +312,28 @@ func skipFile(fileName string, skips map[string]bool) bool {
 		return true
 	}
 }
+
+func shouldDisableService(base, fileName string, disables map[string]bool) bool {
+	relFile := strings.TrimPrefix(fileName, base)
+	namePath := strings.Split(relFile, string(os.PathSeparator))
+	for i := 1; i < len(namePath); i++ {
+		subPath := filepath.Join(namePath[0:i]...)
+		if disables[subPath] {
+			return true
+		}
+	}
+	switch {
+	case strings.HasSuffix(fileName, ".json"):
+	case strings.HasSuffix(fileName, ".yml"):
+	case strings.HasSuffix(fileName, ".yaml"):
+	default:
+		return false
+	}
+	baseFile := filepath.Base(fileName)
+	suffix := filepath.Ext(baseFile)
+	baseName := strings.TrimSuffix(baseFile, suffix)
+	if disables[baseName] {
+		return true
+	}
+	return false
+}

pkg/deploy/stage.go

@@ -11,12 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func Stage(dataDir string, templateVars map[string]string, skipList []string) error {
-	skips := map[string]bool{}
-	for _, skip := range skipList {
-		skips[skip] = true
-	}
-
+func Stage(dataDir string, templateVars map[string]string, skips map[string]bool) error {
 staging:
 	for _, name := range AssetNames() {
 		nameNoExtension := strings.TrimSuffix(name, filepath.Ext(name))

pkg/nodeconfig/nodeconfig.go

+package nodeconfig
+
+import (
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	NodeArgsAnnotation       = "k3s.io/node-args"
+	NodeEnvAnnotation        = "k3s.io/node-env"
+	NodeConfigHashAnnotation = "k3s.io/node-config-hash"
+)
+
+func getNodeArgs() (string, error) {
+	nodeArgsList := []string{}
+	for _, arg := range os.Args[1:] {
+		if strings.Contains(arg, "=") {
+			parsedArg := strings.SplitN(arg, "=", 2)
+			nodeArgsList = append(nodeArgsList, parsedArg...)
+			continue
+		}
+		nodeArgsList = append(nodeArgsList, arg)
+	}
+	for i, arg := range nodeArgsList {
+		if isSecret(arg) {
+			if i+1 < len(nodeArgsList) {
+				nodeArgsList[i+1] = ""
+			}
+		}
+	}
+	nodeArgs, err := json.Marshal(nodeArgsList)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to retrieve argument list for node")
+	}
+	return string(nodeArgs), nil
+}
+
+func getNodeEnv() (string, error) {
+	k3sEnv := make(map[string]string)
+	for _, v := range os.Environ() {
+		keyValue := strings.SplitN(v, "=", 2)
+		if strings.HasPrefix(keyValue[0], "K3S_") {
+			k3sEnv[keyValue[0]] = keyValue[1]
+		}
+	}
+	for key := range k3sEnv {
+		if isSecret(key) {
+			k3sEnv[key] = ""
+		}
+	}
+	k3sEnvJSON, err := json.Marshal(k3sEnv)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to retrieve environment map for node")
+	}
+	return string(k3sEnvJSON), nil
+}
+
+func SetNodeConfigAnnotations(node *corev1.Node) (bool, error) {
+	nodeArgs, err := getNodeArgs()
+	if err != nil {
+		return false, err
+	}
+	nodeEnv, err := getNodeEnv()
+	if err != nil {
+		return false, err
+	}
+	h := sha256.New()
+	_, err = h.Write([]byte(nodeArgs + nodeEnv))
+	if err != nil {
+		return false, fmt.Errorf("Failed to hash the node config: %v", err)
+	}
+	if node.Annotations == nil {
+		node.Annotations = make(map[string]string)
+	}
+	configHash := h.Sum(nil)
+	encoded := base32.StdEncoding.EncodeToString(configHash[:])
+	if node.Annotations[NodeConfigHashAnnotation] == encoded {
+		return false, nil
+	}
+	node.Annotations[NodeEnvAnnotation] = nodeEnv
+	node.Annotations[NodeArgsAnnotation] = nodeArgs
+	node.Annotations[NodeConfigHashAnnotation] = encoded
+	return true, nil
+}
+
+func isSecret(key string) bool {
+	secretData := []string{
+		"K3S_TOKEN",
+		"K3S_DATASTORE_ENDPOINT",
+		"K3S_AGENT_TOKEN",
+		"K3S_CLUSTER_SECRET",
+		"--token",
+		"-t",
+		"--agent-token",
+		"--datastore-endpoint",
+		"--cluster-secret",
+	}
+	for _, secret := range secretData {
+		if key == secret {
+			return true
+		}
+	}
+	return false
+}

pkg/nodeconfig/nodeconfig_test.go

+package nodeconfig
+
+import (
+	"os"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var FakeNodeWithNoAnnotation = &corev1.Node{
+	TypeMeta: metav1.TypeMeta{
+		Kind:       "Node",
+		APIVersion: "v1",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "fakeNode-no-annotation",
+	},
+}
+
+var FakeNodeWithAnnotation = &corev1.Node{
+	TypeMeta: metav1.TypeMeta{
+		Kind:       "Node",
+		APIVersion: "v1",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "fakeNode-with-annotation",
+		Annotations: map[string]string{
+			NodeArgsAnnotation:       `["server","--no-flannel"]`,
+			NodeEnvAnnotation:        `{"K3S_NODE_NAME":"fakeNode-with-annotation"}`,
+			NodeConfigHashAnnotation: "LNQOAOIMOQIBRMEMACW7LYHXUNPZADF6RFGOSPIHJCOS47UVUJAA====",
+		},
+	},
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}) {
+	if a != b {
+		t.Fatalf("[ %v != %v ]", a, b)
+	}
+}
+
+func TestSetEmptyNodeConfigAnnotations(t *testing.T) {
+	os.Args = []string{"k3s", "server", "--no-flannel"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-no-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithNoAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, true, nodeUpdated)
+
+	expectedArgs := `["server","--no-flannel"]`
+	actualArgs := FakeNodeWithNoAnnotation.Annotations[NodeArgsAnnotation]
+	assertEqual(t, expectedArgs, actualArgs)
+
+	expectedEnv := `{"K3S_NODE_NAME":"fakeNode-no-annotation"}`
+	actualEnv := FakeNodeWithNoAnnotation.Annotations[NodeEnvAnnotation]
+	assertEqual(t, expectedEnv, actualEnv)
+
+	expectedHash := "MROOIJGRXUZ53BM74K76TZLRXQOLNNBNJBJOY7JJ22EAEUIBW7YA===="
+	actualHash := FakeNodeWithNoAnnotation.Annotations[NodeConfigHashAnnotation]
+	assertEqual(t, expectedHash, actualHash)
+}
+
+func TestSetExistingNodeConfigAnnotations(t *testing.T) {
+	// adding same config
+	os.Args = []string{"k3s", "server", "--no-flannel"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-with-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, false, nodeUpdated)
+}
+
+func TestSetArgsWithEqual(t *testing.T) {
+	os.Args = []string{"k3s", "server", "--no-flannel", "--write-kubeconfig-mode=777"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-with-no-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithNoAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, true, nodeUpdated)
+	expectedArgs := `["server","--no-flannel","--write-kubeconfig-mode","777"]`
+	actualArgs := FakeNodeWithNoAnnotation.Annotations[NodeArgsAnnotation]
+	assertEqual(t, expectedArgs, actualArgs)
+}

pkg/passwd/passwd.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/rancher/k3s/pkg/token"
+	"github.com/rancher/k3s/pkg/util"
 )
 
 type entry struct {
@@ -133,17 +134,21 @@ func (p *Passwd) Write(passwdFile string) error {
 }
 
 func writePasswords(passwdFile string, records [][]string) error {
-	out, err := os.Create(passwdFile + ".tmp")
-	if err != nil {
-		return err
-	}
-	defer out.Close()
+	err := func() error {
+		// ensure to close tmp file before rename for filesystems like NTFS
+		out, err := os.Create(passwdFile + ".tmp")
+		if err != nil {
+			return err
+		}
+		defer out.Close()
 
-	if err := out.Chmod(0600); err != nil {
-		return err
-	}
+		if err := util.SetFileModeForFile(out, 0600); err != nil {
+			return err
+		}
 
-	if err := csv.NewWriter(out).WriteAll(records); err != nil {
+		return csv.NewWriter(out).WriteAll(records)
+	}()
+	if err != nil {
 		return err
 	}
 

pkg/rootless/mounts.go

+// +build !windows
+
 package rootless
 
 import (
@@ -17,6 +19,8 @@ func setupMounts(stateDir string) error {
 		{"/var/run", ""},
 		{"/var/log", filepath.Join(stateDir, "logs")},
 		{"/var/lib/cni", filepath.Join(stateDir, "cni")},
+		{"/var/lib/kubelet", filepath.Join(stateDir, "kubelet")},
+		{"/etc/rancher", filepath.Join(stateDir, "etc", "rancher")},
 	}
 
 	for _, v := range mountMap {

pkg/rootless/rootless.go

+// +build !windows
+
 package rootless
 
 import (
@@ -89,7 +91,8 @@ func createParentOpt(stateDir string) (*parent.Opt, error) {
 	}
 
 	opt := &parent.Opt{
-		StateDir: stateDir,
+		StateDir:    stateDir,
+		CreatePIDNS: true,
 	}
 
 	mtu := 0
@@ -102,7 +105,7 @@ func createParentOpt(stateDir string) (*parent.Opt, error) {
 	if _, err := exec.LookPath(binary); err != nil {
 		return nil, err
 	}
-	opt.NetworkDriver = slirp4netns.NewParentDriver(binary, mtu, ipnet, disableHostLoopback, "")
+	opt.NetworkDriver = slirp4netns.NewParentDriver(binary, mtu, ipnet, disableHostLoopback, "", false, false)
 	opt.PortDriver, err = portbuiltin.NewParentDriver(&logrusDebugWriter{}, stateDir)
 	if err != nil {
 		return nil, err
@@ -130,5 +133,7 @@ func createChildOpt() (*child.Opt, error) {
 	opt.PortDriver = portbuiltin.NewChildDriver(&logrusDebugWriter{})
 	opt.CopyUpDirs = []string{"/etc", "/run", "/var/lib"}
 	opt.CopyUpDriver = tmpfssymlink.NewChildDriver()
+	opt.MountProcfs = true
+	opt.Reaper = true
 	return opt, nil
 }

pkg/rootless/rootless_windows.go

+package rootless
+
+func Rootless(stateDir string) error {
+	panic("Rootless not supported on windows")
+}

pkg/rootlessports/controller.go

+// +build !windows
+
 package rootlessports
 
 import (
@@ -17,7 +19,7 @@ var (
 	all = "_all_"
 )
 
-func Register(ctx context.Context, serviceController coreClients.ServiceController, httpsPort int) error {
+func Register(ctx context.Context, serviceController coreClients.ServiceController, enabled bool, httpsPort int) error {
 	var (
 		err            error
 		rootlessClient client.Client
@@ -41,6 +43,7 @@ func Register(ctx context.Context, serviceController coreClients.ServiceControll
 	}
 
 	h := &handler{
+		enabled:        enabled,
 		rootlessClient: rootlessClient,
 		serviceClient:  serviceController,
 		serviceCache:   serviceController.Cache(),
@@ -54,6 +57,7 @@ func Register(ctx context.Context, serviceController coreClients.ServiceControll
 }
 
 type handler struct {
+	enabled        bool
 	rootlessClient client.Client
 	serviceClient  coreClients.ServiceController
 	serviceCache   coreClients.ServiceCache
@@ -122,6 +126,11 @@ func (h *handler) toBindPorts() (map[int]int, error) {
 	toBindPorts := map[int]int{
 		h.httpsPort: h.httpsPort,
 	}
+
+	if !h.enabled {
+		return toBindPorts, nil
+	}
+
 	for _, svc := range svcs {
 		for _, ingress := range svc.Status.LoadBalancer.Ingress {
 			if ingress.IP == "" {

pkg/rootlessports/controller_windows.go

+package rootlessports
+
+import (
+	"context"
+	coreClients "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+)
+
+func Register(ctx context.Context, serviceController coreClients.ServiceController, httpsPort int) error {
+	panic("Rootless is not supported on windows")
+}

pkg/server/server.go

@@ -24,10 +24,10 @@ import (
 	"github.com/rancher/k3s/pkg/rootlessports"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
+	"github.com/rancher/k3s/pkg/util"
 	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/resolvehome"
-	"github.com/rancher/wrangler/pkg/slice"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
@@ -127,7 +127,7 @@ func startWrangler(ctx context.Context, config *Config) error {
 }
 
 func masterControllers(ctx context.Context, sc *Context, config *Config) error {
-	if !slice.ContainsString(config.ControlConfig.Skips, "coredns") {
+	if !config.ControlConfig.Skips["coredns"] {
 		if err := node.Register(ctx, sc.Core.Core().V1().ConfigMap(), sc.Core.Core().V1().Node()); err != nil {
 			return err
 		}
@@ -152,8 +152,8 @@ func masterControllers(ctx context.Context, sc *Context, config *Config) error {
 		return err
 	}
 
-	if !config.DisableServiceLB && config.Rootless {
-		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.ControlConfig.HTTPSPort)
+	if config.Rootless {
+		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), !config.DisableServiceLB, config.ControlConfig.HTTPSPort)
 	}
 
 	return nil
@@ -176,7 +176,7 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 		return err
 	}
 
-	return deploy.WatchFiles(ctx, sc.Apply, sc.K3s.K3s().V1().Addon(), dataDir)
+	return deploy.WatchFiles(ctx, sc.Apply, sc.K3s.K3s().V1().Addon(), controlConfig.Disables, dataDir)
 }
 
 func HomeKubeConfig(write, rootless bool) (string, error) {
@@ -264,12 +264,12 @@ func writeKubeConfig(certs string, config *Config) error {
 	if config.ControlConfig.KubeConfigMode != "" {
 		mode, err := strconv.ParseInt(config.ControlConfig.KubeConfigMode, 8, 0)
 		if err == nil {
-			os.Chmod(kubeConfig, os.FileMode(mode))
+			util.SetFileModeForPath(kubeConfig, os.FileMode(mode))
 		} else {
 			logrus.Errorf("failed to set %s to mode %s: %v", kubeConfig, os.FileMode(mode), err)
 		}
 	} else {
-		os.Chmod(kubeConfig, os.FileMode(0600))
+		util.SetFileModeForPath(kubeConfig, os.FileMode(0600))
 	}
 
 	if kubeConfigSymlink != kubeConfig {

pkg/util/file.go

+// +build !windows
+
+package util
+
+import (
+	"os"
+)
+
+func SetFileModeForPath(name string, mode os.FileMode) error {
+	return os.Chmod(name, mode)
+}
+
+func SetFileModeForFile(file *os.File, mode os.FileMode) error {
+	return file.Chmod(mode)
+}

pkg/util/file_windows.go

+package util
+
+import (
+	"os"
+)
+
+func SetFileModeForPath(name string, mode os.FileMode) error {
+	return nil
+}
+
+func SetFileModeForFile(file *os.File, mode os.FileMode) error {
+	return nil
+}

scripts/build

@@ -46,12 +46,20 @@ STATIC_SQLITE="
     -extldflags '-static -lm -ldl -lz -lpthread $DQLITE_STATIC_SQLITE'
 "
 TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless $DQLITE_TAGS"
+RUNC_TAGS="apparmor seccomp"
+RUNC_STATIC="static"
+
+if [ "$SELINUX" = "true" ]; then
+    TAGS="$TAGS selinux"
+    RUNC_TAGS="$RUNC_TAGS selinux"
+fi
 
 if [ "$STATIC_BUILD" != "true" ]; then
     STATIC="
 "
     STATIC_SQLITE="
 "
+    RUNC_STATIC=""
 else
     TAGS="static_build libsqlite3 $TAGS"
 fi
@@ -109,13 +117,16 @@ ln -s containerd ./bin/ctr
 # echo Building containerd
 # CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
 echo Building runc
-make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
+rm -f ./vendor/github.com/opencontainers/runc/runc
+make EXTRA_LDFLAGS="-w -s" BUILDTAGS="$RUNC_TAGS" -C ./vendor/github.com/opencontainers/runc $RUNC_STATIC
 cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
 
 echo Building containerd-shim
+rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim
 make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
 cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim ./bin/containerd-shim
 
 echo Building containerd-shim-runc-v2
+rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2
 make -C ./vendor/github.com/containerd/containerd bin/containerd-shim-runc-v2
 cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2 ./bin/containerd-shim-runc-v2

scripts/package-rpm

+#!/bin/bash
+set -e -x
+
+cd $(dirname $0)/..
+
+ARCH=${DRONE_STAGE_ARCH:-$(arch)}
+. ./scripts/version.sh
+
+if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\-[^\+]*)?\+k3s.+$ ]]; then
+  echo "k3s version $VERSION does not match regex for rpm upload"
+  exit 0
+fi
+
+TMPDIR=$(mktemp -d)
+cleanup() {
+  exit_code=$?
+  trap - EXIT INT
+  rm -rf ${TMPDIR}
+  exit ${exit_code}
+}
+trap cleanup EXIT INT
+
+export HOME=${TMPDIR}
+
+BIN_SUFFIX=""
+if [ ${ARCH} = aarch64 ] || [ ${ARCH} = arm64 ]; then
+    BIN_SUFFIX="-arm64"
+elif [ ${ARCH} = armv7l ] || [ ${ARCH} = arm ]; then
+    BIN_SUFFIX="-armhf"
+fi
+
+# capture version of k3s
+k3s_version=$(sed -E -e 's/^v([^-+]*).*$/\1/' <<< $VERSION)
+# capture pre-release and metadata information of k3s
+k3s_release=$(sed -E -e 's/\+k3s/+/; s/\+/-/g; s/^[^-]*//; s/^--/dev-/; s/-+/./g; s/^\.+//; s/\.+$//;' <<< $VERSION)
+# k3s-selinux policy version needed for functionality
+k3s_policyver=0.1-1
+
+rpmbuild \
+  --define "k3s_version ${k3s_version}" \
+  --define "k3s_release ${k3s_release}" \
+  --define "k3s_policyver ${k3s_policyver}" \
+  --define "k3s_binary k3s${BIN_SUFFIX}" \
+  --define "_sourcedir ${PWD}" \
+  --define "_specdir ${PWD}" \
+  --define "_builddir ${PWD}" \
+  --define "_srcrpmdir ${PWD}" \
+  --define "_rpmdir ${PWD}/dist/rpm" \
+  --define "_buildrootdir ${PWD}/.rpm-build" \
+  -bb package/k3s.spec
+
+if ! grep "BEGIN PGP PRIVATE KEY BLOCK" <<<"$PRIVATE_KEY"; then
+  echo "PRIVATE_KEY not defined, skipping rpm sign and upload"
+  exit 0
+fi
+
+cat <<\EOF >~/.rpmmacros 
+%_signature gpg
+%_gpg_name ci@rancher.com
+EOF
+gpg --import - <<<"$PRIVATE_KEY"
+
+expect <<EOF
+set timeout 60
+spawn sh -c "rpmsign --addsign dist/rpm/**/k3s-*.rpm"
+expect "Enter pass phrase:"
+send -- "$PRIVATE_KEY_PASS_PHRASE\r"
+expect eof
+lassign [wait] _ _ _ code
+exit \$code
+EOF
+
+if [ -z "$AWS_S3_BUCKET" ]; then
+  echo "AWS_S3_BUCKET skipping rpm upload"
+  exit 0
+fi
+
+rpm-s3 --bucket $AWS_S3_BUCKET dist/rpm/**/k3s-*.rpm

scripts/provision/generic/alpine310/vagrant

+#!/bin/bash
+set -ve
+
+apk add -q -f curl libc6-compat tzdata
+download_go
+download_dqlite
+# ---
+cat <<EOF >/etc/profile.d/build.sh
+export SELINUX=true
+export DQLITE=true
+export STATIC_BUILD=true
+EOF
+. /etc/profile.d/build.sh
+# ---
+. /tmp/docker-run
+# ---
+go get -u github.com/go-delve/delve/cmd/dlv
+# ---
+cat <<EOF >/etc/profile.d/docker.sh
+export DOCKER_HOST=tcp://10.0.2.2:2375
+EOF
+. /etc/profile.d/docker.sh
+# ---
+(
+    if ! docker version --format '{{.Server.Version}}' >/tmp/docker-server-version; then
+        echo "WARNING: Unable to connect to the docker socket, to enable docker in vagrant run the following command on the host:"
+        echo "docker run -d -v /var/run/docker.sock:/var/run/docker.sock -p 127.0.0.1:2375:2375 alpine/socat TCP-LISTEN:2375,fork UNIX-CONNECT:/var/run/docker.sock"
+    else
+        echo "Using host docker server v$(cat /tmp/docker-server-version)"
+    fi
+)
\ No newline at end of file

scripts/provision/generic/centos7/gen-gpg-keys

+#!/bin/bash
+
+set -e -x
+
+TMPDIR=$(mktemp -d)
+cleanup() {
+  exit_code=$?
+  trap - EXIT INT
+  rm -rf ${TMPDIR}
+  exit ${exit_code}
+}
+trap cleanup EXIT INT
+
+export HOME=${TMPDIR}
+
+gpg --batch --gen-key - <<EOF
+%echo Generating a default key
+Key-Type: default
+Subkey-Type: default
+Name-Real: Rancher
+Name-Comment: CI
+Name-Email: ci@rancher.com
+Expire-Date: 0
+
+# Key-Length: 4096
+# Subkey-Length: 4096
+Passphrase: $PRIVATE_KEY_PASS_PHRASE
+# %no-protection
+# %no-ask-passphrase
+
+# Do a commit here, so that we can later print "done" :-)
+%commit
+%echo done
+EOF
+
+gpg --armor --export ci@rancher.com >public.key
+gpg --armor --export-secret-key ci@rancher.com >private.key

scripts/provision/generic/centos7/vagrant

+#!/bin/bash
+set -ve
+
+download_go
+# ---
+cat <<EOF >/etc/profile.d/build.sh
+export SELINUX=true
+# export DQLITE=true
+# export STATIC_BUILD=true
+EOF
+. /etc/profile.d/build.sh
+# ---
+sed -E 's|apk( -U)?( --no-cache)?( --repository [^ ]*)? add|yum install -y|g' -i /tmp/docker-run
+sed -E 's/-dev/-devel/g' -i /tmp/docker-run
+. /tmp/docker-run
+# ---
+go get -u github.com/go-delve/delve/cmd/dlv
+# ---
+# docker install instructions slightly changed from https://kubernetes.io/docs/setup/production-environment/container-runtimes/
+# default "exec-opts": ["native.cgroupdriver=cgroupfs"], and set "selinux-enabled": true
+yum remove -y docker docker-common
+yum install -y yum-utils device-mapper-persistent-data lvm2
+yum-config-manager --add-repo   https://download.docker.com/linux/centos/docker-ce.repo
+yum update -y && yum install -y   containerd.io-1.2.10   docker-ce-19.03.4   docker-ce-cli-19.03.4
+mkdir -p /etc/docker
+cat > /etc/docker/daemon.json <<EOF
+{
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "selinux-enabled": true,
+  "storage-driver": "overlay2",
+  "storage-opts": [
+    "overlay2.override_kernel_check=true"
+  ]
+}
+EOF
+# ---
+systemctl daemon-reload
+systemctl disable --now firewalld
+systemctl disable --now docker
+# ---
+# set selinux to permissive for dev & testing purposes only
+setenforce 0
+sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

scripts/provision/generic/centos7/yum-install-rpm-tools

+#!/bin/bash
+
+set -e -x
+
+yum install -y git expect yum-utils rpm-build rpm-sign python-deltarpm epel-release
+yum install -y python2-pip
+pip install git+git://github.com/Voronenko/rpm-s3.git@5695c6ad9a08548141d3713328e1bd3f533d137e

scripts/provision/generic/ubuntu1804/vagrant

+#!/bin/bash
+set -ve
+
+download_go
+# ---
+cat <<EOF >/etc/profile.d/build.sh
+export SELINUX=true
+# export DQLITE=true
+# export STATIC_BUILD=true
+EOF
+. /etc/profile.d/build.sh
+# ---
+apt-get install -y \
+  build-essential \
+  pkg-config \
+  libseccomp-dev \
+  libsqlite3-dev \
+  libselinux1-dev \
+  zlib1g-dev \
+
+# ---
+sed -E 's|apk( -U)?( --no-cache)?( --repository [^ ]*)? add|apt-get install -y|g' -i /tmp/docker-run
+. /tmp/docker-run
+# ---
+go get -u github.com/go-delve/delve/cmd/dlv
+# ---

scripts/provision/vagrant

+#!/bin/bash
+set -ve
+
+PROVISION="scripts/provision/$BOX/vagrant"
+
+if [ ! -f /etc/vagrant_box_build_time ]; then
+    echo "This script should only be called during vagrant provisioning"
+    exit 1
+fi
+
+if [[ $HOME == /go/* ]]; then
+    echo "Must not launch vagrant from /go/"
+    exit 1
+fi 
+cd
+
+# --- Default to root user for vagrant ssh
+cat <<\EOF >/etc/profile.d/root.sh
+[ $EUID -ne 0 ] && exec sudo -i
+EOF
+
+# --- Setup go version
+if [ -z "${GOPATH}" ]; then
+    GOPATH=$(realpath $HOME/../../../..)
+    echo "WARNING: Assuming GOPATH=$GOPATH"
+else
+    echo "Using parent GOPATH=$GOPATH"
+fi
+
+# --- Setup basic env
+cat <<EOF >/etc/profile.d/env.sh
+export ARCH=amd64
+export GO111MODULE=off
+export GOPATH="$GOPATH"
+export PATH=/usr/local/bin:$PATH:/usr/local/go/bin:$GOPATH/bin
+export HOME="$HOME" && cd
+EOF
+. /etc/profile.d/env.sh
+
+# --- Clean go cache
+rm -rf .cache/go-build || true
+
+# --- Set color prompt
+sed -i 's|:/bin/ash$|:/bin/bash|g' /etc/passwd
+cat <<\EOF >/etc/profile.d/color.sh
+alias ls='ls --color=auto'
+export PS1='\033[31m[ \033[90m\D{%F 🐮 %T}\033[31m ]\n\[\033[36m\]\u\[\033[m\]@\[\033[32m\]\h\[\033[35m\]:\[\033[33;1m\]\w\[\033[m\]\$ '
+EOF
+
+# --- Setup install script from docker run commands
+mkdir -p ${GOPATH}/bin
+mkdir -p /go
+ln -sf $GOPATH/bin /go/bin
+sed ':a;N;$!ba;s/\\\n/ /g' <Dockerfile.dapper | grep '^RUN ' | sed -e 's/^RUN //' >/tmp/docker-run
+export BINDIR=/go/bin
+export GOPATH=/go
+export HOME=/tmp
+
+# --- Add k3s motd
+cat <<\EOF >/etc/motd
+               ,        ,
+   ,-----------|'------'| |\    ____
+  /.           '-'@  o|-' | |  /___ \
+ |/|             | .. |   | | __ __) | ____
+   |   .________.'----'   | |/ /|__ < / __/
+   |  ||        |  ||     |   < ___) |\__ \
+   \__|'        \__|'     |_|\_\_____/____/
+
+EOF
+
+# --- Utility function to download go
+download_go() {
+    goversion=$(grep "^FROM " Dockerfile.dapper | sed -e 's/^FROM golang:\(.*\)-.*/\1/')
+    if [ -z "$goversion" ]; then
+        echo "Cannot find version of go to fetch"
+        return 1
+    fi
+    echo "Installing go $goversion"
+    curl -sL https://storage.googleapis.com/golang/go${goversion}.linux-${ARCH}.tar.gz | tar -xzf - -C /usr/local
+}
+
+# --- Utility function to download dqlite
+download_dqlite() {
+    dqliteURL="https://github.com/$(grep dqlite-build Dockerfile.dapper | sed -e 's/^.*--from=\([^ ]*\).*$/\1/' -e 's|:|/releases/download/|')/dqlite-$ARCH.tgz"
+    if [ -z "$dqliteURL" ]; then
+        echo "Cannot find dqlite URL to fetch"
+        return 1
+    fi
+    mkdir -p /usr/src/
+    echo "Downloading DQLITE from $dqliteURL"
+    curl -sL $dqliteURL -o /usr/src/dqlite.tgz
+}
+
+# --- Run vagrant provision script if available
+if [ ! -f "${PROVISION}" ]; then
+  echo 'WARNING: Unable to execute provision script "${PROVISION}"'
+  exit
+fi
+echo "running '${PROVISION}'..." && \
+  . ${PROVISION} && \
+  echo "finished '${PROVISION}'!"

scripts/test

@@ -13,19 +13,18 @@ mkdir -p $artifacts
 
 # ---
 
+[ "$ARCH" = 'arm' ] && \
+  early-exit "Skipping sonobuoy, images not available for $ARCH."
+
 E2E_OUTPUT=$artifacts test-run-sonobuoy
 
 # ---
 
-if [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
-  printf "\033[33mSkipping remaining tests on tag.\033[m\n"
-  exit 0
-fi
+[ "$DRONE_BUILD_EVENT" = 'tag' ] && \
+  early-exit 'Skipping remaining tests on tag.'
 
-if [ "$ARCH" != 'amd64' ]; then
-  printf "\033[33mSkipping remaining tests, images not available for $ARCH.\033[m\n"
-  exit 0
-fi
+[ "$ARCH" != 'amd64' ] && \
+  early-exit "Skipping remaining tests, images not available for $ARCH."
 
 # ---
 

scripts/test-helpers

@@ -437,6 +437,14 @@ export -f provision-cluster
 
 # ---
 
+early-exit() {
+    printf "\033[33m$1\033[m\n"
+    exit $2
+}
+export -f early-exit
+
+# ---
+
 run-test() {
     export PROVISION_LOCK=$(mktemp)
     ./scripts/test-runner $@ &
@@ -454,27 +462,27 @@ export -f run-test
 # ---
 
 e2e-test() {
-  local label=$label
-  if [ -n "$LABEL_SUFFIX" ]; then
-    label="$label-$LABEL_SUFFIX"
-  fi
-  local logOutput=
-  if [ -n "$E2E_OUTPUT" ]; then
-    logOutput=$E2E_OUTPUT/$logName
-  fi
-  LABEL=$label LOG_OUTPUT=$logOutput run-test $@
+    local label=$label
+    if [ -n "$LABEL_SUFFIX" ]; then
+        label="$label-$LABEL_SUFFIX"
+    fi
+    local logOutput=
+    if [ -n "$E2E_OUTPUT" ]; then
+        logOutput=$E2E_OUTPUT/$logName
+    fi
+    LABEL=$label LOG_OUTPUT=$logOutput run-test $@
 }
 
 # ---
 
 run-e2e-tests() {
-  label=PARALLEL \
-      logName=e2e-STATUS-${ARCH}-parallel.log \
-      e2e-test ${sonobuoyParallelArgs[@]}
+    label=PARALLEL \
+        logName=e2e-STATUS-${ARCH}-parallel.log \
+        e2e-test ${sonobuoyParallelArgs[@]}
 
-  label=SERIAL \
-      logName=e2e-STATUS-${ARCH}-serial.log \
-      e2e-test ${sonobuoySerialArgs[@]}
+    label=SERIAL \
+        logName=e2e-STATUS-${ARCH}-serial.log \
+        e2e-test ${sonobuoySerialArgs[@]}
 }
 export -f run-e2e-tests
 

scripts/test-run-basics

 #!/bin/bash
 
+all_services=(
+    coredns
+    local-path-provisioner
+    metrics-server
+    traefik
+)
+
 export NUM_SERVERS=1
 export NUM_AGENTS=1
-export SERVER_ARGS='--no-deploy=traefik,coredns,local-storage,metrics-server'
+export WAIT_SERVICES="${all_services[@]}"
 
 start-test() {
     docker exec $(cat $TEST_DIR/servers/1/metadata/name) check-config || true
     verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name)
+    verify-airgap-images $(cat $TEST_DIR/{servers,agents}/*/metadata/name)
 }
 export -f start-test
 
+# -- check for changes to the airgap image list
+verify-airgap-images() {
+    local airgap_image_list='scripts/airgap/image-list.txt'
+
+    for name in $@; do
+        docker exec $name crictl images -o json \
+            | jq -r '.images[].repoTags[0] | select(. != null)'
+    done | sort -u >$airgap_image_list.tmp
+
+    if ! diff $airgap_image_list{,.tmp}; then
+        echo '[ERROR] Failed airgap image check'
+        return 1
+    fi
+}
+export -f verify-airgap-images
+
 # --- create a basic cluster and check for valid versions
 LABEL=BASICS run-test

scripts/vagrant-provision

-#!/bin/bash
-set -ve
-
-cd $(dirname $0)/..
-
-if [ ! -f /etc/vagrant_box_build_time ]; then
-    echo "This script should only be called during vagrant provisioning"
-    exit 1
-fi
-
-ARCH=amd64
-if [[ $HOME == /go/* ]]; then
-    echo "Must not launch vagrant from /go/"
-    exit 1
-fi 
-
-if [ -z "${GOPATH}" ]; then
-    GOPATH=$(realpath $HOME/../../../..)
-    echo "WARNING: Assuming GOPATH=$GOPATH"
-else
-    echo "Using parent GOPATH=$GOPATH"
-fi
-goversion=$(grep "^FROM " Dockerfile.dapper | sed -e 's/^FROM golang:\(.*\)-.*/\1/')
-
-if [ -z "$goversion" ]; then
-    echo "Cannot find version of go to fetch"
-    exit 1
-fi
-
-echo "Installing go $goversion"
-apk add -q -f curl libc6-compat tzdata
-curl -sL https://storage.googleapis.com/golang/go${goversion}.linux-${ARCH}.tar.gz | tar -xzf - -C /usr/local
-# ---
-cat <<EOF >/etc/profile.d/build.sh
-export STATIC_BUILD=true
-EOF
-# ---
-cat <<EOF >/etc/profile.d/go.sh
-export GOPATH=$GOPATH
-export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
-EOF
-. /etc/profile.d/go.sh
-# ---
-sed -i 's|:/bin/ash$|:/bin/bash|g' /etc/passwd
-cat <<\EOF >/etc/profile.d/color.sh
-alias ls='ls --color=auto'
-export PS1='\033[31m[ \033[90m\D{%F %T}\033[31m ]\n\[\033[36m\]\u\[\033[m\]🐮\[\033[32m\]\h:\[\033[33;1m\]\w\[\033[m\]$ '
-EOF
-# ---
-mkdir -p ${GOPATH}/bin
-mkdir -p /go
-ln -s $GOPATH/bin /go/bin
-sed ':a;N;$!ba;s/\\\n/ /g' <Dockerfile.dapper | grep '^RUN ' | sed -e 's/^RUN //' >/tmp/docker-run
-export BINDIR=/go/bin
-export GOPATH=/go
-export HOME=/tmp && cd
-. /tmp/docker-run
-cd /go
-go get github.com/rancher/trash
-rm -rf /go
-cd
-# ---
-cd /etc/local.d
-cat <<\EOF >cgroup.start
-#!/bin/bash
-mount cgroup_root /sys/fs/cgroup -t tmpfs -o "rw,nosuid,nodev,noexec,relatime,size=10240k,mode=755"
-# blkio freezer net_cls net_prio 
-for d in cpu cpuacct cpuset devices memory pids; do
-    mkdir -p /sys/fs/cgroup/$d
-    mount $d /sys/fs/cgroup/$d -t cgroup -o "rw,nosuid,nodev,noexec,relatime,$d"
-done
-EOF
-chmod 0755 cgroup.start
-rc-update add local default
-rc-service local start
-# ---
-cat <<EOF >/etc/profile.d/docker.sh
-export DOCKER_HOST=tcp://10.0.2.2:2375
-EOF
-. /etc/profile.d/docker.sh
-# ---
-cat <<\EOF >>/etc/motd
-               ,        ,
-   ,-----------|'------'| |\    ____
-  /.           '-'@  o|-' | |  /___ \
- |/|             | .. |   | | __ __) | ____
-   |   .________.'----'   | |/ /|__ < / __/
-   |  ||        |  ||     |   < ___) |\__ \
-   \__|'        \__|'     |_|\_\_____/____/
-
-EOF
-# ---
-set +v
-if ! docker version --format '{{.Server.Version}}' >/tmp/docker-server-version; then
-    echo "WARNING: Unable to connect to the docker socket, to enable docker in vagrant run the following command on the host:"
-    echo "docker run -d -v /var/run/docker.sock:/var/run/docker.sock -p 127.0.0.1:2375:2375 alpine/socat TCP-LISTEN:2375,fork UNIX-CONNECT:/var/run/docker.sock"
-else
-    echo "Using host docker server v$(cat /tmp/docker-server-version)"
-fi

scripts/version.sh

 #!/bin/bash
 
+ARCH=${ARCH:-$(go env GOARCH)}
+SUFFIX="-${ARCH}"
+GIT_TAG=$DRONE_TAG
 TREE_STATE=clean
-if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
-    DIRTY="-dirty"
-    TREE_STATE=dirty
-fi
+COMMIT=$DRONE_COMMIT
 
-COMMIT=$(git log -n3 --pretty=format:"%H %ae" | grep -v ' drone@localhost$' | cut -f1 -d\  | head -1)
-if [ -z "${COMMIT}" ]; then
-  COMMIT=$(git rev-parse HEAD)
-fi
-
-GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
+if [ -d .git ]; then
+    if [ -z "$GIT_TAG" ]; then
+        GIT_TAG=$(git tag -l --contains HEAD | head -n 1)
+    fi
+    if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+        DIRTY="-dirty"
+        TREE_STATE=dirty
+    fi
 
-ARCH=$(go env GOARCH)
-SUFFIX="-${ARCH}"
+    COMMIT=$(git log -n3 --pretty=format:"%H %ae" | grep -v ' drone@localhost$' | cut -f1 -d\  | head -1)
+    if [ -z "${COMMIT}" ]; then
+    COMMIT=$(git rev-parse HEAD || true)
+    fi
+fi
 
 VERSION_CONTAINERD=$(grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}')
 if [ -z "$VERSION_CONTAINERD" ]; then
@@ -40,6 +45,6 @@ if [[ -n "$GIT_TAG" ]]; then
     fi
     VERSION=$GIT_TAG
 else
-    VERSION="$VERSION_K8S+${COMMIT:0:8}$DIRTY"
+    VERSION="$VERSION_K8S+k3s-${COMMIT:0:8}$DIRTY"
 fi
 VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")"

vendor/github.com/containerd/containerd/container.go

@@ -36,6 +36,7 @@ import (
 	prototypes "github.com/gogo/protobuf/types"
 	ver "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 )
 
@@ -242,7 +243,17 @@ func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...N
 		if err != nil {
 			return nil, err
 		}
+		spec, err := c.Spec(ctx)
+		if err != nil {
+			return nil, err
+		}
 		for _, m := range mounts {
+			if spec.Linux != nil && spec.Linux.MountLabel != "" {
+				context := label.FormatMountLabel("", spec.Linux.MountLabel)
+				if context != "" {
+					m.Options = append(m.Options, context)
+				}
+			}
 			request.Rootfs = append(request.Rootfs, &types.Mount{
 				Type:    m.Type,
 				Source:  m.Source,

vendor/github.com/containerd/cri/pkg/server/container_create.go

@@ -39,6 +39,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
@@ -173,6 +174,18 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 		return nil, errors.Wrapf(err, "failed to generate container %q spec", id)
 	}
 
+	meta.ProcessLabel = spec.Process.SelinuxLabel
+	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+		// If privileged don't set the SELinux label but still record it on the container so
+		// the unused MCS label can be release later
+		spec.Process.SelinuxLabel = ""
+	}
+	defer func() {
+		if retErr != nil {
+			_ = label.ReleaseLabel(spec.Process.SelinuxLabel)
+		}
+	}()
+
 	log.G(ctx).Debugf("Container %q spec: %#+v", id, spew.NewFormatter(spec))
 
 	// Set snapshotter before any other options.
@@ -324,7 +337,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 
 func (c *criService) generateContainerSpec(id string, sandboxID string, sandboxPid uint32, config *runtime.ContainerConfig,
 	sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, extraMounts []*runtime.Mount,
-	ociRuntime config.Runtime) (*runtimespec.Spec, error) {
+	ociRuntime config.Runtime) (retSpec *runtimespec.Spec, retErr error) {
 
 	specOpts := []oci.SpecOpts{
 		customopts.WithoutRunMount,
@@ -366,11 +379,27 @@ func (c *criService) generateContainerSpec(id string, sandboxID string, sandboxP
 	specOpts = append(specOpts, oci.WithEnv(env))
 
 	securityContext := config.GetLinux().GetSecurityContext()
-	selinuxOpt := securityContext.GetSelinuxOptions()
-	processLabel, mountLabel, err := initSelinuxOpts(selinuxOpt)
+	labelOptions := toLabel(securityContext.GetSelinuxOptions())
+	if len(labelOptions) == 0 {
+		// Use pod level SELinux config
+		if sandbox, err := c.sandboxStore.Get(sandboxID); err == nil {
+			labelOptions, err = label.DupSecOpt(sandbox.ProcessLabel)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	processLabel, mountLabel, err := label.InitLabels(labelOptions)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
 	}
+	defer func() {
+		if retErr != nil {
+			_ = label.ReleaseLabel(processLabel)
+		}
+	}()
+
 	specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel))
 
 	if !c.config.DisableProcMount {

vendor/github.com/containerd/cri/pkg/server/helpers.go

@@ -298,47 +298,55 @@ func (c *criService) ensureImageExists(ctx context.Context, ref string, config *
 	return &newImage, nil
 }
 
-func initSelinuxOpts(selinuxOpt *runtime.SELinuxOption) (string, string, error) {
-	if selinuxOpt == nil {
-		return "", "", nil
+func toLabel(selinuxOptions *runtime.SELinuxOption) (labels []string) {
+	if selinuxOptions == nil {
+		return nil
 	}
 
-	// Should ignored selinuxOpts if they are incomplete.
-	if selinuxOpt.GetUser() == "" ||
-		selinuxOpt.GetRole() == "" ||
-		selinuxOpt.GetType() == "" {
-		return "", "", nil
+	if selinuxOptions.User != "" {
+		labels = append(labels, "user:"+selinuxOptions.User)
 	}
-
-	// make sure the format of "level" is correct.
-	ok, err := checkSelinuxLevel(selinuxOpt.GetLevel())
-	if err != nil || !ok {
-		return "", "", err
+	if selinuxOptions.Role != "" {
+		labels = append(labels, "role:"+selinuxOptions.Role)
+	}
+	if selinuxOptions.Type != "" {
+		labels = append(labels, "type:"+selinuxOptions.Type)
+	}
+	if selinuxOptions.Level != "" {
+		labels = append(labels, "level:"+selinuxOptions.Level)
 	}
 
-	labelOpts := fmt.Sprintf("%s:%s:%s:%s",
-		selinuxOpt.GetUser(),
-		selinuxOpt.GetRole(),
-		selinuxOpt.GetType(),
-		selinuxOpt.GetLevel())
+	return
+}
 
-	options, err := label.DupSecOpt(labelOpts)
-	if err != nil {
-		return "", "", err
+func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
+	return initLabels(toLabel(selinuxOpts))
+}
+
+func initLabels(options []string) (string, string, error) {
+	for _, opt := range options {
+		if strings.HasPrefix(opt, "level:") {
+			if err := checkSelinuxLevel(strings.TrimPrefix(opt, "level:")); err != nil {
+				return "", "", err
+			}
+		}
 	}
 	return label.InitLabels(options)
 }
 
-func checkSelinuxLevel(level string) (bool, error) {
+func checkSelinuxLevel(level string) error {
 	if len(level) == 0 {
-		return true, nil
+		return nil
 	}
 
 	matched, err := regexp.MatchString(`^s\d(-s\d)??(:c\d{1,4}((.c\d{1,4})?,c\d{1,4})*(.c\d{1,4})?(,c\d{1,4}(.c\d{1,4})?)*)?$`, level)
-	if err != nil || !matched {
-		return false, errors.Wrapf(err, "the format of 'level' %q is not correct", level)
+	if err != nil {
+		return errors.Wrapf(err, "the format of 'level' %q is not correct", level)
+	}
+	if !matched {
+		return fmt.Errorf("the format of 'level' %q is not correct", level)
 	}
-	return true, nil
+	return nil
 }
 
 // isInCRIMounts checks whether a destination is in CRI mount list.

vendor/github.com/containerd/cri/pkg/server/sandbox_run.go

@@ -34,6 +34,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -158,6 +159,18 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, errors.Wrap(err, "failed to generate sandbox container spec")
 	}
 	log.G(ctx).Debugf("Sandbox container %q spec: %#+v", id, spew.NewFormatter(spec))
+	sandbox.ProcessLabel = spec.Process.SelinuxLabel
+	defer func() {
+		if retErr != nil {
+			_ = label.ReleaseLabel(sandbox.ProcessLabel)
+		}
+	}()
+
+	if securityContext.GetPrivileged() {
+		// If privileged don't set selinux label, but we still record the MCS label so that
+		// the unused label can be freed later.
+		spec.Process.SelinuxLabel = ""
+	}
 
 	var specOpts []oci.SpecOpts
 	userstr, err := generateUserString(
@@ -328,7 +341,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 }
 
 func (c *criService) generateSandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
-	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (retSpec *runtimespec.Spec, retErr error) {
 	// Creates a spec Generator with the default spec.
 	// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
 	specOpts := []oci.SpecOpts{
@@ -403,11 +416,15 @@ func (c *criService) generateSandboxContainerSpec(id string, config *runtime.Pod
 		},
 	}))
 
-	selinuxOpt := securityContext.GetSelinuxOptions()
-	processLabel, mountLabel, err := initSelinuxOpts(selinuxOpt)
+	processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions())
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
 	}
+	defer func() {
+		if retErr != nil && processLabel != "" {
+			_ = label.ReleaseLabel(processLabel)
+		}
+	}()
 
 	supplementalGroups := securityContext.GetSupplementalGroups()
 	specOpts = append(specOpts,

vendor/github.com/containerd/cri/pkg/server/service.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/cri/pkg/store/label"
 	cni "github.com/containerd/go-cni"
 	runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
 	runcseccomp "github.com/opencontainers/runc/libcontainer/seccomp"
@@ -104,14 +105,15 @@ type criService struct {
 // NewCRIService returns a new instance of CRIService
 func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIService, error) {
 	var err error
+	labels := label.NewStore()
 	c := &criService{
 		config:             config,
 		client:             client,
 		apparmorEnabled:    runcapparmor.IsEnabled() && !config.DisableApparmor,
 		seccompEnabled:     runcseccomp.IsEnabled(),
 		os:                 osinterface.RealOS{},
-		sandboxStore:       sandboxstore.NewStore(),
-		containerStore:     containerstore.NewStore(),
+		sandboxStore:       sandboxstore.NewStore(labels),
+		containerStore:     containerstore.NewStore(labels),
 		imageStore:         imagestore.NewStore(client),
 		snapshotStore:      snapshotstore.NewStore(),
 		sandboxNameIndex:   registrar.NewRegistrar(),

vendor/github.com/containerd/cri/pkg/store/container/container.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/cri/pkg/store/label"
 	"github.com/docker/docker/pkg/truncindex"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 
@@ -101,13 +102,15 @@ type Store struct {
 	lock       sync.RWMutex
 	containers map[string]Container
 	idIndex    *truncindex.TruncIndex
+	labels     *label.Store
 }
 
 // NewStore creates a container store.
-func NewStore() *Store {
+func NewStore(labels *label.Store) *Store {
 	return &Store{
 		containers: make(map[string]Container),
 		idIndex:    truncindex.NewTruncIndex([]string{}),
+		labels:     labels,
 	}
 }
 
@@ -119,6 +122,9 @@ func (s *Store) Add(c Container) error {
 	if _, ok := s.containers[c.ID]; ok {
 		return store.ErrAlreadyExist
 	}
+	if err := s.labels.Reserve(c.ProcessLabel); err != nil {
+		return err
+	}
 	if err := s.idIndex.Add(c.ID); err != nil {
 		return err
 	}
@@ -165,6 +171,7 @@ func (s *Store) Delete(id string) {
 		// So we need to return if there are error.
 		return
 	}
+	s.labels.Release(s.containers[id].ProcessLabel)
 	s.idIndex.Delete(id) // nolint: errcheck
 	delete(s.containers, id)
 }

vendor/github.com/containerd/cri/pkg/store/container/metadata.go

@@ -61,6 +61,8 @@ type Metadata struct {
 	// StopSignal is the system call signal that will be sent to the container to exit.
 	// TODO(random-liu): Add integration test for stop signal.
 	StopSignal string
+	// ProcessLabel is the SELinux process label for the container
+	ProcessLabel string
 }
 
 // MarshalJSON encodes Metadata into bytes in json format.

vendor/github.com/containerd/cri/pkg/store/label/label.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package label
+
+import (
+	"sync"
+
+	"github.com/opencontainers/selinux/go-selinux"
+)
+
+type Store struct {
+	sync.Mutex
+	levels   map[string]int
+	Releaser func(string)
+	Reserver func(string)
+}
+
+func NewStore() *Store {
+	return &Store{
+		levels:   map[string]int{},
+		Releaser: selinux.ReleaseLabel,
+		Reserver: selinux.ReserveLabel,
+	}
+}
+
+func (s *Store) Reserve(label string) error {
+	s.Lock()
+	defer s.Unlock()
+
+	context, err := selinux.NewContext(label)
+	if err != nil {
+		return err
+	}
+
+	level := context["level"]
+	// no reason to count empty
+	if level == "" {
+		return nil
+	}
+
+	if _, ok := s.levels[level]; !ok {
+		s.Reserver(label)
+	}
+
+	s.levels[level]++
+	return nil
+}
+
+func (s *Store) Release(label string) {
+	s.Lock()
+	defer s.Unlock()
+
+	context, err := selinux.NewContext(label)
+	if err != nil {
+		return
+	}
+
+	level := context["level"]
+	if level == "" {
+		return
+	}
+
+	count, ok := s.levels[level]
+	if !ok {
+		return
+	}
+	switch {
+	case count == 1:
+		s.Releaser(label)
+		delete(s.levels, level)
+	case count < 1:
+		delete(s.levels, level)
+	case count > 1:
+		s.levels[level] = count - 1
+	}
+}

vendor/github.com/containerd/cri/pkg/store/sandbox/metadata.go

@@ -61,6 +61,8 @@ type Metadata struct {
 	RuntimeHandler string
 	// CNIresult resulting configuration for attached network namespace interfaces
 	CNIResult *cni.CNIResult
+	// ProcessLabel is the SELinux process label for the container
+	ProcessLabel string
 }
 
 // MarshalJSON encodes Metadata into bytes in json format.

vendor/github.com/containerd/cri/pkg/store/sandbox/sandbox.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/cri/pkg/store/label"
 	"github.com/docker/docker/pkg/truncindex"
 
 	"github.com/containerd/cri/pkg/netns"
@@ -62,13 +63,15 @@ type Store struct {
 	lock      sync.RWMutex
 	sandboxes map[string]Sandbox
 	idIndex   *truncindex.TruncIndex
+	labels    *label.Store
 }
 
 // NewStore creates a sandbox store.
-func NewStore() *Store {
+func NewStore(labels *label.Store) *Store {
 	return &Store{
 		sandboxes: make(map[string]Sandbox),
 		idIndex:   truncindex.NewTruncIndex([]string{}),
+		labels:    labels,
 	}
 }
 
@@ -79,6 +82,9 @@ func (s *Store) Add(sb Sandbox) error {
 	if _, ok := s.sandboxes[sb.ID]; ok {
 		return store.ErrAlreadyExist
 	}
+	if err := s.labels.Reserve(sb.ProcessLabel); err != nil {
+		return err
+	}
 	if err := s.idIndex.Add(sb.ID); err != nil {
 		return err
 	}
@@ -125,6 +131,7 @@ func (s *Store) Delete(id string) {
 		// So we need to return if there are error.
 		return
 	}
+	s.labels.Release(s.sandboxes[id].ProcessLabel)
 	s.idIndex.Delete(id) // nolint: errcheck
 	delete(s.sandboxes, id)
 }

vendor/github.com/docker/docker/pkg/idtools/idtools_unix.go

-// +build !windows
-
-package idtools // import "github.com/docker/docker/pkg/idtools"
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"syscall"
-
-	"github.com/docker/docker/pkg/system"
-	"github.com/opencontainers/runc/libcontainer/user"
-)
-
-var (
-	entOnce   sync.Once
-	getentCmd string
-)
-
-func mkdirAs(path string, mode os.FileMode, owner Identity, mkAll, chownExisting bool) error {
-	// make an array containing the original path asked for, plus (for mkAll == true)
-	// all path components leading up to the complete path that don't exist before we MkdirAll
-	// so that we can chown all of them properly at the end.  If chownExisting is false, we won't
-	// chown the full directory path if it exists
-
-	var paths []string
-
-	stat, err := system.Stat(path)
-	if err == nil {
-		if !stat.IsDir() {
-			return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
-		}
-		if !chownExisting {
-			return nil
-		}
-
-		// short-circuit--we were called with an existing directory and chown was requested
-		return lazyChown(path, owner.UID, owner.GID, stat)
-	}
-
-	if os.IsNotExist(err) {
-		paths = []string{path}
-	}
-
-	if mkAll {
-		// walk back to "/" looking for directories which do not exist
-		// and add them to the paths array for chown after creation
-		dirPath := path
-		for {
-			dirPath = filepath.Dir(dirPath)
-			if dirPath == "/" {
-				break
-			}
-			if _, err := os.Stat(dirPath); err != nil && os.IsNotExist(err) {
-				paths = append(paths, dirPath)
-			}
-		}
-		if err := system.MkdirAll(path, mode); err != nil {
-			return err
-		}
-	} else {
-		if err := os.Mkdir(path, mode); err != nil && !os.IsExist(err) {
-			return err
-		}
-	}
-	// even if it existed, we will chown the requested path + any subpaths that
-	// didn't exist when we called MkdirAll
-	for _, pathComponent := range paths {
-		if err := lazyChown(pathComponent, owner.UID, owner.GID, nil); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// CanAccess takes a valid (existing) directory and a uid, gid pair and determines
-// if that uid, gid pair has access (execute bit) to the directory
-func CanAccess(path string, pair Identity) bool {
-	statInfo, err := system.Stat(path)
-	if err != nil {
-		return false
-	}
-	fileMode := os.FileMode(statInfo.Mode())
-	permBits := fileMode.Perm()
-	return accessible(statInfo.UID() == uint32(pair.UID),
-		statInfo.GID() == uint32(pair.GID), permBits)
-}
-
-func accessible(isOwner, isGroup bool, perms os.FileMode) bool {
-	if isOwner && (perms&0100 == 0100) {
-		return true
-	}
-	if isGroup && (perms&0010 == 0010) {
-		return true
-	}
-	if perms&0001 == 0001 {
-		return true
-	}
-	return false
-}
-
-// LookupUser uses traditional local system files lookup (from libcontainer/user) on a username,
-// followed by a call to `getent` for supporting host configured non-files passwd and group dbs
-func LookupUser(username string) (user.User, error) {
-	// first try a local system files lookup using existing capabilities
-	usr, err := user.LookupUser(username)
-	if err == nil {
-		return usr, nil
-	}
-	// local files lookup failed; attempt to call `getent` to query configured passwd dbs
-	usr, err = getentUser(fmt.Sprintf("%s %s", "passwd", username))
-	if err != nil {
-		return user.User{}, err
-	}
-	return usr, nil
-}
-
-// LookupUID uses traditional local system files lookup (from libcontainer/user) on a uid,
-// followed by a call to `getent` for supporting host configured non-files passwd and group dbs
-func LookupUID(uid int) (user.User, error) {
-	// first try a local system files lookup using existing capabilities
-	usr, err := user.LookupUid(uid)
-	if err == nil {
-		return usr, nil
-	}
-	// local files lookup failed; attempt to call `getent` to query configured passwd dbs
-	return getentUser(fmt.Sprintf("%s %d", "passwd", uid))
-}
-
-func getentUser(args string) (user.User, error) {
-	reader, err := callGetent(args)
-	if err != nil {
-		return user.User{}, err
-	}
-	users, err := user.ParsePasswd(reader)
-	if err != nil {
-		return user.User{}, err
-	}
-	if len(users) == 0 {
-		return user.User{}, fmt.Errorf("getent failed to find passwd entry for %q", strings.Split(args, " ")[1])
-	}
-	return users[0], nil
-}
-
-// LookupGroup uses traditional local system files lookup (from libcontainer/user) on a group name,
-// followed by a call to `getent` for supporting host configured non-files passwd and group dbs
-func LookupGroup(groupname string) (user.Group, error) {
-	// first try a local system files lookup using existing capabilities
-	group, err := user.LookupGroup(groupname)
-	if err == nil {
-		return group, nil
-	}
-	// local files lookup failed; attempt to call `getent` to query configured group dbs
-	return getentGroup(fmt.Sprintf("%s %s", "group", groupname))
-}
-
-// LookupGID uses traditional local system files lookup (from libcontainer/user) on a group ID,
-// followed by a call to `getent` for supporting host configured non-files passwd and group dbs
-func LookupGID(gid int) (user.Group, error) {
-	// first try a local system files lookup using existing capabilities
-	group, err := user.LookupGid(gid)
-	if err == nil {
-		return group, nil
-	}
-	// local files lookup failed; attempt to call `getent` to query configured group dbs
-	return getentGroup(fmt.Sprintf("%s %d", "group", gid))
-}
-
-func getentGroup(args string) (user.Group, error) {
-	reader, err := callGetent(args)
-	if err != nil {
-		return user.Group{}, err
-	}
-	groups, err := user.ParseGroup(reader)
-	if err != nil {
-		return user.Group{}, err
-	}
-	if len(groups) == 0 {
-		return user.Group{}, fmt.Errorf("getent failed to find groups entry for %q", strings.Split(args, " ")[1])
-	}
-	return groups[0], nil
-}
-
-func callGetent(args string) (io.Reader, error) {
-	entOnce.Do(func() { getentCmd, _ = resolveBinary("getent") })
-	// if no `getent` command on host, can't do anything else
-	if getentCmd == "" {
-		return nil, fmt.Errorf("")
-	}
-	out, err := execCmd(getentCmd, args)
-	if err != nil {
-		exitCode, errC := system.GetExitCode(err)
-		if errC != nil {
-			return nil, err
-		}
-		switch exitCode {
-		case 1:
-			return nil, fmt.Errorf("getent reported invalid parameters/database unknown")
-		case 2:
-			terms := strings.Split(args, " ")
-			return nil, fmt.Errorf("getent unable to find entry %q in %s database", terms[1], terms[0])
-		case 3:
-			return nil, fmt.Errorf("getent database doesn't support enumeration")
-		default:
-			return nil, err
-		}
-
-	}
-	return bytes.NewReader(out), nil
-}
-
-// lazyChown performs a chown only if the uid/gid don't match what's requested
-// Normally a Chown is a no-op if uid/gid match, but in some cases this can still cause an error, e.g. if the
-// dir is on an NFS share, so don't call chown unless we absolutely must.
-func lazyChown(p string, uid, gid int, stat *system.StatT) error {
-	if stat == nil {
-		var err error
-		stat, err = system.Stat(p)
-		if err != nil {
-			return err
-		}
-	}
-	if stat.UID() == uint32(uid) && stat.GID() == uint32(gid) {
-		return nil
-	}
-	return os.Chown(p, uid, gid)
-}

vendor/github.com/docker/docker/pkg/idtools/idtools_windows.go

-package idtools // import "github.com/docker/docker/pkg/idtools"
-
-import (
-	"os"
-
-	"github.com/docker/docker/pkg/system"
-)
-
-// This is currently a wrapper around MkdirAll, however, since currently
-// permissions aren't set through this path, the identity isn't utilized.
-// Ownership is handled elsewhere, but in the future could be support here
-// too.
-func mkdirAs(path string, mode os.FileMode, owner Identity, mkAll, chownExisting bool) error {
-	if err := system.MkdirAll(path, mode); err != nil {
-		return err
-	}
-	return nil
-}
-
-// CanAccess takes a valid (existing) directory and a uid, gid pair and determines
-// if that uid, gid pair has access (execute bit) to the directory
-// Windows does not require/support this function, so always return true
-func CanAccess(path string, identity Identity) bool {
-	return true
-}

vendor/github.com/docker/docker/pkg/idtools/usergroupadd_linux.go

-package idtools // import "github.com/docker/docker/pkg/idtools"
-
-import (
-	"fmt"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-)
-
-// add a user and/or group to Linux /etc/passwd, /etc/group using standard
-// Linux distribution commands:
-// adduser --system --shell /bin/false --disabled-login --disabled-password --no-create-home --group <username>
-// useradd -r -s /bin/false <username>
-
-var (
-	once        sync.Once
-	userCommand string
-
-	cmdTemplates = map[string]string{
-		"adduser": "--system --shell /bin/false --no-create-home --disabled-login --disabled-password --group %s",
-		"useradd": "-r -s /bin/false %s",
-		"usermod": "-%s %d-%d %s",
-	}
-
-	idOutRegexp = regexp.MustCompile(`uid=([0-9]+).*gid=([0-9]+)`)
-	// default length for a UID/GID subordinate range
-	defaultRangeLen   = 65536
-	defaultRangeStart = 100000
-	userMod           = "usermod"
-)
-
-// AddNamespaceRangesUser takes a username and uses the standard system
-// utility to create a system user/group pair used to hold the
-// /etc/sub{uid,gid} ranges which will be used for user namespace
-// mapping ranges in containers.
-func AddNamespaceRangesUser(name string) (int, int, error) {
-	if err := addUser(name); err != nil {
-		return -1, -1, fmt.Errorf("Error adding user %q: %v", name, err)
-	}
-
-	// Query the system for the created uid and gid pair
-	out, err := execCmd("id", name)
-	if err != nil {
-		return -1, -1, fmt.Errorf("Error trying to find uid/gid for new user %q: %v", name, err)
-	}
-	matches := idOutRegexp.FindStringSubmatch(strings.TrimSpace(string(out)))
-	if len(matches) != 3 {
-		return -1, -1, fmt.Errorf("Can't find uid, gid from `id` output: %q", string(out))
-	}
-	uid, err := strconv.Atoi(matches[1])
-	if err != nil {
-		return -1, -1, fmt.Errorf("Can't convert found uid (%s) to int: %v", matches[1], err)
-	}
-	gid, err := strconv.Atoi(matches[2])
-	if err != nil {
-		return -1, -1, fmt.Errorf("Can't convert found gid (%s) to int: %v", matches[2], err)
-	}
-
-	// Now we need to create the subuid/subgid ranges for our new user/group (system users
-	// do not get auto-created ranges in subuid/subgid)
-
-	if err := createSubordinateRanges(name); err != nil {
-		return -1, -1, fmt.Errorf("Couldn't create subordinate ID ranges: %v", err)
-	}
-	return uid, gid, nil
-}
-
-func addUser(userName string) error {
-	once.Do(func() {
-		// set up which commands are used for adding users/groups dependent on distro
-		if _, err := resolveBinary("adduser"); err == nil {
-			userCommand = "adduser"
-		} else if _, err := resolveBinary("useradd"); err == nil {
-			userCommand = "useradd"
-		}
-	})
-	if userCommand == "" {
-		return fmt.Errorf("Cannot add user; no useradd/adduser binary found")
-	}
-	args := fmt.Sprintf(cmdTemplates[userCommand], userName)
-	out, err := execCmd(userCommand, args)
-	if err != nil {
-		return fmt.Errorf("Failed to add user with error: %v; output: %q", err, string(out))
-	}
-	return nil
-}
-
-func createSubordinateRanges(name string) error {
-
-	// first, we should verify that ranges weren't automatically created
-	// by the distro tooling
-	ranges, err := parseSubuid(name)
-	if err != nil {
-		return fmt.Errorf("Error while looking for subuid ranges for user %q: %v", name, err)
-	}
-	if len(ranges) == 0 {
-		// no UID ranges; let's create one
-		startID, err := findNextUIDRange()
-		if err != nil {
-			return fmt.Errorf("Can't find available subuid range: %v", err)
-		}
-		out, err := execCmd(userMod, fmt.Sprintf(cmdTemplates[userMod], "v", startID, startID+defaultRangeLen-1, name))
-		if err != nil {
-			return fmt.Errorf("Unable to add subuid range to user: %q; output: %s, err: %v", name, out, err)
-		}
-	}
-
-	ranges, err = parseSubgid(name)
-	if err != nil {
-		return fmt.Errorf("Error while looking for subgid ranges for user %q: %v", name, err)
-	}
-	if len(ranges) == 0 {
-		// no GID ranges; let's create one
-		startID, err := findNextGIDRange()
-		if err != nil {
-			return fmt.Errorf("Can't find available subgid range: %v", err)
-		}
-		out, err := execCmd(userMod, fmt.Sprintf(cmdTemplates[userMod], "w", startID, startID+defaultRangeLen-1, name))
-		if err != nil {
-			return fmt.Errorf("Unable to add subgid range to user: %q; output: %s, err: %v", name, out, err)
-		}
-	}
-	return nil
-}
-
-func findNextUIDRange() (int, error) {
-	ranges, err := parseSubuid("ALL")
-	if err != nil {
-		return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subuid file: %v", err)
-	}
-	sort.Sort(ranges)
-	return findNextRangeStart(ranges)
-}
-
-func findNextGIDRange() (int, error) {
-	ranges, err := parseSubgid("ALL")
-	if err != nil {
-		return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subgid file: %v", err)
-	}
-	sort.Sort(ranges)
-	return findNextRangeStart(ranges)
-}
-
-func findNextRangeStart(rangeList ranges) (int, error) {
-	startID := defaultRangeStart
-	for _, arange := range rangeList {
-		if wouldOverlap(arange, startID) {
-			startID = arange.Start + arange.Length
-		}
-	}
-	return startID, nil
-}
-
-func wouldOverlap(arange subIDRange, ID int) bool {
-	low := ID
-	high := ID + defaultRangeLen
-	if (low >= arange.Start && low <= arange.Start+arange.Length) ||
-		(high <= arange.Start+arange.Length && high >= arange.Start) {
-		return true
-	}
-	return false
-}

vendor/github.com/docker/docker/pkg/idtools/usergroupadd_unsupported.go

-// +build !linux
-
-package idtools // import "github.com/docker/docker/pkg/idtools"
-
-import "fmt"
-
-// AddNamespaceRangesUser takes a name and finds an unused uid, gid pair
-// and calls the appropriate helper function to add the group and then
-// the user to the group in /etc/group and /etc/passwd respectively.
-func AddNamespaceRangesUser(name string) (int, int, error) {
-	return -1, -1, fmt.Errorf("No support for adding users or groups on this OS")
-}

vendor/github.com/docker/docker/pkg/idtools/utils_unix.go

-// +build !windows
-
-package idtools // import "github.com/docker/docker/pkg/idtools"
-
-import (
-	"fmt"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-func resolveBinary(binname string) (string, error) {
-	binaryPath, err := exec.LookPath(binname)
-	if err != nil {
-		return "", err
-	}
-	resolvedPath, err := filepath.EvalSymlinks(binaryPath)
-	if err != nil {
-		return "", err
-	}
-	//only return no error if the final resolved binary basename
-	//matches what was searched for
-	if filepath.Base(resolvedPath) == binname {
-		return resolvedPath, nil
-	}
-	return "", fmt.Errorf("Binary %q does not resolve to a binary of that name in $PATH (%q)", binname, resolvedPath)
-}
-
-func execCmd(cmd, args string) ([]byte, error) {
-	execCmd := exec.Command(cmd, strings.Split(args, " ")...)
-	return execCmd.CombinedOutput()
-}

vendor/github.com/rootless-containers/rootlesskit/pkg/child/child.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"os/signal"
 	"runtime"
 	"strconv"
 	"syscall"
@@ -43,39 +44,33 @@ func mountSysfs() error {
 		return errors.Wrap(err, "creating a directory under /tmp")
 	}
 	defer os.RemoveAll(tmp)
-	cmds := [][]string{{"mount", "--rbind", "/sys/fs/cgroup", tmp}}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-		return errors.Wrapf(err, "executing %v", cmds)
+	cgroupDir := "/sys/fs/cgroup"
+	if err := unix.Mount(cgroupDir, tmp, "", uintptr(unix.MS_BIND|unix.MS_REC), ""); err != nil {
+		return errors.Wrapf(err, "failed to create bind mount on %s", cgroupDir)
 	}
-	cmds = [][]string{{"mount", "-t", "sysfs", "none", "/sys"}}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
+
+	if err := unix.Mount("none", "/sys", "sysfs", 0, ""); err != nil {
 		// when the sysfs in the parent namespace is RO,
 		// we can't mount RW sysfs even in the child namespace.
 		// https://github.com/rootless-containers/rootlesskit/pull/23#issuecomment-429292632
 		// https://github.com/torvalds/linux/blob/9f203e2f2f065cd74553e6474f0ae3675f39fb0f/fs/namespace.c#L3326-L3328
-		cmdsRo := [][]string{{"mount", "-t", "sysfs", "-o", "ro", "none", "/sys"}}
-		logrus.Warnf("failed to mount sysfs (%v), falling back to read-only mount (%v): %v",
-			cmds, cmdsRo, err)
-		if err := common.Execs(os.Stderr, os.Environ(), cmdsRo); err != nil {
+		logrus.Warnf("failed to mount sysfs, falling back to read-only mount: %v", err)
+		if err := unix.Mount("none", "/sys", "sysfs", uintptr(unix.MS_RDONLY), ""); err != nil {
 			// when /sys/firmware is masked, even RO sysfs can't be mounted
-			logrus.Warnf("failed to mount sysfs (%v): %v", cmdsRo, err)
+			logrus.Warnf("failed to mount sysfs: %v", err)
 		}
 	}
-	cmds = [][]string{{"mount", "-n", "--move", tmp, "/sys/fs/cgroup"}}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-		return errors.Wrapf(err, "executing %v", cmds)
+	if err := unix.Mount(tmp, cgroupDir, "", uintptr(unix.MS_MOVE), ""); err != nil {
+		return errors.Wrapf(err, "failed to move mount point from %s to %s", tmp, cgroupDir)
 	}
 	return nil
 }
 
 func mountProcfs() error {
-	cmds := [][]string{{"mount", "-t", "proc", "none", "/proc"}}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-		cmdsRo := [][]string{{"mount", "-t", "proc", "-o", "ro", "none", "/proc"}}
-		logrus.Warnf("failed to mount procfs (%v), falling back to read-only mount (%v): %v",
-			cmds, cmdsRo, err)
-		if err := common.Execs(os.Stderr, os.Environ(), cmdsRo); err != nil {
-			logrus.Warnf("failed to mount procfs (%v): %v", cmdsRo, err)
+	if err := unix.Mount("none", "/proc", "proc", 0, ""); err != nil {
+		logrus.Warnf("failed to mount procfs, falling back to read-only mount: %v", err)
+		if err := unix.Mount("none", "/proc", "proc", uintptr(unix.MS_RDONLY), ""); err != nil {
+			logrus.Warnf("failed to mount procfs: %v", err)
 		}
 	}
 	return nil
@@ -171,6 +166,7 @@ type Opt struct {
 	CopyUpDirs    []string
 	PortDriver    port.ChildDriver
 	MountProcfs   bool // needs to be set if (and only if) parent.Opt.CreatePIDNS is set
+	Reaper        bool
 }
 
 func Child(opt Opt) error {
@@ -242,8 +238,14 @@ func Child(opt Opt) error {
 	if err != nil {
 		return err
 	}
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "command %v exited", opt.TargetCmd)
+	if opt.Reaper {
+		if err := runAndReap(cmd); err != nil {
+			return errors.Wrapf(err, "command %v exited", opt.TargetCmd)
+		}
+	} else {
+		if err := cmd.Run(); err != nil {
+			return errors.Wrapf(err, "command %v exited", opt.TargetCmd)
+		}
 	}
 	if opt.PortDriver != nil {
 		portQuitCh <- struct{}{}
@@ -251,3 +253,27 @@ func Child(opt Opt) error {
 	}
 	return nil
 }
+
+func runAndReap(cmd *exec.Cmd) error {
+	c := make(chan os.Signal, 32)
+	signal.Notify(c, syscall.SIGCHLD)
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	result := make(chan error)
+	go func() {
+		defer close(result)
+		for range c {
+			for {
+				if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 {
+					break
+				} else {
+					if pid == cmd.Process.Pid {
+						result <- cmd.Wait()
+					}
+				}
+			}
+		}
+	}()
+	return <-result
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/child/hosts.go

@@ -6,9 +6,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 
-	"github.com/rootless-containers/rootlesskit/pkg/common"
+	"github.com/pkg/errors"
 )
 
 // generateEtcHosts makes sure the current hostname is resolved into
@@ -56,11 +56,9 @@ func mountEtcHosts(tempDir string) error {
 	if err := ioutil.WriteFile(myEtcHosts, newEtcHosts, 0644); err != nil {
 		return errors.Wrapf(err, "writing %s", myEtcHosts)
 	}
-	cmds := [][]string{
-		{"mount", "--bind", myEtcHosts, "/etc/hosts"},
-	}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-		return errors.Wrapf(err, "executing %v", cmds)
+
+	if err := unix.Mount(myEtcHosts, "/etc/hosts", "", uintptr(unix.MS_BIND), ""); err != nil {
+		return errors.Wrapf(err, "failed to create bind mount /etc/hosts for %s", myEtcHosts)
 	}
 	return nil
 }

vendor/github.com/rootless-containers/rootlesskit/pkg/child/resolvconf.go

 package child
 
 import (
+	"golang.org/x/sys/unix"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
-
-	"github.com/rootless-containers/rootlesskit/pkg/common"
 )
 
 func generateResolvConf(dns string) []byte {
@@ -36,11 +35,9 @@ func mountResolvConf(tempDir, dns string) error {
 	if err := ioutil.WriteFile(myResolvConf, generateResolvConf(dns), 0644); err != nil {
 		return errors.Wrapf(err, "writing %s", myResolvConf)
 	}
-	cmds := [][]string{
-		{"mount", "--bind", myResolvConf, "/etc/resolv.conf"},
-	}
-	if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-		return errors.Wrapf(err, "executing %v", cmds)
+
+	if err := unix.Mount(myResolvConf, "/etc/resolv.conf", "", uintptr(unix.MS_BIND), ""); err != nil {
+		return errors.Wrapf(err, "failed to create bind mount /etc/resolv.conf for %s", myResolvConf)
 	}
 	return nil
 }

vendor/github.com/rootless-containers/rootlesskit/pkg/copyup/tmpfssymlink/tmpfssymlink.go

@@ -5,9 +5,10 @@ import (
 	"os"
 	"path/filepath"
 
+	"golang.org/x/sys/unix"
+
 	"github.com/pkg/errors"
 
-	"github.com/rootless-containers/rootlesskit/pkg/common"
 	"github.com/rootless-containers/rootlesskit/pkg/copyup"
 )
 
@@ -33,24 +34,23 @@ func (d *childDriver) CopyUp(dirs []string) ([]string, error) {
 			// TODO: we can support copy-up /tmp by changing bind0TempDir
 			return copied, errors.New("/tmp cannot be copied up")
 		}
-		cmds := [][]string{
-			// TODO: read-only bind (does not work well for /run)
-			{"mount", "--rbind", d, bind0},
-			{"mount", "-n", "-t", "tmpfs", "none", d},
+
+		if err := unix.Mount(d, bind0, "", uintptr(unix.MS_BIND|unix.MS_REC), ""); err != nil {
+			return copied, errors.Wrapf(err, "failed to create bind mount on %s", d)
 		}
-		if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-			return copied, errors.Wrapf(err, "executing %v", cmds)
+
+		if err := unix.Mount("none", d, "tmpfs", 0, ""); err != nil {
+			return copied, errors.Wrapf(err, "failed to mount tmpfs on %s", d)
 		}
+
 		bind1, err := ioutil.TempDir(d, ".ro")
 		if err != nil {
 			return copied, errors.Wrapf(err, "creating a directory under %s", d)
 		}
-		cmds = [][]string{
-			{"mount", "-n", "--move", bind0, bind1},
-		}
-		if err := common.Execs(os.Stderr, os.Environ(), cmds); err != nil {
-			return copied, errors.Wrapf(err, "executing %v", cmds)
+		if err := unix.Mount(bind0, bind1, "", uintptr(unix.MS_MOVE), ""); err != nil {
+			return copied, errors.Wrapf(err, "failed to move mount point from %s to %s", bind0, bind1)
 		}
+
 		files, err := ioutil.ReadDir(bind1)
 		if err != nil {
 			return copied, errors.Wrapf(err, "reading dir %s", bind1)

vendor/github.com/rootless-containers/rootlesskit/pkg/network/slirp4netns/slirp4netns.go

@@ -3,12 +3,16 @@ package slirp4netns
 import (
 	"context"
 	"net"
+	"os"
 	"os/exec"
 	"strconv"
+	"strings"
 	"syscall"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 
 	"github.com/rootless-containers/rootlesskit/pkg/common"
 	"github.com/rootless-containers/rootlesskit/pkg/network"
@@ -16,13 +20,62 @@ import (
 	"github.com/rootless-containers/rootlesskit/pkg/network/parentutils"
 )
 
+type Features struct {
+	// SupportsCIDR --cidr (v0.3.0)
+	SupportsCIDR bool
+	// SupportsDisableHostLoopback --disable-host-loopback (v0.3.0)
+	SupportsDisableHostLoopback bool
+	// SupportsAPISocket --api-socket (v0.3.0)
+	SupportsAPISocket bool
+	// SupportsEnableSandbox --enable-sandbox (v0.4.0)
+	SupportsEnableSandbox bool
+	// SupportsEnableSeccomp --enable-seccomp (v0.4.0)
+	SupportsEnableSeccomp bool
+	// KernelSupportsSeccomp whether the kernel supports slirp4netns --enable-seccomp
+	KernelSupportsEnableSeccomp bool
+}
+
+func DetectFeatures(binary string) (*Features, error) {
+	if binary == "" {
+		return nil, errors.New("got empty slirp4netns binary")
+	}
+	realBinary, err := exec.LookPath(binary)
+	if err != nil {
+		return nil, errors.Wrapf(err, "slirp4netns binary %q is not installed", binary)
+	}
+	cmd := exec.Command(realBinary, "--help")
+	cmd.Env = os.Environ()
+	b, err := cmd.CombinedOutput()
+	s := string(b)
+	if err != nil {
+		return nil, errors.Wrapf(err,
+			"command \"%s --help\" failed, make sure slirp4netns v0.2.0+ is installed: %q",
+			realBinary, s)
+	}
+	kernelSupportsEnableSeccomp := false
+	if unix.Prctl(unix.PR_GET_SECCOMP, 0, 0, 0, 0) != unix.EINVAL {
+		kernelSupportsEnableSeccomp = unix.Prctl(unix.PR_SET_SECCOMP, unix.SECCOMP_MODE_FILTER, 0, 0, 0) != unix.EINVAL
+	}
+	f := Features{
+		SupportsCIDR:                strings.Contains(s, "--cidr"),
+		SupportsDisableHostLoopback: strings.Contains(s, "--disable-host-loopback"),
+		SupportsAPISocket:           strings.Contains(s, "--api-socket"),
+		SupportsEnableSandbox:       strings.Contains(s, "--enable-sandbox"),
+		SupportsEnableSeccomp:       strings.Contains(s, "--enable-seccomp"),
+		KernelSupportsEnableSeccomp: kernelSupportsEnableSeccomp,
+	}
+	return &f, nil
+}
+
 // NewParentDriver instantiates new parent driver.
 // ipnet is supported only for slirp4netns v0.3.0+.
 // ipnet MUST be nil for slirp4netns < v0.3.0.
 //
 // disableHostLoopback is supported only for slirp4netns v0.3.0+
 // apiSocketPath is supported only for slirp4netns v0.3.0+
-func NewParentDriver(binary string, mtu int, ipnet *net.IPNet, disableHostLoopback bool, apiSocketPath string) network.ParentDriver {
+// enableSandbox is supported only for slirp4netns v0.4.0+
+// enableSeccomp is supported only for slirp4netns v0.4.0+
+func NewParentDriver(binary string, mtu int, ipnet *net.IPNet, disableHostLoopback bool, apiSocketPath string, enableSandbox, enableSeccomp bool) network.ParentDriver {
 	if binary == "" {
 		panic("got empty slirp4netns binary")
 	}
@@ -38,6 +91,8 @@ func NewParentDriver(binary string, mtu int, ipnet *net.IPNet, disableHostLoopba
 		ipnet:               ipnet,
 		disableHostLoopback: disableHostLoopback,
 		apiSocketPath:       apiSocketPath,
+		enableSandbox:       enableSandbox,
+		enableSeccomp:       enableSeccomp,
 	}
 }
 
@@ -47,6 +102,8 @@ type parentDriver struct {
 	ipnet               *net.IPNet
 	disableHostLoopback bool
 	apiSocketPath       string
+	enableSandbox       bool
+	enableSeccomp       bool
 }
 
 func (d *parentDriver) MTU() int {
@@ -60,7 +117,14 @@ func (d *parentDriver) ConfigureNetwork(childPID int, stateDir string) (*common.
 		return nil, common.Seq(cleanups), errors.Wrapf(err, "setting up tap %s", tap)
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-	opts := []string{"--mtu", strconv.Itoa(d.mtu)}
+	readyR, readyW, err := os.Pipe()
+	if err != nil {
+		return nil, common.Seq(cleanups), err
+	}
+	defer readyR.Close()
+	defer readyW.Close()
+	// -r: readyFD
+	opts := []string{"--mtu", strconv.Itoa(d.mtu), "-r", "3"}
 	if d.disableHostLoopback {
 		opts = append(opts, "--disable-host-loopback")
 	}
@@ -70,10 +134,17 @@ func (d *parentDriver) ConfigureNetwork(childPID int, stateDir string) (*common.
 	if d.apiSocketPath != "" {
 		opts = append(opts, "--api-socket", d.apiSocketPath)
 	}
+	if d.enableSandbox {
+		opts = append(opts, "--enable-sandbox")
+	}
+	if d.enableSeccomp {
+		opts = append(opts, "--enable-seccomp")
+	}
 	cmd := exec.CommandContext(ctx, d.binary, append(opts, []string{strconv.Itoa(childPID), tap}...)...)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Pdeathsig: syscall.SIGKILL,
 	}
+	cmd.ExtraFiles = append(cmd.ExtraFiles, readyW)
 	cleanups = append(cleanups, func() error {
 		logrus.Debugf("killing slirp4netns")
 		cancel()
@@ -84,6 +155,10 @@ func (d *parentDriver) ConfigureNetwork(childPID int, stateDir string) (*common.
 	if err := cmd.Start(); err != nil {
 		return nil, common.Seq(cleanups), errors.Wrapf(err, "executing %v", cmd)
 	}
+
+	if err := waitForReadyFD(cmd.Process.Pid, readyR); err != nil {
+		return nil, common.Seq(cleanups), errors.Wrapf(err, "waiting for ready fd (%v)", cmd)
+	}
 	netmsg := common.NetworkMessage{
 		Dev: tap,
 		MTU: d.mtu,
@@ -115,6 +190,41 @@ func (d *parentDriver) ConfigureNetwork(childPID int, stateDir string) (*common.
 	return &netmsg, common.Seq(cleanups), nil
 }
 
+// waitForReady is from libpod
+// https://github.com/containers/libpod/blob/e6b843312b93ddaf99d0ef94a7e60ff66bc0eac8/libpod/networking_linux.go#L272-L308
+func waitForReadyFD(cmdPid int, r *os.File) error {
+	b := make([]byte, 16)
+	for {
+		if err := r.SetDeadline(time.Now().Add(1 * time.Second)); err != nil {
+			return errors.Wrapf(err, "error setting slirp4netns pipe timeout")
+		}
+		if _, err := r.Read(b); err == nil {
+			break
+		} else {
+			if os.IsTimeout(err) {
+				// Check if the process is still running.
+				var status syscall.WaitStatus
+				pid, err := syscall.Wait4(cmdPid, &status, syscall.WNOHANG, nil)
+				if err != nil {
+					return errors.Wrapf(err, "failed to read slirp4netns process status")
+				}
+				if pid != cmdPid {
+					continue
+				}
+				if status.Exited() {
+					return errors.New("slirp4netns failed")
+				}
+				if status.Signaled() {
+					return errors.New("slirp4netns killed by signal")
+				}
+				continue
+			}
+			return errors.Wrapf(err, "failed to read from slirp4netns sync pipe")
+		}
+	}
+	return nil
+}
+
 func NewChildDriver() network.ChildDriver {
 	return &childDriver{}
 }

vendor/github.com/docker/docker/pkg/idtools/idtools.go → vendor/github.com/rootless-containers/rootlesskit/pkg/parent/idtools/idtools.go

-package idtools // import "github.com/docker/docker/pkg/idtools"
+// Package idtools is forked from https://github.com/moby/moby/tree/c12f09bf99b54f274a5ae241dd154fa74020cbab/pkg/idtools
+package idtools
 
 import (
 	"bufio"
 	"fmt"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -33,28 +35,6 @@ const (
 	subgidFileName = "/etc/subgid"
 )
 
-// MkdirAllAndChown creates a directory (include any along the path) and then modifies
-// ownership to the requested uid/gid.  If the directory already exists, this
-// function will still change ownership to the requested uid/gid pair.
-func MkdirAllAndChown(path string, mode os.FileMode, owner Identity) error {
-	return mkdirAs(path, mode, owner, true, true)
-}
-
-// MkdirAndChown creates a directory and then modifies ownership to the requested uid/gid.
-// If the directory already exists, this function still changes ownership.
-// Note that unlike os.Mkdir(), this function does not return IsExist error
-// in case path already exists.
-func MkdirAndChown(path string, mode os.FileMode, owner Identity) error {
-	return mkdirAs(path, mode, owner, false, true)
-}
-
-// MkdirAllAndChownNew creates a directory (include any along the path) and then modifies
-// ownership ONLY of newly created directories to the requested uid/gid. If the
-// directories along the path exist, no change of ownership will be performed
-func MkdirAllAndChownNew(path string, mode os.FileMode, owner Identity) error {
-	return mkdirAs(path, mode, owner, true, false)
-}
-
 // GetRootUIDGID retrieves the remapped root uid/gid pair from the set of maps.
 // If the maps are empty, then the root uid/gid will default to "real" 0/0
 func GetRootUIDGID(uidMap, gidMap []IDMap) (int, int, error) {
@@ -202,6 +182,8 @@ func (i *IdentityMapping) GIDs() []IDMap {
 func createIDMap(subidRanges ranges) []IDMap {
 	idMap := []IDMap{}
 
+	// sort the ranges by lowest ID first
+	sort.Sort(subidRanges)
 	containerID := 0
 	for _, idrange := range subidRanges {
 		idMap = append(idMap, IDMap{

vendor/github.com/rootless-containers/rootlesskit/pkg/parent/parent.go

@@ -12,7 +12,6 @@ import (
 	"strconv"
 	"syscall"
 
-	"github.com/docker/docker/pkg/idtools"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -22,6 +21,7 @@ import (
 	"github.com/rootless-containers/rootlesskit/pkg/common"
 	"github.com/rootless-containers/rootlesskit/pkg/msgutil"
 	"github.com/rootless-containers/rootlesskit/pkg/network"
+	"github.com/rootless-containers/rootlesskit/pkg/parent/idtools"
 	"github.com/rootless-containers/rootlesskit/pkg/port"
 )
 
@@ -103,10 +103,6 @@ func Parent(opt Opt) error {
 	if err := cmd.Start(); err != nil {
 		return errors.Wrap(err, "failed to start the child")
 	}
-	childPIDPath := filepath.Join(opt.StateDir, StateFileChildPID)
-	if err := ioutil.WriteFile(childPIDPath, []byte(strconv.Itoa(cmd.Process.Pid)), 0444); err != nil {
-		return errors.Wrapf(err, "failed to write the child PID %d to %s", cmd.Process.Pid, childPIDPath)
-	}
 	if err := setupUIDGIDMap(cmd.Process.Pid); err != nil {
 		return errors.Wrap(err, "failed to setup UID/GID map")
 	}
@@ -176,6 +172,12 @@ func Parent(opt Opt) error {
 			logrus.Debugf("published port %v", st)
 		}
 	}
+
+	// after child is fully configured, write PID to child_pid file
+	childPIDPath := filepath.Join(opt.StateDir, StateFileChildPID)
+	if err := ioutil.WriteFile(childPIDPath, []byte(strconv.Itoa(cmd.Process.Pid)), 0444); err != nil {
+		return errors.Wrapf(err, "failed to write the child PID %d to %s", cmd.Process.Pid, childPIDPath)
+	}
 	// listens the API
 	apiSockPath := filepath.Join(opt.StateDir, StateFileAPISock)
 	apiCloser, err := listenServeAPI(apiSockPath, &router.Backend{PortDriver: opt.PortDriver})

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/child/child.go

+package child
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+
+	"github.com/rootless-containers/rootlesskit/pkg/msgutil"
+	"github.com/rootless-containers/rootlesskit/pkg/port"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg"
+	opaquepkg "github.com/rootless-containers/rootlesskit/pkg/port/builtin/opaque"
+)
+
+func NewDriver(logWriter io.Writer) port.ChildDriver {
+	return &childDriver{
+		logWriter: logWriter,
+	}
+}
+
+type childDriver struct {
+	logWriter io.Writer
+}
+
+func (d *childDriver) RunChildDriver(opaque map[string]string, quit <-chan struct{}) error {
+	socketPath := opaque[opaquepkg.SocketPath]
+	if socketPath == "" {
+		return errors.New("socket path not set")
+	}
+	childReadyPipePath := opaque[opaquepkg.ChildReadyPipePath]
+	if childReadyPipePath == "" {
+		return errors.New("child ready pipe path not set")
+	}
+	childReadyPipeW, err := os.OpenFile(childReadyPipePath, os.O_WRONLY, os.ModeNamedPipe)
+	if err != nil {
+		return err
+	}
+	ln, err := net.ListenUnix("unix", &net.UnixAddr{
+		Name: socketPath,
+		Net:  "unix",
+	})
+	if err != nil {
+		return err
+	}
+	// write nothing, just close
+	if err = childReadyPipeW.Close(); err != nil {
+		return err
+	}
+	stopAccept := make(chan struct{}, 1)
+	go func() {
+		<-quit
+		stopAccept <- struct{}{}
+		ln.Close()
+	}()
+	for {
+		c, err := ln.AcceptUnix()
+		if err != nil {
+			select {
+			case <-stopAccept:
+				return nil
+			default:
+			}
+			return err
+		}
+		go func() {
+			if rerr := d.routine(c); rerr != nil {
+				rep := msg.Reply{
+					Error: rerr.Error(),
+				}
+				msgutil.MarshalToWriter(c, &rep)
+			}
+			c.Close()
+		}()
+	}
+	return nil
+}
+
+func (d *childDriver) routine(c *net.UnixConn) error {
+	var req msg.Request
+	if _, err := msgutil.UnmarshalFromReader(c, &req); err != nil {
+		return err
+	}
+	switch req.Type {
+	case msg.RequestTypeInit:
+		return d.handleConnectInit(c, &req)
+	case msg.RequestTypeConnect:
+		return d.handleConnectRequest(c, &req)
+	default:
+		return errors.Errorf("unknown request type %q", req.Type)
+	}
+}
+
+func (d *childDriver) handleConnectInit(c *net.UnixConn, req *msg.Request) error {
+	_, err := msgutil.MarshalToWriter(c, nil)
+	return err
+}
+
+func (d *childDriver) handleConnectRequest(c *net.UnixConn, req *msg.Request) error {
+	switch req.Proto {
+	case "tcp":
+	case "udp":
+	default:
+		return errors.Errorf("unknown proto: %q", req.Proto)
+	}
+	var dialer net.Dialer
+	targetConn, err := dialer.Dial(req.Proto, fmt.Sprintf("127.0.0.1:%d", req.Port))
+	if err != nil {
+		return err
+	}
+	defer targetConn.Close() // no effect on duplicated FD
+	targetConnFiler, ok := targetConn.(filer)
+	if !ok {
+		return errors.Errorf("unknown target connection: %+v", targetConn)
+	}
+	targetConnFile, err := targetConnFiler.File()
+	if err != nil {
+		return err
+	}
+	oob := unix.UnixRights(int(targetConnFile.Fd()))
+	f, err := c.File()
+	if err != nil {
+		return err
+	}
+	err = unix.Sendmsg(int(f.Fd()), []byte("dummy"), oob, nil, 0)
+	return err
+}
+
+// filer is implemented by *net.TCPConn and *net.UDPConn
+type filer interface {
+	File() (f *os.File, err error)
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg/msg.go

+package msg
+
+import (
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+
+	"github.com/rootless-containers/rootlesskit/pkg/msgutil"
+	"github.com/rootless-containers/rootlesskit/pkg/port"
+)
+
+const (
+	RequestTypeInit    = "init"
+	RequestTypeConnect = "connect"
+)
+
+// Request and Response are encoded as JSON with uint32le length header.
+type Request struct {
+	Type  string // "init" or "connect"
+	Proto string // "tcp" or "udp"
+	Port  int
+}
+
+// Reply may contain FD as OOB
+type Reply struct {
+	Error string
+}
+
+// Initiate sends "init" request to the child UNIX socket.
+func Initiate(c *net.UnixConn) error {
+	req := Request{
+		Type: RequestTypeInit,
+	}
+	if _, err := msgutil.MarshalToWriter(c, &req); err != nil {
+		return err
+	}
+	if err := c.CloseWrite(); err != nil {
+		return err
+	}
+	var rep Reply
+	if _, err := msgutil.UnmarshalFromReader(c, &rep); err != nil {
+		return err
+	}
+	return c.CloseRead()
+}
+
+// ConnectToChild connects to the child UNIX socket, and obtains TCP or UDP socket FD
+// that corresponds to the port spec.
+func ConnectToChild(c *net.UnixConn, spec port.Spec) (int, error) {
+	req := Request{
+		Type:  RequestTypeConnect,
+		Proto: spec.Proto,
+		Port:  spec.ChildPort,
+	}
+	if _, err := msgutil.MarshalToWriter(c, &req); err != nil {
+		return 0, err
+	}
+	if err := c.CloseWrite(); err != nil {
+		return 0, err
+	}
+	oobSpace := unix.CmsgSpace(4)
+	oob := make([]byte, oobSpace)
+	_, oobN, _, _, err := c.ReadMsgUnix(nil, oob)
+	if err != nil {
+		return 0, err
+	}
+	if oobN != oobSpace {
+		return 0, errors.Errorf("expected OOB space %d, got %d", oobSpace, oobN)
+	}
+	oob = oob[:oobN]
+	fd, err := parseFDFromOOB(oob)
+	if err != nil {
+		return 0, err
+	}
+	if err := c.CloseRead(); err != nil {
+		return 0, err
+	}
+	return fd, nil
+}
+
+// ConnectToChildWithSocketPath wraps ConnectToChild
+func ConnectToChildWithSocketPath(socketPath string, spec port.Spec) (int, error) {
+	var dialer net.Dialer
+	conn, err := dialer.Dial("unix", socketPath)
+	if err != nil {
+		return 0, err
+	}
+	defer conn.Close()
+	c := conn.(*net.UnixConn)
+	return ConnectToChild(c, spec)
+}
+
+// ConnectToChildWithRetry retries ConnectToChild every (i*5) milliseconds.
+func ConnectToChildWithRetry(socketPath string, spec port.Spec, retries int) (int, error) {
+	for i := 0; i < retries; i++ {
+		fd, err := ConnectToChildWithSocketPath(socketPath, spec)
+		if i == retries-1 && err != nil {
+			return 0, err
+		}
+		if err == nil {
+			return fd, err
+		}
+		// TODO: backoff
+		time.Sleep(time.Duration(i*5) * time.Millisecond)
+	}
+	// NOT REACHED
+	return 0, errors.New("reached max retry")
+}
+
+func parseFDFromOOB(oob []byte) (int, error) {
+	scms, err := unix.ParseSocketControlMessage(oob)
+	if err != nil {
+		return 0, err
+	}
+	if len(scms) != 1 {
+		return 0, errors.Errorf("unexpected scms: %v", scms)
+	}
+	scm := scms[0]
+	fds, err := unix.ParseUnixRights(&scm)
+	if err != nil {
+		return 0, err
+	}
+	if len(fds) != 1 {
+		return 0, errors.Errorf("unexpected fds: %v", fds)
+	}
+	return fds[0], nil
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/opaque/opaque.go

+package opaque
+
+const (
+	SocketPath         = "builtin.socketpath"
+	ChildReadyPipePath = "builtin.readypipepath"
+)

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/parent.go

+package parent
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	"github.com/rootless-containers/rootlesskit/pkg/port"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/opaque"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/tcp"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp"
+	"github.com/rootless-containers/rootlesskit/pkg/port/portutil"
+)
+
+// NewDriver for builtin driver.
+func NewDriver(logWriter io.Writer, stateDir string) (port.ParentDriver, error) {
+	// TODO: consider using socketpair FD instead of socket file
+	socketPath := filepath.Join(stateDir, ".bp.sock")
+	childReadyPipePath := filepath.Join(stateDir, ".bp-ready.pipe")
+	// remove the path just in case the previous rootlesskit instance crashed
+	if err := os.RemoveAll(childReadyPipePath); err != nil {
+		return nil, errors.Wrapf(err, "cannot remove %s", childReadyPipePath)
+	}
+	if err := syscall.Mkfifo(childReadyPipePath, 0600); err != nil {
+		return nil, errors.Wrapf(err, "cannot mkfifo %s", childReadyPipePath)
+	}
+	d := driver{
+		logWriter:          logWriter,
+		socketPath:         socketPath,
+		childReadyPipePath: childReadyPipePath,
+		ports:              make(map[int]*port.Status, 0),
+		stoppers:           make(map[int]func() error, 0),
+		nextID:             1,
+	}
+	return &d, nil
+}
+
+type driver struct {
+	logWriter          io.Writer
+	socketPath         string
+	childReadyPipePath string
+	mu                 sync.Mutex
+	ports              map[int]*port.Status
+	stoppers           map[int]func() error
+	nextID             int
+}
+
+func (d *driver) OpaqueForChild() map[string]string {
+	return map[string]string{
+		opaque.SocketPath:         d.socketPath,
+		opaque.ChildReadyPipePath: d.childReadyPipePath,
+	}
+}
+
+func (d *driver) RunParentDriver(initComplete chan struct{}, quit <-chan struct{}, _ *port.ChildContext) error {
+	childReadyPipeR, err := os.OpenFile(d.childReadyPipePath, os.O_RDONLY, os.ModeNamedPipe)
+	if err != nil {
+		return err
+	}
+	if _, err = ioutil.ReadAll(childReadyPipeR); err != nil {
+		return err
+	}
+	childReadyPipeR.Close()
+	var dialer net.Dialer
+	conn, err := dialer.Dial("unix", d.socketPath)
+	if err != nil {
+		return err
+	}
+	err = msg.Initiate(conn.(*net.UnixConn))
+	conn.Close()
+	if err != nil {
+		return err
+	}
+	initComplete <- struct{}{}
+	<-quit
+	return nil
+}
+
+func (d *driver) AddPort(ctx context.Context, spec port.Spec) (*port.Status, error) {
+	d.mu.Lock()
+	err := portutil.ValidatePortSpec(spec, d.ports)
+	d.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	routineStopCh := make(chan struct{})
+	routineStop := func() error {
+		close(routineStopCh)
+		return nil // FIXME
+	}
+	switch spec.Proto {
+	case "tcp":
+		err = tcp.Run(d.socketPath, spec, routineStopCh, d.logWriter)
+	case "udp":
+		err = udp.Run(d.socketPath, spec, routineStopCh, d.logWriter)
+	default:
+		// NOTREACHED
+		return nil, errors.New("spec was not validated?")
+	}
+	if err != nil {
+		return nil, err
+	}
+	d.mu.Lock()
+	id := d.nextID
+	st := port.Status{
+		ID:   id,
+		Spec: spec,
+	}
+	d.ports[id] = &st
+	d.stoppers[id] = routineStop
+	d.nextID++
+	d.mu.Unlock()
+	return &st, nil
+}
+
+func (d *driver) ListPorts(ctx context.Context) ([]port.Status, error) {
+	var ports []port.Status
+	d.mu.Lock()
+	for _, p := range d.ports {
+		ports = append(ports, *p)
+	}
+	d.mu.Unlock()
+	return ports, nil
+}
+
+func (d *driver) RemovePort(ctx context.Context, id int) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	stop, ok := d.stoppers[id]
+	if !ok {
+		return errors.Errorf("unknown id: %d", id)
+	}
+	err := stop()
+	delete(d.stoppers, id)
+	delete(d.ports, id)
+	return err
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/tcp/tcp.go

+package tcp
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"sync"
+
+	"github.com/rootless-containers/rootlesskit/pkg/port"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg"
+)
+
+func Run(socketPath string, spec port.Spec, stopCh <-chan struct{}, logWriter io.Writer) error {
+	ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", spec.ParentIP, spec.ParentPort))
+	if err != nil {
+		fmt.Fprintf(logWriter, "listen: %v\n", err)
+		return err
+	}
+	newConns := make(chan net.Conn)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				fmt.Fprintf(logWriter, "accept: %v\n", err)
+				close(newConns)
+				return
+			}
+			newConns <- c
+		}
+	}()
+	go func() {
+		defer ln.Close()
+		for {
+			select {
+			case c, ok := <-newConns:
+				if !ok {
+					return
+				}
+				go func() {
+					if err := copyConnToChild(c, socketPath, spec, stopCh); err != nil {
+						fmt.Fprintf(logWriter, "copyConnToChild: %v\n", err)
+						return
+					}
+				}()
+			case <-stopCh:
+				return
+			}
+		}
+	}()
+	// no wait
+	return nil
+}
+
+func copyConnToChild(c net.Conn, socketPath string, spec port.Spec, stopCh <-chan struct{}) error {
+	defer c.Close()
+	// get fd from the child as an SCM_RIGHTS cmsg
+	fd, err := msg.ConnectToChildWithRetry(socketPath, spec, 10)
+	if err != nil {
+		return err
+	}
+	f := os.NewFile(uintptr(fd), "")
+	defer f.Close()
+	fc, err := net.FileConn(f)
+	if err != nil {
+		return err
+	}
+	defer fc.Close()
+	bicopy(c, fc, stopCh)
+	return nil
+}
+
+// bicopy is based on libnetwork/cmd/proxy/tcp_proxy.go .
+// NOTE: sendfile(2) cannot be used for sockets
+func bicopy(x, y net.Conn, quit <-chan struct{}) {
+	var wg sync.WaitGroup
+	var broker = func(to, from net.Conn) {
+		io.Copy(to, from)
+		if fromTCP, ok := from.(*net.TCPConn); ok {
+			fromTCP.CloseRead()
+		}
+		if toTCP, ok := to.(*net.TCPConn); ok {
+			toTCP.CloseWrite()
+		}
+		wg.Done()
+	}
+
+	wg.Add(2)
+	go broker(x, y)
+	go broker(y, x)
+	finish := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(finish)
+	}()
+
+	select {
+	case <-quit:
+	case <-finish:
+	}
+	x.Close()
+	y.Close()
+	<-finish
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp/udp.go

+package udp
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"github.com/rootless-containers/rootlesskit/pkg/port"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg"
+	"github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp/udpproxy"
+)
+
+func Run(socketPath string, spec port.Spec, stopCh <-chan struct{}, logWriter io.Writer) error {
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", spec.ParentIP, spec.ParentPort))
+	if err != nil {
+		return err
+	}
+	c, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		return err
+	}
+	udpp := &udpproxy.UDPProxy{
+		LogWriter: logWriter,
+		Listener:  c,
+		BackendDial: func() (*net.UDPConn, error) {
+			// get fd from the child as an SCM_RIGHTS cmsg
+			fd, err := msg.ConnectToChildWithRetry(socketPath, spec, 10)
+			if err != nil {
+				return nil, err
+			}
+			f := os.NewFile(uintptr(fd), "")
+			defer f.Close()
+			fc, err := net.FileConn(f)
+			if err != nil {
+				return nil, err
+			}
+			uc, ok := fc.(*net.UDPConn)
+			if !ok {
+				return nil, errors.Errorf("file conn doesn't implement *net.UDPConn: %+v", fc)
+			}
+			return uc, nil
+		},
+	}
+	go udpp.Run()
+	go func() {
+		for {
+			select {
+			case <-stopCh:
+				// udpp.Close closes ln as well
+				udpp.Close()
+				return
+			}
+		}
+	}()
+	// no wait
+	return nil
+}

vendor/github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp/udpproxy/udp_proxy.go

+// Package udpproxy is from https://raw.githubusercontent.com/docker/libnetwork/fec6476dfa21380bf8ee4d74048515d968c1ee63/cmd/proxy/udp_proxy.go
+package udpproxy
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+const (
+	// UDPConnTrackTimeout is the timeout used for UDP connection tracking
+	UDPConnTrackTimeout = 90 * time.Second
+	// UDPBufSize is the buffer size for the UDP proxy
+	UDPBufSize = 65507
+)
+
+// A net.Addr where the IP is split into two fields so you can use it as a key
+// in a map:
+type connTrackKey struct {
+	IPHigh uint64
+	IPLow  uint64
+	Port   int
+}
+
+func newConnTrackKey(addr *net.UDPAddr) *connTrackKey {
+	if len(addr.IP) == net.IPv4len {
+		return &connTrackKey{
+			IPHigh: 0,
+			IPLow:  uint64(binary.BigEndian.Uint32(addr.IP)),
+			Port:   addr.Port,
+		}
+	}
+	return &connTrackKey{
+		IPHigh: binary.BigEndian.Uint64(addr.IP[:8]),
+		IPLow:  binary.BigEndian.Uint64(addr.IP[8:]),
+		Port:   addr.Port,
+	}
+}
+
+type connTrackMap map[connTrackKey]*net.UDPConn
+
+// UDPProxy is proxy for which handles UDP datagrams.
+// From libnetwork udp_proxy.go .
+type UDPProxy struct {
+	LogWriter      io.Writer
+	Listener       *net.UDPConn
+	BackendDial    func() (*net.UDPConn, error)
+	connTrackTable connTrackMap
+	connTrackLock  sync.Mutex
+}
+
+func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) {
+	defer func() {
+		proxy.connTrackLock.Lock()
+		delete(proxy.connTrackTable, *clientKey)
+		proxy.connTrackLock.Unlock()
+		proxyConn.Close()
+	}()
+
+	readBuf := make([]byte, UDPBufSize)
+	for {
+		proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout))
+	again:
+		read, err := proxyConn.Read(readBuf)
+		if err != nil {
+			if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED {
+				// This will happen if the last write failed
+				// (e.g: nothing is actually listening on the
+				// proxied port on the container), ignore it
+				// and continue until UDPConnTrackTimeout
+				// expires:
+				goto again
+			}
+			return
+		}
+		for i := 0; i != read; {
+			written, err := proxy.Listener.WriteToUDP(readBuf[i:read], clientAddr)
+			if err != nil {
+				return
+			}
+			i += written
+		}
+	}
+}
+
+// Run starts forwarding the traffic using UDP.
+func (proxy *UDPProxy) Run() {
+	proxy.connTrackTable = make(connTrackMap)
+	readBuf := make([]byte, UDPBufSize)
+	for {
+		read, from, err := proxy.Listener.ReadFromUDP(readBuf)
+		if err != nil {
+			// NOTE: Apparently ReadFrom doesn't return
+			// ECONNREFUSED like Read do (see comment in
+			// UDPProxy.replyLoop)
+			if !isClosedError(err) {
+				fmt.Fprintf(proxy.LogWriter, "Stopping proxy on udp: %v\n", err)
+			}
+			break
+		}
+
+		fromKey := newConnTrackKey(from)
+		proxy.connTrackLock.Lock()
+		proxyConn, hit := proxy.connTrackTable[*fromKey]
+		if !hit {
+			proxyConn, err = proxy.BackendDial()
+			if err != nil {
+				fmt.Fprintf(proxy.LogWriter, "Can't proxy a datagram to udp: %v\n", err)
+				proxy.connTrackLock.Unlock()
+				continue
+			}
+			proxy.connTrackTable[*fromKey] = proxyConn
+			go proxy.replyLoop(proxyConn, from, fromKey)
+		}
+		proxy.connTrackLock.Unlock()
+		for i := 0; i != read; {
+			written, err := proxyConn.Write(readBuf[i:read])
+			if err != nil {
+				fmt.Fprintf(proxy.LogWriter, "Can't proxy a datagram to udp: %v\n", err)
+				break
+			}
+			i += written
+		}
+	}
+}
+
+// Close stops forwarding the traffic.
+func (proxy *UDPProxy) Close() {
+	proxy.Listener.Close()
+	proxy.connTrackLock.Lock()
+	defer proxy.connTrackLock.Unlock()
+	for _, conn := range proxy.connTrackTable {
+		conn.Close()
+	}
+}
+
+func isClosedError(err error) bool {
+	/* This comparison is ugly, but unfortunately, net.go doesn't export errClosing.
+	 * See:
+	 * http://golang.org/src/pkg/net/net.go
+	 * https://code.google.com/p/go/issues/detail?id=4337
+	 * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ
+	 */
+	return strings.HasSuffix(err.Error(), "use of closed network connection")
+}

vendor/modules.txt

@@ -151,7 +151,7 @@ github.com/container-storage-interface/spec/lib/go/csi
 github.com/containerd/cgroups
 # github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 => github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
 github.com/containerd/console
-# github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69 => github.com/rancher/containerd v1.3.3-k3s1
+# github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69 => github.com/rancher/containerd v1.3.3-k3s2
 github.com/containerd/containerd
 github.com/containerd/containerd/api/events
 github.com/containerd/containerd/api/services/containers/v1
@@ -288,7 +288,7 @@ github.com/containerd/continuity/pathdriver
 github.com/containerd/continuity/proto
 github.com/containerd/continuity/syscallx
 github.com/containerd/continuity/sysx
-# github.com/containerd/cri v0.0.0-00010101000000-000000000000 => github.com/rancher/cri v1.3.0-k3s.3
+# github.com/containerd/cri v0.0.0-00010101000000-000000000000 => github.com/rancher/cri v1.3.0-k3s.4
 github.com/containerd/cri
 github.com/containerd/cri/pkg/annotations
 github.com/containerd/cri/pkg/api/runtimeoptions/v1
@@ -306,6 +306,7 @@ github.com/containerd/cri/pkg/server/io
 github.com/containerd/cri/pkg/store
 github.com/containerd/cri/pkg/store/container
 github.com/containerd/cri/pkg/store/image
+github.com/containerd/cri/pkg/store/label
 github.com/containerd/cri/pkg/store/sandbox
 github.com/containerd/cri/pkg/store/snapshot
 github.com/containerd/cri/pkg/util
@@ -383,7 +384,6 @@ github.com/docker/docker/api/types/volume
 github.com/docker/docker/client
 github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog
 github.com/docker/docker/errdefs
-github.com/docker/docker/pkg/idtools
 github.com/docker/docker/pkg/jsonmessage
 github.com/docker/docker/pkg/longpath
 github.com/docker/docker/pkg/mount
@@ -774,7 +774,7 @@ github.com/rancher/wrangler-api/pkg/generated/controllers/rbac
 github.com/rancher/wrangler-api/pkg/generated/controllers/rbac/v1
 # github.com/robfig/cron v1.1.0
 github.com/robfig/cron
-# github.com/rootless-containers/rootlesskit v0.6.0
+# github.com/rootless-containers/rootlesskit v0.7.2
 github.com/rootless-containers/rootlesskit/pkg/api/client
 github.com/rootless-containers/rootlesskit/pkg/api/router
 github.com/rootless-containers/rootlesskit/pkg/child
@@ -787,8 +787,16 @@ github.com/rootless-containers/rootlesskit/pkg/network/iputils
 github.com/rootless-containers/rootlesskit/pkg/network/parentutils
 github.com/rootless-containers/rootlesskit/pkg/network/slirp4netns
 github.com/rootless-containers/rootlesskit/pkg/parent
+github.com/rootless-containers/rootlesskit/pkg/parent/idtools
 github.com/rootless-containers/rootlesskit/pkg/port
 github.com/rootless-containers/rootlesskit/pkg/port/builtin
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/child
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/msg
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/opaque
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/tcp
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp
+github.com/rootless-containers/rootlesskit/pkg/port/builtin/parent/udp/udpproxy
 github.com/rootless-containers/rootlesskit/pkg/port/portutil
 # github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c
 github.com/rubiojr/go-vhd/vhd