Unverified Commit 702fe24a authored by Jacob Blain Christen's avatar Jacob Blain Christen Committed by GitHub

containerd/cri: enable the btrfs snapshotter (#4316)

* vendor: btrfs * enable the btrfs snapshotter * testing: snapshotter/btrfs Signed-off-by: 's avatarJacob Blain Christen <jacob@rancher.com>
parent 3da1bb3a
name: Snapshotter Testing
on:
push:
paths-ignore:
- "install.sh"
- "tests/cgroup2/**"
- "tests/install/**"
- "tests/integration/**"
- "tests/unitcoverage/**"
pull_request:
paths-ignore:
- "install.sh"
- "tests/cgroup2/**"
- "tests/install/**"
- "tests/integration/**"
- "tests/unitcoverage/**"
workflow_dispatch: {}
jobs:
build:
name: "Build"
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: "Checkout"
uses: actions/checkout@v2
with:
fetch-depth: 1
- name: "Build"
run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make
- name: "Upload Binary"
uses: actions/upload-artifact@v2
with:
name: k3s
path: dist/artifacts/k3s
- name: "Upload Images"
uses: actions/upload-artifact@v2
with:
name: k3s-airgap-images-amd64.tar
path: dist/artifacts/k3s-airgap-images-amd64.tar
test:
name: "Test"
# nested virtualization is only available on macOS hosts
runs-on: macos-10.15
needs: build
timeout-minutes: 40
strategy:
matrix:
vm: [opensuse-leap]
steps:
- name: "Checkout"
uses: actions/checkout@v2
with:
fetch-depth: 1
- name: "Download Binary"
uses: actions/download-artifact@v2
with:
name: k3s
path: dist/artifacts/
- name: "Download Images"
uses: actions/download-artifact@v2
with:
name: k3s-airgap-images-amd64.tar
path: dist/artifacts/
- name: "Vagrant Plugin(s)"
working-directory: tests/snapshotter/btrfs/${{ matrix.vm }}
run: vagrant plugin install vagrant-k3s
- name: "Vagrant VM"
working-directory: tests/snapshotter/btrfs/${{ matrix.vm }}
env:
VAGRANT_EXPERIMENTAL: disks
run: vagrant up
......@@ -11,7 +11,7 @@ ENV no_proxy=$no_proxy
RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
zlib-dev tar zip squashfs-tools npm coreutils python3 openssl-dev libffi-dev libseccomp libseccomp-dev \
libseccomp-static make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static \
zstd gzip alpine-sdk binutils-gold
zstd gzip alpine-sdk binutils-gold btrfs-progs-dev btrfs-progs-static
RUN if [ "$(go env GOARCH)" = "arm64" ]; then \
wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz && \
......
......@@ -155,6 +155,7 @@ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u9
github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo=
github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
github.com/containerd/btrfs v1.0.0 h1:osn1exbzdub9L5SouXO5swW4ea/xVdJZ3wokxN5GrnA=
github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
......
......@@ -24,6 +24,7 @@ import (
_ "github.com/containerd/containerd/runtime/v2/runc/options"
_ "github.com/containerd/containerd/snapshots/native/plugin"
_ "github.com/containerd/containerd/snapshots/overlay/plugin"
_ "github.com/containerd/containerd/snapshots/btrfs/plugin"
_ "github.com/containerd/fuse-overlayfs-snapshotter/plugin"
_ "github.com/containerd/stargz-snapshotter/service/plugin"
)
......@@ -42,7 +42,7 @@ STATIC="
STATIC_SQLITE="
-extldflags '-static -lm -ldl -lz -lpthread'
"
TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless"
TAGS="ctrd apparmor seccomp netcgo osusergo providerless"
RUNC_TAGS="apparmor seccomp"
RUNC_STATIC="static"
......
# -*- mode: ruby -*-
# vi: set ft=ruby :
#
ENV['TEST_VM_NAME'] ||= 'snapshotter-btrfs'
ENV['TEST_VM_HOSTNAME'] ||= 'test'
Vagrant.configure("2") do |config|
config.vagrant.plugins = {
'vagrant-k3s' => {:version => '~> 0.1.3'},
}
config.vm.define ENV['TEST_VM_NAME'], primary: true do |test|
test.vm.box = "opensuse/Leap-15.3.x86_64"
test.vm.hostname = ENV['TEST_VM_HOSTNAME']
test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', privileged: true do |sh|
sh.inline = <<~EOF
#!/usr/bin/env bash
set -eu -o pipefail
zypper install -y btrfsprogs hostname
mkdir -p /var/lib/rancher/k3s /etc/rancher/k3s /usr/local/bin
if ! mountpoint -q /var/lib/rancher/k3s; then
: ${BTRFS_DEV:=#{ENV['BTRFS_DEV']}}
for disk in sd[b-d] vd[b-d] xd[b-d]; do
if [ -n "${BTRFS_DEV}" ]; then break; fi
: ${BTRFS_DEV:=$(test -b /dev/$disk && echo $disk)}
done
btrfs filesystem show /dev/${BTRFS_DEV:?unable to determine automatically, please specify} 2>/dev/null || mkfs -t btrfs /dev/${BTRFS_DEV}
mountpoint -q /mnt || mount -t btrfs /dev/${BTRFS_DEV} /mnt
btrfs subvolume show /mnt/@k3s 2>/dev/null || btrfs subvolume create /mnt/@k3s
umount /mnt
mount -t btrfs -o subvol=@k3s /dev/${BTRFS_DEV} /var/lib/rancher/k3s
fi
if [ -e /vagrant/k3s ]; then
cp -vf /vagrant/k3s /usr/local/bin/
chmod -v +x /usr/local/bin/k3s
fi
if [ -e /vagrant/*.tar ]; then
mkdir -vp /var/lib/rancher/k3s/agent/images
for tar in /vagrant/*.tar; do
cp -vf $tar /var/lib/rancher/k3s/agent/images/
done
fi
EOF
end
test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
k3s.args = %w[server --snapshotter=btrfs]
k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant]
k3s.config = {
'disable' => %w[local-storage metrics-server servicelb traefik],
'disable-helm-controller' => true,
'disable-network-policy' => true,
'write-kubeconfig-mode' => '0644',
}
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
end
test.vm.provision "k3s-ready", type: "shell", run: "once" do |sh|
sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" }
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
echo 'Waiting for node to be ready ...'
time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done'
time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done'
SHELL
end
test.vm.provision "k3s-status", type: "shell", run: "once" do |sh|
sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" }
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
kubectl get node,all -A -o wide
btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs
SHELL
end
end
%w[hyperv libvirt virtualbox vmware_desktop].each do |p|
config.vm.provider p do |v|
v.cpus = ENV['CPUS'] || 2
v.memory = ENV['MEMORY'] || 2048
end
end
config.vm.provider :hyperv do |v,o|
o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks"
end
config.vm.provider :libvirt do |v,o|
v.storage :file, :size => '8G'
end
config.vm.provider :virtualbox do |v,o|
v.gui = false
v.check_guest_additions = false
o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks"
end
config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: ['1', 'true'].include?(ENV['RSYNC_DISABLE']),
rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
end
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
bin/
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
# Support running go modules in vendor mode for local development
/vendor/
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
.PHONY: clean binaries generate lint vet test
all: vet lint test binaries
binaries: bin/btrfs-test
vet:
go vet ./...
lint:
golint ./...
test:
go test -v ./...
bin/%: ./cmd/% *.go
go build -o ./$@ ./$<
clean:
rm -rf bin/*
# go-btrfs
[![PkgGoDev](https://pkg.go.dev/badge/github.com/containerd/btrfs)](https://pkg.go.dev/github.com/containerd/btrfs)
[![Build Status](https://github.com/containerd/btrfs/workflows/CI/badge.svg)](https://github.com/containerd/btrfs/actions?query=workflow%3ACI)
[![Go Report Card](https://goreportcard.com/badge/github.com/containerd/btrfs)](https://goreportcard.com/report/github.com/containerd/btrfs)
Native Go bindings for btrfs.
# Status
These are in the early stages. We will try to maintain stability, but please
vendor if you are relying on these directly.
# Contribute
This package may not cover all the use cases for btrfs. If something you need
is missing, please don't hesitate to submit a PR.
Note that due to struct alignment issues, this isn't yet fully native.
Preferably, this could be resolved, so contributions in this direction are
greatly appreciated.
## Applying License Header to New Files
If you submit a contribution that adds a new file, please add the license
header. You can do so manually or use the `ltag` tool:
```console
$ go get github.com/kunalkushwaha/ltag
$ ltag -t ./license-templates
```
The above will add the appropriate licenses to Go files. New templates will
need to be added if other kinds of files are added. Please consult the
documentation at https://github.com/kunalkushwaha/ltag
## Project details
btrfs is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE).
As a containerd sub-project, you will find the:
* [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
information in our [`containerd/project`](https://github.com/containerd/project) repository.
/*
Copyright The containerd Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <stddef.h>
#include <linux/magic.h>
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
#include "btrfs.h"
void unpack_root_item(struct gosafe_btrfs_root_item* dst, struct btrfs_root_item* src) {
memcpy(dst->uuid, src->uuid, BTRFS_UUID_SIZE);
memcpy(dst->parent_uuid, src->parent_uuid, BTRFS_UUID_SIZE);
memcpy(dst->received_uuid, src->received_uuid, BTRFS_UUID_SIZE);
dst->gen = btrfs_root_generation(src);
dst->ogen = btrfs_root_otransid(src);
dst->flags = btrfs_root_flags(src);
}
/* unpack_root_ref(struct gosafe_btrfs_root_ref* dst, struct btrfs_root_ref* src) { */
/*
Copyright The containerd Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <stddef.h>
#include <linux/magic.h>
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
// unfortunately, we need to define "alignment safe" C structs to populate for
// packed structs that aren't handled by cgo. Fields will be added here, as
// needed.
struct gosafe_btrfs_root_item {
u8 uuid[BTRFS_UUID_SIZE];
u8 parent_uuid[BTRFS_UUID_SIZE];
u8 received_uuid[BTRFS_UUID_SIZE];
u64 gen;
u64 ogen;
u64 flags;
};
void unpack_root_item(struct gosafe_btrfs_root_item* dst, struct btrfs_root_item* src);
/* void unpack_root_ref(struct gosafe_btrfs_root_ref* dst, struct btrfs_root_ref* src); */
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package btrfs provides bindings for working with btrfs partitions from Go.
package btrfs
module github.com/containerd/btrfs
go 1.15
require github.com/pkg/errors v0.9.1
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package btrfs
/*
#include <stddef.h>
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
*/
import "C"
import (
"bufio"
"bytes"
"fmt"
"os"
"strings"
"unsafe"
"github.com/pkg/errors"
)
func subvolID(fd uintptr) (uint64, error) {
var args C.struct_btrfs_ioctl_ino_lookup_args
args.objectid = C.BTRFS_FIRST_FREE_OBJECTID
if err := ioctl(fd, C.BTRFS_IOC_INO_LOOKUP, uintptr(unsafe.Pointer(&args))); err != nil {
return 0, err
}
return uint64(args.treeid), nil
}
var (
zeroArray = [16]byte{}
zeros = zeroArray[:]
)
func uuidString(uuid *[C.BTRFS_UUID_SIZE]C.u8) string {
b := (*[maxByteSliceSize]byte)(unsafe.Pointer(uuid))[:C.BTRFS_UUID_SIZE]
if bytes.Equal(b, zeros) {
return ""
}
return fmt.Sprintf("%x-%x-%x-%x-%x", b[:4], b[4:4+2], b[6:6+2], b[8:8+2], b[10:16])
}
func findMountPoint(path string) (string, error) {
fp, err := os.Open("/proc/self/mounts")
if err != nil {
return "", err
}
defer fp.Close()
const (
deviceIdx = 0
pathIdx = 1
typeIdx = 2
options = 3
)
var (
mount string
scanner = bufio.NewScanner(fp)
)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if fields[typeIdx] != "btrfs" {
continue // skip non-btrfs
}
if strings.HasPrefix(path, fields[pathIdx]) {
mount = fields[pathIdx]
}
}
if scanner.Err() != nil {
return "", scanner.Err()
}
if mount == "" {
return "", errors.Errorf("mount point of %v not found", path)
}
return mount, nil
}
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package btrfs
// Info describes metadata about a btrfs subvolume.
type Info struct {
ID uint64 // subvolume id
ParentID uint64 // aka ref_tree
TopLevelID uint64 // not actually clear what this is, not set for now.
Offset uint64 // key offset for root
DirID uint64
Generation uint64
OriginalGeneration uint64
UUID string
ParentUUID string
ReceivedUUID string
Name string
Path string // absolute path of subvolume
Root string // path of root mount point
Readonly bool // true if the snaps hot is readonly, extracted from flags
}
type infosByID []Info
func (b infosByID) Len() int { return len(b) }
func (b infosByID) Less(i, j int) bool { return b[i].ID < b[j].ID }
func (b infosByID) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package btrfs
import "syscall"
func ioctl(fd, request, args uintptr) error {
_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, request, args)
if errno != 0 {
return errno
}
return nil
}
// +build linux,!no_btrfs,cgo
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package plugin
import (
"errors"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/snapshots/btrfs"
)
// Config represents configuration for the btrfs plugin.
type Config struct {
// Root directory for the plugin
RootPath string `toml:"root_path"`
}
func init() {
plugin.Register(&plugin.Registration{
ID: "btrfs",
Type: plugin.SnapshotPlugin,
Config: &Config{},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
ic.Meta.Platforms = []ocispec.Platform{platforms.DefaultSpec()}
config, ok := ic.Config.(*Config)
if !ok {
return nil, errors.New("invalid btrfs configuration")
}
root := ic.Root
if len(config.RootPath) != 0 {
root = config.RootPath
}
ic.Meta.Exports = map[string]string{"root": root}
return btrfs.NewSnapshotter(root)
},
})
}
......@@ -174,6 +174,8 @@ github.com/cilium/ebpf/internal/unix
github.com/cilium/ebpf/link
# github.com/container-storage-interface/spec v1.5.0
github.com/container-storage-interface/spec/lib/go/csi
# github.com/containerd/btrfs v1.0.0 => github.com/containerd/btrfs v1.0.0
github.com/containerd/btrfs
# github.com/containerd/cgroups v1.0.1 => github.com/containerd/cgroups v1.0.1
## explicit
github.com/containerd/cgroups
......@@ -345,6 +347,8 @@ github.com/containerd/containerd/services/snapshots
github.com/containerd/containerd/services/tasks
github.com/containerd/containerd/services/version
github.com/containerd/containerd/snapshots
github.com/containerd/containerd/snapshots/btrfs
github.com/containerd/containerd/snapshots/btrfs/plugin
github.com/containerd/containerd/snapshots/native
github.com/containerd/containerd/snapshots/native/plugin
github.com/containerd/containerd/snapshots/overlay
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment