Merge pull request #71 from AkihiroSuda/dev-vagrant

add Vagrantfile (Fedora 33) for cgroup2 testing

Author: AkihiroSuda

.github/workflows/test.yml

@@ -58,3 +58,31 @@ jobs:
         fetch-depth: 1
     - name: "Ensure that the test suite is compatible with Docker"
       run: go test -v -exec sudo -test.target=docker .
+
+  test-cgroup2:
+    name: "Cgroup2 + rootless"
+    # nested virtualization is only available on macOS hosts
+    runs-on: macos-10.15
+    timeout-minutes: 40
+    steps:
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.15.x
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+    # Vagrant is slow, so we build binaries outside Vagrant
+    - name: "Build binaries"
+      run: |
+        GOOS=linux make binaries
+        GOOS=linux go test -c .
+    - name: "Boot VM"
+      run: |
+        vagrant up
+        vagrant ssh-config >> ~/.ssh/config
+    - name: "Install rootless containerd"
+      run: ssh default -- containerd-rootless-setuptool.sh install
+    - name: "Run tests (rootless)"
+      run: ssh default -- "CONTAINERD_SNAPSHOTTER=native /vagrant/nerdctl.test -test.v"
+    - name: "Uninstall rootless containerd"
+      run: ssh default -- containerd-rootless-setuptool.sh uninstall

.gitignore

@@ -4,3 +4,6 @@ _output
 
 # golangci-lint
 build
+
+# vagrant
+/.vagrant

Dockerfile

@@ -8,7 +8,7 @@ ARG CNI_ISOLATION_VERSION=0.0.3
 ARG BUILDKIT_VERSION=0.8.1
 ARG GO_VERSION=1.15.8
 
-FROM ubuntu:${UBUNTU_VERSION} AS base
+FROM mirror.gcr.io/library/ubuntu:${UBUNTU_VERSION} AS base
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
   apt-get install -qq -y --no-install-recommends \

Vagrantfile

@@ -0,0 +1,63 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrant box for testing cgroup v2
+Vagrant.configure("2") do |config|
+  config.vm.box = "fedora/33-cloud-base"
+  memory = 4096
+  cpus = 2
+  config.vm.provider :virtualbox do |v|
+    v.memory = memory
+    v.cpus = cpus
+  end
+  config.vm.provider :libvirt do |v|
+    v.memory = memory
+    v.cpus = cpus
+  end
+  config.vm.provision "shell", inline: <<-SHELL
+    set -eux -o pipefail
+    if [ ! -x /vagrant/_output/nerdctl ]; then
+      echo "Run 'GOOS=linux make' before running 'vagrant up'"
+      exit 1
+    fi
+    if [ ! -x /vagrant/nerdctl.test ]; then
+      echo "Run 'GOOS=linux go test -c' before running 'vagrant up'"
+      exit 1
+    fi
+
+    # Install RPMs
+    dnf install -y \
+      make \
+      containerd \
+      containernetworking-plugins \
+      iptables \
+      slirp4netns \
+      policycoreutils-python-utils
+
+    # SELinux workaround (https://github.com/moby/moby/issues/41230)
+    semanage permissive -a iptables_t
+
+    # Install runc
+    RUNC_VERSION=1.0.0-rc93
+    # remove rpm version of runc, which doesn't support cgroup v2
+    rm -f /usr/bin/runc
+    curl -o /usr/local/sbin/runc -fsSL https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64
+    chmod +x /usr/local/sbin/runc
+
+    # Install RootlessKit
+    ROOTLESSKIT_VERSION=0.13.1
+    curl -sSL https://github.com/rootless-containers/rootlesskit/releases/download/v${ROOTLESSKIT_VERSION}/rootlesskit-$(uname -m).tar.gz | tar Cxzv /usr/local/bin
+
+    # Delegate cgroup v2 controllers
+    mkdir -p /etc/systemd/system/user@.service.d
+    cat <<EOF >/etc/systemd/system/user@.service.d/delegate.conf
+[Service]
+Delegate=yes
+EOF
+    systemctl daemon-reload
+
+    # Install nerdctl
+    # The binary is built outside Vagrant.
+    make -C /vagrant install
+  SHELL
+end

docs/rootless.md

@@ -15,7 +15,7 @@ $ containerd-rootless-setuptool.sh install
 ...
 [INFO] Installed containerd.service successfully.
 [INFO] To control containerd.service, run: `systemctl --user (start|stop|restart) containerd.service`
-[INFO] To run containerd.service on system startup, run: `sudo loginctl enable-linger suda`
+[INFO] To run containerd.service on system startup, run: `sudo loginctl enable-linger testuser`
 
 [INFO] Use `nerdctl` to connect to the rootless containerd.
 [INFO] You do NOT need to specify $CONTAINERD_ADDRESS explicitly.
@@ -35,3 +35,32 @@ $ nerdctl run -it --rm alpine
 
 Depending on your kernel version, you may need to set `export CONTAINERD_SNAPSHOTTER=native`.
 See https://rootlesscontaine.rs/how-it-works/overlayfs/ .
+
+## Troubleshooting
+
+### Hint to Fedora 33 users
+
+#### runc rpm
+The runc package of Fedora 33 does not support cgroup v2.
+Use the upstream runc binary: https://github.com/opencontainers/runc/releases
+
+The runc package of Fedora 34 will probably support cgroup v2.
+
+Alternatively, you may choose to use `crun` instead of `runc`:
+`nerdctl run --runtime=crun`
+
+#### OverlayFS
+You need to set `export CONTAINERD_SNAPSHOTTER=native` on Fedora 33 because Fedora 33 does not support rootless overlayfs.
+(FUSE-OverlayFS could be used instead, but you might need to recompile containerd, see https://github.com/AkihiroSuda/containerd-fuse-overlayfs)
+
+Fedora 34 (kernel >= 5.11) will probably support rootless overlayfs.
+
+#### SELinux
+If SELinux is enabled on your host, probably you need the following workaround to avoid `can't open lock file /run/xtables.lock:` error:
+```bash
+sudo dnf install -y policycoreutils-python-utils
+sudo semanage permissive -a iptables_t
+```
+
+See https://github.com/moby/moby/issues/41230 .
+This workaround will no longer be needed after the release of Fedora 34.

main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/namespaces"
-	gocni "github.com/containerd/go-cni"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -90,7 +89,7 @@ func newApp() *cli.App {
 			Usage: "Set the cni-plugins binary directory",
 			// CNI_PATH is from https://www.cni.dev/docs/cnitool/
 			EnvVars: []string{"CNI_PATH"},
-			Value:   gocni.DefaultCNIDir,
+			Value:   ncdefaults.CNIPath(),
 		},
 		&cli.StringFlag{
 			Name:  "cni-netconfpath",

pkg/defaults/defaults.go

@@ -19,6 +19,7 @@ package defaults
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/AkihiroSuda/nerdctl/pkg/rootlessutil"
@@ -38,6 +39,31 @@ func DataRoot() string {
 	return filepath.Join(xdh, "nerdctl")
 }
 
+func CNIPath() string {
+	candidates := []string{
+		"/usr/local/libexec/cni",
+		"/usr/libexec/cni", // Fedora
+	}
+	if rootlessutil.IsRootless() {
+		home := os.Getenv("HOME")
+		if home == "" {
+			panic("environment variable HOME is not set")
+		}
+		candidates = append([]string{
+			filepath.Join(home, "opt/cni/bin"),
+		}, candidates...)
+	}
+
+	for _, f := range candidates {
+		if _, err := os.Stat(f); err == nil {
+			return f
+		}
+	}
+
+	// default: /opt/cni/bin
+	return gocni.DefaultCNIDir
+}
+
 func CNINetConfPath() string {
 	if !rootlessutil.IsRootless() {
 		return gocni.DefaultNetDir

pkg/testutil/testutil.go

@@ -133,9 +133,9 @@ func NewBase(t *testing.T) *Base {
 	return base
 }
 
-// TODO: avoid using Docker Hub
+// use GCR mirror to avoid hitting Docker Hub rate limit
 const (
-	AlpineImage                 = "alpine"
-	NginxAlpineImage            = "nginx:1.19.6-alpine"
+	AlpineImage                 = "mirror.gcr.io/library/alpine:3.13"
+	NginxAlpineImage            = "mirror.gcr.io/library/nginx:1.19.6-alpine"
 	NginxAlpineIndexHTMLSnippet = "<title>Welcome to nginx!</title>"
 )