Merge pull request #1 from loft-sh/experiments/in_rootless_container

Experiments/in rootless container

Author: 89luca89

.gitignore

@@ -2,3 +2,4 @@ env*
 devpod-provider-dockerless
 release
 rootlesskit*
+work

README.md

@@ -40,3 +40,15 @@ After the initial setup, just use:
 ```sh
 devpod up .
 ```
+
+## Run in a container
+
+To run in a container, we need CAP_SYS_ADMIN (needed for the unshare, mount and pivot_root syscalls)
+To have custom networking we also need access to /dev/net/tun
+
+We DO NOT require root
+A nice way to run it is:
+
+`docker run --rm -ti --cap-add CAP_SYS_ADMIN --device /dev/net/tun --user 1000:1000 build-alpine:latest`
+
+Using the image in the `image` folder.

cmd/enter.go

@@ -10,7 +10,8 @@ import (
 )
 
 // EnterCmd holds the cmd flags
-type EnterCmd struct{}
+type EnterCmd struct{
+}
 
 // NewEnterCmd defines a command
 func NewEnterCmd() *cobra.Command {

hack/build.sh

@@ -10,6 +10,7 @@ DATE=$(date "+%Y-%m-%d")
 BUILD_PLATFORM=$(uname -a | awk '{print tolower($1);}')
 
 ROOTLESSKIT_VERSION="1.1.1"
+SLIRP4NETNS_VERSION="1.2.2"
 
 echo "Current working directory is $(pwd)"
 echo "PATH is $PATH"
@@ -56,13 +57,19 @@ for OS in ${PROVIDER_BUILD_PLATFORMS[@]}; do
 
 		echo "Building for ${OS}/${ARCH}"
 		if [[ ${ARCH} == "amd64"   ]]; then
+			rm -f rootlesskit slirp4netns
 			wget -c "https://github.com/rootless-containers/rootlesskit/releases/download/v${ROOTLESSKIT_VERSION}/rootlesskit-x86_64.tar.gz"
 		    tar -zxvf rootlesskit-x86_64.tar.gz rootlesskit
+		    wget -c "https://github.com/rootless-containers/slirp4netns/releases/download/v${SLIRP4NETNS_VERSION}/slirp4netns-x86_64" -O slirp4netns
+		    chmod +x slirp4netns
 		elif [[ ${ARCH} == "arm64" ]]; then
+			rm -f rootlesskit slirp4netns
 			wget -c "https://github.com/rootless-containers/rootlesskit/releases/download/v${ROOTLESSKIT_VERSION}/rootlesskit-aarch64.tar.gz"
 		    tar -zxvf rootlesskit-aarch64.tar.gz rootlesskit
+		    wget -c "https://github.com/rootless-containers/slirp4netns/releases/download/v${SLIRP4NETNS_VERSION}/slirp4netns-aarch64" -O slirp4netns
+		    chmod +x slirp4netns
 		fi
-		GOARCH=${ARCH} GOOS=${OS} ${GO_BUILD_CMD} -ldflags "${GO_BUILD_LDFLAGS}" \
+		CGO_ENABLED=0 GOARCH=${ARCH} GOOS=${OS} ${GO_BUILD_CMD} -ldflags "${GO_BUILD_LDFLAGS}" \
 			-o  "${PROVIDER_ROOT}/release/${NAME}" main.go
 		shasum -a 256 "${PROVIDER_ROOT}/release/${NAME}" | cut -d ' ' -f 1 > "${PROVIDER_ROOT}/release/${NAME}".sha256
 	done

hack/provider/provider.yaml

@@ -11,6 +11,8 @@ options:
 agent:
   containerInactivityTimeout: ${INACTIVITY_TIMEOUT}
   local: true
+  docker:
+    install: false
   binaries:
     DOCKERLESS_PROVIDER:
       - os: linux

image/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:latest
+
+
+RUN apk add --update-cache shadow shadow-subids iproute2 && rm -rf /var/cache/apk/*
+
+RUN useradd --shell /bin/sh --home /home/rootless --add-subids-for-system --uid 1000 --password "" rootless && \
+    mkdir -p /home/rootless && chown -R rootless:rootless /home/rootless

main.go

@@ -14,6 +14,11 @@ import (
 var rootlesskit []byte
 var rootlesskitPath = filepath.Join("/tmp/dockerless", "rootlesskit")
 
+//nolint: typecheck
+//go:embed slirp4netns
+var slirp4netns []byte
+var slirp4netnsPath = filepath.Join("/tmp/dockerless", "slirp4netns")
+
 func main() {
 	_, err := os.Stat(rootlesskitPath)
 	if err != nil {
@@ -28,6 +33,19 @@ func main() {
 		}
 	}
 
+	_, err = os.Stat(slirp4netnsPath)
+	if err != nil {
+		err = os.MkdirAll("/tmp/dockerless", 0o755)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = os.WriteFile(slirp4netnsPath, slirp4netns, 0o755)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
 	err = os.Setenv("PATH", os.Getenv("PATH")+":/tmp/dockerless")
 	if err != nil {
 		log.Fatal(err)

pkg/dockerless/enter.go

@@ -61,12 +61,7 @@ func (p *DockerlessProvider) Enter(ctx context.Context, workspaceId string) erro
 		return err
 	}
 
-	err = PivotRoot(containerDIR)
-	if err != nil {
-		return err
-	}
-
-	err = syscall.Chdir("/")
+	err = syscall.Chdir(containerDIR)
 	if err != nil {
 		return err
 	}
@@ -77,28 +72,23 @@ func (p *DockerlessProvider) Enter(ctx context.Context, workspaceId string) erro
 		return fmt.Errorf("error setting hostname for namespace: %w", err)
 	}
 
-	args := []string{
-		"--",
-		runOptions.Entrypoint,
-	}
-	args = append(args, runOptions.Cmd...)
-
-	cmd := exec.Command("/usr/bin/env", args...)
+	cmd := exec.Command(runOptions.Entrypoint, runOptions.Cmd...)
 	cmd.Stdin = os.Stdin
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	cmd.Dir = "/"
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Chroot: containerDIR,
+	}
 	cmd.Env = config.ObjectToList(runOptions.Env)
 
 	return cmd.Run()
 }
 
 func prepareMounts(rootfs string) error {
-	err := MountProc(filepath.Join(rootfs, "/proc"))
+	err := MountBind("/proc", filepath.Join(rootfs, "/proc"))
 	if err != nil {
 		return err
 	}
-
 	err = MountTmpfs(filepath.Join(rootfs, "/tmp"))
 	if err != nil {
 		return err

pkg/dockerless/exec.go

@@ -14,6 +14,8 @@ import (
 )
 
 func (p *DockerlessProvider) ExecuteCommand(ctx context.Context, workspaceId, user, command string, stdin io.Reader, stdout, stderr io.Writer) error {
+	containerDIR := filepath.Join(p.Config.TargetDir, "rootfs", workspaceId)
+
 	ppid, err := GetPid(workspaceId)
 	if err != nil {
 		return fmt.Errorf("container %s is not running", workspaceId)
@@ -33,8 +35,14 @@ func (p *DockerlessProvider) ExecuteCommand(ctx context.Context, workspaceId, us
 		"-u",
 		"-i",
 		"-p",
+		"-r/proc/" + string(pid) + "/root",
+		"-w/proc/" + string(pid) + "/root",
 	}
 
+	_, err = os.Stat("/dev/net/tun")
+	if err == nil {
+		args = append(args, "-n")
+	}
 	// user namespace only if we're rootless
 	if os.Getuid() > 0 {
 		args = append(args, "-U")
@@ -44,23 +52,19 @@ func (p *DockerlessProvider) ExecuteCommand(ctx context.Context, workspaceId, us
 	args = append(args, []string{
 		"-t",
 		string(pid),
+		"sh",
+		"-l",
+		"-c",
 	}...)
 
-	if user == "" {
-		args = append(args, command)
-	} else {
-		containerDIR := filepath.Join(p.Config.TargetDir, "rootfs", workspaceId)
+	if user != "" && user != "0" && user != "root" {
 		uid := findUserPasswd(containerDIR, user)
-
-		args = append(args, []string{"su", "-l", uid, "-c", command}...)
+		command = "su -l " + uid + " -c " + command
 	}
 
+	args = append(args, command)
+
 	cmd := exec.Command(nsenter, args...)
-	environB, err := os.ReadFile(filepath.Join("/proc", string(pid), "environ"))
-	if err == nil {
-		environ := strings.Split(string(environB), "\000")
-		cmd.Env = environ
-	}
 	cmd.Stdin = os.Stdin
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr

pkg/dockerless/start.go

@@ -56,11 +56,24 @@ func (p *DockerlessProvider) Start(ctx context.Context, workspaceId string) erro
 			"--cgroupns",
 			"--utsns",
 			"--ipcns",
-			"--net",
-			"host",
 			"--state-dir",
 			filepath.Join("/tmp", "dockerless", workspaceId),
 		}
+
+		// Default to use slip4netns if we have /dev/net/tun access
+		_, err = os.Stat("/dev/net/tun")
+		if err == nil {
+			args = append(args, []string{
+				"--net",
+				"slirp4netns",
+				"--port-driver",
+				"slirp4netns",
+				"--disable-host-loopback",
+				"--copy-up",
+				"/etc",
+			}...)
+		}
+
 	} else {
 		command = "unshare"
 		args = []string{