Merge pull request #613 from Kern--/remote-sn-rootfs

Add remote snapshotter rootfs (stargz)

Author: Kern--

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "firecracker"]
 	path = _submodules/firecracker
 	url = https://github.com/firecracker-microvm/firecracker.git
+[submodule "stargz-snapshotter"]
+	path = _submodules/stargz-snapshotter
+	url = https://github.com/containerd/stargz-snapshotter

Makefile

@@ -11,7 +11,7 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd snapshotter
+SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd snapshotter docker-credential-mmds
 TEST_SUBDIRS:=$(addprefix test-,$(SUBDIRS))
 INTEG_TEST_SUBDIRS:=$(addprefix integ-test-,$(SUBDIRS))
 
@@ -66,6 +66,10 @@ RUNC_DIR=$(SUBMODULES)/runc
 RUNC_BIN=$(RUNC_DIR)/runc
 RUNC_BUILDER_NAME?=runc-builder
 
+STARGZ_DIR=$(SUBMODULES)/stargz-snapshotter
+STARGZ_BIN=$(STARGZ_DIR)/out/containerd-stargz-grpc
+STARGZ_BUILDER_NAME?=stargz-builder
+
 PROTO_BUILDER_NAME?=proto-builder
 
 # Set this to pass additional commandline flags to the go compiler, e.g. "make test EXTRAGOARGS=-v"
@@ -112,6 +116,7 @@ distclean: clean
 	$(MAKE) -C tools/image-builder distclean
 	$(call rmi-if-exists,localhost/$(RUNC_BUILDER_NAME):$(DOCKER_IMAGE_TAG))
 	$(call rmi-if-exists,localhost/$(FIRECRACKER_BUILDER_NAME):$(DOCKER_IMAGE_TAG))
+	$(call rmi-if-exists,localhost/$(STARGZ_BUILDER_NAME):$(DOCKER_IMAGE_TAG))
 	docker volume rm -f $(CARGO_CACHE_VOLUME_NAME)
 	docker volume rm -f $(GO_CACHE_VOLUME_NAME)
 	$(call rmi-if-exists,$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG))
@@ -134,15 +139,25 @@ deps:
 install:
 	for d in $(SUBDIRS); do $(MAKE) -C $$d install; done
 
-image: $(RUNC_BIN) agent-in-docker
+files_ephemeral: $(RUNC_BIN) agent-in-docker
 	mkdir -p tools/image-builder/files_ephemeral/usr/local/bin
 	mkdir -p tools/image-builder/files_ephemeral/var/firecracker-containerd-test/scripts
 	for f in tools/docker/scripts/*; do test -f $$f && install -m 755 $$f tools/image-builder/files_ephemeral/var/firecracker-containerd-test/scripts; done
 	cp $(RUNC_BIN) tools/image-builder/files_ephemeral/usr/local/bin
 	cp agent/agent tools/image-builder/files_ephemeral/usr/local/bin
 	touch tools/image-builder/files_ephemeral
+
+files_ephemeral_stargz: $(STARGZ_BIN) docker-credential-mmds-in-docker
+	mkdir -p tools/image-builder/files_ephemeral_stargz/usr/local/bin
+	cp docker-credential-mmds/docker-credential-mmds tools/image-builder/files_ephemeral_stargz/usr/local/bin
+	cp $(STARGZ_BIN) tools/image-builder/files_ephemeral_stargz/usr/local/bin
+
+image: files_ephemeral
 	$(MAKE) -C tools/image-builder all-in-docker
 
+image-stargz: files_ephemeral files_ephemeral_stargz
+	$(MAKE) -C tools/image-builder stargz-in-docker
+
 test: $(TEST_SUBDIRS)
 	go test ./... $(EXTRAGOARGS)
 
@@ -223,6 +238,10 @@ ROOTFS_NO_AGENT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-no-agen
 $(ROOTFS_NO_AGENT_INSTALLPATH): tools/image-builder/rootfs-no-agent.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
 	install -D -o root -g root -m400 $< $@
 
+ROOTFS_STARGZ_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-stargz.img
+$(ROOTFS_STARGZ_INSTALLPATH): tools/image-builder/rootfs-stargz.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
+	install -D -o root -g root -m400 $< $@
+
 .PHONY: default-vmlinux
 default-vmlinux: $(DEFAULT_VMLINUX_NAME)
 
@@ -238,6 +257,9 @@ install-default-runc-jailer-config: $(DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH)
 .PHONY: install-test-rootfs
 install-test-rootfs: $(ROOTFS_SLOW_BOOT_INSTALLPATH) $(ROOTFS_SLOW_REBOOT_INSTALLPATH) $(ROOTFS_NO_AGENT_INSTALLPATH)
 
+.PHONY: install-stargz-rootfs
+install-stargz-rootfs: $(ROOTFS_STARGZ_INSTALLPATH)
+
 ##########################
 # CNI Network
 ##########################
@@ -368,3 +390,31 @@ $(RUNC_BIN): $(RUNC_DIR)/VERSION tools/runc-builder-stamp
 .PHONY: install-runc
 install-runc: $(RUNC_BIN)
 	install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $(RUNC_BIN)
+
+##########################
+# Stargz submodule
+##########################
+.PHONY: stargz-snapshotter
+stargz-snapshotter: $(STARGZ_BIN)
+
+$(STARGZ_DIR)/go.mod:
+	git submodule update --init --recursive $(STARGZ_DIR)
+
+tools/stargz-builder-stamp: tools/docker/Dockerfile.stargz-builder
+	docker build \
+		-t localhost/$(STARGZ_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		-f tools/docker/Dockerfile.stargz-builder \
+		tools/
+	touch $@
+
+$(STARGZ_BIN): $(STARGZ_DIR)/go.mod tools/stargz-builder-stamp
+	docker run --rm -it \
+	--user $(UID):$(GID) \
+	--volume $(GO_CACHE_VOLUME_NAME):/go \
+	--volume $(CURDIR):/src \
+	-e HOME=/tmp \
+	-e GOPATH=/go \
+	-e GOPROXY=$(shell go env GOPROXY) \
+	--workdir /src/$(STARGZ_DIR) \
+	localhost/$(STARGZ_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+	make

_submodules/stargz-snapshotter

@@ -0,0 +1 @@
+docker-credential-mmds

docker-credential-mmds/.gitignore

@@ -0,0 +1,39 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+SOURCES := $(shell find mmds . -name '*.go')
+GOMOD := $(shell go env GOMOD)
+GOSUM := $(GOMOD:.mod=.sum)
+
+REVISION=$(shell git rev-parse HEAD)
+
+all: credential-helper
+
+credential-helper: docker-credential-mmds
+
+docker-credential-mmds: $(SOURCES) $(GOMOD) $(GOSUM)
+	go build -o docker-credential-mmds $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)"
+
+test:
+	go test ./... $(EXTRAGOARGS)
+
+integ-test:
+
+# installation is handled by the rootfs building targets in the main Makefile
+install:
+
+clean:
+	rm docker-credential-mmds
+
+.PHONY: all credential-helper clean test integ-test install
+

docker-credential-mmds/Makefile

@@ -0,0 +1,14 @@
+module github.com/firecracker-microvm/firecracker-containerd/docker-credential-mmds
+
+go 1.17
+
+require (
+	github.com/docker/docker-credential-helpers v0.6.4
+	github.com/stretchr/testify v1.5.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v2 v2.2.2 // indirect
+)

docker-credential-mmds/go.mod

@@ -0,0 +1,15 @@
+github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
+github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

docker-credential-mmds/go.sum

@@ -0,0 +1,30 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/firecracker-microvm/firecracker-containerd/docker-credential-mmds/mmds"
+)
+
+func main() {
+	helper, err := mmds.NewHelper()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "docker-credential-mmds: %s\n", err)
+		os.Exit(1)
+	}
+	credentials.Serve(helper)
+}

docker-credential-mmds/main.go

@@ -0,0 +1,141 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+package mmds
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+const (
+	mmdsAddress    = "169.254.169.254"
+	getTokenPath   = "/latest/api/token/"
+	tokenTtlHeader = "X-metadata-token-ttl-seconds"
+	tokenHeader    = "X-metadata-token"
+	tokenTtl       = "21600"
+)
+
+var errUnauthorized = errors.New("Unauthorized")
+
+type version int
+
+const (
+	v1 version = 1
+	v2
+)
+
+// Client is a guest-side MMDS client
+type Client struct {
+	httpClient *http.Client
+	version    version
+	token      string
+}
+
+// NewClient creates a new MMDS client
+// The MMDS client will attempt to get an MMDS v2 token,
+// however if MMDS v2 is not enabled in the guest, it will
+// fall back to MMDS v1 without session tokens.
+func NewClient(httpClient *http.Client) (*Client, error) {
+	client := Client{
+		httpClient: httpClient,
+		version:    v2,
+		token:      "",
+	}
+	err := client.refreshToken()
+	if err != nil {
+		return nil, err
+	}
+	return &client, nil
+}
+
+func (c *Client) refreshToken() error {
+	resp, err := c.httpClient.Do(&http.Request{
+		Method: http.MethodPut,
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   mmdsAddress,
+			Path:   getTokenPath,
+		},
+		Header: http.Header{
+			tokenTtlHeader: []string{tokenTtl},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusMethodNotAllowed {
+		c.version = v1
+		return nil
+	}
+	token, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	c.token = string(token)
+	return nil
+}
+
+// GetMetadata retrieves JSON metadata from MMDS and converts it to a map
+func (c *Client) GetMetadata(path string) (map[string]interface{}, error) {
+	b, err := c.getMetadata(path)
+	if err != nil {
+		return nil, err
+	}
+	var result map[string]interface{}
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func (c *Client) getMetadata(path string) ([]byte, error) {
+	headers := http.Header{
+		"Accept": []string{"application/json"},
+	}
+	if c.version == v2 {
+		headers[tokenHeader] = []string{c.token}
+	}
+	if !strings.HasPrefix(path, "/") {
+		path = "/" + path
+	}
+	resp, err := c.httpClient.Do(&http.Request{
+		Method: http.MethodGet,
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   mmdsAddress,
+			Path:   path,
+		},
+		Header: headers,
+	})
+	if err != nil {
+		return []byte{}, err
+	}
+	defer resp.Body.Close()
+	return readBody(resp)
+}
+
+func readBody(resp *http.Response) ([]byte, error) {
+	switch status := resp.StatusCode; {
+	case status < 300:
+		return ioutil.ReadAll(resp.Body)
+	case status == http.StatusUnauthorized:
+		return []byte{}, errUnauthorized
+	default:
+		body, err := ioutil.ReadAll(resp.Body)
+		return []byte{}, fmt.Errorf("unexpected http response: %d - (err %v) %s", status, err, string(body))
+	}
+}