Merge pull request #123 from cashapp/juho/git-as-package-source

Git repo as package source

Author: jvmakine

.gitignore

@@ -10,3 +10,4 @@ bin/hermit.hcl
 docs/resources/_gen
 docs/public
 .gobin/
+.hermit/

archive/archive.go

@@ -45,6 +45,15 @@ func Extract(b *ui.Task, source string, pkg *manifest.Package) (finalise func()
 	// Do we need to rename the result to the final pkg.Dest?
 	// This is set to false if we are recursively extracting packages within one another
 	renameResult := true
+
+	isDir, err := isDirectory(source)
+	if err != nil {
+		return finalise, errors.WithStack(err)
+	}
+	if isDir {
+		return finalise, installFromDirectory(source, pkg)
+	}
+
 	ext := filepath.Ext(source)
 	switch ext {
 	case ".pkg":
@@ -145,6 +154,72 @@ type hdi struct {
 	SystemEntities []*hdiEntry `plist:"system-entities"`
 }
 
+func isDirectory(path string) (bool, error) {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return false, errors.WithStack(err)
+	}
+	return fileInfo.IsDir(), nil
+}
+
+func installFromDirectory(source string, pkg *manifest.Package) error {
+	dest := pkg.Dest
+	return errors.WithStack(filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
+		relative, err := filepath.Rel(source, path)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		isDir, err := isDirectory(path)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		target, err := makeDestPath(dest, relative, pkg.Strip)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if target == "" {
+			return nil
+		}
+		if !isDir {
+			if err := copyFile(path, target); err != nil {
+				return errors.WithStack(err)
+			}
+		}
+		return nil
+	}))
+}
+
+func copyFile(from, to string) error {
+	err := ensureDirExists(to)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	info, err := os.Stat(from)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	srcFile, err := os.Open(from)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer srcFile.Close() // nolint: gosec
+
+	destFile, err := os.OpenFile(to, os.O_WRONLY|os.O_CREATE, info.Mode())
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer destFile.Close() // nolint: gosec
+
+	_, err = io.Copy(destFile, srcFile)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return errors.WithStack(destFile.Sync())
+}
+
 func installMacDMG(b *ui.Task, source string, pkg *manifest.Package) error {
 	dest := pkg.Dest + "~"
 	err := os.MkdirAll(dest, 0700)

archive/archive_test.go

@@ -29,6 +29,7 @@ func TestExtract(t *testing.T) {
 		{"linux_exe.gz", []string{"linux_exe"}},
 		{"bzip2_1.0.6-9.2_deb10u1_amd64.deb", []string{"/bin/bzip2"}},
 		{"bzip2-1.0.6-13.el7.x86_64.rpm", []string{"/usr/bin/bzip2"}},
+		{"directory", []string{"foo"}},
 	}
 	for _, test := range tests {
 		t.Run(test.file, func(t *testing.T) {

archive/testdata/directory/foo

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "bar"

cache/cache.go

@@ -1,7 +1,6 @@
 package cache
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
@@ -75,22 +74,11 @@ func (c *Cache) Create(checksum, uri string) (*os.File, error) {
 
 // OpenLocal opens a local cached copy of "uri", or errors.
 func (c *Cache) OpenLocal(checksum, uri string) (*os.File, error) {
-	u, err := url.Parse(uri)
+	source, err := getSource(uri)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
-	switch u.Scheme {
-	case "", "file":
-		f, err := os.Open(u.Path)
-		return f, errors.WithStack(err)
-
-	case "http", "https":
-		f, err := os.Open(c.Path(checksum, uri))
-		return f, errors.WithStack(err)
-
-	default:
-		return nil, errors.Errorf("unsupported URI: %s", uri)
-	}
+	return source.OpenLocal(c, checksum)
 }
 
 // Open a local or remote artifact, transparently caching it. Subsequent accesses will use the cached copy.
@@ -119,31 +107,16 @@ func (c *Cache) Open(b *ui.Task, checksum, uri string, mirrors ...string) (*os.F
 //
 // If checksum is present it must be the SHA256 hash of the downloaded artifact.
 func (c *Cache) Download(b *ui.Task, checksum, uri string, mirrors ...string) (path string, etag string, err error) {
-	cachePath := c.Path(checksum, uri)
 	uris := append([]string{uri}, mirrors...)
 	for _, uri := range uris {
 		defer ui.LogElapsed(b, "Download %s", uri)()
-		var u *url.URL
-		u, err = url.Parse(uri)
+		source, err := getSource(uri)
 		if err != nil {
 			return "", "", errors.WithStack(err)
 		}
-
-		// Temporary file location.
-		switch u.Scheme {
-		case "", "file":
-			// TODO: Checksum it again?
-			// Local file, just open it.
-			return u.Path, "", nil
-
-		case "http", "https":
-			path, etag, err = c.downloadHTTP(b, checksum, uri, cachePath)
-
-		default:
-			return "", "", errors.Errorf("unsupported URI %s", uri)
-		}
+		path, etag, err = source.Download(b, c, checksum)
 		if err == nil {
-			return
+			return path, etag, nil
 		}
 		b.Debugf("%s: %s", uri, err)
 	}
@@ -157,30 +130,16 @@ func (c *Cache) Download(b *ui.Task, checksum, uri string, mirrors ...string) (p
 // Otherwise an empty string is returned
 func (c *Cache) ETag(b *ui.Task, uri string, mirrors ...string) (etag string, err error) {
 	for _, uri := range append([]string{uri}, mirrors...) {
-		u, err := url.Parse(uri)
+		source, err := getSource(uri)
 		if err != nil {
 			return "", errors.WithStack(err)
 		}
-		switch u.Scheme {
-		case "http", "https":
-			req, err := http.NewRequestWithContext(context.Background(), http.MethodHead, uri, nil)
-			if err != nil {
-				return "", errors.Wrap(err, uri)
-			}
-			resp, err := c.fastFailHTTPClient.Do(req)
-			if err != nil {
-				b.Debugf("%s failed: %s", uri, err)
-				continue
-			}
-			defer resp.Body.Close()
-			// Normal HTTP error, log and try the next mirror.
-			if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-				b.Debugf("%s failed: %d", uri, resp.StatusCode)
-				continue
-			}
-			etag := resp.Header.Get("ETag")
-			return etag, nil
+		result, err := source.ETag(b, c)
+		if err != nil {
+			b.Debugf("%s failed: %s", uri, err)
+			continue
 		}
+		return result, nil
 	}
 	return "", nil
 }

cache/source.go

@@ -0,0 +1,129 @@
+package cache
+
+import (
+	"context"
+	"github.com/cashapp/hermit/ui"
+	"github.com/cashapp/hermit/util"
+	"github.com/pkg/errors"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type packageSource interface {
+	OpenLocal(cache *Cache, checksum string) (*os.File, error)
+	Download(b *ui.Task, cache *Cache, checksum string) (path string, etag string, err error)
+	ETag(b *ui.Task, cache *Cache) (etag string, err error)
+}
+
+func getSource(uri string) (packageSource, error) {
+	if strings.HasSuffix(uri, ".git") {
+		return &gitSource{URL: uri}, nil
+	}
+
+	u, err := url.Parse(uri)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	switch u.Scheme {
+	case "", "file":
+		return &fileSource{Path: u.Path}, nil
+
+	case "http", "https":
+		return &httpSource{uri}, errors.WithStack(err)
+	default:
+		return nil, errors.Errorf("unsupported URI %s", uri)
+	}
+}
+
+type fileSource struct {
+	Path string
+}
+
+func (s *fileSource) OpenLocal(_ *Cache, _ string) (*os.File, error) {
+	f, err := os.Open(s.Path)
+	return f, errors.WithStack(err)
+}
+
+func (s *fileSource) Download(_ *ui.Task, _ *Cache, _ string) (path string, etag string, err error) {
+	// TODO: Checksum it again?
+	// Local file, just open it.
+	return s.Path, "", nil
+}
+
+func (s *fileSource) ETag(_ *ui.Task, _ *Cache) (etag string, err error) {
+	return "", nil
+}
+
+type httpSource struct {
+	URL string
+}
+
+func (s *httpSource) OpenLocal(c *Cache, checksum string) (*os.File, error) {
+	f, err := os.Open(c.Path(checksum, s.URL))
+	return f, errors.WithStack(err)
+}
+
+func (s *httpSource) Download(b *ui.Task, cache *Cache, checksum string) (path string, etag string, err error) {
+	cachePath := cache.Path(checksum, s.URL)
+	return cache.downloadHTTP(b, checksum, s.URL, cachePath)
+}
+
+func (s *httpSource) ETag(_ *ui.Task, c *Cache) (etag string, err error) {
+	uri := s.URL
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodHead, uri, nil)
+	if err != nil {
+		return "", errors.Wrap(err, uri)
+	}
+	resp, err := c.fastFailHTTPClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, uri)
+	}
+	defer resp.Body.Close()
+	// Normal HTTP error, log and try the next mirror.
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", errors.Errorf("%s failed: %d", uri, resp.StatusCode)
+	}
+	result := resp.Header.Get("ETag")
+	return result, nil
+}
+
+type gitSource struct {
+	URL string
+}
+
+func (s *gitSource) OpenLocal(c *Cache, checksum string) (*os.File, error) {
+	f, err := os.Open(c.Path(checksum, s.URL))
+	return f, errors.WithStack(err)
+}
+
+func (s *gitSource) Download(b *ui.Task, cache *Cache, checksum string) (string, string, error) {
+	base := BasePath(checksum, s.URL)
+	err := util.RunInDir(b, cache.root, "git", "clone", "--depth=1", s.URL, base)
+	if err != nil {
+		return "", "", errors.Wrap(err, s.URL)
+	}
+	etag, err := s.ETag(b, cache)
+	if err != nil {
+		return "", "", errors.Wrap(err, s.URL)
+	}
+
+	return filepath.Join(cache.root, base), etag, nil
+}
+
+func (s *gitSource) ETag(b *ui.Task, c *Cache) (etag string, err error) {
+	bts, err := util.Capture(b, "git", "ls-remote", s.URL, "HEAD")
+	if err != nil {
+		return "", errors.Wrap(err, s.URL)
+	}
+	str := string(bts)
+	parts := strings.Split(str, "\t")
+	if len(parts) != 2 {
+		return "", errors.Errorf("invalid HEAD: %s", str)
+	}
+
+	return parts[0], nil
+}