gb-vendor: parallelize dependencies download in rebuild

Upstreaming FiloSottile/gvt#10 by @petrkotek

Author: FiloSottile

cmd/gb-vendor/restore.go

@@ -5,30 +5,45 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
+	"sync/atomic"
 
 	"github.com/constabulary/gb"
 	"github.com/constabulary/gb/cmd"
 	"github.com/constabulary/gb/fileutils"
 	"github.com/constabulary/gb/vendor"
 )
 
+var (
+	rbInsecure    bool // Allow the use of insecure protocols
+	rbConnections uint // Count of concurrent download connections
+)
+
 func addRestoreFlags(fs *flag.FlagSet) {
-	fs.BoolVar(&insecure, "precaire", false, "allow the use of insecure protocols")
+	fs.BoolVar(&rbInsecure, "precaire", false, "allow the use of insecure protocols")
+	fs.UintVar(&rbConnections, "connections", 8, "count of parallel download connections")
 }
 
 var cmdRestore = &cmd.Command{
 	Name:      "restore",
-	UsageLine: "restore [-precaire]",
-	Short:     "restore dependencies from the manifest",
-	Long: `Restore vendor dependecies.
+	UsageLine: "restore [-precaire] [-connections N]",
+	Short:     "restore dependencies from manifest",
+	Long: `restore fetches the dependencies listed in the manifest.
 
 Flags:
 	-precaire
 		allow the use of insecure protocols.
+	-connections
+		count of parallel download connections.
 
 `,
 	Run: func(ctx *gb.Context, args []string) error {
-		return restore(ctx)
+		switch len(args) {
+		case 0:
+			return restore(ctx)
+		default:
+			return fmt.Errorf("restore takes no arguments")
+		}
 	},
 	AddFlags: addRestoreFlags,
 }
@@ -39,33 +54,61 @@ func restore(ctx *gb.Context) error {
 		return fmt.Errorf("could not load manifest: %v", err)
 	}
 
+	var errors uint32
+	var wg sync.WaitGroup
+	depC := make(chan vendor.Dependency)
+	for i := 0; i < int(rbConnections); i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for d := range depC {
+				if err := downloadDependency(ctx, d); err != nil {
+					fmt.Printf("%s: %v", d.Importpath, err)
+					atomic.AddUint32(&errors, 1)
+				}
+			}
+		}()
+	}
+
 	for _, dep := range m.Dependencies {
-		fmt.Printf("Getting %s\n", dep.Importpath)
-		repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, insecure)
-		if err != nil {
-			return fmt.Errorf("Could not process dependency: %s", err)
-		}
-		wc, err := repo.Checkout("", "", dep.Revision)
-		if err != nil {
-			return fmt.Errorf("Could not retrieve dependency: %s", err)
-		}
-		dst := filepath.Join(ctx.Projectdir(), "vendor", "src", dep.Importpath)
-		src := filepath.Join(wc.Dir(), dep.Path)
+		depC <- dep
+	}
+	close(depC)
+	wg.Wait()
 
-		if _, err := os.Stat(dst); err == nil {
-			if err := fileutils.RemoveAll(dst); err != nil {
-				return fmt.Errorf("dependency could not be deleted: %v", err)
-			}
-		}
+	if errors > 0 {
+		return fmt.Errorf("failed to fetch %d dependencies", errors)
+	}
 
-		if err := fileutils.Copypath(dst, src); err != nil {
-			return err
-		}
+	return nil
+}
+
+func downloadDependency(ctx *gb.Context, dep vendor.Dependency) error {
+	fmt.Printf("Getting %s\n", dep.Importpath)
+	repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, rbInsecure)
+	if err != nil {
+		return fmt.Errorf("dependency could not be processed: %s", err)
+	}
+	wc, err := repo.Checkout("", "", dep.Revision)
+	if err != nil {
+		return fmt.Errorf("dependency could not be fetched: %s", err)
+	}
+	dst := filepath.Join(ctx.Projectdir(), "vendor", "src", dep.Importpath)
+	src := filepath.Join(wc.Dir(), dep.Path)
 
-		if err := wc.Destroy(); err != nil {
-			return err
+	if _, err := os.Stat(dst); err == nil {
+		if err := fileutils.RemoveAll(dst); err != nil {
+			return fmt.Errorf("dependency could not be deleted: %v", err)
 		}
+	}
 
+	if err := fileutils.Copypath(dst, src); err != nil {
+		return err
 	}
+
+	if err := wc.Destroy(); err != nil {
+		return err
+	}
+
 	return nil
 }