Merge pull request #6 from moul/dev/moul/retry

Author: moul

.github/workflows/go.yml

@@ -73,9 +73,6 @@ jobs:
     strategy:
       matrix:
         golang:
-          #- 1.11.13
-          #- 1.12.17
-          #- 1.13.15
           #- 1.14.7
           - 1.15.1
     steps:
@@ -133,9 +130,6 @@ jobs:
     strategy:
       matrix:
         golang:
-          - 1.11.13
-          - 1.12.17
-          - 1.13.15
           - 1.14.7
           - 1.15.1
     env:

Makefile

@@ -1,5 +1,6 @@
 GOPKG ?=	moul.io/testman
 DOCKER_IMAGE ?=	moul/testman
+GOMOD_DIRS ?= .
 GOBINS ?=	.
 NPM_PACKAGES ?=	.
 

examples/Makefile

@@ -1,7 +1,7 @@
 integration:
 	# test with testman
-	testman test -run ^TestStable ./...
-	@# FIXME: test unstable tests
+	testman test -timeout=10s -run ^TestStable ./...  # should always work
+	testman test -timeout=60s -run ^TestUnstable -retry=50 ./...  # should work at least 1/50
 	@# FIXME: test broken tests
 
 	# test with default tools

go.mod

@@ -14,8 +14,8 @@ import (
 	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"moul.io/godev"
 	"moul.io/motd"
+	"moul.io/u"
 )
 
 var opts Opts
@@ -34,8 +34,8 @@ func run(args []string) error {
 	testFlags := flag.NewFlagSet("testman test", flag.ExitOnError)
 	testFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
 	testFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
-	//testFlags.IntVar(&opts.Retry, "retry", 0, "fail after N retries")
-	//testFlags.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "max duration allowed to run the whole suite")
+	testFlags.IntVar(&opts.Retry, "retry", 0, "fail after N retries")
+	testFlags.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "max duration allowed to run the whole suite")
 	listFlags := flag.NewFlagSet("testman list", flag.ExitOnError)
 	listFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
 	listFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
@@ -53,14 +53,14 @@ func run(args []string) error {
 				FlagSet:    testFlags,
 				ShortHelp:  "advanced go test workflows",
 				ShortUsage: "testman test [flags] [packages]",
-				LongHelp:   "EXAMPLES\n   testman test -v ./...",
+				LongHelp:   testLongHelp,
 				Exec:       runTest,
 			}, {
 				Name:       "list",
 				FlagSet:    listFlags,
 				ShortHelp:  "list available tests",
 				ShortUsage: "testman list [packages]",
-				LongHelp:   "EXAMPLE\n   testman list ./...",
+				LongHelp:   listLongHelp,
 				Exec:       runList,
 			},
 		},
@@ -69,11 +69,26 @@ func run(args []string) error {
 	return root.ParseAndRun(context.Background(), args[1:])
 }
 
+const (
+	testLongHelp = `EXAMPLES
+   testman test ./...
+   testman test -v ./...
+   testman test -run ^TestUnstable -timeout=300s -retry=50 ./...`
+	listLongHelp = `EXAMPLES
+   testman list ./...
+   testman list -v ./...
+   testman list -run ^TestStable ./...`
+)
+
 func runList(ctx context.Context, args []string) error {
 	if len(args) == 0 {
 		return flag.ErrHelp
 	}
-	preRun()
+	cleanup, err := preRun()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
 
 	// list packages
 	pkgs, err := listPackagesWithTests(args)
@@ -103,22 +118,28 @@ func runTest(ctx context.Context, args []string) error {
 	if len(args) == 0 {
 		return flag.ErrHelp
 	}
-	preRun()
-	log.Printf("runTest opts=%s args=%s", godev.JSON(opts), godev.JSON(args))
-	start := time.Now()
-
-	// list packages
-	pkgs, err := listPackagesWithTests(args)
+	cleanup, err := preRun()
 	if err != nil {
 		return err
 	}
+	defer cleanup()
 
-	// create temp dir
-	tmpdir, err := ioutil.TempDir("", "testman")
+	log.Printf("runTest opts=%s args=%s", u.JSON(opts), u.JSON(args))
+	start := time.Now()
+
+	if opts.Timeout > 0 {
+		go func() {
+			<-time.After(opts.Timeout)
+			fmt.Printf("FAIL: timed out after %s\n", time.Since(start))
+			os.Exit(1)
+		}()
+	}
+
+	// list packages
+	pkgs, err := listPackagesWithTests(args)
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tmpdir)
 
 	atLeastOneFailure := false
 	// list tests
@@ -133,7 +154,7 @@ func runTest(ctx context.Context, args []string) error {
 
 		pkgStart := time.Now()
 		// compile test binary
-		bin, err := compileTestBin(pkg, tmpdir)
+		bin, err := compileTestBin(pkg, opts.TmpDir)
 		if err != nil {
 			fmt.Printf("FAIL\t%s\t[compile error: %v]\n", pkg.ImportPath, err)
 			return err
@@ -144,40 +165,61 @@ func runTest(ctx context.Context, args []string) error {
 			// FIXME: check if matches run regex
 			args := []string{
 				"-test.count=1",
-				"-test.timeout=300s",
+				fmt.Sprintf("-test.timeout=%s", opts.Timeout),
 			}
 			if opts.Verbose {
 				args = append(args, "-test.v")
 			}
 			args = append(args, "-test.run", fmt.Sprintf("^%s$", test))
-			cmd := exec.Command(bin, args...)
-			log.Println(cmd.String())
-			out, err := cmd.CombinedOutput()
-			if err != nil {
-				fmt.Printf("FAIL\t%s.%s\t[compile error: %v]\n", pkg.ImportPath, test, err)
-				if opts.Verbose {
-					fmt.Println(string(out))
+			for i := opts.Retry; i >= 0; i-- {
+				cmd := exec.Command(bin, args...)
+				log.Println(cmd.String())
+				out, err := cmd.CombinedOutput()
+				if err != nil {
+					if i == 0 {
+						fmt.Printf("FAIL\t%s.%s\t[test error: %v]\n", pkg.ImportPath, test, err)
+						isPackageOK = false
+						atLeastOneFailure = true
+					} else if opts.Verbose {
+						fmt.Printf("RETRY\t%s.%s\t[test error: %v]\n", pkg.ImportPath, test, err)
+					}
+					if opts.Verbose {
+						fmt.Println(string(out))
+					}
+				} else {
+					fmt.Printf("ok\t%s.%s\n", pkg.ImportPath, test)
+					break
 				}
-				isPackageOK = false
-				atLeastOneFailure = true
 			}
 		}
 		if isPackageOK {
 			fmt.Printf("ok\t%s\t%s\n", pkg.ImportPath, time.Since(pkgStart))
 		}
 	}
 
-	fmt.Printf("total: %s\n", time.Since(start))
+	log.Printf("total: %s\n", time.Since(start))
 	if atLeastOneFailure {
 		os.Exit(1)
 	}
 	return nil
 }
 
-func preRun() {
+func preRun() (func(), error) {
 	if !opts.Verbose {
 		log.SetOutput(ioutil.Discard)
 	}
+
+	// create temp dir
+	var err error
+	opts.TmpDir, err = ioutil.TempDir("", "testman")
+	if err != nil {
+		return nil, err
+	}
+
+	cleanup := func() {
+		os.RemoveAll(opts.TmpDir)
+	}
+	return cleanup, nil
 }
 
 func compileTestBin(pkg Package, tempdir string) (string, error) {
@@ -258,8 +300,9 @@ type Package struct {
 type Opts struct {
 	Verbose bool
 	Run     string
-	// Timeout time.Duration
-	// Retry   int
+	Timeout time.Duration
+	Retry   int
+	TmpDir  string
 	// c
 	// debug
 	// continueOnFailure vs failFast

go.sum

@@ -30,6 +30,9 @@ all: help
 ##
 
 rwildcard = $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
+check-program = $(foreach exec,$(1),$(if $(shell PATH="$(PATH)" which $(exec)),,$(error "No $(exec) in PATH")))
+my-filter-out = $(foreach v,$(2),$(if $(findstring $(1),$(v)),,$(v)))
+novendor = $(call my-filter-out,vendor/,$(1))
 
 ##
 ## rules.mk
@@ -70,7 +73,7 @@ GO ?= go
 GOPATH ?= $(HOME)/go
 GO_INSTALL_OPTS ?=
 GO_TEST_OPTS ?= -test.timeout=30s
-GOMOD_DIR ?= .
+GOMOD_DIRS ?= $(sort $(call novendor,$(dir $(call rwildcard,.,*/go.mod go.mod))))
 GOCOVERAGE_FILE ?= ./coverage.txt
 GOTESTJSON_FILE ?= ./go-test.json
 GOBUILDLOG_FILE ?= ./go-build.log
@@ -96,6 +99,7 @@ INSTALL_STEPS += go.install
 
 .PHONY: go.release
 go.release:
+	$(call check-program, goreleaser)
 	goreleaser --snapshot --skip-publish --rm-dist
 	@echo -n "Do you want to release? [y/N] " && read ans && \
 	  if [ $${ans:-N} = y ]; then set -xe; goreleaser --rm-dist; fi
@@ -107,18 +111,19 @@ go.unittest:
 ifeq ($(CI),true)
 	@echo "mode: atomic" > /tmp/gocoverage
 	@rm -f $(GOTESTJSON_FILE)
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do (set -e; (set -euf pipefail; \
+	@set -e; for dir in $(GOMOD_DIRS); do (set -e; (set -euf pipefail; \
 	    cd $$dir; \
-	    ($(GO) test ./... $(GO_TEST_OPTS) -cover -coverprofile=/tmp/profile.out -covermode=atomic -race -json | tee -a $(GOTESTJSON_FILE) 3>&1 1>&2 2>&3 | tee -a $(GOBUILDLOG_FILE); \
+	    (($(GO) test ./... $(GO_TEST_OPTS) -cover -coverprofile=/tmp/profile.out -covermode=atomic -race -json && touch $@.ok) | tee -a $(GOTESTJSON_FILE) 3>&1 1>&2 2>&3 | tee -a $(GOBUILDLOG_FILE); \
 	  ); \
+	  rm $@.ok 2>/dev/null || exit 1; \
 	  if [ -f /tmp/profile.out ]; then \
 	    cat /tmp/profile.out | sed "/mode: atomic/d" >> /tmp/gocoverage; \
 	    rm -f /tmp/profile.out; \
 	  fi)); done
 	@mv /tmp/gocoverage $(GOCOVERAGE_FILE)
 else
 	@echo "mode: atomic" > /tmp/gocoverage
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do (set -e; (set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do (set -e; (set -xe; \
 	  cd $$dir; \
 	  $(GO) test ./... $(GO_TEST_OPTS) -cover -coverprofile=/tmp/profile.out -covermode=atomic -race); \
 	  if [ -f /tmp/profile.out ]; then \
@@ -130,44 +135,44 @@ endif
 
 .PHONY: go.checkdoc
 go.checkdoc:
-	go doc $(GOMOD_DIR)
+	go doc $(first $(GOMOD_DIRS))
 
 .PHONY: go.coverfunc
 go.coverfunc: go.unittest
 	go tool cover -func=$(GOCOVERAGE_FILE) | grep -v .pb.go: | grep -v .pb.gw.go:
 
 .PHONY: go.lint
 go.lint:
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do ( set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do ( set -xe; \
 	  cd $$dir; \
 	  golangci-lint run --verbose ./...; \
 	); done
 
 .PHONY: go.tidy
 go.tidy:
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do ( set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do ( set -xe; \
 	  cd $$dir; \
 	  $(GO)	mod tidy; \
 	); done
 
 .PHONY: go.build
 go.build:
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do ( set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do ( set -xe; \
 	  cd $$dir; \
 	  $(GO)	build ./...; \
 	); done
 
 .PHONY: go.bump-deps
 go.bumpdeps:
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do ( set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do ( set -xe; \
 	  cd $$dir; \
 	  $(GO)	get -u ./...; \
 	); done
 
 .PHONY: go.bump-deps
 go.fmt:
 	if ! command -v goimports &>/dev/null; then GO111MODULE=off go get golang.org/x/tools/cmd/goimports; fi
-	@set -e; for dir in `find $(GOMOD_DIR) -type f -name "go.mod" | grep -v /vendor/ | sed 's@/[^/]*$$@@' | sort | uniq`; do ( set -xe; \
+	@set -e; for dir in $(GOMOD_DIRS); do ( set -xe; \
 	  cd $$dir; \
 	  goimports -w `go list -f '{{.Dir}}' ./...)` \
 	); done
@@ -238,6 +243,7 @@ ifdef DOCKER_IMAGE
 ifneq ($(DOCKER_IMAGE),none)
 .PHONY: docker.build
 docker.build:
+	$(call check-program, docker)
 	$(call docker_build,$(DOCKERFILE_PATH),$(DOCKER_IMAGE))
 
 BUILD_STEPS += docker.build
@@ -316,3 +322,5 @@ help::
 	@[ "$(TIDY_STEPS)" != "" ]      && echo "  tidy"      || true
 	@[ "$(UNITTEST_STEPS)" != "" ]  && echo "  unittest"  || true
 	@# FIXME: list other commands
+
+print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true