Skip to content

Feature Request: Add a subcommand to list all skipped/pending tests #1537

@bartsmykla

Description

@bartsmykla

Hi Ginkgo team! 👋

Our team at Kuma service mesh would love to have an easy way to list all pending or skipped test specs. Here's why:

We run many end-to-end (e2e) tests, and occasionally, some tests become flaky. When this happens, we temporarily skip these tests to keep our CI green. Later, when we have time, we revisit these skipped tests to fix them. Having a quick command to list these pending tests would make our workflow much simpler.

Right now, we use are considering workaround like this:

ginkgo --json-report report.json --dry-run $(find . -type f -name "*suite_test.go*" -exec dirname "{}" \; | uniq)

We then process the output to find skipped/pending tests (see our attempt here). But this takes ~11 minutes to run on MacBook Pro with M3 Max and 36GB of RAM, which isn't ideal.

A simple proof-of-concept we made that directly parses Go files takes less than 1 second:

~ time ginkgo pending ../../kumahq/kuma/...
Executed in  872.21 millis

Would you consider adding something like a ginkgo pending subcommand to list pending/skipped tests? With some guidance we'd be happy to contribute this feature as we've already tested a basic version and it looks straightforward.

ginkgo/pending/pending_command.go
package pending

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"slices"
	"strings"

	"golang.org/x/tools/go/ast/inspector"

	"github.com/onsi/ginkgo/v2/ginkgo/command"
	"github.com/onsi/ginkgo/v2/ginkgo/internal"
	"github.com/onsi/ginkgo/v2/types"
)

func BuildPendingCommand() command.Command {
	var cliConfig = types.NewDefaultCLIConfig()

	flags, err := types.BuildLabelsCommandFlagSet(&cliConfig)
	if err != nil {
		panic(err)
	}
	return command.Command{
		Name:     "pending",
		Usage:    "ginkgo pending",
		Flags:    flags,
		ShortDoc: "Recursively search any pending or excluded tests under the current directory",
		DocLink:  "filtering-specs",
		Command: func(args []string, _ []string) {
			pendingSpecs(args, cliConfig)
		},
	}
}

var prefixes = []string{
	"PDescribe", "PContext", "PIt", "PDescribeTable", "PEntry", "PSpecify", "PWhen",
	"XDescribe", "XContext", "XIt", "XDescribeTable", "XEntry", "XSpecify", "XWhen",
}

func pendingSpecs(args []string, cliConfig types.CLIConfig) {
	fmt.Println("Scanning for pending...")
	suites := internal.FindSuites(args, cliConfig, false)
	if len(suites) == 0 {
		command.AbortWith("Found no test suites")
	}
	for _, suite := range suites {
		res := fetchPendingFromPackage(suite.Path)
		if len(res) > 0 {
			fmt.Printf("%s:\n", suite.PackageName)
			for _, v := range res {
				fmt.Printf("\t%s : %s\n", v.pos.String(), v.name)
			}
		}
	}
}

type out struct {
	pos  token.Position
	name string
}

func fetchPendingFromPackage(packagePath string) []out {
	fset := token.NewFileSet()
	parsedPackages, err := parser.ParseDir(fset, packagePath, nil, 0)
	command.AbortIfError("Failed to parse package source:", err)

	files := []*ast.File{}
	hasTestPackage := false
	for key, pkg := range parsedPackages {
		if strings.HasSuffix(key, "_test") {
			hasTestPackage = true
			for _, file := range pkg.Files {
				files = append(files, file)
			}
		}
	}
	if !hasTestPackage {
		for _, pkg := range parsedPackages {
			for _, file := range pkg.Files {
				files = append(files, file)
			}
		}
	}

	res := []out{}
	ispr := inspector.New(files)
	ispr.Preorder([]ast.Node{&ast.CallExpr{}}, func(n ast.Node) {
		if c, ok := n.(*ast.CallExpr); ok {
			if i, ok := c.Fun.(*ast.Ident); ok && slices.Contains(prefixes, i.Name) {
				pos := fset.Position(c.Pos())
				res = append(res, out{pos: pos, name: c.Args[0].(*ast.BasicLit).Value})
			}
		}
	})
	return res
}

Let us know what you think

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions