code annotations and bookmarks

Author: joonas-fi

cmd/bob/main.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/function61/gokit/app/dynversion"
 	"github.com/function61/gokit/os/osutil"
+	"github.com/function61/turbobob/pkg/bookmarkparser"
 	"github.com/function61/turbobob/pkg/powerline"
 	"github.com/spf13/cobra"
 )
@@ -83,6 +84,7 @@ func toolsEntry() *cobra.Command {
 	// TODO: move powerline here?
 	cmd.AddCommand(initEntry())
 	cmd.AddCommand(langserverEntry())
+	cmd.AddCommand(bookmarkparser.Entrypoint())
 
 	return cmd
 }

pkg/annotationparser/filescanner.go

@@ -0,0 +1,61 @@
+package annotationparser
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+)
+
+type Parser interface {
+	Parse(ctx context.Context, path string, processAnnotation func(Location) error) error
+}
+
+type Annotations map[string]interface{}
+
+// the location of the annotation in a file
+type Location struct {
+	File        string
+	Line        int // the line the annotation is found from. (the line the annotation describes is usually next line)
+	Offset      int
+	Annotations Annotations
+}
+
+func (a Location) LineTarget() int {
+	return a.Line + 1
+}
+
+func ScanDirectoryFilesRecursively(ctx context.Context, dir string, processAnnotation func(Location) error, parsers map[string]Parser) error {
+	if err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if isIgnoredEntryName(info.Name()) {
+			return fs.SkipDir
+		}
+
+		if info.IsDir() { // only process files
+			return nil
+		}
+
+		ext := filepath.Ext(path)
+		if parser, found := parsers[ext]; found {
+			fullPath := filepath.Join(dir, path)
+
+			if err := parser.Parse(ctx, fullPath, processAnnotation); err != nil {
+				return fmt.Errorf("Parse %s: %w", fullPath, err)
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isIgnoredEntryName(name string) bool {
+	return name == ".git"
+}

pkg/annotationparser/helpers.go

@@ -0,0 +1,47 @@
+package annotationparser
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+)
+
+func scanLineBasedContentForRegexMatches(
+	content io.Reader,
+	re *regexp.Regexp,
+	processResult func(match string, fullLine string, lineNumber int, lineOffset int) error,
+) error {
+	// scan line by line to be able to keep track of line numbers and annotation match byte offsets more easily.
+	//
+	// NOTE: docs advise to use a higher-level scanner but returned lines don't contain the line terminators
+	// and since the terminator can be "\r\n" or "\n" we can't calculate line byte offsets unless we know the exact line.
+	lineScanner := bufio.NewReader(content)
+
+	lineNumber := 1
+	lineOffset := 0
+
+	for {
+		// line can end in "\n" (Unix convention) "\r\n" (Windows convention).
+		// both conveniently end in "\n" so that's what we look for.
+		line, err := lineScanner.ReadBytes('\n')
+		isEOF := err == io.EOF
+		if err != nil && !isEOF { // last line has eof set but we need to process its possible content first before stopping
+			return err
+		}
+
+		annotationsMatch := re.FindStringSubmatch(string(line))
+
+		if annotationsMatch != nil {
+			if err := processResult(annotationsMatch[1], annotationsMatch[0], lineNumber, lineOffset); err != nil {
+				return err
+			}
+		}
+
+		lineNumber++
+		lineOffset += len(line)
+
+		if isEOF {
+			return nil
+		}
+	}
+}

pkg/annotationparser/parser.go

@@ -0,0 +1,68 @@
+// Parses annotations from source code
+package annotationparser
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+)
+
+var (
+	// parses lines that look like `// @:{"key": "value"}`.
+	// whitespace is allowed to precede `//`.
+	doubleSlashAnnotationParser = &annotationParserFromRegex{regexp.MustCompile(`^[ \t]*// @:([^\n]+)`)}
+
+	DefaultParsersForFileTypes = map[string]Parser{
+		/*	Annotation format for Go
+
+			established keys are:
+
+			//go:generaste
+			//go:embed
+			//go:build
+			//nolint:<lintername>
+			//easyjson:json
+
+			hence the syntax should be "//" + prefix + ":"
+
+			for prefix seems we cannot use symbols like "@" or "_" because `$ goimports` indents to "// ",
+			so must choose alphanumeric keys.
+		*/
+		".go": doubleSlashAnnotationParser,
+	}
+)
+
+// uses a regex to parse annotations from a line
+type annotationParserFromRegex struct {
+	annotationsRe *regexp.Regexp
+}
+
+func (g *annotationParserFromRegex) Parse(ctx context.Context, path string, processAnnotation func(Location) error) error {
+	// unfortunately we need to buffer this as there is no `FindAllStringSubmatch` that operates on streams
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return scanLineBasedContentForRegexMatches(file, g.annotationsRe, func(match string, fullLine string, lineNumber int, lineOffset int) error {
+		annots := Annotations{}
+
+		if err := json.Unmarshal([]byte(match), &annots); err != nil {
+			return fmt.Errorf("%w\ninput:\n%s", err, fullLine)
+		}
+
+		if err := processAnnotation(Location{
+			File:        path,
+			Line:        lineNumber,
+			Offset:      lineOffset,
+			Annotations: annots,
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}

pkg/bookmarkparser/bookmarkparser.go

@@ -0,0 +1,86 @@
+// Parses code bookmarks from project source code's annotations
+package bookmarkparser
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"regexp"
+
+	"github.com/function61/gokit/encoding/jsonfile"
+	"github.com/function61/turbobob/pkg/annotationparser"
+)
+
+type ID string
+
+var (
+	// starting with URL safe characters (to easily represent in URLs) restrictive set is always safer to widen than the other way around.
+	// subset of https://stackoverflow.com/a/695469
+	idRe = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`)
+)
+
+func (i ID) Validate() error {
+	if !idRe.MatchString(string(i)) {
+		return fmt.Errorf("bookmark ID '%s' does not match regex '%s'", i, idRe.String())
+	}
+
+	return nil
+}
+
+type Bookmark struct {
+	ID   ID     // (hopefully) permanent ID, semantic "symbol" of the bookmark. i.e. favor naming by the concept rather than directly e.g. function name.
+	File string // path relative to repository root
+	Line int    // line being referenced
+}
+
+// @:{"bookmark": "ParseBookmarks"}
+func ParseBookmarks(ctx context.Context, dir string, output io.Writer) error {
+	allAnnotations := []annotationparser.Location{}
+
+	// parses annotations, but not yet mapped to bookmarks
+	if err := annotationparser.ScanDirectoryFilesRecursively(ctx, dir, func(annotation annotationparser.Location) error {
+		allAnnotations = append(allAnnotations, annotation)
+		return nil
+	}, annotationparser.DefaultParsersForFileTypes); err != nil {
+		return err
+	}
+
+	bookmarks, err := annotationsToBookmarks(allAnnotations)
+	if err != nil {
+		return err
+	}
+
+	return jsonfile.Marshal(output, bookmarks)
+}
+
+func annotationsToBookmarks(annotations []annotationparser.Location) ([]Bookmark, error) {
+	withErr := func(err error) ([]Bookmark, error) { return nil, fmt.Errorf("annotationsToBookmarks: %w", err) }
+
+	bookmarks := []Bookmark{}
+	for _, annotation := range annotations {
+		bookmarkID, hasBookmark := annotation.Annotations["bookmark"]
+		if !hasBookmark {
+			continue
+		}
+
+		id_, ok := bookmarkID.(string)
+		if !ok {
+			return withErr(errors.New("bookmark ID not string"))
+		}
+
+		id := ID(id_)
+
+		if err := id.Validate(); err != nil {
+			return withErr(err)
+		}
+
+		bookmarks = append(bookmarks, Bookmark{
+			ID:   id,
+			File: annotation.File,
+			Line: annotation.LineTarget(),
+		})
+	}
+
+	return bookmarks, nil
+}

pkg/bookmarkparser/cli.go

@@ -0,0 +1,21 @@
+package bookmarkparser
+
+import (
+	"context"
+	"log"
+	"os"
+
+	"github.com/function61/gokit/app/cli"
+	"github.com/spf13/cobra"
+)
+
+func Entrypoint() *cobra.Command {
+	return &cobra.Command{
+		Use:   "bookmarks-build",
+		Short: "Build code bookmarks database from current directory",
+		Args:  cobra.NoArgs,
+		Run: cli.RunnerNoArgs(func(ctx context.Context, _ *log.Logger) error {
+			return ParseBookmarks(ctx, ".", os.Stdout)
+		}),
+	}
+}