feat: provenance tracking (#1488)

Author: ThomasRooney

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/getkin/kin-openapi v0.128.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/gofrs/flock v0.12.1
+	github.com/google/go-cmp v0.6.0
 	github.com/google/go-github/v58 v58.0.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.7.0
@@ -38,7 +39,7 @@ require (
 	github.com/speakeasy-api/openapi-overlay v0.10.1
 	github.com/speakeasy-api/sdk-gen-config v1.31.1
 	github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.26.1
-	github.com/speakeasy-api/speakeasy-core v0.19.8
+	github.com/speakeasy-api/speakeasy-core v0.20.0
 	github.com/speakeasy-api/speakeasy-proxy v0.0.2
 	github.com/speakeasy-api/versioning-reports v0.6.0
 	github.com/spf13/cobra v1.9.1
@@ -133,7 +134,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
 	github.com/google/uuid v1.6.0 // indirect

go.sum

@@ -589,8 +589,8 @@ github.com/speakeasy-api/sdk-gen-config v1.31.1 h1:peGHmojOH4OzCpKHoaQ++a7J6hLy6
 github.com/speakeasy-api/sdk-gen-config v1.31.1/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.26.1 h1:cX+ip9YHkunvIHBALjDDIyEvo8rCLUCfIo7ldzcIeU4=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.26.1/go.mod h1:k9JD6Rj0+Iizc5COoLZHyRIOGGITpKZ2qBuFFO8SqNI=
-github.com/speakeasy-api/speakeasy-core v0.19.8 h1:JuHPTV9JRzdC2z0bRRjn9sf3f5K4twxSYBrxtSmGJiQ=
-github.com/speakeasy-api/speakeasy-core v0.19.8/go.mod h1:KqEmQy04aCZJuUed8mt6sywpLcqbeQfRJk9UDWD0cCk=
+github.com/speakeasy-api/speakeasy-core v0.20.0 h1:htH4AsRUIx1dciukesF2ICI81KkmLXg4NHYQ8m66woA=
+github.com/speakeasy-api/speakeasy-core v0.20.0/go.mod h1:KqEmQy04aCZJuUed8mt6sywpLcqbeQfRJk9UDWD0cCk=
 github.com/speakeasy-api/speakeasy-go-sdk v1.8.1 h1:atzohw12oQ5ipaLb1q7ntTu4vvAgKDJsrvaUoOu6sw0=
 github.com/speakeasy-api/speakeasy-go-sdk v1.8.1/go.mod h1:XbzaM0sMjj8bGooz/uEtNkOh1FQiJK7RFuNG3LPBSAU=
 github.com/speakeasy-api/speakeasy-proxy v0.0.2 h1:u4rQ8lXvuYRCSxiLQGb5JxkZRwNIDlyh+pMFYD6OGjA=

integration/multi_test.go

@@ -0,0 +1,86 @@
+package integration_tests
+
+import (
+	"github.com/google/go-cmp/cmp"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/speakeasy-api/sdk-gen-config/workflow"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMultiFileStability(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+	// If windows, skip
+	temp := setupTestDir(t)
+
+	// Copy the multi-file OpenAPI spec files
+	err := copyFile("resources/multi_root.yaml", filepath.Join(temp, "multi_root.yaml"))
+	require.NoError(t, err)
+	err = copyFile("resources/multi_components.yaml", filepath.Join(temp, "multi_components.yaml"))
+	require.NoError(t, err)
+
+	// Create a workflow file with multi-file input
+	workflowFile := &workflow.Workflow{
+		Version: workflow.WorkflowVersion,
+		Sources: map[string]workflow.Source{
+			"multi-file-source": {
+				Inputs: []workflow.Document{
+					{Location: workflow.LocationString("multi_root.yaml")},
+				},
+			},
+		},
+		Targets: map[string]workflow.Target{
+			"multi-file-target": {
+				Target: "typescript",
+				Source: "multi-file-source",
+			},
+		},
+	}
+
+	err = os.MkdirAll(filepath.Join(temp, ".speakeasy"), 0o755)
+	require.NoError(t, err)
+	err = workflow.Save(temp, workflowFile)
+	require.NoError(t, err)
+
+	// Run the initial generation
+	var initialChecksums map[string]string
+	initialArgs := []string{"run", "-t", "all", "--force", "--pinned", "--skip-versioning", "--skip-compile"}
+	cmdErr := execute(t, temp, initialArgs...).Run()
+	require.NoError(t, cmdErr)
+
+	// Calculate checksums of generated files
+	initialChecksums, err = filesToString(temp)
+	require.NoError(t, err)
+
+	// Re-run the generation. We should have stable digests.
+	cmdErr = execute(t, temp, initialArgs...).Run()
+	require.NoError(t, cmdErr)
+	rerunChecksums, err := filesToString(temp)
+	require.NoError(t, err)
+
+	// Compare checksums to ensure stability
+	require.Equal(t, initialChecksums, rerunChecksums, "Generated files should be identical for multi-file OpenAPI specs")
+
+	// Test frozen workflow lock behavior
+	frozenArgs := []string{"run", "-t", "all", "--pinned", "--frozen-workflow-lockfile", "--skip-compile"}
+	cmdErr = execute(t, temp, frozenArgs...).Run()
+	require.NoError(t, cmdErr)
+
+	// Calculate checksums after frozen run
+	frozenChecksums, err := filesToString(temp)
+	require.NoError(t, err)
+
+	// exclude gen.lock -- we could (we do) reformat the document inside the frozen one
+	delete(frozenChecksums, ".speakeasy/gen.lock")
+	delete(initialChecksums, ".speakeasy/gen.lock")
+
+	// Compare checksums
+	if diff := cmp.Diff(initialChecksums, frozenChecksums); diff != "" {
+		t.Fatalf("Generated files should be identical when using --frozen-workflow-lock with multi-file specs. Mismatch (-want +got):\n%s", diff)
+	}
+}

integration/resources/multi_components.yaml

@@ -0,0 +1,18 @@
+components:
+  schemas:
+    TestResponse:
+      type: object
+      properties:
+        status:
+          type: string
+          enum: ["success", "error"]
+        data:
+          type: array
+          items:
+            type: object
+            properties:
+              value:
+                type: string
+              timestamp:
+                type: string
+                format: date-time

integration/resources/multi_root.yaml

@@ -0,0 +1,25 @@
+openapi: 3.1.0
+info:
+  title: Multi-File Test API
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      operationId: getTest
+      summary: Test endpoint
+      responses:
+        "200":
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                $ref: "multi_components.yaml#/components/schemas/TestResponse"
+components:
+  schemas:
+    LocalSchema:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string

integration/workflow_registry_test.go

@@ -1,8 +1,6 @@
 package integration_tests
 
 import (
-	"crypto/md5"
-	"fmt"
 	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
@@ -13,8 +11,6 @@ import (
 )
 
 func TestStability(t *testing.T) {
-	t.Skip("Skipping stability test until we can figure out how to make it work on CI")
-	t.Parallel()
 	temp := setupTestDir(t)
 
 	// Create a basic workflow file
@@ -47,15 +43,15 @@ func TestStability(t *testing.T) {
 	require.NoError(t, cmdErr)
 
 	// Calculate checksums of generated files
-	initialChecksums, err = calculateChecksums(temp)
+	initialChecksums, err = filesToString(temp)
 	require.NoError(t, err)
 
 	// Re-run the generation. We should have stable digests.
 	cmdErr = execute(t, temp, initialArgs...).Run()
 	require.NoError(t, cmdErr)
-	rerunChecksums, err := calculateChecksums(temp)
+	rerunChecksums, err := filesToString(temp)
 	require.NoError(t, err)
-	require.Equal(t, initialChecksums, rerunChecksums, "Generated files should be identical when using --frozen-workflow-lock")
+	require.Equal(t, initialChecksums, rerunChecksums, "Generated files should be identical")
 	// Modify the workflow file to simulate a change
 	// Shouldn't do anything; we'll validate that later.
 	workflowFile.Sources["test-source"].Inputs[0].Location = "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v3.1/petstore.yaml"
@@ -67,9 +63,14 @@ func TestStability(t *testing.T) {
 	require.NoError(t, cmdErr)
 
 	// Calculate checksums after frozen run
-	frozenChecksums, err := calculateChecksums(temp)
+	frozenChecksums, err := filesToString(temp)
 	require.NoError(t, err)
 
+	// exclude gen.lock -- we could (we do) reformat the document inside the frozen one
+	delete(frozenChecksums, ".speakeasy/gen.lock")
+	delete(frozenChecksums, ".speakeasy\\gen.lock") // windows
+	delete(initialChecksums, ".speakeasy/gen.lock")
+	delete(initialChecksums, ".speakeasy\\gen.lock") // windows
 	// Compare checksums
 	require.Equal(t, initialChecksums, frozenChecksums, "Generated files should be identical when using --frozen-workflow-lock")
 }
@@ -168,7 +169,7 @@ func TestRegistryFlow_JSON(t *testing.T) {
 	require.NoError(t, cmdErr)
 }
 
-func calculateChecksums(dir string) (map[string]string, error) {
+func filesToString(dir string) (map[string]string, error) {
 	checksums := make(map[string]string)
 	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -180,7 +181,7 @@ func calculateChecksums(dir string) (map[string]string, error) {
 				return err
 			}
 			relPath, _ := filepath.Rel(dir, path)
-			checksums[relPath] = fmt.Sprintf("%x", md5.Sum(data))
+			checksums[relPath] = string(data)
 		}
 		return nil
 	})

integration/workflow_test.go

@@ -248,6 +248,8 @@ func execute(t *testing.T, wd string, args ...string) Runnable {
 }
 
 // executeI is a helper function to execute the main.go file inline. It can help when debugging integration tests
+// We should not use it on multiple tests at once as they will share memory: this can create issues.
+// so we leave it around as a little helper method: swap out execute for executeI and debug breakpoints work
 var mutex sync.Mutex
 var rootCmd = cmd.CmdForTest(version, artifactArch)
 

internal/run/merge.go

@@ -20,7 +20,10 @@ type Merge struct {
 	ruleset    string
 }
 
-var _ SourceStep = Merge{}
+type MergeResult struct {
+	Location            string
+	InputSchemaLocation []string
+}
 
 func NewMerge(w *Workflow, parentStep *workflowTracking.WorkflowStep, source workflow.Source, ruleset string) Merge {
 	return Merge{
@@ -31,29 +34,29 @@ func NewMerge(w *Workflow, parentStep *workflowTracking.WorkflowStep, source wor
 	}
 }
 
-func (m Merge) Do(ctx context.Context, _ string) (string, error) {
+func (m Merge) Do(ctx context.Context, _ string) (result MergeResult, err error) {
 	mergeStep := m.parentStep.NewSubstep("Merge Documents")
 
-	mergeLocation := m.source.GetTempMergeLocation()
+	result.Location = m.source.GetTempMergeLocation()
 
-	log.From(ctx).Infof("Merging %d schemas into %s...", len(m.source.Inputs), mergeLocation)
+	log.From(ctx).Infof("Merging %d schemas into %s...", len(m.source.Inputs), result.Location)
 
-	inSchemas := []string{}
 	for _, input := range m.source.Inputs {
-		resolvedPath, err := schemas.ResolveDocument(ctx, input, nil, mergeStep)
+		var resolvedPath string
+		resolvedPath, err = schemas.ResolveDocument(ctx, input, nil, mergeStep)
 		if err != nil {
-			return "", err
+			return
 		}
-		inSchemas = append(inSchemas, resolvedPath)
+		result.InputSchemaLocation = append(result.InputSchemaLocation, resolvedPath)
 	}
 
 	mergeStep.NewSubstep(fmt.Sprintf("Merge %d documents", len(m.source.Inputs)))
 
-	if err := mergeDocuments(ctx, inSchemas, mergeLocation, m.ruleset, m.workflow.ProjectDir, m.workflow.SkipGenerateLintReport); err != nil {
-		return "", err
+	if err = mergeDocuments(ctx, result.InputSchemaLocation, result.Location, m.ruleset, m.workflow.ProjectDir, m.workflow.SkipGenerateLintReport); err != nil {
+		return
 	}
 
-	return mergeLocation, nil
+	return result, nil
 }
 
 func mergeDocuments(ctx context.Context, inSchemas []string, outFile, defaultRuleset, workingDir string, skipGenerateLintReport bool) error {

internal/run/overlay.go

@@ -21,36 +21,39 @@ type Overlay struct {
 	source     workflow.Source
 }
 
-var _ SourceStep = Overlay{}
-
 func NewOverlay(parentStep *workflowTracking.WorkflowStep, source workflow.Source) Overlay {
 	return Overlay{
 		parentStep: parentStep,
 		source:     source,
 	}
 }
 
-func (o Overlay) Do(ctx context.Context, inputPath string) (string, error) {
+type OverlayResult struct {
+	Location            string
+	InputSchemaLocation []string
+}
+
+func (o Overlay) Do(ctx context.Context, inputPath string) (result OverlayResult, err error) {
 	overlayStep := o.parentStep.NewSubstep("Applying Overlays")
 
 	overlayLocation := o.source.GetTempOverlayLocation()
+	result.Location = overlayLocation
 
 	log.From(ctx).Infof("Applying %d overlays into %s...", len(o.source.Overlays), overlayLocation)
 
-	var err error
 	var overlaySchemas []string
 	for _, overlay := range o.source.Overlays {
 		overlayFilePath := ""
 		if overlay.Document != nil {
 			overlayFilePath, err = schemas.ResolveDocument(ctx, *overlay.Document, nil, overlayStep)
 			if err != nil {
-				return "", err
+				return
 			}
 		} else if overlay.FallbackCodeSamples != nil {
 			// Make temp file for the overlay output
 			overlayFilePath = filepath.Join(workflow.GetTempDir(), fmt.Sprintf("fallback_code_samples_overlay_%s.yaml", randStringBytes(10)))
-			if err := os.MkdirAll(filepath.Dir(overlayFilePath), 0o755); err != nil {
-				return "", err
+			if err = os.MkdirAll(filepath.Dir(overlayFilePath), 0o755); err != nil {
+				return
 			}
 
 			err = defaultcodesamples.DefaultCodeSamples(ctx, defaultcodesamples.DefaultCodeSamplesFlags{
@@ -60,21 +63,22 @@ func (o Overlay) Do(ctx context.Context, inputPath string) (string, error) {
 			})
 			if err != nil {
 				log.From(ctx).Errorf("failed to generate default code samples: %s", err.Error())
-				return "", err
+				return
 			}
 		}
 
 		overlaySchemas = append(overlaySchemas, overlayFilePath)
+		result.InputSchemaLocation = append(result.InputSchemaLocation, overlayFilePath)
 	}
 
 	overlayStep.NewSubstep(fmt.Sprintf("Apply %d overlay(s)", len(o.source.Overlays)))
 
-	if err := overlayDocument(ctx, inputPath, overlaySchemas, overlayLocation); err != nil {
-		return "", err
+	if err = overlayDocument(ctx, inputPath, overlaySchemas, overlayLocation); err != nil {
+		return
 	}
 
 	overlayStep.Succeed()
-	return overlayLocation, nil
+	return
 }
 
 func overlayDocument(ctx context.Context, schema string, overlayFiles []string, outFile string) error {

internal/run/run.go

@@ -25,6 +25,10 @@ import (
 	"github.com/speakeasy-api/speakeasy/internal/workflowTracking"
 )
 
+type SourceStep interface {
+	Do(ctx context.Context, inputPath string) (string, error)
+}
+
 const speakeasySelf = "speakeasy-self"
 
 func ParseSourcesAndTargets() ([]string, []string, error) {