From 25ca5e02812b5f788ce0b03166228f0398c2a554 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Fri, 30 May 2025 11:32:03 -0400 Subject: [PATCH 01/16] update generation --- .github/workflows/sdk_generation.yaml | 45 ++++++++++++++++++++++----- .speakeasy/gen.yaml | 7 +++++ .speakeasy/workflow.yaml | 4 +-- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/sdk_generation.yaml b/.github/workflows/sdk_generation.yaml index 65342d7..0850868 100644 --- a/.github/workflows/sdk_generation.yaml +++ b/.github/workflows/sdk_generation.yaml @@ -19,11 +19,40 @@ permissions: - cron: 0 0 * * * jobs: generate: - uses: speakeasy-api/sdk-generation-action/.github/workflows/workflow-executor.yaml@v15 - with: - force: ${{ github.event.inputs.force }} - mode: pr - set_version: ${{ github.event.inputs.set_version }} - secrets: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - speakeasy_api_key: ${{ secrets.SPEAKEASY_API_KEY }} + runs-on: ubuntu-latest + steps: + - name: Clone SDK repo + uses: actions/checkout@v3 + + - name: Set up SSH + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + mkdir -p ~/.ssh + ssh-keyscan github.com >> ~/.ssh/known_hosts + echo "${{ secrets.FH_OPS_SSH_KEY }}" > ~/.ssh/id_ed25519 + chmod 400 ~/.ssh/id_ed25519 + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add ~/.ssh/id_ed25519 + + - name: Clone developers repo + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + git clone git@github.com:firehydrant/developers.git /tmp/dev-repo + cd /tmp/dev-repo + git checkout main + + - name: Copy OpenAPI spec + run: | + mkdir -p .speakeasy + cp /tmp/dev-repo/docs/public/openapi3_doc.json ${GITHUB_WORKSPACE}/openapi.json + + - name: Generate SDK + uses: speakeasy-api/sdk-generation-action@v15 + with: + force: ${{ github.event.inputs.force }} + mode: pr + set_version: ${{ github.event.inputs.set_version }} + github_access_token: ${{ secrets.GITHUB_TOKEN }} + SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }} \ No newline at end of file diff --git a/.speakeasy/gen.yaml b/.speakeasy/gen.yaml index 1d4df21..ab66522 100644 --- a/.speakeasy/gen.yaml +++ b/.speakeasy/gen.yaml @@ -1,5 +1,8 @@ configVersion: 2.0.0 generation: + devContainers: + enabled: true + schemaPath: registry.speakeasyapi.dev/firehydrant/firehydrant/firehydrant-oas:main sdkClassName: firehydrant maintainOpenAPIOrder: true usageSnippets: @@ -7,10 +10,14 @@ generation: useClassNamesForArrayFields: true fixes: nameResolutionDec2023: true + nameResolutionFeb2025: true parameterOrderingFeb2024: true requestResponseComponentNamesFeb2024: true + securityFeb2025: false + sharedErrorComponentsApr2025: false auth: oAuth2ClientCredentialsEnabled: true + oAuth2PasswordEnabled: true terraform: version: 0.1.5 additionalDataSources: [] diff --git a/.speakeasy/workflow.yaml b/.speakeasy/workflow.yaml index 82140d3..3626cdf 100644 --- a/.speakeasy/workflow.yaml +++ b/.speakeasy/workflow.yaml @@ -3,9 +3,7 @@ speakeasyVersion: latest sources: firehydrant-oas: inputs: - - location: ./openapi.yaml - overlays: - - location: .speakeasy/speakeasy-suggestions.yaml + - location: ${GITHUB_WORKSPACE}/openapi.json registry: location: registry.speakeasyapi.dev/firehydrant/firehydrant/firehydrant-oas targets: From 80ab267ce42176746dab8b5eaf5b8c5cce73a713 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Wed, 4 Jun 2025 15:58:44 -0400 Subject: [PATCH 02/16] add normalization, overlay generation scripts --- .github/workflows/sdk_generation.yaml | 20 +- .speakeasy/gen.yaml | 13 +- .speakeasy/workflow.yaml | 2 + scripts/normalize/normalize-schema.go | 366 ++++++ scripts/overlay/generate-terraform-overlay.go | 1037 +++++++++++++++++ 5 files changed, 1435 insertions(+), 3 deletions(-) create mode 100644 scripts/normalize/normalize-schema.go create mode 100644 scripts/overlay/generate-terraform-overlay.go diff --git a/.github/workflows/sdk_generation.yaml b/.github/workflows/sdk_generation.yaml index 0850868..8b8500b 100644 --- a/.github/workflows/sdk_generation.yaml +++ b/.github/workflows/sdk_generation.yaml @@ -45,9 +45,25 @@ jobs: - name: Copy OpenAPI spec run: | - mkdir -p .speakeasy - cp /tmp/dev-repo/docs/public/openapi3_doc.json ${GITHUB_WORKSPACE}/openapi.json + cp /tmp/dev-repo/docs/public/openapi3_doc.json ${GITHUB_WORKSPACE}/openapi-raw.json + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + + - name: Process OpenAPI spec + run: | + # Normalize the schema + go run scripts/normalize/normalize-schema.go openapi-raw.json openapi.json + + # Generate Terraform overlay + go run scripts/overlay/generate-terraform-overlay.go openapi.json + # Move overlay to Speakeasy directory + mkdir -p .speakeasy + mv terraform-overlay.yaml .speakeasy/ + - name: Generate SDK uses: speakeasy-api/sdk-generation-action@v15 with: diff --git a/.speakeasy/gen.yaml b/.speakeasy/gen.yaml index ab66522..7fc59cc 100644 --- a/.speakeasy/gen.yaml +++ b/.speakeasy/gen.yaml @@ -7,6 +7,7 @@ generation: maintainOpenAPIOrder: true usageSnippets: optionalPropertyRendering: withExample + sdkInitStyle: constructor useClassNamesForArrayFields: true fixes: nameResolutionDec2023: true @@ -19,13 +20,23 @@ generation: oAuth2ClientCredentialsEnabled: true oAuth2PasswordEnabled: true terraform: - version: 0.1.5 + version: 0.2.6 additionalDataSources: [] additionalDependencies: {} + additionalEphemeralResources: [] + additionalProviderAttributes: + httpHeaders: "" + tlsSkipVerify: "" additionalResources: [] allowUnknownFieldsInWeakUnions: false author: firehydrant defaultErrorName: APIError enableTypeDeduplication: false environmentVariables: [] + operationIdNaming: preserve packageName: firehydrant + providerGeneration: + enabled: true + resourceDetection: + strategy: operationId + tagStrategy: ignore diff --git a/.speakeasy/workflow.yaml b/.speakeasy/workflow.yaml index 3626cdf..c2644ab 100644 --- a/.speakeasy/workflow.yaml +++ b/.speakeasy/workflow.yaml @@ -4,6 +4,8 @@ sources: firehydrant-oas: inputs: - location: ${GITHUB_WORKSPACE}/openapi.json + overlays: + - location: ./.speakeasy/terraform-overlay.yaml registry: location: registry.speakeasyapi.dev/firehydrant/firehydrant/firehydrant-oas targets: diff --git a/scripts/normalize/normalize-schema.go b/scripts/normalize/normalize-schema.go new file mode 100644 index 0000000..55a352d --- /dev/null +++ b/scripts/normalize/normalize-schema.go @@ -0,0 +1,366 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" +) + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + inputPath := os.Args[1] + outputPath := inputPath + if len(os.Args) > 2 { + outputPath = os.Args[2] + } + + fmt.Printf("=== OpenAPI Schema Normalizer ===\n") + fmt.Printf("Input: %s\n", inputPath) + fmt.Printf("Output: %s\n\n", outputPath) + + // Read the spec + specData, err := ioutil.ReadFile(inputPath) + if err != nil { + fmt.Printf("Error reading spec: %v\n", err) + os.Exit(1) + } + + var spec map[string]interface{} + if err := json.Unmarshal(specData, &spec); err != nil { + fmt.Printf("Error parsing JSON: %v\n", err) + os.Exit(1) + } + + // Normalize the spec + report := normalizeSpec(spec) + + // Print report + printNormalizationReport(report) + + // Write normalized spec + normalizedData, err := json.MarshalIndent(spec, "", " ") + if err != nil { + fmt.Printf("Error marshaling normalized spec: %v\n", err) + os.Exit(1) + } + + if err := ioutil.WriteFile(outputPath, normalizedData, 0644); err != nil { + fmt.Printf("Error writing normalized spec: %v\n", err) + os.Exit(1) + } + + fmt.Printf("\n✅ Successfully normalized OpenAPI spec\n") + fmt.Printf(" Total fixes applied: %d\n", report.TotalFixes) +} + +func printUsage() { + fmt.Println("OpenAPI Schema Normalizer") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" openapi-normalize [output.json]") +} + +// ============================================================================ +// NORMALIZATION LOGIC +// ============================================================================ + +type NormalizationReport struct { + TotalFixes int + MapClassFixes int + PropertyFixes int + ConflictDetails []ConflictDetail +} + +type ConflictDetail struct { + Schema string + Property string + ConflictType string + Resolution string +} + +func normalizeSpec(spec map[string]interface{}) NormalizationReport { + report := NormalizationReport{ + ConflictDetails: make([]ConflictDetail, 0), + } + + components, ok := spec["components"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No components found in spec") + return report + } + + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No schemas found in components") + return report + } + + // Build entity relationships + entityMap := buildEntityRelationships(schemas) + + // Normalize each entity and its related schemas + for entityName, related := range entityMap { + fmt.Printf("Analyzing entity: %s\n", entityName) + + entitySchema, ok := schemas[entityName].(map[string]interface{}) + if !ok { + continue + } + + // Check against create schema + if related.CreateSchema != "" { + if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + + // Check against update schema + if related.UpdateSchema != "" { + if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + } + + // Apply global normalizations + globalFixes := applyGlobalNormalizations(schemas) + report.ConflictDetails = append(report.ConflictDetails, globalFixes...) + + // Calculate totals + report.TotalFixes = len(report.ConflictDetails) + for _, detail := range report.ConflictDetails { + if detail.ConflictType == "map-class" { + report.MapClassFixes++ + } else { + report.PropertyFixes++ + } + } + + return report +} + +type EntityRelationship struct { + EntityName string + CreateSchema string + UpdateSchema string +} + +func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { + relationships := make(map[string]EntityRelationship) + + for schemaName := range schemas { + if strings.HasSuffix(schemaName, "Entity") && !strings.Contains(schemaName, "Nullable") && !strings.Contains(schemaName, "Paginated") { + baseName := strings.ToLower(strings.TrimSuffix(schemaName, "Entity")) + + rel := EntityRelationship{ + EntityName: schemaName, + } + + // Look for create schema + createName := "create_" + baseName + if _, exists := schemas[createName]; exists { + rel.CreateSchema = createName + } + + // Look for update schema + updateName := "update_" + baseName + if _, exists := schemas[updateName]; exists { + rel.UpdateSchema = updateName + } + + relationships[schemaName] = rel + } + } + + return relationships +} + +func normalizeSchemas(entityName string, entitySchema map[string]interface{}, + requestName string, requestSchema map[string]interface{}) []ConflictDetail { + + conflicts := make([]ConflictDetail, 0) + + entityProps, _ := entitySchema["properties"].(map[string]interface{}) + requestProps, _ := requestSchema["properties"].(map[string]interface{}) + + if entityProps == nil || requestProps == nil { + return conflicts + } + + fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) + fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) + + // Check each property that exists in both schemas + for propName, requestProp := range requestProps { + if entityProp, exists := entityProps[propName]; exists { + fmt.Printf(" Checking property: %s\n", propName) + conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) + if conflict != nil { + fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) + conflicts = append(conflicts, *conflict) + } + } + } + + return conflicts +} + +func checkAndFixProperty(entityName, propName string, entityProp, requestProp interface{}, + entityProps, requestProps map[string]interface{}) *ConflictDetail { + + entityPropMap, _ := entityProp.(map[string]interface{}) + requestPropMap, _ := requestProp.(map[string]interface{}) + + if entityPropMap == nil || requestPropMap == nil { + return nil + } + + // Check for map vs class conflict + entityType, _ := entityPropMap["type"].(string) + requestType, _ := requestPropMap["type"].(string) + + fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) + + if entityType == "object" && requestType == "object" { + _, entityHasProps := entityPropMap["properties"] + _, entityHasAdditional := entityPropMap["additionalProperties"] + _, requestHasProps := requestPropMap["properties"] + _, requestHasAdditional := requestPropMap["additionalProperties"] + + fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) + fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) + + // Normalize to consistent structure without adding additionalProperties + // Option 1: Both have empty properties - make them consistent + if entityHasProps && requestHasProps { + entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) + requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) + + if len(entityPropsObj) == 0 && len(requestPropsObj) == 0 { + // Both have empty properties - this is already consistent + fmt.Printf(" Both have empty properties - already consistent\n") + return nil + } + } + + // Option 2: One has properties, one has additionalProperties - make them both use properties + if entityHasAdditional && !requestHasAdditional && requestHasProps { + // Entity uses additionalProperties, request uses properties + // Convert entity to use empty properties like request + delete(entityPropMap, "additionalProperties") + entityPropMap["properties"] = map[string]interface{}{} + entityProps[propName] = entityPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted entity from additionalProperties to empty properties", + } + } + + if requestHasAdditional && !entityHasAdditional && entityHasProps { + // Request uses additionalProperties, entity uses properties + // Convert request to use empty properties like entity + delete(requestPropMap, "additionalProperties") + requestPropMap["properties"] = map[string]interface{}{} + requestProps[propName] = requestPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted request from additionalProperties to empty properties", + } + } + } + + return nil +} + +func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) + + // Fix common patterns across all schemas + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + // Apply global normalizations to ALL schemas (don't skip request schemas) + props, ok := schemaMap["properties"].(map[string]interface{}) + if !ok { + continue + } + + fmt.Printf(" Checking schema: %s\n", schemaName) + + // Check for common problematic properties + for propName, prop := range props { + propMap, ok := prop.(map[string]interface{}) + if !ok { + continue + } + + // Fix empty properties objects - but don't add additionalProperties + if propType, _ := propMap["type"].(string); propType == "object" { + if propsObj, hasProps := propMap["properties"].(map[string]interface{}); hasProps && len(propsObj) == 0 { + _, hasAdditional := propMap["additionalProperties"] + + // Debug output for labels property specifically + if propName == "labels" { + fmt.Printf(" Found labels property in %s:\n", schemaName) + fmt.Printf(" Has empty properties: %v\n", hasProps) + fmt.Printf(" Has additionalProperties: %v\n", hasAdditional) + } + + // Only normalize if it has additionalProperties - convert to consistent empty properties + if hasAdditional { + delete(propMap, "additionalProperties") + // Keep the empty properties object for consistency + props[propName] = propMap + + conflicts = append(conflicts, ConflictDetail{ + Schema: schemaName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted additionalProperties to empty properties for consistency", + }) + + if propName == "labels" { + fmt.Printf(" ✅ Converted labels from additionalProperties to empty properties in %s\n", schemaName) + } + } + } + } + } + } + + return conflicts +} + +func printNormalizationReport(report NormalizationReport) { + fmt.Println("\n=== Normalization Report ===") + fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) + fmt.Printf("Map/Class fixes: %d\n", report.MapClassFixes) + fmt.Printf("Other property fixes: %d\n", report.PropertyFixes) + + if len(report.ConflictDetails) > 0 { + fmt.Println("\nDetailed fixes:") + for _, detail := range report.ConflictDetails { + fmt.Printf(" - %s.%s [%s]: %s\n", + detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) + } + } +} diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go new file mode 100644 index 0000000..6fdc22d --- /dev/null +++ b/scripts/overlay/generate-terraform-overlay.go @@ -0,0 +1,1037 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + specPath := os.Args[1] + + fmt.Printf("=== Terraform Overlay Generator ===\n") + fmt.Printf("Input: %s\n", specPath) + + specData, err := ioutil.ReadFile(specPath) + if err != nil { + fmt.Printf("Error reading spec file: %v\n", err) + os.Exit(1) + } + + var spec OpenAPISpec + if err := json.Unmarshal(specData, &spec); err != nil { + fmt.Printf("Error parsing JSON: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) + + // Analyze the spec to find resources + resources := analyzeSpec(spec) + + // Generate overlay + overlay := generateOverlay(resources, spec) + + // Write overlay file + if err := writeOverlay(overlay); err != nil { + fmt.Printf("Error writing overlay: %v\n", err) + os.Exit(1) + } + + // Print summary + printOverlaySummary(resources, overlay) +} + +func printUsage() { + fmt.Println("OpenAPI Terraform Overlay Generator") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" openapi-overlay ") +} + +// ============================================================================ +// OVERLAY GENERATION LOGIC +// ============================================================================ + +type OpenAPISpec struct { + OpenAPI string `json:"openapi"` + Info map[string]interface{} `json:"info"` + Paths map[string]PathItem `json:"paths"` + Components Components `json:"components"` +} + +type Components struct { + Schemas map[string]Schema `json:"schemas"` + SecuritySchemes map[string]interface{} `json:"securitySchemes,omitempty"` +} + +type Schema struct { + Type string `json:"type,omitempty"` + Properties map[string]interface{} `json:"properties,omitempty"` + Required []string `json:"required,omitempty"` + AllOf []interface{} `json:"allOf,omitempty"` + Nullable bool `json:"nullable,omitempty"` + Items interface{} `json:"items,omitempty"` + Raw map[string]interface{} `json:"-"` +} + +type PathItem struct { + Get *Operation `json:"get,omitempty"` + Post *Operation `json:"post,omitempty"` + Put *Operation `json:"put,omitempty"` + Patch *Operation `json:"patch,omitempty"` + Delete *Operation `json:"delete,omitempty"` +} + +type Operation struct { + OperationID string `json:"operationId,omitempty"` + Tags []string `json:"tags,omitempty"` + Parameters []Parameter `json:"parameters,omitempty"` + RequestBody map[string]interface{} `json:"requestBody,omitempty"` + Responses map[string]interface{} `json:"responses,omitempty"` +} + +type Parameter struct { + Name string `json:"name"` + In string `json:"in"` + Required bool `json:"required,omitempty"` + Schema Schema `json:"schema,omitempty"` +} + +type ResourceInfo struct { + EntityName string + SchemaName string + ResourceName string + Operations map[string]OperationInfo + CreateSchema string // Track the create request schema + UpdateSchema string // Track the update request schema +} + +type OperationInfo struct { + OperationID string + Path string + Method string + RequestSchema string // Track request schema separately +} + +type PropertyMismatch struct { + PropertyName string + MismatchType string + Description string +} + +type OverlayAction struct { + Target string `yaml:"target"` + Update map[string]interface{} `yaml:"update,omitempty"` +} + +type Overlay struct { + Overlay string `yaml:"overlay"` + Info struct { + Title string `yaml:"title"` + Version string `yaml:"version"` + Description string `yaml:"description"` + } `yaml:"info"` + Actions []OverlayAction `yaml:"actions"` +} + +func analyzeSpec(spec OpenAPISpec) map[string]*ResourceInfo { + resources := make(map[string]*ResourceInfo) + + // First pass: identify all entity schemas + entitySchemas := identifyEntitySchemas(spec.Components.Schemas) + fmt.Printf("Identified %d entity schemas\n", len(entitySchemas)) + + // Second pass: match operations to entities + for path, pathItem := range spec.Paths { + analyzePathOperations(path, pathItem, entitySchemas, resources, spec) + } + + // Third pass: validate and filter resources + validResources := make(map[string]*ResourceInfo) + for name, resource := range resources { + if isValidTerraformResource(resource) { + validResources[name] = resource + opTypes := make([]string, 0) + for crudType := range resource.Operations { + entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) + opTypes = append(opTypes, fmt.Sprintf("%s->%s", crudType, entityOp)) + } + fmt.Printf("Valid Terraform resource: %s with operations: %v\n", + name, opTypes) + } + } + + return validResources +} + +func identifyEntitySchemas(schemas map[string]Schema) map[string]bool { + entities := make(map[string]bool) + + for name, schema := range schemas { + if isEntitySchema(name, schema) { + entities[name] = true + } + } + + return entities +} + +func isEntitySchema(name string, schema Schema) bool { + // Skip request/response wrappers + lowerName := strings.ToLower(name) + if strings.HasPrefix(lowerName, "create_") || + strings.HasPrefix(lowerName, "update_") || + strings.HasPrefix(lowerName, "delete_") || + strings.Contains(lowerName, "request") || + strings.Contains(lowerName, "response") || + strings.HasSuffix(name, "Paginated") { + return false + } + + // Skip nullable wrapper schemas + if strings.HasPrefix(name, "Nullable") { + return false + } + + // Must be an object with properties + if schema.Type != "object" || len(schema.Properties) == 0 { + return false + } + + // Entities should have an id property and end with "Entity" + _, hasID := schema.Properties["id"] + hasSuffix := strings.HasSuffix(name, "Entity") + + // Be strict: require both conditions + return hasID && hasSuffix +} + +func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[string]bool, + resources map[string]*ResourceInfo, spec OpenAPISpec) { + + operations := []struct { + method string + op *Operation + }{ + {"get", pathItem.Get}, + {"post", pathItem.Post}, + {"put", pathItem.Put}, + {"patch", pathItem.Patch}, + {"delete", pathItem.Delete}, + } + + for _, item := range operations { + if item.op == nil { + continue + } + + resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec) + if resourceInfo != nil { + if existing, exists := resources[resourceInfo.ResourceName]; exists { + fmt.Printf(" Merging operations for %s\n", resourceInfo.ResourceName) + fmt.Printf(" Existing: CreateSchema='%s', UpdateSchema='%s'\n", existing.CreateSchema, existing.UpdateSchema) + fmt.Printf(" New: CreateSchema='%s', UpdateSchema='%s'\n", resourceInfo.CreateSchema, resourceInfo.UpdateSchema) + + // Merge operations + for opType, opInfo := range resourceInfo.Operations { + existing.Operations[opType] = opInfo + } + + // Preserve create/update schema info - don't overwrite with empty values + if resourceInfo.CreateSchema != "" { + existing.CreateSchema = resourceInfo.CreateSchema + } + if resourceInfo.UpdateSchema != "" { + existing.UpdateSchema = resourceInfo.UpdateSchema + } + + fmt.Printf(" After merge: CreateSchema='%s', UpdateSchema='%s'\n", existing.CreateSchema, existing.UpdateSchema) + } else { + resources[resourceInfo.ResourceName] = resourceInfo + fmt.Printf(" New resource: %s with CreateSchema='%s', UpdateSchema='%s'\n", + resourceInfo.ResourceName, resourceInfo.CreateSchema, resourceInfo.UpdateSchema) + } + } + } +} + +func extractResourceInfo(path, method string, op *Operation, + entitySchemas map[string]bool, spec OpenAPISpec) *ResourceInfo { + + // Determine CRUD type + crudType := determineCrudType(path, method, op.OperationID) + if crudType == "" { + return nil + } + + // Find associated entity schema + entityName := findEntityFromOperation(op, entitySchemas, spec) + if entityName == "" { + return nil + } + + resourceName := deriveResourceName(entityName, op.OperationID, path) + + info := &ResourceInfo{ + EntityName: entityName, + SchemaName: entityName, + ResourceName: resourceName, + Operations: make(map[string]OperationInfo), + } + + opInfo := OperationInfo{ + OperationID: op.OperationID, + Path: path, + Method: method, + } + + // Extract request schema for create/update operations + if crudType == "create" || crudType == "update" { + fmt.Printf(" Extracting request schema for %s %s operation\n", entityName, crudType) + fmt.Printf(" Operation: %s %s %s\n", method, path, op.OperationID) + + if op.RequestBody != nil { + fmt.Printf(" RequestBody exists\n") + if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { + fmt.Printf(" Content exists\n") + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + fmt.Printf(" JSON content exists\n") + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + fmt.Printf(" Schema exists: %+v\n", schema) + if ref, ok := schema["$ref"].(string); ok { + requestSchemaName := extractSchemaName(ref) + opInfo.RequestSchema = requestSchemaName + fmt.Printf(" ✅ Found request schema: %s\n", requestSchemaName) + + if crudType == "create" { + info.CreateSchema = requestSchemaName + } else if crudType == "update" { + info.UpdateSchema = requestSchemaName + } + } else { + fmt.Printf(" No $ref found in schema\n") + } + } else { + fmt.Printf(" No schema found in JSON content\n") + } + } else { + fmt.Printf(" No application/json content found\n") + } + } else { + fmt.Printf(" No content found in RequestBody\n") + } + } else { + fmt.Printf(" No RequestBody found\n") + } + } + + info.Operations[crudType] = opInfo + + return info +} + +func determineCrudType(path, method, operationID string) string { + lowerOp := strings.ToLower(operationID) + + // Check operation ID first + if strings.Contains(lowerOp, "create") { + return "create" + } + if strings.Contains(lowerOp, "update") || strings.Contains(lowerOp, "patch") { + return "update" + } + if strings.Contains(lowerOp, "delete") { + return "delete" + } + if strings.Contains(lowerOp, "list") { + return "list" + } + if strings.Contains(lowerOp, "get") && strings.Contains(path, "{") { + return "read" + } + + // Fallback to method-based detection + switch method { + case "post": + if !strings.Contains(path, "{") { + return "create" + } + case "get": + if strings.Contains(path, "{") { + return "read" + } else { + return "list" + } + case "patch", "put": + return "update" + case "delete": + return "delete" + } + + return "" +} + +func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec OpenAPISpec) string { + // Check response schemas first + if op.Responses != nil { + for _, response := range op.Responses { + if respMap, ok := response.(map[string]interface{}); ok { + if content, ok := respMap["content"].(map[string]interface{}); ok { + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + entityName := findEntityInSchema(schema, entitySchemas) + if entityName != "" { + return entityName + } + } + } + } + } + } + } + + // Check tags + if len(op.Tags) > 0 { + for _, tag := range op.Tags { + possibleEntity := tag + "Entity" + if entitySchemas[possibleEntity] { + return possibleEntity + } + } + } + + return "" +} + +func findEntityInSchema(schema map[string]interface{}, entitySchemas map[string]bool) string { + // Direct reference + if ref, ok := schema["$ref"].(string); ok { + schemaName := extractSchemaName(ref) + if entitySchemas[schemaName] { + return schemaName + } + } + + // Check in data array for paginated responses + if props, ok := schema["properties"].(map[string]interface{}); ok { + if data, ok := props["data"].(map[string]interface{}); ok { + if dataType, ok := data["type"].(string); ok && dataType == "array" { + if items, ok := data["items"].(map[string]interface{}); ok { + if ref, ok := items["$ref"].(string); ok { + schemaName := extractSchemaName(ref) + if entitySchemas[schemaName] { + return schemaName + } + } + } + } + } + } + + return "" +} + +func extractSchemaName(ref string) string { + parts := strings.Split(ref, "/") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func deriveResourceName(entityName, operationID, path string) string { + // Remove Entity suffix + resource := strings.TrimSuffix(entityName, "Entity") + + // Convert to snake_case + resource = toSnakeCase(resource) + + // Handle special cases + if strings.Contains(resource, "_") { + parts := strings.Split(resource, "_") + if len(parts) > 1 && parts[0] == parts[1] { + // Remove duplicate prefix (e.g., incidents_incident -> incident) + resource = parts[1] + } + } + + return resource +} + +func toSnakeCase(s string) string { + var result []rune + for i, r := range s { + if i > 0 && isUpper(r) { + if i == len(s)-1 || !isUpper(rune(s[i+1])) { + result = append(result, '_') + } + } + result = append(result, toLower(r)) + } + return string(result) +} + +func isUpper(r rune) bool { + return r >= 'A' && r <= 'Z' +} + +func toLower(r rune) rune { + if r >= 'A' && r <= 'Z' { + return r + 32 + } + return r +} + +func isValidTerraformResource(resource *ResourceInfo) bool { + // Must have at least create and read operations for a full Terraform resource + _, hasCreate := resource.Operations["create"] + _, hasRead := resource.Operations["read"] + + // Some resources might be read-only data sources + if hasRead && !hasCreate { + fmt.Printf(" Note: %s appears to be a read-only data source\n", resource.EntityName) + } + + return hasCreate && hasRead +} + +func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Overlay { + overlay := &Overlay{ + Overlay: "1.0.0", + } + + overlay.Info.Title = "Terraform Provider Overlay" + overlay.Info.Version = "1.0.0" + overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d Terraform resources", len(resources)) + + // Detect property mismatches between request and response schemas + resourceMismatches := detectPropertyMismatches(resources, spec) + + // Report on request/response patterns + fmt.Println("\n=== Request/Response Schema Analysis ===") + for _, resource := range resources { + if resource.CreateSchema != "" || resource.UpdateSchema != "" { + fmt.Printf("%s:\n", resource.EntityName) + if resource.CreateSchema != "" { + fmt.Printf(" Create: %s (request) -> %s (response)\n", resource.CreateSchema, resource.EntityName) + } + if resource.UpdateSchema != "" { + fmt.Printf(" Update: %s (request) -> %s (response)\n", resource.UpdateSchema, resource.EntityName) + } + if mismatches, hasMismatch := resourceMismatches[resource.EntityName]; hasMismatch { + fmt.Printf(" ⚠️ WARNING: Has property mismatches:\n") + for _, mismatch := range mismatches { + fmt.Printf(" - %s: %s\n", mismatch.PropertyName, mismatch.Description) + } + } else { + fmt.Printf(" ✅ No property mismatches detected\n") + } + } + } + + // Generate actions + for _, resource := range resources { + // Mark the response entity schema + entityUpdate := map[string]interface{}{ + "x-speakeasy-entity": resource.EntityName, + } + + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s", resource.SchemaName), + Update: entityUpdate, + }) + + // Add terraform ignore for mismatched properties in request AND response schemas + if mismatches, exists := resourceMismatches[resource.EntityName]; exists { + // Add terraform ignore for create schema properties + if resource.CreateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.CreateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + + // Also ignore in response entity schema + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + + fmt.Printf(" ✅ Added terraform ignore for %s.%s in both request (%s) and response (%s) schemas\n", + resource.EntityName, mismatch.PropertyName, resource.CreateSchema, resource.EntityName) + } + } + + // Add terraform ignore for update schema properties + if resource.UpdateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.UpdateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + + // Also ignore in response entity schema (avoid duplicates) + // Only add if we didn't already add it for create schema + if resource.CreateSchema == "" { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + } + + fmt.Printf(" ✅ Added terraform ignore for %s.%s in update schema\n", resource.EntityName, mismatch.PropertyName) + } + } + } + + // Add entity operations + for crudType, opInfo := range resource.Operations { + entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) + + if crudType == "list" { + fmt.Printf(" List operation: %s -> %s\n", resource.EntityName, entityOp) + } + + operationUpdate := map[string]interface{}{ + "x-speakeasy-entity-operation": entityOp, + } + + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s", opInfo.Path, opInfo.Method), + Update: operationUpdate, + }) + + // Add parameter matches for operations with path parameters + if crudType == "read" || crudType == "update" || crudType == "delete" { + addParameterMatches(overlay, opInfo.Path, opInfo.Method, resource.ResourceName) + } + } + } + + fmt.Println("\n=== Overlay Generation Complete ===") + fmt.Printf("Generated %d actions for %d resources\n", len(overlay.Actions), len(resources)) + + // Count terraform ignore actions + ignoreCount := 0 + for _, action := range overlay.Actions { + if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { + ignoreCount++ + } + } + + if len(resourceMismatches) > 0 { + fmt.Printf("✅ %d resources had property mismatches\n", len(resourceMismatches)) + fmt.Printf("✅ %d terraform ignore actions added\n", ignoreCount) + } + + return overlay +} + +// Enhanced mismatch detection that covers all types of impedance mismatches +func detectPropertyMismatches(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]PropertyMismatch { + mismatches := make(map[string][]PropertyMismatch) + + // We need to work with the raw schemas as map[string]interface{} for proper detection + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + fmt.Printf("Error marshaling spec for mismatch detection: %v\n", err) + return mismatches + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + fmt.Printf("Error unmarshaling spec for mismatch detection: %v\n", err) + return mismatches + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + fmt.Printf("\n=== Mismatch Detection Debug ===\n") + fmt.Printf("Total resources to check: %d\n", len(resources)) + + for _, resource := range resources { + var resourceMismatches []PropertyMismatch + + fmt.Printf("Checking resource: %s\n", resource.EntityName) + fmt.Printf(" CreateSchema: %s\n", resource.CreateSchema) + fmt.Printf(" UpdateSchema: %s\n", resource.UpdateSchema) + + // Check create operation mismatches + if resource.CreateSchema != "" { + if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { + if requestSchema, exists := schemas[resource.CreateSchema].(map[string]interface{}); exists { + fmt.Printf(" Found both entity and create schemas - checking for mismatches\n") + createMismatches := findPropertyMismatches(entitySchema, requestSchema, "create") + resourceMismatches = append(resourceMismatches, createMismatches...) + fmt.Printf(" Create mismatches found: %d\n", len(createMismatches)) + } else { + fmt.Printf(" Warning: Create schema %s not found\n", resource.CreateSchema) + } + } else { + fmt.Printf(" Warning: Entity schema %s not found\n", resource.EntityName) + } + } + + // Check update operation mismatches + if resource.UpdateSchema != "" { + if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { + if requestSchema, exists := schemas[resource.UpdateSchema].(map[string]interface{}); exists { + fmt.Printf(" Found both entity and update schemas - checking for mismatches\n") + updateMismatches := findPropertyMismatches(entitySchema, requestSchema, "update") + resourceMismatches = append(resourceMismatches, updateMismatches...) + fmt.Printf(" Update mismatches found: %d\n", len(updateMismatches)) + } else { + fmt.Printf(" Warning: Update schema %s not found\n", resource.UpdateSchema) + } + } else { + fmt.Printf(" Warning: Entity schema %s not found\n", resource.EntityName) + } + } + + if len(resourceMismatches) > 0 { + mismatches[resource.EntityName] = resourceMismatches + fmt.Printf(" Total mismatches for %s: %d\n", resource.EntityName, len(resourceMismatches)) + for _, mismatch := range resourceMismatches { + fmt.Printf(" - %s: %s\n", mismatch.PropertyName, mismatch.Description) + } + } else { + fmt.Printf(" No mismatches found for %s\n", resource.EntityName) + } + } + + fmt.Printf("=== End Mismatch Detection Debug ===\n\n") + return mismatches +} + +func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, operation string) []PropertyMismatch { + var mismatches []PropertyMismatch + + entityProps, _ := entitySchema["properties"].(map[string]interface{}) + requestProps, _ := requestSchema["properties"].(map[string]interface{}) + + if entityProps == nil || requestProps == nil { + fmt.Printf(" Warning: Could not access properties for %s operation\n", operation) + return mismatches + } + + fmt.Printf(" Checking %d entity properties vs %d request properties for %s\n", + len(entityProps), len(requestProps), operation) + + for propName, entityProp := range entityProps { + if requestProp, exists := requestProps[propName]; exists { + if mismatch := detectPropertyMismatch(propName, entityProp, requestProp, operation); mismatch != nil { + fmt.Printf(" ✅ Found mismatch: %s - %s\n", propName, mismatch.Description) + mismatches = append(mismatches, *mismatch) + } + } + } + + return mismatches +} + +func detectPropertyMismatch(propName string, entityProp, requestProp interface{}, operation string) *PropertyMismatch { + entityMap, _ := entityProp.(map[string]interface{}) + requestMap, _ := requestProp.(map[string]interface{}) + + if entityMap == nil || requestMap == nil { + return nil + } + + entityType, _ := entityMap["type"].(string) + requestType, _ := requestMap["type"].(string) + + fmt.Printf(" Checking property '%s': request=%s, entity=%s\n", propName, requestType, entityType) + + // Type mismatch detection + if entityType != requestType && entityType != "" && requestType != "" { + fmt.Printf(" ✅ Found basic type mismatch: %s != %s\n", requestType, entityType) + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "type-mismatch", + Description: fmt.Sprintf("request type '%s' != response type '%s'", requestType, entityType), + } + } + + // Array item type mismatch detection - this is the key one for string != class + if entityType == "array" && requestType == "array" { + entityItems, _ := entityMap["items"].(map[string]interface{}) + requestItems, _ := requestMap["items"].(map[string]interface{}) + + fmt.Printf(" Both are arrays - checking item types\n") + + // Special debug for environments property + if propName == "environments" { + fmt.Printf(" 🔍 ENVIRONMENTS DEBUG:\n") + fmt.Printf(" Request items: %+v\n", requestItems) + fmt.Printf(" Entity items: %+v\n", entityItems) + } + + if entityItems != nil && requestItems != nil { + requestItemType, _ := requestItems["type"].(string) + entityRef, entityHasRef := entityItems["$ref"].(string) + requestRef, requestHasRef := requestItems["$ref"].(string) + + fmt.Printf(" Request: itemType='%s', hasRef=%v\n", requestItemType, requestHasRef) + fmt.Printf(" Entity: ref='%s', hasRef=%v\n", entityRef, entityHasRef) + + // Case 1: Request uses string[], Entity uses object[] (refs) - THE KEY CASE + if requestItemType == "string" && entityHasRef { + fmt.Printf(" ✅ Found string[] vs object[] mismatch (Case 1)\n") + fmt.Printf(" Request: string[], Entity: %s\n", entityRef) + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "array-item-mismatch", + Description: fmt.Sprintf("request uses string[], response uses object[] (%s)", entityRef), + } + } + + // Case 2: Different object references in arrays + if entityHasRef && requestHasRef && entityRef != requestRef { + fmt.Printf(" ✅ Found different object ref mismatch (Case 2)\n") + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "array-ref-mismatch", + Description: fmt.Sprintf("request references %s, response references %s", requestRef, entityRef), + } + } + + // Case 3: Request uses inline objects, Entity uses refs + requestItemType2, _ := requestItems["type"].(string) + if !requestHasRef && requestItemType2 == "object" && entityHasRef { + fmt.Printf(" ✅ Found inline object vs ref mismatch (Case 3)\n") + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "array-item-mismatch", + Description: fmt.Sprintf("request uses inline objects, response uses object references (%s)", entityRef), + } + } + + // Case 4: Request uses refs, Entity uses string[] (reverse of Case 1) + entityItemType, _ := entityItems["type"].(string) + if requestHasRef && entityItemType == "string" { + fmt.Printf(" ✅ Found object[] vs string[] mismatch (Case 4)\n") + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "array-item-mismatch", + Description: fmt.Sprintf("request uses object[] (%s), response uses string[]", requestRef), + } + } + } else { + fmt.Printf(" Warning: Could not access array items\n") + if entityItems == nil { + fmt.Printf(" Entity items is nil\n") + } + if requestItems == nil { + fmt.Printf(" Request items is nil\n") + } + } + } + + // Object structure mismatch detection (class vs map) + if entityType == "object" && requestType == "object" { + _, entityHasProps := entityMap["properties"] + _, entityHasAdditional := entityMap["additionalProperties"] + _, requestHasProps := requestMap["properties"] + _, requestHasAdditional := requestMap["additionalProperties"] + + fmt.Printf(" Both are objects - checking structure\n") + fmt.Printf(" Entity: hasProps=%v, hasAdditional=%v\n", entityHasProps, entityHasAdditional) + fmt.Printf(" Request: hasProps=%v, hasAdditional=%v\n", requestHasProps, requestHasAdditional) + + // Map vs Class mismatch + if entityHasProps && !entityHasAdditional && requestHasAdditional && !requestHasProps { + fmt.Printf(" ✅ Found map vs class mismatch (entity=class, request=map)\n") + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "object-structure-mismatch", + Description: "request uses map structure, response uses class structure", + } + } + + if requestHasProps && !requestHasAdditional && entityHasAdditional && !entityHasProps { + fmt.Printf(" ✅ Found class vs map mismatch (request=class, entity=map)\n") + return &PropertyMismatch{ + PropertyName: propName, + MismatchType: "object-structure-mismatch", + Description: "request uses class structure, response uses map structure", + } + } + } + + fmt.Printf(" No mismatch detected for %s\n", propName) + return nil +} + +func mapCrudToEntityOperation(crudType, entityName string) string { + switch crudType { + case "create": + return entityName + "#create" + case "read": + return entityName + "#read" + case "update": + return entityName + "#update" + case "delete": + return entityName + "#delete" + case "list": + // For list operations, pluralize the entity name and use #read + pluralEntityName := pluralizeEntityName(entityName) + return pluralEntityName + "#read" + default: + return entityName + "#" + crudType + } +} + +// Simplified pluralization logic - keeping essential rules +func pluralizeEntityName(entityName string) string { + // Remove "Entity" suffix + baseName := strings.TrimSuffix(entityName, "Entity") + + // Essential special cases + specialCases := map[string]string{ + "Person": "People", + "Child": "Children", + "Status": "Statuses", + "Process": "Processes", + "Policy": "Policies", + "Category": "Categories", + "Entry": "Entries", + "Activity": "Activities", + "Property": "Properties", + "Entity": "Entities", + "Query": "Queries", + "Library": "Libraries", + "History": "Histories", + "Summary": "Summaries", + "Country": "Countries", + "City": "Cities", + "Company": "Companies", + } + + if plural, ok := specialCases[baseName]; ok { + return plural + "Entities" + } + + // Simple pluralization rules + if strings.HasSuffix(baseName, "y") && len(baseName) > 1 && !isVowel(baseName[len(baseName)-2]) { + baseName = baseName[:len(baseName)-1] + "ies" + } else if strings.HasSuffix(baseName, "s") || + strings.HasSuffix(baseName, "ss") || + strings.HasSuffix(baseName, "sh") || + strings.HasSuffix(baseName, "ch") || + strings.HasSuffix(baseName, "x") || + strings.HasSuffix(baseName, "z") { + baseName = baseName + "es" + } else { + baseName = baseName + "s" + } + + return baseName + "Entities" +} + +func isVowel(c byte) bool { + return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u' || + c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U' +} + +func addParameterMatches(overlay *Overlay, path, method, resourceName string) { + // Find all path parameters + re := regexp.MustCompile(`\{([^}]+)\}`) + matches := re.FindAllStringSubmatch(path, -1) + + for _, match := range matches { + paramName := match[1] + + // Check if this looks like an ID parameter + if strings.Contains(paramName, "id") || paramName == resourceName { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + path, method, paramName), + Update: map[string]interface{}{ + "x-speakeasy-match": "id", + }, + }) + } + } +} + +func writeOverlay(overlay *Overlay) error { + // Marshal to YAML + data, err := yaml.Marshal(overlay) + if err != nil { + return fmt.Errorf("marshaling overlay: %w", err) + } + + // Write file to current directory + overlayPath := "terraform-overlay.yaml" + if err := ioutil.WriteFile(overlayPath, data, 0644); err != nil { + return fmt.Errorf("writing overlay file: %w", err) + } + + fmt.Printf("Overlay written to: %s\n", overlayPath) + return nil +} + +func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { + fmt.Println("\n=== Summary ===") + fmt.Printf("✅ Successfully generated overlay with %d actions for %d resources\n", + len(overlay.Actions), len(resources)) + + fmt.Println("\nOverlay approach:") + fmt.Println("1. Mark entity schemas with x-speakeasy-entity") + fmt.Println("2. Tag operations with x-speakeasy-entity-operation") + fmt.Println("3. Mark ID parameters with x-speakeasy-match") + fmt.Println("4. Apply x-speakeasy-ignore: true to mismatched properties") + + fmt.Println("\nResources configured:") + for name, resource := range resources { + ops := make([]string, 0, len(resource.Operations)) + for op := range resource.Operations { + ops = append(ops, op) + } + fmt.Printf(" - %s: [%s]\n", name, strings.Join(ops, ", ")) + } +} + +// UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data +func (s *Schema) UnmarshalJSON(data []byte) error { + type Alias Schema + aux := &struct { + *Alias + }{ + Alias: (*Alias)(s), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Also unmarshal into raw map + if err := json.Unmarshal(data, &s.Raw); err != nil { + return err + } + + return nil +} From 6c7ff336dc5292415523602cb3b9032294574f12 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Wed, 4 Jun 2025 17:02:52 -0400 Subject: [PATCH 03/16] more aggressive normalization --- scripts/normalize/normalize-schema.go | 148 +++++++++++++++++++------- 1 file changed, 110 insertions(+), 38 deletions(-) diff --git a/scripts/normalize/normalize-schema.go b/scripts/normalize/normalize-schema.go index 55a352d..0d42137 100644 --- a/scripts/normalize/normalize-schema.go +++ b/scripts/normalize/normalize-schema.go @@ -291,59 +291,131 @@ func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) - // Fix common patterns across all schemas + // First pass: Find and report all additionalProperties instances + fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") for schemaName, schema := range schemas { schemaMap, ok := schema.(map[string]interface{}) if !ok { continue } - // Apply global normalizations to ALL schemas (don't skip request schemas) - props, ok := schemaMap["properties"].(map[string]interface{}) + additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") + if len(additionalPropsFound) > 0 { + fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) + for _, path := range additionalPropsFound { + fmt.Printf(" - %s\n", path) + } + } + } + + // Second pass: Fix all additionalProperties instances + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) if !ok { continue } - fmt.Printf(" Checking schema: %s\n", schemaName) + fmt.Printf(" Normalizing schema: %s\n", schemaName) + schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") + conflicts = append(conflicts, schemaConflicts...) + } + + return conflicts +} + +// Recursively find all additionalProperties in a schema +func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { + var found []string + + switch v := obj.(type) { + case map[string]interface{}: + // Check if this object has additionalProperties + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + fullPath := schemaName + if path != "" { + fullPath += "." + path + } + found = append(found, fullPath) + } - // Check for common problematic properties - for propName, prop := range props { - propMap, ok := prop.(map[string]interface{}) - if !ok { - continue + // Recursively check all nested objects + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key } + nested := findAllAdditionalProperties(schemaName, value, newPath) + found = append(found, nested...) + } + case []interface{}: + // Check array items + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := findAllAdditionalProperties(schemaName, item, newPath) + found = append(found, nested...) + } + } + + return found +} + +// Recursively normalize all additionalProperties in a schema +func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { + var conflicts []ConflictDetail + + switch v := obj.(type) { + case map[string]interface{}: + // Check if this object has additionalProperties + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + objType, _ := v["type"].(string) + _, hasProperties := v["properties"] + + // Remove additionalProperties if: + // 1. It's explicitly type "object", OR + // 2. It has "properties" (implicit object), OR + // 3. It has additionalProperties but no other structure + if objType == "object" || hasProperties || (!hasProperties && hasAdditional) { + // Remove additionalProperties and ensure empty properties + delete(v, "additionalProperties") + if !hasProperties { + v["properties"] = map[string]interface{}{} + } - // Fix empty properties objects - but don't add additionalProperties - if propType, _ := propMap["type"].(string); propType == "object" { - if propsObj, hasProps := propMap["properties"].(map[string]interface{}); hasProps && len(propsObj) == 0 { - _, hasAdditional := propMap["additionalProperties"] - - // Debug output for labels property specifically - if propName == "labels" { - fmt.Printf(" Found labels property in %s:\n", schemaName) - fmt.Printf(" Has empty properties: %v\n", hasProps) - fmt.Printf(" Has additionalProperties: %v\n", hasAdditional) - } - - // Only normalize if it has additionalProperties - convert to consistent empty properties - if hasAdditional { - delete(propMap, "additionalProperties") - // Keep the empty properties object for consistency - props[propName] = propMap - - conflicts = append(conflicts, ConflictDetail{ - Schema: schemaName, - Property: propName, - ConflictType: "map-class", - Resolution: "Converted additionalProperties to empty properties for consistency", - }) - - if propName == "labels" { - fmt.Printf(" ✅ Converted labels from additionalProperties to empty properties in %s\n", schemaName) - } - } + fullPath := schemaName + if path != "" { + fullPath += "." + path } + + conflicts = append(conflicts, ConflictDetail{ + Schema: schemaName, + Property: path, + ConflictType: "map-class", + Resolution: fmt.Sprintf("Converted additionalProperties to empty properties at %s", fullPath), + }) + + fmt.Printf(" ✅ Converted additionalProperties to empty properties at %s\n", fullPath) + } + } + + // Recursively normalize all nested objects + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key } + nested := normalizeAdditionalProperties(schemaName, value, newPath) + conflicts = append(conflicts, nested...) + } + case []interface{}: + // Normalize array items + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := normalizeAdditionalProperties(schemaName, item, newPath) + conflicts = append(conflicts, nested...) } } From 51d5f20f05bb8963c03b6be3437370a7644fab96 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Wed, 4 Jun 2025 18:10:36 -0400 Subject: [PATCH 04/16] more aggressive normalization, speakeasy ignore all mismatching fields --- scripts/normalize/normalize-schema.go | 88 +++++++- scripts/overlay/generate-terraform-overlay.go | 204 ++++++++---------- 2 files changed, 173 insertions(+), 119 deletions(-) diff --git a/scripts/normalize/normalize-schema.go b/scripts/normalize/normalize-schema.go index 0d42137..e9349e7 100644 --- a/scripts/normalize/normalize-schema.go +++ b/scripts/normalize/normalize-schema.go @@ -101,6 +101,12 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { return report } + // Get paths for parameter normalization + paths, pathsOk := spec["paths"].(map[string]interface{}) + if !pathsOk { + fmt.Println("Warning: No paths found in spec") + } + // Build entity relationships entityMap := buildEntityRelationships(schemas) @@ -130,10 +136,16 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { } } - // Apply global normalizations + // Apply global normalizations to schemas globalFixes := applyGlobalNormalizations(schemas) report.ConflictDetails = append(report.ConflictDetails, globalFixes...) + // Normalize path parameters to match entity IDs + if pathsOk { + parameterFixes := normalizePathParameters(paths, schemas) + report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) + } + // Calculate totals report.TotalFixes = len(report.ConflictDetails) for _, detail := range report.ConflictDetails { @@ -422,6 +434,80 @@ func normalizeAdditionalProperties(schemaName string, obj interface{}, path stri return conflicts } +// Normalize path parameters to match entity ID types +func normalizePathParameters(paths map[string]interface{}, schemas map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("\n=== Normalizing Path Parameters ===\n") + + for pathName, pathItem := range paths { + pathMap, ok := pathItem.(map[string]interface{}) + if !ok { + continue + } + + // Check all HTTP methods in this path + methods := []string{"get", "post", "put", "patch", "delete"} + for _, method := range methods { + if operation, exists := pathMap[method]; exists { + opMap, ok := operation.(map[string]interface{}) + if !ok { + continue + } + + // Check parameters in this operation + if parameters, hasParams := opMap["parameters"]; hasParams { + paramsList, ok := parameters.([]interface{}) + if !ok { + continue + } + + for _, param := range paramsList { + paramMap, ok := param.(map[string]interface{}) + if !ok { + continue + } + + // Check if this is a path parameter with integer type that should be string + paramIn, _ := paramMap["in"].(string) + paramName, _ := paramMap["name"].(string) + + if paramIn == "path" && (strings.Contains(paramName, "id") || strings.HasSuffix(paramName, "_id")) { + schema, hasSchema := paramMap["schema"] + if hasSchema { + schemaMap, ok := schema.(map[string]interface{}) + if ok { + paramType, _ := schemaMap["type"].(string) + paramFormat, _ := schemaMap["format"].(string) + + // Convert integer ID parameters to string + if paramType == "integer" { + fmt.Printf(" Found integer ID parameter: %s %s.%s (type: %s, format: %s)\n", + method, pathName, paramName, paramType, paramFormat) + + schemaMap["type"] = "string" + delete(schemaMap, "format") // Remove int32/int64 format + + conflicts = append(conflicts, ConflictDetail{ + Schema: fmt.Sprintf("path:%s", pathName), + Property: fmt.Sprintf("%s.%s", method, paramName), + ConflictType: "parameter-type", + Resolution: fmt.Sprintf("Converted path parameter %s from integer to string", paramName), + }) + + fmt.Printf(" ✅ Converted %s parameter from integer to string\n", paramName) + } + } + } + } + } + } + } + } + } + + return conflicts +} func printNormalizationReport(report NormalizationReport) { fmt.Println("\n=== Normalization Report ===") fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index 6fdc22d..74e6be4 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -552,9 +552,9 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over Update: entityUpdate, }) - // Add terraform ignore for mismatched properties in request AND response schemas + // Add speakeasy ignore for mismatched properties in request AND response schemas if mismatches, exists := resourceMismatches[resource.EntityName]; exists { - // Add terraform ignore for create schema properties + // Add speakeasy ignore for create schema properties if resource.CreateSchema != "" { for _, mismatch := range mismatches { // Ignore in request schema @@ -573,7 +573,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over }, }) - fmt.Printf(" ✅ Added terraform ignore for %s.%s in both request (%s) and response (%s) schemas\n", + fmt.Printf(" ✅ Added speakeasy ignore for %s.%s in both request (%s) and response (%s) schemas\n", resource.EntityName, mismatch.PropertyName, resource.CreateSchema, resource.EntityName) } } @@ -600,7 +600,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over }) } - fmt.Printf(" ✅ Added terraform ignore for %s.%s in update schema\n", resource.EntityName, mismatch.PropertyName) + fmt.Printf(" ✅ Added speakeasy ignore for %s.%s in update schema\n", resource.EntityName, mismatch.PropertyName) } } } @@ -632,7 +632,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over fmt.Println("\n=== Overlay Generation Complete ===") fmt.Printf("Generated %d actions for %d resources\n", len(overlay.Actions), len(resources)) - // Count terraform ignore actions + // Count ignore actions ignoreCount := 0 for _, action := range overlay.Actions { if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { @@ -642,7 +642,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over if len(resourceMismatches) > 0 { fmt.Printf("✅ %d resources had property mismatches\n", len(resourceMismatches)) - fmt.Printf("✅ %d terraform ignore actions added\n", ignoreCount) + fmt.Printf("✅ %d speakeasy ignore actions added\n", ignoreCount) } return overlay @@ -742,9 +742,14 @@ func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, for propName, entityProp := range entityProps { if requestProp, exists := requestProps[propName]; exists { - if mismatch := detectPropertyMismatch(propName, entityProp, requestProp, operation); mismatch != nil { - fmt.Printf(" ✅ Found mismatch: %s - %s\n", propName, mismatch.Description) - mismatches = append(mismatches, *mismatch) + if hasStructuralMismatch(propName, entityProp, requestProp) { + mismatch := PropertyMismatch{ + PropertyName: propName, + MismatchType: "structural-mismatch", + Description: describeStructuralDifference(entityProp, requestProp), + } + fmt.Printf(" ✅ Found structural mismatch: %s - %s\n", propName, mismatch.Description) + mismatches = append(mismatches, mismatch) } } } @@ -752,139 +757,102 @@ func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, return mismatches } -func detectPropertyMismatch(propName string, entityProp, requestProp interface{}, operation string) *PropertyMismatch { - entityMap, _ := entityProp.(map[string]interface{}) - requestMap, _ := requestProp.(map[string]interface{}) +// Check if two property structures are different +func hasStructuralMismatch(propName string, entityProp, requestProp interface{}) bool { + // Convert both to normalized structure representations + entityStructure := getPropertyStructure(entityProp) + requestStructure := getPropertyStructure(requestProp) - if entityMap == nil || requestMap == nil { - return nil - } + fmt.Printf(" Property '%s':\n", propName) + fmt.Printf(" Request structure: %s\n", requestStructure) + fmt.Printf(" Entity structure: %s\n", entityStructure) - entityType, _ := entityMap["type"].(string) - requestType, _ := requestMap["type"].(string) + // If structures are different, we have a mismatch + different := entityStructure != requestStructure + if different { + fmt.Printf(" ✅ Structures differ - will ignore\n") + } else { + fmt.Printf(" ✓ Structures match\n") + } - fmt.Printf(" Checking property '%s': request=%s, entity=%s\n", propName, requestType, entityType) + return different +} - // Type mismatch detection - if entityType != requestType && entityType != "" && requestType != "" { - fmt.Printf(" ✅ Found basic type mismatch: %s != %s\n", requestType, entityType) - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "type-mismatch", - Description: fmt.Sprintf("request type '%s' != response type '%s'", requestType, entityType), - } +// Get a normalized string representation of a property's structure +func getPropertyStructure(prop interface{}) string { + propMap, ok := prop.(map[string]interface{}) + if !ok { + return "unknown" } - // Array item type mismatch detection - this is the key one for string != class - if entityType == "array" && requestType == "array" { - entityItems, _ := entityMap["items"].(map[string]interface{}) - requestItems, _ := requestMap["items"].(map[string]interface{}) + // Check for $ref + if ref, hasRef := propMap["$ref"].(string); hasRef { + return fmt.Sprintf("$ref:%s", ref) + } - fmt.Printf(" Both are arrays - checking item types\n") + propType, _ := propMap["type"].(string) - // Special debug for environments property - if propName == "environments" { - fmt.Printf(" 🔍 ENVIRONMENTS DEBUG:\n") - fmt.Printf(" Request items: %+v\n", requestItems) - fmt.Printf(" Entity items: %+v\n", entityItems) + switch propType { + case "array": + items, hasItems := propMap["items"] + if hasItems { + itemStructure := getPropertyStructure(items) + return fmt.Sprintf("array[%s]", itemStructure) } + return "array[unknown]" - if entityItems != nil && requestItems != nil { - requestItemType, _ := requestItems["type"].(string) - entityRef, entityHasRef := entityItems["$ref"].(string) - requestRef, requestHasRef := requestItems["$ref"].(string) + case "object": + properties, hasProps := propMap["properties"] + additionalProps, hasAdditional := propMap["additionalProperties"] - fmt.Printf(" Request: itemType='%s', hasRef=%v\n", requestItemType, requestHasRef) - fmt.Printf(" Entity: ref='%s', hasRef=%v\n", entityRef, entityHasRef) - - // Case 1: Request uses string[], Entity uses object[] (refs) - THE KEY CASE - if requestItemType == "string" && entityHasRef { - fmt.Printf(" ✅ Found string[] vs object[] mismatch (Case 1)\n") - fmt.Printf(" Request: string[], Entity: %s\n", entityRef) - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "array-item-mismatch", - Description: fmt.Sprintf("request uses string[], response uses object[] (%s)", entityRef), - } + if hasProps { + propsMap, _ := properties.(map[string]interface{}) + if len(propsMap) == 0 { + return "object{empty}" } - // Case 2: Different object references in arrays - if entityHasRef && requestHasRef && entityRef != requestRef { - fmt.Printf(" ✅ Found different object ref mismatch (Case 2)\n") - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "array-ref-mismatch", - Description: fmt.Sprintf("request references %s, response references %s", requestRef, entityRef), - } - } - - // Case 3: Request uses inline objects, Entity uses refs - requestItemType2, _ := requestItems["type"].(string) - if !requestHasRef && requestItemType2 == "object" && entityHasRef { - fmt.Printf(" ✅ Found inline object vs ref mismatch (Case 3)\n") - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "array-item-mismatch", - Description: fmt.Sprintf("request uses inline objects, response uses object references (%s)", entityRef), - } - } - - // Case 4: Request uses refs, Entity uses string[] (reverse of Case 1) - entityItemType, _ := entityItems["type"].(string) - if requestHasRef && entityItemType == "string" { - fmt.Printf(" ✅ Found object[] vs string[] mismatch (Case 4)\n") - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "array-item-mismatch", - Description: fmt.Sprintf("request uses object[] (%s), response uses string[]", requestRef), - } - } - } else { - fmt.Printf(" Warning: Could not access array items\n") - if entityItems == nil { - fmt.Printf(" Entity items is nil\n") - } - if requestItems == nil { - fmt.Printf(" Request items is nil\n") + // Get structure of nested properties + var propStructures []string + for key, value := range propsMap { + propStructures = append(propStructures, fmt.Sprintf("%s:%s", key, getPropertyStructure(value))) } + return fmt.Sprintf("object{%v}", propStructures) } - } - // Object structure mismatch detection (class vs map) - if entityType == "object" && requestType == "object" { - _, entityHasProps := entityMap["properties"] - _, entityHasAdditional := entityMap["additionalProperties"] - _, requestHasProps := requestMap["properties"] - _, requestHasAdditional := requestMap["additionalProperties"] - - fmt.Printf(" Both are objects - checking structure\n") - fmt.Printf(" Entity: hasProps=%v, hasAdditional=%v\n", entityHasProps, entityHasAdditional) - fmt.Printf(" Request: hasProps=%v, hasAdditional=%v\n", requestHasProps, requestHasAdditional) - - // Map vs Class mismatch - if entityHasProps && !entityHasAdditional && requestHasAdditional && !requestHasProps { - fmt.Printf(" ✅ Found map vs class mismatch (entity=class, request=map)\n") - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "object-structure-mismatch", - Description: "request uses map structure, response uses class structure", - } + if hasAdditional { + additionalStructure := getPropertyStructure(additionalProps) + return fmt.Sprintf("object{additional:%s}", additionalStructure) } - if requestHasProps && !requestHasAdditional && entityHasAdditional && !entityHasProps { - fmt.Printf(" ✅ Found class vs map mismatch (request=class, entity=map)\n") - return &PropertyMismatch{ - PropertyName: propName, - MismatchType: "object-structure-mismatch", - Description: "request uses class structure, response uses map structure", + return "object{}" + + case "string", "integer", "number", "boolean": + return propType + + default: + if propType == "" { + // No explicit type - check what we have + if _, hasProps := propMap["properties"]; hasProps { + return "implicit-object" + } + if _, hasItems := propMap["items"]; hasItems { + return "implicit-array" } } + return fmt.Sprintf("type:%s", propType) } +} - fmt.Printf(" No mismatch detected for %s\n", propName) - return nil +// Describe the structural difference for reporting +func describeStructuralDifference(entityProp, requestProp interface{}) string { + entityStructure := getPropertyStructure(entityProp) + requestStructure := getPropertyStructure(requestProp) + + return fmt.Sprintf("request structure '%s' != response structure '%s'", requestStructure, entityStructure) } +// Remove the old detectPropertyMismatch function - replaced with comprehensive structural comparison + func mapCrudToEntityOperation(crudType, entityName string) string { switch crudType { case "create": From 2ce3605edf8ffa11d0bb01d74eb989d0f66db3b4 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Thu, 5 Jun 2025 11:50:51 -0400 Subject: [PATCH 05/16] do not apply match for exact id --- scripts/overlay/generate-terraform-overlay.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index 74e6be4..2529b0b 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -932,8 +932,8 @@ func addParameterMatches(overlay *Overlay, path, method, resourceName string) { for _, match := range matches { paramName := match[1] - // Check if this looks like an ID parameter - if strings.Contains(paramName, "id") || paramName == resourceName { + if paramName != "id" && (strings.Contains(paramName, "id") || paramName == resourceName) { + fmt.Printf(" Adding x-speakeasy-match for parameter: %s (not exact 'id')\n", paramName) overlay.Actions = append(overlay.Actions, OverlayAction{ Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", path, method, paramName), @@ -941,6 +941,10 @@ func addParameterMatches(overlay *Overlay, path, method, resourceName string) { "x-speakeasy-match": "id", }, }) + } else if paramName == "id" { + fmt.Printf(" Skipping x-speakeasy-match for parameter: %s (already exact 'id')\n", paramName) + } else { + fmt.Printf(" Skipping x-speakeasy-match for parameter: %s (not an ID parameter)\n", paramName) } } } From 21e36325fb4da8a9b46e9fc0858f8a3d15fb1510 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Thu, 5 Jun 2025 17:03:05 -0400 Subject: [PATCH 06/16] only add extensions to valid terraform resources --- scripts/overlay/generate-terraform-overlay.go | 639 +++++++++++------- 1 file changed, 389 insertions(+), 250 deletions(-) diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index 2529b0b..f0c0617 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -36,19 +36,15 @@ func main() { fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) - // Analyze the spec to find resources resources := analyzeSpec(spec) - // Generate overlay overlay := generateOverlay(resources, spec) - // Write overlay file if err := writeOverlay(overlay); err != nil { fmt.Printf("Error writing overlay: %v\n", err) os.Exit(1) } - // Print summary printOverlaySummary(resources, overlay) } @@ -59,10 +55,6 @@ func printUsage() { fmt.Println(" openapi-overlay ") } -// ============================================================================ -// OVERLAY GENERATION LOGIC -// ============================================================================ - type OpenAPISpec struct { OpenAPI string `json:"openapi"` Info map[string]interface{} `json:"info"` @@ -113,15 +105,15 @@ type ResourceInfo struct { SchemaName string ResourceName string Operations map[string]OperationInfo - CreateSchema string // Track the create request schema - UpdateSchema string // Track the update request schema + CreateSchema string + UpdateSchema string } type OperationInfo struct { OperationID string Path string Method string - RequestSchema string // Track request schema separately + RequestSchema string } type PropertyMismatch struct { @@ -130,6 +122,13 @@ type PropertyMismatch struct { Description string } +type CRUDInconsistency struct { + PropertyName string + InconsistencyType string + Description string + SchemasToIgnore []string +} + type OverlayAction struct { Target string `yaml:"target"` Update map[string]interface{} `yaml:"update,omitempty"` @@ -157,22 +156,23 @@ func analyzeSpec(spec OpenAPISpec) map[string]*ResourceInfo { analyzePathOperations(path, pathItem, entitySchemas, resources, spec) } - // Third pass: validate and filter resources - validResources := make(map[string]*ResourceInfo) + // Third pass: validate resources but keep all for analysis + fmt.Printf("\n=== Resource Validation ===\n") for name, resource := range resources { - if isValidTerraformResource(resource) { - validResources[name] = resource - opTypes := make([]string, 0) - for crudType := range resource.Operations { - entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) - opTypes = append(opTypes, fmt.Sprintf("%s->%s", crudType, entityOp)) - } - fmt.Printf("Valid Terraform resource: %s with operations: %v\n", - name, opTypes) + opTypes := make([]string, 0) + for crudType := range resource.Operations { + opTypes = append(opTypes, crudType) + } + fmt.Printf("Resource: %s with operations: %v\n", name, opTypes) + + if isTerraformViable(resource, spec) { + fmt.Printf(" ✅ Viable for Terraform\n") + } else { + fmt.Printf(" ❌ Not viable for Terraform - will skip annotations\n") } } - return validResources + return resources } func identifyEntitySchemas(schemas map[string]Schema) map[string]bool { @@ -239,10 +239,6 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec) if resourceInfo != nil { if existing, exists := resources[resourceInfo.ResourceName]; exists { - fmt.Printf(" Merging operations for %s\n", resourceInfo.ResourceName) - fmt.Printf(" Existing: CreateSchema='%s', UpdateSchema='%s'\n", existing.CreateSchema, existing.UpdateSchema) - fmt.Printf(" New: CreateSchema='%s', UpdateSchema='%s'\n", resourceInfo.CreateSchema, resourceInfo.UpdateSchema) - // Merge operations for opType, opInfo := range resourceInfo.Operations { existing.Operations[opType] = opInfo @@ -255,12 +251,8 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str if resourceInfo.UpdateSchema != "" { existing.UpdateSchema = resourceInfo.UpdateSchema } - - fmt.Printf(" After merge: CreateSchema='%s', UpdateSchema='%s'\n", existing.CreateSchema, existing.UpdateSchema) } else { resources[resourceInfo.ResourceName] = resourceInfo - fmt.Printf(" New resource: %s with CreateSchema='%s', UpdateSchema='%s'\n", - resourceInfo.ResourceName, resourceInfo.CreateSchema, resourceInfo.UpdateSchema) } } } @@ -298,46 +290,27 @@ func extractResourceInfo(path, method string, op *Operation, // Extract request schema for create/update operations if crudType == "create" || crudType == "update" { - fmt.Printf(" Extracting request schema for %s %s operation\n", entityName, crudType) - fmt.Printf(" Operation: %s %s %s\n", method, path, op.OperationID) - if op.RequestBody != nil { - fmt.Printf(" RequestBody exists\n") if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { - fmt.Printf(" Content exists\n") if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { - fmt.Printf(" JSON content exists\n") if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { - fmt.Printf(" Schema exists: %+v\n", schema) if ref, ok := schema["$ref"].(string); ok { requestSchemaName := extractSchemaName(ref) opInfo.RequestSchema = requestSchemaName - fmt.Printf(" ✅ Found request schema: %s\n", requestSchemaName) if crudType == "create" { info.CreateSchema = requestSchemaName } else if crudType == "update" { info.UpdateSchema = requestSchemaName } - } else { - fmt.Printf(" No $ref found in schema\n") } - } else { - fmt.Printf(" No schema found in JSON content\n") } - } else { - fmt.Printf(" No application/json content found\n") } - } else { - fmt.Printf(" No content found in RequestBody\n") } - } else { - fmt.Printf(" No RequestBody found\n") } } info.Operations[crudType] = opInfo - return info } @@ -493,17 +466,110 @@ func toLower(r rune) rune { return r } -func isValidTerraformResource(resource *ResourceInfo) bool { - // Must have at least create and read operations for a full Terraform resource +// Check if a resource is viable for Terraform +func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { + // Must have at least create and read operations _, hasCreate := resource.Operations["create"] _, hasRead := resource.Operations["read"] - // Some resources might be read-only data sources - if hasRead && !hasCreate { - fmt.Printf(" Note: %s appears to be a read-only data source\n", resource.EntityName) + if !hasCreate || !hasRead { + return false + } + + // Check for problematic CRUD patterns that can't be handled by property ignoring + if resource.CreateSchema != "" && resource.UpdateSchema != "" { + // Re-parse the spec to get raw schema data for analysis + specData, err := json.Marshal(spec) + if err != nil { + return true // If we can't analyze, assume it's viable + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return true // If we can't analyze, assume it's viable + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + createProps := getSchemaProperties(schemas, resource.CreateSchema) + updateProps := getSchemaProperties(schemas, resource.UpdateSchema) + + // Count manageable properties (non-system fields) + createManageableProps := 0 + updateManageableProps := 0 + commonManageableProps := 0 + + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + } + } + + for prop := range updateProps { + if !isSystemProperty(prop) { + updateManageableProps++ + // Check if this property also exists in create + if createProps[prop] != nil && !isSystemProperty(prop) { + commonManageableProps++ + } + } + } + + // Reject resources with fundamentally incompatible CRUD patterns + if createManageableProps <= 1 && updateManageableProps >= 3 && commonManageableProps == 0 { + fmt.Printf(" Incompatible CRUD pattern: Create=%d manageable, Update=%d manageable, Common=%d\n", + createManageableProps, updateManageableProps, commonManageableProps) + return false + } + } + + return true +} + +func getSchemaProperties(schemas map[string]interface{}, schemaName string) map[string]interface{} { + if schemaName == "" { + return map[string]interface{}{} + } + + schema, exists := schemas[schemaName] + if !exists { + return map[string]interface{}{} + } + + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + return map[string]interface{}{} + } + + properties, ok := schemaMap["properties"].(map[string]interface{}) + if !ok { + return map[string]interface{}{} } - return hasCreate && hasRead + return properties +} + +func isSystemProperty(propName string) bool { + systemProps := []string{ + "id", "created_at", "updated_at", "created_by", "updated_by", + "version", "etag", "revision", "last_modified", + } + + lowerProp := strings.ToLower(propName) + + for _, sysProp := range systemProps { + if lowerProp == sysProp || strings.HasSuffix(lowerProp, "_"+sysProp) { + return true + } + } + + // Also consider ID fields as system properties + if strings.HasSuffix(lowerProp, "_id") { + return true + } + + return false } func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Overlay { @@ -513,35 +579,46 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over overlay.Info.Title = "Terraform Provider Overlay" overlay.Info.Version = "1.0.0" - overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d Terraform resources", len(resources)) + overlay.Info.Description = "Auto-generated overlay for Terraform resources" - // Detect property mismatches between request and response schemas - resourceMismatches := detectPropertyMismatches(resources, spec) + // Separate viable and non-viable resources + viableResources := make(map[string]*ResourceInfo) + skippedResources := make([]string, 0) - // Report on request/response patterns - fmt.Println("\n=== Request/Response Schema Analysis ===") - for _, resource := range resources { - if resource.CreateSchema != "" || resource.UpdateSchema != "" { - fmt.Printf("%s:\n", resource.EntityName) - if resource.CreateSchema != "" { - fmt.Printf(" Create: %s (request) -> %s (response)\n", resource.CreateSchema, resource.EntityName) - } - if resource.UpdateSchema != "" { - fmt.Printf(" Update: %s (request) -> %s (response)\n", resource.UpdateSchema, resource.EntityName) - } - if mismatches, hasMismatch := resourceMismatches[resource.EntityName]; hasMismatch { - fmt.Printf(" ⚠️ WARNING: Has property mismatches:\n") - for _, mismatch := range mismatches { - fmt.Printf(" - %s: %s\n", mismatch.PropertyName, mismatch.Description) - } - } else { - fmt.Printf(" ✅ No property mismatches detected\n") - } + for name, resource := range resources { + if isTerraformViable(resource, spec) { + viableResources[name] = resource + } else { + skippedResources = append(skippedResources, name) } } - // Generate actions - for _, resource := range resources { + fmt.Printf("\n=== Overlay Generation Analysis ===\n") + fmt.Printf("Total resources found: %d\n", len(resources)) + fmt.Printf("Viable for Terraform: %d\n", len(viableResources)) + fmt.Printf("Skipped (non-viable): %d\n", len(skippedResources)) + + if len(skippedResources) > 0 { + fmt.Printf("\nSkipped resources:\n") + for _, skipped := range skippedResources { + fmt.Printf(" - %s\n", skipped) + } + } + + // Update description with actual count + overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(viableResources)) + + // Detect property mismatches for viable resources only + resourceMismatches := detectPropertyMismatches(viableResources, spec) + + // Detect CRUD inconsistencies for viable resources only + resourceCRUDInconsistencies := detectCRUDInconsistencies(viableResources, spec) + + // Track which properties already have ignore actions to avoid duplicates + ignoreTracker := make(map[string]map[string]bool) // map[schemaName][propertyName]bool + + // Generate actions only for viable resources + for _, resource := range viableResources { // Mark the response entity schema entityUpdate := map[string]interface{}{ "x-speakeasy-entity": resource.EntityName, @@ -552,67 +629,31 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over Update: entityUpdate, }) - // Add speakeasy ignore for mismatched properties in request AND response schemas - if mismatches, exists := resourceMismatches[resource.EntityName]; exists { - // Add speakeasy ignore for create schema properties - if resource.CreateSchema != "" { - for _, mismatch := range mismatches { - // Ignore in request schema - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.CreateSchema, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - - // Also ignore in response entity schema - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - - fmt.Printf(" ✅ Added speakeasy ignore for %s.%s in both request (%s) and response (%s) schemas\n", - resource.EntityName, mismatch.PropertyName, resource.CreateSchema, resource.EntityName) - } - } + // Initialize ignore tracker for this resource's schemas + if ignoreTracker[resource.EntityName] == nil { + ignoreTracker[resource.EntityName] = make(map[string]bool) + } + if resource.CreateSchema != "" && ignoreTracker[resource.CreateSchema] == nil { + ignoreTracker[resource.CreateSchema] = make(map[string]bool) + } + if resource.UpdateSchema != "" && ignoreTracker[resource.UpdateSchema] == nil { + ignoreTracker[resource.UpdateSchema] = make(map[string]bool) + } - // Add terraform ignore for update schema properties - if resource.UpdateSchema != "" { - for _, mismatch := range mismatches { - // Ignore in request schema - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.UpdateSchema, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - - // Also ignore in response entity schema (avoid duplicates) - // Only add if we didn't already add it for create schema - if resource.CreateSchema == "" { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - } + // Add speakeasy ignore for property mismatches + if mismatches, exists := resourceMismatches[resource.EntityName]; exists { + addIgnoreActionsForMismatches(overlay, resource, mismatches, ignoreTracker) + } - fmt.Printf(" ✅ Added speakeasy ignore for %s.%s in update schema\n", resource.EntityName, mismatch.PropertyName) - } - } + // Add speakeasy ignore for CRUD inconsistencies + if inconsistencies, exists := resourceCRUDInconsistencies[resource.EntityName]; exists { + addIgnoreActionsForInconsistencies(overlay, resource, inconsistencies, ignoreTracker) } // Add entity operations for crudType, opInfo := range resource.Operations { entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) - if crudType == "list" { - fmt.Printf(" List operation: %s -> %s\n", resource.EntityName, entityOp) - } - operationUpdate := map[string]interface{}{ "x-speakeasy-entity-operation": entityOp, } @@ -629,69 +670,125 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over } } - fmt.Println("\n=== Overlay Generation Complete ===") - fmt.Printf("Generated %d actions for %d resources\n", len(overlay.Actions), len(resources)) + fmt.Printf("\n=== Overlay Generation Complete ===\n") + fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(viableResources)) // Count ignore actions - ignoreCount := 0 + totalIgnores := 0 for _, action := range overlay.Actions { if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { - ignoreCount++ + totalIgnores++ } } - if len(resourceMismatches) > 0 { - fmt.Printf("✅ %d resources had property mismatches\n", len(resourceMismatches)) - fmt.Printf("✅ %d speakeasy ignore actions added\n", ignoreCount) + if totalIgnores > 0 { + fmt.Printf("✅ %d speakeasy ignore actions added for property issues\n", totalIgnores) } return overlay } -// Enhanced mismatch detection that covers all types of impedance mismatches +func addIgnoreActionsForMismatches(overlay *Overlay, resource *ResourceInfo, mismatches []PropertyMismatch, ignoreTracker map[string]map[string]bool) { + // Add speakeasy ignore for create schema properties + if resource.CreateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + if !ignoreTracker[resource.CreateSchema][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.CreateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.CreateSchema][mismatch.PropertyName] = true + } + + // Also ignore in response entity schema + if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.EntityName][mismatch.PropertyName] = true + } + } + } + + // Add speakeasy ignore for update schema properties + if resource.UpdateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + if !ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.UpdateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] = true + } + + // Also ignore in response entity schema (avoid duplicates) + if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.EntityName][mismatch.PropertyName] = true + } + } + } +} + +func addIgnoreActionsForInconsistencies(overlay *Overlay, resource *ResourceInfo, inconsistencies []CRUDInconsistency, ignoreTracker map[string]map[string]bool) { + for _, inconsistency := range inconsistencies { + // Add ignore actions for each schema listed in SchemasToIgnore + for _, schemaName := range inconsistency.SchemasToIgnore { + if !ignoreTracker[schemaName][inconsistency.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", schemaName, inconsistency.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[schemaName][inconsistency.PropertyName] = true + } + } + } +} + +// Enhanced mismatch detection - same as before but only for viable resources func detectPropertyMismatches(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]PropertyMismatch { mismatches := make(map[string][]PropertyMismatch) - // We need to work with the raw schemas as map[string]interface{} for proper detection // Re-parse the spec to get raw schema data specData, err := json.Marshal(spec) if err != nil { - fmt.Printf("Error marshaling spec for mismatch detection: %v\n", err) return mismatches } var rawSpec map[string]interface{} if err := json.Unmarshal(specData, &rawSpec); err != nil { - fmt.Printf("Error unmarshaling spec for mismatch detection: %v\n", err) return mismatches } components, _ := rawSpec["components"].(map[string]interface{}) schemas, _ := components["schemas"].(map[string]interface{}) - fmt.Printf("\n=== Mismatch Detection Debug ===\n") - fmt.Printf("Total resources to check: %d\n", len(resources)) - for _, resource := range resources { var resourceMismatches []PropertyMismatch - fmt.Printf("Checking resource: %s\n", resource.EntityName) - fmt.Printf(" CreateSchema: %s\n", resource.CreateSchema) - fmt.Printf(" UpdateSchema: %s\n", resource.UpdateSchema) - // Check create operation mismatches if resource.CreateSchema != "" { if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { if requestSchema, exists := schemas[resource.CreateSchema].(map[string]interface{}); exists { - fmt.Printf(" Found both entity and create schemas - checking for mismatches\n") createMismatches := findPropertyMismatches(entitySchema, requestSchema, "create") resourceMismatches = append(resourceMismatches, createMismatches...) - fmt.Printf(" Create mismatches found: %d\n", len(createMismatches)) - } else { - fmt.Printf(" Warning: Create schema %s not found\n", resource.CreateSchema) } - } else { - fmt.Printf(" Warning: Entity schema %s not found\n", resource.EntityName) } } @@ -699,30 +796,17 @@ func detectPropertyMismatches(resources map[string]*ResourceInfo, spec OpenAPISp if resource.UpdateSchema != "" { if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { if requestSchema, exists := schemas[resource.UpdateSchema].(map[string]interface{}); exists { - fmt.Printf(" Found both entity and update schemas - checking for mismatches\n") updateMismatches := findPropertyMismatches(entitySchema, requestSchema, "update") resourceMismatches = append(resourceMismatches, updateMismatches...) - fmt.Printf(" Update mismatches found: %d\n", len(updateMismatches)) - } else { - fmt.Printf(" Warning: Update schema %s not found\n", resource.UpdateSchema) } - } else { - fmt.Printf(" Warning: Entity schema %s not found\n", resource.EntityName) } } if len(resourceMismatches) > 0 { mismatches[resource.EntityName] = resourceMismatches - fmt.Printf(" Total mismatches for %s: %d\n", resource.EntityName, len(resourceMismatches)) - for _, mismatch := range resourceMismatches { - fmt.Printf(" - %s: %s\n", mismatch.PropertyName, mismatch.Description) - } - } else { - fmt.Printf(" No mismatches found for %s\n", resource.EntityName) } } - fmt.Printf("=== End Mismatch Detection Debug ===\n\n") return mismatches } @@ -733,13 +817,9 @@ func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, requestProps, _ := requestSchema["properties"].(map[string]interface{}) if entityProps == nil || requestProps == nil { - fmt.Printf(" Warning: Could not access properties for %s operation\n", operation) return mismatches } - fmt.Printf(" Checking %d entity properties vs %d request properties for %s\n", - len(entityProps), len(requestProps), operation) - for propName, entityProp := range entityProps { if requestProp, exists := requestProps[propName]; exists { if hasStructuralMismatch(propName, entityProp, requestProp) { @@ -748,7 +828,6 @@ func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, MismatchType: "structural-mismatch", Description: describeStructuralDifference(entityProp, requestProp), } - fmt.Printf(" ✅ Found structural mismatch: %s - %s\n", propName, mismatch.Description) mismatches = append(mismatches, mismatch) } } @@ -759,23 +838,9 @@ func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, // Check if two property structures are different func hasStructuralMismatch(propName string, entityProp, requestProp interface{}) bool { - // Convert both to normalized structure representations entityStructure := getPropertyStructure(entityProp) requestStructure := getPropertyStructure(requestProp) - - fmt.Printf(" Property '%s':\n", propName) - fmt.Printf(" Request structure: %s\n", requestStructure) - fmt.Printf(" Entity structure: %s\n", entityStructure) - - // If structures are different, we have a mismatch - different := entityStructure != requestStructure - if different { - fmt.Printf(" ✅ Structures differ - will ignore\n") - } else { - fmt.Printf(" ✓ Structures match\n") - } - - return different + return entityStructure != requestStructure } // Get a normalized string representation of a property's structure @@ -803,25 +868,18 @@ func getPropertyStructure(prop interface{}) string { case "object": properties, hasProps := propMap["properties"] - additionalProps, hasAdditional := propMap["additionalProperties"] + _, hasAdditional := propMap["additionalProperties"] if hasProps { propsMap, _ := properties.(map[string]interface{}) if len(propsMap) == 0 { return "object{empty}" } - - // Get structure of nested properties - var propStructures []string - for key, value := range propsMap { - propStructures = append(propStructures, fmt.Sprintf("%s:%s", key, getPropertyStructure(value))) - } - return fmt.Sprintf("object{%v}", propStructures) + return "object{defined}" } if hasAdditional { - additionalStructure := getPropertyStructure(additionalProps) - return fmt.Sprintf("object{additional:%s}", additionalStructure) + return "object{additional}" } return "object{}" @@ -831,7 +889,6 @@ func getPropertyStructure(prop interface{}) string { default: if propType == "" { - // No explicit type - check what we have if _, hasProps := propMap["properties"]; hasProps { return "implicit-object" } @@ -847,11 +904,123 @@ func getPropertyStructure(prop interface{}) string { func describeStructuralDifference(entityProp, requestProp interface{}) string { entityStructure := getPropertyStructure(entityProp) requestStructure := getPropertyStructure(requestProp) - return fmt.Sprintf("request structure '%s' != response structure '%s'", requestStructure, entityStructure) } -// Remove the old detectPropertyMismatch function - replaced with comprehensive structural comparison +// Detect CRUD inconsistencies - same as before but only for viable resources +func detectCRUDInconsistencies(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]CRUDInconsistency { + inconsistencies := make(map[string][]CRUDInconsistency) + + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return inconsistencies + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return inconsistencies + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + for _, resource := range resources { + // Get properties from each schema + entityProps := getSchemaProperties(schemas, resource.EntityName) + createProps := map[string]interface{}{} + updateProps := map[string]interface{}{} + + if resource.CreateSchema != "" { + createProps = getSchemaProperties(schemas, resource.CreateSchema) + } + if resource.UpdateSchema != "" { + updateProps = getSchemaProperties(schemas, resource.UpdateSchema) + } + + // Collect all property names across CRUD operations + allProps := make(map[string]bool) + for prop := range entityProps { + allProps[prop] = true + } + for prop := range createProps { + allProps[prop] = true + } + for prop := range updateProps { + allProps[prop] = true + } + + var resourceInconsistencies []CRUDInconsistency + + // Check each property for consistency across CRUD operations + for propName := range allProps { + // Skip ID properties - they have separate handling logic + if propName == "id" { + continue + } + + entityHas := entityProps[propName] != nil + createHas := createProps[propName] != nil + updateHas := updateProps[propName] != nil + + // Check for CRUD inconsistencies + var schemasToIgnore []string + var inconsistencyType string + var description string + hasInconsistency := false + + if resource.CreateSchema != "" && resource.UpdateSchema != "" { + // Full CRUD resource - all three must be consistent + if !(entityHas && createHas && updateHas) { + hasInconsistency = true + inconsistencyType = "crud-property-mismatch" + description = fmt.Sprintf("Property not present in all CRUD operations (Entity:%v, Create:%v, Update:%v)", entityHas, createHas, updateHas) + + // Ignore in schemas where property exists but shouldn't for consistency + if entityHas && (!createHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) + } + if createHas && (!entityHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) + } + if updateHas && (!entityHas || !createHas) { + schemasToIgnore = append(schemasToIgnore, resource.UpdateSchema) + } + } + } else if resource.CreateSchema != "" { + // Create + Read resource - both must be consistent + if !(entityHas && createHas) { + hasInconsistency = true + inconsistencyType = "create-read-mismatch" + description = fmt.Sprintf("Property not present in both CREATE and READ (Entity:%v, Create:%v)", entityHas, createHas) + + if entityHas && !createHas { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) + } + if createHas && !entityHas { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) + } + } + } + + if hasInconsistency { + inconsistency := CRUDInconsistency{ + PropertyName: propName, + InconsistencyType: inconsistencyType, + Description: description, + SchemasToIgnore: schemasToIgnore, + } + resourceInconsistencies = append(resourceInconsistencies, inconsistency) + } + } + + if len(resourceInconsistencies) > 0 { + inconsistencies[resource.EntityName] = resourceInconsistencies + } + } + + return inconsistencies +} func mapCrudToEntityOperation(crudType, entityName string) string { switch crudType { @@ -872,37 +1041,12 @@ func mapCrudToEntityOperation(crudType, entityName string) string { } } -// Simplified pluralization logic - keeping essential rules +// Simplified pluralization logic func pluralizeEntityName(entityName string) string { // Remove "Entity" suffix baseName := strings.TrimSuffix(entityName, "Entity") - // Essential special cases - specialCases := map[string]string{ - "Person": "People", - "Child": "Children", - "Status": "Statuses", - "Process": "Processes", - "Policy": "Policies", - "Category": "Categories", - "Entry": "Entries", - "Activity": "Activities", - "Property": "Properties", - "Entity": "Entities", - "Query": "Queries", - "Library": "Libraries", - "History": "Histories", - "Summary": "Summaries", - "Country": "Countries", - "City": "Cities", - "Company": "Companies", - } - - if plural, ok := specialCases[baseName]; ok { - return plural + "Entities" - } - - // Simple pluralization rules + // Simple pluralization if strings.HasSuffix(baseName, "y") && len(baseName) > 1 && !isVowel(baseName[len(baseName)-2]) { baseName = baseName[:len(baseName)-1] + "ies" } else if strings.HasSuffix(baseName, "s") || @@ -933,7 +1077,6 @@ func addParameterMatches(overlay *Overlay, path, method, resourceName string) { paramName := match[1] if paramName != "id" && (strings.Contains(paramName, "id") || paramName == resourceName) { - fmt.Printf(" Adding x-speakeasy-match for parameter: %s (not exact 'id')\n", paramName) overlay.Actions = append(overlay.Actions, OverlayAction{ Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", path, method, paramName), @@ -941,10 +1084,6 @@ func addParameterMatches(overlay *Overlay, path, method, resourceName string) { "x-speakeasy-match": "id", }, }) - } else if paramName == "id" { - fmt.Printf(" Skipping x-speakeasy-match for parameter: %s (already exact 'id')\n", paramName) - } else { - fmt.Printf(" Skipping x-speakeasy-match for parameter: %s (not an ID parameter)\n", paramName) } } } @@ -967,24 +1106,24 @@ func writeOverlay(overlay *Overlay) error { } func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { + viableCount := 0 + for _, resource := range resources { + if isTerraformViable(resource, OpenAPISpec{}) { + viableCount++ + } + } + fmt.Println("\n=== Summary ===") - fmt.Printf("✅ Successfully generated overlay with %d actions for %d resources\n", - len(overlay.Actions), len(resources)) + fmt.Printf("✅ Successfully generated overlay with %d actions\n", len(overlay.Actions)) + fmt.Printf("📊 Resources: %d total, %d viable for Terraform, %d skipped\n", + len(resources), viableCount, len(resources)-viableCount) fmt.Println("\nOverlay approach:") - fmt.Println("1. Mark entity schemas with x-speakeasy-entity") - fmt.Println("2. Tag operations with x-speakeasy-entity-operation") - fmt.Println("3. Mark ID parameters with x-speakeasy-match") - fmt.Println("4. Apply x-speakeasy-ignore: true to mismatched properties") - - fmt.Println("\nResources configured:") - for name, resource := range resources { - ops := make([]string, 0, len(resource.Operations)) - for op := range resource.Operations { - ops = append(ops, op) - } - fmt.Printf(" - %s: [%s]\n", name, strings.Join(ops, ", ")) - } + fmt.Println("1. Skip annotations for non-viable resources") + fmt.Println("2. Mark viable entity schemas with x-speakeasy-entity") + fmt.Println("3. Tag operations with x-speakeasy-entity-operation") + fmt.Println("4. Mark ID parameters with x-speakeasy-match") + fmt.Println("5. Apply x-speakeasy-ignore: true to problematic properties") } // UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data From bd27e1e42681ccd9677b6d6dc847e4f5ebc65a18 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Mon, 9 Jun 2025 15:25:56 -0400 Subject: [PATCH 07/16] improve path param id logic, skip more non-viable resources --- scripts/overlay/generate-terraform-overlay.go | 682 ++++++++++++++---- 1 file changed, 553 insertions(+), 129 deletions(-) diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index f0c0617..d4873bd 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -107,6 +107,7 @@ type ResourceInfo struct { Operations map[string]OperationInfo CreateSchema string UpdateSchema string + PrimaryID string // Store the identified primary ID parameter } type OperationInfo struct { @@ -144,6 +145,42 @@ type Overlay struct { Actions []OverlayAction `yaml:"actions"` } +// IDPattern represents the ID parameter pattern for an operation +type IDPattern struct { + Path string + Method string + Operation string + Parameters []string +} + +// PathParameterInconsistency represents inconsistencies in path parameters +type PathParameterInconsistency struct { + InconsistencyType string + Description string + Operations []string +} + +// UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data +func (s *Schema) UnmarshalJSON(data []byte) error { + type Alias Schema + aux := &struct { + *Alias + }{ + Alias: (*Alias)(s), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Also unmarshal into raw map + if err := json.Unmarshal(data, &s.Raw); err != nil { + return err + } + + return nil +} + func analyzeSpec(spec OpenAPISpec) map[string]*ResourceInfo { resources := make(map[string]*ResourceInfo) @@ -466,6 +503,63 @@ func toLower(r rune) rune { return r } +// Extract path parameters in order from a path string +func extractPathParameters(path string) []string { + re := regexp.MustCompile(`\{([^}]+)\}`) + matches := re.FindAllStringSubmatch(path, -1) + + var params []string + for _, match := range matches { + if len(match) > 1 { + params = append(params, match[1]) + } + } + + return params +} + +// Get entity properties for field existence checking +func getEntityProperties(entityName string, spec OpenAPISpec) map[string]interface{} { + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return map[string]interface{}{} + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return map[string]interface{}{} + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + return getSchemaProperties(schemas, entityName) +} + +// Check if a field exists in the entity properties +func checkFieldExistsInEntity(paramName string, entityProps map[string]interface{}) bool { + // Direct field name match + if _, exists := entityProps[paramName]; exists { + return true + } + + // Check for common variations + variations := []string{ + paramName, + strings.TrimSuffix(paramName, "_id"), // Remove _id suffix + strings.TrimSuffix(paramName, "Id"), // Remove Id suffix + } + + for _, variation := range variations { + if _, exists := entityProps[variation]; exists { + return true + } + } + + return false +} + // Check if a resource is viable for Terraform func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { // Must have at least create and read operations @@ -473,6 +567,42 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { _, hasRead := resource.Operations["read"] if !hasCreate || !hasRead { + fmt.Printf(" Missing required operations: Create=%v, Read=%v\n", hasCreate, hasRead) + return false + } + + // Must have a create schema to be manageable by Terraform + if resource.CreateSchema == "" { + fmt.Printf(" No create schema found\n") + return false + } + + // Identify the primary ID for this entity + primaryID, validPrimaryID := identifyEntityPrimaryID(resource) + if !validPrimaryID { + fmt.Printf(" Cannot identify valid primary ID parameter\n") + return false + } + + // Validate all operations against the primary ID + validOperations := validateOperationParameters(resource, primaryID) + + // Must still have CREATE and READ after validation + _, hasValidCreate := validOperations["create"] + _, hasValidRead := validOperations["read"] + + if !hasValidCreate || !hasValidRead { + fmt.Printf(" Lost required operations after parameter validation: Create=%v, Read=%v\n", hasValidCreate, hasValidRead) + return false + } + + // Update resource with only valid operations and primary ID + resource.Operations = validOperations + resource.PrimaryID = primaryID + + // Check for overlapping properties between create and entity schemas + if !hasValidCreateReadConsistency(resource, spec) { + fmt.Printf(" Create and Read operations have incompatible schemas\n") return false } @@ -527,6 +657,196 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { return true } +// Identify the primary ID parameter that belongs to this specific entity +func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { + // Get all unique path parameters across operations + allParams := make(map[string]bool) + + for crudType, opInfo := range resource.Operations { + if crudType == "create" || crudType == "list" { + continue // Skip operations that typically don't have entity-specific IDs + } + + pathParams := extractPathParameters(opInfo.Path) + for _, param := range pathParams { + allParams[param] = true + } + } + + if len(allParams) == 0 { + return "", false // No path parameters found + } + + // Find the parameter that matches this entity + var entityPrimaryID string + matchCount := 0 + + for param := range allParams { + if mapsToEntityID(param, resource.EntityName) { + entityPrimaryID = param + matchCount++ + } + } + + if matchCount == 0 { + // No parameter maps to this entity - check for generic 'id' parameter + if allParams["id"] { + fmt.Printf(" Using generic 'id' parameter for entity %s\n", resource.EntityName) + return "id", true + } + fmt.Printf(" No parameter maps to entity %s\n", resource.EntityName) + return "", false + } + + if matchCount > 1 { + // Multiple parameters claim to map to this entity - ambiguous + fmt.Printf(" Multiple parameters map to entity %s: ambiguous primary ID\n", resource.EntityName) + return "", false + } + + fmt.Printf(" Identified primary ID '%s' for entity %s\n", entityPrimaryID, resource.EntityName) + return entityPrimaryID, true +} + +// Check if a parameter name maps to a specific entity's ID field +func mapsToEntityID(paramName, entityName string) bool { + // Extract base name from entity (e.g., "ChangeEvent" from "ChangeEventEntity") + entityBase := strings.TrimSuffix(entityName, "Entity") + + // Convert to snake_case and add _id suffix + expectedParam := toSnakeCase(entityBase) + "_id" + + return strings.ToLower(paramName) == strings.ToLower(expectedParam) +} + +// Check if parameter looks like an entity ID +func isEntityID(paramName string) bool { + return strings.HasSuffix(strings.ToLower(paramName), "_id") || strings.ToLower(paramName) == "id" +} + +// Validate operations against the identified primary ID +func validateOperationParameters(resource *ResourceInfo, primaryID string) map[string]OperationInfo { + validOperations := make(map[string]OperationInfo) + + for crudType, opInfo := range resource.Operations { + pathParams := extractPathParameters(opInfo.Path) + + if crudType == "create" || crudType == "list" { + // These operations should not have the entity's primary ID in path + hasPrimaryID := false + for _, param := range pathParams { + if param == primaryID { + hasPrimaryID = true + break + } + } + + if hasPrimaryID { + fmt.Printf(" Skipping %s operation %s: unexpectedly has primary ID %s in path\n", + crudType, opInfo.Path, primaryID) + continue + } + + validOperations[crudType] = opInfo + continue + } + + // READ, UPDATE, DELETE should have exactly the primary ID (and possibly other non-entity params) + hasPrimaryID := false + hasConflictingEntityIDs := false + + for _, param := range pathParams { + if param == primaryID { + hasPrimaryID = true + } else if isEntityID(param) { + // This is a different entity ID - check if it conflicts + if !mapsToEntityID(param, resource.EntityName) { + fmt.Printf(" Skipping %s operation %s: has conflicting entity ID %s (expected %s)\n", + crudType, opInfo.Path, param, primaryID) + hasConflictingEntityIDs = true + break + } + } + // Non-entity parameters (like filters, versions, etc.) are OK + } + + if !hasPrimaryID { + fmt.Printf(" Skipping %s operation %s: missing primary ID %s\n", + crudType, opInfo.Path, primaryID) + continue + } + + if hasConflictingEntityIDs { + continue // Already logged above + } + + validOperations[crudType] = opInfo + } + + fmt.Printf(" Valid operations after parameter validation: %v\n", getOperationTypes(validOperations)) + return validOperations +} + +// Helper function to get operation types for logging +func getOperationTypes(operations map[string]OperationInfo) []string { + var types []string + for opType := range operations { + types = append(types, opType) + } + return types +} + +// Check if create and read operations have compatible schemas +func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) bool { + if resource.CreateSchema == "" { + return false + } + + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return false + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return false + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + entityProps := getSchemaProperties(schemas, resource.EntityName) + createProps := getSchemaProperties(schemas, resource.CreateSchema) + + if len(entityProps) == 0 || len(createProps) == 0 { + return false + } + + // Count overlapping manageable properties + commonManageableProps := 0 + createManageableProps := 0 + + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + if entityProps[prop] != nil { + commonManageableProps++ + } + } + } + + // Need at least some manageable properties + if createManageableProps == 0 { + return false + } + + // Require at least 30% overlap of create properties to exist in entity + // This is more lenient than the 50% I had before + overlapRatio := float64(commonManageableProps) / float64(createManageableProps) + return overlapRatio >= 0.3 +} + func getSchemaProperties(schemas map[string]interface{}, schemaName string) map[string]interface{} { if schemaName == "" { return map[string]interface{}{} @@ -605,20 +925,24 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over } } + // Filter operations with unmappable path parameters + fmt.Printf("\n=== Operation-Level Filtering ===\n") + filteredResources := filterOperationsWithUnmappableParameters(viableResources, spec) + // Update description with actual count - overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(viableResources)) + overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(filteredResources)) - // Detect property mismatches for viable resources only - resourceMismatches := detectPropertyMismatches(viableResources, spec) + // Detect property mismatches for filtered resources only + resourceMismatches := detectPropertyMismatches(filteredResources, spec) - // Detect CRUD inconsistencies for viable resources only - resourceCRUDInconsistencies := detectCRUDInconsistencies(viableResources, spec) + // Detect CRUD inconsistencies for filtered resources only + resourceCRUDInconsistencies := detectCRUDInconsistencies(filteredResources, spec) // Track which properties already have ignore actions to avoid duplicates ignoreTracker := make(map[string]map[string]bool) // map[schemaName][propertyName]bool - // Generate actions only for viable resources - for _, resource := range viableResources { + // Generate actions only for filtered resources + for _, resource := range filteredResources { // Mark the response entity schema entityUpdate := map[string]interface{}{ "x-speakeasy-entity": resource.EntityName, @@ -650,7 +974,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over addIgnoreActionsForInconsistencies(overlay, resource, inconsistencies, ignoreTracker) } - // Add entity operations + // Add entity operations and parameter matching for crudType, opInfo := range resource.Operations { entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) @@ -663,27 +987,50 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over Update: operationUpdate, }) - // Add parameter matches for operations with path parameters - if crudType == "read" || crudType == "update" || crudType == "delete" { - addParameterMatches(overlay, opInfo.Path, opInfo.Method, resource.ResourceName) + // Apply parameter matching for operations that use the primary ID + if resource.PrimaryID != "" && (crudType == "read" || crudType == "update" || crudType == "delete") { + pathParams := extractPathParameters(opInfo.Path) + for _, param := range pathParams { + if param == resource.PrimaryID { + // Apply x-speakeasy-match to the primary ID parameter + fmt.Printf(" Applying x-speakeasy-match to %s in %s %s\n", param, opInfo.Method, opInfo.Path) + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + opInfo.Path, opInfo.Method, param), + Update: map[string]interface{}{ + "x-speakeasy-match": "id", + }, + }) + } + } } } } + // Process parameter matching is now handled inline above + // No need for separate addEntityLevelParameterMatches call + fmt.Printf("\n=== Overlay Generation Complete ===\n") - fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(viableResources)) + fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(filteredResources)) - // Count ignore actions + // Count ignore actions and match actions totalIgnores := 0 + totalMatches := 0 for _, action := range overlay.Actions { if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { totalIgnores++ } + if _, hasMatch := action.Update["x-speakeasy-match"]; hasMatch { + totalMatches++ + } } if totalIgnores > 0 { fmt.Printf("✅ %d speakeasy ignore actions added for property issues\n", totalIgnores) } + if totalMatches > 0 { + fmt.Printf("✅ %d speakeasy match actions added for primary ID parameters\n", totalMatches) + } return overlay } @@ -907,7 +1254,7 @@ func describeStructuralDifference(entityProp, requestProp interface{}) string { return fmt.Sprintf("request structure '%s' != response structure '%s'", requestStructure, entityStructure) } -// Detect CRUD inconsistencies - same as before but only for viable resources +// Detect schema property inconsistencies (extracted from detectCRUDInconsistencies) func detectCRUDInconsistencies(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]CRUDInconsistency { inconsistencies := make(map[string][]CRUDInconsistency) @@ -926,96 +1273,202 @@ func detectCRUDInconsistencies(resources map[string]*ResourceInfo, spec OpenAPIS schemas, _ := components["schemas"].(map[string]interface{}) for _, resource := range resources { - // Get properties from each schema - entityProps := getSchemaProperties(schemas, resource.EntityName) - createProps := map[string]interface{}{} - updateProps := map[string]interface{}{} + resourceInconsistencies := detectSchemaPropertyInconsistencies(resource, schemas) + + // Check if we have fundamental validation errors that make the resource non-viable + for _, inconsistency := range resourceInconsistencies { + if inconsistency.PropertyName == "RESOURCE_VALIDATION" { + fmt.Printf("⚠️ Resource %s (%s) validation failed: %s\n", + resource.ResourceName, resource.EntityName, inconsistency.Description) + // Mark the entire resource as having issues but don't add to inconsistencies + // as this will be handled in the viability check + continue + } + } - if resource.CreateSchema != "" { - createProps = getSchemaProperties(schemas, resource.CreateSchema) + // Only add property-level inconsistencies for viable resources + var validInconsistencies []CRUDInconsistency + for _, inconsistency := range resourceInconsistencies { + if inconsistency.PropertyName != "RESOURCE_VALIDATION" { + validInconsistencies = append(validInconsistencies, inconsistency) + } } - if resource.UpdateSchema != "" { - updateProps = getSchemaProperties(schemas, resource.UpdateSchema) + + if len(validInconsistencies) > 0 { + inconsistencies[resource.EntityName] = validInconsistencies } + } + + return inconsistencies +} + +// Detect schema property inconsistencies (simplified CRUD detection) +func detectSchemaPropertyInconsistencies(resource *ResourceInfo, schemas map[string]interface{}) []CRUDInconsistency { + var inconsistencies []CRUDInconsistency + + // First, validate that we have the minimum required operations for Terraform + _, hasCreate := resource.Operations["create"] + _, hasRead := resource.Operations["read"] - // Collect all property names across CRUD operations - allProps := make(map[string]bool) - for prop := range entityProps { - allProps[prop] = true + if !hasCreate || !hasRead { + // Return a fundamental inconsistency - resource is not viable for Terraform + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "missing-required-operations", + Description: fmt.Sprintf("Resource missing required operations: Create=%v, Read=%v", hasCreate, hasRead), + SchemasToIgnore: []string{}, // Don't ignore anything, this makes the whole resource invalid } - for prop := range createProps { - allProps[prop] = true + return []CRUDInconsistency{inconsistency} + } + + // Validate that we have a create schema + if resource.CreateSchema == "" { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "missing-create-schema", + Description: "Resource has CREATE operation but no request schema defined", + SchemasToIgnore: []string{}, } - for prop := range updateProps { - allProps[prop] = true + return []CRUDInconsistency{inconsistency} + } + + // Get properties from each schema + entityProps := getSchemaProperties(schemas, resource.EntityName) + createProps := getSchemaProperties(schemas, resource.CreateSchema) + updateProps := map[string]interface{}{} + + if resource.UpdateSchema != "" { + updateProps = getSchemaProperties(schemas, resource.UpdateSchema) + } + + // Validate that schemas exist and have properties + if len(entityProps) == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "invalid-entity-schema", + Description: fmt.Sprintf("Entity schema '%s' not found or has no properties", resource.EntityName), + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + if len(createProps) == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "invalid-create-schema", + Description: fmt.Sprintf("Create schema '%s' not found or has no properties", resource.CreateSchema), + SchemasToIgnore: []string{}, } + return []CRUDInconsistency{inconsistency} + } - var resourceInconsistencies []CRUDInconsistency + // Check for minimum viable overlap between create and entity schemas + commonManageableProps := 0 + createManageableProps := 0 - // Check each property for consistency across CRUD operations - for propName := range allProps { - // Skip ID properties - they have separate handling logic - if propName == "id" { - continue + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + if entityProps[prop] != nil { + commonManageableProps++ } + } + } - entityHas := entityProps[propName] != nil - createHas := createProps[propName] != nil - updateHas := updateProps[propName] != nil - - // Check for CRUD inconsistencies - var schemasToIgnore []string - var inconsistencyType string - var description string - hasInconsistency := false - - if resource.CreateSchema != "" && resource.UpdateSchema != "" { - // Full CRUD resource - all three must be consistent - if !(entityHas && createHas && updateHas) { - hasInconsistency = true - inconsistencyType = "crud-property-mismatch" - description = fmt.Sprintf("Property not present in all CRUD operations (Entity:%v, Create:%v, Update:%v)", entityHas, createHas, updateHas) - - // Ignore in schemas where property exists but shouldn't for consistency - if entityHas && (!createHas || !updateHas) { - schemasToIgnore = append(schemasToIgnore, resource.EntityName) - } - if createHas && (!entityHas || !updateHas) { - schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) - } - if updateHas && (!entityHas || !createHas) { - schemasToIgnore = append(schemasToIgnore, resource.UpdateSchema) - } + if createManageableProps == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "no-manageable-properties", + Description: "Create schema has no manageable properties (all are system properties)", + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Require reasonable overlap between create and entity schemas + overlapRatio := float64(commonManageableProps) / float64(createManageableProps) + if overlapRatio < 0.3 { // At least 30% overlap required + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "insufficient-schema-overlap", + Description: fmt.Sprintf("Insufficient overlap between create and entity schemas: %.1f%% (%d/%d properties)", overlapRatio*100, commonManageableProps, createManageableProps), + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Now check individual property inconsistencies for viable resources + // Collect all property names across CRUD operations + allProps := make(map[string]bool) + for prop := range entityProps { + allProps[prop] = true + } + for prop := range createProps { + allProps[prop] = true + } + for prop := range updateProps { + allProps[prop] = true + } + + // Check each property for consistency across CRUD operations + for propName := range allProps { + // Skip ID properties - they have separate handling logic + if propName == "id" { + continue + } + + entityHas := entityProps[propName] != nil + createHas := createProps[propName] != nil + updateHas := updateProps[propName] != nil + + // Check for CRUD inconsistencies + var schemasToIgnore []string + var inconsistencyType string + var description string + hasInconsistency := false + + if resource.CreateSchema != "" && resource.UpdateSchema != "" { + // Full CRUD resource - all three must be consistent + if !(entityHas && createHas && updateHas) { + hasInconsistency = true + inconsistencyType = "crud-property-mismatch" + description = fmt.Sprintf("Property not present in all CRUD operations (Entity:%v, Create:%v, Update:%v)", entityHas, createHas, updateHas) + + // Ignore in schemas where property exists but shouldn't for consistency + if entityHas && (!createHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) } - } else if resource.CreateSchema != "" { - // Create + Read resource - both must be consistent - if !(entityHas && createHas) { - hasInconsistency = true - inconsistencyType = "create-read-mismatch" - description = fmt.Sprintf("Property not present in both CREATE and READ (Entity:%v, Create:%v)", entityHas, createHas) - - if entityHas && !createHas { - schemasToIgnore = append(schemasToIgnore, resource.EntityName) - } - if createHas && !entityHas { - schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) - } + if createHas && (!entityHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) + } + if updateHas && (!entityHas || !createHas) { + schemasToIgnore = append(schemasToIgnore, resource.UpdateSchema) } } - - if hasInconsistency { - inconsistency := CRUDInconsistency{ - PropertyName: propName, - InconsistencyType: inconsistencyType, - Description: description, - SchemasToIgnore: schemasToIgnore, + } else if resource.CreateSchema != "" { + // Create + Read resource - both must be consistent + if !(entityHas && createHas) { + hasInconsistency = true + inconsistencyType = "create-read-mismatch" + description = fmt.Sprintf("Property not present in both CREATE and READ (Entity:%v, Create:%v)", entityHas, createHas) + + if entityHas && !createHas { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) + } + if createHas && !entityHas { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) } - resourceInconsistencies = append(resourceInconsistencies, inconsistency) } } - if len(resourceInconsistencies) > 0 { - inconsistencies[resource.EntityName] = resourceInconsistencies + if hasInconsistency { + inconsistency := CRUDInconsistency{ + PropertyName: propName, + InconsistencyType: inconsistencyType, + Description: description, + SchemasToIgnore: schemasToIgnore, + } + inconsistencies = append(inconsistencies, inconsistency) } } @@ -1068,24 +1521,12 @@ func isVowel(c byte) bool { c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U' } -func addParameterMatches(overlay *Overlay, path, method, resourceName string) { - // Find all path parameters - re := regexp.MustCompile(`\{([^}]+)\}`) - matches := re.FindAllStringSubmatch(path, -1) - - for _, match := range matches { - paramName := match[1] - - if paramName != "id" && (strings.Contains(paramName, "id") || paramName == resourceName) { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", - path, method, paramName), - Update: map[string]interface{}{ - "x-speakeasy-match": "id", - }, - }) - } - } +// Filter out operations with unmappable path parameters - simplified now that we handle this in viability check +func filterOperationsWithUnmappableParameters(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string]*ResourceInfo { + // The complex parameter filtering is now handled in isTerraformViable() + // This function now just returns the resources as-is since they've already been validated + fmt.Printf("Operation filtering handled during viability check\n") + return resources } func writeOverlay(overlay *Overlay) error { @@ -1120,29 +1561,12 @@ func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { fmt.Println("\nOverlay approach:") fmt.Println("1. Skip annotations for non-viable resources") - fmt.Println("2. Mark viable entity schemas with x-speakeasy-entity") - fmt.Println("3. Tag operations with x-speakeasy-entity-operation") - fmt.Println("4. Mark ID parameters with x-speakeasy-match") - fmt.Println("5. Apply x-speakeasy-ignore: true to problematic properties") -} - -// UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data -func (s *Schema) UnmarshalJSON(data []byte) error { - type Alias Schema - aux := &struct { - *Alias - }{ - Alias: (*Alias)(s), - } - - if err := json.Unmarshal(data, &aux); err != nil { - return err - } - - // Also unmarshal into raw map - if err := json.Unmarshal(data, &s.Raw); err != nil { - return err - } - - return nil + fmt.Println("2. Filter out operations with unmappable path parameters") + fmt.Println("3. Mark viable entity schemas with x-speakeasy-entity") + fmt.Println("4. Tag viable operations with x-speakeasy-entity-operation") + fmt.Println("5. Analyze ID patterns across filtered operations per entity") + fmt.Println("6. Choose consistent primary ID for each entity") + fmt.Println("7. Mark chosen primary ID with x-speakeasy-match") + fmt.Println("8. Ignore secondary ID parameters not in entity schema") + fmt.Println("9. Apply x-speakeasy-ignore: true to problematic properties") } From 96a77be4c6a7f4931680e0a9acb74648cfdd12d0 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Mon, 9 Jun 2025 16:47:58 -0400 Subject: [PATCH 08/16] ignore routes with manual mappings --- .github/workflows/sdk_generation.yaml | 2 +- scripts/overlay/generate-terraform-overlay.go | 339 ++++++++++++++---- scripts/overlay/manual-mappings.yaml | 33 ++ 3 files changed, 306 insertions(+), 68 deletions(-) create mode 100644 scripts/overlay/manual-mappings.yaml diff --git a/.github/workflows/sdk_generation.yaml b/.github/workflows/sdk_generation.yaml index 8b8500b..a330900 100644 --- a/.github/workflows/sdk_generation.yaml +++ b/.github/workflows/sdk_generation.yaml @@ -58,7 +58,7 @@ jobs: go run scripts/normalize/normalize-schema.go openapi-raw.json openapi.json # Generate Terraform overlay - go run scripts/overlay/generate-terraform-overlay.go openapi.json + go run scripts/overlay/generate-terraform-overlay.go openapi.json scripts/overlay/manual-mappings.yaml # Move overlay to Speakeasy directory mkdir -p .speakeasy diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index d4873bd..4b74c63 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -11,48 +11,16 @@ import ( "gopkg.in/yaml.v3" ) -func main() { - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - specPath := os.Args[1] - - fmt.Printf("=== Terraform Overlay Generator ===\n") - fmt.Printf("Input: %s\n", specPath) - - specData, err := ioutil.ReadFile(specPath) - if err != nil { - fmt.Printf("Error reading spec file: %v\n", err) - os.Exit(1) - } - - var spec OpenAPISpec - if err := json.Unmarshal(specData, &spec); err != nil { - fmt.Printf("Error parsing JSON: %v\n", err) - os.Exit(1) - } - - fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) - - resources := analyzeSpec(spec) - - overlay := generateOverlay(resources, spec) - - if err := writeOverlay(overlay); err != nil { - fmt.Printf("Error writing overlay: %v\n", err) - os.Exit(1) - } - - printOverlaySummary(resources, overlay) +// Manual mapping configuration +type ManualMapping struct { + Path string `yaml:"path"` + Method string `yaml:"method"` + Action string `yaml:"action"` // "ignore", "entity", "match" + Value string `yaml:"value,omitempty"` } -func printUsage() { - fmt.Println("OpenAPI Terraform Overlay Generator") - fmt.Println() - fmt.Println("Usage:") - fmt.Println(" openapi-overlay ") +type ManualMappings struct { + Operations []ManualMapping `yaml:"operations"` } type OpenAPISpec struct { @@ -160,6 +128,115 @@ type PathParameterInconsistency struct { Operations []string } +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + specPath := os.Args[1] + var mappingsPath string + if len(os.Args) > 2 { + mappingsPath = os.Args[2] + } else { + mappingsPath = "manual-mappings.yaml" + } + + fmt.Printf("=== Terraform Overlay Generator ===\n") + fmt.Printf("Input: %s\n", specPath) + + // Load manual mappings + manualMappings := loadManualMappings(mappingsPath) + + specData, err := ioutil.ReadFile(specPath) + if err != nil { + fmt.Printf("Error reading spec file: %v\n", err) + os.Exit(1) + } + + var spec OpenAPISpec + if err := json.Unmarshal(specData, &spec); err != nil { + fmt.Printf("Error parsing JSON: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) + + resources := analyzeSpec(spec, manualMappings) + + overlay := generateOverlay(resources, spec, manualMappings) + + if err := writeOverlay(overlay); err != nil { + fmt.Printf("Error writing overlay: %v\n", err) + os.Exit(1) + } + + printOverlaySummary(resources, overlay) +} + +func printUsage() { + fmt.Println("OpenAPI Terraform Overlay Generator") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" openapi-overlay ") +} + +// Load manual mappings from file +func loadManualMappings(mappingsPath string) *ManualMappings { + data, err := ioutil.ReadFile(mappingsPath) + if err != nil { + // File doesn't exist - return empty mappings + fmt.Printf("No manual mappings file found at %s (this is optional)\n", mappingsPath) + return &ManualMappings{} + } + + var mappings ManualMappings + if err := yaml.Unmarshal(data, &mappings); err != nil { + fmt.Printf("Error parsing manual mappings file: %v\n", err) + return &ManualMappings{} + } + + fmt.Printf("Loaded %d manual mappings from %s\n", len(mappings.Operations), mappingsPath) + return &mappings +} + +// Check if an operation should be manually ignored +func shouldIgnoreOperation(path, method string, manualMappings *ManualMappings) bool { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "ignore" { + fmt.Printf(" Manual mapping: Ignoring operation %s %s\n", method, path) + return true + } + } + return false +} + +// Check if an operation has a manual entity mapping +func getManualEntityMapping(path, method string, manualMappings *ManualMappings) (string, bool) { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "entity" { + fmt.Printf(" Manual mapping: Operation %s %s -> Entity %s\n", method, path, mapping.Value) + return mapping.Value, true + } + } + return "", false +} + +// Check if a parameter has a manual match mapping +func getManualParameterMatch(path, method, paramName string, manualMappings *ManualMappings) (string, bool) { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "match" { + // For match mappings, we expect the value to be in format "param_name:field_name" + parts := strings.SplitN(mapping.Value, ":", 2) + if len(parts) == 2 && parts[0] == paramName { + fmt.Printf(" Manual mapping: Parameter %s in %s %s -> %s\n", paramName, method, path, parts[1]) + return parts[1], true + } + } + } + return "", false +} + // UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data func (s *Schema) UnmarshalJSON(data []byte) error { type Alias Schema @@ -181,7 +258,7 @@ func (s *Schema) UnmarshalJSON(data []byte) error { return nil } -func analyzeSpec(spec OpenAPISpec) map[string]*ResourceInfo { +func analyzeSpec(spec OpenAPISpec, manualMappings *ManualMappings) map[string]*ResourceInfo { resources := make(map[string]*ResourceInfo) // First pass: identify all entity schemas @@ -190,7 +267,7 @@ func analyzeSpec(spec OpenAPISpec) map[string]*ResourceInfo { // Second pass: match operations to entities for path, pathItem := range spec.Paths { - analyzePathOperations(path, pathItem, entitySchemas, resources, spec) + analyzePathOperations(path, pathItem, entitySchemas, resources, spec, manualMappings) } // Third pass: validate resources but keep all for analysis @@ -255,7 +332,7 @@ func isEntitySchema(name string, schema Schema) bool { } func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[string]bool, - resources map[string]*ResourceInfo, spec OpenAPISpec) { + resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) { operations := []struct { method string @@ -273,7 +350,12 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str continue } - resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec) + // Check if this operation should be manually ignored + if shouldIgnoreOperation(path, item.method, manualMappings) { + continue + } + + resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec, manualMappings) if resourceInfo != nil { if existing, exists := resources[resourceInfo.ResourceName]; exists { // Merge operations @@ -296,7 +378,7 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str } func extractResourceInfo(path, method string, op *Operation, - entitySchemas map[string]bool, spec OpenAPISpec) *ResourceInfo { + entitySchemas map[string]bool, spec OpenAPISpec, manualMappings *ManualMappings) *ResourceInfo { // Determine CRUD type crudType := determineCrudType(path, method, op.OperationID) @@ -304,7 +386,52 @@ func extractResourceInfo(path, method string, op *Operation, return nil } - // Find associated entity schema + // Check for manual entity mapping first + if manualEntityName, hasManual := getManualEntityMapping(path, method, manualMappings); hasManual { + // Use manual entity mapping + entityName := manualEntityName + resourceName := deriveResourceName(entityName, op.OperationID, path) + + info := &ResourceInfo{ + EntityName: entityName, + SchemaName: entityName, + ResourceName: resourceName, + Operations: make(map[string]OperationInfo), + } + + opInfo := OperationInfo{ + OperationID: op.OperationID, + Path: path, + Method: method, + } + + // Extract request schema for create/update operations + if crudType == "create" || crudType == "update" { + if op.RequestBody != nil { + if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + if ref, ok := schema["$ref"].(string); ok { + requestSchemaName := extractSchemaName(ref) + opInfo.RequestSchema = requestSchemaName + + if crudType == "create" { + info.CreateSchema = requestSchemaName + } else if crudType == "update" { + info.UpdateSchema = requestSchemaName + } + } + } + } + } + } + } + + info.Operations[crudType] = opInfo + return info + } + + // Find associated entity schema using automatic detection entityName := findEntityFromOperation(op, entitySchemas, spec) if entityName == "" { return nil @@ -892,7 +1019,7 @@ func isSystemProperty(propName string) bool { return false } -func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Overlay { +func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) *Overlay { overlay := &Overlay{ Overlay: "1.0.0", } @@ -901,11 +1028,14 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over overlay.Info.Version = "1.0.0" overlay.Info.Description = "Auto-generated overlay for Terraform resources" + // Clean up resources by removing manually ignored operations + cleanedResources := cleanResourcesWithManualMappings(resources, manualMappings) + // Separate viable and non-viable resources viableResources := make(map[string]*ResourceInfo) skippedResources := make([]string, 0) - for name, resource := range resources { + for name, resource := range cleanedResources { if isTerraformViable(resource, spec) { viableResources[name] = resource } else { @@ -914,7 +1044,7 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over } fmt.Printf("\n=== Overlay Generation Analysis ===\n") - fmt.Printf("Total resources found: %d\n", len(resources)) + fmt.Printf("Total resources found: %d\n", len(cleanedResources)) fmt.Printf("Viable for Terraform: %d\n", len(viableResources)) fmt.Printf("Skipped (non-viable): %d\n", len(skippedResources)) @@ -976,6 +1106,12 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over // Add entity operations and parameter matching for crudType, opInfo := range resource.Operations { + // Double-check that this specific operation isn't in the ignore list + if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { + fmt.Printf(" Skipping ignored operation during overlay generation: %s %s\n", opInfo.Method, opInfo.Path) + continue + } + entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) operationUpdate := map[string]interface{}{ @@ -992,15 +1128,36 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over pathParams := extractPathParameters(opInfo.Path) for _, param := range pathParams { if param == resource.PrimaryID { - // Apply x-speakeasy-match to the primary ID parameter - fmt.Printf(" Applying x-speakeasy-match to %s in %s %s\n", param, opInfo.Method, opInfo.Path) - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", - opInfo.Path, opInfo.Method, param), - Update: map[string]interface{}{ - "x-speakeasy-match": "id", - }, - }) + // Check for manual parameter mapping first + if manualMatch, hasManual := getManualParameterMatch(opInfo.Path, opInfo.Method, param, manualMappings); hasManual { + // Only apply manual match if it's different from the parameter name + if manualMatch != param { + fmt.Printf(" Manual parameter mapping: %s in %s %s -> %s\n", param, opInfo.Method, opInfo.Path, manualMatch) + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + opInfo.Path, opInfo.Method, param), + Update: map[string]interface{}{ + "x-speakeasy-match": manualMatch, + }, + }) + } else { + fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s\n", param, manualMatch) + } + } else { + // Only apply automatic x-speakeasy-match if parameter name doesn't already match "id" + if param != "id" { + fmt.Printf(" Applying x-speakeasy-match to %s in %s %s -> id\n", param, opInfo.Method, opInfo.Path) + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + opInfo.Path, opInfo.Method, param), + Update: map[string]interface{}{ + "x-speakeasy-match": "id", + }, + }) + } else { + fmt.Printf(" Skipping x-speakeasy-match: parameter %s already matches target field 'id'\n", param) + } + } } } } @@ -1035,6 +1192,52 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec) *Over return overlay } +// Clean up resources by removing operations that are manually ignored +func cleanResourcesWithManualMappings(resources map[string]*ResourceInfo, manualMappings *ManualMappings) map[string]*ResourceInfo { + cleanedResources := make(map[string]*ResourceInfo) + + fmt.Printf("\n=== Cleaning Resources with Manual Mappings ===\n") + + for name, resource := range resources { + cleanedResource := &ResourceInfo{ + EntityName: resource.EntityName, + SchemaName: resource.SchemaName, + ResourceName: resource.ResourceName, + Operations: make(map[string]OperationInfo), + CreateSchema: resource.CreateSchema, + UpdateSchema: resource.UpdateSchema, + PrimaryID: resource.PrimaryID, + } + + operationsRemoved := 0 + + // Copy operations that aren't manually ignored + for crudType, opInfo := range resource.Operations { + if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { + fmt.Printf(" Removing manually ignored operation: %s %s (was %s for %s)\n", + opInfo.Method, opInfo.Path, crudType, resource.EntityName) + operationsRemoved++ + } else { + cleanedResource.Operations[crudType] = opInfo + } + } + + // Only include resource if it still has operations after cleaning + if len(cleanedResource.Operations) > 0 { + cleanedResources[name] = cleanedResource + if operationsRemoved > 0 { + fmt.Printf(" Resource %s: kept %d operations, removed %d manually ignored\n", + name, len(cleanedResource.Operations), operationsRemoved) + } + } else { + fmt.Printf(" Resource %s: removed entirely (all operations were manually ignored)\n", name) + } + } + + fmt.Printf("Manual mapping cleanup: %d → %d resources\n", len(resources), len(cleanedResources)) + return cleanedResources +} + func addIgnoreActionsForMismatches(overlay *Overlay, resource *ResourceInfo, mismatches []PropertyMismatch, ignoreTracker map[string]map[string]bool) { // Add speakeasy ignore for create schema properties if resource.CreateSchema != "" { @@ -1560,13 +1763,15 @@ func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { len(resources), viableCount, len(resources)-viableCount) fmt.Println("\nOverlay approach:") - fmt.Println("1. Skip annotations for non-viable resources") - fmt.Println("2. Filter out operations with unmappable path parameters") - fmt.Println("3. Mark viable entity schemas with x-speakeasy-entity") - fmt.Println("4. Tag viable operations with x-speakeasy-entity-operation") - fmt.Println("5. Analyze ID patterns across filtered operations per entity") - fmt.Println("6. Choose consistent primary ID for each entity") - fmt.Println("7. Mark chosen primary ID with x-speakeasy-match") - fmt.Println("8. Ignore secondary ID parameters not in entity schema") - fmt.Println("9. Apply x-speakeasy-ignore: true to problematic properties") + fmt.Println("1. Load manual mappings for edge cases") + fmt.Println("2. Identify entity schemas and match operations to entities") + fmt.Println("3. Apply manual ignore/entity/match mappings during analysis") + fmt.Println("4. Clean resources by removing manually ignored operations") + fmt.Println("5. Analyze ID patterns and choose consistent primary ID per entity") + fmt.Println("6. Filter operations with unmappable path parameters") + fmt.Println("7. Skip annotations for non-viable resources") + fmt.Println("8. Mark viable entity schemas with x-speakeasy-entity") + fmt.Println("9. Tag viable operations with x-speakeasy-entity-operation") + fmt.Println("10. Mark chosen primary ID with x-speakeasy-match") + fmt.Println("11. Apply x-speakeasy-ignore: true to problematic properties") } diff --git a/scripts/overlay/manual-mappings.yaml b/scripts/overlay/manual-mappings.yaml new file mode 100644 index 0000000..9bfcc10 --- /dev/null +++ b/scripts/overlay/manual-mappings.yaml @@ -0,0 +1,33 @@ +# Action types: +# - "ignore": Skip this operation entirely (won't get any x-speakeasy annotations) +# - "entity": Force operation to map to specific entity (value = entity name) +# - "match": Override parameter mapping (value = "param_name:field_name") + +operations: + # IGNORE: Skip operations that shouldn't be treated as entity operations + # Contains a parameter we cannot reconcile across entity operations + - path: "/v1/incidents/{incident_id}/impact" + method: "patch" + action: "ignore" + - path: "v1/incidents/{incident_id}/resolve" + method: "put" + action: "ignore" + - path: "/v1/incidents/{incident_id}/impact" + method: "put" + action: "ignore" + + # ENTITY: Force specific operations to map to a particular entity + # example: + # - path: "/v1/custom/{custom_id}/special" + # method: "get" + # action: "entity" + # value: "CustomEntity" + + # MATCH: Override parameter matching (format: "param_name:field_name") + # example: + # - path: "/v1/organizations/{org_id}/projects/{project_id}" + # method: "get" + # action: "match" + # value: "project_id:id" # Map project_id parameter to "id" field + + From 04e6d82fdda317cf5bf1731c413b2b9d95659ab8 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Mon, 9 Jun 2025 17:26:30 -0400 Subject: [PATCH 09/16] more manual mappings --- scripts/overlay/generate-terraform-overlay.go | 52 +++++++++++++------ scripts/overlay/manual-mappings.yaml | 20 +++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go index 4b74c63..bc4d7af 100644 --- a/scripts/overlay/generate-terraform-overlay.go +++ b/scripts/overlay/generate-terraform-overlay.go @@ -712,7 +712,7 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { } // Validate all operations against the primary ID - validOperations := validateOperationParameters(resource, primaryID) + validOperations := validateOperationParameters(resource, primaryID, spec) // Must still have CREATE and READ after validation _, hasValidCreate := validOperations["create"] @@ -852,9 +852,12 @@ func isEntityID(paramName string) bool { } // Validate operations against the identified primary ID -func validateOperationParameters(resource *ResourceInfo, primaryID string) map[string]OperationInfo { +func validateOperationParameters(resource *ResourceInfo, primaryID string, spec OpenAPISpec) map[string]OperationInfo { validOperations := make(map[string]OperationInfo) + // Get entity properties once for this resource + entityProps := getEntityProperties(resource.EntityName, spec) + for crudType, opInfo := range resource.Operations { pathParams := extractPathParameters(opInfo.Path) @@ -878,7 +881,7 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string) map[s continue } - // READ, UPDATE, DELETE should have exactly the primary ID (and possibly other non-entity params) + // READ, UPDATE, DELETE should have exactly the primary ID hasPrimaryID := false hasConflictingEntityIDs := false @@ -886,15 +889,30 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string) map[s if param == primaryID { hasPrimaryID = true } else if isEntityID(param) { - // This is a different entity ID - check if it conflicts - if !mapsToEntityID(param, resource.EntityName) { - fmt.Printf(" Skipping %s operation %s: has conflicting entity ID %s (expected %s)\n", - crudType, opInfo.Path, param, primaryID) - hasConflictingEntityIDs = true - break + // This is another ID-like parameter + // Check if it maps to a field in the entity (not the primary id field) + if checkFieldExistsInEntity(param, entityProps) { + // This parameter maps to a real entity field - it's valid + fmt.Printf(" Parameter %s maps to entity field - keeping operation %s %s\n", + param, crudType, opInfo.Path) + } else { + // This ID parameter doesn't map to any entity field + if mapsToEntityID(param, resource.EntityName) { + // This would also try to map to the primary ID - CONFLICT! + fmt.Printf(" Skipping %s operation %s: parameter %s would conflict with primary ID %s (both map to entity.id)\n", + crudType, opInfo.Path, param, primaryID) + hasConflictingEntityIDs = true + break + } else { + // This is an unmappable ID parameter + fmt.Printf(" Skipping %s operation %s: unmappable ID parameter %s (not in entity schema)\n", + crudType, opInfo.Path, param) + hasConflictingEntityIDs = true + break + } } } - // Non-entity parameters (like filters, versions, etc.) are OK + // Non-ID parameters are always OK } if !hasPrimaryID { @@ -914,6 +932,8 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string) map[s return validOperations } +// Remove the helper function since we don't need it anymore + // Helper function to get operation types for logging func getOperationTypes(operations map[string]OperationInfo) []string { var types []string @@ -1141,11 +1161,15 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua }, }) } else { - fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s\n", param, manualMatch) + fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s (would create circular reference)\n", param, manualMatch) } } else { - // Only apply automatic x-speakeasy-match if parameter name doesn't already match "id" - if param != "id" { + // Skip x-speakeasy-match when parameter name would map to itself + // This prevents circular references like {id} -> id + if param == "id" { + fmt.Printf(" Skipping x-speakeasy-match: parameter %s maps to same field (avoiding circular reference)\n", param) + } else { + // Apply x-speakeasy-match for parameters that need mapping (e.g., change_event_id -> id) fmt.Printf(" Applying x-speakeasy-match to %s in %s %s -> id\n", param, opInfo.Method, opInfo.Path) overlay.Actions = append(overlay.Actions, OverlayAction{ Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", @@ -1154,8 +1178,6 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua "x-speakeasy-match": "id", }, }) - } else { - fmt.Printf(" Skipping x-speakeasy-match: parameter %s already matches target field 'id'\n", param) } } } diff --git a/scripts/overlay/manual-mappings.yaml b/scripts/overlay/manual-mappings.yaml index 9bfcc10..0f039e6 100644 --- a/scripts/overlay/manual-mappings.yaml +++ b/scripts/overlay/manual-mappings.yaml @@ -16,6 +16,26 @@ operations: method: "put" action: "ignore" + # Resource type path params are causing issues with entity resolution + # TODO: Figure out how to include + # Terraform propably doesn't need this + # but the pattern isn't that odd so this could easily reappear + - path: /v1/saved_searches/{resource_type}/{saved_search_id}"" + method: "delete" + action: "ignore" + - path: /v1/saved_searches/{resource_type}/{saved_search_id}" + method: "get" + action: "ignore" + - path: "/v1/saved_searches/{resource_type}/{saved_search_id}" + method: "patch" + action: "ignore" + - path: "/v1/saved_searches/{resource_type}" + method: "post" + action: "ignore" + - path: "/v1/saved_searches/{resource_type}" + method: "get" + action: "ignore" + # ENTITY: Force specific operations to map to a particular entity # example: # - path: "/v1/custom/{custom_id}/special" From 17f540d08ecc79877913044b405bd45be092664d Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Tue, 10 Jun 2025 10:55:40 -0400 Subject: [PATCH 10/16] normalize enums, extraction and refs --- scripts/normalize/normalize-schema.go | 519 +++++++++++++++++++++++++- 1 file changed, 500 insertions(+), 19 deletions(-) diff --git a/scripts/normalize/normalize-schema.go b/scripts/normalize/normalize-schema.go index e9349e7..fc794a4 100644 --- a/scripts/normalize/normalize-schema.go +++ b/scripts/normalize/normalize-schema.go @@ -24,7 +24,6 @@ func main() { fmt.Printf("Input: %s\n", inputPath) fmt.Printf("Output: %s\n\n", outputPath) - // Read the spec specData, err := ioutil.ReadFile(inputPath) if err != nil { fmt.Printf("Error reading spec: %v\n", err) @@ -37,13 +36,10 @@ func main() { os.Exit(1) } - // Normalize the spec report := normalizeSpec(spec) - // Print report printNormalizationReport(report) - // Write normalized spec normalizedData, err := json.MarshalIndent(spec, "", " ") if err != nil { fmt.Printf("Error marshaling normalized spec: %v\n", err) @@ -57,6 +53,7 @@ func main() { fmt.Printf("\n✅ Successfully normalized OpenAPI spec\n") fmt.Printf(" Total fixes applied: %d\n", report.TotalFixes) + fmt.Printf(" Enum extractions: %d\n", report.EnumExtractions) } func printUsage() { @@ -66,15 +63,13 @@ func printUsage() { fmt.Println(" openapi-normalize [output.json]") } -// ============================================================================ -// NORMALIZATION LOGIC -// ============================================================================ - type NormalizationReport struct { TotalFixes int MapClassFixes int PropertyFixes int + EnumExtractions int ConflictDetails []ConflictDetail + ExtractedEnums []EnumExtraction } type ConflictDetail struct { @@ -84,9 +79,38 @@ type ConflictDetail struct { Resolution string } +type EntityRelationship struct { + EntityName string + CreateSchema string + UpdateSchema string +} + +// EnumInfo represents an enum found in the spec +type EnumInfo struct { + Type string `json:"type"` + Values []string `json:"values"` + Description string `json:"description"` + Signature string `json:"signature"` // For deduplication +} + +// EnumLocation tracks where an enum is used +type EnumLocation struct { + SchemaName string `json:"schema_name"` + PropertyPath string `json:"property_path"` + EnumInfo EnumInfo `json:"enum_info"` +} + +// EnumExtraction represents the result of enum extraction +type EnumExtraction struct { + ExtractedName string `json:"extracted_name"` + EnumInfo EnumInfo `json:"enum_info"` + Locations []EnumLocation `json:"locations"` +} + func normalizeSpec(spec map[string]interface{}) NormalizationReport { report := NormalizationReport{ ConflictDetails: make([]ConflictDetail, 0), + ExtractedEnums: make([]EnumExtraction, 0), } components, ok := spec["components"].(map[string]interface{}) @@ -107,10 +131,15 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { fmt.Println("Warning: No paths found in spec") } - // Build entity relationships + // Step 1: Extract and normalize enums + extractions := normalizeEnums(spec) + report.ExtractedEnums = extractions + report.EnumExtractions = len(extractions) + + // Step 2: Build entity relationships entityMap := buildEntityRelationships(schemas) - // Normalize each entity and its related schemas + // Step 3: Normalize each entity and its related schemas for entityName, related := range entityMap { fmt.Printf("Analyzing entity: %s\n", entityName) @@ -136,18 +165,18 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { } } - // Apply global normalizations to schemas + // Step 4: Apply global normalizations to schemas globalFixes := applyGlobalNormalizations(schemas) report.ConflictDetails = append(report.ConflictDetails, globalFixes...) - // Normalize path parameters to match entity IDs + // Step 5: Normalize path parameters to match entity IDs if pathsOk { parameterFixes := normalizePathParameters(paths, schemas) report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) } // Calculate totals - report.TotalFixes = len(report.ConflictDetails) + report.TotalFixes = len(report.ConflictDetails) + report.EnumExtractions for _, detail := range report.ConflictDetails { if detail.ConflictType == "map-class" { report.MapClassFixes++ @@ -159,10 +188,449 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { return report } -type EntityRelationship struct { - EntityName string - CreateSchema string - UpdateSchema string +func normalizeEnums(spec map[string]interface{}) []EnumExtraction { + fmt.Printf("\n=== Extracting and Normalizing Enums ===\n") + + components, ok := spec["components"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No components found in spec") + return []EnumExtraction{} + } + + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No schemas found in components") + return []EnumExtraction{} + } + + // Step 1: Collect all enum usages + enumLocations := collectAllEnums(schemas) + fmt.Printf("Found %d enum usages across schemas\n", len(enumLocations)) + + if len(enumLocations) == 0 { + return []EnumExtraction{} + } + + // Step 2: Group by signature for deduplication + enumGroups := groupEnumsBySignature(enumLocations) + fmt.Printf("Found %d unique enum signatures\n", len(enumGroups)) + + // Step 3: Extract enums that appear in multiple locations or are complex + extractions := []EnumExtraction{} + + for signature, locations := range enumGroups { + if len(locations) > 1 { + // Multiple usages - extract to shared enum + extraction := extractSharedEnum(signature, locations, schemas) + extractions = append(extractions, extraction) + fmt.Printf("🔄 Extracted shared enum: %s (used in %d locations)\n", + extraction.ExtractedName, len(locations)) + } else if shouldExtractSingleEnum(locations[0]) { + // Single usage but complex enough to warrant extraction + extraction := extractSingleEnum(locations[0], schemas) + extractions = append(extractions, extraction) + fmt.Printf("📤 Extracted single enum: %s\n", extraction.ExtractedName) + } + } + + return extractions +} + +// Collect all enum usages from schemas +func collectAllEnums(schemas map[string]interface{}) []EnumLocation { + var locations []EnumLocation + + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + // Recursively find enums in this schema + enumsInSchema := findEnumsInSchema(schemaName, schemaMap, "") + locations = append(locations, enumsInSchema...) + } + + return locations +} + +// Recursively find enums in a schema object +func findEnumsInSchema(schemaName string, obj interface{}, path string) []EnumLocation { + var locations []EnumLocation + + switch v := obj.(type) { + case map[string]interface{}: + // Check if this object is an enum + if enumValues, hasEnum := v["enum"]; hasEnum { + if enumArray, ok := enumValues.([]interface{}); ok && len(enumArray) > 0 { + // Convert enum values to strings + var values []string + for _, val := range enumArray { + if str, ok := val.(string); ok { + values = append(values, str) + } + } + + if len(values) > 0 { + enumType := "string" + if typeVal, hasType := v["type"].(string); hasType { + enumType = typeVal + } + + description := "" + if desc, hasDesc := v["description"].(string); hasDesc { + description = desc + } + + location := EnumLocation{ + SchemaName: schemaName, + PropertyPath: path, + EnumInfo: EnumInfo{ + Type: enumType, + Values: values, + Description: description, + Signature: createEnumSignature(enumType, values), + }, + } + locations = append(locations, location) + } + } + } + + // Recursively check nested objects (but skip certain keys) + for key, value := range v { + // Skip certain keys that aren't property definitions + if key == "enum" || key == "type" || key == "description" || key == "example" { + continue + } + + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key + } + nested := findEnumsInSchema(schemaName, value, newPath) + locations = append(locations, nested...) + } + + case []interface{}: + // Check array items + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := findEnumsInSchema(schemaName, item, newPath) + locations = append(locations, nested...) + } + } + + return locations +} + +// Create a signature for deduplication +func createEnumSignature(enumType string, values []string) string { + // Sort values for consistent signature + sortedValues := make([]string, len(values)) + copy(sortedValues, values) + + // Simple bubble sort + for i := 0; i < len(sortedValues); i++ { + for j := i + 1; j < len(sortedValues); j++ { + if sortedValues[i] > sortedValues[j] { + sortedValues[i], sortedValues[j] = sortedValues[j], sortedValues[i] + } + } + } + + return fmt.Sprintf("%s:[%s]", enumType, strings.Join(sortedValues, ",")) +} + +// Group enum locations by signature +func groupEnumsBySignature(locations []EnumLocation) map[string][]EnumLocation { + groups := make(map[string][]EnumLocation) + + for _, location := range locations { + signature := location.EnumInfo.Signature + groups[signature] = append(groups[signature], location) + } + + return groups +} + +// Check if a single enum should be extracted +func shouldExtractSingleEnum(location EnumLocation) bool { + // Extract if: + // - Has more than 3 values (complex enum) + // - Values suggest it's a standard enum (encoding, status, etc.) + if len(location.EnumInfo.Values) > 3 { + return true + } + + // Check for common enum patterns + return isStandardEnumType(location.EnumInfo.Values) +} + +// Check if enum values represent standard types +func isStandardEnumType(values []string) bool { + return isEncodingEnum(values) || isStatusEnum(values) || isLevelEnum(values) || isTypeEnum(values) +} + +// Extract a shared enum used in multiple locations +func extractSharedEnum(signature string, locations []EnumLocation, schemas map[string]interface{}) EnumExtraction { + // Generate a meaningful name for the extracted enum + extractedName := generateExtractedEnumName(locations) + + // Ensure the name is unique + extractedName = ensureUniqueName(extractedName, schemas) + + // Use the first location's enum info as the canonical definition + canonicalEnum := locations[0].EnumInfo + + // Create the extracted enum schema + enumSchema := map[string]interface{}{ + "type": canonicalEnum.Type, + "enum": convertStringsToInterfaces(canonicalEnum.Values), + } + + if canonicalEnum.Description != "" { + enumSchema["description"] = canonicalEnum.Description + } + + // Add to schemas + schemas[extractedName] = enumSchema + + // Replace all usages with $ref + for _, location := range locations { + replaceEnumWithRef(schemas, location, extractedName) + } + + return EnumExtraction{ + ExtractedName: extractedName, + EnumInfo: canonicalEnum, + Locations: locations, + } +} + +// Extract a single complex enum +func extractSingleEnum(location EnumLocation, schemas map[string]interface{}) EnumExtraction { + extractedName := generateSingleEnumName(location) + extractedName = ensureUniqueName(extractedName, schemas) + + // Create the extracted enum schema + enumSchema := map[string]interface{}{ + "type": location.EnumInfo.Type, + "enum": convertStringsToInterfaces(location.EnumInfo.Values), + } + + if location.EnumInfo.Description != "" { + enumSchema["description"] = location.EnumInfo.Description + } + + // Add to schemas + schemas[extractedName] = enumSchema + + // Replace usage with $ref + replaceEnumWithRef(schemas, location, extractedName) + + return EnumExtraction{ + ExtractedName: extractedName, + EnumInfo: location.EnumInfo, + Locations: []EnumLocation{location}, + } +} + +// Generate name for shared enum +func generateExtractedEnumName(locations []EnumLocation) string { + // Try to infer from enum values + if len(locations) > 0 { + values := locations[0].EnumInfo.Values + if isEncodingEnum(values) { + return "EncodingType" + } + if isStatusEnum(values) { + return "StatusType" + } + if isLevelEnum(values) { + return "LevelType" + } + if isTypeEnum(values) { + return "ItemType" + } + } + + // Fallback to generic name + return fmt.Sprintf("SharedEnum%d", len(locations[0].EnumInfo.Values)) +} + +// Generate name for single enum +func generateSingleEnumName(location EnumLocation) string { + // Use property path to create meaningful name + parts := strings.Split(location.PropertyPath, ".") + if len(parts) > 0 { + lastPart := parts[len(parts)-1] + if lastPart != "properties" && lastPart != "" { + return toPascalCase(lastPart) + "Type" + } + } + + // Try to infer from enum values + values := location.EnumInfo.Values + if isEncodingEnum(values) { + return "EncodingType" + } + if isStatusEnum(values) { + return "StatusType" + } + if isLevelEnum(values) { + return "LevelType" + } + + return "ExtractedEnumType" +} + +// Ensure extracted enum name is unique +func ensureUniqueName(baseName string, schemas map[string]interface{}) string { + if _, exists := schemas[baseName]; !exists { + return baseName + } + + // Add counter suffix if name exists + counter := 1 + for { + candidateName := fmt.Sprintf("%s%d", baseName, counter) + if _, exists := schemas[candidateName]; !exists { + return candidateName + } + counter++ + } +} + +// Helper functions for enum type detection +func isEncodingEnum(values []string) bool { + encodingPatterns := []string{"json", "yaml", "xml", "text", "application"} + for _, value := range values { + lowerValue := strings.ToLower(value) + for _, pattern := range encodingPatterns { + if strings.Contains(lowerValue, pattern) { + return true + } + } + } + return false +} + +func isStatusEnum(values []string) bool { + statusPatterns := []string{"active", "inactive", "pending", "completed", "failed", "success", "error", "open", "closed"} + for _, value := range values { + lowerValue := strings.ToLower(value) + for _, pattern := range statusPatterns { + if lowerValue == pattern { + return true + } + } + } + return false +} + +func isLevelEnum(values []string) bool { + levelPatterns := []string{"low", "medium", "high", "debug", "info", "warn", "error", "critical"} + for _, value := range values { + lowerValue := strings.ToLower(value) + for _, pattern := range levelPatterns { + if lowerValue == pattern { + return true + } + } + } + return false +} + +func isTypeEnum(values []string) bool { + // Check if any value contains "type" or represents common type patterns + for _, value := range values { + lowerValue := strings.ToLower(value) + if strings.Contains(lowerValue, "type") { + return true + } + } + + // Check for common type patterns + typePatterns := []string{"string", "number", "boolean", "object", "array", "null"} + for _, value := range values { + lowerValue := strings.ToLower(value) + for _, pattern := range typePatterns { + if lowerValue == pattern { + return true + } + } + } + return false +} + +// Convert string slice to interface slice (for JSON marshaling) +func convertStringsToInterfaces(strings []string) []interface{} { + result := make([]interface{}, len(strings)) + for i, s := range strings { + result[i] = s + } + return result +} + +// Replace enum definition with $ref +func replaceEnumWithRef(schemas map[string]interface{}, location EnumLocation, refName string) { + schema, ok := schemas[location.SchemaName].(map[string]interface{}) + if !ok { + return + } + + // Navigate to the enum location and replace it + if location.PropertyPath == "" { + // Enum is at schema root (shouldn't happen for properties, but handle it) + return + } + + // Navigate through the path to find the enum + parts := strings.Split(location.PropertyPath, ".") + current := schema + + // Navigate to parent of the enum + for i := 0; i < len(parts)-1; i++ { + part := parts[i] + if next, ok := current[part].(map[string]interface{}); ok { + current = next + } else { + return // Path not found + } + } + + // Replace the enum with $ref + finalPart := parts[len(parts)-1] + current[finalPart] = map[string]interface{}{ + "$ref": fmt.Sprintf("#/components/schemas/%s", refName), + } +} + +// Convert string to PascalCase +func toPascalCase(s string) string { + if s == "" { + return s + } + + // Handle snake_case and kebab-case + parts := strings.FieldsFunc(s, func(r rune) bool { + return r == '_' || r == '-' || r == ' ' + }) + + var result strings.Builder + for _, part := range parts { + if len(part) > 0 { + result.WriteString(strings.ToUpper(string(part[0]))) + if len(part) > 1 { + result.WriteString(strings.ToLower(part[1:])) + } + } + } + + return result.String() } func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { @@ -508,14 +976,27 @@ func normalizePathParameters(paths map[string]interface{}, schemas map[string]in return conflicts } + func printNormalizationReport(report NormalizationReport) { fmt.Println("\n=== Normalization Report ===") fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) fmt.Printf("Map/Class fixes: %d\n", report.MapClassFixes) - fmt.Printf("Other property fixes: %d\n", report.PropertyFixes) + fmt.Printf("Property fixes: %d\n", report.PropertyFixes) + fmt.Printf("Enum extractions: %d\n", report.EnumExtractions) + + if len(report.ExtractedEnums) > 0 { + fmt.Println("\nExtracted enums:") + for _, extraction := range report.ExtractedEnums { + fmt.Printf(" - %s: %v (used in %d locations)\n", + extraction.ExtractedName, extraction.EnumInfo.Values, len(extraction.Locations)) + for _, location := range extraction.Locations { + fmt.Printf(" └─ %s.%s\n", location.SchemaName, location.PropertyPath) + } + } + } if len(report.ConflictDetails) > 0 { - fmt.Println("\nDetailed fixes:") + fmt.Println("\nOther fixes applied:") for _, detail := range report.ConflictDetails { fmt.Printf(" - %s.%s [%s]: %s\n", detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) From 8c3ec9edd1ff1bc627f666b1fd3a6dd9ceed21a0 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Thu, 12 Jun 2025 13:49:17 -0400 Subject: [PATCH 11/16] manual mappings for event sources, alert grouping --- scripts/overlay/manual-mappings.yaml | 69 ++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/scripts/overlay/manual-mappings.yaml b/scripts/overlay/manual-mappings.yaml index 0f039e6..01b9de9 100644 --- a/scripts/overlay/manual-mappings.yaml +++ b/scripts/overlay/manual-mappings.yaml @@ -22,7 +22,7 @@ operations: # but the pattern isn't that odd so this could easily reappear - path: /v1/saved_searches/{resource_type}/{saved_search_id}"" method: "delete" - action: "ignore" + action: "ignore" - path: /v1/saved_searches/{resource_type}/{saved_search_id}" method: "get" action: "ignore" @@ -36,18 +36,61 @@ operations: method: "get" action: "ignore" - # ENTITY: Force specific operations to map to a particular entity - # example: - # - path: "/v1/custom/{custom_id}/special" - # method: "get" - # action: "entity" - # value: "CustomEntity" + # Manual Entity Mappings + # Signals Event Sources + - path: "/v1/signals/event_sources/{transposer_slug}" + method: "get" + action: "entity" + value: "Signals_API_TransposerEntity" + + - path: "/v1/signals/event_sources" + method: "put" + action: "entity" + value: "Signals_API_TransposerEntity" + + - path: "/v1/signals/event_sources/{transposer_slug}" + method: "delete" + action: "entity" + value: "Signals_API_TransposerEntity" + + - path: "/v1/signals/event_sources" + method: "get" + action: "entity" + value: "Signals_API_TransposerEntity" - # MATCH: Override parameter matching (format: "param_name:field_name") - # example: - # - path: "/v1/organizations/{org_id}/projects/{project_id}" - # method: "get" - # action: "match" - # value: "project_id:id" # Map project_id parameter to "id" field + # Parameter matching for transposer_slug -> slug field + - path: "/v1/signals/event_sources/{transposer_slug}" + method: "get" + action: "match" + value: "transposer_slug:slug" + - path: "/v1/signals/event_sources/{transposer_slug}" + method: "delete" + action: "match" + value: "transposer_slug:slug" +# Signals Alert Grouping + - path: "/v1/signals/grouping" + method: "get" + action: "entity" + value: "Signals_API_GroupingEntity" + + - path: "/v1/signals/grouping" + method: "post" + action: "entity" + value: "Signals_API_GroupingEntity" + + - path: "/v1/signals/grouping/{id}" + method: "get" + action: "entity" + value: "Signals_API_GroupingEntity" + + - path: "/v1/signals/grouping/{id}" + method: "patch" + action: "entity" + value: "Signals_API_GroupingEntity" + + - path: "/v1/signals/grouping/{id}" + method: "delete" + action: "entity" + value: "Signals_API_GroupingEntity" \ No newline at end of file From 588ed4159a368047e00603563137591253aeee0a Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Thu, 12 Jun 2025 19:15:06 -0400 Subject: [PATCH 12/16] refactor --- .github/workflows/sdk_generation.yaml | 4 +- scripts/normalize/main.go | 491 +++++ scripts/normalize/normalize-schema.go | 1005 --------- scripts/overlay/add-ignores.go | 76 + scripts/overlay/analyze-spec.go | 401 ++++ scripts/overlay/detect.go | 384 ++++ scripts/overlay/generate-overlay.go | 189 ++ scripts/overlay/generate-terraform-overlay.go | 1799 ----------------- scripts/overlay/link-entity-operations.go | 49 + scripts/overlay/main.go | 170 ++ scripts/overlay/manual-mappings.go | 118 ++ scripts/overlay/terraform-viable.go | 397 ++++ 12 files changed, 2277 insertions(+), 2806 deletions(-) create mode 100644 scripts/normalize/main.go delete mode 100644 scripts/normalize/normalize-schema.go create mode 100644 scripts/overlay/add-ignores.go create mode 100644 scripts/overlay/analyze-spec.go create mode 100644 scripts/overlay/detect.go create mode 100644 scripts/overlay/generate-overlay.go delete mode 100644 scripts/overlay/generate-terraform-overlay.go create mode 100644 scripts/overlay/link-entity-operations.go create mode 100644 scripts/overlay/main.go create mode 100644 scripts/overlay/manual-mappings.go create mode 100644 scripts/overlay/terraform-viable.go diff --git a/.github/workflows/sdk_generation.yaml b/.github/workflows/sdk_generation.yaml index a330900..ee6b38e 100644 --- a/.github/workflows/sdk_generation.yaml +++ b/.github/workflows/sdk_generation.yaml @@ -55,10 +55,10 @@ jobs: - name: Process OpenAPI spec run: | # Normalize the schema - go run scripts/normalize/normalize-schema.go openapi-raw.json openapi.json + go run ./scripts/normalize openapi-raw.json openapi.json # Generate Terraform overlay - go run scripts/overlay/generate-terraform-overlay.go openapi.json scripts/overlay/manual-mappings.yaml + go run ./scripts/overlay openapi.json ./scripts/overlay/manual-mappings.yaml # Move overlay to Speakeasy directory mkdir -p .speakeasy diff --git a/scripts/normalize/main.go b/scripts/normalize/main.go new file mode 100644 index 0000000..1ec78c4 --- /dev/null +++ b/scripts/normalize/main.go @@ -0,0 +1,491 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" +) + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + inputPath := os.Args[1] + outputPath := inputPath + if len(os.Args) > 2 { + outputPath = os.Args[2] + } + + fmt.Printf("=== OpenAPI Schema Normalizer ===\n") + fmt.Printf("Input: %s\n", inputPath) + fmt.Printf("Output: %s\n\n", outputPath) + + specData, err := ioutil.ReadFile(inputPath) + if err != nil { + fmt.Printf("Error reading spec: %v\n", err) + os.Exit(1) + } + + var spec map[string]interface{} + if err := json.Unmarshal(specData, &spec); err != nil { + fmt.Printf("Error parsing JSON: %v\n", err) + os.Exit(1) + } + + report := normalizeSpec(spec) + + printNormalizationReport(report) + + normalizedData, err := json.MarshalIndent(spec, "", " ") + if err != nil { + fmt.Printf("Error marshaling normalized spec: %v\n", err) + os.Exit(1) + } + + if err := ioutil.WriteFile(outputPath, normalizedData, 0644); err != nil { + fmt.Printf("Error writing normalized spec: %v\n", err) + os.Exit(1) + } + + fmt.Printf("\n✅ Successfully normalized OpenAPI spec\n") + fmt.Printf(" Total fixes applied: %d\n", report.TotalFixes) +} + +func printUsage() { + fmt.Println("OpenAPI Schema Normalizer") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" openapi-normalize [output.json]") +} + +type NormalizationReport struct { + TotalFixes int + MapClassFixes int + PropertyFixes int + ConflictDetails []ConflictDetail +} + +type ConflictDetail struct { + Schema string + Property string + ConflictType string + Resolution string +} + +func normalizeSpec(spec map[string]interface{}) NormalizationReport { + report := NormalizationReport{ + ConflictDetails: make([]ConflictDetail, 0), + } + + components, ok := spec["components"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No components found in spec") + return report + } + + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No schemas found in components") + return report + } + + paths, pathsOk := spec["paths"].(map[string]interface{}) + if !pathsOk { + fmt.Println("Warning: No paths found in spec") + } + + entityMap := buildEntityRelationships(schemas) + + for entityName, related := range entityMap { + fmt.Printf("Analyzing entity: %s\n", entityName) + + entitySchema, ok := schemas[entityName].(map[string]interface{}) + if !ok { + continue + } + + // Check against create schema + if related.CreateSchema != "" { + if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + + // Check against update schema + if related.UpdateSchema != "" { + if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + } + + // Apply global normalizations to schemas + globalFixes := applyGlobalNormalizations(schemas) + report.ConflictDetails = append(report.ConflictDetails, globalFixes...) + + // Normalize path parameters to match entity IDs + if pathsOk { + parameterFixes := normalizePathParameters(paths) + report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) + } + + // Calculate totals + report.TotalFixes = len(report.ConflictDetails) + for _, detail := range report.ConflictDetails { + if detail.ConflictType == "map-class" { + report.MapClassFixes++ + } else { + report.PropertyFixes++ + } + } + + return report +} + +type EntityRelationship struct { + EntityName string + CreateSchema string + UpdateSchema string +} + +func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { + relationships := make(map[string]EntityRelationship) + + for schemaName := range schemas { + if strings.HasSuffix(schemaName, "Entity") && !strings.Contains(schemaName, "Nullable") && !strings.Contains(schemaName, "Paginated") { + baseName := strings.ToLower(strings.TrimSuffix(schemaName, "Entity")) + + rel := EntityRelationship{ + EntityName: schemaName, + } + + createName := "create_" + baseName + if _, exists := schemas[createName]; exists { + rel.CreateSchema = createName + } + + updateName := "update_" + baseName + if _, exists := schemas[updateName]; exists { + rel.UpdateSchema = updateName + } + + relationships[schemaName] = rel + } + } + + return relationships +} + +func normalizeSchemas(entityName string, entitySchema map[string]interface{}, + requestName string, requestSchema map[string]interface{}) []ConflictDetail { + + conflicts := make([]ConflictDetail, 0) + + entityProps, _ := entitySchema["properties"].(map[string]interface{}) + requestProps, _ := requestSchema["properties"].(map[string]interface{}) + + if entityProps == nil || requestProps == nil { + return conflicts + } + + fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) + fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) + + // Check each property that exists in both schemas + // Terraform requires exact matches for properties across requests and responses + for propName, requestProp := range requestProps { + if entityProp, exists := entityProps[propName]; exists { + fmt.Printf(" Checking property: %s\n", propName) + conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) + if conflict != nil { + fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) + conflicts = append(conflicts, *conflict) + } + } + } + + return conflicts +} + +func checkAndFixProperty(entityName, propName string, entityProp, requestProp interface{}, + entityProps, requestProps map[string]interface{}) *ConflictDetail { + + entityPropMap, _ := entityProp.(map[string]interface{}) + requestPropMap, _ := requestProp.(map[string]interface{}) + + if entityPropMap == nil || requestPropMap == nil { + return nil + } + + // Check for map vs class conflict + entityType, _ := entityPropMap["type"].(string) + requestType, _ := requestPropMap["type"].(string) + + fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) + + if entityType == "object" && requestType == "object" { + _, entityHasProps := entityPropMap["properties"] + _, entityHasAdditional := entityPropMap["additionalProperties"] + _, requestHasProps := requestPropMap["properties"] + _, requestHasAdditional := requestPropMap["additionalProperties"] + + fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) + fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) + + if entityHasProps && requestHasProps { + entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) + requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) + + if len(entityPropsObj) == 0 && len(requestPropsObj) == 0 { + // already the same - both empty properties + return nil + } + } + + if entityHasAdditional && !requestHasAdditional && requestHasProps { + delete(entityPropMap, "additionalProperties") + entityPropMap["properties"] = map[string]interface{}{} + entityProps[propName] = entityPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted entity from additionalProperties to empty properties", + } + } + + if requestHasAdditional && !entityHasAdditional && entityHasProps { + delete(requestPropMap, "additionalProperties") + requestPropMap["properties"] = map[string]interface{}{} + requestProps[propName] = requestPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted request from additionalProperties to empty properties", + } + } + } + + return nil +} + +func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) + + fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") + if len(additionalPropsFound) > 0 { + fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) + for _, path := range additionalPropsFound { + fmt.Printf(" - %s\n", path) + } + } + } + + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + fmt.Printf(" Normalizing schema: %s\n", schemaName) + schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") + conflicts = append(conflicts, schemaConflicts...) + } + + return conflicts +} + +// Recursively find all additionalProperties in a schema +func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { + var found []string + + switch v := obj.(type) { + case map[string]interface{}: + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + fullPath := schemaName + if path != "" { + fullPath += "." + path + } + found = append(found, fullPath) + } + + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key + } + nested := findAllAdditionalProperties(schemaName, value, newPath) + found = append(found, nested...) + } + case []interface{}: + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := findAllAdditionalProperties(schemaName, item, newPath) + found = append(found, nested...) + } + } + + return found +} + +// Recursively normalize all additionalProperties in a schema +func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { + var conflicts []ConflictDetail + + switch v := obj.(type) { + case map[string]interface{}: + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + objType, _ := v["type"].(string) + _, hasProperties := v["properties"] + + if objType == "object" || hasProperties || (!hasProperties && hasAdditional) { + delete(v, "additionalProperties") + if !hasProperties { + v["properties"] = map[string]interface{}{} + } + + fullPath := schemaName + if path != "" { + fullPath += "." + path + } + + conflicts = append(conflicts, ConflictDetail{ + Schema: schemaName, + Property: path, + ConflictType: "map-class", + Resolution: fmt.Sprintf("Converted additionalProperties to empty properties at %s", fullPath), + }) + + fmt.Printf(" ✅ Converted additionalProperties to empty properties at %s\n", fullPath) + } + } + + // Recursively normalize all nested objects + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key + } + nested := normalizeAdditionalProperties(schemaName, value, newPath) + conflicts = append(conflicts, nested...) + } + case []interface{}: + // Normalize array items + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := normalizeAdditionalProperties(schemaName, item, newPath) + conflicts = append(conflicts, nested...) + } + } + + return conflicts +} + +// Normalize path parameters to match entity ID types +func normalizePathParameters(paths map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("\n=== Normalizing Path Parameters ===\n") + + for pathName, pathItem := range paths { + pathMap, ok := pathItem.(map[string]interface{}) + if !ok { + continue + } + + methods := []string{"get", "post", "put", "patch", "delete"} + for _, method := range methods { + if operation, exists := pathMap[method]; exists { + opMap, ok := operation.(map[string]interface{}) + if !ok { + continue + } + + if parameters, hasParams := opMap["parameters"]; hasParams { + paramsList, ok := parameters.([]interface{}) + if !ok { + continue + } + + for _, param := range paramsList { + paramMap, ok := param.(map[string]interface{}) + if !ok { + continue + } + + // normailze int and string parameters + paramIn, _ := paramMap["in"].(string) + paramName, _ := paramMap["name"].(string) + + if paramIn == "path" && (strings.Contains(paramName, "id") || strings.HasSuffix(paramName, "_id")) { + schema, hasSchema := paramMap["schema"] + if hasSchema { + schemaMap, ok := schema.(map[string]interface{}) + if ok { + paramType, _ := schemaMap["type"].(string) + paramFormat, _ := schemaMap["format"].(string) + + if paramType == "integer" { + fmt.Printf(" Found integer ID parameter: %s %s.%s (type: %s, format: %s)\n", + method, pathName, paramName, paramType, paramFormat) + + schemaMap["type"] = "string" + delete(schemaMap, "format") + + conflicts = append(conflicts, ConflictDetail{ + Schema: fmt.Sprintf("path:%s", pathName), + Property: fmt.Sprintf("%s.%s", method, paramName), + ConflictType: "parameter-type", + Resolution: fmt.Sprintf("Converted path parameter %s from integer to string", paramName), + }) + + fmt.Printf(" ✅ Converted %s parameter from integer to string\n", paramName) + } + } + } + } + } + } + } + } + } + + return conflicts +} + +func printNormalizationReport(report NormalizationReport) { + fmt.Println("\n=== Normalization Report ===") + fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) + fmt.Printf("Map/Class fixes: %d\n", report.MapClassFixes) + fmt.Printf("Other property fixes: %d\n", report.PropertyFixes) + + if len(report.ConflictDetails) > 0 { + fmt.Println("\nDetailed fixes:") + for _, detail := range report.ConflictDetails { + fmt.Printf(" - %s.%s [%s]: %s\n", + detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) + } + } +} diff --git a/scripts/normalize/normalize-schema.go b/scripts/normalize/normalize-schema.go deleted file mode 100644 index fc794a4..0000000 --- a/scripts/normalize/normalize-schema.go +++ /dev/null @@ -1,1005 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "strings" -) - -func main() { - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - inputPath := os.Args[1] - outputPath := inputPath - if len(os.Args) > 2 { - outputPath = os.Args[2] - } - - fmt.Printf("=== OpenAPI Schema Normalizer ===\n") - fmt.Printf("Input: %s\n", inputPath) - fmt.Printf("Output: %s\n\n", outputPath) - - specData, err := ioutil.ReadFile(inputPath) - if err != nil { - fmt.Printf("Error reading spec: %v\n", err) - os.Exit(1) - } - - var spec map[string]interface{} - if err := json.Unmarshal(specData, &spec); err != nil { - fmt.Printf("Error parsing JSON: %v\n", err) - os.Exit(1) - } - - report := normalizeSpec(spec) - - printNormalizationReport(report) - - normalizedData, err := json.MarshalIndent(spec, "", " ") - if err != nil { - fmt.Printf("Error marshaling normalized spec: %v\n", err) - os.Exit(1) - } - - if err := ioutil.WriteFile(outputPath, normalizedData, 0644); err != nil { - fmt.Printf("Error writing normalized spec: %v\n", err) - os.Exit(1) - } - - fmt.Printf("\n✅ Successfully normalized OpenAPI spec\n") - fmt.Printf(" Total fixes applied: %d\n", report.TotalFixes) - fmt.Printf(" Enum extractions: %d\n", report.EnumExtractions) -} - -func printUsage() { - fmt.Println("OpenAPI Schema Normalizer") - fmt.Println() - fmt.Println("Usage:") - fmt.Println(" openapi-normalize [output.json]") -} - -type NormalizationReport struct { - TotalFixes int - MapClassFixes int - PropertyFixes int - EnumExtractions int - ConflictDetails []ConflictDetail - ExtractedEnums []EnumExtraction -} - -type ConflictDetail struct { - Schema string - Property string - ConflictType string - Resolution string -} - -type EntityRelationship struct { - EntityName string - CreateSchema string - UpdateSchema string -} - -// EnumInfo represents an enum found in the spec -type EnumInfo struct { - Type string `json:"type"` - Values []string `json:"values"` - Description string `json:"description"` - Signature string `json:"signature"` // For deduplication -} - -// EnumLocation tracks where an enum is used -type EnumLocation struct { - SchemaName string `json:"schema_name"` - PropertyPath string `json:"property_path"` - EnumInfo EnumInfo `json:"enum_info"` -} - -// EnumExtraction represents the result of enum extraction -type EnumExtraction struct { - ExtractedName string `json:"extracted_name"` - EnumInfo EnumInfo `json:"enum_info"` - Locations []EnumLocation `json:"locations"` -} - -func normalizeSpec(spec map[string]interface{}) NormalizationReport { - report := NormalizationReport{ - ConflictDetails: make([]ConflictDetail, 0), - ExtractedEnums: make([]EnumExtraction, 0), - } - - components, ok := spec["components"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No components found in spec") - return report - } - - schemas, ok := components["schemas"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No schemas found in components") - return report - } - - // Get paths for parameter normalization - paths, pathsOk := spec["paths"].(map[string]interface{}) - if !pathsOk { - fmt.Println("Warning: No paths found in spec") - } - - // Step 1: Extract and normalize enums - extractions := normalizeEnums(spec) - report.ExtractedEnums = extractions - report.EnumExtractions = len(extractions) - - // Step 2: Build entity relationships - entityMap := buildEntityRelationships(schemas) - - // Step 3: Normalize each entity and its related schemas - for entityName, related := range entityMap { - fmt.Printf("Analyzing entity: %s\n", entityName) - - entitySchema, ok := schemas[entityName].(map[string]interface{}) - if !ok { - continue - } - - // Check against create schema - if related.CreateSchema != "" { - if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) - report.ConflictDetails = append(report.ConflictDetails, conflicts...) - } - } - - // Check against update schema - if related.UpdateSchema != "" { - if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) - report.ConflictDetails = append(report.ConflictDetails, conflicts...) - } - } - } - - // Step 4: Apply global normalizations to schemas - globalFixes := applyGlobalNormalizations(schemas) - report.ConflictDetails = append(report.ConflictDetails, globalFixes...) - - // Step 5: Normalize path parameters to match entity IDs - if pathsOk { - parameterFixes := normalizePathParameters(paths, schemas) - report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) - } - - // Calculate totals - report.TotalFixes = len(report.ConflictDetails) + report.EnumExtractions - for _, detail := range report.ConflictDetails { - if detail.ConflictType == "map-class" { - report.MapClassFixes++ - } else { - report.PropertyFixes++ - } - } - - return report -} - -func normalizeEnums(spec map[string]interface{}) []EnumExtraction { - fmt.Printf("\n=== Extracting and Normalizing Enums ===\n") - - components, ok := spec["components"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No components found in spec") - return []EnumExtraction{} - } - - schemas, ok := components["schemas"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No schemas found in components") - return []EnumExtraction{} - } - - // Step 1: Collect all enum usages - enumLocations := collectAllEnums(schemas) - fmt.Printf("Found %d enum usages across schemas\n", len(enumLocations)) - - if len(enumLocations) == 0 { - return []EnumExtraction{} - } - - // Step 2: Group by signature for deduplication - enumGroups := groupEnumsBySignature(enumLocations) - fmt.Printf("Found %d unique enum signatures\n", len(enumGroups)) - - // Step 3: Extract enums that appear in multiple locations or are complex - extractions := []EnumExtraction{} - - for signature, locations := range enumGroups { - if len(locations) > 1 { - // Multiple usages - extract to shared enum - extraction := extractSharedEnum(signature, locations, schemas) - extractions = append(extractions, extraction) - fmt.Printf("🔄 Extracted shared enum: %s (used in %d locations)\n", - extraction.ExtractedName, len(locations)) - } else if shouldExtractSingleEnum(locations[0]) { - // Single usage but complex enough to warrant extraction - extraction := extractSingleEnum(locations[0], schemas) - extractions = append(extractions, extraction) - fmt.Printf("📤 Extracted single enum: %s\n", extraction.ExtractedName) - } - } - - return extractions -} - -// Collect all enum usages from schemas -func collectAllEnums(schemas map[string]interface{}) []EnumLocation { - var locations []EnumLocation - - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - // Recursively find enums in this schema - enumsInSchema := findEnumsInSchema(schemaName, schemaMap, "") - locations = append(locations, enumsInSchema...) - } - - return locations -} - -// Recursively find enums in a schema object -func findEnumsInSchema(schemaName string, obj interface{}, path string) []EnumLocation { - var locations []EnumLocation - - switch v := obj.(type) { - case map[string]interface{}: - // Check if this object is an enum - if enumValues, hasEnum := v["enum"]; hasEnum { - if enumArray, ok := enumValues.([]interface{}); ok && len(enumArray) > 0 { - // Convert enum values to strings - var values []string - for _, val := range enumArray { - if str, ok := val.(string); ok { - values = append(values, str) - } - } - - if len(values) > 0 { - enumType := "string" - if typeVal, hasType := v["type"].(string); hasType { - enumType = typeVal - } - - description := "" - if desc, hasDesc := v["description"].(string); hasDesc { - description = desc - } - - location := EnumLocation{ - SchemaName: schemaName, - PropertyPath: path, - EnumInfo: EnumInfo{ - Type: enumType, - Values: values, - Description: description, - Signature: createEnumSignature(enumType, values), - }, - } - locations = append(locations, location) - } - } - } - - // Recursively check nested objects (but skip certain keys) - for key, value := range v { - // Skip certain keys that aren't property definitions - if key == "enum" || key == "type" || key == "description" || key == "example" { - continue - } - - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := findEnumsInSchema(schemaName, value, newPath) - locations = append(locations, nested...) - } - - case []interface{}: - // Check array items - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := findEnumsInSchema(schemaName, item, newPath) - locations = append(locations, nested...) - } - } - - return locations -} - -// Create a signature for deduplication -func createEnumSignature(enumType string, values []string) string { - // Sort values for consistent signature - sortedValues := make([]string, len(values)) - copy(sortedValues, values) - - // Simple bubble sort - for i := 0; i < len(sortedValues); i++ { - for j := i + 1; j < len(sortedValues); j++ { - if sortedValues[i] > sortedValues[j] { - sortedValues[i], sortedValues[j] = sortedValues[j], sortedValues[i] - } - } - } - - return fmt.Sprintf("%s:[%s]", enumType, strings.Join(sortedValues, ",")) -} - -// Group enum locations by signature -func groupEnumsBySignature(locations []EnumLocation) map[string][]EnumLocation { - groups := make(map[string][]EnumLocation) - - for _, location := range locations { - signature := location.EnumInfo.Signature - groups[signature] = append(groups[signature], location) - } - - return groups -} - -// Check if a single enum should be extracted -func shouldExtractSingleEnum(location EnumLocation) bool { - // Extract if: - // - Has more than 3 values (complex enum) - // - Values suggest it's a standard enum (encoding, status, etc.) - if len(location.EnumInfo.Values) > 3 { - return true - } - - // Check for common enum patterns - return isStandardEnumType(location.EnumInfo.Values) -} - -// Check if enum values represent standard types -func isStandardEnumType(values []string) bool { - return isEncodingEnum(values) || isStatusEnum(values) || isLevelEnum(values) || isTypeEnum(values) -} - -// Extract a shared enum used in multiple locations -func extractSharedEnum(signature string, locations []EnumLocation, schemas map[string]interface{}) EnumExtraction { - // Generate a meaningful name for the extracted enum - extractedName := generateExtractedEnumName(locations) - - // Ensure the name is unique - extractedName = ensureUniqueName(extractedName, schemas) - - // Use the first location's enum info as the canonical definition - canonicalEnum := locations[0].EnumInfo - - // Create the extracted enum schema - enumSchema := map[string]interface{}{ - "type": canonicalEnum.Type, - "enum": convertStringsToInterfaces(canonicalEnum.Values), - } - - if canonicalEnum.Description != "" { - enumSchema["description"] = canonicalEnum.Description - } - - // Add to schemas - schemas[extractedName] = enumSchema - - // Replace all usages with $ref - for _, location := range locations { - replaceEnumWithRef(schemas, location, extractedName) - } - - return EnumExtraction{ - ExtractedName: extractedName, - EnumInfo: canonicalEnum, - Locations: locations, - } -} - -// Extract a single complex enum -func extractSingleEnum(location EnumLocation, schemas map[string]interface{}) EnumExtraction { - extractedName := generateSingleEnumName(location) - extractedName = ensureUniqueName(extractedName, schemas) - - // Create the extracted enum schema - enumSchema := map[string]interface{}{ - "type": location.EnumInfo.Type, - "enum": convertStringsToInterfaces(location.EnumInfo.Values), - } - - if location.EnumInfo.Description != "" { - enumSchema["description"] = location.EnumInfo.Description - } - - // Add to schemas - schemas[extractedName] = enumSchema - - // Replace usage with $ref - replaceEnumWithRef(schemas, location, extractedName) - - return EnumExtraction{ - ExtractedName: extractedName, - EnumInfo: location.EnumInfo, - Locations: []EnumLocation{location}, - } -} - -// Generate name for shared enum -func generateExtractedEnumName(locations []EnumLocation) string { - // Try to infer from enum values - if len(locations) > 0 { - values := locations[0].EnumInfo.Values - if isEncodingEnum(values) { - return "EncodingType" - } - if isStatusEnum(values) { - return "StatusType" - } - if isLevelEnum(values) { - return "LevelType" - } - if isTypeEnum(values) { - return "ItemType" - } - } - - // Fallback to generic name - return fmt.Sprintf("SharedEnum%d", len(locations[0].EnumInfo.Values)) -} - -// Generate name for single enum -func generateSingleEnumName(location EnumLocation) string { - // Use property path to create meaningful name - parts := strings.Split(location.PropertyPath, ".") - if len(parts) > 0 { - lastPart := parts[len(parts)-1] - if lastPart != "properties" && lastPart != "" { - return toPascalCase(lastPart) + "Type" - } - } - - // Try to infer from enum values - values := location.EnumInfo.Values - if isEncodingEnum(values) { - return "EncodingType" - } - if isStatusEnum(values) { - return "StatusType" - } - if isLevelEnum(values) { - return "LevelType" - } - - return "ExtractedEnumType" -} - -// Ensure extracted enum name is unique -func ensureUniqueName(baseName string, schemas map[string]interface{}) string { - if _, exists := schemas[baseName]; !exists { - return baseName - } - - // Add counter suffix if name exists - counter := 1 - for { - candidateName := fmt.Sprintf("%s%d", baseName, counter) - if _, exists := schemas[candidateName]; !exists { - return candidateName - } - counter++ - } -} - -// Helper functions for enum type detection -func isEncodingEnum(values []string) bool { - encodingPatterns := []string{"json", "yaml", "xml", "text", "application"} - for _, value := range values { - lowerValue := strings.ToLower(value) - for _, pattern := range encodingPatterns { - if strings.Contains(lowerValue, pattern) { - return true - } - } - } - return false -} - -func isStatusEnum(values []string) bool { - statusPatterns := []string{"active", "inactive", "pending", "completed", "failed", "success", "error", "open", "closed"} - for _, value := range values { - lowerValue := strings.ToLower(value) - for _, pattern := range statusPatterns { - if lowerValue == pattern { - return true - } - } - } - return false -} - -func isLevelEnum(values []string) bool { - levelPatterns := []string{"low", "medium", "high", "debug", "info", "warn", "error", "critical"} - for _, value := range values { - lowerValue := strings.ToLower(value) - for _, pattern := range levelPatterns { - if lowerValue == pattern { - return true - } - } - } - return false -} - -func isTypeEnum(values []string) bool { - // Check if any value contains "type" or represents common type patterns - for _, value := range values { - lowerValue := strings.ToLower(value) - if strings.Contains(lowerValue, "type") { - return true - } - } - - // Check for common type patterns - typePatterns := []string{"string", "number", "boolean", "object", "array", "null"} - for _, value := range values { - lowerValue := strings.ToLower(value) - for _, pattern := range typePatterns { - if lowerValue == pattern { - return true - } - } - } - return false -} - -// Convert string slice to interface slice (for JSON marshaling) -func convertStringsToInterfaces(strings []string) []interface{} { - result := make([]interface{}, len(strings)) - for i, s := range strings { - result[i] = s - } - return result -} - -// Replace enum definition with $ref -func replaceEnumWithRef(schemas map[string]interface{}, location EnumLocation, refName string) { - schema, ok := schemas[location.SchemaName].(map[string]interface{}) - if !ok { - return - } - - // Navigate to the enum location and replace it - if location.PropertyPath == "" { - // Enum is at schema root (shouldn't happen for properties, but handle it) - return - } - - // Navigate through the path to find the enum - parts := strings.Split(location.PropertyPath, ".") - current := schema - - // Navigate to parent of the enum - for i := 0; i < len(parts)-1; i++ { - part := parts[i] - if next, ok := current[part].(map[string]interface{}); ok { - current = next - } else { - return // Path not found - } - } - - // Replace the enum with $ref - finalPart := parts[len(parts)-1] - current[finalPart] = map[string]interface{}{ - "$ref": fmt.Sprintf("#/components/schemas/%s", refName), - } -} - -// Convert string to PascalCase -func toPascalCase(s string) string { - if s == "" { - return s - } - - // Handle snake_case and kebab-case - parts := strings.FieldsFunc(s, func(r rune) bool { - return r == '_' || r == '-' || r == ' ' - }) - - var result strings.Builder - for _, part := range parts { - if len(part) > 0 { - result.WriteString(strings.ToUpper(string(part[0]))) - if len(part) > 1 { - result.WriteString(strings.ToLower(part[1:])) - } - } - } - - return result.String() -} - -func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { - relationships := make(map[string]EntityRelationship) - - for schemaName := range schemas { - if strings.HasSuffix(schemaName, "Entity") && !strings.Contains(schemaName, "Nullable") && !strings.Contains(schemaName, "Paginated") { - baseName := strings.ToLower(strings.TrimSuffix(schemaName, "Entity")) - - rel := EntityRelationship{ - EntityName: schemaName, - } - - // Look for create schema - createName := "create_" + baseName - if _, exists := schemas[createName]; exists { - rel.CreateSchema = createName - } - - // Look for update schema - updateName := "update_" + baseName - if _, exists := schemas[updateName]; exists { - rel.UpdateSchema = updateName - } - - relationships[schemaName] = rel - } - } - - return relationships -} - -func normalizeSchemas(entityName string, entitySchema map[string]interface{}, - requestName string, requestSchema map[string]interface{}) []ConflictDetail { - - conflicts := make([]ConflictDetail, 0) - - entityProps, _ := entitySchema["properties"].(map[string]interface{}) - requestProps, _ := requestSchema["properties"].(map[string]interface{}) - - if entityProps == nil || requestProps == nil { - return conflicts - } - - fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) - fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) - - // Check each property that exists in both schemas - for propName, requestProp := range requestProps { - if entityProp, exists := entityProps[propName]; exists { - fmt.Printf(" Checking property: %s\n", propName) - conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) - if conflict != nil { - fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) - conflicts = append(conflicts, *conflict) - } - } - } - - return conflicts -} - -func checkAndFixProperty(entityName, propName string, entityProp, requestProp interface{}, - entityProps, requestProps map[string]interface{}) *ConflictDetail { - - entityPropMap, _ := entityProp.(map[string]interface{}) - requestPropMap, _ := requestProp.(map[string]interface{}) - - if entityPropMap == nil || requestPropMap == nil { - return nil - } - - // Check for map vs class conflict - entityType, _ := entityPropMap["type"].(string) - requestType, _ := requestPropMap["type"].(string) - - fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) - - if entityType == "object" && requestType == "object" { - _, entityHasProps := entityPropMap["properties"] - _, entityHasAdditional := entityPropMap["additionalProperties"] - _, requestHasProps := requestPropMap["properties"] - _, requestHasAdditional := requestPropMap["additionalProperties"] - - fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) - fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) - - // Normalize to consistent structure without adding additionalProperties - // Option 1: Both have empty properties - make them consistent - if entityHasProps && requestHasProps { - entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) - requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) - - if len(entityPropsObj) == 0 && len(requestPropsObj) == 0 { - // Both have empty properties - this is already consistent - fmt.Printf(" Both have empty properties - already consistent\n") - return nil - } - } - - // Option 2: One has properties, one has additionalProperties - make them both use properties - if entityHasAdditional && !requestHasAdditional && requestHasProps { - // Entity uses additionalProperties, request uses properties - // Convert entity to use empty properties like request - delete(entityPropMap, "additionalProperties") - entityPropMap["properties"] = map[string]interface{}{} - entityProps[propName] = entityPropMap - - return &ConflictDetail{ - Schema: entityName, - Property: propName, - ConflictType: "map-class", - Resolution: "Converted entity from additionalProperties to empty properties", - } - } - - if requestHasAdditional && !entityHasAdditional && entityHasProps { - // Request uses additionalProperties, entity uses properties - // Convert request to use empty properties like entity - delete(requestPropMap, "additionalProperties") - requestPropMap["properties"] = map[string]interface{}{} - requestProps[propName] = requestPropMap - - return &ConflictDetail{ - Schema: entityName, - Property: propName, - ConflictType: "map-class", - Resolution: "Converted request from additionalProperties to empty properties", - } - } - } - - return nil -} - -func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail { - conflicts := make([]ConflictDetail, 0) - - fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) - - // First pass: Find and report all additionalProperties instances - fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") - if len(additionalPropsFound) > 0 { - fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) - for _, path := range additionalPropsFound { - fmt.Printf(" - %s\n", path) - } - } - } - - // Second pass: Fix all additionalProperties instances - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - fmt.Printf(" Normalizing schema: %s\n", schemaName) - schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") - conflicts = append(conflicts, schemaConflicts...) - } - - return conflicts -} - -// Recursively find all additionalProperties in a schema -func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { - var found []string - - switch v := obj.(type) { - case map[string]interface{}: - // Check if this object has additionalProperties - if _, hasAdditional := v["additionalProperties"]; hasAdditional { - fullPath := schemaName - if path != "" { - fullPath += "." + path - } - found = append(found, fullPath) - } - - // Recursively check all nested objects - for key, value := range v { - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := findAllAdditionalProperties(schemaName, value, newPath) - found = append(found, nested...) - } - case []interface{}: - // Check array items - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := findAllAdditionalProperties(schemaName, item, newPath) - found = append(found, nested...) - } - } - - return found -} - -// Recursively normalize all additionalProperties in a schema -func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { - var conflicts []ConflictDetail - - switch v := obj.(type) { - case map[string]interface{}: - // Check if this object has additionalProperties - if _, hasAdditional := v["additionalProperties"]; hasAdditional { - objType, _ := v["type"].(string) - _, hasProperties := v["properties"] - - // Remove additionalProperties if: - // 1. It's explicitly type "object", OR - // 2. It has "properties" (implicit object), OR - // 3. It has additionalProperties but no other structure - if objType == "object" || hasProperties || (!hasProperties && hasAdditional) { - // Remove additionalProperties and ensure empty properties - delete(v, "additionalProperties") - if !hasProperties { - v["properties"] = map[string]interface{}{} - } - - fullPath := schemaName - if path != "" { - fullPath += "." + path - } - - conflicts = append(conflicts, ConflictDetail{ - Schema: schemaName, - Property: path, - ConflictType: "map-class", - Resolution: fmt.Sprintf("Converted additionalProperties to empty properties at %s", fullPath), - }) - - fmt.Printf(" ✅ Converted additionalProperties to empty properties at %s\n", fullPath) - } - } - - // Recursively normalize all nested objects - for key, value := range v { - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := normalizeAdditionalProperties(schemaName, value, newPath) - conflicts = append(conflicts, nested...) - } - case []interface{}: - // Normalize array items - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := normalizeAdditionalProperties(schemaName, item, newPath) - conflicts = append(conflicts, nested...) - } - } - - return conflicts -} - -// Normalize path parameters to match entity ID types -func normalizePathParameters(paths map[string]interface{}, schemas map[string]interface{}) []ConflictDetail { - conflicts := make([]ConflictDetail, 0) - - fmt.Printf("\n=== Normalizing Path Parameters ===\n") - - for pathName, pathItem := range paths { - pathMap, ok := pathItem.(map[string]interface{}) - if !ok { - continue - } - - // Check all HTTP methods in this path - methods := []string{"get", "post", "put", "patch", "delete"} - for _, method := range methods { - if operation, exists := pathMap[method]; exists { - opMap, ok := operation.(map[string]interface{}) - if !ok { - continue - } - - // Check parameters in this operation - if parameters, hasParams := opMap["parameters"]; hasParams { - paramsList, ok := parameters.([]interface{}) - if !ok { - continue - } - - for _, param := range paramsList { - paramMap, ok := param.(map[string]interface{}) - if !ok { - continue - } - - // Check if this is a path parameter with integer type that should be string - paramIn, _ := paramMap["in"].(string) - paramName, _ := paramMap["name"].(string) - - if paramIn == "path" && (strings.Contains(paramName, "id") || strings.HasSuffix(paramName, "_id")) { - schema, hasSchema := paramMap["schema"] - if hasSchema { - schemaMap, ok := schema.(map[string]interface{}) - if ok { - paramType, _ := schemaMap["type"].(string) - paramFormat, _ := schemaMap["format"].(string) - - // Convert integer ID parameters to string - if paramType == "integer" { - fmt.Printf(" Found integer ID parameter: %s %s.%s (type: %s, format: %s)\n", - method, pathName, paramName, paramType, paramFormat) - - schemaMap["type"] = "string" - delete(schemaMap, "format") // Remove int32/int64 format - - conflicts = append(conflicts, ConflictDetail{ - Schema: fmt.Sprintf("path:%s", pathName), - Property: fmt.Sprintf("%s.%s", method, paramName), - ConflictType: "parameter-type", - Resolution: fmt.Sprintf("Converted path parameter %s from integer to string", paramName), - }) - - fmt.Printf(" ✅ Converted %s parameter from integer to string\n", paramName) - } - } - } - } - } - } - } - } - } - - return conflicts -} - -func printNormalizationReport(report NormalizationReport) { - fmt.Println("\n=== Normalization Report ===") - fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) - fmt.Printf("Map/Class fixes: %d\n", report.MapClassFixes) - fmt.Printf("Property fixes: %d\n", report.PropertyFixes) - fmt.Printf("Enum extractions: %d\n", report.EnumExtractions) - - if len(report.ExtractedEnums) > 0 { - fmt.Println("\nExtracted enums:") - for _, extraction := range report.ExtractedEnums { - fmt.Printf(" - %s: %v (used in %d locations)\n", - extraction.ExtractedName, extraction.EnumInfo.Values, len(extraction.Locations)) - for _, location := range extraction.Locations { - fmt.Printf(" └─ %s.%s\n", location.SchemaName, location.PropertyPath) - } - } - } - - if len(report.ConflictDetails) > 0 { - fmt.Println("\nOther fixes applied:") - for _, detail := range report.ConflictDetails { - fmt.Printf(" - %s.%s [%s]: %s\n", - detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) - } - } -} diff --git a/scripts/overlay/add-ignores.go b/scripts/overlay/add-ignores.go new file mode 100644 index 0000000..7d13681 --- /dev/null +++ b/scripts/overlay/add-ignores.go @@ -0,0 +1,76 @@ +package main + +import "fmt" + +func addIgnoreActionsForMismatches(overlay *Overlay, resource *ResourceInfo, mismatches []PropertyMismatch, ignoreTracker map[string]map[string]bool) { + // Add speakeasy ignore for create schema properties + if resource.CreateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + if !ignoreTracker[resource.CreateSchema][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.CreateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.CreateSchema][mismatch.PropertyName] = true + } + + // Also ignore in response entity schema + if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.EntityName][mismatch.PropertyName] = true + } + } + } + + // Add speakeasy ignore for update schema properties + if resource.UpdateSchema != "" { + for _, mismatch := range mismatches { + // Ignore in request schema + if !ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.UpdateSchema, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] = true + } + + // Also ignore in response entity schema (avoid duplicates) + if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[resource.EntityName][mismatch.PropertyName] = true + } + } + } +} + +func addIgnoreActionsForInconsistencies(overlay *Overlay, resource *ResourceInfo, inconsistencies []CRUDInconsistency, ignoreTracker map[string]map[string]bool) { + for _, inconsistency := range inconsistencies { + // Add ignore actions for each schema listed in SchemasToIgnore + for _, schemaName := range inconsistency.SchemasToIgnore { + if !ignoreTracker[schemaName][inconsistency.PropertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", schemaName, inconsistency.PropertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[schemaName][inconsistency.PropertyName] = true + } + } + } +} diff --git a/scripts/overlay/analyze-spec.go b/scripts/overlay/analyze-spec.go new file mode 100644 index 0000000..ac591e4 --- /dev/null +++ b/scripts/overlay/analyze-spec.go @@ -0,0 +1,401 @@ +package main + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +type ResourceInfo struct { + EntityName string + SchemaName string + ResourceName string + Operations map[string]OperationInfo + CreateSchema string + UpdateSchema string + // Store the identified primary ID parameter + // We need to track this so that we use the correct ID field where there are multiple IDs in path params + PrimaryID string +} + +type OperationInfo struct { + OperationID string + Path string + Method string + RequestSchema string +} + +func analyzeSpec(spec OpenAPISpec, manualMappings *ManualMappings) map[string]*ResourceInfo { + resources := make(map[string]*ResourceInfo) + + // First pass: identify all entity schemas + entitySchemas := identifyEntitySchemas(spec.Components.Schemas) + fmt.Printf("Identified %d entity schemas\n", len(entitySchemas)) + + // Second pass: match operations to entities + for path, pathItem := range spec.Paths { + analyzePathOperations(path, pathItem, entitySchemas, resources, spec, manualMappings) + } + + // Third pass: validate resources but keep all for analysis + fmt.Printf("\n=== Resource Validation ===\n") + for name, resource := range resources { + opTypes := make([]string, 0) + for crudType := range resource.Operations { + opTypes = append(opTypes, crudType) + } + fmt.Printf("Resource: %s with operations: %v\n", name, opTypes) + + if isTerraformViable(resource, spec) { + fmt.Printf(" ✅ Viable for Terraform\n") + } else { + fmt.Printf(" ❌ Not viable for Terraform - will skip annotations\n") + } + } + + return resources +} + +func identifyEntitySchemas(schemas map[string]Schema) map[string]bool { + entities := make(map[string]bool) + + for name, schema := range schemas { + if isEntitySchema(name, schema) { + entities[name] = true + } + } + + return entities +} + +func isEntitySchema(name string, schema Schema) bool { + // Skip request/response wrappers + lowerName := strings.ToLower(name) + if strings.HasPrefix(lowerName, "create_") || + strings.HasPrefix(lowerName, "update_") || + strings.HasPrefix(lowerName, "delete_") || + strings.Contains(lowerName, "request") || + strings.Contains(lowerName, "response") || + strings.HasSuffix(name, "Paginated") { + return false + } + + // Skip nullable wrapper schemas + if strings.HasPrefix(name, "Nullable") { + return false + } + + // Must be an object with properties + if schema.Type != "object" || len(schema.Properties) == 0 { + return false + } + + // Entities should have an id property and end with "Entity" + _, hasID := schema.Properties["id"] + _, hasSlug := schema.Properties["slug"] + hasSuffix := strings.HasSuffix(name, "Entity") + + hasIdentifier := hasID || hasSlug + // Be strict: require both conditions + return hasIdentifier && hasSuffix +} + +func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[string]bool, + resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) { + + operations := []struct { + method string + op *Operation + }{ + {"get", pathItem.Get}, + {"post", pathItem.Post}, + {"put", pathItem.Put}, + {"patch", pathItem.Patch}, + {"delete", pathItem.Delete}, + } + + for _, item := range operations { + if item.op == nil { + continue + } + + // Check if this operation should be manually ignored + if shouldIgnoreOperation(path, item.method, manualMappings) { + continue + } + + resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec, manualMappings) + if resourceInfo != nil { + if existing, exists := resources[resourceInfo.ResourceName]; exists { + // Merge operations + for opType, opInfo := range resourceInfo.Operations { + existing.Operations[opType] = opInfo + } + + // Preserve create/update schema info - don't overwrite with empty values + if resourceInfo.CreateSchema != "" { + existing.CreateSchema = resourceInfo.CreateSchema + } + if resourceInfo.UpdateSchema != "" { + existing.UpdateSchema = resourceInfo.UpdateSchema + } + } else { + resources[resourceInfo.ResourceName] = resourceInfo + } + } + } +} + +func extractResourceInfo(path, method string, op *Operation, + entitySchemas map[string]bool, spec OpenAPISpec, manualMappings *ManualMappings) *ResourceInfo { + + // Determine CRUD type + crudType := determineCrudType(path, method, op.OperationID) + if crudType == "" { + return nil + } + + // Check for manual entity mapping first + if manualEntityName, hasManual := getManualEntityMapping(path, method, manualMappings); hasManual { + // Use manual entity mapping + entityName := manualEntityName + resourceName := deriveResourceName(entityName, op.OperationID, path) + + info := &ResourceInfo{ + EntityName: entityName, + SchemaName: entityName, + ResourceName: resourceName, + Operations: make(map[string]OperationInfo), + } + + opInfo := OperationInfo{ + OperationID: op.OperationID, + Path: path, + Method: method, + } + + // Extract request schema for create/update operations + if crudType == "create" || crudType == "update" { + if op.RequestBody != nil { + if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + if ref, ok := schema["$ref"].(string); ok { + requestSchemaName := extractSchemaName(ref) + opInfo.RequestSchema = requestSchemaName + + if crudType == "create" { + info.CreateSchema = requestSchemaName + } else if crudType == "update" { + info.UpdateSchema = requestSchemaName + } + } + } + } + } + } + } + + info.Operations[crudType] = opInfo + return info + } + + // Find associated entity schema using automatic detection + entityName := findEntityFromOperation(op, entitySchemas, spec) + if entityName == "" { + return nil + } + + resourceName := deriveResourceName(entityName, op.OperationID, path) + + info := &ResourceInfo{ + EntityName: entityName, + SchemaName: entityName, + ResourceName: resourceName, + Operations: make(map[string]OperationInfo), + } + + opInfo := OperationInfo{ + OperationID: op.OperationID, + Path: path, + Method: method, + } + + // Extract request schema for create/update operations + if crudType == "create" || crudType == "update" { + if op.RequestBody != nil { + if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + if ref, ok := schema["$ref"].(string); ok { + requestSchemaName := extractSchemaName(ref) + opInfo.RequestSchema = requestSchemaName + + if crudType == "create" { + info.CreateSchema = requestSchemaName + } else if crudType == "update" { + info.UpdateSchema = requestSchemaName + } + } + } + } + } + } + } + + info.Operations[crudType] = opInfo + return info +} + +func determineCrudType(path, method, operationID string) string { + lowerOp := strings.ToLower(operationID) + + // Check operation ID first + if strings.Contains(lowerOp, "create") { + return "create" + } + if strings.Contains(lowerOp, "update") || strings.Contains(lowerOp, "patch") { + return "update" + } + if strings.Contains(lowerOp, "delete") { + return "delete" + } + if strings.Contains(lowerOp, "list") { + return "list" + } + if strings.Contains(lowerOp, "get") && strings.Contains(path, "{") { + return "read" + } + + // Fallback to method-based detection + switch method { + case "post": + if !strings.Contains(path, "{") { + return "create" + } + case "get": + if strings.Contains(path, "{") { + return "read" + } else { + return "list" + } + case "patch", "put": + return "update" + case "delete": + return "delete" + } + + return "" +} + +func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec OpenAPISpec) string { + // Check response schemas first + if op.Responses != nil { + for _, response := range op.Responses { + if respMap, ok := response.(map[string]interface{}); ok { + if content, ok := respMap["content"].(map[string]interface{}); ok { + if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { + if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { + entityName := findEntityInSchema(schema, entitySchemas) + if entityName != "" { + return entityName + } + } + } + } + } + } + } + + // Check tags + if len(op.Tags) > 0 { + for _, tag := range op.Tags { + possibleEntity := tag + "Entity" + if entitySchemas[possibleEntity] { + return possibleEntity + } + } + } + + return "" +} + +func findEntityInSchema(schema map[string]interface{}, entitySchemas map[string]bool) string { + // Direct reference + if ref, ok := schema["$ref"].(string); ok { + schemaName := extractSchemaName(ref) + if entitySchemas[schemaName] { + return schemaName + } + } + + // Check in data array for paginated responses + if props, ok := schema["properties"].(map[string]interface{}); ok { + if data, ok := props["data"].(map[string]interface{}); ok { + if dataType, ok := data["type"].(string); ok && dataType == "array" { + if items, ok := data["items"].(map[string]interface{}); ok { + if ref, ok := items["$ref"].(string); ok { + schemaName := extractSchemaName(ref) + if entitySchemas[schemaName] { + return schemaName + } + } + } + } + } + } + + return "" +} + +func extractSchemaName(ref string) string { + parts := strings.Split(ref, "/") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func deriveResourceName(entityName, operationID, path string) string { + resource := strings.TrimSuffix(entityName, "Entity") + + resource = toSnakeCase(resource) + + if strings.Contains(resource, "_") { + parts := strings.Split(resource, "_") + if len(parts) > 1 && parts[0] == parts[1] { + // Remove duplicate prefix (e.g., incidents_incident -> incident) + resource = parts[1] + } + } + + return resource +} + +func toSnakeCase(s string) string { + var result []rune + for i, r := range s { + if i > 0 && unicode.IsUpper(r) { + if i == len(s)-1 || !unicode.IsUpper(rune(s[i+1])) { + result = append(result, '_') + } + } + result = append(result, []rune(strings.ToLower(string(r)))...) + } + return string(result) +} + +func extractPathParameters(path string) []string { + re := regexp.MustCompile(`\{([^}]+)\}`) + matches := re.FindAllStringSubmatch(path, -1) + + var params []string + for _, match := range matches { + if len(match) > 1 { + params = append(params, match[1]) + } + } + + return params +} diff --git a/scripts/overlay/detect.go b/scripts/overlay/detect.go new file mode 100644 index 0000000..8432e23 --- /dev/null +++ b/scripts/overlay/detect.go @@ -0,0 +1,384 @@ +package main + +import ( + "encoding/json" + "fmt" +) + +type PropertyMismatch struct { + PropertyName string + MismatchType string + Description string +} + +type CRUDInconsistency struct { + PropertyName string + InconsistencyType string + Description string + SchemasToIgnore []string +} + +func detectPropertyMismatches(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]PropertyMismatch { + mismatches := make(map[string][]PropertyMismatch) + + specData, err := json.Marshal(spec) + if err != nil { + return mismatches + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return mismatches + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + for _, resource := range resources { + var resourceMismatches []PropertyMismatch + + if resource.CreateSchema != "" { + if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { + if requestSchema, exists := schemas[resource.CreateSchema].(map[string]interface{}); exists { + createMismatches := findPropertyMismatches(entitySchema, requestSchema, "create") + resourceMismatches = append(resourceMismatches, createMismatches...) + } + } + } + + // Check update operation mismatches + if resource.UpdateSchema != "" { + if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { + if requestSchema, exists := schemas[resource.UpdateSchema].(map[string]interface{}); exists { + updateMismatches := findPropertyMismatches(entitySchema, requestSchema, "update") + resourceMismatches = append(resourceMismatches, updateMismatches...) + } + } + } + + if len(resourceMismatches) > 0 { + mismatches[resource.EntityName] = resourceMismatches + } + } + + return mismatches +} + +func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, operation string) []PropertyMismatch { + var mismatches []PropertyMismatch + + entityProps, _ := entitySchema["properties"].(map[string]interface{}) + requestProps, _ := requestSchema["properties"].(map[string]interface{}) + + if entityProps == nil || requestProps == nil { + return mismatches + } + + for propName, entityProp := range entityProps { + if requestProp, exists := requestProps[propName]; exists { + if hasStructuralMismatch(entityProp, requestProp) { + mismatch := PropertyMismatch{ + PropertyName: propName, + MismatchType: "structural-mismatch", + Description: describeStructuralDifference(entityProp, requestProp), + } + mismatches = append(mismatches, mismatch) + } + } + } + + return mismatches +} + +// Check if two property structures are different +func hasStructuralMismatch(entityProp, requestProp interface{}) bool { + entityStructure := getPropertyStructure(entityProp) + requestStructure := getPropertyStructure(requestProp) + return entityStructure != requestStructure +} + +// Get a normalized string representation of a property's structure +func getPropertyStructure(prop interface{}) string { + propMap, ok := prop.(map[string]interface{}) + if !ok { + return "unknown" + } + + if ref, hasRef := propMap["$ref"].(string); hasRef { + return fmt.Sprintf("$ref:%s", ref) + } + + propType, _ := propMap["type"].(string) + + switch propType { + case "array": + items, hasItems := propMap["items"] + if hasItems { + itemStructure := getPropertyStructure(items) + return fmt.Sprintf("array[%s]", itemStructure) + } + return "array[unknown]" + + case "object": + properties, hasProps := propMap["properties"] + _, hasAdditional := propMap["additionalProperties"] + + if hasProps { + propsMap, _ := properties.(map[string]interface{}) + if len(propsMap) == 0 { + return "object{empty}" + } + return "object{defined}" + } + + if hasAdditional { + return "object{additional}" + } + + return "object{}" + + case "string", "integer", "number", "boolean": + return propType + + default: + if propType == "" { + if _, hasProps := propMap["properties"]; hasProps { + return "implicit-object" + } + if _, hasItems := propMap["items"]; hasItems { + return "implicit-array" + } + } + return fmt.Sprintf("type:%s", propType) + } +} + +// Describe the structural difference for reporting +func describeStructuralDifference(entityProp, requestProp interface{}) string { + entityStructure := getPropertyStructure(entityProp) + requestStructure := getPropertyStructure(requestProp) + return fmt.Sprintf("request structure '%s' != response structure '%s'", requestStructure, entityStructure) +} + +// Detect schema property inconsistencies (extracted from detectCRUDInconsistencies) +func detectCRUDInconsistencies(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]CRUDInconsistency { + inconsistencies := make(map[string][]CRUDInconsistency) + + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return inconsistencies + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return inconsistencies + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + for _, resource := range resources { + resourceInconsistencies := detectSchemaPropertyInconsistencies(resource, schemas) + + // Check if we have fundamental validation errors that make the resource non-viable + for _, inconsistency := range resourceInconsistencies { + if inconsistency.PropertyName == "RESOURCE_VALIDATION" { + fmt.Printf("⚠️ Resource %s (%s) validation failed: %s\n", + resource.ResourceName, resource.EntityName, inconsistency.Description) + // Mark the entire resource as having issues but don't add to inconsistencies + // as this will be handled in the viability check + continue + } + } + + // Only add property-level inconsistencies for viable resources + var validInconsistencies []CRUDInconsistency + for _, inconsistency := range resourceInconsistencies { + if inconsistency.PropertyName != "RESOURCE_VALIDATION" { + validInconsistencies = append(validInconsistencies, inconsistency) + } + } + + if len(validInconsistencies) > 0 { + inconsistencies[resource.EntityName] = validInconsistencies + } + } + + return inconsistencies +} + +// Detect schema property inconsistencies (simplified CRUD detection) +func detectSchemaPropertyInconsistencies(resource *ResourceInfo, schemas map[string]interface{}) []CRUDInconsistency { + var inconsistencies []CRUDInconsistency + + // First, validate that we have the minimum required operations for Terraform + _, hasCreate := resource.Operations["create"] + _, hasPut := resource.Operations["put"] + _, hasRead := resource.Operations["read"] + + createOrPut := hasCreate || hasPut + if !createOrPut || !hasRead { + // Return a fundamental inconsistency - resource is not viable for Terraform + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "missing-required-operations", + Description: fmt.Sprintf("Resource missing required operations: Create=%v, Read=%v", hasCreate, hasRead), + SchemasToIgnore: []string{}, // Don't ignore anything, this makes the whole resource invalid since Terraform needs a create and a read, at minimum + } + return []CRUDInconsistency{inconsistency} + } + + // Validate that we have a create schema + if resource.CreateSchema == "" { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "missing-create-schema", + Description: "Resource has CREATE operation but no request schema defined", + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Get properties from each schema + entityProps := getSchemaProperties(schemas, resource.EntityName) + createProps := getSchemaProperties(schemas, resource.CreateSchema) + updateProps := map[string]interface{}{} + + if resource.UpdateSchema != "" { + updateProps = getSchemaProperties(schemas, resource.UpdateSchema) + } + + // Validate that schemas exist and have properties + if len(entityProps) == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "invalid-entity-schema", + Description: fmt.Sprintf("Entity schema '%s' not found or has no properties", resource.EntityName), + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + if len(createProps) == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "invalid-create-schema", + Description: fmt.Sprintf("Create schema '%s' not found or has no properties", resource.CreateSchema), + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Check for minimum viable overlap between create and entity schemas + commonManageableProps := 0 + createManageableProps := 0 + + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + if entityProps[prop] != nil { + commonManageableProps++ + } + } + } + + if createManageableProps == 0 { + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "no-manageable-properties", + Description: "Create schema has no manageable properties (all are system properties)", + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Require reasonable overlap between create and entity schemas + overlapRatio := float64(commonManageableProps) / float64(createManageableProps) + if overlapRatio < 0.3 { // At least 30% overlap required + inconsistency := CRUDInconsistency{ + PropertyName: "RESOURCE_VALIDATION", + InconsistencyType: "insufficient-schema-overlap", + Description: fmt.Sprintf("Insufficient overlap between create and entity schemas: %.1f%% (%d/%d properties)", overlapRatio*100, commonManageableProps, createManageableProps), + SchemasToIgnore: []string{}, + } + return []CRUDInconsistency{inconsistency} + } + + // Now check individual property inconsistencies for viable resources + // Collect all property names across CRUD operations + allProps := make(map[string]bool) + for prop := range entityProps { + allProps[prop] = true + } + for prop := range createProps { + allProps[prop] = true + } + for prop := range updateProps { + allProps[prop] = true + } + + // Check each property for consistency across CRUD operations + for propName := range allProps { + // Skip ID properties - they have separate handling logic + if propName == "id" { + continue + } + + entityHas := entityProps[propName] != nil + createHas := createProps[propName] != nil + updateHas := updateProps[propName] != nil + + // Check for CRUD inconsistencies + var schemasToIgnore []string + var inconsistencyType string + var description string + hasInconsistency := false + + if resource.CreateSchema != "" && resource.UpdateSchema != "" { + // Full CRUD resource - all three must be consistent + if !(entityHas && createHas && updateHas) { + hasInconsistency = true + inconsistencyType = "crud-property-mismatch" + description = fmt.Sprintf("Property not present in all CRUD operations (Entity:%v, Create:%v, Update:%v)", entityHas, createHas, updateHas) + + // Ignore in schemas where property exists but shouldn't for consistency + if entityHas && (!createHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) + } + if createHas && (!entityHas || !updateHas) { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) + } + if updateHas && (!entityHas || !createHas) { + schemasToIgnore = append(schemasToIgnore, resource.UpdateSchema) + } + } + } else if resource.CreateSchema != "" { + // Create + Read resource - both must be consistent + if !(entityHas && createHas) { + hasInconsistency = true + inconsistencyType = "create-read-mismatch" + description = fmt.Sprintf("Property not present in both CREATE and READ (Entity:%v, Create:%v)", entityHas, createHas) + + if entityHas && !createHas { + schemasToIgnore = append(schemasToIgnore, resource.EntityName) + } + if createHas && !entityHas { + schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) + } + } + } + + if hasInconsistency { + inconsistency := CRUDInconsistency{ + PropertyName: propName, + InconsistencyType: inconsistencyType, + Description: description, + SchemasToIgnore: schemasToIgnore, + } + inconsistencies = append(inconsistencies, inconsistency) + } + } + + return inconsistencies +} diff --git a/scripts/overlay/generate-overlay.go b/scripts/overlay/generate-overlay.go new file mode 100644 index 0000000..e511685 --- /dev/null +++ b/scripts/overlay/generate-overlay.go @@ -0,0 +1,189 @@ +package main + +import "fmt" + +type OverlayAction struct { + Target string `yaml:"target"` + Update map[string]interface{} `yaml:"update,omitempty"` +} + +type Overlay struct { + Overlay string `yaml:"overlay"` + Info struct { + Title string `yaml:"title"` + Version string `yaml:"version"` + Description string `yaml:"description"` + } `yaml:"info"` + Actions []OverlayAction `yaml:"actions"` +} + +func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) *Overlay { + overlay := &Overlay{ + Overlay: "1.0.0", + } + + overlay.Info.Title = "Terraform Provider Overlay" + overlay.Info.Version = "1.0.0" + overlay.Info.Description = "Auto-generated overlay for Terraform resources" + + // Clean up resources by removing manually ignored operations + mappedResources := applyManualMappings(resources, manualMappings) + + // Separate viable and non-viable resources + viableResources := make(map[string]*ResourceInfo) + skippedResources := make([]string, 0) + + for name, resource := range mappedResources { + if isTerraformViable(resource, spec) { + viableResources[name] = resource + } else { + skippedResources = append(skippedResources, name) + } + } + + fmt.Printf("\n=== Overlay Generation Analysis ===\n") + fmt.Printf("Resources after Manual Mapping: %d\n", len(mappedResources)) + fmt.Printf("Viable for Terraform: %d\n", len(viableResources)) + fmt.Printf("Skipped (non-viable): %d\n", len(skippedResources)) + + if len(skippedResources) > 0 { + fmt.Printf("\nSkipped resources:\n") + for _, skipped := range skippedResources { + fmt.Printf(" - %s\n", skipped) + } + } + + // Filter operations with unmappable path parameters + fmt.Printf("\n=== Operation-Level Filtering ===\n") + + // Update description with actual count + overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(viableResources)) + + // Detect property mismatches for filtered resources only + resourceMismatches := detectPropertyMismatches(viableResources, spec) + + // Detect CRUD inconsistencies for filtered resources only + resourceCRUDInconsistencies := detectCRUDInconsistencies(viableResources, spec) + + // Track which properties already have ignore actions to avoid duplicates + ignoreTracker := make(map[string]map[string]bool) // map[schemaName][propertyName]bool + + // Generate actions only for filtered resources + for _, resource := range viableResources { + // Mark the response entity schema + entityUpdate := map[string]interface{}{ + "x-speakeasy-entity": resource.EntityName, + } + + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s", resource.SchemaName), + Update: entityUpdate, + }) + + // Initialize ignore tracker for this resource's schemas + if ignoreTracker[resource.EntityName] == nil { + ignoreTracker[resource.EntityName] = make(map[string]bool) + } + if resource.CreateSchema != "" && ignoreTracker[resource.CreateSchema] == nil { + ignoreTracker[resource.CreateSchema] = make(map[string]bool) + } + if resource.UpdateSchema != "" && ignoreTracker[resource.UpdateSchema] == nil { + ignoreTracker[resource.UpdateSchema] = make(map[string]bool) + } + + // Add speakeasy ignore for property mismatches + if mismatches, exists := resourceMismatches[resource.EntityName]; exists { + addIgnoreActionsForMismatches(overlay, resource, mismatches, ignoreTracker) + } + + // Add speakeasy ignore for CRUD inconsistencies + if inconsistencies, exists := resourceCRUDInconsistencies[resource.EntityName]; exists { + addIgnoreActionsForInconsistencies(overlay, resource, inconsistencies, ignoreTracker) + } + + // Add entity operations and parameter matching + for crudType, opInfo := range resource.Operations { + // Double-check that this specific operation isn't in the ignore list + if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { + fmt.Printf(" Skipping ignored operation during overlay generation: %s %s\n", opInfo.Method, opInfo.Path) + continue + } + + entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) + + operationUpdate := map[string]interface{}{ + "x-speakeasy-entity-operation": entityOp, + } + + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s", opInfo.Path, opInfo.Method), + Update: operationUpdate, + }) + + // Apply parameter matching for operations that use the primary ID + if resource.PrimaryID != "" && (crudType == "read" || crudType == "update" || crudType == "delete") { + pathParams := extractPathParameters(opInfo.Path) + for _, param := range pathParams { + if param == resource.PrimaryID { + // Check for manual parameter mapping first + if manualMatch, hasManual := getManualParameterMatch(opInfo.Path, opInfo.Method, param, manualMappings); hasManual { + // Only apply manual match if it's different from the parameter name + if manualMatch != param { + fmt.Printf(" Manual parameter mapping: %s in %s %s -> %s\n", param, opInfo.Method, opInfo.Path, manualMatch) + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + opInfo.Path, opInfo.Method, param), + Update: map[string]interface{}{ + "x-speakeasy-match": manualMatch, + }, + }) + } else { + fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s (would create circular reference)\n", param, manualMatch) + } + } else { + // Skip x-speakeasy-match when parameter name would map to itself + // This prevents circular references like {id} -> id + if param == "id" { + fmt.Printf(" Skipping x-speakeasy-match: parameter %s maps to same field (avoiding circular reference)\n", param) + } else { + // Apply x-speakeasy-match for parameters that need mapping (e.g., change_event_id -> id) + fmt.Printf(" Applying x-speakeasy-match to %s in %s %s -> id\n", param, opInfo.Method, opInfo.Path) + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", + opInfo.Path, opInfo.Method, param), + Update: map[string]interface{}{ + "x-speakeasy-match": "id", + }, + }) + } + } + } + } + } + } + } + + fmt.Printf("\n=== Overlay Generation Complete ===\n") + fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(viableResources)) + + // Count ignore actions and match actions + totalIgnores := 0 + totalMatches := 0 + for _, action := range overlay.Actions { + if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { + totalIgnores++ + } + if _, hasMatch := action.Update["x-speakeasy-match"]; hasMatch { + totalMatches++ + } + } + + if totalIgnores > 0 { + fmt.Printf("✅ %d speakeasy ignore actions added for property issues\n", totalIgnores) + } + if totalMatches > 0 { + fmt.Printf("✅ %d speakeasy match actions added for primary ID parameters\n", totalMatches) + } + + return overlay +} diff --git a/scripts/overlay/generate-terraform-overlay.go b/scripts/overlay/generate-terraform-overlay.go deleted file mode 100644 index bc4d7af..0000000 --- a/scripts/overlay/generate-terraform-overlay.go +++ /dev/null @@ -1,1799 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "regexp" - "strings" - - "gopkg.in/yaml.v3" -) - -// Manual mapping configuration -type ManualMapping struct { - Path string `yaml:"path"` - Method string `yaml:"method"` - Action string `yaml:"action"` // "ignore", "entity", "match" - Value string `yaml:"value,omitempty"` -} - -type ManualMappings struct { - Operations []ManualMapping `yaml:"operations"` -} - -type OpenAPISpec struct { - OpenAPI string `json:"openapi"` - Info map[string]interface{} `json:"info"` - Paths map[string]PathItem `json:"paths"` - Components Components `json:"components"` -} - -type Components struct { - Schemas map[string]Schema `json:"schemas"` - SecuritySchemes map[string]interface{} `json:"securitySchemes,omitempty"` -} - -type Schema struct { - Type string `json:"type,omitempty"` - Properties map[string]interface{} `json:"properties,omitempty"` - Required []string `json:"required,omitempty"` - AllOf []interface{} `json:"allOf,omitempty"` - Nullable bool `json:"nullable,omitempty"` - Items interface{} `json:"items,omitempty"` - Raw map[string]interface{} `json:"-"` -} - -type PathItem struct { - Get *Operation `json:"get,omitempty"` - Post *Operation `json:"post,omitempty"` - Put *Operation `json:"put,omitempty"` - Patch *Operation `json:"patch,omitempty"` - Delete *Operation `json:"delete,omitempty"` -} - -type Operation struct { - OperationID string `json:"operationId,omitempty"` - Tags []string `json:"tags,omitempty"` - Parameters []Parameter `json:"parameters,omitempty"` - RequestBody map[string]interface{} `json:"requestBody,omitempty"` - Responses map[string]interface{} `json:"responses,omitempty"` -} - -type Parameter struct { - Name string `json:"name"` - In string `json:"in"` - Required bool `json:"required,omitempty"` - Schema Schema `json:"schema,omitempty"` -} - -type ResourceInfo struct { - EntityName string - SchemaName string - ResourceName string - Operations map[string]OperationInfo - CreateSchema string - UpdateSchema string - PrimaryID string // Store the identified primary ID parameter -} - -type OperationInfo struct { - OperationID string - Path string - Method string - RequestSchema string -} - -type PropertyMismatch struct { - PropertyName string - MismatchType string - Description string -} - -type CRUDInconsistency struct { - PropertyName string - InconsistencyType string - Description string - SchemasToIgnore []string -} - -type OverlayAction struct { - Target string `yaml:"target"` - Update map[string]interface{} `yaml:"update,omitempty"` -} - -type Overlay struct { - Overlay string `yaml:"overlay"` - Info struct { - Title string `yaml:"title"` - Version string `yaml:"version"` - Description string `yaml:"description"` - } `yaml:"info"` - Actions []OverlayAction `yaml:"actions"` -} - -// IDPattern represents the ID parameter pattern for an operation -type IDPattern struct { - Path string - Method string - Operation string - Parameters []string -} - -// PathParameterInconsistency represents inconsistencies in path parameters -type PathParameterInconsistency struct { - InconsistencyType string - Description string - Operations []string -} - -func main() { - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - specPath := os.Args[1] - var mappingsPath string - if len(os.Args) > 2 { - mappingsPath = os.Args[2] - } else { - mappingsPath = "manual-mappings.yaml" - } - - fmt.Printf("=== Terraform Overlay Generator ===\n") - fmt.Printf("Input: %s\n", specPath) - - // Load manual mappings - manualMappings := loadManualMappings(mappingsPath) - - specData, err := ioutil.ReadFile(specPath) - if err != nil { - fmt.Printf("Error reading spec file: %v\n", err) - os.Exit(1) - } - - var spec OpenAPISpec - if err := json.Unmarshal(specData, &spec); err != nil { - fmt.Printf("Error parsing JSON: %v\n", err) - os.Exit(1) - } - - fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) - - resources := analyzeSpec(spec, manualMappings) - - overlay := generateOverlay(resources, spec, manualMappings) - - if err := writeOverlay(overlay); err != nil { - fmt.Printf("Error writing overlay: %v\n", err) - os.Exit(1) - } - - printOverlaySummary(resources, overlay) -} - -func printUsage() { - fmt.Println("OpenAPI Terraform Overlay Generator") - fmt.Println() - fmt.Println("Usage:") - fmt.Println(" openapi-overlay ") -} - -// Load manual mappings from file -func loadManualMappings(mappingsPath string) *ManualMappings { - data, err := ioutil.ReadFile(mappingsPath) - if err != nil { - // File doesn't exist - return empty mappings - fmt.Printf("No manual mappings file found at %s (this is optional)\n", mappingsPath) - return &ManualMappings{} - } - - var mappings ManualMappings - if err := yaml.Unmarshal(data, &mappings); err != nil { - fmt.Printf("Error parsing manual mappings file: %v\n", err) - return &ManualMappings{} - } - - fmt.Printf("Loaded %d manual mappings from %s\n", len(mappings.Operations), mappingsPath) - return &mappings -} - -// Check if an operation should be manually ignored -func shouldIgnoreOperation(path, method string, manualMappings *ManualMappings) bool { - for _, mapping := range manualMappings.Operations { - if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "ignore" { - fmt.Printf(" Manual mapping: Ignoring operation %s %s\n", method, path) - return true - } - } - return false -} - -// Check if an operation has a manual entity mapping -func getManualEntityMapping(path, method string, manualMappings *ManualMappings) (string, bool) { - for _, mapping := range manualMappings.Operations { - if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "entity" { - fmt.Printf(" Manual mapping: Operation %s %s -> Entity %s\n", method, path, mapping.Value) - return mapping.Value, true - } - } - return "", false -} - -// Check if a parameter has a manual match mapping -func getManualParameterMatch(path, method, paramName string, manualMappings *ManualMappings) (string, bool) { - for _, mapping := range manualMappings.Operations { - if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "match" { - // For match mappings, we expect the value to be in format "param_name:field_name" - parts := strings.SplitN(mapping.Value, ":", 2) - if len(parts) == 2 && parts[0] == paramName { - fmt.Printf(" Manual mapping: Parameter %s in %s %s -> %s\n", paramName, method, path, parts[1]) - return parts[1], true - } - } - } - return "", false -} - -// UnmarshalJSON custom unmarshaler for Schema to handle both structured and raw data -func (s *Schema) UnmarshalJSON(data []byte) error { - type Alias Schema - aux := &struct { - *Alias - }{ - Alias: (*Alias)(s), - } - - if err := json.Unmarshal(data, &aux); err != nil { - return err - } - - // Also unmarshal into raw map - if err := json.Unmarshal(data, &s.Raw); err != nil { - return err - } - - return nil -} - -func analyzeSpec(spec OpenAPISpec, manualMappings *ManualMappings) map[string]*ResourceInfo { - resources := make(map[string]*ResourceInfo) - - // First pass: identify all entity schemas - entitySchemas := identifyEntitySchemas(spec.Components.Schemas) - fmt.Printf("Identified %d entity schemas\n", len(entitySchemas)) - - // Second pass: match operations to entities - for path, pathItem := range spec.Paths { - analyzePathOperations(path, pathItem, entitySchemas, resources, spec, manualMappings) - } - - // Third pass: validate resources but keep all for analysis - fmt.Printf("\n=== Resource Validation ===\n") - for name, resource := range resources { - opTypes := make([]string, 0) - for crudType := range resource.Operations { - opTypes = append(opTypes, crudType) - } - fmt.Printf("Resource: %s with operations: %v\n", name, opTypes) - - if isTerraformViable(resource, spec) { - fmt.Printf(" ✅ Viable for Terraform\n") - } else { - fmt.Printf(" ❌ Not viable for Terraform - will skip annotations\n") - } - } - - return resources -} - -func identifyEntitySchemas(schemas map[string]Schema) map[string]bool { - entities := make(map[string]bool) - - for name, schema := range schemas { - if isEntitySchema(name, schema) { - entities[name] = true - } - } - - return entities -} - -func isEntitySchema(name string, schema Schema) bool { - // Skip request/response wrappers - lowerName := strings.ToLower(name) - if strings.HasPrefix(lowerName, "create_") || - strings.HasPrefix(lowerName, "update_") || - strings.HasPrefix(lowerName, "delete_") || - strings.Contains(lowerName, "request") || - strings.Contains(lowerName, "response") || - strings.HasSuffix(name, "Paginated") { - return false - } - - // Skip nullable wrapper schemas - if strings.HasPrefix(name, "Nullable") { - return false - } - - // Must be an object with properties - if schema.Type != "object" || len(schema.Properties) == 0 { - return false - } - - // Entities should have an id property and end with "Entity" - _, hasID := schema.Properties["id"] - hasSuffix := strings.HasSuffix(name, "Entity") - - // Be strict: require both conditions - return hasID && hasSuffix -} - -func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[string]bool, - resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) { - - operations := []struct { - method string - op *Operation - }{ - {"get", pathItem.Get}, - {"post", pathItem.Post}, - {"put", pathItem.Put}, - {"patch", pathItem.Patch}, - {"delete", pathItem.Delete}, - } - - for _, item := range operations { - if item.op == nil { - continue - } - - // Check if this operation should be manually ignored - if shouldIgnoreOperation(path, item.method, manualMappings) { - continue - } - - resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec, manualMappings) - if resourceInfo != nil { - if existing, exists := resources[resourceInfo.ResourceName]; exists { - // Merge operations - for opType, opInfo := range resourceInfo.Operations { - existing.Operations[opType] = opInfo - } - - // Preserve create/update schema info - don't overwrite with empty values - if resourceInfo.CreateSchema != "" { - existing.CreateSchema = resourceInfo.CreateSchema - } - if resourceInfo.UpdateSchema != "" { - existing.UpdateSchema = resourceInfo.UpdateSchema - } - } else { - resources[resourceInfo.ResourceName] = resourceInfo - } - } - } -} - -func extractResourceInfo(path, method string, op *Operation, - entitySchemas map[string]bool, spec OpenAPISpec, manualMappings *ManualMappings) *ResourceInfo { - - // Determine CRUD type - crudType := determineCrudType(path, method, op.OperationID) - if crudType == "" { - return nil - } - - // Check for manual entity mapping first - if manualEntityName, hasManual := getManualEntityMapping(path, method, manualMappings); hasManual { - // Use manual entity mapping - entityName := manualEntityName - resourceName := deriveResourceName(entityName, op.OperationID, path) - - info := &ResourceInfo{ - EntityName: entityName, - SchemaName: entityName, - ResourceName: resourceName, - Operations: make(map[string]OperationInfo), - } - - opInfo := OperationInfo{ - OperationID: op.OperationID, - Path: path, - Method: method, - } - - // Extract request schema for create/update operations - if crudType == "create" || crudType == "update" { - if op.RequestBody != nil { - if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { - if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { - if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { - if ref, ok := schema["$ref"].(string); ok { - requestSchemaName := extractSchemaName(ref) - opInfo.RequestSchema = requestSchemaName - - if crudType == "create" { - info.CreateSchema = requestSchemaName - } else if crudType == "update" { - info.UpdateSchema = requestSchemaName - } - } - } - } - } - } - } - - info.Operations[crudType] = opInfo - return info - } - - // Find associated entity schema using automatic detection - entityName := findEntityFromOperation(op, entitySchemas, spec) - if entityName == "" { - return nil - } - - resourceName := deriveResourceName(entityName, op.OperationID, path) - - info := &ResourceInfo{ - EntityName: entityName, - SchemaName: entityName, - ResourceName: resourceName, - Operations: make(map[string]OperationInfo), - } - - opInfo := OperationInfo{ - OperationID: op.OperationID, - Path: path, - Method: method, - } - - // Extract request schema for create/update operations - if crudType == "create" || crudType == "update" { - if op.RequestBody != nil { - if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { - if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { - if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { - if ref, ok := schema["$ref"].(string); ok { - requestSchemaName := extractSchemaName(ref) - opInfo.RequestSchema = requestSchemaName - - if crudType == "create" { - info.CreateSchema = requestSchemaName - } else if crudType == "update" { - info.UpdateSchema = requestSchemaName - } - } - } - } - } - } - } - - info.Operations[crudType] = opInfo - return info -} - -func determineCrudType(path, method, operationID string) string { - lowerOp := strings.ToLower(operationID) - - // Check operation ID first - if strings.Contains(lowerOp, "create") { - return "create" - } - if strings.Contains(lowerOp, "update") || strings.Contains(lowerOp, "patch") { - return "update" - } - if strings.Contains(lowerOp, "delete") { - return "delete" - } - if strings.Contains(lowerOp, "list") { - return "list" - } - if strings.Contains(lowerOp, "get") && strings.Contains(path, "{") { - return "read" - } - - // Fallback to method-based detection - switch method { - case "post": - if !strings.Contains(path, "{") { - return "create" - } - case "get": - if strings.Contains(path, "{") { - return "read" - } else { - return "list" - } - case "patch", "put": - return "update" - case "delete": - return "delete" - } - - return "" -} - -func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec OpenAPISpec) string { - // Check response schemas first - if op.Responses != nil { - for _, response := range op.Responses { - if respMap, ok := response.(map[string]interface{}); ok { - if content, ok := respMap["content"].(map[string]interface{}); ok { - if jsonContent, ok := content["application/json"].(map[string]interface{}); ok { - if schema, ok := jsonContent["schema"].(map[string]interface{}); ok { - entityName := findEntityInSchema(schema, entitySchemas) - if entityName != "" { - return entityName - } - } - } - } - } - } - } - - // Check tags - if len(op.Tags) > 0 { - for _, tag := range op.Tags { - possibleEntity := tag + "Entity" - if entitySchemas[possibleEntity] { - return possibleEntity - } - } - } - - return "" -} - -func findEntityInSchema(schema map[string]interface{}, entitySchemas map[string]bool) string { - // Direct reference - if ref, ok := schema["$ref"].(string); ok { - schemaName := extractSchemaName(ref) - if entitySchemas[schemaName] { - return schemaName - } - } - - // Check in data array for paginated responses - if props, ok := schema["properties"].(map[string]interface{}); ok { - if data, ok := props["data"].(map[string]interface{}); ok { - if dataType, ok := data["type"].(string); ok && dataType == "array" { - if items, ok := data["items"].(map[string]interface{}); ok { - if ref, ok := items["$ref"].(string); ok { - schemaName := extractSchemaName(ref) - if entitySchemas[schemaName] { - return schemaName - } - } - } - } - } - } - - return "" -} - -func extractSchemaName(ref string) string { - parts := strings.Split(ref, "/") - if len(parts) > 0 { - return parts[len(parts)-1] - } - return "" -} - -func deriveResourceName(entityName, operationID, path string) string { - // Remove Entity suffix - resource := strings.TrimSuffix(entityName, "Entity") - - // Convert to snake_case - resource = toSnakeCase(resource) - - // Handle special cases - if strings.Contains(resource, "_") { - parts := strings.Split(resource, "_") - if len(parts) > 1 && parts[0] == parts[1] { - // Remove duplicate prefix (e.g., incidents_incident -> incident) - resource = parts[1] - } - } - - return resource -} - -func toSnakeCase(s string) string { - var result []rune - for i, r := range s { - if i > 0 && isUpper(r) { - if i == len(s)-1 || !isUpper(rune(s[i+1])) { - result = append(result, '_') - } - } - result = append(result, toLower(r)) - } - return string(result) -} - -func isUpper(r rune) bool { - return r >= 'A' && r <= 'Z' -} - -func toLower(r rune) rune { - if r >= 'A' && r <= 'Z' { - return r + 32 - } - return r -} - -// Extract path parameters in order from a path string -func extractPathParameters(path string) []string { - re := regexp.MustCompile(`\{([^}]+)\}`) - matches := re.FindAllStringSubmatch(path, -1) - - var params []string - for _, match := range matches { - if len(match) > 1 { - params = append(params, match[1]) - } - } - - return params -} - -// Get entity properties for field existence checking -func getEntityProperties(entityName string, spec OpenAPISpec) map[string]interface{} { - // Re-parse the spec to get raw schema data - specData, err := json.Marshal(spec) - if err != nil { - return map[string]interface{}{} - } - - var rawSpec map[string]interface{} - if err := json.Unmarshal(specData, &rawSpec); err != nil { - return map[string]interface{}{} - } - - components, _ := rawSpec["components"].(map[string]interface{}) - schemas, _ := components["schemas"].(map[string]interface{}) - - return getSchemaProperties(schemas, entityName) -} - -// Check if a field exists in the entity properties -func checkFieldExistsInEntity(paramName string, entityProps map[string]interface{}) bool { - // Direct field name match - if _, exists := entityProps[paramName]; exists { - return true - } - - // Check for common variations - variations := []string{ - paramName, - strings.TrimSuffix(paramName, "_id"), // Remove _id suffix - strings.TrimSuffix(paramName, "Id"), // Remove Id suffix - } - - for _, variation := range variations { - if _, exists := entityProps[variation]; exists { - return true - } - } - - return false -} - -// Check if a resource is viable for Terraform -func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { - // Must have at least create and read operations - _, hasCreate := resource.Operations["create"] - _, hasRead := resource.Operations["read"] - - if !hasCreate || !hasRead { - fmt.Printf(" Missing required operations: Create=%v, Read=%v\n", hasCreate, hasRead) - return false - } - - // Must have a create schema to be manageable by Terraform - if resource.CreateSchema == "" { - fmt.Printf(" No create schema found\n") - return false - } - - // Identify the primary ID for this entity - primaryID, validPrimaryID := identifyEntityPrimaryID(resource) - if !validPrimaryID { - fmt.Printf(" Cannot identify valid primary ID parameter\n") - return false - } - - // Validate all operations against the primary ID - validOperations := validateOperationParameters(resource, primaryID, spec) - - // Must still have CREATE and READ after validation - _, hasValidCreate := validOperations["create"] - _, hasValidRead := validOperations["read"] - - if !hasValidCreate || !hasValidRead { - fmt.Printf(" Lost required operations after parameter validation: Create=%v, Read=%v\n", hasValidCreate, hasValidRead) - return false - } - - // Update resource with only valid operations and primary ID - resource.Operations = validOperations - resource.PrimaryID = primaryID - - // Check for overlapping properties between create and entity schemas - if !hasValidCreateReadConsistency(resource, spec) { - fmt.Printf(" Create and Read operations have incompatible schemas\n") - return false - } - - // Check for problematic CRUD patterns that can't be handled by property ignoring - if resource.CreateSchema != "" && resource.UpdateSchema != "" { - // Re-parse the spec to get raw schema data for analysis - specData, err := json.Marshal(spec) - if err != nil { - return true // If we can't analyze, assume it's viable - } - - var rawSpec map[string]interface{} - if err := json.Unmarshal(specData, &rawSpec); err != nil { - return true // If we can't analyze, assume it's viable - } - - components, _ := rawSpec["components"].(map[string]interface{}) - schemas, _ := components["schemas"].(map[string]interface{}) - - createProps := getSchemaProperties(schemas, resource.CreateSchema) - updateProps := getSchemaProperties(schemas, resource.UpdateSchema) - - // Count manageable properties (non-system fields) - createManageableProps := 0 - updateManageableProps := 0 - commonManageableProps := 0 - - for prop := range createProps { - if !isSystemProperty(prop) { - createManageableProps++ - } - } - - for prop := range updateProps { - if !isSystemProperty(prop) { - updateManageableProps++ - // Check if this property also exists in create - if createProps[prop] != nil && !isSystemProperty(prop) { - commonManageableProps++ - } - } - } - - // Reject resources with fundamentally incompatible CRUD patterns - if createManageableProps <= 1 && updateManageableProps >= 3 && commonManageableProps == 0 { - fmt.Printf(" Incompatible CRUD pattern: Create=%d manageable, Update=%d manageable, Common=%d\n", - createManageableProps, updateManageableProps, commonManageableProps) - return false - } - } - - return true -} - -// Identify the primary ID parameter that belongs to this specific entity -func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { - // Get all unique path parameters across operations - allParams := make(map[string]bool) - - for crudType, opInfo := range resource.Operations { - if crudType == "create" || crudType == "list" { - continue // Skip operations that typically don't have entity-specific IDs - } - - pathParams := extractPathParameters(opInfo.Path) - for _, param := range pathParams { - allParams[param] = true - } - } - - if len(allParams) == 0 { - return "", false // No path parameters found - } - - // Find the parameter that matches this entity - var entityPrimaryID string - matchCount := 0 - - for param := range allParams { - if mapsToEntityID(param, resource.EntityName) { - entityPrimaryID = param - matchCount++ - } - } - - if matchCount == 0 { - // No parameter maps to this entity - check for generic 'id' parameter - if allParams["id"] { - fmt.Printf(" Using generic 'id' parameter for entity %s\n", resource.EntityName) - return "id", true - } - fmt.Printf(" No parameter maps to entity %s\n", resource.EntityName) - return "", false - } - - if matchCount > 1 { - // Multiple parameters claim to map to this entity - ambiguous - fmt.Printf(" Multiple parameters map to entity %s: ambiguous primary ID\n", resource.EntityName) - return "", false - } - - fmt.Printf(" Identified primary ID '%s' for entity %s\n", entityPrimaryID, resource.EntityName) - return entityPrimaryID, true -} - -// Check if a parameter name maps to a specific entity's ID field -func mapsToEntityID(paramName, entityName string) bool { - // Extract base name from entity (e.g., "ChangeEvent" from "ChangeEventEntity") - entityBase := strings.TrimSuffix(entityName, "Entity") - - // Convert to snake_case and add _id suffix - expectedParam := toSnakeCase(entityBase) + "_id" - - return strings.ToLower(paramName) == strings.ToLower(expectedParam) -} - -// Check if parameter looks like an entity ID -func isEntityID(paramName string) bool { - return strings.HasSuffix(strings.ToLower(paramName), "_id") || strings.ToLower(paramName) == "id" -} - -// Validate operations against the identified primary ID -func validateOperationParameters(resource *ResourceInfo, primaryID string, spec OpenAPISpec) map[string]OperationInfo { - validOperations := make(map[string]OperationInfo) - - // Get entity properties once for this resource - entityProps := getEntityProperties(resource.EntityName, spec) - - for crudType, opInfo := range resource.Operations { - pathParams := extractPathParameters(opInfo.Path) - - if crudType == "create" || crudType == "list" { - // These operations should not have the entity's primary ID in path - hasPrimaryID := false - for _, param := range pathParams { - if param == primaryID { - hasPrimaryID = true - break - } - } - - if hasPrimaryID { - fmt.Printf(" Skipping %s operation %s: unexpectedly has primary ID %s in path\n", - crudType, opInfo.Path, primaryID) - continue - } - - validOperations[crudType] = opInfo - continue - } - - // READ, UPDATE, DELETE should have exactly the primary ID - hasPrimaryID := false - hasConflictingEntityIDs := false - - for _, param := range pathParams { - if param == primaryID { - hasPrimaryID = true - } else if isEntityID(param) { - // This is another ID-like parameter - // Check if it maps to a field in the entity (not the primary id field) - if checkFieldExistsInEntity(param, entityProps) { - // This parameter maps to a real entity field - it's valid - fmt.Printf(" Parameter %s maps to entity field - keeping operation %s %s\n", - param, crudType, opInfo.Path) - } else { - // This ID parameter doesn't map to any entity field - if mapsToEntityID(param, resource.EntityName) { - // This would also try to map to the primary ID - CONFLICT! - fmt.Printf(" Skipping %s operation %s: parameter %s would conflict with primary ID %s (both map to entity.id)\n", - crudType, opInfo.Path, param, primaryID) - hasConflictingEntityIDs = true - break - } else { - // This is an unmappable ID parameter - fmt.Printf(" Skipping %s operation %s: unmappable ID parameter %s (not in entity schema)\n", - crudType, opInfo.Path, param) - hasConflictingEntityIDs = true - break - } - } - } - // Non-ID parameters are always OK - } - - if !hasPrimaryID { - fmt.Printf(" Skipping %s operation %s: missing primary ID %s\n", - crudType, opInfo.Path, primaryID) - continue - } - - if hasConflictingEntityIDs { - continue // Already logged above - } - - validOperations[crudType] = opInfo - } - - fmt.Printf(" Valid operations after parameter validation: %v\n", getOperationTypes(validOperations)) - return validOperations -} - -// Remove the helper function since we don't need it anymore - -// Helper function to get operation types for logging -func getOperationTypes(operations map[string]OperationInfo) []string { - var types []string - for opType := range operations { - types = append(types, opType) - } - return types -} - -// Check if create and read operations have compatible schemas -func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) bool { - if resource.CreateSchema == "" { - return false - } - - // Re-parse the spec to get raw schema data - specData, err := json.Marshal(spec) - if err != nil { - return false - } - - var rawSpec map[string]interface{} - if err := json.Unmarshal(specData, &rawSpec); err != nil { - return false - } - - components, _ := rawSpec["components"].(map[string]interface{}) - schemas, _ := components["schemas"].(map[string]interface{}) - - entityProps := getSchemaProperties(schemas, resource.EntityName) - createProps := getSchemaProperties(schemas, resource.CreateSchema) - - if len(entityProps) == 0 || len(createProps) == 0 { - return false - } - - // Count overlapping manageable properties - commonManageableProps := 0 - createManageableProps := 0 - - for prop := range createProps { - if !isSystemProperty(prop) { - createManageableProps++ - if entityProps[prop] != nil { - commonManageableProps++ - } - } - } - - // Need at least some manageable properties - if createManageableProps == 0 { - return false - } - - // Require at least 30% overlap of create properties to exist in entity - // This is more lenient than the 50% I had before - overlapRatio := float64(commonManageableProps) / float64(createManageableProps) - return overlapRatio >= 0.3 -} - -func getSchemaProperties(schemas map[string]interface{}, schemaName string) map[string]interface{} { - if schemaName == "" { - return map[string]interface{}{} - } - - schema, exists := schemas[schemaName] - if !exists { - return map[string]interface{}{} - } - - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - return map[string]interface{}{} - } - - properties, ok := schemaMap["properties"].(map[string]interface{}) - if !ok { - return map[string]interface{}{} - } - - return properties -} - -func isSystemProperty(propName string) bool { - systemProps := []string{ - "id", "created_at", "updated_at", "created_by", "updated_by", - "version", "etag", "revision", "last_modified", - } - - lowerProp := strings.ToLower(propName) - - for _, sysProp := range systemProps { - if lowerProp == sysProp || strings.HasSuffix(lowerProp, "_"+sysProp) { - return true - } - } - - // Also consider ID fields as system properties - if strings.HasSuffix(lowerProp, "_id") { - return true - } - - return false -} - -func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) *Overlay { - overlay := &Overlay{ - Overlay: "1.0.0", - } - - overlay.Info.Title = "Terraform Provider Overlay" - overlay.Info.Version = "1.0.0" - overlay.Info.Description = "Auto-generated overlay for Terraform resources" - - // Clean up resources by removing manually ignored operations - cleanedResources := cleanResourcesWithManualMappings(resources, manualMappings) - - // Separate viable and non-viable resources - viableResources := make(map[string]*ResourceInfo) - skippedResources := make([]string, 0) - - for name, resource := range cleanedResources { - if isTerraformViable(resource, spec) { - viableResources[name] = resource - } else { - skippedResources = append(skippedResources, name) - } - } - - fmt.Printf("\n=== Overlay Generation Analysis ===\n") - fmt.Printf("Total resources found: %d\n", len(cleanedResources)) - fmt.Printf("Viable for Terraform: %d\n", len(viableResources)) - fmt.Printf("Skipped (non-viable): %d\n", len(skippedResources)) - - if len(skippedResources) > 0 { - fmt.Printf("\nSkipped resources:\n") - for _, skipped := range skippedResources { - fmt.Printf(" - %s\n", skipped) - } - } - - // Filter operations with unmappable path parameters - fmt.Printf("\n=== Operation-Level Filtering ===\n") - filteredResources := filterOperationsWithUnmappableParameters(viableResources, spec) - - // Update description with actual count - overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(filteredResources)) - - // Detect property mismatches for filtered resources only - resourceMismatches := detectPropertyMismatches(filteredResources, spec) - - // Detect CRUD inconsistencies for filtered resources only - resourceCRUDInconsistencies := detectCRUDInconsistencies(filteredResources, spec) - - // Track which properties already have ignore actions to avoid duplicates - ignoreTracker := make(map[string]map[string]bool) // map[schemaName][propertyName]bool - - // Generate actions only for filtered resources - for _, resource := range filteredResources { - // Mark the response entity schema - entityUpdate := map[string]interface{}{ - "x-speakeasy-entity": resource.EntityName, - } - - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s", resource.SchemaName), - Update: entityUpdate, - }) - - // Initialize ignore tracker for this resource's schemas - if ignoreTracker[resource.EntityName] == nil { - ignoreTracker[resource.EntityName] = make(map[string]bool) - } - if resource.CreateSchema != "" && ignoreTracker[resource.CreateSchema] == nil { - ignoreTracker[resource.CreateSchema] = make(map[string]bool) - } - if resource.UpdateSchema != "" && ignoreTracker[resource.UpdateSchema] == nil { - ignoreTracker[resource.UpdateSchema] = make(map[string]bool) - } - - // Add speakeasy ignore for property mismatches - if mismatches, exists := resourceMismatches[resource.EntityName]; exists { - addIgnoreActionsForMismatches(overlay, resource, mismatches, ignoreTracker) - } - - // Add speakeasy ignore for CRUD inconsistencies - if inconsistencies, exists := resourceCRUDInconsistencies[resource.EntityName]; exists { - addIgnoreActionsForInconsistencies(overlay, resource, inconsistencies, ignoreTracker) - } - - // Add entity operations and parameter matching - for crudType, opInfo := range resource.Operations { - // Double-check that this specific operation isn't in the ignore list - if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { - fmt.Printf(" Skipping ignored operation during overlay generation: %s %s\n", opInfo.Method, opInfo.Path) - continue - } - - entityOp := mapCrudToEntityOperation(crudType, resource.EntityName) - - operationUpdate := map[string]interface{}{ - "x-speakeasy-entity-operation": entityOp, - } - - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.paths[\"%s\"].%s", opInfo.Path, opInfo.Method), - Update: operationUpdate, - }) - - // Apply parameter matching for operations that use the primary ID - if resource.PrimaryID != "" && (crudType == "read" || crudType == "update" || crudType == "delete") { - pathParams := extractPathParameters(opInfo.Path) - for _, param := range pathParams { - if param == resource.PrimaryID { - // Check for manual parameter mapping first - if manualMatch, hasManual := getManualParameterMatch(opInfo.Path, opInfo.Method, param, manualMappings); hasManual { - // Only apply manual match if it's different from the parameter name - if manualMatch != param { - fmt.Printf(" Manual parameter mapping: %s in %s %s -> %s\n", param, opInfo.Method, opInfo.Path, manualMatch) - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", - opInfo.Path, opInfo.Method, param), - Update: map[string]interface{}{ - "x-speakeasy-match": manualMatch, - }, - }) - } else { - fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s (would create circular reference)\n", param, manualMatch) - } - } else { - // Skip x-speakeasy-match when parameter name would map to itself - // This prevents circular references like {id} -> id - if param == "id" { - fmt.Printf(" Skipping x-speakeasy-match: parameter %s maps to same field (avoiding circular reference)\n", param) - } else { - // Apply x-speakeasy-match for parameters that need mapping (e.g., change_event_id -> id) - fmt.Printf(" Applying x-speakeasy-match to %s in %s %s -> id\n", param, opInfo.Method, opInfo.Path) - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", - opInfo.Path, opInfo.Method, param), - Update: map[string]interface{}{ - "x-speakeasy-match": "id", - }, - }) - } - } - } - } - } - } - } - - // Process parameter matching is now handled inline above - // No need for separate addEntityLevelParameterMatches call - - fmt.Printf("\n=== Overlay Generation Complete ===\n") - fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(filteredResources)) - - // Count ignore actions and match actions - totalIgnores := 0 - totalMatches := 0 - for _, action := range overlay.Actions { - if _, hasIgnore := action.Update["x-speakeasy-ignore"]; hasIgnore { - totalIgnores++ - } - if _, hasMatch := action.Update["x-speakeasy-match"]; hasMatch { - totalMatches++ - } - } - - if totalIgnores > 0 { - fmt.Printf("✅ %d speakeasy ignore actions added for property issues\n", totalIgnores) - } - if totalMatches > 0 { - fmt.Printf("✅ %d speakeasy match actions added for primary ID parameters\n", totalMatches) - } - - return overlay -} - -// Clean up resources by removing operations that are manually ignored -func cleanResourcesWithManualMappings(resources map[string]*ResourceInfo, manualMappings *ManualMappings) map[string]*ResourceInfo { - cleanedResources := make(map[string]*ResourceInfo) - - fmt.Printf("\n=== Cleaning Resources with Manual Mappings ===\n") - - for name, resource := range resources { - cleanedResource := &ResourceInfo{ - EntityName: resource.EntityName, - SchemaName: resource.SchemaName, - ResourceName: resource.ResourceName, - Operations: make(map[string]OperationInfo), - CreateSchema: resource.CreateSchema, - UpdateSchema: resource.UpdateSchema, - PrimaryID: resource.PrimaryID, - } - - operationsRemoved := 0 - - // Copy operations that aren't manually ignored - for crudType, opInfo := range resource.Operations { - if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { - fmt.Printf(" Removing manually ignored operation: %s %s (was %s for %s)\n", - opInfo.Method, opInfo.Path, crudType, resource.EntityName) - operationsRemoved++ - } else { - cleanedResource.Operations[crudType] = opInfo - } - } - - // Only include resource if it still has operations after cleaning - if len(cleanedResource.Operations) > 0 { - cleanedResources[name] = cleanedResource - if operationsRemoved > 0 { - fmt.Printf(" Resource %s: kept %d operations, removed %d manually ignored\n", - name, len(cleanedResource.Operations), operationsRemoved) - } - } else { - fmt.Printf(" Resource %s: removed entirely (all operations were manually ignored)\n", name) - } - } - - fmt.Printf("Manual mapping cleanup: %d → %d resources\n", len(resources), len(cleanedResources)) - return cleanedResources -} - -func addIgnoreActionsForMismatches(overlay *Overlay, resource *ResourceInfo, mismatches []PropertyMismatch, ignoreTracker map[string]map[string]bool) { - // Add speakeasy ignore for create schema properties - if resource.CreateSchema != "" { - for _, mismatch := range mismatches { - // Ignore in request schema - if !ignoreTracker[resource.CreateSchema][mismatch.PropertyName] { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.CreateSchema, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - ignoreTracker[resource.CreateSchema][mismatch.PropertyName] = true - } - - // Also ignore in response entity schema - if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - ignoreTracker[resource.EntityName][mismatch.PropertyName] = true - } - } - } - - // Add speakeasy ignore for update schema properties - if resource.UpdateSchema != "" { - for _, mismatch := range mismatches { - // Ignore in request schema - if !ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.UpdateSchema, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - ignoreTracker[resource.UpdateSchema][mismatch.PropertyName] = true - } - - // Also ignore in response entity schema (avoid duplicates) - if !ignoreTracker[resource.EntityName][mismatch.PropertyName] { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", resource.EntityName, mismatch.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - ignoreTracker[resource.EntityName][mismatch.PropertyName] = true - } - } - } -} - -func addIgnoreActionsForInconsistencies(overlay *Overlay, resource *ResourceInfo, inconsistencies []CRUDInconsistency, ignoreTracker map[string]map[string]bool) { - for _, inconsistency := range inconsistencies { - // Add ignore actions for each schema listed in SchemasToIgnore - for _, schemaName := range inconsistency.SchemasToIgnore { - if !ignoreTracker[schemaName][inconsistency.PropertyName] { - overlay.Actions = append(overlay.Actions, OverlayAction{ - Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", schemaName, inconsistency.PropertyName), - Update: map[string]interface{}{ - "x-speakeasy-ignore": true, - }, - }) - ignoreTracker[schemaName][inconsistency.PropertyName] = true - } - } - } -} - -// Enhanced mismatch detection - same as before but only for viable resources -func detectPropertyMismatches(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]PropertyMismatch { - mismatches := make(map[string][]PropertyMismatch) - - // Re-parse the spec to get raw schema data - specData, err := json.Marshal(spec) - if err != nil { - return mismatches - } - - var rawSpec map[string]interface{} - if err := json.Unmarshal(specData, &rawSpec); err != nil { - return mismatches - } - - components, _ := rawSpec["components"].(map[string]interface{}) - schemas, _ := components["schemas"].(map[string]interface{}) - - for _, resource := range resources { - var resourceMismatches []PropertyMismatch - - // Check create operation mismatches - if resource.CreateSchema != "" { - if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { - if requestSchema, exists := schemas[resource.CreateSchema].(map[string]interface{}); exists { - createMismatches := findPropertyMismatches(entitySchema, requestSchema, "create") - resourceMismatches = append(resourceMismatches, createMismatches...) - } - } - } - - // Check update operation mismatches - if resource.UpdateSchema != "" { - if entitySchema, exists := schemas[resource.EntityName].(map[string]interface{}); exists { - if requestSchema, exists := schemas[resource.UpdateSchema].(map[string]interface{}); exists { - updateMismatches := findPropertyMismatches(entitySchema, requestSchema, "update") - resourceMismatches = append(resourceMismatches, updateMismatches...) - } - } - } - - if len(resourceMismatches) > 0 { - mismatches[resource.EntityName] = resourceMismatches - } - } - - return mismatches -} - -func findPropertyMismatches(entitySchema, requestSchema map[string]interface{}, operation string) []PropertyMismatch { - var mismatches []PropertyMismatch - - entityProps, _ := entitySchema["properties"].(map[string]interface{}) - requestProps, _ := requestSchema["properties"].(map[string]interface{}) - - if entityProps == nil || requestProps == nil { - return mismatches - } - - for propName, entityProp := range entityProps { - if requestProp, exists := requestProps[propName]; exists { - if hasStructuralMismatch(propName, entityProp, requestProp) { - mismatch := PropertyMismatch{ - PropertyName: propName, - MismatchType: "structural-mismatch", - Description: describeStructuralDifference(entityProp, requestProp), - } - mismatches = append(mismatches, mismatch) - } - } - } - - return mismatches -} - -// Check if two property structures are different -func hasStructuralMismatch(propName string, entityProp, requestProp interface{}) bool { - entityStructure := getPropertyStructure(entityProp) - requestStructure := getPropertyStructure(requestProp) - return entityStructure != requestStructure -} - -// Get a normalized string representation of a property's structure -func getPropertyStructure(prop interface{}) string { - propMap, ok := prop.(map[string]interface{}) - if !ok { - return "unknown" - } - - // Check for $ref - if ref, hasRef := propMap["$ref"].(string); hasRef { - return fmt.Sprintf("$ref:%s", ref) - } - - propType, _ := propMap["type"].(string) - - switch propType { - case "array": - items, hasItems := propMap["items"] - if hasItems { - itemStructure := getPropertyStructure(items) - return fmt.Sprintf("array[%s]", itemStructure) - } - return "array[unknown]" - - case "object": - properties, hasProps := propMap["properties"] - _, hasAdditional := propMap["additionalProperties"] - - if hasProps { - propsMap, _ := properties.(map[string]interface{}) - if len(propsMap) == 0 { - return "object{empty}" - } - return "object{defined}" - } - - if hasAdditional { - return "object{additional}" - } - - return "object{}" - - case "string", "integer", "number", "boolean": - return propType - - default: - if propType == "" { - if _, hasProps := propMap["properties"]; hasProps { - return "implicit-object" - } - if _, hasItems := propMap["items"]; hasItems { - return "implicit-array" - } - } - return fmt.Sprintf("type:%s", propType) - } -} - -// Describe the structural difference for reporting -func describeStructuralDifference(entityProp, requestProp interface{}) string { - entityStructure := getPropertyStructure(entityProp) - requestStructure := getPropertyStructure(requestProp) - return fmt.Sprintf("request structure '%s' != response structure '%s'", requestStructure, entityStructure) -} - -// Detect schema property inconsistencies (extracted from detectCRUDInconsistencies) -func detectCRUDInconsistencies(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string][]CRUDInconsistency { - inconsistencies := make(map[string][]CRUDInconsistency) - - // Re-parse the spec to get raw schema data - specData, err := json.Marshal(spec) - if err != nil { - return inconsistencies - } - - var rawSpec map[string]interface{} - if err := json.Unmarshal(specData, &rawSpec); err != nil { - return inconsistencies - } - - components, _ := rawSpec["components"].(map[string]interface{}) - schemas, _ := components["schemas"].(map[string]interface{}) - - for _, resource := range resources { - resourceInconsistencies := detectSchemaPropertyInconsistencies(resource, schemas) - - // Check if we have fundamental validation errors that make the resource non-viable - for _, inconsistency := range resourceInconsistencies { - if inconsistency.PropertyName == "RESOURCE_VALIDATION" { - fmt.Printf("⚠️ Resource %s (%s) validation failed: %s\n", - resource.ResourceName, resource.EntityName, inconsistency.Description) - // Mark the entire resource as having issues but don't add to inconsistencies - // as this will be handled in the viability check - continue - } - } - - // Only add property-level inconsistencies for viable resources - var validInconsistencies []CRUDInconsistency - for _, inconsistency := range resourceInconsistencies { - if inconsistency.PropertyName != "RESOURCE_VALIDATION" { - validInconsistencies = append(validInconsistencies, inconsistency) - } - } - - if len(validInconsistencies) > 0 { - inconsistencies[resource.EntityName] = validInconsistencies - } - } - - return inconsistencies -} - -// Detect schema property inconsistencies (simplified CRUD detection) -func detectSchemaPropertyInconsistencies(resource *ResourceInfo, schemas map[string]interface{}) []CRUDInconsistency { - var inconsistencies []CRUDInconsistency - - // First, validate that we have the minimum required operations for Terraform - _, hasCreate := resource.Operations["create"] - _, hasRead := resource.Operations["read"] - - if !hasCreate || !hasRead { - // Return a fundamental inconsistency - resource is not viable for Terraform - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "missing-required-operations", - Description: fmt.Sprintf("Resource missing required operations: Create=%v, Read=%v", hasCreate, hasRead), - SchemasToIgnore: []string{}, // Don't ignore anything, this makes the whole resource invalid - } - return []CRUDInconsistency{inconsistency} - } - - // Validate that we have a create schema - if resource.CreateSchema == "" { - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "missing-create-schema", - Description: "Resource has CREATE operation but no request schema defined", - SchemasToIgnore: []string{}, - } - return []CRUDInconsistency{inconsistency} - } - - // Get properties from each schema - entityProps := getSchemaProperties(schemas, resource.EntityName) - createProps := getSchemaProperties(schemas, resource.CreateSchema) - updateProps := map[string]interface{}{} - - if resource.UpdateSchema != "" { - updateProps = getSchemaProperties(schemas, resource.UpdateSchema) - } - - // Validate that schemas exist and have properties - if len(entityProps) == 0 { - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "invalid-entity-schema", - Description: fmt.Sprintf("Entity schema '%s' not found or has no properties", resource.EntityName), - SchemasToIgnore: []string{}, - } - return []CRUDInconsistency{inconsistency} - } - - if len(createProps) == 0 { - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "invalid-create-schema", - Description: fmt.Sprintf("Create schema '%s' not found or has no properties", resource.CreateSchema), - SchemasToIgnore: []string{}, - } - return []CRUDInconsistency{inconsistency} - } - - // Check for minimum viable overlap between create and entity schemas - commonManageableProps := 0 - createManageableProps := 0 - - for prop := range createProps { - if !isSystemProperty(prop) { - createManageableProps++ - if entityProps[prop] != nil { - commonManageableProps++ - } - } - } - - if createManageableProps == 0 { - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "no-manageable-properties", - Description: "Create schema has no manageable properties (all are system properties)", - SchemasToIgnore: []string{}, - } - return []CRUDInconsistency{inconsistency} - } - - // Require reasonable overlap between create and entity schemas - overlapRatio := float64(commonManageableProps) / float64(createManageableProps) - if overlapRatio < 0.3 { // At least 30% overlap required - inconsistency := CRUDInconsistency{ - PropertyName: "RESOURCE_VALIDATION", - InconsistencyType: "insufficient-schema-overlap", - Description: fmt.Sprintf("Insufficient overlap between create and entity schemas: %.1f%% (%d/%d properties)", overlapRatio*100, commonManageableProps, createManageableProps), - SchemasToIgnore: []string{}, - } - return []CRUDInconsistency{inconsistency} - } - - // Now check individual property inconsistencies for viable resources - // Collect all property names across CRUD operations - allProps := make(map[string]bool) - for prop := range entityProps { - allProps[prop] = true - } - for prop := range createProps { - allProps[prop] = true - } - for prop := range updateProps { - allProps[prop] = true - } - - // Check each property for consistency across CRUD operations - for propName := range allProps { - // Skip ID properties - they have separate handling logic - if propName == "id" { - continue - } - - entityHas := entityProps[propName] != nil - createHas := createProps[propName] != nil - updateHas := updateProps[propName] != nil - - // Check for CRUD inconsistencies - var schemasToIgnore []string - var inconsistencyType string - var description string - hasInconsistency := false - - if resource.CreateSchema != "" && resource.UpdateSchema != "" { - // Full CRUD resource - all three must be consistent - if !(entityHas && createHas && updateHas) { - hasInconsistency = true - inconsistencyType = "crud-property-mismatch" - description = fmt.Sprintf("Property not present in all CRUD operations (Entity:%v, Create:%v, Update:%v)", entityHas, createHas, updateHas) - - // Ignore in schemas where property exists but shouldn't for consistency - if entityHas && (!createHas || !updateHas) { - schemasToIgnore = append(schemasToIgnore, resource.EntityName) - } - if createHas && (!entityHas || !updateHas) { - schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) - } - if updateHas && (!entityHas || !createHas) { - schemasToIgnore = append(schemasToIgnore, resource.UpdateSchema) - } - } - } else if resource.CreateSchema != "" { - // Create + Read resource - both must be consistent - if !(entityHas && createHas) { - hasInconsistency = true - inconsistencyType = "create-read-mismatch" - description = fmt.Sprintf("Property not present in both CREATE and READ (Entity:%v, Create:%v)", entityHas, createHas) - - if entityHas && !createHas { - schemasToIgnore = append(schemasToIgnore, resource.EntityName) - } - if createHas && !entityHas { - schemasToIgnore = append(schemasToIgnore, resource.CreateSchema) - } - } - } - - if hasInconsistency { - inconsistency := CRUDInconsistency{ - PropertyName: propName, - InconsistencyType: inconsistencyType, - Description: description, - SchemasToIgnore: schemasToIgnore, - } - inconsistencies = append(inconsistencies, inconsistency) - } - } - - return inconsistencies -} - -func mapCrudToEntityOperation(crudType, entityName string) string { - switch crudType { - case "create": - return entityName + "#create" - case "read": - return entityName + "#read" - case "update": - return entityName + "#update" - case "delete": - return entityName + "#delete" - case "list": - // For list operations, pluralize the entity name and use #read - pluralEntityName := pluralizeEntityName(entityName) - return pluralEntityName + "#read" - default: - return entityName + "#" + crudType - } -} - -// Simplified pluralization logic -func pluralizeEntityName(entityName string) string { - // Remove "Entity" suffix - baseName := strings.TrimSuffix(entityName, "Entity") - - // Simple pluralization - if strings.HasSuffix(baseName, "y") && len(baseName) > 1 && !isVowel(baseName[len(baseName)-2]) { - baseName = baseName[:len(baseName)-1] + "ies" - } else if strings.HasSuffix(baseName, "s") || - strings.HasSuffix(baseName, "ss") || - strings.HasSuffix(baseName, "sh") || - strings.HasSuffix(baseName, "ch") || - strings.HasSuffix(baseName, "x") || - strings.HasSuffix(baseName, "z") { - baseName = baseName + "es" - } else { - baseName = baseName + "s" - } - - return baseName + "Entities" -} - -func isVowel(c byte) bool { - return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u' || - c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U' -} - -// Filter out operations with unmappable path parameters - simplified now that we handle this in viability check -func filterOperationsWithUnmappableParameters(resources map[string]*ResourceInfo, spec OpenAPISpec) map[string]*ResourceInfo { - // The complex parameter filtering is now handled in isTerraformViable() - // This function now just returns the resources as-is since they've already been validated - fmt.Printf("Operation filtering handled during viability check\n") - return resources -} - -func writeOverlay(overlay *Overlay) error { - // Marshal to YAML - data, err := yaml.Marshal(overlay) - if err != nil { - return fmt.Errorf("marshaling overlay: %w", err) - } - - // Write file to current directory - overlayPath := "terraform-overlay.yaml" - if err := ioutil.WriteFile(overlayPath, data, 0644); err != nil { - return fmt.Errorf("writing overlay file: %w", err) - } - - fmt.Printf("Overlay written to: %s\n", overlayPath) - return nil -} - -func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { - viableCount := 0 - for _, resource := range resources { - if isTerraformViable(resource, OpenAPISpec{}) { - viableCount++ - } - } - - fmt.Println("\n=== Summary ===") - fmt.Printf("✅ Successfully generated overlay with %d actions\n", len(overlay.Actions)) - fmt.Printf("📊 Resources: %d total, %d viable for Terraform, %d skipped\n", - len(resources), viableCount, len(resources)-viableCount) - - fmt.Println("\nOverlay approach:") - fmt.Println("1. Load manual mappings for edge cases") - fmt.Println("2. Identify entity schemas and match operations to entities") - fmt.Println("3. Apply manual ignore/entity/match mappings during analysis") - fmt.Println("4. Clean resources by removing manually ignored operations") - fmt.Println("5. Analyze ID patterns and choose consistent primary ID per entity") - fmt.Println("6. Filter operations with unmappable path parameters") - fmt.Println("7. Skip annotations for non-viable resources") - fmt.Println("8. Mark viable entity schemas with x-speakeasy-entity") - fmt.Println("9. Tag viable operations with x-speakeasy-entity-operation") - fmt.Println("10. Mark chosen primary ID with x-speakeasy-match") - fmt.Println("11. Apply x-speakeasy-ignore: true to problematic properties") -} diff --git a/scripts/overlay/link-entity-operations.go b/scripts/overlay/link-entity-operations.go new file mode 100644 index 0000000..a8e92b8 --- /dev/null +++ b/scripts/overlay/link-entity-operations.go @@ -0,0 +1,49 @@ +package main + +import "strings" + +func mapCrudToEntityOperation(crudType, entityName string) string { + switch crudType { + case "create": + return entityName + "#create" + case "read": + return entityName + "#read" + case "update": + return entityName + "#update" + case "delete": + return entityName + "#delete" + case "list": + // For list operations, pluralize the entity name and use #read + pluralEntityName := pluralizeEntityName(entityName) + return pluralEntityName + "#read" + default: + return entityName + "#" + crudType + } +} + +// Simplified pluralization logic +func pluralizeEntityName(entityName string) string { + // Remove "Entity" suffix + baseName := strings.TrimSuffix(entityName, "Entity") + + // Simple pluralization + if strings.HasSuffix(baseName, "y") && len(baseName) > 1 && !isVowel(baseName[len(baseName)-2]) { + baseName = baseName[:len(baseName)-1] + "ies" + } else if strings.HasSuffix(baseName, "s") || + strings.HasSuffix(baseName, "ss") || + strings.HasSuffix(baseName, "sh") || + strings.HasSuffix(baseName, "ch") || + strings.HasSuffix(baseName, "x") || + strings.HasSuffix(baseName, "z") { + baseName = baseName + "es" + } else { + baseName = baseName + "s" + } + + return baseName + "Entities" +} + +func isVowel(c byte) bool { + return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u' || + c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U' +} diff --git a/scripts/overlay/main.go b/scripts/overlay/main.go new file mode 100644 index 0000000..702d499 --- /dev/null +++ b/scripts/overlay/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "gopkg.in/yaml.v3" +) + +type OpenAPISpec struct { + OpenAPI string `json:"openapi"` + Info map[string]interface{} `json:"info"` + Paths map[string]PathItem `json:"paths"` + Components Components `json:"components"` +} + +type Components struct { + Schemas map[string]Schema `json:"schemas"` + SecuritySchemes map[string]interface{} `json:"securitySchemes,omitempty"` +} + +type Schema struct { + Type string `json:"type,omitempty"` + Properties map[string]interface{} `json:"properties,omitempty"` + Required []string `json:"required,omitempty"` + AllOf []interface{} `json:"allOf,omitempty"` + Nullable bool `json:"nullable,omitempty"` + Items interface{} `json:"items,omitempty"` + Raw map[string]interface{} `json:"-"` +} + +type PathItem struct { + Get *Operation `json:"get,omitempty"` + Post *Operation `json:"post,omitempty"` + Put *Operation `json:"put,omitempty"` + Patch *Operation `json:"patch,omitempty"` + Delete *Operation `json:"delete,omitempty"` +} + +type Operation struct { + OperationID string `json:"operationId,omitempty"` + Tags []string `json:"tags,omitempty"` + Parameters []Parameter `json:"parameters,omitempty"` + RequestBody map[string]interface{} `json:"requestBody,omitempty"` + Responses map[string]interface{} `json:"responses,omitempty"` +} + +type Parameter struct { + Name string `json:"name"` + In string `json:"in"` + Required bool `json:"required,omitempty"` + Schema Schema `json:"schema,omitempty"` +} + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + specPath := os.Args[1] + var mappingsPath string + if len(os.Args) > 2 { + mappingsPath = os.Args[2] + } else { + mappingsPath = "manual-mappings.yaml" + } + + fmt.Printf("=== Terraform Overlay Generator ===\n") + fmt.Printf("Input: %s\n", specPath) + + manualMappings := loadManualMappings(mappingsPath) + + specData, err := ioutil.ReadFile(specPath) + if err != nil { + fmt.Printf("Error reading spec file: %v\n", err) + os.Exit(1) + } + + var spec OpenAPISpec + if err := json.Unmarshal(specData, &spec); err != nil { + fmt.Printf("Error parsing JSON: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Found %d paths and %d schemas\n\n", len(spec.Paths), len(spec.Components.Schemas)) + + resources := analyzeSpec(spec, manualMappings) + + overlay := generateOverlay(resources, spec, manualMappings) + + if err := writeOverlay(overlay); err != nil { + fmt.Printf("Error writing overlay: %v\n", err) + os.Exit(1) + } + + printOverlaySummary(resources, overlay) +} + +func printUsage() { + fmt.Println("OpenAPI Terraform Overlay Generator") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" openapi-overlay ") +} + +func (s *Schema) UnmarshalJSON(data []byte) error { + type Alias Schema + aux := &struct { + *Alias + }{ + Alias: (*Alias)(s), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Also unmarshal into raw map + if err := json.Unmarshal(data, &s.Raw); err != nil { + return err + } + + return nil +} + +func writeOverlay(overlay *Overlay) error { + data, err := yaml.Marshal(overlay) + if err != nil { + return fmt.Errorf("marshaling overlay: %w", err) + } + + overlayPath := "terraform-overlay.yaml" + if err := ioutil.WriteFile(overlayPath, data, 0644); err != nil { + return fmt.Errorf("writing overlay file: %w", err) + } + + fmt.Printf("Overlay written to: %s\n", overlayPath) + return nil +} + +func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { + viableCount := 0 + for _, resource := range resources { + if isTerraformViable(resource, OpenAPISpec{}) { + viableCount++ + } + } + + fmt.Println("\n=== Summary ===") + fmt.Printf("✅ Successfully generated overlay with %d actions\n", len(overlay.Actions)) + fmt.Printf("📊 Resources: %d total, %d viable for Terraform, %d skipped\n", + len(resources), viableCount, len(resources)-viableCount) + + fmt.Println("\nOverlay approach:") + fmt.Println("1. Load manual mappings for edge cases") + fmt.Println("2. Identify entity schemas and match operations to entities") + fmt.Println("3. Apply manual ignore/entity/match mappings during analysis") + fmt.Println("4. Clean resources by removing manually ignored operations") + fmt.Println("5. Analyze ID patterns and choose consistent primary ID per entity") + fmt.Println("6. Filter operations with unmappable path parameters") + fmt.Println("7. Skip annotations for non-viable resources") + fmt.Println("8. Mark viable entity schemas with x-speakeasy-entity") + fmt.Println("9. Tag viable operations with x-speakeasy-entity-operation") + fmt.Println("10. Mark chosen primary ID with x-speakeasy-match") + fmt.Println("11. Apply x-speakeasy-ignore: true to problematic properties") + fmt.Println("12. Apply x-speakeasy-enums: true to enum schemas for proper code generation") +} diff --git a/scripts/overlay/manual-mappings.go b/scripts/overlay/manual-mappings.go new file mode 100644 index 0000000..a2ed343 --- /dev/null +++ b/scripts/overlay/manual-mappings.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "io/ioutil" + "strings" + + "gopkg.in/yaml.v3" +) + +// Manual mapping configuration +type ManualMapping struct { + Path string `yaml:"path"` + Method string `yaml:"method"` + Action string `yaml:"action"` // "ignore", "entity", "match" + Value string `yaml:"value,omitempty"` +} + +type ManualMappings struct { + Operations []ManualMapping `yaml:"operations"` +} + +func loadManualMappings(mappingsPath string) *ManualMappings { + data, err := ioutil.ReadFile(mappingsPath) + if err != nil { + // File doesn't exist - return empty mappings + fmt.Printf("No manual mappings file found at %s (this is optional)\n", mappingsPath) + return &ManualMappings{} + } + + var mappings ManualMappings + if err := yaml.Unmarshal(data, &mappings); err != nil { + fmt.Printf("Error parsing manual mappings file: %v\n", err) + return &ManualMappings{} + } + + fmt.Printf("Loaded %d manual mappings from %s\n", len(mappings.Operations), mappingsPath) + return &mappings +} + +func applyManualMappings(resources map[string]*ResourceInfo, manualMappings *ManualMappings) map[string]*ResourceInfo { + cleanedResources := make(map[string]*ResourceInfo) + + fmt.Printf("\n=== Cleaning Resources with Manual Mappings ===\n") + + for name, resource := range resources { + cleanedResource := &ResourceInfo{ + EntityName: resource.EntityName, + SchemaName: resource.SchemaName, + ResourceName: resource.ResourceName, + Operations: make(map[string]OperationInfo), + CreateSchema: resource.CreateSchema, + UpdateSchema: resource.UpdateSchema, + PrimaryID: resource.PrimaryID, + } + + operationsRemoved := 0 + + // Copy operations that aren't manually ignored + for crudType, opInfo := range resource.Operations { + if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { + fmt.Printf(" Removing manually ignored operation: %s %s (was %s for %s)\n", + opInfo.Method, opInfo.Path, crudType, resource.EntityName) + operationsRemoved++ + } else { + cleanedResource.Operations[crudType] = opInfo + } + } + + // Only include resource if it still has operations after cleaning + if len(cleanedResource.Operations) > 0 { + cleanedResources[name] = cleanedResource + if operationsRemoved > 0 { + fmt.Printf(" Resource %s: kept %d operations, removed %d manually ignored\n", + name, len(cleanedResource.Operations), operationsRemoved) + } + } else { + fmt.Printf(" Resource %s: removed entirely (all operations were manually ignored)\n", name) + } + } + + fmt.Printf("Manual mapping cleanup: %d → %d resources\n", len(resources), len(cleanedResources)) + return cleanedResources +} + +func getManualParameterMatch(path, method, paramName string, manualMappings *ManualMappings) (string, bool) { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.ToLower(mapping.Method) == strings.ToLower(method) && mapping.Action == "match" { + // For match mappings, we expect the value to be in format "param_name:field_name" + parts := strings.SplitN(mapping.Value, ":", 2) + if len(parts) == 2 && parts[0] == paramName { + fmt.Printf(" Manual mapping: Parameter %s in %s %s -> %s\n", paramName, method, path, parts[1]) + return parts[1], true + } + } + } + return "", false +} + +func shouldIgnoreOperation(path, method string, manualMappings *ManualMappings) bool { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.EqualFold(mapping.Method, method) && mapping.Action == "ignore" { + fmt.Printf(" Manual mapping: Ignoring operation %s %s\n", method, path) + return true + } + } + return false +} + +func getManualEntityMapping(path, method string, manualMappings *ManualMappings) (string, bool) { + for _, mapping := range manualMappings.Operations { + if mapping.Path == path && strings.EqualFold(mapping.Method, method) && mapping.Action == "entity" { + fmt.Printf(" Manual mapping: Operation %s %s -> Entity %s\n", method, path, mapping.Value) + return mapping.Value, true + } + } + return "", false +} diff --git a/scripts/overlay/terraform-viable.go b/scripts/overlay/terraform-viable.go new file mode 100644 index 0000000..798078f --- /dev/null +++ b/scripts/overlay/terraform-viable.go @@ -0,0 +1,397 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" +) + +// Check if a resource is viable for Terraform +func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { + // Must have at least create and read operations + _, hasCreate := resource.Operations["create"] + _, hasRead := resource.Operations["read"] + + if !hasCreate || !hasRead { + fmt.Printf(" Missing required operations: Create=%v, Read=%v\n", hasCreate, hasRead) + return false + } + + // Must have a create schema to be manageable by Terraform + if resource.CreateSchema == "" { + fmt.Printf(" No create schema found\n") + return false + } + + // Identify the primary ID for this entity + primaryID, validPrimaryID := identifyEntityPrimaryID(resource) + if !validPrimaryID { + fmt.Printf(" Cannot identify valid primary ID parameter\n") + return false + } + + // Validate all operations against the primary ID + validOperations := validateOperationParameters(resource, primaryID, spec) + + // Must still have CREATE and READ after validation + _, hasValidCreate := validOperations["create"] + _, hasValidRead := validOperations["read"] + + if !hasValidCreate || !hasValidRead { + fmt.Printf(" Lost required operations after parameter validation: Create=%v, Read=%v\n", hasValidCreate, hasValidRead) + return false + } + + // Update resource with only valid operations and primary ID + resource.Operations = validOperations + resource.PrimaryID = primaryID + + // Check for overlapping properties between create and entity schemas + if !hasValidCreateReadConsistency(resource, spec) { + fmt.Printf(" Create and Read operations have incompatible schemas\n") + return false + } + + // Check for problematic CRUD patterns that can't be handled by property ignoring + if resource.CreateSchema != "" && resource.UpdateSchema != "" { + // Re-parse the spec to get raw schema data for analysis + specData, err := json.Marshal(spec) + if err != nil { + return true // If we can't analyze, assume it's viable + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return true // If we can't analyze, assume it's viable + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + createProps := getSchemaProperties(schemas, resource.CreateSchema) + updateProps := getSchemaProperties(schemas, resource.UpdateSchema) + + // Count manageable properties (non-system fields) + createManageableProps := 0 + updateManageableProps := 0 + commonManageableProps := 0 + + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + } + } + + for prop := range updateProps { + if !isSystemProperty(prop) { + updateManageableProps++ + // Check if this property also exists in create + if createProps[prop] != nil && !isSystemProperty(prop) { + commonManageableProps++ + } + } + } + + // Reject resources with fundamentally incompatible CRUD patterns + if createManageableProps <= 1 && updateManageableProps >= 3 && commonManageableProps == 0 { + fmt.Printf(" Incompatible CRUD pattern: Create=%d manageable, Update=%d manageable, Common=%d\n", + createManageableProps, updateManageableProps, commonManageableProps) + return false + } + } + + return true +} + +// Validate operations against the identified primary ID +func validateOperationParameters(resource *ResourceInfo, primaryID string, spec OpenAPISpec) map[string]OperationInfo { + validOperations := make(map[string]OperationInfo) + + // Get entity properties once for this resource + entityProps := getEntityProperties(resource.EntityName, spec) + + for crudType, opInfo := range resource.Operations { + pathParams := extractPathParameters(opInfo.Path) + + if crudType == "create" || crudType == "list" { + // These operations should not have the entity's primary ID in path + hasPrimaryID := false + for _, param := range pathParams { + if param == primaryID { + hasPrimaryID = true + break + } + } + + if hasPrimaryID { + fmt.Printf(" Skipping %s operation %s: unexpectedly has primary ID %s in path\n", + crudType, opInfo.Path, primaryID) + continue + } + + validOperations[crudType] = opInfo + continue + } + + // READ, UPDATE, DELETE should have exactly the primary ID + hasPrimaryID := false + hasConflictingEntityIDs := false + + for _, param := range pathParams { + if param == primaryID { + hasPrimaryID = true + } else if isEntityID(param) { + // This is another ID-like parameter + // Check if it maps to a field in the entity (not the primary id field) + if checkFieldExistsInEntity(param, entityProps) { + // This parameter maps to a real entity field - it's valid + fmt.Printf(" Parameter %s maps to entity field - keeping operation %s %s\n", + param, crudType, opInfo.Path) + } else { + // This ID parameter doesn't map to any entity field + if mapsToEntityID(param, resource.EntityName) { + fmt.Printf(" Skipping %s operation %s: parameter %s would conflict with primary ID %s (both map to entity.id)\n", + crudType, opInfo.Path, param, primaryID) + hasConflictingEntityIDs = true + break + } else { + // This is an unmappable ID parameter + fmt.Printf(" Skipping %s operation %s: unmappable ID parameter %s (not in entity schema)\n", + crudType, opInfo.Path, param) + hasConflictingEntityIDs = true + break + } + } + } + // Non-ID parameters are always OK + } + + if !hasPrimaryID { + fmt.Printf(" Skipping %s operation %s: missing primary ID %s\n", + crudType, opInfo.Path, primaryID) + continue + } + + if hasConflictingEntityIDs { + continue + } + + validOperations[crudType] = opInfo + } + + fmt.Printf(" Valid operations after parameter validation: %v\n", getOperationTypes(validOperations)) + return validOperations +} + +// Get entity properties for field existence checking +func getEntityProperties(entityName string, spec OpenAPISpec) map[string]interface{} { + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return map[string]interface{}{} + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return map[string]interface{}{} + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + return getSchemaProperties(schemas, entityName) +} + +// Check if create and read operations have compatible schemas +func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) bool { + if resource.CreateSchema == "" { + return false + } + + // Re-parse the spec to get raw schema data + specData, err := json.Marshal(spec) + if err != nil { + return false + } + + var rawSpec map[string]interface{} + if err := json.Unmarshal(specData, &rawSpec); err != nil { + return false + } + + components, _ := rawSpec["components"].(map[string]interface{}) + schemas, _ := components["schemas"].(map[string]interface{}) + + entityProps := getSchemaProperties(schemas, resource.EntityName) + createProps := getSchemaProperties(schemas, resource.CreateSchema) + + if len(entityProps) == 0 || len(createProps) == 0 { + return false + } + + // Count overlapping manageable properties + commonManageableProps := 0 + createManageableProps := 0 + + for prop := range createProps { + if !isSystemProperty(prop) { + createManageableProps++ + if entityProps[prop] != nil { + commonManageableProps++ + } + } + } + + // Need at least some manageable properties + if createManageableProps == 0 { + return false + } + + // Require at least 30% overlap of create properties to exist in entity + // This is more lenient than the 50% I had before + overlapRatio := float64(commonManageableProps) / float64(createManageableProps) + return overlapRatio >= 0.3 +} + +func getSchemaProperties(schemas map[string]interface{}, schemaName string) map[string]interface{} { + if schemaName == "" { + return map[string]interface{}{} + } + + schema, exists := schemas[schemaName] + if !exists { + return map[string]interface{}{} + } + + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + return map[string]interface{}{} + } + + properties, ok := schemaMap["properties"].(map[string]interface{}) + if !ok { + return map[string]interface{}{} + } + + return properties +} + +func isSystemProperty(propName string) bool { + systemProps := []string{ + "id", "created_at", "updated_at", "created_by", "updated_by", + "version", "etag", "revision", "last_modified", + } + + lowerProp := strings.ToLower(propName) + + for _, sysProp := range systemProps { + if lowerProp == sysProp || strings.HasSuffix(lowerProp, "_"+sysProp) { + return true + } + } + + // Also consider ID fields as system properties + if strings.HasSuffix(lowerProp, "_id") { + return true + } + + return false +} + +// Check if a field exists in the entity properties +func checkFieldExistsInEntity(paramName string, entityProps map[string]interface{}) bool { + // Direct field name match + if _, exists := entityProps[paramName]; exists { + return true + } + + // Check for common variations + variations := []string{ + paramName, + strings.TrimSuffix(paramName, "_id"), // Remove _id suffix + strings.TrimSuffix(paramName, "Id"), // Remove Id suffix + } + + for _, variation := range variations { + if _, exists := entityProps[variation]; exists { + return true + } + } + + return false +} + +// Identify the primary ID parameter that belongs to this specific entity +func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { + // Get all unique path parameters across operations + allParams := make(map[string]bool) + + for crudType, opInfo := range resource.Operations { + if crudType == "create" || crudType == "list" { + continue // Skip operations that typically don't have entity-specific IDs + } + + pathParams := extractPathParameters(opInfo.Path) + for _, param := range pathParams { + allParams[param] = true + } + } + + if len(allParams) == 0 { + return "", false // No path parameters found + } + + // Find the parameter that matches this entity + var entityPrimaryID string + matchCount := 0 + + for param := range allParams { + if mapsToEntityID(param, resource.EntityName) { + entityPrimaryID = param + matchCount++ + } + } + + if matchCount == 0 { + // No parameter maps to this entity - check for generic 'id' parameter + if allParams["id"] { + fmt.Printf(" Using generic 'id' parameter for entity %s\n", resource.EntityName) + return "id", true + } + fmt.Printf(" No parameter maps to entity %s\n", resource.EntityName) + return "", false + } + + if matchCount > 1 { + // Multiple parameters claim to map to this entity - ambiguous + fmt.Printf(" Multiple parameters map to entity %s: ambiguous primary ID\n", resource.EntityName) + return "", false + } + + fmt.Printf(" Identified primary ID '%s' for entity %s\n", entityPrimaryID, resource.EntityName) + return entityPrimaryID, true +} + +// Check if a parameter name maps to a specific entity's ID field +func mapsToEntityID(paramName, entityName string) bool { + // Extract base name from entity (e.g., "ChangeEvent" from "ChangeEventEntity") + entityBase := strings.TrimSuffix(entityName, "Entity") + + // Convert to snake_case and add _id suffix + expectedParam := toSnakeCase(entityBase) + "_id" + + return strings.ToLower(paramName) == strings.ToLower(expectedParam) +} + +// Check if parameter looks like an entity ID +func isEntityID(paramName string) bool { + return strings.HasSuffix(strings.ToLower(paramName), "_id") || strings.ToLower(paramName) == "id" +} + +func getOperationTypes(operations map[string]OperationInfo) []string { + var types []string + for opType := range operations { + types = append(types, opType) + } + return types +} From 507c0f1081789b156e62340f64de2649ea260965 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Thu, 12 Jun 2025 20:49:46 -0400 Subject: [PATCH 13/16] normalize enums --- scripts/normalize/enums.go | 280 ++++++++++++++++++++ scripts/normalize/main.go | 425 ++----------------------------- scripts/normalize/normalize.go | 246 ++++++++++++++++++ scripts/normalize/path-params.go | 78 ++++++ scripts/normalize/properties.go | 93 +++++++ 5 files changed, 720 insertions(+), 402 deletions(-) create mode 100644 scripts/normalize/enums.go create mode 100644 scripts/normalize/normalize.go create mode 100644 scripts/normalize/path-params.go create mode 100644 scripts/normalize/properties.go diff --git a/scripts/normalize/enums.go b/scripts/normalize/enums.go new file mode 100644 index 0000000..91c88db --- /dev/null +++ b/scripts/normalize/enums.go @@ -0,0 +1,280 @@ +package main + +import ( + "fmt" + "strings" + "unicode" +) + +// EnumProperty represents an enum found during normalization +type EnumNormalizationInfo struct { + SchemaName string + PropertyPath string + PropertyName string + EnumValues []string + Target map[string]interface{} +} + +// Main enum normalization function +func normalizeEnums(schemas map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("\n=== Normalizing Enum Properties ===\n") + + // Find all enum properties across all schemas + allEnums := findAllEnumProperties(schemas) + + if len(allEnums) == 0 { + fmt.Printf("No enum properties found to normalize\n") + return conflicts + } + + fmt.Printf("Found %d enum properties to normalize\n", len(allEnums)) + + // Transform each enum property + for _, enumInfo := range allEnums { + conflict := transformEnumProperty(enumInfo) + if conflict != nil { + conflicts = append(conflicts, *conflict) + } + } + + fmt.Printf("Successfully normalized %d enum properties\n", len(conflicts)) + return conflicts +} + +// Find all enum properties recursively across all schemas +func findAllEnumProperties(schemas map[string]interface{}) []EnumNormalizationInfo { + var allEnums []EnumNormalizationInfo + + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + schemaEnums := findEnumsInSchemaRecursive(schemaName, schemaMap, "", schemaMap) + allEnums = append(allEnums, schemaEnums...) + } + + return allEnums +} + +// Recursively find enum properties within a single schema +func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interface{}, path string, rootSchema map[string]interface{}) []EnumNormalizationInfo { + var enums []EnumNormalizationInfo + + // Check if current object has enum values + if enumValues, hasEnum := currentObj["enum"]; hasEnum { + if enumArray, ok := enumValues.([]interface{}); ok && len(enumArray) > 0 { + // Convert enum values to strings + var stringValues []string + for _, val := range enumArray { + if str, ok := val.(string); ok { + stringValues = append(stringValues, str) + } + } + + if len(stringValues) > 0 { + // Extract property name from path + propertyName := extractPropertyNameFromNormalizationPath(path, schemaName) + + enumInfo := EnumNormalizationInfo{ + SchemaName: schemaName, + PropertyPath: path, + PropertyName: propertyName, + EnumValues: stringValues, + Target: currentObj, // Reference to modify + } + + enums = append(enums, enumInfo) + fmt.Printf("📋 Found enum in %s.%s: %d values\n", schemaName, propertyName, len(stringValues)) + } + } + } + + // Recursively check properties + if properties, hasProps := currentObj["properties"].(map[string]interface{}); hasProps { + for propName, propValue := range properties { + if propMap, ok := propValue.(map[string]interface{}); ok { + newPath := propName + if path != "" { + newPath = fmt.Sprintf("%s.properties.%s", path, propName) + } else { + newPath = fmt.Sprintf("properties.%s", propName) + } + + propEnums := findEnumsInSchemaRecursive(schemaName, propMap, newPath, rootSchema) + enums = append(enums, propEnums...) + } + } + } + + // Check array items + if items, hasItems := currentObj["items"].(map[string]interface{}); hasItems { + newPath := path + if path != "" { + newPath = fmt.Sprintf("%s.items", path) + } else { + newPath = "items" + } + + itemEnums := findEnumsInSchemaRecursive(schemaName, items, newPath, rootSchema) + enums = append(enums, itemEnums...) + } + + return enums +} + +// Transform a single enum property +func transformEnumProperty(enumInfo EnumNormalizationInfo) *ConflictDetail { + // Generate x-speakeasy-enums members + var members []map[string]interface{} + + for _, value := range enumInfo.EnumValues { + memberName := generateEnumMemberName(value, enumInfo.PropertyName, enumInfo.SchemaName) + members = append(members, map[string]interface{}{ + "name": memberName, + "value": value, + }) + } + + // Remove original enum and add x-speakeasy-enums + delete(enumInfo.Target, "enum") + enumInfo.Target["x-speakeasy-enums"] = members + + // Create conflict detail for reporting + targetPath := enumInfo.SchemaName + if enumInfo.PropertyPath != "" { + targetPath = fmt.Sprintf("%s.%s", enumInfo.SchemaName, enumInfo.PropertyPath) + } + + return &ConflictDetail{ + Schema: enumInfo.SchemaName, + Property: enumInfo.PropertyName, + ConflictType: "enum-normalization", + Resolution: fmt.Sprintf("Replaced enum array with x-speakeasy-enums at %s (%d values)", targetPath, len(members)), + } +} + +// Generate unique enum member name using EntityName+FieldName+EnumValue +func generateEnumMemberName(value, fieldName, entityName string) string { + // Handle empty/whitespace-only values + originalValue := value + if strings.TrimSpace(value) == "" { + value = "Empty" + } + + // Clean and convert parts + cleanEntityName := convertToEnumMemberName(strings.TrimSuffix(entityName, "Entity")) + cleanFieldName := convertToEnumMemberName(fieldName) + cleanValue := convertToEnumMemberName(value) + + // Combine all three parts + memberName := cleanEntityName + cleanFieldName + cleanValue + + // Ensure it starts with a letter (Go requirement) + if len(memberName) > 0 && unicode.IsDigit(rune(memberName[0])) { + memberName = "Value" + memberName + } + + // Final fallback + if memberName == "" { + memberName = cleanEntityName + cleanFieldName + "Unknown" + } + + // Debug logging for empty strings + if originalValue == "" { + fmt.Printf("🔍 Empty string enum: entity=%s, field=%s, value='%s' -> memberName='%s'\n", + entityName, fieldName, originalValue, memberName) + } + + return memberName +} + +// Extract property name from normalization path +func extractPropertyNameFromNormalizationPath(path, schemaName string) string { + if path == "" { + // Schema-level enum, use schema name without Entity suffix + return strings.TrimSuffix(schemaName, "Entity") + } + + // Extract the last property name from the path + // Examples: + // "properties.status" -> "status" + // "properties.integration.properties.type" -> "type" + // "properties.alerts.items" -> "alerts" + + parts := strings.Split(path, ".") + + // Work backwards to find the actual property name + for i := len(parts) - 1; i >= 0; i-- { + part := parts[i] + + // Skip structural keywords + if part == "properties" || part == "items" { + continue + } + + // This should be the actual property name + return part + } + + // Fallback to schema name if we can't determine property name + return strings.TrimSuffix(schemaName, "Entity") +} + +// Convert arbitrary string to Go-style enum member name component +func convertToEnumMemberName(value string) string { + // Handle empty or whitespace-only strings + if strings.TrimSpace(value) == "" { + return "Empty" + } + + // Replace special characters with underscores + cleaned := strings.Map(func(r rune) rune { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + return r + } + return '_' + }, value) + + // Remove leading/trailing underscores and collapse multiple underscores + cleaned = strings.Trim(cleaned, "_") + // Simple approach to collapse multiple underscores + for strings.Contains(cleaned, "__") { + cleaned = strings.ReplaceAll(cleaned, "__", "_") + } + + // Split by underscores and convert to PascalCase + if cleaned == "" { + return "Empty" + } + + parts := strings.Split(cleaned, "_") + var result strings.Builder + + for _, part := range parts { + if len(part) > 0 { + // Capitalize first letter, lowercase the rest + result.WriteString(strings.ToUpper(string(part[0]))) + if len(part) > 1 { + result.WriteString(strings.ToLower(part[1:])) + } + } + } + + memberName := result.String() + + // Ensure it starts with a letter (Go requirement) + if len(memberName) > 0 && unicode.IsDigit(rune(memberName[0])) { + memberName = "Value" + memberName + } + + // Handle empty result after cleaning + if memberName == "" { + memberName = "Empty" + } + + return memberName +} diff --git a/scripts/normalize/main.go b/scripts/normalize/main.go index 1ec78c4..2fc2481 100644 --- a/scripts/normalize/main.go +++ b/scripts/normalize/main.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "os" - "strings" ) func main() { @@ -76,410 +75,32 @@ type ConflictDetail struct { Resolution string } -func normalizeSpec(spec map[string]interface{}) NormalizationReport { - report := NormalizationReport{ - ConflictDetails: make([]ConflictDetail, 0), - } - - components, ok := spec["components"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No components found in spec") - return report - } - - schemas, ok := components["schemas"].(map[string]interface{}) - if !ok { - fmt.Println("Warning: No schemas found in components") - return report - } - - paths, pathsOk := spec["paths"].(map[string]interface{}) - if !pathsOk { - fmt.Println("Warning: No paths found in spec") - } - - entityMap := buildEntityRelationships(schemas) - - for entityName, related := range entityMap { - fmt.Printf("Analyzing entity: %s\n", entityName) - - entitySchema, ok := schemas[entityName].(map[string]interface{}) - if !ok { - continue - } - - // Check against create schema - if related.CreateSchema != "" { - if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) - report.ConflictDetails = append(report.ConflictDetails, conflicts...) - } - } - - // Check against update schema - if related.UpdateSchema != "" { - if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) - report.ConflictDetails = append(report.ConflictDetails, conflicts...) - } - } - } - - // Apply global normalizations to schemas - globalFixes := applyGlobalNormalizations(schemas) - report.ConflictDetails = append(report.ConflictDetails, globalFixes...) - - // Normalize path parameters to match entity IDs - if pathsOk { - parameterFixes := normalizePathParameters(paths) - report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) - } - - // Calculate totals - report.TotalFixes = len(report.ConflictDetails) - for _, detail := range report.ConflictDetails { - if detail.ConflictType == "map-class" { - report.MapClassFixes++ - } else { - report.PropertyFixes++ - } - } - - return report -} - -type EntityRelationship struct { - EntityName string - CreateSchema string - UpdateSchema string -} - -func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { - relationships := make(map[string]EntityRelationship) - - for schemaName := range schemas { - if strings.HasSuffix(schemaName, "Entity") && !strings.Contains(schemaName, "Nullable") && !strings.Contains(schemaName, "Paginated") { - baseName := strings.ToLower(strings.TrimSuffix(schemaName, "Entity")) - - rel := EntityRelationship{ - EntityName: schemaName, - } - - createName := "create_" + baseName - if _, exists := schemas[createName]; exists { - rel.CreateSchema = createName - } - - updateName := "update_" + baseName - if _, exists := schemas[updateName]; exists { - rel.UpdateSchema = updateName - } - - relationships[schemaName] = rel - } - } - - return relationships -} - -func normalizeSchemas(entityName string, entitySchema map[string]interface{}, - requestName string, requestSchema map[string]interface{}) []ConflictDetail { - - conflicts := make([]ConflictDetail, 0) - - entityProps, _ := entitySchema["properties"].(map[string]interface{}) - requestProps, _ := requestSchema["properties"].(map[string]interface{}) - - if entityProps == nil || requestProps == nil { - return conflicts - } - - fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) - fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) - - // Check each property that exists in both schemas - // Terraform requires exact matches for properties across requests and responses - for propName, requestProp := range requestProps { - if entityProp, exists := entityProps[propName]; exists { - fmt.Printf(" Checking property: %s\n", propName) - conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) - if conflict != nil { - fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) - conflicts = append(conflicts, *conflict) - } - } - } - - return conflicts -} - -func checkAndFixProperty(entityName, propName string, entityProp, requestProp interface{}, - entityProps, requestProps map[string]interface{}) *ConflictDetail { - - entityPropMap, _ := entityProp.(map[string]interface{}) - requestPropMap, _ := requestProp.(map[string]interface{}) - - if entityPropMap == nil || requestPropMap == nil { - return nil - } - - // Check for map vs class conflict - entityType, _ := entityPropMap["type"].(string) - requestType, _ := requestPropMap["type"].(string) - - fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) - - if entityType == "object" && requestType == "object" { - _, entityHasProps := entityPropMap["properties"] - _, entityHasAdditional := entityPropMap["additionalProperties"] - _, requestHasProps := requestPropMap["properties"] - _, requestHasAdditional := requestPropMap["additionalProperties"] - - fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) - fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) - - if entityHasProps && requestHasProps { - entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) - requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) - - if len(entityPropsObj) == 0 && len(requestPropsObj) == 0 { - // already the same - both empty properties - return nil - } - } - - if entityHasAdditional && !requestHasAdditional && requestHasProps { - delete(entityPropMap, "additionalProperties") - entityPropMap["properties"] = map[string]interface{}{} - entityProps[propName] = entityPropMap - - return &ConflictDetail{ - Schema: entityName, - Property: propName, - ConflictType: "map-class", - Resolution: "Converted entity from additionalProperties to empty properties", - } - } - - if requestHasAdditional && !entityHasAdditional && entityHasProps { - delete(requestPropMap, "additionalProperties") - requestPropMap["properties"] = map[string]interface{}{} - requestProps[propName] = requestPropMap - - return &ConflictDetail{ - Schema: entityName, - Property: propName, - ConflictType: "map-class", - Resolution: "Converted request from additionalProperties to empty properties", - } - } - } - - return nil -} - -func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail { - conflicts := make([]ConflictDetail, 0) - - fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) - - fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") - if len(additionalPropsFound) > 0 { - fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) - for _, path := range additionalPropsFound { - fmt.Printf(" - %s\n", path) - } - } - } - - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - fmt.Printf(" Normalizing schema: %s\n", schemaName) - schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") - conflicts = append(conflicts, schemaConflicts...) - } - - return conflicts -} - -// Recursively find all additionalProperties in a schema -func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { - var found []string - - switch v := obj.(type) { - case map[string]interface{}: - if _, hasAdditional := v["additionalProperties"]; hasAdditional { - fullPath := schemaName - if path != "" { - fullPath += "." + path - } - found = append(found, fullPath) - } - - for key, value := range v { - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := findAllAdditionalProperties(schemaName, value, newPath) - found = append(found, nested...) - } - case []interface{}: - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := findAllAdditionalProperties(schemaName, item, newPath) - found = append(found, nested...) - } - } - - return found -} - -// Recursively normalize all additionalProperties in a schema -func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { - var conflicts []ConflictDetail - - switch v := obj.(type) { - case map[string]interface{}: - if _, hasAdditional := v["additionalProperties"]; hasAdditional { - objType, _ := v["type"].(string) - _, hasProperties := v["properties"] - - if objType == "object" || hasProperties || (!hasProperties && hasAdditional) { - delete(v, "additionalProperties") - if !hasProperties { - v["properties"] = map[string]interface{}{} - } - - fullPath := schemaName - if path != "" { - fullPath += "." + path - } - - conflicts = append(conflicts, ConflictDetail{ - Schema: schemaName, - Property: path, - ConflictType: "map-class", - Resolution: fmt.Sprintf("Converted additionalProperties to empty properties at %s", fullPath), - }) - - fmt.Printf(" ✅ Converted additionalProperties to empty properties at %s\n", fullPath) - } - } - - // Recursively normalize all nested objects - for key, value := range v { - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := normalizeAdditionalProperties(schemaName, value, newPath) - conflicts = append(conflicts, nested...) - } - case []interface{}: - // Normalize array items - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := normalizeAdditionalProperties(schemaName, item, newPath) - conflicts = append(conflicts, nested...) - } - } - - return conflicts -} - -// Normalize path parameters to match entity ID types -func normalizePathParameters(paths map[string]interface{}) []ConflictDetail { - conflicts := make([]ConflictDetail, 0) - - fmt.Printf("\n=== Normalizing Path Parameters ===\n") - - for pathName, pathItem := range paths { - pathMap, ok := pathItem.(map[string]interface{}) - if !ok { - continue - } - - methods := []string{"get", "post", "put", "patch", "delete"} - for _, method := range methods { - if operation, exists := pathMap[method]; exists { - opMap, ok := operation.(map[string]interface{}) - if !ok { - continue - } - - if parameters, hasParams := opMap["parameters"]; hasParams { - paramsList, ok := parameters.([]interface{}) - if !ok { - continue - } - - for _, param := range paramsList { - paramMap, ok := param.(map[string]interface{}) - if !ok { - continue - } - - // normailze int and string parameters - paramIn, _ := paramMap["in"].(string) - paramName, _ := paramMap["name"].(string) - - if paramIn == "path" && (strings.Contains(paramName, "id") || strings.HasSuffix(paramName, "_id")) { - schema, hasSchema := paramMap["schema"] - if hasSchema { - schemaMap, ok := schema.(map[string]interface{}) - if ok { - paramType, _ := schemaMap["type"].(string) - paramFormat, _ := schemaMap["format"].(string) - - if paramType == "integer" { - fmt.Printf(" Found integer ID parameter: %s %s.%s (type: %s, format: %s)\n", - method, pathName, paramName, paramType, paramFormat) - - schemaMap["type"] = "string" - delete(schemaMap, "format") - - conflicts = append(conflicts, ConflictDetail{ - Schema: fmt.Sprintf("path:%s", pathName), - Property: fmt.Sprintf("%s.%s", method, paramName), - ConflictType: "parameter-type", - Resolution: fmt.Sprintf("Converted path parameter %s from integer to string", paramName), - }) - - fmt.Printf(" ✅ Converted %s parameter from integer to string\n", paramName) - } - } - } - } - } - } - } - } - } - - return conflicts -} - func printNormalizationReport(report NormalizationReport) { fmt.Println("\n=== Normalization Report ===") fmt.Printf("Total fixes applied: %d\n", report.TotalFixes) - fmt.Printf("Map/Class fixes: %d\n", report.MapClassFixes) - fmt.Printf("Other property fixes: %d\n", report.PropertyFixes) + + mapClassFixes := 0 + parameterFixes := 0 + enumFixes := 0 + otherFixes := 0 + + for _, detail := range report.ConflictDetails { + switch detail.ConflictType { + case "map-class": + mapClassFixes++ + case "parameter-type": + parameterFixes++ + case "enum-normalization": + enumFixes++ + default: + otherFixes++ + } + } + + fmt.Printf("Map/Class fixes: %d\n", mapClassFixes) + fmt.Printf("Parameter type fixes: %d\n", parameterFixes) + fmt.Printf("Enum normalization fixes: %d\n", enumFixes) + fmt.Printf("Other fixes: %d\n", otherFixes) if len(report.ConflictDetails) > 0 { fmt.Println("\nDetailed fixes:") diff --git a/scripts/normalize/normalize.go b/scripts/normalize/normalize.go new file mode 100644 index 0000000..873570d --- /dev/null +++ b/scripts/normalize/normalize.go @@ -0,0 +1,246 @@ +package main + +import ( + "fmt" + "strings" +) + +func normalizeSpec(spec map[string]interface{}) NormalizationReport { + report := NormalizationReport{ + ConflictDetails: make([]ConflictDetail, 0), + } + + components, ok := spec["components"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No components found in spec") + return report + } + + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + fmt.Println("Warning: No schemas found in components") + return report + } + + paths, pathsOk := spec["paths"].(map[string]interface{}) + if !pathsOk { + fmt.Println("Warning: No paths found in spec") + } + + entityMap := buildEntityRelationships(schemas) + + for entityName, related := range entityMap { + fmt.Printf("Analyzing entity: %s\n", entityName) + + entitySchema, ok := schemas[entityName].(map[string]interface{}) + if !ok { + continue + } + + // Check against create schema + if related.CreateSchema != "" { + if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + + // Check against update schema + if related.UpdateSchema != "" { + if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { + conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) + report.ConflictDetails = append(report.ConflictDetails, conflicts...) + } + } + } + + // Apply global normalizations to schemas + globalFixes := applyGlobalNormalizations(schemas) + report.ConflictDetails = append(report.ConflictDetails, globalFixes...) + + enumFixes := normalizeEnums(schemas) + report.ConflictDetails = append(report.ConflictDetails, enumFixes...) + + // Normalize path parameters to match entity IDs + if pathsOk { + parameterFixes := normalizePathParameters(paths) + report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) + } + + // Calculate totals + report.TotalFixes = len(report.ConflictDetails) + for _, detail := range report.ConflictDetails { + if detail.ConflictType == "map-class" { + report.MapClassFixes++ + } else { + report.PropertyFixes++ + } + } + + return report +} + +func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) + + fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") + if len(additionalPropsFound) > 0 { + fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) + for _, path := range additionalPropsFound { + fmt.Printf(" - %s\n", path) + } + } + } + + for schemaName, schema := range schemas { + schemaMap, ok := schema.(map[string]interface{}) + if !ok { + continue + } + + fmt.Printf(" Normalizing schema: %s\n", schemaName) + schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") + conflicts = append(conflicts, schemaConflicts...) + } + + return conflicts +} + +type EntityRelationship struct { + EntityName string + CreateSchema string + UpdateSchema string +} + +func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityRelationship { + relationships := make(map[string]EntityRelationship) + + for schemaName := range schemas { + if strings.HasSuffix(schemaName, "Entity") && !strings.Contains(schemaName, "Nullable") && !strings.Contains(schemaName, "Paginated") { + baseName := strings.ToLower(strings.TrimSuffix(schemaName, "Entity")) + + rel := EntityRelationship{ + EntityName: schemaName, + } + + createName := "create_" + baseName + if _, exists := schemas[createName]; exists { + rel.CreateSchema = createName + } + + updateName := "update_" + baseName + if _, exists := schemas[updateName]; exists { + rel.UpdateSchema = updateName + } + + relationships[schemaName] = rel + } + } + + return relationships +} + +func normalizeSchemas(entityName string, entitySchema map[string]interface{}, + requestName string, requestSchema map[string]interface{}) []ConflictDetail { + + conflicts := make([]ConflictDetail, 0) + + entityProps, _ := entitySchema["properties"].(map[string]interface{}) + requestProps, _ := requestSchema["properties"].(map[string]interface{}) + + if entityProps == nil || requestProps == nil { + return conflicts + } + + fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) + fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) + + // Check each property that exists in both schemas + // Terraform requires exact matches for properties across requests and responses + for propName, requestProp := range requestProps { + if entityProp, exists := entityProps[propName]; exists { + fmt.Printf(" Checking property: %s\n", propName) + conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) + if conflict != nil { + fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) + conflicts = append(conflicts, *conflict) + } + } + } + + return conflicts +} + +func checkAndFixProperty(entityName, propName string, entityProp, requestProp interface{}, + entityProps, requestProps map[string]interface{}) *ConflictDetail { + + entityPropMap, _ := entityProp.(map[string]interface{}) + requestPropMap, _ := requestProp.(map[string]interface{}) + + if entityPropMap == nil || requestPropMap == nil { + return nil + } + + // Check for map vs class conflict + entityType, _ := entityPropMap["type"].(string) + requestType, _ := requestPropMap["type"].(string) + + fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) + + if entityType == "object" && requestType == "object" { + _, entityHasProps := entityPropMap["properties"] + _, entityHasAdditional := entityPropMap["additionalProperties"] + _, requestHasProps := requestPropMap["properties"] + _, requestHasAdditional := requestPropMap["additionalProperties"] + + fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) + fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) + + if entityHasProps && requestHasProps { + entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) + requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) + + if len(entityPropsObj) == 0 && len(requestPropsObj) == 0 { + // already the same - both empty properties + return nil + } + } + + if entityHasAdditional && !requestHasAdditional && requestHasProps { + delete(entityPropMap, "additionalProperties") + entityPropMap["properties"] = map[string]interface{}{} + entityProps[propName] = entityPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted entity from additionalProperties to empty properties", + } + } + + if requestHasAdditional && !entityHasAdditional && entityHasProps { + delete(requestPropMap, "additionalProperties") + requestPropMap["properties"] = map[string]interface{}{} + requestProps[propName] = requestPropMap + + return &ConflictDetail{ + Schema: entityName, + Property: propName, + ConflictType: "map-class", + Resolution: "Converted request from additionalProperties to empty properties", + } + } + } + + return nil +} diff --git a/scripts/normalize/path-params.go b/scripts/normalize/path-params.go new file mode 100644 index 0000000..7718747 --- /dev/null +++ b/scripts/normalize/path-params.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "strings" +) + +// Normalize path parameters to match entity ID types +func normalizePathParameters(paths map[string]interface{}) []ConflictDetail { + conflicts := make([]ConflictDetail, 0) + + fmt.Printf("\n=== Normalizing Path Parameters ===\n") + + for pathName, pathItem := range paths { + pathMap, ok := pathItem.(map[string]interface{}) + if !ok { + continue + } + + methods := []string{"get", "post", "put", "patch", "delete"} + for _, method := range methods { + if operation, exists := pathMap[method]; exists { + opMap, ok := operation.(map[string]interface{}) + if !ok { + continue + } + + if parameters, hasParams := opMap["parameters"]; hasParams { + paramsList, ok := parameters.([]interface{}) + if !ok { + continue + } + + for _, param := range paramsList { + paramMap, ok := param.(map[string]interface{}) + if !ok { + continue + } + + // normailze int and string parameters + paramIn, _ := paramMap["in"].(string) + paramName, _ := paramMap["name"].(string) + + if paramIn == "path" && (strings.Contains(paramName, "id") || strings.HasSuffix(paramName, "_id")) { + schema, hasSchema := paramMap["schema"] + if hasSchema { + schemaMap, ok := schema.(map[string]interface{}) + if ok { + paramType, _ := schemaMap["type"].(string) + paramFormat, _ := schemaMap["format"].(string) + + if paramType == "integer" { + fmt.Printf(" Found integer ID parameter: %s %s.%s (type: %s, format: %s)\n", + method, pathName, paramName, paramType, paramFormat) + + schemaMap["type"] = "string" + delete(schemaMap, "format") + + conflicts = append(conflicts, ConflictDetail{ + Schema: fmt.Sprintf("path:%s", pathName), + Property: fmt.Sprintf("%s.%s", method, paramName), + ConflictType: "parameter-type", + Resolution: fmt.Sprintf("Converted path parameter %s from integer to string", paramName), + }) + + fmt.Printf(" ✅ Converted %s parameter from integer to string\n", paramName) + } + } + } + } + } + } + } + } + } + + return conflicts +} diff --git a/scripts/normalize/properties.go b/scripts/normalize/properties.go new file mode 100644 index 0000000..4d47cca --- /dev/null +++ b/scripts/normalize/properties.go @@ -0,0 +1,93 @@ +package main + +import "fmt" + +// Recursively find all additionalProperties in a schema +func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { + var found []string + + switch v := obj.(type) { + case map[string]interface{}: + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + fullPath := schemaName + if path != "" { + fullPath += "." + path + } + found = append(found, fullPath) + } + + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key + } + nested := findAllAdditionalProperties(schemaName, value, newPath) + found = append(found, nested...) + } + case []interface{}: + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := findAllAdditionalProperties(schemaName, item, newPath) + found = append(found, nested...) + } + } + + return found +} + +// Recursively normalize all additionalProperties in a schema +func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { + var conflicts []ConflictDetail + + switch v := obj.(type) { + case map[string]interface{}: + if _, hasAdditional := v["additionalProperties"]; hasAdditional { + objType, _ := v["type"].(string) + _, hasProperties := v["properties"] + + if objType == "object" || hasProperties || (!hasProperties && hasAdditional) { + delete(v, "additionalProperties") + if !hasProperties { + v["properties"] = map[string]interface{}{} + } + + fullPath := schemaName + if path != "" { + fullPath += "." + path + } + + conflicts = append(conflicts, ConflictDetail{ + Schema: schemaName, + Property: path, + ConflictType: "map-class", + Resolution: fmt.Sprintf("Converted additionalProperties to empty properties at %s", fullPath), + }) + + fmt.Printf(" ✅ Converted additionalProperties to empty properties at %s\n", fullPath) + } + } + + // Recursively normalize all nested objects + for key, value := range v { + newPath := path + if newPath != "" { + newPath += "." + key + } else { + newPath = key + } + nested := normalizeAdditionalProperties(schemaName, value, newPath) + conflicts = append(conflicts, nested...) + } + case []interface{}: + // Normalize array items + for i, item := range v { + newPath := fmt.Sprintf("%s[%d]", path, i) + nested := normalizeAdditionalProperties(schemaName, item, newPath) + conflicts = append(conflicts, nested...) + } + } + + return conflicts +} From 3f50abb2c5fb6eb1910f2dde7edc8c73ec62d538 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Fri, 13 Jun 2025 09:54:43 -0400 Subject: [PATCH 14/16] cleanup normalization debug output --- scripts/normalize/enums.go | 58 +++------------------------------ scripts/normalize/main.go | 15 +++++---- scripts/normalize/normalize.go | 42 +++--------------------- scripts/normalize/properties.go | 37 --------------------- 4 files changed, 16 insertions(+), 136 deletions(-) diff --git a/scripts/normalize/enums.go b/scripts/normalize/enums.go index 91c88db..990a9d4 100644 --- a/scripts/normalize/enums.go +++ b/scripts/normalize/enums.go @@ -6,7 +6,6 @@ import ( "unicode" ) -// EnumProperty represents an enum found during normalization type EnumNormalizationInfo struct { SchemaName string PropertyPath string @@ -15,13 +14,11 @@ type EnumNormalizationInfo struct { Target map[string]interface{} } -// Main enum normalization function func normalizeEnums(schemas map[string]interface{}) []ConflictDetail { conflicts := make([]ConflictDetail, 0) fmt.Printf("\n=== Normalizing Enum Properties ===\n") - // Find all enum properties across all schemas allEnums := findAllEnumProperties(schemas) if len(allEnums) == 0 { @@ -31,7 +28,6 @@ func normalizeEnums(schemas map[string]interface{}) []ConflictDetail { fmt.Printf("Found %d enum properties to normalize\n", len(allEnums)) - // Transform each enum property for _, enumInfo := range allEnums { conflict := transformEnumProperty(enumInfo) if conflict != nil { @@ -43,7 +39,6 @@ func normalizeEnums(schemas map[string]interface{}) []ConflictDetail { return conflicts } -// Find all enum properties recursively across all schemas func findAllEnumProperties(schemas map[string]interface{}) []EnumNormalizationInfo { var allEnums []EnumNormalizationInfo @@ -60,14 +55,11 @@ func findAllEnumProperties(schemas map[string]interface{}) []EnumNormalizationIn return allEnums } -// Recursively find enum properties within a single schema func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interface{}, path string, rootSchema map[string]interface{}) []EnumNormalizationInfo { var enums []EnumNormalizationInfo - // Check if current object has enum values if enumValues, hasEnum := currentObj["enum"]; hasEnum { if enumArray, ok := enumValues.([]interface{}); ok && len(enumArray) > 0 { - // Convert enum values to strings var stringValues []string for _, val := range enumArray { if str, ok := val.(string); ok { @@ -76,7 +68,6 @@ func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interfa } if len(stringValues) > 0 { - // Extract property name from path propertyName := extractPropertyNameFromNormalizationPath(path, schemaName) enumInfo := EnumNormalizationInfo{ @@ -84,16 +75,14 @@ func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interfa PropertyPath: path, PropertyName: propertyName, EnumValues: stringValues, - Target: currentObj, // Reference to modify + Target: currentObj, } enums = append(enums, enumInfo) - fmt.Printf("📋 Found enum in %s.%s: %d values\n", schemaName, propertyName, len(stringValues)) } } } - // Recursively check properties if properties, hasProps := currentObj["properties"].(map[string]interface{}); hasProps { for propName, propValue := range properties { if propMap, ok := propValue.(map[string]interface{}); ok { @@ -110,7 +99,6 @@ func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interfa } } - // Check array items if items, hasItems := currentObj["items"].(map[string]interface{}); hasItems { newPath := path if path != "" { @@ -126,9 +114,7 @@ func findEnumsInSchemaRecursive(schemaName string, currentObj map[string]interfa return enums } -// Transform a single enum property func transformEnumProperty(enumInfo EnumNormalizationInfo) *ConflictDetail { - // Generate x-speakeasy-enums members var members []map[string]interface{} for _, value := range enumInfo.EnumValues { @@ -139,11 +125,9 @@ func transformEnumProperty(enumInfo EnumNormalizationInfo) *ConflictDetail { }) } - // Remove original enum and add x-speakeasy-enums delete(enumInfo.Target, "enum") enumInfo.Target["x-speakeasy-enums"] = members - // Create conflict detail for reporting targetPath := enumInfo.SchemaName if enumInfo.PropertyPath != "" { targetPath = fmt.Sprintf("%s.%s", enumInfo.SchemaName, enumInfo.PropertyPath) @@ -157,81 +141,51 @@ func transformEnumProperty(enumInfo EnumNormalizationInfo) *ConflictDetail { } } -// Generate unique enum member name using EntityName+FieldName+EnumValue func generateEnumMemberName(value, fieldName, entityName string) string { - // Handle empty/whitespace-only values - originalValue := value - if strings.TrimSpace(value) == "" { - value = "Empty" - } - - // Clean and convert parts cleanEntityName := convertToEnumMemberName(strings.TrimSuffix(entityName, "Entity")) cleanFieldName := convertToEnumMemberName(fieldName) cleanValue := convertToEnumMemberName(value) - // Combine all three parts memberName := cleanEntityName + cleanFieldName + cleanValue - // Ensure it starts with a letter (Go requirement) + // Ensure it starts with a letter if len(memberName) > 0 && unicode.IsDigit(rune(memberName[0])) { memberName = "Value" + memberName } - // Final fallback + // Handle empty result after cleaning if memberName == "" { memberName = cleanEntityName + cleanFieldName + "Unknown" } - // Debug logging for empty strings - if originalValue == "" { - fmt.Printf("🔍 Empty string enum: entity=%s, field=%s, value='%s' -> memberName='%s'\n", - entityName, fieldName, originalValue, memberName) - } - return memberName } -// Extract property name from normalization path func extractPropertyNameFromNormalizationPath(path, schemaName string) string { if path == "" { - // Schema-level enum, use schema name without Entity suffix return strings.TrimSuffix(schemaName, "Entity") } - // Extract the last property name from the path - // Examples: - // "properties.status" -> "status" - // "properties.integration.properties.type" -> "type" - // "properties.alerts.items" -> "alerts" - parts := strings.Split(path, ".") - // Work backwards to find the actual property name for i := len(parts) - 1; i >= 0; i-- { part := parts[i] - // Skip structural keywords if part == "properties" || part == "items" { continue } - // This should be the actual property name return part } - // Fallback to schema name if we can't determine property name return strings.TrimSuffix(schemaName, "Entity") } -// Convert arbitrary string to Go-style enum member name component func convertToEnumMemberName(value string) string { - // Handle empty or whitespace-only strings if strings.TrimSpace(value) == "" { return "Empty" } - // Replace special characters with underscores cleaned := strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return r @@ -239,14 +193,11 @@ func convertToEnumMemberName(value string) string { return '_' }, value) - // Remove leading/trailing underscores and collapse multiple underscores cleaned = strings.Trim(cleaned, "_") - // Simple approach to collapse multiple underscores for strings.Contains(cleaned, "__") { cleaned = strings.ReplaceAll(cleaned, "__", "_") } - // Split by underscores and convert to PascalCase if cleaned == "" { return "Empty" } @@ -256,7 +207,6 @@ func convertToEnumMemberName(value string) string { for _, part := range parts { if len(part) > 0 { - // Capitalize first letter, lowercase the rest result.WriteString(strings.ToUpper(string(part[0]))) if len(part) > 1 { result.WriteString(strings.ToLower(part[1:])) @@ -266,7 +216,7 @@ func convertToEnumMemberName(value string) string { memberName := result.String() - // Ensure it starts with a letter (Go requirement) + // Ensure it starts with a letter if len(memberName) > 0 && unicode.IsDigit(rune(memberName[0])) { memberName = "Value" + memberName } diff --git a/scripts/normalize/main.go b/scripts/normalize/main.go index 2fc2481..e51ac6b 100644 --- a/scripts/normalize/main.go +++ b/scripts/normalize/main.go @@ -102,11 +102,12 @@ func printNormalizationReport(report NormalizationReport) { fmt.Printf("Enum normalization fixes: %d\n", enumFixes) fmt.Printf("Other fixes: %d\n", otherFixes) - if len(report.ConflictDetails) > 0 { - fmt.Println("\nDetailed fixes:") - for _, detail := range report.ConflictDetails { - fmt.Printf(" - %s.%s [%s]: %s\n", - detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) - } - } + // Helpful for debugging + // if len(report.ConflictDetails) > 0 { + // fmt.Println("\nDetailed fixes:") + // for _, detail := range report.ConflictDetails { + // fmt.Printf(" - %s.%s [%s]: %s\n", + // detail.Schema, detail.Property, detail.ConflictType, detail.Resolution) + // } + // } } diff --git a/scripts/normalize/normalize.go b/scripts/normalize/normalize.go index 873570d..c4ef54b 100644 --- a/scripts/normalize/normalize.go +++ b/scripts/normalize/normalize.go @@ -30,44 +30,38 @@ func normalizeSpec(spec map[string]interface{}) NormalizationReport { entityMap := buildEntityRelationships(schemas) for entityName, related := range entityMap { - fmt.Printf("Analyzing entity: %s\n", entityName) entitySchema, ok := schemas[entityName].(map[string]interface{}) if !ok { continue } - // Check against create schema if related.CreateSchema != "" { if createSchema, ok := schemas[related.CreateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.CreateSchema, createSchema) + conflicts := normalizeSchemas(entityName, entitySchema, createSchema) report.ConflictDetails = append(report.ConflictDetails, conflicts...) } } - // Check against update schema if related.UpdateSchema != "" { if updateSchema, ok := schemas[related.UpdateSchema].(map[string]interface{}); ok { - conflicts := normalizeSchemas(entityName, entitySchema, related.UpdateSchema, updateSchema) + conflicts := normalizeSchemas(entityName, entitySchema, updateSchema) report.ConflictDetails = append(report.ConflictDetails, conflicts...) } } } - // Apply global normalizations to schemas globalFixes := applyGlobalNormalizations(schemas) report.ConflictDetails = append(report.ConflictDetails, globalFixes...) enumFixes := normalizeEnums(schemas) report.ConflictDetails = append(report.ConflictDetails, enumFixes...) - // Normalize path parameters to match entity IDs if pathsOk { parameterFixes := normalizePathParameters(paths) report.ConflictDetails = append(report.ConflictDetails, parameterFixes...) } - // Calculate totals report.TotalFixes = len(report.ConflictDetails) for _, detail := range report.ConflictDetails { if detail.ConflictType == "map-class" { @@ -85,29 +79,12 @@ func applyGlobalNormalizations(schemas map[string]interface{}) []ConflictDetail fmt.Printf("Applying global normalizations to %d schemas\n", len(schemas)) - fmt.Printf("\n=== Scanning for all additionalProperties instances ===\n") for schemaName, schema := range schemas { schemaMap, ok := schema.(map[string]interface{}) if !ok { continue } - additionalPropsFound := findAllAdditionalProperties(schemaName, schemaMap, "") - if len(additionalPropsFound) > 0 { - fmt.Printf("Schema %s has additionalProperties at:\n", schemaName) - for _, path := range additionalPropsFound { - fmt.Printf(" - %s\n", path) - } - } - } - - for schemaName, schema := range schemas { - schemaMap, ok := schema.(map[string]interface{}) - if !ok { - continue - } - - fmt.Printf(" Normalizing schema: %s\n", schemaName) schemaConflicts := normalizeAdditionalProperties(schemaName, schemaMap, "") conflicts = append(conflicts, schemaConflicts...) } @@ -149,8 +126,7 @@ func buildEntityRelationships(schemas map[string]interface{}) map[string]EntityR return relationships } -func normalizeSchemas(entityName string, entitySchema map[string]interface{}, - requestName string, requestSchema map[string]interface{}) []ConflictDetail { +func normalizeSchemas(entityName string, entitySchema map[string]interface{}, requestSchema map[string]interface{}) []ConflictDetail { conflicts := make([]ConflictDetail, 0) @@ -161,17 +137,12 @@ func normalizeSchemas(entityName string, entitySchema map[string]interface{}, return conflicts } - fmt.Printf(" Comparing %s vs %s\n", entityName, requestName) - fmt.Printf(" Entity properties: %d, Request properties: %d\n", len(entityProps), len(requestProps)) - // Check each property that exists in both schemas // Terraform requires exact matches for properties across requests and responses for propName, requestProp := range requestProps { if entityProp, exists := entityProps[propName]; exists { - fmt.Printf(" Checking property: %s\n", propName) conflict := checkAndFixProperty(entityName, propName, entityProp, requestProp, entityProps, requestProps) if conflict != nil { - fmt.Printf(" ✅ Fixed: %s - %s\n", propName, conflict.Resolution) conflicts = append(conflicts, *conflict) } } @@ -190,21 +161,16 @@ func checkAndFixProperty(entityName, propName string, entityProp, requestProp in return nil } - // Check for map vs class conflict + // Check for map vs class conflict - event if these are the same shape, they need to be the same type or generation will fail entityType, _ := entityPropMap["type"].(string) requestType, _ := requestPropMap["type"].(string) - fmt.Printf(" Property types - Entity: %s, Request: %s\n", entityType, requestType) - if entityType == "object" && requestType == "object" { _, entityHasProps := entityPropMap["properties"] _, entityHasAdditional := entityPropMap["additionalProperties"] _, requestHasProps := requestPropMap["properties"] _, requestHasAdditional := requestPropMap["additionalProperties"] - fmt.Printf(" Entity - hasProps: %v, hasAdditional: %v\n", entityHasProps, entityHasAdditional) - fmt.Printf(" Request - hasProps: %v, hasAdditional: %v\n", requestHasProps, requestHasAdditional) - if entityHasProps && requestHasProps { entityPropsObj, _ := entityPropMap["properties"].(map[string]interface{}) requestPropsObj, _ := requestPropMap["properties"].(map[string]interface{}) diff --git a/scripts/normalize/properties.go b/scripts/normalize/properties.go index 4d47cca..d3ea1cc 100644 --- a/scripts/normalize/properties.go +++ b/scripts/normalize/properties.go @@ -2,42 +2,6 @@ package main import "fmt" -// Recursively find all additionalProperties in a schema -func findAllAdditionalProperties(schemaName string, obj interface{}, path string) []string { - var found []string - - switch v := obj.(type) { - case map[string]interface{}: - if _, hasAdditional := v["additionalProperties"]; hasAdditional { - fullPath := schemaName - if path != "" { - fullPath += "." + path - } - found = append(found, fullPath) - } - - for key, value := range v { - newPath := path - if newPath != "" { - newPath += "." + key - } else { - newPath = key - } - nested := findAllAdditionalProperties(schemaName, value, newPath) - found = append(found, nested...) - } - case []interface{}: - for i, item := range v { - newPath := fmt.Sprintf("%s[%d]", path, i) - nested := findAllAdditionalProperties(schemaName, item, newPath) - found = append(found, nested...) - } - } - - return found -} - -// Recursively normalize all additionalProperties in a schema func normalizeAdditionalProperties(schemaName string, obj interface{}, path string) []ConflictDetail { var conflicts []ConflictDetail @@ -69,7 +33,6 @@ func normalizeAdditionalProperties(schemaName string, obj interface{}, path stri } } - // Recursively normalize all nested objects for key, value := range v { newPath := path if newPath != "" { From 7493d213fba7b889640ae58bc8920b41829b8bf3 Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Fri, 13 Jun 2025 12:34:16 -0400 Subject: [PATCH 15/16] support put based creation, slug based entities --- scripts/overlay/analyze-spec.go | 66 +++++---------- scripts/overlay/detect.go | 4 +- scripts/overlay/generate-overlay.go | 68 ++++++--------- scripts/overlay/main.go | 1 - scripts/overlay/manual-mappings.go | 12 --- scripts/overlay/terraform-viable.go | 125 ++++++++++------------------ 6 files changed, 93 insertions(+), 183 deletions(-) diff --git a/scripts/overlay/analyze-spec.go b/scripts/overlay/analyze-spec.go index ac591e4..21c76bb 100644 --- a/scripts/overlay/analyze-spec.go +++ b/scripts/overlay/analyze-spec.go @@ -35,23 +35,7 @@ func analyzeSpec(spec OpenAPISpec, manualMappings *ManualMappings) map[string]*R // Second pass: match operations to entities for path, pathItem := range spec.Paths { - analyzePathOperations(path, pathItem, entitySchemas, resources, spec, manualMappings) - } - - // Third pass: validate resources but keep all for analysis - fmt.Printf("\n=== Resource Validation ===\n") - for name, resource := range resources { - opTypes := make([]string, 0) - for crudType := range resource.Operations { - opTypes = append(opTypes, crudType) - } - fmt.Printf("Resource: %s with operations: %v\n", name, opTypes) - - if isTerraformViable(resource, spec) { - fmt.Printf(" ✅ Viable for Terraform\n") - } else { - fmt.Printf(" ❌ Not viable for Terraform - will skip annotations\n") - } + analyzePathOperations(path, pathItem, entitySchemas, resources, manualMappings) } return resources @@ -70,7 +54,6 @@ func identifyEntitySchemas(schemas map[string]Schema) map[string]bool { } func isEntitySchema(name string, schema Schema) bool { - // Skip request/response wrappers lowerName := strings.ToLower(name) if strings.HasPrefix(lowerName, "create_") || strings.HasPrefix(lowerName, "update_") || @@ -81,28 +64,26 @@ func isEntitySchema(name string, schema Schema) bool { return false } - // Skip nullable wrapper schemas + // Many of our schemas are nullable wrappers, to enable optional child schemas as properties in our sdks + // skip these if strings.HasPrefix(name, "Nullable") { return false } - // Must be an object with properties if schema.Type != "object" || len(schema.Properties) == 0 { return false } - // Entities should have an id property and end with "Entity" _, hasID := schema.Properties["id"] _, hasSlug := schema.Properties["slug"] hasSuffix := strings.HasSuffix(name, "Entity") hasIdentifier := hasID || hasSlug - // Be strict: require both conditions - return hasIdentifier && hasSuffix + return hasSuffix && hasIdentifier } func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[string]bool, - resources map[string]*ResourceInfo, spec OpenAPISpec, manualMappings *ManualMappings) { + resources map[string]*ResourceInfo, manualMappings *ManualMappings) { operations := []struct { method string @@ -120,20 +101,17 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str continue } - // Check if this operation should be manually ignored if shouldIgnoreOperation(path, item.method, manualMappings) { continue } - resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, spec, manualMappings) + resourceInfo := extractResourceInfo(path, item.method, item.op, entitySchemas, manualMappings) if resourceInfo != nil { if existing, exists := resources[resourceInfo.ResourceName]; exists { - // Merge operations for opType, opInfo := range resourceInfo.Operations { existing.Operations[opType] = opInfo } - // Preserve create/update schema info - don't overwrite with empty values if resourceInfo.CreateSchema != "" { existing.CreateSchema = resourceInfo.CreateSchema } @@ -148,19 +126,16 @@ func analyzePathOperations(path string, pathItem PathItem, entitySchemas map[str } func extractResourceInfo(path, method string, op *Operation, - entitySchemas map[string]bool, spec OpenAPISpec, manualMappings *ManualMappings) *ResourceInfo { + entitySchemas map[string]bool, manualMappings *ManualMappings) *ResourceInfo { - // Determine CRUD type crudType := determineCrudType(path, method, op.OperationID) if crudType == "" { return nil } - // Check for manual entity mapping first if manualEntityName, hasManual := getManualEntityMapping(path, method, manualMappings); hasManual { - // Use manual entity mapping entityName := manualEntityName - resourceName := deriveResourceName(entityName, op.OperationID, path) + resourceName := deriveResourceName(entityName) info := &ResourceInfo{ EntityName: entityName, @@ -175,7 +150,6 @@ func extractResourceInfo(path, method string, op *Operation, Method: method, } - // Extract request schema for create/update operations if crudType == "create" || crudType == "update" { if op.RequestBody != nil { if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { @@ -201,13 +175,12 @@ func extractResourceInfo(path, method string, op *Operation, return info } - // Find associated entity schema using automatic detection - entityName := findEntityFromOperation(op, entitySchemas, spec) + entityName := findEntityFromOperation(op, entitySchemas) if entityName == "" { return nil } - resourceName := deriveResourceName(entityName, op.OperationID, path) + resourceName := deriveResourceName(entityName) info := &ResourceInfo{ EntityName: entityName, @@ -222,7 +195,6 @@ func extractResourceInfo(path, method string, op *Operation, Method: method, } - // Extract request schema for create/update operations if crudType == "create" || crudType == "update" { if op.RequestBody != nil { if content, ok := op.RequestBody["content"].(map[string]interface{}); ok { @@ -251,7 +223,6 @@ func extractResourceInfo(path, method string, op *Operation, func determineCrudType(path, method, operationID string) string { lowerOp := strings.ToLower(operationID) - // Check operation ID first if strings.Contains(lowerOp, "create") { return "create" } @@ -268,7 +239,6 @@ func determineCrudType(path, method, operationID string) string { return "read" } - // Fallback to method-based detection switch method { case "post": if !strings.Contains(path, "{") { @@ -280,7 +250,13 @@ func determineCrudType(path, method, operationID string) string { } else { return "list" } - case "patch", "put": + case "put": + if !strings.Contains(path, "{") { + return "create" + } + + return "update" + case "patch": return "update" case "delete": return "delete" @@ -289,8 +265,7 @@ func determineCrudType(path, method, operationID string) string { return "" } -func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec OpenAPISpec) string { - // Check response schemas first +func findEntityFromOperation(op *Operation, entitySchemas map[string]bool) string { if op.Responses != nil { for _, response := range op.Responses { if respMap, ok := response.(map[string]interface{}); ok { @@ -308,7 +283,6 @@ func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec } } - // Check tags if len(op.Tags) > 0 { for _, tag := range op.Tags { possibleEntity := tag + "Entity" @@ -322,7 +296,6 @@ func findEntityFromOperation(op *Operation, entitySchemas map[string]bool, spec } func findEntityInSchema(schema map[string]interface{}, entitySchemas map[string]bool) string { - // Direct reference if ref, ok := schema["$ref"].(string); ok { schemaName := extractSchemaName(ref) if entitySchemas[schemaName] { @@ -330,7 +303,6 @@ func findEntityInSchema(schema map[string]interface{}, entitySchemas map[string] } } - // Check in data array for paginated responses if props, ok := schema["properties"].(map[string]interface{}); ok { if data, ok := props["data"].(map[string]interface{}); ok { if dataType, ok := data["type"].(string); ok && dataType == "array" { @@ -357,7 +329,7 @@ func extractSchemaName(ref string) string { return "" } -func deriveResourceName(entityName, operationID, path string) string { +func deriveResourceName(entityName string) string { resource := strings.TrimSuffix(entityName, "Entity") resource = toSnakeCase(resource) diff --git a/scripts/overlay/detect.go b/scripts/overlay/detect.go index 8432e23..c611478 100644 --- a/scripts/overlay/detect.go +++ b/scripts/overlay/detect.go @@ -214,10 +214,10 @@ func detectSchemaPropertyInconsistencies(resource *ResourceInfo, schemas map[str // First, validate that we have the minimum required operations for Terraform _, hasCreate := resource.Operations["create"] - _, hasPut := resource.Operations["put"] + // _, hasPut := resource.Operations["put"] _, hasRead := resource.Operations["read"] - createOrPut := hasCreate || hasPut + createOrPut := hasCreate //|| hasPut if !createOrPut || !hasRead { // Return a fundamental inconsistency - resource is not viable for Terraform inconsistency := CRUDInconsistency{ diff --git a/scripts/overlay/generate-overlay.go b/scripts/overlay/generate-overlay.go index e511685..e1aa0bd 100644 --- a/scripts/overlay/generate-overlay.go +++ b/scripts/overlay/generate-overlay.go @@ -1,6 +1,9 @@ package main -import "fmt" +import ( + "fmt" + "strings" +) type OverlayAction struct { Target string `yaml:"target"` @@ -27,13 +30,16 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua overlay.Info.Description = "Auto-generated overlay for Terraform resources" // Clean up resources by removing manually ignored operations - mappedResources := applyManualMappings(resources, manualMappings) - // Separate viable and non-viable resources + // In general, we don't need to alter the spec at this point + // We can simply not apply x-speakeasy extentions to these entities + // Without the extensions, they will not be registered in the Terraform provider + cleanedResources := applyManualMappings(resources, manualMappings) + viableResources := make(map[string]*ResourceInfo) skippedResources := make([]string, 0) - for name, resource := range mappedResources { + for name, resource := range cleanedResources { if isTerraformViable(resource, spec) { viableResources[name] = resource } else { @@ -42,35 +48,27 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua } fmt.Printf("\n=== Overlay Generation Analysis ===\n") - fmt.Printf("Resources after Manual Mapping: %d\n", len(mappedResources)) + fmt.Printf("Resources after Manual Mappings: %d\n", len(cleanedResources)) fmt.Printf("Viable for Terraform: %d\n", len(viableResources)) fmt.Printf("Skipped (non-viable): %d\n", len(skippedResources)) - if len(skippedResources) > 0 { - fmt.Printf("\nSkipped resources:\n") - for _, skipped := range skippedResources { - fmt.Printf(" - %s\n", skipped) - } - } - - // Filter operations with unmappable path parameters - fmt.Printf("\n=== Operation-Level Filtering ===\n") + // Helpful for debugging + // if len(skippedResources) > 0 { + // fmt.Printf("\nSkipped resources:\n") + // for _, skipped := range skippedResources { + // fmt.Printf(" - %s\n", skipped) + // } + // } - // Update description with actual count overlay.Info.Description = fmt.Sprintf("Auto-generated overlay for %d viable Terraform resources", len(viableResources)) - // Detect property mismatches for filtered resources only resourceMismatches := detectPropertyMismatches(viableResources, spec) - // Detect CRUD inconsistencies for filtered resources only resourceCRUDInconsistencies := detectCRUDInconsistencies(viableResources, spec) - // Track which properties already have ignore actions to avoid duplicates - ignoreTracker := make(map[string]map[string]bool) // map[schemaName][propertyName]bool + ignoreTracker := make(map[string]map[string]bool) - // Generate actions only for filtered resources for _, resource := range viableResources { - // Mark the response entity schema entityUpdate := map[string]interface{}{ "x-speakeasy-entity": resource.EntityName, } @@ -80,7 +78,6 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua Update: entityUpdate, }) - // Initialize ignore tracker for this resource's schemas if ignoreTracker[resource.EntityName] == nil { ignoreTracker[resource.EntityName] = make(map[string]bool) } @@ -91,21 +88,16 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua ignoreTracker[resource.UpdateSchema] = make(map[string]bool) } - // Add speakeasy ignore for property mismatches if mismatches, exists := resourceMismatches[resource.EntityName]; exists { addIgnoreActionsForMismatches(overlay, resource, mismatches, ignoreTracker) } - // Add speakeasy ignore for CRUD inconsistencies if inconsistencies, exists := resourceCRUDInconsistencies[resource.EntityName]; exists { addIgnoreActionsForInconsistencies(overlay, resource, inconsistencies, ignoreTracker) } - // Add entity operations and parameter matching for crudType, opInfo := range resource.Operations { - // Double-check that this specific operation isn't in the ignore list if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { - fmt.Printf(" Skipping ignored operation during overlay generation: %s %s\n", opInfo.Method, opInfo.Path) continue } @@ -120,16 +112,12 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua Update: operationUpdate, }) - // Apply parameter matching for operations that use the primary ID if resource.PrimaryID != "" && (crudType == "read" || crudType == "update" || crudType == "delete") { pathParams := extractPathParameters(opInfo.Path) for _, param := range pathParams { if param == resource.PrimaryID { - // Check for manual parameter mapping first if manualMatch, hasManual := getManualParameterMatch(opInfo.Path, opInfo.Method, param, manualMappings); hasManual { - // Only apply manual match if it's different from the parameter name if manualMatch != param { - fmt.Printf(" Manual parameter mapping: %s in %s %s -> %s\n", param, opInfo.Method, opInfo.Path, manualMatch) overlay.Actions = append(overlay.Actions, OverlayAction{ Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", opInfo.Path, opInfo.Method, param), @@ -137,22 +125,19 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua "x-speakeasy-match": manualMatch, }, }) - } else { - fmt.Printf(" Skipping manual parameter mapping: %s already matches target field %s (would create circular reference)\n", param, manualMatch) } } else { - // Skip x-speakeasy-match when parameter name would map to itself - // This prevents circular references like {id} -> id - if param == "id" { - fmt.Printf(" Skipping x-speakeasy-match: parameter %s maps to same field (avoiding circular reference)\n", param) - } else { - // Apply x-speakeasy-match for parameters that need mapping (e.g., change_event_id -> id) - fmt.Printf(" Applying x-speakeasy-match to %s in %s %s -> id\n", param, opInfo.Method, opInfo.Path) + // Avoid creating circular reference by not mapping id -> id or slug -> slug + if param != "id" && param != "slug" { + primaryIdentifier := "id" + if strings.Contains(resource.PrimaryID, "slug") { + primaryIdentifier = "slug" + } overlay.Actions = append(overlay.Actions, OverlayAction{ Target: fmt.Sprintf("$.paths[\"%s\"].%s.parameters[?(@.name==\"%s\")]", opInfo.Path, opInfo.Method, param), Update: map[string]interface{}{ - "x-speakeasy-match": "id", + "x-speakeasy-match": primaryIdentifier, }, }) } @@ -166,7 +151,6 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua fmt.Printf("\n=== Overlay Generation Complete ===\n") fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(viableResources)) - // Count ignore actions and match actions totalIgnores := 0 totalMatches := 0 for _, action := range overlay.Actions { diff --git a/scripts/overlay/main.go b/scripts/overlay/main.go index 702d499..7f04060 100644 --- a/scripts/overlay/main.go +++ b/scripts/overlay/main.go @@ -166,5 +166,4 @@ func printOverlaySummary(resources map[string]*ResourceInfo, overlay *Overlay) { fmt.Println("9. Tag viable operations with x-speakeasy-entity-operation") fmt.Println("10. Mark chosen primary ID with x-speakeasy-match") fmt.Println("11. Apply x-speakeasy-ignore: true to problematic properties") - fmt.Println("12. Apply x-speakeasy-enums: true to enum schemas for proper code generation") } diff --git a/scripts/overlay/manual-mappings.go b/scripts/overlay/manual-mappings.go index a2ed343..2c9c521 100644 --- a/scripts/overlay/manual-mappings.go +++ b/scripts/overlay/manual-mappings.go @@ -34,7 +34,6 @@ func loadManualMappings(mappingsPath string) *ManualMappings { return &ManualMappings{} } - fmt.Printf("Loaded %d manual mappings from %s\n", len(mappings.Operations), mappingsPath) return &mappings } @@ -56,26 +55,18 @@ func applyManualMappings(resources map[string]*ResourceInfo, manualMappings *Man operationsRemoved := 0 - // Copy operations that aren't manually ignored for crudType, opInfo := range resource.Operations { if shouldIgnoreOperation(opInfo.Path, opInfo.Method, manualMappings) { - fmt.Printf(" Removing manually ignored operation: %s %s (was %s for %s)\n", - opInfo.Method, opInfo.Path, crudType, resource.EntityName) operationsRemoved++ } else { cleanedResource.Operations[crudType] = opInfo } } - // Only include resource if it still has operations after cleaning if len(cleanedResource.Operations) > 0 { cleanedResources[name] = cleanedResource if operationsRemoved > 0 { - fmt.Printf(" Resource %s: kept %d operations, removed %d manually ignored\n", - name, len(cleanedResource.Operations), operationsRemoved) } - } else { - fmt.Printf(" Resource %s: removed entirely (all operations were manually ignored)\n", name) } } @@ -89,7 +80,6 @@ func getManualParameterMatch(path, method, paramName string, manualMappings *Man // For match mappings, we expect the value to be in format "param_name:field_name" parts := strings.SplitN(mapping.Value, ":", 2) if len(parts) == 2 && parts[0] == paramName { - fmt.Printf(" Manual mapping: Parameter %s in %s %s -> %s\n", paramName, method, path, parts[1]) return parts[1], true } } @@ -100,7 +90,6 @@ func getManualParameterMatch(path, method, paramName string, manualMappings *Man func shouldIgnoreOperation(path, method string, manualMappings *ManualMappings) bool { for _, mapping := range manualMappings.Operations { if mapping.Path == path && strings.EqualFold(mapping.Method, method) && mapping.Action == "ignore" { - fmt.Printf(" Manual mapping: Ignoring operation %s %s\n", method, path) return true } } @@ -110,7 +99,6 @@ func shouldIgnoreOperation(path, method string, manualMappings *ManualMappings) func getManualEntityMapping(path, method string, manualMappings *ManualMappings) (string, bool) { for _, mapping := range manualMappings.Operations { if mapping.Path == path && strings.EqualFold(mapping.Method, method) && mapping.Action == "entity" { - fmt.Printf(" Manual mapping: Operation %s %s -> Entity %s\n", method, path, mapping.Value) return mapping.Value, true } } diff --git a/scripts/overlay/terraform-viable.go b/scripts/overlay/terraform-viable.go index 798078f..6ee54a6 100644 --- a/scripts/overlay/terraform-viable.go +++ b/scripts/overlay/terraform-viable.go @@ -8,25 +8,25 @@ import ( // Check if a resource is viable for Terraform func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { + fmt.Printf("Validating resource %v (%v)\n", resource.ResourceName, resource.EntityName) // Must have at least create and read operations _, hasCreate := resource.Operations["create"] _, hasRead := resource.Operations["read"] - if !hasCreate || !hasRead { - fmt.Printf(" Missing required operations: Create=%v, Read=%v\n", hasCreate, hasRead) + if (!hasCreate) || !hasRead { + fmt.Printf(" Missing required operations in %v: Create=%v Read=%v\n", resource.ResourceName, hasCreate, hasRead) return false } // Must have a create schema to be manageable by Terraform if resource.CreateSchema == "" { - fmt.Printf(" No create schema found\n") + fmt.Printf(" No create schema found for %v\n", resource.ResourceName) return false } - // Identify the primary ID for this entity - primaryID, validPrimaryID := identifyEntityPrimaryID(resource) + primaryID, validPrimaryID := identifyEntityPrimaryID(resource, spec) if !validPrimaryID { - fmt.Printf(" Cannot identify valid primary ID parameter\n") + fmt.Printf(" Cannot identify valid primary ID parameter for %v\n", resource.EntityName) return false } @@ -48,7 +48,7 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { // Check for overlapping properties between create and entity schemas if !hasValidCreateReadConsistency(resource, spec) { - fmt.Printf(" Create and Read operations have incompatible schemas\n") + fmt.Printf(" %v Create and Read operations have incompatible schemas\n", resource.EntityName) return false } @@ -71,7 +71,6 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { createProps := getSchemaProperties(schemas, resource.CreateSchema) updateProps := getSchemaProperties(schemas, resource.UpdateSchema) - // Count manageable properties (non-system fields) createManageableProps := 0 updateManageableProps := 0 commonManageableProps := 0 @@ -85,14 +84,12 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { for prop := range updateProps { if !isSystemProperty(prop) { updateManageableProps++ - // Check if this property also exists in create if createProps[prop] != nil && !isSystemProperty(prop) { commonManageableProps++ } } } - // Reject resources with fundamentally incompatible CRUD patterns if createManageableProps <= 1 && updateManageableProps >= 3 && commonManageableProps == 0 { fmt.Printf(" Incompatible CRUD pattern: Create=%d manageable, Update=%d manageable, Common=%d\n", createManageableProps, updateManageableProps, commonManageableProps) @@ -103,11 +100,9 @@ func isTerraformViable(resource *ResourceInfo, spec OpenAPISpec) bool { return true } -// Validate operations against the identified primary ID func validateOperationParameters(resource *ResourceInfo, primaryID string, spec OpenAPISpec) map[string]OperationInfo { validOperations := make(map[string]OperationInfo) - // Get entity properties once for this resource entityProps := getEntityProperties(resource.EntityName, spec) for crudType, opInfo := range resource.Operations { @@ -133,10 +128,16 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string, spec continue } - // READ, UPDATE, DELETE should have exactly the primary ID hasPrimaryID := false hasConflictingEntityIDs := false + // We need to validate that we do not have conflicting ID path parameters + // At time of this comment, we only map a single ID parameter to the corresponding entity id, using x-speakeasy-match + + // If this constraint prevents us from adding resources that customers need/want + // we'll need to figure out how to map additional IDs to right field and entities + // for the time, being, multiple ID params in path are not supported and corresponding operations are ignored, + // unless they are already exact match to the field name on the entity for _, param := range pathParams { if param == primaryID { hasPrimaryID = true @@ -145,22 +146,11 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string, spec // Check if it maps to a field in the entity (not the primary id field) if checkFieldExistsInEntity(param, entityProps) { // This parameter maps to a real entity field - it's valid - fmt.Printf(" Parameter %s maps to entity field - keeping operation %s %s\n", - param, crudType, opInfo.Path) + continue } else { // This ID parameter doesn't map to any entity field - if mapsToEntityID(param, resource.EntityName) { - fmt.Printf(" Skipping %s operation %s: parameter %s would conflict with primary ID %s (both map to entity.id)\n", - crudType, opInfo.Path, param, primaryID) - hasConflictingEntityIDs = true - break - } else { - // This is an unmappable ID parameter - fmt.Printf(" Skipping %s operation %s: unmappable ID parameter %s (not in entity schema)\n", - crudType, opInfo.Path, param) - hasConflictingEntityIDs = true - break - } + hasConflictingEntityIDs = true + break } } // Non-ID parameters are always OK @@ -173,19 +163,18 @@ func validateOperationParameters(resource *ResourceInfo, primaryID string, spec } if hasConflictingEntityIDs { + fmt.Printf(" Skipping %s operation %s: has conflicting entity ID parameters\n", + crudType, opInfo.Path) continue } validOperations[crudType] = opInfo } - fmt.Printf(" Valid operations after parameter validation: %v\n", getOperationTypes(validOperations)) return validOperations } -// Get entity properties for field existence checking func getEntityProperties(entityName string, spec OpenAPISpec) map[string]interface{} { - // Re-parse the spec to get raw schema data specData, err := json.Marshal(spec) if err != nil { return map[string]interface{}{} @@ -203,12 +192,13 @@ func getEntityProperties(entityName string, spec OpenAPISpec) map[string]interfa } // Check if create and read operations have compatible schemas +// We need to ensure that the create and read operations are exactly the same, after accounting for ignored properties and normalization func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) bool { if resource.CreateSchema == "" { return false } + fmt.Printf(" Checking create/read consistency for %v\n", resource.ResourceName) - // Re-parse the spec to get raw schema data specData, err := json.Marshal(spec) if err != nil { return false @@ -229,7 +219,6 @@ func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) boo return false } - // Count overlapping manageable properties commonManageableProps := 0 createManageableProps := 0 @@ -247,10 +236,8 @@ func hasValidCreateReadConsistency(resource *ResourceInfo, spec OpenAPISpec) boo return false } - // Require at least 30% overlap of create properties to exist in entity - // This is more lenient than the 50% I had before - overlapRatio := float64(commonManageableProps) / float64(createManageableProps) - return overlapRatio >= 0.3 + // If there is any overlap, try to use the schemas + return true } func getSchemaProperties(schemas map[string]interface{}, schemaName string) map[string]interface{} { @@ -290,26 +277,18 @@ func isSystemProperty(propName string) bool { } } - // Also consider ID fields as system properties - if strings.HasSuffix(lowerProp, "_id") { - return true - } - - return false + return strings.HasSuffix(lowerProp, "_id") } -// Check if a field exists in the entity properties func checkFieldExistsInEntity(paramName string, entityProps map[string]interface{}) bool { - // Direct field name match if _, exists := entityProps[paramName]; exists { return true } - // Check for common variations variations := []string{ paramName, - strings.TrimSuffix(paramName, "_id"), // Remove _id suffix - strings.TrimSuffix(paramName, "Id"), // Remove Id suffix + strings.TrimSuffix(paramName, "_id"), + strings.TrimSuffix(paramName, "Id"), } for _, variation := range variations { @@ -321,14 +300,12 @@ func checkFieldExistsInEntity(paramName string, entityProps map[string]interface return false } -// Identify the primary ID parameter that belongs to this specific entity -func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { - // Get all unique path parameters across operations +func identifyEntityPrimaryID(resource *ResourceInfo, spec OpenAPISpec) (string, bool) { allParams := make(map[string]bool) for crudType, opInfo := range resource.Operations { if crudType == "create" || crudType == "list" { - continue // Skip operations that typically don't have entity-specific IDs + continue } pathParams := extractPathParameters(opInfo.Path) @@ -338,10 +315,9 @@ func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { } if len(allParams) == 0 { - return "", false // No path parameters found + return "", false } - // Find the parameter that matches this entity var entityPrimaryID string matchCount := 0 @@ -352,46 +328,37 @@ func identifyEntityPrimaryID(resource *ResourceInfo) (string, bool) { } } - if matchCount == 0 { - // No parameter maps to this entity - check for generic 'id' parameter - if allParams["id"] { - fmt.Printf(" Using generic 'id' parameter for entity %s\n", resource.EntityName) - return "id", true + if matchCount == 1 { + return entityPrimaryID, true + } + + entityProps := getEntityProperties(resource.EntityName, spec) + _, hasID := entityProps["id"] + _, hasSlug := entityProps["slug"] + + if hasSlug && !hasID { + for param := range allParams { + if strings.Contains(strings.ToLower(param), "slug") { + return param, true + } } - fmt.Printf(" No parameter maps to entity %s\n", resource.EntityName) - return "", false } - if matchCount > 1 { - // Multiple parameters claim to map to this entity - ambiguous - fmt.Printf(" Multiple parameters map to entity %s: ambiguous primary ID\n", resource.EntityName) - return "", false + if allParams["id"] { + return "id", true } - fmt.Printf(" Identified primary ID '%s' for entity %s\n", entityPrimaryID, resource.EntityName) - return entityPrimaryID, true + return "", false } -// Check if a parameter name maps to a specific entity's ID field func mapsToEntityID(paramName, entityName string) bool { - // Extract base name from entity (e.g., "ChangeEvent" from "ChangeEventEntity") entityBase := strings.TrimSuffix(entityName, "Entity") - // Convert to snake_case and add _id suffix expectedParam := toSnakeCase(entityBase) + "_id" - return strings.ToLower(paramName) == strings.ToLower(expectedParam) + return strings.EqualFold(paramName, expectedParam) } -// Check if parameter looks like an entity ID func isEntityID(paramName string) bool { return strings.HasSuffix(strings.ToLower(paramName), "_id") || strings.ToLower(paramName) == "id" } - -func getOperationTypes(operations map[string]OperationInfo) []string { - var types []string - for opType := range operations { - types = append(types, opType) - } - return types -} From 7764e4134b5e3822b6d838c0390df32d1c48490f Mon Sep 17 00:00:00 2001 From: Jonah Stewart Date: Fri, 13 Jun 2025 17:12:53 -0400 Subject: [PATCH 16/16] add manual entity-level property ignores --- scripts/overlay/generate-overlay.go | 24 ++++++++++++++++++++++++ scripts/overlay/manual-mappings.go | 17 +++++++++++++++++ scripts/overlay/manual-mappings.yaml | 21 ++++++++++++++++++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/scripts/overlay/generate-overlay.go b/scripts/overlay/generate-overlay.go index e1aa0bd..e28baad 100644 --- a/scripts/overlay/generate-overlay.go +++ b/scripts/overlay/generate-overlay.go @@ -148,6 +148,30 @@ func generateOverlay(resources map[string]*ResourceInfo, spec OpenAPISpec, manua } } + // Add this after the existing ignore actions in generateOverlay + // Apply manual property ignores + manualPropertyIgnores := getManualPropertyIgnores(manualMappings) + for schemaName, properties := range manualPropertyIgnores { + for _, propertyName := range properties { + // Initialize ignore tracker if needed + if ignoreTracker[schemaName] == nil { + ignoreTracker[schemaName] = make(map[string]bool) + } + + // Only add if not already ignored + if !ignoreTracker[schemaName][propertyName] { + overlay.Actions = append(overlay.Actions, OverlayAction{ + Target: fmt.Sprintf("$.components.schemas.%s.properties.%s", schemaName, propertyName), + Update: map[string]interface{}{ + "x-speakeasy-ignore": true, + }, + }) + ignoreTracker[schemaName][propertyName] = true + fmt.Printf("✅ Added manual property ignore: %s.%s\n", schemaName, propertyName) + } + } + } + fmt.Printf("\n=== Overlay Generation Complete ===\n") fmt.Printf("Generated %d actions for %d viable resources\n", len(overlay.Actions), len(viableResources)) diff --git a/scripts/overlay/manual-mappings.go b/scripts/overlay/manual-mappings.go index 2c9c521..dc97f0b 100644 --- a/scripts/overlay/manual-mappings.go +++ b/scripts/overlay/manual-mappings.go @@ -14,6 +14,10 @@ type ManualMapping struct { Method string `yaml:"method"` Action string `yaml:"action"` // "ignore", "entity", "match" Value string `yaml:"value,omitempty"` + + // For entity mappings + Schema string `yaml:"schema,omitempty"` + Property string `yaml:"property,omitempty"` } type ManualMappings struct { @@ -104,3 +108,16 @@ func getManualEntityMapping(path, method string, manualMappings *ManualMappings) } return "", false } + +func getManualPropertyIgnores(manualMappings *ManualMappings) map[string][]string { + ignores := make(map[string][]string) // map[schemaName][]propertyNames + + for _, mapping := range manualMappings.Operations { + if mapping.Action == "ignore_property" && mapping.Schema != "" && mapping.Property != "" { + fmt.Printf(" Manual property ignore: %s.%s\n", mapping.Schema, mapping.Property) + ignores[mapping.Schema] = append(ignores[mapping.Schema], mapping.Property) + } + } + + return ignores +} diff --git a/scripts/overlay/manual-mappings.yaml b/scripts/overlay/manual-mappings.yaml index 01b9de9..819381c 100644 --- a/scripts/overlay/manual-mappings.yaml +++ b/scripts/overlay/manual-mappings.yaml @@ -2,6 +2,7 @@ # - "ignore": Skip this operation entirely (won't get any x-speakeasy annotations) # - "entity": Force operation to map to specific entity (value = entity name) # - "match": Override parameter mapping (value = "param_name:field_name") +# - "ignore_property": Ignore a specific property on an entity, specify schema and property operations: # IGNORE: Skip operations that shouldn't be treated as entity operations @@ -36,6 +37,10 @@ operations: method: "get" action: "ignore" + # The template Property on this entity causes collision with the incidents type template + # TODO: Figure out why this is happening + - path: + # Manual Entity Mappings # Signals Event Sources - path: "/v1/signals/event_sources/{transposer_slug}" @@ -93,4 +98,18 @@ operations: - path: "/v1/signals/grouping/{id}" method: "delete" action: "entity" - value: "Signals_API_GroupingEntity" \ No newline at end of file + value: "Signals_API_GroupingEntity" + + +# Entity Level Property Ignoring - only work for top level properties + - action: "ignore_property" + schema: "PublicAPI_V1_FormConfigurations_SelectedValueEntity" + property: "template" + + - action: "ignore_property" + schema: "MediaImageEntity" + property: "versions_urls" + + - action: "ignore_property" + schema: "IncidentTypeEntity_TemplateValuesEntity" + property: "runbooks" \ No newline at end of file