Generate Terraform (#16)

* Generate terraform

* Refactor to Terraform Configurations

* Allow configuring the provider version

* Mandate PROJECT_ID on workspaces

* Feature flag terraform

* Add some README

* with_terraform instead generate_terraform

* bump tf

* consistently case variable

* Bump minimum ruby

* Capitalize Dimensions

Author: claytongentry

.github/workflows/ci.yml

@@ -14,9 +14,9 @@ jobs:
     strategy:
       matrix:
         ruby:
-          - '3.1.6'
           - '3.2.6'
           - '3.3.6'
+          - '3.4.1'
 
     steps:
       - uses: actions/checkout@v4

.rubocop.yml

@@ -4,7 +4,7 @@ require: rubocop-rspec
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 3.1
+  TargetRubyVersion: 3.2
 
 Style/StringLiterals:
   EnforcedStyle: double_quotes

README.md

@@ -61,7 +61,29 @@ manifold add <data_project_name>
 After you fill out the manifold.yml file, this command generates the necessary BigQuery schema files based on the specified dimensions and metrics.
 
 ```bash
-manifold generate <data_project_name> bq
+manifold generate
+```
+
+4. **Generate Terraform Configuration (Optional)**
+
+Manifold can optionally generate Terraform configurations for managing your BigQuery resources. To generate both BigQuery schemas and Terraform configurations, use the `--tf` flag:
+
+```bash
+manifold generate --tf
+```
+
+This will create:
+
+- A root `main.tf.json` file that sets up the Google Cloud provider and workspace modules
+- Individual workspace configurations in `workspaces/<workspace_name>/main.tf.json`
+- Dataset and table definitions that reference your generated BigQuery schemas
+
+The generated Terraform configurations use the Google Cloud provider and expect a `project_id` variable to be set. You can apply these configurations using standard Terraform commands:
+
+```bash
+terraform init
+terraform plan -var="project_id=your-project-id"
+terraform apply -var="project_id=your-project-id"
 ```
 
 ## Manifold Configuration

lib/manifold/api/project.rb

@@ -22,8 +22,9 @@ def workspaces
         @workspaces ||= workspace_directories.map { |dir| Workspace.from_directory(dir, logger:) }
       end
 
-      def generate
-        workspaces.each(&:generate)
+      def generate(with_terraform: false)
+        workspaces.each { |w| w.generate(with_terraform:) }
+        generate_terraform_entrypoint if with_terraform
       end
 
       def workspaces_directory
@@ -39,6 +40,11 @@ def vectors_directory
       def workspace_directories
         workspaces_directory.children.select(&:directory?)
       end
+
+      def generate_terraform_entrypoint
+        config = Terraform::ProjectConfiguration.new(workspaces)
+        config.write(directory.join("main.tf.json"))
+      end
     end
   end
 end

lib/manifold/api/workspace.rb

@@ -26,10 +26,11 @@ def add
         FileUtils.cp(template_path, manifold_path)
       end
 
-      def generate
+      def generate(with_terraform: false)
         return unless manifold_exists? && any_vectors?
 
         generate_dimensions
+        generate_terraform if with_terraform
         logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
       end
 
@@ -99,6 +100,15 @@ def any_vectors?
       def vectors
         manifold_yaml["vectors"]
       end
+
+      def generate_terraform
+        config = Terraform::WorkspaceConfiguration.new(name)
+        config.write(terraform_main_path)
+      end
+
+      def terraform_main_path
+        directory.join("main.tf.json")
+      end
     end
   end
 end

lib/manifold/cli.rb

@@ -46,10 +46,12 @@ def add(name)
     end
 
     desc "generate", "Generate BigQuery schema for all workspaces in the project"
+    method_option :tf, type: :boolean, desc: "Generate Terraform configurations"
     def generate
-      name = Pathname.pwd.basename.to_s
-      project = API::Project.new(name, directory: Pathname.pwd, logger:)
-      project.generate
+      path = Pathname.pwd
+      name = path.basename.to_s
+      project = API::Project.new(name, directory: path, logger:)
+      project.generate(with_terraform: options[:tf])
       logger.info "Generated BigQuery schema for all workspaces in the project."
     end
   end

lib/manifold/terraform/configuration.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Provides a base class for Terraform configuration files.
+    class Configuration
+      def as_json
+        raise NotImplementedError, "#{self.class} must implement #as_json"
+      end
+
+      def write(path)
+        path.write("#{JSON.pretty_generate(as_json)}\n")
+      end
+    end
+  end
+end

lib/manifold/terraform/project_configuration.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Represents a Terraform configuration for a Manifold project.
+    class ProjectConfiguration < Configuration
+      attr_reader :workspaces, :provider_version
+
+      DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.18.1"
+
+      def initialize(workspaces, provider_version: DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION)
+        super()
+        @workspaces = workspaces
+        @provider_version = provider_version
+      end
+
+      def as_json
+        {
+          "terraform" => terraform_block,
+          "provider" => provider_block,
+          "variable" => variables_block,
+          "module" => workspace_modules
+        }
+      end
+
+      private
+
+      def terraform_block
+        {
+          "required_providers" => {
+            "google" => {
+              "source" => "hashicorp/google",
+              "version" => provider_version
+            }
+          }
+        }
+      end
+
+      def provider_block
+        {
+          "google" => {
+            "project" => "${var.project_id}"
+          }
+        }
+      end
+
+      def variables_block
+        {
+          "project_id" => {
+            "description" => "The GCP project ID where resources will be created",
+            "type" => "string"
+          }
+        }
+      end
+
+      def workspace_modules
+        workspaces.each_with_object({}) do |workspace, modules|
+          modules[workspace.name] = {
+            "source" => "./workspaces/#{workspace.name}",
+            "project_id" => "${var.project_id}"
+          }
+        end
+      end
+    end
+  end
+end

lib/manifold/terraform/workspace_configuration.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Represents a Terraform configuration for a Manifold workspace.
+    class WorkspaceConfiguration < Configuration
+      attr_reader :name
+
+      def initialize(name)
+        super()
+        @name = name
+      end
+
+      def as_json
+        {
+          "variable" => variables_block,
+          "resource" => {
+            "google_bigquery_dataset" => dataset_config,
+            "google_bigquery_table" => table_config
+          }
+        }
+      end
+
+      private
+
+      def variables_block
+        {
+          "project_id" => {
+            "description" => "The GCP project ID where resources will be created",
+            "type" => "string"
+          }
+        }
+      end
+
+      def dataset_config
+        {
+          name => {
+            "dataset_id" => name,
+            "project" => "${var.project_id}",
+            "location" => "US"
+          }
+        }
+      end
+
+      def table_config
+        {
+          "dimensions" => {
+            "dataset_id" => name,
+            "project" => "${var.project_id}",
+            "table_id" => "Dimensions",
+            "schema" => "${file(\"${path.module}/tables/dimensions.json\")}",
+            "depends_on" => ["google_bigquery_dataset.#{name}"]
+          }
+        }
+      end
+    end
+  end
+end

manifold.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.summary = "A CLI for managing data infrastructures in BigQuery"
   spec.homepage = "https://github.com/bustle/manifold"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 3.1.0"
+  spec.required_ruby_version = ">= 3.2.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/bustle/manifold"