Generate Metrics Tables

Author: claytongentry

lib/manifold/api/schema_manager.rb

@@ -17,6 +17,7 @@ def write_schemas(tables_directory)
         tables_directory.mkpath
         write_dimensions_schema(tables_directory)
         write_manifold_schema(tables_directory)
+        write_metrics_schemas(tables_directory)
       end
 
       # Returns the dimensions schema structure
@@ -52,6 +53,35 @@ def write_manifold_schema(tables_directory)
         manifold_path.write(manifold_schema_json.concat("\n"))
       end
 
+      def write_metrics_schemas(tables_directory)
+        return unless @manifold_yaml["metrics"]
+
+        @manifold_yaml["metrics"].each do |group_name, group_config|
+          metrics_table_path = tables_directory.join("metrics_#{group_name}.json")
+          metrics_table_schema = metrics_table_schema(group_name, group_config)
+          metrics_table_path.write(JSON.pretty_generate(metrics_table_schema).concat("\n"))
+          @logger.info("Generated metrics table schema for '#{group_name}'.")
+        end
+      end
+
+      def metrics_table_schema(group_name, group_config)
+        [
+          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
+          { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
+          { "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
+            "fields" => [metrics_group_field(group_name, group_config)] }
+        ]
+      end
+
+      def metrics_group_field(group_name, group_config)
+        {
+          "name" => group_name,
+          "type" => "RECORD",
+          "mode" => "NULLABLE",
+          "fields" => group_metrics_fields(group_config)
+        }
+      end
+
       def dimensions_fields
         @dimensions_fields ||= @vectors.filter_map do |vector|
           @logger.info("Loading vector schema for '#{vector}'.")

spec/manifold/api/workspace_spec.rb

@@ -178,6 +178,39 @@
         expect(schema_fields[:metrics]["mode"]).to eq("REQUIRED")
       end
 
+      it "generates a metrics table schema file for each metrics group" do
+        expect(workspace.tables_directory.join("metrics_taps.json")).to be_file
+      end
+
+      it "includes required id field in metrics table schema" do
+        metrics_schema = parse_metrics_schema("taps")
+        expect(metrics_schema).to include(
+          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" }
+        )
+      end
+
+      it "includes required timestamp field in metrics table schema" do
+        metrics_schema = parse_metrics_schema("taps")
+        expect(metrics_schema).to include(
+          { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" }
+        )
+      end
+
+      it "includes required metrics field in metrics table schema" do
+        metrics_schema = parse_metrics_schema("taps")
+        metrics_field = metrics_schema.find { |f| f["name"] == "metrics" }
+        expect(metrics_field["type"]).to eq("RECORD")
+        expect(metrics_field["mode"]).to eq("REQUIRED")
+      end
+
+      it "includes the correct metrics group in the metrics table schema" do
+        metrics_schema = parse_metrics_schema("taps")
+        metrics_field = metrics_schema.find { |f| f["name"] == "metrics" }
+        group_field = metrics_field["fields"].first
+        expect(group_field["name"]).to eq("taps")
+        expect(group_field["type"]).to eq("RECORD")
+      end
+
       shared_examples "breakout metrics" do |breakout_name|
         let(:breakout) do
           schema_fields[:metrics]["fields"]
@@ -244,6 +277,10 @@ def get_dimension(field)
         dimensions = parse_dimensions_schema.find { |f| f["name"] == "dimensions" }
         dimensions["fields"].find { |f| f["name"] == field }
       end
+
+      def parse_metrics_schema(group_name)
+        JSON.parse(workspace.tables_directory.join("metrics_#{group_name}.json").read)
+      end
     end
 
     context "when the manifold configuration is missing" do

spec/manifold/terraform/workspace_configuration_spec.rb

@@ -103,6 +103,18 @@
         expect(dimensions_routine_details[:sql_content]).to include(source_sql)
       end
     end
+
+    context "when metrics configuration is present" do
+      before do
+        config.manifold_config = manifold_config
+      end
+
+      it "includes metrics table configurations" do
+        expect(json["resource"]["google_bigquery_table"]).to include(
+          "metrics_taps" => expected_metrics_table("taps")
+        )
+      end
+    end
   end
 
   context "when manifold configuration is present" do
@@ -208,6 +220,16 @@ def expected_routine_config
     }
   end
 
+  def expected_metrics_table(group_name)
+    {
+      "dataset_id" => name,
+      "project" => "${var.project_id}",
+      "table_id" => "Metrics_#{group_name}",
+      "schema" => "${file(\"${path.module}/tables/metrics_#{group_name}.json\")}",
+      "depends_on" => ["google_bigquery_dataset.#{name}"]
+    }
+  end
+
   def setup_merge_vector_config
     Pathname.pwd.join("lib/routines").mkpath
     Pathname.pwd.join("lib/routines/select_pages.sql").write(source_sql)