fix(tesseract): Fix issues with member expressions over multi stage (#9416)

Author: waralexrom

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -697,9 +697,11 @@ export class BaseQuery {
       rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null,
       offset: this.options.offset ? this.options.offset.toString() : null,
       baseTools: this,
-      ungrouped: this.options.ungrouped
+      ungrouped: this.options.ungrouped,
+      exportAnnotatedSql: exportAnnotatedSql === true
 
     };
+
     const buildResult = nativeBuildSqlAndParams(queryParams);
 
     if (buildResult.error) {

packages/cubejs-schema-compiler/test/integration/postgres/member-expression.test.ts

@@ -5,7 +5,7 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery';
 import { prepareYamlCompiler } from '../../unit/PrepareCompiler';
 import { dbRunner } from './PostgresDBRunner';
 
-describe('Member Expression', () => {
+describe('Member Expression Multistage', () => {
   jest.setTimeout(200000);
 
   const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(`
@@ -37,6 +37,170 @@ cubes:
       - name: count
         type: count
 
+  - name: orders
+    sql: >
+      select 10 AS ID, 'complited' AS STATUS, '2021-01-05 00:00:00'::timestamp AS CREATED_AT, 100 AS CUSTOMER_ID, 50.0 as revenue
+      UNION ALL
+      select 11 AS ID, 'complited' AS STATUS, '2021-05-01 00:00:00'::timestamp AS CREATED_AT, 100 AS CUSTOMER_ID, 150.0 as revenue
+      UNION ALL
+      select 12 AS ID, 'complited' AS STATUS, '2021-06-01 00:00:00'::timestamp AS CREATED_AT, 100 AS CUSTOMER_ID, 200.0 as revenue
+      UNION ALL
+      select 13 AS ID, 'complited' AS STATUS, '2022-01-04 00:00:00'::timestamp AS CREATED_AT, 100 AS CUSTOMER_ID, 10.0 as revenue
+      UNION ALL
+      select 14 AS ID, 'complited' AS STATUS, '2022-05-04 00:00:00'::timestamp AS CREATED_AT, 100 AS CUSTOMER_ID, 30.0 as revenue
+    public: false
+
+    joins:
+      - name: line_items
+        sql: "{CUBE}.ID = {line_items}.order_id"
+        relationship: many_to_one
+
+      - name: customers
+        sql: "{CUBE}.CUSTOMER_ID = {customers}.ID"
+        relationship: many_to_one
+
+    dimensions:
+      - name: id
+        sql: ID
+        type: number
+        primary_key: true
+
+      - name: status
+        sql: STATUS
+        type: string
+
+      - name: date
+        sql: CREATED_AT
+        type: time
+
+      - name: amount
+        sql: '{line_items.total_amount}'
+        type: number
+        sub_query: true
+
+    measures:
+      - name: count
+        type: count
+
+      - name: completed_count
+        type: count
+        filters:
+          - sql: "{CUBE}.STATUS = 'completed'"
+
+      - name: returned_count
+        type: count
+        filters:
+          - sql: "{CUBE}.STATUS = 'returned'"
+
+      - name: return_rate
+        type: number
+        sql: "({returned_count} / NULLIF({completed_count}, 0)) * 100.0"
+        description: "Percentage of returned orders out of completed, exclude just placed orders."
+        format: percent
+
+      - name: total_amount
+        sql: '{CUBE.amount}'
+        type: sum
+
+      - name: revenue
+        sql: "revenue"
+        type: sum
+        format: currency
+
+      - name: average_order_value
+        sql: '{CUBE.amount}'
+        type: avg
+
+      - name: revenue_1_y_ago
+        sql: "{revenue}"
+        multi_stage: true
+        type: number
+        format: currency
+        time_shift:
+          - time_dimension: date
+            interval: 1 year
+            type: prior
+          - time_dimension: orders_view.date
+            interval: 1 year
+            type: prior
+
+      - name: cagr_1_y
+        sql: "(({revenue} / {revenue_1_y_ago}) - 1)"
+        type: number
+        format: percent
+        description: "Annual CAGR, year over year growth in revenue"
+
+  - name: line_items
+    sql: >
+      SELECT 10 AS ID, 10 AS PRODUCT_ID, '2021-01-01 00:00:00'::timestamp AS CREATED_AT, 10 as order_id
+      UNION ALL
+      SELECT 11 AS ID, 10 AS PRODUCT_ID, '2021-01-01 00:00:00'::timestamp AS CREATED_AT, 11 as order_id
+      UNION ALL
+      SELECT 12 AS ID, 10 AS PRODUCT_ID, '2021-01-01 00:00:00'::timestamp AS CREATED_AT, 11 as order_id
+      UNION ALL
+      SELECT 13 AS ID, 10 AS PRODUCT_ID, '2021-01-01 00:00:00'::timestamp AS CREATED_AT, 12 as order_id
+    public: false
+
+    joins:
+      - name: products
+        sql: "{CUBE}.PRODUCT_ID = {products}.ID"
+        relationship: many_to_one
+
+    dimensions:
+      - name: id
+        sql: ID
+        type: number
+        primary_key: true
+
+      - name: created_at
+        sql: CREATED_AT
+        type: time
+
+      - name: price
+        sql: "{products.price}"
+        type: number
+
+    measures:
+      - name: count
+        type: count
+
+      - name: total_amount
+        sql: "{price}"
+        type: sum
+  - name: products
+    sql: >
+      SELECT 10 AS ID, 'Clothes' AS PRODUCT_CATEGORY, 'Shirt' AS NAME, 10 AS PRICE
+      UNION ALL
+      SELECT 11 AS ID, 'Clothes' AS PRODUCT_CATEGORY, 'Shirt' AS NAME, 20 AS PRICE
+    public: false
+    description: > 
+      Products and categories in our e-commerce store.
+
+    dimensions:
+      - name: id
+        sql: ID
+        type: number
+        primary_key: true
+
+      - name: product_category
+        sql: PRODUCT_CATEGORY
+        type: string
+
+      - name: name
+        sql: NAME
+        type: string
+
+      - name: price
+        sql: PRICE
+        type: number
+
+    measures:
+      - name: count
+        type: count
+
+
+
+
 views:
   - name: customers_view
 
@@ -47,6 +211,29 @@ views:
 
           - city
 
+  - name: orders_view
+    cubes:
+      - join_path: orders
+        includes:
+          - count
+          - date
+          - revenue
+          - cagr_1_y
+          - return_rate
+
+      - join_path: line_items.products
+        prefix: true
+        includes:
+          - product_category
+      
+      - join_path: orders.customers
+        prefix: true
+        includes:
+          - city
+          - count
+          - id
+
+
     `);
 
   async function runQueryTest(q, expectedResult) {
@@ -124,4 +311,77 @@ views:
   },
 
   [{ count: 1, city: 'New York', cubejoinfield: 'NULL' }, { count: 1, city: 'New York', cubejoinfield: 'NULL' }]));
+  if (getEnv('nativeSqlPlanner')) {
+    it('member expression multi stage', async () => runQueryTest({
+      measures: [
+        {
+        // eslint-disable-next-line no-new-func
+          expression: new Function(
+            'orders',
+            // eslint-disable-next-line no-template-curly-in-string
+            'return `${orders.cagr_1_y}`'
+          ),
+          // eslint-disable-next-line no-template-curly-in-string
+          definition: '${orders.cagr_1_y}',
+          expressionName: 'orders__cagr_2023',
+          cubeName: 'orders',
+        },
+      ],
+      timeDimensions: [
+        {
+          dimension: 'orders.date',
+          dateRange: ['2022-01-01', '2022-10-31'],
+        },
+      ],
+      timezone: 'America/Los_Angeles'
+    },
+
+    [{ orders__cagr_2023: '-0.90000000000000000000' }]));
+  } else {
+    it.skip('member expression multi stage', () => {
+      // Skipping because it works only in Tesseract
+    });
+  }
+
+  if (getEnv('nativeSqlPlanner')) {
+    it('member expression multi stage with time dimension segment', async () => runQueryTest({
+      measures: [
+        {
+        // eslint-disable-next-line no-new-func
+          expression: new Function(
+            'orders',
+            // eslint-disable-next-line no-template-curly-in-string
+            'return `${orders.cagr_1_y}`'
+          ),
+          // eslint-disable-next-line no-template-curly-in-string
+          definition: '${orders.cagr_1_y}',
+          expressionName: 'orders__cagr_2023',
+          cubeName: 'orders',
+        },
+      ],
+      segments: [
+        {
+          cubeName: 'orders',
+          name: 'orders_date____c',
+          expressionName: 'orders_date____c',
+          // eslint-disable-next-line no-new-func
+          expression: new Function(
+            'orders',
+            // eslint-disable-next-line no-template-curly-in-string
+            'return `((${orders.date} >= CAST(\'2022-01-01\' AS TIMESTAMP)) AND (${orders.date} < CAST(\'2022-10-31\' AS TIMESTAMP)))`'
+          ),
+          // eslint-disable-next-line no-template-curly-in-string
+          definition: '{"cube_name":"orders","alias":"orders_date____c","cube_params":["orders"],"expr":"((${orders.date} >= CAST($0$ AS TIMESTAMP)) AND (${orders.date} < CAST($1$ AS TIMESTAMP)))","grouping_set":null}',
+        }
+      ],
+
+      timezone: 'America/Los_Angeles'
+    },
+
+    [{ orders__cagr_2023: '-0.90000000000000000000' }]));
+  } else {
+    it.skip('member expression multi stage with time dimension segment', () => {
+      // Skipping because it works only in Tesseract
+    });
+  }
 });

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -59,6 +59,8 @@ pub struct BaseQueryOptionsStatic {
     pub row_limit: Option<String>,
     pub offset: Option<String>,
     pub ungrouped: Option<bool>,
+    #[serde(rename = "exportAnnotatedSql")]
+    pub export_annotated_sql: bool,
 }
 
 #[nativebridge::native_bridge(BaseQueryOptionsStatic)]

rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs

@@ -81,6 +81,23 @@ impl BaseDimension {
                     default_alias,
                 }))
             }
+            MemberSymbol::MemberExpression(expression_symbol) => {
+                let full_name = expression_symbol.full_name();
+                let cube_name = expression_symbol.cube_name().clone();
+                let name = expression_symbol.name().clone();
+                let member_expression_definition = expression_symbol.definition().clone();
+                let default_alias = PlanSqlTemplates::alias_name(&name);
+                Some(Rc::new(Self {
+                    dimension: full_name,
+                    query_tools: query_tools.clone(),
+                    member_evaluator: evaluation_node,
+                    definition: None,
+                    cube_name,
+                    name,
+                    member_expression_definition,
+                    default_alias,
+                }))
+            }
             _ => None,
         };
         Ok(result)

rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs

@@ -113,7 +113,7 @@ impl BaseMember for BaseMeasure {
     }
 
     fn full_name(&self) -> String {
-        format!("{}.{}", self.cube_name, self.name)
+        self.member_evaluator.full_name()
     }
 
     fn cube_name(&self) -> &String {
@@ -151,6 +151,24 @@ impl BaseMeasure {
                     default_alias,
                 }))
             }
+            MemberSymbol::MemberExpression(expression_symbol) => {
+                let full_name = expression_symbol.full_name();
+                let cube_name = expression_symbol.cube_name().clone();
+                let name = expression_symbol.name().clone();
+                let member_expression_definition = expression_symbol.definition().clone();
+                let default_alias = PlanSqlTemplates::alias_name(&name);
+                Some(Rc::new(Self {
+                    measure: full_name,
+                    query_tools: query_tools.clone(),
+                    member_evaluator: evaluation_node,
+                    definition: None,
+                    cube_name,
+                    name,
+                    member_expression_definition,
+                    default_alias,
+                    time_shifts: vec![],
+                }))
+            }
             _ => None,
         };
         Ok(res)

rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs

@@ -27,6 +27,7 @@ impl<IT: InnerTypes> BaseQuery<IT> {
             options.base_tools()?,
             options.join_graph()?,
             options.static_data().timezone.clone(),
+            options.static_data().export_annotated_sql,
         )?;
 
         let request = QueryProperties::try_new(query_tools.clone(), options)?;