fix(schema-compiler): Fix incorrect truncated time dimensions over time series queries for BigQuery

.github/workflows/rust-cubestore.yml

@@ -114,7 +114,7 @@ jobs:
           - x86_64-apple-darwin
           - aarch64-apple-darwin
         include:
-          - os: windows-2019
+          - os: windows-2022
             target: x86_64-pc-windows-msvc
             executable_name: cubestored.exe
             strip: true

packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

@@ -142,10 +142,6 @@ export class BigqueryQuery extends BaseQuery {
     return new BigqueryFilter(this, filter);
   }
 
-  public dateSeriesSql(timeDimension: BaseTimeDimension) {
-    return `${timeDimension.dateSeriesAliasName()} AS (${this.seriesSql(timeDimension)})`;
-  }
-
   public seriesSql(timeDimension: BaseTimeDimension) {
     const values = timeDimension.timeSeries().map(
       ([from, to]) => `select '${from}' f, '${to}' t`
@@ -161,26 +157,6 @@ export class BigqueryQuery extends BaseQuery {
     return 6;
   }
 
-  public overTimeSeriesSelect(cumulativeMeasures, dateSeriesSql, baseQuery, dateJoinConditionSql, baseQueryAlias) {
-    const forSelect = this.overTimeSeriesForSelect(cumulativeMeasures);
-    const outerSeriesAlias = this.cubeAlias('outer_series');
-    const outerBase = this.cubeAlias('outer_base');
-    const timeDimensionAlias = this.timeDimensions.map(d => d.aliasName()).filter(d => !!d)[0];
-    const aliasesForSelect = this.timeDimensions.map(d => d.dateSeriesSelectColumn(outerSeriesAlias)).concat(
-      this.dimensions.concat(cumulativeMeasures).map(s => s.aliasName())
-    ).filter(c => !!c).join(', ');
-    const dateSeriesAlias = this.timeDimensions.map(d => `${d.dateSeriesAliasName()}`).filter(c => !!c)[0];
-    return `
-    WITH ${dateSeriesSql} SELECT ${aliasesForSelect} FROM
-    ${dateSeriesAlias} ${outerSeriesAlias}
-    LEFT JOIN (
-      SELECT ${forSelect} FROM ${dateSeriesAlias}
-      INNER JOIN (${baseQuery}) AS ${baseQueryAlias} ON ${dateJoinConditionSql}
-      ${this.groupByClause()}
-    ) AS ${outerBase} ON ${outerSeriesAlias}.${this.escapeColumnName('date_from')} = ${outerBase}.${timeDimensionAlias}
-    `;
-  }
-
   public subtractInterval(date, interval) {
     const [intervalFormatted, timeUnit] = this.formatInterval(interval);
     if (['YEAR', 'MONTH', 'QUARTER'].includes(timeUnit) || intervalFormatted.includes('WEEK')) {
@@ -346,6 +322,7 @@ export class BigqueryQuery extends BaseQuery {
     // DATEADD is being rewritten to DATE_ADD
     templates.functions.DATE_ADD = 'DATETIME_ADD(DATETIME({{ args[0] }}), INTERVAL {{ interval }} {{ date_part }})';
     templates.functions.CURRENTDATE = 'CURRENT_DATE';
+    templates.functions.DATE = 'TIMESTAMP({{ args_concat }})';
     delete templates.functions.TO_CHAR;
     delete templates.functions.PERCENTILECONT;
     templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}';

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -150,10 +150,9 @@
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
-    "--------------------",
 
+    "--------------------",
     "week granularity is not supported for intervals",
     "--------------------",
     "querying BigECommerce: rolling window by 2 week",
@@ -209,6 +208,11 @@
     "SQL API: Nested Rollup over asterisk",
     "SQL API: Extended nested Rollup over asterisk",
     "SQL API: Timeshift measure from cube",
-    "SQL API: SQL push down push to cube quoted alias"
+    "SQL API: SQL push down push to cube quoted alias",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+    "querying BigECommerce: rolling window YTD without granularity",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -163,8 +163,8 @@
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
     "querying BigECommerce: rolling window YTD without date range",
-    "--------------------",
 
+    "---------------------------------------",
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",
     "SQL API: reuse params",
@@ -198,14 +198,9 @@
     "SQL API: Rollup with aliases",
     "SQL API: Nested Rollup over asterisk",
     "SQL API: Extended nested Rollup over asterisk",
-    "querying Products: dimensions -- doesn't work wo ordering",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
-    "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
     "querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + no dimension",
     "querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension",
     "pre-aggregations Customers: running total without time dimension",
-    "querying BigECommerce: null boolean",
     "querying BigECommerce: rolling count_distinct_approx window by 2 day",
     "querying BigECommerce: rolling count_distinct_approx window by 2 week",
     "querying BigECommerce: rolling count_distinct_approx window by 2 month",
@@ -216,6 +211,13 @@
     "SQL API: Nested Rollup",
     "SQL API: Nested Rollup with aliases",
     "SQL API: Timeshift measure from cube",
-    "SQL API: SQL push down push to cube quoted alias"
+
+    "---- Different results comparing to baseQuery version. Need to investigate ----",
+    "SQL API: SQL push down push to cube quoted alias",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+    "querying BigECommerce: rolling window YTD without granularity",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -179,20 +179,25 @@
     "querying BigECommerce: partitioned pre-agg",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+
+    "---------------------------------------",
     "Unsupported JOIN ON conditions. Unexpected 'big_e_commerce__order_date_month > subtractWeeks(date_to, 2)'",
-    "--------------------",
+    "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day",
     "querying BigECommerce: rolling window by 2 week",
     "querying BigECommerce: rolling window by 2 month",
+    "querying BigECommerce: rolling window YTD (month)",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+    "querying BigECommerce: rolling window YTD (month + week + day)",
+    "querying BigECommerce: rolling window YTD (month + week + day + no gran)",
 
     "---------------------------------------",
     "Requires Tesseract.                    ",
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
-    "--------------------",
 
     "---------------------------------------",
     "Custom Granularities                   ",
@@ -210,6 +215,11 @@
     "SQL API: Rollup over exprs",
     "SQL API: Rollup with aliases",
     "SQL API: Simple Rollup",
-    "SQL API: SQL push down push to cube quoted alias"
+    "SQL API: SQL push down push to cube quoted alias",
+
+    "SKIPPED SQL API (due to inconsistency)",
+    "---------------------------------------",
+    "Below doesn't work probably due to strict type comparison in ClickHouse, but tests run across all DBs",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -215,9 +215,7 @@
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
-    "--------------------",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -141,9 +141,8 @@
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
-    "--------------------",
+
     "---------------------------------------",
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",
@@ -163,6 +162,12 @@
     "SQL API: NULLS FIRST/LAST SQL push down",
     "SQL API: SQL push down push to cube quoted alias",
     "SQL API: Date/time comparison with SQL push down",
-    "SQL API: Date/time comparison with date_trunc with SQL push down"
+    "SQL API: Date/time comparison with date_trunc with SQL push down",
+
+    "---------------------------------------",
+    "Error during rewrite: Can't detect Cube query and it may be not supported yet.",
+    "---------------------------------------",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -131,12 +131,12 @@
     "querying BigECommerce: partitioned pre-agg",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+
     "---------------------------------------",
     "Requires Tesseract.                    ",
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
 
     "---------------------------------------",
@@ -158,6 +158,12 @@
     "SQL API: Nested Rollup over asterisk",
     "SQL API: Extended nested Rollup over asterisk",
     "SQL API: SQL push down push to cube quoted alias",
-    "SQL API: Date/time comparison with date_trunc with SQL push down"
+    "SQL API: Date/time comparison with date_trunc with SQL push down",
+
+    "---------------------------------------",
+    "Error during rewrite: Can't detect Cube query and it may be not supported yet.",
+    "---------------------------------------",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/postgres.json

@@ -156,6 +156,7 @@
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+
     "---------------------------------------",
     "Requires Tesseract.                    ",
     "---------------------------------------",
@@ -181,6 +182,13 @@
     "SQL API: Nested Rollup over asterisk",
     "SQL API: Extended nested Rollup over asterisk",
     "SQL API: Timeshift measure from cube",
-    "SQL API: SQL push down push to cube quoted alias"
+    "SQL API: SQL push down push to cube quoted alias",
+
+    "---- Different results comparing to baseQuery version. Need to investigate ----",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+    "querying BigECommerce: rolling window YTD without granularity",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }

packages/cubejs-testing-drivers/fixtures/redshift.json

@@ -167,14 +167,14 @@
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+
     "---------------------------------------",
     "Requires Tesseract.                    ",
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range",
-    
+
     "---------------------------------------",
     "SKIPPED SQL API (Need work)            ",
     "---------------------------------------",

packages/cubejs-testing-drivers/fixtures/snowflake.json

@@ -232,12 +232,12 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+
     "---------------------------------------",
     "Requires Tesseract.                    ",
     "---------------------------------------",
     "querying BigECommerce: rolling window by 2 day without date range",
     "querying BigECommerce: rolling window by 2 month without date range",
-    "querying BigECommerce: rolling window YTD",
     "querying BigECommerce: rolling window YTD without date range"
   ]
 }

packages/cubejs-testing-drivers/package.json

@@ -59,6 +59,7 @@
     "redshift-core": "yarn test-driver -i dist/test/redshift-core.test.js",
     "redshift-full": "yarn test-driver -i dist/test/redshift-full.test.js",
     "redshift-export-bucket-s3-full": "yarn test-driver -i dist/test/redshift-export-bucket-s3-full.test.js",
+    "update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-export-bucket-s3-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-full --mode=local -u",
     "tst": "clear && yarn tsc && yarn bigquery-core"
   },
   "files": [