Skip to content

Commit 07884d1

Browse files
committed
Update BigQuery temp relation naming to use microsecond precision
Changes the datetime format from %j%H%M%S (9 digits) to %H%M%S%f (12 digits) to provide microsecond-level uniqueness for temp table names during concurrent dbt execution, matching PostgreSQL adapter implementation. - Update bigquery__make_relation_with_suffix to use %H%M%S%f format - Add functional tests for BigQuery and PostgreSQL to verify 12-digit format
1 parent f4dfd35 commit 07884d1

File tree

3 files changed

+139
-0
lines changed

3 files changed

+139
-0
lines changed

dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,3 +209,34 @@ having count(*) > 1
209209
{% do adapter.upload_file(local_file_path, database, table_schema, table_name, kwargs=kwargs) %}
210210

211211
{% endmacro %}
212+
213+
{% macro bigquery__make_relation_with_suffix(base_relation, suffix, dstring) %}
214+
{% if dstring %}
215+
{% set dt = modules.datetime.datetime.now() %}
216+
{% set dtstring = dt.strftime("%H%M%S%f") %}
217+
{% set suffix = suffix ~ dtstring %}
218+
{% endif %}
219+
{% set suffix_length = suffix|length %}
220+
{% set relation_max_name_length = 1024 %} {# BigQuery limit #}
221+
{% if suffix_length > relation_max_name_length %}
222+
{% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}
223+
{% endif %}
224+
{% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}
225+
226+
{{ return(base_relation.incorporate(path={"identifier": identifier })) }}
227+
228+
{% endmacro %}
229+
230+
{% macro bigquery__make_temp_relation(base_relation, suffix) %}
231+
{% set temp_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=True) %}
232+
{{ return(temp_relation) }}
233+
{% endmacro %}
234+
235+
{% macro bigquery__make_intermediate_relation(base_relation, suffix) %}
236+
{{ return(bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}
237+
{% endmacro %}
238+
239+
{% macro bigquery__make_backup_relation(base_relation, backup_relation_type, suffix) %}
240+
{% set backup_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False) %}
241+
{{ return(backup_relation.incorporate(type=backup_relation_type)) }}
242+
{% endmacro %}
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import pytest
2+
import re
3+
from pathlib import Path
4+
from dbt.tests.util import run_dbt
5+
6+
7+
_MODEL_INCREMENTAL = """
8+
{{
9+
config(
10+
materialized='incremental',
11+
unique_key='id',
12+
on_schema_change='append_new_columns'
13+
)
14+
}}
15+
16+
select
17+
1 as id,
18+
'value' as name
19+
"""
20+
21+
22+
class TestTempRelationNaming:
23+
@pytest.fixture(scope="class")
24+
def models(self):
25+
return {"my_incremental_model.sql": _MODEL_INCREMENTAL}
26+
27+
def test_incremental_with_schema_change_creates_temp_relation(self, project):
28+
results = run_dbt(["run"])
29+
assert len(results) == 1
30+
31+
results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"])
32+
assert len(results) == 1
33+
34+
log_dir = Path(project.project_root) / "logs"
35+
if log_dir.exists():
36+
log_files = sorted(
37+
log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True
38+
)
39+
if log_files:
40+
log_content = log_files[0].read_text()
41+
temp_table_pattern = r"__dbt_tmp\d{12}\b"
42+
matches = re.findall(temp_table_pattern, log_content)
43+
44+
assert len(matches) > 0, (
45+
f"Expected temp table with pattern '{temp_table_pattern}' "
46+
f"(12 digits from %H%M%S%f format, not old 9-digit %j%H%M%S format)"
47+
)
48+
49+
for match in matches:
50+
suffix = match.replace("__dbt_tmp", "")
51+
assert len(suffix) == 12 and suffix.isdigit()
52+
53+
results = run_dbt(["run"])
54+
assert len(results) == 1
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import pytest
2+
import re
3+
from pathlib import Path
4+
from dbt.tests.util import run_dbt
5+
6+
7+
_MODEL_INCREMENTAL = """
8+
{{
9+
config(
10+
materialized='incremental',
11+
unique_key='id',
12+
on_schema_change='append_new_columns'
13+
)
14+
}}
15+
16+
select
17+
1 as id,
18+
'value' as name
19+
"""
20+
21+
22+
class TestTempRelationNaming:
23+
@pytest.fixture(scope="class")
24+
def models(self):
25+
return {"my_incremental_model.sql": _MODEL_INCREMENTAL}
26+
27+
def test_incremental_with_schema_change_creates_temp_relation(self, project):
28+
results = run_dbt(["run"])
29+
assert len(results) == 1
30+
31+
results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"])
32+
assert len(results) == 1
33+
34+
log_dir = Path(project.project_root) / "logs"
35+
if log_dir.exists():
36+
log_files = sorted(
37+
log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True
38+
)
39+
if log_files:
40+
log_content = log_files[0].read_text()
41+
temp_table_pattern = r"__dbt_tmp\d{12}\b"
42+
matches = re.findall(temp_table_pattern, log_content)
43+
44+
assert len(matches) > 0, (
45+
f"Expected temp table with pattern '{temp_table_pattern}' "
46+
f"(12 digits from %H%M%S%f format)"
47+
)
48+
49+
for match in matches:
50+
suffix = match.replace("__dbt_tmp", "")
51+
assert len(suffix) == 12 and suffix.isdigit()
52+
53+
results = run_dbt(["run"])
54+
assert len(results) == 1

0 commit comments

Comments
 (0)