Merge pull request #438 from lucaslortiz/master

Fix calc_batch_size maximum number of parameters allowed

Author: schlich

dbt/include/sqlserver/macros/materializations/seeds/helpers.sql

@@ -0,0 +1,57 @@
+{% macro sqlserver__get_binding_char() %}
+  {{ return('?') }}
+{% endmacro %}
+
+{% macro sqlserver__get_batch_size() %}
+  {{ return(400) }}
+{% endmacro %}
+
+{% macro calc_batch_size(num_columns) %}
+    {#
+        SQL Server allows for a max of 2098 parameters in a single statement.
+        Check if the max_batch_size fits with the number of columns, otherwise
+        reduce the batch size so it fits.
+    #}
+    {% set max_batch_size = get_batch_size() %}
+    {% set calculated_batch = (2098 / num_columns)|int %}
+    {% set batch_size = [max_batch_size, calculated_batch] | min %}
+
+    {{ return(batch_size) }}
+{%  endmacro %}
+
+{% macro sqlserver__load_csv_rows(model, agate_table) %}
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set batch_size = calc_batch_size(agate_table.column_names|length) %}
+  {% set bindings = [] %}
+  {% set statements = [] %}
+
+  {{ log("Inserting batches of " ~ batch_size ~ " records") }}
+
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
+
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
+
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
+
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
+
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
+{% endmacro %}

tests/functional/adapter/test_seed.py

@@ -243,7 +243,7 @@ def test_custom_batch_size(self, project, logs_dir):
             logs = "".join(fp.readlines())
         # this is changed from 350.
         # Fabric goes -1 of min batch of (2100/number of columns -1) or 400
-        assert "Inserting batches of 349.0 records" in logs
+        assert "Inserting batches of 349 records" in logs
 
 
 class SeedConfigBase: