Bugfix/standard columns (#7)

* Remove unnecessary models

* Remove source references

* remove feature-based columns

* Update readme

* Update changelog

Author: phillem15

CHANGELOG.md

@@ -1,2 +1,13 @@
+# dbt_netsuite_source v0.2.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #7](https://github.com/swishbi/dbt_netsuite_source/pull/7) includes the following breaking changes:
+- Remove columns from staging models that are feature dependent in NetSuite. These columns can be re-added via the pass through columns.
+
+## 🎉 Feature Updates 🎉
+
+## Contributors
+- [phillem15](https://github.com/phillem15) ([#7](https://github.com/swishbi/dbt_netsuite_source/pull/7))
+
 # dbt_netsuite_source v0.1.0
 Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you!

README.md

@@ -35,21 +35,27 @@ To use this dbt package, you must have the **Netsuite2** (netsuite2.com) ODBC co
 - accountingperiodfiscalcalendar
 - accountingperiod
 - accountingbook
-- consolidatedexchangerate
+- classification
 - currency
 - customer
-- classification
 - department
+- employee
+- employeestatus
+- employeestatuscategory
+- employeetype
+- employeetypecategory
 - entity
 - entityaddress
 - item
-- job
+- itemsubtype
+- itemtype
 - location
 - locationmainaddress
+- subsidiary
+- transaction
 - transactionaccountingline
 - transactionline
-- transaction
-- subsidiary
+- transactionstatus
 - vendor
 - vendorcategory
 

dbt_project.yml

@@ -27,10 +27,7 @@ vars:
     netsuite_classes: "{{ source('netsuite', 'classification') }}"
     netsuite_consolidated_exchange_rates: "{{ source('netsuite', 'consolidated_exchange_rate') }}"
     netsuite_currencies: "{{ source('netsuite', 'currency') }}"
-    netsuite_customer_address_book_entity_address: "{{ source('netsuite', 'customer_address_book_entity_address') }}"
-    netsuite_customer_address_books: "{{ source('netsuite', 'customer_address_books') }}"
     netsuite_customers: "{{ source('netsuite', 'customer') }}"
-    netsuite_deleted_records: "{{ source('netsuite', 'deleted_records') }}"
     netsuite_departments: "{{ source('netsuite', 'department') }}"
     netsuite_employee_status_categories: "{{ source('netsuite', 'employee_status_categories') }}"
     netsuite_employee_statuses: "{{ source('netsuite', 'employee_statuses') }}"
@@ -48,8 +45,6 @@ vars:
     netsuite_jobs: "{{ source('netsuite', 'job') }}"
     netsuite_location_main_address: "{{ source('netsuite', 'location_main_address') }}"
     netsuite_locations: "{{ source('netsuite', 'location') }}"
-    netsuite_next_transaction_line_links: "{{ source('netsuite', 'next_transaction_line_links') }}"
-    netsuite_previous_transaction_line_links: "{{ source('netsuite', 'previous_transaction_line_links') }}"
     netsuite_project_task_assignees: "{{ source('netsuite', 'project_task_assignees') }}"
     netsuite_project_tasks: "{{ source('netsuite', 'project_tasks') }}"
     netsuite_revenue_elements: "{{ source('netsuite', 'revenue_elements') }}"

models/_netsuite__sources.yml

@@ -618,10 +618,6 @@ sources:
           - name: category
             description: "{{ doc('vendor_category_id') }}"
 
-      - name: customer_address_books
-        identifier: "{{ var('netsuite_customer_address_book_identifier', 'customeraddressbook') }}"
-      - name: customer_address_book_entity_address
-        identifier: "{{ var('netsuite_customer_address_book_entity_address_identifier', 'customeraddressbookentityaddress') }}"
       - name: employees
         identifier: "{{ var('netsuite_employee_identifier', 'employee') }}"
       - name: employee_statuses
@@ -642,8 +638,6 @@ sources:
         identifier: "{{ var('netsuite_job_status_identifier', 'jobstatus') }}"
       - name: job_types
         identifier: "{{ var('netsuite_job_type_identifier', 'jobtype') }}"
-      - name: previous_transaction_line_links
-        identifier: "{{ var('netsuite_previous_transaction_line_link_identifier', 'previoustransactionlinelink') }}"
       - name: project_tasks
         identifier: "{{ var('netsuite_project_task_identifier', 'projecttask') }}"
       - name: project_task_assignees

models/base/base_netsuite__customers.sql

@@ -2,7 +2,7 @@ with customers as (
     select * from {{ ref('stg_netsuite__customers') }}
 ),
 entity_address as (
-    select * from {{ ref('stg_netsuite__customer_address_book_entity_address') }}
+    select * from {{ ref('stg_netsuite__entity_address') }}
 ),
 employees as (
     select * from {{ ref('base_netsuite__employees') }}
@@ -11,10 +11,10 @@ joined as (
 
     select
         customers.*,
-        entity_address.customer_city,
-        entity_address.customer_state,
-        entity_address.customer_zip_code,
-        entity_address.customer_country,
+        entity_address.city as customer_city,
+        entity_address.state as customer_state,
+        entity_address.zip_code as customer_zip_code,
+        entity_address.country as customer_country,
         sales_rep.employee_name_last_first as sales_rep_name_last_first,
         sales_rep.employee_name_first_last as sales_rep_name_first_last,
         coalesce(parent_customer.company_name, customers.company_name) as parent_company_name,
@@ -23,7 +23,7 @@ joined as (
     from customers
 
     left join entity_address 
-        on coalesce(customers.default_billing_address_id, customers.default_shipping_address_id) = entity_address.customer_address_book_address_id
+        on coalesce(customers.default_billing_address_id, customers.default_shipping_address_id) = entity_address.entity_address_id
 
     left join employees as sales_rep 
         on sales_rep.employee_id = customers.sales_rep_id

models/stg_netsuite__accounts.sql

@@ -6,13 +6,12 @@ renamed as (
         accountsearchdisplaynamecopy as account_name,
         acctnumber as account_number,
         accttype as account_type_id,
-        cashflowrate as cash_flow_rate_type,
         class as class_id,
         {% if var('netsuite__multiple_currencies_enabled', false) %}
+        cashflowrate as cash_flow_rate_type,
         currency as currency_id,
         generalrate as general_rate_type,
         {% endif %}
-        deferralacct as deferral_account_id,
         department as department_id,
         description as account_description,
         externalid as account_external_id,

models/stg_netsuite__customer_address_book_entity_address.sql

@@ -6,17 +6,13 @@ renamed as (
         assetaccount as asset_account_id,
         class as class_id,
         deferralaccount as deferral_account_id,
-        deferredrevenueaccount as deferred_revenue_account_id,
         department as department_id,
         description as item_sales_description,
         expenseaccount as expense_account_id,
         fullname as item_name,
         gainlossaccount as gain_loss_account_id,
         id as item_id,
         incomeaccount as income_account_id,
-        intercoexpenseaccount as intercompany_expense_account_id,
-        intercoincomeaccount as intercompany_income_account_id,
-        isfulfillable = 'T' as is_fulfillable,
         itemtype as item_type_id,
         lastmodifieddate as last_modified_date,
         location as location_id,

models/stg_netsuite__customer_address_books.sql

@@ -3,39 +3,26 @@ with source as (
 ),
 renamed as (
     select
-        actualshipdate as actual_ship_date,
+        id as transaction_line_id,
+        transaction as transaction_id,
         class as class_id,
-        costestimaterate as cost_estimate_rate,
-        createdfrom as created_from_id,
-        createdpo as created_po_id,
         department as department_id,
-        dropship = 'T' as is_drop_shipment,
         entity as entity_id,
-        id as transaction_line_id,
-        isbillable = 'T' as is_billable,
-        isclosed = 'T' as is_closed,
-        iscogs = 'T' as is_cogs,
-        isfullyshipped = 'T' as is_fully_shipped,
-        isinventoryaffecting = 'T' as is_inventory_affecting,
+        location as location_id,
         item as item_id,
+        subsidiary as  subsidiary_id,
         linelastmodifieddate as line_last_modified_date,
         linesequencenumber as line_sequence_number,
-        location as location_id,
-        mainline = 'T' as is_main_line,
         memo,
         quantity,
-        quantitybackordered as quantity_back_ordered,
-        quantitybilled as quantity_billed,
-        quantitycommitted as quantity_committed,
-        quantitypacked as quantity_packed,
-        quantitypicked as quantity_picked,
-        quantityshiprecv as quantity_shipped_received,
         rate,
-        specialorder = 'T' as is_special_order,
-        subsidiary as  subsidiary_id,
-        taxline = 'T' as is_tax_line,
-        transaction as transaction_id,
-        uniquekey as transaction_line_unique_key
+        cleared = 'T' as is_cleared,
+        commitmentfirm = 'T' as is_commitment_firm,
+        isbillable = 'T' as is_billable,
+        isclosed = 'T' as is_closed,
+        iscogs = 'T' as is_cogs,
+        mainline = 'T' as is_main_line,
+        taxline = 'T' as is_tax_line
 
         --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }}

models/stg_netsuite__items.sql

@@ -11,9 +11,7 @@ renamed as (
         duedate as transaction_due_date,
         employee as employee_id,
         entity as entity_id,
-        expectedclosedate as transaction_expected_close_date,
         id as transaction_id,
-        intercoadj = 'T' as is_transaction_intercompany_adjustment,
         isreversal = 'T' as is_transaction_reversal,
         lastmodifieddate as last_modified_date,
         memo as transaction_memo,
@@ -24,7 +22,6 @@ renamed as (
         tranid as transaction_name,
         transactionnumber as transaction_number,
         type as transaction_type,
-        voided = 'T' as is_transaction_voided,
 
         concat('https://{{ var("netsuite_account_id", "123456") }}.app.netsuite.com/app/accounting/transactions/', lower(type), '.nl?id=', id) as transaction_url_link,
         concat(type,'_',id) as reference_id