Skip to content

Update/explicit union #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
fivetran-catfritz wants to merge 2 commits into from
Closed

Update/explicit union #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
323523c
update/explicit-union
fivetran-catfritz Jun 11, 2026
4dd897d
simplify casts
fivetran-catfritz Jun 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion integration_tests/ci/test_scenarios.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@ test_scenarios:
using_transfer: false
using_vendor_credit: false
using_customer_type: false
include_incremental: false
include_incremental: false
57 changes: 57 additions & 0 deletions integration_tests/tests/consistency/consistency_column_types.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{{ config(
tags="fivetran_validations",
enabled=var('fivetran_validation_tests_enabled', false)
) }}

-- BigQuery only: uses INFORMATION_SCHEMA.COLUMNS which is dataset-scoped in BQ

{% set end_models = [
'quickbooks__ap_ar_enhanced',
'quickbooks__balance_sheet',
'quickbooks__cash_flow_statement',
'quickbooks__expenses_sales_enhanced',
'quickbooks__general_ledger',
'quickbooks__general_ledger_by_period',
'quickbooks__profit_and_loss'
] %}

with prod_columns as (
{% for model in end_models %}
select '{{ model }}' as model_name, column_name, data_type
from `{{ target.schema }}_quickbooks_prod.INFORMATION_SCHEMA.COLUMNS`
where table_name = '{{ model }}'
{{ 'union all' if not loop.last }}
{% endfor %}
),

dev_columns as (
{% for model in end_models %}
select '{{ model }}' as model_name, column_name, data_type
from `{{ target.schema }}_quickbooks_dev.INFORMATION_SCHEMA.COLUMNS`
where table_name = '{{ model }}'
{{ 'union all' if not loop.last }}
{% endfor %}
),

final as (
select
coalesce(prod.model_name, dev.model_name) as model_name,
coalesce(prod.column_name, dev.column_name) as column_name,
prod.data_type as prod_type,
dev.data_type as dev_type,
case
when prod.column_name is null then 'column added in dev'
when dev.column_name is null then 'column removed in dev'
else 'type mismatch'
end as issue
from prod_columns prod
full outer join dev_columns dev
on prod.model_name = dev.model_name
and prod.column_name = dev.column_name
where prod.data_type != dev.data_type
or prod.column_name is null
or dev.column_name is null
)

select *
from final
18 changes: 18 additions & 0 deletions macros/explicit_union.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{% macro explicit_union(relations, columns) %}
{{ return(adapter.dispatch('explicit_union', 'quickbooks')(relations, columns)) }}
{% endmacro %}

{% macro default__explicit_union(relations, columns) %}

{%- for rel_name, enabled in relations.items() if enabled == 'enabled' %}

{{ 'union all' if not loop.first }}

select
{%- for col_name in columns %}
{{ col_name }}{{ ',' if not loop.last }}
{%- endfor %}
from {{ ref(rel_name) }}
{%- endfor %}

{% endmacro %}
13 changes: 12 additions & 1 deletion macros/quickbooks_macros.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,15 @@ macros:
description: List of strings referring to particular models to then be appended and eventually unioned if enabled.
- name: enabled_unioned_models
type: list
description: List of relations of models that are enabled to eventually be unioned into a general ledger model.
description: List of relations of models that are enabled to eventually be unioned into a general ledger model.

- name: explicit_union
description: >
Generates a UNION ALL across a set of models with explicit column casts. Designed for use with ephemeral models where `dbt_utils.union_relations` cannot introspect column types at compile time. Only relations with value `'enabled'` are included in the union.
arguments:
- name: relations
type: dict
description: Dict mapping model name (passed to ref()) to either `'enabled'` or `'disabled'`.
- name: columns
type: list
description: Ordered list of column names to select in the union.
68 changes: 33 additions & 35 deletions models/intermediate/int_quickbooks__expenses_union.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,33 @@
with expense_union as (

select *
from {{ ref('int_quickbooks__purchase_transactions') }}

{% if var('using_bill', True) %}
union all

select *
from {{ ref('int_quickbooks__bill_transactions') }}
{% endif %}

{% if var('using_journal_entry', True) %}
union all

select *
from {{ ref('int_quickbooks__journal_entry_transactions')}}
{% endif %}

{% if var('using_deposit', True) %}
union all

select *
from {{ ref('int_quickbooks__deposit_transactions')}}
{% endif %}
{%- set expense_relations = {
'int_quickbooks__purchase_transactions': 'enabled',
'int_quickbooks__bill_transactions': 'enabled' if var('using_bill', True) else 'disabled',
'int_quickbooks__journal_entry_transactions': 'enabled' if var('using_journal_entry', True) else 'disabled',
'int_quickbooks__deposit_transactions': 'enabled' if var('using_deposit', True) else 'disabled',
'int_quickbooks__vendor_credit_transactions': 'enabled' if var('using_vendor_credit', True) else 'disabled'
} -%}

{%- set expense_transaction_columns = [
'transaction_id',
'source_relation',
'transaction_line_id',
'doc_number',
'transaction_type',
'transaction_date',
'account_id',
'class_id',
'department_id',
'customer_id',
'vendor_id',
'billable_status',
'description',
'amount',
'converted_amount',
'total_amount',
'total_converted_amount'
] -%}

{% if var('using_vendor_credit', True) %}
union all

select *
from {{ ref('int_quickbooks__vendor_credit_transactions') }}
{% endif %}
with expense_union as (
{{ explicit_union(expense_relations, expense_transaction_columns) }}
),

customers as (
Expand All @@ -48,7 +46,7 @@ customer_types as (
{% endif %}

{% if var('using_department', True) %}
departments as (
departments as (

select *
from {{ ref('stg_quickbooks__department') }}
Expand All @@ -70,8 +68,8 @@ expense_accounts as (

final as (

select
'expense' as transaction_source,
select
cast('expense' as {{ dbt.type_string() }}) as transaction_source,
expense_union.transaction_id,
expense_union.source_relation,
expense_union.transaction_line_id,
Expand Down Expand Up @@ -134,4 +132,4 @@ final as (
)

select *
from final
from final
73 changes: 35 additions & 38 deletions models/intermediate/int_quickbooks__sales_union.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,40 +1,37 @@
{{ config(enabled=fivetran_utils.enabled_vars_one_true(['using_sales_receipt','using_invoice'])) }}

with sales_union as (

{% if var('using_sales_receipt', True) %}
select *
from {{ ref('int_quickbooks__sales_receipt_transactions') }}
{% endif %}

{% if fivetran_utils.enabled_vars(['using_sales_receipt','using_invoice']) %}
union all

select *
from {{ ref('int_quickbooks__invoice_transactions') }}

{% else %}

{% if var('using_invoice', True) %}
select *
from {{ ref('int_quickbooks__invoice_transactions') }}
{% endif %}

{% endif %}

{% if var('using_refund_receipt', True) %}
union all

select *
from {{ ref('int_quickbooks__refund_receipt_transactions') }}
{% endif %}
{%- set sales_relations = {
'int_quickbooks__sales_receipt_transactions': 'enabled' if var('using_sales_receipt', True) else 'disabled',
'int_quickbooks__invoice_transactions': 'enabled' if var('using_invoice', True) else 'disabled',
'int_quickbooks__refund_receipt_transactions': 'enabled' if var('using_refund_receipt', True) else 'disabled',
'int_quickbooks__credit_memo_transactions': 'enabled' if var('using_credit_memo', True) else 'disabled'
} -%}

{%- set sales_transaction_columns = [
'transaction_id',
'source_relation',
'transaction_line_id',
'doc_number',
'transaction_type',
'transaction_date',
'item_id',
'item_quantity',
'item_unit_price',
'account_id',
'class_id',
'department_id',
'customer_id',
'vendor_id',
'billable_status',
'description',
'amount',
'converted_amount',
'total_amount',
'total_converted_amount'
] -%}

{% if var('using_credit_memo', True) %}
union all

select *
from {{ ref('int_quickbooks__credit_memo_transactions') }}
{% endif %}
with sales_union as (
{{ explicit_union(sales_relations, sales_transaction_columns) }}
),

customers as (
Expand All @@ -53,7 +50,7 @@ customer_types as (
{% endif %}

{% if var('using_department', True) %}
departments as (
departments as (

select *
from {{ ref('stg_quickbooks__department') }}
Expand All @@ -75,8 +72,8 @@ income_accounts as (

final as (

select
'sales' as transaction_source,
select
cast('sales' as {{ dbt.type_string() }}) as transaction_source,
sales_union.transaction_id,
sales_union.source_relation,
sales_union.transaction_line_id,
Expand Down Expand Up @@ -138,4 +135,4 @@ final as (
)

select *
from final
from final
63 changes: 39 additions & 24 deletions models/quickbooks__expenses_sales_enhanced.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,44 @@
with expenses as (
{%- set enhanced_relations = {
'int_quickbooks__expenses_union': 'enabled',
'int_quickbooks__sales_union': 'enabled' if fivetran_utils.enabled_vars_one_true(['using_sales_receipt', 'using_invoice']) else 'disabled'
} -%}

select *
from {{ ref('int_quickbooks__expenses_union') }}
),
{%- set enhanced_columns = [
'transaction_source',
'transaction_id',
'source_relation',
'transaction_line_id',
'doc_number',
'transaction_type',
'transaction_date',
'item_id',
'item_quantity',
'item_unit_price',
'account_id',
'account_name',
'account_sub_type',
'account_number',
'parent_account_number',
'class_id',
'department_id'
] -%}
{%- do enhanced_columns.extend(['department_name']) if var('using_department', True) -%}
{%- do enhanced_columns.extend(['customer_id', 'customer_name', 'customer_website']) -%}
{%- do enhanced_columns.extend(['customer_type_name']) if var('using_customer_type', True) -%}
{%- do enhanced_columns.extend([
'vendor_id',
'vendor_name',
'billable_status',
'description',
'amount',
'converted_amount',
'total_amount',
'total_converted_amount'
]) -%}

{% if fivetran_utils.enabled_vars_one_true(['using_sales_receipt','using_invoice']) %}
sales as (

select *
from {{ ref('int_quickbooks__sales_union') }}
),
{% endif %}

final as (

select *
from expenses

{% if fivetran_utils.enabled_vars_one_true(['using_sales_receipt','using_invoice']) %}
union all

select *
from sales
{% endif %}
with final as (
{{ explicit_union(enhanced_relations, enhanced_columns) }}
)

select *
from final
from final
Loading