Schema Change: Field deprecations/sunsetting

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_shopify v1.8.0
+
+[PR #155](https://github.com/fivetran/dbt_shopify/pull/155) includes the following updates:
+
+## Schema/Data Changes
+**5 total changes • 5 possible breaking changes**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ------------- | ----------- | --- | --- | ----- |
+| `shopify__order_lines` | Sunset columns | `variant_fulfillment_service`<br>`variant_grams`<br>`variant_weight`<br>`variant_weight_unit`<br>`variant_option_1`<br>`variant_option_2`<br>`variant_option_3` | | [Completes sunset of deprecated fields from January updates](https://fivetran.com/docs/connectors/applications/shopify/changelog#january2025). |
+| `shopify__inventory_levels` | Sunset columns | `variant_fulfillment_service`<br>`variant_inventory_management`<br>`variant_grams`<br>`variant_weight`<br>`variant_weight_unit`<br>`variant_option_1`<br>`variant_option_2`<br>`variant_option_3` | | [Completes sunset of deprecated fields from January updates](https://fivetran.com/docs/connectors/applications/shopify/changelog#january2025). |
+| `shopify__daily_shop`<br>`shopify_gql__daily_shop` | Data changes | Order metrics may be inflated | Order metrics now correctly calculated | Soft-deleted staging shop records are now filtered out, which previously could have incorrectly inflated metrics. |
+| `shopify__order_lines` | Deprecated columns | `pre_tax_price`<br>`pre_tax_price_set` | | These fields now return null and will be sunset in a future release. |
+| `shopify__products` | Deprecated column | `published_scope` | | This field now returns null and will be sunset in a future release. |
+
 # dbt_shopify v1.7.0
 
 This release includes the following updates:

README.md

@@ -81,7 +81,7 @@ If you are **not** using the [Shopify Holistic reporting package](https://github
 ```yml
 packages:
   - package: fivetran/shopify
-    version: [">=1.7.0", "<1.8.0"]
+    version: [">=1.8.0", "<1.9.0"]
 ```
 
 > All required sources and staging models are now bundled into this transformation package. Do not include `fivetran/shopify_source` in your `packages.yml` since this package has been deprecated.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '1.7.0'
+version: '1.8.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '1.7.0'
+version: '1.8.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -494,4 +494,4 @@ seeds:
 
 flags:
   send_anonymous_usage_stats: False
-  use_colors: True
+  use_colors: True

integration_tests/tests/integrity/horizontal_integrity_daily_shop_orders.sql

@@ -32,19 +32,21 @@ daily_shop_metrics as (
 
     select 
         source_relation,
+        shop_id,
         date_day,
         sum(count_orders) as count_orders,
         sum(order_adjusted_total) as order_adjusted_total
 
     from daily_shop
     where date_day >= '2020-01-01' and date_day < '2024-06-10'
-    group by 1,2
+    group by 1,2,3
 ),
 
 final as (
 
     select
         daily_shop_metrics.source_relation,
+        daily_shop_metrics.date_day,
         coalesce(daily_shop_metrics.count_orders, 0) as daily_shop_count_orders,
         coalesce(order_metrics.count_orders, 0) as order_count_orders,
         coalesce(daily_shop_metrics.order_adjusted_total, 0) as daily_shop_order_adjusted_total,

macros/staging/get_inventory_level_columns.sql

@@ -1,6 +1,5 @@
 {% macro get_inventory_level_columns() %}
 
-{# Columns below line 13 to be deprecated. #}
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "inventory_item_id", "datatype": dbt.type_int()},
@@ -10,8 +9,6 @@
     {"name": "created_at", "datatype": dbt.type_timestamp()},
     {"name": "deactivation_alert", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_int()}
-    ,
-    {"name": "available", "datatype": dbt.type_int()}
 ] %}
 
 {{ return(columns) }}

macros/staging/get_product_variant_columns.sql

@@ -1,6 +1,5 @@
 {% macro get_product_variant_columns() %}
 
-{# Columns below line 28 to be deprecated. #}
 {% set columns = [
     {"name": "id", "datatype": dbt.type_numeric()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
@@ -16,22 +15,13 @@
     {"name": "compare_at_price", "datatype": dbt.type_float()},
     {"name": "taxable", "datatype": dbt.type_boolean()},
     {"name": "barcode", "datatype": dbt.type_string()},
-    {"name": "old_inventory_quantity", "datatype": dbt.type_numeric()},
     {"name": "inventory_quantity", "datatype": dbt.type_numeric()},
     {"name": "tax_code", "datatype": dbt.type_string()},
     {"name": "available_for_sale", "datatype": dbt.type_boolean()},
     {"name": "display_name", "datatype": dbt.type_string()},
     {"name": "legacy_resource_id", "datatype": dbt.type_bigint()},
     {"name": "requires_components", "datatype": dbt.type_boolean()},
-    {"name": "sellable_online_quantity", "datatype": dbt.type_int()},
-    {"name": "fulfillment_service", "datatype": dbt.type_string()},
-    {"name": "grams", "datatype": dbt.type_float()},
-    {"name": "inventory_management", "datatype": dbt.type_string()},
-    {"name": "weight", "datatype": dbt.type_float()},
-    {"name": "weight_unit", "datatype": dbt.type_string()},
-    {"name": "option_1", "datatype": dbt.type_string()},
-    {"name": "option_2", "datatype": dbt.type_string()},
-    {"name": "option_3", "datatype": dbt.type_string()}
+    {"name": "sellable_online_quantity", "datatype": dbt.type_int()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }}

models/graphql/staging/src_shopify_graphql.yml

@@ -1561,7 +1561,7 @@ sources:
           - name: applies_once_per_customer
             description: Whether a customer can only use the discount once.
           - name: async_usage_count
-            description: The number of times the discount has been used, updated asynchronously.
+            description: (DEPRECATED) The number of times the discount has been used, updated asynchronously. This field now returns null following a connector update and will be removed in a future release.
           - name: usage_limit
             description: The maximum number of times the discount can be used across all customers.
           - name: codes_count_count
@@ -1608,7 +1608,7 @@ sources:
           - name: applies_once_per_customer
             description: Whether a customer can only use the discount once.
           - name: async_usage_count
-            description: The number of times the discount has been used, updated asynchronously.
+            description: (DEPRECATED) The number of times the discount has been used, updated asynchronously. This field now returns null following a connector update and will be removed in a future release.
           - name: usage_limit
             description: The maximum number of times the discount can be used.
           - name: codes_count
@@ -1652,7 +1652,7 @@ sources:
           - name: applies_once_per_customer
             description: Whether a customer can only use the discount once.
           - name: async_usage_count
-            description: The number of times the discount has been used, updated asynchronously.
+            description: (DEPRECATED) The number of times the discount has been used, updated asynchronously. This field now returns null following a connector update and will be removed in a future release.
           - name: usage_limit
             description: The maximum number of times the discount can be used across all customers.
           - name: codes_count
@@ -1698,7 +1698,7 @@ sources:
           - name: applies_once_per_customer
             description: Whether a customer can only use the discount once.
           - name: async_usage_count
-            description: The number of times the discount has been used, updated asynchronously.
+            description: (DEPRECATED) The number of times the discount has been used, updated asynchronously. This field now returns null following a connector update and will be removed in a future release.
           - name: usage_limit
             description: The maximum number of times the discount can be used across all customers.
           - name: codes_count
@@ -1729,8 +1729,7 @@ sources:
           - name: id
             description: The unique ID of the redeem code.
           - name: async_usage_count
-            description: >
-              The number of times that the redeem code has been used. This value is updated asynchronously and may not match real-time usage counts.
+            description: (DEPRECATED) The number of times that the redeem code has been used. This field now returns null following a connector update and will be removed in a future release.
           - name: code
             description: The redeemable code that a customer can enter at checkout (e.g., `SUMMER20`).
           - name: created_by_description

models/graphql/staging/stg_shopify_gql__shop.sql

@@ -78,3 +78,4 @@ final as (
 
 select *
 from final
+where not coalesce(is_deleted, false)

models/graphql/staging/stg_shopify_graphql.yml

@@ -830,7 +830,7 @@ models:
         tests:
           - not_null
       - name: async_usage_count
-        description: The number of times that the redeem code has been used. This is updated asynchronously and may differ from real-time usage.
+        description: (DEPRECATED) The number of times that the redeem code has been used. This field now returns null following a connector update and will be removed in a future release.
       - name: code
         description: The redeemable code that a customer can enter at checkout (e.g., `SUMMER20`).
       - name: created_by_description

models/rest/shopify__inventory_levels.sql

@@ -62,19 +62,11 @@ inventory_quantity_aggregated as (
         {% set inventory_states = var('shopify_inventory_states', ['incoming', 'on_hand', 'available', 'committed', 'reserved', 'damaged', 'safety_stock', 'quality_control']) -%}
         {% for inventory_state in inventory_states -%}
             , sum(case when lower(inventory_state_name) = {{ "'" ~ inventory_state|lower ~ "'" }}
-                {% if inventory_state|lower == 'available ' -%}
-                then coalesce(inventory_quantity.quantity, inventory_level.available_quantity)
-                {% else -%}
                 then inventory_quantity.quantity
-                {% endif -%}
                 end) as {{ inventory_state }}_quantity
         {% endfor -%}
 
     from inventory_quantity
-    left join inventory_level
-        on inventory_quantity.inventory_item_id = inventory_level.inventory_item_id
-        and inventory_quantity.inventory_level_id = inventory_level.inventory_level_id
-        and inventory_quantity.source_relation = inventory_level.source_relation
     group by 1,2,3
 ),
 
@@ -137,17 +129,9 @@ joined_info as (
         product_variant_media.media_id as variant_media_id,
         {% endif %}
 
-        product_variant.fulfillment_service as variant_fulfillment_service,
-        product_variant.inventory_management as variant_inventory_management,
         product_variant.is_taxable as is_variant_taxable,
         product_variant.barcode as variant_barcode,
-        product_variant.grams as variant_grams, 
         product_variant.inventory_quantity as variant_inventory_quantity,
-        product_variant.weight as variant_weight,
-        product_variant.weight_unit as variant_weight_unit,
-        product_variant.option_1 as variant_option_1,
-        product_variant.option_2 as variant_option_2,
-        product_variant.option_3 as variant_option_3,
         product_variant.tax_code as variant_tax_code,
         product_variant.created_timestamp as variant_created_at,
         product_variant.updated_timestamp as variant_updated_at,

models/rest/shopify__order_lines.sql

@@ -78,17 +78,10 @@ refunds_aggregated as (
         product_variants.position as variant_position,
         product_variants.inventory_policy as variant_inventory_policy,
         product_variants.compare_at_price as variant_compare_at_price,
-        product_variants.fulfillment_service as variant_fulfillment_service,
 
         product_variants.is_taxable as variant_is_taxable,
         product_variants.barcode as variant_barcode,
-        product_variants.grams as variant_grams,
         product_variants.inventory_quantity as variant_inventory_quantity,
-        product_variants.weight as variant_weight,
-        product_variants.weight_unit as variant_weight_unit,
-        product_variants.option_1 as variant_option_1,
-        product_variants.option_2 as variant_option_2,
-        product_variants.option_3 as variant_option_3,
         product_variants.tax_code as variant_tax_code,
         product_variants.is_available_for_sale as variant_is_available_for_sale,
         product_variants.display_name as variant_display_name,