Refactor customers mart rollup

.gitignore

@@ -14,3 +14,4 @@ profiles.yml
 
 .ruff_cache
 __pycache__
+dbt_internal_packages/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "dbt.enableNewLineagePanel": true
+}

dbt_project.yml

@@ -15,8 +15,8 @@ test-paths: ["data-tests"]
 seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
-
 target-path: "target"
+function-paths: ["functions"]
 clean-targets:
   - "target"
   - "dbt_packages"
@@ -36,3 +36,5 @@ models:
       +materialized: view
     marts:
       +materialized: table
+flags:
+  require_generic_test_arguments_property: true

functions/is_positive_int.py

@@ -0,0 +1,4 @@
+import re
+
+def main(a_string):
+    return 1 if re.search(r'^[0-9]+$', a_string or '') else 0

functions/schema.yml

@@ -0,0 +1,15 @@
+functions:
+    - name: is_positive_int # required
+      description: My UDF that returns 1 if a string represents a naked positive integer (like "10", "+8" is not allowed). # optional
+      config:
+        runtime_version: "3.11"   # required
+        entry_point: main         # required
+        schema: udf_schema
+        database: udf_db
+        volatility: deterministic  
+      arguments:                   # optional
+        - name: a_string           # required if arguments is specified
+          data_type: string        # required if arguments is specified
+          description: The string that I want to check if it's representing a positive integer (like "10") 
+      returns:                    # required
+        data_type: integer        # required

models/marts/customers.sql

@@ -15,7 +15,7 @@ orders as (
 customer_orders_summary as (
 
     select
-        orders.customer_id,
+        customers.customer_id,
 
         count(distinct orders.order_id) as count_lifetime_orders,
         count(distinct orders.order_id) > 1 as is_repeat_buyer,
@@ -25,7 +25,10 @@ customer_orders_summary as (
         sum(orders.tax_paid) as lifetime_tax_paid,
         sum(orders.order_total) as lifetime_spend
 
-    from orders
+    from customers
+
+    left join orders
+        on customers.customer_id = orders.customer_id
 
     group by 1
 

models/marts/customers.yml

@@ -3,7 +3,8 @@ models:
     description: Customer overview data mart, offering key details for each unique customer. One row per customer.
     data_tests:
       - dbt_utils.expression_is_true:
-          expression: "lifetime_spend_pretax + lifetime_tax_paid = lifetime_spend"
+          arguments:
+            expression: "lifetime_spend_pretax + lifetime_tax_paid = lifetime_spend"
     columns:
       - name: customer_id
         description: The unique key of the orders mart.
@@ -28,7 +29,8 @@ models:
         description: Options are 'new' or 'returning', indicating if a customer has ordered more than once or has only placed their first order to date.
         data_tests:
           - accepted_values:
-              values: ["new", "returning"]
+              arguments:
+                values: ["new", "returning"]
 
 semantic_models:
   - name: customers
@@ -104,4 +106,4 @@ saved_queries:
     exports:
       - name: customer_order_metrics
         config:
-          export_as: table
+          export_as: table

models/marts/models.sql

@@ -0,0 +1,4 @@
+select
+    maybe_positive_int_column,
+    {{ function('is_positive_int') }}(maybe_positive_int_column) as is_positive_int
+from {{ ref('customers') }}

models/marts/order_items.sql

@@ -1,12 +1,9 @@
 with
 
 order_items as (
-
-    select * from {{ ref('stg_order_items') }}
-
+    select * from {{ref ('stg_order_items') }}
 ),
 
-
 orders as (
 
     select * from {{ ref('stg_orders') }}
@@ -19,6 +16,7 @@ products as (
 
 ),
 
+
 supplies as (
 
     select * from {{ ref('stg_supplies') }}
@@ -30,7 +28,7 @@ order_supplies_summary as (
     select
         product_id,
 
-        sum(supply_cost) as supply_cost
+        sum(distinct supply_cost) as supply_cost
 
     from supplies
 
@@ -54,7 +52,7 @@ joined as (
 
     from order_items
 
-    left join orders on order_items.order_id = orders.order_id
+    join orders on order_items.order_id = orders.order_id
 
     left join products on order_items.product_id = products.product_id
 
@@ -63,4 +61,5 @@ joined as (
 
 )
 
-select * from joined
+select *,123 as new_column from joined
+where is_drink_item=true

models/marts/order_items.yml

@@ -8,37 +8,37 @@ models:
       - name: order_id
         data_tests:
           - relationships:
-              to: ref('orders')
-              field: order_id
-
+              arguments:
+                to: ref('orders')
+                field: order_id
 unit_tests:
   - name: test_supply_costs_sum_correctly
     description: "Test that the counts of drinks and food orders convert to booleans properly."
     model: order_items
     given:
       - input: ref('stg_supplies')
         rows:
-          - { product_id: 1, supply_cost: 4.50 }
-          - { product_id: 2, supply_cost: 3.50 }
-          - { product_id: 2, supply_cost: 5.00 }
+          - {product_id: 1, supply_cost: 4.50}
+          - {product_id: 2, supply_cost: 3.50}
+          - {product_id: 2, supply_cost: 5.00}
       - input: ref('stg_products')
         rows:
-          - { product_id: 1 }
-          - { product_id: 2 }
+          - {product_id: 1}
+          - {product_id: 2}
       - input: ref('stg_order_items')
         rows:
-          - { order_id: 1, product_id: 1 }
-          - { order_id: 2, product_id: 2 }
-          - { order_id: 2, product_id: 2 }
+          - {order_id: 1, product_id: 1}
+          - {order_id: 2, product_id: 2}
+          - {order_id: 2, product_id: 2}
       - input: ref('stg_orders')
         rows:
-          - { order_id: 1 }
-          - { order_id: 2 }
+          - {order_id: 1}
+          - {order_id: 2}
     expect:
       rows:
-        - { order_id: 1, product_id: 1, supply_cost: 4.50 }
-        - { order_id: 2, product_id: 2, supply_cost: 8.50 }
-        - { order_id: 2, product_id: 2, supply_cost: 8.50 }
+        - {order_id: 1, product_id: 1, supply_cost: 4.50}
+        - {order_id: 2, product_id: 2, supply_cost: 8.50}
+        - {order_id: 2, product_id: 2, supply_cost: 8.50}
 
 semantic_models:
   - name: order_item
@@ -178,4 +178,4 @@ saved_queries:
     exports:
       - name: revenue_metrics
         config:
-          export_as: table
+          export_as: table

models/marts/orders.sql

@@ -50,11 +50,12 @@ compute_booleans as (
         order_items_summary.count_drink_items,
         order_items_summary.count_order_items,
         order_items_summary.count_food_items > 0 as is_food_order,
-        order_items_summary.count_drink_items > 0 as is_drink_order
+        order_items_summary.count_drink_items > 0 as is_drink_order,
+
 
     from orders
 
-    left join
+    inner join
         order_items_summary
         on orders.order_id = order_items_summary.order_id
 

models/marts/orders.yml

@@ -3,9 +3,11 @@ models:
     description: Order overview data mart, offering key details for each order inlcluding if it's a customer's first order and a food vs. drink item breakdown. One row per order.
     data_tests:
       - dbt_utils.expression_is_true:
-          expression: "order_items_subtotal = subtotal"
+          arguments:
+            expression: "order_items_subtotal = subtotal"
       - dbt_utils.expression_is_true:
-          expression: "order_total = subtotal + tax_paid"
+          arguments:
+            expression: "order_total = subtotal + tax_paid"
     columns:
       - name: order_id
         description: The unique key of the orders mart.
@@ -16,8 +18,9 @@ models:
         description: The foreign key relating to the customer who placed the order.
         data_tests:
           - relationships:
-              to: ref('stg_customers')
-              field: customer_id
+              arguments:
+                to: ref('stg_customers')
+                field: customer_id
       - name: order_total
         description: The total amount of the order in USD including tax.
       - name: ordered_at
@@ -36,44 +39,17 @@ unit_tests:
     given:
       - input: ref('order_items')
         rows:
-          - {
-              order_id: 1,
-              order_item_id: 1,
-              is_drink_item: false,
-              is_food_item: true,
-            }
-          - {
-              order_id: 1,
-              order_item_id: 2,
-              is_drink_item: true,
-              is_food_item: false,
-            }
-          - {
-              order_id: 2,
-              order_item_id: 3,
-              is_drink_item: false,
-              is_food_item: true,
-            }
+          - {order_id: 1, order_item_id: 1, is_drink_item: false, is_food_item: true}
+          - {order_id: 1, order_item_id: 2, is_drink_item: true, is_food_item: false}
+          - {order_id: 2, order_item_id: 3, is_drink_item: false, is_food_item: true}
       - input: ref('stg_orders')
         rows:
-          - { order_id: 1 }
-          - { order_id: 2 }
+          - {order_id: 1}
+          - {order_id: 2}
     expect:
       rows:
-        - {
-            order_id: 1,
-            count_food_items: 1,
-            count_drink_items: 1,
-            is_drink_order: true,
-            is_food_order: true,
-          }
-        - {
-            order_id: 2,
-            count_food_items: 1,
-            count_drink_items: 0,
-            is_drink_order: false,
-            is_food_order: true,
-          }
+        - {order_id: 1, count_food_items: 1, count_drink_items: 1, is_drink_order: true, is_food_order: true}
+        - {order_id: 2, count_food_items: 1, count_drink_items: 0, is_drink_order: false, is_food_order: true}
 
 semantic_models:
   - name: orders
@@ -180,4 +156,4 @@ saved_queries:
     exports:
       - name: order_metrics
         config:
-          export_as: table
+          export_as: table

models/staging/__sources.yml

@@ -9,12 +9,14 @@ sources:
         description: One record per person who has purchased one or more items
       - name: raw_orders
         description: One record per order (consisting of one or more order items)
-        loaded_at_field: ordered_at
+        config:
+          loaded_at_field: ordered_at
       - name: raw_items
         description: Items included in an order
       - name: raw_stores
-        loaded_at_field: opened_at
+        config:
+          loaded_at_field: opened_at
       - name: raw_products
         description: One record per SKU for items sold in stores
       - name: raw_supplies
-        description: One record per supply per SKU of items sold in stores
+        description: One record per supply per SKU of items sold in stores

models/staging/stg_order_items.yml

@@ -12,5 +12,6 @@ models:
         data_tests:
           - not_null
           - relationships:
-              to: ref('stg_orders')
-              field: order_id
+              arguments:
+                to: ref('stg_orders')
+                field: order_id

models/staging/stg_orders.yml

@@ -3,10 +3,11 @@ models:
     description: Order data with basic cleaning and transformation applied, one row per order.
     data_tests:
       - dbt_utils.expression_is_true:
-          expression: "order_total - tax_paid = subtotal"
+          arguments:
+            expression: "order_total - tax_paid = subtotal"
     columns:
       - name: order_id
         description: The unique key for each order.
         data_tests:
           - not_null
-          - unique
+          - unique

package-lock.yml

@@ -1,8 +1,5 @@
 packages:
-  - package: dbt-labs/dbt_utils
-    version: 1.1.1
-  - package: godatadriven/dbt_date
-    version: 0.10.0
-  - git: https://github.com/dbt-labs/dbt-audit-helper.git
-    revision: b8f3a3348ce0ff8afc3aa4b9ade2123b00772473
-sha1_hash: f2a73967505955af30d18f70489a423fd47cdf79
+  - name: dbt_utils
+    package: dbt-labs/dbt_utils
+    version: 1.3.2
+sha1_hash: ccc8b17d4482f52f5cce6cf4ab0518ae8c147118

packages.yml

@@ -1,7 +1,3 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: 1.1.1
-  - package: godatadriven/dbt_date
-    version: 0.10.0
-  - git: "https://github.com/dbt-labs/dbt-audit-helper.git"
-    revision: main
+    version: 1.3.2