diff --git a/CHANGELOG.md b/CHANGELOG.md index afa7d72fa76..31d75feb340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and this project adheres to ### Added +- Report monthly active users (MAU) — distinct users active in the trailing 30 + days — in the usage tracker submission, alongside the existing 90-day active + user count. Reported at both instance and project level, and bumps the usage + report schema to version 3. + [#4826](https://github.com/OpenFn/lightning/issues/4826) + ### Changed - Stop reporting expected credential-resolution failures (OAuth re-auth needed, diff --git a/lib/lightning/usage_tracking/project_metrics_service.ex b/lib/lightning/usage_tracking/project_metrics_service.ex index e1f6ff4eea9..d24dea7eb17 100644 --- a/lib/lightning/usage_tracking/project_metrics_service.ex +++ b/lib/lightning/usage_tracking/project_metrics_service.ex @@ -27,6 +27,8 @@ defmodule Lightning.UsageTracking.ProjectMetricsService do %{ no_of_active_users: UserService.no_of_active_users(date, users), + no_of_monthly_active_users: + UserService.no_of_monthly_active_users(date, users), no_of_users: UserService.no_of_users(date, users), workflows: instrument_workflows(project, cleartext_enabled, date) } diff --git a/lib/lightning/usage_tracking/report_data.ex b/lib/lightning/usage_tracking/report_data.ex index 82a3644f357..e59531987fa 100644 --- a/lib/lightning/usage_tracking/report_data.ex +++ b/lib/lightning/usage_tracking/report_data.ex @@ -15,7 +15,7 @@ defmodule Lightning.UsageTracking.ReportData do instance: instrument_instance(configuration, cleartext_enabled, date), projects: instrument_projects(cleartext_enabled, date), report_date: date, - version: "2" + version: "3" } end @@ -26,6 +26,7 @@ defmodule Lightning.UsageTracking.ReportData do |> instrument_identity(cleartext_enabled) |> Map.merge(%{ no_of_active_users: UserService.no_of_active_users(date), + no_of_monthly_active_users: UserService.no_of_monthly_active_users(date), no_of_users: UserService.no_of_users(date), operating_system: operating_system_name(), version: UsageTracking.lightning_version() diff --git a/lib/lightning/usage_tracking/user_queries.ex b/lib/lightning/usage_tracking/user_queries.ex index 524cc1c58e7..a010808c42d 100644 --- a/lib/lightning/usage_tracking/user_queries.ex +++ b/lib/lightning/usage_tracking/user_queries.ex @@ -9,6 +9,12 @@ defmodule Lightning.UsageTracking.UserQueries do alias Lightning.Accounts.User alias Lightning.Accounts.UserToken + # Trailing window (in days) used for the two active-user metrics. The 90-day + # figure is the original `no_of_active_users` series; the 30-day figure backs + # the standard SaaS "monthly active users" (MAU) metric. + @active_window_days 90 + @monthly_active_window_days 30 + def existing_users(date) do report_time = report_date_as_time(date) @@ -16,17 +22,32 @@ defmodule Lightning.UsageTracking.UserQueries do end def existing_users(date, user_list) do - list_ids = user_list |> Enum.map(& &1.id) - - from eu in existing_users(date), where: eu.id in ^list_ids + existing_users(date) |> filter_by_users(user_list) end def active_users(date) do + active_users_within(date, @active_window_days) + end + + def active_users(date, user_list) do + active_users_within(date, @active_window_days) |> filter_by_users(user_list) + end + + def monthly_active_users(date) do + active_users_within(date, @monthly_active_window_days) + end + + def monthly_active_users(date, user_list) do + active_users_within(date, @monthly_active_window_days) + |> filter_by_users(user_list) + end + + defp active_users_within(date, window_days) do report_time = report_date_as_time(date) {:ok, threshold_time, _offset} = date - |> Date.add(-90) + |> Date.add(-window_days) |> then(&"#{&1}T23:59:59Z") |> DateTime.from_iso8601() @@ -38,10 +59,10 @@ defmodule Lightning.UsageTracking.UserQueries do where: ut.inserted_at > ^threshold_time and ut.inserted_at <= ^report_time end - def active_users(date, user_list) do + defp filter_by_users(query, user_list) do list_ids = user_list |> Enum.map(& &1.id) - from au in active_users(date), where: au.id in ^list_ids + from u in query, where: u.id in ^list_ids end defp report_date_as_time(date) do diff --git a/lib/lightning/usage_tracking/user_service.ex b/lib/lightning/usage_tracking/user_service.ex index e1318d67769..a038f8cd0b6 100644 --- a/lib/lightning/usage_tracking/user_service.ex +++ b/lib/lightning/usage_tracking/user_service.ex @@ -22,4 +22,12 @@ defmodule Lightning.UsageTracking.UserService do def no_of_active_users(date, user_list) do UserQueries.active_users(date, user_list) |> Repo.aggregate(:count) end + + def no_of_monthly_active_users(date) do + UserQueries.monthly_active_users(date) |> Repo.aggregate(:count) + end + + def no_of_monthly_active_users(date, user_list) do + UserQueries.monthly_active_users(date, user_list) |> Repo.aggregate(:count) + end end diff --git a/test/lightning/usage_tracking/project_metrics_service_test.exs b/test/lightning/usage_tracking/project_metrics_service_test.exs index ed815e9cf19..41a2c135bff 100644 --- a/test/lightning/usage_tracking/project_metrics_service_test.exs +++ b/test/lightning/usage_tracking/project_metrics_service_test.exs @@ -136,6 +136,50 @@ defmodule Lightning.UsageTracking.ProjectMetricsServiceTest do } = ProjectMetricsService.generate_metrics(project, enabled, date) end + test "includes the number of monthly active users (trailing 30 days)", %{ + date: date, + enabled: enabled, + project: project + } do + # The project's active users last logged in at 2023-11-08, which is + # within the 90-day window but outside the 30-day monthly window. + assert %{ + no_of_monthly_active_users: 0 + } = ProjectMetricsService.generate_metrics(project, enabled, date) + end + + test "counts project users active within the trailing 30 days", %{ + date: date, + enabled: enabled + } do + {:ok, within_30_days, _offset} = + DateTime.from_iso8601("#{Date.add(date, -10)}T10:00:00Z") + + project = + insert(:project, + project_users: [ + build(:project_user, + user: fn -> + user = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + user: user, + inserted_at: within_30_days + ) + + user + end + ) + ] + ) + |> Repo.preload([:users, workflows: [:jobs, :runs]]) + + assert %{ + no_of_monthly_active_users: 1 + } = ProjectMetricsService.generate_metrics(project, enabled, date) + end + test "includes data for workflows existing on or before date", %{ date: date, eligible_workflow_1: eligible_workflow_1, diff --git a/test/lightning/usage_tracking/report_data_test.exs b/test/lightning/usage_tracking/report_data_test.exs index 93e1a9e86af..11c06af59b1 100644 --- a/test/lightning/usage_tracking/report_data_test.exs +++ b/test/lightning/usage_tracking/report_data_test.exs @@ -144,6 +144,39 @@ defmodule Lightning.UsageTracking.ReportDataTest do } = ReportData.generate(report_config, enabled, date) end + test "includes the number of monthly active users (trailing 30 days)", %{ + cleartext_enabled: enabled, + config: report_config, + date: date + } do + {:ok, within_30_days, _offset} = + DateTime.from_iso8601("#{Date.add(date, -29)}T10:00:00Z") + + {:ok, within_90_but_not_30_days, _offset} = + DateTime.from_iso8601("#{Date.add(date, -45)}T10:00:00Z") + + monthly_active_user = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: within_30_days, + user: monthly_active_user + ) + + quarterly_only_user = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: within_90_but_not_30_days, + user: quarterly_only_user + ) + + # The 90-day window counts both; only one is active within 30 days. + assert %{ + instance: %{no_of_active_users: 2, no_of_monthly_active_users: 1} + } = ReportData.generate(report_config, enabled, date) + end + test "includes the operating system details", %{ cleartext_enabled: enabled, config: report_config, @@ -196,7 +229,7 @@ defmodule Lightning.UsageTracking.ReportDataTest do config: report_config, date: date } do - assert %{version: "2"} = ReportData.generate(report_config, enabled, date) + assert %{version: "3"} = ReportData.generate(report_config, enabled, date) end test "indicates the applicable report date", %{ @@ -399,7 +432,7 @@ defmodule Lightning.UsageTracking.ReportDataTest do config: report_config, date: date } do - assert %{version: "2"} = ReportData.generate(report_config, enabled, date) + assert %{version: "3"} = ReportData.generate(report_config, enabled, date) end test "indicates the applicable report date", %{ diff --git a/test/lightning/usage_tracking/user_queries_test.exs b/test/lightning/usage_tracking/user_queries_test.exs index 983a92445f4..4163f818873 100644 --- a/test/lightning/usage_tracking/user_queries_test.exs +++ b/test/lightning/usage_tracking/user_queries_test.exs @@ -231,6 +231,147 @@ defmodule Lightning.UsageTracking.UserQueriesTest do end end + describe "monthly_active_users/1" do + test "returns users that have logged in in the last 30 days" do + user_within_window = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _active_token = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-07 00:00:00Z], + user: user_within_window + ) + + user_on_report_date = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _active_token = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-02-05 23:59:59Z], + user: user_on_report_date + ) + + user_older_than_30_days = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _ineligible_token_older_than_30_days = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-06 23:59:59Z], + user: user_older_than_30_days + ) + + user_newer_than_report_date = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _ineligible_token_newer_than_report_date = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-02-06 00:00:01Z], + user: user_newer_than_report_date + ) + + user_with_non_session_token = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _ineligible_token_not_session = + insert( + :user_token, + context: "api", + inserted_at: ~U[2024-02-05 00:00:01Z], + user: user_with_non_session_token + ) + + result = UserQueries.monthly_active_users(@date) |> Repo.all() + + assert(result |> contains(user_within_window)) + assert(result |> contains(user_on_report_date)) + refute(result |> contains(user_older_than_30_days)) + refute(result |> contains(user_newer_than_report_date)) + refute(result |> contains(user_with_non_session_token)) + end + + test "if user has more than one token, only includes user once" do + user = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _active_token_1 = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-07 00:00:00Z], + user: user + ) + + _active_token_2 = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-07 00:00:01Z], + user: user + ) + + result = UserQueries.monthly_active_users(@date) |> Repo.all() + + assert(result |> contains(user)) + assert(length(result) == 1) + end + end + + describe "monthly_active_users/2" do + test "returns subset of user list that have logged in the last 30 days" do + user_in_list_within_window = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _active_token = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-07 00:00:00Z], + user: user_in_list_within_window + ) + + user_not_in_list = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _active_token = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-07 00:00:00Z], + user: user_not_in_list + ) + + user_in_list_older_than_30_days = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + _ineligible_token_older_than_30_days = + insert( + :user_token, + context: "session", + inserted_at: ~U[2024-01-06 23:59:59Z], + user: user_in_list_older_than_30_days + ) + + user_list = [ + user_in_list_within_window, + user_in_list_older_than_30_days + ] + + result = UserQueries.monthly_active_users(@date, user_list) |> Repo.all() + + assert(result |> contains(user_in_list_within_window)) + refute(result |> contains(user_not_in_list)) + refute(result |> contains(user_in_list_older_than_30_days)) + end + end + defp contains(result, desired_user) do result |> Enum.find(fn user -> user.id == desired_user.id end) end diff --git a/test/lightning/usage_tracking/user_service_test.exs b/test/lightning/usage_tracking/user_service_test.exs index 39e5030237f..0f5eead055a 100644 --- a/test/lightning/usage_tracking/user_service_test.exs +++ b/test/lightning/usage_tracking/user_service_test.exs @@ -189,4 +189,71 @@ defmodule Lightning.UsageTracking.UserServiceTest do assert UserService.no_of_active_users(@date, user_list) == 2 end end + + describe ".no_of_monthly_active_users/1" do + test "counts users active within the trailing 30 days" do + {:ok, within_threshold_time, _offset} = + DateTime.from_iso8601("#{Date.add(@date, -29)}T10:00:00Z") + + {:ok, outside_threshold_time, _offset} = + DateTime.from_iso8601("#{Date.add(@date, -30)}T10:00:00Z") + + user_within_window = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: within_threshold_time, + user: user_within_window + ) + + user_outside_window = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: outside_threshold_time, + user: user_outside_window + ) + + assert UserService.no_of_monthly_active_users(@date) == 1 + end + end + + describe ".no_of_monthly_active_users/2" do + test "counts the active subset of the user list within the trailing 30 days" do + {:ok, within_threshold_time, _offset} = + DateTime.from_iso8601("#{Date.add(@date, -29)}T10:00:00Z") + + {:ok, outside_threshold_time, _offset} = + DateTime.from_iso8601("#{Date.add(@date, -30)}T10:00:00Z") + + user_in_list_active = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: within_threshold_time, + user: user_in_list_active + ) + + user_not_in_list = insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: within_threshold_time, + user: user_not_in_list + ) + + user_in_list_inactive = + insert(:user, inserted_at: ~U[2024-02-04 01:00:00Z]) + + insert(:user_token, + context: "session", + inserted_at: outside_threshold_time, + user: user_in_list_inactive + ) + + user_list = [user_in_list_active, user_in_list_inactive] + + assert UserService.no_of_monthly_active_users(@date, user_list) == 1 + end + end end