From 600e4a49d9c3c3d9c4888e2ce49a42877ff03f6f Mon Sep 17 00:00:00 2001 From: bluebird42 <2034043+bluebird42@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:48:45 +0200 Subject: [PATCH] Show how far and when people watch on the data overview Add an engagement section with the share of views that reach 25%, 50%, 75% and the end of the video, and views by weekday and hour (UTC). Sources gain operating system and website breakdowns, the latter taken from the host of the embedding page. "Show more" on the source panels never appeared because the panel compared the full list with itself; it now expands any breakdown with more than five rows. Device names are grouped case-insensitively. Co-Authored-By: Claude Opus 5.5 --- lib/mave_core/analytics/overview.ex | 75 +++++++++++- .../live/dashboard/data/index.ex | 106 ++++++++++------- .../dashboard/data/overview_components.ex | 109 ++++++++++++++++++ test/mave_core/analytics/overview_test.exs | 21 +++- .../live/dashboard/data/index_test.exs | 42 ++++++- 5 files changed, 302 insertions(+), 51 deletions(-) diff --git a/lib/mave_core/analytics/overview.ex b/lib/mave_core/analytics/overview.ex index 4135cc7..4497b64 100644 --- a/lib/mave_core/analytics/overview.ex +++ b/lib/mave_core/analytics/overview.ex @@ -14,6 +14,7 @@ defmodule MaveCore.Analytics.Overview do @periods ~w(24h 7d 30d 12m) @default_period "30d" @min_watch_ms 1_000 + @reach_percents [0, 25, 50, 75, 95] @clickhouse_identifier ~r/^[A-Za-z_][A-Za-z0-9_]*$/ def periods, do: @periods @@ -30,8 +31,16 @@ defmodule MaveCore.Analytics.Overview do previous: %{views: 100, ...}, series: [%{start: ~U[...], views: 4, watch_ms: 120_000}, ...], devices: [%{device: "desktop", views: 80}, ...], - browsers: [%{browser: "Chrome", views: 60}, ...] + browsers: [%{browser: "Chrome", views: 60}, ...], + operating_systems: [%{os: "Mac OS", views: 50}, ...], + websites: [%{website: "example.com", views: 70}, ...], + reach: [%{percent: 0, share: 1.0}, %{percent: 25, share: 0.8}, ...], + hours: %{{1, 9} => 12, ...} }} + + `reach` is the share of views that got at least that far into the video; + 95% counts as finished. `hours` maps `{weekday, hour}` (Monday is 1) to + views started in that hour. """ def space_overview(space_hash, period, now \\ DateTime.utc_now()) @@ -98,6 +107,31 @@ defmodule MaveCore.Analytics.Overview do #{dimension_sql("device", "device", window)} UNION ALL #{dimension_sql("browser", "browser", window)} + UNION ALL + #{dimension_sql("os", "os", window)} + UNION ALL + #{dimension_sql("website", "website", window)} + UNION ALL + SELECT + 'reach' AS kind, + toString(percent) AS key, + toUInt64(countIf(completion * 100 >= percent)) AS views, + toInt64(0) AS watch_ms, + toFloat64(0) AS value + FROM views + ARRAY JOIN #{reach_sql()} AS percent + WHERE started_at >= #{datetime_literal(window.from)} AND video_duration > 0 + GROUP BY percent + UNION ALL + SELECT + 'hour' AS kind, + concat(toString(toDayOfWeek(started_at, 0, 'UTC')), '-', toString(toHour(started_at, 'UTC'))) AS key, + toUInt64(count()) AS views, + toInt64(0) AS watch_ms, + toFloat64(0) AS value + FROM views + WHERE started_at >= #{datetime_literal(window.from)} + GROUP BY key """ query_clickhouse(sql, {:error, :analytics_unavailable}, fn rows -> @@ -118,6 +152,8 @@ defmodule MaveCore.Analytics.Overview do duration, device, browser, + os, + source_url, leadInFrame(timestamp) OVER ( PARTITION BY embed_hash, session_id ORDER BY timestamp @@ -146,8 +182,10 @@ defmodule MaveCore.Analytics.Overview do max(duration) AS video_duration, max(video_time) AS furthest, if(video_duration > 0, least(1, furthest / video_duration), 0) AS completion, - nullIf(argMin(device, timestamp), '') AS device, - nullIf(argMin(browser, timestamp), '') AS browser + nullIf(lower(argMin(device, timestamp)), '') AS device, + nullIf(argMin(browser, timestamp), '') AS browser, + nullIf(argMin(os, timestamp), '') AS os, + nullIf(domainWithoutWWW(argMin(source_url, timestamp)), '') AS website FROM ordered GROUP BY embed_hash, session_id HAVING watch_ms >= #{@min_watch_ms} @@ -193,10 +231,33 @@ defmodule MaveCore.Analytics.Overview do %{start: start, views: to_int(views), watch_ms: to_int(watch_ms)} end), devices: dimension_rows(grouped, "device", :device), - browsers: dimension_rows(grouped, "browser", :browser) + browsers: dimension_rows(grouped, "browser", :browser), + operating_systems: dimension_rows(grouped, "os", :os), + websites: dimension_rows(grouped, "website", :website), + reach: reach(Map.get(grouped, "reach", [])), + hours: hours(Map.get(grouped, "hour", [])) } end + defp reach(rows) do + counts = Map.new(rows, fn [percent, views, _watch_ms, _value] -> {percent, to_int(views)} end) + started = Map.get(counts, "0", 0) + + Enum.map(@reach_percents, fn percent -> + views = Map.get(counts, Integer.to_string(percent), 0) + %{percent: percent, share: if(started > 0, do: views / started, else: 0.0)} + end) + end + + defp hours(rows) do + Map.new(rows, fn [key, views, _watch_ms, _value] -> + [weekday, hour] = key |> String.split("-") |> Enum.map(&String.to_integer/1) + {{weekday, hour}, to_int(views)} + end) + end + + defp reach_sql, do: "[" <> Enum.map_join(@reach_percents, ", ", &Integer.to_string/1) <> "]" + defp totals([views, watch_ms, completion]) do views = to_int(views) watch_ms = to_int(watch_ms) @@ -229,7 +290,11 @@ defmodule MaveCore.Analytics.Overview do previous: totals([0, 0, 0.0]), series: Enum.map(window.buckets, &%{start: &1, views: 0, watch_ms: 0}), devices: [], - browsers: [] + browsers: [], + operating_systems: [], + websites: [], + reach: Enum.map(@reach_percents, &%{percent: &1, share: 0.0}), + hours: %{} } end diff --git a/lib/mave_core_web/live/dashboard/data/index.ex b/lib/mave_core_web/live/dashboard/data/index.ex index 1c363d8..a7324ea 100644 --- a/lib/mave_core_web/live/dashboard/data/index.ex +++ b/lib/mave_core_web/live/dashboard/data/index.ex @@ -24,8 +24,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do |> assign(:settings_open, false) |> assign(:route_scope, :unscoped) |> assign(:period, Overview.default_period()) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false) + |> assign(:expanded_sources, MapSet.new()) |> assign(:data_access_notice, nil) |> assign(:data_access_help, nil) |> assign_async_defaults()} @@ -84,8 +83,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do {:noreply, socket |> assign(:overview, AsyncResult.ok(socket.assigns.overview, overview)) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false)} + |> assign(:expanded_sources, MapSet.new())} end def handle_async(:fetch_overview, _result, socket) do @@ -110,12 +108,16 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do )} end - def handle_event("toggle_show_all_devices", _, socket) do - {:noreply, assign(socket, :show_all_devices, !socket.assigns.show_all_devices)} - end + def handle_event("toggle_source", %{"source" => source}, socket) + when source in ["device", "browser", "os", "website"] do + expanded = socket.assigns.expanded_sources + + expanded = + if MapSet.member?(expanded, source), + do: MapSet.delete(expanded, source), + else: MapSet.put(expanded, source) - def handle_event("toggle_show_all_browsers", _, socket) do - {:noreply, assign(socket, :show_all_browsers, !socket.assigns.show_all_browsers)} + {:noreply, assign(socket, :expanded_sources, expanded)} end def render(assigns) do @@ -189,6 +191,17 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.divider /> +
+ <.subtitle label={gettext("Engagement")} icon="hero-play-circle" /> +
+ +
+ + +
+ + <.divider /> +
<.subtitle label={gettext("Sources")} icon="hero-globe-alt" />
@@ -197,9 +210,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.source_panel title={gettext("Device")} assign={overview_field(@overview, :devices)} - rows={visible_devices(overview_field(@overview, :devices), @show_all_devices)} - toggle="toggle_show_all_devices" - expanded={@show_all_devices} + expanded={MapSet.member?(@expanded_sources, "device")} key_name={:device} empty_icon="device" /> @@ -207,12 +218,28 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.source_panel title={gettext("Browser")} assign={overview_field(@overview, :browsers)} - rows={visible_browsers(overview_field(@overview, :browsers), @show_all_browsers)} - toggle="toggle_show_all_browsers" - expanded={@show_all_browsers} + expanded={MapSet.member?(@expanded_sources, "browser")} key_name={:browser} empty_icon="browser" /> + + <.source_panel + title={gettext("Operating system")} + assign={overview_field(@overview, :operating_systems)} + expanded={MapSet.member?(@expanded_sources, "os")} + key_name={:os} + empty_icon="os" + capitalize={false} + /> + + <.source_panel + title={gettext("Websites")} + assign={overview_field(@overview, :websites)} + expanded={MapSet.member?(@expanded_sources, "website")} + key_name={:website} + empty_icon="website" + capitalize={false} + /> <% end %> @@ -294,11 +321,10 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do attr :title, :string, required: true attr :assign, :any, required: true - attr :rows, :list, required: true - attr :toggle, :string, required: true attr :expanded, :boolean, required: true attr :key_name, :atom, required: true attr :empty_icon, :string, required: true + attr :capitalize, :boolean, default: true defp source_panel(assigns) do ~H""" @@ -374,23 +400,25 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do
{gettext("No data yet")}
<% else %> - <%= for row <- @rows do %> -
-
- {row |> Map.fetch!(@key_name) |> to_string() |> String.capitalize()} + <%= for row <- visible_rows(rows, @key_name, @expanded) do %> +
+
+ {source_label(Map.fetch!(row, @key_name), @capitalize)}
-
{format_number(row.views)}
+
{format_number(row.views)}
<% end %> - <%= if length(rows) < length(@assign.result || []) do %> -
+
+ <% end %> <% end %> @@ -428,20 +456,17 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do defp format_number(number), do: Formatter.format_number(number) - defp visible_devices(%AsyncResult{ok?: true, result: devices}, true), do: devices - defp visible_devices(%AsyncResult{ok?: true, result: devices}, false), do: Enum.take(devices, 5) - defp visible_devices(_, _), do: [] - - defp visible_browsers(%AsyncResult{ok?: true, result: browsers}, expanded?) do - filtered = - Enum.filter(browsers, fn %{browser: type, views: count} -> - not (type in ["ie", "firefox", "brave", "opera"] and count == 0) - end) - - if expanded?, do: filtered, else: Enum.take(filtered, 5) + defp visible_rows(rows, :browser, expanded?) do + rows + |> Enum.reject(&(&1.browser in ["ie", "firefox", "brave", "opera"] and &1.views == 0)) + |> visible_rows(nil, expanded?) end - defp visible_browsers(_, _), do: [] + defp visible_rows(rows, _key, true), do: rows + defp visible_rows(rows, _key, false), do: Enum.take(rows, 5) + + defp source_label(value, true), do: value |> to_string() |> String.capitalize() + defp source_label(value, false), do: to_string(value) defp assign_async_defaults(socket) do socket @@ -456,8 +481,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do AsyncResult.ok(AsyncResult.loading(), Overview.empty(socket.assigns.period)) ) |> assign(:top_videos, AsyncResult.ok(AsyncResult.loading(), [])) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false) + |> assign(:expanded_sources, MapSet.new()) end defp data_access_notice(reason) do diff --git a/lib/mave_core_web/live/dashboard/data/overview_components.ex b/lib/mave_core_web/live/dashboard/data/overview_components.ex index c300e9f..f7b44e9 100644 --- a/lib/mave_core_web/live/dashboard/data/overview_components.ex +++ b/lib/mave_core_web/live/dashboard/data/overview_components.ex @@ -130,6 +130,115 @@ defmodule MaveCoreWeb.Dashboard.Data.OverviewComponents do """ end + attr :overview, AsyncResult, required: true + + def reach(assigns) do + ~H""" + <.video_data_section label={gettext("How far people watch")}> + <.async_result :let={overview} assign={@overview}> + <:loading> +
+ + {gettext("Loading data")} +
+ + <:failed> +
+ {gettext("Data is unavailable")} +
+ +
+
+ {reach_label(step.percent)} +
+
+
+
+ {round(step.share * 100)}% +
+
+ + + """ + end + + attr :overview, AsyncResult, required: true + + def hours(assigns) do + ~H""" + <.video_data_section label={gettext("When people watch")}> + <.async_result :let={overview} assign={@overview}> + <:loading> +
+ + {gettext("Loading data")} +
+ + <:failed> +
+ {gettext("Data is unavailable")} +
+ +
+ + {if rem(hour, 6) == 0, do: hour} + <%= for {weekday, label} <- weekdays() do %> + {label} +
+
+ <% end %> +
+

{gettext("Hours in UTC")}

+ + + """ + end + + defp reach_label(0), do: gettext("Started") + defp reach_label(95), do: gettext("Finished") + defp reach_label(percent), do: "#{percent}%" + + defp weekdays do + [ + {1, gettext("Mon")}, + {2, gettext("Tue")}, + {3, gettext("Wed")}, + {4, gettext("Thu")}, + {5, gettext("Fri")}, + {6, gettext("Sat")}, + {7, gettext("Sun")} + ] + end + + defp hour_opacity(hours, weekday, hour) do + busiest = hours |> Map.values() |> Enum.max(fn -> 0 end) + + case Map.get(hours, {weekday, hour}, 0) do + 0 -> 0.06 + views -> Float.round(0.15 + 0.85 * views / busiest, 2) + end + end + attr :chart, :map, required: true defp trend_chart(assigns) do diff --git a/test/mave_core/analytics/overview_test.exs b/test/mave_core/analytics/overview_test.exs index 5953eda..dda0979 100644 --- a/test/mave_core/analytics/overview_test.exs +++ b/test/mave_core/analytics/overview_test.exs @@ -64,7 +64,26 @@ defmodule MaveCore.Analytics.OverviewTest do assert length(overview.series) == 24 assert Enum.sum(Enum.map(overview.series, & &1.views)) == 3 - assert [%{device: "Desktop", views: 3}] = overview.devices + assert [%{device: "desktop", views: 3}] = overview.devices + assert [%{os: "Mac OS", views: 3}] = overview.operating_systems + assert [%{website: "example.com", views: 3}] = overview.websites + + # Reached 50%, 25% and 100% of the video. + assert [ + %{percent: 0, share: 1.0}, + %{percent: 25, share: 1.0}, + %{percent: 50, share: two_thirds}, + %{percent: 75, share: one_third}, + %{percent: 95, share: finished} + ] = overview.reach + + assert_in_delta two_thirds, 2 / 3, 0.001 + assert_in_delta one_third, 1 / 3, 0.001 + assert_in_delta finished, 1 / 3, 0.001 + + assert overview.hours |> Map.values() |> Enum.sum() == 3 + + assert Enum.all?(Map.keys(overview.hours), fn {day, hour} -> day in 1..7 and hour in 0..23 end) end test "rejects unknown periods" do diff --git a/test/mave_core_web/live/dashboard/data/index_test.exs b/test/mave_core_web/live/dashboard/data/index_test.exs index bed4187..7f9c0c4 100644 --- a/test/mave_core_web/live/dashboard/data/index_test.exs +++ b/test/mave_core_web/live/dashboard/data/index_test.exs @@ -102,8 +102,7 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do devices: [%{device: "desktop", views: 12_345}], browsers: [%{browser: "chrome", views: 67_890}] }), - show_all_devices: false, - show_all_browsers: false + expanded_sources: MapSet.new() } |> Index.render() |> rendered_to_string() @@ -128,8 +127,7 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do data_access_notice: nil, data_access_help: nil, overview: loading, - show_all_devices: false, - show_all_browsers: false + expanded_sources: MapSet.new() } |> Index.render() |> rendered_to_string() @@ -143,6 +141,42 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do refute html =~ "..." end + test "/data shows engagement and lets long breakdowns expand", %{conn: conn, db_name: db_name} do + {conn, space} = authenticated_conn(conn) + + values = + Enum.map_join(1..7, ",\n", fn n -> + session = "00000000-0000-0000-0000-00000000300#{n}" + site = "https://site#{n}.example.com/page" + + """ + (addSeconds(now64(6), -#{n * 60}), 'play', '#{session}', '#{space.hash}', 'CCCCCCCCCC', 0.0, 10, '#{site}', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple'), + (addSeconds(now64(6), -#{n * 60 - 5}), 'pause', '#{session}', '#{space.hash}', 'CCCCCCCCCC', 5.0, 10, '#{site}', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple') + """ + end) + + {:ok, _} = + ClickHouseRepo.query(""" + INSERT INTO #{db_name}.events + (timestamp, name, session_id, space_hash, embed_hash, video_time, duration, source_url, browser, browser_version, os, os_version, device, device_brand) + VALUES + #{values} + """) + + {:ok, view, _html} = live(conn, "/data") + _ = render_async(view, 5_000) + + assert view |> element("#data-reach") |> render() =~ "50%" + assert has_element?(view, "#data-hours") + assert render(view) =~ "Operating system" + + # Seven websites: five are shown until the panel expands. + assert render(view) =~ "site1.example.com" + refute render(view) =~ "site7.example.com" + view |> element(~s(button[phx-value-source="website"])) |> render_click() + assert render(view) =~ "site7.example.com" + end + test "/data shows overview and top list tabs", %{conn: conn} do {conn, _space} = authenticated_conn(conn)