diff --git a/lib/mave_core/analytics/overview.ex b/lib/mave_core/analytics/overview.ex index 4135cc7..4497b64 100644 --- a/lib/mave_core/analytics/overview.ex +++ b/lib/mave_core/analytics/overview.ex @@ -14,6 +14,7 @@ defmodule MaveCore.Analytics.Overview do @periods ~w(24h 7d 30d 12m) @default_period "30d" @min_watch_ms 1_000 + @reach_percents [0, 25, 50, 75, 95] @clickhouse_identifier ~r/^[A-Za-z_][A-Za-z0-9_]*$/ def periods, do: @periods @@ -30,8 +31,16 @@ defmodule MaveCore.Analytics.Overview do previous: %{views: 100, ...}, series: [%{start: ~U[...], views: 4, watch_ms: 120_000}, ...], devices: [%{device: "desktop", views: 80}, ...], - browsers: [%{browser: "Chrome", views: 60}, ...] + browsers: [%{browser: "Chrome", views: 60}, ...], + operating_systems: [%{os: "Mac OS", views: 50}, ...], + websites: [%{website: "example.com", views: 70}, ...], + reach: [%{percent: 0, share: 1.0}, %{percent: 25, share: 0.8}, ...], + hours: %{{1, 9} => 12, ...} }} + + `reach` is the share of views that got at least that far into the video; + 95% counts as finished. `hours` maps `{weekday, hour}` (Monday is 1) to + views started in that hour. """ def space_overview(space_hash, period, now \\ DateTime.utc_now()) @@ -98,6 +107,31 @@ defmodule MaveCore.Analytics.Overview do #{dimension_sql("device", "device", window)} UNION ALL #{dimension_sql("browser", "browser", window)} + UNION ALL + #{dimension_sql("os", "os", window)} + UNION ALL + #{dimension_sql("website", "website", window)} + UNION ALL + SELECT + 'reach' AS kind, + toString(percent) AS key, + toUInt64(countIf(completion * 100 >= percent)) AS views, + toInt64(0) AS watch_ms, + toFloat64(0) AS value + FROM views + ARRAY JOIN #{reach_sql()} AS percent + WHERE started_at >= #{datetime_literal(window.from)} AND video_duration > 0 + GROUP BY percent + UNION ALL + SELECT + 'hour' AS kind, + concat(toString(toDayOfWeek(started_at, 0, 'UTC')), '-', toString(toHour(started_at, 'UTC'))) AS key, + toUInt64(count()) AS views, + toInt64(0) AS watch_ms, + toFloat64(0) AS value + FROM views + WHERE started_at >= #{datetime_literal(window.from)} + GROUP BY key """ query_clickhouse(sql, {:error, :analytics_unavailable}, fn rows -> @@ -118,6 +152,8 @@ defmodule MaveCore.Analytics.Overview do duration, device, browser, + os, + source_url, leadInFrame(timestamp) OVER ( PARTITION BY embed_hash, session_id ORDER BY timestamp @@ -146,8 +182,10 @@ defmodule MaveCore.Analytics.Overview do max(duration) AS video_duration, max(video_time) AS furthest, if(video_duration > 0, least(1, furthest / video_duration), 0) AS completion, - nullIf(argMin(device, timestamp), '') AS device, - nullIf(argMin(browser, timestamp), '') AS browser + nullIf(lower(argMin(device, timestamp)), '') AS device, + nullIf(argMin(browser, timestamp), '') AS browser, + nullIf(argMin(os, timestamp), '') AS os, + nullIf(domainWithoutWWW(argMin(source_url, timestamp)), '') AS website FROM ordered GROUP BY embed_hash, session_id HAVING watch_ms >= #{@min_watch_ms} @@ -193,10 +231,33 @@ defmodule MaveCore.Analytics.Overview do %{start: start, views: to_int(views), watch_ms: to_int(watch_ms)} end), devices: dimension_rows(grouped, "device", :device), - browsers: dimension_rows(grouped, "browser", :browser) + browsers: dimension_rows(grouped, "browser", :browser), + operating_systems: dimension_rows(grouped, "os", :os), + websites: dimension_rows(grouped, "website", :website), + reach: reach(Map.get(grouped, "reach", [])), + hours: hours(Map.get(grouped, "hour", [])) } end + defp reach(rows) do + counts = Map.new(rows, fn [percent, views, _watch_ms, _value] -> {percent, to_int(views)} end) + started = Map.get(counts, "0", 0) + + Enum.map(@reach_percents, fn percent -> + views = Map.get(counts, Integer.to_string(percent), 0) + %{percent: percent, share: if(started > 0, do: views / started, else: 0.0)} + end) + end + + defp hours(rows) do + Map.new(rows, fn [key, views, _watch_ms, _value] -> + [weekday, hour] = key |> String.split("-") |> Enum.map(&String.to_integer/1) + {{weekday, hour}, to_int(views)} + end) + end + + defp reach_sql, do: "[" <> Enum.map_join(@reach_percents, ", ", &Integer.to_string/1) <> "]" + defp totals([views, watch_ms, completion]) do views = to_int(views) watch_ms = to_int(watch_ms) @@ -229,7 +290,11 @@ defmodule MaveCore.Analytics.Overview do previous: totals([0, 0, 0.0]), series: Enum.map(window.buckets, &%{start: &1, views: 0, watch_ms: 0}), devices: [], - browsers: [] + browsers: [], + operating_systems: [], + websites: [], + reach: Enum.map(@reach_percents, &%{percent: &1, share: 0.0}), + hours: %{} } end diff --git a/lib/mave_core_web/live/dashboard/data/index.ex b/lib/mave_core_web/live/dashboard/data/index.ex index 1c363d8..a7324ea 100644 --- a/lib/mave_core_web/live/dashboard/data/index.ex +++ b/lib/mave_core_web/live/dashboard/data/index.ex @@ -24,8 +24,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do |> assign(:settings_open, false) |> assign(:route_scope, :unscoped) |> assign(:period, Overview.default_period()) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false) + |> assign(:expanded_sources, MapSet.new()) |> assign(:data_access_notice, nil) |> assign(:data_access_help, nil) |> assign_async_defaults()} @@ -84,8 +83,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do {:noreply, socket |> assign(:overview, AsyncResult.ok(socket.assigns.overview, overview)) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false)} + |> assign(:expanded_sources, MapSet.new())} end def handle_async(:fetch_overview, _result, socket) do @@ -110,12 +108,16 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do )} end - def handle_event("toggle_show_all_devices", _, socket) do - {:noreply, assign(socket, :show_all_devices, !socket.assigns.show_all_devices)} - end + def handle_event("toggle_source", %{"source" => source}, socket) + when source in ["device", "browser", "os", "website"] do + expanded = socket.assigns.expanded_sources + + expanded = + if MapSet.member?(expanded, source), + do: MapSet.delete(expanded, source), + else: MapSet.put(expanded, source) - def handle_event("toggle_show_all_browsers", _, socket) do - {:noreply, assign(socket, :show_all_browsers, !socket.assigns.show_all_browsers)} + {:noreply, assign(socket, :expanded_sources, expanded)} end def render(assigns) do @@ -189,6 +191,17 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.divider /> +
+ <.subtitle label={gettext("Engagement")} icon="hero-play-circle" /> +
+ +
+ + +
+ + <.divider /> +
<.subtitle label={gettext("Sources")} icon="hero-globe-alt" />
@@ -197,9 +210,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.source_panel title={gettext("Device")} assign={overview_field(@overview, :devices)} - rows={visible_devices(overview_field(@overview, :devices), @show_all_devices)} - toggle="toggle_show_all_devices" - expanded={@show_all_devices} + expanded={MapSet.member?(@expanded_sources, "device")} key_name={:device} empty_icon="device" /> @@ -207,12 +218,28 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do <.source_panel title={gettext("Browser")} assign={overview_field(@overview, :browsers)} - rows={visible_browsers(overview_field(@overview, :browsers), @show_all_browsers)} - toggle="toggle_show_all_browsers" - expanded={@show_all_browsers} + expanded={MapSet.member?(@expanded_sources, "browser")} key_name={:browser} empty_icon="browser" /> + + <.source_panel + title={gettext("Operating system")} + assign={overview_field(@overview, :operating_systems)} + expanded={MapSet.member?(@expanded_sources, "os")} + key_name={:os} + empty_icon="os" + capitalize={false} + /> + + <.source_panel + title={gettext("Websites")} + assign={overview_field(@overview, :websites)} + expanded={MapSet.member?(@expanded_sources, "website")} + key_name={:website} + empty_icon="website" + capitalize={false} + /> <% end %> @@ -294,11 +321,10 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do attr :title, :string, required: true attr :assign, :any, required: true - attr :rows, :list, required: true - attr :toggle, :string, required: true attr :expanded, :boolean, required: true attr :key_name, :atom, required: true attr :empty_icon, :string, required: true + attr :capitalize, :boolean, default: true defp source_panel(assigns) do ~H""" @@ -374,23 +400,25 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do
{gettext("No data yet")}
<% else %> - <%= for row <- @rows do %> -
-
- {row |> Map.fetch!(@key_name) |> to_string() |> String.capitalize()} + <%= for row <- visible_rows(rows, @key_name, @expanded) do %> +
+
+ {source_label(Map.fetch!(row, @key_name), @capitalize)}
-
{format_number(row.views)}
+
{format_number(row.views)}
<% end %> - <%= if length(rows) < length(@assign.result || []) do %> -
+
+ <% end %> <% end %> @@ -428,20 +456,17 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do defp format_number(number), do: Formatter.format_number(number) - defp visible_devices(%AsyncResult{ok?: true, result: devices}, true), do: devices - defp visible_devices(%AsyncResult{ok?: true, result: devices}, false), do: Enum.take(devices, 5) - defp visible_devices(_, _), do: [] - - defp visible_browsers(%AsyncResult{ok?: true, result: browsers}, expanded?) do - filtered = - Enum.filter(browsers, fn %{browser: type, views: count} -> - not (type in ["ie", "firefox", "brave", "opera"] and count == 0) - end) - - if expanded?, do: filtered, else: Enum.take(filtered, 5) + defp visible_rows(rows, :browser, expanded?) do + rows + |> Enum.reject(&(&1.browser in ["ie", "firefox", "brave", "opera"] and &1.views == 0)) + |> visible_rows(nil, expanded?) end - defp visible_browsers(_, _), do: [] + defp visible_rows(rows, _key, true), do: rows + defp visible_rows(rows, _key, false), do: Enum.take(rows, 5) + + defp source_label(value, true), do: value |> to_string() |> String.capitalize() + defp source_label(value, false), do: to_string(value) defp assign_async_defaults(socket) do socket @@ -456,8 +481,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do AsyncResult.ok(AsyncResult.loading(), Overview.empty(socket.assigns.period)) ) |> assign(:top_videos, AsyncResult.ok(AsyncResult.loading(), [])) - |> assign(:show_all_devices, false) - |> assign(:show_all_browsers, false) + |> assign(:expanded_sources, MapSet.new()) end defp data_access_notice(reason) do diff --git a/lib/mave_core_web/live/dashboard/data/overview_components.ex b/lib/mave_core_web/live/dashboard/data/overview_components.ex index c300e9f..f7b44e9 100644 --- a/lib/mave_core_web/live/dashboard/data/overview_components.ex +++ b/lib/mave_core_web/live/dashboard/data/overview_components.ex @@ -130,6 +130,115 @@ defmodule MaveCoreWeb.Dashboard.Data.OverviewComponents do """ end + attr :overview, AsyncResult, required: true + + def reach(assigns) do + ~H""" + <.video_data_section label={gettext("How far people watch")}> + <.async_result :let={overview} assign={@overview}> + <:loading> +
+ + {gettext("Loading data")} +
+ + <:failed> +
+ {gettext("Data is unavailable")} +
+ +
+
+ {reach_label(step.percent)} +
+
+
+
+ {round(step.share * 100)}% +
+
+ + + """ + end + + attr :overview, AsyncResult, required: true + + def hours(assigns) do + ~H""" + <.video_data_section label={gettext("When people watch")}> + <.async_result :let={overview} assign={@overview}> + <:loading> +
+ + {gettext("Loading data")} +
+ + <:failed> +
+ {gettext("Data is unavailable")} +
+ +
+ + {if rem(hour, 6) == 0, do: hour} + <%= for {weekday, label} <- weekdays() do %> + {label} +
+
+ <% end %> +
+

{gettext("Hours in UTC")}

+ + + """ + end + + defp reach_label(0), do: gettext("Started") + defp reach_label(95), do: gettext("Finished") + defp reach_label(percent), do: "#{percent}%" + + defp weekdays do + [ + {1, gettext("Mon")}, + {2, gettext("Tue")}, + {3, gettext("Wed")}, + {4, gettext("Thu")}, + {5, gettext("Fri")}, + {6, gettext("Sat")}, + {7, gettext("Sun")} + ] + end + + defp hour_opacity(hours, weekday, hour) do + busiest = hours |> Map.values() |> Enum.max(fn -> 0 end) + + case Map.get(hours, {weekday, hour}, 0) do + 0 -> 0.06 + views -> Float.round(0.15 + 0.85 * views / busiest, 2) + end + end + attr :chart, :map, required: true defp trend_chart(assigns) do diff --git a/test/mave_core/analytics/overview_test.exs b/test/mave_core/analytics/overview_test.exs index 5953eda..dda0979 100644 --- a/test/mave_core/analytics/overview_test.exs +++ b/test/mave_core/analytics/overview_test.exs @@ -64,7 +64,26 @@ defmodule MaveCore.Analytics.OverviewTest do assert length(overview.series) == 24 assert Enum.sum(Enum.map(overview.series, & &1.views)) == 3 - assert [%{device: "Desktop", views: 3}] = overview.devices + assert [%{device: "desktop", views: 3}] = overview.devices + assert [%{os: "Mac OS", views: 3}] = overview.operating_systems + assert [%{website: "example.com", views: 3}] = overview.websites + + # Reached 50%, 25% and 100% of the video. + assert [ + %{percent: 0, share: 1.0}, + %{percent: 25, share: 1.0}, + %{percent: 50, share: two_thirds}, + %{percent: 75, share: one_third}, + %{percent: 95, share: finished} + ] = overview.reach + + assert_in_delta two_thirds, 2 / 3, 0.001 + assert_in_delta one_third, 1 / 3, 0.001 + assert_in_delta finished, 1 / 3, 0.001 + + assert overview.hours |> Map.values() |> Enum.sum() == 3 + + assert Enum.all?(Map.keys(overview.hours), fn {day, hour} -> day in 1..7 and hour in 0..23 end) end test "rejects unknown periods" do diff --git a/test/mave_core_web/live/dashboard/data/index_test.exs b/test/mave_core_web/live/dashboard/data/index_test.exs index bed4187..7f9c0c4 100644 --- a/test/mave_core_web/live/dashboard/data/index_test.exs +++ b/test/mave_core_web/live/dashboard/data/index_test.exs @@ -102,8 +102,7 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do devices: [%{device: "desktop", views: 12_345}], browsers: [%{browser: "chrome", views: 67_890}] }), - show_all_devices: false, - show_all_browsers: false + expanded_sources: MapSet.new() } |> Index.render() |> rendered_to_string() @@ -128,8 +127,7 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do data_access_notice: nil, data_access_help: nil, overview: loading, - show_all_devices: false, - show_all_browsers: false + expanded_sources: MapSet.new() } |> Index.render() |> rendered_to_string() @@ -143,6 +141,42 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do refute html =~ "..." end + test "/data shows engagement and lets long breakdowns expand", %{conn: conn, db_name: db_name} do + {conn, space} = authenticated_conn(conn) + + values = + Enum.map_join(1..7, ",\n", fn n -> + session = "00000000-0000-0000-0000-00000000300#{n}" + site = "https://site#{n}.example.com/page" + + """ + (addSeconds(now64(6), -#{n * 60}), 'play', '#{session}', '#{space.hash}', 'CCCCCCCCCC', 0.0, 10, '#{site}', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple'), + (addSeconds(now64(6), -#{n * 60 - 5}), 'pause', '#{session}', '#{space.hash}', 'CCCCCCCCCC', 5.0, 10, '#{site}', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple') + """ + end) + + {:ok, _} = + ClickHouseRepo.query(""" + INSERT INTO #{db_name}.events + (timestamp, name, session_id, space_hash, embed_hash, video_time, duration, source_url, browser, browser_version, os, os_version, device, device_brand) + VALUES + #{values} + """) + + {:ok, view, _html} = live(conn, "/data") + _ = render_async(view, 5_000) + + assert view |> element("#data-reach") |> render() =~ "50%" + assert has_element?(view, "#data-hours") + assert render(view) =~ "Operating system" + + # Seven websites: five are shown until the panel expands. + assert render(view) =~ "site1.example.com" + refute render(view) =~ "site7.example.com" + view |> element(~s(button[phx-value-source="website"])) |> render_click() + assert render(view) =~ "site7.example.com" + end + test "/data shows overview and top list tabs", %{conn: conn} do {conn, _space} = authenticated_conn(conn)