From be9154c67d2185f75432610e1f9e4e3e1d1ef266 Mon Sep 17 00:00:00 2001 From: bluebird42 <2034043+bluebird42@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:51:04 +0200 Subject: [PATCH] Add watch time, completion and trends to the data top list The top list follows the selected period and shows watch time, average completion and a views trend per video next to views. Views, watch time and completion columns sort the list. The ranking query moves into Analytics.Overview so it shares the overview's definition of a view. Co-Authored-By: Claude Opus 5.5 --- lib/mave_core/analytics.ex | 79 -------- lib/mave_core/analytics/overview.ex | 90 ++++++++- lib/mave_core_web/dashboard_routes.ex | 9 +- .../live/dashboard/data/index.ex | 114 ++++-------- .../dashboard/data/top_list_components.ex | 175 ++++++++++++++++++ test/mave_core/analytics/overview_test.exs | 26 +++ .../live/dashboard/data/index_test.exs | 51 ++++- 7 files changed, 380 insertions(+), 164 deletions(-) create mode 100644 lib/mave_core_web/live/dashboard/data/top_list_components.ex diff --git a/lib/mave_core/analytics.ex b/lib/mave_core/analytics.ex index 577bbf6..ec1406f 100644 --- a/lib/mave_core/analytics.ex +++ b/lib/mave_core/analytics.ex @@ -98,85 +98,6 @@ defmodule MaveCore.Analytics do }} end - @doc """ - Returns the space's most viewed video embeds this year as - `%{embed_hash: hash, views: count}` maps, ordered by views. - """ - def space_top_videos(space_hash, limit \\ 25) - - def space_top_videos(space_hash, limit) - when is_binary(space_hash) and is_integer(limit) and limit > 0 do - with {:ok, space_hash} <- parse_space_hash(space_hash) do - if SharedStorageSpace.hash?(space_hash) do - {:ok, []} - else - do_space_top_videos(space_hash, limit) - end - end - end - - def space_top_videos(_space_hash, _limit), do: {:error, :invalid_space_hash} - - defp do_space_top_videos(space_hash, limit) do - now = DateTime.utc_now() - year_from = DateTime.new!(Date.new!(now.year, 1, 1), ~T[00:00:00], "Etc/UTC") - - db_name = clickhouse_database_name() - threshold = @min_watch_seconds * 1000 - - sql = """ - /* mave_perf:spaces.top_videos */ - WITH ordered AS ( - SELECT - embed_hash, - session_id, - timestamp, - name, - leadInFrame(timestamp) OVER ( - PARTITION BY embed_hash, session_id - ORDER BY timestamp - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS next_ts, - leadInFrame(name) OVER ( - PARTITION BY embed_hash, session_id - ORDER BY timestamp - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS next_name - FROM #{db_name}.events - PREWHERE space_hash = #{string_literal(space_hash)} - AND timestamp >= #{datetime_literal(year_from)} - AND timestamp < #{datetime_literal(now)} - WHERE name IN ('play', 'pause') - ), - session_aggregates AS ( - SELECT - embed_hash, - session_id, - sumIf( - greatest(toInt64(0), toUnixTimestamp64Milli(next_ts) - toUnixTimestamp64Milli(timestamp)), - name = 'play' AND next_name = 'pause' - ) AS watch_ms - FROM ordered - GROUP BY embed_hash, session_id - ) - SELECT - embed_hash, - countIf(watch_ms >= #{threshold}) AS views - FROM session_aggregates - GROUP BY embed_hash - HAVING views > 0 - ORDER BY views DESC, embed_hash ASC - LIMIT #{limit} - """ - - query_clickhouse(sql, {:error, :analytics_unavailable}, fn rows -> - {:ok, - Enum.map(rows, fn [embed_hash, views] -> - %{embed_hash: embed_hash, views: to_int(views)} - end)} - end) - end - def space_monthly_watch_time(space_hash) when is_binary(space_hash) do with {:ok, space_hash} <- parse_space_hash(space_hash) do if SharedStorageSpace.hash?(space_hash) do diff --git a/lib/mave_core/analytics/overview.ex b/lib/mave_core/analytics/overview.ex index 4497b64..a5097c9 100644 --- a/lib/mave_core/analytics/overview.ex +++ b/lib/mave_core/analytics/overview.ex @@ -81,6 +81,90 @@ defmodule MaveCore.Analytics.Overview do } end + @doc """ + Returns the space's most viewed videos in `period`, ordered by views, as + maps with `:embed_hash`, `:views`, `:watch_ms`, `:avg_completion` and + `:series` (views per hour, day or month, oldest first). + """ + def top_videos(space_hash, period, limit \\ 25, now \\ DateTime.utc_now()) + + def top_videos(space_hash, period, limit, now) + when is_binary(space_hash) and period in @periods and is_integer(limit) and limit > 0 do + window = window(period, now) + + with {:ok, space_hash} <- parse_space_hash(space_hash) do + if SharedStorageSpace.hash?(space_hash), + do: {:ok, []}, + else: query_top_videos(space_hash, window, limit) + end + end + + def top_videos(_space_hash, _period, _limit, _now), do: {:error, :invalid_request} + + defp query_top_videos(space_hash, window, limit) do + sql = """ + /* mave_perf:spaces.top_videos */ + #{views_cte(space_hash, %{window | previous_from: window.from})}, + top AS ( + SELECT + embed_hash, + count() AS views, + sum(watch_ms) AS watch_ms, + ifNotFinite(avgIf(completion, video_duration > 0), 0) AS completion + FROM views + GROUP BY embed_hash + ORDER BY views DESC, embed_hash ASC + LIMIT #{limit} + ) + SELECT + 'video' AS kind, + embed_hash, + '' AS key, + toUInt64(views) AS views, + toInt64(watch_ms) AS watch_ms, + toFloat64(completion) AS value + FROM top + UNION ALL + SELECT + 'bucket' AS kind, + embed_hash, + #{bucket_key_sql(window.bucket)} AS key, + toUInt64(count()) AS views, + toInt64(0) AS watch_ms, + toFloat64(0) AS value + FROM views + WHERE embed_hash IN (SELECT embed_hash FROM top) + GROUP BY embed_hash, key + """ + + query_clickhouse(sql, {:error, :analytics_unavailable}, fn rows -> + {:ok, build_top_videos(rows, window)} + end) + end + + defp build_top_videos(rows, window) do + {videos, buckets} = Enum.split_with(rows, &(hd(&1) == "video")) + + series = + Enum.group_by(buckets, &Enum.at(&1, 1), fn [_, _, key, views, _, _] -> + {key, to_int(views)} + end) + + videos + |> Enum.map(fn [_, hash, _, views, watch_ms, completion] -> + counts = Map.new(Map.get(series, hash, [])) + + %{ + embed_hash: hash, + views: to_int(views), + watch_ms: to_int(watch_ms), + avg_completion: to_float(completion), + series: Enum.map(window.buckets, &Map.get(counts, bucket_key(&1), 0)) + } + end) + |> Enum.sort_by(&{-&1.views, &1.embed_hash}) + end + defp query_overview(space_hash, period, window) do sql = """ /* mave_perf:spaces.overview */ @@ -96,7 +180,7 @@ defmodule MaveCore.Analytics.Overview do UNION ALL SELECT 'series' AS kind, - formatDateTime(toDateTime(#{bucket_sql(window.bucket)}(started_at, 'UTC'), 'UTC'), '%Y-%m-%d %H:%i:%S', 'UTC') AS key, + #{bucket_key_sql(window.bucket)} AS key, toUInt64(count()) AS views, toInt64(sum(watch_ms)) AS watch_ms, toFloat64(0) AS value @@ -317,6 +401,10 @@ defmodule MaveCore.Analytics.Overview do %{dt | year: div(index, 12), month: rem(index, 12) + 1} end + defp bucket_key_sql(bucket) do + "formatDateTime(toDateTime(#{bucket_sql(bucket)}(started_at, 'UTC'), 'UTC'), '%Y-%m-%d %H:%i:%S', 'UTC')" + end + defp bucket_sql(:hour), do: "toStartOfHour" defp bucket_sql(:day), do: "toStartOfDay" defp bucket_sql(:month), do: "toStartOfMonth" diff --git a/lib/mave_core_web/dashboard_routes.ex b/lib/mave_core_web/dashboard_routes.ex index d23b357..ac480c6 100644 --- a/lib/mave_core_web/dashboard_routes.ex +++ b/lib/mave_core_web/dashboard_routes.ex @@ -24,7 +24,7 @@ defmodule MaveCoreWeb.DashboardRoutes do end def data_top_list_path(space_or_user, opts \\ []) do - path_for(space_or_user, "/data/top-list", period_params(opts), opts) + path_for(space_or_user, "/data/top-list", period_params(opts) ++ sort_params(opts), opts) end def settings_path(space_or_user, tab \\ nil, opts \\ []) do @@ -65,6 +65,13 @@ defmodule MaveCoreWeb.DashboardRoutes do end end + defp sort_params(opts) do + case Keyword.get(opts, :sort) do + sort when is_binary(sort) and sort != "" -> [sort: sort] + _ -> [] + end + end + defp maybe_put_param(params, _key, false, _value), do: params defp maybe_put_param(params, key, true, value), do: Keyword.put(params, key, value) diff --git a/lib/mave_core_web/live/dashboard/data/index.ex b/lib/mave_core_web/live/dashboard/data/index.ex index a7324ea..9735308 100644 --- a/lib/mave_core_web/live/dashboard/data/index.ex +++ b/lib/mave_core_web/live/dashboard/data/index.ex @@ -8,11 +8,11 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do # Deleted videos keep their analytics, so query extra rows to fill the list. @top_list_query_size 50 - alias MaveCore.Analytics alias MaveCore.Analytics.Overview alias MaveCore.Embeds alias MaveCore.UsageLimits alias MaveCoreWeb.Dashboard.Data.OverviewComponents + alias MaveCoreWeb.Dashboard.Data.TopListComponents alias MaveCoreWeb.DashboardRoutes alias MaveCoreWeb.Formatter alias Phoenix.LiveView.AsyncResult @@ -24,6 +24,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do |> assign(:settings_open, false) |> assign(:route_scope, :unscoped) |> assign(:period, Overview.default_period()) + |> assign(:sort, TopListComponents.default_sort()) |> assign(:expanded_sources, MapSet.new()) |> assign(:data_access_notice, nil) |> assign(:data_access_help, nil) @@ -32,7 +33,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do def handle_params(params, _url, socket) do space = socket.assigns.current_space - socket = assign(socket, :period, period(params)) + socket = assign(socket, period: period(params), sort: sort(params)) socket = case UsageLimits.can_view_space_data?(space) do @@ -53,9 +54,11 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do end defp start_tab_fetch(socket, :top_list, space) do + %{period: period, sort: sort} = socket.assigns + socket |> assign(:top_videos, AsyncResult.loading()) - |> start_async(:fetch_top_videos, fn -> fetch_top_videos(space) end) + |> start_async(:fetch_top_videos, fn -> fetch_top_videos(space, period, sort) end) end defp start_tab_fetch(socket, _overview, space) do @@ -66,16 +69,16 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do |> start_async(:fetch_overview, fn -> Overview.space_overview(space.hash, period) end) end - defp fetch_top_videos(space) do - with {:ok, rows} <- Analytics.space_top_videos(space.hash, @top_list_query_size) do + defp fetch_top_videos(space, period, sort) do + with {:ok, rows} <- Overview.top_videos(space.hash, period, @top_list_query_size) do videos = Embeds.dashboard_videos_by_hash(space, Enum.map(rows, & &1.embed_hash)) top_videos = - for %{embed_hash: hash, views: views} <- rows, + for %{embed_hash: hash} = row <- rows, Map.has_key?(videos, hash), - do: Map.put(videos[hash], :views, views) + do: Map.merge(row, videos[hash]) - {:ok, Enum.take(top_videos, @top_list_size)} + {:ok, top_videos |> Enum.take(@top_list_size) |> TopListComponents.sort(sort)} end end @@ -151,11 +154,7 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do {gettext("Top list")}
- +
<%= if @live_action == :top_list do %> - <.top_list + <% else %> @@ -251,74 +252,6 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do """ end - attr :top_videos, AsyncResult, required: true - attr :current_space, :any, required: true - attr :route_scope, :atom, required: true - - defp top_list(assigns) do - ~H""" -
- <.subtitle label={gettext("Most viewed this year")} icon="hero-trophy" /> -
- - <.async_result :let={top_videos} assign={@top_videos}> - <:loading> -
- <.video_data_section> - <%= for width <- [46, 38, 52, 30, 42, 35, 48, 33] do %> -
- -
- -
- -
- <% end %> - {gettext("Loading data")} - -
- - - <:failed> - <.video_data_section> -
- {gettext("Data is unavailable")} -
- - - - <.video_data_section> -
- <.icon name="hero-trophy" class="h-6 w-6 mb-2 opacity-50" /> -
{gettext("No data yet")}
-
-
- <.link - :for={{video, rank} <- Enum.with_index(top_videos, 1)} - navigate={DashboardRoutes.video_path(@current_space, video.id, :all, scope: @route_scope)} - class="flex items-center gap-4 px-3 py-2.5 border-b border-stone-100 last:border-none hover:bg-stone-50 active:bg-blue-50" - > -
{rank}
-
{video.name}
-
- {format_number(video.views)} -
- -
- - - """ - end - attr :title, :string, required: true attr :assign, :any, required: true attr :expanded, :boolean, required: true @@ -437,6 +370,9 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do defp period(_params), do: Overview.default_period() + defp period_path(%{live_action: :top_list} = assigns, period), + do: top_list_path(assigns, period, assigns.sort) + defp period_path(assigns, period) do DashboardRoutes.data_path(assigns.current_space, scope: assigns.route_scope, @@ -444,6 +380,20 @@ defmodule MaveCoreWeb.Dashboard.Data.Index do ) end + defp top_list_path(assigns, period, sort) do + DashboardRoutes.data_top_list_path(assigns.current_space, + scope: assigns.route_scope, + period: period_param(period), + sort: if(sort != TopListComponents.default_sort(), do: sort) + ) + end + + defp sort(%{"sort" => sort}) do + if sort in TopListComponents.sorts(), do: sort, else: TopListComponents.default_sort() + end + + defp sort(_params), do: TopListComponents.default_sort() + # Keeps URLs for the default period free of a query string. defp period_param(period) do if period == Overview.default_period(), do: nil, else: period diff --git a/lib/mave_core_web/live/dashboard/data/top_list_components.ex b/lib/mave_core_web/live/dashboard/data/top_list_components.ex new file mode 100644 index 0000000..db93b91 --- /dev/null +++ b/lib/mave_core_web/live/dashboard/data/top_list_components.ex @@ -0,0 +1,175 @@ +defmodule MaveCoreWeb.Dashboard.Data.TopListComponents do + @moduledoc false + + use MaveCoreWeb, :html + import MaveCoreWeb.DashboardComponents, except: [icon: 1] + + alias MaveCoreWeb.DashboardRoutes + alias MaveCoreWeb.Formatter + alias Phoenix.LiveView.AsyncResult + + @sorts ~w(views watch_time completion) + + def sorts, do: @sorts + def default_sort, do: "views" + + @doc "Sorts top list rows by `sort`, highest first." + def sort(videos, "watch_time"), do: Enum.sort_by(videos, &{-&1.watch_ms, -&1.views}) + def sort(videos, "completion"), do: Enum.sort_by(videos, &{-&1.avg_completion, -&1.views}) + def sort(videos, _views), do: Enum.sort_by(videos, &(-&1.views)) + + attr :top_videos, AsyncResult, required: true + attr :current_space, :any, required: true + attr :route_scope, :atom, required: true + attr :sort, :string, required: true + attr :sort_path, :any, required: true, doc: "function that builds the path for a sort" + + def top_list(assigns) do + ~H""" +
+ <.subtitle label={gettext("Top videos")} icon="hero-trophy" /> +
+ + <.async_result :let={top_videos} assign={@top_videos}> + <:loading> +
+ <.video_data_section> + <%= for width <- [46, 38, 52, 30, 42, 35, 48, 33] do %> +
+ +
+ +
+ +
+ <% end %> + {gettext("Loading data")} + +
+ + + <:failed> + <.video_data_section> +
+ {gettext("Data is unavailable")} +
+ + + + <.video_data_section> +
+ <.icon name="hero-trophy" class="h-6 w-6 mb-2 opacity-50" /> +
{gettext("No data yet")}
+
+
+ + + + + + + + + + + + + + + + + + + +
#{gettext("Video")} + <.link + patch={@sort_path.(sort)} + data-sort={sort} + aria-sort={if sort == @sort, do: "descending"} + class={[ + "whitespace-nowrap hover:text-stone-600", + sort == @sort && "text-stone-600" + ]} + > + {label}{if sort == @sort, do: " ▾"} + + {gettext("Trend")}
{rank} + <.link + navigate={ + DashboardRoutes.video_path(@current_space, video.id, :all, scope: @route_scope) + } + class="block truncate text-stone-500 hover:text-stone-700" + > + {video.name} + + + {format_number(video.views)} + + {format_watch_time(video.watch_ms)} + + {round(video.avg_completion * 100)}% + + <.sparkline values={video.series} /> +
+
+ + + """ + end + + attr :values, :list, required: true + + defp sparkline(assigns) do + max = Enum.max(assigns.values, fn -> 0 end) + step = 88 / max(length(assigns.values) - 1, 1) + + points = + assigns.values + |> Enum.with_index() + |> Enum.map_join(" ", fn {value, index} -> + y = if max > 0, do: 15 - 13 * value / max, else: 15 + "#{Float.round(index * step, 2)},#{Float.round(y, 2)}" + end) + + assigns = assign(assigns, :points, points) + + ~H""" + + """ + end + + defp sort_columns do + [ + {"views", gettext("Views")}, + {"watch_time", gettext("Watch time")}, + {"completion", gettext("Completion")} + ] + end + + defp format_watch_time(ms) when ms >= 3_600_000, + do: "#{format_number(Float.round(ms / 3_600_000, 1))} h" + + defp format_watch_time(ms), do: "#{format_number(round(ms / 60_000))} min" + + defp format_number(number), do: Formatter.format_number(number) +end diff --git a/test/mave_core/analytics/overview_test.exs b/test/mave_core/analytics/overview_test.exs index dda0979..dc3409c 100644 --- a/test/mave_core/analytics/overview_test.exs +++ b/test/mave_core/analytics/overview_test.exs @@ -86,8 +86,34 @@ defmodule MaveCore.Analytics.OverviewTest do assert Enum.all?(Map.keys(overview.hours), fn {day, hour} -> day in 1..7 and hour in 0..23 end) end + test "ranks videos with watch time, completion and views per bucket", %{db_name: db_name} do + now = DateTime.utc_now() + + insert_events(db_name, [ + {-3_600, "play", 1, "AAAAAAAAAA", 0.0}, + {-3_540, "pause", 1, "AAAAAAAAAA", 60.0}, + {-1_800, "play", 2, "BBBBBBBBBB", 0.0}, + {-1_680, "pause", 2, "BBBBBBBBBB", 120.0}, + {-600, "play", 3, "BBBBBBBBBB", 0.0}, + {-570, "pause", 3, "BBBBBBBBBB", 30.0}, + # Outside the 7-day period. + {-8 * 86_400, "play", 4, "AAAAAAAAAA", 0.0}, + {-8 * 86_400 + 60, "pause", 4, "AAAAAAAAAA", 60.0} + ]) + + assert {:ok, [first, second]} = Overview.top_videos("space", "7d", 25, now) + + assert %{embed_hash: "BBBBBBBBBB", views: 2, watch_ms: 150_000} = first + assert_in_delta first.avg_completion, (1.0 + 0.25) / 2, 0.001 + assert length(first.series) == 7 + assert Enum.sum(first.series) == 2 + + assert %{embed_hash: "AAAAAAAAAA", views: 1, watch_ms: 60_000} = second + end + test "rejects unknown periods" do assert {:error, :invalid_request} = Overview.space_overview("space", "5y") + assert {:error, :invalid_request} = Overview.top_videos("space", "5y") end defp insert_events(db_name, events) do diff --git a/test/mave_core_web/live/dashboard/data/index_test.exs b/test/mave_core_web/live/dashboard/data/index_test.exs index 7f9c0c4..da5f493 100644 --- a/test/mave_core_web/live/dashboard/data/index_test.exs +++ b/test/mave_core_web/live/dashboard/data/index_test.exs @@ -230,7 +230,7 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do view |> render() |> LazyHTML.from_fragment() - |> LazyHTML.query("#data-top-list a") + |> LazyHTML.query("#data-top-list tbody a") |> Enum.map(&LazyHTML.attribute(&1, "href")) |> List.flatten() @@ -240,6 +240,55 @@ defmodule MaveCoreWeb.Live.Dashboard.Data.IndexTest do ] end + test "/data/top-list sorts by watch time and keeps the period", %{ + conn: conn, + db_name: db_name + } do + {conn, space} = authenticated_conn(conn) + {:ok, short} = Embeds.create_video_embed(space) + {:ok, long} = Embeds.create_video_embed(space) + + # Two short views against one long view. + rows = + [{short.hash, 1, 3}, {short.hash, 2, 3}, {long.hash, 3, 9}] + |> Enum.map_join(",\n", fn {embed_hash, session, seconds} -> + session_id = "00000000-0000-0000-0000-00000000400#{session}" + + """ + (addSeconds(now64(6), -30), 'play', '#{session_id}', '#{space.hash}', '#{embed_hash}', 0.0, 10, 'https://example.com', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple'), + (addSeconds(now64(6), #{seconds - 30}), 'pause', '#{session_id}', '#{space.hash}', '#{embed_hash}', #{seconds}.0, 10, 'https://example.com', 'Chrome', '100', 'Mac', '14', 'Desktop', 'Apple') + """ + end) + + {:ok, _} = + ClickHouseRepo.query(""" + INSERT INTO #{db_name}.events + (timestamp, name, session_id, space_hash, embed_hash, video_time, duration, source_url, browser, browser_version, os, os_version, device, device_brand) + VALUES + #{rows} + """) + + {:ok, view, _html} = live(conn, "/data/top-list?period=7d") + _ = render_async(view, 5_000) + assert top_list_links(view) == ["/videos/#{short.id}", "/videos/#{long.id}"] + + view |> element(~s(#data-top-list a[data-sort="watch_time"])) |> render_click() + assert_patch(view, "/data/top-list?period=7d&sort=watch_time") + _ = render_async(view, 5_000) + + assert top_list_links(view) == ["/videos/#{long.id}", "/videos/#{short.id}"] + assert has_element?(view, ~s(#data-top-list a[aria-sort="descending"]), "Watch time") + end + + defp top_list_links(view) do + view + |> render() + |> LazyHTML.from_fragment() + |> LazyHTML.query("#data-top-list tbody a") + |> Enum.map(&LazyHTML.attribute(&1, "href")) + |> List.flatten() + end + defp authenticated_conn(conn, space_hash \\ nil) do email = "data-index-#{System.unique_integer([:positive])}@example.com" {:ok, user} = Accounts.create_user(email)