Skip to content

Add period selection and engagement totals to the data overview #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bluebird42 wants to merge 1 commit into
base: feat/data-top-list
Choose a base branch
from
Open

Add period selection and engagement totals to the data overview #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 320 additions & 0 deletions lib/mave_core/analytics/overview.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,320 @@
defmodule MaveCore.Analytics.Overview do
@moduledoc """
Space-wide engagement statistics for the dashboard data page.

A view is one playback session of one video with at least one second
watched. Statistics cover a selectable period and are compared with the
period of equal length right before it. Times are in UTC.
"""

alias Ecto.Adapters.SQL
alias MaveCore.ClickHouseRepo
alias MaveCore.SharedStorageSpace

@periods ~w(24h 7d 30d 12m)
@default_period "30d"
@min_watch_ms 1_000
@clickhouse_identifier ~r/^[A-Za-z_][A-Za-z0-9_]*$/

def periods, do: @periods
def default_period, do: @default_period

@doc """
Returns the overview for `period` (one of `periods/0`).

{:ok,
%{
period: "30d",
bucket: :day,
current: %{views: 120, watch_ms: 3_600_000, avg_watch_ms: 30_000, avg_completion: 0.58},
previous: %{views: 100, ...},
series: [%{start: ~U[...], views: 4, watch_ms: 120_000}, ...],
devices: [%{device: "desktop", views: 80}, ...],
browsers: [%{browser: "Chrome", views: 60}, ...]
}}
"""
def space_overview(space_hash, period, now \\ DateTime.utc_now())

def space_overview(space_hash, period, now) when is_binary(space_hash) and period in @periods do
window = window(period, now)

with {:ok, space_hash} <- parse_space_hash(space_hash) do
if SharedStorageSpace.hash?(space_hash) do
{:ok, empty(period, now)}
else
query_overview(space_hash, period, window)
end
end
end

def space_overview(_space_hash, _period, _now), do: {:error, :invalid_request}

@doc false
def window(period, %DateTime{} = now) do
now = DateTime.truncate(now, :second)

{bucket, from} =
case period do
"24h" -> {:hour, now |> start_of(:hour) |> DateTime.add(-23, :hour)}
"7d" -> {:day, now |> start_of(:day) |> DateTime.add(-6, :day)}
"30d" -> {:day, now |> start_of(:day) |> DateTime.add(-29, :day)}
"12m" -> {:month, now |> start_of(:month) |> shift_months(-11)}
end

length = DateTime.diff(now, from)

%{
bucket: bucket,
from: from,
to: now,
previous_from: DateTime.add(from, -length),
buckets: buckets(bucket, from, now)
}
end

defp query_overview(space_hash, period, window) do
sql = """
/* mave_perf:spaces.overview */
#{views_cte(space_hash, window)}
SELECT
'total' AS kind,
if(started_at >= #{datetime_literal(window.from)}, 'current', 'previous') AS key,
toUInt64(count()) AS views,
toInt64(sum(watch_ms)) AS watch_ms,
toFloat64(ifNotFinite(avgIf(completion, video_duration > 0), 0)) AS value
FROM views
GROUP BY key
UNION ALL
SELECT
'series' AS kind,
formatDateTime(toDateTime(#{bucket_sql(window.bucket)}(started_at, 'UTC'), 'UTC'), '%Y-%m-%d %H:%i:%S', 'UTC') AS key,
toUInt64(count()) AS views,
toInt64(sum(watch_ms)) AS watch_ms,
toFloat64(0) AS value
FROM views
WHERE started_at >= #{datetime_literal(window.from)}
GROUP BY key
UNION ALL
#{dimension_sql("device", "device", window)}
UNION ALL
#{dimension_sql("browser", "browser", window)}
"""

query_clickhouse(sql, {:error, :analytics_unavailable}, fn rows ->
{:ok, build_overview(rows, period, window)}
end)
end

# One row per (video, session) with at least one second watched.
defp views_cte(space_hash, window) do
"""
WITH ordered AS (
SELECT
embed_hash,
session_id,
timestamp,
name,
video_time,
duration,
device,
browser,
leadInFrame(timestamp) OVER (
PARTITION BY embed_hash, session_id
ORDER BY timestamp
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS next_ts,
leadInFrame(name) OVER (
PARTITION BY embed_hash, session_id
ORDER BY timestamp
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS next_name
FROM #{clickhouse_database_name()}.events
PREWHERE space_hash = #{string_literal(space_hash)}
AND timestamp >= #{datetime_literal(window.previous_from)}
AND timestamp < #{datetime_literal(window.to)}
WHERE name IN ('play', 'pause')
),
views AS (
SELECT
embed_hash,
session_id,
min(timestamp) AS started_at,
sumIf(
greatest(toInt64(0), toUnixTimestamp64Milli(next_ts) - toUnixTimestamp64Milli(timestamp)),
name = 'play' AND next_name = 'pause'
) AS watch_ms,
max(duration) AS video_duration,
max(video_time) AS furthest,
if(video_duration > 0, least(1, furthest / video_duration), 0) AS completion,
nullIf(argMin(device, timestamp), '') AS device,
nullIf(argMin(browser, timestamp), '') AS browser
FROM ordered
GROUP BY embed_hash, session_id
HAVING watch_ms >= #{@min_watch_ms}
)
"""
end

defp dimension_sql(kind, column, window) do
"""
SELECT
#{string_literal(kind)} AS kind,
ifNull(#{column}, '') AS key,
toUInt64(count()) AS views,
toInt64(0) AS watch_ms,
toFloat64(0) AS value
FROM views
WHERE started_at >= #{datetime_literal(window.from)}
GROUP BY key
ORDER BY views DESC
LIMIT 50
"""
end

defp build_overview(rows, period, window) do
grouped = Enum.group_by(rows, &hd/1, &tl/1)

totals =
Map.new(Map.get(grouped, "total", []), fn [key | values] -> {key, totals(values)} end)

series =
Map.new(Map.get(grouped, "series", []), fn [key, views, watch_ms, _] ->
{key, {views, watch_ms}}
end)

%{
period: period,
bucket: window.bucket,
current: Map.get(totals, "current", totals([0, 0, 0.0])),
previous: Map.get(totals, "previous", totals([0, 0, 0.0])),
series:
Enum.map(window.buckets, fn start ->
{views, watch_ms} = Map.get(series, bucket_key(start), {0, 0})
%{start: start, views: to_int(views), watch_ms: to_int(watch_ms)}
end),
devices: dimension_rows(grouped, "device", :device),
browsers: dimension_rows(grouped, "browser", :browser)
}
end

defp totals([views, watch_ms, completion]) do
views = to_int(views)
watch_ms = to_int(watch_ms)

%{
views: views,
watch_ms: watch_ms,
avg_watch_ms: if(views > 0, do: div(watch_ms, views), else: 0),
avg_completion: to_float(completion)
}
end

defp dimension_rows(grouped, kind, field) do
grouped
|> Map.get(kind, [])
|> Enum.map(fn [key, views, _watch_ms, _value] ->
%{field => if(key in [nil, ""], do: "Unknown", else: key), views: to_int(views)}
end)
|> Enum.sort_by(& &1.views, :desc)
end

@doc "An overview without views, for spaces that cannot view statistics."
def empty(period, now \\ DateTime.utc_now()) when period in @periods do
window = window(period, now)

%{
period: period,
bucket: window.bucket,
current: totals([0, 0, 0.0]),
previous: totals([0, 0, 0.0]),
series: Enum.map(window.buckets, &%{start: &1, views: 0, watch_ms: 0}),
devices: [],
browsers: []
}
end

defp buckets(bucket, from, to) do
from
|> Stream.iterate(&next_bucket(&1, bucket))
|> Enum.take_while(&(DateTime.compare(&1, to) != :gt))
end

defp next_bucket(start, :hour), do: DateTime.add(start, 1, :hour)
defp next_bucket(start, :day), do: DateTime.add(start, 1, :day)
defp next_bucket(start, :month), do: shift_months(start, 1)

defp start_of(%DateTime{} = dt, :hour), do: %{dt | minute: 0, second: 0, microsecond: {0, 0}}
defp start_of(%DateTime{} = dt, :day), do: %{start_of(dt, :hour) | hour: 0}
defp start_of(%DateTime{} = dt, :month), do: %{start_of(dt, :day) | day: 1}

defp shift_months(%DateTime{year: year, month: month} = dt, months) do
index = year * 12 + month - 1 + months
%{dt | year: div(index, 12), month: rem(index, 12) + 1}
end

defp bucket_sql(:hour), do: "toStartOfHour"
defp bucket_sql(:day), do: "toStartOfDay"
defp bucket_sql(:month), do: "toStartOfMonth"

# Matches the formatted bucket start returned by ClickHouse.
defp bucket_key(%DateTime{} = start) do
start |> DateTime.to_naive() |> NaiveDateTime.to_string()
end

defp parse_space_hash(space_hash) do
space_hash = String.trim(space_hash)

if String.match?(space_hash, ~r/^[A-Za-z0-9]{5}$/),
do: {:ok, space_hash},
else: {:error, :invalid_space_hash}
end

defp to_int(nil), do: 0
defp to_int(v) when is_integer(v), do: v
defp to_int(v) when is_float(v), do: trunc(v)
defp to_int(%Decimal{} = v), do: Decimal.to_integer(v)

defp to_int(v) when is_binary(v) do
case Integer.parse(v) do
{i, _} -> i
_ -> 0
end
end

defp to_float(v) when is_float(v), do: v
defp to_float(v) when is_integer(v), do: v * 1.0
defp to_float(%Decimal{} = v), do: Decimal.to_float(v)
defp to_float(_v), do: 0.0

defp clickhouse_database_name do
case ClickHouseRepo.config()[:database] || "mave_metrics" do
db_name when is_binary(db_name) ->
if db_name =~ @clickhouse_identifier, do: db_name, else: "mave_metrics"

_ ->
"mave_metrics"
end
end

# sobelow_skip ["SQL.Query"]
defp query_clickhouse(sql, default, mapper) do
case SQL.query(ClickHouseRepo, sql, []) do
{:ok, %{rows: rows}} -> mapper.(rows)
_ -> default
end
end

defp datetime_literal(dt) do
"parseDateTime64BestEffort(#{string_literal(DateTime.to_iso8601(dt))})"
end

defp string_literal(value) do
escaped =
value
|> to_string()
|> String.replace("\\", "\\\\")
|> String.replace("'", "\\'")

"'#{escaped}'"
end
end
22 changes: 20 additions & 2 deletions lib/mave_core_web/components/dashboard_components/video_detail.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,9 @@ defmodule MaveCoreWeb.DashboardComponents.VideoDetail do
attr :label, :string, required: true
attr :value, :string, required: true
attr :loading, :boolean, default: false
attr :unit, :string, default: nil, doc: "text above the value; defaults to \"views\""
attr :delta, :string, default: nil, doc: "optional comparison shown below the value"
attr :delta_tone, :atom, values: [:up, :down, :flat], default: :flat

def video_metric_card(assigns) do
~H"""
Expand All @@ -549,8 +552,11 @@ defmodule MaveCoreWeb.DashboardComponents.VideoDetail do
aria-busy={to_string(@loading)}
>
<.section_title label={@label} />
<div class="text-sm text-stone-400 pt-6 px-6">views</div>
<div class="text-5xl font-extralight text-stone-500 pt-3 pb-6 px-6">
<div class="text-sm text-stone-400 pt-6 px-6">{@unit || gettext("views")}</div>
<div class={[
"text-5xl font-extralight text-stone-500 pt-3 px-6",
if(@delta && !@loading, do: "pb-2", else: "pb-6")
]}>
<div
:if={@loading}
class="data-loading-skeleton h-12 w-24 rounded-md"
Expand All @@ -560,6 +566,18 @@ defmodule MaveCoreWeb.DashboardComponents.VideoDetail do
<span :if={@loading} class="sr-only">{gettext("Loading")}</span>
<span :if={!@loading}>{@value}</span>
</div>
<div
:if={@delta && !@loading}
data-metric-delta={@delta_tone}
class={[
"px-6 pb-5 text-xs tabular-nums",
@delta_tone == :up && "text-green-600",
@delta_tone == :down && "text-red-500",
@delta_tone == :flat && "text-stone-400"
]}
>
{@delta}
</div>
</div>
"""
end
Expand Down
11 changes: 9 additions & 2 deletions lib/mave_core_web/dashboard_routes.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ defmodule MaveCoreWeb.DashboardRoutes do
end

def data_path(space_or_user, opts \\ []) do
path_for(space_or_user, "/data", [], opts)
path_for(space_or_user, "/data", period_params(opts), opts)
end

def data_top_list_path(space_or_user, opts \\ []) do
path_for(space_or_user, "/data/top-list", [], opts)
path_for(space_or_user, "/data/top-list", period_params(opts), opts)
end

def settings_path(space_or_user, tab \\ nil, opts \\ []) do
Expand Down Expand Up @@ -58,6 +58,13 @@ defmodule MaveCoreWeb.DashboardRoutes do
|> maybe_put_param(:page, page > 1, page)
end

defp period_params(opts) do
case Keyword.get(opts, :period) do
period when is_binary(period) and period != "" -> [period: period]
_ -> []
end
end

defp maybe_put_param(params, _key, false, _value), do: params
defp maybe_put_param(params, key, true, value), do: Keyword.put(params, key, value)

Expand Down
Loading
Loading