Add v2 metrics schema and API with Spotify plays

Makefile

@@ -73,13 +73,13 @@ integration-test: ## Run integration tests locally
 	npx jest ./tests/api_e2e --verbose true
 
 .PHONY: dev-sql
-dev-sql: ## Watch SQL files and output ready to run queries (specify podcast id as first param)
+dev-sql: ## Watch SQL files and output ready to run queries (PODCAST_ID required, optional START_DATE, END_DATE, VERSION=v1)
 ifndef PODCAST_ID
 	@echo "Error: Podcast ID is missing. Please specify the PODCAST_ID variable. Example: make dev-sql PODCAST_ID=3"
 	@exit 1
 else
 	@echo "Podcast ID: $(PODCAST_ID)"
-	$(SHELL) ./watch-sql-queries.sh "$(PODCAST_ID)" "$(START_DATE)" "$(END_DATE)"
+	$(SHELL) ./watch-sql-queries.sh "$(PODCAST_ID)" "$(START_DATE)" "$(END_DATE)" "$(VERSION)"
 endif
 
 .PHONY: e2e-tests
@@ -106,8 +106,8 @@ send-api-req-local: ## Send a request to the local running server
 	curl -X POST http://localhost:8080/connector -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr' --data-binary "@./fixtures/anchorEpisodeAggregatedPerformance.json" 
 
 .PHONY: send-analytics-%
-send-analytics-%: ## Send analytics request to the local running server
-	curl http://localhost:8080/analytics/v1/3/$*?start=$$(date -d "2 months ago" +%Y-%m-%d)\&end=$$(date -d "2 days ago" +%Y-%m-%d) -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr'
+send-analytics-%: ## Send analytics request to the local running server (optional VERSION=v1)
+	curl http://localhost:8080/analytics/$(or $(VERSION),v1)/3/$*?start=$$(date -d "2 months ago" +%Y-%m-%d)\&end=$$(date -d "2 days ago" +%Y-%m-%d) -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr'
 
 .PHONY: send-api-req-prod
 send-api-req-prod: ## Send request to production

db_schema/migrations/19_platform_independent_metrics.sql

@@ -0,0 +1,129 @@
+-- Migration 19: platform independent metrics schema (used by v2 API)
+-- NOTE: the file name must not contain digits after the migration number
+-- Platform independent structure with own podcast and episode ids,
+-- mapping tables to platform ids and metric tables based on date ranges.
+-- Starts with Spotify plays (= Spotify streams) and backfills the existing history.
+-- NOTE: statements are split by semicolon, so do not use semicolons in comments or strings
+
+-- platforms data is fetched from (ids are referenced in code, see src/types/metrics.ts)
+CREATE TABLE IF NOT EXISTS platforms (
+  platform_id TINYINT UNSIGNED NOT NULL,
+  name VARCHAR(32) NOT NULL,
+  PRIMARY KEY (platform_id),
+  UNIQUE KEY (name)
+);
+
+INSERT INTO platforms (platform_id, name) VALUES
+  (1, 'spotify'),
+  (2, 'apple'),
+  (3, 'anchor'),
+  (4, 'podigee');
+
+-- metric definitions (ids are referenced in code, see src/types/metrics.ts)
+-- additive metrics (e.g. plays) are stored as daily values and can be summed up
+-- non additive metrics (e.g. unique listeners) are stored with explicit date ranges
+CREATE TABLE IF NOT EXISTS metrics (
+  metric_id SMALLINT UNSIGNED NOT NULL,
+  name VARCHAR(64) NOT NULL,
+  additive BOOLEAN NOT NULL,
+  PRIMARY KEY (metric_id),
+  UNIQUE KEY (name)
+);
+
+INSERT INTO metrics (metric_id, name, additive) VALUES
+  (1, 'plays', 1);
+
+-- episodes with an id which is unique across all podcasts
+CREATE TABLE IF NOT EXISTS episodes (
+  episode_id INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  podcast_id INTEGER NOT NULL,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (episode_id),
+  KEY (podcast_id)
+);
+
+-- maps the podcast id (podcasts.account_id) to the id of the podcast on a platform
+CREATE TABLE IF NOT EXISTS podcastPlatformIds (
+  podcast_id INTEGER NOT NULL,
+  platform_id TINYINT UNSIGNED NOT NULL,
+  platform_podcast_id VARCHAR(128) NOT NULL,
+  PRIMARY KEY (podcast_id, platform_id),
+  KEY (platform_id, platform_podcast_id)
+);
+
+-- maps the id of an episode on a platform to our episode id
+-- scoped by podcast as the same platform show might be connected to multiple podcasts
+CREATE TABLE IF NOT EXISTS episodePlatformIds (
+  podcast_id INTEGER NOT NULL,
+  platform_id TINYINT UNSIGNED NOT NULL,
+  platform_episode_id VARCHAR(128) NOT NULL,
+  episode_id INT UNSIGNED NOT NULL,
+  PRIMARY KEY (podcast_id, platform_id, platform_episode_id),
+  KEY (episode_id)
+);
+
+-- metric values of a podcast for a date range
+-- daily values: start_date = end_date
+-- lifetime values: start_date = release date, end_date = date the data was fetched
+CREATE TABLE IF NOT EXISTS podcastMetrics (
+  podcast_id INTEGER NOT NULL,
+  metric_id SMALLINT UNSIGNED NOT NULL,
+  platform_id TINYINT UNSIGNED NOT NULL,
+  start_date DATE NOT NULL,
+  end_date DATE NOT NULL,
+  value BIGINT UNSIGNED NOT NULL,
+  PRIMARY KEY (podcast_id, metric_id, platform_id, start_date, end_date)
+);
+
+-- metric values of an episode for a date range (same semantics as podcastMetrics)
+CREATE TABLE IF NOT EXISTS episodeMetrics (
+  episode_id INT UNSIGNED NOT NULL,
+  metric_id SMALLINT UNSIGNED NOT NULL,
+  platform_id TINYINT UNSIGNED NOT NULL,
+  start_date DATE NOT NULL,
+  end_date DATE NOT NULL,
+  value BIGINT UNSIGNED NOT NULL,
+  PRIMARY KEY (episode_id, metric_id, platform_id, start_date, end_date)
+);
+
+-- backfill podcast platform ids from the podcasts table
+INSERT IGNORE INTO podcastPlatformIds (podcast_id, platform_id, platform_podcast_id)
+SELECT account_id, 1, spotify_id FROM podcasts WHERE spotify_id IS NOT NULL AND spotify_id <> ''
+UNION ALL
+SELECT account_id, 2, apple_id FROM podcasts WHERE apple_id IS NOT NULL AND apple_id <> ''
+UNION ALL
+SELECT account_id, 3, anchor_id FROM podcasts WHERE anchor_id IS NOT NULL AND anchor_id <> ''
+UNION ALL
+SELECT account_id, 4, podigee_id FROM podcasts WHERE podigee_id IS NOT NULL AND podigee_id <> '';
+
+-- backfill episode ids for all known Spotify episodes
+-- the tables are empty, so ids can be assigned by row number
+INSERT INTO episodePlatformIds (podcast_id, platform_id, platform_episode_id, episode_id)
+SELECT account_id, 1, episode_id, ROW_NUMBER() OVER (ORDER BY account_id, episode_id)
+FROM (
+  SELECT account_id, episode_id FROM spotifyEpisodeMetadata
+  UNION
+  SELECT DISTINCT account_id, episode_id FROM spotifyEpisodeDetailedStreams
+) spotifyEpisodes
+WHERE episode_id IS NOT NULL AND episode_id <> '';
+
+-- auto increment continues after the highest inserted id
+INSERT INTO episodes (episode_id, podcast_id)
+SELECT episode_id, podcast_id FROM episodePlatformIds;
+
+-- backfill daily Spotify plays (= streams)
+INSERT INTO podcastMetrics (podcast_id, metric_id, platform_id, start_date, end_date, value)
+SELECT account_id, 1, 1, sps_date, sps_date, sps_streams
+FROM spotifyPodcastDetailedStreams
+WHERE sps_streams IS NOT NULL AND sps_streams >= 0;
+
+INSERT INTO episodeMetrics (episode_id, metric_id, platform_id, start_date, end_date, value)
+SELECT m.episode_id, 1, 1, s.sps_date, s.sps_date, s.sps_streams
+FROM spotifyEpisodeDetailedStreams s
+JOIN episodePlatformIds m
+  ON m.podcast_id = s.account_id
+  AND m.platform_id = 1
+  AND m.platform_episode_id = s.episode_id
+WHERE s.sps_streams IS NOT NULL AND s.sps_streams >= 0;
+
+INSERT INTO migrations (migration_id, migration_name) VALUES (19, 'v2 metrics');

db_schema/queries/v2/episodePlays.sql

@@ -0,0 +1,21 @@
+-- @doc
+-- Returns daily plays per episode and platform (Spotify: streams of 60+ seconds).
+-- Each row is a daily value (startDate = endDate) between start and end.
+-- Fields: episodeId, metric, platform, startDate, endDate, value
+
+SELECT
+  m.episode_id AS episodeId,
+  mt.name AS metric,
+  p.name AS platform,
+  m.start_date AS startDate,
+  m.end_date AS endDate,
+  m.value
+FROM episodes e
+JOIN episodeMetrics m ON m.episode_id = e.episode_id
+JOIN metrics mt ON mt.metric_id = m.metric_id
+JOIN platforms p ON p.platform_id = m.platform_id
+WHERE e.podcast_id = @podcast_id
+  AND mt.name = 'plays'
+  AND m.start_date = m.end_date
+  AND m.start_date BETWEEN @start AND @end
+ORDER BY m.episode_id, m.start_date, p.name

db_schema/queries/v2/episodePlaysLifetime.sql

@@ -0,0 +1,27 @@
+-- @doc
+-- Returns the lifetime plays per episode and platform as reported by the platform.
+-- The range starts at the release date and ends at the date the data was fetched.
+-- The latest value fetched on or before end is returned (start is ignored).
+-- Fields: episodeId, metric, platform, startDate, endDate, value
+
+SELECT episodeId, metric, platform, startDate, endDate, value
+FROM (
+  SELECT
+    m.episode_id AS episodeId,
+    mt.name AS metric,
+    p.name AS platform,
+    m.start_date AS startDate,
+    m.end_date AS endDate,
+    m.value,
+    ROW_NUMBER() OVER (PARTITION BY m.episode_id, m.platform_id ORDER BY m.end_date DESC, m.start_date) AS rn
+  FROM episodes e
+  JOIN episodeMetrics m ON m.episode_id = e.episode_id
+  JOIN metrics mt ON mt.metric_id = m.metric_id
+  JOIN platforms p ON p.platform_id = m.platform_id
+  WHERE e.podcast_id = @podcast_id
+    AND mt.name = 'plays'
+    AND m.start_date < m.end_date
+    AND m.end_date <= @end
+) lifetime
+WHERE rn = 1
+ORDER BY episodeId, platform

db_schema/queries/v2/episodePlaysTotal.sql

@@ -0,0 +1,23 @@
+-- @doc
+-- Returns the plays per episode and platform summed up from start to end.
+-- days is the number of days with data in the range, to make gaps visible.
+-- Fields: episodeId, metric, platform, startDate, endDate, value, days
+
+SELECT
+  m.episode_id AS episodeId,
+  mt.name AS metric,
+  p.name AS platform,
+  DATE_FORMAT(@start, '%Y-%m-%d') AS startDate,
+  DATE_FORMAT(@end, '%Y-%m-%d') AS endDate,
+  CAST(SUM(m.value) AS UNSIGNED) AS value,
+  COUNT(*) AS days
+FROM episodes e
+JOIN episodeMetrics m ON m.episode_id = e.episode_id
+JOIN metrics mt ON mt.metric_id = m.metric_id
+JOIN platforms p ON p.platform_id = m.platform_id
+WHERE e.podcast_id = @podcast_id
+  AND mt.name = 'plays'
+  AND m.start_date = m.end_date
+  AND m.start_date BETWEEN @start AND @end
+GROUP BY m.episode_id, mt.name, p.name
+ORDER BY m.episode_id, p.name

db_schema/queries/v2/episodes.sql

@@ -0,0 +1,23 @@
+-- @doc
+-- Returns all episodes of the podcast with their episode id and the ids on each platform.
+-- Use it to map the episodeId of other v2 queries. start and end are ignored.
+-- Fields: episodeId, name, releaseDate, spotifyId, appleId, anchorId, podigeeId
+
+SELECT
+  e.episode_id AS episodeId,
+  MAX(sm.ep_name) AS name,
+  MAX(sm.ep_release_date) AS releaseDate,
+  MAX(CASE WHEN p.name = 'spotify' THEN ep.platform_episode_id END) AS spotifyId,
+  MAX(CASE WHEN p.name = 'apple' THEN ep.platform_episode_id END) AS appleId,
+  MAX(CASE WHEN p.name = 'anchor' THEN ep.platform_episode_id END) AS anchorId,
+  MAX(CASE WHEN p.name = 'podigee' THEN ep.platform_episode_id END) AS podigeeId
+FROM episodes e
+JOIN episodePlatformIds ep ON ep.episode_id = e.episode_id
+JOIN platforms p ON p.platform_id = ep.platform_id
+LEFT JOIN spotifyEpisodeMetadata sm
+  ON p.name = 'spotify'
+  AND sm.account_id = e.podcast_id
+  AND sm.episode_id = ep.platform_episode_id
+WHERE e.podcast_id = @podcast_id
+GROUP BY e.episode_id
+ORDER BY releaseDate DESC, e.episode_id DESC

db_schema/queries/v2/podcastPlays.sql

@@ -0,0 +1,20 @@
+-- @doc
+-- Returns daily plays of the podcast per platform (Spotify: streams of 60+ seconds).
+-- Each row is a daily value (startDate = endDate) between start and end.
+-- Fields: podcastId, metric, platform, startDate, endDate, value
+
+SELECT
+  m.podcast_id AS podcastId,
+  mt.name AS metric,
+  p.name AS platform,
+  m.start_date AS startDate,
+  m.end_date AS endDate,
+  m.value
+FROM podcastMetrics m
+JOIN metrics mt ON mt.metric_id = m.metric_id
+JOIN platforms p ON p.platform_id = m.platform_id
+WHERE m.podcast_id = @podcast_id
+  AND mt.name = 'plays'
+  AND m.start_date = m.end_date
+  AND m.start_date BETWEEN @start AND @end
+ORDER BY m.start_date, p.name

db_schema/queries/v2/podcastPlaysLifetime.sql

@@ -0,0 +1,26 @@
+-- @doc
+-- Returns the lifetime plays of the podcast per platform as reported by the platform.
+-- The range starts at the release date and ends at the date the data was fetched.
+-- The latest value fetched on or before end is returned (start is ignored).
+-- Fields: podcastId, metric, platform, startDate, endDate, value
+
+SELECT podcastId, metric, platform, startDate, endDate, value
+FROM (
+  SELECT
+    m.podcast_id AS podcastId,
+    mt.name AS metric,
+    p.name AS platform,
+    m.start_date AS startDate,
+    m.end_date AS endDate,
+    m.value,
+    ROW_NUMBER() OVER (PARTITION BY m.platform_id ORDER BY m.end_date DESC, m.start_date) AS rn
+  FROM podcastMetrics m
+  JOIN metrics mt ON mt.metric_id = m.metric_id
+  JOIN platforms p ON p.platform_id = m.platform_id
+  WHERE m.podcast_id = @podcast_id
+    AND mt.name = 'plays'
+    AND m.start_date < m.end_date
+    AND m.end_date <= @end
+) lifetime
+WHERE rn = 1
+ORDER BY platform

db_schema/queries/v2/podcastPlaysMissingDays.sql

@@ -0,0 +1,28 @@
+-- @doc
+-- Returns the days between start and end without daily Spotify plays of the podcast.
+-- Can be used to fetch missing days and fill gaps.
+-- Fields: podcastId, metric, platform, date
+
+WITH RECURSIVE days (day) AS (
+  SELECT CAST(@start AS DATE)
+  UNION ALL
+  SELECT day + INTERVAL 1 DAY FROM days WHERE day < CAST(@end AS DATE)
+)
+SELECT /*+ SET_VAR(cte_max_recursion_depth = 100000) */
+  @podcast_id AS podcastId,
+  'plays' AS metric,
+  'spotify' AS platform,
+  d.day AS date
+FROM days d
+WHERE NOT EXISTS (
+  SELECT 1
+  FROM podcastMetrics m
+  JOIN metrics mt ON mt.metric_id = m.metric_id
+  JOIN platforms p ON p.platform_id = m.platform_id
+  WHERE m.podcast_id = @podcast_id
+    AND mt.name = 'plays'
+    AND p.name = 'spotify'
+    AND m.start_date = d.day
+    AND m.end_date = d.day
+)
+ORDER BY d.day

db_schema/queries/v2/podcastPlaysTotal.sql

@@ -0,0 +1,22 @@
+-- @doc
+-- Returns the plays of the podcast per platform summed up from start to end.
+-- days is the number of days with data in the range, to make gaps visible.
+-- Fields: podcastId, metric, platform, startDate, endDate, value, days
+
+SELECT
+  m.podcast_id AS podcastId,
+  mt.name AS metric,
+  p.name AS platform,
+  DATE_FORMAT(@start, '%Y-%m-%d') AS startDate,
+  DATE_FORMAT(@end, '%Y-%m-%d') AS endDate,
+  CAST(SUM(m.value) AS UNSIGNED) AS value,
+  COUNT(*) AS days
+FROM podcastMetrics m
+JOIN metrics mt ON mt.metric_id = m.metric_id
+JOIN platforms p ON p.platform_id = m.platform_id
+WHERE m.podcast_id = @podcast_id
+  AND mt.name = 'plays'
+  AND m.start_date = m.end_date
+  AND m.start_date BETWEEN @start AND @end
+GROUP BY m.podcast_id, mt.name, p.name
+ORDER BY p.name