From f3bf21a1caecb3d427497a5b7e435641799305d4 Mon Sep 17 00:00:00 2001 From: Wolfgang Gassler Date: Tue, 20 Oct 2026 13:38:26 +0200 Subject: [PATCH] Add v2 metrics schema and API with Spotify plays Introduce a platform independent metrics structure with own episode ids, platform id mappings and podcast/episode metrics stored by date range. Spotify streams are written as daily and lifetime plays alongside v1, existing history is backfilled in migration 19 and exposed via /analytics/v2 queries. Also apply the podcast access check to all API versions, return after 401 responses and await migrations on startup. Co-Authored-By: Claude Opus 5.5 --- Makefile | 8 +- .../19_platform_independent_metrics.sql | 129 ++++++++++++ db_schema/queries/v2/episodePlays.sql | 21 ++ db_schema/queries/v2/episodePlaysLifetime.sql | 27 +++ db_schema/queries/v2/episodePlaysTotal.sql | 23 +++ db_schema/queries/v2/episodes.sql | 23 +++ db_schema/queries/v2/podcastPlays.sql | 20 ++ db_schema/queries/v2/podcastPlaysLifetime.sql | 26 +++ .../queries/v2/podcastPlaysMissingDays.sql | 28 +++ db_schema/queries/v2/podcastPlaysTotal.sql | 22 +++ db_schema/schema.sql | 88 ++++++++- src/api/connectors/SpotifyConnector.test.ts | 131 +++++++++++++ src/api/connectors/SpotifyConnector.ts | 97 ++++++++- src/config/generate-query-docs.ts | 36 ++-- src/config/query-examples.ts | 84 +++++++- src/db/DBInitializer.ts | 2 +- src/db/MetricsRepository.test.ts | 165 ++++++++++++++++ src/db/MetricsRepository.ts | 143 ++++++++++++++ src/index.ts | 13 +- src/types/connector.ts | 1 + src/types/metrics.ts | 23 +++ src/types/provider/spotify.ts | 1 + tests/api_e2e/analytics_v2.test.js | 185 ++++++++++++++++++ watch-sql-queries.sh | 5 +- 24 files changed, 1269 insertions(+), 32 deletions(-) create mode 100644 db_schema/migrations/19_platform_independent_metrics.sql create mode 100644 db_schema/queries/v2/episodePlays.sql create mode 100644 db_schema/queries/v2/episodePlaysLifetime.sql create mode 100644 db_schema/queries/v2/episodePlaysTotal.sql create mode 100644 db_schema/queries/v2/episodes.sql create mode 100644 db_schema/queries/v2/podcastPlays.sql create mode 100644 db_schema/queries/v2/podcastPlaysLifetime.sql create mode 100644 db_schema/queries/v2/podcastPlaysMissingDays.sql create mode 100644 db_schema/queries/v2/podcastPlaysTotal.sql create mode 100644 src/api/connectors/SpotifyConnector.test.ts create mode 100644 src/db/MetricsRepository.test.ts create mode 100644 src/db/MetricsRepository.ts create mode 100644 src/types/metrics.ts create mode 100644 tests/api_e2e/analytics_v2.test.js diff --git a/Makefile b/Makefile index ed684528..906f1472 100644 --- a/Makefile +++ b/Makefile @@ -73,13 +73,13 @@ integration-test: ## Run integration tests locally npx jest ./tests/api_e2e --verbose true .PHONY: dev-sql -dev-sql: ## Watch SQL files and output ready to run queries (specify podcast id as first param) +dev-sql: ## Watch SQL files and output ready to run queries (PODCAST_ID required, optional START_DATE, END_DATE, VERSION=v1) ifndef PODCAST_ID @echo "Error: Podcast ID is missing. Please specify the PODCAST_ID variable. Example: make dev-sql PODCAST_ID=3" @exit 1 else @echo "Podcast ID: $(PODCAST_ID)" - $(SHELL) ./watch-sql-queries.sh "$(PODCAST_ID)" "$(START_DATE)" "$(END_DATE)" + $(SHELL) ./watch-sql-queries.sh "$(PODCAST_ID)" "$(START_DATE)" "$(END_DATE)" "$(VERSION)" endif .PHONY: e2e-tests @@ -106,8 +106,8 @@ send-api-req-local: ## Send a request to the local running server curl -X POST http://localhost:8080/connector -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr' --data-binary "@./fixtures/anchorEpisodeAggregatedPerformance.json" .PHONY: send-analytics-% -send-analytics-%: ## Send analytics request to the local running server - curl http://localhost:8080/analytics/v1/3/$*?start=$$(date -d "2 months ago" +%Y-%m-%d)\&end=$$(date -d "2 days ago" +%Y-%m-%d) -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr' +send-analytics-%: ## Send analytics request to the local running server (optional VERSION=v1) + curl http://localhost:8080/analytics/$(or $(VERSION),v1)/3/$*?start=$$(date -d "2 months ago" +%Y-%m-%d)\&end=$$(date -d "2 days ago" +%Y-%m-%d) -H 'Content-Type: application/json' -H 'Authorization: Bearer dummy-cn389ncoiwuencr' .PHONY: send-api-req-prod send-api-req-prod: ## Send request to production diff --git a/db_schema/migrations/19_platform_independent_metrics.sql b/db_schema/migrations/19_platform_independent_metrics.sql new file mode 100644 index 00000000..0d098f61 --- /dev/null +++ b/db_schema/migrations/19_platform_independent_metrics.sql @@ -0,0 +1,129 @@ +-- Migration 19: platform independent metrics schema (used by v2 API) +-- NOTE: the file name must not contain digits after the migration number +-- Platform independent structure with own podcast and episode ids, +-- mapping tables to platform ids and metric tables based on date ranges. +-- Starts with Spotify plays (= Spotify streams) and backfills the existing history. +-- NOTE: statements are split by semicolon, so do not use semicolons in comments or strings + +-- platforms data is fetched from (ids are referenced in code, see src/types/metrics.ts) +CREATE TABLE IF NOT EXISTS platforms ( + platform_id TINYINT UNSIGNED NOT NULL, + name VARCHAR(32) NOT NULL, + PRIMARY KEY (platform_id), + UNIQUE KEY (name) +); + +INSERT INTO platforms (platform_id, name) VALUES + (1, 'spotify'), + (2, 'apple'), + (3, 'anchor'), + (4, 'podigee'); + +-- metric definitions (ids are referenced in code, see src/types/metrics.ts) +-- additive metrics (e.g. plays) are stored as daily values and can be summed up +-- non additive metrics (e.g. unique listeners) are stored with explicit date ranges +CREATE TABLE IF NOT EXISTS metrics ( + metric_id SMALLINT UNSIGNED NOT NULL, + name VARCHAR(64) NOT NULL, + additive BOOLEAN NOT NULL, + PRIMARY KEY (metric_id), + UNIQUE KEY (name) +); + +INSERT INTO metrics (metric_id, name, additive) VALUES + (1, 'plays', 1); + +-- episodes with an id which is unique across all podcasts +CREATE TABLE IF NOT EXISTS episodes ( + episode_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + podcast_id INTEGER NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (episode_id), + KEY (podcast_id) +); + +-- maps the podcast id (podcasts.account_id) to the id of the podcast on a platform +CREATE TABLE IF NOT EXISTS podcastPlatformIds ( + podcast_id INTEGER NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + platform_podcast_id VARCHAR(128) NOT NULL, + PRIMARY KEY (podcast_id, platform_id), + KEY (platform_id, platform_podcast_id) +); + +-- maps the id of an episode on a platform to our episode id +-- scoped by podcast as the same platform show might be connected to multiple podcasts +CREATE TABLE IF NOT EXISTS episodePlatformIds ( + podcast_id INTEGER NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + platform_episode_id VARCHAR(128) NOT NULL, + episode_id INT UNSIGNED NOT NULL, + PRIMARY KEY (podcast_id, platform_id, platform_episode_id), + KEY (episode_id) +); + +-- metric values of a podcast for a date range +-- daily values: start_date = end_date +-- lifetime values: start_date = release date, end_date = date the data was fetched +CREATE TABLE IF NOT EXISTS podcastMetrics ( + podcast_id INTEGER NOT NULL, + metric_id SMALLINT UNSIGNED NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + start_date DATE NOT NULL, + end_date DATE NOT NULL, + value BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (podcast_id, metric_id, platform_id, start_date, end_date) +); + +-- metric values of an episode for a date range (same semantics as podcastMetrics) +CREATE TABLE IF NOT EXISTS episodeMetrics ( + episode_id INT UNSIGNED NOT NULL, + metric_id SMALLINT UNSIGNED NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + start_date DATE NOT NULL, + end_date DATE NOT NULL, + value BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (episode_id, metric_id, platform_id, start_date, end_date) +); + +-- backfill podcast platform ids from the podcasts table +INSERT IGNORE INTO podcastPlatformIds (podcast_id, platform_id, platform_podcast_id) +SELECT account_id, 1, spotify_id FROM podcasts WHERE spotify_id IS NOT NULL AND spotify_id <> '' +UNION ALL +SELECT account_id, 2, apple_id FROM podcasts WHERE apple_id IS NOT NULL AND apple_id <> '' +UNION ALL +SELECT account_id, 3, anchor_id FROM podcasts WHERE anchor_id IS NOT NULL AND anchor_id <> '' +UNION ALL +SELECT account_id, 4, podigee_id FROM podcasts WHERE podigee_id IS NOT NULL AND podigee_id <> ''; + +-- backfill episode ids for all known Spotify episodes +-- the tables are empty, so ids can be assigned by row number +INSERT INTO episodePlatformIds (podcast_id, platform_id, platform_episode_id, episode_id) +SELECT account_id, 1, episode_id, ROW_NUMBER() OVER (ORDER BY account_id, episode_id) +FROM ( + SELECT account_id, episode_id FROM spotifyEpisodeMetadata + UNION + SELECT DISTINCT account_id, episode_id FROM spotifyEpisodeDetailedStreams +) spotifyEpisodes +WHERE episode_id IS NOT NULL AND episode_id <> ''; + +-- auto increment continues after the highest inserted id +INSERT INTO episodes (episode_id, podcast_id) +SELECT episode_id, podcast_id FROM episodePlatformIds; + +-- backfill daily Spotify plays (= streams) +INSERT INTO podcastMetrics (podcast_id, metric_id, platform_id, start_date, end_date, value) +SELECT account_id, 1, 1, sps_date, sps_date, sps_streams +FROM spotifyPodcastDetailedStreams +WHERE sps_streams IS NOT NULL AND sps_streams >= 0; + +INSERT INTO episodeMetrics (episode_id, metric_id, platform_id, start_date, end_date, value) +SELECT m.episode_id, 1, 1, s.sps_date, s.sps_date, s.sps_streams +FROM spotifyEpisodeDetailedStreams s +JOIN episodePlatformIds m + ON m.podcast_id = s.account_id + AND m.platform_id = 1 + AND m.platform_episode_id = s.episode_id +WHERE s.sps_streams IS NOT NULL AND s.sps_streams >= 0; + +INSERT INTO migrations (migration_id, migration_name) VALUES (19, 'v2 metrics'); diff --git a/db_schema/queries/v2/episodePlays.sql b/db_schema/queries/v2/episodePlays.sql new file mode 100644 index 00000000..03a4526e --- /dev/null +++ b/db_schema/queries/v2/episodePlays.sql @@ -0,0 +1,21 @@ +-- @doc +-- Returns daily plays per episode and platform (Spotify: streams of 60+ seconds). +-- Each row is a daily value (startDate = endDate) between start and end. +-- Fields: episodeId, metric, platform, startDate, endDate, value + +SELECT + m.episode_id AS episodeId, + mt.name AS metric, + p.name AS platform, + m.start_date AS startDate, + m.end_date AS endDate, + m.value +FROM episodes e +JOIN episodeMetrics m ON m.episode_id = e.episode_id +JOIN metrics mt ON mt.metric_id = m.metric_id +JOIN platforms p ON p.platform_id = m.platform_id +WHERE e.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date = m.end_date + AND m.start_date BETWEEN @start AND @end +ORDER BY m.episode_id, m.start_date, p.name diff --git a/db_schema/queries/v2/episodePlaysLifetime.sql b/db_schema/queries/v2/episodePlaysLifetime.sql new file mode 100644 index 00000000..97d8c454 --- /dev/null +++ b/db_schema/queries/v2/episodePlaysLifetime.sql @@ -0,0 +1,27 @@ +-- @doc +-- Returns the lifetime plays per episode and platform as reported by the platform. +-- The range starts at the release date and ends at the date the data was fetched. +-- The latest value fetched on or before end is returned (start is ignored). +-- Fields: episodeId, metric, platform, startDate, endDate, value + +SELECT episodeId, metric, platform, startDate, endDate, value +FROM ( + SELECT + m.episode_id AS episodeId, + mt.name AS metric, + p.name AS platform, + m.start_date AS startDate, + m.end_date AS endDate, + m.value, + ROW_NUMBER() OVER (PARTITION BY m.episode_id, m.platform_id ORDER BY m.end_date DESC, m.start_date) AS rn + FROM episodes e + JOIN episodeMetrics m ON m.episode_id = e.episode_id + JOIN metrics mt ON mt.metric_id = m.metric_id + JOIN platforms p ON p.platform_id = m.platform_id + WHERE e.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date < m.end_date + AND m.end_date <= @end +) lifetime +WHERE rn = 1 +ORDER BY episodeId, platform diff --git a/db_schema/queries/v2/episodePlaysTotal.sql b/db_schema/queries/v2/episodePlaysTotal.sql new file mode 100644 index 00000000..c46b5b65 --- /dev/null +++ b/db_schema/queries/v2/episodePlaysTotal.sql @@ -0,0 +1,23 @@ +-- @doc +-- Returns the plays per episode and platform summed up from start to end. +-- days is the number of days with data in the range, to make gaps visible. +-- Fields: episodeId, metric, platform, startDate, endDate, value, days + +SELECT + m.episode_id AS episodeId, + mt.name AS metric, + p.name AS platform, + DATE_FORMAT(@start, '%Y-%m-%d') AS startDate, + DATE_FORMAT(@end, '%Y-%m-%d') AS endDate, + CAST(SUM(m.value) AS UNSIGNED) AS value, + COUNT(*) AS days +FROM episodes e +JOIN episodeMetrics m ON m.episode_id = e.episode_id +JOIN metrics mt ON mt.metric_id = m.metric_id +JOIN platforms p ON p.platform_id = m.platform_id +WHERE e.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date = m.end_date + AND m.start_date BETWEEN @start AND @end +GROUP BY m.episode_id, mt.name, p.name +ORDER BY m.episode_id, p.name diff --git a/db_schema/queries/v2/episodes.sql b/db_schema/queries/v2/episodes.sql new file mode 100644 index 00000000..4958753d --- /dev/null +++ b/db_schema/queries/v2/episodes.sql @@ -0,0 +1,23 @@ +-- @doc +-- Returns all episodes of the podcast with their episode id and the ids on each platform. +-- Use it to map the episodeId of other v2 queries. start and end are ignored. +-- Fields: episodeId, name, releaseDate, spotifyId, appleId, anchorId, podigeeId + +SELECT + e.episode_id AS episodeId, + MAX(sm.ep_name) AS name, + MAX(sm.ep_release_date) AS releaseDate, + MAX(CASE WHEN p.name = 'spotify' THEN ep.platform_episode_id END) AS spotifyId, + MAX(CASE WHEN p.name = 'apple' THEN ep.platform_episode_id END) AS appleId, + MAX(CASE WHEN p.name = 'anchor' THEN ep.platform_episode_id END) AS anchorId, + MAX(CASE WHEN p.name = 'podigee' THEN ep.platform_episode_id END) AS podigeeId +FROM episodes e +JOIN episodePlatformIds ep ON ep.episode_id = e.episode_id +JOIN platforms p ON p.platform_id = ep.platform_id +LEFT JOIN spotifyEpisodeMetadata sm + ON p.name = 'spotify' + AND sm.account_id = e.podcast_id + AND sm.episode_id = ep.platform_episode_id +WHERE e.podcast_id = @podcast_id +GROUP BY e.episode_id +ORDER BY releaseDate DESC, e.episode_id DESC diff --git a/db_schema/queries/v2/podcastPlays.sql b/db_schema/queries/v2/podcastPlays.sql new file mode 100644 index 00000000..c18e3f93 --- /dev/null +++ b/db_schema/queries/v2/podcastPlays.sql @@ -0,0 +1,20 @@ +-- @doc +-- Returns daily plays of the podcast per platform (Spotify: streams of 60+ seconds). +-- Each row is a daily value (startDate = endDate) between start and end. +-- Fields: podcastId, metric, platform, startDate, endDate, value + +SELECT + m.podcast_id AS podcastId, + mt.name AS metric, + p.name AS platform, + m.start_date AS startDate, + m.end_date AS endDate, + m.value +FROM podcastMetrics m +JOIN metrics mt ON mt.metric_id = m.metric_id +JOIN platforms p ON p.platform_id = m.platform_id +WHERE m.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date = m.end_date + AND m.start_date BETWEEN @start AND @end +ORDER BY m.start_date, p.name diff --git a/db_schema/queries/v2/podcastPlaysLifetime.sql b/db_schema/queries/v2/podcastPlaysLifetime.sql new file mode 100644 index 00000000..7a113f2a --- /dev/null +++ b/db_schema/queries/v2/podcastPlaysLifetime.sql @@ -0,0 +1,26 @@ +-- @doc +-- Returns the lifetime plays of the podcast per platform as reported by the platform. +-- The range starts at the release date and ends at the date the data was fetched. +-- The latest value fetched on or before end is returned (start is ignored). +-- Fields: podcastId, metric, platform, startDate, endDate, value + +SELECT podcastId, metric, platform, startDate, endDate, value +FROM ( + SELECT + m.podcast_id AS podcastId, + mt.name AS metric, + p.name AS platform, + m.start_date AS startDate, + m.end_date AS endDate, + m.value, + ROW_NUMBER() OVER (PARTITION BY m.platform_id ORDER BY m.end_date DESC, m.start_date) AS rn + FROM podcastMetrics m + JOIN metrics mt ON mt.metric_id = m.metric_id + JOIN platforms p ON p.platform_id = m.platform_id + WHERE m.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date < m.end_date + AND m.end_date <= @end +) lifetime +WHERE rn = 1 +ORDER BY platform diff --git a/db_schema/queries/v2/podcastPlaysMissingDays.sql b/db_schema/queries/v2/podcastPlaysMissingDays.sql new file mode 100644 index 00000000..abf44a1d --- /dev/null +++ b/db_schema/queries/v2/podcastPlaysMissingDays.sql @@ -0,0 +1,28 @@ +-- @doc +-- Returns the days between start and end without daily Spotify plays of the podcast. +-- Can be used to fetch missing days and fill gaps. +-- Fields: podcastId, metric, platform, date + +WITH RECURSIVE days (day) AS ( + SELECT CAST(@start AS DATE) + UNION ALL + SELECT day + INTERVAL 1 DAY FROM days WHERE day < CAST(@end AS DATE) +) +SELECT /*+ SET_VAR(cte_max_recursion_depth = 100000) */ + @podcast_id AS podcastId, + 'plays' AS metric, + 'spotify' AS platform, + d.day AS date +FROM days d +WHERE NOT EXISTS ( + SELECT 1 + FROM podcastMetrics m + JOIN metrics mt ON mt.metric_id = m.metric_id + JOIN platforms p ON p.platform_id = m.platform_id + WHERE m.podcast_id = @podcast_id + AND mt.name = 'plays' + AND p.name = 'spotify' + AND m.start_date = d.day + AND m.end_date = d.day +) +ORDER BY d.day diff --git a/db_schema/queries/v2/podcastPlaysTotal.sql b/db_schema/queries/v2/podcastPlaysTotal.sql new file mode 100644 index 00000000..32a81279 --- /dev/null +++ b/db_schema/queries/v2/podcastPlaysTotal.sql @@ -0,0 +1,22 @@ +-- @doc +-- Returns the plays of the podcast per platform summed up from start to end. +-- days is the number of days with data in the range, to make gaps visible. +-- Fields: podcastId, metric, platform, startDate, endDate, value, days + +SELECT + m.podcast_id AS podcastId, + mt.name AS metric, + p.name AS platform, + DATE_FORMAT(@start, '%Y-%m-%d') AS startDate, + DATE_FORMAT(@end, '%Y-%m-%d') AS endDate, + CAST(SUM(m.value) AS UNSIGNED) AS value, + COUNT(*) AS days +FROM podcastMetrics m +JOIN metrics mt ON mt.metric_id = m.metric_id +JOIN platforms p ON p.platform_id = m.platform_id +WHERE m.podcast_id = @podcast_id + AND mt.name = 'plays' + AND m.start_date = m.end_date + AND m.start_date BETWEEN @start AND @end +GROUP BY m.podcast_id, mt.name, p.name +ORDER BY p.name diff --git a/db_schema/schema.sql b/db_schema/schema.sql index b2eba686..eda5c88c 100644 --- a/db_schema/schema.sql +++ b/db_schema/schema.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS migrations ( -- ----------------------------------------- -- IMPORTANT: this is the schema version -- ID has to be incremented for each change -INSERT INTO migrations (migration_id, migration_name) VALUES (18, 'episode mapping indexes'); +INSERT INTO migrations (migration_id, migration_name) VALUES (19, 'v2 metrics'); -- ----------------------------------------- CREATE TABLE IF NOT EXISTS events ( @@ -806,3 +806,89 @@ CREATE TABLE IF NOT EXISTS hosterEpisodeMetrics ( FOREIGN KEY (subdimension) REFERENCES subdimensions(dim_id) ON DELETE RESTRICT ); + +-- ----------------------------------------- +-- v2 metrics schema (own podcast/episode ids, platform id mappings, metrics by date range) +-- ----------------------------------------- + +-- platforms data is fetched from (ids are referenced in code, see src/types/metrics.ts) +CREATE TABLE IF NOT EXISTS platforms ( + platform_id TINYINT UNSIGNED NOT NULL, + name VARCHAR(32) NOT NULL, + PRIMARY KEY (platform_id), + UNIQUE KEY (name) +); + +INSERT INTO platforms (platform_id, name) VALUES + (1, 'spotify'), + (2, 'apple'), + (3, 'anchor'), + (4, 'podigee'); + +-- metric definitions (ids are referenced in code, see src/types/metrics.ts) +-- additive metrics (e.g. plays) are stored as daily values and can be summed up +-- non additive metrics (e.g. unique listeners) are stored with explicit date ranges +CREATE TABLE IF NOT EXISTS metrics ( + metric_id SMALLINT UNSIGNED NOT NULL, + name VARCHAR(64) NOT NULL, + additive BOOLEAN NOT NULL, + PRIMARY KEY (metric_id), + UNIQUE KEY (name) +); + +INSERT INTO metrics (metric_id, name, additive) VALUES + (1, 'plays', 1); + +-- episodes with an id which is unique across all podcasts +CREATE TABLE IF NOT EXISTS episodes ( + episode_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + podcast_id INTEGER NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (episode_id), + KEY (podcast_id) +); + +-- maps the podcast id (podcasts.account_id) to the id of the podcast on a platform +CREATE TABLE IF NOT EXISTS podcastPlatformIds ( + podcast_id INTEGER NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + platform_podcast_id VARCHAR(128) NOT NULL, + PRIMARY KEY (podcast_id, platform_id), + KEY (platform_id, platform_podcast_id) +); + +-- maps the id of an episode on a platform to our episode id +-- scoped by podcast as the same platform show might be connected to multiple podcasts +CREATE TABLE IF NOT EXISTS episodePlatformIds ( + podcast_id INTEGER NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + platform_episode_id VARCHAR(128) NOT NULL, + episode_id INT UNSIGNED NOT NULL, + PRIMARY KEY (podcast_id, platform_id, platform_episode_id), + KEY (episode_id) +); + +-- metric values of a podcast for a date range +-- daily values: start_date = end_date +-- lifetime values: start_date = release date, end_date = date the data was fetched +CREATE TABLE IF NOT EXISTS podcastMetrics ( + podcast_id INTEGER NOT NULL, + metric_id SMALLINT UNSIGNED NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + start_date DATE NOT NULL, + end_date DATE NOT NULL, + value BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (podcast_id, metric_id, platform_id, start_date, end_date) +); + +-- metric values of an episode for a date range (same semantics as podcastMetrics) +CREATE TABLE IF NOT EXISTS episodeMetrics ( + episode_id INT UNSIGNED NOT NULL, + metric_id SMALLINT UNSIGNED NOT NULL, + platform_id TINYINT UNSIGNED NOT NULL, + start_date DATE NOT NULL, + end_date DATE NOT NULL, + value BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (episode_id, metric_id, platform_id, start_date, end_date) +); + diff --git a/src/api/connectors/SpotifyConnector.test.ts b/src/api/connectors/SpotifyConnector.test.ts new file mode 100644 index 00000000..67004fab --- /dev/null +++ b/src/api/connectors/SpotifyConnector.test.ts @@ -0,0 +1,131 @@ +import { SpotifyConnector } from './SpotifyConnector' +import { SpotifyRepository } from '../../db/SpotifyRepository' +import { MetricsRepository } from '../../db/MetricsRepository' +import { METRIC, PLATFORM } from '../../types/metrics' +import podcastDetailedStreams from '../../../fixtures/spotifyPodcastDetailedStreams.json' +import episodeDetailedStreams from '../../../fixtures/spotifyEpisodeDetailedStreams.json' +import podcastMetadata from '../../../fixtures/spotifyPodcastMetadata.json' +import episodesMetadata from '../../../fixtures/spotifyEpisodesMetadata.json' + +describe('SpotifyConnector v2 metrics', () => { + let repo: jest.Mocked + let metricsRepo: jest.Mocked + let connector: SpotifyConnector + + beforeEach(() => { + repo = { + storePodcastDetailedStreams: jest.fn(), + storeEpisodeDetailedStreams: jest.fn(), + storePodcastMetadata: jest.fn(), + storeEpisodeMetadata: jest.fn(), + } as any + metricsRepo = { + getOrCreateEpisodeId: jest.fn().mockResolvedValue(42), + upsertPodcastPlatformId: jest.fn(), + storePodcastMetrics: jest.fn(), + storeEpisodeMetrics: jest.fn(), + } as any + connector = new SpotifyConnector(repo, metricsRepo) + }) + + const expectedDailyPlays = (payload: typeof podcastDetailedStreams) => + payload.data.detailedStreams.map((entry) => ({ + start: entry.date, + end: entry.date, + value: entry.streams, + })) + + it('should store podcast streams as daily plays', async () => { + await connector.handleRequest(3, podcastDetailedStreams as any) + + expect(repo.storePodcastDetailedStreams).toHaveBeenCalled() + expect(metricsRepo.storePodcastMetrics).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + METRIC.plays, + expectedDailyPlays(podcastDetailedStreams) + ) + }) + + it('should store episode streams as daily plays of our episode id', async () => { + await connector.handleRequest(3, episodeDetailedStreams as any) + + expect(repo.storeEpisodeDetailedStreams).toHaveBeenCalled() + expect(metricsRepo.getOrCreateEpisodeId).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + episodeDetailedStreams.meta.episode + ) + expect(metricsRepo.storeEpisodeMetrics).toHaveBeenCalledWith( + 42, + PLATFORM.spotify, + METRIC.plays, + expectedDailyPlays(episodeDetailedStreams) + ) + }) + + it('should store lifetime episode plays from release until fetch date', async () => { + await connector.handleRequest(3, episodesMetadata as any) + + expect(repo.storeEpisodeMetadata).toHaveBeenCalled() + expect(metricsRepo.getOrCreateEpisodeId).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + episodesMetadata.data.id + ) + expect(metricsRepo.storeEpisodeMetrics).toHaveBeenCalledWith( + 42, + PLATFORM.spotify, + METRIC.plays, + [ + { + start: episodesMetadata.data.releaseDate, + end: episodesMetadata.retrieved.substring(0, 10), + value: episodesMetadata.data.streams, + }, + ] + ) + }) + + it('should store the podcast platform id and lifetime plays', async () => { + const payload = { + ...podcastMetadata, + retrieved: '2026-10-18T06:00:00.000Z', + } + await connector.handleRequest(3, payload as any) + + expect(repo.storePodcastMetadata).toHaveBeenCalled() + expect(metricsRepo.upsertPodcastPlatformId).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + podcastMetadata.data.id + ) + expect(metricsRepo.storePodcastMetrics).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + METRIC.plays, + [ + { + start: podcastMetadata.data.releaseDate, + end: '2026-10-18', + value: podcastMetadata.data.streams, + }, + ] + ) + }) + + it('should skip lifetime plays if the release date is after the fetch date', async () => { + const payload = { + ...podcastMetadata, + retrieved: '2022-01-01T06:00:00.000Z', + } + await connector.handleRequest(3, payload as any) + + expect(metricsRepo.storePodcastMetrics).toHaveBeenCalledWith( + 3, + PLATFORM.spotify, + METRIC.plays, + [] + ) + }) +}) diff --git a/src/api/connectors/SpotifyConnector.ts b/src/api/connectors/SpotifyConnector.ts index 9aa1b0c0..b4fe0a2d 100644 --- a/src/api/connectors/SpotifyConnector.ts +++ b/src/api/connectors/SpotifyConnector.ts @@ -28,12 +28,44 @@ import impressionsFacetedSchema from '../../schema/spotify/impressionsFaceted.js import impressionsFunnelSchema from '../../schema/spotify/impressionsFunnel.json' import { validateJsonApiPayload } from '../JsonPayloadValidator' import { SpotifyRepository } from '../../db/SpotifyRepository' +import { MetricsRepository } from '../../db/MetricsRepository' +import { METRIC, MetricValue, PLATFORM } from '../../types/metrics' + +// date (YYYY-MM-DD) the data was fetched by the connector +const getFetchDate = (payload: ConnectorPayload): string => + (payload.retrieved ?? new Date().toISOString()).substring(0, 10) + +// lifetime value from the release date until the data was fetched +// returns no value if the release date is unknown or after the fetch date +const getLifetimeValue = ( + releaseDate: string | undefined, + fetchDate: string, + value: number +): MetricValue[] => { + const start = releaseDate?.substring(0, 10) + if (!start || start > fetchDate) { + return [] + } + return [{ start, end: fetchDate, value }] +} + +// v2: Spotify streams (60+ seconds) are stored as plays +const streamsToDailyPlays = ( + payload: SpotifyDetailedStreamsPayload +): MetricValue[] => + payload.detailedStreams.map((entry) => ({ + start: entry.date, + end: entry.date, + value: entry.streams, + })) class SpotifyConnector implements ConnectorHandler { repo: SpotifyRepository + metricsRepo: MetricsRepository - constructor(repo: SpotifyRepository) { + constructor(repo: SpotifyRepository, metricsRepo: MetricsRepository) { this.repo = repo + this.metricsRepo = metricsRepo } async handleRequest( @@ -43,10 +75,26 @@ class SpotifyConnector implements ConnectorHandler { if (payload.meta.endpoint === 'metadata') { // metadata of podcast validateJsonApiPayload(podcastMetadataSchema, payload.data) + const data = payload.data as SpotifyPodcastMetadataPayload - return await this.repo.storePodcastMetadata( + await this.repo.storePodcastMetadata(accountId, data) + + if (data.id) { + await this.metricsRepo.upsertPodcastPlatformId( + accountId, + PLATFORM.spotify, + data.id + ) + } + return await this.metricsRepo.storePodcastMetrics( accountId, - payload.data as SpotifyPodcastMetadataPayload + PLATFORM.spotify, + METRIC.plays, + getLifetimeValue( + data.releaseDate, + getFetchDate(payload), + data.streams + ) ) } else if (payload.meta.endpoint === 'detailedStreams') { // validates the payload and throws an error if it is not valid @@ -54,25 +102,56 @@ class SpotifyConnector implements ConnectorHandler { // detailedStreams API has the same format for episodes and podcasts // it is distinguished by the presence/absence of the episode id + const data = payload.data as SpotifyDetailedStreamsPayload if (payload.meta.episode !== undefined) { - return await this.repo.storeEpisodeDetailedStreams( + await this.repo.storeEpisodeDetailedStreams( accountId, payload.meta.episode, - payload.data as SpotifyDetailedStreamsPayload + data + ) + + const episodeId = await this.metricsRepo.getOrCreateEpisodeId( + accountId, + PLATFORM.spotify, + payload.meta.episode + ) + return await this.metricsRepo.storeEpisodeMetrics( + episodeId, + PLATFORM.spotify, + METRIC.plays, + streamsToDailyPlays(data) ) } else { - return await this.repo.storePodcastDetailedStreams( + await this.repo.storePodcastDetailedStreams(accountId, data) + + return await this.metricsRepo.storePodcastMetrics( accountId, - payload.data as SpotifyDetailedStreamsPayload + PLATFORM.spotify, + METRIC.plays, + streamsToDailyPlays(data) ) } } else if (payload.meta.endpoint === 'episodeMetadata') { // metadata of single episode validateJsonApiPayload(episodeMetadataSchema, payload.data) + const data = payload.data as SpotifyEpisodeMetadataPayload + + await this.repo.storeEpisodeMetadata(accountId, data) - return await this.repo.storeEpisodeMetadata( + const episodeId = await this.metricsRepo.getOrCreateEpisodeId( accountId, - payload.data as SpotifyEpisodeMetadataPayload + PLATFORM.spotify, + data.id + ) + return await this.metricsRepo.storeEpisodeMetrics( + episodeId, + PLATFORM.spotify, + METRIC.plays, + getLifetimeValue( + data.releaseDate, + getFetchDate(payload), + data.streams + ) ) } else if (payload.meta.endpoint === 'followers') { // follower count per day diff --git a/src/config/generate-query-docs.ts b/src/config/generate-query-docs.ts index b16e9895..496565e5 100644 --- a/src/config/generate-query-docs.ts +++ b/src/config/generate-query-docs.ts @@ -5,13 +5,16 @@ import { getQueryDataSchema, getQueryResponseExample } from './query-examples' /** * Generates OpenAPI documentation for SQL query endpoints - * Reads SQL files from db_schema/queries/v1/ and creates endpoint specs + * Reads SQL files from db_schema/queries// and creates endpoint specs */ -const QUERIES_DIR = path.join(__dirname, '../../db_schema/queries/v1') +const QUERIES_BASE_DIR = path.join(__dirname, '../../db_schema/queries') +const QUERY_VERSIONS = ['v1', 'v2'] type QueryDoc = { name: string + // key of the response example: name for v1, / otherwise + exampleKey: string category: string description: string path: string @@ -53,23 +56,27 @@ const extractDescription = (sqlContent: string): string => { return commentLines.join('
') || 'Analytics query endpoint' } -export const generateQueryDocs = (): QueryDoc[] => { +const generateVersionQueryDocs = (version: string): QueryDoc[] => { const queries: QueryDoc[] = [] + const queriesDir = path.join(QUERIES_BASE_DIR, version) - if (!fs.existsSync(QUERIES_DIR)) { - console.warn(`Queries directory not found: ${QUERIES_DIR}`) + if (!fs.existsSync(queriesDir)) { + console.warn(`Queries directory not found: ${queriesDir}`) return queries } - const files = fs.readdirSync(QUERIES_DIR).filter((f) => f.endsWith('.sql')) + const files = fs.readdirSync(queriesDir).filter((f) => f.endsWith('.sql')) for (const file of files) { const queryName = file.replace('.sql', '') - const filePath = path.join(QUERIES_DIR, file) + const filePath = path.join(queriesDir, file) const sqlContent = fs.readFileSync(filePath, 'utf-8') let description = extractDescription(sqlContent) - const category = categorizeQuery(queryName) + const category = + version === 'v1' + ? categorizeQuery(queryName) + : `Metrics (${version})` // First line has to start with `-- @doc`. If not, ignore the file if (!description.startsWith('@doc')) { @@ -81,15 +88,20 @@ export const generateQueryDocs = (): QueryDoc[] => { queries.push({ name: queryName, + exampleKey: + version === 'v1' ? queryName : `${version}/${queryName}`, category, description, - path: `/analytics/v1/{podcastId}/${queryName}`, + path: `/analytics/${version}/{podcastId}/${queryName}`, }) } return queries } +export const generateQueryDocs = (): QueryDoc[] => + QUERY_VERSIONS.flatMap(generateVersionQueryDocs) + // Generate OpenAPI paths object for all queries export const generateQueryPaths = () => { const queries = generateQueryDocs() @@ -105,7 +117,7 @@ export const generateQueryPaths = () => { for (const query of queries) { const exampleDate = getTwoDaysAgo() const responseExample = getQueryResponseExample( - query.name, + query.exampleKey, exampleDate, exampleDate ) @@ -173,7 +185,9 @@ export const generateQueryPaths = () => { result: { type: 'string' }, }, }, - data: getQueryDataSchema(query.name), + data: getQueryDataSchema( + query.exampleKey + ), }, }, ...(responseExample diff --git a/src/config/query-examples.ts b/src/config/query-examples.ts index 0918c932..68f450d4 100644 --- a/src/config/query-examples.ts +++ b/src/config/query-examples.ts @@ -19,6 +19,87 @@ type OpenApiSchema = { } export const queryDataExamples: QueryExampleMap = { + 'v2/podcastPlays': [ + { + podcastId: 42, + metric: 'plays', + platform: 'spotify', + startDate: '2026-07-01', + endDate: '2026-07-01', + value: 128, + }, + ], + 'v2/podcastPlaysTotal': [ + { + podcastId: 42, + metric: 'plays', + platform: 'spotify', + startDate: '2026-07-01', + endDate: '2026-07-31', + value: 3912, + days: 31, + }, + ], + 'v2/podcastPlaysLifetime': [ + { + podcastId: 42, + metric: 'plays', + platform: 'spotify', + startDate: '2022-10-25', + endDate: '2026-07-31', + value: 85120, + }, + ], + 'v2/podcastPlaysMissingDays': [ + { + podcastId: 42, + metric: 'plays', + platform: 'spotify', + date: '2026-07-14', + }, + ], + 'v2/episodePlays': [ + { + episodeId: 1017, + metric: 'plays', + platform: 'spotify', + startDate: '2026-07-01', + endDate: '2026-07-01', + value: 37, + }, + ], + 'v2/episodePlaysTotal': [ + { + episodeId: 1017, + metric: 'plays', + platform: 'spotify', + startDate: '2026-07-01', + endDate: '2026-07-31', + value: 845, + days: 31, + }, + ], + 'v2/episodePlaysLifetime': [ + { + episodeId: 1017, + metric: 'plays', + platform: 'spotify', + startDate: '2026-06-24', + endDate: '2026-07-31', + value: 1290, + }, + ], + 'v2/episodes': [ + { + episodeId: 1017, + name: 'How Small Teams Measure Growth', + releaseDate: '2026-06-24', + spotifyId: '2ebg5MhyOK9rotpCBfe830', + appleId: null, + anchorId: null, + podigeeId: null, + }, + ], appleEpisodesLTR: [ { name: 'How Small Teams Measure Growth', @@ -516,7 +597,8 @@ export const getQueryResponseExample = ( return { meta: { - query: queryName, + // strip the version prefix of keys like v2/podcastPlays + query: queryName.split('/').pop() ?? queryName, accountId: '42', podcastId: '42', date: '2026-07-09 12:00:00', diff --git a/src/db/DBInitializer.ts b/src/db/DBInitializer.ts index be92b6ce..243f6e14 100644 --- a/src/db/DBInitializer.ts +++ b/src/db/DBInitializer.ts @@ -146,7 +146,7 @@ class DBInitializer { migrationId++ ) { console.log(`Running migration number ${migrationId} ...`) - this.runMigration(migrationId) + await this.runMigration(migrationId) console.log(`Migration ${migrationId} done`) } console.log('All migration work finished') diff --git a/src/db/MetricsRepository.test.ts b/src/db/MetricsRepository.test.ts new file mode 100644 index 00000000..63d46d7c --- /dev/null +++ b/src/db/MetricsRepository.test.ts @@ -0,0 +1,165 @@ +import { MetricsRepository } from './MetricsRepository' +import { Pool } from 'mysql2/promise' +import { METRIC, PLATFORM } from '../types/metrics' + +describe('MetricsRepository', () => { + let repo: MetricsRepository + let mockPool: jest.Mocked + + beforeEach(() => { + mockPool = { + query: jest.fn(), + } as any + + repo = new MetricsRepository(mockPool) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + describe('getOrCreateEpisodeId', () => { + it('should return the existing episode id', async () => { + mockPool.query.mockResolvedValueOnce([ + [{ episode_id: 7 }], + [], + ] as any) + + const result = await repo.getOrCreateEpisodeId( + 3, + PLATFORM.spotify, + 'spotifyEpisode' + ) + + expect(result).toBe(7) + expect(mockPool.query).toHaveBeenCalledTimes(1) + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining( + 'SELECT episode_id FROM episodePlatformIds' + ), + [3, PLATFORM.spotify, 'spotifyEpisode'] + ) + }) + + it('should create a new episode and mapping if unknown', async () => { + mockPool.query + .mockResolvedValueOnce([[], []] as any) + .mockResolvedValueOnce([{ insertId: 42 }, []] as any) + .mockResolvedValueOnce([{ affectedRows: 1 }, []] as any) + + const result = await repo.getOrCreateEpisodeId( + 3, + PLATFORM.spotify, + 'spotifyEpisode' + ) + + expect(result).toBe(42) + expect(mockPool.query).toHaveBeenNthCalledWith( + 2, + 'INSERT INTO episodes (podcast_id) VALUES (?)', + [3] + ) + expect(mockPool.query).toHaveBeenNthCalledWith( + 3, + expect.stringContaining( + 'INSERT IGNORE INTO episodePlatformIds' + ), + [3, PLATFORM.spotify, 'spotifyEpisode', 42] + ) + }) + + it('should use the concurrently created episode and remove the orphan', async () => { + mockPool.query + .mockResolvedValueOnce([[], []] as any) + .mockResolvedValueOnce([{ insertId: 42 }, []] as any) + .mockResolvedValueOnce([{ affectedRows: 0 }, []] as any) + .mockResolvedValueOnce([{ affectedRows: 1 }, []] as any) + .mockResolvedValueOnce([[{ episode_id: 41 }], []] as any) + + const result = await repo.getOrCreateEpisodeId( + 3, + PLATFORM.spotify, + 'spotifyEpisode' + ) + + expect(result).toBe(41) + expect(mockPool.query).toHaveBeenNthCalledWith( + 4, + 'DELETE FROM episodes WHERE episode_id = ?', + [42] + ) + }) + }) + + describe('storePodcastMetrics', () => { + it('should store all values with one bulk upsert', async () => { + mockPool.query.mockResolvedValue([{ affectedRows: 2 }, []] as any) + + await repo.storePodcastMetrics(3, PLATFORM.spotify, METRIC.plays, [ + { start: '2022-09-26', end: '2022-09-26', value: 11 }, + { start: '2022-09-27', end: '2022-09-27', value: 32 }, + ]) + + expect(mockPool.query).toHaveBeenCalledTimes(1) + const [sql, params] = mockPool.query.mock.calls[0] as any + expect(sql).toContain('INSERT INTO podcastMetrics (podcast_id,') + expect(sql).toContain('ON DUPLICATE KEY UPDATE') + expect(params).toEqual([ + [ + [ + 3, + METRIC.plays, + PLATFORM.spotify, + '2022-09-26', + '2022-09-26', + 11, + ], + [ + 3, + METRIC.plays, + PLATFORM.spotify, + '2022-09-27', + '2022-09-27', + 32, + ], + ], + ]) + }) + + it('should not query without values', async () => { + await repo.storePodcastMetrics( + 3, + PLATFORM.spotify, + METRIC.plays, + [] + ) + + expect(mockPool.query).not.toHaveBeenCalled() + }) + }) + + describe('storeEpisodeMetrics', () => { + it('should store values for the episode id', async () => { + mockPool.query.mockResolvedValue([{ affectedRows: 1 }, []] as any) + + await repo.storeEpisodeMetrics(42, PLATFORM.spotify, METRIC.plays, [ + { start: '2022-09-13', end: '2022-10-04', value: 120 }, + ]) + + const [sql, params] = mockPool.query.mock.calls[0] as any + expect(sql).toContain('INSERT INTO episodeMetrics (episode_id,') + expect(params).toEqual([ + [ + [ + 42, + METRIC.plays, + PLATFORM.spotify, + '2022-09-13', + '2022-10-04', + 120, + ], + ], + ]) + }) + }) +}) diff --git a/src/db/MetricsRepository.ts b/src/db/MetricsRepository.ts new file mode 100644 index 00000000..40d70761 --- /dev/null +++ b/src/db/MetricsRepository.ts @@ -0,0 +1,143 @@ +import { Pool, ResultSetHeader, RowDataPacket } from 'mysql2/promise' +import { MetricId, MetricValue, PlatformId } from '../types/metrics' + +// v2 metrics: platform independent podcast/episode ids and metric values +// stored by date range (see podcastMetrics and episodeMetrics tables) +class MetricsRepository { + pool: Pool + + constructor(pool: Pool) { + this.pool = pool + } + + private async findEpisodeId( + podcastId: number, + platformId: PlatformId, + platformEpisodeId: string + ): Promise { + const [rows] = await this.pool.query( + 'SELECT episode_id FROM episodePlatformIds WHERE podcast_id = ? AND platform_id = ? AND platform_episode_id = ?', + [podcastId, platformId, platformEpisodeId] + ) + return rows.length > 0 ? rows[0].episode_id : null + } + + // returns our episode id for the episode id of a platform + // and creates a new episode if it is not known yet + async getOrCreateEpisodeId( + podcastId: number, + platformId: PlatformId, + platformEpisodeId: string + ): Promise { + const existingId = await this.findEpisodeId( + podcastId, + platformId, + platformEpisodeId + ) + if (existingId !== null) { + return existingId + } + + const [episodeResult] = await this.pool.query( + 'INSERT INTO episodes (podcast_id) VALUES (?)', + [podcastId] + ) + const episodeId = episodeResult.insertId + + const [mappingResult] = await this.pool.query( + 'INSERT IGNORE INTO episodePlatformIds (podcast_id, platform_id, platform_episode_id, episode_id) VALUES (?,?,?,?)', + [podcastId, platformId, platformEpisodeId, episodeId] + ) + if (mappingResult.affectedRows > 0) { + return episodeId + } + + // a concurrent request created the mapping in the meantime, + // remove the orphaned episode and use the existing one + await this.pool.query('DELETE FROM episodes WHERE episode_id = ?', [ + episodeId, + ]) + const concurrentId = await this.findEpisodeId( + podcastId, + platformId, + platformEpisodeId + ) + if (concurrentId === null) { + throw new Error( + `Could not create episode id for ${platformEpisodeId} of podcast ${podcastId}` + ) + } + return concurrentId + } + + async upsertPodcastPlatformId( + podcastId: number, + platformId: PlatformId, + platformPodcastId: string + ): Promise { + await this.pool.query( + 'INSERT INTO podcastPlatformIds (podcast_id, platform_id, platform_podcast_id) VALUES (?,?,?) ON DUPLICATE KEY UPDATE platform_podcast_id = VALUES(platform_podcast_id)', + [podcastId, platformId, platformPodcastId] + ) + } + + private async storeMetrics( + table: 'podcastMetrics' | 'episodeMetrics', + idColumn: 'podcast_id' | 'episode_id', + id: number, + platformId: PlatformId, + metricId: MetricId, + values: MetricValue[] + ): Promise { + if (values.length === 0) { + return + } + const rows = values.map((v) => [ + id, + metricId, + platformId, + v.start, + v.end, + v.value, + ]) + // existing values are overwritten, so refetching data is idempotent + await this.pool.query( + `INSERT INTO ${table} (${idColumn}, metric_id, platform_id, start_date, end_date, value) VALUES ? ON DUPLICATE KEY UPDATE value = VALUES(value)`, + [rows] + ) + } + + async storePodcastMetrics( + podcastId: number, + platformId: PlatformId, + metricId: MetricId, + values: MetricValue[] + ): Promise { + return this.storeMetrics( + 'podcastMetrics', + 'podcast_id', + podcastId, + platformId, + metricId, + values + ) + } + + async storeEpisodeMetrics( + episodeId: number, + platformId: PlatformId, + metricId: MetricId, + values: MetricValue[] + ): Promise { + return this.storeMetrics( + 'episodeMetrics', + 'episode_id', + episodeId, + platformId, + metricId, + values + ) + } +} + +export { MetricsRepository } diff --git a/src/index.ts b/src/index.ts index 7c152330..dbe8156e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,7 @@ import bodyParser from 'body-parser' import { EventsApi, ConnectorApi } from './api' import { EventRepository } from './db/EventRepository' import { SpotifyRepository } from './db/SpotifyRepository' +import { MetricsRepository } from './db/MetricsRepository' import { AuthController } from './auth/AuthController' import { AuthError, HttpError, PayloadError } from './types/api' import { SpotifyConnector } from './api/connectors/SpotifyConnector' @@ -148,7 +149,8 @@ const eventRepo = new EventRepository(pool) const eventsApi = new EventsApi(eventRepo) const spotifyRepo = new SpotifyRepository(pool) -const spotifyConnector = new SpotifyConnector(spotifyRepo) +const metricsRepo = new MetricsRepository(pool) +const spotifyConnector = new SpotifyConnector(spotifyRepo, metricsRepo) const appleRepo = new AppleRepository(pool) const appleConnector = new AppleConnector(appleRepo) @@ -372,11 +374,13 @@ app.get( // Throw error if accountIds is not set or empty if (!accountIds || accountIds.length === 0) { res.status(401).send('Not authorized: no account access') + return } const podcastId = req.params.podcastId if (podcastId === '') { res.status(401).send('Not authorized: podcastId not set') + return } const version = req.params.version @@ -390,12 +394,13 @@ app.get( } // Check if the user has access to the requested podcast ID - // For v1, the podcast id must be one of the user's account ids + // The podcast id must be one of the user's account ids const podcastIdNum = parseInt(podcastId, 10) - if (version === 'v1' && !accountIds.includes(podcastIdNum)) { + if (!accountIds.includes(podcastIdNum)) { res.status(401).send( - 'Not authorized: podcastId must be one of your accessible account IDs in v1' + 'Not authorized: podcastId must be one of your accessible account IDs' ) + return } // Get date from query parameters diff --git a/src/types/connector.ts b/src/types/connector.ts index 604e2eee..adb19185 100644 --- a/src/types/connector.ts +++ b/src/types/connector.ts @@ -10,4 +10,5 @@ export interface ConnectorPayload { } data: object provider: string + retrieved?: string } diff --git a/src/types/metrics.ts b/src/types/metrics.ts new file mode 100644 index 00000000..d415af33 --- /dev/null +++ b/src/types/metrics.ts @@ -0,0 +1,23 @@ +// ids have to match the seeded rows in the platforms and metrics tables +// (see db_schema/schema.sql and db_schema/migrations/19_v2_metrics.sql) +export const PLATFORM = { + spotify: 1, + apple: 2, + anchor: 3, + podigee: 4, +} as const + +export const METRIC = { + plays: 1, +} as const + +export type PlatformId = typeof PLATFORM[keyof typeof PLATFORM] +export type MetricId = typeof METRIC[keyof typeof METRIC] + +// value of a metric for a date range +// daily values have the same start and end date +export interface MetricValue { + start: string + end: string + value: number +} diff --git a/src/types/provider/spotify.ts b/src/types/provider/spotify.ts index a3625d72..d1bbdef3 100644 --- a/src/types/provider/spotify.ts +++ b/src/types/provider/spotify.ts @@ -73,6 +73,7 @@ export interface SpotifyEpisodeAggregatePayload { } export interface SpotifyPodcastMetadataPayload { + id?: string name: string totalEpisodes: number starts: number diff --git a/tests/api_e2e/analytics_v2.test.js b/tests/api_e2e/analytics_v2.test.js new file mode 100644 index 00000000..11ca72ca --- /dev/null +++ b/tests/api_e2e/analytics_v2.test.js @@ -0,0 +1,185 @@ +const request = require('supertest') +const auth = require('./authheader') +const podcastDetailedStreamsPayload = require('../../fixtures/spotifyPodcastDetailedStreams.json') +const episodeDetailedStreamsPayload = require('../../fixtures/spotifyEpisodeDetailedStreams.json') +const podcastMetadataPayload = require('../../fixtures/spotifyPodcastMetadata.json') +const episodesMetadataPayload = require('../../fixtures/spotifyEpisodesMetadata.json') + +const baseURL = process.env.API_BASE_URL || 'http://localhost:8080' + +// hardcoded for testing and is set in db_auth_data.sql file +const podcast_test_id = 3 + +const path_prefix = `/analytics/v2/${podcast_test_id}/` + +const start = '2022-09-26' +const end = '2022-10-02' +const streams = podcastDetailedStreamsPayload.data.detailedStreams + +const postPayload = async (payload) => { + const response = await request(baseURL) + .post('/connector') + .set(auth) + .send(payload) + expect(response.statusCode).toBe(200) +} + +const getQuery = async (query, queryStart = start, queryEnd = end) => { + const response = await request(baseURL) + .get(`${path_prefix}${query}?start=${queryStart}&end=${queryEnd}`) + .set(auth) + .send() + expect(response.statusCode).toBe(200) + expect(response.body.meta.result).toBe('success') + return response.body.data +} + +const getEpisodeId = async (spotifyId) => { + const episodes = await getQuery('episodes') + const episode = episodes.find((e) => e.spotifyId === spotifyId) + expect(episode).toBeDefined() + return episode.episodeId +} + +beforeAll(async () => { + await postPayload(podcastDetailedStreamsPayload) + await postPayload(episodeDetailedStreamsPayload) + await postPayload(episodesMetadataPayload) + // release date of the fixture is after its retrieved date + await postPayload({ + ...podcastMetadataPayload, + retrieved: '2022-10-30T10:00:00.000Z', + }) +}) + +describe('v2 podcast plays', () => { + it('should return daily Spotify streams as plays', async () => { + const data = await getQuery('podcastPlays') + + const spotify = data.filter((row) => row.platform === 'spotify') + expect(spotify).toEqual( + streams.map((entry) => ({ + podcastId: podcast_test_id, + metric: 'plays', + platform: 'spotify', + startDate: entry.date, + endDate: entry.date, + value: entry.streams, + })) + ) + }) + + it('should sum up the plays of the date range', async () => { + const data = await getQuery('podcastPlaysTotal') + + expect(data).toContainEqual({ + podcastId: podcast_test_id, + metric: 'plays', + platform: 'spotify', + startDate: start, + endDate: end, + value: streams.reduce((sum, entry) => sum + entry.streams, 0), + days: streams.length, + }) + }) + + it('should return lifetime plays from release until fetch date', async () => { + const data = await getQuery('podcastPlaysLifetime', start, '2022-10-30') + + expect(data).toContainEqual({ + podcastId: podcast_test_id, + metric: 'plays', + platform: 'spotify', + startDate: podcastMetadataPayload.data.releaseDate, + endDate: '2022-10-30', + value: podcastMetadataPayload.data.streams, + }) + }) + + it('should return days without plays', async () => { + const data = await getQuery('podcastPlaysMissingDays', '2022-09-22', end) + + expect(data.map((row) => row.date)).toEqual([ + '2022-09-22', + '2022-09-23', + '2022-09-24', + '2022-09-25', + ]) + }) +}) + +describe('v2 episode plays', () => { + it('should return daily plays with our numeric episode id', async () => { + const episodeId = await getEpisodeId( + episodeDetailedStreamsPayload.meta.episode + ) + expect(typeof episodeId).toBe('number') + + const data = await getQuery('episodePlays') + const episodeRows = data.filter((row) => row.episodeId === episodeId) + expect(episodeRows).toEqual( + episodeDetailedStreamsPayload.data.detailedStreams.map((entry) => ({ + episodeId, + metric: 'plays', + platform: 'spotify', + startDate: entry.date, + endDate: entry.date, + value: entry.streams, + })) + ) + }) + + it('should keep the episode id when data is sent again', async () => { + const spotifyId = episodeDetailedStreamsPayload.meta.episode + const episodeId = await getEpisodeId(spotifyId) + + await postPayload(episodeDetailedStreamsPayload) + + expect(await getEpisodeId(spotifyId)).toBe(episodeId) + const episodes = await getQuery('episodes') + expect(episodes.filter((e) => e.spotifyId === spotifyId)).toHaveLength(1) + }) + + it('should sum up the plays per episode', async () => { + const episodeId = await getEpisodeId( + episodeDetailedStreamsPayload.meta.episode + ) + const data = await getQuery('episodePlaysTotal') + + expect(data).toContainEqual({ + episodeId, + metric: 'plays', + platform: 'spotify', + startDate: start, + endDate: end, + value: streams.reduce((sum, entry) => sum + entry.streams, 0), + days: streams.length, + }) + }) + + it('should return lifetime plays of an episode', async () => { + const episodeId = await getEpisodeId(episodesMetadataPayload.data.id) + const fetchDate = episodesMetadataPayload.retrieved.substring(0, 10) + + const data = await getQuery('episodePlaysLifetime', start, fetchDate) + + expect(data).toContainEqual({ + episodeId, + metric: 'plays', + platform: 'spotify', + startDate: episodesMetadataPayload.data.releaseDate, + endDate: fetchDate, + value: episodesMetadataPayload.data.streams, + }) + }) +}) + +describe('v2 access', () => { + it('should return status 401 for a podcast of another account', async () => { + const response = await request(baseURL) + .get('/analytics/v2/10/podcastPlays') + .set(auth) + .send() + expect(response.statusCode).toBe(401) + }) +}) diff --git a/watch-sql-queries.sh b/watch-sql-queries.sh index 75ad7b7d..ab663fed 100644 --- a/watch-sql-queries.sh +++ b/watch-sql-queries.sh @@ -23,8 +23,11 @@ DEFAULT_END_DATE=$(date -d "$(date +%Y-%m-01) -1 day" +%Y-%m-%d) START_DATE=${2:-$DEFAULT_START_DATE} END_DATE=${3:-$DEFAULT_END_DATE} +# Query version (v1, v2, ...), defaults to v1 +VERSION=${4:-v1} + # Directory to watch -WATCH_DIR="./db_schema/queries/v1/" +WATCH_DIR="./db_schema/queries/$VERSION/" echo "Watching directory: $WATCH_DIR" echo "Using podcast ID: $PODCAST_ID in $START_DATE to $END_DATE"