From c1b1a914f6952ba67e7a90c17d34f8d15a3c455e Mon Sep 17 00:00:00 2001 From: LeeGordon83 Date: Fri, 12 Jun 2026 08:46:21 +0100 Subject: [PATCH 1/2] altered time scale to delete river level data to 3 years --- lib/constructed-queries.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/constructed-queries.js b/lib/constructed-queries.js index 317dddf0..6f7a75c4 100644 --- a/lib/constructed-queries.js +++ b/lib/constructed-queries.js @@ -32,7 +32,7 @@ module.exports = { const sql = ` DELETE FROM u_flood.sls_telemetry_value_parent - WHERE imported < to_timestamp(EXTRACT(EPOCH FROM now()) - 432000) at time zone 'utc'; + WHERE imported < to_timestamp(EXTRACT(EPOCH FROM now()) - 94608000) at time zone 'utc'; ` return Query.fromString(sql) }, From d32145839f799f45f36505bed01ed13656d23698 Mon Sep 17 00:00:00 2001 From: LeeGordon83 Date: Thu, 8 Oct 2026 16:50:29 +0100 Subject: [PATCH 2/2] added lambda to add historical telemetry data --- lib/constructed-queries.js | 6 +++- lib/functions/telemetry-history.js | 10 +++++++ readme.md | 5 ++++ test/unit/constructed-queries.js | 13 +++++++- test/unit/functions/telemetry-history.js | 38 ++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 lib/functions/telemetry-history.js create mode 100644 test/unit/functions/telemetry-history.js diff --git a/lib/constructed-queries.js b/lib/constructed-queries.js index 6f7a75c4..2bff6a3c 100644 --- a/lib/constructed-queries.js +++ b/lib/constructed-queries.js @@ -32,10 +32,14 @@ module.exports = { const sql = ` DELETE FROM u_flood.sls_telemetry_value_parent - WHERE imported < to_timestamp(EXTRACT(EPOCH FROM now()) - 94608000) at time zone 'utc'; + WHERE imported < to_timestamp(EXTRACT(EPOCH FROM now()) - 432000) at time zone 'utc'; ` return Query.fromString(sql) }, + updateTelemetryHistory: () => { + const sql = 'SELECT u_flood.update_telemetry_history();' + return Query.fromString(sql) + }, refreshStationMviews: () => { const sql = ` REFRESH MATERIALIZED VIEW CONCURRENTLY u_flood.telemetry_context_mview WITH DATA; diff --git a/lib/functions/telemetry-history.js b/lib/functions/telemetry-history.js new file mode 100644 index 00000000..1220449d --- /dev/null +++ b/lib/functions/telemetry-history.js @@ -0,0 +1,10 @@ +const { Pool } = require('../helpers/pool') + +module.exports.handler = async () => { + const pool = new Pool({ connectionString: process.env.LFW_DATA_DB_CONNECTION }) + try { + await pool.query('updateTelemetryHistory') + } finally { + await pool.end() + } +} diff --git a/readme.md b/readme.md index 0bc174ac..7be9a3f0 100644 --- a/readme.md +++ b/readme.md @@ -32,6 +32,10 @@ brings in the observed telemtery data from the S3 bucket and processes it into t refreshes the materialized views and clears out the old telemetry. +### telemetry-history + +Runs `u_flood.update_telemetry_history()` to maintain historical telemetry. Its schedule is configured separately in Terraform and should run at every day but at a minimum must run at least once every 5 days. This function will also clear any historical data over 3 years old. + ### station-process Uses the exports context file from sharepoint to update the stations in the database. @@ -91,6 +95,7 @@ Cron configurations (e.g., thresholds, data syncs) are managed per environment v export LFW_DATA_IMTD_THRESHOLD_SCHEDULE= export LFW_DATA_DTS_SCHEDULE= export LFW_DATA_FWIS_SCHEDULE= +export LFW_DATA_TELEMETRY_HISTORY_SCHEDULE= ``` These are defined in the Terraform modules and should be updated through the infrastructure as code process. diff --git a/test/unit/constructed-queries.js b/test/unit/constructed-queries.js index 67d45892..8fee1c93 100644 --- a/test/unit/constructed-queries.js +++ b/test/unit/constructed-queries.js @@ -1,7 +1,7 @@ const { expect } = require('@hapi/code') const Lab = require('@hapi/lab') const { - slsTelemetryValues, slsTelemetryStation, slsTelemetryValueParent, deleteOldTelemetry, refreshStationMviews, + slsTelemetryValues, slsTelemetryStation, slsTelemetryValueParent, deleteOldTelemetry, updateTelemetryHistory, refreshStationMviews, deleteStations, insertStations, upsertFfoiMax, deleteCurrentFwis, insertFloodWarnings, refreshFloodWarningsMview, updateTimestamp } = require('../../lib/constructed-queries') @@ -101,6 +101,17 @@ describe('constructedQueries', () => { }) }) + describe('updateTelemetryHistory', () => { + it('returns the history update sql in a query object', () => { + // Act + const actual = updateTelemetryHistory() + + // Assert + expect(actual.text).to.equal('SELECT u_flood.update_telemetry_history();') + expect(actual.values).to.be.undefined() + }) + }) + describe('refreshStationMviews', () => { it('returns appropriate sql in a query object', () => { // Act diff --git a/test/unit/functions/telemetry-history.js b/test/unit/functions/telemetry-history.js new file mode 100644 index 00000000..ae067946 --- /dev/null +++ b/test/unit/functions/telemetry-history.js @@ -0,0 +1,38 @@ +const Lab = require('@hapi/lab') +const lab = exports.lab = Lab.script() +const Code = require('@hapi/code') +const sinon = require('sinon') +const proxyquire = require('proxyquire') + +lab.experiment('telemetry history', () => { + lab.test('runs the history update query before ending the pool', async () => { + const pool = { + query: sinon.stub().resolves(), + end: sinon.stub().resolves() + } + const Pool = sinon.stub().returns(pool) + const { handler } = proxyquire('../../../lib/functions/telemetry-history', { + '../helpers/pool': { Pool } + }) + + await handler() + + sinon.assert.calledOnceWithExactly(pool.query, 'updateTelemetryHistory') + sinon.assert.callOrder(pool.query, pool.end) + }) + + lab.test('ends the pool and propagates a history update failure', async () => { + const error = new Error('history update failed') + const pool = { + query: sinon.stub().rejects(error), + end: sinon.stub().resolves() + } + const Pool = sinon.stub().returns(pool) + const { handler } = proxyquire('../../../lib/functions/telemetry-history', { + '../helpers/pool': { Pool } + }) + + await Code.expect(handler()).to.reject() + sinon.assert.calledOnce(pool.end) + }) +})