From 0b68814ed58be830d0967e8d28ae34f0e574ef30 Mon Sep 17 00:00:00 2001 From: James Ray Date: Wed, 16 Sep 2026 13:08:20 -0700 Subject: [PATCH] Cover the minutes between GOES imports The GOES import runs hourly and slices exactly one hour back from Time.now(). NOAA publishes each minute a few minutes late, so the newest point available when the job fires at :54:56 is around :49, and the next run's window starts at :54:56, after the minutes that arrived in the meantime. Nothing ever fetches them again. Timestream shows the hole: over 24 hours every minute of the hour has 23 or 24 samples except :50 with 13, :51 with 6, and :52 through :54 with none at all. That is roughly two hours of X-ray flux a day that never lands. Look back two hours instead. The feed already returns six hours on every call, so this costs no extra request. Timestream deduplicates identical records and record_timeseries logs a rejected duplicate rather than raising, so the overlap is free, and a run that fails now backfills itself on the next hour. --- lambda_function/src/executor/executor.py | 13 ++++++-- lambda_function/tests/test_executor.py | 40 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lambda_function/src/executor/executor.py b/lambda_function/src/executor/executor.py index 4a8dab4..e09ff10 100755 --- a/lambda_function/src/executor/executor.py +++ b/lambda_function/src/executor/executor.py @@ -405,7 +405,14 @@ def import_GOES_data_to_timestream() -> None: goes_json_data = pd.read_json( "https://services.swpc.noaa.gov/json/goes/primary/xrays-6-hour.json" ) - last_hour = Time.now() - TimeDelta(1 * u.hour) + # NOAA publishes each minute a few minutes late, and this job runs + # hourly, so a window of exactly one hour ends before the most + # recent minutes arrive and the next run starts after them. Look + # back two hours so every minute is covered by two runs. Timestream + # deduplicates identical records, and record_timeseries logs a + # rejected duplicate instead of raising, so the overlap is free and + # a missed run backfills itself. + lookback_start = Time.now() - TimeDelta(2 * u.hour) goes_short = goes_json_data[goes_json_data["energy"] == "0.05-0.4nm"] goes_long = goes_json_data[goes_json_data["energy"] == "0.1-0.8nm"] @@ -420,8 +427,8 @@ def import_GOES_data_to_timestream() -> None: time=time_tags, data={"xrsb": goes_long["flux"].values * u.W / u.m**2} ) - tsa_last = tsa.loc[last_hour : Time.now()] - tsb_last = tsb.loc[last_hour : Time.now()] + tsa_last = tsa.loc[lookback_start : Time.now()] + tsb_last = tsb.loc[lookback_start : Time.now()] if len(tsa_last) > 0: record_timeseries(tsa_last, ts_name="GOES", instrument_name="goes xrsa") diff --git a/lambda_function/tests/test_executor.py b/lambda_function/tests/test_executor.py index b0da1a5..88222c4 100755 --- a/lambda_function/tests/test_executor.py +++ b/lambda_function/tests/test_executor.py @@ -5,6 +5,7 @@ import boto3 import numpy as np +import pandas as pd import pytest from astropy.time import Time, TimeDelta from src.executor.executor import Executor @@ -220,6 +221,45 @@ def fake_record_timeseries(*args, **kwargs): } +def test_import_GOES_data_covers_the_minutes_between_runs(monkeypatch) -> None: + """The import window reaches back far enough to cover the previous run's tail. + + The job runs hourly and NOAA publishes each minute a few minutes late, so a + window of exactly one hour drops the minutes that arrive after a run starts: + the next run's window begins after them and nothing fetches them again. + """ + recorded_calls = [] + + def fake_record_timeseries(*args, **kwargs): + recorded_calls.append((args, kwargs)) + + now = Time.now() + # A minute that the previous hourly run would have been too early to see. + stale = now - TimeDelta(70 * 60, format="sec") + fresh = now - TimeDelta(5 * 60, format="sec") + + rows = [] + for stamp in (stale, fresh): + tag = stamp.isot[:19] + "Z" + rows.append({"time_tag": tag, "energy": "0.05-0.4nm", "flux": 1e-7}) + rows.append({"time_tag": tag, "energy": "0.1-0.8nm", "flux": 2e-7}) + + monkeypatch.setattr( + "src.executor.executor.pd.read_json", lambda *a, **k: pd.DataFrame(rows) + ) + monkeypatch.setattr( + "src.executor.executor.record_timeseries", fake_record_timeseries + ) + + Executor.import_GOES_data_to_timestream() + + assert len(recorded_calls) == 2, "expected one xrsa and one xrsb record call" + recorded = recorded_calls[0][0][0] + assert len(recorded) == 2, ( + "the window dropped the minute published after the previous run started" + ) + + def test_create_GOES_data_annotations(monkeypatch) -> None: """GOES annotations uses live flare feed and only mocks annotation writes.""" annotation_calls = []