diff --git a/lambda_function/src/executor/executor.py b/lambda_function/src/executor/executor.py index 4a8dab4..e09ff10 100755 --- a/lambda_function/src/executor/executor.py +++ b/lambda_function/src/executor/executor.py @@ -405,7 +405,14 @@ def import_GOES_data_to_timestream() -> None: goes_json_data = pd.read_json( "https://services.swpc.noaa.gov/json/goes/primary/xrays-6-hour.json" ) - last_hour = Time.now() - TimeDelta(1 * u.hour) + # NOAA publishes each minute a few minutes late, and this job runs + # hourly, so a window of exactly one hour ends before the most + # recent minutes arrive and the next run starts after them. Look + # back two hours so every minute is covered by two runs. Timestream + # deduplicates identical records, and record_timeseries logs a + # rejected duplicate instead of raising, so the overlap is free and + # a missed run backfills itself. + lookback_start = Time.now() - TimeDelta(2 * u.hour) goes_short = goes_json_data[goes_json_data["energy"] == "0.05-0.4nm"] goes_long = goes_json_data[goes_json_data["energy"] == "0.1-0.8nm"] @@ -420,8 +427,8 @@ def import_GOES_data_to_timestream() -> None: time=time_tags, data={"xrsb": goes_long["flux"].values * u.W / u.m**2} ) - tsa_last = tsa.loc[last_hour : Time.now()] - tsb_last = tsb.loc[last_hour : Time.now()] + tsa_last = tsa.loc[lookback_start : Time.now()] + tsb_last = tsb.loc[lookback_start : Time.now()] if len(tsa_last) > 0: record_timeseries(tsa_last, ts_name="GOES", instrument_name="goes xrsa") diff --git a/lambda_function/tests/test_executor.py b/lambda_function/tests/test_executor.py index b0da1a5..88222c4 100755 --- a/lambda_function/tests/test_executor.py +++ b/lambda_function/tests/test_executor.py @@ -5,6 +5,7 @@ import boto3 import numpy as np +import pandas as pd import pytest from astropy.time import Time, TimeDelta from src.executor.executor import Executor @@ -220,6 +221,45 @@ def fake_record_timeseries(*args, **kwargs): } +def test_import_GOES_data_covers_the_minutes_between_runs(monkeypatch) -> None: + """The import window reaches back far enough to cover the previous run's tail. + + The job runs hourly and NOAA publishes each minute a few minutes late, so a + window of exactly one hour drops the minutes that arrive after a run starts: + the next run's window begins after them and nothing fetches them again. + """ + recorded_calls = [] + + def fake_record_timeseries(*args, **kwargs): + recorded_calls.append((args, kwargs)) + + now = Time.now() + # A minute that the previous hourly run would have been too early to see. + stale = now - TimeDelta(70 * 60, format="sec") + fresh = now - TimeDelta(5 * 60, format="sec") + + rows = [] + for stamp in (stale, fresh): + tag = stamp.isot[:19] + "Z" + rows.append({"time_tag": tag, "energy": "0.05-0.4nm", "flux": 1e-7}) + rows.append({"time_tag": tag, "energy": "0.1-0.8nm", "flux": 2e-7}) + + monkeypatch.setattr( + "src.executor.executor.pd.read_json", lambda *a, **k: pd.DataFrame(rows) + ) + monkeypatch.setattr( + "src.executor.executor.record_timeseries", fake_record_timeseries + ) + + Executor.import_GOES_data_to_timestream() + + assert len(recorded_calls) == 2, "expected one xrsa and one xrsb record call" + recorded = recorded_calls[0][0][0] + assert len(recorded) == 2, ( + "the window dropped the minute published after the previous run started" + ) + + def test_create_GOES_data_annotations(monkeypatch) -> None: """GOES annotations uses live flare feed and only mocks annotation writes.""" annotation_calls = []