Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions lambda_function/src/executor/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,14 @@ def import_GOES_data_to_timestream() -> None:
goes_json_data = pd.read_json(
"https://services.swpc.noaa.gov/json/goes/primary/xrays-6-hour.json"
)
last_hour = Time.now() - TimeDelta(1 * u.hour)
# NOAA publishes each minute a few minutes late, and this job runs
# hourly, so a window of exactly one hour ends before the most
# recent minutes arrive and the next run starts after them. Look
# back two hours so every minute is covered by two runs. Timestream
# deduplicates identical records, and record_timeseries logs a
# rejected duplicate instead of raising, so the overlap is free and
# a missed run backfills itself.
lookback_start = Time.now() - TimeDelta(2 * u.hour)

goes_short = goes_json_data[goes_json_data["energy"] == "0.05-0.4nm"]
goes_long = goes_json_data[goes_json_data["energy"] == "0.1-0.8nm"]
Expand All @@ -420,8 +427,8 @@ def import_GOES_data_to_timestream() -> None:
time=time_tags, data={"xrsb": goes_long["flux"].values * u.W / u.m**2}
)

tsa_last = tsa.loc[last_hour : Time.now()]
tsb_last = tsb.loc[last_hour : Time.now()]
tsa_last = tsa.loc[lookback_start : Time.now()]
tsb_last = tsb.loc[lookback_start : Time.now()]

if len(tsa_last) > 0:
record_timeseries(tsa_last, ts_name="GOES", instrument_name="goes xrsa")
Expand Down
40 changes: 40 additions & 0 deletions lambda_function/tests/test_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import boto3
import numpy as np
import pandas as pd
import pytest
from astropy.time import Time, TimeDelta
from src.executor.executor import Executor
Expand Down Expand Up @@ -220,6 +221,45 @@ def fake_record_timeseries(*args, **kwargs):
}


def test_import_GOES_data_covers_the_minutes_between_runs(monkeypatch) -> None:
"""The import window reaches back far enough to cover the previous run's tail.

The job runs hourly and NOAA publishes each minute a few minutes late, so a
window of exactly one hour drops the minutes that arrive after a run starts:
the next run's window begins after them and nothing fetches them again.
"""
recorded_calls = []

def fake_record_timeseries(*args, **kwargs):
recorded_calls.append((args, kwargs))

now = Time.now()
# A minute that the previous hourly run would have been too early to see.
stale = now - TimeDelta(70 * 60, format="sec")
fresh = now - TimeDelta(5 * 60, format="sec")

rows = []
for stamp in (stale, fresh):
tag = stamp.isot[:19] + "Z"
rows.append({"time_tag": tag, "energy": "0.05-0.4nm", "flux": 1e-7})
rows.append({"time_tag": tag, "energy": "0.1-0.8nm", "flux": 2e-7})

monkeypatch.setattr(
"src.executor.executor.pd.read_json", lambda *a, **k: pd.DataFrame(rows)
)
monkeypatch.setattr(
"src.executor.executor.record_timeseries", fake_record_timeseries
)

Executor.import_GOES_data_to_timestream()

assert len(recorded_calls) == 2, "expected one xrsa and one xrsb record call"
recorded = recorded_calls[0][0][0]
assert len(recorded) == 2, (
"the window dropped the minute published after the previous run started"
)


def test_create_GOES_data_annotations(monkeypatch) -> None:
"""GOES annotations uses live flare feed and only mocks annotation writes."""
annotation_calls = []
Expand Down
Loading