diff --git a/docroot/schema/image_layer.schema.json b/docroot/schema/image_layer.schema.json index ec78ea3f..07b99550 100644 --- a/docroot/schema/image_layer.schema.json +++ b/docroot/schema/image_layer.schema.json @@ -19,15 +19,34 @@ }, "opacity": { "oneOf": [ - { "type": "number", "minimum": 0, "maximum": 100 }, - { "type": "string", "pattern": "^\\d{0,3}$" } + { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + { + "type": "string", + "pattern": "^\\d{0,3}$" + } ] }, "visible": { "oneOf": [ - {"type": "boolean" }, - {"type": "integer", "minimum": 0, "maximum": 1}, - {"type": "string", "enum": ["1", "0"]} + { + "type": "boolean" + }, + { + "type": "integer", + "minimum": 0, + "maximum": 1 + }, + { + "type": "string", + "enum": [ + "1", + "0" + ] + } ] }, "diffTime": { @@ -38,11 +57,180 @@ "nickname": { "type": "string", "enum": [ - "GOES-R SUVI 94","GOES-R SUVI 131","GOES-R SUVI 171","GOES-R SUVI 195","GOES-R SUVI 284","GOES-R SUVI 304","EIT 171","EIT 195","EIT 284","EIT 304","LASCO C2","LASCO C3","MDI Mag","MDI Int","AIA 94","AIA 131","AIA 171","AIA 193","AIA 211","AIA 304","AIA 335","AIA 1600","AIA 1700","AIA 4500","HMI Int","HMI Mag","EUVI-A 171","EUVI-A 195","EUVI-A 284","EUVI-A 304","EUVI-B 171","EUVI-B 195","EUVI-B 284","EUVI-B 304","COR1-A","COR2-A","COR1-B","COR2-B","SWAP 174","SXT AlMgMn","SXT thin-Al","SXT white-light","XRT Al_med/Al_mesh","XRT Al_med/Al_thick","XRT Al_med/Be_thick","XRT Al_med/Gband","XRT Al_med/Open","XRT Al_med/Ti_poly","XRT Al_poly/Al_mesh","XRT Al_poly/Al_thick","XRT Al_poly/Be_thick","XRT Al_poly/Gband","XRT Al_poly/Open","XRT Al_poly/Ti_poly","XRT Be_med/Al_mesh","XRT Be_med/Al_thick","XRT Be_med/Be_thick","XRT Be_med/Gband","XRT Be_med/Open","XRT Be_med/Ti_poly","XRT Be_thin/Al_mesh","XRT Be_thin/Al_thick","XRT Be_thin/Be_thick","XRT Be_thin/Gband","XRT Be_thin/Open","XRT Be_thin/Ti_poly","XRT C_poly/Al_mesh","XRT C_poly/Al_thick","XRT C_poly/Be_thick","XRT C_poly/Gband","XRT C_poly/Open","XRT C_poly/Ti_poly","XRT Mispositioned/Mispositioned","XRT Open/Al_mesh","XRT Open/Al_thick","XRT Open/Be_thick","XRT Open/Gband","XRT Open/Open","XRT Open/Ti_poly","TRACE 171","TRACE 195","TRACE 284","TRACE 1216","TRACE 1550","TRACE 1600","TRACE 1700","TRACE white-light","COSMO KCor","EUI FSI 174","EUI FSI 304","EUI HRI 174","EUI HRI 1216","IRIS SJI 1330","IRIS SJI 2796","IRIS SJI 1400","IRIS SJI 1600","IRIS SJI 2832","IRIS SJI 5000","GONG H-alpha","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","PUNCH","GOES","SWFO-L1","PUNCH","GOES-R SUVI 94","GOES-R SUVI 131","GOES-R SUVI 171","GOES-R SUVI 195","GOES-R SUVI 284","GOES-R SUVI 304","XRT Any/Any","XRT Any/Al_mesh","XRT Any/Al_thick","XRT Any/Be_thick","XRT Any/Gband","XRT Any/Open","XRT Any/Ti_poly","XRT Al_med/Any","XRT Al_poly/Any","XRT Be_med/Any","XRT Be_thin/Any","XRT C_poly/Any","XRT Open/Any" + "GOES-R SUVI 94", + "GOES-R SUVI 131", + "GOES-R SUVI 171", + "GOES-R SUVI 195", + "GOES-R SUVI 284", + "GOES-R SUVI 304", + "EIT 171", + "EIT 195", + "EIT 284", + "EIT 304", + "LASCO C2", + "LASCO C3", + "MDI Mag", + "MDI Int", + "AIA 94", + "AIA 131", + "AIA 171", + "AIA 193", + "AIA 211", + "AIA 304", + "AIA 335", + "AIA 1600", + "AIA 1700", + "AIA 4500", + "HMI Int", + "HMI Mag", + "EUVI-A 171", + "EUVI-A 195", + "EUVI-A 284", + "EUVI-A 304", + "EUVI-B 171", + "EUVI-B 195", + "EUVI-B 284", + "EUVI-B 304", + "COR1-A", + "COR2-A", + "COR1-B", + "COR2-B", + "SWAP 174", + "SXT AlMgMn", + "SXT thin-Al", + "SXT white-light", + "XRT Al_med/Al_mesh", + "XRT Al_med/Al_thick", + "XRT Al_med/Be_thick", + "XRT Al_med/Gband", + "XRT Al_med/Open", + "XRT Al_med/Ti_poly", + "XRT Al_poly/Al_mesh", + "XRT Al_poly/Al_thick", + "XRT Al_poly/Be_thick", + "XRT Al_poly/Gband", + "XRT Al_poly/Open", + "XRT Al_poly/Ti_poly", + "XRT Be_med/Al_mesh", + "XRT Be_med/Al_thick", + "XRT Be_med/Be_thick", + "XRT Be_med/Gband", + "XRT Be_med/Open", + "XRT Be_med/Ti_poly", + "XRT Be_thin/Al_mesh", + "XRT Be_thin/Al_thick", + "XRT Be_thin/Be_thick", + "XRT Be_thin/Gband", + "XRT Be_thin/Open", + "XRT Be_thin/Ti_poly", + "XRT C_poly/Al_mesh", + "XRT C_poly/Al_thick", + "XRT C_poly/Be_thick", + "XRT C_poly/Gband", + "XRT C_poly/Open", + "XRT C_poly/Ti_poly", + "XRT Mispositioned/Mispositioned", + "XRT Open/Al_mesh", + "XRT Open/Al_thick", + "XRT Open/Be_thick", + "XRT Open/Gband", + "XRT Open/Open", + "XRT Open/Ti_poly", + "TRACE 171", + "TRACE 195", + "TRACE 284", + "TRACE 1216", + "TRACE 1550", + "TRACE 1600", + "TRACE 1700", + "TRACE white-light", + "COSMO KCor", + "EUI FSI 174", + "EUI FSI 304", + "EUI HRI 174", + "EUI HRI 1216", + "IRIS SJI 1330", + "IRIS SJI 2796", + "IRIS SJI 1400", + "IRIS SJI 1600", + "IRIS SJI 2832", + "IRIS SJI 5000", + "GONG H-alpha", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "RHESSI", + "PUNCH", + "GOES", + "SWFO-L1", + "PUNCH", + "GOES-R SUVI 94", + "GOES-R SUVI 131", + "GOES-R SUVI 171", + "GOES-R SUVI 195", + "GOES-R SUVI 284", + "GOES-R SUVI 304", + "XRT Any/Any", + "XRT Any/Al_mesh", + "XRT Any/Al_thick", + "XRT Any/Be_thick", + "XRT Any/Gband", + "XRT Any/Open", + "XRT Any/Ti_poly", + "XRT Al_med/Any", + "XRT Al_poly/Any", + "XRT Be_med/Any", + "XRT Be_thin/Any", + "XRT C_poly/Any", + "XRT Open/Any", + "SPICE 77.04", + "SPICE 103.19", + "SPICE 70.38", + "SPICE 78.77", + "SPICE 70.6", + "SPICE 97.25", + "SPICE 76.51", + "SPICE 97.7" ] }, - "sourceId": { "type": "integer" }, - "layeringOrder": { "type": "integer" }, + "sourceId": { + "type": "integer" + }, + "layeringOrder": { + "type": "integer" + }, "uiLabels": { "type": "array", "items": { @@ -52,21 +240,139 @@ "name": { "type": "string", "enum": [ - "", "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","6-12keV","6562","735","94","AIA","Al_med","Al_mesh","Al_poly","Al_thick","AlMgMn","Any","Back Projection","Be_med","Be_thick","Be_thin","C_poly","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","Polarized Brightness","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","Total Brightness","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh" + "", + "GOES-R", + "100-300keV", + "12-25keV", + "1216", + "131", + "1330", + "1400", + "1550", + "1600", + "1700", + "171", + "174", + "193", + "195", + "211", + "25-50keV", + "2796", + "2832", + "284", + "3-6keV", + "304", + "335", + "4500", + "50-100keV", + "5000", + "6-12keV", + "6562", + "735", + "94", + "AIA", + "Al_med", + "Al_mesh", + "Al_poly", + "Al_thick", + "AlMgMn", + "Any", + "Back Projection", + "Be_med", + "Be_thick", + "Be_thin", + "C_poly", + "C2", + "C3", + "CCOR-1", + "CCOR-2", + "Clean", + "Clean 59", + "continuum", + "COR1", + "COR2", + "COSMO", + "EIT", + "EUI", + "EUVI", + "FSI", + "Gband", + "GOES", + "GONG", + "H-alpha", + "Hinode", + "HMI", + "HRI", + "IRIS", + "KCor", + "LASCO", + "magnetogram", + "MDI", + "MEM GE", + "Mispositioned", + "MLSO", + "Open", + "Polarized Brightness", + "PROBA2", + "PUNCH", + "RHESSI", + "SDO", + "SECCHI", + "SJI", + "SOHO", + "SOLO", + "STEREO_A", + "STEREO_B", + "SUVI", + "SWAP", + "SWFO-L1", + "SXT", + "thin-Al", + "Ti_poly", + "Total Brightness", + "TRACE", + "VIS CS", + "VIS FWDFIT", + "WFI+NFI", + "white-light", + "XRT", + "Yohkoh", + "SPICE", + "intensity", + "Ne VIII 77.04 nm", + "O VI 103.19 nm", + "O III 70.38 nm", + "O IV 78.77 nm", + "Mg IX 70.60 nm", + "H Ly gamma 97.25 nm", + "N IV 76.51 nm", + "C III 97.70 nm" ], "$comment": "Autogenerated from available sources" }, "label": { "type": "string", "enum": [ - "","Detector","Energy Band","Filter","Filter Wheel 1","Filter Wheel 2","Instrument","Measurement","Observatory","Reconstruction" + "", + "Detector", + "Energy Band", + "Filter", + "Filter Wheel 1", + "Filter Wheel 2", + "Instrument", + "Measurement", + "Observatory", + "Reconstruction", + "Line" ], "$comment": "Autogenerated from available sources" } } } }, - "diffCount": { "type": "number" }, + "diffCount": { + "type": "number" + }, "difference": { "type": "integer", "minimum": 0, @@ -78,12 +384,116 @@ } }, "patternProperties": { - "^(Detector|Energy Band|Filter|Filter Wheel 1|Filter Wheel 2|Instrument|Measurement|Observatory|Reconstruction)$": { + "^(Detector|Energy Band|Filter|Filter Wheel 1|Filter Wheel 2|Instrument|Line|Measurement|Observatory|Reconstruction)$": { "type": "string", "enum": [ - "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","6-12keV","6562","735","94","AIA","Al_med","Al_mesh","Al_poly","Al_thick","AlMgMn","Any","Back Projection","Be_med","Be_thick","Be_thin","C_poly","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","Polarized Brightness","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","Total Brightness","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh" + "GOES-R", + "100-300keV", + "12-25keV", + "1216", + "131", + "1330", + "1400", + "1550", + "1600", + "1700", + "171", + "174", + "193", + "195", + "211", + "25-50keV", + "2796", + "2832", + "284", + "3-6keV", + "304", + "335", + "4500", + "50-100keV", + "5000", + "6-12keV", + "6562", + "735", + "94", + "AIA", + "Al_med", + "Al_mesh", + "Al_poly", + "Al_thick", + "AlMgMn", + "Any", + "Back Projection", + "Be_med", + "Be_thick", + "Be_thin", + "C_poly", + "C2", + "C3", + "CCOR-1", + "CCOR-2", + "Clean", + "Clean 59", + "continuum", + "COR1", + "COR2", + "COSMO", + "EIT", + "EUI", + "EUVI", + "FSI", + "Gband", + "GOES", + "GONG", + "H-alpha", + "Hinode", + "HMI", + "HRI", + "IRIS", + "KCor", + "LASCO", + "magnetogram", + "MDI", + "MEM GE", + "Mispositioned", + "MLSO", + "Open", + "Polarized Brightness", + "PROBA2", + "PUNCH", + "RHESSI", + "SDO", + "SECCHI", + "SJI", + "SOHO", + "SOLO", + "STEREO_A", + "STEREO_B", + "SUVI", + "SWAP", + "SWFO-L1", + "SXT", + "thin-Al", + "Ti_poly", + "Total Brightness", + "TRACE", + "VIS CS", + "VIS FWDFIT", + "WFI+NFI", + "white-light", + "XRT", + "Yohkoh", + "SPICE", + "intensity", + "Ne VIII 77.04 nm", + "O VI 103.19 nm", + "O III 70.38 nm", + "O IV 78.77 nm", + "Mg IX 70.60 nm", + "H Ly gamma 97.25 nm", + "N IV 76.51 nm", + "C III 97.70 nm" ] } } -} - +} \ No newline at end of file diff --git a/docs/src/source/appendix/data_sources/solo.rst b/docs/src/source/appendix/data_sources/solo.rst index e8d74676..c6b980f9 100644 --- a/docs/src/source/appendix/data_sources/solo.rst +++ b/docs/src/source/appendix/data_sources/solo.rst @@ -3,14 +3,40 @@ Solar Orbiter .. table:: Solar Orbiter Data Sources - +-----------+-------------+----------------+------------+-------------+ - | Source ID | Description | Observatory | Instrument | Measurement | - +===========+=============+================+============+=============+ - | 84 | EUI FSI 174 | Solar Orbiter | EUI | 174 | - +-----------+-------------+----------------+------------+-------------+ - | 85 | EUI FSI 304 | Solar Orbiter | EUI | 304 | - +-----------+-------------+----------------+------------+-------------+ - | 86 | EUI HRI 174 | Solar Orbiter | EUI | 174 | - +-----------+-------------+----------------+------------+-------------+ - | 87 | EUI HRI 1216| Solar Orbiter | EUI | 1216 | - +-----------+-------------+----------------+------------+-------------+ + +-----------+----------------+----------------+------------+-----------+---------------------+ + | Source ID | Description | Observatory | Instrument | Measurement| Line | + +===========+================+================+============+===========+=====================+ + | 84 | EUI FSI 174 | Solar Orbiter | EUI | 174 | | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 85 | EUI FSI 304 | Solar Orbiter | EUI | 304 | | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 86 | EUI HRI 174 | Solar Orbiter | EUI | 174 | | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 87 | EUI HRI 1216 | Solar Orbiter | EUI | 1216 | | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 601 | SPICE 77.04 | Solar Orbiter | SPICE | intensity | Ne VIII 77.04 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 602 | SPICE 103.19 | Solar Orbiter | SPICE | intensity | O VI 103.19 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 603 | SPICE 70.38 | Solar Orbiter | SPICE | intensity | O III 70.38 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 604 | SPICE 78.77 | Solar Orbiter | SPICE | intensity | O IV 78.77 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 605 | SPICE 70.6 | Solar Orbiter | SPICE | intensity | Mg IX 70.60 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 606 | SPICE 97.25 | Solar Orbiter | SPICE | intensity | H Ly gamma 97.25 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 607 | SPICE 76.51 | Solar Orbiter | SPICE | intensity | N IV 76.51 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + | 608 | SPICE 97.7 | Solar Orbiter | SPICE | intensity | C III 97.70 nm | + +-----------+----------------+----------------+------------+-----------+---------------------+ + +SPICE Import +^^^^^^^^^^^^ + +The SPICE downloader reads JP2 files from the directory configured by +``SPICE_DATA_PATH``. The expected layout under that root is ``YYYY/MM/DD``. + +Example:: + + SPICE_DATA_PATH=/path/to/spice/jp2 python install/downloader.py -b local -m localmove -d spice diff --git a/install/__test__/test_spice.py b/install/__test__/test_spice.py new file mode 100644 index 00000000..e2c9fefe --- /dev/null +++ b/install/__test__/test_spice.py @@ -0,0 +1,127 @@ +import datetime +import os + +import pytest + +from helioviewer.hvpull.net.daemon import ImageRetrievalDaemon +from helioviewer.hvpull.servers.spice import SpiceDataServer +from helioviewer.jp2parser import JP2parser + + +class FakeImageData: + observatory = "Solar Orbiter" + instrument = "SPICE" + detector = "SW" + nickname = "SPICE" + measurement = "103.19" + meta = {"cmpnam": "O VI 103.19 nm"} + date = datetime.datetime(2023, 12, 2, 6, 19, 22) + + +def test_spice_data_path_required(monkeypatch): + monkeypatch.delenv("SPICE_DATA_PATH", raising=False) + + with pytest.raises(ValueError, match="SPICE_DATA_PATH"): + SpiceDataServer() + + +def test_spice_data_path_defined(monkeypatch): + monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2") + + spice = SpiceDataServer() + + assert spice.uri == "/data/spice/jp2" + assert spice.name == "SPICE" + + +def test_compute_directories(monkeypatch): + monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2") + spice = SpiceDataServer() + + dirs = spice.compute_directories( + datetime.datetime(2023, 12, 1), + datetime.datetime(2023, 12, 3) + ) + + assert dirs == [ + "/data/spice/jp2/2023/12/03", + "/data/spice/jp2/2023/12/02", + "/data/spice/jp2/2023/12/01", + ] + + +def test_get_datetime_from_file(monkeypatch): + monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2") + spice = SpiceDataServer() + + assert spice.get_datetime_from_file( + "solo_L4_spice-n-ras_20231202T061922_218104664-000_0_3.jp2" + ) == datetime.datetime(2023, 12, 2, 6, 19, 22) + assert spice.get_datetime_from_file( + "solo_L4_spice-n-ras_20231202T181921_218104668-000_5_0.jp2" + ) == datetime.datetime(2023, 12, 2, 18, 19, 21) + + +def test_spice_registered_in_daemon(): + assert ImageRetrievalDaemon.get_servers()["spice"] == "SpiceDataServer" + + +def test_spice_loads_from_spice_module(): + daemon = ImageRetrievalDaemon.__new__(ImageRetrievalDaemon) + + cls = daemon._load_class( + "helioviewer.hvpull.servers", + "spice", + ImageRetrievalDaemon.get_servers()["spice"] + ) + + assert cls is SpiceDataServer + + +def test_spice_detection_keys_ignore_detector(): + img = { + "observatory": "Solar_Orbiter", + "instrument": "SPICE", + "detector": "SW", + "measurement": "intensity", + "line": "O VI 103.19 nm", + } + + assert JP2parser.get_detection_keys(img) == [ + "observatory", + "instrument", + "measurement", + "line", + ] + + +def test_spice_metadata_extraction(monkeypatch): + parser = JP2parser.__new__(JP2parser) + parser._filepath = "solo_L4_spice-n-ras_20231202T061922_218104664-000_0_3.jp2" + parser._data = {} + + monkeypatch.setattr(parser, "getImageMap", lambda: FakeImageData()) + monkeypatch.setattr(parser, "getImageDimensions", lambda: {"width": 1024, "height": 1024}) + monkeypatch.setattr(parser, "getRefPixelCoords", lambda: {"x": 512, "y": 512}) + monkeypatch.setattr(parser, "getImagePlateScale", lambda: 1) + monkeypatch.setattr(parser, "getDSun", lambda: 149597870700) + monkeypatch.setattr(parser, "getLayeringOrder", lambda: 1) + monkeypatch.setattr(parser, "_get_date", lambda image_data: image_data.date) + monkeypatch.setattr(parser, "get_storage_suffix", lambda image: os.path.join(image["nickname"], "2023/12/02", image["measurement"])) + + image = parser.getData() + + assert image["observatory"] == "Solar_Orbiter" + assert image["instrument"] == "SPICE" + assert image["detector"] == "SW" + assert image["measurement"] == "intensity" + assert image["line"] == "O VI 103.19 nm" + + +def test_spice_missing_cmpnam_message(): + parser = JP2parser.__new__(JP2parser) + image_data = FakeImageData() + image_data.meta = {} + + with pytest.raises(ValueError, match="SPICE JP2 missing required CMPNAM metadata"): + parser._get_spice_line(image_data) diff --git a/install/database/2026_07_23_spice_source.sql b/install/database/2026_07_23_spice_source.sql new file mode 100644 index 00000000..69deae14 --- /dev/null +++ b/install/database/2026_07_23_spice_source.sql @@ -0,0 +1,68 @@ +INSERT INTO datasources ( + id, + name, + description, + units, + layeringOrder, + enabled, + sourceIdGroup, + displayOrder +) +VALUES + (601, 'SPICE 77.04', 'Solar Orbiter SPICE 77.04', 'NM', 1, 0, '', 0), + (602, 'SPICE 103.19', 'Solar Orbiter SPICE 103.19', 'NM', 1, 0, '', 0), + (603, 'SPICE 70.38', 'Solar Orbiter SPICE 70.38', 'NM', 1, 0, '', 0), + (604, 'SPICE 78.77', 'Solar Orbiter SPICE 78.77', 'NM', 1, 0, '', 0), + (605, 'SPICE 70.6', 'Solar Orbiter SPICE 70.6', 'NM', 1, 0, '', 0), + (606, 'SPICE 97.25', 'Solar Orbiter SPICE 97.25', 'NM', 1, 0, '', 0), + (607, 'SPICE 76.51', 'Solar Orbiter SPICE 76.51', 'NM', 1, 0, '', 0), + (608, 'SPICE 97.7', 'Solar Orbiter SPICE 97.7', 'NM', 1, 0, '', 0); + +INSERT INTO datasource_property ( + sourceId, + label, + name, + fitsName, + description, + uiOrder +) +VALUES + (601, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (601, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (601, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (601, 'Line', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 4), + + (602, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (602, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (602, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (602, 'Line', 'O VI 103.19 nm', 'O VI 103.19 nm', 'O VI 103.19 nm', 4), + + (603, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (603, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (603, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (603, 'Line', 'O III 70.38 nm', 'O III 70.38 nm', 'O III 70.38 nm', 4), + + (604, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (604, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (604, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (604, 'Line', 'O IV 78.77 nm', 'O IV 78.77 nm', 'O IV 78.77 nm', 4), + + (605, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (605, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (605, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (605, 'Line', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 4), + + (606, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (606, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (606, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (606, 'Line', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 4), + + (607, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (607, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (607, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (607, 'Line', 'N IV 76.51 nm', 'N IV 76.51 nm', 'N IV 76.51 nm', 4), + + (608, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), + (608, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), + (608, 'Measurement', 'intensity', 'intensity', 'intensity', 3), + (608, 'Line', 'C III 97.70 nm', 'C III 97.70 nm', 'C III 97.70 nm', 4); diff --git a/install/helioviewer/db.py b/install/helioviewer/db.py index 316b8efb..cebb7a5c 100644 --- a/install/helioviewer/db.py +++ b/install/helioviewer/db.py @@ -365,6 +365,14 @@ def create_datasource_table(cursor): (134, 'PUNCH', 'PUNCH WFI+NFI PAM Mosaic', NULL, 1, 0, '', 0, 0, 0, 0), (132, 'GOES', 'CCOR-1', NULL, 1, 0, '', 0, 0, 0, 0), (133, 'SWFO-L1', 'CCOR-2', NULL, 1, 0, '', 0, 0, 0, 0), +(601, 'SPICE 77.04', 'Solar Orbiter SPICE 77.04', 'NM', 1, 0, '', 0, 0, 0, 0), +(602, 'SPICE 103.19', 'Solar Orbiter SPICE 103.19', 'NM', 1, 0, '', 0, 0, 0, 0), +(603, 'SPICE 70.38', 'Solar Orbiter SPICE 70.38', 'NM', 1, 0, '', 0, 0, 0, 0), +(604, 'SPICE 78.77', 'Solar Orbiter SPICE 78.77', 'NM', 1, 0, '', 0, 0, 0, 0), +(605, 'SPICE 70.6', 'Solar Orbiter SPICE 70.6', 'NM', 1, 0, '', 0, 0, 0, 0), +(606, 'SPICE 97.25', 'Solar Orbiter SPICE 97.25', 'NM', 1, 0, '', 0, 0, 0, 0), +(607, 'SPICE 76.51', 'Solar Orbiter SPICE 76.51', 'NM', 1, 0, '', 0, 0, 0, 0), +(608, 'SPICE 97.7', 'Solar Orbiter SPICE 97.7', 'NM', 1, 0, '', 0, 0, 0, 0), (2000, 'GOES-R SUVI 94', 'GOES-R SUVI 94', NULL, 1, 0, '', 0, 0, 0, 0), (2001, 'GOES-R SUVI 131', 'GOES-R SUVI 131', NULL, 1, 0, '', 0, 0, 0, 0), (2002, 'GOES-R SUVI 171', 'GOES-R SUVI 171', NULL, 1, 0, '', 0, 0, 0, 0), @@ -403,7 +411,7 @@ def create_datasource_property_table(cursor): """Inserts properties about datasource @sourceId: unique sidentifier which matches datasource table id - @label: one of the following Observatory/Instrument/Detector/Measurement + @label: one of the following Observatory/Instrument/Detector/Measurement/Line @name: the name that is embedded in the jp2 file @fitsName the name that is embedded in the jp2 file @description: verbal description of datasource @@ -933,7 +941,39 @@ def create_datasource_property_table(cursor): (133, 'Observatory', 'SWFO-L1', 'SWFO-L1', 'SWFO-L1 Observatory',1), (133, 'Instrument', 'CCOR-2', 'CCOR2', 'CCOR-2',2), (133, 'Detector', 'CCOR-2', '1', 'CCOR-2',2), -(133, 'Measurement', 'white-light', 'white-light', 'White Light',3);""") +(133, 'Measurement', 'white-light', 'white-light', 'White Light',3), +(601, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(601, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(601, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(601, 'Line', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 4), +(602, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(602, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(602, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(602, 'Line', 'O VI 103.19 nm', 'O VI 103.19 nm', 'O VI 103.19 nm', 4), +(603, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(603, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(603, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(603, 'Line', 'O III 70.38 nm', 'O III 70.38 nm', 'O III 70.38 nm', 4), +(604, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(604, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(604, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(604, 'Line', 'O IV 78.77 nm', 'O IV 78.77 nm', 'O IV 78.77 nm', 4), +(605, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(605, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(605, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(605, 'Line', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 4), +(606, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(606, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(606, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(606, 'Line', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 4), +(607, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(607, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(607, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(607, 'Line', 'N IV 76.51 nm', 'N IV 76.51 nm', 'N IV 76.51 nm', 4), +(608, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(608, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2), +(608, 'Measurement', 'intensity', 'intensity', 'intensity', 3), +(608, 'Line', 'C III 97.70 nm', 'C III 97.70 nm', 'C III 97.70 nm', 4);""") def create_movies_table(cursor): """Creates movie table diff --git a/install/helioviewer/hvpull/browser/basebrowser.py b/install/helioviewer/hvpull/browser/basebrowser.py index ef5de03a..24973ab9 100644 --- a/install/helioviewer/hvpull/browser/basebrowser.py +++ b/install/helioviewer/hvpull/browser/basebrowser.py @@ -1,6 +1,5 @@ """Base data browser definition""" -from typing import Callable, Optional - +from collections.abc import Callable class BaseDataBrowser: """BaseDataBrowser""" @@ -10,8 +9,8 @@ def __init__(self, server): def get_directories(self, start_time, end_time): """Gets a list of directories to be queried for the given time range""" return None - - def get_files(self, uri, extension, filter_func: Optional[Callable] = None): + + def get_files(self, uri, extension, filter_func: Callable | None = None): """Get all the files that end with specified extension at the uri""" return None diff --git a/install/helioviewer/hvpull/browser/httpbrowser.py b/install/helioviewer/hvpull/browser/httpbrowser.py index 8fee7af8..6c65d370 100644 --- a/install/helioviewer/hvpull/browser/httpbrowser.py +++ b/install/helioviewer/hvpull/browser/httpbrowser.py @@ -2,7 +2,7 @@ import sys import os import socket -from typing import Callable, Optional +from collections.abc import Callable from helioviewer.hvpull.browser.basebrowser import BaseDataBrowser, NetworkError if (sys.version_info >= (3, 0)): @@ -82,7 +82,7 @@ def get_directories(self, start_date, end_date): # filter(lambda url: url.endswith("/"), self._query(location)) return self.server.compute_directories(start_date, end_date) - def get_files(self, location, extension, filter_func: Optional[Callable] = None): + def get_files(self, location, extension, filter_func: Callable | None = None): """Get all the files that end with specified extension at the uri""" files = None num_retries = 0 @@ -126,4 +126,3 @@ def _query(self, location): urls = filter(lambda url: url[0] != "/" and url[0] != "?", result) return [os.path.join(location, url) for url in urls] - diff --git a/install/helioviewer/hvpull/browser/localbrowser.py b/install/helioviewer/hvpull/browser/localbrowser.py index ef347479..10db5c15 100644 --- a/install/helioviewer/hvpull/browser/localbrowser.py +++ b/install/helioviewer/hvpull/browser/localbrowser.py @@ -1,6 +1,6 @@ """Local data browser""" import os -from typing import Callable, Optional +from collections.abc import Callable from helioviewer.hvpull.browser.basebrowser import BaseDataBrowser @@ -15,7 +15,7 @@ def get_directories(self, start_date, end_date): """Get a list of directories at the passed uri""" return self.server.compute_directories(start_date, end_date) - def get_files(self, location, extension, filter_func: Optional[Callable] = None): + def get_files(self, location, extension, filter_func: Callable | None = None): """Get all the files that end with specified extension at the uri""" # ensure the location exists diff --git a/install/helioviewer/hvpull/net/daemon.py b/install/helioviewer/hvpull/net/daemon.py index 336f71f0..e1ed453d 100644 --- a/install/helioviewer/hvpull/net/daemon.py +++ b/install/helioviewer/hvpull/net/daemon.py @@ -876,6 +876,7 @@ def get_servers(cls): "kcor": "KCORDataServer", "hv_kcor": "HVKCORDataServer", "solar_orbiter": "SolarOrbiterDataServer", + "spice": "SpiceDataServer", "suvi": "SUVIDataServer", "iris": "IRISDataServer", "hv_iris": "HvIRISDataServer", diff --git a/install/helioviewer/hvpull/servers/spice.py b/install/helioviewer/hvpull/servers/spice.py new file mode 100644 index 00000000..d87fbc89 --- /dev/null +++ b/install/helioviewer/hvpull/servers/spice.py @@ -0,0 +1,16 @@ +import os +from helioviewer.hvpull.servers import DataServer + +class SpiceDataServer(DataServer): + """Solar Orbiter/SPICE Datasource definition""" + def __init__(self): + """Defines the root directory where SPICE JP2 files are kept.""" + spice_data_path = os.environ.get('SPICE_DATA_PATH') + if not spice_data_path: + raise ValueError("SPICE_DATA_PATH must be set to the SPICE JP2 root directory") + + DataServer.__init__(self, spice_data_path, "SPICE") + + def compute_directories(self, start_date, end_date): + """Computes a list of remote directories expected to contain files""" + return [os.path.join(self.uri, date) for date in self.get_dates(start_date, end_date)] diff --git a/install/helioviewer/jp2parser.py b/install/helioviewer/jp2parser.py index 9b72c01a..40700915 100644 --- a/install/helioviewer/jp2parser.py +++ b/install/helioviewer/jp2parser.py @@ -126,6 +126,9 @@ def getData(self): image['measurement'] = 'white-light' elif image['instrument'] == "CCOR2": image['measurement'] = 'white-light' + elif image['instrument'] == "SPICE": + image['measurement'] = 'intensity' + image['line'] = self._get_spice_line(imageData) else: image['measurement'] = measurement image['date'] = self._get_date(imageData) @@ -178,10 +181,26 @@ def get_detection_keys(img): leafs = ["observatory", "energy_band", "reconstruction_method"] elif img["observatory"] == "PUNCH": leafs = ["observatory", "instrument", "title"] + elif img["instrument"] == "SPICE": + leafs = ["observatory", "instrument", "measurement", "line"] else: leafs = ["observatory", "instrument", "detector", "measurement"] return leafs + def _get_spice_line(self, imageData): + """ + Returns the SPICE spectral line name from JP2 metadata. + """ + for key in ["cmpnam", "CMPNAM"]: + try: + line = imageData.meta.get(key) + except AttributeError: + line = imageData.meta[key] if key in imageData.meta else None + if line is not None and str(line).strip(): + return str(line).strip() + + raise ValueError("SPICE JP2 missing required CMPNAM metadata; cannot determine datasource line") + def _get_punch_file_type(self, filepath): """ Returns the PUNCH file type e.g. CAM/PAM based on the filename. @@ -504,4 +523,3 @@ def _process_rhessi_extras(self, image: dict, imageData: GenericMap) -> dict: "HV_RHESSI_IMAGE_RECONSTRUCTION_METHOD" ] return image -