diff --git a/changelog/164.feature.rst b/changelog/164.feature.rst new file mode 100644 index 00000000..566ebfaa --- /dev/null +++ b/changelog/164.feature.rst @@ -0,0 +1 @@ +Added remote FITS header fetching for the e-Callisto client to provide accurate start and end times in search results. \ No newline at end of file diff --git a/radiospectra/net/sources/ecallisto.py b/radiospectra/net/sources/ecallisto.py index 11c62658..8cccb953 100644 --- a/radiospectra/net/sources/ecallisto.py +++ b/radiospectra/net/sources/ecallisto.py @@ -3,7 +3,14 @@ from sunpy.net.dataretriever.client import GenericClient from radiospectra.net.attrs import Observatory +import zlib +import urllib.request +import urllib.error +import logging +from astropy.io import fits +from astropy.time import Time +log = logging.getLogger(__name__) class eCALLISTOClient(GenericClient): """ @@ -41,8 +48,8 @@ class eCALLISTOClient(GenericClient): pattern = ( r"http://soleil80.cs.technik.fhnw.ch/solarradio/data/2002-20yy_Callisto/" - r"{{year:4d}}/{{month:2d}}/{{day:2d}}/{obs}_{{year:4d}}{{month:2d}}{{day:2d}}" - r"_{{hour:2d}}{{minute:2d}}{{second:2d}}{{suffix}}.fit.gz" + r"{year:4d}/{month:2d}/{day:2d}/{obs}_{year:4d}{month:2d}{day:2d}" + r"_{hour:2d}{minute:2d}{second:2d}{suffix}.fit.gz" ) @classmethod @@ -50,10 +57,10 @@ def pre_search_hook(cls, *args, **kwargs): baseurl, pattern, matchdict = super().pre_search_hook(*args, **kwargs) obs = matchdict["Observatory"] if obs[0] == "*": - pattern = pattern.replace("{obs}", "{{Observatory}}") + pattern = pattern.replace("{obs}", "{Observatory}") matchdict.pop("Observatory") else: - # Need case sensitive so have to override + obs_attr = [a for a in args if isinstance(a, Observatory)][0] pattern = pattern.replace("{obs}", obs_attr.value) return baseurl, pattern, matchdict @@ -62,10 +69,28 @@ def post_search_hook(self, exdict, matchdict): original = super().post_search_hook(exdict, matchdict) original["ID"] = original["suffix"].replace("_", "") del original["suffix"] - # We don't know the end time for all files - # https://github.com/sunpy/radiospectra/issues/60 - del original["End Time"] - return original + url = exdict.get('url') + if url: + start_h, end_h = self._fetch_remote_header(url) + if start_h: + try: + original["Start Time"] = Time(start_h) + except (ValueError, TypeError): + pass + + if end_h: + try: + original["End Time"] = Time(end_h) + except (ValueError, TypeError): + original["End Time"] = end_h + else: + if "End Time" in original: + del original["End Time"] + else: + if "End Time" in original: + del original["End Time"] + + return original @classmethod def register_values(cls): @@ -97,3 +122,34 @@ def _can_handle_query(cls, *query): ): return False return True + + def _fetch_remote_header(self, url): + headers = {'User-Agent': 'SunPy/Radiospectra', 'Range': 'bytes=0-15360'} + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=5) as response: + if response.getcode() == 206: + compressed_data = response.read() + # e-Callisto files are .gz, so we need to decompress + d = zlib.decompressobj(16 + zlib.MAX_WBITS) + header_bytes = d.decompress(compressed_data) + header = fits.Header.fromstring(header_bytes[:2880].decode('ascii', errors='ignore')) + + # Get Date and Time keywords + date_obs = header.get('DATE-OBS') + time_obs = header.get('TIME-OBS', '') + date_end = header.get('DATE-END', date_obs) + time_end = header.get('TIME-END', '') + + # Combine into strings for Astropy Time + start = f"{date_obs} {time_obs}".strip() + end = f"{date_end} {time_end}".strip() + + return (start if date_obs else None), (end if date_end else None) + + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + log.warning(f"Network error fetching e-Callisto header from {url}: {e}") + except (zlib.error, ValueError, KeyError) as e: + log.warning(f"Metadata parsing error for {url}: {e}") + + return None, None \ No newline at end of file diff --git a/radiospectra/net/sources/tests/test_ecallisto.py b/radiospectra/net/sources/tests/test_ecallisto.py new file mode 100644 index 00000000..549331ea --- /dev/null +++ b/radiospectra/net/sources/tests/test_ecallisto.py @@ -0,0 +1,49 @@ +import pytest +import zlib +from unittest.mock import patch, MagicMock +from astropy.io import fits +from radiospectra.net.sources.ecallisto import eCALLISTOClient + +# 1. MOCK TEST: +@patch('urllib.request.urlopen') +def test_fetch_remote_header_mock(mock_urlopen): + mock_response = MagicMock() + mock_response.getcode.return_value = 206 + + # Create a VALID fake FITS header using astropy + hdr = fits.Header() + hdr['SIMPLE'] = True + hdr['DATE-OBS'] = '2026-03-30' + hdr['TIME-OBS'] = '10:00:00' + hdr['DATE-END'] = '2026-03-30' + hdr['TIME-END'] = '10:15:00' + + header_bytes = hdr.tostring().encode('ascii') + + gzip_compressor = zlib.compressobj(wbits=16+zlib.MAX_WBITS) + fake_gz_data = gzip_compressor.compress(header_bytes) + gzip_compressor.flush() + + mock_response.read.return_value = fake_gz_data + mock_urlopen.return_value.__enter__.return_value = mock_response + + client = eCALLISTOClient() + start, end = client._fetch_remote_header("http://fake-url.fit.gz") + + assert start == "2026-03-30 10:00:00" + assert end == "2026-03-30 10:15:00" + + args, kwargs = mock_urlopen.call_args + assert args[0].headers['Range'] == 'bytes=0-15360' + +# 2. REAL SERVER TEST: +@pytest.mark.remote_data +def test_fetch_remote_header_real_server(): + client = eCALLISTOClient() + # Using a known stable URL from the archive + url = "http://soleil80.cs.technik.fhnw.ch/solarradio/data/2002-20yy_Callisto/2019/10/05/ALASKA_20191005_230000_59.fit.gz" + + start, end = client._fetch_remote_header(url) + + + assert start == "2019/10/05 23:00:00.757" + assert end is not None \ No newline at end of file