diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index aca7b6e74c..3d5729935d 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -176,6 +176,7 @@ A solar radiation network in the USA, run by NOAA. :toctree: generated/ iotools.read_surfrad + iotools.get_surfrad MIDC diff --git a/docs/sphinx/source/whatsnew/v0.15.3.rst b/docs/sphinx/source/whatsnew/v0.15.3.rst index adfeff1dfe..519c9099b6 100644 --- a/docs/sphinx/source/whatsnew/v0.15.3.rst +++ b/docs/sphinx/source/whatsnew/v0.15.3.rst @@ -34,6 +34,9 @@ Enhancements :py:func:`~pvlib.iotools.get_nsrdb_psm4_polar_tmy`. (:issue:`2639`, :pull:`2807`) * Ensure all timezones are available in all OSs. (:issue:`2795`, :pull:`2809`) +* Added :py:func:`pvlib.iotools.get_surfrad`, which requests and reads NOAA + SURFRAD data. + (:issue:`1155`, :pull:`2836`) Documentation @@ -76,3 +79,4 @@ Contributors * Jason Lun Leung (:ghuser:`jason-rpkt`) * Leonardo Scappatura (:ghuser:`Leonard013`) * Carolina Crespo (:ghuser:`cbcrespo`) +* Ioannis Sifnaios (:ghuser:`IoannisSifnaios`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 5c4f2deb8d..28ff223554 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -3,6 +3,7 @@ from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import get_srml # noqa: F401 from pvlib.iotools.surfrad import read_surfrad # noqa: F401 +from pvlib.iotools.surfrad import get_surfrad # noqa: F401 from pvlib.iotools.midc import read_midc # noqa: F401 from pvlib.iotools.midc import read_midc_raw_data_from_nrel # noqa: F401 from pvlib.iotools.crn import read_crn # noqa: F401 diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index 77d9833034..5ca3f0e010 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -5,6 +5,9 @@ from urllib.request import urlopen, Request import pandas as pd import numpy as np +import warnings +import urllib.error + SURFRAD_COLUMNS = [ 'year', 'jday', 'month', 'day', 'hour', 'minute', 'dt', 'zen', @@ -181,3 +184,98 @@ def _format_index(data): data.index = index data = data.tz_localize('UTC') return data + + +def get_surfrad(station, start, end, map_variables=True, + url="https://gml.noaa.gov/aftp/data/radiation/surfrad/"): + """ + Request data from NOAA SURFRAD and read it into a DataFrame. + + The SURFRAD network is described in [1]_. The README files are located in + the station directories in the SURFRAD data archives [2]_. In addition to + the FTP server, the SURFRAD files are also available via HTTP access [3]_. + + Data is returned for complete days, including ``start`` and ``end``. + + Parameters + ---------- + station : str + Three-letter SURFRAD station abbreviation. + start : datetime-like + First day of the requested period. + end : datetime-like + Last day of the requested period. + map_variables : bool, default True + Passed through to read_surfrad: whether to rename columns to + pvlib variable names (e.g. 'dw_solar' -> 'ghi'). + url : str, default 'https://gml.noaa.gov/aftp/data/radiation/surfrad/' + Base URL of the SURFRAD archive. + + Returns + ------- + data : pd.DataFrame + Dataframe with data from SURFRAD. + meta : dict + Metadata. + + See Also + -------- + pvlib.iotools.read_surfrad + + Notes + ----- + Missing days (e.g. before a station's operational start date, or gaps + in the archive) are skipped with a warning rather than raising an error. + + Examples + -------- + >>> data, meta = get_surfrad( + ... station='bon', start='2020-01-01', end='2020-01-31') + + References + ---------- + .. [1] NOAA Earth System Research Laboratory Surface Radiation Budget + Network + `SURFRAD Homepage `_ + .. [2] NOAA SURFRAD Data Archive + `SURFRAD Archive `_ + .. [3] `NOAA SURFRAD HTTP Index + `_ + """ + start = pd.to_datetime(start) + end = pd.to_datetime(end) + + dates = pd.date_range(start.floor('D'), end, freq='D') + station = station.lower() + + filenames = [ + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" # noqa: E231,E501 + for d in dates + ] + + dfs = [] + file_metadata = None + for f in filenames: + try: + dfi, file_metadata = read_surfrad(url + f, + map_variables=VARIABLE_MAP) + dfs.append(dfi) + + except urllib.error.HTTPError: + warnings.warn(f"The following file was not found: {f}") + + if not dfs: + raise ValueError( + f"No data retrieved for station '{station}' between " + f"{start.date()} and {end.date()}. Check the station code " + "and date range." + ) + + data = pd.concat(dfs, axis='rows') + meta = { + 'station': station, + 'filenames': filenames, + # all files should share metadata, so just take it from the last one + **file_metadata, + } + return data, meta diff --git a/tests/iotools/test_surfrad.py b/tests/iotools/test_surfrad.py index ce12428713..ea4ea5b876 100644 --- a/tests/iotools/test_surfrad.py +++ b/tests/iotools/test_surfrad.py @@ -2,7 +2,12 @@ import pytest from pvlib.iotools import surfrad -from tests.conftest import TESTS_DATA_DIR, RERUNS, RERUNS_DELAY +from tests.conftest import ( + TESTS_DATA_DIR, + assert_frame_equal, + RERUNS, + RERUNS_DELAY, +) testfile = TESTS_DATA_DIR / 'surfrad-slv16001.dat' network_testfile = ('ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/' @@ -73,3 +78,39 @@ def test_read_surfrad_metadata(): 'tz': 'UTC'} _, metadata = surfrad.read_surfrad(testfile) assert metadata == expected + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_surfrad(): + df, meta = surfrad.get_surfrad('slv', '2016-01-01', '2016-01-01') + + assert meta['station'] == 'slv' + assert isinstance(meta['filenames'], list) + + assert len(df) == 1440 + assert df.index[0] == pd.to_datetime('2016-01-01 00:00+00:00') + assert df.index[-1] == pd.to_datetime('2016-01-01 23:59+00:00') + + expected, _ = surfrad.read_surfrad(testfile) + assert_frame_equal(df, expected) + + +@pytest.mark.remote_data +def test_get_surfrad_missing_day(): + # SURFRAD's Alamosa station data begins 2014-07-28 (slv14209.dat), so + # requesting the day before that will raise a warning + message = 'The following file was not found: slv/2014/slv14208.dat' + with pytest.warns(UserWarning, match=message): + df, meta = surfrad.get_surfrad('slv', '2014-07-27', '2014-07-28') + + # but the data for 2014-07-28 is still returned + assert not df.empty + + +@pytest.mark.remote_data +def test_get_surfrad_no_data(): + message = "No data retrieved for station 'xxx'" + with pytest.warns(UserWarning): + with pytest.raises(ValueError, match=message): + surfrad.get_surfrad('xxx', '2016-01-01', '2016-01-01')