From 2556725c795d488251bf7cc228d178a0e8b082b1 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 17:29:59 +0200 Subject: [PATCH 01/13] generate get_surfrad --- pvlib/iotools/surfrad.py | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index 77d9833034..aa1d06a7a2 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -5,6 +5,9 @@ from urllib.request import urlopen, Request import pandas as pd import numpy as np +import warnings +import urllib.error + SURFRAD_COLUMNS = [ 'year', 'jday', 'month', 'day', 'hour', 'minute', 'dt', 'zen', @@ -181,3 +184,83 @@ def _format_index(data): data.index = index data = data.tz_localize('UTC') return data + + +def get_surfrad(station, start, end, map_variables=True, + url="https://gml.noaa.gov/aftp/data/radiation/surfrad/"): + """ + Request data from NOAA SURFRAD and read it into a DataFrame. + + Data is returned for complete days, including ``start`` and ``end``. + + Parameters + ---------- + station : str + Three-letter SURFRAD station abbreviation. + start : datetime-like + First day of the requested period. + end : datetime-like + Last day of the requested period. + map_variables : bool, default True + Passed through to read_surfrad: whether to rename columns to + pvlib variable names (e.g. 'dw_solar' -> 'ghi'). + url : str, default 'https://gml.noaa.gov/aftp/data/radiation/surfrad/' + Base URL of the SURFRAD archive. + + Returns + ------- + data : pd.DataFrame + Dataframe with data from SURFRAD. + meta : dict + Metadata. + + See Also + -------- + pvlib.iotools.read_surfrad + + Notes + ----- + Missing days (e.g. before a station's operational start date, or gaps + in the archive) are skipped with a warning rather than raising an error. + + Examples + -------- + >>> data, meta = get_surfrad( + ... station='bon', start='2020-01-01', end='2020-01-31') + """ + start = pd.to_datetime(start) + end = pd.to_datetime(end) + + dates = pd.date_range(start.floor('D'), end, freq='D') + station = station.lower() + + filenames = [ + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" + for d in dates + ] + + dfs = [] + file_metadata = None + for f in filenames: + try: + dfi, file_metadata = read_surfrad(url + f, map_variables=map_variables) + dfs.append(dfi) + + except urllib.error.HTTPError: + warnings.warn(f"The following file was not found: {f}") + + if not dfs: + raise ValueError( + f"No data retrieved for station '{station}' between " + f"{start.date()} and {end.date()}. Check the station code " + "and date range." + ) + + data = pd.concat(dfs, axis='rows') + meta = { + 'station': station, + 'filenames': filenames, + # all files should share metadata, so just take it from the last one + **file_metadata, + } + return data, meta From f1ccbe0b0dc79d0a98166be5e470eb46992861b5 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 17:44:15 +0200 Subject: [PATCH 02/13] function documentation --- docs/sphinx/source/reference/iotools.rst | 1 + docs/sphinx/source/whatsnew/v0.15.3.rst | 4 ++++ pvlib/iotools/__init__.py | 1 + 3 files changed, 6 insertions(+) diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index aca7b6e74c..3d5729935d 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -176,6 +176,7 @@ A solar radiation network in the USA, run by NOAA. :toctree: generated/ iotools.read_surfrad + iotools.get_surfrad MIDC diff --git a/docs/sphinx/source/whatsnew/v0.15.3.rst b/docs/sphinx/source/whatsnew/v0.15.3.rst index adfeff1dfe..519c9099b6 100644 --- a/docs/sphinx/source/whatsnew/v0.15.3.rst +++ b/docs/sphinx/source/whatsnew/v0.15.3.rst @@ -34,6 +34,9 @@ Enhancements :py:func:`~pvlib.iotools.get_nsrdb_psm4_polar_tmy`. (:issue:`2639`, :pull:`2807`) * Ensure all timezones are available in all OSs. (:issue:`2795`, :pull:`2809`) +* Added :py:func:`pvlib.iotools.get_surfrad`, which requests and reads NOAA + SURFRAD data. + (:issue:`1155`, :pull:`2836`) Documentation @@ -76,3 +79,4 @@ Contributors * Jason Lun Leung (:ghuser:`jason-rpkt`) * Leonardo Scappatura (:ghuser:`Leonard013`) * Carolina Crespo (:ghuser:`cbcrespo`) +* Ioannis Sifnaios (:ghuser:`IoannisSifnaios`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 5c4f2deb8d..28ff223554 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -3,6 +3,7 @@ from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import get_srml # noqa: F401 from pvlib.iotools.surfrad import read_surfrad # noqa: F401 +from pvlib.iotools.surfrad import get_surfrad # noqa: F401 from pvlib.iotools.midc import read_midc # noqa: F401 from pvlib.iotools.midc import read_midc_raw_data_from_nrel # noqa: F401 from pvlib.iotools.crn import read_crn # noqa: F401 From 1cbb2aeea361f5085b54b500bf536623b4910a04 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 17:53:31 +0200 Subject: [PATCH 03/13] fix linter --- pvlib/iotools/surfrad.py | 42 +++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index aa1d06a7a2..a87c6a37c5 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -187,12 +187,17 @@ def _format_index(data): def get_surfrad(station, start, end, map_variables=True, - url="https://gml.noaa.gov/aftp/data/radiation/surfrad/"): + url="https://gml.noaa.gov/aftp/data/radiation/surfrad/"): """ Request data from NOAA SURFRAD and read it into a DataFrame. - + + The SURFRAD network is described in [1]_. + See README files located in the station directories in the SURFRAD + data archives [2]_ for details on SURFRAD daily data files. In addition to + the FTP server, the SURFRAD files are also available via HTTP access [3]_. + Data is returned for complete days, including ``start`` and ``end``. - + Parameters ---------- station : str @@ -206,56 +211,67 @@ def get_surfrad(station, start, end, map_variables=True, pvlib variable names (e.g. 'dw_solar' -> 'ghi'). url : str, default 'https://gml.noaa.gov/aftp/data/radiation/surfrad/' Base URL of the SURFRAD archive. - + Returns ------- data : pd.DataFrame Dataframe with data from SURFRAD. meta : dict Metadata. - + See Also -------- pvlib.iotools.read_surfrad - + Notes ----- Missing days (e.g. before a station's operational start date, or gaps in the archive) are skipped with a warning rather than raising an error. - + Examples -------- >>> data, meta = get_surfrad( ... station='bon', start='2020-01-01', end='2020-01-31') + + References + ---------- + .. [1] NOAA Earth System Research Laboratory Surface Radiation Budget + Network + `SURFRAD Homepage `_ + .. [2] NOAA SURFRAD Data Archive + `SURFRAD Archive `_ + .. [3] `NOAA SURFRAD HTTP Index + `_ """ start = pd.to_datetime(start) end = pd.to_datetime(end) - + dates = pd.date_range(start.floor('D'), end, freq='D') station = station.lower() - + filenames = [ f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" for d in dates ] - + dfs = [] file_metadata = None for f in filenames: try: - dfi, file_metadata = read_surfrad(url + f, map_variables=map_variables) + dfi, file_metadata = read_surfrad(url + f, + map_variables=VARIABLE_MAP) dfs.append(dfi) except urllib.error.HTTPError: warnings.warn(f"The following file was not found: {f}") - + if not dfs: raise ValueError( f"No data retrieved for station '{station}' between " f"{start.date()} and {end.date()}. Check the station code " "and date range." ) - + data = pd.concat(dfs, axis='rows') meta = { 'station': station, From 9927aeb281f3937e2d01c070ab53c31fef005d70 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 17:58:18 +0200 Subject: [PATCH 04/13] linter v2 --- pvlib/iotools/surfrad.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index a87c6a37c5..ffb5cef514 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -191,9 +191,8 @@ def get_surfrad(station, start, end, map_variables=True, """ Request data from NOAA SURFRAD and read it into a DataFrame. - The SURFRAD network is described in [1]_. - See README files located in the station directories in the SURFRAD - data archives [2]_ for details on SURFRAD daily data files. In addition to + The SURFRAD network is described in [1]_. The README files are located in + the station directories in the SURFRAD data archives [2]_. In addition to the FTP server, the SURFRAD files are also available via HTTP access [3]_. Data is returned for complete days, including ``start`` and ``end``. @@ -250,7 +249,7 @@ def get_surfrad(station, start, end, map_variables=True, station = station.lower() filenames = [ - f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear: 03}.dat" for d in dates ] From 2784b2f2ab752f44faba9aa77f0cacd92fd025bd Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 18:41:40 +0200 Subject: [PATCH 05/13] add tests --- tests/iotools/test_surfrad.py | 45 ++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/iotools/test_surfrad.py b/tests/iotools/test_surfrad.py index ce12428713..0e7ed636a1 100644 --- a/tests/iotools/test_surfrad.py +++ b/tests/iotools/test_surfrad.py @@ -1,8 +1,15 @@ +import urllib.error + import pandas as pd import pytest from pvlib.iotools import surfrad -from tests.conftest import TESTS_DATA_DIR, RERUNS, RERUNS_DELAY +from tests.conftest import ( + TESTS_DATA_DIR, + assert_frame_equal, + RERUNS, + RERUNS_DELAY, +) testfile = TESTS_DATA_DIR / 'surfrad-slv16001.dat' network_testfile = ('ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/' @@ -73,3 +80,39 @@ def test_read_surfrad_metadata(): 'tz': 'UTC'} _, metadata = surfrad.read_surfrad(testfile) assert metadata == expected + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_surfrad(): + df, meta = surfrad.get_surfrad('slv', '2016-01-01', '2016-01-01') + + assert meta['station'] == 'slv' + assert isinstance(meta['filenames'], list) + + assert len(df) == 1440 + assert df.index[0] == pd.to_datetime('2016-01-01 00:00+00:00') + assert df.index[-1] == pd.to_datetime('2016-01-01 23:59+00:00') + + expected, _ = surfrad.read_surfrad(testfile) + assert_frame_equal(df, expected) + + +@pytest.mark.remote_data +def test_get_surfrad_missing_day(): + # SURFRAD's Alamosa station data begins 2014-07-28 (slv14209.dat), so + # requesting the day before that will raise a warning + message = 'The following file was not found: slv/2014/slv14208.dat' + with pytest.warns(UserWarning, match=message): + df, meta = surfrad.get_surfrad('slv', '2014-07-27', '2014-07-28') + + # but the data for 2014-07-28 is still returned + assert not df.empty + + +@pytest.mark.remote_data +def test_get_surfrad_no_data(): + message = "No data retrieved for station 'xxx'" + with pytest.warns(UserWarning): + with pytest.raises(ValueError, match=message): + surfrad.get_surfrad('xxx', '2016-01-01', '2016-01-01') From 2fb99a77983a08007158fb97637f945cc70ead59 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 18:48:31 +0200 Subject: [PATCH 06/13] fix test linter --- tests/iotools/test_surfrad.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/iotools/test_surfrad.py b/tests/iotools/test_surfrad.py index 0e7ed636a1..ea4ea5b876 100644 --- a/tests/iotools/test_surfrad.py +++ b/tests/iotools/test_surfrad.py @@ -1,5 +1,3 @@ -import urllib.error - import pandas as pd import pytest From d27759bfee382c5e58793f5e51b6e8fa857f5855 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 19:04:25 +0200 Subject: [PATCH 07/13] minor error --- pvlib/iotools/surfrad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index ffb5cef514..d0ff97f2b6 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -249,7 +249,7 @@ def get_surfrad(station, start, end, map_variables=True, station = station.lower() filenames = [ - f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear: 03}.dat" + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" # noqa: E231 for d in dates ] From b30411c540e2ddc3d8495835ec4ee8e934b638c8 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Tue, 4 Aug 2026 19:06:33 +0200 Subject: [PATCH 08/13] linter again --- pvlib/iotools/surfrad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index d0ff97f2b6..5ca3f0e010 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -249,7 +249,7 @@ def get_surfrad(station, start, end, map_variables=True, station = station.lower() filenames = [ - f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" # noqa: E231 + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" # noqa: E231,E501 for d in dates ] From 759dd8fcf614ef342f08dd1e2e77a134b061480e Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios <88548539+IoannisSifnaios@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:30:18 +0200 Subject: [PATCH 09/13] Update docs/sphinx/source/whatsnew/v0.15.3.rst Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com> --- docs/sphinx/source/whatsnew/v0.15.3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.15.3.rst b/docs/sphinx/source/whatsnew/v0.15.3.rst index 25d3446d89..3d89dcb293 100644 --- a/docs/sphinx/source/whatsnew/v0.15.3.rst +++ b/docs/sphinx/source/whatsnew/v0.15.3.rst @@ -47,8 +47,8 @@ Enhancements :py:func:`~pvlib.iotools.get_nsrdb_psm4_polar_tmy`. (:issue:`2639`, :pull:`2807`) * Ensure all timezones are available in all OSs. (:issue:`2795`, :pull:`2809`) -* Added :py:func:`pvlib.iotools.get_surfrad`, which requests and reads NOAA - SURFRAD data. +* Add :py:func:`pvlib.iotools.get_surfrad` for retrieving irradiance data + from NOAA's SURFRAD network. (:issue:`1155`, :pull:`2836`) * Implement the ANTS-2D bifacial irradiance model in :py:func:`pvlib.bifacial.ants2d.get_irradiance`. (:pull:`2740`) From 0deed99428b24184332c3dd31dbe5a0bbbc2532e Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios <88548539+IoannisSifnaios@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:30:29 +0200 Subject: [PATCH 10/13] Update pvlib/iotools/surfrad.py Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com> --- pvlib/iotools/surfrad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index 5ca3f0e010..6764f3e202 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -229,7 +229,7 @@ def get_surfrad(station, start, end, map_variables=True, Examples -------- - >>> data, meta = get_surfrad( + >>> data, meta = pvlib.iotools.get_surfrad( ... station='bon', start='2020-01-01', end='2020-01-31') References From 1d02f71216667d4b51f3d5c452e3d8e46ae63609 Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios <88548539+IoannisSifnaios@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:31:59 +0200 Subject: [PATCH 11/13] Update tests/iotools/test_surfrad.py Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com> --- tests/iotools/test_surfrad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/iotools/test_surfrad.py b/tests/iotools/test_surfrad.py index ea4ea5b876..03a5dc9b87 100644 --- a/tests/iotools/test_surfrad.py +++ b/tests/iotools/test_surfrad.py @@ -97,6 +97,7 @@ def test_get_surfrad(): @pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_surfrad_missing_day(): # SURFRAD's Alamosa station data begins 2014-07-28 (slv14209.dat), so # requesting the day before that will raise a warning From 5403d156d3247d2e20b0e8ae33a27b3b2be00e6f Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios Date: Wed, 5 Aug 2026 16:33:24 +0200 Subject: [PATCH 12/13] Update test_surfrad.py --- tests/iotools/test_surfrad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/iotools/test_surfrad.py b/tests/iotools/test_surfrad.py index 03a5dc9b87..f377a292af 100644 --- a/tests/iotools/test_surfrad.py +++ b/tests/iotools/test_surfrad.py @@ -110,6 +110,7 @@ def test_get_surfrad_missing_day(): @pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_surfrad_no_data(): message = "No data retrieved for station 'xxx'" with pytest.warns(UserWarning): From dcf5864a38f37bd33f05e4c1837c097b9b85b23b Mon Sep 17 00:00:00 2001 From: Ioannis Sifnaios <88548539+IoannisSifnaios@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:00:05 +0200 Subject: [PATCH 13/13] Update pvlib/iotools/surfrad.py Co-authored-by: Kevin Anderson --- pvlib/iotools/surfrad.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/surfrad.py b/pvlib/iotools/surfrad.py index 6764f3e202..948ec5e09e 100644 --- a/pvlib/iotools/surfrad.py +++ b/pvlib/iotools/surfrad.py @@ -206,8 +206,9 @@ def get_surfrad(station, start, end, map_variables=True, end : datetime-like Last day of the requested period. map_variables : bool, default True - Passed through to read_surfrad: whether to rename columns to - pvlib variable names (e.g. 'dw_solar' -> 'ghi'). + Passed through to :py:func:`~pvlib.iotools.read_surfrad`: + whether to rename columns to pvlib variable names + (e.g. ``'dw_solar'`` -> ``'ghi'``). url : str, default 'https://gml.noaa.gov/aftp/data/radiation/surfrad/' Base URL of the SURFRAD archive.