From d00588bb64c6fd4d3dac959033309f99669b3a01 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 15:59:06 -0600 Subject: [PATCH 01/11] Add some basic unit tests of vegtype_str2int(). One fails. --- python/ctsm/test/test_unit_cropcal_utils.py | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 python/ctsm/test/test_unit_cropcal_utils.py diff --git a/python/ctsm/test/test_unit_cropcal_utils.py b/python/ctsm/test/test_unit_cropcal_utils.py new file mode 100755 index 0000000000..8baf83cffe --- /dev/null +++ b/python/ctsm/test/test_unit_cropcal_utils.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +"""Unit tests for cropcal_utils.py""" + +import unittest + +from ctsm import unit_testing +from ctsm.crop_calendars import cropcal_utils as ccu + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +class TestCropCalUtils(unittest.TestCase): + """Tests of cropcal_utils.py""" + + def setUp(self): + self.vegtype_mainlist = ["crop_1", "crop_2", "crop_3"] + + def test_vegtype_str2int_1string(self): + """ + Tests vegtype_str2int() for a single string + """ + ccu.vegtype_str2int("crop_1", vegtype_mainlist=self.vegtype_mainlist) + + def test_vegtype_str2int_2strings(self): + """ + Tests vegtype_str2int() for two strings + """ + result = ccu.vegtype_str2int(["crop_1", "crop_3"], vegtype_mainlist=self.vegtype_mainlist) + self.assertListEqual(result, [0, 2]) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 471086a3d5297a010059499b87aa64a0a7dbc8b9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:00:06 -0600 Subject: [PATCH 02/11] vegtype_str2int(): Fix for single string inputs. --- python/ctsm/crop_calendars/cropcal_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index c7e8b6ac52..84d129e453 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -265,6 +265,7 @@ def vegtype_str2int(vegtype_str, vegtype_mainlist=None): convert_to_ndarray = not isinstance(vegtype_str, np.ndarray) if convert_to_ndarray: vegtype_str = np.array(vegtype_str) + vegtype_str = np.atleast_1d(vegtype_str) if isinstance(vegtype_mainlist, xr.Dataset): vegtype_mainlist = vegtype_mainlist.vegtype_str.values From 5bd62977cea26edb76cc5b66d31d037ecbc1aaab Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:07:34 -0600 Subject: [PATCH 03/11] grid_one_variable: Replace np.NaN with np.nan. --- python/ctsm/crop_calendars/grid_one_variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/crop_calendars/grid_one_variable.py b/python/ctsm/crop_calendars/grid_one_variable.py index d7c7126e54..b83f37ae13 100644 --- a/python/ctsm/crop_calendars/grid_one_variable.py +++ b/python/ctsm/crop_calendars/grid_one_variable.py @@ -100,7 +100,7 @@ def create_filled_array(this_ds, fill_value, thisvar_da, new_dims): if fill_value: thisvar_gridded[:] = fill_value else: - thisvar_gridded[:] = np.NaN + thisvar_gridded[:] = np.nan return thisvar_gridded From e0bac6d653affd9a232a5ad51637e52f3bbe92f7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:09:00 -0600 Subject: [PATCH 04/11] When calling vegtype_str2int for a single string, return a single int, not list. --- python/ctsm/crop_calendars/check_rx_obeyed.py | 2 +- python/ctsm/crop_calendars/cropcal_utils.py | 5 +++++ python/ctsm/crop_calendars/generate_gdd20_baseline.py | 4 ++-- python/ctsm/crop_calendars/generate_gdds_functions.py | 4 ++-- python/ctsm/test/test_unit_cropcal_utils.py | 7 ++++--- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/python/ctsm/crop_calendars/check_rx_obeyed.py b/python/ctsm/crop_calendars/check_rx_obeyed.py index c8c5410faf..68a37fb146 100644 --- a/python/ctsm/crop_calendars/check_rx_obeyed.py +++ b/python/ctsm/crop_calendars/check_rx_obeyed.py @@ -144,7 +144,7 @@ def check_rx_obeyed( continue ds_thisveg = dates_ds.isel(patch=thisveg_patches) - vegtype_int = utils.vegtype_str2int(vegtype_str)[0] + vegtype_int = utils.vegtype_str2int(vegtype_str) rx_da = rx_ds[f"gs1_{vegtype_int}"] rx_array = rx_da.values[ ds_thisveg.patches1d_jxy.values.astype(int) - 1, diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index 84d129e453..4f2929de37 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -265,6 +265,7 @@ def vegtype_str2int(vegtype_str, vegtype_mainlist=None): convert_to_ndarray = not isinstance(vegtype_str, np.ndarray) if convert_to_ndarray: vegtype_str = np.array(vegtype_str) + was_0d = vegtype_str.ndim == 0 vegtype_str = np.atleast_1d(vegtype_str) if isinstance(vegtype_mainlist, xr.Dataset): @@ -290,6 +291,10 @@ def vegtype_str2int(vegtype_str, vegtype_mainlist=None): indices[np.where(vegtype_str == vegtype_str_2)] = vegtype_mainlist.index(vegtype_str_2) if convert_to_ndarray: indices = [int(x) for x in indices] + + if was_0d: + indices = indices[0] + return indices diff --git a/python/ctsm/crop_calendars/generate_gdd20_baseline.py b/python/ctsm/crop_calendars/generate_gdd20_baseline.py index ee942eba21..ea2519db88 100644 --- a/python/ctsm/crop_calendars/generate_gdd20_baseline.py +++ b/python/ctsm/crop_calendars/generate_gdd20_baseline.py @@ -182,7 +182,7 @@ def _get_gddn_for_cft(cft_str, variable): def _get_output_varname(cft_str): - cft_int = utils.vegtype_str2int(cft_str)[0] + cft_int = utils.vegtype_str2int(cft_str) return f"gdd20bl_{cft_int}" @@ -275,7 +275,7 @@ def generate_gdd20_baseline(input_files, output_file, author, time_slice, variab # Process all crops encoding_dict = {} for cft_str in MGDCROP_LIST: - cft_int = utils.vegtype_str2int(cft_str)[0] + cft_int = utils.vegtype_str2int(cft_str) print(f"{cft_str} ({cft_int})") # Which GDDN history variable does this crop use? E.g., GDD0, GDD10 diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index f80f1e55f7..de737a6077 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -607,7 +607,7 @@ def import_and_process_1yr( log(logger, f" SKIPPING {vegtype_str}") continue - vegtype_int = utils.vegtype_str2int(vegtype_str)[0] + vegtype_int = utils.vegtype_str2int(vegtype_str) this_crop_full_patchlist = list(xr_flexsel(h2_ds, vegtype=vegtype_str).patch.values) # Get time series for each patch of this type @@ -1166,7 +1166,7 @@ def make_figures( raise RuntimeError(f"If mapping {vegtype_str}, you must provide land use dataset") else: vegtypes_str = [x for x in incl_vegtypes_str if vegtype_str.lower() in x] - vegtypes_int = [utils.vegtype_str2int(x)[0] for x in vegtypes_str] + vegtypes_int = [utils.vegtype_str2int(x) for x in vegtypes_str] # Crop fraction map (for masking and weighting) if lu_ds: diff --git a/python/ctsm/test/test_unit_cropcal_utils.py b/python/ctsm/test/test_unit_cropcal_utils.py index 8baf83cffe..edbb81bd7b 100755 --- a/python/ctsm/test/test_unit_cropcal_utils.py +++ b/python/ctsm/test/test_unit_cropcal_utils.py @@ -20,13 +20,14 @@ def setUp(self): def test_vegtype_str2int_1string(self): """ - Tests vegtype_str2int() for a single string + Tests vegtype_str2int() for a single string. Result should be an int. """ - ccu.vegtype_str2int("crop_1", vegtype_mainlist=self.vegtype_mainlist) + result = ccu.vegtype_str2int("crop_1", vegtype_mainlist=self.vegtype_mainlist) + self.assertEqual(result, 0) def test_vegtype_str2int_2strings(self): """ - Tests vegtype_str2int() for two strings + Tests vegtype_str2int() for two strings. result should be a list of ints. """ result = ccu.vegtype_str2int(["crop_1", "crop_3"], vegtype_mainlist=self.vegtype_mainlist) self.assertListEqual(result, [0, 2]) From 11a471cd486ba88d12ff7cf10d62bb7d1c5c51c5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:32:18 -0600 Subject: [PATCH 05/11] Add some basic unit tests of create_filled_array(). 1 failing. --- .../ctsm/test/test_unit_grid_one_variable.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 python/ctsm/test/test_unit_grid_one_variable.py diff --git a/python/ctsm/test/test_unit_grid_one_variable.py b/python/ctsm/test/test_unit_grid_one_variable.py new file mode 100755 index 0000000000..e5d8f9b7a0 --- /dev/null +++ b/python/ctsm/test/test_unit_grid_one_variable.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +""" +Unit tests for grid_one_variable +""" + +import unittest + +import numpy as np +import xarray as xr + +from ctsm import unit_testing +from ctsm.crop_calendars import grid_one_variable as g1v + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + +# pylint: disable=protected-access + +## Too many instant variables as part of the class (too many self. in the SetUp) +# pylint: disable=too-many-instance-attributes + + +class TestCreateFilledArray(unittest.TestCase): + """Unit tests for create_filled_array""" + + def setUp(self): + # Set up this_ds, which will provide us with sizes of dimensions in most cases + lat_vals = [55.0, 56.0, 57.0] + lat_da = xr.DataArray( + data=lat_vals, + dims=["lat"], + coords={"lat": lat_vals}, + ) + lon_vals = [255.0, 256.0, 257.0] + lon_da = xr.DataArray( + data=lon_vals, + dims=["lon"], + coords={"lon": lon_vals}, + ) + self.this_ds = xr.Dataset( + data_vars={ + "lat": lat_da, + "lon": lon_da, + } + ) + + def test_create_filled_array_fillNone(self): + """ + Test create_filled_array() with fill_value None: Should be filled with NaN + """ + + fill_value = None + thisvar_da_dummy = xr.DataArray() + new_dims = ["lat", "lon"] + + result = g1v.create_filled_array(self.this_ds, fill_value, thisvar_da_dummy, new_dims) + + self.assertTrue(np.all(np.isnan(result))) + + def test_create_filled_array_fill1(self): + """ + Test create_filled_array() with fill_value 1: Should be filled with 1 + """ + + fill_value = 1.0 + thisvar_da_dummy = xr.DataArray() + new_dims = ["lat", "lon"] + + result = g1v.create_filled_array(self.this_ds, fill_value, thisvar_da_dummy, new_dims) + + self.assertTrue(np.all(result == fill_value)) + + def test_create_filled_array_fill0(self): + """ + Test create_filled_array() with fill_value 0: Should be filled with 0 + """ + + fill_value = 0.0 + thisvar_da_dummy = xr.DataArray() + new_dims = ["lat", "lon"] + + result = g1v.create_filled_array(self.this_ds, fill_value, thisvar_da_dummy, new_dims) + + self.assertTrue(np.all(result == fill_value)) From c3f9be28ef0c067d61dc7d048ac7573c88771489 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:34:06 -0600 Subject: [PATCH 06/11] create_filled_array(): Fix behavior with fill_value=0. --- python/ctsm/crop_calendars/grid_one_variable.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ctsm/crop_calendars/grid_one_variable.py b/python/ctsm/crop_calendars/grid_one_variable.py index b83f37ae13..01eaf84ca0 100644 --- a/python/ctsm/crop_calendars/grid_one_variable.py +++ b/python/ctsm/crop_calendars/grid_one_variable.py @@ -87,6 +87,10 @@ def create_filled_array(this_ds, fill_value, thisvar_da, new_dims): """ Create a Numpy array to be filled with gridded data """ + + if fill_value is None: + fill_value = np.nan + dim_size_list = [] for dim in new_dims: if dim == "ivt_str": @@ -97,10 +101,7 @@ def create_filled_array(this_ds, fill_value, thisvar_da, new_dims): dim_size = this_ds.sizes[dim] dim_size_list = dim_size_list + [dim_size] thisvar_gridded = np.empty(dim_size_list) - if fill_value: - thisvar_gridded[:] = fill_value - else: - thisvar_gridded[:] = np.nan + thisvar_gridded[:] = fill_value return thisvar_gridded From a73eb84a97a3794fce9f962a621644a49de08558 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 16:35:23 -0600 Subject: [PATCH 07/11] grid_one_variable(): Fix behavior with fill_value=0. --- python/ctsm/crop_calendars/grid_one_variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/crop_calendars/grid_one_variable.py b/python/ctsm/crop_calendars/grid_one_variable.py index 01eaf84ca0..c06ed50b16 100644 --- a/python/ctsm/crop_calendars/grid_one_variable.py +++ b/python/ctsm/crop_calendars/grid_one_variable.py @@ -161,7 +161,7 @@ def grid_one_variable(this_ds, var, fill_value=None, **kwargs): # Get DataArrays needed for gridding thisvar_da, vt_da, spatial_unit, ixy_da, jxy_da = get_ixy_jxy_das(this_ds, var) - if not fill_value and "_FillValue" in thisvar_da.attrs: + if fill_value is None and "_FillValue" in thisvar_da.attrs: fill_value = thisvar_da.attrs["_FillValue"] # Renumber vt_da to work as indices on new ivt dimension, if needed. From a728bda43ab16c9535bee4ca9692ce7b869059ed Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 17:12:36 -0600 Subject: [PATCH 08/11] generate_gdd20_baseline: Move time slice from _parse_args() to generate_gdd20_baseline(). --- .../crop_calendars/generate_gdd20_baseline.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdd20_baseline.py b/python/ctsm/crop_calendars/generate_gdd20_baseline.py index ea2519db88..5fa066fe79 100644 --- a/python/ctsm/crop_calendars/generate_gdd20_baseline.py +++ b/python/ctsm/crop_calendars/generate_gdd20_baseline.py @@ -109,22 +109,7 @@ def _parse_args(): if not os.path.exists(filename): raise FileNotFoundError(f"Input file not found: {filename}") - # Process time slice - # Assumes CESM behavior where data for e.g. 1987 is saved as 1988-01-01. - # It would be more robust, accounting for upcoming behavior (where timestamp for a year is the - # middle of that year), to do slice("YEAR1-01-03", "YEARN-01-02"), but that's not compatible - # with ctsm_pylib as of the version using python 3.7.9. See safer_timeslice() in cropcal_utils. - if args.first_year is not None: - date_1 = f"{args.first_year+1}-01-01" - else: - date_1 = "0000-01-01" - if args.last_year is not None: - date_n = f"{args.last_year+1}-01-01" - else: - date_n = "9999-12-31" - time_slice = slice(date_1, date_n) - - return args, time_slice + return args def _get_cft_list(crop_list): @@ -232,7 +217,7 @@ def setup_output_dataset(input_files, author, variable, year_args, ds_in): return ds_out -def generate_gdd20_baseline(input_files, output_file, author, time_slice, variable, year_args): +def generate_gdd20_baseline(input_files, output_file, author, variable, year_args): """ Generate stream_fldFileName_gdd20_baseline file from CTSM outputs """ @@ -252,6 +237,23 @@ def generate_gdd20_baseline(input_files, output_file, author, time_slice, variab input_files = list(set(input_files)) input_files.sort() + # Process time slice + # Assumes CESM behavior where data for e.g. 1987 is saved as 1988-01-01. + # It would be more robust, accounting for upcoming behavior (where timestamp for a year is the + # middle of that year), to do slice("YEAR1-01-03", "YEARN-01-02"), but that's not compatible + # with ctsm_pylib as of the version using python 3.7.9. See safer_timeslice() in cropcal_utils. + first_year = year_args[0] + last_year = year_args[1] + if first_year is not None: + date_1 = f"{first_year+1}-01-01" + else: + date_1 = "0000-01-01" + if last_year is not None: + date_n = f"{last_year+1}-01-01" + else: + date_n = "9999-12-31" + time_slice = slice(date_1, date_n) + # Import history files and ensure they have lat/lon dims ds_in = import_ds(input_files, my_vars=var_list_in + GRIDDING_VAR_LIST, time_slice=time_slice) if not all(x in ds_in.dims for x in ["lat", "lon"]): @@ -323,12 +325,11 @@ def main(): """ main() function for calling generate_gdd20_baseline.py from command line. """ - args, time_slice = _parse_args() + args = _parse_args() generate_gdd20_baseline( args.input_files, args.output_file, args.author, - time_slice, args.variable, [args.first_year, args.last_year], ) From fd55e311455d363fea61db7aa6cb92d05479fe5e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 10 Oct 2025 17:14:46 -0600 Subject: [PATCH 09/11] generate_gdd20_baseline: Simplify time slice, assuming modern ctsm_pylib. --- .../ctsm/crop_calendars/generate_gdd20_baseline.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdd20_baseline.py b/python/ctsm/crop_calendars/generate_gdd20_baseline.py index 5fa066fe79..c41c607a08 100644 --- a/python/ctsm/crop_calendars/generate_gdd20_baseline.py +++ b/python/ctsm/crop_calendars/generate_gdd20_baseline.py @@ -238,19 +238,13 @@ def generate_gdd20_baseline(input_files, output_file, author, variable, year_arg input_files.sort() # Process time slice - # Assumes CESM behavior where data for e.g. 1987 is saved as 1988-01-01. - # It would be more robust, accounting for upcoming behavior (where timestamp for a year is the - # middle of that year), to do slice("YEAR1-01-03", "YEARN-01-02"), but that's not compatible - # with ctsm_pylib as of the version using python 3.7.9. See safer_timeslice() in cropcal_utils. first_year = year_args[0] last_year = year_args[1] - if first_year is not None: - date_1 = f"{first_year+1}-01-01" - else: + date_1 = f"{first_year}-01-01" + date_n = f"{last_year}-12-31" + if first_year is None: date_1 = "0000-01-01" - if last_year is not None: - date_n = f"{last_year+1}-01-01" - else: + if last_year is None: date_n = "9999-12-31" time_slice = slice(date_1, date_n) From 9c9a313967a55ffd566baa2495d8b88b8286621b Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 29 Oct 2025 11:17:40 -0600 Subject: [PATCH 10/11] generate_gdd20_baseline.py: Refactor to new function _get_time_slice(). --- .../crop_calendars/generate_gdd20_baseline.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdd20_baseline.py b/python/ctsm/crop_calendars/generate_gdd20_baseline.py index c41c607a08..fa42eb58b4 100644 --- a/python/ctsm/crop_calendars/generate_gdd20_baseline.py +++ b/python/ctsm/crop_calendars/generate_gdd20_baseline.py @@ -217,6 +217,22 @@ def setup_output_dataset(input_files, author, variable, year_args, ds_in): return ds_out +def _get_time_slice(year_args): + """ + Based on years from input arguments, return a time slice for selecting from dataset + """ + first_year = year_args[0] + last_year = year_args[1] + date_1 = f"{first_year}-01-01" + date_n = f"{last_year}-12-31" + if first_year is None: + date_1 = "0000-01-01" + if last_year is None: + date_n = "9999-12-31" + time_slice = slice(date_1, date_n) + return time_slice + + def generate_gdd20_baseline(input_files, output_file, author, variable, year_args): """ Generate stream_fldFileName_gdd20_baseline file from CTSM outputs @@ -238,15 +254,7 @@ def generate_gdd20_baseline(input_files, output_file, author, variable, year_arg input_files.sort() # Process time slice - first_year = year_args[0] - last_year = year_args[1] - date_1 = f"{first_year}-01-01" - date_n = f"{last_year}-12-31" - if first_year is None: - date_1 = "0000-01-01" - if last_year is None: - date_n = "9999-12-31" - time_slice = slice(date_1, date_n) + time_slice = _get_time_slice(year_args) # Import history files and ensure they have lat/lon dims ds_in = import_ds(input_files, my_vars=var_list_in + GRIDDING_VAR_LIST, time_slice=time_slice) From 5bf0c522ae1a0e2eaf3202cc31f2932979bdd2ea Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 29 Oct 2025 11:18:06 -0600 Subject: [PATCH 11/11] test_unit_grid_one_variable: Add unit_testing.setup_for_tests(). --- python/ctsm/test/test_unit_grid_one_variable.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/ctsm/test/test_unit_grid_one_variable.py b/python/ctsm/test/test_unit_grid_one_variable.py index e5d8f9b7a0..400b4ede7e 100755 --- a/python/ctsm/test/test_unit_grid_one_variable.py +++ b/python/ctsm/test/test_unit_grid_one_variable.py @@ -84,3 +84,8 @@ def test_create_filled_array_fill0(self): result = g1v.create_filled_array(self.this_ds, fill_value, thisvar_da_dummy, new_dims) self.assertTrue(np.all(result == fill_value)) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main()