[develop] Add MPI for UFS_FIRE, add a check for incorrectly placed envvars: in config file, remove obsolete FV3_RRFS_v1beta suite (ufs-community#1376)

This PR:
 * Updates to the latest UFS Weather Model hash, which includes the addition of MPI capability for the CFBM fire model (UFS_FIRE). The MPI parallelization is activated by setting FIRE_NUM_TASKS to a value greater than the default of 1. In addition, a new setting FIRE_GEO_EM_FILE is provided in case the user wants to use an input fire geo_em file with a different name. This supersedes the old FIRE_INPUT_DIR which assumed the filename would be geo_em.d01.nc, though the default values are set so that config files using the old setting will still work as expected
* Adds a check for if the user incorrectly places a setting outside of the envvars: section for all config variables. This will work on all sections, not just fire:
* Incorporates changes from ufs-community#1375, removing the obsolete FV3_RRFS_v1beta CCPP suite.

---------

Co-authored-by: michael.lueken <Michael.Lueken@noaa.gov>

Author: mkavulich

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 826f6ff
+hash = 0de77d26
 local_path = sorc/ufs-weather-model
 required = True
 

doc/UsersGuide/BuildingRunningTesting/FIRE.rst

@@ -98,13 +98,44 @@ The fire module has the ability to print out additional messages to the log file
    fire:
      FIRE_PRINT_MSG: 1
 
-The fire module now supports OpenMP parallelization. By default, parallelization is off, but it can be enabled by setting ``OMP_NUM_THREADS_FIRE`` to a value greater than 1. In testing with the default case (200x200 domain) we have observed a 10-25% speedup with ``OMP_NUM_THREADS_FIRE: 2`` vs 1, with only marginal additional speedup for ``OMP_NUM_THREADS_FIRE>2``. However, larger domains may benefit from additional parallel threads.
+The fire module supports both OpenMP and MPI parallelization; they can be used individually or together. By default, parallelization is off, but it can be enabled at runtime with the following settings.
+
+OpenMP parallelization can be activated by setting ``OMP_NUM_THREADS_FIRE`` to a value greater than 1.
 
 .. code-block:: console
 
    fire:
      OMP_NUM_THREADS_FIRE: 2
 
+MPI parallelization can be activated by setting ``FIRE_NUM_TASKS`` to a value greater than 1.
+
+.. code-block:: console
+
+   fire:
+     FIRE_NUM_TASKS: 2
+
+We tested several combinations of parallelization methods using a 15-hour one-way coupled (ATM->FIRE) forecast; the results are shown in the table below. As a summary, MPI parallelization shows more efficient speedup per core than OpenMP parallelization, though multiple combinations resulted in significant speedup. Scaling from single-core to 2 MPI tasks resulted in nearly linear speedup (45% reduction in wallclock time). It is likely that larger domains will benefit more from additional tasks, both for OpenMP and MPI.
+
+.. list-table:: Tests of different parallelization options with 200x200 3km atmospheric grid, 200x200 100m fire grid
+    :header-rows: 1
+    :stub-columns: 1
+
+    * - Timings for parallel tests (s) 
+      - MPI 1
+      - MPI 2
+      - MPI 4
+    * - OpenMP 1
+      - 2820.20
+      - 1577.75
+      - 1343.32
+    * - OpenMP 2
+      - 2168.50
+      - 1372.19
+      - 1334.16
+    * - OpenMP 4
+      - 1867.51
+      - 1373.24
+      - 1340.81
 
 Additional boundary conditions file
 -----------------------------------
@@ -119,7 +150,9 @@ Once the file is acquired/created, you will need to specify its location in your
 
    fire:
      envvars:
-       FIRE_INPUT_DIR: /directory/containing/geo_em/file
+       FIRE_GEO_EM_FILE: /directory/containing/geo_em/file/geo_em.d01.nc
+
+Note that you can use a different filename if you prefer.
 
 Specifying a fire ignition
 ---------------------------
@@ -138,7 +171,7 @@ Here is one example of settings that can be specified for a UFS FIRE simulation:
    fire:
      envvars:
        UFS_FIRE: True
-       FIRE_INPUT_DIR: /home/fire_input
+       FIRE_GEO_EM_FILE: /home/fire_input/geo_em.d01.nc
      DT_FIRE: 0.5
      OUTPUT_DT_FIRE: 1800
      FIRE_NUM_IGNITIONS: 1

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -2130,6 +2130,9 @@ Non-default parameters for the Community Fire Behavior Model (CFBM) in SRW are s
 ``FIRE_INPUT_DIR``: (Default: "")
    Directory where fire input file (geo_em.d01.nc) can be found
 
+``FIRE_GEO_EM_FILE``: (default: '{{ [fire.envvars.FIRE_INPUT_DIR, "geo_em.d01.nc"]|path_join}}')
+   Full path to fire input geo_em netCDF file
+
 ``DT_FIRE``: (Default: 0.5)
    The fire behavior component’s integration timestep in seconds
 

parm/FV3.input.yml

@@ -7,28 +7,6 @@
 # run with.
 #
 
-FV3_RRFS_v1beta:
-  gfs_physics_nml: 
-    do_deep: False
-    do_mynnsfclay: True
-    imfdeepcnv: -1
-    imfshalcnv: -1
-    iopt_alb: 2
-    iopt_btr: 1
-    iopt_crs: 1
-    iopt_dveg: 2
-    iopt_frz: 1
-    iopt_inf: 1
-    iopt_rad: 1
-    iopt_run: 1
-    iopt_sfc: 1
-    iopt_snf: 4
-    iopt_stc: 1
-    iopt_tbot: 2
-    iopt_trs: 2
-    lsm: 2
-    lsoil_lsm: 4
-
 FV3_WoFS_v0:
   gfs_physics_nml:
     do_deep: False

parm/diag_table.FV3_RRFS_v1beta

@@ -200,7 +200,6 @@ case "${CCPP_PHYS_SUITE}" in
     varmap_file="GFSphys_var_map.txt"
     ;;
 #
-  "FV3_RRFS_v1beta" | \
   "FV3_GFS_v15_thompson_mynn_lam3km" | \
   "FV3_GFS_v17_p8" | \
   "FV3_WoFS_v0" | \

parm/field_table.FV3_RRFS_v1beta

@@ -194,7 +194,6 @@ case "${CCPP_PHYS_SUITE}" in
     varmap_file="GFSphys_var_map.txt"
     ;;
 #
-  "FV3_RRFS_v1beta" | \
   "FV3_GFS_v15_thompson_mynn_lam3km" | \
   "FV3_GFS_v17_p8" | \
   "FV3_WoFS_v0" | \

scripts/exregional_make_ics.sh

@@ -831,7 +831,7 @@ if [ $(boolify "${UFS_FIRE}") = "TRUE" ]; then
 $settings"
   fi
   # Link fire input file
-  create_symlink_to_file ${FIRE_INPUT_DIR}/geo_em.d01.nc geo_em.d01.nc FALSE
+  create_symlink_to_file ${FIRE_GEO_EM_FILE} geo_em.d01.nc FALSE
 fi
 #
 #-----------------------------------------------------------------------

scripts/exregional_make_lbcs.sh

@@ -30,7 +30,7 @@ if (BUILD_UFS)
     if(CPL_AQM)
       set(CCPP_SUITES "FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8")
     else()
-	    set(CCPP_SUITES "FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8,FV3_RRFS_v1beta,FV3_HRRR,FV3_HRRR_gf,FV3_RAP,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0,RRFS_sas")
+	    set(CCPP_SUITES "FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8,FV3_HRRR,FV3_HRRR_gf,FV3_RAP,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0,RRFS_sas")
     endif()
   endif()