diff --git a/src/coreComponents/integrationTests/solverStatisticsTests/CMakeLists.txt b/src/coreComponents/integrationTests/solverStatisticsTests/CMakeLists.txt index 75a2b9e5aea..9cb6cbf273e 100644 --- a/src/coreComponents/integrationTests/solverStatisticsTests/CMakeLists.txt +++ b/src/coreComponents/integrationTests/solverStatisticsTests/CMakeLists.txt @@ -1,6 +1,7 @@ # Specify list of tests set( solverStatisticsTests_sources - testSolverStats.cpp ) + testSolverStats.cpp + testCoupledSolverStats.cpp ) set( dependencyList mainInterface gtest ${parallelDeps} ) @@ -16,3 +17,11 @@ foreach(test ${solverStatisticsTests_sources}) COMMAND ${test_name} ) endforeach() + +# Check that only one rank writes the statistics CSV files +if( ENABLE_MPI ) + set( nranks 2 ) + geos_add_test( NAME testCoupledSolverStats_mpi + COMMAND testCoupledSolverStats + NUM_MPI_TASKS ${nranks} ) +endif() diff --git a/src/coreComponents/integrationTests/solverStatisticsTests/testCoupledSolverStats.cpp b/src/coreComponents/integrationTests/solverStatisticsTests/testCoupledSolverStats.cpp new file mode 100644 index 00000000000..a8d203e9c41 --- /dev/null +++ b/src/coreComponents/integrationTests/solverStatisticsTests/testCoupledSolverStats.cpp @@ -0,0 +1,281 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2024 TotalEnergies + * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2023-2024 Chevron + * Copyright (c) 2019- GEOS/GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +#include "integrationTests/fluidFlowTests/testCompFlowUtils.hpp" +#include "mainInterface/initialization.hpp" +#include "mainInterface/GeosxState.hpp" +#include "fileIO/Outputs/OutputBase.hpp" +#include "common/MpiWrapper.hpp" +#include "common/Path.hpp" + +#include +#include +#include + +using namespace geos; +using namespace geos::dataRepository; +using namespace geos::testing; + +CommandLineOptions g_commandLineOptions; + +//////////////////////////////// Utilities //////////////////////////////// + +string replaceAll( string str, string_view token, string_view value ) +{ + for( size_t pos = 0; ( pos = str.find( token, pos ) ) != string::npos; pos += value.size() ) + str.replace( pos, token.size(), value ); + return str; +} + +struct CSVStats +{ + bool exists = false; + integer headerCount = 0; + integer dataRowCount = 0; +}; + +CSVStats inspectCSV( string const & path, string_view headerToken ) +{ + CSVStats stats; + std::ifstream file( path ); + if( !file.is_open() ) + return stats; + stats.exists = true; + string line; + while( std::getline( file, line ) ) + { + if( line.find( headerToken ) != string::npos ) + stats.headerCount++; + else if( !line.empty() ) + stats.dataRowCount++; + } + return stats; +} + +string statsFilePath( string_view solverName, string_view suffix ) +{ + return joinPath( OutputBase::getOutputDirectory(), + "convergence", + GEOS_FMT( "{}_{}.csv", solverName, suffix ) ); +} + +void expectSingleHeaderCSV( string const & path, string_view headerToken, integer const minDataRows ) +{ + CSVStats const stats = inspectCSV( path, headerToken ); + EXPECT_TRUE( stats.exists ) << path; + EXPECT_EQ( stats.headerCount, 1 ) << path; + EXPECT_GE( stats.dataRowCount, minDataRows ) << path; +} + +//////////////////////////////// Input deck //////////////////////////////// + +// Mandel-type consolidation problem (PoroElastic_Mandel_smoke), self-contained: +// initial-displacement table files removed, constant step load instead of loadFunction. +static char const * xmlTemplate = + R"xml( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + )xml"; + +//////////////////////////////// Tests //////////////////////////////// + +struct Params +{ + string couplingType; // "FullyImplicit" or "Sequential" + string csvMode; // "all", "none", ... + string suffix; // unique per variant so output files do not collide +}; + +// Run a single coupled-statistics scenario and verify its CSV output. +static void runScenario( Params const & p ) +{ + SCOPED_TRACE( GEOS_FMT( "{}_{}", p.couplingType, p.csvMode ) ); + + // the serial and parallel variants of this test may run concurrently: keep their output files apart + string const suffix = GEOS_FMT( "{}_np{}", p.suffix, MpiWrapper::commSize() ); + stdVector< string > const solverNames = { "poroSolve" + suffix, "flowSolve" + suffix, "lagsolve" + suffix }; + + // remove files left over by a previous run, so that the checks below only see this run's output + if( MpiWrapper::commRank() == 0 ) + { + for( string const & solverName : solverNames ) + { + std::filesystem::remove( statsFilePath( solverName, "iterations" ) ); + std::filesystem::remove( statsFilePath( solverName, "convergence" ) ); + } + } + MpiWrapper::barrier(); + + // Sequential coupling forbids line search; use "None" there. + string const lineSearchAction = ( p.couplingType == "Sequential" ) ? "None" : "Attempt"; + + string xmlInput = replaceAll( xmlTemplate, "@COUPLING_TYPE@", p.couplingType ); + xmlInput = replaceAll( xmlInput, "@CSV_MODE@", p.csvMode ); + xmlInput = replaceAll( xmlInput, "@SUFFIX@", suffix ); + xmlInput = replaceAll( xmlInput, "@LINE_SEARCH_ACTION@", lineSearchAction ); + + GeosxState state( std::make_unique< CommandLineOptions >( g_commandLineOptions ) ); + ProblemManager & problem = state.getProblemManager(); + setupProblemFromXML( problem, xmlInput.data() ); + + EXPECT_FALSE( problem.runSimulation() ) << "Simulation exited early."; + + MpiWrapper::barrier(); + if( MpiWrapper::commRank() != 0 ) + return; + + string_view const iterationHeaderToken = "Number of time steps"; + string_view const convergenceHeaderToken = "Cycle number"; + string const & coupledName = solverNames[0]; + + // 1) Coupled solver CSVs: exist, exactly ONE header (a single rank writes), and data rows. + expectSingleHeaderCSV( statsFilePath( coupledName, "iterations" ), iterationHeaderToken, 2 ); // 2 forced steps + // Without subcycling, the Sequential scheme does a single pass and computes no outer-loop residual. + if( p.couplingType == "FullyImplicit" ) + { + expectSingleHeaderCSV( statsFilePath( coupledName, "convergence" ), convergenceHeaderToken, 2 ); + } + + // 2) Sub-solver CSVs: in Sequential mode the sub-solvers iterate on their own, + // so their iteration and convergence files must contain data. + if( p.couplingType == "Sequential" ) + { + for( string const & solverName : { solverNames[1], solverNames[2] } ) + { + expectSingleHeaderCSV( statsFilePath( solverName, "iterations" ), iterationHeaderToken, 2 ); + expectSingleHeaderCSV( statsFilePath( solverName, "convergence" ), convergenceHeaderToken, 2 ); + } + } + // In FullyImplicit mode sub-solvers take no independent time steps, + // so writeIterationStatsToTable() early-returns: no iteration file expected. + else + { + for( string const & solverName : { solverNames[1], solverNames[2] } ) + { + EXPECT_FALSE( inspectCSV( statsFilePath( solverName, "iterations" ), iterationHeaderToken ).exists ) << solverName; + } + } +} + +TEST( CoupledSolverStatisticsCSVTest, checkCSVOutput ) +{ + stdVector< Params > const scenarios = { + { "FullyImplicit", "all", "FIM" }, + { "Sequential", "all", "Seq" }, + }; + + for( Params const & p : scenarios ) + { + runScenario( p ); + } +} + +int main( int argc, char * * argv ) +{ + ::testing::InitGoogleTest( &argc, argv ); + g_commandLineOptions = *geos::basicSetup( argc, argv ); + int const result = RUN_ALL_TESTS(); + geos::basicCleanup(); + return result; +} diff --git a/src/coreComponents/integrationTests/solverStatisticsTests/testSolverStats.cpp b/src/coreComponents/integrationTests/solverStatisticsTests/testSolverStats.cpp index b781f669cfb..5920d78e679 100644 --- a/src/coreComponents/integrationTests/solverStatisticsTests/testSolverStats.cpp +++ b/src/coreComponents/integrationTests/solverStatisticsTests/testSolverStats.cpp @@ -79,7 +79,7 @@ static const string solverCSVOutput = name="SinglePhaseFlow" discretization="singlePhaseTPFA" targetRegions="{ Channel }" - writeStatistics="convergence" > + writeStatistics="all" > @@ -284,6 +284,7 @@ TEST( testSolverStats, testOutputFiles ) EXPECT_EQ( csvLines2[0], "Cycle number,time_n (s),dt (s),iteration,R,Rflow" ); + ASSERT_TRUE( std::remove( iterationStat.getFilename().c_str() ) == 0 ); ASSERT_TRUE( std::remove( convergenceStat.getFilename().c_str() ) == 0 ); } diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index f5dc5cdb3b1..3712dd6d203 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -1598,9 +1598,13 @@ void PhysicsSolverBase::cleanup( real64 const GEOS_UNUSED_PARAM( time_n ), getIterationStats().outputStatistics(); } - bool wrongIterationCSVOutputRequest = getIterationStats().getCSVOutputRequest() && + // CSV files are only written by rank 0 + bool const isRankZero = MpiWrapper::commRank() == 0; + bool wrongIterationCSVOutputRequest = isRankZero && + getIterationStats().getCSVOutputRequest() && !getIterationStats().getCSVOutputOpened(); - bool wrongConvergenceCSVOutputRequest = getConvergenceStats().getCSVOutputRequest() && + bool wrongConvergenceCSVOutputRequest = isRankZero && + getConvergenceStats().getCSVOutputRequest() && !getConvergenceStats().getCSVOutputOpened(); GEOS_WARNING_IF( wrongIterationCSVOutputRequest, diff --git a/src/coreComponents/physicsSolvers/SolverStatistics.cpp b/src/coreComponents/physicsSolvers/SolverStatistics.cpp index 4324bc684ef..4c1c6da0203 100644 --- a/src/coreComponents/physicsSolvers/SolverStatistics.cpp +++ b/src/coreComponents/physicsSolvers/SolverStatistics.cpp @@ -20,6 +20,7 @@ #include "SolverStatistics.hpp" #include "fileIO/Outputs/OutputBase.hpp" +#include "common/MpiWrapper.hpp" namespace geos { @@ -145,6 +146,16 @@ void IterationsStatistics::writeIterationStatsToTable() if( m_numTimeSteps == 0 || !m_logOutputRequest ) return; + // linear solver timings are reported per row, so they are reset on every rank, even when no CSV is written. + // Timings of attempts made before the first converged time step are accumulated into the first row. + real64 const setupTime = m_setupTime; + real64 const solveTime = m_solveTime; + resetSolverLinearTime(); + + // only rank 0 writes the CSV file + if( !m_CSVOutputRequest || MpiWrapper::commRank() != 0 ) + return; + m_iterationData.addRow( m_numTimeSteps, m_numTimeStepCuts, m_numSuccessfulConfigIterations, @@ -153,22 +164,20 @@ void IterationsStatistics::writeIterationStatsToTable() m_numDiscardedConfigIterations, m_numDiscardedNonlinearIterations, m_numDiscardedLinearIterations, - m_setupTime, - m_solveTime ); + setupTime, + solveTime ); if( !m_CSVOutputOpened ) { - m_logStream.open( m_iterationsFilename ); + m_csvStream.open( m_iterationsFilename ); m_iterationCSVFormatter = std::make_unique< TableCSVFormatter >( m_iterationCSVLayout ); - m_logStream << m_iterationCSVFormatter->headerToString( ); - m_CSVOutputOpened = true; + m_csvStream << m_iterationCSVFormatter->headerToString( ); + m_CSVOutputOpened = true; } - m_logStream << m_iterationCSVFormatter->dataToString( m_iterationData ); - m_logStream.flush(); + m_csvStream << m_iterationCSVFormatter->dataToString( m_iterationData ); + m_csvStream.flush(); m_iterationData.clear(); - - resetSolverLinearTime(); } void IterationsStatistics::outputStatistics() const @@ -204,8 +213,10 @@ ConvergenceStatistics::ConvergenceStatistics(): void ConvergenceStatistics::writeConvergenceStatsToTable() { - if( !m_CSVOutputRequest ) + // residual norms are global quantities: only rank 0 writes the CSV file + if( !m_CSVOutputRequest || MpiWrapper::commRank() != 0 ) return; + stdVector< TableData::CellData > residualsNormCells; residualsNormCells.emplace_back( TableData::CellData( {CellType::Value, @@ -237,14 +248,14 @@ void ConvergenceStatistics::writeConvergenceStatsToTable() } m_convergenceLayout.addColumns( header ); - m_logStream.open( m_convergenceFilename ); + m_csvStream.open( m_convergenceFilename ); m_convergenceFormatter = std::make_unique< TableCSVFormatter >( m_convergenceLayout ); - m_logStream << m_convergenceFormatter->headerToString( ); + m_csvStream << m_convergenceFormatter->headerToString( ); m_CSVOutputOpened = true; } - m_logStream << m_convergenceFormatter->dataToString( m_convergenceData ); - m_logStream.flush(); + m_csvStream << m_convergenceFormatter->dataToString( m_convergenceData ); + m_csvStream.flush(); m_convergenceData.clear(); } diff --git a/src/coreComponents/physicsSolvers/SolverStatistics.hpp b/src/coreComponents/physicsSolvers/SolverStatistics.hpp index 0cef5c39d20..56ff32b9aa4 100644 --- a/src/coreComponents/physicsSolvers/SolverStatistics.hpp +++ b/src/coreComponents/physicsSolvers/SolverStatistics.hpp @@ -182,7 +182,7 @@ class IterationsStatistics : public dataRepository::Group * @brief Close the stream output file if a file was created */ void closeFile() - { if( m_CSVOutputOpened ) { m_logStream.close(); m_CSVOutputOpened = false; } } + { if( m_CSVOutputOpened ) { m_csvStream.close(); m_CSVOutputOpened = false; } } /** * @brief Get the number of time steps @@ -238,7 +238,7 @@ class IterationsStatistics : public dataRepository::Group /// A boolean indicating whether the iteration CSV file was opened or not bool m_CSVOutputOpened = false; /// Stream output for the iteration statistics - std::ofstream m_logStream; + std::ofstream m_csvStream; /// Table Layout contenaning header for both CSV and log TableLayout m_iterationCSVLayout; /// Contain the iteration data for both CSV and log output @@ -337,7 +337,7 @@ class ConvergenceStatistics * @brief Close the stream output file if a file was created */ void closeFile() - { if( m_CSVOutputOpened ) { m_logStream.close(); m_CSVOutputOpened = false; } } + { if( m_CSVOutputOpened ) { m_csvStream.close(); m_CSVOutputOpened = false; } } private: /// The time at the beginning of the step @@ -360,7 +360,7 @@ class ConvergenceStatistics /// A boolean indicating whether the CSV file was opened or not bool m_CSVOutputOpened = false; /// Stream output for the convergence statistics - std::ofstream m_logStream; + std::ofstream m_csvStream; /// Contain the layout for both the CSV and log output. /// For a solver, output all residuals residuals name available TableLayout m_convergenceLayout; diff --git a/src/coreComponents/physicsSolvers/multiphysics/CoupledSolver.hpp b/src/coreComponents/physicsSolvers/multiphysics/CoupledSolver.hpp index c5cddf6acd1..79e628cec33 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/CoupledSolver.hpp +++ b/src/coreComponents/physicsSolvers/multiphysics/CoupledSolver.hpp @@ -576,6 +576,7 @@ class CoupledSolver : public PhysicsSolverBase forEachArgInTuple( m_solvers, [&]( auto & solver, auto ) { solver->getIterationStats().iterateTimeStepStatistics(); + solver->getIterationStats().writeIterationStatsToTable(); } ); // get out of the time loop break; @@ -589,9 +590,11 @@ class CoupledSolver : public PhysicsSolverBase // notify the solver statistics counter that this is a time step cut getIterationStats().updateTimeStepCut(); + getIterationStats().writeIterationStatsToTable(); forEachArgInTuple( m_solvers, [&]( auto & solver, auto ) { solver->getIterationStats().updateTimeStepCut(); + solver->getIterationStats().writeIterationStatsToTable(); } ); } }