From 9a7dfd8b984a86e3d29e587b6d692fa031b05f17 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Tue, 27 Jun 2023 13:25:02 +0200 Subject: [PATCH 01/18] TSMP-PDAF: change amps layer `mpi1` usage of communicator `dacomm` instead of `MPI_COMM_WORLD` --- pfsimulator/amps/mpi1/amps_finalize.c | 2 +- pfsimulator/amps/mpi1/amps_init.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pfsimulator/amps/mpi1/amps_finalize.c b/pfsimulator/amps/mpi1/amps_finalize.c index 4fee01906..89a06d82d 100644 --- a/pfsimulator/amps/mpi1/amps_finalize.c +++ b/pfsimulator/amps/mpi1/amps_finalize.c @@ -69,7 +69,7 @@ int amps_Finalize() MPI_Comm_free(&s_CommWrite); MPI_Comm_free(&s_CommWorld); - MPI_Finalize(); + /* MPI_Finalize(); */ } #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS) amps_gpu_finalize(); diff --git a/pfsimulator/amps/mpi1/amps_init.c b/pfsimulator/amps/mpi1/amps_init.c index ea8c9f8ec..e0569459f 100644 --- a/pfsimulator/amps/mpi1/amps_init.c +++ b/pfsimulator/amps/mpi1/amps_init.c @@ -35,6 +35,8 @@ #include #include +MPI_Comm dacomm; /* kuw */ + /* Global flag indicating if AMPS has been initialized */ int amps_mpi_initialized = FALSE; @@ -114,10 +116,10 @@ int amps_Init(int *argc, char **argv[]) int length; #endif - MPI_Init(argc, argv); + /* MPI_Init(argc, argv); */ amps_mpi_initialized = TRUE; - MPI_Comm_dup(MPI_COMM_WORLD, &s_CommWorld); + MPI_Comm_dup(dacomm, &s_CommWorld); MPI_Comm_size(amps_CommWorld, &s_size); MPI_Comm_rank(amps_CommWorld, &s_rank); From 48a3dce68bf1c1c0f4666f4fff8596f3c3cd7f5b Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Tue, 27 Jun 2023 13:41:53 +0200 Subject: [PATCH 02/18] TSMP-PDAF: changes to `parflow_lib` Add Vector: - PhaseRelPermGetAlpha - PhaseRelPermGetN Add PFModule - SaturationGetAlpha - SaturationGetN - GetPhaseRelPerm - GetSaturation Add function - PseudoAdvanceRichards Change time step control in `solver_richards.f90` If environment variable `FOR2131` is defined: - do a vector update of saturations in `solver_richards.f90:1718-1752` --- pfsimulator/parflow_lib/parflow_proto.h | 16 + .../parflow_lib/problem_phase_rel_perm.c | 28 ++ pfsimulator/parflow_lib/problem_saturation.c | 28 ++ pfsimulator/parflow_lib/solver_richards.c | 359 ++++++++++++++++-- 4 files changed, 396 insertions(+), 35 deletions(-) diff --git a/pfsimulator/parflow_lib/parflow_proto.h b/pfsimulator/parflow_lib/parflow_proto.h index cddf5b5b3..398525c6d 100644 --- a/pfsimulator/parflow_lib/parflow_proto.h +++ b/pfsimulator/parflow_lib/parflow_proto.h @@ -919,6 +919,8 @@ void PhaseRelPermFreeInstanceXtra(void); PFModule *PhaseRelPermNewPublicXtra(void); void PhaseRelPermFreePublicXtra(void); int PhaseRelPermSizeOfTempData(void); +Vector *PhaseRelPermGetAlpha(PFModule *this_module); +Vector *PhaseRelPermGetN(PFModule *this_module); typedef void (*PhaseSourceInvoke) (Vector *phase_source, int phase, Problem *problem, ProblemData *problem_data, double time); typedef PFModule *(*PhaseSourceNewPublicXtraInvoke) (int num_phases); @@ -972,6 +974,8 @@ typedef PFModule *(*SaturationOutputStaticInvoke) (char *file_prefix, ProblemDat /* problem_saturation.c */ void Saturation(Vector *phase_saturation, Vector *phase_pressure, Vector *phase_density, double gravity, ProblemData *problem_data, int fcn); +PFModule *SaturationGetAlpha(PFModule *this_module); +PFModule *SaturationGetN(PFModule *this_module); PFModule *SaturationInitInstanceXtra(Grid *grid, double *temp_data); void SaturationFreeInstanceXtra(void); PFModule *SaturationNewPublicXtra(void); @@ -1148,6 +1152,8 @@ PFModule *SolverRichardsNewPublicXtra(char *name); void SolverRichardsFreePublicXtra(void); int SolverRichardsSizeOfTempData(void); ProblemData *GetProblemDataRichards(PFModule *this_module); +PFModule *GetPhaseRelPerm(PFModule *this_module); +PFModule *GetSaturation(PFModule *this_module); Problem *GetProblemRichards(PFModule *this_module); PFModule *GetICPhasePressureRichards(PFModule *this_module); void AdvanceRichards(PFModule *this_module, @@ -1159,6 +1165,16 @@ void AdvanceRichards(PFModule *this_module, Vector ** porosity_out, Vector ** saturation_out ); +// PDAF: this function is for initialization of OASIS only +void PseudoAdvanceRichards(PFModule *this_module, + double start_time, /* Starting time */ + double stop_time, /* Stopping time */ + PFModule *time_step_control, /* Use this module to control timestep if supplied */ + Vector * evap_trans, /* Flux from land surface model */ + Vector ** pressure_out, /* Output vars */ + Vector ** porosity_out, + Vector ** saturation_out + ); void SetupRichards(PFModule *this_module); diff --git a/pfsimulator/parflow_lib/problem_phase_rel_perm.c b/pfsimulator/parflow_lib/problem_phase_rel_perm.c index bfbd696a5..89287123c 100644 --- a/pfsimulator/parflow_lib/problem_phase_rel_perm.c +++ b/pfsimulator/parflow_lib/problem_phase_rel_perm.c @@ -2175,3 +2175,31 @@ int PhaseRelPermSizeOfTempData() return sz; } + +/*----------------------------------------------------------------------- + * PhaseRelPermGetAlpha + *-----------------------------------------------------------------------*/ + +Vector *PhaseRelPermGetAlpha(PFModule *this_module) +{ + PublicXtra *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module); + + Type1 *dummy1; + dummy1 = (Type1*)(public_xtra->data); + + return (dummy1 -> alpha_values); +} + +/*----------------------------------------------------------------------- + * PhaseRelPermGetN + *-----------------------------------------------------------------------*/ + +Vector *PhaseRelPermGetN(PFModule *this_module) +{ + PublicXtra *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module); + + Type1 *dummy1; + dummy1 = (Type1*)(public_xtra->data); + + return (dummy1 -> n_values); +} diff --git a/pfsimulator/parflow_lib/problem_saturation.c b/pfsimulator/parflow_lib/problem_saturation.c index 3828da1d6..0320251a6 100644 --- a/pfsimulator/parflow_lib/problem_saturation.c +++ b/pfsimulator/parflow_lib/problem_saturation.c @@ -654,6 +654,34 @@ void Saturation( } /* End switch */ } +/*-------------------------------------------------------------------------- + * SaturationGetAlpha + *--------------------------------------------------------------------------*/ + +PFModule *SaturationGetAlpha(PFModule *this_module) +{ + PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module); + + Type1 *dummy1; + dummy1 = (Type1 *)(public_xtra -> data); + + return (dummy1 -> alpha_values); +} + +/*-------------------------------------------------------------------------- + * SaturationGetN + *--------------------------------------------------------------------------*/ + +PFModule *SaturationGetN(PFModule *this_module) +{ + PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module); + + Type1 *dummy1; + dummy1 = (Type1 *)(public_xtra -> data); + + return (dummy1 -> n_values); +} + /*-------------------------------------------------------------------------- * SaturationInitInstanceXtra *--------------------------------------------------------------------------*/ diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index 3c9c86f9b..a8619dcb8 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -1577,6 +1577,9 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time double sw_lon = .0; #endif +int istep = 1; +char filename[2048]; + #ifdef HAVE_CLM Grid *grid = (instance_xtra->grid); Subgrid *subgrid; @@ -1597,8 +1600,8 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time int fflag, fstart, fstop; // IMF: index w/in 3D forcing array corresponding to istep int n, c; // IMF: index vars for looping over subgrid data BH: added c int ind_veg; /*BH: temporary variable to store vegetation index */ - int Stepcount = 0; /* Added for transient EvapTrans file management - NBE */ - int Loopcount = 0; /* Added for transient EvapTrans file management - NBE */ + /* int Stepcount = 0; /\* Added for transient EvapTrans file management - NBE *\/ */ + /* int Loopcount = 0; /\* Added for transient EvapTrans file management - NBE *\/ */ double sw=NAN, lw=NAN, prcp=NAN, tas=NAN, u=NAN, v=NAN, patm=NAN, qatm=NAN; // IMF: 1D forcing vars (local to AdvanceRichards) double lai[18], sai[18], z0m[18], displa[18]; /*BH: array with lai/sai/z0m/displa values for each veg class */ double *sw_data = NULL; @@ -1663,6 +1666,8 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time char file_prefix[2048], file_type[2048], file_postfix[2048]; char nc_postfix[2048]; + int Stepcount = 0; /* Added for transient EvapTrans file management - NBE */ + int Loopcount = 0; /* Added for transient EvapTrans file management - NBE */ int first_tstep = 1; sprintf(file_prefix, "%s", GlobalsOutFileName); @@ -1673,6 +1678,8 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time int nlat = GetInt("ComputationalGrid.NY"); double pfl_step = GetDouble("TimeStep.Value"); double pfl_stop = GetDouble("TimingInfo.StopTime"); + // PDAF: getting start time + double pfl_start = GetDouble("TimingInfo.StartTime"); int is; ForSubgridI(is, GridSubgrids(grid)) @@ -1691,8 +1698,8 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time dx = SubgridDX(subgrid); dy = SubgridDY(subgrid); - CALL_oas_pfl_define(nx, ny, dx, dy, ix, iy, sw_lon, sw_lat, nlon, nlat, - pfl_step, pfl_stop); + // CALL_oas_pfl_define(nx, ny, dx, dy, ix, iy, sw_lon, sw_lat, nlon, nlat, + // pfl_step, pfl_stop); } amps_Sync(amps_CommWorld); #endif // end to HAVE_OAS3 CALL @@ -1718,6 +1725,23 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time } dt = cdt; +#ifdef FOR2131 + PFModuleInvokeType(SaturationInvoke, problem_saturation, + (instance_xtra -> saturation, instance_xtra -> pressure, + instance_xtra -> density, gravity, problem_data, + CALCFCN)); + + handle = InitVectorUpdate(instance_xtra -> saturation, VectorUpdateAll); + FinalizeVectorUpdate(handle); + + //sprintf(file_postfix, "pressIC.%05d",instance_xtra -> file_number); +// WritePFBinary(file_prefix, file_postfix, instance_xtra -> pressure); + //sprintf(file_postfix, "saturIC.%05d",instance_xtra -> file_number); +// WritePFBinary(file_prefix, file_postfix, instance_xtra -> saturation); + // sprintf(file_postfix, "permIC.%05d",instance_xtra -> file_number); + // WritePFBinary(file_prefix, file_postfix, ProblemDataPermeabilityX(problem_data)); + +#endif /* * Check to see if pressure solves are requested * start_count < 0 implies that subsurface data ONLY is requested @@ -1745,6 +1769,19 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time { if (t == ct) { + if (time_step_control) + { + PFModuleInvokeType(SelectTimeStepInvoke, time_step_control, + (&dt, &dt_info, t, problem, + problem_data)); + } + else + { + PFModuleInvokeType(SelectTimeStepInvoke, select_time_step, + (&dt, &dt_info, t, problem, + problem_data)); + } + ct += cdt; //CPS oasis exchange @@ -2438,7 +2475,30 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time //#endif //End of call to CLM + /* NBE counter for reusing CLM input files */ + clm_next += 1; + if (clm_next > clm_skip) + { + istep = istep + 1; + clm_next = 1; + } // NBE + + //istep = istep + 1; + EndTiming(CLMTimingIndex); + + + /* ============================================================= + * NBE: It looks like the time step isn't really scaling the CLM + * inputs, but the looping flag is working as intended as + * of 2014-04-06. + * + * It is using the different time step counter BUT then it + * isn't scaling the inputs properly. + * ============================================================= */ +#endif // End of call to CLM + + istep = istep + 1; /******************************************/ /* read transient evap trans flux file */ /******************************************/ @@ -2492,28 +2552,28 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time } - /* NBE counter for reusing CLM input files */ - clm_next += 1; - if (clm_next > clm_skip) - { - istep = istep + 1; - clm_next = 1; - } // NBE +// /* NBE counter for reusing CLM input files */ +// clm_next += 1; +// if (clm_next > clm_skip) +// { +// istep = istep + 1; +// clm_next = 1; +// } // NBE - //istep = istep + 1; +// //istep = istep + 1; - EndTiming(CLMTimingIndex); +// EndTiming(CLMTimingIndex); - /* ============================================================= - * NBE: It looks like the time step isn't really scaling the CLM - * inputs, but the looping flag is working as intended as - * of 2014-04-06. - * - * It is using the different time step counter BUT then it - * isn't scaling the inputs properly. - * ============================================================= */ -#endif +// /* ============================================================= +// * NBE: It looks like the time step isn't really scaling the CLM +// * inputs, but the looping flag is working as intended as +// * of 2014-04-06. +// * +// * It is using the different time step counter BUT then it +// * isn't scaling the inputs properly. +// * ============================================================= */ +// #endif } //Endif to check whether an entire dt is complete converged = 1; @@ -2535,18 +2595,18 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time /*******************************************************************/ if (converged) { - if (time_step_control) - { - PFModuleInvokeType(SelectTimeStepInvoke, time_step_control, - (&dt, &dt_info, t, problem, - problem_data)); - } - else - { - PFModuleInvokeType(SelectTimeStepInvoke, select_time_step, - (&dt, &dt_info, t, problem, - problem_data)); - } + // if (time_step_control) + // { + // PFModuleInvokeType(SelectTimeStepInvoke, time_step_control, + // (&dt, &dt_info, t, problem, + // problem_data)); + // } + // else + // { + // PFModuleInvokeType(SelectTimeStepInvoke, select_time_step, + // (&dt, &dt_info, t, problem, + // problem_data)); + // } PFVCopy(instance_xtra->density, instance_xtra->old_density); PFVCopy(instance_xtra->saturation, @@ -2969,7 +3029,7 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time /*************************************************************** * modify land surface pressures * ***************************************************************/ - + if (public_xtra->reset_surface_pressure == 1) { GrGeomSolid *gr_domain = ProblemDataGrDomain(problem_data); @@ -4048,6 +4108,185 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time *saturation_out = instance_xtra->saturation; } +// Compare to AdvanceRichards() in parflow v3.9.0 +PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Starting time */ + double stop_time, /* Stopping time */ + PFModule * time_step_control, /* Use this module to control timestep if supplied */ + Vector * evap_trans, /* Flux from land surface model */ + Vector ** pressure_out, /* Output vars */ + Vector ** porosity_out, Vector ** saturation_out) +{ + //>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) + printf("Pseudo richard for OAS init\n"); + //>>TSMP-PDAF internal change end + + PublicXtra *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module); + InstanceXtra *instance_xtra = + (InstanceXtra*)PFModuleInstanceXtra(this_module); + Problem *problem = (public_xtra->problem); + + int max_iterations = (public_xtra->max_iterations); + int print_satur = (public_xtra->print_satur); + int print_wells = (public_xtra->print_wells); + + PFModule *problem_saturation = (instance_xtra->problem_saturation); + PFModule *phase_density = (instance_xtra->phase_density); + PFModule *select_time_step = (instance_xtra->select_time_step); + PFModule *l2_error_norm = (instance_xtra->l2_error_norm); + PFModule *nonlin_solver = (instance_xtra->nonlin_solver); + + ProblemData *problem_data = (instance_xtra->problem_data); + + int start_count = ProblemStartCount(problem); + double dump_interval = ProblemDumpInterval(problem); + + Vector *porosity = ProblemDataPorosity(problem_data); + Vector *evap_trans_sum = instance_xtra->evap_trans_sum; + Vector *overland_sum = instance_xtra->overland_sum; /* sk: Vector of outflow at the boundary */ + + if (evap_trans == NULL) + { + evap_trans = instance_xtra->evap_trans; + } + +#ifdef HAVE_OAS3 + Grid *grid = (instance_xtra->grid); + Subgrid *subgrid; + Subvector *p_sub, *s_sub, *et_sub, *m_sub, *po_sub, *dz_sub; + double *pp, *sp, *et, *ms, *po_dat, *dz_dat; + double sw_lat = .0; + double sw_lon = .0; +#endif + +//>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) +// #ifdef HAVE_CLM +// Grid *grid = (instance_xtra->grid); +// Subgrid *subgrid; +// Subvector *p_sub, *s_sub, *et_sub, *m_sub, *po_sub, *dz_sub; +// double *pp, *sp, *et, *ms, *po_dat, *dz_dat; + +// /* IMF: For CLM met forcing (local to AdvanceRichards) */ +// int istep; // IMF: counter for clm output times + +// /* NBE added for clm reuse of inputs */ +// int clm_next = 1; //NBE: Counter for reuse loop +// int clm_skip = public_xtra->clm_reuse_count; // NBE:defaults to 1 +// int clm_write_logs = public_xtra->clm_write_logs; // NBE: defaults to 1, disables log file writing if 0 +// int clm_last_rst = public_xtra->clm_last_rst; // Reuse of the RST file +// int clm_daily_rst = public_xtra->clm_daily_rst; // Daily or hourly RST files, defaults to daily + +// int fstep = INT_MIN; +// int fflag, fstart, fstop; // IMF: index w/in 3D forcing array corresponding to istep +// int n, c; // IMF: index vars for looping over subgrid data BH: added c +// int ind_veg; /*BH: temporary variable to store vegetation index */ +// double sw=NAN, lw=NAN, prcp=NAN, tas=NAN, u=NAN, v=NAN, patm=NAN, qatm=NAN; // IMF: 1D forcing vars (local to AdvanceRichards) +// double lai[18], sai[18], z0m[18], displa[18]; /*BH: array with lai/sai/z0m/displa values for each veg class */ +// double *sw_data = NULL; +// double *lw_data = NULL; +// double *prcp_data = NULL; // IMF: 2D forcing vars (SubvectorData) (local to AdvanceRichards) +// double *tas_data = NULL; +// double *u_data = NULL; +// double *v_data = NULL; +// double *patm_data = NULL; +// double *qatm_data = NULL; +// double *lai_data = NULL; +// /*BH*/ double *sai_data = NULL; +// /*BH*/ double *z0m_data = NULL; +// /*BH*/ double *displa_data = NULL; +// /*BH*/ double *veg_map_data = NULL; +// /*BH*/ /*will fail if veg_map_data is declared as int */ +// char filename[2048]; // IMF: 1D input file name *or* 2D/3D input file base name +// Subvector *sw_forc_sub, *lw_forc_sub, *prcp_forc_sub, *tas_forc_sub, *u_forc_sub, *v_forc_sub, *patm_forc_sub, *qatm_forc_sub, *lai_forc_sub, *sai_forc_sub, *z0m_forc_sub, *displa_forc_sub, *veg_map_forc_sub; /*BH: added LAI/SAI/Z0M/DISPLA/vegmap */ + +// /* Slopes */ +// Subvector *slope_x_sub, *slope_y_sub; +// double *slope_x_data, *slope_y_data; + +// /* IMF: For writing CLM output */ +// Subvector *eflx_lh_tot_sub, *eflx_lwrad_out_sub, *eflx_sh_tot_sub, +// *eflx_soil_grnd_sub, *qflx_evap_tot_sub, *qflx_evap_grnd_sub, +// *qflx_evap_soi_sub, *qflx_evap_veg_sub, *qflx_tran_veg_sub, +// *qflx_infl_sub, *swe_out_sub, *t_grnd_sub, *tsoil_sub, *irr_flag_sub, +// *qflx_qirr_sub, *qflx_qirr_inst_sub; + +// double *eflx_lh, *eflx_lwrad, *eflx_sh, *eflx_grnd, *qflx_tot, *qflx_grnd, +// *qflx_soi, *qflx_eveg, *qflx_tveg, *qflx_in, *swe, *t_g, *t_soi, *iflag, +// *qirr, *qirr_inst; +// int clm_file_dir_length; +// #endif +//>>TSMP-PDAF internal change end + + int any_file_dumped; + int clm_file_dumped; + int dump_files = 0; + + int retval; + int converged; + int take_more_time_steps; + int conv_failures; + int max_failures = public_xtra->max_convergence_failures; + + double t; + double dt = 0.0; + double ct = 0.0; + double cdt = 0.0; + double print_dt; + double dtmp, err_norm; + double gravity = ProblemGravity(problem); + + VectorUpdateCommHandle *handle; + + char dt_info; + char file_prefix[2048], file_type[2048], file_postfix[2048]; + char nc_postfix[2048]; + +//>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) + // int Stepcount = 0; /* Added for transient EvapTrans file management - NBE */ + // int Loopcount = 0; /* Added for transient EvapTrans file management - NBE */ + // int first_tstep = 1; +//>>TSMP-PDAF internal change end + + sprintf(file_prefix, "%s", GlobalsOutFileName); + + //CPS oasis definition phase +#ifdef HAVE_OAS3 + int nlon = GetInt("ComputationalGrid.NX"); + int nlat = GetInt("ComputationalGrid.NY"); + double pfl_step = GetDouble("TimeStep.Value"); + double pfl_stop = GetDouble("TimingInfo.StopTime"); + //>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) + double pfl_start = GetDouble("TimingInfo.StartTime"); + //>>TSMP-PDAF internal change end + + int is; + ForSubgridI(is, GridSubgrids(grid)) + { + double dx, dy; + int nx, ny, ix, iy; + + subgrid = GridSubgrid(grid, is); + + nx = SubgridNX(subgrid); + ny = SubgridNY(subgrid); + + ix = SubgridIX(subgrid); + iy = SubgridIY(subgrid); + + dx = SubgridDX(subgrid); + dy = SubgridDY(subgrid); + + //>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) + // gw only init once + // kuw + CALL_oas_pfl_define(nx, ny, dx, dy, ix, iy, sw_lon, sw_lat, nlon, nlat, + pfl_step, pfl_stop); + // kuw end + // gw end + //>>TSMP-PDAF internal change end + } + amps_Sync(amps_CommWorld); +#endif // end to HAVE_OAS3 CALL +} void TeardownRichards(PFModule * this_module) @@ -5794,6 +6033,16 @@ GetProblemDataRichards(PFModule * this_module) return(instance_xtra->problem_data); } +PFModule *GetPhaseRelPerm(PFModule *this_module){ + InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module); + return (instance_xtra -> phase_rel_perm); +} + +PFModule *GetSaturation(PFModule *this_module){ + InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module); + return (instance_xtra -> problem_saturation); +} + Problem * GetProblemRichards(PFModule * this_module) { @@ -5810,3 +6059,43 @@ GetICPhasePressureRichards(PFModule * this_module) return(instance_xtra->ic_phase_pressure); } + +Vector *GetPressureRichards(PFModule *this_module) +{ + InstanceXtra *instance_xtra = + (InstanceXtra *)PFModuleInstanceXtra(this_module); + + return (instance_xtra -> pressure); +} + +Vector *GetSaturationRichards(PFModule *this_module) +{ + InstanceXtra *instance_xtra = + (InstanceXtra *)PFModuleInstanceXtra(this_module); + + return (instance_xtra -> saturation); +} + +Vector *GetDensityRichards(PFModule *this_module) +{ + InstanceXtra *instance_xtra = + (InstanceXtra *)PFModuleInstanceXtra(this_module); + + return (instance_xtra -> density); +} + +int GetEvapTransFile(PFModule *this_module) +{ + PublicXtra *public_xtra = + (PublicXtra *)PFModulePublicXtra(this_module); + + return (public_xtra -> evap_trans_file); +} + +char *GetEvapTransFilename(PFModule *this_module) +{ + PublicXtra *public_xtra = + (PublicXtra *)PFModulePublicXtra(this_module); + + return (public_xtra -> evap_trans_filename); +} From 9307ae5cfeb7afc0989f17122a590077dcdf4ea1 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 13 Jul 2023 07:40:53 +0200 Subject: [PATCH 03/18] TSMP-PDAF: comment out saturation update in solver_richards General idea: Removal of environment variable FOR2131 in TSMP-PDAF --- pfsimulator/parflow_lib/solver_richards.c | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index a8619dcb8..195f348cc 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -1725,23 +1725,23 @@ char filename[2048]; } dt = cdt; -#ifdef FOR2131 - PFModuleInvokeType(SaturationInvoke, problem_saturation, - (instance_xtra -> saturation, instance_xtra -> pressure, - instance_xtra -> density, gravity, problem_data, - CALCFCN)); - - handle = InitVectorUpdate(instance_xtra -> saturation, VectorUpdateAll); - FinalizeVectorUpdate(handle); - - //sprintf(file_postfix, "pressIC.%05d",instance_xtra -> file_number); -// WritePFBinary(file_prefix, file_postfix, instance_xtra -> pressure); - //sprintf(file_postfix, "saturIC.%05d",instance_xtra -> file_number); -// WritePFBinary(file_prefix, file_postfix, instance_xtra -> saturation); - // sprintf(file_postfix, "permIC.%05d",instance_xtra -> file_number); - // WritePFBinary(file_prefix, file_postfix, ProblemDataPermeabilityX(problem_data)); - -#endif +/* #ifdef FOR2131 */ +/* PFModuleInvokeType(SaturationInvoke, problem_saturation, */ +/* (instance_xtra -> saturation, instance_xtra -> pressure, */ +/* instance_xtra -> density, gravity, problem_data, */ +/* CALCFCN)); */ + +/* handle = InitVectorUpdate(instance_xtra -> saturation, VectorUpdateAll); */ +/* FinalizeVectorUpdate(handle); */ + +/* //sprintf(file_postfix, "pressIC.%05d",instance_xtra -> file_number); */ +/* // WritePFBinary(file_prefix, file_postfix, instance_xtra -> pressure); */ +/* //sprintf(file_postfix, "saturIC.%05d",instance_xtra -> file_number); */ +/* // WritePFBinary(file_prefix, file_postfix, instance_xtra -> saturation); */ +/* // sprintf(file_postfix, "permIC.%05d",instance_xtra -> file_number); */ +/* // WritePFBinary(file_prefix, file_postfix, ProblemDataPermeabilityX(problem_data)); */ + +/* #endif */ /* * Check to see if pressure solves are requested * start_count < 0 implies that subsurface data ONLY is requested From f5ed22a9068aca4fe745c0918ab3506c51d4b6c2 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 18 Oct 2023 13:54:18 +0200 Subject: [PATCH 04/18] parflow: fix compilation error, correct type should be Vector ``` eTSMP/parflow/pfsimulator/parflow_lib/problem_saturation.c(668): error #120: return value type does not match the function type return (dummy1 -> alpha_values); ^ eTSMP/parflow/pfsimulator/parflow_lib/problem_saturation.c(682): error #120: return value type does not match the function type return (dummy1 -> n_values); ^ ``` --- pfsimulator/parflow_lib/parflow_proto.h | 4 ++-- pfsimulator/parflow_lib/problem_saturation.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfsimulator/parflow_lib/parflow_proto.h b/pfsimulator/parflow_lib/parflow_proto.h index 32f4cd12a..975d861c8 100644 --- a/pfsimulator/parflow_lib/parflow_proto.h +++ b/pfsimulator/parflow_lib/parflow_proto.h @@ -974,8 +974,8 @@ typedef PFModule *(*SaturationOutputStaticInvoke) (char *file_prefix, ProblemDat /* problem_saturation.c */ void Saturation(Vector *phase_saturation, Vector *phase_pressure, Vector *phase_density, double gravity, ProblemData *problem_data, int fcn); -PFModule *SaturationGetAlpha(PFModule *this_module); -PFModule *SaturationGetN(PFModule *this_module); +Vector *SaturationGetAlpha(PFModule *this_module); +Vector *SaturationGetN(PFModule *this_module); PFModule *SaturationInitInstanceXtra(Grid *grid, double *temp_data); void SaturationFreeInstanceXtra(void); PFModule *SaturationNewPublicXtra(void); diff --git a/pfsimulator/parflow_lib/problem_saturation.c b/pfsimulator/parflow_lib/problem_saturation.c index 0320251a6..86a4494da 100644 --- a/pfsimulator/parflow_lib/problem_saturation.c +++ b/pfsimulator/parflow_lib/problem_saturation.c @@ -658,7 +658,7 @@ void Saturation( * SaturationGetAlpha *--------------------------------------------------------------------------*/ -PFModule *SaturationGetAlpha(PFModule *this_module) +Vector *SaturationGetAlpha(PFModule *this_module) { PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module); @@ -672,7 +672,7 @@ PFModule *SaturationGetAlpha(PFModule *this_module) * SaturationGetN *--------------------------------------------------------------------------*/ -PFModule *SaturationGetN(PFModule *this_module) +Vector *SaturationGetN(PFModule *this_module) { PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module); From 5e9bd79b936c38f8c5abfaf82ac84f982112aee1 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 18 Oct 2023 14:04:01 +0200 Subject: [PATCH 05/18] parflow: fix compilation error, correct type of PseudoAdvanceRichards ``` eTSMP/parflow/pfsimulator/parflow_lib/solver_richards.c(4112): error #303: explicit type is missing ("int" assumed) PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Starting time */ ^ eTSMP/parflow/pfsimulator/parflow_lib/solver_richards.c(4112): error #147: declaration is incompatible with "void PseudoAdvanceRichards(PFModule *, double, double, PFModule *, Vector *, Vector **, Vector **, Vector **)" (declared at line 1169 of "/p/scratch/cjibg36/jibg3683/DATAASSIMILATION/TSMP-PDAF/eTSMP/parflow/pfsimulator/parflow_lib/parflow_proto.h") PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Starting time */ ^ ``` --- pfsimulator/parflow_lib/solver_richards.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index faf863256..55da14d4e 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -4109,6 +4109,7 @@ char filename[2048]; } // Compare to AdvanceRichards() in parflow v3.9.0 +void PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Starting time */ double stop_time, /* Stopping time */ PFModule * time_step_control, /* Use this module to control timestep if supplied */ From 3aefeff7ad970a74d9ac91ae949ace0e900ea4e2 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Sat, 11 Nov 2023 11:34:16 +0100 Subject: [PATCH 06/18] Installed header files --- CMakeLists.txt | 5 ++ pfsimulator/amps/CMakeLists.txt | 6 +++ pfsimulator/parflow_lib/CMakeLists.txt | 64 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb45fc071..9d7a42ac2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ set (CMAKE_C_STANDARD 11) include (CheckCCompilerFlag) include (CheckCXXCompilerFlag) +include(GNUInstallDirs) # Check if compiler supports extra debugging flags foreach (DEBUG_FLAG "-ggdb" "-g3") @@ -546,6 +547,10 @@ if ( ${PARFLOW_HAVE_CLM} ) configure_file (pfsimulator/clm/parflow_config.F90.in ${PROJECT_BINARY_DIR}/pfsimulator/clm/parflow_config.F90) endif ( ${PARFLOW_HAVE_CLM} ) +# Install headers +install(FILES ${CMAKE_BINARY_DIR}/include/parflow_config.h + ${CMAKE_BINARY_DIR}/include/pfversion.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) #----------------------------------------------------------------------------- # CMAKE Subdirectories #----------------------------------------------------------------------------- diff --git a/pfsimulator/amps/CMakeLists.txt b/pfsimulator/amps/CMakeLists.txt index 35cdacce1..b626e6de6 100644 --- a/pfsimulator/amps/CMakeLists.txt +++ b/pfsimulator/amps/CMakeLists.txt @@ -46,5 +46,11 @@ set(AMPS_SCRIPTS run) string(REGEX REPLACE "([^;]+)" "${PARFLOW_AMPS_LAYER}/\\1" AMPS_SCRIPTS "${AMPS_SCRIPTS}") install(FILES ${AMPS_SCRIPTS} DESTINATION bin) +# Install headers +install(FILES common/amps_common.h + ${PARFLOW_AMPS_LAYER}/amps.h + ${PARFLOW_AMPS_LAYER}/amps_proto.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) + add_subdirectory (test/src) diff --git a/pfsimulator/parflow_lib/CMakeLists.txt b/pfsimulator/parflow_lib/CMakeLists.txt index 206a157ca..d09385305 100644 --- a/pfsimulator/parflow_lib/CMakeLists.txt +++ b/pfsimulator/parflow_lib/CMakeLists.txt @@ -282,3 +282,67 @@ if (${PARFLOW_HAVE_NETCDF}) target_include_directories (pfsimulator PUBLIC "${netCDF_INCLUDE_DIRS}") target_include_directories (pfsimulator PUBLIC "${NETCDF_INCLUDE_DIRS}") endif (${PARFLOW_HAVE_NETCDF}) + +# Install headers +set (HEADER_FILES + backend_mapping.h + background.h + bc_pressure.h + char_vector.h + clustering.h + communication.h + computation.h + drv_gridmodule.h + drv_module.h + drv_tilemodule.h + f2c.h + fgetopt.h + file_versions.h + general.h + geometry.h + geostats.h + globals.h + grgeometry.h + grgeom_list.h + grgeom_octree.h + grid.h + hbt.h + hypre_dependences.h + index_space.h + info_header.h + input_database.h + kinsol_dependences.h + lb.h + llnlmath.h + llnltyps.h + logging.h + loops.h + matrix.h + metadata.h + nl_function_eval.h + n_vector.h + parflow.h + parflow_netcdf.h + parflow_proto_f90.h + parflow_proto_f.h + parflow_proto.h + pf_cudaloops.h + pf_cudamalloc.h + pf_devices.h + pf_hypre.h + pf_kokkosloops.h + pf_kokkosmalloc.h + pf_module.h + pf_omploops.h + problem_bc.h + problem_eval.h + problem.h + region.h + solver.h + time_cycle_data.h + timing.h + vector.h + well.h +) +install(FILES ${HEADER_FILES} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) From 6707e801a8c7a9d4c250ec7e80ea819babb1adea Mon Sep 17 00:00:00 2001 From: kvrigor Date: Sat, 11 Nov 2023 11:34:16 +0100 Subject: [PATCH 07/18] Installed header files --- CMakeLists.txt | 5 ++ pfsimulator/amps/CMakeLists.txt | 6 +++ pfsimulator/parflow_lib/CMakeLists.txt | 64 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb45fc071..9d7a42ac2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ set (CMAKE_C_STANDARD 11) include (CheckCCompilerFlag) include (CheckCXXCompilerFlag) +include(GNUInstallDirs) # Check if compiler supports extra debugging flags foreach (DEBUG_FLAG "-ggdb" "-g3") @@ -546,6 +547,10 @@ if ( ${PARFLOW_HAVE_CLM} ) configure_file (pfsimulator/clm/parflow_config.F90.in ${PROJECT_BINARY_DIR}/pfsimulator/clm/parflow_config.F90) endif ( ${PARFLOW_HAVE_CLM} ) +# Install headers +install(FILES ${CMAKE_BINARY_DIR}/include/parflow_config.h + ${CMAKE_BINARY_DIR}/include/pfversion.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) #----------------------------------------------------------------------------- # CMAKE Subdirectories #----------------------------------------------------------------------------- diff --git a/pfsimulator/amps/CMakeLists.txt b/pfsimulator/amps/CMakeLists.txt index 35cdacce1..b626e6de6 100644 --- a/pfsimulator/amps/CMakeLists.txt +++ b/pfsimulator/amps/CMakeLists.txt @@ -46,5 +46,11 @@ set(AMPS_SCRIPTS run) string(REGEX REPLACE "([^;]+)" "${PARFLOW_AMPS_LAYER}/\\1" AMPS_SCRIPTS "${AMPS_SCRIPTS}") install(FILES ${AMPS_SCRIPTS} DESTINATION bin) +# Install headers +install(FILES common/amps_common.h + ${PARFLOW_AMPS_LAYER}/amps.h + ${PARFLOW_AMPS_LAYER}/amps_proto.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) + add_subdirectory (test/src) diff --git a/pfsimulator/parflow_lib/CMakeLists.txt b/pfsimulator/parflow_lib/CMakeLists.txt index 206a157ca..d09385305 100644 --- a/pfsimulator/parflow_lib/CMakeLists.txt +++ b/pfsimulator/parflow_lib/CMakeLists.txt @@ -282,3 +282,67 @@ if (${PARFLOW_HAVE_NETCDF}) target_include_directories (pfsimulator PUBLIC "${netCDF_INCLUDE_DIRS}") target_include_directories (pfsimulator PUBLIC "${NETCDF_INCLUDE_DIRS}") endif (${PARFLOW_HAVE_NETCDF}) + +# Install headers +set (HEADER_FILES + backend_mapping.h + background.h + bc_pressure.h + char_vector.h + clustering.h + communication.h + computation.h + drv_gridmodule.h + drv_module.h + drv_tilemodule.h + f2c.h + fgetopt.h + file_versions.h + general.h + geometry.h + geostats.h + globals.h + grgeometry.h + grgeom_list.h + grgeom_octree.h + grid.h + hbt.h + hypre_dependences.h + index_space.h + info_header.h + input_database.h + kinsol_dependences.h + lb.h + llnlmath.h + llnltyps.h + logging.h + loops.h + matrix.h + metadata.h + nl_function_eval.h + n_vector.h + parflow.h + parflow_netcdf.h + parflow_proto_f90.h + parflow_proto_f.h + parflow_proto.h + pf_cudaloops.h + pf_cudamalloc.h + pf_devices.h + pf_hypre.h + pf_kokkosloops.h + pf_kokkosmalloc.h + pf_module.h + pf_omploops.h + problem_bc.h + problem_eval.h + problem.h + region.h + solver.h + time_cycle_data.h + timing.h + vector.h + well.h +) +install(FILES ${HEADER_FILES} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) From dbeda10622f0abaa9b2690f1304832b474ab30f2 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 22 Nov 2023 19:49:42 +0100 Subject: [PATCH 08/18] Installed missing header files: oas3_coupler.h, oas3_external.h, Parflow.hxx --- pfsimulator/amps/CMakeLists.txt | 10 +++++++--- pfsimulator/parflow_lib/CMakeLists.txt | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pfsimulator/amps/CMakeLists.txt b/pfsimulator/amps/CMakeLists.txt index b626e6de6..43ad9f22e 100644 --- a/pfsimulator/amps/CMakeLists.txt +++ b/pfsimulator/amps/CMakeLists.txt @@ -4,6 +4,11 @@ set(COMMON_SRC_FILES amps_abort.c amps_clear.c amps_clock.c amps_exchange.c amps_ffopen.c amps_find_powers.c amps_fopen.c amps_invoice.c amps_io.c amps_newhandle.c amps_newpackage.c amps_sfbcast.c amps_sfclose.c amps_sfopen.c amps_wait.c signal.c amps_print.c) + +list(APPEND HEADER_FILES common/amps_common.h + ${PARFLOW_AMPS_LAYER}/amps.h + ${PARFLOW_AMPS_LAYER}/amps_proto.h) + string(REGEX REPLACE "([^;]+)" "common/\\1" COMMON_SRC_FILES "${COMMON_SRC_FILES}") # Sets AMPS_SRC_FILES for AMPS transport layer being used in build @@ -36,6 +41,7 @@ if(${PARFLOW_HAVE_OAS3}) target_link_libraries(amps ${NETCDF_Fortran_LIBRARY}) target_include_directories(amps PUBLIC "${NETCDF_Fortran_INCLUDES}") + list(APPEND HEADER_FILES oas3/oas3_coupler.h oas3/oas3_external.h) endif(${PARFLOW_HAVE_OAS3}) @@ -47,9 +53,7 @@ string(REGEX REPLACE "([^;]+)" "${PARFLOW_AMPS_LAYER}/\\1" AMPS_SCRIPTS "${AMPS_ install(FILES ${AMPS_SCRIPTS} DESTINATION bin) # Install headers -install(FILES common/amps_common.h - ${PARFLOW_AMPS_LAYER}/amps.h - ${PARFLOW_AMPS_LAYER}/amps_proto.h +install(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) add_subdirectory (test/src) diff --git a/pfsimulator/parflow_lib/CMakeLists.txt b/pfsimulator/parflow_lib/CMakeLists.txt index d09385305..340032ee0 100644 --- a/pfsimulator/parflow_lib/CMakeLists.txt +++ b/pfsimulator/parflow_lib/CMakeLists.txt @@ -321,6 +321,7 @@ set (HEADER_FILES metadata.h nl_function_eval.h n_vector.h + Parflow.hxx parflow.h parflow_netcdf.h parflow_proto_f90.h From 0ce06f95136b4e6615c7331eb2e8c60fb12e9db6 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 23 Nov 2023 13:05:41 +0100 Subject: [PATCH 09/18] amps: If `oas3`, move `oas3_*.h` header files to install directory --- pfsimulator/amps/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pfsimulator/amps/CMakeLists.txt b/pfsimulator/amps/CMakeLists.txt index d29db3076..f78401c3e 100644 --- a/pfsimulator/amps/CMakeLists.txt +++ b/pfsimulator/amps/CMakeLists.txt @@ -57,6 +57,12 @@ install(FILES common/amps_common.h ${PARFLOW_AMPS_LAYER}/amps.h ${PARFLOW_AMPS_LAYER}/amps_proto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) +if(${PARFLOW_HAVE_OAS3}) + install(FILES common/amps_common.h + ${PARFLOW_AMPS_LAYER}/oas3_coupler.h + ${PARFLOW_AMPS_LAYER}/oas3_external.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) +endif(${PARFLOW_HAVE_OAS3}) add_subdirectory (test/src) From d67aa13c831087d6f74ea07a1c5b6504d59b3d82 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 23 Nov 2023 17:51:55 +0100 Subject: [PATCH 10/18] amps: use `HEADER_FILES` variable in install step --- pfsimulator/amps/CMakeLists.txt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pfsimulator/amps/CMakeLists.txt b/pfsimulator/amps/CMakeLists.txt index f78401c3e..43ad9f22e 100644 --- a/pfsimulator/amps/CMakeLists.txt +++ b/pfsimulator/amps/CMakeLists.txt @@ -53,16 +53,8 @@ string(REGEX REPLACE "([^;]+)" "${PARFLOW_AMPS_LAYER}/\\1" AMPS_SCRIPTS "${AMPS_ install(FILES ${AMPS_SCRIPTS} DESTINATION bin) # Install headers -install(FILES common/amps_common.h - ${PARFLOW_AMPS_LAYER}/amps.h - ${PARFLOW_AMPS_LAYER}/amps_proto.h +install(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) -if(${PARFLOW_HAVE_OAS3}) - install(FILES common/amps_common.h - ${PARFLOW_AMPS_LAYER}/oas3_coupler.h - ${PARFLOW_AMPS_LAYER}/oas3_external.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) -endif(${PARFLOW_HAVE_OAS3}) add_subdirectory (test/src) From 1f6fa82fea72a2af207a04d82d8755671d581c2b Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 14 Nov 2024 15:34:40 +0100 Subject: [PATCH 11/18] README: Warning message for users looking for pure ParFlow --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 3f6c835ec..7ac3200bc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ +# Fork of ParFlow pre-patched for TSMP-PDAF + +**DO NOT USE WITHOUT PDAF** + +This is a fork of ParFlow with changes added to run ParFlow in the +TSMP-PDAF framework. + +It is built via the TSMP2 repository: +https://github.com/HPSCTerrSys/TSMP2 + + # ParFlow ![ParFlow CI Test](https://github.com/parflow/parflow/workflows/ParFlow%20CI%20Test/badge.svg) From 1822baf71a06cad0496b7fce8b1bb87404baf7f1 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 28 Nov 2024 13:59:26 +0100 Subject: [PATCH 12/18] PDAF: remove redefinition of `filename` ``` $TSMP/models/parflow_pdaf/pfsimulator/parflow_lib/solver_richards.c:1715:15: error: redefinition of 'filename' char filename[2048]; ``` --- pfsimulator/parflow_lib/solver_richards.c | 1 - 1 file changed, 1 deletion(-) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index e22e9fa56..b85b7d31e 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -1712,7 +1712,6 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time #endif int istep = 1; -char filename[2048]; #ifdef HAVE_CLM Grid *grid = (instance_xtra->grid); From 338a27003ff1d97cb84102c23381dc6d335b24b4 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 28 Nov 2024 14:29:55 +0100 Subject: [PATCH 13/18] PDAF: comment unused loop to avoid `unused-but-set-variable` error ``` $TSMP/models/parflow_pdaf/pfsimulator/parflow_lib/solver_richards.c:1821:12: error: variable 'dx' set but not used [-Werror,-Wunused-but-set-variable] double dx, dy; ^ ``` --- pfsimulator/parflow_lib/solver_richards.c | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index b85b7d31e..828166490 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -1815,25 +1815,25 @@ int istep = 1; double pfl_start = GetDouble("TimingInfo.StartTime"); int is; - ForSubgridI(is, GridSubgrids(grid)) - { - double dx, dy; - int nx, ny, ix, iy; + /* ForSubgridI(is, GridSubgrids(grid)) */ + /* { */ + /* double dx, dy; */ + /* int nx, ny, ix, iy; */ - subgrid = GridSubgrid(grid, is); + /* subgrid = GridSubgrid(grid, is); */ - nx = SubgridNX(subgrid); - ny = SubgridNY(subgrid); + /* nx = SubgridNX(subgrid); */ + /* ny = SubgridNY(subgrid); */ - ix = SubgridIX(subgrid); - iy = SubgridIY(subgrid); + /* ix = SubgridIX(subgrid); */ + /* iy = SubgridIY(subgrid); */ - dx = SubgridDX(subgrid); - dy = SubgridDY(subgrid); + /* dx = SubgridDX(subgrid); */ + /* dy = SubgridDY(subgrid); */ // CALL_oas_pfl_define(nx, ny, dx, dy, ix, iy, sw_lon, sw_lat, nlon, nlat, // pfl_step, pfl_stop); - } + /* } */ amps_Sync(amps_CommWorld); #endif // end to HAVE_OAS3 CALL From 24fd719aa5c20356b44ef902db8a6d0b7b945847 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 28 Nov 2024 15:53:46 +0100 Subject: [PATCH 14/18] PDAF: add `reservoir.h` to `HEADER_FILES` Error: ``` $TSMP/bin/JUWELS_ParFlow-CLM3.5-PDAF/include/parflow/parflow.h:69:10: fatal error: 'reservoir.h' file not found ^~~~~~~~~~~~~ ``` --- pfsimulator/parflow_lib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pfsimulator/parflow_lib/CMakeLists.txt b/pfsimulator/parflow_lib/CMakeLists.txt index a4f6080ba..4cfa9055e 100644 --- a/pfsimulator/parflow_lib/CMakeLists.txt +++ b/pfsimulator/parflow_lib/CMakeLists.txt @@ -360,6 +360,7 @@ set (HEADER_FILES timing.h vector.h well.h + reservoir.h ) install(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow) From 7e918bcba1613a892931508e8e09727c2025efb0 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Fri, 11 Jul 2025 11:03:06 +0200 Subject: [PATCH 15/18] PDAF: comment out unused variables in solver_richards.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Error: ``` parflow_pdaf/pfsimulator/parflow_lib/solver_richards.c:1817:16: error: unused variable ‘pfl_start’ [-Werror=unused-variable] ``` Background: For PDAF, the subgrid-loops, where these variables are originally used is commented out. However, up-to-now the (for PDAF) unused variable were still declared. --- pfsimulator/parflow_lib/solver_richards.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index 828166490..977ecd8b8 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -1707,8 +1707,8 @@ AdvanceRichards(PFModule * this_module, double start_time, /* Starting time Subgrid *subgrid; Subvector *p_sub, *s_sub, *et_sub, *m_sub, *po_sub, *dz_sub; double *pp, *sp, *et, *ms, *po_dat, *dz_dat; - double sw_lat = .0; - double sw_lon = .0; + /* double sw_lat = .0; */ + /* double sw_lon = .0; */ #endif int istep = 1; @@ -1807,12 +1807,12 @@ int istep = 1; //CPS oasis definition phase #ifdef HAVE_OAS3 - int nlon = GetInt("ComputationalGrid.NX"); - int nlat = GetInt("ComputationalGrid.NY"); - double pfl_step = GetDouble("TimeStep.Value"); - double pfl_stop = GetDouble("TimingInfo.StopTime"); - // PDAF: getting start time - double pfl_start = GetDouble("TimingInfo.StartTime"); + /* int nlon = GetInt("ComputationalGrid.NX"); */ + /* int nlat = GetInt("ComputationalGrid.NY"); */ + /* double pfl_step = GetDouble("TimeStep.Value"); */ + /* double pfl_stop = GetDouble("TimingInfo.StopTime"); */ + /* // PDAF: getting start time */ + /* double pfl_start = GetDouble("TimingInfo.StartTime"); */ int is; /* ForSubgridI(is, GridSubgrids(grid)) */ From 6ced8f8a33df23a2bdfc49eb728824583b63d4fe Mon Sep 17 00:00:00 2001 From: Johannes Keller <16795031+jjokella@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:23:37 +0200 Subject: [PATCH 16/18] FindOASIS.cmake: Add NetCDF_Fortran_INCLUDES (#9) --- cmake/modules/FindOASIS.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/modules/FindOASIS.cmake b/cmake/modules/FindOASIS.cmake index a876f5109..ca42f127a 100644 --- a/cmake/modules/FindOASIS.cmake +++ b/cmake/modules/FindOASIS.cmake @@ -22,5 +22,6 @@ if(OASIS_FOUND AND NOT TARGET OASIS3MCT::OASIS3MCT) target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE ${OASIS_Fortran_LIBRARY} ${MCT_Fortran_LIBRARY} ${MPEU_Fortran_LIBRARY} ${SCRIP_Fortran_LIBRARY}) target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE OpenMP::OpenMP_Fortran) target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE ${NETCDF_Fortran_LIBRARY}) + target_include_directories(OASIS3MCT::OASIS3MCT INTERFACE ${NETCDF_Fortran_INCLUDES}) endif() From 4476a1eded78f502837bfb0bfa78c08c85355fcf Mon Sep 17 00:00:00 2001 From: Johannes Keller <16795031+jjokella@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:24:02 +0200 Subject: [PATCH 17/18] Commenting out unused variables in PseudoAdvanceRichards (#10) --- pfsimulator/parflow_lib/solver_richards.c | 76 +++++++++++------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c index 8d7a7687d..e1088d2b0 100644 --- a/pfsimulator/parflow_lib/solver_richards.c +++ b/pfsimulator/parflow_lib/solver_richards.c @@ -4928,29 +4928,29 @@ PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Startin printf("Pseudo richard for OAS init\n"); //>>TSMP-PDAF internal change end - PublicXtra *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module); + /* PublicXtra *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module); */ InstanceXtra *instance_xtra = (InstanceXtra*)PFModuleInstanceXtra(this_module); - Problem *problem = (public_xtra->problem); + /* Problem *problem = (public_xtra->problem); */ - int max_iterations = (public_xtra->max_iterations); - int print_satur = (public_xtra->print_satur); - int print_wells = (public_xtra->print_wells); + /* int max_iterations = (public_xtra->max_iterations); */ + /* int print_satur = (public_xtra->print_satur); */ + /* int print_wells = (public_xtra->print_wells); */ - PFModule *problem_saturation = (instance_xtra->problem_saturation); - PFModule *phase_density = (instance_xtra->phase_density); - PFModule *select_time_step = (instance_xtra->select_time_step); - PFModule *l2_error_norm = (instance_xtra->l2_error_norm); - PFModule *nonlin_solver = (instance_xtra->nonlin_solver); + /* PFModule *problem_saturation = (instance_xtra->problem_saturation); */ + /* PFModule *phase_density = (instance_xtra->phase_density); */ + /* PFModule *select_time_step = (instance_xtra->select_time_step); */ + /* PFModule *l2_error_norm = (instance_xtra->l2_error_norm); */ + /* PFModule *nonlin_solver = (instance_xtra->nonlin_solver); */ - ProblemData *problem_data = (instance_xtra->problem_data); + /* ProblemData *problem_data = (instance_xtra->problem_data); */ - int start_count = ProblemStartCount(problem); - double dump_interval = ProblemDumpInterval(problem); + /* int start_count = ProblemStartCount(problem); */ + /* double dump_interval = ProblemDumpInterval(problem); */ - Vector *porosity = ProblemDataPorosity(problem_data); - Vector *evap_trans_sum = instance_xtra->evap_trans_sum; - Vector *overland_sum = instance_xtra->overland_sum; /* sk: Vector of outflow at the boundary */ + /* Vector *porosity = ProblemDataPorosity(problem_data); */ + /* Vector *evap_trans_sum = instance_xtra->evap_trans_sum; */ + /* Vector *overland_sum = instance_xtra->overland_sum; /\* sk: Vector of outflow at the boundary *\/ */ if (evap_trans == NULL) { @@ -4960,8 +4960,8 @@ PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Startin #ifdef HAVE_OAS3 Grid *grid = (instance_xtra->grid); Subgrid *subgrid; - Subvector *p_sub, *s_sub, *et_sub, *m_sub, *po_sub, *dz_sub; - double *pp, *sp, *et, *ms, *po_dat, *dz_dat; + /* Subvector *p_sub, *s_sub, *et_sub, *m_sub, *po_sub, *dz_sub; */ + /* double *pp, *sp, *et, *ms, *po_dat, *dz_dat; */ double sw_lat = .0; double sw_lon = .0; #endif @@ -5024,29 +5024,29 @@ PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Startin // #endif //>>TSMP-PDAF internal change end - int any_file_dumped; - int clm_file_dumped; - int dump_files = 0; + /* int any_file_dumped; */ + /* int clm_file_dumped; */ + /* int dump_files = 0; */ - int retval; - int converged; - int take_more_time_steps; - int conv_failures; - int max_failures = public_xtra->max_convergence_failures; + /* int retval; */ + /* int converged; */ + /* int take_more_time_steps; */ + /* int conv_failures; */ + /* int max_failures = public_xtra->max_convergence_failures; */ - double t; - double dt = 0.0; - double ct = 0.0; - double cdt = 0.0; - double print_dt; - double dtmp, err_norm; - double gravity = ProblemGravity(problem); + /* double t; */ + /* double dt = 0.0; */ + /* double ct = 0.0; */ + /* double cdt = 0.0; */ + /* double print_dt; */ + /* double dtmp, err_norm; */ + /* double gravity = ProblemGravity(problem); */ - VectorUpdateCommHandle *handle; + /* VectorUpdateCommHandle *handle; */ - char dt_info; - char file_prefix[2048], file_type[2048], file_postfix[2048]; - char nc_postfix[2048]; + /* char dt_info; */ + char file_prefix[2048];//, file_type[2048], file_postfix[2048]; + /* char nc_postfix[2048]; */ //>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) // int Stepcount = 0; /* Added for transient EvapTrans file management - NBE */ @@ -5063,7 +5063,7 @@ PseudoAdvanceRichards(PFModule * this_module, double start_time, /* Startin double pfl_step = GetDouble("TimeStep.Value"); double pfl_stop = GetDouble("TimingInfo.StopTime"); //>>TSMP-PDAF internal change beginning (compare to AdvanceRichards) - double pfl_start = GetDouble("TimingInfo.StartTime"); + /* double pfl_start = GetDouble("TimingInfo.StartTime"); */ //>>TSMP-PDAF internal change end int is; From fa51ec6265e1d95b64c423c3dc4cb2b998573e15 Mon Sep 17 00:00:00 2001 From: Johannes Keller <16795031+jjokella@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:24:29 +0200 Subject: [PATCH 18/18] wrf_parflow: declare Solver_module as extern (#11) --- pfsimulator/parflow_lib/wrf_parflow.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pfsimulator/parflow_lib/wrf_parflow.c b/pfsimulator/parflow_lib/wrf_parflow.c index cad5b77c2..f85bde0dd 100644 --- a/pfsimulator/parflow_lib/wrf_parflow.c +++ b/pfsimulator/parflow_lib/wrf_parflow.c @@ -36,7 +36,8 @@ #include -amps_ThreadLocalDcl(PFModule *, Solver_module); +extern PFModule *Solver_module; /* defined in solver.c */ +/* amps_ThreadLocalDcl(PFModule *, Solver_module); */ amps_ThreadLocalDcl(PFModule *, solver); amps_ThreadLocalDcl(Vector *, evap_trans);