### OpenFOAM (https://www.openfoam.com/) is free, open source CFD software.
###
### This configuration times the execution of the OpenFOAM Motorbike tutorial
### based on the OpenFOAM-v1812 branch of the git repository. By default,
### the tutorial uses 6 processes for the calculations based on a 3x2 xdivision
### and ydivision (6 subdomains) which is modified depending on the machine.
###
### Solver:  Default
### Faces:   Default
### Steps:   All
###
### At time of writing, the openMPI version used is openMPI 3.1.6-1.19
### if using SLE 15 SP2 and openMPI 3.1.4-lp151.7.3.1 if using
### openSUSE Leap 15.1.
###
### For a given number of subdomains many decompositions (ie. division
### in X-, and/or Y-, and/or Z-dimension) exist.  Runtime for each of
### those decompositions might differ significantly from each
### other. For a two NUMA node system (marvin4, 2x24 CPUs) optimal
### decompositions (those with shortest runtime; probably they are
### specific for this system only) for different numbers of subdomains
### were determined:
###
###               divison   runtime
### subdomains    X  Y  Z  (min:sec)
###     3         3  1  1   ~ 5:58
###     4         1  2  2   ~ 4:38
###     6         3  2  1   ~ 3:35
###     8         4  2  1   ~ 3:03
###    12         3  2  2   ~ 2:26
###    16         4  2  2   ~ 2:07
###    18         3  2  3   ~ 2:00
###    20        10  2  1   ~ 1:53
###    24         6  2  2   ~ 1:46
###    32        16  1  2   ~ 2:19
###    48         8  2  3   ~ 2:10
###
### Shortest overall runtime was seen with 24 subdomains (6, 2, 2).
### Outliers with excessive runtime were decomposition (1, 4, 2) for 8
### subdomains with more than 50 minutes (and then test was aborted)
### and decomposition (8, 4, 1) for 32 subdomains with runtime of
### about 27 minutes.

. $SHELLPACK_INCLUDE/include-sizes.sh
get_numa_details

export MMTESTS="openfoam"

# List of monitors
export RUN_MONITOR=yes
export MONITORS_ALWAYS=
export MONITORS_GZIP="proc-vmstat mpstat turbostat"
export MONITORS_WITH_LATENCY="vmstat"
export MONITOR_UPDATE_FREQUENCY=10

# OpenFOAM
export OPENFOAM_OPENMPI_VERSION=openmpi3
export OPENFOAM_MPI_PATH=/usr/$MMTESTS_LIBDIR/mpi/gcc/$OPENFOAM_OPENMPI_VERSION/bin
export OPENFOAM_MPI_LIBPATH=/usr/$MMTESTS_LIBDIR/mpi/gcc/$OPENFOAM_OPENMPI_VERSION/$MMTESTS_LIBDIR
export OPENFOAM_MODEL="motorbike-custom"
export OPENFOAM_MAXCPUS=$NUMCPUS
export OPENFOAM_LOCAL_CELLS=100000
export OPENFOAM_GLOBAL_CELLS=100000000
export OPENFOAM_REFINEMENT_CELLS=10
export OPENFOAM_CELL_LEVELS=3
export OPENFOAM_ITERATIONS=3
export OPENFOAM_XDIVISION=6
export OPENFOAM_YDIVISION=2
export OPENFOAM_ZDIVISION=2

# Set additional mpi options if required
grep -q " ht " /proc/cpuinfo
if [ $? -eq 0 ]; then
	if [ $OPENFOAM_MAXCPUS -gt $((NUMCPUS/2)) ]; then
		echo WARNING: Setting use-hwthread-cpus because HT siblings must be used
		OPENFOAM_MPI_OPTS+=" --use-hwthread-cpus"
	fi
fi
if [ "`whoami`" = "root" ]; then
	OPENFOAM_MPI_OPTS+=" --allow-run-as-root"
fi
export OPENFOAM_MPI_OPTS
