#!/usr/bin/bash
# This is the script for running the NAS Parallel Benchmark
#
###SHELLPACK preamble nas-bench 3.3.1

if [ "$NAS_OPENMPI_VERSION" != "" ]; then
	install-depends $NAS_OPENMPI_VERSION $NAS_OPENMPI_VERSION-devel
	if [ $? -ne 0 ]; then
		echo WARNING: Falling back to openmpi1
		export NAS_OPENMPI_VERSION_NEW=openmpi
		export NAS_MPI_PATH=`echo $NAS_MPI_PATH | sed -e "s@$NAS_OPENMPI_VERSION@$NAS_OPENMPI_VERSION_NEW@g"`
		export NAS_MPI_LIBPATH=`echo $NAS_MPI_LIBPATH | sed -e "s@$NAS_OPENMPI_VERSION@$NAS_OPENMPI_VERSION_NEW@g"`
		export NAS_OPENMPI_VERSION=$NAS_OPENMPI_VERSION_NEW
		install-depends $NAS_OPENMPI_VERSION $NAS_OPENMPI_VERSION-devel
	fi
	###SHELLPACK mpi_setup_deps NAS_OPENMPI_VERSION
	###SHELLPACK mpi_setup_env $NAS_OPENMPI_VERSION NAS_MPI_PATH NAS_MPI_LIBPATH NAS_MPI_OPTS
fi

if [ "$NAS_MPICH_VERSION" != "" ]; then
	###SHELLPACK mpich_setup_deps NAS_MPICH_VERSION
	###SHELLPACK mpi_setup_env NAS_MPICH_VERSION NAS_MPICH_PATH NAS_MPICH_LIBPATH
fi

install-depends libpsm_infinipath1
install-depends gfortran gcc

NAS_TYPE=SER
RUNBITS=
OPENMP=
OPENMPI=
MPICPUS=
MEMTOTAL_MBYTES=$((MEMTOTAL_BYTES/1048576))
NAS_BIND_PLACE=${NAS_BIND_PLACE:-socket}

if [ "$NAS_MAX_CPUS" = "" ]; then
	NAS_MAX_CPUS=$NUMCPUS
fi

if [ "$NAS_MPICH_PATH" != "" ]; then
	export PATH=$NAS_MPICH_PATH:$PATH
fi
if [ "$NAS_MPICH_LIBPATH" != "" ]; then
	export LD_LIBRARY_PATH=$NAS_MPICH_LIBPATH
fi

###SHELLPACK parseargBegin
	--type)
		NAS_TYPE=$2
		if [ "$NAS_TYPE" = "OMP" ]; then
			OPENMP=--openmp
		fi
		if [ "$NAS_TYPE" = "MPI" -o "$NAS_TYPE" = "MPICH" ]; then
			export OMPI_TMPDIR=$SHELLPACK_DATA
			export TMPDIR=$SHELLPACK_DATA
			case $NAS_CLASS in
			C)
				if [ $MEMTOTAL_MBYTES -lt 6144 ]; then
					echo INFO: Machine has too little memory to be useful
					exit $SHELLPACK_SUCCESS
				fi
				;;
			*)
				echo Machine has sufficient memory for testing.
				;;
			esac
			if [ "$NAS_TYPE" = "MPI" ]; then
				MPIOPT="-mca btl ^openib,udapl"
				OPENMPI="--openmpi"
			else
				MPICH="--mpich"
			fi
		fi
		shift 2
		;;
	--bitness)
		RUNBITS=$2
		shift 2
		;;
	--cma)
		MPIOPT="-mca btl btl_sm_use_cma 1"
		shift
		;;
###SHELLPACK parseargParam --max-cpus   NAS_MAX_CPUS
###SHELLPACK parseargParam --iterations NAS_ITERATIONS
###SHELLPACK parseargParam --joblist    NAS_JOBLIST
###SHELLPACK parseargEnd
###SHELLPACK monitor_hooks

###SHELLPACK check_install_required nas-$VERSION
###SHELLPACK init_complete
###SHELLPACK build_start nas-$VERSION-installed

JOBLIST="bt cg dc ep ft is lu mg sp ua"
if [ "$NAS_JOBLIST" != "" ]; then
	JOBLIST=$NAS_JOBLIST
fi

# Additional test parameters
case $NAS_TYPE in
SER)
	echo
	;;
MPI|MPICH)
	REMOVE_LIST="dc ua"
	if [ "$NAS_CLASS" = "D" ]; then
		REMOVE_LIST="$REMOVE_LIST ft"
	fi
	for REMOVE in $REMOVE_LIST; do
		JOBLIST=`echo $JOBLIST | sed -e "s/$REMOVE//"`
	done
	if [ "$NAS_BIND" = "yes" ]; then
		MMTESTS_MPI_OPTS+=" --bind-to $NAS_BIND_PLACE"
	fi
	MPIRUN="mpirun $NAS_MPI_OPTS $MMTESTS_MPI_OPTS"
	;;
OMP)
	REMOVE_LIST="dc"
	if [ "$NAS_CLASS" = "D" ]; then
		REMOVE_LIST="ft dc"
	fi
	for REMOVE in $REMOVE_LIST; do
		JOBLIST=`echo $JOBLIST | sed -e "s/$REMOVE//"`
	done
	export OMP_NUM_THREADS=$NAS_MAX_CPUS
	if [ "$NAS_BIND" = "yes" ]; then
		export OMP_PROC_BIND=true
	fi
	;;
*)
	die "Unrecognised NAS type $NAS_TYPE"
	;;
esac

case $NAS_TYPE in
MPI)
	log_cmd "OpenMPI Environment"
	log_cmd " NAS_OPENMPI_VERSION=$NAS_OPENMPI_VERSION"
	log_cmd " PATH=$NAS_MPI_PATH:\$PATH"
	log_cmd " LD_LIBRARY_PATH=$NAS_MPI_LIBPATH"
	log_cmd " OMPI_TMPDIR=$OMPI_TMPDIR"
	log_cmd " TMPDIR=$TMPDIR"
	log_cmd " OPENMPI=$OPENMPI"
	log_cmd ""
	;;
MPICH)
	log_cmd "MPICH Environment"
	log_cmd " NAS_MPICH_VERSION=$NAS_MPICH_VERSION"
	log_cmd " PATH=$NAS_MPICH_PATH:\$PATH"
	log_cmd " LD_LIBRARY_PATH=$NAS_MPICH_LIBPATH"
	log_cmd " OMPI_TMPDIR=$OMPI_TMPDIR"
	log_cmd " TMPDIR=$TMPDIR"
	log_cmd " OPENMPI=$OPENMPI"
	log_cmd ""
	;;

OMP)
	log_cmd "OpenMP Environment"
	log_cmd " OMP_NUM_THREADS=$OMP_NUM_THREADS"
	log_cmd " OMP_PROC_BIND=$OMP_PROC_BIND"
	log_cmd ""
	;;
esac

if [ "$RUNBITS" = "" ]; then
	case `uname -m` in
		i?86)
			RUNBITS=32
			;;
		*)
			RUNBITS=64
			;;
	esac
fi
export PATH=$SHELLPACK_TOPLEVEL/nas/bin:$PATH

function lookup_mpicpus
{
	local NAS_JOB=$1

	JOB_MAX_CPUS=$NAS_MAX_CPUS
	for SPEC in $NAS_JOB_MAX_CPUS; do
		SPEC_JOB=`echo $SPEC | awk -F = '{print $1}'`
		SPEC_CPUS=`echo $SPEC | awk -F = '{print $2}'`

		if [ "$SPEC_JOB" = "$NAS_JOB" ]; then
			JOB_MAX_CPUS=$SPEC_CPUS
		fi
	done

	# OLD POWER 2: cg ft is mg
	# OLD SQUARE:  bt ep lu sp
	# No limit:    none
	#
	# NEW POWER 2: cg ft is mg
	# NEW SQUARE:  bt sp
	# No limit:    ep lu

	case $NAS_JOB in
	cg|ft|is|mg)
		# MPI requires a power of two e.g. 2, 4, 8, 16 etc
		MPICPUS=$(round_down_power_2 $JOB_MAX_CPUS)
		;;
	bt|sp)
		# MPI requires a square e.g. 4, 9, 16, 25 etc
		MPICPUS=$(round_down_nearest_square $JOB_MAX_CPUS)
		;;
	*)
		# No restrictions on parallelisation
		MPICPUS=$JOB_MAX_CPUS
		;;
	esac
}

# Generate suite.def
SUBVERSION=$VERSION
NPB_TYPE=`echo $NAS_TYPE | sed -e 's/MPICH/MPI/'`
if [ ! -e $SHELLPACK_SOURCES/nas-$VERSION-installed/NPB$VERSION-$NPB_TYPE ]; then
	SUBVERSION=`echo $VERSION | sed -e 's/\.[0-9]*$//'`
fi
pushd $SHELLPACK_SOURCES/nas-$VERSION-installed/NPB$SUBVERSION-$NPB_TYPE > /dev/null || die "Failed to cd to nas-$VERSION-installed/NPB$SUBVERSION-$NPB_TYPE"
rm -rf bin
mkdir -p bin
echo -n > config/suite.def
for JOB in $JOBLIST; do
	lookup_mpicpus $JOB
	echo "$JOB      $NAS_CLASS $MPICPUS" >> config/suite.def
done

# Run the benchmark ##########################################################
export GCC_OPTIMISE=$NAS_OPTIMISE_FLAGS
if [ "$GCC_OPTIMISE" = "" ]; then
	export GCC_OPTIMISE=$DEFAULT_OPTIMISATION_LEVEL
fi
GENSPECSH="generate-nas.sh --bitness ${RUNBITS} $OPENMP $OPENMPI $MPICH"
$GENSPECSH > config/make.def
cp config/make.def $LOGDIR_RESULTS || die "Failed to generate make.def"
ulimit -n 4096

make clean || die "Failed to clean"
make suite || die "Failed to build"

for JOB in $JOBLIST; do
	mmtests_activity $JOB.$NAS_CLASS
	case $NAS_TYPE in
	SER|OMP)
		BENCH_CMD="$NAS_WRAPPER ./bin/$JOB.$NAS_CLASS.x"
		;;
	MPI|MPICH)
		lookup_mpicpus $JOB
		BENCH_CMD="$SLURM_CMD $MPIRUN -np $MPICPUS $MMTESTS_MPI_OPTS $NAS_WRAPPER ./bin/$JOB.$NAS_CLASS.$MPICPUS"
		cluster_replicate_file ./bin/$JOB.$NAS_CLASS.$MPICPUS
	esac
	log_cmd $BENCH_CMD

	monitor_pre_hook $LOGDIR_RESULTS $JOB.$NAS_CLASS
	###SHELLPACK iteration_begin $NAS_ITERATIONS
		if [ "$RUNBITS" = "32" -a ! -x ./bin/$JOB.$NAS_CLASS ]; then
			echo "Skipping $JOB.$NAS_CLASS"
			continue
		fi

		echo Executing $JOB.$NAS_CLASS iteration $ITERATION/$NAS_ITERATIONS
		case $NAS_TYPE in
		SER|OMP)
			eval $TIME_CMD -o $LOGDIR_RESULTS/time-$JOB.$NAS_CLASS.$ITERATION	\
				$BENCH_CMD > $LOGDIR_RESULTS/$JOB.$NAS_CLASS.log.$ITERATION 2>&1
			;;
		MPI|MPICH)
			eval $TIME_CMD -o $LOGDIR_RESULTS/time-$JOB.$NAS_CLASS.$ITERATION	\
				$BENCH_CMD > $LOGDIR_RESULTS/$JOB.$NAS_CLASS.log.$ITERATION 2>&1
			;;
		esac

		if [ $? -ne 0 ]; then
			die "Failed to execute $JOB.$NAS_CLASS"
		fi

		rm `find -name "ADC.view.dat*"` 2> /dev/null
	###SHELLPACK iteration_end $NAS_ITERATIONS
	monitor_post_hook $LOGDIR_RESULTS $JOB.$NAS_CLASS
done

exit $SHELLPACK_SUCCESS
