#!/bin/bash
###SHELLPACK preamble specfem3d-bench v7.0.2
###SHELLPACK parseargBegin
###SHELLPACK parseargParam --model   SPECFEM3D_MODEL
###SHELLPACK parseargEnd
###SHELLPACK monitor_hooks

if [ "$SPECFEM3D_OPENMPI_VERSION" != "" ]; then
	###SHELLPACK mpi_setup_deps SPECFEM3D_OPENMPI_VERSION
	###SHELLPACK mpi_setup_env SPECFEM3D_OPENMPI_VERSION SPECFEM3D_MPI_PATH SPECFEM3D_MPI_LIBPATH SPECFEM3D_MPI_OPTS
	###SHELLPACK mpi_setup_shmem
fi

###SHELLPACK check_install_required_continue specfem3d-$VERSION

if [ "$INSTALL_ONLY" = "yes" ]; then
	rm -rf $SHELLPACK_DATA/model-$SPECFEM3D_MODEL
fi

ulimit -n 1048576

if [ ! -e $SHELLPACK_DATA/model-$SPECFEM3D_MODEL ]; then
	case $SPECFEM3D_MODEL in
	small_s362)
		# Copy model
		cd $SHELLPACK_SOURCES/specfem3d-${VERSION}-installed/EXAMPLES || die "Failed to change to EXAMPLES directory"
		rm -rf model-$SPECFEM3D_MODEL
		cp -r global_small model-$SPECFEM3D_MODEL || die "Failed to copy model"
		cd model-$SPECFEM3D_MODEL || die "Failed to change to model"
		export PBS_O_WORKDIR=`pwd`

		# Configure model
		log_runtime "Add MPI flags to example script: $SPECFEM3D_MPI_OPTS $MMTESTS_MPI_OPTS"
		sed -i -e "s/mpirun/mpirun $SPECFEM3D_MPI_OPTS $MMTESTS_MPI_OPTS/" go_*_pbs.bash
		sed -i -e "s/^NCHUNKS.*/NCHUNKS = $SPECFEM3D_NCHUNKS/" DATA/Par_file
		sed -i -e "s/^NEX_XI.*/NEX_XI = $SPECFEM3D_NEX_XI/" DATA/Par_file
		sed -i -e "s/^NEX_ETA.*/NEX_ETA = $SPECFEM3D_NEX_ETA/" DATA/Par_file
		sed -i -e "s/^NPROC_XI.*/NPROC_XI = $SPECFEM3D_NPROC_XI/" DATA/Par_file
		sed -i -e "s/^NPROC_ETA.*/NPROC_ETA = $SPECFEM3D_NPROC_ETA/" DATA/Par_file
		cp DATA/Par_file $LOGDIR_RESULTS/Par_file || die "Failed to copy DATA/Par_file"

		# Build
		echo Building global_small
		ROOTDIR=$SHELLPACK_SOURCES/specfem3d-${VERSION}-installed
		ROOTDIR_ESCAPED=`echo "$ROOTDIR" | sed -e 's/\//\\\\\//g'`
		sed -i -e "s/rootdir=.*/rootdir=$ROOTDIR_ESCAPED/" run_this_example.sh
		./run_this_example.sh || die "Failed to execute global_small run_this_example.sh"

		# Make a copy
		cd $SHELLPACK_SOURCES/specfem3d-${VERSION}-installed/EXAMPLES || die "Failed to change to EXAMPLES directory"
		cp -r model-$SPECFEM3D_MODEL $SHELLPACK_DATA/model-$SPECFEM3D_MODEL
		cd $SHELLPACK_DATA/model-$SPECFEM3D_MODEL

		echo Preparing mesh
		$TIME_CMD -o $LOGDIR_RESULTS/prepare-time.mesher ./go_mesher_pbs.bash \
			&> $LOGDIR_RESULTS/prepare-log.mesher
		;;
	global_s362)
		# Copy model
		cd $SHELLPACK_SOURCES/specfem3d-${VERSION}-installed/EXAMPLES || die "Failed to change to EXAMPLES directory"
		rm -rf model-$SPECFEM3D_MODEL
		cp -r global_s362ani_shakemovie model-$SPECFEM3D_MODEL || die "Failed to copy model"
		cd model-$SPECFEM3D_MODEL || die "Failed to change to model"
		export PBS_O_WORKDIR=`pwd`

		# Configure model
		log_runtime "Add MPI flags to example script: $SPECFEM3D_MPI_OPTS $MMTESTS_MPI_OPTS"
		sed -i -e /compute_nodes/d go_*_pbs.bash
		sed -i -e /jobid/d go_*_pbs.bash
		sed -i -e "s/mpiexec/mpiexec $SPECFEM3D_MPI_OPTS $MMTESTS_MPI_OPTS/" go_*_pbs.bash
		sed -i -e "s/^NCHUNKS.*/NCHUNKS = $SPECFEM3D_NCHUNKS/" DATA/Par_file
		sed -i -e "s/^NEX_XI.*/NEX_XI = $SPECFEM3D_NEX_XI/" DATA/Par_file
		sed -i -e "s/^NEX_ETA.*/NEX_ETA = $SPECFEM3D_NEX_ETA/" DATA/Par_file
		sed -i -e "s/^NPROC_XI.*/NPROC_XI = $SPECFEM3D_NPROC_XI/" DATA/Par_file
		sed -i -e "s/^NPROC_ETA.*/NPROC_ETA = $SPECFEM3D_NPROC_ETA/" DATA/Par_file
		cp DATA/Par_file $LOGDIR_RESULTS/Par_file || die "Failed to copy DATA/Par_file"

		# Build
		echo Building global_s362ani_shakemovie
		ROOTDIR=$SHELLPACK_SOURCES/specfem3d-${VERSION}-installed
		ROOTDIR_ESCAPED=`echo "$ROOTDIR" | sed -e 's/\//\\\\\//g'`
		sed -i -e "s/rootdir=.*/rootdir=$ROOTDIR_ESCAPED/" run_this_example.sh
		./run_this_example.sh || die "Failed to execute global_s362ani_shakemovie run_this_example.sh"

		# Make a copy
		cd $SHELLPACK_SOURCES/specfem3d-${VERSION}-installed/EXAMPLES || die "Failed to change to EXAMPLES directory"
		cp -r model-$SPECFEM3D_MODEL $SHELLPACK_DATA/model-$SPECFEM3D_MODEL
		cd $SHELLPACK_DATA/model-$SPECFEM3D_MODEL

		echo Preparing mesh
		$TIME_CMD -o $LOGDIR_RESULTS/prepare-time.mesher ./go_mesher_pbs.bash \
			&> $LOGDIR_RESULTS/prepare-log.mesher
		;;
	*)
		die "Unrecognised model $SPECFEM3D_MODEL"
		;;
	esac
fi
###SHELLPACK init_complete

case $SPECFEM3D_MODEL in
small_s362)
	JOB_LIST="solver:go_solver_pbs.bash"
	;;
global_s362)
	JOB_LIST="solver:go_solver_pbs.bash"
	;;
*)
	die "Unrecognised model $SPECFEM3D_MODEL"
	;;
esac

cd $SHELLPACK_DATA/model-$SPECFEM3D_MODEL
export PBS_O_WORKDIR=`pwd`

monitor_pre_hook $LOGDIR_RESULTS $SPECFEM3D_MODEL
for JOB in $JOB_LIST; do
	echo Executing $JOB
	NAME=`echo $JOB | awk -F : '{print $1}'`
	SCRIPT=`echo $JOB | awk -F : '{print $2}'`

	BENCH_CMD="./$SCRIPT"
	log_cmd "$BENCH_CMD"
	log_linkages $SHELLPACK_SOURCES/specfem3d-${VERSION}-installed/bin/xmeshfem3D

	save_rc $TIME_CMD -o $LOGDIR_RESULTS/specfem3d-time.$NAME   \
		$BENCH_CMD &> $LOGDIR_RESULTS/specfem3d-log.$NAME
	recover_rc
	if [ $? -ne 0 ]; then
		die "Failed to execute $SCRIPT"
	fi
	gzip $LOGDIR_RESULTS/specfem3d-log.$NAME
	tar -czf $LOGDIR_RESULTS/output-$NAME.tar.gz OUTPUT_FILES
done
monitor_post_hook $LOGDIR_RESULTS $SPECFEM3D_MODEL
cd $SHELLPACK_DATA
rm -rf $SHELLPACK_DATA/model-$SPECFEM3D_MODEL
exit $SHELLPACK_SUCCESS
