#!/usr/bin/bash
# This is the script for running the HPC Challenge
#
###SHELLPACK preamble hpcc-bench 1.5.0b

###SHELLPACK parseargBegin
###SHELLPACK parseargParam	--workload-size		HPCC_WORKLOAD_SIZE
###SHELLPACK parseargParam	--iterations		HPCC_ITERATIONS
###SHELLPACK parseargEnd
###SHELLPACK monitor_hooks

###SHELLPACK check_install_required hpcc-${VERSION}
###SHELLPACK init_complete

ulimit -n 4096
MPICPUS=$(round_down_power_2 $HPCC_MAX_CPUS)

# Calculate basic parameters
N=`echo "scale=0; sqrt ($HPCC_WORKLOAD_SIZE/8)" | bc -l`
P=`echo "scale=0; sqrt ($MPICPUS)" | bc -l`
Q=$P

# Choose NB
FACTOR=1
while [ $((P*Q*FACTOR)) -lt 100 ]; do
	FACTOR=$((FACTOR+1))
done
NB=$((P*Q*FACTOR))

# Write hpccinf.txt
cat > hpccinf.txt <<EOF
HPLinpack benchmark input file
Innovative Computing Laboratory, University of Tennessee
HPL.out      output file name (if any)
8            device out (6=stdout,7=stderr,file)
1            # of problems sizes (N)
$N           Ns
1            # of NBs
$NB          NBs
0            PMAP process mapping (0=Row-,1=Column-major)
1            # of process grids (P x Q)
$P           Ps
$Q           Qs
16.0         threshold
1            # of panel fact
2            PFACTs (0=left, 1=Crout, 2=Right)
1            # of recursive stopping criterium
4            NBMINs (>= 1)
1            # of panels in recursion
2            NDIVs
1            # of recursive panel fact.
1            RFACTs (0=left, 1=Crout, 2=Right)
1            # of broadcast
1            BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)
1            # of lookahead depth
1            DEPTHs (>=0)
2            SWAP (0=bin-exch,1=long,2=mix)
64           swapping threshold
0            L1 in (0=transposed,1=no-transposed) form
0            U  in (0=transposed,1=no-transposed) form
1            Equilibration (0=no,1=yes)
8            memory alignment in double (> 0)
##### This line (no. 32) is ignored (it serves as a separator). ######
0                               Number of additional problem sizes for PTRANS
1200 10000 30000                values of N
0                               number of additional blocking sizes for PTRANS
40 9 8 13 13 20 16 32 64        values of NB
EOF
cp hpccinf.txt $LOGDIR_RESULTS/

log_cmd "OpenMPI Environment"
log_cmd " HPCC_OPENMPI_VERSION=$HPCC_OPENMPI_VERSION"
log_cmd " PATH=$HPCC_MPI_PATH:\$PATH"
log_cmd " LD_LIBRARY_PATH=$HPCC_MPI_LIBPATH"
log_cmd ""

export PATH=$HPCC_MPI_PATH:$PATH
export LD_LIBRARY_PATH=$HPCC_MPI_LIBPATH

cluster_replicate_file ./hpcc
BENCH_CMD="$SLURM_CMD mpirun $HPCC_MPI_OPTS $MMTESTS_MPI_OPTS -np $MPICPUS ./hpcc"
log_cmd $BENCH_CMD

monitor_pre_hook $LOGDIR_RESULTS $P
###SHELLPACK iteration_begin $HPCC_ITERATIONS
	echo Running hpcc iteration $ITERATION/$HPCC_ITERATIONS
	eval $TIME_CMD -o $LOGDIR_RESULTS/time-${ITERATION} \
		$BENCH_CMD 2>&1 | tee $LOGDIR_RESULTS/hpcc-${ITERATION}.log
	mv hpccoutf.txt $LOGDIR_RESULTS/hpccoutf-${ITERATION}.txt
###SHELLPACK iteration_end
monitor_post_hook $LOGDIR_RESULTS $P

exit $SHELLPACK_SUCCESS
