# Example script for submitting Quantum Espresso calculations
# on BU SCC using the $TMPDIR to avoid network problems.
# Using the parallelization (?)
# and environment variables to set options matching script.

# Gaussian input file is expected in qe.in

# Output directly from this script is in $JOB_ID-qsub.out,
# Output from QW and additional information is in
# $JOB_ID/qe-$JOB_ID.out

# Besides avoiding network/backup/mirroring/indexing
# problems relating to scratch files, there is a smaller memory
# footprint in the directory containing the job submission and output.

# Many user specific qsub options such as -M user_email have been
# removed, see link for their description to add them back
# https://www.bu.edu/tech/support/research/system-usage/running-jobs/submitting-jobs/

# Being paranoid, the time limit is at 8 hours, feel free to adjust.

# Gaussian jobs should be run with the AVX option. 
# Questions on exactly what this does should be directed to SCC help.

# Script collated by:
# Luke Nambi Mohanam, lmohanam<at>bu.edu

#!/bin/bash -l
#$ -N qe_omp
#$ -pe omp 16
#$ -l h_rt=8:00:00
#$ -l mem_per_core=8G
#$ -l avx
#$ -cwd
#$ -j y
#$ -o $JOB_ID-qsub.out
#$ -e $JOB_ID-qsub.err


# get code to exit if error encountered
set -e

# Set environment variables for Gaussian matching the above options
# Note utility programs have different options.

# Number of OMP 'slots'
export GAUSS_CDEF=16 
# memory is mem_per_core times number of omp 'Slots'
# 8 times 16 in the above settings
export GAUSS_MDEF=248GB 
# old checkpoint file
export GAUSS_ICDEF=old.chk
# checkpoint file
export GAUSS_CDEF=g16.chk
# RWF, read write file,  that contains wave function
export GAUSS_ZDEF=g16.rwf



# Check gaussian input file is correctly named.
if [ ! -e qe.in ] ; then
  echo "Quantum Espresso input file does not exist!"
  exit 1
fi

# Identify submission directory, matching -cwd option.
#  this is the simplest way to ensure files are copied from and back from
#   the correct path. And locks the standard output file path.
SUBMIT_DIR=$PWD

# create output directory
OUTPUT_DIR=$PWD/$JOB_ID
mkdir $OUTPUT_DIR

# create scratch directory
TMP_FREE_SPACE=$(df $TMPDIR | awk '{print $4}' | tail -n 1) #free space on $TMPDIR in KB
if (( $TMP_FREE_SPACE < 150000000 )); then # make sure there is 150GB of space
  echo "not enough space on TMPDIR"
  exit 1
fi
echo "qsub output for Quantum Espresso job"
echo ""
echo "SPACE ON TMPDIR:"
echo $TMP_FREE_SPACE


# cp is used instead of rsync to avoid extra scanning --
# failed copying can not be restarted!

#keep copy of this script in scratch and in output
cp $SUBMIT_DIR/example.qsub $TMPDIR/$JOB_ID.qsub
#populate scratch directory
## input file
cp $SUBMIT_DIR/qe.in $TMPDIR/.
## checkpoint file(?), will overwrite with newer file if present
wait
echo "Scratch input created!"

cd $TMPDIR
echo "tmp dir contents at start:"
ls -al

echo "Quantum Espresso OMP qsub call with environment variables:" &> qe-$JOB_ID.out
echo "Job ID:    $JOB_ID" >> qe-$JOB_ID.out
echo "Job Name:  $JOB_NAME" >> qe-$JOB_ID.out
echo "Host Name: $HOSTNAME" >> qe-$JOB_ID.out
echo "Submit Dir:   $SUBMIT_DIR" >> qe-$JOB_ID.out
echo "Output Dir:   $OUTPUT_DIR" >> qe-$JOB_ID.out
echo "Tmp Dir:   $TMPDIR" >> qe-$JOB_ID.out

#load qe and run calculation
module load (?)
echo "Quantum Espresso module 7.1" >> qe-$JOB_ID.out
echo "" >> qe-$JOB_ID.out

echo "Starting Quantum Espresso pw.x Job!"
pw.x < $TMPDIR/qe.in >> qe-$JOB_ID.out 2>&1

echo ""
echo "pw.x Job Done"
echo ""
echo "tmp dir contents at end:"
ls -al

# Function to always call upon script termination

function end_job_steps {

#Copy back individual files
     cp $TMPDIR/qe-$JOB_ID.out $OUTPUT_DIR/.
# Copying checkpoint file can be 
#  uncommented below if desired.
#     cp $TMPDIR/$GAUSS_CDEF $OUTPUT_DIR/.

     wait
}

trap end_job_steps 0