# Copyright 2016-2023 Intel Corporation
#
# This software and the related documents are Intel copyrighted materials, and
# your use of them is governed by the express license under which they were
# provided to you (License). Unless the License provides otherwise, you may
# not use, modify, copy, publish, distribute, disclose, or transmit this
# software or the related documents without Intel's prior written permission.
#
# This software and the related documents are provided as is, with no express
# or implied warranties, other than those that are expressly stated in the
# License.

# Detects presence of all required LSB tools
function lsb_tools () {
  get_tool_presence "["
  get_tool_presence "ar"
  get_tool_presence "awk"
  get_tool_presence "basename"
  get_tool_presence "bc"
  get_tool_presence "cat"
  get_tool_presence "chmod"
  get_tool_presence "chown"
  get_tool_presence "cksum"
  get_tool_presence "cmp"
  get_tool_presence "comm"
  get_tool_presence "cp"
  get_tool_presence "cpio"
  get_tool_presence "csplit"
  get_tool_presence "cut"
  get_tool_presence "date"
  get_tool_presence "dd"
  get_tool_presence "df"
  get_tool_presence "diff"
  get_tool_presence "dirname"
  get_tool_presence "dmesg"
  get_tool_presence "du"
  get_tool_presence "echo"
  get_tool_presence "ed"
  get_tool_presence "egrep"
  get_tool_presence "env"
  get_tool_presence "ex"
  get_tool_presence "expr"
  get_tool_presence "false"
  get_tool_presence "fgrep"
  get_tool_presence "file"
  get_tool_presence "find"
  get_tool_presence "fold"
  get_tool_presence "fuser"
  get_tool_presence "getconf"
  get_tool_presence "grep"
  get_tool_presence "gunzip"
  get_tool_presence "gzip"
  get_tool_presence "head"
  get_tool_presence "hostname"
  get_tool_presence "iconv"
  get_tool_presence "id"
  get_tool_presence "join"
  get_tool_presence "kill"
  get_tool_presence "killall"
  get_tool_presence "ln"
  get_tool_presence "locale"
  get_tool_presence "logger"
  get_tool_presence "logname"
  get_tool_presence "ls"
  get_tool_presence "mkdir"
  get_tool_presence "mkfifo"
  get_tool_presence "mktemp"
  get_tool_presence "more"
  get_tool_presence "mv"
  get_tool_presence "nice"
  get_tool_presence "nl"
  get_tool_presence "nohup"
  get_tool_presence "od"
  get_tool_presence "paste"
  get_tool_presence "patch"
  get_tool_presence "pathchk"
  get_tool_presence "pidof"
  get_tool_presence "printf"
  get_tool_presence "ps"
  get_tool_presence "pwd"
  get_tool_presence "rm"
  get_tool_presence "rmdir"
  get_tool_presence "sed"
  get_tool_presence "seq"
  get_tool_presence "sh"
  get_tool_presence "sleep"
  get_tool_presence "sort"
  get_tool_presence "split"
  get_tool_presence "strings"
  get_tool_presence "tail"
  get_tool_presence "tar"
  get_tool_presence "tee"
  get_tool_presence "test"
  get_tool_presence "time"
  get_tool_presence "touch"
  get_tool_presence "tr"
  get_tool_presence "true"
  get_tool_presence "uname"
  get_tool_presence "uniq"
  get_tool_presence "vi"
  get_tool_presence "wc"
  get_tool_presence "xargs"
}

# Defines a variable named $memory_size that contains the memory size
# in megabytes (default).  If not successful, $memory_size = 0 and the
# exit status is 1.
function memory_size() {
    local conversion=1024 # default to MB
    memory_size=0

    # silently ignore unrecognized options
    case "$1" in
        "KB" | "kb")
            conversion=1
            ;;
        "MB" | "mb")
            conversion=1024
            ;;
    esac

    # /proc/meminfo units are KB
    local mem=`\grep "MemTotal:" /proc/meminfo | awk '{printf "%d", $2}'`
    if [[ $? == 0 && -n ${mem} ]] ; then
        memory_size=$((${mem} / ${conversion})) # convert KB to ...
        return 0
    fi

    _stderr "could not determine the memory size"
    return 1
}

# Defines a variable named $memory_size_cluster_min that contains the
# minimum memory size across a set of nodes.  If not successful,
# $memory_size_cluster_min = 0 and the exit status is 1.  Uses pdsh.
function memory_size_cluster_min() {

    local nodefile=$1
    local prefix=$2
    local units=$3
    memory_size_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local mem=`$pdsh -N -w ^$nodefile "bash -c '. $self && memory_size $units && echo \\\$memory_size'" | sort -un | head -1`
    if [[ $? == 0 && -n ${mem} && ${mem} =~ ^[0-9]+$ ]] ; then
        memory_size_cluster_min=$mem
        return 0
    fi

    _stderr "could not determine the minimum cluster memory size"
    return 1
}

# Defines a variable named $memory_size_cluster_min that contains the
# minimum memory size across a set of nodes.  If not successful,
# $memory_size_cluster_min = 0 and the exit status is 1.  Uses mpirun.
function memory_size_cluster_min_mpirun() {
    local nodefile=$1
    local self="$2/../../functions"
    local units=$3
    memory_size_cluster_min=0

    local mem=`mpirun -ppn 1 -f $nodefile bash -c ". $self && memory_size $units && echo \\\$memory_size" | grep -E ^[0-9]+$ | sort -un | head -1`
    if [[ $? == 0 && -n ${mem} ]] ; then
        memory_size_cluster_min=$mem
        return 0
    fi

    _stderr "could not determine the minimum cluster memory size"
    return 1
}

# Defines a variable named $min_mic_count that contains the min number
# of coprocessors installed on any one node in the nodefile. If not
# successful min_mic_count will be set to zero and the exit status is 1.  Uses pdsh.
function mic_coprocessor_count_cluster_min() {

    local nodefile=$1
    local prefix=$2

    min_mic_count=0

    _arch_token

    get_pdsh

    local mic_count=`$pdsh -N -w ^$nodefile '/sbin/lspci -x | grep -c "Co-processor: Intel"' | sort | head -n 1`
    if [[ $? == 0 && -n ${mic_count} && ${mic_count} =~ ^-?[0-9]+$ ]] ; then
          min_mic_count=${mic_count}
          return 0
    fi

    _stderr "could not determine if Intel(R) Xeon Phi(TM) coprocessors were installed."
    return 1
}

# Defines a variable named $min_mic_count that contains the min number
# of coprocessors installed on any one node in the nodefile. If not
# successful min_mic_count will be set to zero and the exit status is 1.  Uses mpirun.
function mic_coprocessor_count_cluster_min_mpirun() {
    local nodefile=$1
    min_mic_count=0

    local mic_count=`mpirun -ppn 1 -f $nodefile bash -c '/sbin/lspci -x | grep -c "Co-processor: Intel"' | grep -E ^[0-9]+$ | sort | head -1`
    if [[ $? == 0 && -n ${mic_count} ]] ; then
          min_mic_count=${mic_count}
          return 0
    fi

    _stderr "could not determine if Intel(R) Xeon Phi(TM) coprocessors were installed."
    return 1
}

# Defines a variable named $num_logical_cores that contains the
# number of logical cores.  If not successful,
# $num_logical_cores = 0 and the exit status is 1.
function num_logical_cores() {
    local ncores=`lscpu -p=core | grep -v "^#" | wc -l`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_logical_cores=$ncores
        return 0
    fi

    _stderr "could not determine the number of logical cores"
    return 1
}

# Defines a variable named $num_physical_cores that contains the
# number of physical (not logical) cores.  If not successful,
# $num_physical_cores = 0 and the exit status is 1.
function num_physical_cores() {
    local ncores=`lscpu -p=core | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_physical_cores=$ncores
        return 0
    fi

    _stderr "could not determine the number of physical cores"
    return 1
}

# Defines a variable named $num_physical_cores_cluster_min that
# contains the minimum number of physical cores across a set of nodes.
# If not successful,$num_physical_cores_cluster_min = 0 and the exit
# status is 1.  Uses pdsh.
function num_physical_cores_cluster_min() {

    local nodefile=$1
    local prefix=$2
    num_physical_cores_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local ncores=`$pdsh -N -w ^$nodefile "bash -c '. $self && num_physical_cores && echo \\\$num_physical_cores'" | sort -un | head -1`
    if [[ $? == 0 && -n ${ncores} && ${ncores} =~ ^[0-9]+$ ]] ; then
        num_physical_cores_cluster_min=$ncores
        return 0
    fi

    _stderr "could not determine the minimum cluster number of physical cores"
    return 1
}

# Defines a variable named $num_physical_cores_cluster_min that
# contains the minimum number of physical cores across a set of nodes.
# If not successful,$num_physical_cores_cluster_min = 0 and the exit
# status is 1.  Uses mpirun.
function num_physical_cores_cluster_min_mpirun() {
    local nodefile=$1
    local self="$2/../../functions"
    num_physical_cores_cluster_min=0

    local ncores=`mpirun -ppn 1 -f $nodefile bash -c ". $self && num_physical_cores && echo \\\$num_physical_cores" | grep -E ^[0-9]+$ | sort -un | head -1`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_physical_cores_cluster_min_mpirun=$ncores
        return 0
    fi

    _stderr "could not determine the minimum cluster number of physical cores"
    return 1
}

# Defines a variable named $num_omp_threads that
# contains the number of OMP threads that needs to be used across a set of nodes.
# If not successful, $num_omp_threads = 0 and the exit
# status is 1.
function num_omp_threads() {
    local nodefile=$1
    local self="$2/../../functions"
    num_omp_threads=0

    num_physical_cores_cluster_min $nodefile $2
    num_sockets_cluster_min $nodefile $2

    if [[ ${num_physical_cores_cluster_min} -gt 0 && ${num_sockets_cluster_min} -gt 0 ]] ; then
        num_omp_threads=$((num_physical_cores_cluster_min/$num_sockets_cluster_min))
        return 0
    fi

    # Fallback
    echo "checking whether environment variable OMP_NUM_THREADS is set..."
    if [[ $num_omp_threads == 0 && ! -z "${OMP_NUM_THREADS}" ]] ; then
        echo "found OMP_NUM_THREADS set to value '${OMP_NUM_THREADS}'"
        num_omp_threads=${OMP_NUM_THREADS}
        return 0
    fi

    _stderr "could not determine the number of OMP threads"
    return 1
}

# Defines a variable named $num_omp_threads_mpirun that
# contains the number of OMP threads that needs to be used across a set of nodes.
# If not successful, $num_omp_threads_mpirun = 0 and the exit
# status is 1. Uses mpirun.
function num_omp_threads_mpirun() {
    local nodefile=$1
    local self="$2/../../functions"
    num_omp_threads_mpirun=0

    num_physical_cores_cluster_min_mpirun $nodefile $2
    num_sockets_cluster_min_mpirun $nodefile $2

    if [[ ${num_physical_cores_cluster_min_mpirun} -gt 0 && ${num_sockets_cluster_min_mpirun} -gt 0 ]] ; then
        num_omp_threads_mpirun=$((num_physical_cores_cluster_min_mpirun/$num_sockets_cluster_min_mpirun))
        return 0
    fi

    # Fallback
    echo "checking whether environment variable OMP_NUM_THREADS is set..."
    if [[ $num_omp_threads_mpirun == 0 && ! -z "${OMP_NUM_THREADS}" ]] ; then
        echo "found OMP_NUM_THREADS set to value '${OMP_NUM_THREADS}'"
        num_omp_threads_mpirun=${OMP_NUM_THREADS}
        return 0
    fi

    _stderr "could not determine the number of OMP threads"
    return 1
}

# Look for which mpi we are running.  If we are running IMPI, set
# IS INTEL MPI to true.  Otherwise, false.
function getmpi() {
    MPIRUN_LOCATION=`which mpirun`
    MPIRUN_VERSION=`mpirun --version`
    if [[ "${MPIRUN_VERSION}" == *"Intel(R) MPI Library"* ]]; then
        IS_INTEL_MPI=true
    else
        IS_INTEL_MPI=false
    fi
}

# this function is currently used for osu providers to print the mpi location.
function getmpi_and_print_path() {
    getmpi
    echo "<==================================================>"
    echo "mpirun location                 : " ${MPIRUN_LOCATION}
    echo "<==================================================>"
}

function set_mpi_timeout(){
    # Configure MPI to timeout before CLCK does to give MPI a chance to cleanup
    if [[ ! -z $CLCK_TIMEOUT && $CLCK_TIMEOUT -gt 5 ]]; then
        export I_MPI_JOB_TIMEOUT=$(( $CLCK_TIMEOUT - 5 ))
        export MPIEXEC_TIMEOUT=$(( $CLCK_TIMEOUT - 5 ))
    fi
}

function check_for_nodefile() {
    # This environment variable will be defined for us
    # CLCK_DP_NODEFILE
    if [ ! -f "${CLCK_DP_NODEFILE}" ] ; then
        echo "Missing or invalid nodefile: '${CLCK_DP_NODEFILE}'" >&2
        exit 1
    fi
}

# this function is currently used for osu providers.  It is a cheat so only the correct arguments may
# be passed in and run.  It gives us a single place to make any corrections.
function run_osu_mpi() {
    check_for_nodefile
    set_mpi_timeout
    getmpi_and_print_path
    if [ $IS_INTEL_MPI == "true" ] ; then
        # Configure MPI to cleanup
        export I_MPI_MPIRUN_CLEANUP=1
    fi

	EXEC_CMD="mpirun $MY_MPI_OPTIONS -f ${CLCK_DP_NODEFILE} -ppn $PPN $BINARY --message-size ${MY_MIN_MESSAGE_SIZE}:${MY_MAX_MESSAGE_SIZE} --full"
    echo $EXEC_CMD
    $EXEC_CMD
    return $?
}

# this function is currently used for imb providers.  It is a cheat so only the correct arguments may
# be passed in and run.  It gives us a single place to make any corrections.
function run_imb_mpi() {
    check_for_nodefile
    set_mpi_timeout
    getmpi

    #if not initialized, do so.
    if [ "$BENCHMARK_OPTIONS" == "" ]; then
        BENCHMARK_OPTIONS=""
    fi

    if [ "$BENCHMARK_BINARY" == "" ]; then
        BENCHMARK_BINARY="IMB-MPI1"
    fi

    if [ "$BENCHMARK_PPN" == "" ]; then
        BENCHMARK_PPN=1
    fi

    # Note: If the user options has a I_MPI_DEBUG, we over ride it with 12 anyway
    if [ $IS_INTEL_MPI == "true" ] ; then
        # Configure MPI to cleanup
        export I_MPI_MPIRUN_CLEANUP=1
        CLCK_I_MPI_ROOT=${I_MPI_ROOT}
        EXEC_CMD="mpirun -ppn $BENCHMARK_PPN $MY_OPTIONS $MY_DAPL $MY_OFI $MY_OFA $MY_TCP $MY_TMI $MY_FABRICS $MY_MPI_PIN -f ${CLCK_DP_NODEFILE} -genv I_MPI_DEBUG=12 $XIMB $BENCHMARK_BINARY ${CLCK_I_MPI_ROOT} $BENCHMARK_NAME $BENCHMARK_OPTIONS -msglen $MSGLEN_PATH"
    else
        echo "missing Intel(R) MPI runtime environment" >&2
        exit 255
    fi

    echo $EXEC_CMD
    $EXEC_CMD
    return $?
}

########
# All functions below this point are prefixed with an underscore.
# This denotes that the functions are private and not intended to be
# used outside this file.  Warning: these functions may change without
# warning.
########

# For the architecture, return the corresponding token used by Intel
# tools, x86_64 -> intel64.
function _arch_token() {
    local arch=`uname -m`

    if [[ $arch != "x86_64" ]] ; then
        _stderr "only intel64 architecture supported"
        return 1;
    fi

    _arch_token="intel64"

    return 0
}

# Echo to STDERR.
function _stderr() {
    echo $* >&2
}

# Compare two version strings of the format a.b.c.d.  Returns 1 if the
# first version is greater, 0 if it less or the same.
function _vercmp() {
    local ver1=`echo $1 | awk -F. '{printf "%03d%03d%03d%03d\n", $1, $2, $3, $4}'`
    local ver2=`echo $2 | awk -F. '{printf "%03d%03d%03d%03d\n", $1, $2, $3, $4}'`

    if [ $ver1 -gt $ver2 ] ; then
        return 1
    else
        return 0
    fi
}


function get_lustre_filesystems {
    LFS_BIN=$1

    #number of filesystems
    num_fs=0

    #filesystems as array of strings
    fs_list=()

    #number of osts per filesystem as array of integers
    num_ost_list=()

    #list of mount points as array of strings
    mount_points=()

    num_osts=0

    #causes for loop to iterate using newline as the delimiter
    IFS=$'\n'

    for line in $(${LFS_BIN} df)
    do
	#if line only contains uuid, then fs count is 0 and fs name is emtied
	echo $line | grep -q "^UUID" &> /dev/null
	if [ $? -eq 0 ]
	then
	    num_osts=0
	fi

	#  if fs count is 0, then set the name and increment it, else increment
	echo $line | grep -q "\[OST:" &> /dev/null
	if [ $? -eq 0 ]
	then
	    if [ $num_osts -eq 0 ]
	    then
		#add filesystem to fs list
		fs=$(echo $line | awk 'BEGIN { FS="-" }; {print $1}')
		fs_list[${num_fs}]=$fs
	    fi
	    num_osts=$((num_osts + 1))
	fi

	#if last line, then add number of osts to list and increment number of
	# filesystems
	echo $line | grep -q "^filesystem summary:" &> /dev/null
	if [ $? -eq 0 ]
	then
	    num_ost_list[${num_fs}]=$num_osts

	    echo $line | awk '{print $NF}' &> /dev/null
	    mount_points[${num_fs}]=$(echo $line | awk '{print $NF}')

	    num_fs=$((num_fs + 1))
	fi
    done

    unset IFS
}

# Defines a variable named $num_threads_per_core that contains the
# number of logical threads per core.  If not successful,
# $num_threads_per_core = 0 and the exit status is 1.
function num_threads_per_core() {
    local ncores=`lscpu -p=core | grep -v "^#" | sort -u | wc -l`
    local ntotalthreads=`lscpu -p=cpu | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${ncores}  && -n ${ntotalthreads} ]] ; then
        let num_threads_per_core=$ntotalthreads/$ncores
        return 0
    fi

    _stderr "could not determine the number of threads per core"
    return 1
}

# Prints the tool name and fully qualified path in the format:
# <tool_name>: <fully qualified path>
function get_tool_presence() {
    if [ $# -eq 2 ]; then
        get_tool_path $1 $2
    else
        get_tool_path $1
    fi

    echo "${tool}: $tool_path"
}

# Defines a variable named tool_path that contains a fully qualified path
# for a given tool/command
function get_tool_path() {
    declare -i ret=0
    tool=$1
    path="/usr/sbin:/sbin"
    if [ $# -eq 2 ]; then
        path=$2
    fi
    tool_path=`PATH="$PATH":$path \which --skip-functions --skip-alias $tool 2>/dev/null`
    ret=$?
    if [[ ! ( "$tool_path" =~ "$tool" ) ]] ; then
      tool_path=`PATH="$PATH":$path \which $tool 2>/dev/null`
      ret=$?
    fi
    if [ $ret != 0 ]; then
        tool_path="CLCK_LOCATION_NOT_FOUND"
    fi
}

# Defines variables  clck_cpu_family, clck_cpu_model and clck_cpu_vendors based
# on /proc/cpuinfo output
function detect_model_family_vendor () {
    clck_cpu_family=`\cat /proc/cpuinfo  | \grep -E "^cpu family\s+:" | \sort -u | \awk '{print $NF}'`
    clck_cpu_model=`\cat /proc/cpuinfo  | \grep -E "^model\s+:" | \sort -u | \awk '{print $NF}'`
    clck_cpu_vendor=`\cat /proc/cpuinfo  | \grep -E "^vendor_id\s+" | \sort -u | \awk '{print $NF}'`
    clck_cpu_stepping=`\cat /proc/cpuinfo  | \grep -E "^stepping\s+" | \sort -u | \awk '{print $NF}'`
    clck_cpu_model_number=`\cat /proc/cpuinfo  | \grep -E "^model name\s+" | \sort -u | \awk '{print $7}'`

    if [[ $clck_cpu_family == "" || $clck_cpu_model == "" || $clck_cpu_vendor == "" || $clck_cpu_stepping == "" ]] ; then
        _stderr "Could not determine model/family/vendorid/stepping of the cpu"
        return 1
    fi
    return 0
}

# Defines variable clck_cpu_type based on model, family, vendor and architecture
function get_cpu_type () {
    _arch_token
    detect_model_family_vendor

    clck_cpu_type="generic"

    # Enable easier identification of Intel(R) Xeon(R) Scalable Processors
    # of any generation
    is_xeon_scalable=0

    # Identify Intel(R) Processors
    # second-generation Intel(R) Xeon(R) Scalable Processors
    if [[ $_arch_token == "intel64" && $clck_cpu_vendor == GenuineIntel ]] ; then
      if [[ $clck_cpu_family == 6 && $clck_cpu_model == 87 ]] ; then
        clck_cpu_type="xeonphi"
        return 0
      elif [[ $clck_cpu_family == 6 && $clck_cpu_model == 133 ]] ; then
        clck_cpu_type="xeonphi_m"
        return 0
    # third-generation Intel(R) Xeon(R) Scalable Processors
      elif [[ $clck_cpu_family -eq 6 && $clck_cpu_model -eq 106 ]] ; then
        clck_cpu_type="xeon_scalable_tg"
        is_xeon_scalable=1
        return 0
    # second-generation Intel(R) Xeon(R) Scalable Processors
      elif [[ $clck_cpu_family -eq 6 && $clck_cpu_model -eq 85 ]] ; then
        if [[ $clck_cpu_stepping -eq 2 ||  $clck_cpu_stepping -eq 4 ]] ; then
          clck_cpu_type="xeon_scalable"
          is_xeon_scalable=1
          return 0
        elif [[ $clck_cpu_stepping -ge 5 || $clck_cpu_stepping -le 7 ]] ; then
          if [[ $clck_cpu_model_number =~ ^[0-9]+$ && $clck_cpu_model_number -gt 9200 ]]; then
            clck_cpu_type="xeon_scalable_csp_a"
          else
            clck_cpu_type="xeon_scalable_csp"
          fi
          is_xeon_scalable=1
          return 0
        fi
      fi
    fi
    return 1
}

# Defines the variable pdsh contained in the pdsh variable.
# If successful returns 0 otherwise returns 1
function get_pdsh() {
   CURRENT_DIR=$(dirname $0)

   PATH_TO_PDSH="$CURRENT_DIR/../../../libexec/intel64"

   if [[ $PATH_TO_PDSH != 0 ]] ; then
     pdsh="$PATH_TO_PDSH/pdsh"
     return 0
   fi

   _stderr "could not determine the pdsh path"
   return 1
}

# Defines a variable named $num_sockets that contains the
# number of sockets.  If not successful,
# $num_sockets = 0 and the exit status is 1.
function num_sockets() {
    local nsockets=`lscpu -p=socket | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${nsockets} ]] ; then
        num_sockets=$nsockets
        return 0
    fi

    _stderr "could not determine the number of sockets"
    return 1
}

# Defines a variable named $num_sockets_cluster_min that
# contains the minimum number of sockets across a set of nodes.
# If not successful,$num_sockets_cluster_min = 0 and the exit
# status is 1.
function num_sockets_cluster_min() {

    local nodefile=$1
    local prefix=$2
    num_sockets_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local nsockets=`$pdsh -N -w ^$nodefile "bash -c '. $self && num_sockets && echo \\\$num_sockets'" | sort -un | head -1`
    if [[ $? == 0 && -n ${nsockets} && ${nsockets} =~ ^[0-9]+$ ]] ; then
        num_sockets_cluster_min=$nsockets
        return 0
    fi

    _stderr "could not determine the minimum cluster number of sockets"
    return 1
}

# Defines a variable named $num_sockets_cluster_min_mpirun that
# contains the minimum number of sockets across a set of nodes.
# If not successful,$num_sockets_cluster_min_mpirun = 0 and the exit
# status is 1.
function num_sockets_cluster_min_mpirun() {

    local nodefile=$1
    local prefix=$2
    num_sockets_cluster_min_mpirun=0

    _arch_token

    local self="${prefix}/../../functions"

    local nsockets=`mpirun -ppn 1 -f $nodefile bash -c ". $self && num_sockets && echo \\\$num_sockets" | sort -un | head -1`
    if [[ $? == 0 && -n ${nsockets} && ${nsockets} =~ ^[0-9]+$ ]] ; then
        num_sockets_cluster_min_mpirun=$nsockets
        return 0
    fi

    _stderr "could not determine the minimum cluster number of sockets"
    return 1
}

function e810_with_rdma() {
  # Check lspci for the presence of E810
  num_of_e810=$(lspci -n | grep -cE "8086:159[1|2|3]")
  # check if the ice driver is loaded
  ice_loaded=$(lsmod |grep -ce "^ice\ ")
  # check if the irdma driver is loaded
  irdma_loaded=$(lsmod |grep -ce "^irdma\ ")
  #if irdma is loaded, check if roce is enabled
  if [[ ${irdma_loaded} -ne 0 ]] ; then
    if [[ -e /sys/module/irdma/parameters/roce_ena ]] ; then
      irdma_roce_enabled=$(cat /sys/module/irdma/parameters/roce_ena)
    fi
  fi
  e810_has_rdma=0
  if [[ ${num_of_e810} -ne 0  && \
        ${ice_loaded} -ne 0 && \
        ${irdma_loaded} -ne 0 && \
        ${irdma_roce_enabled} -ne 0 ]] ; then
    e810_has_rdma=1
  fi
}

function e810_with_iefs() {
  # Check lspci for the presence of E810
  num_of_e810=$(lspci -n | grep -cE "8086:159[1|2|3]")
  # check if the rv driver is loaded
  rv_loaded=$(lsmod |grep -ce "^rv\ ")
  e810_has_iefs=0
  if [[ ${num_of_e810} -ne 0  && \
        ${rv_loaded} -ne 0 && \
        -e /dev/rv ]] ; then
    e810_has_iefs=1
  fi
}
