#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_prog		"$test_name.prog"
set nodename_base	"tux"
set nodename		"${nodename_base}0"
set nodename_diff	"${nodename_base}1"
set test_ulong_prog	"$test_name.ulong"
set file_in		"$test_dir/test_script"
set file_out		"$test_dir/output"
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64
set dup_err		"error: gpu duplicate device file name"
set cpu_set_err		"fatal: _set_cpu_set_bitstr: cpu_set_bitstr is empty"
set mismatch_err	"error: This GPU specified in \\\[slurm\\\|gres\\\].conf has mismatching Cores or Links"
set no_name_err		"error: Invalid GRES data, no type name"
set parse_err		"error: Parse error in file"
set links_err		"error: gres.conf: Ignoring invalid Link"
set conf_mismatch_err	"warning: A line in gres.conf for GRES $re_word_str has $number more configured than expected in slurm.conf"
set slurm_conf_type_err	"GRES in slurm.conf have a type while others do not"
set gres_conf_type_err	"fatal: gres.conf for $re_word_str, some records have \"Type=\" specification while others do not"
set gres_conf_file_err	"fatal: gres.conf for $re_word_str, some records have \"File\" specification while others do not"
set flags_mismatch_err	"fatal: Invalid GRES record name=${re_word_str} type=${re_word_str}: Flags (${re_word_str}) does not match env flags for previous GRES of same node and name"
set flags_no_gpu_err	"fatal: Invalid GRES record name=${re_word_str} type=${re_word_str}: Flags (${re_word_str}) contains \"no_gpu_env\", which must be mutually exclusive to all other GRES env flags of same node and name"
set key_parse_err	"error: _parse_next_key: Parsing error at unrecognized key"
set flags_default	"HAS_FILE,ENV_NVML,ENV_RSMI,ENV_ONEAPI,ENV_OPENCL,ENV_DEFAULT"
set flags_default_type	"HAS_FILE,HAS_TYPE,ENV_NVML,ENV_RSMI,ENV_ONEAPI,ENV_OPENCL,ENV_DEFAULT"
set flags_default_type_shared "HAS_FILE,HAS_TYPE,ENV_NVML,ENV_RSMI,ENV_ONEAPI,ENV_OPENCL,ENV_DEFAULT,SHARED,ONE_SHARING"
set flags_type		"HAS_TYPE"
set flags_file		"HAS_FILE"
set flags_file_type	"HAS_FILE,HAS_TYPE"
set flags_file_type_shared "HAS_FILE,HAS_TYPE,SHARED,ONE_SHARING"

set debug_mode 0

if {[get_config_param "SwitchType"] eq "switch/cray"} {
	# Many Cray-specific changes required in slurm.conf test files
	skip "This test can not run on a Cray system"
}

#
## testproc configuration
#
# Reduce verbosity
set testsuite_testproc_log_calls no

proc cleanup {} {
	global test_prog test_ulong_prog

	# Delete generated files
	file delete $test_prog
	file delete $test_ulong_prog
}

# Run the test under valgrind if any arguments are given
set use_valgrind 0

# log_level_t values in log.h
# NOTE: The test requires output from log level INFO, so can't go any lower
set SLURM_LOG_LEVEL_INFO 3
set SLURM_LOG_LEVEL_VERBOSE 4
set SLURM_LOG_LEVEL_DEBUG 5
set SLURM_LOG_LEVEL_DEBUG2 6
set SLURM_LOG_LEVEL_DEBUG3 7

set debug_level $SLURM_LOG_LEVEL_INFO

if {[lindex $argv 0] == "v" || [lindex $argv 0] == "valgrind"} {
	set use_valgrind 1
	log_info "Running tests under valgrind!"
} elseif {[lindex $argv 0] == "i" || [lindex $argv 0] == "info"} {
	set debug_mode 1
	log_info "Running tests under debug mode with log level INFO"
	set debug_level $SLURM_LOG_LEVEL_INFO
} elseif {[lindex $argv 0] == "verbose"} {
	set debug_mode 1
	log_info "Running tests under debug mode with log level VERBOSE"
	set debug_level $SLURM_LOG_LEVEL_VERBOSE
} elseif {[lindex $argv 0] == "d" || [lindex $argv 0] == "d1" || [lindex $argv 0] == "debug"} {
	set debug_mode 1
	log_info "Running tests under debug mode with log level DEBUG"
	set debug_level $SLURM_LOG_LEVEL_DEBUG
} elseif {[lindex $argv 0] == "d2" || [lindex $argv 0] == "debug2"} {
	set debug_mode 1
	log_info "Running tests under debug mode with log level DEBUG2"
	set debug_level $SLURM_LOG_LEVEL_DEBUG2
} elseif {[lindex $argv 0] == "d3" || [lindex $argv 0] == "debug3"} {
	set debug_mode 1
	log_info "Running tests under debug mode with log level DEBUG3"
	set debug_level $SLURM_LOG_LEVEL_DEBUG3
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Create or overwrite the slurm.conf file used by the test runner program
proc create_slurm_conf {} {
	global test_name test_dir
	# Contain slurm.conf data to this file
	# This is all we need to trigger loading the GRES GPU plugin
	set slurm_conf "
	# This file was autogenerated by $test_name
	ControlMachine=test_machine
	ClusterName=test_cluster
	GresTypes=gpu,mps,nic,mic,tmpdisk
	"

	# NOTE: There are no node definitions - node name and node gres info
	# are passed into the test program directly, not through this slurm.conf

	generate_file $slurm_conf $test_dir/slurm.conf
}

create_slurm_conf
# Set up dummy device files for testing. They just need to exist
set dev "$test_dir/nvidia"
set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
if {$use_valgrind} {
	set rc [compile_against_libslurm -full $test_prog "-DUSING_VALGRIND"]
} else {
	set rc [compile_against_libslurm -full $test_prog]
}
if {$rc} {
	fail "Cannot compile test program"
}

# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
log_debug "Build command: $build_cmd"
eval exec $build_cmd

set output [run_command_output -fail -nolog "./$test_ulong_prog"]
if {$output != $ulong_bits} {
	log_warn "We are running on a non-64-bit architecture, where sizeof(unsigned long) * 8 = $output bits"
	set ulong_bits $output
}

#
# Tests if $test_prog actual output matches expected output
#
# test_minor -		The minor test number (39.18.X).
# slurm_conf_gres -     This is the `Gres=` value that would be in
# 			slurm.conf for this node (it's easier to pass it into
# 			the program directly than to regenerate a slurm.conf
# 			file with different node definitions).
# gres_conf -		The gres.conf to use.
# fake_gpus_conf -	The fake_gpus.conf to use. This file tells Slurm to
# 			pretend to have devices detected by system, in effect
# 			emulating the result of AutoDetect=nvml.
# output_expected -	The expected "GRES_PARSABLE" output.
# err_msgs (optional) -	The error message regex to look for. Use | for multiple.
# 			If using with $output_expected, it will only reliably
# 			match if $err_msgs appear BEFORE lines with
# 			GRES_PARSABLE.
# errs_expected (optional) - The number of times you expect to see $err_msgs in
# 			the output. Defaults to 0. If this number isn't hit,
# 			then the test will fail.
#
proc test_cfg {test_minor slurm_conf_gres gres_conf fake_gpus_conf output_expected {err_msgs ""} {errs_expected 0} } {
	global test_dir nodename test_prog re_word_str
	global test_name use_valgrind file_in file_out
	global bin_cat debug_mode debug_level

	log_info "====$test_name.$test_minor===="

	generate_file $gres_conf $test_dir/gres.conf
	generate_file $fake_gpus_conf $test_dir/fake_gpus.conf

	if {$use_valgrind} {
		make_bash_script $file_in "
valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog \"$test_dir\" \"$nodename\" \"$slurm_conf_gres\" $debug_level > $file_out 2>&1
$bin_cat $file_out
rm -rf $file_out"
	} else {
		make_bash_script $file_in "
./$test_prog \"$test_dir\" \"$nodename\" \"$slurm_conf_gres\" $debug_level > $file_out 2>&1
$bin_cat $file_out
rm -rf $file_out"
	}

	##
	### Parse the `GRES_PARSABLE` output
	##
	# The order of GPUs is important because it directly corresponds to the
	# bits of the GRES bitmaps used to track the GPUs, and this shouldn't
	# change unless we deliberately change it.
	#
	# Also, we will eventually want to sort GPUs by PCI bus ID if AutoDetect
	# is used, so we need to test the order to make sure that works. We may
	# also want to guarantee that the GPU order in gres.conf is preserved if
	# AutoDetect is NOT used, so users can manually specify the PCI bus
	# order.
	if {$debug_mode} {
		set output [run_command_output -fail "$file_in"]
	} else {
		set output [run_command_output -fail -nolog "$file_in"]
	}
	set actual [regexp -all -inline "GRES_PARSABLE$re_word_str" $output]

	# Check for expected errors
	if {$err_msgs != ""} {
		set errs_actual [regexp -all "$err_msgs" $output]
	} else {
		set errs_actual 0
	}

	set expected [regexp -all -inline "GRES_PARSABLE$re_word_str" $output_expected]

	file delete $file_in

	# Check the output with what is expected
	log_debug "Actual:"
	foreach {value} $actual {
		log_debug "$value"
	}
	if {![subtest {$actual == $expected} "Expected output does not match actual output"]} {
		log_debug "Expected:"
		foreach {value} $expected {
			log_debug "$value"
		}
	}

	#  Check for expected errors, if applicable
	if {$err_msgs != ""} {
		subtest {$errs_actual == $errs_expected} "Verify expected errors" "$errs_actual != $errs_expected (regexp='$err_msgs')"
	}
}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# GPU device:
#
#     type|cpu_count|cpu_range|links|device_files[|unique_id]
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
# GRES_PARSABLE[name](device_count):type|cpu_count|cpu_range|links|device_files[|unique_id]
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = SLURM_LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################



# ##############################################################################
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ##############################################################################

# ##############################################################################
# # Test a2 - Type-less gpu specification in slurm.conf and empty gres.conf
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|(null)|${dev}1
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
tesla|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|$flags_file
"
testproc_alias "a2" test_cfg "a2" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a4 - Test empty and null identifiers in fake_gpus.conf
# ##############################################################################

set slurm_conf_gres "gpu:8"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|(null)|(null)|(null)
tesla|4|(null)|(null)|${dev}1
tesla|4|(null)|-1,0|(null)
tesla|4|(null)|-1,0|${dev}3
tesla|4|0-1|(null)|(null)
tesla|4|0-1|(null)|${dev}2
tesla|4|0-1|-1,0|(null)
tesla|4|0-1|-1,0|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|-1,0|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|-1,0|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|-1,0|${dev}3|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|-1,0|${dev}4|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|$flags_file
"
testproc_alias "a4" test_cfg "a4" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# The following tests are NVML-specific.
# They will only execute if HAVE_NVML = 1 in config.h.
# In fake_gpus_conf, a CPU range starting with `~` will trigger the GPU plugin's
# gpu_p_test_cpu_conv(). In order for this to exercise gpu/nvml-specific code,
# explicitly specify `AutoDetect=nvml` in the ad-hoc gres.conf. If not,
# gpu/generic will be used and CPU ranges will be set to null, failing the tests
# ##############################################################################

if [have_nvml] {

# ##############################################################################
# # Test a6 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 20 ulongs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by $test_name
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|$flags_file
"
testproc_alias "a6" test_cfg "a6" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a8 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 1 ulong with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by $test_name
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|$flags_file
"
testproc_alias "a8" test_cfg "a8" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a10 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity of 3 ulongs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by $test_name
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|$flags_file
"
testproc_alias "a10" test_cfg "a10" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a12 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for max # of CPUs supported (2^15)
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by $test_name
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|$flags_file
"
testproc_alias "a12" test_cfg "a12" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a14 - Test conversion of NVML-style CPU affinity array to Slurm bitstr
# 		Convert device CPU affinity for all zeros (expect this to fatal)
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|0|~zero|(null)|${dev}1
"
set expected_output ""
set err_msgs $cpu_set_err
set expected_errs 1
testproc_alias "a14" test_cfg "a14" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test a16 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPU 0 set out of 16 total CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|0|(null)|${dev}1|$flags_file
"
testproc_alias "a16" test_cfg "a16" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a18 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 0-1 set out of 4 total CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|$flags_file
"
testproc_alias "a18" test_cfg "a18" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a20 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 8-15 set out of 16 CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|8-15|(null)|${dev}1|$flags_file
"
testproc_alias "a20" test_cfg "a20" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# End NVML-specific tests
# ##############################################################################

} else {
	log_warn "====Omitting NVML-specific tests===="
}

# ##############################################################################
# # Test a22 - Test that when no GRES type is specified in slurm.conf,
#               the GRES type is NOT set to what the system detects, but rather
#               is set to NULL.
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|$flags_file
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|$flags_file
"
testproc_alias "a22" test_cfg "a22" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a24 - Test that the GRES type specified in slurm.conf overrides the
#               GRES type specified in the system, if it's a substring.
# ##############################################################################

set slurm_conf_gres "gpu:v100:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}4|$flags_file_type
"
testproc_alias "a24" test_cfg "a24" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test a26 - Test that the GRES type specified in slurm.conf doesn't need
#               to match the case of the system-detected GRES type, and that it
#               can also match the first part.
# ##############################################################################

set slurm_conf_gres "gpu:TESLA_V100:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}4|$flags_file_type
"
testproc_alias "a26" test_cfg "a26" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test a28 - Test that the GRES type specified in slurm.conf can match the
#               last part of the system-detected GRES type
# ##############################################################################

set slurm_conf_gres "gpu:pcie-16gb:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}4|$flags_file_type
"
testproc_alias "a28" test_cfg "a28" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a30 - Test that the GRES type specified in slurm.conf can match the
#               full system-detected GRES type
# ##############################################################################

set slurm_conf_gres "gpu:tesla_v100-pcie-16gb:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4|$flags_file_type
"
testproc_alias "a30" test_cfg "a30" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a32 - Test that GRES types work as expected when one GRES type is a
#               substring of another GRES type (e.g. k20m + k20m1). See bug 7345
# ##############################################################################

set slurm_conf_gres "gpu:k20m:1,gpu:k20m1:1,gpu:v100:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla_k20m1|4|0-1|(null)|${dev}1
tesla_k20m|4|0-1|(null)|${dev}2
tesla_v100-sxm2-32gb|4|0-1|(null)|${dev}3
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m1|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}3|$flags_file_type
"
testproc_alias "a32" test_cfg "a32" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a34 - Test that only the first system device that matches a conf gres
#               is used, and that the gres type is truncated.
# ##############################################################################

set slurm_conf_gres "gpu:p100:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
nvidia-p100|4|0-1|(null)|${dev}2
nvidia-p100|4|0-1|(null)|${dev}6
nvidia-p100|4|0-1|(null)|${dev}3
nvidia-p100|4|0-1|(null)|${dev}4
nvidia-p100|4|0-1|(null)|${dev}1
nvidia-p100|4|0-1|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):p100|4|0-1|(null)|${dev}1|$flags_file_type
"
testproc_alias "a34" test_cfg "a34" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a36 - Test that the shorter GRES types don't match against the longer
#               GRES types.
# ##############################################################################

set slurm_conf_gres "gpu:aaaa:1,gpu:a:1,gpu:aa:1,gpu:aaa:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
aa|4|0-1|(null)|${dev}2
aaa|4|0-1|(null)|${dev}1
a|4|0-1|(null)|${dev}3
aaaa|4|0-1|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):aa|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):a|4|0-1|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):aaaa|4|0-1|(null)|${dev}4|$flags_file_type
"

testproc_alias "a36" test_cfg "a36" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a38 - Test that the shorter GRES types don't match against the longer
#               GRES types prematurely.
# ##############################################################################

set slurm_conf_gres "gpu:k20:2,gpu:k20m:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
k20|4|0-1|(null)|${dev}2
k20m|4|0-1|(null)|${dev}1
k20|4|0-1|(null)|${dev}3
k20m|4|0-1|(null)|${dev}4
k20|4|0-1|(null)|${dev}5
k20m|4|0-1|(null)|${dev}7
k20|4|0-1|(null)|${dev}6
k20m|4|0-1|(null)|${dev}8
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}4|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}7|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}8|$flags_file_type
"

testproc_alias "a38" test_cfg "a38" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a39 - Show that shorter types will match against longer types if the
# 		longer types aren't used up, and that the longer type will be
# 		truncated accordingly.
# ##############################################################################

set slurm_conf_gres "gpu:k20:2,gpu:k20m:2"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
k20m|4|0-1|(null)|${dev}1
k20m|4|0-1|(null)|${dev}2
k20m|4|0-1|(null)|${dev}3
k20|4|0-1|(null)|${dev}4
k20|4|0-1|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}4|$flags_file_type
"

testproc_alias "a39" test_cfg "a39" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a40 - Test GRES types that are the same length
# 		Note: In a real system, the controller would set the node to
# 		drain, since the node reports one less aaa GPU than the
# 		controller is expecting.
# ##############################################################################

set slurm_conf_gres "gpu:aaa:3,gpu:bbb:2,gpu:ccc:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
test-bbb-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-ccc-test|4|0-1|(null)|${dev}5
test-aaa-test|4|0-1|(null)|${dev}1
test-bbb-test|4|0-1|(null)|${dev}4
test-ccc-test|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}4|$flags_file_type
GRES_PARSABLE\[gpu\](1):ccc|4|0-1|(null)|${dev}5|$flags_file_type
"

testproc_alias "a40" test_cfg "a40" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test a42 - Test multiple identical GRES type specifications
# ##############################################################################

set slurm_conf_gres "gpu:aaa:1,gpu:aaa:1,gpu:aaa:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
test-aaa-test|4|0-1|(null)|${dev}4
test-aaa-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-aaa-test|4|0-1|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}3|$flags_file_type
"

testproc_alias "a42" test_cfg "a42" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# Test a52 - Test slurm.conf, gres.conf, and detected device interaction
#
# The node returns a list of only 3 GPUs, not 4 (since gpu:special is not
# detected on the system). So the controller should set the node to drain.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|$flags_file_type
"

testproc_alias "a52" test_cfg "a52" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# Test a53 - Test slurm.conf, gres.conf, and detected device interaction
#
# gpu:special is added on via gres.conf. The final GPU list count matches what
# is expected in slurm.conf. No errors.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|$flags_default_type
"

testproc_alias "a53" test_cfg "a53" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test a54 - Test slurm.conf, gres.conf, and detected device interaction
#
# The gres conf record for nvidia1 *does* match a system GPU with the same Type
# and File. However, Cores is mismatched, so an error is printed in slurmd and
# the system device is omitted from the final GPU list. The total GPU count is 3
# instead of 4, so the controller sets the node to drain.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|$flags_default_type
"

testproc_alias "a54" test_cfg "a54" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test a55 - Test slurm.conf, gres.conf, and detected device interaction
#
# nvidia[0-2] matches exactly what is found on the system, so no problem there.
# tesla + nvidia3 does not match any type and file combo found in the system
# GPUs, so this is assumed to be an “extra” GPU. However, it is not added, since
# there are already 3 teslas found as outlined in slurm.conf.
# The total GPU count is 5 instead of 4, which is fine (i.e. won't set node to
# drain).
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[0-2\] Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|$flags_default_type
"
set err_msgs $conf_mismatch_err
set expected_errs 1

testproc_alias "a55" test_cfg "a55" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# Test a56 - Test slurm.conf, gres.conf, and detected device interaction
#
# nvidia[0-2] are found on the system, but nvidia3 is not. However, nvidia3 is
# assumed to be an extra GPU (like gpu:special), so it’s ok.
# The total GPU count is 5, so there are no errors or warnings.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4,gpu:special:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[0-3\] Cores=0-1
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|$flags_default_type
"

testproc_alias "a56" test_cfg "a56" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test a57 - Test that an empty slurm.conf yields no devices
# ##############################################################################

set slurm_conf_gres ""
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[0-3\] Cores=0-1
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output ""

set err_msgs $conf_mismatch_err
set expected_errs 2

testproc_alias "a57" test_cfg "a57" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# Test a58 - Test that a non-GPU GRES doesn't need an explicit entry in
# gres.conf. Also test that GPUs are rejected unless they have a File
# specification in gres.conf when AutoDetect is turned off.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1,tmpdisk:100"
# Nothing in gres.conf
set gres_conf ""
# Effectively "turn off" AutoDetect
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](100):(null)|4|(null)|(null)|(null)|CountOnly
"
testproc_alias "a58" test_cfg "a58" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test a59 - Test over-specified gres.conf compared to slurm.conf
#
# This tests that a larger gres.conf line is truncated down to match what
# slurm.conf specifies. This also tests that user errors are printed whenever
# more GRES are found in gres.conf than specified in slurm.conf.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:gtx:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[0-4\]
Name=gpu Type=tesla File=${dev}5
Name=gpu Type=special File=${dev}6
Name=gpu Type=gtx File=${dev}7
Name=nic Count=100
Name=tmpdisk Count=10G
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):gtx|4|(null)|(null)|${dev}7|$flags_default_type
"

set err_msgs $conf_mismatch_err
set expected_errs 5

testproc_alias "a59" test_cfg "a59" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# Test a60 - Test unique_id parsing (for MIGs)
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|(null)|${dev}1|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/7/0
tesla|4|0-1|(null)|${dev}2|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/10/0
tesla|4|2-3|(null)|${dev}3|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/8/0
tesla|4|2-3|(null)|${dev}4|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/9/0
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/7/0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/10/0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/8/0|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/9/0|$flags_file_type
"
testproc_alias "a60" test_cfg "a60" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ##############################################################################



# ##############################################################################
# # Test b2 - Test that all MPS is distributed across multiple GPU types
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:1080:1,gpu:gtx560:1,mps:300"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
gtx560|4|0-1|(null)|${dev}0
"
set expected_output "
GRES_PARSABLE\[mps\](100):gtx560|4|0-1|(null)|${dev}0|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):gtx560|4|0-1|(null)|${dev}0|$flags_file_type
GRES_PARSABLE\[mps\](100):tesla|4|0-1|(null)|${dev}1|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[mps\](100):1080|4|0-1|(null)|${dev}2|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}2|$flags_file_type
"

testproc_alias "b2" test_cfg "b2" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test b4 - Test that errors are emitted and configured devices omitted when
#	       Cores or Links mismatch with the corresponding system devices
# ##############################################################################

set slurm_conf_gres "gpu:ti:3,gpu:gtx:3"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=ti File=${dev}0 COREs=0
Name=gpu Type=ti File=${dev}\[1-2\] COREs=0-1
Name=gpu Type=gtx File=${dev}\[3-5\] COREs=0-1 Links=-1,0,0,0,0,0
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
1080ti|4|0-1|(null)|${dev}0
1080ti|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
gtx560|4|0-1|0,0,0,-1,0,0|${dev}3
gtx560|4|0-1|0,0,0,0,-1,0|${dev}4
gtx560|4|0-1|-1,0,0,0,0,0|${dev}5
"

set expected_output "
GRES_PARSABLE\[gpu\](1):gtx|4|0-1|-1,0,0,0,0,0|${dev}5|$flags_file_type
GRES_PARSABLE\[gpu\](1):ti|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):ti|4|0-1|(null)|${dev}2|$flags_file_type
"

set err_msgs $mismatch_err
set expected_errs 3
testproc_alias "b4" test_cfg "b4" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# # Test b5 - Test that GPUs are sorted according to links, not device file
# ##############################################################################

set slurm_conf_gres "gpu:a100:8"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 COREs=0-1  Links=0,0,0,-1,0,0,0,0
Name=gpu Type=a100 File=${dev}1 COREs=0-1  Links=0,0,-1,0,0,0,0,0
Name=gpu Type=a100 File=${dev}2 COREs=0-1  Links=0,-1,0,0,0,0,0,0
Name=gpu Type=a100 File=${dev}3 COREs=0-1  Links=-1,0,0,0,0,0,0,0
Name=gpu Type=a100 File=${dev}4 COREs=2-3 Links=0,0,0,0,0,0,0,-1
Name=gpu Type=a100 File=${dev}5 COREs=2-3 Links=0,0,0,0,0,0,-1,0
Name=gpu Type=a100 File=${dev}6 COREs=2-3 Links=0,0,0,0,0,-1,0,0
Name=gpu Type=a100 File=${dev}7 COREs=2-3 Links=0,0,0,0,-1,0,0,0
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|-1,0,0,0,0,0,0,0|${dev}3
a100|4|0-1|0,-1,0,0,0,0,0,0|${dev}2
a100|4|0-1|0,0,-1,0,0,0,0,0|${dev}1
a100|4|0-1|0,0,0,-1,0,0,0,0|${dev}0
a100|4|2-3|0,0,0,0,-1,0,0,0|${dev}7
a100|4|2-3|0,0,0,0,0,-1,0,0|${dev}6
a100|4|2-3|0,0,0,0,0,0,-1,0|${dev}5
a100|4|2-3|0,0,0,0,0,0,0,-1|${dev}4
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|-1,0,0,0,0,0,0,0|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,-1,0,0,0,0,0,0|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,0,-1,0,0,0,0,0|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,0,0,-1,0,0,0,0|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|0,0,0,0,-1,0,0,0|${dev}7|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|0,0,0,0,0,-1,0,0|${dev}6|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|0,0,0,0,0,0,-1,0|${dev}5|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|0,0,0,0,0,0,0,-1|${dev}4|$flags_file_type
"

testproc_alias "b5" test_cfg "b5" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test b6 - Test that "extra" GPUs are still used when not found on system
# ##############################################################################

set slurm_conf_gres "gpu:1080:3"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=1080 File=${dev}\[0-2\] COREs=0-1
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}2|$flags_default_type
"

testproc_alias "b6" test_cfg "b6" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# # Test b7 - Test that GPUs are sorted according links, if specified, and
# 	       device file, if not specified. Test also that null links are
# 	       sorted after non-null links.
# ##############################################################################

set slurm_conf_gres "gpu:a100:8"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 COREs=0-1  Links=0,0,0,-1
Name=gpu Type=a100 File=${dev}1 COREs=0-1  Links=0,0,-1,0
Name=gpu Type=a100 File=${dev}2 COREs=0-1  Links=0,-1,0,0
Name=gpu Type=a100 File=${dev}3 COREs=0-1  Links=-1,0,0,0
Name=gpu Type=a100 File=${dev}4 COREs=2-3
Name=gpu Type=a100 File=${dev}5 COREs=2-3
Name=gpu Type=a100 File=${dev}6 COREs=2-3
Name=gpu Type=a100 File=${dev}7 COREs=2-3
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|-1,0,0,0|${dev}3
a100|4|0-1|0,-1,0,0|${dev}2
a100|4|0-1|0,0,-1,0|${dev}1
a100|4|0-1|0,0,0,-1|${dev}0
a100|4|2-3|(null)|${dev}7
a100|4|2-3|(null)|${dev}6
a100|4|2-3|(null)|${dev}5
a100|4|2-3|(null)|${dev}4
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|-1,0,0,0|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,-1,0,0|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,0,-1,0|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|0,0,0,-1|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|(null)|${dev}4|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|(null)|${dev}5|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|(null)|${dev}6|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|2-3|(null)|${dev}7|$flags_file_type
"

testproc_alias "b7" test_cfg "b7" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# # Test b8 - Test that separate "extra" GPUs in gres.conf with different Cores
# 	       and Links are properly accounted under the typeless slurm.conf
# 	       gres specification.
# ##############################################################################

set slurm_conf_gres "gpu:5"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu File=${dev}\[0-1\] Cores=0,1
Name=gpu File=${dev}\[2-3\] Cores=0,1 Links=-1
Name=gpu File=${dev}4 Cores=0
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|-1|${dev}2|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|-1|${dev}3|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|(null)|${dev}0|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|(null)|${dev}1|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0|(null)|${dev}4|$flags_default
"

testproc_alias "b8" test_cfg "b8" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test b10 - Test that slurm.conf doesn't allow a mix of Type and no Type
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:1"
set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
set err_msgs $slurm_conf_type_err
set expected_errs 1

testproc_alias "b10" test_cfg "b10" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test b12 - Test that gres.conf doesn't allow a mix of Type and no Type
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:v100:2"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu            File=${dev}\[0-1\]
Name=gpu Type=v100  File=${dev}\[2-3\]
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $gres_conf_type_err
set expected_errs 1

testproc_alias "b12" test_cfg "b12" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test b14 - Test that gres.conf doesn't allow a mix of File and no File
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:v100:2"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla  File=${dev}\[0-1\]
Name=gpu Type=v100
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $gres_conf_file_err
set expected_errs 1

testproc_alias "b14" test_cfg "b14" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# ------------------------------------------------------------------------------
# Test gres.conf parsing
# ------------------------------------------------------------------------------
# ##############################################################################


# ##############################################################################
# # Test c2 - Test gres/gpu plus gres/mps with count
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:gtx560:1,mps:200"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|$flags_default_type
"
testproc_alias "c2" test_cfg "c2" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c4 - Test gres/mps with count and no file names
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:gtx560:1,mps:210"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=mps Count=210
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[mps\](105):gtx560|4|0,1|(null)|${dev}0|$flags_default_type_shared
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[mps\](105):tesla|4|2,3|(null)|${dev}1|$flags_default_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|$flags_default_type
"
testproc_alias "c4" test_cfg "c4" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c6 - Test using only a subset of the system devices found
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
Name=gpu Type=tesla File=${dev}1 Cores=0-1
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|(null)|${dev}1
tesla|4|0-1|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|$flags_file_type
"
testproc_alias "c6" test_cfg "c6" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c8 - Test disjoint sets of conf and system devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:6"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=2-3
Name=gpu Type=tesla File=${dev}2 Cores=2-3
Name=gpu Type=tesla File=${dev}3 Cores=2-3
Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by $test_name
tesla|4|0-1|(null)|${dev}5
tesla|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}5|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}6|$flags_file_type
"
testproc_alias "c8" test_cfg "c8" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c10 - Different links, different records
# ##############################################################################

# Devices 0-2 are all doubly linked to each other
# Device 5 is singly linked to 3-4
# Devices 7-8 are doubly linked to each other
set slurm_conf_gres "gpu:tesla:8"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|$flags_default_type
"
testproc_alias "c10" test_cfg "c10" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c12 - Empty everything
# ##############################################################################

set slurm_conf_gres ""
set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
testproc_alias "c12" test_cfg "c12" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c14 - Empty system devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|$flags_default_type
"
testproc_alias "c14" test_cfg "c14" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test c16 - Test non-GPU GRESs with types
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,tmpdisk:disky:1,nic:nikki:1,mic:mickey:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=disky File=${dev}2 Cores=0-3
Name=nic     Type=nikki File=${dev}3 Cores=0-3
Name=mic     Type=mickey File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[nic\](1):nikki|4|0-3|(null)|${dev}3|$flags_file_type
GRES_PARSABLE\[mic\](1):mickey|4|0-3|(null)|${dev}4|$flags_file_type
GRES_PARSABLE\[tmpdisk\](1):disky|4|0-3|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|$flags_default_type
"
testproc_alias "c16" test_cfg "c16" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c17 - Test non-GPU GRESs without types
# ##############################################################################

set slurm_conf_gres "gpu:1,tmpdisk:1,nic:1,mic:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu     File=${dev}1 Cores=0-3
Name=tmpdisk File=${dev}2 Cores=0-3
Name=nic     File=${dev}3 Cores=0-3
Name=mic     File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[nic\](1):(null)|4|0-3|(null)|${dev}3|HAS_FILE
GRES_PARSABLE\[mic\](1):(null)|4|0-3|(null)|${dev}4|HAS_FILE
GRES_PARSABLE\[tmpdisk\](1):(null)|4|0-3|(null)|${dev}2|HAS_FILE
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|$flags_default
"
testproc_alias "c17" test_cfg "c17" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c18 - Test NodeName when Name isn't specified
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2"
set gres_conf "
# This file was autogenerated by $test_name
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|$flags_default_type
"

set err_msgs $no_name_err
set expected_errs 1
testproc_alias "c18" test_cfg "c18" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c20 - Test that empty Name does not parse and produces error
# ##############################################################################

set slurm_conf_gres ""
set gres_conf "
# This file was autogenerated by $test_name
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $key_parse_err
set expected_errs 1
testproc_alias "c20" test_cfg "c20" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c22 - Ensure no malloc error for large count with non-GPU GRES
#               See bug 6014
# ##############################################################################

set slurm_conf_gres "tmpdisk:10g"
set gres_conf "Name=tmpdisk count=10g"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):(null)|4|(null)|(null)|(null)|
"
testproc_alias "c22" test_cfg "c22" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c23 - Ensure no errors for large count with non-GPU GRES *and* with
# 		Type and Cores specified in gres.conf.
# ##############################################################################

set slurm_conf_gres "tmpdisk:tempy:10g"
set gres_conf "Name=tmpdisk Type=tempy Count=10g Cores=0-1"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):tempy|4|0-1|(null)|(null)|HAS_TYPE
"
testproc_alias "c23" test_cfg "c23" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c24 - Different types, different records
# ##############################################################################

set slurm_conf_gres "gpu:a:2,gpu:b:2"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a File=${dev}1 Cores=0-3
Name=gpu Type=a File=${dev}2 Cores=0-3
Name=gpu Type=b File=${dev}5 Cores=0-3
Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}5|$flags_default_type
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}6|$flags_default_type
"
testproc_alias "c24" test_cfg "c24" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c28 - Test alternating device/cpu ranges
# ##############################################################################

set slurm_conf_gres "gpu:tesla_a:4,gpu:tesla_b:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}4|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}5|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}6|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}7|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}8|$flags_default_type
"
testproc_alias "c28" test_cfg "c28" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c30 - Test duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:6"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
testproc_alias "c30" test_cfg "c30" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c32 - Test increasing duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:10"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1       Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
testproc_alias "c32" test_cfg "c32" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c34 - Test decreasing duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:10"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
testproc_alias "c34" test_cfg "c34" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c36 - Test duplicate devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}1 Cores=0-1
Name=gpu Type=tesla File=${dev}1 Cores=0-2
Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
testproc_alias "c36" test_cfg "c36" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c38 - Test overlapping Core ranges
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}2 Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|$flags_default_type
"
testproc_alias "c38" test_cfg "c38" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test c40 - Test that nulled, empty, and zeroed links produce errors
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|$flags_default_type
"
set err_msgs $links_err
set expected_errs 3
testproc_alias "c40" test_cfg "c40" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test c42 - Show link ranges do NOT work - only commas
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|$flags_default_type
"
testproc_alias "c42" test_cfg "c42" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c46 - Type not specified
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3
Name=gpu File=${dev}3 Cores=0-3
Name=gpu File=${dev}4 Cores=0-3

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}2|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}3|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}4|$flags_default
"
testproc_alias "c46" test_cfg "c46" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c48 - Cores not specified or null
# ##############################################################################

set slurm_conf_gres "gpu:tesla:5"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1
Name=gpu Type=tesla File=${dev}2
Name=gpu Type=tesla File=${dev}3 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Cores=null
Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4||(null)|${dev}3|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|$flags_default_type
"
testproc_alias "c48" test_cfg "c48" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# TODO: What other tests should we add?
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# ##############################################################################
# # Test c50 - Test examples in gres.conf docs
# ##############################################################################

set slurm_conf_gres "gpu:gtx560:1,gpu:tesla:1,mps:200"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}0
Name=mps Count=100 File=${dev}1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|$flags_default_type
"
testproc_alias "c50" test_cfg "c50" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c52 - Test examples in gres.conf docs
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}\[0-1\] COREs=0,1
# NOTE: nvidia2 device is out of service
# Name=gpu Type=tesla File=${dev}\[2-3\] COREs=2,3
Name=gpu Type=tesla File=${dev}3 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|$flags_default_type
"
testproc_alias "c52" test_cfg "c52" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c54 - Test examples in gres.conf docs
# #		Note: ${nodename_base}0 == $nodename, so only node 0's GRES
# #		definitions will apply to this test
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf "
# This file was autogenerated by $test_name
## Explicitly specify devices on nodes ${nodename_base}0-${nodename_base}15
# NodeName=${nodename_base}\[0-15\] Name=gpu File=${dev}\[0-3\]
# NOTE: ${nodename_base}3 ${dev}1 device is out of service
NodeName=${nodename_base}\[0-2\] Name=gpu File=${dev}\[0-3\]
NodeName=${nodename_base}3 Name=gpu File=${dev}\[0,2-3\]
NodeName=${nodename_base}\[4-15\] Name=gpu File=${dev}\[0-3\]
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}0|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}2|$flags_default
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}3|$flags_default
"
testproc_alias "c54" test_cfg "c54" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test c56 - Test gres/mps with count on only some devices
# # NOTE the device numbers being out of order, one GPU without a gres/mps and
# #      a gres/mps with a device file not valid for any configured GPU
# ##############################################################################

set slurm_conf_gres "gpu:gtx560:1,gpu:tesla:2,mps:900"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}3 COREs=2,3
Name=mps Count=200 File=${dev}1
Name=mps Count=300 File=${dev}3
Name=mps Count=400 File=${dev}8
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[mps\](0):gtx560|4|0,1|(null)|${dev}0|$flags_default_type_shared
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[mps\](200):tesla|4|2,3|(null)|${dev}1|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[mps\](300):tesla|4|2,3|(null)|${dev}3|$flags_file_type_shared
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|$flags_default_type
"
testproc_alias "c56" test_cfg "c56" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ==============================================================================
# ==============================================================================
# # Tests dX - Test Flags parsing with and without AutoDetect
# ==============================================================================
# ==============================================================================

# ##############################################################################
# # Test d1 - Default (all envs set)
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 Cores=0-1
Name=gpu Type=a100 File=${dev}1 Cores=0-1
Name=gpu Type=a100 File=${dev}2 Cores=0-1
Name=gpu Type=a100 File=${dev}3 Cores=0-1
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_default_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_default_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_default_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_default_type
"

testproc_alias "d1" test_cfg "d1" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test d2 - No envs set
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 Cores=0-1 Flags=no_gpu_env
Name=gpu Type=a100 File=${dev}1 Cores=0-1 Flags=no_gpu_env
Name=gpu Type=a100 File=${dev}2 Cores=0-1 Flags=no_gpu_env
Name=gpu Type=a100 File=${dev}3 Cores=0-1 Flags=no_gpu_env
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type
"

testproc_alias "d2" test_cfg "d2" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test d3 - Test with NodeName
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
NodeName=$nodename      Name=gpu Type=a100 File=${dev}0 Cores=0-1 Flags=amd_gpu_env
NodeName=$nodename      Name=gpu Type=a100 File=${dev}1 Cores=0-1 Flags=amd_gpu_env
NodeName=$nodename_diff Name=gpu Type=a100 File=${dev}2 Cores=0-1 Flags=nvidia_gpu_env
NodeName=$nodename_diff Name=gpu Type=a100 File=${dev}3 Cores=0-1 Flags=nvidia_gpu_env
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_RSMI
"

testproc_alias "d3" test_cfg "d3" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test d4 - AutoDetect match with conf does NOT overwrite no_gpu_env
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0       Cores=0-1 Flags=no_gpu_env
Name=gpu Type=a100 File=${dev}\[1-3\] Cores=0-1 Flags=no_gpu_env
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|(null)|${dev}0|(null)|nvidia_gpu_env
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type
"

testproc_alias "d4" test_cfg "d4" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test d5 - Test that AutoDetect and conf can match with correct Flags
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0       Cores=0-1 Flags=nvidia_gpu_env
Name=gpu Type=a100 File=${dev}\[1-3\] Cores=0-1 Flags=nvidia_gpu_env
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|(null)|${dev}0|(null)|nvidia_gpu_env
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type,ENV_NVML
"

testproc_alias "d5" test_cfg "d5" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test d6 - Test that AutoDetect works with no conf
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}\[1-3\] Cores=0-1 Flags=nvidia_gpu_env
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|(null)|${dev}0|(null)|nvidia_gpu_env
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type,ENV_NVML
"

testproc_alias "d6" test_cfg "d6" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test d7 - Test that AutoDetect works with no conf, and other conf different
#              NOTE: The stepd will combine all env flags of records on the same
#              node, so in reality, both CUDA_* and ROCR_* would be set on all.
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}\[1-3\] Cores=0-1 Flags=amd_gpu_env
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|(null)|${dev}0|(null)|nvidia_gpu_env
"

set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_NVML
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type,ENV_RSMI
"

testproc_alias "d7" test_cfg "d7" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test d8 - Test that flags propagate to next GRES line
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 Cores=0-1 Flags=amd_gpu_env,nvidia_gpu_env
Name=gpu Type=a100 File=${dev}1 Cores=0-1
Name=gpu Type=a100 File=${dev}2 Cores=0-1
Name=gpu Type=a100 File=${dev}3 Cores=0-1
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_NVML,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_NVML,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}2|$flags_file_type,ENV_NVML,ENV_RSMI
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}3|$flags_file_type,ENV_NVML,ENV_RSMI
"

testproc_alias "d8" test_cfg "d8" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test d9 - Fatal on env flag mismatch
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}\[0-1\] Cores=0-1 Flags=no_gpu_env
Name=gpu Type=a100 File=${dev}\[2-3\] Cores=0-1 Flags=nvidia_gpu_env
"

set fake_gpus_conf ""
set expected_output ""
set err_msgs $flags_mismatch_err
set expected_errs 1
testproc_alias "d9" test_cfg "d9" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test d10 - Fatal on env flag mismatch with default
# ##############################################################################

set slurm_conf_gres "gpu:a100:4"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0       Cores=0-1
Name=gpu Type=a100 File=${dev}\[1-3\] Cores=0-1 Flags=nvidia_gpu_env
"

set fake_gpus_conf "
# This file was autogenerated by $test_name
a100|4|0-1|(null)|${dev}0|(null)|nvidia_gpu_env
"

set expected_output ""
set err_msgs $flags_mismatch_err
set expected_errs 1

testproc_alias "d10" test_cfg "d10" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test d11 - Make sure that env flag propagation works with different types
#               of the same GRES name
# ##############################################################################

set slurm_conf_gres "gpu:a100:1,gpu:b100:1,gpu:c100:1,gpu:d100:1"
set gres_conf "
# This file was autogenerated by $test_name
Name=gpu Type=a100 File=${dev}0 Cores=0-1 Flags=amd_gpu_env
Name=gpu Type=b100 File=${dev}1 Cores=0-1
Name=gpu Type=c100 File=${dev}2 Cores=0-1
Name=gpu Type=d100 File=${dev}3 Cores=0-1
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a100|4|0-1|(null)|${dev}0|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):b100|4|0-1|(null)|${dev}1|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):c100|4|0-1|(null)|${dev}2|$flags_file_type,ENV_RSMI
GRES_PARSABLE\[gpu\](1):d100|4|0-1|(null)|${dev}3|$flags_file_type,ENV_RSMI
"

testproc_alias "d11" test_cfg "d11" $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output
