#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test of job array suspend/resume.
############################################################################
# Copyright (C) 2005-2007 The Regents of the University of California.
# Copyright (C) 2008 Lawrence Livermore National Security.
# Copyright (C) 2015 SchedMD LLC.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette@schedmd.com>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set array_size     4
set file_script    "$test_dir/script"
set job_id         0
set not_supported  0

if {[get_config_param "MaxArraySize"] < [expr $array_size + 1]} {
	skip "MaxArraySize is to small for this test"
}
if [param_contains [get_config_param "PreemptMode"] "GANG"] {
	skip "This test can't be run with gang scheduling"
}
if {![is_super_user]} {
	skip "This test can't be run except as SlurmUser"
}

proc cleanup {} {
	global job_id

	cancel_job $job_id
}

run_command -fail "$bin_cc -o $test_dir/prog test3.7.prog.c"
make_bash_script $file_script "$srun $test_dir/prog 10"

set job_id [submit_job -fail "-N1 --array=0-[expr $array_size -1] --output=$test_dir/%A_%a.out -t1 $file_script"]
wait_for_job -fail ${job_id}_1 RUNNING

# Allow some initial countdown in prog
sleep 3

run_command -fail "$scontrol suspend $job_id"
wait_for_job -fail ${job_id}_1 SUSPENDED

# Needs >2 secs to print JobSuspended
sleep 3

run_command -fail "$scontrol resume $job_id"
wait_for_job -fail ${job_id}_1 RUNNING
wait_for_job -fail ${job_id}_1 DONE

for {set idx 0} {$idx < $array_size} {incr idx} {
	set filename "$test_dir/${job_id}_${idx}.out"
	wait_for_file -fail $filename
	set output [run_command_output -fail "$bin_cat $filename"]
	subtest {[regexp -all "JobSuspended" $output] == 1} "Verify job (${job_id}) index $idx was suspended"
	subtest {[regexp -all "AllDone" $output] == 1} "Verify job (${job_id}) index $idx resumed and ended"
}
