#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm sacct with -s R, -S and -E options
############################################################################
# Copyright (C) 2017 SchedMD LLC.
# Written by Albert Gil <albert.gil@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals

set timeout         60
set jobid           0
set jobid_dependent 0
set jobid_queue     0

############################################################################
#
# Proc: sacct_query
#
# Purpose: Runs sacct with the given args and subtest if job and expected_err
#	   is returned or not as expected.
#
# Input: the jobname to match, the args to pass to sacct, and booleans
#        to specify if job is expected to be returned or not, and if an
#        error is expected or not.
#
############################################################################

proc sacct_query {jobname sacct_args expected_job {expected_err false}} {
	global sacct

	set result [run_command "$sacct -v -X -p --format=job,jobname,eligible,start,end,state $sacct_args"]
	set output [dict get $result output]
	set job_found [regexp "($jobname)" $output]
	set err_found [regexp "sacct: error:" $output]

	regsub -- {-j [^ ]+} $sacct_args "-j <job_id>" filtered_args
	regsub -- {-S [^ ]+} $filtered_args "-S <start_time>" filtered_args
	regsub -- {-E [^ ]+} $filtered_args "-E <end_time>" filtered_args

	if {$expected_job} {
		subtest {$job_found} "Verify job is shown by sacct $filtered_args"
	} else {
		subtest {!$job_found} "Verify job is NOT shown by sacct $filtered_args"
	}

	if {$expected_err} {
		subtest {$err_found} "Verify error message is printed by sacct $filtered_args"
		subtest {[dict get $result exit_code]} "Verify exit code is non-zero for sacct $filtered_args"
	} else {
		subtest {!$err_found} "Verify error message is NOT printed by sacct $filtered_args"
		subtest {![dict get $result exit_code]} "Verify exit code is zero for sacct $filtered_args"
	}
}

############################################################################
#
# Proc: get_time
#
# Purpose: Obtain the current time with the right format
#          (YYYY-MM-DDTHH:MM:SS) with the date command.
#
# Returns: The date, or 0.
#
############################################################################

proc get_time { } {
	global bin_date

	set output [run_command_output -fail "$bin_date '+%FT%T#'"]
	if {![regexp "(.+)#" $output - date]} {
		fail "Unable to parse date"
	}

	return $date
}

############################################################################
#
# Check preconditions
#
############################################################################

if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType."
}

proc cleanup {} {
	global jobid jobid_dependent jobid_queue

	cancel_job [list $jobid $jobid_dependent $jobid_queue]
}

############################################################################
#
# Main Test
#
# The followind diagram shows the timeline of a running job used in this
# test:
#
#  <-------S------>|<------E------>|<------P------>|<------R------>|<------C-------->
#   ----|-----|----|----|-----|----|----|-----|----|----|-----|----|----|-----|----> t
#      QsS   QeS   |   QsE   QeE   |   QsP   QeP   |   QsR   QeR   |   QsC   QeC
#               Job Submitted   Job Eligible    Job Starts      Job Ends
#
# As you can see we define 3 time zones:
#   S) Before job is Submited
#   E) Job is NonEligible
#   P) Job is Pending (Eligible)
#   R) Job is Running
#   C) Job is Completed/Cancelled
#
# In each time zone we define two times to be used in the sacct queries:
#
#   Qs[S,E,P,R,C]: Used mainly with -S
#   Qe[S,E,P,R,C]: Used mainly with -E
#
# The test basically does:
#
# 1) Get the QsS and QeS
# 2) Submit a job forcing a Dependency (NonEligible) and later Pending
# 3) Get the QsE and QeE
# 4) Do all the test with Q[s,e][S,E] while the job is still NonEligible
# 3) Get the QsP and QeP
# 4) Do all the test with Q[s,e][S,E,P] while the job is still Pending (Eligible)
# 5) Make the job to Run
# 6) Get the QsR and QeR
# 7) Do all the test with Q[s,e][S,E,P,R] while the job is still Running
# 8) Cancel the job
# 9) Get the QsC and QeC
# 10) Do all the test with Q[s,e][S,E,P,R,C] with a Cancelled/Completed job.
#
# The expected behavior of default values of -S and -E are the ones defined
# in the function slurmdb_job_cond_def_start_end():
#
#   - with -j and -s:
#     -S defaults to Epoch 0
#     -E defaults to -S (unless no -S then Now)
#   - with only -j (NOT -s)
#     -S defaults to Epoch 0
#     -E defaults to Now
#   - with only -s (NOT -j):
#     -S defaults to Now
#     -E defaults to -S
#   - without either -j nor -s:
#     -S defaults to Midnight
#     -E defaults to Now
#
############################################################################


############################################################################

#
# Get Random name for the job, to avoid errors with old/duplicated jobs
#
set current_timestamp [clock seconds]
set jobname "test${test_id}-${current_timestamp}"
log_info "Unique job name used to avoid false test errors with old/duplicated jobs: $jobname."

#
# Get query times before any job is run (QsS and QeS)
#
set QsS [get_time]
sleep 1
set QeS [get_time]
sleep 1

#
# Start a long job (will be cancelled)
# Test job will depend on it to test non-eligible
#
set def_node [lindex [get_nodes_by_state] 0]
set jobid_dependent [submit_job -fail "-t5 -w $def_node --exclusive -J queue-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]
wait_for_job -fail $jobid_dependent "RUNNING"

#
# Start a long job (will be cancelled)
# Test job will wait until this job, to test eligible
#
set def_node [lindex [get_nodes_by_state] 0]
set jobid_queue [submit_job -fail "-t5 -w $def_node --exclusive -J queue-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]

#
# Start main job used for testing
#
set jobid [submit_job -fail "-t5 --dependency=afterany:$jobid_dependent -w $def_node -J $jobname-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]

# Ensure that job is actually pending (non-eligible)
wait_for_job $jobid "PENDING"

# Wait for the submit time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid submit

set QsE [get_time]
sleep 1
set QeE [get_time]
sleep 1

log_info "Test cases while the job is still NonEligible..."

testproc sacct_query $jobname "-j $jobid -s R" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false

#
# Test -s PD
#
testproc sacct_query $jobname "-j $jobid -s PD" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false

#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true


#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true

# For -s PD
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true

#
# Without none -s nor -j
# -S defaults to Mindnight
# -E defaults to Now
#
testproc sacct_query $jobname "" false
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsS" false
testproc sacct_query $jobname "-S $QsE" false
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false

# cancel the first job and wait until the second one starts running
# main job should be then eligible
cancel_job $jobid_dependent
wait_for_job $jobid_queue "RUNNING"

# Wait for the eligible time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid eligible

set QsP [get_time]
sleep 1
set QeP [get_time]
sleep 1

#
# Test cases while the job is still pending
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases while the job is still PENDING..."

testproc sacct_query $jobname "-j $jobid -s R" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false

# Test -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true

#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true

#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true

# For -s PD
testproc sacct_query $jobname "-s PD" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true

#
# Without none -s nor -j
# -S defaults to Midnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true


#
# Canceling the blocking job and wait for the actual one to run
#
cancel_job $jobid_queue
wait_for_job $jobid "RUNNING"

# Wait for the start time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid start

#
# Get query times while the job is running (QsR and QeR)
#
set QsR [get_time]
sleep 1
set QeR [get_time]
sleep 1

#
# Test cases while the job is still running
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases while the job is still running..."
testproc sacct_query $jobname "-j $jobid -s R" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsR" true
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeR" true

# Test -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeR" true

#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -S $QsR" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true
testproc sacct_query $jobname "-j $jobid -E $QeR" true

#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" true
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -S $QsR" true
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true
testproc sacct_query $jobname "-s R -E $QeR" false true

# For -s PD
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -S $QsR" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true
testproc sacct_query $jobname "-s PD -E $QeR" false true

#
# Without none -s nor -j
# -S defaults to Mindnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsS -E $QeR" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeR" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeR" true
testproc sacct_query $jobname "-S $QsR -E $QeS" false true
testproc sacct_query $jobname "-S $QsR -E $QeE" false true
testproc sacct_query $jobname "-S $QsR -E $QeP" false true
testproc sacct_query $jobname "-S $QsR -E $QeR" true
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-S $QsR" true
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true
testproc sacct_query $jobname "-E $QeR" true

#
# Cancel running job
#
cancel_job $jobid

# Wait for the end time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid end

#
# Get query times when job is no more running
#
set QsC [get_time]
sleep 1
set QeC [get_time]
sleep 1

#
# Test cases for non-running job
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases when the job is not running..."
testproc sacct_query $jobname "-j $jobid -s R" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -E $QeC" true

# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
# For -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeC" true

#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -S $QsR" true
testproc sacct_query $jobname "-j $jobid -S $QsC" false
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true
testproc sacct_query $jobname "-j $jobid -E $QeR" true
testproc sacct_query $jobname "-j $jobid -E $QeC" true

#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsS -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeC" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -S $QsR" true
testproc sacct_query $jobname "-s R -S $QsC" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true
testproc sacct_query $jobname "-s R -E $QeR" false true
testproc sacct_query $jobname "-s R -E $QeC" false true

#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
# For -s PD
#
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-s PD -S $QsR -E $QeC" false
testproc sacct_query $jobname "-s PD -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeC" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -S $QsR" false
testproc sacct_query $jobname "-s PD -S $QsC" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true
testproc sacct_query $jobname "-s PD -E $QeR" false true
testproc sacct_query $jobname "-s PD -E $QeC" false true

#
# Without none -s nor -j
# -S defaults to Midnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsS -E $QeR" true
testproc sacct_query $jobname "-S $QsS -E $QeC" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeR" true
testproc sacct_query $jobname "-S $QsE -E $QeC" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeR" true
testproc sacct_query $jobname "-S $QsP -E $QeC" true
testproc sacct_query $jobname "-S $QsR -E $QeS" false true
testproc sacct_query $jobname "-S $QsR -E $QeE" false true
testproc sacct_query $jobname "-S $QsR -E $QeP" false true
testproc sacct_query $jobname "-S $QsR -E $QeR" true
testproc sacct_query $jobname "-S $QsR -E $QeC" true
testproc sacct_query $jobname "-S $QsC -E $QeS" false true
testproc sacct_query $jobname "-S $QsC -E $QeE" false true
testproc sacct_query $jobname "-S $QsC -E $QeP" false true
testproc sacct_query $jobname "-S $QsC -E $QeR" false true
testproc sacct_query $jobname "-S $QsC -E $QeC" false
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-S $QsR" true
testproc sacct_query $jobname "-S $QsC" false
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true
testproc sacct_query $jobname "-E $QeR" true
testproc sacct_query $jobname "-E $QeC" true

log_info "Jobid    : $jobid"
log_info "QsS      : $QsS"
log_info "QeS      : $QeS"
log_info "QsE      : $QsE"
log_info "QeE      : $QeE"
log_info "QsP      : $QsP"
log_info "QeP      : $QeP"
log_info "QsR      : $QsR"
log_info "QeR      : $QeR"
log_info "QsC      : $QsC"
log_info "QeC      : $QeC"
