#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          sacctmgr modify account, descriptions, limits       
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2008 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Joseph Donaghy <donaghy1@llnl.gov>
# LLNL-CODE-402394.
# 
# This file is part of SLURM, a resource management program.
# For details, see <http://www.llnl.gov/linux/slurm/>.
#  
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
# 
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# 
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id     "21.13"
set exit_code   0
set tc1		tcluster1
set tc2		tcluster2
set tc3		tcluster3
set fs1		2500
set fs2		1700
set mc1		1000000
set mc2		700000
set mj1		50
set mj2		70
set mn1		300
set mn2		700
set mw1		01:00:00
set mw2		00:07:00
set acc		account
set ass		associations
set nm1		testaccta1
set nm2		testaccta2
set nm3		testaccta3
set des		Description
set ds1		testaccounta1
set ds2		testacct
set org		Organization
set or1		accountorga1
set or2		acttrg
set qs		QosLevel
set qs1		normal
set par		parent
set access_err  0

print_header $test_id

set timeout 60

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountStorageType\n"
	exit 0
}

#
# Verify if Administrator privileges
#
 if { [string compare [check_accounting_admin_level] "Administrator"] } {
	send_user "\nWARNING: This test can't be run without being an Accounting administrator.\nUse sacctmgr mod user \$USER_NAME admin=admin.\n"
	exit 0
}

#
# Use sacctmgr to create a cluster
#	
proc _add_cluster {name} {
        global sacctmgr timeout

	set exit_code 0
	set matches 0

	if { ![string length $name] } {
		send_user "FAILURE: we need a name to add\n"
		return 1
	}

	set my_pid [spawn $sacctmgr -i add cluster $name]
	expect {
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr exit_code 1
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr exit_code 1
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknwon problem\n"
			incr exit_code 1
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr exit_code 1
		}
		-re "Adding Cluster" {
			incr matches
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr add not responding\n"
			slow_kill $my_pid
			exit_code 1
		}
		eof {
			wait
		}
	}

	if {$matches != 1} {
		send_user "\nFAILURE:  sacctmgr had a problem adding clusters
	got $matches\n"
		incr exit_code 1
	}

	if { ![check_acct_associations] } {
		send_user "\nFAILURE:  Our associations don't line up\n"
		incr exit_code 1
	}
	
	return $exit_code
}

#
# Use sacctmgr to remove the test cluster
#
proc _remove_cluster {name} {
        global access_err sacctmgr timeout

	set exit_code 0
	set matches 0
	set nothing 0

	if { ![string length $name] } {
		send_user "FAILURE: we need a name to remove\n"
		return 1
	}

	set my_pid [spawn $sacctmgr -i delete cluster $name]
	expect {
		-re "privilege to preform this action" {
			set access_err 1
			exp_continue
		}
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr exit_code 1
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr exit_code 1
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknwon problem\n"
			incr exit_code 1
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr exit_code 1
		}
		-re "Deleting clusters" {
			incr matches
			exp_continue
		}
		-re " Nothing deleted" {
			incr matches
			set nothing 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr delete not responding\n"
			slow_kill $my_pid
			incr exit_code 1
		}
		eof {
			wait
		}
	}
	if {$access_err != 0} {
		return 1
	}
	if {$matches != 1} {
		send_user "\nFAILURE: sacctmgr had a problem deleting cluster got $matches\n"
		incr exit_code 1
	}
	if { !$nothing } {
		if { ![check_acct_associations] } {
			send_user "\nFAILURE:  Our associations don't line up\n"
			incr exit_code 1
		}
	}

	return $exit_code
}

#
# Use sacctmgr to add an account
#
proc _add_acct { cluster name } {
        global sacctmgr timeout

	set exit_code 0
	set matches 0

	if { ![string length $name] } {
		send_user "FAILURE: we need a name to add\n"
		return 1
	}

	set command "$name"

	if { [string length $cluster] } {
		set command "$command cluster=$cluster"
	}

	set my_pid [eval spawn $sacctmgr -i add account $command]
	expect {
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr exit_code 1
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr exit_code 1
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknwon problem\n"
			incr exit_code 1
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr exit_code 1
		}
		-re "Adding Account" {
			incr matches
			exp_continue
		}
		-re "Associations" {
			incr matches
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr add not responding\n"
			slow_kill $my_pid
			incr exit_code 1
		}
		eof {
			wait
		}
	}

	if {$matches != 2} {
		send_user "\nFAILURE:  sacctmgr had a problem adding account.
	got $matches\n"
		incr exit_code 1
	}

	if { ![check_acct_associations] } {
		send_user "\nFAILURE:  Our associations don't line up\n"
		incr exit_code 1
	}
	return $exit_code
}

#
# Use sacctmgr to remove an account
#
proc _remove_acct { cluster name } {
        global sacctmgr timeout

	set exit_code 0
	set matches 0
	set nothing 1
	set check "Deleting account"

	if { ![string length $name] } {
		send_user "FAILURE: we need a name to remove\n"
		return 1
	}

	set command "$name"

	if { [string length $cluster] } {
		set command "$command cluster=$cluster"
		set check "Deleting account associations"
	}

	set my_pid [eval spawn $sacctmgr -i delete account $command]
	expect {
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr exit_code 1
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr exit_code 1
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknwon problem\n"
			incr exit_code 1
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr exit_code 1
		}
		-re "$check" {
			incr matches
			exp_continue
		}
		-re " Nothing deleted" {
			incr matches
			set nothing 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr add not responding\n"
			slow_kill $my_pid
			incr exit_code 1
		}
		eof {
			wait
		}
	}

	if {$matches != 1} {
		send_user "\nFAILURE:  sacctmgr had a problem deleting account.
	got $matches\n"
		incr exit_code 1
	}

	if { !$nothing } {
		if { ![check_acct_associations] } {
			send_user "\nFAILURE:  Our associations don't line up\n"
			incr exit_code 1
		}
	}

	return $exit_code
}

#
# Use sacctmgr to modify an account
#
proc _mod_acct { cluster name desc org parent fs maxcpu maxjob maxnodes maxwall wdesc worg} {
	global sacctmgr timeout
	
	set exit_code 0
	set matches 0
	set expected 0
	set acct_stuff 0
	set assoc_stuff 0

	if { ![string length $name] } {
		send_user "FAILURE: we need a name to modify\n"
		return 1
	}

	#set up the where
	set wcommand "where $name"

	if { [string length $cluster] } {
		set wcommand "$wcommand cluster=$cluster"
	}

	if { [string length $wdesc] } {
		set wcommand "$wcommand description='$wdesc'"
	}

	if { [string length $worg] } {
		set wcommand "$wcommand organization='$worg'"
	}

	#set up the set
	set scommand "set"
	if { [string length $parent] } {
		set scommand "$scommand parent=$parent"
		set assoc_stuff 1
	}

	if { [string length $fs] } {
		set scommand "$scommand fairshare=$fs"
		set assoc_stuff 1
	}

	if { [string length $maxcpu] } {
		set scommand "$scommand maxcpum=$maxcpu"
		set assoc_stuff 1
	}

	if { [string length $maxjob] } {
		set scommand "$scommand maxj=$maxjob"
		set assoc_stuff 1
	}

	if { [string length $maxnodes] } {
		set scommand "$scommand maxn=$maxnodes"
		set assoc_stuff 1
	}

	if { [string length $maxwall] } {
		set scommand "$scommand maxw=$maxwall"
		set assoc_stuff 1
	}

	if { [string length $desc] } {
		set scommand "$scommand description='$desc'"
		set acct_stuff 1
	}

	if { [string length $org] } {
		set scommand "$scommand organization='$org'"
		set acct_stuff 1
	}

	incr expected $acct_stuff
	incr expected $assoc_stuff

	set my_pid [eval spawn $sacctmgr -i modify account $scommand $wcommand ]
	expect {
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr exit_code 1
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr exit_code 1
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknwon problem\n"
			incr exit_code 1
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr exit_code 1
		}
		-re "Modified accounts" {
			incr matches
			exp_continue
		}
		-re "Modified account associations" {
			incr matches
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr add not responding\n"
			slow_kill $my_pid
			incr exit_code 1
		}
		eof {
			wait
		}
	}

	if {$matches != $expected} {
		send_user "\nFAILURE:  sacctmgr had a problem modifying account.
	got $matches needed $expected\n"
		incr exit_code 1
	}
	
	if { ![check_acct_associations] } {
		send_user "\nFAILURE:  Our associations don't line up\n"
		incr exit_code 1
	}
	return $exit_code
}

#make sure we have a clean system and permission to do this work
_remove_cluster "$tc1,$tc2"
if {$access_err != 0} {
	send_user "\nWARNING: not authorized to perform this test\n"
	exit $exit_code
}
_remove_acct "" "$nm1,$nm2,$nm3"

#add cluster
incr exit_code [_add_cluster "$tc1,$tc2"]
if { $exit_code } {
	_remove_acct "" "$nm1,$nm2,$nm3"
	_remove_cluster "$tc1,$tc2"
	exit $exit_code
}

#add accounts
incr exit_code [_add_acct "$tc1,$tc2" "$nm1,$nm2,$nm3"]
if { $exit_code } {
	_remove_acct "" "$nm1,$nm2,$nm3"
	_remove_cluster "$tc1,$tc2"
	exit $exit_code
}

# OK This is the real test, first test change the desc and org of the accounts 
incr exit_code [_mod_acct "" "$nm1,$nm2,$nm3" $ds2 $or2 "" "" "" "" "" "" "" ""]
if { $exit_code } {
	_remove_acct "" "$nm1,$nm2,$nm3"
	_remove_cluster "$tc1,$tc2"
	exit $exit_code
}

#
# Use sacctmgr to list the test account modifications
#
set matches 0
set my_pid [spawn $sacctmgr -n -p list account acc=$nm1,$nm2,$nm3 format="Account,Desc,Org"]
expect {
	-re "There was a problem" {
	        send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "($nm1|$nm2|$nm3).$ds2.$or2." {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr list associations not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	send_user "\nFAILURE:  Account modification 1 incorrect with only $matches.\n"
	incr exit_code 1
}

# Next, test change the limits of the accounts 
incr exit_code [_mod_acct "" "$nm1" "" "" "" $fs2 $mc2 $mj2 $mn2 $mw2 "" ""]
if { $exit_code } {
	_remove_acct "" "$nm1,$nm2,$nm3"
	_remove_cluster "$tc1,$tc2"
	exit $exit_code
}

#
# Use sacctmgr to list the test account modifications
#
set matches 0
set my_pid [spawn $sacctmgr -n -p list assoc acc=$nm1,$nm2,$nm3 format="Account,Cluster,Fairshare,MaxCPUM,MaxJ,MaxN,MaxW"]
expect {
	-re "There was a problem" {
	        send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "($nm1.($tc1|$tc2).$fs2.$mc2.$mj2.$mn2.$mw2.|($nm2|$nm3).($tc1|$tc2).1)" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr list associations not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {$matches != 6} {
	send_user "\nFAILURE:  Account modification 2 incorrect with only $matches.\n"
	incr exit_code 1
}

# Next, test change the desc and limits of the accounts 
incr exit_code [_mod_acct "" "$nm1" $ds1 $or1 "" -1 -1 -1 -1 -1 "" ""]
if { $exit_code } {
	_remove_acct "" "$nm1,$nm2,$nm3"
	_remove_cluster "$tc1,$tc2"
	exit $exit_code
}

#
# Use sacctmgr to list the test account modifications
#
set matches 0
set my_pid [eval spawn $sacctmgr -n -p list account withassoc acc=$nm1,$nm2,$nm3 format="Account,Desc,Org,Cluster,Fairshare,MaxCPUM,MaxJ,MaxN,MaxW"]
expect {
	-re "There was a problem" {
	        send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	# Any time you use () around something you need to combine all 
	# those things together since you can miss some and they be thrown away
	-re "($nm1.$ds1.$or1.($tc1|$tc2).1.....|($nm2|$nm3).$ds2.$or2.($tc1|$tc2).1.....)" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr list associations not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {$matches != 6} {
	send_user "\nFAILURE: HERE ???  Account modification 2 incorrect with $matches.\n"
	incr exit_code 1
}

# This is the end below here

incr exit_code [_remove_acct "$tc1,$tc2" "$nm1,$nm2,$nm3"]
incr exit_code [_remove_acct "" "$nm1,$nm2,$nm3"]
incr exit_code [_remove_cluster "$tc1,$tc2"]

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "\nFAILURE\n"
}
exit $exit_code

