diff --git a/t/Makefile.am b/t/Makefile.am index 58b6baf2..daa87f2e 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -40,6 +40,7 @@ TESTSCRIPTS = \ t1038-hierarchy-small-tie-all-db.t \ t1039-issue476.t \ t1040-mf-priority-projects.t \ + t1041-mf-priority-queue-limits.t \ t5000-valgrind.t \ python/t1000-example.py \ python/t1001_db.py \ diff --git a/t/t1041-mf-priority-queue-limits.t b/t/t1041-mf-priority-queue-limits.t new file mode 100755 index 00000000..4e7ae6a0 --- /dev/null +++ b/t/t1041-mf-priority-queue-limits.t @@ -0,0 +1,207 @@ +#!/bin/bash + +test_description='test multi-factor priority plugin queue limits' + +. `dirname $0`/sharness.sh + +mkdir -p conf.d + +MULTI_FACTOR_PRIORITY=${FLUX_BUILD_DIR}/src/plugins/.libs/mf_priority.so +SUBMIT_AS=${SHARNESS_TEST_SRCDIR}/scripts/submit_as.py +DB_PATH=$(pwd)/FluxAccountingTest.db + +export TEST_UNDER_FLUX_SCHED_SIMPLE_MODE="limited=1" +test_under_flux 16 job -o,--config-path=$(pwd)/conf.d + +flux setattr log-stderr-level 1 + +test_expect_success 'allow guest access to testexec' ' + flux config load <<-EOF + [exec.testexec] + allow-guests = true + EOF +' + +test_expect_success 'create flux-accounting DB' ' + flux account -p ${DB_PATH} create-db +' + +test_expect_success 'start flux-accounting service' ' + flux account-service -p ${DB_PATH} -t +' + +test_expect_success 'load multi-factor priority plugin' ' + flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} +' + +test_expect_success 'check that mf_priority plugin is loaded' ' + flux jobtap list | grep mf_priority +' + +test_expect_success 'add some banks' ' + flux account add-bank root 1 && + flux account add-bank --parent-bank=root bankA 1 +' + +test_expect_success 'add queues with different running jobs limits' ' + flux account add-queue bronze --priority=200 --max-running-jobs=3 && + flux account add-queue silver --priority=300 --max-running-jobs=2 && + flux account add-queue gold --priority=400 --max-running-jobs=1 +' + +test_expect_success 'add a user' ' + flux account add-user \ + --username=user1 \ + --userid=5001 \ + --bank=bankA \ + --queues="bronze,silver,gold" \ + --max-running-jobs=100 \ + --max-active-jobs=100 +' + +test_expect_success 'send the user and queue information to the plugin' ' + flux account-priority-update -p ${DB_PATH} +' + +test_expect_success 'configure flux with those queues' ' + cat >conf.d/queues.toml <<-EOT && + [queues.bronze] + [queues.silver] + [queues.gold] + EOT + flux config reload && + flux queue start --all +' + +# In this set of tests, an association belongs to all three available queues, +# and each queue has a different limit on the number of running jobs available +# per-association. The association will submit the max number of running jobs +# to the silver queue (2 jobs). A dependency specific to the number of running +# jobs per-queue is added to the third submitted job in the silver queue, but +# jobs submitted to other queues will still receive an alloc event. +# +# Once one of the currently running jobs in the silver queue completes and is +# cleaned up, the job with a dependency added to it will have its dependency +# removed and will receive its alloc event. +test_expect_success 'submit max number of jobs to silver queue' ' + job1=$(flux python ${SUBMIT_AS} 5001 --queue=silver sleep 60) && + job2=$(flux python ${SUBMIT_AS} 5001 --queue=silver sleep 60) && + flux job wait-event -vt 5 ${job1} alloc && + flux job wait-event -vt 5 ${job2} alloc +' + +test_expect_success 'running jobs count for the queues are incremented once jobs start' ' + flux jobtap query mf_priority.so > silver.json && + jq -e ".mf_priority_map[] | \ + select(.userid == 5001) | \ + .banks[0].queue_usage.silver == 2" user1.json && + jq -e ".mf_priority_map[] | \ + select(.userid == 5001) | \ + .banks[0].cur_run_jobs == 3" query.json && + jq -e ".mf_priority_map[] | \ + select(.userid == 5001) | \ + .banks[0].queue_usage.silver == 0" user1.json && + jq -e ".mf_priority_map[] | \ + select(.userid == 5001) | \ + .banks[0].held_jobs | length == 1"