diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4514941e..e8826197 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -497,6 +497,24 @@ jobs: - name: Test enter and exit maintainenace mode with set noout and stop osd run: ~/actionutils.sh test_maintenance_enter_set_noout_stop_osds_and_exit node-wrk1 + - name: Scale down the cluster to 3 nodes + run: | + set -uex + ~/actionutils.sh headexec remove_disk osd.4 + ~/actionutils.sh headexec remove_node node-wrk3 + + if ~/actionutils.sh headexec "microceph cluster maintenance enter node-wrk1"; then + echo "Unexpected to succeed entering maintenance mode with less than 4 nodes" + exit 1 + fi + echo "It's expected to fail entering maintenance mode with less than 4 nodes" + + - name: Test force enter and exit maintainenace mode without set noout and stop osd + run: ~/actionutils.sh test_maintenance_enter_and_exit_force node-wrk1 + + - name: Test force enter and exit maintainenace mode with set noout and stop osd + run: ~/actionutils.sh test_maintenance_enter_set_noout_stop_osds_and_exit_force node-wrk1 + loop-file-tests: name: Test with loopback file OSDs runs-on: ubuntu-22.04 diff --git a/tests/scripts/actionutils.sh b/tests/scripts/actionutils.sh index 958e69f8..68c7d91d 100755 --- a/tests/scripts/actionutils.sh +++ b/tests/scripts/actionutils.sh @@ -751,6 +751,23 @@ function remove_node() { sudo microceph status } +function remove_disk() { + local disk="${1?missing}" + sudo microceph disk remove $disk + for i in $(seq 1 8); do + if sudo microceph.ceph osd info $disk ; then + echo -n '.' + sleep 5 + else + echo "$disk does not exist" + break + fi + done + sleep 1 + sudo microceph.ceph -s + sudo microceph status +} + function test_migration() { local src="${1?missing}" local dst="${2?missing}" @@ -862,6 +879,7 @@ function test_dry_run_maintenance_enter() { set -xe # Debug + nodeexec $node "microceph status" nodeexec $node "microceph.ceph -s" # Count expected steps when --set-noout=false --stop-osds=false @@ -893,6 +911,7 @@ function test_dry_run_maintenance_exit() { set -xe # Debug + nodeexec $node "microceph status" nodeexec $node "microceph.ceph -s" # Count expected steps @@ -901,13 +920,14 @@ function test_dry_run_maintenance_exit() { } # Test `microceph cluster maintenance enter --set-noout=false --stop-osds=false` and then `microceph cluster maintenance exit`. -# Usage: test_maintenance_enter +# Usage: test_maintenance_enter_and_exit function test_maintenance_enter_and_exit() { local node="${1?missing}" set -xe # Debug + nodeexec $node "microceph status" nodeexec $node "microceph.ceph -s" # Enter idempotently @@ -935,6 +955,7 @@ function test_maintenance_enter_set_noout_stop_osds_and_exit() { set -xe # Debug + nodeexec $node "microceph status" nodeexec $node "microceph.ceph -s" # Enter idempotently @@ -954,6 +975,62 @@ function test_maintenance_enter_set_noout_stop_osds_and_exit() { done } +# Test `microceph cluster maintenance enter --set-noout=false --stop-osds=false --force` and then `microceph cluster maintenance exit` succeeds. +# Usage: test_maintenance_enter_and_exit_force +function test_maintenance_enter_and_exit_force() { + local node="${1?missing}" + + set -xe + + # Debug + nodeexec $node "microceph status" + nodeexec $node "microceph.ceph -s" + + # Enter idempotently + for i in $(seq 1 3); do + echo "Enter counts: $i" + nodeexec $node "microceph cluster maintenance enter --set-noout=false --stop-osds=false --force $node" + [ ! $(nodeexec $node is_osd_noout_set) ] # assert noout is unset + nodeexec $node check_snap_service_active_enabled osd # assert osd service is still active and enabled + done + + # Exit idempotently + for i in $(seq 1 3); do + echo "Exit counts: $i" + nodeexec $node "microceph cluster maintenance exit $node" + [ ! $(nodeexec $node is_osd_noout_set) ] # assert noout is unset + nodeexec $node check_snap_service_active_enabled osd # assert osd service is active and enabled + done +} + +# Test `microceph cluster maintenance enter --set-noout=true --stop-osds=true --force` and then `microceph cluster maintenance exit` succeeds. +# Usage: test_maintenance_enter_set_noout_stop_osds_and_exit +function test_maintenance_enter_set_noout_stop_osds_and_exit_force() { + local node="${1?missing}" + + set -xe + + # Debug + nodeexec $node "microceph status" + nodeexec $node "microceph.ceph -s" + + # Enter idempotently + for i in $(seq 1 3); do + echo "Enter counts: $i" + nodeexec $node "microceph cluster maintenance enter --set-noout=true --stop-osds=true --force $node" + nodeexec $node is_osd_noout_set # assert noout is set + [ ! $(nodeexec $node check_snap_service_active_enabled osd) ] # assert osd service is not active and not enabled + done + + # Exit idempotently + for i in $(seq 1 3); do + echo "Exit counts: $i" + nodeexec $node "microceph cluster maintenance exit $node" + [ ! $(nodeexec $node is_osd_noout_set) ] # assert noout is unset + nodeexec $node check_snap_service_active_enabled osd # assert osd service is active and enabled + done +} + run="${1}" shift