Fix test_pfcwd_function on Mellanox platform (#16118)

tests/common/helpers/pfcwd_helper.py

@@ -569,10 +569,24 @@ def has_neighbor_device(setup_pfc_test):
     return True
 
 
-def check_pfc_storm_state(dut, port, queue, expected_state):
+def check_pfc_storm_state(dut, port, queue):
     """
     Helper function to check if PFC storm is detected/restored on a given queue
     """
+    pfcwd_stats = dut.show_and_parse("show pfcwd stats")
+    queue_name = str(port) + ":" + str(queue)
+    for entry in pfcwd_stats:
+        if entry["queue"] == queue_name:
+            logger.info("PFCWD status on queue {} stats: {}".format(queue_name, entry))
+            return entry['storm detected/restored']
+    logger.info("PFCWD not triggered on queue {}".format(queue_name))
+    return None
+
+
+def verify_pfc_storm_in_expected_state(dut, port, queue, expected_state):
+    """
+    Helper function to verify if PFC storm on a specific queue is in expected state
+    """
     pfcwd_stat = parser_show_pfcwd_stat(dut, port, queue)
     if expected_state == "storm":
         if ("storm" in pfcwd_stat[0]['status']) and \

tests/pfcwd/test_pfcwd_cli.py

@@ -12,7 +12,7 @@
 from tests.common import constants
 from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled, dualtor_ports # noqa F401
 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401, E501
-from tests.common.helpers.pfcwd_helper import send_background_traffic, check_pfc_storm_state, parser_show_pfcwd_stat
+from tests.common.helpers.pfcwd_helper import send_background_traffic, verify_pfc_storm_in_expected_state, parser_show_pfcwd_stat # noqa E501
 from tests.common.utilities import wait_until
 from tests.common.cisco_data import is_cisco_device
 
@@ -298,7 +298,7 @@ def storm_detect_path(self, dut, port, action):
 
         logger.info("Verify if PFC storm is detected on port {}".format(port))
         pytest_assert(
-            wait_until(30, 2, 5, check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "storm"),
+            wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, dut, port, self.storm_hndle.pfc_queue_idx, "storm"), # noqa E501
             "PFC storm state did not change as expected"
         )
 
@@ -317,7 +317,7 @@ def storm_restore_path(self, dut, port):
         # storm restore
         logger.info("Verify if PFC storm is restored on port {}".format(port))
         pytest_assert(
-            wait_until(30, 2, 5, check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "restore"),
+            wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, dut, port, self.storm_hndle.pfc_queue_idx, "restore"), # noqa E501
             "PFC storm state did not change as expected"
         )
 

tests/pfcwd/test_pfcwd_function.py

@@ -724,6 +724,10 @@ def storm_detect_path(self, dut, port, action):
         test_ports_info = {self.pfc_wd['rx_port'][0]: self.pfc_wd}
         queues = [self.storm_hndle.pfc_queue_idx]
 
+        if dut.facts['asic_type'] == "mellanox":
+            PFC_STORM_TIMEOUT = 30
+            pfcwd_stats_before_test = check_pfc_storm_state(dut, port, self.storm_hndle.pfc_queue_idx)
+
         with send_background_traffic(dut, self.ptf, queues, selected_test_ports, test_ports_info):
             if action != "dontcare":
                 start_wd_on_ports(dut, port, restore_time, detect_time, action)
@@ -743,14 +747,9 @@ def storm_detect_path(self, dut, port, action):
             if dut.facts['asic_type'] in ["mellanox", "cisco-8000"]:
                 # On Mellanox platform, more time is required for PFC storm being triggered
                 # as PFC pause sent from Non-Mellanox leaf fanout is not continuous sometimes.
-                PFC_STORM_TIMEOUT = 30
-                pytest_assert(
-                    wait_until(
-                        PFC_STORM_TIMEOUT, 2, 5,
-                        check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "storm"
-                    ),
-                    "PFC storm state did not change as expected"
-                )
+                pytest_assert(wait_until(PFC_STORM_TIMEOUT, 2, 0,
+                                        lambda: check_pfc_storm_state(dut, port, self.storm_hndle.pfc_queue_idx) != pfcwd_stats_before_test),  # noqa: E501, E128
+                                        "PFC storm state did not change as expected")  # noqa: E127
             else:
                 time.sleep(5)