scripts: Update regression test fail condition

Make testbench throw 'fatal' on fail by default. Add a 'tb_silent_fail' flag to force the exit status to 0 (for RISCV-DV tests). Signed-off-by: Wiktoria Kuna <[email protected]>
chipsalliance · Jan 24, 2025 · 320966b · 320966b
1 parent bcfad71
commit 320966b
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 11 deletions.
diff --git a/.github/scripts/run_regression_test.sh b/.github/scripts/run_regression_test.sh
@@ -3,6 +3,19 @@
 SELF_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 . ${SELF_DIR}/common.inc.sh
 
+trap report_status EXIT
+
+report_status(){
+    rc=$?
+    if [ $rc != 0 ]; then
+        echo -e "${COLOR_WHITE}Test '${NAME}' ${COLOR_RED}FAILED${COLOR_CLEAR}"
+    else
+        mv ${DIR}/coverage.dat ${RESULTS_DIR}/
+        echo -e "${COLOR_WHITE}Test '${NAME}' ${COLOR_GREEN}SUCCEEDED${COLOR_CLEAR}"
+    fi
+    exit $rc
+}
+
 run_regression_test(){
     # Run a regression test with coverage collection enabled
     # Args:
@@ -60,14 +73,6 @@ run_regression_test(){
     # Run the test
     mkdir -p ${DIR}
     make -j`nproc` -C ${DIR} -f $RV_ROOT/tools/Makefile verilator $EXTRA_ARGS CONF_PARAMS="${PARAMS}" TEST=${NAME} COVERAGE=${COVERAGE} 2>&1 | tee ${LOG}
-    if [ ! -f "${DIR}/coverage.dat" ]; then
-        echo -e "${COLOR_WHITE}Test '${NAME}' ${COLOR_RED}FAILED${COLOR_CLEAR}"
-        exit 1
-    else
-        mv ${DIR}/coverage.dat ${RESULTS_DIR}/
-        echo -e "${COLOR_WHITE}Test '${NAME}' ${COLOR_GREEN}SUCCEEDED${COLOR_CLEAR}"
-        exit 0
-    fi
 }
 
 # Example usage

diff --git a/testbench/tb_top.sv b/testbench/tb_top.sv
@@ -868,7 +868,11 @@ module tb_top
         end
         else if(mailbox_write && mailbox_data[7:0] == 8'h1) begin
             $display("TEST_FAILED");
-            $finish;
+            `ifdef TB_SILENT_FAIL
+                $finish;
+            `else
+                $fatal;
+            `endif // TB_SILENT_FAIL
         end
 
         // Custom test commands

diff --git a/tools/Makefile b/tools/Makefile
@@ -119,6 +119,11 @@ ifdef assert
  ASSERT_DEFINES = +define+RV_ASSERT_ON
 endif
 
+# Prevent testbench from returning a non-zero exit code
+ifdef tb_silent_fail
+ TB_SILENT_FAIL = +define+TB_SILENT_FAIL
+endif
+
 # provide specific link file
 ifeq (,$(wildcard $(TEST_DIR)/$(TEST).ld))
 	LINK = $(BUILD_DIR)/link.ld
@@ -193,7 +198,7 @@ ${BUILD_DIR}/defines.h:
 verilator-build: ${TBFILES} ${BUILD_DIR}/defines.h $(TB_VERILATOR_SRCS)
 	$(VERILATOR)  --cc -CFLAGS "${CFLAGS}" --coverage-max-width 20000 $(defines) \
 	  $(includes) -I${RV_ROOT}/testbench -f ${RV_ROOT}/testbench/flist \
-	  $(VERILATOR_SKIP_WARNINGS) $(VERILATOR_EXTRA_ARGS) ${TBFILES} --top-module tb_top \
+	  $(VERILATOR_SKIP_WARNINGS) $(VERILATOR_EXTRA_ARGS) ${TB_SILENT_FAIL} ${TBFILES} --top-module tb_top \
 	  -exe $(TB_VERILATOR_SRCS) --autoflush --timing $(VERILATOR_DEBUG) $(VERILATOR_COVERAGE) -fno-table
 	cp ${RV_ROOT}/testbench/test_tb_top.cpp obj_dir/
 	$(MAKE) -e -C obj_dir/ -f Vtb_top.mk $(VERILATOR_MAKE_FLAGS)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
@@ -122,8 +122,10 @@ $(WORK_DIR)/verilator/Vtb_top.mk: $(WORK_DIR)/defines.h
 		--timing $(VERILATOR_COVERAGE) -fno-table -Wno-LATCH\
 		-Mdir $(WORK_DIR)/verilator
 
+# Set `tb_silent_fail` as generated instruction sequences may cause TB errors
+# Errors are to be reported when execution flows discrepancy is encountered
 $(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
-	$(MAKE) -C $(WORK_DIR)/verilator -f Vtb_top.mk OPT_FAST="-O3"
+	$(MAKE) -C $(WORK_DIR)/verilator -f Vtb_top.mk OPT_FAST="-O3" tb_silent_fail=1
 
 # Code generation
 $(TEST_DIR)/generate.log: | $(TEST_DIR)