Add semgrep to gerrit validation

Add test targets to call semgrep to tests/Makefile.
Integrate those targets with the gerrit validation pipeline.

There is no need to integrate with our standard .venv, so we create a
tiny dedicated venv to run semgrep.

CMK-19530

Change-Id: I0c66c53aa2b70994ab13c3c75661e4e13ff18c33

buildscripts/scripts/stages.yml

@@ -96,6 +96,9 @@ VARIABLES:
       SH: "echo '${CHANGED_FILES_REL}' | grep 'checkman/' || true"
       REPLACE_NEWLINES: true
 
+    - NAME: CHANGED_SEMGREP_RULE_FILES
+      SH: "echo '${CHANGED_FILES_REL}' | grep '^tests/semgrep/rules' || true"
+
     ########## packages ########################################################
 
     - NAME: CHANGED_CHECK_CERT_FILES
@@ -308,6 +311,16 @@ STAGES:
       RESULT_CHECK_TYPE: "JUNIT"
       RESULT_CHECK_FILE_PATTERN: "results/bandit_results.xml"
 
+    - NAME: "Python semgrep"
+      ONLY_WHEN_NOT_EMPTY: CHANGED_REFERENCE_IMAGE,CHANGED_PYTHON_FILES,CHANGED_PIPFILE_LOCK,CHANGED_PYPROJECT_TOML_FILE,CHANGED_TESTS_MAKEFILE,CHANGED_RUN_PIPENV_SCRIPT,CHANGED_SEMGREP_RULE_FILES
+      DIR: "tests"
+      ENV_VARS:
+          SEMGREP_OUTPUT_ARGS: "--junit-xml --junit-xml-output=${RESULTS}/semgrep_results.xml"
+      COMMAND: "make test-semgrep-docker"
+      TEXT_ON_SKIP: "No Python files changed"
+      RESULT_CHECK_TYPE: "JUNIT"
+      RESULT_CHECK_FILE_PATTERN: "results/semgrep_results.xml"
+
     - NAME: "Shell Format"
       ONLY_WHEN_NOT_EMPTY: CHANGED_REFERENCE_IMAGE,CHANGED_SHELL_FILES
       DIR: "tests"

scripts/run-in-docker.sh

@@ -138,6 +138,7 @@ docker run -a stdout -a stderr \
     -e USER \
     -e CI \
     -e BANDIT_OUTPUT_ARGS \
+    -e SEMGREP_OUTPUT_ARGS \
     -e GROOVYLINT_OUTPUT_ARGS \
     -e JUNIT_XML \
     -e PYLINT_ARGS \

tests/Makefile

@@ -13,6 +13,7 @@ PYTEST_OPTS_UNIT_SKIP_SLOW = -m "not slow"
 PYTEST_OPTS_UNIT_SLOW_ONLY = -m "slow"
 THREE_TO_TWO := $(PIPENV) run 3to2
 BANDIT := $(PIPENV) run bandit
+SEMGREP := $(realpath semgrep)/run-semgrep
 FIND_INVALID_TEST_FILES := $(SCRIPTS)/find-invalid-autogen-files
 ADDITIONAL_MYPY_ARGS := $(JUNIT_XML)
 # Please note: We can currently not include defines.make as it would override
@@ -66,7 +67,8 @@ AGENT_PLUGIN_UNIT_TEST_FILES_PY2 := $(subst agent-plugin-unit,agent-plugin-unit/
 	test-unit-shell test-unit-shell-docker test-shellcheck test-shellcheck-docker test-cycles test-cycles-docker \
 	test-unit-omdlib \
 	test-tidy-core test-tidy-docker test-iwyu-core test-iwyu-docker \
-	test-unit-neb test-unit-core test-flake8
+	test-unit-neb test-unit-core test-flake8 \
+	test-semgrep test-semgrep-docker
 
 clean:
 	$(RM) -r .mypy_cache $(AGENT_PLUGIN_UNIT_TEST_FILES_PY2)
@@ -128,6 +130,8 @@ help:
 	@echo "test-unit-testlib-docker            - Run doctests present in testlib-modules within docker container."
 	@echo "test-werks                          - Run Werks tests"
 	@echo "test-werks-docker                   - Run Werks tests in docker"
+	@echo "test-semgrep                        - Run semgrep code quality tests"
+	@echo "test-semgrep-docker                 - Run semgrep code quality tests in docker"
 
 bandit.ini:
 	( echo -n -e "[bandit]\ntargets: "; \
@@ -175,6 +179,20 @@ $(foreach VERS,$(AGENT_PLUGIN_PYTHON_VERSIONS),test-agent-plugin-unit-py$(VERS)-
 	    $$IMAGE_HASH \
 	    python$$PYTHON_VERSION -m pytest --ignore=tests/py2/ "/tests"
 
+test-semgrep:
+	$(SEMGREP) scan \
+	    $(SEMGREP_OUTPUT_ARGS) \
+	    --config "$(realpath semgrep/rules)" \
+	    --quiet \
+	    --oss-only \
+	    --use-git-ignore \
+	    --exclude=$(realpath ..)/packages \
+	    --disable-version-check \
+	    $(realpath ..)
+
+test-semgrep-docker:
+	../scripts/run-in-docker.sh TERM=xterm make test-semgrep
+
 test-bandit: bandit.ini
 # Currently only care about high severity reported issues. Once this is reached,
 # go and enable the medium/low checks.

tests/semgrep/.gitignore

@@ -0,0 +1 @@
+.venv

tests/semgrep/BUILD

@@ -0,0 +1,18 @@
+load("@rules_uv//uv:pip.bzl", "pip_compile")
+load("@rules_uv//uv:venv.bzl", "create_venv")
+
+exports_files([
+    "requirements.txt",
+])
+
+pip_compile(
+    name = "requirements_lock",
+    requirements_in = ":requirements.txt",
+    requirements_txt = ":requirements_lock.txt",
+)
+
+create_venv(
+    name = "venv",
+    destination_folder = "tests/semgrep/.venv",
+    requirements_txt = ":requirements_lock.txt",
+)

tests/semgrep/README.md

@@ -0,0 +1,11 @@
+Update requirements with:
+
+```
+bazel run :requirements_lock
+```
+
+Run semgrep (also creates & updates venv):
+
+```
+./run-semgrep
+```

tests/semgrep/requirements.txt

@@ -0,0 +1,2 @@
+semgrep==1.93.0
+

tests/semgrep/run-semgrep

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Create a venv with semgrep installed and run semgrep
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VENV_DIR="$SCRIPT_DIR/.venv"
+SEMGREP_EXEC="$VENV_DIR/bin/semgrep"
+
+bazel run //tests/semgrep:venv
+
+# shellcheck source=/dev/null
+source "$VENV_DIR"/bin/activate
+
+"$SEMGREP_EXEC" "$@"