Merge pull request #297 from AikidoSec/AIK-4292

Create new matrix strategy for end2end tests
AikidoSec · Jan 23, 2025 · 2c0b1c3 · 2c0b1c3
2 parents 0136a6b + a4093f9
commit 2c0b1c3
Show file tree

Hide file tree

Showing 158 changed files with 12,070 additions and 1,360 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -39,11 +39,13 @@ jobs:
       - name: Start databases
         working-directory:  ./sample-apps/databases
         run: docker compose up --build -d
+      - name: Install dependencies & build
+        run: |
+          python -m pip install --upgrade pip
+          make install && make build
       - name: Start flask-mysql
         working-directory: ./sample-apps/flask-mysql
-        run: |
-          cp .env.benchmark .env.example
-          docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
+        run: nohup make runBenchmark & nohup make runZenDisabled &
       - name: Install K6
         uses: grafana/setup-k6-action@v1
       - name: Run flask-mysql k6 Benchmark
@@ -58,11 +60,13 @@ jobs:
       - name: Start databases
         working-directory:  ./sample-apps/databases
         run: docker compose up --build -d
-      - name: Start starlette multi-threaded
-        working-directory: ./sample-apps/starlette-postgres-uvicorn
+      - name: Install dependencies & build
         run: |
-          cp .env.benchmark .env.example
-          docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
+          python -m pip install --upgrade pip
+          make install && make build
+      - name: Start starlette
+        working-directory: ./sample-apps/starlette-postgres-uvicorn
+        run: nohup make runBenchmark & nohup make runZenDisabled &
       - name: Install wrk
         run: |
           sudo apt-get update

diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -3,126 +3,68 @@ name: End-to-End tests
 on: [pull_request]
 
 jobs:
-  test:
+  build:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-
-    - name: Start databases
-      working-directory:  ./sample-apps/databases
-      run: docker compose up --build -d
-    - name: Build and start aikido mock server
-      working-directory: ./end2end/server
-      run: docker build -t mock-core . && docker run --name mock_core -d -p 5000:5000 mock-core
+      - uses: actions/checkout@v4
+      - name: Install poetry
+        run: pip install poetry
+      - name: Build
+        run: make build
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pkg-build
+          path: ./
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        make install
-
-    # django-mysql
-    - name: Start django-mysql
-      working-directory: ./sample-apps/django-mysql
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-mysql
-      run: sleep 5 && poetry run pytest ./end2end/django_mysql_test.py
-
-    # django-mysql-gunicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start django-mysql-gunicorn
-      working-directory: ./sample-apps/django-mysql-gunicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-mysql-gunicorn
-      run: sleep 5 && poetry run pytest ./end2end/django_mysql_gunicorn_test.py
-
-    # django-postgres-gunicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start django-postgres-gunicorn
-      working-directory: ./sample-apps/django-postgres-gunicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-postgres-gunicorn
-      run: sleep 5 && poetry run pytest ./end2end/django_postgres_gunicorn_test.py
-
-    # flask-mongo
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mongo
-      working-directory: ./sample-apps/flask-mongo
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mongo
-      run: sleep 5 && poetry run pytest ./end2end/flask_mongo_test.py
-
-    # flask-mysql
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mysql
-      working-directory: ./sample-apps/flask-mysql
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mysql
-      run: sleep 5 && poetry run pytest ./end2end/flask_mysql_test.py
-
-    # flask-mysql-uwsgi
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mysql-uwsgi
-      working-directory: ./sample-apps/flask-mysql-uwsgi
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mysql-uwsgi
-      run: sleep 5 && poetry run pytest ./end2end/flask_mysql_uwsgi_test.py
-
-    # flask-postgres
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-postgres
-      working-directory: ./sample-apps/flask-postgres
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-postgres
-      run: sleep 5 && poetry run pytest ./end2end/flask_postgres_test.py
+  end2end-test:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    needs: build
+    strategy:
+      matrix:
+        app:
+          - { name: django-mysql, testfile: end2end/django_mysql_test.py }
+          - { name: django-mysql-gunicorn, testfile: end2end/django_mysql_gunicorn_test.py }
+          - { name: django-postgres-gunicorn, testfile: end2end/django_postgres_gunicorn_test.py }
+          - { name: flask-mongo, testfile: end2end/flask_mongo_test.py }
+          - { name: flask-mysql, testfile: end2end/flask_mysql_test.py }
+          - { name: flask-mysql-uwsgi, testfile: end2end/flask_mysql_uwsgi_test.py }
+          - { name: flask-postgres, testfile: end2end/flask_postgres_test.py }
+          - { name: flask-postgres-xml, testfile: end2end/flask_postgres_xml_test.py }
+          - { name: flask-postgres-xml, testfile: end2end/flask_postgres_xml_lxml_test.py }
+          - { name: quart-postgres-uvicorn, testfile: end2end/quart_postgres_uvicorn_test.py }
+          - { name: starlette-postgres-uvicorn, testfile: end2end/starlette_postgres_uvicorn_test.py }
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - name: Install packages
+        run: sudo apt update && sudo apt install python3-dev libmysqlclient-dev
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pkg-build
 
-    # flask-postgres-xml
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-postgres-xml
-      working-directory: ./sample-apps/flask-postgres-xml
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-postgres-xml
-      run: | 
-        sleep 5
-        poetry run pytest ./end2end/flask_postgres_xml_test.py
-        docker restart mock_core
-        poetry run pytest ./end2end/flask_postgres_xml_lxml_test.py
+      - name: Start databases
+        working-directory: ./sample-apps/databases
+        run: docker compose up --build -d
+      - name: Build and start aikido mock server
+        run: |
+            cd end2end/server && docker build -t mock_core .
+            docker run --name mock_core -d -p 5000:5000 mock_core
 
-    # quart-postgres-uvicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start quart-postgres-uvicorn
-      working-directory: ./sample-apps/quart-postgres-uvicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for quart-postgres-uvicorn
-      run: sleep 5 && poetry run pytest ./end2end/quart_postgres_uvicorn_test.py
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make install
 
-    # starlette-postgres-uvicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start starlette-postgres-uvicorn
-      working-directory: ./sample-apps/starlette-postgres-uvicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for starlette-postgres-uvicorn
-      run: sleep 5 && poetry run pytest ./end2end/starlette_postgres_uvicorn_test.py
+      - name: Start application
+        working-directory: ./sample-apps/${{ matrix.app.name }}
+        run: |
+          nohup make run > output.log & tail -f output.log & sleep 20
+          nohup make runZenDisabled & sleep 20
+      - name: Run end2end tests for application
+        run: tail -f  ./sample-apps/${{ matrix.app.name }}/output.log & poetry run pytest ./${{ matrix.app.testfile }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -48,8 +48,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install poetry
       run: pip install poetry
-    - name: Download binaries
-      run: make binaries
+    - name: Download binaries & build
+      run: make build
     - name: Publish to PyPI
       env:
         POETRY_HTTP_BASIC_PYPI_USERNAME: __token__

diff --git a/.github/workflows/smoke-test-ffi.yml b/.github/workflows/smoke-test-ffi.yml
@@ -38,7 +38,7 @@ jobs:
         run: sudo apt-get install libmysqlclient-dev
 
       - name: Install dependencies
-        run: make install
+        run: make install && cp -r .cache/binaries/* aikido_zen/libs
 
       - name: Run tests
         run: poetry run pytest aikido_zen/vulnerabilities/sql_injection/init_test.py
diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml
@@ -49,11 +49,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install poetry
       run: pip install poetry
-    - name: Download binaries
-      run: make binaries
     - name: Install dependencies
       run: poetry install
-
+    - name: Download binaries & build
+      run: make build
     - name: Publish to TestPyPI
       env:
         POETRY_HTTP_BASIC_PYPI_USERNAME: __token__

diff --git a/Makefile b/Makefile
@@ -1,41 +1,47 @@
-build: binaries _build
-_build:
-	poetry build
+CACHE_DIR := .cache/binaries
+BUILD_DIR := dist/
 
+# Build/clean/lint/install :
+.PHONY: build
+build: check_binaries
+	@echo "Building using poetry, creates $(BUILD_DIR) folder."
+	@echo "Copying binaries from $(CACHE_DIR) folder : "
+	cp -r .cache/binaries/* aikido_zen/libs
+	poetry build
 .PHONY: clean
 clean:
-	poetry env remove python
-
+	@echo "Cleaning up: Removing build and cache directories, remove poetry env"
+	rm -rf $(BUILD_DIR)
+	rm -rf $(CACHE_DIR)
+	@poetry env remove $(shell poetry env list | grep 'Activated' | awk '{print $$1}')
 .PHONY: lint
 lint:
 	poetry run black aikido_zen/
 	poetry run pylint aikido_zen/
-
-install: binaries
+install: check_binaries
 	pip install poetry
 	poetry install
-
-dev_install: install _dev_install
-_dev_install:
+.PHONY: dev_install
+dev_install: install
 	poetry install --with=dev
 
 
+# Testing/Benchmarks :
 .PHONY: test
-test:
+test: build
 	poetry run pytest aikido_zen/
-
 .PHONY: end2end
 end2end:
-	poetry run pytest end2end/ 	
-
+	poetry run pytest end2end/
 .PHONY: cov
-cov:
+cov: build
 	poetry run pytest aikido_zen/ --cov=aikido_zen --cov-report=xml
-
 .PHONY: benchmark
 benchmark:
 	k6 run -q ./benchmarks/flask-mysql-benchmarks.js
 
+
+# Binaries cache :
 BASE_URL = https://github.com/AikidoSec/zen-internals/releases/download/v0.1.35
 FILES = \
     libzen_internals_aarch64-apple-darwin.dylib \
@@ -49,10 +55,19 @@ FILES = \
     libzen_internals_x86_64-unknown-linux-gnu.so \
     libzen_internals_x86_64-unknown-linux-gnu.so.sha256sum
 
-binaries: binaries_make_dir $(addprefix aikido_zen/lib/, $(FILES))
+binaries: binaries_make_dir $(addprefix .cache/binaries/, $(FILES))
 binaries_make_dir:
-	rm -rf aikido_zen/lib
-	mkdir -p aikido_zen/lib/
-aikido_zen/lib/%:
+	rm -rf .cache/binaries
+	mkdir -p .cache/binaries/
+.cache/binaries/%:
 	@echo "Downloading $*..."
 	curl -L -o $@ $(BASE_URL)/$*
+
+.PHONY: check_binaries
+check_binaries:
+	@if [ -d "$(CACHE_DIR)" ]; then \
+  		echo "Directory $(CACHE_DIR) exists."; \
+	else \
+		echo "Directory $(CACHE_DIR) is empty. Running 'make binaries'..."; \
+		$(MAKE) binaries; \
+	fi
diff --git a/aikido_zen/helpers/aikido_disabled_flag_active.py b/aikido_zen/helpers/aikido_disabled_flag_active.py
@@ -7,9 +7,9 @@
 
 def aikido_disabled_flag_active():
     """
-    Checks the environment variable "AIKIDO_DISABLED"
+    Checks the environment variable "AIKIDO_DISABLE"
     """
-    aikido_disabled_env = os.getenv("AIKIDO_DISABLED")
+    aikido_disabled_env = os.getenv("AIKIDO_DISABLE")
     if aikido_disabled_env is not None:
         return aikido_disabled_env.lower() in ["true", "1"]
     return False
diff --git a/aikido_zen/helpers/aikido_disabled_flag_active_test.py b/aikido_zen/helpers/aikido_disabled_flag_active_test.py
@@ -3,35 +3,35 @@
 
 
 def test_aikido_disabled_flag_does_not_exist(monkeypatch):
-    monkeypatch.setenv("AIKIDO_DISABLED", None)
+    monkeypatch.setenv("AIKIDO_DISABLE", None)
     assert aikido_disabled_flag_active() == False
 
 
 def test_aikido_disabled_flag_exists_but_invalid(monkeypatch):
-    monkeypatch.setenv("AIKIDO_DISABLED", "token")
+    monkeypatch.setenv("AIKIDO_DISABLE", "token")
     assert aikido_disabled_flag_active() == False
 
 
 def test_aikido_disabled_flag_exists_and_off(monkeypatch):
-    monkeypatch.setenv("AIKIDO_DISABLED", "0")
+    monkeypatch.setenv("AIKIDO_DISABLE", "0")
     assert aikido_disabled_flag_active() == False
 
-    monkeypatch.setenv("AIKIDO_DISABLED", "false")
+    monkeypatch.setenv("AIKIDO_DISABLE", "false")
     assert aikido_disabled_flag_active() == False
 
-    monkeypatch.setenv("AIKIDO_DISABLED", "False")
+    monkeypatch.setenv("AIKIDO_DISABLE", "False")
     assert aikido_disabled_flag_active() == False
 
 
 def test_aikido_disabled_flag_exists_and_on(monkeypatch):
-    monkeypatch.setenv("AIKIDO_DISABLED", "1")
+    monkeypatch.setenv("AIKIDO_DISABLE", "1")
     assert aikido_disabled_flag_active() == True
 
-    monkeypatch.setenv("AIKIDO_DISABLED", "True")
+    monkeypatch.setenv("AIKIDO_DISABLE", "True")
     assert aikido_disabled_flag_active() == True
 
-    monkeypatch.setenv("AIKIDO_DISABLED", "TRUE")
+    monkeypatch.setenv("AIKIDO_DISABLE", "TRUE")
     assert aikido_disabled_flag_active() == True
 
-    monkeypatch.setenv("AIKIDO_DISABLED", "true")
+    monkeypatch.setenv("AIKIDO_DISABLE", "true")
     assert aikido_disabled_flag_active() == True