From a759665439c6776e0655401c1dbc91b370f4c298 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dawid=20Urba=C5=84ski?= <dawurb@gmail.com>
Date: Tue, 20 Jun 2023 11:54:07 +0200
Subject: [PATCH 1/3] docker init moved to parsedmarc, updated docker files,
 updated kibana and elasticsearch version to ARM

---
 data/Dockerfiles/parsedmarc/init-script.sh | 92 ++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 data/Dockerfiles/parsedmarc/init-script.sh

diff --git a/data/Dockerfiles/parsedmarc/init-script.sh b/data/Dockerfiles/parsedmarc/init-script.sh
new file mode 100644
index 0000000..d611f87
--- /dev/null
+++ b/data/Dockerfiles/parsedmarc/init-script.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Copyright 2020, Patrik Kernstock.
+
+set -x
+
+echo "## ELASTICSEARCH"
+echo "Setting permissions..."
+chmod g+rwx -R /usr/share/elasticsearch/data/
+chgrp 0 -R /usr/share/elasticsearch/data/
+
+# echo "## NGINX"
+# echo "Checking nginx certs..."
+# cd /etc/nginx/ssl/
+# if [ ! -f "/etc/nginx/ssl/kibana.crt" ] || [ ! -f "/etc/nginx/ssl/kibana.key" ]; then
+# 	echo "No certs found. Generating..."
+# 	openssl req -x509 -nodes -days 365 -newkey rsa:3072 -keyout kibana.key -out kibana.crt \
+# 		-subj "/CN=parsedmarc" -addext "subjectAltName=DNS:parsedmarc"
+# 	echo "Certs generated."
+# fi
+
+echo "## KIBANA"
+exportFile="/etc/parsedmarc/kibana_export.ndjson"
+if [ ! -f "${exportFile}" ]; then
+	# trigger empty file to trigger below update logic.
+	touch ${exportFile}
+fi
+echo "Downloading dashboard from GitHub..."
+rm /etc/parsedmarc/kibana_export.ndjson.tmp
+curl https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson \
+	-o /etc/parsedmarc/kibana_export.ndjson.tmp
+if [ ${?} -ne 0 ]; then
+	echo "Downloading kibana export failed."
+else
+	fileNew=$(wc -c "${exportFile}.tmp" | awk -F' ' '{ print $1 }')
+	fileOld=$(wc -c "${exportFile}" | awk -F' ' '{ print $1 }')
+
+	if [ "$fileNew" == "$fileOld" ]; then
+		echo "File size is the same. Not proceeding."
+	else
+		echo "File size is different... updating..."
+
+		while ! curl -s -f -I http://kibana:5601 >/dev/null; do
+			echo "Kibana not responding... waiting 5 secs..."
+			sleep 5
+		done
+
+		echo "Kibana responded. Waiting 10s, then proceeding with dashboard update..."
+		sleep 10
+		rm ${exportFile}
+		mv ${exportFile}.tmp ${exportFile}
+		RES=$(curl -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true \
+			-H "kbn-xsrf: true" --form file=@${exportFile})
+		echo "Result: $RES"
+		if [ ${?} -ne 0 ]; then
+			echo "[!!!] Import might have failed. Manual check recommended."
+		else
+			# if the flag exists, we already set the defaultRoute once. So we don't do that again.
+			if [ ! -f "/etc/parsedmarc/flag.defaultRouteSet" ]; then
+				DEF_DASHBOARD_NAME="DMARC Summary"
+				echo "Setting '${DEF_DASHBOARD_NAME}' dashboard as default route..."
+				DEF_DASHBOARD_ID=$(cat "${exportFile}" | jq --arg DBNAME "${DEF_DASHBOARD_NAME}" 'select(.attributes.title == $DBNAME) | .id' | tr -d '"')
+				if [ "$DEF_DASHBOARD_ID" != "" ]; then
+					echo "Found dashboard ID: ${DEF_DASHBOARD_ID}"
+					DEFAULT_ROUTE="/app/kibana#/dashboard/${DEF_DASHBOARD_ID}"
+					echo "DefaultRoute being set to: ${DEFAULT_ROUTE}"
+					curl -X POST -H "kbn-xsrf: true" -H "Content-Type: application/json" \
+						"http://kibana:5601/api/kibana/settings/defaultRoute" \
+						-d "{\"value\": \"${DEFAULT_ROUTE}\"}"
+					if [ ${?} -ne 0 ]; then
+						echo "[!!!] Setting defaultRoute seems to gone wrong. Manual check recommended."
+					else
+						echo "DefaultRoute set."
+						echo "Notice: This might require a restart of Kibana to take effect. Not done automatically as part of this script."
+						echo -e "# This is a flag to remember which defaultRoute we set in the past:\n${DEFAULT_ROUTE}" \
+							> /etc/parsedmarc/flag.defaultRouteSet
+					fi
+				else
+					echo "[!] Default dashboard with name '${DEF_DASHBOARD_NAME}' could not be found."
+				fi
+			fi
+		fi
+		echo "Dashboard import done."
+	fi
+fi
+
+sleep 3
+
+# Create empty file to let other containers know that we're ready.
+touch /ready
+#sleep infinity # or while true; do sleep 86400; done
+parsedmarc -c /etc/parsedmarc/config.ini
+exit 0

From e3dd9470b788281d1bce1eb983097428af2c8e0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dawid=20Urba=C5=84ski?= <dawurb@gmail.com>
Date: Tue, 20 Jun 2023 12:08:33 +0200
Subject: [PATCH 2/3] compose and dockerfile update

---
 data/Dockerfiles/parsedmarc/Dockerfile | 15 +++---
 docker-compose.yml                     | 67 +++++++++-----------------
 2 files changed, 31 insertions(+), 51 deletions(-)

diff --git a/data/Dockerfiles/parsedmarc/Dockerfile b/data/Dockerfiles/parsedmarc/Dockerfile
index e3949c5..22d488a 100644
--- a/data/Dockerfiles/parsedmarc/Dockerfile
+++ b/data/Dockerfiles/parsedmarc/Dockerfile
@@ -1,11 +1,10 @@
 FROM pypy:3-slim
 
-ADD start.sh /start.sh
+COPY init-script.sh /start.sh
 RUN apt-get update \
-    && apt-get install -y libxslt-dev libz-dev libxml2-dev gcc libemail-outlook-message-perl \
-    && pip install -U parsedmarc \
-    && apt-get purge --yes gcc && apt autoremove --yes && apt-get clean \
-    && rm -Rf /var/lib/{apt,dpkg}/ && rm -Rf /root/.cache/ \
-    && chmod +x /start.sh
-
-ENTRYPOINT [ "/start.sh" ]
+    && apt-get install -y --no-install-recommends libxslt-dev libz-dev libxml2-dev gcc libemail-outlook-message-perl curl openssl jq bash \
+    && pip install --no-cache-dir -U parsedmarc \
+    && apt-get purge --yes gcc && apt-get autoremove --yes && apt-get clean \
+    && rm -Rf /var/lib/{apt,dpkg}/ && rm -Rf /root/.cache/ && rm -rf /var/lib/apt/lists/* \
+    && chmod +x /start.sh 
+ENTRYPOINT ["./start.sh"]
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 42bad18..3bc3532 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,27 +1,14 @@
-version: '2.4'
+version: '3.8'
 
 services:
-  parsedmarc-init:
-    image: patschi/parsedmarc:init
-    restart: always
-    volumes:
-      - ./data/conf/parsedmarc/:/etc/parsedmarc/:rw
-      - ./data/conf/nginx/ssl/:/etc/nginx/ssl/:rw
-      - ./data/data/elasticsearch:/usr/share/elasticsearch/data/:rw
-    networks:
-      - parsedmarc-network
-    healthcheck:
-      test: [ "CMD", "test", "-f", "/ready" ]
-      interval: 10s
-      timeout: 5s
-      retries: 9999
-      start_period: 10s
-
   parsedmarc:
-    image: patschi/parsedmarc:latest
+    build:
+      context: data/Dockerfiles/parsedmarc
+      dockerfile: Dockerfile
     volumes:
       - ./data/conf/parsedmarc/:/etc/parsedmarc/
       - ./data/data/geoipupdate/:/usr/share/GeoIP:z,ro
+      - ./data/data/elasticsearch:/usr/share/elasticsearch/data/:rw
     restart: always
     networks:
       - parsedmarc-network
@@ -30,7 +17,7 @@ services:
         condition: service_healthy
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1-arm64
     environment:
       - cluster.name=parsedmarc
       - discovery.type=single-node
@@ -54,24 +41,18 @@ services:
       timeout: 10s
       retries: 3
       start_period: 30s
-    depends_on:
-      parsedmarc-init:
-        condition: service_started
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.16.2
+    image: docker.elastic.co/kibana/kibana:8.8.1-arm64
     environment:
       - elasticsearch.hosts=http://elasticsearch:9200
       - telemetry.enabled=false
       - telemetry.optIn=false
-    expose: # only expose docker-internally
-      - 5601
+    ports: # only expose docker-internally
+      - "8080:5601"
     restart: always
     networks:
       - parsedmarc-network
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
     healthcheck:
       test: [ "CMD", "curl","-s" ,"-f", "http://localhost:5601/" ]
       interval: 1m
@@ -90,21 +71,21 @@ services:
     volumes:
       - ./data/data/geoipupdate/:/usr/share/GeoIP:z,rw
 
-  nginx:
-    image: nginx:alpine
-    restart: always
-    ports:
-      - "9999:443"
-    volumes:
-      - ./data/conf/nginx/site.conf:/etc/nginx/conf.d/default.conf:ro
-      - ./data/conf/nginx/ssl/:/etc/nginx/ssl/:ro
-    networks:
-      - parsedmarc-network
-    depends_on:
-      kibana:
-        condition: service_healthy
-      parsedmarc-init:
-        condition: service_healthy
+  # nginx:
+  #   image: nginx:alpine
+  #   restart: always
+  #   ports:
+  #     - "9999:443"
+  #   volumes:
+  #     - ./data/conf/nginx/site.conf:/etc/nginx/conf.d/default.conf:ro
+  #     - ./data/conf/nginx/ssl/:/etc/nginx/ssl/:ro
+  #   networks:
+  #     - parsedmarc-network
+  #   depends_on:
+  #     kibana:
+  #       condition: service_healthy
+  #     parsedmarc-init:
+  #       condition: service_healthy
 
 networks:
   parsedmarc-network:

From e9be4ea4fdadb37fb9a432eb46c02f98a5228a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dawid=20Urba=C5=84ski?= <dawurb@gmail.com>
Date: Wed, 21 Jun 2023 10:52:22 +0200
Subject: [PATCH 3/3] IF added to begining of script to run only on first setup
 or after container deletion, fixed curl return code validation

---
 data/Dockerfiles/parsedmarc/init-script.sh | 149 +++++++++++----------
 1 file changed, 77 insertions(+), 72 deletions(-)

diff --git a/data/Dockerfiles/parsedmarc/init-script.sh b/data/Dockerfiles/parsedmarc/init-script.sh
index d611f87..b08636b 100644
--- a/data/Dockerfiles/parsedmarc/init-script.sh
+++ b/data/Dockerfiles/parsedmarc/init-script.sh
@@ -2,91 +2,96 @@
 # Copyright 2020, Patrik Kernstock.
 
 set -x
+if [ -f "ready" ]
+then
+	echo "It seems file 'ready' exist, so I have nothing to do here"
+else	
+	echo "## ELASTICSEARCH"
+	echo "Setting permissions..."
+	chmod g+rwx -R /usr/share/elasticsearch/data/
+	chgrp 0 -R /usr/share/elasticsearch/data/
 
-echo "## ELASTICSEARCH"
-echo "Setting permissions..."
-chmod g+rwx -R /usr/share/elasticsearch/data/
-chgrp 0 -R /usr/share/elasticsearch/data/
+	# echo "## NGINX"
+	# echo "Checking nginx certs..."
+	# cd /etc/nginx/ssl/
+	# if [ ! -f "/etc/nginx/ssl/kibana.crt" ] || [ ! -f "/etc/nginx/ssl/kibana.key" ]; then
+	# 	echo "No certs found. Generating..."
+	# 	openssl req -x509 -nodes -days 365 -newkey rsa:3072 -keyout kibana.key -out kibana.crt \
+	# 		-subj "/CN=parsedmarc" -addext "subjectAltName=DNS:parsedmarc"
+	# 	echo "Certs generated."
+	# fi
 
-# echo "## NGINX"
-# echo "Checking nginx certs..."
-# cd /etc/nginx/ssl/
-# if [ ! -f "/etc/nginx/ssl/kibana.crt" ] || [ ! -f "/etc/nginx/ssl/kibana.key" ]; then
-# 	echo "No certs found. Generating..."
-# 	openssl req -x509 -nodes -days 365 -newkey rsa:3072 -keyout kibana.key -out kibana.crt \
-# 		-subj "/CN=parsedmarc" -addext "subjectAltName=DNS:parsedmarc"
-# 	echo "Certs generated."
-# fi
-
-echo "## KIBANA"
-exportFile="/etc/parsedmarc/kibana_export.ndjson"
-if [ ! -f "${exportFile}" ]; then
-	# trigger empty file to trigger below update logic.
-	touch ${exportFile}
-fi
-echo "Downloading dashboard from GitHub..."
-rm /etc/parsedmarc/kibana_export.ndjson.tmp
-curl https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson \
-	-o /etc/parsedmarc/kibana_export.ndjson.tmp
-if [ ${?} -ne 0 ]; then
-	echo "Downloading kibana export failed."
-else
-	fileNew=$(wc -c "${exportFile}.tmp" | awk -F' ' '{ print $1 }')
-	fileOld=$(wc -c "${exportFile}" | awk -F' ' '{ print $1 }')
-
-	if [ "$fileNew" == "$fileOld" ]; then
-		echo "File size is the same. Not proceeding."
+	echo "## KIBANA"
+	exportFile="/etc/parsedmarc/kibana_export.ndjson"
+	if [ ! -f "${exportFile}" ]; then
+		# trigger empty file to trigger below update logic.
+		touch ${exportFile}
+	fi
+	echo "Downloading dashboard from GitHub..."
+	rm /etc/parsedmarc/kibana_export.ndjson.tmp
+	curl https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson \
+		-o /etc/parsedmarc/kibana_export.ndjson.tmp
+	if [ ${?} -ne 0 ]; then
+		echo "Downloading kibana export failed."
 	else
-		echo "File size is different... updating..."
-
-		while ! curl -s -f -I http://kibana:5601 >/dev/null; do
-			echo "Kibana not responding... waiting 5 secs..."
-			sleep 5
-		done
+		fileNew=$(wc -c "${exportFile}.tmp" | awk -F' ' '{ print $1 }')
+		fileOld=$(wc -c "${exportFile}" | awk -F' ' '{ print $1 }')
 
-		echo "Kibana responded. Waiting 10s, then proceeding with dashboard update..."
-		sleep 10
-		rm ${exportFile}
-		mv ${exportFile}.tmp ${exportFile}
-		RES=$(curl -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true \
-			-H "kbn-xsrf: true" --form file=@${exportFile})
-		echo "Result: $RES"
-		if [ ${?} -ne 0 ]; then
-			echo "[!!!] Import might have failed. Manual check recommended."
+		if [ "$fileNew" == "$fileOld" ]; then
+			echo "File size is the same. Not proceeding."
 		else
-			# if the flag exists, we already set the defaultRoute once. So we don't do that again.
-			if [ ! -f "/etc/parsedmarc/flag.defaultRouteSet" ]; then
-				DEF_DASHBOARD_NAME="DMARC Summary"
-				echo "Setting '${DEF_DASHBOARD_NAME}' dashboard as default route..."
-				DEF_DASHBOARD_ID=$(cat "${exportFile}" | jq --arg DBNAME "${DEF_DASHBOARD_NAME}" 'select(.attributes.title == $DBNAME) | .id' | tr -d '"')
-				if [ "$DEF_DASHBOARD_ID" != "" ]; then
-					echo "Found dashboard ID: ${DEF_DASHBOARD_ID}"
-					DEFAULT_ROUTE="/app/kibana#/dashboard/${DEF_DASHBOARD_ID}"
-					echo "DefaultRoute being set to: ${DEFAULT_ROUTE}"
-					curl -X POST -H "kbn-xsrf: true" -H "Content-Type: application/json" \
-						"http://kibana:5601/api/kibana/settings/defaultRoute" \
-						-d "{\"value\": \"${DEFAULT_ROUTE}\"}"
-					if [ ${?} -ne 0 ]; then
-						echo "[!!!] Setting defaultRoute seems to gone wrong. Manual check recommended."
+			echo "File size is different... updating..."
+
+			while ! curl -s -f -I http://kibana:5601 >/dev/null; do
+				echo "Kibana not responding... waiting 5 secs..."
+				sleep 5
+			done
+
+			echo "Kibana responded. Waiting 10s, then proceeding with dashboard update..."
+			sleep 10
+			rm ${exportFile}
+			mv ${exportFile}.tmp ${exportFile}
+			RES=$(curl -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true \
+				-H "kbn-xsrf: true" --form file=@${exportFile})
+			RET=$?
+			echo "Result: $RES"
+			if [ "$RET" -ne 0 ]; then
+				echo "[!!!] Import might have failed. Manual check recommended."
+			else
+				# if the flag exists, we already set the defaultRoute once. So we don't do that again.
+				if [ ! -f "/etc/parsedmarc/flag.defaultRouteSet" ]; then
+					DEF_DASHBOARD_NAME="DMARC Summary"
+					echo "Setting '${DEF_DASHBOARD_NAME}' dashboard as default route..."
+					DEF_DASHBOARD_ID=$(cat "${exportFile}" | jq --arg DBNAME "${DEF_DASHBOARD_NAME}" 'select(.attributes.title == $DBNAME) | .id' | tr -d '"')
+					if [ "$DEF_DASHBOARD_ID" != "" ]; then
+						echo "Found dashboard ID: ${DEF_DASHBOARD_ID}"
+						DEFAULT_ROUTE="/app/kibana#/dashboard/${DEF_DASHBOARD_ID}"
+						echo "DefaultRoute being set to: ${DEFAULT_ROUTE}"
+						curl -X POST -H "kbn-xsrf: true" -H "Content-Type: application/json" \
+							"http://kibana:5601/api/kibana/settings/defaultRoute" \
+							-d "{\"value\": \"${DEFAULT_ROUTE}\"}"
+						if [ ${?} -ne 0 ]; then
+							echo "[!!!] Setting defaultRoute seems to gone wrong. Manual check recommended."
+						else
+							echo "DefaultRoute set."
+							echo "Notice: This might require a restart of Kibana to take effect. Not done automatically as part of this script."
+							echo -e "# This is a flag to remember which defaultRoute we set in the past:\n${DEFAULT_ROUTE}" \
+								> /etc/parsedmarc/flag.defaultRouteSet
+						fi
 					else
-						echo "DefaultRoute set."
-						echo "Notice: This might require a restart of Kibana to take effect. Not done automatically as part of this script."
-						echo -e "# This is a flag to remember which defaultRoute we set in the past:\n${DEFAULT_ROUTE}" \
-							> /etc/parsedmarc/flag.defaultRouteSet
+						echo "[!] Default dashboard with name '${DEF_DASHBOARD_NAME}' could not be found."
 					fi
-				else
-					echo "[!] Default dashboard with name '${DEF_DASHBOARD_NAME}' could not be found."
 				fi
 			fi
+			echo "Dashboard import done."
 		fi
-		echo "Dashboard import done."
 	fi
-fi
 
-sleep 3
+	sleep 3
 
-# Create empty file to let other containers know that we're ready.
-touch /ready
+	# Create empty file to let other containers know that we're ready.
+	touch /ready
+fi
 #sleep infinity # or while true; do sleep 86400; done
 parsedmarc -c /etc/parsedmarc/config.ini
 exit 0