Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARM SETUP #13

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions data/Dockerfiles/parsedmarc/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
FROM pypy:3-slim

ADD start.sh /start.sh
COPY init-script.sh /start.sh
RUN apt-get update \
&& apt-get install -y libxslt-dev libz-dev libxml2-dev gcc libemail-outlook-message-perl \
&& pip install -U parsedmarc \
&& apt-get purge --yes gcc && apt autoremove --yes && apt-get clean \
&& rm -Rf /var/lib/{apt,dpkg}/ && rm -Rf /root/.cache/ \
&& chmod +x /start.sh

ENTRYPOINT [ "/start.sh" ]
&& apt-get install -y --no-install-recommends libxslt-dev libz-dev libxml2-dev gcc libemail-outlook-message-perl curl openssl jq bash \
&& pip install --no-cache-dir -U parsedmarc \
&& apt-get purge --yes gcc && apt-get autoremove --yes && apt-get clean \
&& rm -Rf /var/lib/{apt,dpkg}/ && rm -Rf /root/.cache/ && rm -rf /var/lib/apt/lists/* \
&& chmod +x /start.sh
ENTRYPOINT ["./start.sh"]
97 changes: 97 additions & 0 deletions data/Dockerfiles/parsedmarc/init-script.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/bin/bash
# Copyright 2020, Patrik Kernstock.

set -x
if [ -f "ready" ]
then
echo "It seems file 'ready' exist, so I have nothing to do here"
else
echo "## ELASTICSEARCH"
echo "Setting permissions..."
chmod g+rwx -R /usr/share/elasticsearch/data/
chgrp 0 -R /usr/share/elasticsearch/data/

# echo "## NGINX"
# echo "Checking nginx certs..."
# cd /etc/nginx/ssl/
# if [ ! -f "/etc/nginx/ssl/kibana.crt" ] || [ ! -f "/etc/nginx/ssl/kibana.key" ]; then
# echo "No certs found. Generating..."
# openssl req -x509 -nodes -days 365 -newkey rsa:3072 -keyout kibana.key -out kibana.crt \
# -subj "/CN=parsedmarc" -addext "subjectAltName=DNS:parsedmarc"
# echo "Certs generated."
# fi

echo "## KIBANA"
exportFile="/etc/parsedmarc/kibana_export.ndjson"
if [ ! -f "${exportFile}" ]; then
# trigger empty file to trigger below update logic.
touch ${exportFile}
fi
echo "Downloading dashboard from GitHub..."
rm /etc/parsedmarc/kibana_export.ndjson.tmp
curl https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson \
-o /etc/parsedmarc/kibana_export.ndjson.tmp
if [ ${?} -ne 0 ]; then
echo "Downloading kibana export failed."
else
fileNew=$(wc -c "${exportFile}.tmp" | awk -F' ' '{ print $1 }')
fileOld=$(wc -c "${exportFile}" | awk -F' ' '{ print $1 }')

if [ "$fileNew" == "$fileOld" ]; then
echo "File size is the same. Not proceeding."
else
echo "File size is different... updating..."

while ! curl -s -f -I http://kibana:5601 >/dev/null; do
echo "Kibana not responding... waiting 5 secs..."
sleep 5
done

echo "Kibana responded. Waiting 10s, then proceeding with dashboard update..."
sleep 10
rm ${exportFile}
mv ${exportFile}.tmp ${exportFile}
RES=$(curl -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true \
-H "kbn-xsrf: true" --form file=@${exportFile})
RET=$?
echo "Result: $RES"
if [ "$RET" -ne 0 ]; then
echo "[!!!] Import might have failed. Manual check recommended."
else
# if the flag exists, we already set the defaultRoute once. So we don't do that again.
if [ ! -f "/etc/parsedmarc/flag.defaultRouteSet" ]; then
DEF_DASHBOARD_NAME="DMARC Summary"
echo "Setting '${DEF_DASHBOARD_NAME}' dashboard as default route..."
DEF_DASHBOARD_ID=$(cat "${exportFile}" | jq --arg DBNAME "${DEF_DASHBOARD_NAME}" 'select(.attributes.title == $DBNAME) | .id' | tr -d '"')
if [ "$DEF_DASHBOARD_ID" != "" ]; then
echo "Found dashboard ID: ${DEF_DASHBOARD_ID}"
DEFAULT_ROUTE="/app/kibana#/dashboard/${DEF_DASHBOARD_ID}"
echo "DefaultRoute being set to: ${DEFAULT_ROUTE}"
curl -X POST -H "kbn-xsrf: true" -H "Content-Type: application/json" \
"http://kibana:5601/api/kibana/settings/defaultRoute" \
-d "{\"value\": \"${DEFAULT_ROUTE}\"}"
if [ ${?} -ne 0 ]; then
echo "[!!!] Setting defaultRoute seems to gone wrong. Manual check recommended."
else
echo "DefaultRoute set."
echo "Notice: This might require a restart of Kibana to take effect. Not done automatically as part of this script."
echo -e "# This is a flag to remember which defaultRoute we set in the past:\n${DEFAULT_ROUTE}" \
> /etc/parsedmarc/flag.defaultRouteSet
fi
else
echo "[!] Default dashboard with name '${DEF_DASHBOARD_NAME}' could not be found."
fi
fi
fi
echo "Dashboard import done."
fi
fi

sleep 3

# Create empty file to let other containers know that we're ready.
touch /ready
fi
#sleep infinity # or while true; do sleep 86400; done
parsedmarc -c /etc/parsedmarc/config.ini
exit 0
67 changes: 24 additions & 43 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,27 +1,14 @@
version: '2.4'
version: '3.8'

services:
parsedmarc-init:
image: patschi/parsedmarc:init
restart: always
volumes:
- ./data/conf/parsedmarc/:/etc/parsedmarc/:rw
- ./data/conf/nginx/ssl/:/etc/nginx/ssl/:rw
- ./data/data/elasticsearch:/usr/share/elasticsearch/data/:rw
networks:
- parsedmarc-network
healthcheck:
test: [ "CMD", "test", "-f", "/ready" ]
interval: 10s
timeout: 5s
retries: 9999
start_period: 10s

parsedmarc:
image: patschi/parsedmarc:latest
build:
context: data/Dockerfiles/parsedmarc
dockerfile: Dockerfile
volumes:
- ./data/conf/parsedmarc/:/etc/parsedmarc/
- ./data/data/geoipupdate/:/usr/share/GeoIP:z,ro
- ./data/data/elasticsearch:/usr/share/elasticsearch/data/:rw
restart: always
networks:
- parsedmarc-network
Expand All @@ -30,7 +17,7 @@ services:
condition: service_healthy

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1-arm64
environment:
- cluster.name=parsedmarc
- discovery.type=single-node
Expand All @@ -54,24 +41,18 @@ services:
timeout: 10s
retries: 3
start_period: 30s
depends_on:
parsedmarc-init:
condition: service_started

kibana:
image: docker.elastic.co/kibana/kibana:7.16.2
image: docker.elastic.co/kibana/kibana:8.8.1-arm64
environment:
- elasticsearch.hosts=http://elasticsearch:9200
- telemetry.enabled=false
- telemetry.optIn=false
expose: # only expose docker-internally
- 5601
ports: # only expose docker-internally
- "8080:5601"
restart: always
networks:
- parsedmarc-network
depends_on:
elasticsearch:
condition: service_healthy
healthcheck:
test: [ "CMD", "curl","-s" ,"-f", "http://localhost:5601/" ]
interval: 1m
Expand All @@ -90,21 +71,21 @@ services:
volumes:
- ./data/data/geoipupdate/:/usr/share/GeoIP:z,rw

nginx:
image: nginx:alpine
restart: always
ports:
- "9999:443"
volumes:
- ./data/conf/nginx/site.conf:/etc/nginx/conf.d/default.conf:ro
- ./data/conf/nginx/ssl/:/etc/nginx/ssl/:ro
networks:
- parsedmarc-network
depends_on:
kibana:
condition: service_healthy
parsedmarc-init:
condition: service_healthy
# nginx:
# image: nginx:alpine
# restart: always
# ports:
# - "9999:443"
# volumes:
# - ./data/conf/nginx/site.conf:/etc/nginx/conf.d/default.conf:ro
# - ./data/conf/nginx/ssl/:/etc/nginx/ssl/:ro
# networks:
# - parsedmarc-network
# depends_on:
# kibana:
# condition: service_healthy
# parsedmarc-init:
# condition: service_healthy

networks:
parsedmarc-network:
Expand Down