From 7ceeb504719d3b039aedb55b8c8066db17409418 Mon Sep 17 00:00:00 2001 From: Uladzislau Muraveika Date: Tue, 14 Jan 2025 10:20:13 +0300 Subject: [PATCH] feat: store localnet logs between runs Details: - change rm and clean commands to remove only subfolder for tmp_log folder- justification - tmp_log folder needed for the log aggregator, prevent removal on the go when log stack is running - fix promtail config to point it to the right place - add named volume to the docker-compose to store Loki indexed between localnet runs - echance debug-start-log - it checking that localnet is running, exits if not - add debug-delete-volume-log to make, it removes loki named volume, warns about sudo rm for the loki files --- .gitignore | 1 + Makefile | 18 +++++++++++++----- test/debug-external.sh | 2 +- test/debug.sh | 2 +- test/logs_aggregator/docker-compose.yml | 13 +++++++++++-- test/logs_aggregator/promtail-config.yml | 2 +- test/logs_aggregator/start_log_aggregator.sh | 17 ++++++++++++----- 7 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 5b3554c8d8..c86cdb7108 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,4 @@ cache_*_db # log aggregation logs folder variable file, no sense to track it test/logs_aggregator/.env +test/logs_aggregator/loki diff --git a/Makefile b/Makefile index e32a57b91a..b2f7312619 100644 --- a/Makefile +++ b/Makefile @@ -55,10 +55,13 @@ help: @echo "travis_rpc_checker - run the Travis RPC checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH" @echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH" @echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes" + @echo "debug-multi-bls - cleans up environment, rebuilds the binary, and deploys with external nodes in configuration 1 harmony process -> 2 validators" @echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network" - @echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, needs docker compose and started localnet" + @echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, creates"\ + "persistent volume to store parsed logs between localnet runs, needs docker compose and started localnet" @echo "debug-stop-log - stops a docker compose Promtail->Loki->Grafana stack" @echo "debug-restart-log - restart a docker compose Promtail->Loki->Grafana stack" + @echo "debug-delete-volume-log - removes persistent volume for the Loki and host folder for it" libs: make -C $(TOP)/mcl -j8 @@ -101,7 +104,7 @@ debug-multi-bls: echo sleep 10s before creating the external validator sleep 10 bash ./test/build-localnet-validator.sh - + debug-multi-bls-with-terminal: # add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging # add LEGACY_SYNC=true before bash or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync @@ -111,7 +114,7 @@ debug-multi-bls-with-terminal: bash ./test/build-localnet-validator.sh screen -r localnet -debug-multi-bls-multi-ext-node: pre-external +debug-multi-bls-multi-ext-node: # add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging # add LEGACY_SYNC=true before bash or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync ./test/debug.sh ./test/configs/local-multi-bls-multi-ext-node.txt & @@ -120,7 +123,7 @@ debug-multi-bls-multi-ext-node: pre-external bash ./test/build-localnet-validator.sh clean: - rm -rf ./tmp_log* + rm -rf ./tmp_log/* rm -rf ./.dht* rm -rf ./db-* rm -rf ./latest @@ -236,7 +239,7 @@ travis_rosetta_checker: TEST_REPO_BRANCH='master' bash ./scripts/travis_rosetta_checker.sh -debug_external: clean +debug_external: bash test/debug-external.sh build_localnet_validator: @@ -252,3 +255,8 @@ debug-stop-log: bash ./test/logs_aggregator/stop_log_aggregator.sh debug-restart-log: debug-stop-log debug-start-log + +debug-delete-volume-log: + docker volume rm logs_aggregator_loki_data + @echo "[WARN] - it needs sudo to remove folder created with loki docker image user" + sudo rm -rf test/logs_aggregator/loki diff --git a/test/debug-external.sh b/test/debug-external.sh index 4a75003dec..07879e8250 100755 --- a/test/debug-external.sh +++ b/test/debug-external.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash ./test/kill_node.sh -rm -rf tmp_log* 2> /dev/null +rm -rf tmp_log/* 2> /dev/null rm *.rlp 2> /dev/null rm -rf .dht* 2> /dev/null scripts/go_executable_build.sh -S || exit 1 # dynamic builds are faster for debug iteration... diff --git a/test/debug.sh b/test/debug.sh index 2792b3df5e..6b75c65959 100755 --- a/test/debug.sh +++ b/test/debug.sh @@ -6,7 +6,7 @@ Localnet_Blocks_Per_Epoch=$2 Localnet_Blocks_Per_Epoch_V2=$3 ./test/kill_node.sh -rm -rf tmp_log* 2> /dev/null +rm -rf tmp_log/* 2> /dev/null rm *.rlp 2> /dev/null rm -rf .dht* 2> /dev/null scripts/go_executable_build.sh -S || exit 1 # dynamic builds are faster for debug iteration... diff --git a/test/logs_aggregator/docker-compose.yml b/test/logs_aggregator/docker-compose.yml index 5ca06f64e4..122d02d05c 100644 --- a/test/logs_aggregator/docker-compose.yml +++ b/test/logs_aggregator/docker-compose.yml @@ -6,12 +6,13 @@ services: - 3100:3100 volumes: - ./loki-config.yml:/etc/loki/loki-config.yaml + - loki_data:/loki promtail: container_name: promtail image: grafana/promtail:latest volumes: - - ./promtail-config.yml:/etc/promtail/promtail-config.yaml - - ${CURRENT_SESSION_LOGS}:/var/log/ + - ./promtail-config.yml:/etc/promtail/config.yml + - ${LOG_FOLDER}:/var/log/ grafana: container_name: grafana image: grafana/grafana @@ -23,3 +24,11 @@ services: - 3000:3000 volumes: - ./loki-datasource.yaml:/etc/grafana/provisioning/datasources/loki-datasource.yaml + +volumes: + loki_data: + driver: local + driver_opts: + type: none + o: bind + device: ${LOKI_FOLDER} diff --git a/test/logs_aggregator/promtail-config.yml b/test/logs_aggregator/promtail-config.yml index 5a1547d0d5..f94e28bb6f 100644 --- a/test/logs_aggregator/promtail-config.yml +++ b/test/logs_aggregator/promtail-config.yml @@ -15,4 +15,4 @@ scrape_configs: - "localhost" labels: job: varlogs - __path__: /var/log/*.log + __path__: /var/log/log*/*.log diff --git a/test/logs_aggregator/start_log_aggregator.sh b/test/logs_aggregator/start_log_aggregator.sh index 8d0d405610..da90255e29 100644 --- a/test/logs_aggregator/start_log_aggregator.sh +++ b/test/logs_aggregator/start_log_aggregator.sh @@ -5,13 +5,20 @@ set -eou pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" echo "working in ${DIR}" cd $DIR && pwd -logs="$(ls -1 '../../tmp_log/')" path="$(readlink -f '../../tmp_log/')" -echo "Current localnet logs are placed into '${path}/${logs}'" -echo "CURRENT_SESSION_LOGS='${path}/${logs}'" > .env -echo "CURRENT_SESSION_LOGS='${path}/${logs}'" -echo "starting docker compose lor log aggregation" +test -d "${path}" || (echo "logs folder do not exist - ${path},"\ + "please create a localnet first, exiting" && exit 1) +log_path="$(find ${path} -type d | sort | tail -n 1)" +log_path=$(echo "${log_path}" | sed "s#.*/tmp_log/#/var/log/#") +loki_path="${DIR}/loki" +mkdir -p "${loki_path}" +echo "LOG_FOLDER='${path}'" > .env +echo "LOKI_FOLDER='${loki_path}'" >> .env +echo "starting docker compose for log aggregation" docker compose up --detach sleep 5 +echo "Whole list of log folders" +find ${path} -type d | sort | grep 'log-' echo "Opening Grafana" python3 -m webbrowser "http://localhost:3000/explore" +echo "Please, use {filename=~\"${log_path}/.*\"} to get the latest run localnet logs"