From b0cac9ff759556ae909bb2c5c48456a759dc3d3d Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Fri, 26 May 2023 13:42:09 +0300 Subject: [PATCH 01/10] fix: remove concurrency from slo.yml --- .github/workflows/slo.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index b5ceb504..8cd4a56a 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -41,8 +41,6 @@ on: SLO_S3_ENDPOINT: required: true -concurrency: slo - jobs: obtain-mutex: runs-on: ubuntu-latest From 59bc92fd4192e3386920212a5446b3151ff90490 Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Tue, 30 May 2023 15:05:40 +0300 Subject: [PATCH 02/10] feat: add annotations on second dashboard --- .github/actions/grafana-annotate/action.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/actions/grafana-annotate/action.yml b/.github/actions/grafana-annotate/action.yml index 505cdb0d..0e65dcb3 100644 --- a/.github/actions/grafana-annotate/action.yml +++ b/.github/actions/grafana-annotate/action.yml @@ -8,6 +8,10 @@ inputs: description: "Dashboard UID" required: false default: "7CzMl5t4k" + dashboard1: + description: "Dashboard1 UID" + required: false + default: "SaEJlEwVk" text: description: "Text to set" required: true @@ -19,6 +23,7 @@ inputs: description: "Grafana annotation URL" required: false default: http://localhost:3000/api/annotations +description: "Github Action for annotating in grafana" runs: using: "composite" steps: @@ -26,17 +31,6 @@ runs: shell: bash run: echo TAGS='${{ inputs.tags }}' >> $GITHUB_ENV - - name: Echo command - shell: bash - run: > - echo kubectl exec -it - `kubectl get pods -l "app.kubernetes.io/name=grafana" -o jsonpath="{.items[0].metadata.name}"` - --container grafana -- /bin/bash -c " - echo '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}'; - wget -qS -O- --header 'Accept: application/json' --header 'Content-Type: application/json' - --post-data '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}' - '${{ inputs.annotations-url }}' " - - name: Push annotation shell: bash run: > @@ -46,4 +40,7 @@ runs: echo '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}'; wget -qS -O- --header 'Accept: application/json' --header 'Content-Type: application/json' --post-data '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}' - '${{ inputs.annotations-url }}' " + '${{ inputs.annotations-url }}'; + wget -qS -O- --header 'Accept: application/json' --header 'Content-Type: application/json' + --post-data '{\"dashboardUID\":\"${{ inputs.dashboard1 }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}' + '${{ inputs.annotations-url }}'; " From c2412308c5cb983d8bfbe65d4fd768b1a22e353f Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Thu, 1 Jun 2023 13:23:37 +0300 Subject: [PATCH 03/10] feat: add attempts on grafana dashboard --- .github/actions/grafana-annotate/action.yml | 3 - k8s/helms/grafana.yaml | 85 +++++++-------------- 2 files changed, 26 insertions(+), 62 deletions(-) diff --git a/.github/actions/grafana-annotate/action.yml b/.github/actions/grafana-annotate/action.yml index 0e65dcb3..4934c3d1 100644 --- a/.github/actions/grafana-annotate/action.yml +++ b/.github/actions/grafana-annotate/action.yml @@ -40,7 +40,4 @@ runs: echo '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}'; wget -qS -O- --header 'Accept: application/json' --header 'Content-Type: application/json' --post-data '{\"dashboardUID\":\"${{ inputs.dashboard }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}' - '${{ inputs.annotations-url }}'; - wget -qS -O- --header 'Accept: application/json' --header 'Content-Type: application/json' - --post-data '{\"dashboardUID\":\"${{ inputs.dashboard1 }}\", \"text\":\"${{ inputs.text }}\", \"tags\":${TAGS}}' '${{ inputs.annotations-url }}'; " diff --git a/k8s/helms/grafana.yaml b/k8s/helms/grafana.yaml index 749918f6..b14cd59b 100644 --- a/k8s/helms/grafana.yaml +++ b/k8s/helms/grafana.yaml @@ -96,17 +96,6 @@ dashboards: "links": [], "liveNow": false, "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "title": "Row title", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -168,14 +157,14 @@ dashboards: "h": 9, "w": 12, "x": 0, - "y": 1 + "y": 0 }, - "id": 2, + "id": 12, "options": { "legend": { "calcs": [], - "displayMode": "table", - "placement": "right", + "displayMode": "list", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -190,7 +179,7 @@ dashboards: "uid": "prometheus" }, "editorMode": "builder", - "expr": "inflight > 0", + "expr": "rate(inflight[$__rate_interval]) > 0", "format": "time_series", "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} In flight", "range": true, @@ -202,7 +191,7 @@ dashboards: "uid": "prometheus" }, "editorMode": "builder", - "expr": "oks > 0", + "expr": "rate(oks[$__rate_interval]) > 0", "hide": false, "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} OK", "range": true, @@ -214,14 +203,14 @@ dashboards: "uid": "prometheus" }, "editorMode": "builder", - "expr": "not_oks > 0", + "expr": "rate(not_oks[$__rate_interval]) > 0", "hide": false, "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} not OK", "range": true, "refId": "C" } ], - "title": "SLO Requests", + "title": "SLO Requests RPS", "transformations": [], "type": "timeseries" }, @@ -240,6 +229,7 @@ dashboards: "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -286,14 +276,14 @@ dashboards: "h": 9, "w": 12, "x": 12, - "y": 1 + "y": 0 }, - "id": 12, + "id": 14, "options": { "legend": { "calcs": [], - "displayMode": "table", - "placement": "right", + "displayMode": "list", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -308,38 +298,14 @@ dashboards: "uid": "prometheus" }, "editorMode": "builder", - "expr": "delta(inflight[2s]) / 2 > 0", - "format": "time_series", - "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} In flight", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "builder", - "expr": "delta(oks[2s]) / 2 > 0", + "expr": "histogram_quantile(1, rate(attempts_bucket[$__rate_interval]))", "hide": false, - "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} OK", + "legendFormat": "{{sdk}}-{{sdkVersion}} {{jobName}}-{{status}}", "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "builder", - "expr": "changes(not_oks[2s]) > 0", - "hide": false, - "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} not OK", - "range": true, - "refId": "C" + "refId": "A" } ], - "title": "SLO Requests RPS", + "title": "Attempts", "transformations": [], "type": "timeseries" }, @@ -349,7 +315,7 @@ dashboards: "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 9 }, "id": 7, "panels": [], @@ -417,7 +383,7 @@ dashboards: "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 10 }, "id": 4, "options": { @@ -510,7 +476,7 @@ dashboards: "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 10 }, "id": 5, "options": { @@ -602,7 +568,7 @@ dashboards: "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 18 }, "id": 10, "options": { @@ -694,7 +660,7 @@ dashboards: "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 18 }, "id": 11, "options": { @@ -727,15 +693,16 @@ dashboards: } ], "refresh": false, - "schemaVersion": 37, + "revision": 1, + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { - "from": "2023-03-30T12:36:47.587Z", - "to": "2023-03-30T12:43:11.246Z" + "from": "now-15m", + "to": "now" }, "timepicker": {}, "timezone": "", From 6cafbb096c0fe93ee365857393826a375a64b0b0 Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Fri, 2 Jun 2023 15:23:15 +0300 Subject: [PATCH 04/10] feat: add variable on grafana dashboard to filter results by url parameter --- k8s/helms/grafana.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/k8s/helms/grafana.yaml b/k8s/helms/grafana.yaml index b14cd59b..9fd15cb4 100644 --- a/k8s/helms/grafana.yaml +++ b/k8s/helms/grafana.yaml @@ -698,7 +698,20 @@ dashboards: "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "filters": [ ], + "hide": 2, + "label": "", + "name": "filter", + "skipUrlSync": false, + "type": "adhoc" + } + ] }, "time": { "from": "now-15m", From a766501dc0f1e7828c9ba70e95209d105e4904f3 Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Fri, 2 Jun 2023 18:42:54 +0300 Subject: [PATCH 05/10] fix: remove decimals from attempts graph --- k8s/helms/grafana.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/helms/grafana.yaml b/k8s/helms/grafana.yaml index 9fd15cb4..fc62fcd1 100644 --- a/k8s/helms/grafana.yaml +++ b/k8s/helms/grafana.yaml @@ -255,6 +255,7 @@ dashboards: "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", From 3a31e9070f82e27312cdac0eeae231bfc6f4160e Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Fri, 9 Jun 2023 01:29:07 +0300 Subject: [PATCH 06/10] feat: add playground to run slo workloads locally --- playground/.gitignore | 1 + playground/README.md | 40 ++ .../provisioning/dashboards/dashboard.yml | 6 + .../grafana/provisioning/dashboards/slo.json | 658 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 11 + playground/configs/prometheus/prometheus.yml | 8 + playground/docker-compose.yml | 63 ++ 7 files changed, 787 insertions(+) create mode 100644 playground/.gitignore create mode 100644 playground/README.md create mode 100644 playground/configs/grafana/provisioning/dashboards/dashboard.yml create mode 100644 playground/configs/grafana/provisioning/dashboards/slo.json create mode 100644 playground/configs/grafana/provisioning/datasources/datasource.yml create mode 100644 playground/configs/prometheus/prometheus.yml create mode 100644 playground/docker-compose.yml diff --git a/playground/.gitignore b/playground/.gitignore new file mode 100644 index 00000000..8fce6030 --- /dev/null +++ b/playground/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 00000000..eefb5cf0 --- /dev/null +++ b/playground/README.md @@ -0,0 +1,40 @@ +# SLO playground + +Playground may be used for testing SLO workloads locally + +It has several services: + +- `prometheus` - storage for metrics +- `prometheus-pushgateway` - push acceptor for prometheus +- `grafana` - provides chats for metrics +- `ydb` - local instance of ydb-database to run workload with + +## Network addresses + +- Grafana dashboard: http://localhost:3000 +- Prometheus pushgateway: http://localhost:9091 +- YDB monitoring: http://localhost:8765 +- YDB GRPC: grpc://localhost:2136 +- YDB GRPC TLS: grpcs://localhost:2135 + +## Start + +```shell +docker-compose up -d +``` + +## Stop + +```shell +docker-compose down +``` + +## Configs + +Grafana's dashboards stored in `configs/grafana/provisioning/dashboards` + +## Data + +YDB databases are not persistent + +All other data like metrics and certs stored in `data/` \ No newline at end of file diff --git a/playground/configs/grafana/provisioning/dashboards/dashboard.yml b/playground/configs/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..c6784142 --- /dev/null +++ b/playground/configs/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,6 @@ +apiVersion: 1 + +providers: + - name: 'SLO' + options: + path: /etc/grafana/provisioning/dashboards diff --git a/playground/configs/grafana/provisioning/dashboards/slo.json b/playground/configs/grafana/provisioning/dashboards/slo.json new file mode 100644 index 00000000..8b8c4e1f --- /dev/null +++ b/playground/configs/grafana/provisioning/dashboards/slo.json @@ -0,0 +1,658 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "rate(inflight[$__rate_interval]) > 0", + "format": "time_series", + "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} In flight", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "rate(oks[$__rate_interval]) > 0", + "hide": false, + "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} OK", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "rate(not_oks[$__rate_interval]) > 0", + "hide": false, + "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} not OK", + "range": true, + "refId": "C" + } + ], + "title": "SLO Requests RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "histogram_quantile(1, rate(attempts_bucket[$__rate_interval]))", + "hide": false, + "legendFormat": "{{sdk}}-{{sdkVersion}} {{jobName}}-{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Attempts", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 7, + "panels": [], + "title": "Latencies", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"read\", status=\"ok\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Read Latencies (OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"write\", status=\"ok\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Latencies (OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"read\", status=\"err\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Read Latencies (NOT OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"write\", status=\"err\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Latencies (NOT OK)", + "type": "timeseries" + } + ], + "refresh": false, + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "filters": [ ], + "hide": 2, + "label": "", + "name": "filter", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "SLO", + "uid": "7CzMl5t4k", + "version": 1, + "weekStart": "" +} diff --git a/playground/configs/grafana/provisioning/datasources/datasource.yml b/playground/configs/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..0b62b9c3 --- /dev/null +++ b/playground/configs/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true diff --git a/playground/configs/prometheus/prometheus.yml b/playground/configs/prometheus/prometheus.yml new file mode 100644 index 00000000..281b390b --- /dev/null +++ b/playground/configs/prometheus/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: 'slo' + static_configs: + - targets: ['prometheus-pushgateway:9091'] diff --git a/playground/docker-compose.yml b/playground/docker-compose.yml new file mode 100644 index 00000000..fa213250 --- /dev/null +++ b/playground/docker-compose.yml @@ -0,0 +1,63 @@ +version: '2.1' + +networks: + monitor-net: + driver: bridge + +services: + prometheus: + image: prom/prometheus:v2.44.0 + container_name: prometheus + volumes: + - ./configs/prometheus:/etc/prometheus + - ./data/prometheus:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + expose: + - 9090 + networks: + - monitor-net + + prometheus-pushgateway: + image: prom/pushgateway:v1.6.0 + container_name: prometheus-pushgateway + ports: + - "9091:9091" + networks: + - monitor-net + + grafana: + image: grafana/grafana:9.5.3 + container_name: grafana + volumes: + - ./configs/grafana/provisioning:/etc/grafana/provisioning + - ./data/grafana:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=passw0rD + restart: unless-stopped + ports: + - "3000:3000" + networks: + - monitor-net + + ydb: + image: cr.yandex/yc/yandex-docker-local-ydb:23.2 + container_name: ydb-local + hostname: localhost + environment: + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_DEFAULT_LOG_LEVEL=NOTICE + ports: + - "2135:2135" + - "2136:2136" + - "8765:8765" + volumes: + - ./data/ydb_certs:/ydb_certs From f41b6bc9ac86fee027f609ee8031652cdc83b5ad Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Tue, 13 Jun 2023 16:19:34 +0300 Subject: [PATCH 07/10] feat: remove inflight from slo dashboard --- k8s/helms/grafana.yaml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/k8s/helms/grafana.yaml b/k8s/helms/grafana.yaml index fc62fcd1..139edd96 100644 --- a/k8s/helms/grafana.yaml +++ b/k8s/helms/grafana.yaml @@ -173,18 +173,6 @@ dashboards: } }, "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "builder", - "expr": "rate(inflight[$__rate_interval]) > 0", - "format": "time_series", - "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} In flight", - "range": true, - "refId": "A" - }, { "datasource": { "type": "prometheus", @@ -693,7 +681,7 @@ dashboards: "type": "timeseries" } ], - "refresh": false, + "refresh": "", "revision": 1, "schemaVersion": 38, "style": "dark", @@ -705,8 +693,8 @@ dashboards: "type": "prometheus", "uid": "prometheus" }, - "filters": [ ], - "hide": 2, + "filters": [], + "hide": 0, "label": "", "name": "filter", "skipUrlSync": false, From 9c245d31bbaee7440ea3adc4d6531de702b6c221 Mon Sep 17 00:00:00 2001 From: Vadim Vladimirov Date: Wed, 21 Jun 2023 12:50:31 +0300 Subject: [PATCH 08/10] feat: update playground dashboard --- .../grafana/provisioning/dashboards/slo.json | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/playground/configs/grafana/provisioning/dashboards/slo.json b/playground/configs/grafana/provisioning/dashboards/slo.json index 8b8c4e1f..69d76bf7 100644 --- a/playground/configs/grafana/provisioning/dashboards/slo.json +++ b/playground/configs/grafana/provisioning/dashboards/slo.json @@ -104,18 +104,6 @@ } }, "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "builder", - "expr": "rate(inflight[$__rate_interval]) > 0", - "format": "time_series", - "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} In flight", - "range": true, - "refId": "A" - }, { "datasource": { "type": "prometheus", @@ -624,7 +612,7 @@ "type": "timeseries" } ], - "refresh": false, + "refresh": "", "revision": 1, "schemaVersion": 38, "style": "dark", @@ -636,8 +624,8 @@ "type": "prometheus", "uid": "prometheus" }, - "filters": [ ], - "hide": 2, + "filters": [], + "hide": 0, "label": "", "name": "filter", "skipUrlSync": false, From c91ef5952c7d4aab71418e9e2f47410b9f579a0a Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Fri, 19 Jan 2024 12:56:01 +0300 Subject: [PATCH 09/10] branch for SLO debug --- .github/actions/mutex/action.yml | 1 + .gitignore | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .gitignore diff --git a/.github/actions/mutex/action.yml b/.github/actions/mutex/action.yml index 7c4d3c82..34c1ce4e 100644 --- a/.github/actions/mutex/action.yml +++ b/.github/actions/mutex/action.yml @@ -26,6 +26,7 @@ runs: - name: Check if locked if: inputs.command == 'obtain' run: | + kubectl get configmaps slo-mutex -o=jsonpath={.data.busy} if test -n "`kubectl get configmaps slo-mutex -o=jsonpath={.data.busy}`"; then echo "Locked by `kubectl get configmaps slo-mutex -o=jsonpath={.data.lockedBy}`, wait"; fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..071d86cf --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +#npm +npm-debug.log* +node_modules + +#temporary +tmp/ +temp/ + +# typescript build (next line will be removed by pre-release-branch-update GitHub action to be included in dist branch) +build/ + +#personal +.idea/ +.vscode/ +.DS_Store + +# git merge leftovers +*.orig + +# secrets +secrets/* From cfda13dd731121532dd7fdf69b9b9f755de4524e Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Fri, 19 Jan 2024 12:57:42 +0300 Subject: [PATCH 10/10] Revert "branch for SLO debug" --- .github/actions/mutex/action.yml | 1 - .gitignore | 21 --------------------- 2 files changed, 22 deletions(-) delete mode 100644 .gitignore diff --git a/.github/actions/mutex/action.yml b/.github/actions/mutex/action.yml index 34c1ce4e..7c4d3c82 100644 --- a/.github/actions/mutex/action.yml +++ b/.github/actions/mutex/action.yml @@ -26,7 +26,6 @@ runs: - name: Check if locked if: inputs.command == 'obtain' run: | - kubectl get configmaps slo-mutex -o=jsonpath={.data.busy} if test -n "`kubectl get configmaps slo-mutex -o=jsonpath={.data.busy}`"; then echo "Locked by `kubectl get configmaps slo-mutex -o=jsonpath={.data.lockedBy}`, wait"; fi diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 071d86cf..00000000 --- a/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -#npm -npm-debug.log* -node_modules - -#temporary -tmp/ -temp/ - -# typescript build (next line will be removed by pre-release-branch-update GitHub action to be included in dist branch) -build/ - -#personal -.idea/ -.vscode/ -.DS_Store - -# git merge leftovers -*.orig - -# secrets -secrets/*