Merge branch 'main' into Runbooks-queried-by-github-and-gitlb

.github/workflows/test-docs.yml

@@ -2,10 +2,12 @@ name: Test docs
 on:
   push:
     paths:
-      - 'keep/**'
+      - 'keep/poviders/**'
+      - 'docs/**'
   pull_request:
     paths:
-      - 'keep/**'
+      - 'keep/poviders/**'
+      - 'docs/**'
   workflow_dispatch:
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}-${{ github.job }}
@@ -15,7 +17,7 @@ env:
   STORAGE_MANAGER_DIRECTORY: /tmp/storage-manager
 
 jobs:
-  tests:
+  tests-docs:
     runs-on: ubuntu-latest
 
     steps:
@@ -28,13 +30,35 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - name: Install mintlify and validate docs
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: cache deps
+        id: cache-deps
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: pydeps-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies using poetry
+        run: poetry install --no-interaction --no-root --with dev
+
+      - name: Validate docs for providers
+        run: |
+          cd scripts;
+          poetry run python ./docs_get_providers_list.py --validate    
+
+      - name: Install deps and validate docs
         run: |
-          npm i -g mintlify
+          npm i -g mintlify;
+          
           cd docs && mintlify broken-links;
           cd ../scripts;
           ./docs_validate_navigation.sh;
-          python3 docs_get_providers_list.py --validate;
 
           # Todo: validate if openapi schema is matching with the code
   

.github/workflows/test-pr-e2e.yml

@@ -26,7 +26,7 @@ env:
   EE_ENABLED: true
 
 jobs:
-  tests:
+  tests-e2e:
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/test-pr.yml

@@ -64,26 +64,31 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - uses: chartboost/ruff-action@v1
         with:
           src: "./keep"
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
+
       - name: cache deps
         id: cache-deps
         uses: actions/cache@v2
         with:
           path: .venv
           key: pydeps-${{ hashFiles('**/poetry.lock') }}
+
       - name: Install dependencies using poetry
-        run: poetry install --no-interaction --no-root
+        run: poetry install --no-interaction --no-root --with dev
 
       - name: Run unit tests and report coverage
         run: |
@@ -97,6 +102,7 @@ jobs:
 
       - name: Convert coverage results to JSON (for CodeCov support)
         run: poetry run coverage json --omit="keep/providers/*"
+
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         with:

.gitignore

@@ -198,6 +198,8 @@ tempo-data/
 # docs
 docs/node_modules/
 
+oauth2.cfg
+
 
 scripts/automatic_extraction_rules.py
 
@@ -209,3 +211,4 @@ ee/experimental/ai_temp/*
 
 oauth2.cfg
 scripts/keep_slack_bot.py
+keepnew.db

docs/api-ref/topology/create-application.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /topology/applications
+---

docs/api-ref/topology/delete-application.mdx

@@ -0,0 +1,3 @@
+---
+openapi: delete /topology/applications/{application_id}
+---

docs/api-ref/topology/get-applications.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /topology/applications
+---

docs/api-ref/topology/update-application.mdx

@@ -0,0 +1,3 @@
+---
+openapi: put /topology/applications/{application_id}
+---

docs/deployment/authentication/oauth2proxy-auth.mdx

@@ -0,0 +1,32 @@
+---
+title: "OAuth2Proxy Authentication"
+---
+
+Delegate authentication to Oauth2Proxy.
+
+
+### When to Use
+
+- **oauth2-proxy user:** Use this authentication method if you want to delegate authentication to an external Oauth2Proxy service.
+
+### Setup Instructions
+
+To start Keep with Oauth2Proxy authentication, set the following environment variables:
+
+#### Frontend Environment Variables
+
+| Environment Variable | Description | Required | Default Value |
+|--------------------|-----------|:--------:|:-------------:|
+| AUTH_TYPE | Set to 'OAUTH2PROXY' for OAUTH2PROXY authentication | Yes | - |
+
+#### Backend Environment Variables
+
+| Environment Variable | Description | Required | Default Value |
+|--------------------|-----------|:--------:|:-------------:|
+| AUTH_TYPE | Set to 'OAUTH2PROXY' for OAUTH2PROXY authentication | Yes | - |
+| KEEP_OAUTH2_PROXY_USER_HEADER | Header for the authenticated user's email | Yes | x-forwarded-email |
+| KEEP_OAUTH2_PROXY_ROLE_HEADER | Header for the authenticated user's role | Yes | x-forwarded-groups |
+| KEEP_OAUTH2_PROXY_AUTO_CREATE_USER | Automatically create user if not exists | No | true |
+| KEEP_OAUTH2_PROXY_ADMIN_ROLE | Role name for admin users | No | admin |
+| KEEP_OAUTH2_PROXY_NOC_ROLE | Role name for NOC (Network Operations Center) users | No | noc |
+| KEEP_OAUTH2_PROXY_WEBHOOK_ROLE | Role name for webhook users | No | webhook |

docs/deployment/authentication/overview.mdx

@@ -20,12 +20,13 @@ Choosing the right authentication strategy depends on your specific use case, se
 
 ### Authentication Features Comparison
 
-| Identity Provider | RBAC | SAML/OIDC | SSO | LDAP | Resource-based permission | User Management | Group Management | On Prem | License |
-|:---:|:----:|:---------:|:---:|:----:|:-------------------------:|:----------------:|:-----------------:|:-------:|:-------:|
-| **No Auth** |  ❌   |     ❌     |  ❌  |  ❌   |             ❌            |        ❌        |         ❌         |    ✅    |   **OSS**   |
-| **DB** |  ✅ <br />(Predefiend roles)  |     ❌     |  ❌  |  ❌   |             ✅            |        ✅        |         ❌         |    ✅    |   **OSS**   |
-| **Auth0**             |  ✅ <br />(Predefiend roles)  |     ✅     |  ✅  |  🚧  |             🚧            |        ✅        |         🚧        |    ❌    |   **EE**   |
-| **Keycloak**          |  ✅ <br />(Custom roles)  |     ✅     |  ✅  |  ✅   |             ✅            |        ✅        |         ✅         |    ✅    |   **EE**    |
+| Identity Provider | RBAC | SAML/OIDC/SSO | LDAP | Resource-based permission | User Management | Group Management | On Prem | License |
+|:---:|:----:|:---------:|:----:|:-------------------------:|:----------------:|:-----------------:|:-------:|:-------:|
+| **No Auth** |  ❌   |     ❌     |  ❌   |             ❌            |        ❌        |         ❌         |    ✅    |   **OSS**   |
+| **DB** |  ✅ <br />(Predefiend roles)  |     ❌     |  ❌   |             ✅            |        ✅        |         ❌         |    ✅    |   **OSS**   |
+| **Auth0**             |  ✅ <br />(Predefiend roles)  |     ✅     |  🚧  |             🚧            |        ✅        |         🚧        |    ❌    |   **EE**   |
+| **Keycloak**          |  ✅ <br />(Custom roles)  |     ✅     |  ✅   |             ✅            |        ✅        |         ✅         |    ✅    |   **EE**    |
+| **Oauth2Proxy**       |  ✅ <br />(Predefiend roles)  |     ✅     |  ❌   |             ❌            |        N/A        |         N/A         |    ✅    |   **OSS**   |
 ### How To Configure
 <Tip>
 Some authentication providers require additional environment variables. These will be covered in detail on the specific authentication provider pages.
@@ -39,5 +40,6 @@ The authentication scheme on Keep is controlled with environment variables both
 | **DB** | `AUTH_TYPE=DB`  | `KEEP_JWT_SECRET` |
 | **Auth0**                           |  `AUTH_TYPE=AUTH0`  | `AUTH0_DOMAIN`, `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET` |
 | **Keycloak** | `AUTH_TYPE=KEYCLOAK` | `KEYCLOAK_URL`, `KEYCLOAK_REALM`, `KEYCLOAK_CLIENT_ID`, `KEYCLOAK_CLIENT_SECRET` |
+| **Oauth2Proxy** | `AUTH_TYPE=OAUTH2PROXY` | `OAUTH2_PROXY_USER_HEADER`, `OAUTH2_PROXY_ROLE_HEADER`, `OAUTH2_PROXY_AUTO_CREATE_USER` |
 
 For more details on each authentication strategy, including setup instructions and implications, refer to the respective sections.

docs/deployment/kubernetes.mdx

@@ -18,8 +18,8 @@ helm install keep keephq/keep
 
 Notice for it to work locally, you'll need this port forwarding:
 ```
+# expose the UI
 kubectl port-forward svc/keep-frontend 3000:3000
-kubectl port-forward svc/keep-backend 8080:8080
 ```
 
 To learn more about Keep's helm chart, see https://github.com/keephq/helm-charts/blob/main/README.md

docs/deployment/stress-testing.mdx

@@ -35,18 +35,18 @@ The primary parameters that affect the specification requirements for Keep are:
 
 ### Testing Scenarios:
 
-- **Low Volume (< 10,000 total alerts, 100's of alerts per day)**:
+- **Low Volume (< 10,000 total alerts, hundreds of alerts per day)**:
    - **Setup**: Use a standard relational database (e.g., MySQL, PostgreSQL) with default configurations.
    - **Expectations**: Keep should handle queries and alert ingestion with minimal resource usage.
 
-- **Medium Volume (10,000 - 100,000 total alerts, 1000's of alerts per day)**:
+- **Medium Volume (10,000 - 100,000 total alerts, thousands of alerts per day)**:
    - **Setup**: Scale the database to larger instances or clusters. Adjust best practices to the DB (e.g. increasing innodb_buffer_pool_size)
    - **Expectations**: CPU and RAM usage should increase proportionally but remain within acceptable limits.
 
-3. **High Volume (100,000 - 1,000,000 total alerts, 5000's of alerts per day)**:
+3. **High Volume (100,000 - 1,000,000 total alerts, >five thousands of alerts per day)**:
    - **Setup**: Deploy Keep with Elasticsearch for storing alerts as documents.
    - **Expectations**: The system should maintain performance levels despite the large alert volume, with increased resource usage managed through scaling strategies.
-4. **Very High Volume (> 1,000,000 total alerts, 10k's of alerts per day)**:
+4. **Very High Volume (> 1,000,000 total alerts, tens of thousands of alerts per day)**:
     - **Setup**: Deploy Keep with Elasticsearch for storing alerts as documents.
     - **Setup #2**: Deploy Keep with Redis and with ARQ to use Redis as a queue.
 

docs/mint.json

@@ -63,7 +63,8 @@
             "deployment/authentication/no-auth",
             "deployment/authentication/db-auth",
             "deployment/authentication/auth0-auth",
-            "deployment/authentication/keycloak-auth"
+            "deployment/authentication/keycloak-auth",
+            "deployment/authentication/oauth2proxy-auth"
           ]
         },
         {
@@ -128,6 +129,7 @@
             "providers/documentation/incidentio-provider",
             "providers/documentation/incidentmanager-provider",
             "providers/documentation/jira-provider",
+            "providers/documentation/jira-on-prem-provider",
             "providers/documentation/kibana-provider",
             "providers/documentation/kubernetes-provider",
             "providers/documentation/linear_provider",
@@ -280,7 +282,13 @@
         },
         {
           "group": "topology",
-          "pages": ["api-ref/topology/get-topology-data"]
+          "pages": [
+            "api-ref/topology/get-topology-data",
+            "api-ref/topology/create-application",
+            "api-ref/topology/delete-application",
+            "api-ref/topology/get-applications",
+            "api-ref/topology/update-application"
+          ]
         },
         {
           "group": "alerts",

docs/providers/documentation/azuremonitoring-provider.mdx

@@ -1,16 +1,16 @@
 ---
-title: "Azure Monitoring"
-sidebarTitle: "Azure Monitoring Provider"
-description: "Azure Monitoring provider allows you to get alerts from Azure Monitoring via webhooks."
+title: "Azure Monitor"
+sidebarTitle: "Azure Monitor Provider"
+description: "Azure Monitorg provider allows you to get alerts from Azure Monitor via webhooks."
 ---
 
 ## Overview
 
-The Azure Monitoring Provider integrates Keep with Azure Monitoring, allowing you to receive alerts within Keep's platform. By setting up a webhook in Azure, you can ensure that critical alerts are sent to Keep, allowing for efficient monitoring and response.
+The Azure Monitor Provider integrates Keep with Azure Monitor, allowing you to receive alerts within Keep's platform. By setting up a webhook in Azure, you can ensure that critical alerts are sent to Keep, allowing for efficient monitoring and response.
 
-## Connecting Azure Monitoring to Keep
+## Connecting Azure Monitor to Keep
 
-Connecting Azure Monitoring to Keep involves creating an Action Group in Azure, adding a webhook action, and configuring the Alert Rule to use the new Action Group.
+Connecting Azure Monitor to Keep involves creating an Action Group in Azure, adding a webhook action, and configuring the Alert Rule to use the new Action Group.
 
 ### Step 1: Navigate an Action Group
 1. Log in to your Azure portal.

docs/providers/documentation/console-provider.mdx

@@ -5,24 +5,31 @@ description: "Console provider is sort of a mock provider that projects given al
 ---
 
 ## Inputs
-- alert_message: The alert message to print to the console
+
+- message: The alert message to print to the console
 
 ## Outputs
+
 This provider has no outputs
 
 ## Authentication Parameters
+
 This provider has no authentication
 
 ## Connecting with the Provider
+
 This provider doesn't require any connection
 
 ## Notes
-*No information yet, feel free to contribute it using the "Edit this page" link the buttom of the page*
+
+_No information yet, feel free to contribute it using the "Edit this page" link the buttom of the page_
 
 ## Useful Links
-*No information yet, feel free to contribute it using the "Edit this page" link the buttom of the page*
+
+_No information yet, feel free to contribute it using the "Edit this page" link the buttom of the page_
 
 ## Example
+
 ```python
 config = {
         "description": "Console Output Provider",
@@ -32,7 +39,7 @@ provider = ProvidersFactory.get_provider(
     provider_id='mock', provider_type="console", provider_config=config
 )
 provider.notify(
-    alert_message="Simple alert showing context with name: {name}".format(
+    message="Simple alert showing context with name: {name}".format(
         name="John Doe"
     )
 )

docs/providers/documentation/grafana_oncall-provider.mdx

@@ -1,6 +1,6 @@
 ---
-title: "Grafana Oncall Provider"
-description: "Grafana Oncall Provider is a class that allows to ingest/digest data from Grafana On-Call."
+title: "Grafana OnCall Provider"
+description: "Grafana Oncall Provider is a class that allows to ingest/digest data from Grafana OnCall."
 ---
 
 ## Inputs
@@ -21,24 +21,24 @@ Grafana Oncall Provider does not currently support the `query` function.
 
 ## Authentication Parameters
 
-The Grafana Oncall Provider uses API token authentication. You need to provide the following authentication parameters to connect to Grafana On-Call:
+The Grafana Oncall Provider uses API token authentication. You need to provide the following authentication parameters to connect to Grafana OnCall:
 
-- **token** (required): Your Grafana On-Call API Token.
-- **host** (required): The URL of your Grafana On-Call host (e.g., https://keephq.grafana.net).
+- **token** (required): Your Grafana OnCall API Token.
+- **host** (required): The URL of your Grafana OnCall host (e.g., https://keephq.grafana.net).
 
 ## Connecting with the Provider
 
-To connect to Grafana On-Call, you need to create an API Token:
+To connect to Grafana OnCall, you need to create an API Token:
 
-1. Log in to your Grafana On-Call account.
+1. Log in to your Grafana OnCall account.
 2. Go to the **API Tokens** page.
 3. Click the **Generate Token** button and provide a name for your token.
 4. Copy the token value and keep it secure.
 5. Add the token value to the `authentication` section in the Grafana Oncall Provider configuration.
 
 ## Notes
 
-- This provider allows you to interact with Grafana On-Call to create or update incidents.
+- This provider allows you to interact with Grafana OnCall to create or update incidents.
 - The `random_color` function generates a random color for incident labels.
 - The `startTime` and `endTime` parameters use ISO-8601 format.
 - The `notify` function returns information about the incident created or updated.
@@ -97,5 +97,5 @@ Payload example:
 
 ## Useful Links
 
-- [Grafana On-Call](https://keephq.grafana.net)
-- [Grafana On-Call API Documentation](https://keephq.grafana.net/docs/api)
+- [Grafana OnCall](https://keephq.grafana.net)
+- [Grafana OnCall API Documentation](https://keephq.grafana.net/docs/api)

docs/providers/documentation/jira-on-prem-provider.mdx

@@ -0,0 +1,7 @@
+---
+title: "Jira On-Prem Provider"
+sidebarTitle: "Jira On-Prem Provider"
+description: "Jira On-Prem Provider is a provider used to query data and creating issues in Jira"
+---
+
+Keep supports Jira OnPrem as a provider. Please check [Jira Provider](./jira-provider.md) for documentation.