stabilize e2e tests (#3349)

# What this PR does
Stabilize e2e tests by:
- improve usage of locators
- fix unreliable selectors
- prevent parallelism within the same test file

Additionally:
- configure eslint for e2e tests and fix existing errors/warnings
- bump Playwright version to latest stable

## Which issue(s) this PR fixes
#3217

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

grafana-plugin/e2e-tests/.eslintrc

@@ -0,0 +1,6 @@
+{
+  "rules": {
+    "rulesdir/no-relative-import-paths": "off",
+    "no-console": "off"
+  }
+}

grafana-plugin/e2e-tests/alerts/directPaging.test.ts

@@ -13,8 +13,8 @@ test('we can directly page a user', async ({ adminRolePage }) => {
   const { page } = adminRolePage;
 
   await goToOnCallPage(page, 'alert-groups');
+  await page.waitForTimeout(1000);
   await clickButton({ page, buttonText: 'Escalation' });
-
   await fillInInput(page, 'textarea[name="message"]', message);
   await clickButton({ page, buttonText: 'Invite' });
 
@@ -23,8 +23,14 @@ test('we can directly page a user', async ({ adminRolePage }) => {
   await addRespondersPopup.getByText('Users').click();
   await addRespondersPopup.getByText(adminRolePage.userName).click();
 
-  await clickButton({ page, buttonText: 'Create' });
+  // If user is not on call, confirm invitation
+  await page.waitForTimeout(1000);
+  const isConfirmationModalShown = await page.getByText('Confirm Participant Invitation').isVisible();
+  if (isConfirmationModalShown) {
+    await page.getByTestId('confirm-non-oncall').click();
+  }
 
+  await clickButton({ page, buttonText: 'Create' });
   // Check we are redirected to the alert group page
   await page.waitForURL('**/alert-groups/I*'); // Alert group IDs always start with "I"
   await expect(page.getByTestId('incident-message')).toContainText(message);

grafana-plugin/e2e-tests/escalationChains/escalationPolicy.test.ts

@@ -1,6 +1,6 @@
 import {expect, test} from "../fixtures";
-import {generateRandomValue} from "../utils/forms";
 import {createEscalationChain, EscalationStep, selectEscalationStepValue} from "../utils/escalationChain";
+import {generateRandomValue} from "../utils/forms";
 
 test('escalation policy does not go back to "Default" after adding users to notify', async ({ adminRolePage }) => {
   const { page, userName } = adminRolePage;

grafana-plugin/e2e-tests/escalationChains/searching.test.ts

@@ -1,6 +1,6 @@
 import { test, expect, Page } from '../fixtures';
-import { generateRandomValue } from '../utils/forms';
 import { createEscalationChain } from '../utils/escalationChain';
+import { generateRandomValue } from '../utils/forms';
 
 const assertEscalationChainSearchWorks = async (
   page: Page,

grafana-plugin/e2e-tests/fixtures.ts

@@ -1,10 +1,14 @@
-import * as fs from 'fs';
-import * as path from 'path';
 import { test as base, Browser, Page, TestInfo } from '@playwright/test';
 
-import { GRAFANA_ADMIN_USERNAME, GRAFANA_EDITOR_USERNAME, GRAFANA_VIEWER_USERNAME } from './utils/constants';
 import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';
 
+import { GRAFANA_ADMIN_USERNAME, GRAFANA_EDITOR_USERNAME, GRAFANA_VIEWER_USERNAME } from './utils/constants';
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+
+
 export class BaseRolePage {
   page: Page;
   userName: string;

grafana-plugin/e2e-tests/globalSetup.ts

@@ -1,5 +1,8 @@
+import { OrgRole } from '@grafana/data';
 import { test as setup, chromium, expect, Page, BrowserContext, FullConfig, APIRequestContext } from '@playwright/test';
 
+import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';
+
 import GrafanaAPIClient from './utils/clients/grafana';
 import {
   GRAFANA_ADMIN_PASSWORD,
@@ -14,8 +17,6 @@ import {
 } from './utils/constants';
 import { clickButton, getInputByName } from './utils/forms';
 import { goToGrafanaPage } from './utils/navigation';
-import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';
-import { OrgRole } from '@grafana/data';
 
 const grafanaApiClient = new GrafanaAPIClient(GRAFANA_ADMIN_USERNAME, GRAFANA_ADMIN_PASSWORD);
 

grafana-plugin/e2e-tests/integrations/heartbeat.test.ts

@@ -1,16 +1,12 @@
 import { test, Page, expect } from '../fixtures';
-
 import { generateRandomValue, selectDropdownValue } from '../utils/forms';
 import { createIntegration } from '../utils/integrations';
 
 const HEARTBEAT_SETTINGS_FORM_TEST_ID = 'heartbeat-settings-form';
 
 test.describe("updating an integration's heartbeat interval works", async () => {
   const _openHeartbeatSettingsForm = async (page: Page) => {
-    const integrationSettingsPopupElement = page.getByTestId('integration-settings-context-menu');
-    await integrationSettingsPopupElement.waitFor({ state: 'visible' });
-    await integrationSettingsPopupElement.click();
-
+    await page.getByTestId('integration-settings-context-menu-wrapper').getByRole('img').click();
     await page.getByTestId('integration-heartbeat-settings').click();
   };
 
@@ -60,6 +56,6 @@ test.describe("updating an integration's heartbeat interval works", async () =>
      */
     await page.request.get(endpoint);
     await page.reload({ waitUntil: 'networkidle' });
-    await page.getByTestId('heartbeat-badge').waitFor({ state: 'visible' });
+    await page.getByTestId('heartbeat-badge').waitFor();
   });
 });

grafana-plugin/e2e-tests/integrations/integrationsTable.test.ts

@@ -1,47 +1,73 @@
-import { test, expect } from '../fixtures';
+import { test } from '../fixtures';
 import { generateRandomValue } from '../utils/forms';
-import { createIntegration } from '../utils/integrations';
+import { createIntegration, searchIntegrationAndAssertItsPresence } from '../utils/integrations';
 
 test('Integrations table shows data in Connections and Direct Paging tabs', async ({ adminRolePage: { page } }) => {
-  //   // Create 2 integrations that are not Direct Paging
   const ID = generateRandomValue();
   const WEBHOOK_INTEGRATION_NAME = `Webhook-${ID}`;
   const ALERTMANAGER_INTEGRATION_NAME = `Alertmanager-${ID}`;
-  const DIRECT_PAGING_INTEGRATION_NAME = `Direct paging`;
+  const DIRECT_PAGING_INTEGRATION_NAME = `Direct paging integration name`;
 
+  // Create 2 integrations that are not Direct Paging
   await createIntegration({ page, integrationSearchText: 'Webhook', integrationName: WEBHOOK_INTEGRATION_NAME });
+  await page.waitForTimeout(1000);
   await page.getByRole('tab', { name: 'Tab Integrations' }).click();
-
   await createIntegration({
     page,
     integrationSearchText: 'Alertmanager',
     shouldGoToIntegrationsPage: false,
     integrationName: ALERTMANAGER_INTEGRATION_NAME,
   });
+  await page.waitForTimeout(1000);
   await page.getByRole('tab', { name: 'Tab Integrations' }).click();
 
   // Create 1 Direct Paging integration if it doesn't exist
-  const integrationsTable = page.getByTestId('integrations-table');
   await page.getByRole('tab', { name: 'Tab Direct Paging' }).click();
-  const isDirectPagingAlreadyCreated = await page.getByText('Direct paging').isVisible();
+  const integrationsTable = page.getByTestId('integrations-table');
+  await page.waitForTimeout(2000);
+  const isDirectPagingAlreadyCreated = (await integrationsTable.getByText('Direct paging').count()) >= 1;
   if (!isDirectPagingAlreadyCreated) {
     await createIntegration({
       page,
       integrationSearchText: 'Direct paging',
       shouldGoToIntegrationsPage: false,
       integrationName: DIRECT_PAGING_INTEGRATION_NAME,
     });
+    await page.waitForTimeout(1000);
   }
   await page.getByRole('tab', { name: 'Tab Integrations' }).click();
 
   // By default Connections tab is opened and newly created integrations are visible except Direct Paging one
-  await expect(integrationsTable.getByText(WEBHOOK_INTEGRATION_NAME)).toBeVisible();
-  await expect(integrationsTable.getByText(ALERTMANAGER_INTEGRATION_NAME)).toBeVisible();
-  await expect(integrationsTable).not.toContainText(DIRECT_PAGING_INTEGRATION_NAME);
+  await searchIntegrationAndAssertItsPresence({ page, integrationsTable, integrationName: WEBHOOK_INTEGRATION_NAME });
+  await searchIntegrationAndAssertItsPresence({
+    page,
+    integrationsTable,
+    integrationName: ALERTMANAGER_INTEGRATION_NAME,
+  });
+  await searchIntegrationAndAssertItsPresence({
+    page,
+    integrationsTable,
+    integrationName: DIRECT_PAGING_INTEGRATION_NAME,
+    visibleExpected: false,
+  });
 
   // Then after switching to Direct Paging tab only Direct Paging integration is visible
   await page.getByRole('tab', { name: 'Tab Direct Paging' }).click();
-  await expect(integrationsTable.getByText(WEBHOOK_INTEGRATION_NAME)).not.toBeVisible();
-  await expect(integrationsTable.getByText(ALERTMANAGER_INTEGRATION_NAME)).not.toBeVisible();
-  await expect(integrationsTable).toContainText(DIRECT_PAGING_INTEGRATION_NAME);
+  await searchIntegrationAndAssertItsPresence({
+    page,
+    integrationsTable,
+    integrationName: WEBHOOK_INTEGRATION_NAME,
+    visibleExpected: false,
+  });
+  await searchIntegrationAndAssertItsPresence({
+    page,
+    integrationsTable,
+    integrationName: ALERTMANAGER_INTEGRATION_NAME,
+    visibleExpected: false,
+  });
+  await searchIntegrationAndAssertItsPresence({
+    page,
+    integrationsTable,
+    integrationName: 'Direct paging',
+  });
 });

grafana-plugin/e2e-tests/integrations/maintenanceMode.test.ts

@@ -13,36 +13,34 @@ import { goToOnCallPage } from '../utils/navigation';
 type MaintenanceModeType = 'Debug' | 'Maintenance';
 
 test.describe('maintenance mode works', () => {
-  test.slow(); // this test is doing a good amount of work, give it time
-
   const MAINTENANCE_DURATION = '1 hour';
   const REMAINING_TIME_TEXT = '59m left';
   const REMAINING_TIME_TOOLTIP_TEST_ID = 'maintenance-mode-remaining-time-tooltip';
 
   const createRoutedText = (escalationChainName: string): string =>
     `alert group assigned to route "default" with escalation chain "${escalationChainName}"`;
 
-  const _openIntegrationSettingsPopup = async (page: Page): Promise<Locator> => {
-    const integrationSettingsPopupElement = page.getByTestId('integration-settings-context-menu');
-    await integrationSettingsPopupElement.waitFor({ state: 'visible' });
+  const _openIntegrationSettingsPopup = async (page: Page, shouldDoubleClickSettingsIcon = false): Promise<void> => {
+    await page.waitForTimeout(2000);
+    const integrationSettingsPopupElement = page
+      .getByTestId('integration-settings-context-menu-wrapper')
+      .getByRole('img');
     await integrationSettingsPopupElement.click();
-    return integrationSettingsPopupElement;
+    /**
+     * sometimes we need to click twice (e.g. adding the escalation chain route
+     * doesn't unfocus out of the select element after selecting an option)
+     */
+    if (shouldDoubleClickSettingsIcon) {
+      await integrationSettingsPopupElement.click();
+    }
   };
 
   const getRemainingTimeTooltip = (page: Page): Locator => page.getByTestId(REMAINING_TIME_TOOLTIP_TEST_ID);
 
   const enableMaintenanceMode = async (page: Page, mode: MaintenanceModeType): Promise<void> => {
-    const integrationSettingsPopupElement = await _openIntegrationSettingsPopup(page);
-    /**
-     * we need to click twice here, because adding the escalation chain route
-     * doesn't unfocus out of the select element after selecting an option
-     */
-    await integrationSettingsPopupElement.click();
-
+    await _openIntegrationSettingsPopup(page, true);
     // open the maintenance mode settings drawer + fill in the maintenance details
-    const startMaintenanceModeButton = page.getByTestId('integration-start-maintenance');
-    await startMaintenanceModeButton.waitFor({ state: 'visible' });
-    await startMaintenanceModeButton.click();
+    await page.getByTestId('integration-start-maintenance').click();
 
     // fill in the form
     const maintenanceModeDrawer = page.getByTestId('maintenance-mode-drawer');
@@ -77,12 +75,10 @@ test.describe('maintenance mode works', () => {
     await goToOnCallPage(page, 'integrations');
 
     await filterIntegrationsTableAndGoToDetailPage(page, integrationName);
-    await _openIntegrationSettingsPopup(page);
+    await _openIntegrationSettingsPopup(page, true);
 
     // click the stop maintenance button
-    const stopMaintenanceModeButton = page.getByTestId('integration-stop-maintenance');
-    await stopMaintenanceModeButton.waitFor({ state: 'visible' });
-    await stopMaintenanceModeButton.click();
+    await page.getByTestId('integration-stop-maintenance').click();
 
     // in the modal popup, confirm that we want to stop it
     await clickButton({
@@ -114,6 +110,8 @@ test.describe('maintenance mode works', () => {
   };
 
   test('debug mode', async ({ adminRolePage: { page, userName } }) => {
+    test.slow();
+
     const { escalationChainName, integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
       page,
       userName,
@@ -130,6 +128,7 @@ test.describe('maintenance mode works', () => {
   });
 
   test('"maintenance" mode', async ({ adminRolePage: { page, userName } }) => {
+    test.slow();
     const { integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
       page,
       userName,

grafana-plugin/e2e-tests/integrations/uniqueIntegrationNames.test.ts

@@ -1,8 +1,10 @@
 import { test, expect } from '../fixtures';
 import { openCreateIntegrationModal } from '../utils/integrations';
+import { goToOnCallPage } from '../utils/navigation';
 
 test('integrations have unique names', async ({ adminRolePage }) => {
   const { page } = adminRolePage;
+  await goToOnCallPage(page, 'integrations');
   await openCreateIntegrationModal(page);
 
   const integrationNames = await page.getByTestId('integration-display-name').allInnerTexts();

grafana-plugin/e2e-tests/schedules/addOverride.test.ts

@@ -1,7 +1,8 @@
+import dayjs from 'dayjs';
+
 import { test, expect } from '../fixtures';
 import { clickButton, generateRandomValue } from '../utils/forms';
 import { createOnCallSchedule, getOverrideFormDateInputs } from '../utils/schedule';
-import dayjs from 'dayjs';
 
 test('default dates in override creation modal are correct', async ({ adminRolePage }) => {
   const { page, userName } = adminRolePage;