test: Add configurable test expectations. (#16496)

* Add a sample "expectations" config.
* feat: Add configurable expectation for dial in.
* Add JaaS unauthenticatedJoins expectation.
* test: Move grantModerator to moderation/, add expectation.
* test: Move kick test to moderation/, fix p2p enabled case.
* test: Add a test case for non-moderator kick.

tests/helpers/TestsConfig.ts

@@ -4,18 +4,7 @@
 export const config = {
     /** Enable debug logging. Note this includes private information from .env */
     debug: Boolean(process.env.JITSI_DEBUG?.trim()),
-    /** Whether to expect the environment to automatically elect a new moderator when the existing moderator leaves. */
-    autoModerator: (() => {
-        if (typeof process.env.AUTO_MODERATOR !== 'undefined') {
-            return process.env.AUTO_MODERATOR?.trim() === 'true';
-        }
-
-        // If not explicitly configured, fallback to recognizing whether we're running against one of the JaaS
-        // environments which are known to have the setting disabled.
-        return !Boolean(
-            process.env.JWT_PRIVATE_KEY_PATH && process.env.JWT_KID?.startsWith('vpaas-magic-cookie-')
-        );
-    })(),
+    expectationsFile: process.env.EXPECTATIONS?.trim(),
     jaas: {
         customerId: (() => {
             if (typeof process.env.JAAS_TENANT !== 'undefined') {

tests/helpers/expectations.ts

@@ -0,0 +1,50 @@
+import fs from 'fs';
+import { merge } from 'lodash-es';
+
+import { config } from './TestsConfig';
+
+const defaultExpectations = {
+    dialIn: {
+        /*
+         * The dial-in functionality is enabled.
+         * true -> assert the config is enabled, the UI elements are displayed, and the feature works.
+         * false -> assert the config is disabled and the UI elements are not displayed.
+         * null -> if the config is enabled, assert the UI elements are displayed and the feature works.
+         */
+        enabled: null,
+    },
+    jaas: {
+        /**
+         * Whether the jaas account is configured with the account-level setting to allow unauthenticated users to join.
+         */
+        unauthenticatedJoins: false
+    },
+    moderation: {
+        // Everyone is a moderator.
+        allModerators: false,
+        // When a moderator leaves, another one is elected.
+        autoModerator: true,
+        // The first to join is a moderator.
+        firstModerator: true,
+        // The grantOwner function is available.
+        grantModerator: true
+    }
+};
+
+let overrides: any = {};
+
+if (config.expectationsFile) {
+    try {
+        const str = fs.readFileSync(config.expectationsFile, 'utf8');
+
+        // Remove comments and multiline comments.
+        overrides = JSON.parse(str.replace(/\/\/.*|\/\*[\s\S]*?\*\//g, ''));
+    } catch (e) {
+        console.error('Error reading expectations file', e);
+    }
+    console.log('Loaded expectations from', config.expectationsFile);
+}
+
+export const expectations = merge(defaultExpectations, overrides);
+
+console.log('Expectations:', expectations);

tests/specs/2way/fakeDialInAudio.spec.ts

@@ -1,5 +1,6 @@
 import process from 'node:process';
 
+import { expectations } from '../../helpers/expectations';
 import { ensureOneParticipant, ensureTwoParticipants } from '../../helpers/participants';
 import { cleanup, isDialInEnabled, waitForAudioFromDialInParticipant } from '../helpers/DialIn';
 
@@ -16,8 +17,15 @@ describe('Fake Dial-In', () => {
 
         await ensureOneParticipant();
 
+        const configEnabled = await isDialInEnabled(ctx.p1);
+
+        if (expectations.dialIn.enabled !== null) {
+            expect(configEnabled).toBe(expectations.dialIn.enabled);
+        }
+
         // check dial-in is enabled, so skip
-        if (await isDialInEnabled(ctx.p1)) {
+        if (configEnabled) {
+            console.log('Dial in config is enabled, skipping fake dial in');
             ctx.skipSuiteTests = true;
         }
     });

tests/specs/2way/grantModerator.spec.ts

@@ -1,42 +0,0 @@
-import { ensureOneParticipant, ensureTwoParticipants } from '../../helpers/participants';
-
-describe('Grant moderator', () => {
-    it('joining the meeting', async () => {
-        await ensureOneParticipant();
-
-        if (await ctx.p1.execute(() => typeof APP.conference._room.grantOwner !== 'function')) {
-            ctx.skipSuiteTests = true;
-
-            return;
-        }
-
-        await ensureTwoParticipants();
-    });
-
-    it('grant moderator and validate', async () => {
-        const { p1, p2 } = ctx;
-
-        if (!await p1.isModerator()) {
-            ctx.skipSuiteTests = true;
-
-            return;
-        }
-
-        if (await p2.isModerator()) {
-            ctx.skipSuiteTests = true;
-
-            return;
-        }
-
-        await p1.getFilmstrip().grantModerator(p2);
-
-        await p2.driver.waitUntil(
-            () => p2.isModerator(),
-            {
-                timeout: 3000,
-                timeoutMsg: 'p2 did not become moderator'
-            }
-        );
-
-    });
-});

tests/specs/2way/kick.spec.ts

@@ -1,44 +0,0 @@
-import { ensureTwoParticipants } from '../../helpers/participants';
-
-describe('Kick', () => {
-    it('joining the meeting', async () => {
-        await ensureTwoParticipants();
-
-        if (!await ctx.p1.isModerator()) {
-            ctx.skipSuiteTests = true;
-        }
-    });
-
-    it('kick and check', () => kickParticipant2AndCheck());
-
-    it('kick p2p and check', async () => {
-        await ensureTwoParticipants({
-            configOverwrite: {
-                p2p: {
-                    enabled: true
-                }
-            }
-        });
-
-        await kickParticipant2AndCheck();
-    });
-});
-
-/**
- * Kicks the second participant and checks that the participant is removed from the conference and that dialog is open.
- */
-async function kickParticipant2AndCheck() {
-    const { p1, p2 } = ctx;
-
-    await p1.getFilmstrip().kickParticipant(await p2.getEndpointId());
-
-    await p1.waitForParticipants(0);
-
-    // check that the kicked participant sees the kick reason dialog
-    // let's wait for this to appear at least 2 seconds
-    await p2.driver.waitUntil(
-        async () => p2.isLeaveReasonDialogOpen(), {
-            timeout: 2000,
-            timeoutMsg: 'No leave reason dialog shown for p2'
-        });
-}

tests/specs/3way/audioVideoModeration.spec.ts

@@ -1,5 +1,5 @@
 import { Participant } from '../../helpers/Participant';
-import { config } from '../../helpers/TestsConfig';
+import { expectations } from '../../helpers/expectations';
 import {
     ensureOneParticipant,
     ensureThreeParticipants, ensureTwoParticipants,
@@ -79,7 +79,7 @@ describe('AVModeration', () => {
     it('hangup and change moderator', async () => {
         // The test below is only correct when the environment is configured to automatically elect a new moderator
         // when the moderator leaves. For environments where this is not the case, the test is skipped.
-        if (!config.autoModerator) {
+        if (!expectations.autoModerator) {
             return;
         }
 

tests/specs/3way/lobby.spec.ts

@@ -1,5 +1,5 @@
 import { P1, P3, Participant } from '../../helpers/Participant';
-import { config } from '../../helpers/TestsConfig';
+import { expectations } from '../../helpers/expectations';
 import {
     ensureOneParticipant,
     ensureThreeParticipants,
@@ -198,7 +198,7 @@ describe('Lobby', () => {
     it('change of moderators in lobby', async () => {
         // The test below is only correct when the environment is configured to automatically elect a new moderator
         // when the moderator leaves. For environments where this is not the case, the test is skipped.
-        if (!config.autoModerator) {
+        if (!expectations.autoModerator) {
             return;
         }
         await hangupAllParticipants();
@@ -291,7 +291,7 @@ describe('Lobby', () => {
     it('moderator leaves while lobby enabled', async () => {
         // The test below is only correct when the environment is configured to automatically elect a new moderator
         // when the moderator leaves. For environments where this is not the case, the test is skipped.
-        if (!config.autoModerator) {
+        if (!expectations.autoModerator) {
             return;
         }
         const { p1, p2, p3 } = ctx;

tests/specs/alone/dialInAudio.spec.ts

@@ -1,6 +1,7 @@
 import process from 'node:process';
 
 import { config as testsConfig } from '../../helpers/TestsConfig';
+import { expectations } from '../../helpers/expectations';
 import { ensureOneParticipant } from '../../helpers/participants';
 import { cleanup, dialIn, isDialInEnabled, waitForAudioFromDialInParticipant } from '../helpers/DialIn';
 
@@ -25,7 +26,14 @@ describe('Dial-In', () => {
 
         expect(await ctx.p1.isInMuc()).toBe(true);
 
-        if (!await isDialInEnabled(ctx.p1)) {
+        const configEnabled = await isDialInEnabled(ctx.p1);
+
+        if (expectations.dialIn.enabled !== null) {
+            expect(configEnabled).toBe(expectations.dialIn.enabled);
+        }
+
+        if (!configEnabled) {
+            console.log('Dial in config is disabled, skipping dial-in tests');
             ctx.skipSuiteTests = true;
         }
     });

tests/specs/alone/invite.spec.ts

@@ -1,11 +1,11 @@
 import { Participant } from '../../helpers/Participant';
 import { config as testsConfig } from '../../helpers/TestsConfig';
+import { expectations } from '../../helpers/expectations';
 import { ensureOneParticipant } from '../../helpers/participants';
 import { assertDialInDisplayed, assertUrlDisplayed, isDialInEnabled, verifyMoreNumbersPage } from '../helpers/DialIn';
 
 describe('Invite', () => {
     let p1: Participant;
-    let dialInEnabled: boolean;
 
     it('setup', async () => {
         // This is a temporary hack to avoid failing when running against a jaas env. The same cases are covered in
@@ -19,23 +19,30 @@ describe('Invite', () => {
         await ensureOneParticipant();
 
         p1 = ctx.p1;
-        dialInEnabled = await isDialInEnabled(p1);
+
     });
 
+    // The URL should always be displayed.
     it('url displayed', () => assertUrlDisplayed(p1));
 
-    it('dial-in displayed', async () => {
-        if (!dialInEnabled) {
-            return;
+    it('config values', async () => {
+        const dialInEnabled = await isDialInEnabled(p1);
+
+        if (expectations.dialIn.enabled !== null) {
+            expect(dialInEnabled).toBe(expectations.dialIn.enabled);
+        }
+    });
+
+    it('dial-in displayed', async () => {
+        if (expectations.dialIn.enabled !== null) {
+            await assertDialInDisplayed(p1, expectations.dialIn.enabled);
         }
-        await assertDialInDisplayed(p1);
     });
 
     it('view more numbers page', async () => {
-        if (!dialInEnabled) {
-            return;
+        if (expectations.dialIn.enabled === true) {
+            // TODO: assert the page is NOT shown when the expectation is false.
+            await verifyMoreNumbersPage(p1);
         }
-
-        await verifyMoreNumbersPage(p1);
     });
 });

tests/specs/helpers/DialIn.ts

@@ -95,14 +95,14 @@ export async function assertUrlDisplayed(p: Participant) {
     await inviteDialog.waitTillOpen(true);
 }
 
-export async function assertDialInDisplayed(p: Participant) {
+export async function assertDialInDisplayed(p: Participant, displayed: boolean = false) {
     const inviteDialog = p.getInviteDialog();
 
     await inviteDialog.open();
     await inviteDialog.waitTillOpen();
 
-    expect((await inviteDialog.getDialInNumber()).length > 0).toBe(true);
-    expect((await inviteDialog.getPinNumber()).length > 0).toBe(true);
+    expect((await inviteDialog.getDialInNumber()).length > 0).toBe(displayed);
+    expect((await inviteDialog.getPinNumber()).length > 0).toBe(displayed);
 }
 
 export async function verifyMoreNumbersPage(p: Participant) {

tests/specs/jaas/dial/dialin.spec.ts

@@ -2,6 +2,7 @@ import type { Participant } from '../../../helpers/Participant';
 import { setTestProperties } from '../../../helpers/TestProperties';
 import { config as testsConfig } from '../../../helpers/TestsConfig';
 import WebhookProxy from '../../../helpers/WebhookProxy';
+import { expectations } from '../../../helpers/expectations';
 import { joinJaasMuc, generateJaasToken as t } from '../../../helpers/jaas';
 import {
     assertDialInDisplayed, assertUrlDisplayed,
@@ -34,14 +35,21 @@ describe('Dial-in', () => {
         webhooksProxy = ctx.webhooksProxy;
 
         expect(await p1.isInMuc()).toBe(true);
-        expect(await isDialInEnabled(p1)).toBe(true);
+        if (expectations.dialIn.enabled !== null) {
+            expect(await isDialInEnabled(p1)).toBe(expectations.dialIn.enabled);
+        }
         expect(customerId).toBeDefined();
     });
 
     it ('Invite UI', async () => {
         await assertUrlDisplayed(p1);
-        await assertDialInDisplayed(p1);
-        await verifyMoreNumbersPage(p1);
+        if (expectations.dialIn.enabled !== null) {
+            await assertDialInDisplayed(p1, expectations.dialIn.enabled);
+        }
+        if (expectations.dialIn.enabled === true) {
+            // TODO: assert the page is NOT shown when the expectation is false.
+            await verifyMoreNumbersPage(p1);
+        }
     });
 
     it('dial-in', async () => {

tests/specs/jaas/joinMuc.spec.ts

@@ -1,4 +1,5 @@
 import { setTestProperties } from '../../helpers/TestProperties';
+import { expectations } from '../../helpers/expectations';
 import { joinJaasMuc, generateJaasToken as t } from '../../helpers/jaas';
 import { TOKEN_AUTH_FAILED_TEST_ID, TOKEN_AUTH_FAILED_TITLE_TEST_ID } from '../../pageobjects/Notifications';
 
@@ -86,11 +87,15 @@ describe('XMPP login and MUC join test', () => {
         console.log('Joining a MUC without a token');
         const p = await joinJaasMuc();
 
-        expect(Boolean(await p.isInMuc())).toBe(false);
+        if (expectations.jaas.unauthenticatedJoins) {
+            expect(Boolean(await p.isInMuc())).toBe(true);
+        } else {
+            expect(Boolean(await p.isInMuc())).toBe(false);
 
-        const errorText = await p.getNotifications().getNotificationText(TOKEN_AUTH_FAILED_TEST_ID);
+            const errorText = await p.getNotifications().getNotificationText(TOKEN_AUTH_FAILED_TEST_ID);
 
-        expect(errorText).toContain('not allowed to join');
+            expect(errorText).toContain('not allowed to join');
+        }
     });
 
     // it('without sending a conference-request', async () => {

tests/specs/moderation/grantModerator.spec.ts

@@ -0,0 +1,55 @@
+import { Participant } from '../../helpers/Participant';
+import { setTestProperties } from '../../helpers/TestProperties';
+import { expectations } from '../../helpers/expectations';
+import { ensureOneParticipant, ensureTwoParticipants } from '../../helpers/participants';
+
+setTestProperties(__filename, {
+    usesBrowsers: [ 'p1', 'p2' ]
+});
+
+describe('Grant moderator', () => {
+    let p1: Participant, p2: Participant;
+
+    it('setup', async () => {
+        if (expectations.moderation.allModerators) {
+            ctx.skipSuiteTests = true;
+            console.log('Skipping because allModerators is expected.');
+
+            return;
+        }
+
+        await ensureOneParticipant();
+        p1 = ctx.p1;
+        expect(await p1.isModerator()).toBe(true);
+
+        const functionAvailable = await p1.execute(() => typeof APP.conference._room.grantOwner === 'function');
+
+        if (expectations.moderation.grantModerator) {
+            expect(functionAvailable).toBe(true);
+        } else {
+            if (!functionAvailable) {
+                ctx.skipSuiteTests = true;
+                console.log('Skipping because the grant moderator function is not available and not expected.');
+
+                return;
+            }
+        }
+
+        await ensureTwoParticipants();
+        p2 = ctx.p2;
+        expect(await p2.isModerator()).toBe(false);
+    });
+
+    it('grant moderator', async () => {
+        await p1.getFilmstrip().grantModerator(p2);
+
+        await p2.driver.waitUntil(
+            () => p2.isModerator(),
+            {
+                timeout: 3000,
+                timeoutMsg: 'p2 did not become moderator'
+            }
+        );
+
+    });
+});

tests/specs/moderation/kick.spec.ts

@@ -0,0 +1,75 @@
+import { Participant } from '../../helpers/Participant';
+import { setTestProperties } from '../../helpers/TestProperties';
+import { expectations } from '../../helpers/expectations';
+import { ensureTwoParticipants } from '../../helpers/participants';
+
+setTestProperties(__filename, {
+    usesBrowsers: [ 'p1', 'p2' ]
+});
+
+describe('Kick', () => {
+    let p1: Participant, p2: Participant;
+
+    it('setup', async () => {
+        await ensureTwoParticipants();
+        p1 = ctx.p1;
+        p2 = ctx.p2;
+
+        // We verify elsewhere (moderation.spec.ts) that the first participant is a moderator.
+        if (!await p1.isModerator()) {
+            ctx.skipSuiteTests = true;
+        }
+    });
+
+    it('kick (p2p disabled)', () => kickAndCheck(p1, p2));
+
+    it('setup (p2p enabled)', async () => {
+        await p1.hangup();
+        await p2.hangup();
+
+        await ensureTwoParticipants({
+            configOverwrite: {
+                p2p: {
+                    enabled: true
+                }
+            }
+        });
+        p1 = ctx.p1;
+        p2 = ctx.p2;
+    });
+
+    it('kick (p2p enabled)', async () => {
+        await kickAndCheck(p1, p2);
+    });
+
+    it('non-moderator cannot kick', async () => {
+        if (!expectations.moderation.allModerators) {
+            await ensureTwoParticipants();
+            p2 = ctx.p2;
+            expect(await p2.isModerator()).toBe(false);
+
+            await p2.execute(
+                epId => APP.conference._room.kickParticipant(epId, 'for funzies'),
+                await p1.getEndpointId()
+            );
+
+            await p1.driver.pause(3000);
+            expect(await p1.isInMuc()).toBe(true);
+        }
+    });
+});
+
+/**
+ * Kicks the second participant and checks that the participant is removed from the conference and that dialogue is open.
+ */
+async function kickAndCheck(kicker: Participant, kickee: Participant) {
+    await kicker.getFilmstrip().kickParticipant(await kickee.getEndpointId());
+    await kicker.waitForParticipants(0);
+
+    // check that the kicked participant sees the kick reason dialog
+    await kickee.driver.waitUntil(
+        async () => kickee.isLeaveReasonDialogOpen(), {
+            timeout: 2000,
+            timeoutMsg: 'No leave reason dialog shown for p2'
+        });
+}

tests/specs/moderation/moderation.spec.ts

@@ -0,0 +1,39 @@
+import { Participant } from '../../helpers/Participant';
+import { setTestProperties } from '../../helpers/TestProperties';
+import { expectations } from '../../helpers/expectations';
+import { joinMuc } from '../../helpers/joinMuc';
+
+setTestProperties(__filename, {
+    usesBrowsers: [ 'p1', 'p2' ]
+});
+
+// Just make sure that users are given moderator rights as specified in the expectations config.
+describe('Moderation', () => {
+    let p1: Participant, p2: Participant;
+
+    it('setup', async () => {
+        p1 = await joinMuc({ name: 'p1' });
+        p2 = await joinMuc({ name: 'p2' });
+    });
+    it('first moderator', async () => {
+        if (expectations.moderation.firstModerator) {
+            expect(await p1.isModerator()).toBe(true);
+        } else {
+            expect(await p1.isModerator()).toBe(false);
+        }
+    });
+    it('all moderators', async () => {
+        if (expectations.moderation.allModerators) {
+            expect(await p1.isModerator()).toBe(true);
+            expect(await p2.isModerator()).toBe(true);
+        }
+    });
+    it('auto moderator promotion', async () => {
+        if (expectations.moderation.autoModerator && !expectations.moderation.allModerators) {
+            expect(await p1.isModerator()).toBe(true);
+            expect(await p2.isModerator()).toBe(false);
+            await p1.hangup();
+            await p2.driver.waitUntil(async () => (await p2.isModerator()));
+        }
+    });
+});