test: Retry some tests (#16888)

* chore: Update wdio to 9.23.2.

* test: Add a retry test property.

* test: Retry some of the flaky tests.

package.json

@@ -164,12 +164,12 @@
     "@types/w3c-image-capture": "1.0.6",
     "@types/w3c-web-hid": "1.0.3",
     "@types/zxcvbn": "4.4.1",
-    "@wdio/allure-reporter": "9.22.0",
-    "@wdio/cli": "9.22.0",
-    "@wdio/globals": "9.17.0",
-    "@wdio/junit-reporter": "9.21.0",
-    "@wdio/local-runner": "9.22.0",
-    "@wdio/mocha-framework": "9.22.0",
+    "@wdio/allure-reporter": "9.23.2",
+    "@wdio/cli": "9.23.2",
+    "@wdio/globals": "9.23.0",
+    "@wdio/junit-reporter": "9.23.2",
+    "@wdio/local-runner": "9.23.2",
+    "@wdio/mocha-framework": "9.23.2",
     "babel-loader": "9.1.0",
     "babel-plugin-optional-require": "0.3.1",
     "circular-dependency-plugin": "5.2.0",

resources/prosody-plugins/mod_muc_meeting_id.lua

@@ -212,39 +212,39 @@ local function filterTranscriptionResult(event)
             -- Do not fire the event, but forward the message
             return
         end
-    end
 
-    if msg_obj.transcript ~= nil then
-        local transcription = msg_obj;
+        if msg_obj.transcript ~= nil then
+            local transcription = msg_obj;
 
-        -- in case of the string matching optimization above failed
-        if transcription.is_interim then
-            return;
+            -- in case of the string matching optimization above failed
+            if transcription.is_interim then
+                return;
+            end
+
+            -- TODO what if we have multiple alternative transcriptions not just 1
+            local text_message = transcription.transcript[1].text;
+            --do not send empty messages
+            if text_message == '' then
+                return;
+            end
+
+            local user_id = transcription.participant.id;
+            local who = room:get_occupant_by_nick(jid.bare(room.jid)..'/'..user_id);
+
+            transcription.jid = who and who.jid;
+            transcription.session_id = room._data.meetingId;
+
+            local tenant, conference_name, id = extract_subdomain(jid.node(room.jid));
+            if tenant then
+                transcription.fqn = tenant..'/'..conference_name;
+            else
+                transcription.fqn = conference_name;
+            end
+            transcription.customer_id = id;
+
+            return module:fire_event('jitsi-transcript-received', {
+                room = room, occupant = occupant, transcription = transcription, stanza = stanza });
         end
-
-        -- TODO what if we have multiple alternative transcriptions not just 1
-        local text_message = transcription.transcript[1].text;
-        --do not send empty messages
-        if text_message == '' then
-            return;
-        end
-
-        local user_id = transcription.participant.id;
-        local who = room:get_occupant_by_nick(jid.bare(room.jid)..'/'..user_id);
-
-        transcription.jid = who and who.jid;
-        transcription.session_id = room._data.meetingId;
-
-        local tenant, conference_name, id = extract_subdomain(jid.node(room.jid));
-        if tenant then
-            transcription.fqn = tenant..'/'..conference_name;
-        else
-            transcription.fqn = conference_name;
-        end
-        transcription.customer_id = id;
-
-        return module:fire_event('jitsi-transcript-received', {
-            room = room, occupant = occupant, transcription = transcription, stanza = stanza });
     end
 
     return module:fire_event('jitsi-endpoint-message-received', {

tests/helpers/Participant.ts

@@ -674,6 +674,10 @@ export class Participant {
         return new IframeAPI(this);
     }
 
+    async getRoomMetadata() {
+        return this.execute(() => window.APP?.conference?._room?.getMetadataHandler()?.getMetadata());
+    }
+
     /**
      * Hangups the participant by leaving the page. base.html is an empty page on all deployments.
      */

tests/helpers/TestProperties.ts

@@ -8,6 +8,8 @@ export type ITestProperties = {
     description?: string;
     /** The test requires the webhook proxy to be available. */
     requireWebhookProxy: boolean;
+    /** Whether the test should be retried. */
+    retry: boolean;
     /** The test requires jaas, it should be skipped when the jaas configuration is not enabled. */
     useJaas: boolean;
     /** The test uses the webhook proxy if available. */
@@ -18,6 +20,7 @@ export type ITestProperties = {
 const defaultProperties: ITestProperties = {
     useWebhookProxy: false,
     requireWebhookProxy: false,
+    retry: false,
     useJaas: false,
     usesBrowsers: [ 'p1' ]
 };

tests/helpers/WebhookProxy.ts

@@ -176,6 +176,7 @@ export default class WebhookProxy {
      * @param value
      */
     set defaultMeetingSettings(value: {
+        asyncTranscriptions?: boolean;
         autoAudioRecording?: boolean;
         autoTranscriptions?: boolean;
         autoVideoRecording?: boolean;

tests/specs/jaas/recording.spec.ts

@@ -7,6 +7,7 @@ import { joinJaasMuc, generateJaasToken as t } from '../../helpers/jaas';
 
 setTestProperties(__filename, {
     requireWebhookProxy: true,
+    retry: true,
     useJaas: true
 });
 

tests/specs/jaas/transcriptions.spec.ts

@@ -8,182 +8,201 @@ import { joinJaasMuc, generateJaasToken as t } from '../../helpers/jaas';
 
 setTestProperties(__filename, {
     requireWebhookProxy: true,
+    retry: true,
     useJaas: true,
     usesBrowsers: [ 'p1', 'p2' ]
 });
 
-describe('Transcription', () => {
-    let p1: Participant, p2: Participant;
-    let webhooksProxy: WebhookProxy;
+for (const asyncTranscriptions of [ false, true ]) {
+    describe(`Transcription (async=${asyncTranscriptions})`, () => {
+        let p1: Participant, p2: Participant;
+        let webhooksProxy: WebhookProxy;
 
-    it('setup', async () => {
-        const room = ctx.roomName;
+        it('setup', async () => {
+            const room = ctx.roomName;
 
-        webhooksProxy = ctx.webhooksProxy;
+            webhooksProxy = ctx.webhooksProxy;
+            webhooksProxy.defaultMeetingSettings = { asyncTranscriptions };
 
-        p1 = await joinJaasMuc({
-            name: 'p1',
-            token: t({ room, moderator: true }),
-            iFrameApi: true });
+            p1 = await joinJaasMuc({
+                name: 'p1',
+                token: t({ room, moderator: true }),
+                iFrameApi: true
+            });
 
-        const transcriptionEnabled = await p1.execute(() => config.transcription?.enabled);
+            const transcriptionEnabled = await p1.execute(() => config.transcription?.enabled);
 
-        expect(transcriptionEnabled).toBe(expectations.jaas.transcriptionEnabled);
+            expect(transcriptionEnabled).toBe(expectations.jaas.transcriptionEnabled);
 
-        p2 = await joinJaasMuc({
-            name: 'p2',
-            token: t({ room }),
-            iFrameApi: true }, {
-            configOverwrite: {
-                startWithAudioMuted: true
+            const roomMetadata = await p1.getRoomMetadata();
+
+            if (asyncTranscriptions) {
+                expect(roomMetadata.asyncTranscription).toBe(true);
+            } else {
+                expect(roomMetadata.asyncTranscription).toBeFalsy();
             }
+
+            p2 = await joinJaasMuc({
+                name: 'p2',
+                token: t({ room }),
+                iFrameApi: true
+            }, {
+                configOverwrite: {
+                    startWithAudioMuted: true
+                }
+            });
+
+            await Promise.all([
+                p1.switchToMainFrame(),
+                p2.switchToMainFrame(),
+            ]);
+
+            expect(await p1.getIframeAPI().getEventResult('isModerator')).toBe(true);
+            expect(await p1.getIframeAPI().getEventResult('videoConferenceJoined')).toBeDefined();
         });
 
-        await Promise.all([
-            p1.switchToMainFrame(),
-            p2.switchToMainFrame(),
-        ]);
+        it('toggle subtitles', async () => {
+            await p1.getIframeAPI().addEventListener('transcriptionChunkReceived');
+            await p2.getIframeAPI().addEventListener('transcriptionChunkReceived');
+            await p1.getIframeAPI().executeCommand('toggleSubtitles');
 
-        expect(await p1.getIframeAPI().getEventResult('isModerator')).toBe(true);
-        expect(await p1.getIframeAPI().getEventResult('videoConferenceJoined')).toBeDefined();
-    });
+            await checkReceivingChunks(p1, p2, webhooksProxy, !asyncTranscriptions);
 
-    it('toggle subtitles', async () => {
-        await p1.getIframeAPI().addEventListener('transcriptionChunkReceived');
-        await p2.getIframeAPI().addEventListener('transcriptionChunkReceived');
-        await p1.getIframeAPI().executeCommand('toggleSubtitles');
+            await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
+            await p1.getIframeAPI().addEventListener('transcribingStatusChanged');
 
-        await checkReceivingChunks(p1, p2, webhooksProxy);
+            await p1.getIframeAPI().executeCommand('toggleSubtitles');
 
-        await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
-        await p1.getIframeAPI().addEventListener('transcribingStatusChanged');
-
-        await p1.getIframeAPI().executeCommand('toggleSubtitles');
-
-        await p1.driver.waitUntil(() => p1.getIframeAPI()
-            .getEventResult('transcribingStatusChanged'), {
-            timeout: 15000,
-            timeoutMsg: 'transcribingStatusChanged event not received by p1'
-        });
-    });
-
-    it('set subtitles on and off', async () => {
-        // we need to clear results or the last one will be used, from the previous time subtitles were on
-        await p1.getIframeAPI().clearEventResults('transcriptionChunkReceived');
-        await p2.getIframeAPI().clearEventResults('transcriptionChunkReceived');
-
-        await p1.getIframeAPI().executeCommand('setSubtitles', true, true);
-
-        await checkReceivingChunks(p1, p2, webhooksProxy);
-
-        await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
-
-        await p1.getIframeAPI().executeCommand('setSubtitles', false);
-
-        await p1.driver.waitUntil(() => p1.getIframeAPI()
-            .getEventResult('transcribingStatusChanged'), {
-            timeout: 15000,
-            timeoutMsg: 'transcribingStatusChanged event not received by p1'
-        });
-    });
-
-    it('start/stop transcriptions via recording', async () => {
-        // we need to clear results or the last one will be used, from the previous time subtitles were on
-        await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
-        await p1.getIframeAPI().clearEventResults('transcriptionChunkReceived');
-        await p2.getIframeAPI().clearEventResults('transcriptionChunkReceived');
-
-        await p2.getIframeAPI().addEventListener('transcribingStatusChanged');
-
-        await p1.getIframeAPI().executeCommand('startRecording', { transcription: true });
-
-        let allTranscriptionStatusChanged: Promise<any>[] = [];
-
-        allTranscriptionStatusChanged.push(await p1.driver.waitUntil(() => p1.getIframeAPI()
+            await p1.driver.waitUntil(() => p1.getIframeAPI()
                 .getEventResult('transcribingStatusChanged'), {
-            timeout: 10000,
-            timeoutMsg: 'transcribingStatusChanged event not received on p1'
-        }));
-        allTranscriptionStatusChanged.push(await p2.driver.waitUntil(() => p2.getIframeAPI()
+                timeout: 15000,
+                timeoutMsg: 'transcribingStatusChanged event not received by p1'
+            });
+        });
+
+        it('set subtitles on and off', async () => {
+            // we need to clear results or the last one will be used, from the previous time subtitles were on
+            await p1.getIframeAPI().clearEventResults('transcriptionChunkReceived');
+            await p2.getIframeAPI().clearEventResults('transcriptionChunkReceived');
+
+            await p1.getIframeAPI().executeCommand('setSubtitles', true, true);
+
+            await checkReceivingChunks(p1, p2, webhooksProxy, !asyncTranscriptions);
+
+            await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
+
+            await p1.getIframeAPI().executeCommand('setSubtitles', false);
+
+            await p1.driver.waitUntil(() => p1.getIframeAPI()
                 .getEventResult('transcribingStatusChanged'), {
-            timeout: 10000,
-            timeoutMsg: 'transcribingStatusChanged event not received on p2'
-        }));
-
-        let result = await Promise.allSettled(allTranscriptionStatusChanged);
-
-        expect(result.length).toBe(2);
-
-        result.forEach(e => {
-            // @ts-ignore
-            expect(e.value.on).toBe(true);
+                timeout: 15000,
+                timeoutMsg: 'transcribingStatusChanged event not received by p1'
+            });
         });
 
-        await checkReceivingChunks(p1, p2, webhooksProxy);
+        it('start/stop transcriptions via recording', async () => {
+            // we need to clear results or the last one will be used, from the previous time subtitles were on
+            await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
+            await p1.getIframeAPI().clearEventResults('transcriptionChunkReceived');
+            await p2.getIframeAPI().clearEventResults('transcriptionChunkReceived');
 
-        await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
-        await p2.getIframeAPI().clearEventResults('transcribingStatusChanged');
+            await p2.getIframeAPI().addEventListener('transcribingStatusChanged');
 
-        await p1.getIframeAPI().executeCommand('stopRecording', 'file', true);
+            await p1.getIframeAPI().executeCommand('startRecording', { transcription: true });
 
-        allTranscriptionStatusChanged = [];
+            let allTranscriptionStatusChanged: Promise<any>[] = [];
 
-        allTranscriptionStatusChanged.push(await p1.driver.waitUntil(() => p1.getIframeAPI()
-            .getEventResult('transcribingStatusChanged'), {
-            timeout: 10000,
-            timeoutMsg: 'transcribingStatusChanged event not received on p1'
-        }));
-        allTranscriptionStatusChanged.push(await p2.driver.waitUntil(() => p2.getIframeAPI()
-            .getEventResult('transcribingStatusChanged'), {
-            timeout: 10000,
-            timeoutMsg: 'transcribingStatusChanged event not received on p2'
-        }));
+            allTranscriptionStatusChanged.push(await p1.driver.waitUntil(() => p1.getIframeAPI()
+                .getEventResult('transcribingStatusChanged'), {
+                timeout: 10000,
+                timeoutMsg: 'transcribingStatusChanged event not received on p1'
+            }));
+            allTranscriptionStatusChanged.push(await p2.driver.waitUntil(() => p2.getIframeAPI()
+                .getEventResult('transcribingStatusChanged'), {
+                timeout: 10000,
+                timeoutMsg: 'transcribingStatusChanged event not received on p2'
+            }));
 
-        result = await Promise.allSettled(allTranscriptionStatusChanged);
+            let result = await Promise.allSettled(allTranscriptionStatusChanged);
 
-        expect(result.length).toBe(2);
+            expect(result.length).toBe(2);
 
-        result.forEach(e => {
-            // @ts-ignore
-            expect(e.value.on).toBe(false);
+            result.forEach(e => {
+                // @ts-ignore
+                expect(e.value.on).toBe(true);
+            });
+
+            await checkReceivingChunks(p1, p2, webhooksProxy, !asyncTranscriptions);
+
+            await p1.getIframeAPI().clearEventResults('transcribingStatusChanged');
+            await p2.getIframeAPI().clearEventResults('transcribingStatusChanged');
+
+            await p1.getIframeAPI().executeCommand('stopRecording', 'file', true);
+
+            allTranscriptionStatusChanged = [];
+
+            allTranscriptionStatusChanged.push(await p1.driver.waitUntil(() => p1.getIframeAPI()
+                .getEventResult('transcribingStatusChanged'), {
+                timeout: 10000,
+                timeoutMsg: 'transcribingStatusChanged event not received on p1'
+            }));
+            allTranscriptionStatusChanged.push(await p2.driver.waitUntil(() => p2.getIframeAPI()
+                .getEventResult('transcribingStatusChanged'), {
+                timeout: 10000,
+                timeoutMsg: 'transcribingStatusChanged event not received on p2'
+            }));
+
+            result = await Promise.allSettled(allTranscriptionStatusChanged);
+
+            expect(result.length).toBe(2);
+
+            result.forEach(e => {
+                // @ts-ignore
+                expect(e.value.on).toBe(false);
+            });
+
+            await p1.getIframeAPI().executeCommand('hangup');
+            await p2.getIframeAPI().executeCommand('hangup');
+
+            // sometimes events are not immediately received,
+            // let's wait for destroy event before waiting for those that depends on it
+            await webhooksProxy.waitForEvent('ROOM_DESTROYED');
+
+            const event: {
+                data: {
+                    preAuthenticatedLink: string;
+                };
+                eventType: string;
+            } = await webhooksProxy.waitForEvent('TRANSCRIPTION_UPLOADED');
+
+            expect(event.eventType).toBe('TRANSCRIPTION_UPLOADED');
+            expect(event.data.preAuthenticatedLink).toBeDefined();
         });
-
-        await p1.getIframeAPI().executeCommand('hangup');
-        await p2.getIframeAPI().executeCommand('hangup');
-
-        // sometimes events are not immediately received,
-        // let's wait for destroy event before waiting for those that depends on it
-        await webhooksProxy.waitForEvent('ROOM_DESTROYED');
-
-        const event: {
-            data: {
-                preAuthenticatedLink: string;
-            };
-            eventType: string;
-        } = await webhooksProxy.waitForEvent('TRANSCRIPTION_UPLOADED');
-
-        expect('TRANSCRIPTION_UPLOADED').toBe(event.eventType);
-        expect(event.data.preAuthenticatedLink).toBeDefined();
     });
-});
+}
 
-async function checkReceivingChunks(p1: Participant, p2: Participant, webhooksProxy: WebhookProxy) {
-    const allTranscripts: Promise<any>[] = [];
-
-    allTranscripts.push(await p1.driver.waitUntil(() => p1.getIframeAPI()
+/**
+ *
+ * @param p1
+ * @param p2
+ * @param webhooksProxy
+ * @param expectName Whether to expect the events to contain the name of the participant. Currently, async
+ * transcriptions do not include the name. TODO: remove this parameter when async transcription events are fixed.
+ */
+async function checkReceivingChunks(p1: Participant, p2: Participant, webhooksProxy: WebhookProxy, expectName = true) {
+    const p1Promise = p1.driver.waitUntil(() => p1.getIframeAPI()
             .getEventResult('transcriptionChunkReceived'), {
         timeout: 60000,
         timeoutMsg: 'transcriptionChunkReceived event not received on p1 side'
-    }));
+    });
 
-    allTranscripts.push(await p2.driver.waitUntil(() => p2.getIframeAPI()
+    const p2Promise = p2.driver.waitUntil(() => p2.getIframeAPI()
             .getEventResult('transcriptionChunkReceived'), {
         timeout: 60000,
         timeoutMsg: 'transcriptionChunkReceived event not received on p2 side'
-    }));
+    });
 
-    // TRANSCRIPTION_CHUNK_RECEIVED webhook
-    allTranscripts.push((async () => {
+    const webhookPromise = async () => {
         const event: {
             data: {
                 final: string;
@@ -198,42 +217,36 @@ async function checkReceivingChunks(p1: Participant, p2: Participant, webhooksPr
             eventType: string;
         } = await webhooksProxy.waitForEvent('TRANSCRIPTION_CHUNK_RECEIVED');
 
-        expect('TRANSCRIPTION_CHUNK_RECEIVED').toBe(event.eventType);
+        expect(event.eventType).toBe('TRANSCRIPTION_CHUNK_RECEIVED');
 
         event.data.stable = event.data.final;
 
         return event;
-    })());
+    };
 
-    const result = await Promise.allSettled(allTranscripts);
-
-    expect(result.length).toBeGreaterThan(0);
-
-    // @ts-ignore
-    const firstEntryData = result[0].value.data;
-    const stable = firstEntryData.stable || firstEntryData.final;
-    const language = firstEntryData.language;
-    const messageID = firstEntryData.messageID;
+    const [ p1Event, p2Event, webhookEvent ] = await Promise.all([ p1Promise, p2Promise, await webhookPromise() ]);
     const p1Id = await p1.getEndpointId();
 
-    result.map(r => {
-        // @ts-ignore
-        const v = r.value;
+    const p1Transcript = p1Event.data.stable || p1Event.data.final;
+    const p2Transcript = p2Event.data.stable || p2Event.data.final;
+    const webhookTranscript = webhookEvent.data.stable || webhookEvent.data.final;
 
-        expect(v).toBeDefined();
+    expect(p2Transcript.includes(p1Transcript) || p1Transcript.includes(p2Transcript)).toBe(true);
+    expect(webhookTranscript.includes(p1Transcript) || p1Transcript.includes(webhookTranscript)).toBe(true);
 
-        return v.data;
-    }).forEach(tr => {
-        const checkTranscripts = stable.includes(tr.stable || tr.final) || (tr.stable || tr.final).includes(stable);
+    expect(p2Event.data.language).toBe(p1Event.data.language);
+    expect(webhookEvent.data.language).toBe(p1Event.data.language);
 
-        if (!checkTranscripts) {
-            console.log('received events', JSON.stringify(result));
-        }
+    expect(p2Event.data.messageID).toBe(p1Event.data.messageID);
+    expect(webhookEvent.data.messageID).toBe(p1Event.data.messageID);
 
-        expect(checkTranscripts).toBe(true);
-        expect(tr.language).toBe(language);
-        expect(tr.messageID).toBe(messageID);
-        expect(tr.participant.id).toBe(p1Id);
-        expect(tr.participant.name).toBe(p1.name);
-    });
+    expect(p1Event.data.participant.id).toBe(p1Id);
+    expect(p2Event.data.participant.id).toBe(p1Id);
+    expect(webhookEvent.data.participant.id).toBe(p1Id);
+
+    if (expectName) {
+        expect(p1Event.data.participant.name).toBe(p1.name);
+        expect(p2Event.data.participant.name).toBe(p1.name);
+        expect(webhookEvent.data.participant.name).toBe(p1.name);
+    }
 }

tests/specs/misc/breakoutRooms.spec.ts

@@ -10,6 +10,7 @@ import {
 } from '../../helpers/participants';
 
 setTestProperties(__filename, {
+    retry: true,
     usesBrowsers: [ 'p1', 'p2', 'p3' ]
 });
 

tests/wdio.conf.ts

@@ -288,9 +288,23 @@ export const config: WebdriverIO.MultiremoteConfig = {
         keepAlive.forEach(clearInterval);
     },
 
-    beforeSession(c, capabilities_, specs_, cid) {
+    async beforeSession(c, capabilities_, spec, cid) {
         const originalBefore = c.before;
 
+        if (spec && spec.length == 1) {
+            const testFilePath = spec[0].replace(/^file:\/\//, '');
+            const testProperties = await getTestProperties(testFilePath);
+
+            if (testProperties.retry) {
+                c.specFileRetries = 1;
+                c.specFileRetriesDeferred = true;
+                c.specFileRetriesDelay = 1;
+                console.log(`Enabling retry for ${testFilePath}`);
+            }
+        } else {
+            console.log('No test file or multiple test files specified, will not enable retries');
+        }
+
         if (!originalBefore || !Array.isArray(originalBefore) || originalBefore.length !== 1) {
             console.warn('No before hook found or more than one found, skipping');