Updates ljm to use release branch directly.

It is showing Login/Logout buttons which does nothing in this case.

Makefile

@@ -55,6 +55,8 @@ deploy-appbundle:
 		$(BUILD_DIR)/face-landmarks-worker.min.js.map \
 		$(BUILD_DIR)/noise-suppressor-worklet.min.js \
 		$(BUILD_DIR)/noise-suppressor-worklet.min.js.map \
+		$(BUILD_DIR)/screenshot-capture-worker.min.js \
+		$(BUILD_DIR)/screenshot-capture-worker.min.js.map \
 		$(DEPLOY_DIR)
 	cp \
 		$(BUILD_DIR)/close3.min.js \

conference.js

@@ -2042,10 +2042,6 @@ export default {
 
                     return this.useVideoStream(stream);
                 })
-                .then(() => {
-                    logger.info(`Switched local video device to ${cameraDeviceId}.`);
-                    this._updateVideoDeviceId();
-                })
                 .catch(error => {
                     logger.error(`Failed to switch to selected camera:${cameraDeviceId}, error:${error}`);
 
@@ -2100,8 +2096,6 @@ export default {
                         // above mentioned chrome bug.
                         localAudio._realDeviceId = localAudio.deviceId = 'default';
                     }
-                    logger.info(`switched local audio input device to: ${selectedDeviceId}`);
-                    this._updateAudioDeviceId();
                 })
                 .catch(err => {
                     logger.error(`Failed to switch to selected audio input device ${selectedDeviceId}, error=${err}`);
@@ -2186,13 +2180,6 @@ export default {
 
             return dispatch(getAvailableDevices())
                 .then(devices => {
-                    // Ugly way to synchronize real device IDs with local
-                    // storage and settings menu. This is a workaround until
-                    // getConstraints() method will be implemented in browsers.
-                    this._updateAudioDeviceId();
-
-                    this._updateVideoDeviceId();
-
                     APP.UI.onAvailableDevicesChanged(devices);
                 });
         }
@@ -2200,36 +2187,6 @@ export default {
         return Promise.resolve();
     },
 
-    /**
-     * Updates the settings for the currently used video device, extracting
-     * the device id from the used track.
-     * @private
-     */
-    _updateVideoDeviceId() {
-        const localVideo = getLocalJitsiVideoTrack(APP.store.getState());
-
-        if (localVideo && localVideo.videoType === 'camera') {
-            APP.store.dispatch(updateSettings({
-                cameraDeviceId: localVideo.getDeviceId()
-            }));
-        }
-    },
-
-    /**
-     * Updates the settings for the currently used audio device, extracting
-     * the device id from the used track.
-     * @private
-     */
-    _updateAudioDeviceId() {
-        const localAudio = getLocalJitsiAudioTrack(APP.store.getState());
-
-        if (localAudio) {
-            APP.store.dispatch(updateSettings({
-                micDeviceId: localAudio.getDeviceId()
-            }));
-        }
-    },
-
     /**
      * Event listener for JitsiMediaDevicesEvents.DEVICE_LIST_CHANGED to
      * handle change of available media devices.
@@ -2378,14 +2335,10 @@ export default {
                         this.useAudioStream(track)
                             .then(() => {
                                 hasDefaultMicChanged && (track._realDeviceId = track.deviceId = 'default');
-                                this._updateAudioDeviceId();
                             }));
                 } else {
                     promises.push(
-                        this.useVideoStream(track)
-                            .then(() => {
-                                this._updateVideoDeviceId();
-                            }));
+                        this.useVideoStream(track));
                 }
             }
         }

package.json

@@ -22,7 +22,7 @@
     "@giphy/js-fetch-api": "4.7.1",
     "@giphy/react-components": "6.8.1",
     "@giphy/react-native-sdk": "2.3.0",
-    "@jitsi/excalidraw": "https://github.com/jitsi/excalidraw/releases/download/v0.0.15/jitsi-excalidraw-0.0.15.tgz",
+    "@jitsi/excalidraw": "https://github.com/jitsi/excalidraw/releases/download/v0.0.16/jitsi-excalidraw-0.0.16.tgz",
     "@jitsi/js-utils": "2.2.1",
     "@jitsi/logger": "2.0.2",
     "@jitsi/rnnoise-wasm": "0.1.0",
@@ -65,12 +65,13 @@
     "js-md5": "0.6.1",
     "js-sha512": "0.8.0",
     "jwt-decode": "2.2.0",
-    "lib-jitsi-meet": "https://github.com/jitsi/lib-jitsi-meet/releases/download/v1720.0.0+b3173832/lib-jitsi-meet.tgz",
+    "lib-jitsi-meet": "git+https://github.com/jitsi/lib-jitsi-meet#release-7629",
     "lodash": "4.17.21",
     "moment": "2.29.4",
     "moment-duration-format": "2.2.2",
     "null-loader": "4.0.1",
     "optional-require": "1.0.3",
+    "pixelmatch": "5.3.0",
     "promise.allsettled": "1.0.4",
     "punycode": "2.3.0",
     "react": "18.2.0",
@@ -113,7 +114,6 @@
     "react-youtube": "10.1.0",
     "redux": "4.0.4",
     "redux-thunk": "2.4.1",
-    "resemblejs": "4.0.0",
     "seamless-scroll-polyfill": "2.1.8",
     "semver": "7.5.4",
     "tss-react": "4.4.4",
@@ -136,6 +136,8 @@
     "@types/dom-screen-wake-lock": "1.0.1",
     "@types/js-md5": "0.4.3",
     "@types/lodash": "4.14.182",
+    "@types/offscreencanvas": "2019.7.2",
+    "@types/pixelmatch": "5.2.5",
     "@types/punycode": "2.1.0",
     "@types/react": "17.0.14",
     "@types/react-dom": "17.0.14",
@@ -145,7 +147,6 @@
     "@types/react-native-video": "5.0.14",
     "@types/react-redux": "7.1.24",
     "@types/react-window": "1.8.5",
-    "@types/resemblejs": "^4.1.0",
     "@types/unorm": "1.3.28",
     "@types/uuid": "8.3.4",
     "@types/w3c-image-capture": "1.0.6",

react/features/app/middlewares.any.ts

@@ -1,7 +1,6 @@
 import '../analytics/middleware';
 import '../authentication/middleware';
 import '../av-moderation/middleware';
-import '../base/app/middleware';
 import '../base/conference/middleware';
 import '../base/config/middleware';
 import '../base/jwt/middleware';

react/features/app/middlewares.web.ts

@@ -1,3 +1,4 @@
+import '../base/app/middleware';
 import '../base/connection/middleware';
 import '../base/i18n/middleware';
 import '../base/devices/middleware';

react/features/base/app/middleware.web.ts

@@ -1,6 +1,7 @@
 import { AnyAction } from 'redux';
 
-import MiddlewareRegistry from '../../base/redux/MiddlewareRegistry';
+import MiddlewareRegistry from '../redux/MiddlewareRegistry';
+import { inIframe } from '../util/iframeUtils';
 
 import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from './actionTypes';
 import logger from './logger';
@@ -21,7 +22,9 @@ MiddlewareRegistry.register(() => (next: Function) => async (action: AnyAction)
 
     switch (action.type) {
     case APP_WILL_MOUNT: {
-        if ('PressureObserver' in globalThis) {
+        // Disable it inside an iframe until Google fixes the origin trial for 3rd party sources:
+        // https://bugs.chromium.org/p/chromium/issues/detail?id=1504167
+        if (!inIframe() && 'PressureObserver' in globalThis) {
             pressureObserver = new window.PressureObserver(
                     (records: typeof window.PressureRecord) => {
                         logger.info('Compute pressure state changed:', JSON.stringify(records));

react/features/base/conference/middleware.any.ts

@@ -14,6 +14,7 @@ import { reloadNow } from '../../app/actions';
 import { IStore } from '../../app/types';
 import { removeLobbyChatParticipant } from '../../chat/actions.any';
 import { openDisplayNamePrompt } from '../../display-name/actions';
+import { isVpaasMeeting } from '../../jaas/functions';
 import { showErrorNotification } from '../../notifications/actions';
 import { NOTIFICATION_TIMEOUT_TYPE } from '../../notifications/constants';
 import { hasDisplayName } from '../../prejoin/utils';
@@ -306,7 +307,7 @@ async function _connectionEstablished({ dispatch, getState }: IStore, next: Func
 
     // if there is token auth URL defined and local participant is using jwt
     // this means it is logged in when connection is established, so we can change the state
-    if (tokenAuthUrl) {
+    if (tokenAuthUrl && !isVpaasMeeting(getState())) {
         let email;
 
         if (getState()['features/base/jwt'].jwt) {

react/features/base/config/configType.ts

@@ -460,6 +460,7 @@ export interface IConfig {
     inviteServiceCallFlowsUrl?: string;
     inviteServiceUrl?: string;
     jaasActuatorUrl?: string;
+    jaasConferenceCreatorUrl?: string;
     jaasFeedbackMetadataURL?: string;
     jaasTokenUrl?: string;
     legalUrls?: {

react/features/base/devices/actions.web.ts

@@ -43,7 +43,7 @@ const DEVICE_TYPE_TO_SETTINGS_KEYS = {
         userSelectedDeviceLabel: 'userSelectedAudioOutputDeviceLabel'
     },
     videoInput: {
-        currentDeviceId: 'audioOutputDeviceId',
+        currentDeviceId: 'cameraDeviceId',
         userSelectedDeviceId: 'userSelectedCameraDeviceId',
         userSelectedDeviceLabel: 'userSelectedCameraDeviceLabel'
     }

react/features/base/settings/middleware.web.ts

@@ -3,9 +3,14 @@ import { PREJOIN_INITIALIZED } from '../../prejoin/actionTypes';
 import { setPrejoinPageVisibility } from '../../prejoin/actions';
 import { APP_WILL_MOUNT } from '../app/actionTypes';
 import { getJwtName } from '../jwt/functions';
+import { MEDIA_TYPE } from '../media/constants';
 import MiddlewareRegistry from '../redux/MiddlewareRegistry';
+import { TRACK_ADDED } from '../tracks/actionTypes';
+import { ITrack } from '../tracks/types';
 
 import { updateSettings } from './actions';
+import logger from './logger';
+
 
 import './middleware.any';
 
@@ -27,6 +32,9 @@ MiddlewareRegistry.register(store => next => action => {
     case PREJOIN_INITIALIZED:
         _maybeUpdateDisplayName(store);
         break;
+    case TRACK_ADDED:
+        _maybeUpdateDeviceId(store, action.track);
+        break;
     }
 
     return result;
@@ -68,3 +76,30 @@ function _maybeUpdateDisplayName({ dispatch, getState }: IStore) {
         }
     }
 }
+
+/**
+ * Maybe update the camera or mic device id when local track is added or updated.
+ *
+ * @param {Store} store - The redux store.
+ * @param {ITrack} track - The potential local track.
+ * @private
+ * @returns {void}
+ */
+function _maybeUpdateDeviceId({ dispatch, getState }: IStore, track: ITrack) {
+    if (track.local) {
+        const { cameraDeviceId, micDeviceId } = getState()['features/base/settings'];
+        const deviceId = track.jitsiTrack.getDeviceId();
+
+        if (track.mediaType === MEDIA_TYPE.VIDEO && track.videoType === 'camera' && cameraDeviceId !== deviceId) {
+            dispatch(updateSettings({
+                cameraDeviceId: track.jitsiTrack.getDeviceId()
+            }));
+            logger.info(`switched local video device to: ${deviceId}`);
+        } else if (track.mediaType === MEDIA_TYPE.AUDIO && micDeviceId !== deviceId) {
+            dispatch(updateSettings({
+                micDeviceId: track.jitsiTrack.getDeviceId()
+            }));
+            logger.info(`switched local audio input device to: ${deviceId}`);
+        }
+    }
+}

react/features/conference/actions.web.ts

@@ -1,5 +1,5 @@
 import { IStore } from '../app/types';
-import { configureInitialDevices } from '../base/devices/actions.web';
+import { configureInitialDevices, getAvailableDevices } from '../base/devices/actions.web';
 import { openDialog } from '../base/dialog/actions';
 import { getBackendSafeRoomName } from '../base/util/uri';
 
@@ -34,6 +34,18 @@ export function dismissCalendarNotification() {
     };
 }
 
+/**
+ * Setups initial devices. Makes sure we populate availableDevices list before configuring.
+ *
+ * @returns {Promise<any>}
+ */
+export function setupInitialDevices() {
+    return async (dispatch: IStore['dispatch']) => {
+        await dispatch(getAvailableDevices());
+        await dispatch(configureInitialDevices());
+    };
+}
+
 /**
  * Init.
  *
@@ -45,7 +57,7 @@ export function init() {
 
         // XXX For web based version we use conference initialization logic
         // from the old app (at the moment of writing).
-        return dispatch(configureInitialDevices()).then(
+        return dispatch(setupInitialDevices()).then(
             () => APP.conference.init({
                 roomName: room
             }).catch((error: Error) => {

react/features/desktop-picker/components/DesktopPicker.tsx

@@ -127,13 +127,6 @@ class DesktopPicker extends PureComponent<IProps, IState> {
         types: []
     };
 
-    /**
-     * Stores the type of the selected tab.
-     *
-     * @type {string}
-     */
-    _selectedTabType = DEFAULT_TAB_TYPE;
-
     /**
      * Initializes a new DesktopPicker instance.
      *
@@ -223,17 +216,20 @@ class DesktopPicker extends PureComponent<IProps, IState> {
      * Computes the selected source.
      *
      * @param {Object} sources - The available sources.
+     * @param {string} selectedTab - The selected tab.
      * @returns {Object} The selectedSource value.
      */
-    _getSelectedSource(sources: any = {}) {
+    _getSelectedSource(sources: any = {}, selectedTab?: string) {
         const { selectedSource } = this.state;
 
+        const currentSelectedTab = selectedTab ?? this.state.selectedTab;
+
         /**
          * If there are no sources for this type (or no sources for any type)
          * we can't select anything.
          */
-        if (!Array.isArray(sources[this._selectedTabType as keyof typeof sources])
-            || sources[this._selectedTabType as keyof typeof sources].length <= 0) {
+        if (!Array.isArray(sources[currentSelectedTab as keyof typeof sources])
+            || sources[currentSelectedTab as keyof typeof sources].length <= 0) {
             return {};
         }
 
@@ -245,12 +241,12 @@ class DesktopPicker extends PureComponent<IProps, IState> {
          * 3) The selected source is no longer available.
          */
         if (!selectedSource // scenario 1)
-                || selectedSource.type !== this._selectedTabType // scenario 2)
-                || !sources[this._selectedTabType].some( // scenario 3)
+                || selectedSource.type !== currentSelectedTab // scenario 2)
+                || !sources[currentSelectedTab].some( // scenario 3)
                         (source: any) => source.id === selectedSource.id)) {
             return {
-                id: sources[this._selectedTabType][0].id,
-                type: this._selectedTabType
+                id: sources[currentSelectedTab][0].id,
+                type: currentSelectedTab
             };
         }
 
@@ -319,10 +315,10 @@ class DesktopPicker extends PureComponent<IProps, IState> {
         // use the option from one tab when sharing from another.
         this.setState({
             screenShareAudio: false,
-            selectedSource: this._getSelectedSource(sources),
+            selectedSource: this._getSelectedSource(sources, id),
 
             // select type `window` or `screen` from id
-            selectedTab: id.split('-')[0]
+            selectedTab: id
         });
     }
 
@@ -410,8 +406,8 @@ class DesktopPicker extends PureComponent<IProps, IState> {
                     const selectedSource = this._getSelectedSource(sources);
 
                     this.setState({
-                        sources,
-                        selectedSource
+                        selectedSource,
+                        sources
                     });
                 })
                 .catch((error: any) => logger.log(error));

react/features/device-selection/functions.web.ts

@@ -109,7 +109,7 @@ export function getVideoDeviceSelectionDialogProps(stateful: IStateful, isDispla
     const framerate = state['features/screen-share'].captureFrameRate ?? SS_DEFAULT_FRAME_RATE;
 
     let disableVideoInputSelect = !inputDeviceChangeSupported;
-    let selectedVideoInputId = settings.cameraDeviceId;
+    let selectedVideoInputId = settings.cameraDeviceId || userSelectedCamera;
 
     // audio input change will be a problem only when we are in a
     // conference and this is not supported, when we open device selection on
@@ -180,8 +180,8 @@ export function processExternalDeviceRequest( // eslint-disable-line max-params
                 };
                 const currentlyUsedDeviceIds = new Set([
                     getAudioOutputDeviceId(),
-                    settings.micDeviceId,
-                    settings.cameraDeviceId
+                    settings.micDeviceId ?? getUserSelectedMicDeviceId(state),
+                    settings.cameraDeviceId ?? getUserSelectedCameraDeviceId(state)
                 ]);
 
                 devices.forEach(device => {

react/features/feedback/actions.web.ts

@@ -4,7 +4,6 @@ import { IStore } from '../app/types';
 import { IJitsiConference } from '../base/conference/reducer';
 import { openDialog } from '../base/dialog/actions';
 import { extractFqnFromPath } from '../dynamic-branding/functions.any';
-import { isVpaasMeeting } from '../jaas/functions';
 
 import {
     CANCEL_FEEDBACK,
@@ -12,7 +11,7 @@ import {
     SUBMIT_FEEDBACK_SUCCESS
 } from './actionTypes';
 import FeedbackDialog from './components/FeedbackDialog.web';
-import { sendFeedbackToJaaSRequest } from './functions.web';
+import { sendFeedbackToJaaSRequest, shouldSendJaaSFeedbackMetadata } from './functions.web';
 
 /**
  * Caches the passed in feedback in the redux store.
@@ -72,7 +71,9 @@ export function maybeOpenFeedbackDialog(conference: IJitsiConference, title?: st
                 showThankYou: true,
                 wasDialogShown: false
             });
-        } else if (conference.isCallstatsEnabled() && feedbackPercentage > Math.random() * 100) {
+        } else if (
+            (conference.isCallstatsEnabled() || shouldSendJaaSFeedbackMetadata(state))
+                && feedbackPercentage > Math.random() * 100) {
             return new Promise(resolve => {
                 dispatch(openFeedbackDialog(conference, title, () => {
                     const { submitted } = getState()['features/feedback'];
@@ -125,16 +126,15 @@ export function openFeedbackDialog(conference?: IJitsiConference, title?: string
  * @returns {Promise}
  */
 export function sendJaasFeedbackMetadata(conference: IJitsiConference, feedback: Object) {
-    return (dispatch: IStore['dispatch'], getState: IStore['getState']): Promise<any> => {
+    return (_dispatch: IStore['dispatch'], getState: IStore['getState']) => {
         const state = getState();
-        const { jaasFeedbackMetadataURL } = state['features/base/config'];
 
-        const { jwt, user, tenant } = state['features/base/jwt'];
-
-        if (!isVpaasMeeting(state) || !jaasFeedbackMetadataURL) {
+        if (!shouldSendJaaSFeedbackMetadata(state)) {
             return Promise.resolve();
         }
 
+        const { jaasFeedbackMetadataURL } = state['features/base/config'];
+        const { jwt, user, tenant } = state['features/base/jwt'];
         const meetingFqn = extractFqnFromPath();
         const feedbackData = {
             ...feedback,
@@ -164,17 +164,39 @@ export function submitFeedback(
         score: number,
         message: string,
         conference: IJitsiConference) {
-    return (dispatch: IStore['dispatch']) =>
-        conference.sendFeedback(score, message)
-        .then(() => dispatch({ type: SUBMIT_FEEDBACK_SUCCESS }))
-        .then(() => dispatch(sendJaasFeedbackMetadata(conference, { score,
-            message }))
-        .catch(error => {
+    return (dispatch: IStore['dispatch'], getState: IStore['getState']) => {
+        const state = getState();
+        const promises = [];
+
+        if (conference.isCallstatsEnabled()) {
+            promises.push(conference.sendFeedback(score, message));
+        }
+
+        if (shouldSendJaaSFeedbackMetadata(state)) {
+            promises.push(dispatch(sendJaasFeedbackMetadata(conference, {
+                score,
+                message
+            })));
+        }
+
+        return Promise.allSettled(promises)
+        .then(results => {
+            const rejected = results.find((result): result is PromiseRejectedResult => result?.status === 'rejected');
+
+            if (typeof rejected === 'undefined') {
+                dispatch({ type: SUBMIT_FEEDBACK_SUCCESS });
+
+                return Promise.resolve();
+            }
+
+            const error = rejected.reason;
+
             dispatch({
                 type: SUBMIT_FEEDBACK_ERROR,
                 error
             });
 
             return Promise.reject(error);
-        }));
+        });
+    };
 }

react/features/feedback/components/FeedbackButton.web.ts

@@ -9,6 +9,7 @@ import { translate } from '../../base/i18n/functions';
 import { IconFeedback } from '../../base/icons/svg';
 import AbstractButton, { IProps as AbstractButtonProps } from '../../base/toolbox/components/AbstractButton';
 import { openFeedbackDialog } from '../actions';
+import { shouldSendJaaSFeedbackMetadata } from '../functions.web';
 
 /**
  * The type of the React {@code Component} props of {@link FeedbackButton}.
@@ -45,11 +46,11 @@ class FeedbackButton extends AbstractButton<IProps> {
 }
 
 const mapStateToProps = (state: IReduxState) => {
-    const { callStatsID } = state['features/base/config'];
+    const { conference } = state['features/base/conference'];
 
     return {
-        _conference: state['features/base/conference'].conference,
-        visible: Boolean(callStatsID)
+        _conference: conference,
+        visible: conference?.isCallstatsEnabled() || shouldSendJaaSFeedbackMetadata(state)
     };
 };
 

react/features/feedback/functions.web.ts

@@ -1,16 +1,23 @@
+import { IReduxState } from '../app/types';
+import { isVpaasMeeting } from '../jaas/functions';
+
 import logger from './logger';
 
 /**
  * Sends feedback metadata to JaaS endpoints.
  *
- * @param {string} url - The JaaS metadata endpoint URL.
+ * @param {string|undefined} url - The JaaS metadata endpoint URL.
  * @param {Object} feedbackData - The feedback data object.
  * @returns {Promise}
  */
-export async function sendFeedbackToJaaSRequest(url: string, feedbackData: {
+export async function sendFeedbackToJaaSRequest(url: string | undefined, feedbackData: {
     jwt?: string; meetingFqn: string; message?: string; score?: number;
     sessionId: string; tenant?: string; userId?: string;
 }) {
+    if (!url) {
+        throw new TypeError('Trying to send jaas feedback request to an undefined URL!');
+    }
+
     const {
         jwt,
         sessionId,
@@ -49,3 +56,15 @@ export async function sendFeedbackToJaaSRequest(url: string, feedbackData: {
     }
 }
 
+/**
+ * Returns whether jaas feedback metadata should be send or not.
+ *
+ * @param {IReduxState} state - The redux state.
+ * @returns {boolean} - True if jaas feedback metadata should be send and false otherwise.
+ */
+export function shouldSendJaaSFeedbackMetadata(state: IReduxState) {
+    const { jaasFeedbackMetadataURL } = state['features/base/config'];
+
+    return Boolean(isVpaasMeeting(state) && jaasFeedbackMetadataURL);
+}
+

react/features/jaas/functions.ts

@@ -1,4 +1,5 @@
 import { IReduxState } from '../app/types';
+import { IJitsiConference } from '../base/conference/reducer';
 
 import { VPAAS_TENANT_PREFIX } from './constants';
 import logger from './logger';
@@ -47,6 +48,31 @@ export function isVpaasMeeting(state: IReduxState) {
     return false;
 }
 
+/**
+ * Sends a request for retrieving the conference creator's customer id.
+ *
+ * @param {IJitsiConference} conference - The conference state.
+ * @param {IReduxState} state - The state of the app.
+ * @returns {Object} - Object containing customerId field.
+ */
+export async function sendGetCustomerIdRequest(conference: IJitsiConference, state: IReduxState) {
+    const { jaasConferenceCreatorUrl } = state['features/base/config'];
+
+    const roomJid = conference?.room?.roomjid;
+
+    if (jaasConferenceCreatorUrl && roomJid) {
+        const fullUrl = `${jaasConferenceCreatorUrl}?conference=${encodeURIComponent(roomJid)}`;
+        const response = await fetch(fullUrl);
+        const responseBody = await response.json();
+
+        if (response.ok) {
+            return responseBody;
+        }
+
+        logger.error(`Failed to fetch ${fullUrl}. with: ${JSON.stringify(responseBody)}`);
+    }
+}
+
 /**
  * Sends a request for retrieving jaas customer details.
  *

react/features/prejoin/components/web/PrejoinApp.tsx

@@ -8,6 +8,7 @@ import { createPrejoinTracks } from '../../../base/tracks/functions.web';
 import GlobalStyles from '../../../base/ui/components/GlobalStyles.web';
 import JitsiThemeProvider from '../../../base/ui/components/JitsiThemeProvider.web';
 import DialogContainer from '../../../base/ui/components/web/DialogContainer';
+import { setupInitialDevices } from '../../../conference/actions.web';
 import { initPrejoin, makePrecallTest } from '../../actions.web';
 
 import PrejoinThirdParty from './PrejoinThirdParty';
@@ -59,6 +60,7 @@ export default class PrejoinApp extends BaseApp<Props> {
             startWithVideoMuted
         }));
 
+        await dispatch?.(setupInitialDevices());
         const { tryCreateLocalTracks, errors } = createPrejoinTracks();
 
         const tracks = await tryCreateLocalTracks;

react/features/rtcstats/RTCStats.ts

@@ -69,6 +69,16 @@ class RTCStats {
         JitsiMeetJS.rtcstats.sendStatsEntry('e2eRtt', e2eRttData);
     }
 
+    /**
+     * Send identity data, the data will be processed by rtcstats-server and saved in the dump file.
+     *
+     * @param {Object} identityData - The object that holds the identity data.
+     * @returns {void}
+     */
+    sendIdentityData(identityData: Object) {
+        JitsiMeetJS.rtcstats.sendIdentityEntry(identityData);
+    }
+
     /**
      * Send the timestamp of the start of the conference, the data will be processed by the rtcstats-server
      * and saved in the dump file.

react/features/rtcstats/middleware.ts

@@ -11,12 +11,14 @@ import MiddlewareRegistry from '../base/redux/MiddlewareRegistry';
 import { TRACK_ADDED, TRACK_UPDATED } from '../base/tracks/actionTypes';
 import { ADD_FACE_LANDMARKS } from '../face-landmarks/actionTypes';
 import { FaceLandmarks } from '../face-landmarks/types';
+import { sendGetCustomerIdRequest } from '../jaas/functions';
 
 import RTCStats from './RTCStats';
 import {
     canSendFaceLandmarksRTCStatsData,
     isRTCStatsEnabled
 } from './functions';
+import logger from './logger';
 
 /**
  * Middleware which intercepts lib-jitsi-meet initialization and conference join in order init the
@@ -33,6 +35,16 @@ MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: AnyA
     case CONFERENCE_JOINED: {
         if (isRTCStatsEnabled(state)) {
             RTCStats.init();
+
+            sendGetCustomerIdRequest(action?.conference, state)
+                .then(customerData => {
+                    const { customerId } = customerData ?? {};
+
+                    customerId && RTCStats.sendIdentityData({ customerId });
+                })
+                .catch(error => {
+                    logger.error('Error while getting customer id:', error);
+                });
         }
         break;
     }

react/features/screenshot-capture/ScreenshotCaptureSummary.tsx

@@ -1,4 +1,3 @@
-import resemble from 'resemblejs';
 import 'image-capture';
 import './createImageBitmap';
 
@@ -7,20 +6,20 @@ import { sendAnalytics } from '../analytics/functions';
 import { IReduxState } from '../app/types';
 import { getCurrentConference } from '../base/conference/functions';
 import { getLocalParticipant, getRemoteParticipants } from '../base/participants/functions';
-import { ITrack } from '../base/tracks/types';
+import { getBaseUrl } from '../base/util/helpers';
 import { extractFqnFromPath } from '../dynamic-branding/functions.any';
 
 import {
-    CLEAR_INTERVAL,
-    INTERVAL_TIMEOUT,
-    PERCENTAGE_LOWER_BOUND,
+    CLEAR_TIMEOUT,
     POLL_INTERVAL,
-    SET_INTERVAL
+    SCREENSHOT_QUEUE_LIMIT,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
 } from './constants';
+import logger from './logger';
 // eslint-disable-next-line lines-around-comment
 // @ts-ignore
 import { processScreenshot } from './processScreenshot';
-import { timerWorkerScript } from './worker';
 
 declare let ImageCapture: any;
 
@@ -30,14 +29,10 @@ declare let ImageCapture: any;
  */
 export default class ScreenshotCaptureSummary {
     _state: IReduxState;
-    _currentCanvas: HTMLCanvasElement;
-    _currentCanvasContext: CanvasRenderingContext2D | null;
     _initializedRegion: boolean;
-    _imageCapture: any;
+    _imageCapture: ImageCapture;
     _streamWorker: Worker;
-    _streamHeight: any;
-    _streamWidth: any;
-    _storedImageData?: ImageData;
+    _queue: Blob[];
 
     /**
      * Initializes a new {@code ScreenshotCaptureEffect} instance.
@@ -46,16 +41,23 @@ export default class ScreenshotCaptureSummary {
      */
     constructor(state: IReduxState) {
         this._state = state;
-        this._currentCanvas = document.createElement('canvas');
-        this._currentCanvasContext = this._currentCanvas.getContext('2d');
 
         // Bind handlers such that they access the same instance.
         this._handleWorkerAction = this._handleWorkerAction.bind(this);
-        this._initScreenshotCapture = this._initScreenshotCapture.bind(this);
-        this._streamWorker = new Worker(timerWorkerScript, { name: 'Screenshot capture worker' });
+        const baseUrl = `${getBaseUrl()}libs/`;
+
+        let workerUrl = `${baseUrl}screenshot-capture-worker.min.js`;
+
+        // @ts-ignore
+        const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });
+
+        // @ts-ignore
+        workerUrl = window.URL.createObjectURL(workerBlob);
+        this._streamWorker = new Worker(workerUrl, { name: 'Screenshot capture worker' });
         this._streamWorker.onmessage = this._handleWorkerAction;
 
         this._initializedRegion = false;
+        this._queue = [];
     }
 
     /**
@@ -77,10 +79,17 @@ export default class ScreenshotCaptureSummary {
             ...jwt && { 'Authorization': `Bearer ${jwt}` }
         };
 
-        await fetch(`${_screenshotHistoryRegionUrl}/${sessionId}`, {
-            method: 'POST',
-            headers
-        });
+        try {
+            await fetch(`${_screenshotHistoryRegionUrl}/${sessionId}`, {
+                method: 'POST',
+                headers
+            });
+        } catch (err) {
+            logger.warn(`Could not create screenshot region: ${err}`);
+
+            return;
+        }
+
 
         this._initializedRegion = true;
     }
@@ -88,31 +97,27 @@ export default class ScreenshotCaptureSummary {
     /**
      * Starts the screenshot capture event on a loop.
      *
-     * @param {Track} track - The track that contains the stream from which screenshots are to be sent.
+     * @param {JitsiTrack} jitsiTrack - The track that contains the stream from which screenshots are to be sent.
      * @returns {Promise} - Promise that resolves once effect has started or rejects if the
      * videoType parameter is not desktop.
      */
-    async start(track: ITrack) {
-        const { videoType } = track;
-        const stream = track.getOriginalStream();
+    async start(jitsiTrack: any) {
+        if (!window.OffscreenCanvas) {
+            logger.warn('Can\'t start screenshot capture, OffscreenCanvas is not available');
+
+            return;
+        }
+        const { videoType, track } = jitsiTrack;
 
         if (videoType !== 'desktop') {
             return;
         }
-        const desktopTrack = stream.getVideoTracks()[0];
-        const { height, width }
-            = desktopTrack.getSettings() ?? desktopTrack.getConstraints();
-
-        this._streamHeight = height;
-        this._streamWidth = width;
-        this._currentCanvas.height = parseInt(height, 10);
-        this._currentCanvas.width = parseInt(width, 10);
-        this._imageCapture = new ImageCapture(desktopTrack);
+        this._imageCapture = new ImageCapture(track);
 
         if (!this._initializedRegion) {
             await this._initRegionSelection();
         }
-        this._initScreenshotCapture();
+        this.sendTimeout();
     }
 
     /**
@@ -121,28 +126,34 @@ export default class ScreenshotCaptureSummary {
      * @returns {void}
      */
     stop() {
-        this._streamWorker.postMessage({ id: CLEAR_INTERVAL });
+        this._streamWorker.postMessage({ id: CLEAR_TIMEOUT });
     }
 
     /**
-     * Method that is called as soon as the first frame of the video loads from stream.
-     * The method is used to store the {@code ImageData} object from the first frames
-     * in order to use it for future comparisons based on which we can process only certain
-     * screenshots.
+     * Sends to worker the imageBitmap for the next timeout.
      *
-     * @private
-     * @returns {void}
+     * @returns {Promise<void>}
      */
-    async _initScreenshotCapture() {
-        const imageBitmap = await this._imageCapture.grabFrame();
+    async sendTimeout() {
+        let imageBitmap: ImageBitmap | undefined;
 
-        this._currentCanvasContext?.drawImage(imageBitmap, 0, 0, this._streamWidth, this._streamHeight);
-        const imageData = this._currentCanvasContext?.getImageData(0, 0, this._streamWidth, this._streamHeight);
+        if (!this._imageCapture.track || this._imageCapture.track.readyState !== 'live') {
+            logger.warn('Track is in invalid state');
+            this.stop();
+
+            return;
+        }
+
+        try {
+            imageBitmap = await this._imageCapture.grabFrame();
+        } catch (e) {
+            // ignore error
+        }
 
-        this._storedImageData = imageData;
         this._streamWorker.postMessage({
-            id: SET_INTERVAL,
-            timeMs: POLL_INTERVAL
+            id: SET_TIMEOUT,
+            timeMs: POLL_INTERVAL,
+            imageBitmap
         });
     }
 
@@ -153,18 +164,24 @@ export default class ScreenshotCaptureSummary {
      * @param {EventHandler} message - Message received from the Worker.
      * @returns {void}
      */
-    _handleWorkerAction(message: { data: { id: number; }; }) {
-        return message.data.id === INTERVAL_TIMEOUT && this._handleScreenshot();
+    _handleWorkerAction(message: { data: { id: number; imageBlob?: Blob; }; }) {
+        const { id, imageBlob } = message.data;
+
+        this.sendTimeout();
+        if (id === TIMEOUT_TICK && imageBlob && this._queue.length < SCREENSHOT_QUEUE_LIMIT) {
+            this._doProcessScreenshot(imageBlob);
+        }
     }
 
     /**
      * Method that processes the screenshot.
      *
      * @private
-     * @param {ImageData} imageData - The image data of the new screenshot.
+     * @param {Blob} imageBlob - The blob for the current screenshot.
      * @returns {void}
      */
-    _doProcessScreenshot(imageData?: ImageData) {
+    _doProcessScreenshot(imageBlob: Blob) {
+        this._queue.push(imageBlob);
         sendAnalytics(createScreensharingCaptureTakenEvent());
 
         const conference = getCurrentConference(this._state);
@@ -175,41 +192,24 @@ export default class ScreenshotCaptureSummary {
         const { jwt } = this._state['features/base/jwt'];
         const meetingFqn = extractFqnFromPath();
         const remoteParticipants = getRemoteParticipants(this._state);
-        const participants = [];
+        const participants: Array<string | undefined> = [];
 
         participants.push(getLocalParticipant(this._state)?.id);
         remoteParticipants.forEach(p => participants.push(p.id));
-        this._storedImageData = imageData;
 
-        processScreenshot(this._currentCanvas, {
+        processScreenshot(imageBlob, {
             jid,
             jwt,
             sessionId,
             timestamp,
             meetingFqn,
             participants
+        }).then(() => {
+            const index = this._queue.indexOf(imageBlob);
+
+            if (index > -1) {
+                this._queue.splice(index, 1);
+            }
         });
     }
-
-    /**
-     * Screenshot handler.
-     *
-     * @private
-     * @returns {void}
-     */
-    async _handleScreenshot() {
-        const imageBitmap = await this._imageCapture.grabFrame();
-
-        this._currentCanvasContext?.drawImage(imageBitmap, 0, 0, this._streamWidth, this._streamHeight);
-        const imageData = this._currentCanvasContext?.getImageData(0, 0, this._streamWidth, this._streamHeight);
-
-        resemble(imageData ?? '')
-            .compareTo(this._storedImageData ?? '')
-            .setReturnEarlyThreshold(PERCENTAGE_LOWER_BOUND)
-            .onComplete(resultData => {
-                if (resultData.rawMisMatchPercentage > PERCENTAGE_LOWER_BOUND) {
-                    this._doProcessScreenshot(imageData);
-                }
-            });
-    }
 }

react/features/screenshot-capture/constants.ts

@@ -1,40 +1,44 @@
 /**
  * Percent of pixels that signal if two images should be considered different.
  */
-export const PERCENTAGE_LOWER_BOUND = 5;
+export const PERCENTAGE_LOWER_BOUND = 4;
 
 /**
  * Number of milliseconds that represent how often screenshots should be taken.
  */
-export const POLL_INTERVAL = 4000;
+export const POLL_INTERVAL = 2000;
 
 /**
- * SET_INTERVAL constant is used to set interval and it is set in
+ * SET_TIMEOUT constant is used to set interval and it is set in
  * the id property of the request.data property. TimeMs property must
  * also be set. Request.data example:
  *
  * {
- *      id: SET_INTERVAL,
+ *      id: SET_TIMEOUT,
  *      timeMs: 33
  * }.
  */
-export const SET_INTERVAL = 1;
+export const SET_TIMEOUT = 1;
 
 /**
- * CLEAR_INTERVAL constant is used to clear the interval and it is set in
+ * CLEAR_TIMEOUT constant is used to clear the interval and it is set in
  * the id property of the request.data property.
  *
  * {
- *      id: CLEAR_INTERVAL
+ *      id: CLEAR_TIMEOUT
  * }.
  */
-export const CLEAR_INTERVAL = 2;
+export const CLEAR_TIMEOUT = 2;
 
 /**
- * INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
+ * TIMEOUT_TICK constant is used as response and it is set in the id property.
  *
  * {
- *      id: INTERVAL_TIMEOUT
+ *      id: TIMEOUT_TICK
  * }.
  */
-export const INTERVAL_TIMEOUT = 3;
+export const TIMEOUT_TICK = 3;
+
+export const SCREENSHOT_QUEUE_LIMIT = 3;
+
+export const MAX_FILE_SIZE = 1000000;

react/features/screenshot-capture/processScreenshot.js

@@ -1,10 +1,10 @@
 /**
  * Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}.
  *
- * @param {HTMLCanvasElement} canvas - The canvas containing a screenshot to be processed.
+ * @param {Blob} imageBlob - The blob of the screenshot that has to be processed.
  * @param {Object} options - Custom options required for processing.
- * @returns {void}
+ * @returns {Promise<void>}
  */
-export function processScreenshot(canvas, options) { // eslint-disable-line no-unused-vars
+export async function processScreenshot(imageBlob, options) { // eslint-disable-line no-unused-vars
     return;
 }

react/features/screenshot-capture/worker.ts

@@ -1,29 +1,132 @@
+import pixelmatch from 'pixelmatch';
+
 import {
-    CLEAR_INTERVAL,
-    INTERVAL_TIMEOUT,
-    SET_INTERVAL
+    CLEAR_TIMEOUT,
+    MAX_FILE_SIZE,
+    PERCENTAGE_LOWER_BOUND,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
 } from './constants';
 
-const code = `
-    var timer;
 
-    onmessage = function(request) {
-        switch (request.data.id) {
-        case ${SET_INTERVAL}: {
-            timer = setInterval(() => {
-                postMessage({ id: ${INTERVAL_TIMEOUT} });
-            }, request.data.timeMs);
-            break;
-        }
-        case ${CLEAR_INTERVAL}: {
-            if (timer) {
-                clearInterval(timer);
+let timer: ReturnType<typeof setTimeout>;
+const canvas = new OffscreenCanvas(0, 0);
+const ctx = canvas.getContext('2d');
+let storedImageData: ImageData | undefined;
+
+/**
+ * Sends Blob with the screenshot to main thread.
+ *
+  * @param {ImageData} imageData - The image of the screenshot.
+ * @returns {void}
+ */
+async function sendBlob(imageData: ImageData) {
+    let imageBlob = await canvas.convertToBlob({ type: 'image/jpeg' });
+
+    if (imageBlob.size > MAX_FILE_SIZE) {
+        const quality = Number((MAX_FILE_SIZE / imageBlob.size).toFixed(2)) * 0.92;
+
+        imageBlob = await canvas.convertToBlob({ type: 'image/jpeg',
+            quality });
+    }
+
+    storedImageData = imageData;
+
+    postMessage({
+        id: TIMEOUT_TICK,
+        imageBlob
+    });
+}
+
+/**
+ * Sends empty message to main thread.
+ *
+ * @returns {void}
+ */
+function sendEmpty() {
+    postMessage({
+        id: TIMEOUT_TICK
+    });
+}
+
+/**
+ * Draws the image bitmap on the canvas and checks the difference percent with the previous image
+ * if there is no previous image the percentage is not calculated.
+ *
+ * @param {ImageBitmap} imageBitmap - The image bitmap that is drawn on canvas.
+ * @returns {void}
+ */
+function checkScreenshot(imageBitmap: ImageBitmap) {
+    const { height, width } = imageBitmap;
+
+    if (canvas.width !== width) {
+        canvas.width = width;
+    }
+
+    if (canvas.height !== height) {
+        canvas.height = height;
+    }
+
+    ctx?.drawImage(imageBitmap, 0, 0, width, height);
+    const imageData = ctx?.getImageData(0, 0, width, height);
+
+    imageBitmap.close();
+
+    if (!imageData) {
+        sendEmpty();
+
+        return;
+    }
+
+    if (!storedImageData || imageData.data.length !== storedImageData.data.length) {
+        sendBlob(imageData);
+
+        return;
+    }
+
+    let numOfPixels = 0;
+
+    try {
+        numOfPixels = pixelmatch(
+            imageData.data,
+            storedImageData.data,
+            null,
+            width,
+            height);
+    } catch {
+        sendEmpty();
+
+        return;
+    }
+
+    const percent = numOfPixels / imageData.data.length * 100;
+
+    if (percent >= PERCENTAGE_LOWER_BOUND) {
+        sendBlob(imageData);
+    } else {
+        sendEmpty();
+    }
+}
+
+onmessage = function(request) {
+    switch (request.data.id) {
+    case SET_TIMEOUT: {
+        timer = setTimeout(async () => {
+            const imageBitmap = request.data.imageBitmap;
+
+            if (imageBitmap) {
+                checkScreenshot(imageBitmap);
+            } else {
+                sendEmpty();
             }
-            break;
+        }, request.data.timeMs);
+        break;
+    }
+    case CLEAR_TIMEOUT: {
+        if (timer) {
+            clearTimeout(timer);
         }
-        }
-    };
-`;
-
-// @ts-ignore
-export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));
+        break;
+    }
+    }
+};

webpack.config.js

@@ -383,6 +383,17 @@ module.exports = (_env, argv) => {
 
                 globalObject: 'AudioWorkletGlobalScope'
             }
+        }),
+
+        Object.assign({}, config, {
+            entry: {
+                'screenshot-capture-worker': './react/features/screenshot-capture/worker.ts'
+            },
+            plugins: [
+                ...config.plugins,
+                ...getBundleAnalyzerPlugin(analyzeBundle, 'screenshot-capture-worker')
+            ],
+            performance: getPerformanceHints(perfHintOptions, 4 * 1024)
         })
     ];
 };