From d06b847319c11974eebf51517ded10188f07983c Mon Sep 17 00:00:00 2001 From: Hristo Terezov Date: Mon, 24 Nov 2025 23:02:28 -0600 Subject: [PATCH] feat(pip): Add Picture-in-Picture support for Electron Implements Picture-in-Picture functionality for the Electron wrapper to maintain video engagement when users are not actively focused on the conference window. This feature addresses the need to keep users visually connected to the conference even when multitasking. Key features: - Automatic PiP mode activation and deactivation based on user interaction - Displays large video participant's stream or renders their avatar on canvas when video unavailable - Provides audio/video mute controls via MediaSession API directly in PiP window - Adds API events (_pip-requested) for Electron wrapper integration Implementation includes new pip feature module with Redux architecture, canvas-based avatar rendering with custom backgrounds support, and integration with existing mute/unmute logic. Depends on jitsi-meet-electron-sdk#479 for proper user gesture handling in Electron. --- custom.d.ts | 5 + modules/API/API.js | 41 ++ modules/API/external/external_api.js | 99 +++- package-lock.json | 32 ++ react/features/app/components/App.web.tsx | 2 + react/features/app/middlewares.web.ts | 1 + react/features/app/reducers.web.ts | 1 + react/features/app/types.ts | 2 + .../avatar/components/web/StatelessAvatar.tsx | 19 +- .../base/avatar/components/web/styles.ts | 26 + react/features/base/config/configType.ts | 4 + react/features/base/config/configWhitelist.ts | 1 + react/features/base/config/middleware.web.ts | 7 +- react/features/base/util/helpers.ts | 16 +- .../components/web/DisplayName.tsx | 6 +- .../display-name/components/web/styles.ts | 16 + react/features/filmstrip/functions.web.ts | 12 + react/features/pip/actionTypes.ts | 4 + react/features/pip/actions.ts | 193 ++++++++ react/features/pip/components/PiP.tsx | 24 + .../pip/components/PiPVideoElement.tsx | 208 ++++++++ react/features/pip/external-api.shared.ts | 31 ++ react/features/pip/functions.ts | 461 ++++++++++++++++++ react/features/pip/hooks.ts | 174 +++++++ react/features/pip/logger.ts | 3 + react/features/pip/middleware.ts | 1 + react/features/pip/reducer.ts | 30 ++ react/features/pip/subscriber.ts | 61 +++ react/features/pip/types.ts | 14 + tsconfig.native.json | 1 + webpack.config.js | 7 + 31 files changed, 1492 insertions(+), 10 deletions(-) create mode 100644 react/features/base/avatar/components/web/styles.ts create mode 100644 react/features/pip/actionTypes.ts create mode 100644 react/features/pip/actions.ts create mode 100644 react/features/pip/components/PiP.tsx create mode 100644 react/features/pip/components/PiPVideoElement.tsx create mode 100644 react/features/pip/external-api.shared.ts create mode 100644 react/features/pip/functions.ts create mode 100644 react/features/pip/hooks.ts create mode 100644 react/features/pip/logger.ts create mode 100644 react/features/pip/middleware.ts create mode 100644 react/features/pip/reducer.ts create mode 100644 react/features/pip/subscriber.ts create mode 100644 react/features/pip/types.ts diff --git a/custom.d.ts b/custom.d.ts index 60bd434c6f..69f38596d4 100644 --- a/custom.d.ts +++ b/custom.d.ts @@ -2,3 +2,8 @@ declare module '*.svg' { const content: any; export default content; } + +declare module '*.svg?raw' { + const content: string; + export default content; +} diff --git a/modules/API/API.js b/modules/API/API.js index 4a8d892ebf..108b610467 100755 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -107,6 +107,7 @@ import { open as openParticipantsPane } from '../../react/features/participants-pane/actions'; import { getParticipantsPaneOpen } from '../../react/features/participants-pane/functions'; +import { hidePiP, showPiP } from '../../react/features/pip/actions'; import { startLocalVideoRecording, stopLocalVideoRecording } from '../../react/features/recording/actions.any'; import { grantRecordingConsent, grantRecordingConsentAndUnmute } from '../../react/features/recording/actions.web'; import { RECORDING_METADATA_ID, RECORDING_TYPES } from '../../react/features/recording/constants'; @@ -907,6 +908,12 @@ function initCommands() { backgroundType: VIRTUAL_BACKGROUND_TYPE.IMAGE, virtualSource: backgroundImage }, jitsiTrack)); + }, + 'show-pip': () => { + APP.store.dispatch(showPiP()); + }, + 'hide-pip': () => { + APP.store.dispatch(hidePiP()); } }; transport.on('event', ({ data, name }) => { @@ -2249,6 +2256,40 @@ class API { }); } + /** + * Notify external application (if API is enabled) that Picture-in-Picture was requested. + * Used by Electron to handle PiP requests with proper user gesture context. + * + * @returns {void} + */ + notifyPictureInPictureRequested() { + this._sendEvent({ + name: '_pip-requested' + }); + } + + /** + * Notify external application (if API is enabled) that Picture-in-Picture mode was entered. + * + * @returns {void} + */ + notifyPictureInPictureEntered() { + this._sendEvent({ + name: 'pip-entered' + }); + } + + /** + * Notify external application (if API is enabled) that Picture-in-Picture mode was exited. + * + * @returns {void} + */ + notifyPictureInPictureLeft() { + this._sendEvent({ + name: 'pip-left' + }); + } + /** * Notify external application ( if API is enabled) that a participant menu button was clicked. * diff --git a/modules/API/external/external_api.js b/modules/API/external/external_api.js index 9a821e2739..f0c6d66745 100644 --- a/modules/API/external/external_api.js +++ b/modules/API/external/external_api.js @@ -2,6 +2,7 @@ import { jitsiLocalStorage } from '@jitsi/js-utils/jitsi-local-storage'; import EventEmitter from 'events'; import { urlObjectToString } from '../../../react/features/base/util/uri'; +import { isPiPEnabled } from '../../../react/features/pip/external-api.shared'; import { PostMessageTransportBackend, Transport @@ -94,7 +95,9 @@ const commands = { toggleTileView: 'toggle-tile-view', toggleVirtualBackgroundDialog: 'toggle-virtual-background', toggleVideo: 'toggle-video', - toggleWhiteboard: 'toggle-whiteboard' + toggleWhiteboard: 'toggle-whiteboard', + showPiP: 'show-pip', + hidePiP: 'hide-pip' }; /** @@ -102,6 +105,9 @@ const commands = { * events expected by jitsi-meet. */ const events = { + '_pip-requested': '_pipRequested', + 'pip-entered': 'pipEntered', + 'pip-left': 'pipLeft', 'avatar-changed': 'avatarChanged', 'audio-availability-changed': 'audioAvailabilityChanged', 'audio-mute-status-changed': 'audioMuteStatusChanged', @@ -330,6 +336,7 @@ export default class JitsiMeetExternalAPI extends EventEmitter { this._myUserID = undefined; this._onStageParticipant = undefined; this._iAmvisitor = undefined; + this._pipConfig = configOverwrite?.pip; this._setupListeners(); id++; } @@ -625,6 +632,26 @@ export default class JitsiMeetExternalAPI extends EventEmitter { // Since this is internal event we don't need to emit it to the consumer of the API. return true; + case 'config-overwrite': { + // Update stored pip config when overwriteConfig is called. + if (data.pip !== undefined) { + const wasEnabled = isPiPEnabled(this._pipConfig); + + this._pipConfig = data.pip; + + const isEnabled = isPiPEnabled(this._pipConfig); + + // Handle enable/disable transitions. + if (!wasEnabled && isEnabled) { + this._setupIntersectionObserver(); + } else if (wasEnabled && !isEnabled) { + this._teardownIntersectionObserver(); + } + } + + // Since this is internal event we don't need to emit it to the consumer of the API. + return true; + } } const eventName = events[name]; @@ -649,6 +676,56 @@ export default class JitsiMeetExternalAPI extends EventEmitter { this.emit(requestName, data, callback); } }); + + this._setupIntersectionObserver(); + } + + /** + * Sets up IntersectionObserver to monitor iframe visibility. + * Calls showPiP/hidePiP based on visibility. + * + * @private + * @returns {void} + */ + _setupIntersectionObserver() { + if (!isPiPEnabled(this._pipConfig)) { + return; + } + + // Don't create duplicate observers. + if (this._intersectionObserver) { + return; + } + + this._isIntersecting = true; + + this._intersectionObserver = new IntersectionObserver(entries => { + const entry = entries[entries.length - 1]; + const wasIntersecting = this._isIntersecting; + + this._isIntersecting = entry.isIntersecting; + + if (!entry.isIntersecting && wasIntersecting) { + this.showPiP(); + } else if (entry.isIntersecting && !wasIntersecting) { + this.hidePiP(); + } + }); + + this._intersectionObserver.observe(this._frame); + } + + /** + * Tears down IntersectionObserver. + * + * @private + * @returns {void} + */ + _teardownIntersectionObserver() { + if (this._intersectionObserver) { + this._intersectionObserver.disconnect(); + this._intersectionObserver = null; + } } /** @@ -851,6 +928,8 @@ export default class JitsiMeetExternalAPI extends EventEmitter { this.emit('_willDispose'); this._transport.dispose(); this.removeAllListeners(); + this._teardownIntersectionObserver(); + if (this._frame && this._frame.parentNode) { this._frame.parentNode.removeChild(this._frame); } @@ -1496,6 +1575,24 @@ export default class JitsiMeetExternalAPI extends EventEmitter { this.executeCommand('setVirtualBackground', enabled, backgroundImage); } + /** + * Shows Picture-in-Picture window. + * + * @returns {void} + */ + showPiP() { + this.executeCommand('showPiP'); + } + + /** + * Hides Picture-in-Picture window. + * + * @returns {void} + */ + hidePiP() { + this.executeCommand('hidePiP'); + } + /** * Opens the desktop picker. This is invoked by the Electron SDK when gDM is used. * diff --git a/package-lock.json b/package-lock.json index 4ebf24b527..7e52a9f9d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18279,6 +18279,17 @@ "webrtc-adapter": "8.1.1" } }, + "node_modules/lib-jitsi-meet/node_modules/@jitsi/js-utils": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/@jitsi/js-utils/-/js-utils-2.8.2.tgz", + "integrity": "sha512-tMFuci2lPmbQIFF/f3b5QdkL1vzY6sii9nj0e+K0EEJcFUJiX/QVkv5GbI6pMZ74BYAQXGFZqeASo8hKItniUA==", + "license": "Apache-2.0", + "dependencies": { + "@hapi/bourne": "3.0.0", + "js-md5": "0.7.3", + "ua-parser-js": "1.0.35" + } + }, "node_modules/lib-jitsi-meet/node_modules/@jitsi/logger": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@jitsi/logger/-/logger-2.0.2.tgz", @@ -18301,6 +18312,12 @@ "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==", "license": "MIT" }, + "node_modules/lib-jitsi-meet/node_modules/js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==", + "license": "MIT" + }, "node_modules/lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -39694,6 +39711,16 @@ "webrtc-adapter": "8.1.1" }, "dependencies": { + "@jitsi/js-utils": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/@jitsi/js-utils/-/js-utils-2.8.2.tgz", + "integrity": "sha512-tMFuci2lPmbQIFF/f3b5QdkL1vzY6sii9nj0e+K0EEJcFUJiX/QVkv5GbI6pMZ74BYAQXGFZqeASo8hKItniUA==", + "requires": { + "@hapi/bourne": "3.0.0", + "js-md5": "0.7.3", + "ua-parser-js": "1.0.35" + } + }, "@jitsi/logger": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@jitsi/logger/-/logger-2.0.2.tgz", @@ -39714,6 +39741,11 @@ "version": "10.4.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.4.0.tgz", "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==" + }, + "js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==" } } }, diff --git a/react/features/app/components/App.web.tsx b/react/features/app/components/App.web.tsx index b3fc07c1ef..ca38235ef0 100644 --- a/react/features/app/components/App.web.tsx +++ b/react/features/app/components/App.web.tsx @@ -5,6 +5,7 @@ import JitsiThemeProvider from '../../base/ui/components/JitsiThemeProvider.web' import DialogContainer from '../../base/ui/components/web/DialogContainer'; import ChromeExtensionBanner from '../../chrome-extension-banner/components/ChromeExtensionBanner.web'; import OverlayContainer from '../../overlay/components/web/OverlayContainer'; +import PiP from '../../pip/components/PiP'; import { AbstractApp } from './AbstractApp'; @@ -47,6 +48,7 @@ export class App extends AbstractApp { + { super._createMainElement(component, props) } ); diff --git a/react/features/app/middlewares.web.ts b/react/features/app/middlewares.web.ts index 11dc496100..9f46023269 100644 --- a/react/features/app/middlewares.web.ts +++ b/react/features/app/middlewares.web.ts @@ -12,6 +12,7 @@ import '../no-audio-signal/middleware'; import '../notifications/middleware'; import '../noise-detection/middleware'; import '../old-client-notification/middleware'; +import '../pip/middleware'; import '../power-monitor/middleware'; import '../prejoin/middleware'; import '../remote-control/middleware'; diff --git a/react/features/app/reducers.web.ts b/react/features/app/reducers.web.ts index f81950b13d..238a196793 100644 --- a/react/features/app/reducers.web.ts +++ b/react/features/app/reducers.web.ts @@ -8,6 +8,7 @@ import '../keyboard-shortcuts/reducer'; import '../no-audio-signal/reducer'; import '../noise-detection/reducer'; import '../participants-pane/reducer'; +import '../pip/reducer'; import '../power-monitor/reducer'; import '../prejoin/reducer'; import '../remote-control/reducer'; diff --git a/react/features/app/types.ts b/react/features/app/types.ts index 3b2a830144..0ef5d57350 100644 --- a/react/features/app/types.ts +++ b/react/features/app/types.ts @@ -57,6 +57,7 @@ import { INoiseDetectionState } from '../noise-detection/reducer'; import { INoiseSuppressionState } from '../noise-suppression/reducer'; import { INotificationsState } from '../notifications/reducer'; import { IParticipantsPaneState } from '../participants-pane/reducer'; +import { IPipState } from '../pip/reducer'; import { IPollsState } from '../polls/reducer'; import { IPollsHistoryState } from '../polls-history/reducer'; import { IPowerMonitorState } from '../power-monitor/reducer'; @@ -145,6 +146,7 @@ export interface IReduxState { 'features/noise-suppression': INoiseSuppressionState; 'features/notifications': INotificationsState; 'features/participants-pane': IParticipantsPaneState; + 'features/pip': IPipState; 'features/polls': IPollsState; 'features/polls-history': IPollsHistoryState; 'features/power-monitor': IPowerMonitorState; diff --git a/react/features/base/avatar/components/web/StatelessAvatar.tsx b/react/features/base/avatar/components/web/StatelessAvatar.tsx index 29eb4252e0..40690fb255 100644 --- a/react/features/base/avatar/components/web/StatelessAvatar.tsx +++ b/react/features/base/avatar/components/web/StatelessAvatar.tsx @@ -5,7 +5,18 @@ import Icon from '../../../icons/components/Icon'; import { pixelsToRem } from '../../../ui/functions.any'; import { isIcon } from '../../functions'; import { IAvatarProps } from '../../types'; -import { PRESENCE_AVAILABLE_COLOR, PRESENCE_AWAY_COLOR, PRESENCE_BUSY_COLOR, PRESENCE_IDLE_COLOR } from '../styles'; +import { + PRESENCE_AVAILABLE_COLOR, + PRESENCE_AWAY_COLOR, + PRESENCE_BUSY_COLOR, + PRESENCE_IDLE_COLOR +} from '../styles'; + +import { + AVATAR_DEFAULT_BACKGROUND_COLOR, + getAvatarFont, + getAvatarInitialsColor +} from './styles'; interface IProps extends IAvatarProps { @@ -48,10 +59,10 @@ interface IProps extends IAvatarProps { const useStyles = makeStyles()(theme => { return { avatar: { - backgroundColor: '#AAA', + backgroundColor: AVATAR_DEFAULT_BACKGROUND_COLOR, borderRadius: '50%', - color: theme.palette?.text01 || '#fff', - ...(theme.typography?.heading1 ?? {}), + color: getAvatarInitialsColor(theme), + ...getAvatarFont(theme), fontSize: 'inherit', objectFit: 'cover', textAlign: 'center', diff --git a/react/features/base/avatar/components/web/styles.ts b/react/features/base/avatar/components/web/styles.ts new file mode 100644 index 0000000000..6da6d1c3ea --- /dev/null +++ b/react/features/base/avatar/components/web/styles.ts @@ -0,0 +1,26 @@ +import { Theme } from '@mui/material/styles'; + +// Default avatar background color +export const AVATAR_DEFAULT_BACKGROUND_COLOR = '#AAA'; + +/** + * Returns the avatar font style from the theme. + * + * @param {Theme} theme - The MUI theme. + * @returns {Object} The font style object containing fontFamily, fontWeight, etc. + */ +export const getAvatarFont = (theme: Theme) => theme.typography?.heading1 ?? {}; + +/** + * Default text color for avatar initials. + */ +export const AVATAR_DEFAULT_INITIALS_COLOR = '#FFFFFF'; + +/** + * Returns the text color for avatar initials from the theme. + * + * @param {Theme} theme - The MUI theme. + * @returns {string} The text color. + */ +export const getAvatarInitialsColor = (theme: Theme): string => + theme.palette?.text01 || AVATAR_DEFAULT_INITIALS_COLOR; diff --git a/react/features/base/config/configType.ts b/react/features/base/config/configType.ts index e8aa5ab579..27ed2aceb5 100644 --- a/react/features/base/config/configType.ts +++ b/react/features/base/config/configType.ts @@ -517,6 +517,10 @@ export interface IConfig { peopleSearchQueryTypes?: string[]; peopleSearchTokenLocation?: string; peopleSearchUrl?: string; + pip?: { + disabled?: boolean; + showOnPrejoin?: boolean; + }; preferBosh?: boolean; preferVisitor?: boolean; preferredTranscribeLanguage?: string; diff --git a/react/features/base/config/configWhitelist.ts b/react/features/base/config/configWhitelist.ts index e5e78ecc85..f86158f545 100644 --- a/react/features/base/config/configWhitelist.ts +++ b/react/features/base/config/configWhitelist.ts @@ -199,6 +199,7 @@ export default [ 'participantMenuButtonsWithNotifyClick', 'participantsPane', 'pcStatsInterval', + 'pip', 'preferBosh', 'preferVisitor', 'prejoinConfig.enabled', diff --git a/react/features/base/config/middleware.web.ts b/react/features/base/config/middleware.web.ts index 0009f714fa..1b197505b2 100644 --- a/react/features/base/config/middleware.web.ts +++ b/react/features/base/config/middleware.web.ts @@ -13,11 +13,12 @@ import './middleware.any'; * @returns {Function} */ MiddlewareRegistry.register(store => next => action => { + const result = next(action); + switch (action.type) { case SET_CONFIG: { const { initialWidth, stageFilmstripParticipants } = action.config.filmstrip || {}; const { dispatch, getState } = store; - const result = next(action); const state = getState(); if (stageFilmstripParticipants !== undefined) { @@ -39,7 +40,9 @@ MiddlewareRegistry.register(store => next => action => { window.config = state['features/base/config']; } - return result; + break; } } + + return result; }); diff --git a/react/features/base/util/helpers.ts b/react/features/base/util/helpers.ts index 97a1113294..bb489a9026 100644 --- a/react/features/base/util/helpers.ts +++ b/react/features/base/util/helpers.ts @@ -22,7 +22,6 @@ export function assignIfDefined(target: Object, source: Object) { return to; } - const MATCH_OPERATOR_REGEXP = /[|\\{}()[\]^$+*?.-]/g; /** @@ -79,6 +78,21 @@ export function getJitsiMeetGlobalNS() { return window.JitsiMeetJS.app; } +/** + * Gets the Electron-specific global namespace. + * + * @returns {Object} The Electron namespace. + */ +export function getElectronGlobalNS() { + const globalNS = getJitsiMeetGlobalNS(); + + if (!globalNS.electron) { + globalNS.electron = {}; + } + + return globalNS.electron; +} + /** * Returns the object that stores the connection times. * diff --git a/react/features/display-name/components/web/DisplayName.tsx b/react/features/display-name/components/web/DisplayName.tsx index b89428ffac..8e63e0375a 100644 --- a/react/features/display-name/components/web/DisplayName.tsx +++ b/react/features/display-name/components/web/DisplayName.tsx @@ -13,6 +13,8 @@ import Tooltip from '../../../base/tooltip/components/Tooltip'; import { getIndicatorsTooltipPosition } from '../../../filmstrip/functions.web'; import { appendSuffix } from '../../functions'; +import { getDisplayNameColor } from './styles'; + /** * The type of the React {@code Component} props of {@link DisplayName}. */ @@ -49,7 +51,7 @@ const useStyles = makeStyles()(theme => { return { displayName: { ...theme.typography.labelBold, - color: theme.palette.text01, + color: getDisplayNameColor(theme), overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' @@ -62,7 +64,7 @@ const useStyles = makeStyles()(theme => { boxShadow: 'none', padding: 0, ...theme.typography.labelBold, - color: theme.palette.text01 + color: getDisplayNameColor(theme) } }; }); diff --git a/react/features/display-name/components/web/styles.ts b/react/features/display-name/components/web/styles.ts index f00d25d63e..100b1210d0 100644 --- a/react/features/display-name/components/web/styles.ts +++ b/react/features/display-name/components/web/styles.ts @@ -121,3 +121,19 @@ export function scaleFontProperty( return parseFloat(calculatedRemValue.toFixed(3)); } + + +/** + * Default text color for display name. + */ +export const DISPLAY_NAME_DEFAULT_COLOR = '#FFFFFF'; + +/** + * Returns the text color for display name from the theme. + * + * @param {Theme} theme - The MUI theme. + * @returns {string} The text color. + */ +export const getDisplayNameColor = (theme: Theme): string => + theme.palette?.text01 || DISPLAY_NAME_DEFAULT_COLOR; + diff --git a/react/features/filmstrip/functions.web.ts b/react/features/filmstrip/functions.web.ts index 665c4f88b9..61ae85b1b6 100644 --- a/react/features/filmstrip/functions.web.ts +++ b/react/features/filmstrip/functions.web.ts @@ -1,3 +1,5 @@ +import { Theme } from '@mui/material/styles'; + import { IReduxState } from '../app/types'; import { IStateful } from '../base/app/types'; import { isMobileBrowser } from '../base/environment/utils'; @@ -830,3 +832,13 @@ export function isTopPanelEnabled(state: IReduxState) { return !filmstrip?.disableTopPanel && participantsCount >= (filmstrip?.minParticipantCountForTopPanel ?? 50); } + +/** + * Returns the thumbnail background color from the theme. + * + * @param {Theme} theme - The MUI theme. + * @returns {string} The background color. + */ +export function getThumbnailBackgroundColor(theme: Theme): string { + return theme.palette.uiBackground; +} diff --git a/react/features/pip/actionTypes.ts b/react/features/pip/actionTypes.ts new file mode 100644 index 0000000000..6c8f2c36a2 --- /dev/null +++ b/react/features/pip/actionTypes.ts @@ -0,0 +1,4 @@ +/** + * Action type to set Picture-in-Picture active state. + */ +export const SET_PIP_ACTIVE = 'SET_PIP_ACTIVE'; diff --git a/react/features/pip/actions.ts b/react/features/pip/actions.ts new file mode 100644 index 0000000000..183dbbae02 --- /dev/null +++ b/react/features/pip/actions.ts @@ -0,0 +1,193 @@ +import { IStore } from '../app/types'; +import { MEDIA_TYPE } from '../base/media/constants'; +import { isLocalTrackMuted } from '../base/tracks/functions.any'; +import { handleToggleVideoMuted } from '../toolbox/actions.any'; +import { muteLocal } from '../video-menu/actions.any'; + +import { SET_PIP_ACTIVE } from './actionTypes'; +import { + cleanupMediaSessionHandlers, + enterPiP, + setupMediaSessionHandlers, + shouldShowPiP +} from './functions'; +import logger from './logger'; + +/** + * Action to set Picture-in-Picture active state. + * + * @param {boolean} isPiPActive - Whether PiP is active. + * @returns {{ + * type: SET_PIP_ACTIVE, + * isPiPActive: boolean + * }} + */ +export function setPiPActive(isPiPActive: boolean) { + return { + type: SET_PIP_ACTIVE, + isPiPActive + }; +} + +/** + * Toggles audio mute from PiP MediaSession controls. + * Uses exact same logic as toolbar audio button including GUM pending state. + * + * @returns {Function} + */ +export function toggleAudioFromPiP() { + return (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const audioMuted = isLocalTrackMuted(state['features/base/tracks'], MEDIA_TYPE.AUDIO); + + // Use the exact same action as toolbar button. + dispatch(muteLocal(!audioMuted, MEDIA_TYPE.AUDIO)); + }; +} + +/** + * Toggles video mute from PiP MediaSession controls. + * Uses exact same logic as toolbar video button including GUM pending state. + * + * @returns {Function} + */ +export function toggleVideoFromPiP() { + return (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const videoMuted = isLocalTrackMuted(state['features/base/tracks'], MEDIA_TYPE.VIDEO); + + // Use the exact same action as toolbar button (showUI=true, ensureTrack=true). + dispatch(handleToggleVideoMuted(!videoMuted, true, true)); + }; +} + +/** + * Action to exit Picture-in-Picture mode. + * + * @returns {Function} + */ +export function exitPiP() { + return (dispatch: IStore['dispatch']) => { + if (document.pictureInPictureElement) { + document.exitPictureInPicture() + .then(() => { + logger.debug('Exited Picture-in-Picture mode'); + }) + .catch((err: Error) => { + logger.error(`Error while exiting PiP: ${err.message}`); + }); + } + + dispatch(setPiPActive(false)); + cleanupMediaSessionHandlers(); + }; +} + +/** + * Action to handle window blur or tab switch. + * Enters PiP mode if not already active. + * + * @param {HTMLVideoElement} videoElement - The video element we will use for PiP. + * @returns {Function} + */ +export function handleWindowBlur(videoElement: HTMLVideoElement) { + return (_dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const isPiPActive = state['features/pip']?.isPiPActive; + + if (!isPiPActive) { + enterPiP(videoElement); + } + }; +} + +/** + * Action to handle window focus. + * Exits PiP mode if currently active (matches old AOT behavior). + * + * @returns {Function} + */ +export function handleWindowFocus() { + return (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const isPiPActive = state['features/pip']?.isPiPActive; + + if (isPiPActive) { + dispatch(exitPiP()); + } + }; +} + +/** + * Action to handle the browser's leavepictureinpicture event. + * Updates state and cleans up MediaSession handlers. + * + * @returns {Function} + */ +export function handlePiPLeaveEvent() { + return (dispatch: IStore['dispatch']) => { + logger.log('Left Picture-in-Picture mode'); + + dispatch(setPiPActive(false)); + cleanupMediaSessionHandlers(); + APP.API.notifyPictureInPictureLeft(); + }; +} + +/** + * Action to handle the browser's enterpictureinpicture event. + * Updates state and sets up MediaSession handlers. + * + * @returns {Function} + */ +export function handlePipEnterEvent() { + return (dispatch: IStore['dispatch']) => { + logger.log('Entered Picture-in-Picture mode'); + + dispatch(setPiPActive(true)); + setupMediaSessionHandlers(dispatch); + APP.API.notifyPictureInPictureEntered(); + }; +} + +/** + * Shows Picture-in-Picture window. + * Called from external API when iframe becomes not visible (IntersectionObserver). + * + * @returns {Function} + */ +export function showPiP() { + return (_dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const isPiPActive = state['features/pip']?.isPiPActive; + + if (!shouldShowPiP(state)) { + return; + } + + if (!isPiPActive) { + const videoElement = document.getElementById('pipVideo') as HTMLVideoElement; + + if (videoElement) { + enterPiP(videoElement); + } + } + }; +} + +/** + * Hides Picture-in-Picture window. + * Called from external API when iframe becomes visible. + * + * @returns {Function} + */ +export function hidePiP() { + return (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + const isPiPActive = state['features/pip']?.isPiPActive; + + if (isPiPActive) { + dispatch(exitPiP()); + } + }; +} diff --git a/react/features/pip/components/PiP.tsx b/react/features/pip/components/PiP.tsx new file mode 100644 index 0000000000..c8863733f7 --- /dev/null +++ b/react/features/pip/components/PiP.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { useSelector } from 'react-redux'; + +import { shouldShowPiP } from '../functions'; + +import PiPVideoElement from './PiPVideoElement'; + +/** + * Wrapper component that conditionally renders PiPVideoElement. + * Prevents mounting when PiP is disabled or on prejoin without showOnPrejoin flag. + * + * @returns {React.ReactElement | null} + */ +function PiP() { + const showPiP = useSelector(shouldShowPiP); + + if (!showPiP) { + return null; + } + + return ; +} + +export default PiP; diff --git a/react/features/pip/components/PiPVideoElement.tsx b/react/features/pip/components/PiPVideoElement.tsx new file mode 100644 index 0000000000..fe18af6690 --- /dev/null +++ b/react/features/pip/components/PiPVideoElement.tsx @@ -0,0 +1,208 @@ +import React, { useEffect, useRef } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { makeStyles } from 'tss-react/mui'; + +import { IReduxState, IStore } from '../../app/types'; +import { getAvatarFont, getAvatarInitialsColor } from '../../base/avatar/components/web/styles'; +import { getLocalParticipant, getParticipantDisplayName } from '../../base/participants/functions'; +import { isTrackStreamingStatusActive } from '../../connection-indicator/functions'; +import { getDisplayNameColor } from '../../display-name/components/web/styles'; +import { getThumbnailBackgroundColor } from '../../filmstrip/functions.web'; +import { getLargeVideoParticipant } from '../../large-video/functions'; +import { isPrejoinPageVisible } from '../../prejoin/functions.any'; +import { handlePiPLeaveEvent, handlePipEnterEvent, handleWindowBlur, handleWindowFocus } from '../actions'; +import { getPiPVideoTrack } from '../functions'; +import { useCanvasAvatar } from '../hooks'; +import logger from '../logger'; + +const useStyles = makeStyles()(() => { + return { + hiddenVideo: { + position: 'absolute' as const, + width: '1px', + height: '1px', + opacity: 0, + pointerEvents: 'none' as const, + left: '-9999px', + top: '-9999px' + } + }; +}); + +/** + * Component that renders a hidden video element for Picture-in-Picture. + * Automatically switches between real video track and canvas-based avatar + * depending on video availability. + * + * @returns {JSX.Element} The hidden video element. + */ +const PiPVideoElement: React.FC = () => { + const { classes, theme } = useStyles(); + const videoRef = useRef(null); + const previousTrackRef = useRef(null); + + // Redux selectors. + const isOnPrejoin = useSelector(isPrejoinPageVisible); + const localParticipant = useSelector(getLocalParticipant); + const largeVideoParticipant = useSelector(getLargeVideoParticipant); + + // Use local participant during prejoin, otherwise large video participant. + const participant = isOnPrejoin ? localParticipant : largeVideoParticipant; + + // Get appropriate video track based on prejoin state. + const videoTrack = useSelector((state: IReduxState) => + getPiPVideoTrack(state, participant) + ); + const displayName = useSelector((state: IReduxState) => + participant?.id + ? getParticipantDisplayName(state, participant.id) + : '' + ); + const customAvatarBackgrounds = useSelector((state: IReduxState) => + state['features/dynamic-branding']?.avatarBackgrounds || [] + ); + + const dispatch: IStore['dispatch'] = useDispatch(); + const videoElement = videoRef.current; + const avatarFont = getAvatarFont(theme); + const fontFamily = (avatarFont as any).fontFamily ?? 'Inter, sans-serif'; + const initialsColor = getAvatarInitialsColor(theme); + const displayNameColor = getDisplayNameColor(theme); + const { canvasStream } = useCanvasAvatar({ + participant, + displayName, + customAvatarBackgrounds, + backgroundColor: getThumbnailBackgroundColor(theme), + fontFamily, + initialsColor, + displayNameColor + }); + + // Determine if we should show avatar instead of video. + const shouldShowAvatar = !videoTrack + || videoTrack.muted + || (!videoTrack.local && !isTrackStreamingStatusActive(videoTrack)); + + /** + * Effect: Handle switching between real video track and canvas avatar stream. + */ + useEffect(() => { + if (!videoElement) { + return; + } + + const previousTrack = previousTrackRef.current; + + // Detach previous track. + if (previousTrack?.jitsiTrack) { + try { + previousTrack.jitsiTrack.detach(videoElement); + } catch (error) { + logger.error('Error detaching previous track:', error); + } + } + + if (shouldShowAvatar) { + // Use canvas stream for avatar. + // Only set srcObject if it's different to avoid interrupting playback. + if (canvasStream && videoElement.srcObject !== canvasStream) { + videoElement.srcObject = canvasStream; + } + } else if (videoTrack?.jitsiTrack) { + // Attach real video track. + videoTrack.jitsiTrack.attach(videoElement) + .catch((error: Error) => { + logger.error('Error attaching video track:', error); + }); + } + + previousTrackRef.current = videoTrack; + + // Cleanup on unmount or track change. + return () => { + if (videoTrack?.jitsiTrack && videoElement) { + try { + videoTrack.jitsiTrack.detach(videoElement); + } catch (error) { + logger.error('Error during cleanup:', error); + } + } + }; + }, [ videoTrack, shouldShowAvatar, canvasStream, videoElement ]); + + /** + * Effect: Window blur/focus and visibility change listeners. + * Enters PiP on blur, exits on focus (matches old AOT behavior). + */ + useEffect(() => { + if (!videoElement) { + return; + } + + const onWindowBlur = () => dispatch(handleWindowBlur(videoElement)); + const onWindowFocus = () => { + + // In the use case where the PiP is closed by the 'X' or 'back to main window' buttons, this handler is + // called before the leavepictureinpicture handler. From there we call document.exitPictureInPicture() + // which seems to put Chrome into a weird state - document.exitPictureInPicture() never resolves, the + // leavepictureinpicture is never triggered and it is not possible to display PiP again. + // This is probably a browser bug. To workaround it we have the 100ms timeout here. This way this event + // is triggered after the leavepictureinpicture event and everythink seems to work well. + setTimeout(() => { + dispatch(handleWindowFocus()); + }, 100); + }; + const onVisibilityChange = () => { + if (document.hidden) { + onWindowBlur(); + } + }; + + window.addEventListener('blur', onWindowBlur); + window.addEventListener('focus', onWindowFocus); + document.addEventListener('visibilitychange', onVisibilityChange); + + return () => { + window.removeEventListener('blur', onWindowBlur); + window.removeEventListener('focus', onWindowFocus); + document.removeEventListener('visibilitychange', onVisibilityChange); + }; + }, [ dispatch, videoElement ]); + + /** + * Effect: PiP enter/leave event listeners. + * Updates Redux state when browser PiP events occur. + */ + useEffect(() => { + if (!videoElement) { + return; + } + + const onEnterPiP = () => { + dispatch(handlePipEnterEvent()); + }; + const onLeavePiP = () => { + dispatch(handlePiPLeaveEvent()); + }; + + videoElement.addEventListener('enterpictureinpicture', onEnterPiP); + videoElement.addEventListener('leavepictureinpicture', onLeavePiP); + + return () => { + videoElement.removeEventListener('enterpictureinpicture', onEnterPiP); + videoElement.removeEventListener('leavepictureinpicture', onLeavePiP); + }; + }, [ videoElement, dispatch ]); + + return ( +