ZF/jitsi-meet
1
0
Fork 0
mirror of https://gitcode.com/GitHub_Trending/ji/jitsi-meet.git synced 2025-12-30 03:12:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

ref(transcriptions): refactor transcriptions api (#14144)

Browse Source 
* ref(transcriptions): refactor transcriptions api

* ref(transcriptions): refactor usage of translation label

Extend IFrame API to allow adding a transcriber in the room without the subtitles needing to be visible.
Allow transcription chunk messages to be passed through the IFrame API if a transcriber is present.
Clean-up transcription messages sent through the IFrame API to not include timeout field and possible conflicting states (stable / unstable /final)

* fix linting

* code review: extend api message to match webhook format
This commit is contained in:
Avram Tudor
2023-12-12 14:36:54 +02:00
committed by GitHub
parent 4c6cadea6d
commit 72b4c8123a
13 changed files with 105 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modules/API/API.js
View File

@@ -466,8 +466,8 @@ function initCommands() {
'toggle-subtitles': () => {
APP.store.dispatch(toggleRequestingSubtitles());
},
'set-subtitles': enabled => {
APP.store.dispatch(setRequestingSubtitles(enabled));
'set-subtitles': (enabled, displaySubtitles, language) => {
APP.store.dispatch(setRequestingSubtitles(enabled, displaySubtitles, language));
},
'toggle-tile-view': () => {
sendAnalytics(createApiEvent('tile-view.toggled'));

16
react/features/base/i18n/i18next.ts
View File

@@ -40,14 +40,6 @@ export const LANGUAGES: Array<string> = Object.keys(LANGUAGES_RESOURCES);
*/
export const TRANSLATION_LANGUAGES: Array<string> = Object.keys(TRANSLATION_LANGUAGES_RESOURCES);
/**
* The available/supported translation languages head. (Languages displayed on the top ).
*
* @public
* @type {Array<string>}
*/
export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ 'en' ];
/**
* The default language.
*
@@ -58,6 +50,14 @@ export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ 'en' ];
*/
export const DEFAULT_LANGUAGE = 'en';
/**
* The available/supported translation languages head. (Languages displayed on the top ).
*
* @public
* @type {Array<string>}
*/
export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ DEFAULT_LANGUAGE ];
/**
* The options to initialize i18next with.
*

7
react/features/mobile/external-api/middleware.ts
View File

@@ -379,9 +379,10 @@ function _registerForNativeEvents(store: IStore) {
dispatch(sendMessage(message));
});
eventEmitter.addListener(ExternalAPI.SET_CLOSED_CAPTIONS_ENABLED, ({ enabled }: any) => {
dispatch(setRequestingSubtitles(enabled));
});
eventEmitter.addListener(ExternalAPI.SET_CLOSED_CAPTIONS_ENABLED,
({ enabled, displaySubtitles, language }: any) => {
dispatch(setRequestingSubtitles(enabled, displaySubtitles, language));
});
eventEmitter.addListener(ExternalAPI.TOGGLE_CAMERA, () => {
dispatch(toggleCameraFacingMode());

12
react/features/subtitles/actionTypes.ts
View File

@@ -34,18 +34,6 @@ export const REMOVE_TRANSCRIPT_MESSAGE = 'REMOVE_TRANSCRIPT_MESSAGE';
*/
export const UPDATE_TRANSCRIPT_MESSAGE = 'UPDATE_TRANSCRIPT_MESSAGE';
/**
* The type of (redux) action which indicates that a transcript with an
* given message_id to be added or updated is received.
*
* {
* type: UPDATE_TRANSLATION_LANGUAGE,
* transcriptMessageID: string,
* newTranscriptMessage: Object
* }
*/
export const UPDATE_TRANSLATION_LANGUAGE = 'UPDATE_TRANSLATION_LANGUAGE';
/**
* The type of (redux) action which indicates that the user pressed the
* ClosedCaption button, to either enable or disable subtitles based on the

35
react/features/subtitles/actions.any.ts
View File

@@ -1,10 +1,11 @@
import { DEFAULT_LANGUAGE } from '../base/i18n/i18next';
import {
ENDPOINT_MESSAGE_RECEIVED,
REMOVE_TRANSCRIPT_MESSAGE,
SET_REQUESTING_SUBTITLES,
TOGGLE_REQUESTING_SUBTITLES,
UPDATE_TRANSCRIPT_MESSAGE,
UPDATE_TRANSLATION_LANGUAGE
UPDATE_TRANSCRIPT_MESSAGE
} from './actionTypes';
/**
@@ -80,29 +81,23 @@ export function toggleRequestingSubtitles() {
* Signals that the local user has enabled or disabled the subtitles.
*
* @param {boolean} enabled - The new state of the subtitles.
* @param {boolean} displaySubtitles - Whether to display subtitles or not.
* @param {string} language - The language of the subtitles.
* @returns {{
* type: SET_REQUESTING_SUBTITLES,
* enabled: boolean
* enabled: boolean,
* displaySubtitles: boolean,
* language: string
* }}
*/
export function setRequestingSubtitles(enabled: boolean) {
export function setRequestingSubtitles(
enabled: boolean,
displaySubtitles = true,
language: string | null = DEFAULT_LANGUAGE) {
return {
type: SET_REQUESTING_SUBTITLES,
enabled
};
}
/**
* Signals that the local user has selected language for the translation.
*
* @param {string} value - The selected language for translation.
* @returns {{
* type: UPDATE_TRANSLATION_LANGUAGE
* }}
*/
export function updateTranslationLanguage(value: string) {
return {
type: UPDATE_TRANSLATION_LANGUAGE,
value
displaySubtitles,
enabled,
language: `translation-languages:${language}`
};
}

4
react/features/subtitles/actions.web.ts
View File

@@ -8,9 +8,7 @@ export * from './actions.any';
/**
* Signals that the local user has toggled the LanguageSelector button.
*
* @returns {{
* type: UPDATE_TRANSLATION_LANGUAGE
* }}
* @returns {Function}
*/
export function toggleLanguageSelectorDialog() {
return function(dispatch: IStore['dispatch']) {

16
react/features/subtitles/components/AbstractCaptions.tsx
View File

@@ -9,7 +9,12 @@ import { IReduxState } from '../../app/types';
export interface IAbstractCaptionsProps {
/**
* Whether local participant is requesting to see subtitles.
* Whether local participant is displaying subtitles.
*/
_displaySubtitles: boolean;
/**
* Whether local participant is requesting subtitles.
*/
_requestingSubtitles: boolean;
@@ -34,9 +39,9 @@ export class AbstractCaptions<P extends IAbstractCaptionsProps> extends Componen
* @returns {ReactElement}
*/
render(): any {
const { _requestingSubtitles, _transcripts } = this.props;
const { _displaySubtitles, _requestingSubtitles, _transcripts } = this.props;
if (!_requestingSubtitles || !_transcripts || !_transcripts.size) {
if (!_requestingSubtitles || !_displaySubtitles || !_transcripts || !_transcripts.size) {
return null;
}
@@ -95,7 +100,7 @@ function _constructTranscripts(state: IReduxState): Map<string, string> {
for (const [ id, transcriptMessage ] of _transcriptMessages) {
if (transcriptMessage) {
let text = `${transcriptMessage.participantName}: `;
let text = `${transcriptMessage.participant.name}: `;
if (transcriptMessage.final) {
text += transcriptMessage.final;
@@ -125,10 +130,11 @@ function _constructTranscripts(state: IReduxState): Map<string, string> {
* }}
*/
export function _abstractMapStateToProps(state: IReduxState) {
const { _requestingSubtitles } = state['features/subtitles'];
const { _displaySubtitles, _requestingSubtitles } = state['features/subtitles'];
const transcripts = _constructTranscripts(state);
return {
_displaySubtitles,
_requestingSubtitles,
// avoid re-renders by setting to prop new empty Map instances.

2
react/features/subtitles/components/AbstractClosedCaptionButton.tsx
View File

@@ -8,7 +8,7 @@ import { maybeShowPremiumFeatureDialog } from '../../jaas/actions';
export interface IAbstractProps extends AbstractButtonProps {
_language: string;
_language: string | null;
/**
* Whether the local participant is currently requesting subtitles.

24
react/features/subtitles/components/AbstractLanguageSelectorDialog.tsx
View File

@@ -1,4 +1,4 @@
import React, { ComponentType, useCallback, useEffect, useState } from 'react';
import React, { ComponentType, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
@@ -7,12 +7,12 @@ import {
TRANSLATION_LANGUAGES,
TRANSLATION_LANGUAGES_HEAD
} from '../../base/i18n/i18next';
import { setRequestingSubtitles, updateTranslationLanguage } from '../actions.any';
import { setRequestingSubtitles } from '../actions.any';
export interface IAbstractLanguageSelectorDialogProps {
dispatch: IStore['dispatch'];
language: string;
language: string | null;
listItems: Array<any>;
onLanguageSelected: (e: string) => void;
subtitles: string;
@@ -30,10 +30,10 @@ export interface IAbstractLanguageSelectorDialogProps {
const AbstractLanguageSelectorDialog = (Component: ComponentType<IAbstractLanguageSelectorDialogProps>) => () => {
const dispatch = useDispatch();
const { t } = useTranslation();
const off = 'transcribing.subtitlesOff';
const noLanguageLabel = 'transcribing.subtitlesOff';
const [ subtitles, setSubtiles ] = useState(off);
const language = useSelector((state: IReduxState) => state['features/subtitles']._language);
const subtitles = language ?? noLanguageLabel;
const transcription = useSelector((state: IReduxState) => state['features/base/config'].transcription);
const translationLanguagesHead = transcription?.translationLanguagesHead ?? TRANSLATION_LANGUAGES_HEAD;
@@ -42,7 +42,7 @@ const AbstractLanguageSelectorDialog = (Component: ComponentType<IAbstractLangua
// The off and the head languages are always on the top of the list. But once you are selecting
// a language from the translationLanguages, that language is moved under the fixedItems list,
// until a new languages is selected. FixedItems keep their positions.
const fixedItems = [ off, ...languagesHead ];
const fixedItems = [ noLanguageLabel, ...languagesHead ];
const translationLanguages = transcription?.translationLanguages ?? TRANSLATION_LANGUAGES;
const languages = translationLanguages
.map((lang: string) => `translation-languages:${lang}`)
@@ -58,14 +58,12 @@ const AbstractLanguageSelectorDialog = (Component: ComponentType<IAbstractLangua
};
});
useEffect(() => {
language ? setSubtiles(language) : setSubtiles(off);
}, []);
const onLanguageSelected = useCallback((value: string) => {
const selectedLanguage = value === noLanguageLabel ? null : value;
const enabled = Boolean(selectedLanguage);
const displaySubtitles = enabled;
const onLanguageSelected = useCallback((e: string) => {
setSubtiles(e);
dispatch(updateTranslationLanguage(e));
dispatch(setRequestingSubtitles(e !== off));
dispatch(setRequestingSubtitles(enabled, displaySubtitles, selectedLanguage));
}, [ language ]);
return (

2
react/features/subtitles/components/native/ClosedCaptionButton.tsx
View File

@@ -22,7 +22,7 @@ class ClosedCaptionButton
icon = IconSubtitles;
label = 'toolbar.startSubtitles';
labelProps = {
language: this.props.t(this.props._language),
language: this.props.t(this.props._language ?? 'transcribing.subtitlesOff'),
languages: this.props.t(this.props.languages ?? ''),
languagesHead: this.props.t(this.props.languagesHead ?? '')
};

2
react/features/subtitles/components/web/ClosedCaptionButton.tsx
View File

@@ -18,7 +18,7 @@ class ClosedCaptionButton
tooltip = 'transcribing.ccButtonTooltip';
label = 'toolbar.startSubtitles';
labelProps = {
language: this.props.t(this.props._language),
language: this.props.t(this.props._language ?? 'transcribing.subtitlesOff'),
languages: this.props.t(this.props.languages ?? ''),
languagesHead: this.props.t(this.props.languagesHead ?? '')
};

81
react/features/subtitles/middleware.ts
View File

@@ -55,12 +55,15 @@ MiddlewareRegistry.register(store => next => action => {
case ENDPOINT_MESSAGE_RECEIVED:
return _endpointMessageReceived(store, next, action);
case TOGGLE_REQUESTING_SUBTITLES:
_requestingSubtitlesChange(store);
case TOGGLE_REQUESTING_SUBTITLES: {
const state = store.getState()['features/subtitles'];
const toggledValue = !state._requestingSubtitles;
_requestingSubtitlesChange(store, toggledValue, state._language);
break;
}
case SET_REQUESTING_SUBTITLES:
_requestingSubtitlesChange(store);
_requestingSubtitlesSet(store, action.enabled);
_requestingSubtitlesChange(store, action.enabled, action.language);
break;
}
@@ -91,23 +94,28 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
}
const state = getState();
const translationLanguage
const language
= state['features/base/conference'].conference
?.getLocalParticipantProperty(P_NAME_TRANSLATION_LANGUAGE);
try {
const transcriptMessageID = json.message_id;
const participantName = json.participant.name;
const { name, id, avatar_url: avatarUrl } = json.participant;
const participant = {
avatarUrl,
id,
name
};
if (json.type === JSON_TYPE_TRANSLATION_RESULT
&& json.language === translationLanguage) {
&& json.language === language) {
// Displays final results in the target language if translation is
// enabled.
const newTranscriptMessage = {
clearTimeOut: undefined,
final: json.text,
participantName
participant
};
_setClearerOnTranscriptMessage(dispatch,
@@ -115,8 +123,7 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
} else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT
&& json.language.slice(0, 2) === translationLanguage) {
} else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT && json.language.slice(0, 2) === language) {
// Displays interim and final results without any translation if
// translations are disabled.
@@ -125,10 +132,11 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
// We update the previous transcript message with the same
// message ID or adds a new transcript message if it does not
// exist in the map.
const existingMessage = state['features/subtitles']._transcriptMessages.get(transcriptMessageID);
const newTranscriptMessage: any = {
...state['features/subtitles']._transcriptMessages
.get(transcriptMessageID)
|| { participantName }
clearTimeOut: existingMessage?.clearTimeOut,
language,
participant
};
_setClearerOnTranscriptMessage(dispatch,
@@ -144,7 +152,6 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
// stable field of the state and remove the previously
// unstable results
newTranscriptMessage.stable = text;
newTranscriptMessage.unstable = undefined;
} else {
// Otherwise, this result has an unstable result, which we
@@ -157,9 +164,13 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
// Notify the external API too.
if (typeof APP !== 'undefined') {
const sanitizedTranscriptMessage = { ...newTranscriptMessage };
delete sanitizedTranscriptMessage.clearTimeOut;
APP.API.notifyTranscriptionChunkReceived({
messageID: transcriptMessageID,
...newTranscriptMessage
...sanitizedTranscriptMessage
});
}
}
@@ -175,43 +186,27 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
* and Jigasi to decide whether the transcriber needs to be in the room.
*
* @param {Store} store - The redux store.
* @param {boolean} enabled - Whether subtitles should be enabled or not.
* @param {string} language - The language to use for translation.
* @private
* @returns {void}
*/
function _requestingSubtitlesChange({ getState }: IStore) {
const state = getState();
const { _language } = state['features/subtitles'];
const { conference } = state['features/base/conference'];
const requestingSubtitles = _language !== 'transcribing.subtitlesOff';
conference?.setLocalParticipantProperty(
P_NAME_REQUESTING_TRANSCRIPTION,
requestingSubtitles);
if (requestingSubtitles) {
conference?.setLocalParticipantProperty(
P_NAME_TRANSLATION_LANGUAGE,
_language.replace('translation-languages:', ''));
}
}
/**
* Set the local property 'requestingTranscription'. This will cause Jicofo
* and Jigasi to decide whether the transcriber needs to be in the room.
*
* @param {Store} store - The redux store.
* @param {boolean} enabled - The new state of the subtitles.
* @private
* @returns {void}
*/
function _requestingSubtitlesSet({ getState }: IStore, enabled: boolean) {
function _requestingSubtitlesChange(
{ getState }: IStore,
enabled: boolean,
language?: string | null) {
const state = getState();
const { conference } = state['features/base/conference'];
conference?.setLocalParticipantProperty(
P_NAME_REQUESTING_TRANSCRIPTION,
enabled);
if (enabled && language) {
conference?.setLocalParticipantProperty(
P_NAME_TRANSLATION_LANGUAGE,
language.replace('translation-languages:', ''));
}
}
/**

20
react/features/subtitles/reducer.ts
View File

@@ -2,16 +2,17 @@ import ReducerRegistry from '../base/redux/ReducerRegistry';
import {
REMOVE_TRANSCRIPT_MESSAGE,
SET_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE, UPDATE_TRANSLATION_LANGUAGE
SET_REQUESTING_SUBTITLES, TOGGLE_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE
} from './actionTypes';
/**
* Default State for 'features/transcription' feature.
*/
const defaultState = {
_displaySubtitles: true,
_transcriptMessages: new Map(),
_requestingSubtitles: false,
_language: 'transcribing.subtitlesOff'
_language: null
};
interface ITranscriptMessage {
@@ -22,7 +23,8 @@ interface ITranscriptMessage {
}
export interface ISubtitlesState {
_language: string;
_displaySubtitles: boolean;
_language: string | null;
_requestingSubtitles: boolean;
_transcriptMessages: Map<string, ITranscriptMessage> | any;
}
@@ -38,16 +40,18 @@ ReducerRegistry.register<ISubtitlesState>('features/subtitles', (
return _removeTranscriptMessage(state, action);
case UPDATE_TRANSCRIPT_MESSAGE:
return _updateTranscriptMessage(state, action);
case UPDATE_TRANSLATION_LANGUAGE:
return {
...state,
_language: action.value
};
case SET_REQUESTING_SUBTITLES:
return {
...state,
_displaySubtitles: action.displaySubtitles,
_language: action.language,
_requestingSubtitles: action.enabled
};
case TOGGLE_REQUESTING_SUBTITLES:
return {
...state,
_requestingSubtitles: !state._requestingSubtitles
};
}
return state;