ZF/jitsi-meet
1
0
Fork 0
mirror of https://gitcode.com/GitHub_Trending/ji/jitsi-meet.git synced 2025-12-30 03:12:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ios/android/sdk): delegate transcription chunk received event (#14516)

Browse Source 
* feat(subtitles): created separate helpers for delegating transcription chunks events 
on web and mobile
This commit is contained in:
Calinteodor
2024-03-28 14:10:02 +02:00
committed by GitHub
parent 47598222ce
commit 5247fbdc0f
12 changed files with 106 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
android/sdk/src/main/java/org/jitsi/meet/sdk/BroadcastEvent.java
View File

@@ -89,7 +89,8 @@ public class BroadcastEvent {
CHAT_MESSAGE_RECEIVED("org.jitsi.meet.CHAT_MESSAGE_RECEIVED"),
CHAT_TOGGLED("org.jitsi.meet.CHAT_TOGGLED"),
VIDEO_MUTED_CHANGED("org.jitsi.meet.VIDEO_MUTED_CHANGED"),
READY_TO_CLOSE("org.jitsi.meet.READY_TO_CLOSE");
READY_TO_CLOSE("org.jitsi.meet.READY_TO_CLOSE"),
TRANSCRIPTION_CHUNK_RECEIVED("org.jitsi.meet.TRANSCRIPTION_CHUNK_RECEIVED");
private static final String CONFERENCE_BLURRED_NAME = "CONFERENCE_BLURRED";
private static final String CONFERENCE_FOCUSED_NAME = "CONFERENCE_FOCUSED";
@@ -106,6 +107,7 @@ public class BroadcastEvent {
private static final String CHAT_TOGGLED_NAME = "CHAT_TOGGLED";
private static final String VIDEO_MUTED_CHANGED_NAME = "VIDEO_MUTED_CHANGED";
private static final String READY_TO_CLOSE_NAME = "READY_TO_CLOSE";
private static final String TRANSCRIPTION_CHUNK_RECEIVED_NAME = "TRANSCRIPTION_CHUNK_RECEIVED";
private final String action;
@@ -158,6 +160,8 @@ public class BroadcastEvent {
return VIDEO_MUTED_CHANGED;
case READY_TO_CLOSE_NAME:
return READY_TO_CLOSE;
case TRANSCRIPTION_CHUNK_RECEIVED_NAME:
return TRANSCRIPTION_CHUNK_RECEIVED;
}
return null;

7
android/sdk/src/main/java/org/jitsi/meet/sdk/JitsiMeetActivity.java
View File

@@ -255,6 +255,10 @@ public class JitsiMeetActivity extends AppCompatActivity
finish();
}
// protected void onTranscriptionChunkReceived(HashMap<String, Object> extraData) {
// JitsiMeetLogger.i("Transcription chunk received: " + extraData);
// }
// Activity lifecycle methods
//
@@ -338,6 +342,9 @@ public class JitsiMeetActivity extends AppCompatActivity
case READY_TO_CLOSE:
onReadyToClose();
break;
// case TRANSCRIPTION_CHUNK_RECEIVED:
// onTranscriptionChunkReceived(event.getData());
// break;
}
}
}

5
ios/app/src/ViewController.m
View File

@@ -98,6 +98,10 @@
[self _onJitsiMeetViewDelegateEvent:@"READY_TO_CLOSE" withData:data];
}
// - (void)transcriptionChunkReceived:(NSDictionary *)data {
// [self _onJitsiMeetViewDelegateEvent:@"TRANSCRIPTION_CHUNK_RECEIVED" withData:data];
// }
- (void)participantJoined:(NSDictionary *)data {
NSLog(@"%@%@", @"Participant joined: ", data[@"participantId"]);
}
@@ -130,6 +134,7 @@
NSLog(@"%@%@", @"Video muted changed: ", data[@"muted"]);
}
#pragma mark - Helpers
- (void)terminate {

2
ios/sdk/src/JitsiMeetView.h
View File

@@ -26,7 +26,7 @@
@property (nonatomic, nullable, weak) id<JitsiMeetViewDelegate> delegate;
/**
* Joins the conference specified by the given options. The gievn options will
* Joins the conference specified by the given options. The given options will
* be merged with the defaultConferenceOptions (if set) in JitsiMeet. If there
* is an already active conference it will be automatically left prior to
* joining the new one.

7
ios/sdk/src/JitsiMeetViewDelegate.h
View File

@@ -116,4 +116,11 @@
*/
- (void)readyToClose:(NSDictionary *)data;
/**
* Called when the transcription chunk was received.
*
* The `data` dictionary contains a `messageID`, `language`, `participant` key.
*/
- (void)transcriptionChunkReceived:(NSDictionary *)data;
@end

2
react/features/mobile/external-api/middleware.ts
View File

@@ -86,7 +86,7 @@ const CONFERENCE_TERMINATED = 'CONFERENCE_TERMINATED';
const ENDPOINT_TEXT_MESSAGE_RECEIVED = 'ENDPOINT_TEXT_MESSAGE_RECEIVED';
/**
* Event which will be emitted on the native side to indicate a participant togggles
* Event which will be emitted on the native side to indicate a participant toggles
* the screen share.
*/
const SCREEN_SHARE_TOGGLED = 'SCREEN_SHARE_TOGGLED';

2
react/features/subtitles/components/AbstractClosedCaptionButton.tsx
View File

@@ -4,7 +4,7 @@ import { IReduxState } from '../../app/types';
import { MEET_FEATURES } from '../../base/jwt/constants';
import AbstractButton, { IProps as AbstractButtonProps } from '../../base/toolbox/components/AbstractButton';
import { maybeShowPremiumFeatureDialog } from '../../jaas/actions';
import { canStartSubtitles } from '../functions';
import { canStartSubtitles } from '../functions.any';
export interface IAbstractProps extends AbstractButtonProps {

0
react/features/subtitles/functions.ts → react/features/subtitles/functions.any.ts
View File

30
react/features/subtitles/functions.native.ts Normal file
View File

@@ -0,0 +1,30 @@
/* eslint-disable max-params, max-len */
import { sendEvent } from '../mobile/external-api/functions';
/**
* Event which will be emitted on the native side to indicate that the transcription chunk was received.
*/
const TRANSCRIPTION_CHUNK_RECEIVED = 'TRANSCRIPTION_CHUNK_RECEIVED';
/**
* Logs when about the received transcription chunk.
*
* @param {string} transcriptMessageID - Transcription message id.
* @param {string} language - The language of the transcribed message.
* @param {Object} participant - The participant who send the message.
* @param {any} text - The message text.
* @param {any} _store - The store.
* @returns {Event}
*/
export const notifyTranscriptionChunkReceived = (transcriptMessageID: string, language: string, participant: Object, text: any, _store?: any) =>
sendEvent(
_store,
TRANSCRIPTION_CHUNK_RECEIVED,
{
messageID: transcriptMessageID,
language,
participant,
text
});

19
react/features/subtitles/functions.web.ts Normal file
View File

@@ -0,0 +1,19 @@
/* eslint-disable max-params, max-len */
/**
* Logs when about the received transcription chunk.
*
* @param {string} transcriptMessageID - Transcription message id.
* @param {string} language - The language of the transcribed message.
* @param {Object} participant - The participant who send the message.
* @param {any} text - The message text.
* @param {any} _store - The store.
* @returns {Event}
*/
export const notifyTranscriptionChunkReceived = (transcriptMessageID: string, language: string, participant: Object, text: any, _store?: any) =>
APP.API.notifyTranscriptionChunkReceived({
messageID: transcriptMessageID,
language,
participant,
...text
});

46
react/features/subtitles/middleware.ts
View File

@@ -12,6 +12,8 @@ import {
removeTranscriptMessage,
updateTranscriptMessage
} from './actions.any';
import { notifyTranscriptionChunkReceived } from './functions';
/**
* The type of json-message which indicates that json carries a
@@ -43,7 +45,7 @@ const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';
const REMOVE_AFTER_MS = 3000;
/**
* Stability factor for a trancription. We'll treat a transcript as stable
* Stability factor for a transcription. We'll treat a transcript as stable
* beyond this value.
*/
const STABLE_TRANSCRIPTION_FACTOR = 0.85;
@@ -89,13 +91,14 @@ MiddlewareRegistry.register(store => next => action => {
* @private
* @returns {Object} The value returned by {@code next(action)}.
*/
function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function, action: AnyAction) {
function _endpointMessageReceived(store: IStore, next: Function, action: AnyAction) {
const { data: json } = action;
if (![ JSON_TYPE_TRANSCRIPTION_RESULT, JSON_TYPE_TRANSLATION_RESULT ].includes(json?.type)) {
return next(action);
}
const { dispatch, getState } = store;
const state = getState();
const language
= state['features/base/conference'].conference
@@ -129,7 +132,7 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
const { text } = json.transcript[0];
// First, notify the external API.
if (typeof APP !== 'undefined' && !(json.is_interim && skipInterimTranscriptions)) {
if (!(json.is_interim && skipInterimTranscriptions)) {
const txt: any = {};
if (!json.is_interim) {
@@ -140,26 +143,31 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
txt.unstable = text;
}
APP.API.notifyTranscriptionChunkReceived({
messageID: transcriptMessageID,
language: json.language,
notifyTranscriptionChunkReceived(
transcriptMessageID,
json.language,
participant,
...txt
});
txt,
store
);
// Dump transcript in a <transcript> element for debugging purposes.
if (!json.is_interim && dumpTranscript) {
try {
let elem = document.body.getElementsByTagName('transcript')[0];
if (navigator.product !== 'ReactNative') {
if (!elem) {
elem = document.createElement('transcript');
document.body.appendChild(elem);
// Dump transcript in a <transcript> element for debugging purposes.
if (!json.is_interim && dumpTranscript) {
try {
let elem = document.body.getElementsByTagName('transcript')[0];
// eslint-disable-next-line max-depth
if (!elem) {
elem = document.createElement('transcript');
document.body.appendChild(elem);
}
elem.append(`${new Date(json.timestamp).toISOString()} ${participant.name}: ${text}`);
} catch (_) {
// Ignored.
}
elem.append(`${new Date(json.timestamp).toISOString()} ${participant.name}: ${text}`);
} catch (_) {
// Ignored.
}
}
}

4
react/features/subtitles/reducer.ts
View File

@@ -2,7 +2,9 @@ import ReducerRegistry from '../base/redux/ReducerRegistry';
import {
REMOVE_TRANSCRIPT_MESSAGE,
SET_REQUESTING_SUBTITLES, TOGGLE_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE
SET_REQUESTING_SUBTITLES,
TOGGLE_REQUESTING_SUBTITLES,
UPDATE_TRANSCRIPT_MESSAGE
} from './actionTypes';
/**