init

react/features/screenshot-capture/CameraCaptureDialog.tsx

@@ -0,0 +1,154 @@
+import { Theme } from '@mui/material';
+import React, { useCallback, useRef } from 'react';
+import { WithTranslation } from 'react-i18next';
+import { connect, useDispatch } from 'react-redux';
+import { makeStyles } from 'tss-react/mui';
+
+import { hideDialog } from '../base/dialog/actions';
+import { translate } from '../base/i18n/functions';
+import Label from '../base/label/components/web/Label';
+import { CAMERA_FACING_MODE } from '../base/media/constants';
+import Button from '../base/ui/components/web/Button';
+import Dialog from '../base/ui/components/web/Dialog';
+import { BUTTON_TYPES } from '../base/ui/constants.any';
+
+import { ICameraCapturePayload } from './actionTypes';
+
+const useStyles = makeStyles()((theme: Theme) => ({
+    container: {
+        display: 'flex',
+        height: '100%',
+        width: '100%',
+        justifyContent: 'center',
+        alignItems: 'center',
+        flexDirection: 'column',
+        gap: theme.spacing(3),
+        textAlign: 'center'
+    },
+    buttonsContainer: {
+        display: 'flex',
+        flexDirection: 'column',
+        gap: theme.spacing(3),
+        width: '100%',
+        maxWidth: '100%'
+    },
+
+    hidden: {
+        display: 'none'
+    },
+    label: {
+        background: 'transparent',
+        margin: `${theme.spacing(3)} 0 ${theme.spacing(6)}`,
+    },
+    button: {
+        width: '100%',
+        height: '48px',
+        maxWidth: '400px'
+    }
+}));
+
+/**
+ * The type of {@link CameraCaptureDialog}'s React {@code Component} props.
+ */
+interface IProps extends WithTranslation {
+    /**
+     * Callback function on file input changed.
+     */
+    callback: ({ error, dataURL }: { dataURL?: string; error?: string; }) => void;
+
+    /**
+     * The camera capture payload.
+     */
+    componentProps: ICameraCapturePayload;
+}
+
+/**
+ * Implements the Camera capture dialog.
+ *
+ * @param {Object} props - The props of the component.
+ * @returns {React$Element}
+ */
+const CameraCaptureDialog = ({
+    callback,
+    componentProps,
+    t,
+}: IProps) => {
+    const { cameraFacingMode = CAMERA_FACING_MODE.ENVIRONMENT,
+        descriptionText,
+        titleText } = componentProps;
+    const dispatch = useDispatch();
+    const { classes } = useStyles();
+    const inputRef = useRef<HTMLInputElement>(null);
+    const onCancel = useCallback(() => {
+        callback({
+            error: 'User canceled!'
+        });
+        dispatch(hideDialog());
+    }, []);
+
+    const onSubmit = useCallback(() => {
+        inputRef.current?.click();
+    }, []);
+
+    const onInputChange = useCallback(event => {
+        const reader = new FileReader();
+        const files = event.target.files;
+
+        if (!files?.[0]) {
+            callback({
+                error: 'No picture selected!'
+            });
+
+            return;
+        }
+
+        reader.onload = () => {
+            callback({ dataURL: reader.result as string });
+            dispatch(hideDialog());
+        };
+        reader.onerror = () => {
+            callback({ error: 'Failed generating base64 URL!' });
+            dispatch(hideDialog());
+        };
+
+        reader.readAsDataURL(files[0]);
+    }, []);
+
+    return (
+        <Dialog
+            cancel = {{ hidden: true }}
+            disableAutoHideOnSubmit = { true }
+            ok = {{ hidden: true }}
+            onCancel = { onCancel }
+            titleKey = { titleText || t('dialog.cameraCaptureDialog.title') }>
+            <div className = { classes.container }>
+                <Label
+                    aria-label = { descriptionText || t('dialog.cameraCaptureDialog.description') }
+                    className = { classes.label }
+                    text = { descriptionText || t('dialog.cameraCaptureDialog.description') } />
+                <div className = { classes.buttonsContainer } >
+                    <Button
+                        accessibilityLabel = { t('dialog.cameraCaptureDialog.ok') }
+                        className = { classes.button }
+                        labelKey = { 'dialog.cameraCaptureDialog.ok' }
+                        onClick = { onSubmit } />
+                    <Button
+                        accessibilityLabel = { t('dialog.cameraCaptureDialog.reject') }
+                        className = { classes.button }
+                        labelKey = { 'dialog.cameraCaptureDialog.reject' }
+                        onClick = { onCancel }
+                        type = { BUTTON_TYPES.TERTIARY } />
+                    <input
+                        accept = 'image/*'
+                        capture = { cameraFacingMode }
+                        className = { classes.hidden }
+                        onChange = { onInputChange }
+                        ref = { inputRef }
+                        type = 'file' />
+                </div>
+            </div>
+        </Dialog>
+    );
+};
+
+export default translate(connect()(CameraCaptureDialog));

react/features/screenshot-capture/ScreenshotCaptureSummary.tsx

@@ -0,0 +1,214 @@
+import 'image-capture';
+
+import { createScreensharingCaptureTakenEvent } from '../analytics/AnalyticsEvents';
+import { sendAnalytics } from '../analytics/functions';
+import { IReduxState } from '../app/types';
+import { getCurrentConference } from '../base/conference/functions';
+import { getLocalParticipant, getRemoteParticipants } from '../base/participants/functions';
+import { getBaseUrl } from '../base/util/helpers';
+import { extractFqnFromPath } from '../dynamic-branding/functions.any';
+
+import {
+    CLEAR_TIMEOUT,
+    POLL_INTERVAL,
+    SCREENSHOT_QUEUE_LIMIT,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
+} from './constants';
+import logger from './logger';
+// eslint-disable-next-line lines-around-comment
+// @ts-ignore
+import { processScreenshot } from './processScreenshot';
+
+declare let ImageCapture: any;
+
+/**
+ * Effect that wraps {@code MediaStream} adding periodic screenshot captures.
+ * Manipulates the original desktop stream and performs custom processing operations, if implemented.
+ */
+export default class ScreenshotCaptureSummary {
+    _state: IReduxState;
+    _initializedRegion: boolean;
+    _imageCapture: ImageCapture;
+    _streamWorker: Worker;
+    _queue: Blob[];
+
+    /**
+     * Initializes a new {@code ScreenshotCaptureEffect} instance.
+     *
+     * @param {Object} state - The redux state.
+     */
+    constructor(state: IReduxState) {
+        this._state = state;
+
+        // Bind handlers such that they access the same instance.
+        this._handleWorkerAction = this._handleWorkerAction.bind(this);
+        const baseUrl = `${getBaseUrl()}libs/`;
+
+        let workerUrl = `${baseUrl}screenshot-capture-worker.min.js`;
+
+        // @ts-ignore
+        const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });
+
+        // @ts-ignore
+        workerUrl = window.URL.createObjectURL(workerBlob);
+        this._streamWorker = new Worker(workerUrl, { name: 'Screenshot capture worker' });
+        this._streamWorker.onmessage = this._handleWorkerAction;
+
+        this._initializedRegion = false;
+        this._queue = [];
+    }
+
+    /**
+     * Make a call to backend for region selection.
+     *
+     * @returns {void}
+     */
+    async _initRegionSelection() {
+        const { _screenshotHistoryRegionUrl } = this._state['features/base/config'];
+        const conference = getCurrentConference(this._state);
+        const sessionId = conference?.getMeetingUniqueId();
+        const { jwt } = this._state['features/base/jwt'];
+
+        if (!_screenshotHistoryRegionUrl) {
+            return;
+        }
+
+        const headers = {
+            ...jwt && { 'Authorization': `Bearer ${jwt}` }
+        };
+
+        try {
+            await fetch(`${_screenshotHistoryRegionUrl}/${sessionId}`, {
+                method: 'POST',
+                headers
+            });
+        } catch (err) {
+            logger.warn(`Could not create screenshot region: ${err}`);
+
+            return;
+        }
+
+
+        this._initializedRegion = true;
+    }
+
+    /**
+     * Starts the screenshot capture event on a loop.
+     *
+     * @param {JitsiTrack} jitsiTrack - The track that contains the stream from which screenshots are to be sent.
+     * @returns {Promise} - Promise that resolves once effect has started or rejects if the
+     * videoType parameter is not desktop.
+     */
+    async start(jitsiTrack: any) {
+        if (!window.OffscreenCanvas) {
+            logger.warn('Can\'t start screenshot capture, OffscreenCanvas is not available');
+
+            return;
+        }
+        const { videoType, track } = jitsiTrack;
+
+        if (videoType !== 'desktop') {
+            return;
+        }
+        this._imageCapture = new ImageCapture(track);
+
+        if (!this._initializedRegion) {
+            await this._initRegionSelection();
+        }
+        this.sendTimeout();
+    }
+
+    /**
+     * Stops the ongoing {@code ScreenshotCaptureEffect} by clearing the {@code Worker} interval.
+     *
+     * @returns {void}
+     */
+    stop() {
+        this._streamWorker.postMessage({ id: CLEAR_TIMEOUT });
+    }
+
+    /**
+     * Sends to worker the imageBitmap for the next timeout.
+     *
+     * @returns {Promise<void>}
+     */
+    async sendTimeout() {
+        let imageBitmap: ImageBitmap | undefined;
+
+        if (!this._imageCapture.track || this._imageCapture.track.readyState !== 'live') {
+            logger.warn('Track is in invalid state');
+            this.stop();
+
+            return;
+        }
+
+        try {
+            imageBitmap = await this._imageCapture.grabFrame();
+        } catch (e) {
+            // ignore error
+        }
+
+        this._streamWorker.postMessage({
+            id: SET_TIMEOUT,
+            timeMs: POLL_INTERVAL,
+            imageBitmap
+        });
+    }
+
+    /**
+     * Handler of the {@code EventHandler} message that calls the appropriate method based on the parameter's id.
+     *
+     * @private
+     * @param {EventHandler} message - Message received from the Worker.
+     * @returns {void}
+     */
+    _handleWorkerAction(message: { data: { id: number; imageBlob?: Blob; }; }) {
+        const { id, imageBlob } = message.data;
+
+        this.sendTimeout();
+        if (id === TIMEOUT_TICK && imageBlob && this._queue.length < SCREENSHOT_QUEUE_LIMIT) {
+            this._doProcessScreenshot(imageBlob);
+        }
+    }
+
+    /**
+     * Method that processes the screenshot.
+     *
+     * @private
+     * @param {Blob} imageBlob - The blob for the current screenshot.
+     * @returns {void}
+     */
+    _doProcessScreenshot(imageBlob: Blob) {
+        this._queue.push(imageBlob);
+        sendAnalytics(createScreensharingCaptureTakenEvent());
+
+        const conference = getCurrentConference(this._state);
+        const sessionId = conference?.getMeetingUniqueId();
+        const { connection } = this._state['features/base/connection'];
+        const jid = connection?.getJid();
+        const timestamp = Date.now();
+        const { jwt } = this._state['features/base/jwt'];
+        const meetingFqn = extractFqnFromPath();
+        const remoteParticipants = getRemoteParticipants(this._state);
+        const participants: Array<string | undefined> = [];
+
+        participants.push(getLocalParticipant(this._state)?.id);
+        remoteParticipants.forEach(p => participants.push(p.id));
+
+        processScreenshot(imageBlob, {
+            jid,
+            jwt,
+            sessionId,
+            timestamp,
+            meetingFqn,
+            participants
+        }).then(() => {
+            const index = this._queue.indexOf(imageBlob);
+
+            if (index > -1) {
+                this._queue.splice(index, 1);
+            }
+        });
+    }
+}

react/features/screenshot-capture/actionTypes.ts

@@ -0,0 +1,29 @@
+/**
+ * Redux action type dispatched in order to toggle screenshot captures.
+ *
+ * {
+ *      type: SET_SCREENSHOT_CAPTURE
+ * }
+ */
+
+export const SET_SCREENSHOT_CAPTURE = 'SET_SCREENSHOT_CAPTURE';
+
+/**
+ * The camera capture payload.
+ */
+export interface ICameraCapturePayload {
+  /**
+   * Selected camera on open.
+   */
+  cameraFacingMode?: string;
+
+  /**
+   * Custom explanatory text to show.
+   */
+  descriptionText?: string,
+
+  /**
+   * Custom dialog title text.
+   */
+  titleText?: string
+};

react/features/screenshot-capture/actions.ts

@@ -0,0 +1,86 @@
+import { IStore } from '../app/types';
+import { openDialog } from '../base/dialog/actions';
+import { isMobileBrowser } from '../base/environment/utils';
+import { getLocalJitsiDesktopTrack } from '../base/tracks/functions';
+
+import CameraCaptureDialog from './CameraCaptureDialog';
+import { ICameraCapturePayload, SET_SCREENSHOT_CAPTURE } from './actionTypes';
+import { createScreenshotCaptureSummary } from './functions';
+import logger from './logger';
+
+let screenshotSummary: any;
+
+/**
+ * Marks the on-off state of screenshot captures.
+ *
+ * @param {boolean} enabled - Whether to turn screen captures on or off.
+ * @returns {{
+    *      type: START_SCREENSHOT_CAPTURE,
+    *      payload: enabled
+    * }}
+*/
+function setScreenshotCapture(enabled: boolean) {
+    return {
+        type: SET_SCREENSHOT_CAPTURE,
+        payload: enabled
+    };
+}
+
+/**
+* Action that toggles the screenshot captures.
+*
+* @param {boolean} enabled - Bool that represents the intention to start/stop screenshot captures.
+* @returns {Promise}
+*/
+export function toggleScreenshotCaptureSummary(enabled: boolean) {
+    return async function(dispatch: IStore['dispatch'], getState: IStore['getState']) {
+        const state = getState();
+
+        if (state['features/screenshot-capture'].capturesEnabled !== enabled) {
+            if (!screenshotSummary) {
+                try {
+                    screenshotSummary = await createScreenshotCaptureSummary(state);
+                } catch (err) {
+                    logger.error('Cannot create screenshotCaptureSummary', err);
+                }
+            }
+
+            if (enabled) {
+                try {
+                    const jitsiTrack = getLocalJitsiDesktopTrack(state);
+
+                    await screenshotSummary.start(jitsiTrack);
+                    dispatch(setScreenshotCapture(enabled));
+                } catch {
+                    // Handle promise rejection from {@code start} due to stream type not being desktop.
+                    logger.error('Unsupported stream type.');
+                }
+            } else {
+                screenshotSummary.stop();
+                dispatch(setScreenshotCapture(enabled));
+            }
+        }
+
+        return Promise.resolve();
+    };
+}
+
+/**
+ * Opens {@code CameraCaptureDialog}.
+ *
+ * @param {Function} callback - The callback to execute on picture taken.
+ * @param {ICameraCapturePayload} componentProps - The camera capture payload.
+ * @returns {Function}
+ */
+export function openCameraCaptureDialog(callback: Function, componentProps: ICameraCapturePayload) {
+    return (dispatch: IStore['dispatch']) => {
+        if (!isMobileBrowser()) {
+            return;
+        }
+
+        dispatch(openDialog(CameraCaptureDialog, {
+            callback,
+            componentProps
+        }));
+    };
+}

react/features/screenshot-capture/constants.ts

@@ -0,0 +1,51 @@
+/**
+ * Percent of pixels that signal if two images should be considered different.
+ */
+export const PERCENTAGE_LOWER_BOUND = 4;
+
+/**
+ * Number of milliseconds that represent how often screenshots should be taken.
+ */
+export const POLL_INTERVAL = 2000;
+
+/**
+ * SET_TIMEOUT constant is used to set interval and it is set in
+ * the id property of the request.data property. TimeMs property must
+ * also be set.
+ *
+ * ```
+ * Request.data example:
+ * {
+ *      id: SET_TIMEOUT,
+ *      timeMs: 33
+ * }
+ * ```
+ */
+export const SET_TIMEOUT = 1;
+
+/**
+ * CLEAR_TIMEOUT constant is used to clear the interval and it is set in
+ * the id property of the request.data property.
+ *
+ * ```
+ * {
+ *      id: CLEAR_TIMEOUT
+ * }
+ * ```
+ */
+export const CLEAR_TIMEOUT = 2;
+
+/**
+ * TIMEOUT_TICK constant is used as response and it is set in the id property.
+ *
+ * ```
+ * {
+ *      id: TIMEOUT_TICK
+ * }
+ * ```
+ */
+export const TIMEOUT_TICK = 3;
+
+export const SCREENSHOT_QUEUE_LIMIT = 3;
+
+export const MAX_FILE_SIZE = 1000000;

react/features/screenshot-capture/functions.tsx

@@ -0,0 +1,55 @@
+import { IReduxState } from '../app/types';
+import { IStateful } from '../base/app/types';
+import { toState } from '../base/redux/functions';
+import { isCloudRecordingRunning } from '../recording/functions';
+import { isScreenVideoShared } from '../screen-share/functions';
+
+import ScreenshotCaptureSummary from './ScreenshotCaptureSummary';
+
+/**
+ * Creates a new instance of ScreenshotCapture.
+ *
+ * @param {Object | Function} stateful - The redux store, state, or
+ * {@code getState} function.
+ * @returns {Promise<ScreenshotCapture>}
+ */
+export function createScreenshotCaptureSummary(stateful: IStateful) {
+    if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
+        return Promise.reject(new Error('ScreenshotCaptureSummary not supported!'));
+    }
+
+    return new ScreenshotCaptureSummary(toState(stateful));
+}
+
+/**
+ * Checks if the screenshot capture is enabled based on the config.
+ *
+ * @param {Object} state - Redux state.
+ * @param {boolean} checkSharing - Whether to check if screensharing is on.
+ * @param {boolean} checkRecording - Whether to check is recording is on.
+ * @returns {boolean}
+ */
+export function isScreenshotCaptureEnabled(state: IReduxState, checkSharing?: boolean, checkRecording?: boolean) {
+    const { screenshotCapture } = state['features/base/config'];
+
+    if (!screenshotCapture?.enabled) {
+        return false;
+    }
+
+    if (checkSharing && !isScreenVideoShared(state)) {
+        return false;
+    }
+
+    if (checkRecording) {
+        // Feature enabled always.
+        if (screenshotCapture.mode === 'always') {
+            return true;
+        }
+
+        // Feature enabled only when recording is also on.
+        return isCloudRecordingRunning(state);
+    }
+
+    return true;
+
+}

react/features/screenshot-capture/logger.ts

@@ -0,0 +1,3 @@
+import { getLogger } from '../base/logging/functions';
+
+export default getLogger('features/screenshot-capture');

react/features/screenshot-capture/processScreenshot.js

@@ -0,0 +1,10 @@
+/**
+ * Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}.
+ *
+ * @param {Blob} imageBlob - The blob of the screenshot that has to be processed.
+ * @param {Object} options - Custom options required for processing.
+ * @returns {Promise<void>}
+ */
+export async function processScreenshot(imageBlob, options) { // eslint-disable-line no-unused-vars
+    return await Promise.resolve();
+}

react/features/screenshot-capture/reducer.ts

@@ -0,0 +1,30 @@
+import PersistenceRegistry from '../base/redux/PersistenceRegistry';
+import ReducerRegistry from '../base/redux/ReducerRegistry';
+
+import { SET_SCREENSHOT_CAPTURE } from './actionTypes';
+
+PersistenceRegistry.register('features/screnshot-capture', true, {
+    capturesEnabled: false
+});
+
+const DEFAULT_STATE = {
+    capturesEnabled: false
+};
+
+export interface IScreenshotCaptureState {
+    capturesEnabled: boolean;
+}
+
+ReducerRegistry.register<IScreenshotCaptureState>('features/screenshot-capture',
+(state = DEFAULT_STATE, action): IScreenshotCaptureState => {
+    switch (action.type) {
+    case SET_SCREENSHOT_CAPTURE: {
+        return {
+            ...state,
+            capturesEnabled: action.payload
+        };
+    }
+    }
+
+    return state;
+});

react/features/screenshot-capture/worker.ts

@@ -0,0 +1,132 @@
+import pixelmatch from 'pixelmatch';
+
+import {
+    CLEAR_TIMEOUT,
+    MAX_FILE_SIZE,
+    PERCENTAGE_LOWER_BOUND,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
+} from './constants';
+
+
+let timer: ReturnType<typeof setTimeout>;
+const canvas = new OffscreenCanvas(0, 0);
+const ctx = canvas.getContext('2d');
+let storedImageData: ImageData | undefined;
+
+/**
+ * Sends Blob with the screenshot to main thread.
+ *
+  * @param {ImageData} imageData - The image of the screenshot.
+ * @returns {void}
+ */
+async function sendBlob(imageData: ImageData) {
+    let imageBlob = await canvas.convertToBlob({ type: 'image/jpeg' });
+
+    if (imageBlob.size > MAX_FILE_SIZE) {
+        const quality = Number((MAX_FILE_SIZE / imageBlob.size).toFixed(2)) * 0.92;
+
+        imageBlob = await canvas.convertToBlob({ type: 'image/jpeg',
+            quality });
+    }
+
+    storedImageData = imageData;
+
+    postMessage({
+        id: TIMEOUT_TICK,
+        imageBlob
+    });
+}
+
+/**
+ * Sends empty message to main thread.
+ *
+ * @returns {void}
+ */
+function sendEmpty() {
+    postMessage({
+        id: TIMEOUT_TICK
+    });
+}
+
+/**
+ * Draws the image bitmap on the canvas and checks the difference percent with the previous image
+ * if there is no previous image the percentage is not calculated.
+ *
+ * @param {ImageBitmap} imageBitmap - The image bitmap that is drawn on canvas.
+ * @returns {void}
+ */
+function checkScreenshot(imageBitmap: ImageBitmap) {
+    const { height, width } = imageBitmap;
+
+    if (canvas.width !== width) {
+        canvas.width = width;
+    }
+
+    if (canvas.height !== height) {
+        canvas.height = height;
+    }
+
+    ctx?.drawImage(imageBitmap, 0, 0, width, height);
+    const imageData = ctx?.getImageData(0, 0, width, height);
+
+    imageBitmap.close();
+
+    if (!imageData) {
+        sendEmpty();
+
+        return;
+    }
+
+    if (!storedImageData || imageData.data.length !== storedImageData.data.length) {
+        sendBlob(imageData);
+
+        return;
+    }
+
+    let numOfPixels = 0;
+
+    try {
+        numOfPixels = pixelmatch(
+            imageData.data,
+            storedImageData.data,
+            null,
+            width,
+            height);
+    } catch {
+        sendEmpty();
+
+        return;
+    }
+
+    const percent = numOfPixels / imageData.data.length * 100;
+
+    if (percent >= PERCENTAGE_LOWER_BOUND) {
+        sendBlob(imageData);
+    } else {
+        sendEmpty();
+    }
+}
+
+onmessage = function(request) {
+    switch (request.data.id) {
+    case SET_TIMEOUT: {
+        timer = setTimeout(async () => {
+            const imageBitmap = request.data.imageBitmap;
+
+            if (imageBitmap) {
+                checkScreenshot(imageBitmap);
+            } else {
+                sendEmpty();
+            }
+        }, request.data.timeMs);
+        break;
+    }
+    case CLEAR_TIMEOUT: {
+        if (timer) {
+            clearTimeout(timer);
+        }
+        break;
+    }
+    }
+};