diff --git a/functional-samples/cookbook.offscreen-user-media/README.md b/functional-samples/cookbook.offscreen-user-media/README.md new file mode 100644 index 0000000000..1f7691adb2 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/README.md @@ -0,0 +1,34 @@ +This recipe shows how to use User Media in an Extension Service Worker using the [Offscreen document][1]. + +## Context + +- The extension aims to capture audio from the user in the context of extension (globally) via a media recorder in the offscreen document. +- Service worker no longer have access to window objects and APIs. Hence, it was difficult for the extension to fetch permissions and capture audio. +- Offscreen document handles permission checks, audio devices management and recording using navigator API and media recorder respectively. + +## Steps + +1. User presses START/STOP recording from extension popup +2. Popup sends message to background service worker. +3. If STOP, Service worker sends message to offscreen to stop mediarecorder. +4. If START, Service worker sends message to the active tab's content script to intiate recording process. +5. The content script sends message to offscreen to check audio permissions. + - If GRANTED, send message to offscreen to start media recorder. + - If DENIED, show alert on window + - If PROMPT, + - inject an IFrame to request permission from the user. + - Listen to the user'e response on the iFrame + - If allowed, move to GRANTED step. Else, DENIED. + +## Running this extension + +1. Clone this repository. +2. Load this directory in Chrome as an [unpacked extension][2]. +3. Open the Extension menu and click the extension named "Offscreen API - User media". + +Click on the extension popup for START and STOP recording buttons. + +Inspect the offscreen html page to view logs from media recorder and audio chunk management. + +[1]: https://developer.chrome.com/docs/extensions/reference/offscreen/ +[2]: https://developer.chrome.com/docs/extensions/mv3/getstarted/development-basics/#load-unpacked diff --git a/functional-samples/cookbook.offscreen-user-media/background.js b/functional-samples/cookbook.offscreen-user-media/background.js new file mode 100644 index 0000000000..aa6ffaa665 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/background.js @@ -0,0 +1,133 @@ +/** + * Path to the offscreen HTML document. + * @type {string} + */ +const OFFSCREEN_DOCUMENT_PATH = 'offscreen/offscreen.html'; + +/** + * Reason for creating the offscreen document. + * @type {string} + */ +const OFFSCREEN_REASON = 'USER_MEDIA'; + +/** + * Listener for extension installation. + */ +chrome.runtime.onInstalled.addListener(handleInstall); + +/** + * Listener for messages from the extension. + * @param {Object} request - The message request. + * @param {Object} sender - The sender of the message. + * @param {function} sendResponse - Callback function to send a response. + */ +chrome.runtime.onMessage.addListener((request) => { + switch (request.message.type) { + case 'TOGGLE_RECORDING': + switch (request.message.data) { + case 'START': + initateRecordingStart(); + break; + case 'STOP': + initateRecordingStop(); + break; + } + break; + } +}); + +/** + * Handles the installation of the extension. + */ +async function handleInstall() { + console.log('Extension installed...'); + if (!(await hasDocument())) { + // create offscreen document + await createOffscreenDocument(); + } +} + +/** + * Sends a message to the offscreen document. + * @param {string} type - The type of the message. + * @param {Object} data - The data to be sent with the message. + */ +async function sendMessageToOffscreenDocument(type, data) { + // Create an offscreen document if one doesn't exist yet + try { + if (!(await hasDocument())) { + await createOffscreenDocument(); + } + } finally { + // Now that we have an offscreen document, we can dispatch the message. + chrome.runtime.sendMessage({ + message: { + type: type, + target: 'offscreen', + data: data + } + }); + } +} + +/** + * Initiates the stop recording process. + */ +function initateRecordingStop() { + console.log('Recording stopped at offscreen'); + sendMessageToOffscreenDocument('STOP_OFFSCREEN_RECORDING'); +} + +/** + * Initiates the start recording process. + */ +function initateRecordingStart() { + chrome.tabs.query({ active: true, lastFocusedWindow: true }, ([tab]) => { + if (chrome.runtime.lastError || !tab) { + console.error('No valid webpage or tab opened'); + return; + } + + chrome.tabs.sendMessage( + tab.id, + { + // Send message to content script of the specific tab to check and/or prompt mic permissions + message: { type: 'PROMPT_MICROPHONE_PERMISSION' } + }, + (response) => { + // If user allows the mic permissions, we continue the recording procedure. + if (response.message.status === 'success') { + console.log('Recording started at offscreen'); + sendMessageToOffscreenDocument('START_OFFSCREEN_RECORDING'); + } + } + ); + }); +} + +/** + * Checks if there is an offscreen document. + * @returns {Promise} - Promise that resolves to a boolean indicating if an offscreen document exists. + */ +async function hasDocument() { + // Check all windows controlled by the service worker if one of them is the offscreen document + const matchedClients = await clients.matchAll(); + for (const client of matchedClients) { + if (client.url.endsWith(OFFSCREEN_DOCUMENT_PATH)) { + return true; + } + } + return false; +} + +/** + * Creates the offscreen document. + * @returns {Promise} - Promise that resolves when the offscreen document is created. + */ +async function createOffscreenDocument() { + await chrome.offscreen.createDocument({ + url: OFFSCREEN_DOCUMENT_PATH, + reasons: [OFFSCREEN_REASON], + justification: 'To interact with user media' + }); +} diff --git a/functional-samples/cookbook.offscreen-user-media/contentScript.js b/functional-samples/cookbook.offscreen-user-media/contentScript.js new file mode 100644 index 0000000000..2864fe0768 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/contentScript.js @@ -0,0 +1,79 @@ +/** + * Listener for messages from the background script. + * @param {Object} request - The message request. + * @param {Object} sender - The sender of the message. + * @param {function} sendResponse - Callback function to send a response. + * @returns {boolean} - Whether the response should be sent asynchronously (true by default). + */ +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + switch (request.message.type) { + case 'PROMPT_MICROPHONE_PERMISSION': + // Check for mic permissions. If not found, prompt + checkMicPermissions() + .then(() => { + sendResponse({ message: { status: 'success' } }); + }) + .catch(() => { + promptMicPermissions(); + const iframe = document.getElementById('PERMISSION_IFRAME_ID'); + window.addEventListener('message', (event) => { + if (event.source === iframe.contentWindow && event.data) { + if (event.data.type === 'permissionsGranted') { + sendResponse({ + message: { status: 'success' } + }); + } else { + sendResponse({ + message: { + status: 'failure' + } + }); + } + document.body.removeChild(iframe); + } + }); + }); + break; + + default: + // Do nothing for other message types + break; + } + return true; +}); + +/** + * Checks microphone permissions using a message to the background script. + * @returns {Promise} - Promise that resolves if permissions are granted, rejects otherwise. + */ +function checkMicPermissions() { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage( + { + message: { + type: 'CHECK_PERMISSIONS', + target: 'offscreen' + } + }, + (response) => { + if (response.message.status === 'success') { + resolve(); + } else { + reject(response.message.data); + } + } + ); + }); +} + +/** + * Prompts the user for microphone permissions using an iframe. + */ +function promptMicPermissions() { + const iframe = document.createElement('iframe'); + iframe.setAttribute('hidden', 'hidden'); + iframe.setAttribute('allow', 'microphone'); + iframe.setAttribute('id', 'PERMISSION_IFRAME_ID'); + iframe.src = chrome.runtime.getURL('requestPermissions.html'); + document.body.appendChild(iframe); +} diff --git a/functional-samples/cookbook.offscreen-user-media/manifest.json b/functional-samples/cookbook.offscreen-user-media/manifest.json new file mode 100644 index 0000000000..195be36e41 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/manifest.json @@ -0,0 +1,25 @@ +{ + "name": "Offscreen API - User media", + "version": "1.0", + "manifest_version": 3, + "description": "Shows how to record audio in a chrome extension via offscreen document.", + "background": { + "service_worker": "background.js" + }, + "content_scripts": [ + { + "matches": [""], + "js": ["contentScript.js"] + } + ], + "action": { + "default_popup": "popup/popup.html" + }, + "permissions": ["offscreen", "activeTab", "tabs"], + "web_accessible_resources": [ + { + "resources": ["requestPermissions.html", "requestPermissions.js"], + "matches": [""] + } + ] +} diff --git a/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.html b/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.html new file mode 100644 index 0000000000..a495ee9598 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.html @@ -0,0 +1,10 @@ + + + + + + Offscreen Document + + + + diff --git a/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.js b/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.js new file mode 100644 index 0000000000..882e6858f5 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.js @@ -0,0 +1,152 @@ +/** + * MediaRecorder instance for audio recording. + * @type {MediaRecorder} + */ +let mediaRecorder; + +/** + * Event listener for messages from the extension. + * @param {Object} request - The message request. + * @param {Object} sender - The sender of the message. + * @param {function} sendResponse - Callback function to send a response. + * @returns {boolean} - Indicates whether the response should be asynchronous. + */ +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.message.target !== 'offscreen') { + return; + } + + switch (request.message.type) { + case 'START_OFFSCREEN_RECORDING': + // Start recording + handleRecording(); + sendResponse({}); + break; + case 'STOP_OFFSCREEN_RECORDING': + // Stop recording + stopRecording(); + sendResponse({}); + break; + case 'CHECK_PERMISSIONS': + checkAudioPermissions() + .then((data) => sendResponse(data)) + .catch((errorData) => sendResponse(errorData)); + break; + default: + break; + } + + return true; +}); + +/** + * Stops the recording if the MediaRecorder is in the recording state. + */ +function stopRecording() { + if (mediaRecorder && mediaRecorder.state === 'recording') { + console.log('Stopped recording in offscreen...'); + mediaRecorder.stop(); + } +} + +/** + * Initiates the audio recording process using MediaRecorder. + */ +async function handleRecording() { + getAudioInputDevices().then((audioInputDevices) => { + const deviceId = audioInputDevices[0].deviceId; + navigator.mediaDevices + .getUserMedia({ + audio: { + deviceId: { exact: deviceId } + } + }) + .then((audioStream) => { + try { + mediaRecorder = new MediaRecorder(audioStream); + mediaRecorder.ondataavailable = (event) => { + if (mediaRecorder.state === 'recording') { + saveAudioChunks([event.data]); + } + }; + mediaRecorder.onstop = handleStopRecording; + + // Start MediaRecorder and capture chunks every 10s. + mediaRecorder.start(10000); + + console.log('Started recording in offscreen...'); + } catch (error) { + console.error( + 'Unable to initiate MediaRecorder and/or streams', + error + ); + } + }); + }); +} + +/** + * Saves audio chunks captured by MediaRecorder. + * @param {Blob[]} chunkData - Array of audio chunks in Blob format. + */ +function saveAudioChunks(chunkData) { + console.log('Chunk captured from MediaRecorder'); + // Manage audio chunks accordingly as per your needs +} + +/** + * Event handler for when MediaRecorder is stopped. + */ +function handleStopRecording() { + // Handle cases when MediaRecorder is stopped if needed +} + +/** + * Fetches audio input devices using the `navigator.mediaDevices.enumerateDevices` API. + * @returns {Promise} - Promise that resolves to an array of audio input devices. + */ +function getAudioInputDevices() { + return new Promise((resolve, reject) => { + navigator.mediaDevices + .enumerateDevices() + .then((devices) => { + // Filter the devices to include only audio input devices + const audioInputDevices = devices.filter( + (device) => device.kind === 'audioinput' + ); + resolve(audioInputDevices); + }) + .catch((error) => { + console.log('Error getting audio input devices', error); + reject(error); + }); + }); +} + +/** + * Checks microphone permissions using the `navigator.permissions.query` API. + * @returns {Promise} - Promise that resolves to an object containing permission status. + */ +function checkAudioPermissions() { + return new Promise((resolve, reject) => { + navigator.permissions + .query({ name: 'microphone' }) + .then((result) => { + if (result.state === 'granted') { + console.log('Mic permissions granted'); + resolve({ message: { status: 'success' } }); + } else { + console.log('Mic permissions missing', result.state); + reject({ + message: { status: 'error', data: result.state } + }); + } + }) + .catch((error) => { + console.warn('Permissions error', error); + reject({ + message: { status: 'error', data: { error: error } } + }); + }); + }); +} diff --git a/functional-samples/cookbook.offscreen-user-media/popup/popup.css b/functional-samples/cookbook.offscreen-user-media/popup/popup.css new file mode 100644 index 0000000000..677e20e2b4 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/popup/popup.css @@ -0,0 +1,29 @@ +body { + display: flex; + justify-content: center; + align-items: center; + height: 100px; + margin: 0; +} + +#popup-container { + text-align: center; +} + +#start-button, +#stop-button { + padding: 10px; + font-size: 14px; + cursor: pointer; + margin: 5px; +} + +#start-button { + background-color: #4caf50; /* Green */ + color: white; +} + +#stop-button { + background-color: #f44336; /* Red */ + color: white; +} diff --git a/functional-samples/cookbook.offscreen-user-media/popup/popup.html b/functional-samples/cookbook.offscreen-user-media/popup/popup.html new file mode 100644 index 0000000000..80277962c4 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/popup/popup.html @@ -0,0 +1,16 @@ + + + + + + + Record Extension + + + + + + diff --git a/functional-samples/cookbook.offscreen-user-media/popup/popup.js b/functional-samples/cookbook.offscreen-user-media/popup/popup.js new file mode 100644 index 0000000000..397cc35f3d --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/popup/popup.js @@ -0,0 +1,25 @@ +document.addEventListener('DOMContentLoaded', function () { + const startButton = document.getElementById('start-button'); + const stopButton = document.getElementById('stop-button'); + + // Function to send a message to the background script + const sendMessageToBackground = (message) => { + chrome.runtime.sendMessage({ + message: { + type: 'TOGGLE_RECORDING', + target: 'background', + data: message + } + }); + }; + + // Add a click event listener to the start button + startButton.addEventListener('click', function () { + sendMessageToBackground('START'); + }); + + // Add a click event listener to the stop button + stopButton.addEventListener('click', function () { + sendMessageToBackground('STOP'); + }); +}); diff --git a/functional-samples/cookbook.offscreen-user-media/requestPermissions.html b/functional-samples/cookbook.offscreen-user-media/requestPermissions.html new file mode 100644 index 0000000000..a644b8bd85 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/requestPermissions.html @@ -0,0 +1,8 @@ + + + + + Request Permissions + + + diff --git a/functional-samples/cookbook.offscreen-user-media/requestPermissions.js b/functional-samples/cookbook.offscreen-user-media/requestPermissions.js new file mode 100644 index 0000000000..ef95d91043 --- /dev/null +++ b/functional-samples/cookbook.offscreen-user-media/requestPermissions.js @@ -0,0 +1,45 @@ +/** + * Requests user permission for microphone access and sends a message to the parent window. + */ +function getUserPermission() { + console.info('Getting user permission for microphone access...'); + + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((response) => { + if (response.id !== null && response.id !== undefined) { + console.log('Microphone access granted'); + // Post a message to the parent window indicating successful permission + window.parent.postMessage({ type: 'permissionsGranted' }, '*'); + return; + } + // Post a message to the parent window indicating failed permission + window.parent.postMessage( + { + type: 'permissionsFailed' + }, + '*' + ); + }) + .catch((error) => { + console.warn('Error requesting microphone permission: ', error); + if (error.message === 'Permission denied') { + // Show an alert if permission is denied + window.alert( + 'Please allow microphone access. Highlight uses your microphone to record audio during meetings.' + ); + } + + // Post a message to the parent window indicating failed permission with an optional error message + window.parent.postMessage( + { + type: 'permissionsFailed', + message: error.message + }, + '*' + ); + }); +} + +// Call the function to request microphone permission +getUserPermission();