diff --git a/packages/aws-amplify-react/src/Interactions/ChatBot.tsx b/packages/aws-amplify-react/src/Interactions/ChatBot.tsx index b2a6f812816..bca891ebdf4 100644 --- a/packages/aws-amplify-react/src/Interactions/ChatBot.tsx +++ b/packages/aws-amplify-react/src/Interactions/ChatBot.tsx @@ -7,7 +7,7 @@ import { } from '../AmplifyUI'; import { Input, Button } from '../AmplifyTheme'; -import { I18n } from '@aws-amplify/core'; +import { I18n, AudioRecorder } from '@aws-amplify/core'; import { Interactions } from '@aws-amplify/interactions'; import { ConsoleLogger as Logger } from '@aws-amplify/core'; import { chatBot } from '../Amplify-UI/data-test-attributes'; @@ -55,13 +55,11 @@ const STATES = { const defaultVoiceConfig = { silenceDetectionConfig: { - time: 2000, + time: 1500, amplitude: 0.2, }, }; -let audioControl; - export interface IChatBotProps { botName?: string; clearOnComplete?: boolean; @@ -94,13 +92,14 @@ export interface IChatBotState { export class ChatBot extends React.Component { private listItemsRef: any; + private audioRecorder: AudioRecorder; constructor(props) { super(props); if (this.props.voiceEnabled) { - require('./aws-lex-audio'); - // @ts-ignore - audioControl = new global.LexAudio.audioControl(); + this.audioRecorder = new AudioRecorder( + this.props.voiceConfig.silenceDetectionConfig + ); } if (!this.props.textEnabled && this.props.voiceEnabled) { STATES.INITIAL.MESSAGE = 'Click the mic button'; @@ -158,22 +157,18 @@ export class ChatBot extends React.Component { micButtonDisabled: false, }, () => { - audioControl.startRecording( - this.onSilenceHandler, - null, - this.props.voiceConfig.silenceDetectionConfig - ); + this.audioRecorder.startRecording(this.onSilenceHandler); } ); } } onSilenceHandler() { - audioControl.stopRecording(); + this.audioRecorder.stopRecording(); if (!this.state.continueConversation) { return; } - audioControl.exportWAV(blob => { + this.audioRecorder.exportWAV(blob => { this.setState( { currentVoiceState: STATES.SENDING, @@ -235,7 +230,7 @@ export class ChatBot extends React.Component { return; } if (this.state.lexResponse.contentType === 'audio/mpeg') { - audioControl.play(this.state.lexResponse.audioStream, () => { + this.audioRecorder.play(this.state.lexResponse.audioStream, () => { if ( this.state.lexResponse.dialogState === 'ReadyForFulfillment' || this.state.lexResponse.dialogState === 'Fulfilled' || @@ -257,11 +252,7 @@ export class ChatBot extends React.Component { micButtonDisabled: false, }, () => { - audioControl.startRecording( - this.onSilenceHandler, - null, - this.props.voiceConfig.silenceDetectionConfig - ); + this.audioRecorder.startRecording(this.onSilenceHandler); } ); } @@ -288,7 +279,7 @@ export class ChatBot extends React.Component { micButtonDisabled: false, }, () => { - audioControl.clear(); + this.audioRecorder.clear(); } ); } @@ -424,7 +415,6 @@ export class ChatBot extends React.Component { render() { const { title, theme, onComplete } = this.props; - return ( {title && ( diff --git a/packages/aws-amplify-react/src/Interactions/aws-lex-audio.tsx b/packages/aws-amplify-react/src/Interactions/aws-lex-audio.tsx deleted file mode 100644 index 96e668af106..00000000000 --- a/packages/aws-amplify-react/src/Interactions/aws-lex-audio.tsx +++ /dev/null @@ -1,1011 +0,0 @@ -// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// Permission is hereby granted, free of charge, to any person obtaining a copy of this -// software and associated documentation files (the "Software"), to deal in the Software -// without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// @ts-ignore -(function e(t, n, r) { - function s(o, u) { - if (!n[o]) { - if (!t[o]) { - var a = typeof require == 'function' && require; - // @ts-ignore - if (!u && a) return a(o, !0); - // @ts-ignore - if (i) return i(o, !0); - var f = new Error("Cannot find module '" + o + "'"); - // @ts-ignore - throw ((f.code = 'MODULE_NOT_FOUND'), f); - } - var l = (n[o] = { exports: {} }); - t[o][0].call( - l.exports, - function(e) { - var n = t[o][1][e]; - // @ts-ignore - return s(n ? n : e); - }, - l, - l.exports, - e, - t, - n, - r - ); - } - return n[o].exports; - } - var i = typeof require == 'function' && require; - // @ts-ignore - for (var o = 0; o < r.length; o++) s(r[o]); - return s; -})( - { - 1: [ - function(require, module, exports) { - (function() { - 'use strict'; - var rec = require('./recorder.js'); - var recorder, - audioRecorder, - checkAudioSupport, - audioSupported, - playbackSource, - UNSUPPORTED = 'Audio is not supported.'; - - /** - * Represents an audio control that can start and stop recording, - * export captured audio, play an audio buffer, and check if audio - * is supported. - */ - exports.audioControl = function(options) { - options = options || {}; - this.checkAudioSupport = options.checkAudioSupport !== false; - - /** - * This callback type is called `onSilenceCallback`. - * - * @callback onSilenceCallback - */ - - /** - * Visualize callback: `visualizerCallback`. - * - * @callback visualizerCallback - * @param {Uint8Array} dataArray - * @param {number} bufferLength - */ - - /** - * Clears the previous buffer and starts buffering audio. - * - * @param {?onSilenceCallback} onSilence - Called when silence is detected. - * @param {?visualizerCallback} visualizer - Can be used to visualize the captured buffer. - * @param {silenceDetectionConfig} - Specify custom silence detection values. - * @throws {Error} If audio is not supported. - */ - var startRecording = function( - onSilence, - visualizer, - silenceDetectionConfig - ) { - onSilence = - onSilence || - function() { - /* no op */ - }; - visualizer = - visualizer || - function() { - /* no op */ - }; - audioSupported = audioSupported !== false; - if (!audioSupported) { - throw new Error(UNSUPPORTED); - } - const context = audioRecorder.audioContext(); - context.resume().then(() => { - recorder = audioRecorder.createRecorder(silenceDetectionConfig); - recorder.record(onSilence, visualizer); - }); - }; - - /** - * Stops buffering audio. - * - * @throws {Error} If audio is not supported. - */ - var stopRecording = function() { - audioSupported = audioSupported !== false; - if (!audioSupported) { - throw new Error(UNSUPPORTED); - } - recorder.stop(); - }; - - /** - * On export complete callback: `onExportComplete`. - * - * @callback onExportComplete - * @param {Blob} blob The exported audio as a Blob. - */ - - /** - * Exports the captured audio buffer. - * - * @param {onExportComplete} callback - Called when the export is complete. - * @param {sampleRate} The sample rate to use in the export. - * @throws {Error} If audio is not supported. - */ - var exportWAV = function(callback, sampleRate) { - audioSupported = audioSupported !== false; - if (!audioSupported) { - throw new Error(UNSUPPORTED); - } - if (!(callback && typeof callback === 'function')) { - throw new Error('You must pass a callback function to export.'); - } - sampleRate = - typeof sampleRate !== 'undefined' ? sampleRate : 16000; - recorder.exportWAV(callback, sampleRate); - recorder.clear(); - }; - - /** - * On playback complete callback: `onPlaybackComplete`. - * - * @callback onPlaybackComplete - */ - - /** - * Plays the audio buffer with an HTML5 audio tag. - * @param {Uint8Array} buffer - The audio buffer to play. - * @param {?onPlaybackComplete} callback - Called when audio playback is complete. - */ - var playHtmlAudioElement = function(buffer, callback) { - if (typeof buffer === 'undefined') { - return; - } - var myBlob = new Blob([buffer]); - var audio = document.createElement('audio'); - var objectUrl = window.URL.createObjectURL(myBlob); - audio.src = objectUrl; - audio.addEventListener('ended', function() { - audio.currentTime = 0; - if (typeof callback === 'function') { - callback(); - } - }); - audio.play(); - }; - - /** - * On playback complete callback: `onPlaybackComplete`. - * - * @callback onPlaybackComplete - */ - - /** - * Plays the audio buffer with a WebAudio AudioBufferSourceNode. - * @param {Uint8Array} buffer - The audio buffer to play. - * @param {?onPlaybackComplete} callback - Called when audio playback is complete. - */ - var play = function(buffer, callback) { - if (typeof buffer === 'undefined') { - return; - } - var myBlob = new Blob([buffer]); - // We'll use a FileReader to create and ArrayBuffer out of the audio response. - var fileReader = new FileReader(); - fileReader.onload = function() { - // Once we have an ArrayBuffer we can create our BufferSource and decode the result as an AudioBuffer. - playbackSource = audioRecorder - .audioContext() - .createBufferSource(); - audioRecorder - .audioContext() - .decodeAudioData(this.result, function(buf) { - // Set the source buffer as our new AudioBuffer. - playbackSource.buffer = buf; - // Set the destination (the actual audio-rendering device--your device's speakers). - playbackSource.connect( - audioRecorder.audioContext().destination - ); - // Add an "on ended" callback. - playbackSource.onended = function(event) { - if (typeof callback === 'function') { - callback(); - } - }; - // Start the playback. - playbackSource.start(0); - }); - }; - fileReader.readAsArrayBuffer(myBlob); - }; - - /** - * Stops the playback source (created by the play method) if it exists. The `onPlaybackComplete` - * callback will be called. - */ - var stop = function() { - if (typeof playbackSource === 'undefined') { - return; - } - playbackSource.stop(); - }; - - /** - * Clear the recording buffer. - */ - var clear = function() { - recorder.clear(); - }; - - /** - * On audio supported callback: `onAudioSupported`. - * - * @callback onAudioSupported - * @param {boolean} - */ - - /** - * Checks that getUserMedia is supported and the user has given us access to the mic. - * @param {onAudioSupported} callback - Called with the result. - */ - var supportsAudio = function(callback) { - callback = - callback || - function() { - /* no op */ - }; - if ( - navigator.mediaDevices && - navigator.mediaDevices.getUserMedia - ) { - audioRecorder = rec.audioRecorder(); - audioRecorder - .requestDevice() - .then(function(stream) { - audioSupported = true; - callback(audioSupported); - }) - .catch(function(error) { - audioSupported = false; - callback(audioSupported); - }); - } else { - audioSupported = false; - callback(audioSupported); - } - }; - - if (this.checkAudioSupport) { - // @ts-ignore - supportsAudio(); - } - - return { - startRecording: startRecording, - stopRecording: stopRecording, - exportWAV: exportWAV, - play: play, - stop: stop, - clear: clear, - playHtmlAudioElement: playHtmlAudioElement, - supportsAudio: supportsAudio, - }; - }; - })(); - }, - { './recorder.js': 5 }, - ], - 2: [ - function(require, module, exports) { - (function() { - 'use strict'; - var AudioControl = require('./control.js').audioControl; - - var DEFAULT_LATEST = '$LATEST'; - var DEFAULT_CONTENT_TYPE = 'audio/x-l16; sample-rate=16000'; - var DEFAULT_USER_ID = 'userId'; - var DEFAULT_ACCEPT_HEADER_VALUE = 'audio/mpeg'; - var MESSAGES = Object.freeze({ - PASSIVE: 'Passive', - LISTENING: 'Listening', - SENDING: 'Sending', - SPEAKING: 'Speaking', - }); - - var lexruntime, - audioControl = new AudioControl({ checkAudioSupport: false }); - - exports.conversation = function( - config, - onStateChange, - onSuccess, - onError, - onAudioData - ) { - var currentState; - - // Apply default values. - this.config = applyDefaults(config); - this.lexConfig = this.config.lexConfig; - this.messages = MESSAGES; - onStateChange = - onStateChange || - function() { - /* no op */ - }; - this.onSuccess = - onSuccess || - function() { - /* no op */ - }; - this.onError = - onError || - function() { - /* no op */ - }; - this.onAudioData = - onAudioData || - function() { - /* no op */ - }; - - // Validate input. - if (!this.config.lexConfig.botName) { - this.onError('A Bot name must be provided.'); - return; - } - // @ts-ignore - if (!AWS.config.credentials) { - this.onError('AWS Credentials must be provided.'); - return; - } - // @ts-ignore - if (!AWS.config.region) { - this.onError('A Region value must be provided.'); - return; - } - - // @ts-ignore - lexruntime = new AWS.LexRuntime(); - - this.onSilence = function() { - if (config.silenceDetection) { - audioControl.stopRecording(); - currentState.advanceConversation(); - } - }; - - this.transition = function(conversation) { - currentState = conversation; - var state = currentState.state; - onStateChange(state.message); - - // If we are transitioning into SENDING or SPEAKING we want to immediately advance the conversation state - // to start the service call or playback. - if ( - state.message === state.messages.SENDING || - state.message === state.messages.SPEAKING - ) { - currentState.advanceConversation(); - } - // If we are transitioning in to sending and we are not detecting silence (this was a manual state change) - // we need to do some cleanup: stop recording, and stop rendering. - if ( - state.message === state.messages.SENDING && - !this.config.silenceDetection - ) { - audioControl.stopRecording(); - } - }; - - this.advanceConversation = function() { - audioControl.supportsAudio(function(supported) { - if (supported) { - currentState.advanceConversation(); - } else { - onError('Audio is not supported.'); - } - }); - }; - - this.updateConfig = function(newValue) { - this.config = applyDefaults(newValue); - this.lexConfig = this.config.lexConfig; - }; - - this.reset = function() { - audioControl.clear(); - currentState = new Initial(currentState.state); - }; - - currentState = new Initial(this); - - return { - advanceConversation: this.advanceConversation, - updateConfig: this.updateConfig, - reset: this.reset, - }; - }; - - var Initial = function(state) { - this.state = state; - state.message = state.messages.PASSIVE; - this.advanceConversation = function() { - audioControl.startRecording( - state.onSilence, - state.onAudioData, - state.config.silenceDetectionConfig - ); - state.transition(new Listening(state)); - }; - }; - - var Listening = function(state) { - this.state = state; - state.message = state.messages.LISTENING; - this.advanceConversation = function() { - audioControl.exportWAV(function(blob) { - state.audioInput = blob; - state.transition(new Sending(state)); - }); - }; - }; - - var Sending = function(state) { - this.state = state; - state.message = state.messages.SENDING; - this.advanceConversation = function() { - state.lexConfig.inputStream = state.audioInput; - lexruntime.postContent(state.lexConfig, function(err, data) { - if (err) { - state.onError(err); - state.transition(new Initial(state)); - } else { - state.audioOutput = data; - state.transition(new Speaking(state)); - state.onSuccess(data); - } - }); - }; - }; - - var Speaking = function(state) { - this.state = state; - state.message = state.messages.SPEAKING; - this.advanceConversation = function() { - if (state.audioOutput.contentType === 'audio/mpeg') { - audioControl.play(state.audioOutput.audioStream, function() { - if ( - state.audioOutput.dialogState === 'ReadyForFulfillment' || - state.audioOutput.dialogState === 'Fulfilled' || - state.audioOutput.dialogState === 'Failed' || - !state.config.silenceDetection - ) { - state.transition(new Initial(state)); - } else { - audioControl.startRecording( - state.onSilence, - state.onAudioData, - state.config.silenceDetectionConfig - ); - state.transition(new Listening(state)); - } - }); - } else { - state.transition(new Initial(state)); - } - }; - }; - - var applyDefaults = function(config) { - config = config || {}; - config.silenceDetection = config.hasOwnProperty('silenceDetection') - ? config.silenceDetection - : true; - - var lexConfig = config.lexConfig || {}; - lexConfig.botAlias = lexConfig.hasOwnProperty('botAlias') - ? lexConfig.botAlias - : DEFAULT_LATEST; - lexConfig.botName = lexConfig.hasOwnProperty('botName') - ? lexConfig.botName - : ''; - lexConfig.contentType = lexConfig.hasOwnProperty('contentType') - ? lexConfig.contentType - : DEFAULT_CONTENT_TYPE; - lexConfig.userId = lexConfig.hasOwnProperty('userId') - ? lexConfig.userId - : DEFAULT_USER_ID; - lexConfig.accept = lexConfig.hasOwnProperty('accept') - ? lexConfig.accept - : DEFAULT_ACCEPT_HEADER_VALUE; - config.lexConfig = lexConfig; - - return config; - }; - })(); - }, - { './control.js': 1 }, - ], - 3: [ - function(require, module, exports) { - (function(global) { - /** - * @module LexAudio - * @description The global namespace for Amazon Lex Audio - */ - global.LexAudio = global.LexAudio || {}; - global.LexAudio.audioControl = require('./control.js').audioControl; - global.LexAudio.conversation = require('./conversation.js').conversation; - module.exports = global.LexAudio; - }.call( - this, - typeof global !== 'undefined' - ? global - : typeof self !== 'undefined' - ? self - : typeof window !== 'undefined' - ? window - : {} - )); - }, - { './control.js': 1, './conversation.js': 2 }, - ], - 4: [ - function(require, module, exports) { - var bundleFn = arguments[3]; - var sources = arguments[4]; - var cache = arguments[5]; - - var stringify = JSON.stringify; - - module.exports = function(fn, options) { - var wkey; - var cacheKeys = Object.keys(cache); - - for (var i = 0, l = cacheKeys.length; i < l; i++) { - var key = cacheKeys[i]; - var exp = cache[key].exports; - // Using babel as a transpiler to use esmodule, the export will always - // be an object with the default export as a property of it. To ensure - // the existing api and babel esmodule exports are both supported we - // check for both - if (exp === fn || (exp && exp.default === fn)) { - wkey = key; - break; - } - } - - if (!wkey) { - wkey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16); - var wcache = {}; - for (var i = 0, l = cacheKeys.length; i < l; i++) { - var key = cacheKeys[i]; - wcache[key] = key; - } - sources[wkey] = [ - // @ts-ignore - Function(['require', 'module', 'exports'], '(' + fn + ')(self)'), - wcache, - ]; - } - var skey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16); - - var scache = {}; - scache[wkey] = wkey; - sources[skey] = [ - // @ts-ignore - Function( - // @ts-ignore - ['require'], - // try to call default if defined to also support babel esmodule - // exports - 'var f = require(' + - stringify(wkey) + - ');' + - '(f.default ? f.default : f)(self);' - ), - scache, - ]; - - var workerSources = {}; - resolveSources(skey); - - function resolveSources(key) { - workerSources[key] = true; - - for (var depPath in sources[key][1]) { - var depKey = sources[key][1][depPath]; - if (!workerSources[depKey]) { - resolveSources(depKey); - } - } - } - - var src = - '(' + - bundleFn + - ')({' + - Object.keys(workerSources) - .map(function(key) { - return ( - stringify(key) + - ':[' + - sources[key][0] + - ',' + - stringify(sources[key][1]) + - ']' - ); - }) - .join(',') + - '},{},[' + - stringify(skey) + - '])'; - - // @ts-ignore - var URL = - // @ts-ignore - window.URL || window.webkitURL || window.mozURL || window.msURL; - - var blob = new Blob([src], { type: 'text/javascript' }); - if (options && options.bare) { - return blob; - } - var workerUrl = URL.createObjectURL(blob); - var worker = new Worker(workerUrl); - // @ts-ignore - worker.objectURL = workerUrl; - return worker; - }; - }, - {}, - ], - 5: [ - function(require, module, exports) { - (function() { - 'use strict'; - var work = require('webworkify'); - var worker = work(require('./worker.js')); - var audio_context, audio_stream; - - /** - * The Recorder object. Sets up the onaudioprocess callback and communicates - * with the web worker to perform audio actions. - */ - var recorder = function(source, silenceDetectionConfig) { - silenceDetectionConfig = silenceDetectionConfig || {}; - silenceDetectionConfig.time = silenceDetectionConfig.hasOwnProperty( - 'time' - ) - ? silenceDetectionConfig.time - : 1500; - silenceDetectionConfig.amplitude = silenceDetectionConfig.hasOwnProperty( - 'amplitude' - ) - ? silenceDetectionConfig.amplitude - : 0.2; - - var recording = false, - currCallback, - start, - silenceCallback, - visualizationCallback; - - // Create a ScriptProcessorNode with a bufferSize of 4096 and a single input and output channel - var node = source.context.createScriptProcessor(4096, 1, 1); - - worker.onmessage = function(message) { - var blob = message.data; - currCallback(blob); - }; - - worker.postMessage({ - command: 'init', - config: { - sampleRate: source.context.sampleRate, - }, - }); - - /** - * Sets the silence and viz callbacks, resets the silence start time, and sets recording to true. - * @param {?onSilenceCallback} onSilence - Called when silence is detected. - * @param {?visualizerCallback} visualizer - Can be used to visualize the captured buffer. - */ - var record = function(onSilence, visualizer) { - silenceCallback = onSilence; - visualizationCallback = visualizer; - start = Date.now(); - recording = true; - }; - - /** - * Sets recording to false. - */ - var stop = function() { - recording = false; - }; - - /** - * Posts "clear" message to the worker. - */ - var clear = function() { - stop(); - worker.postMessage({ command: 'clear' }); - }; - - /** - * Sets the export callback and posts an "export" message to the worker. - * @param {onExportComplete} callback - Called when the export is complete. - * @param {sampleRate} The sample rate to use in the export. - */ - var exportWAV = function(callback, sampleRate) { - currCallback = callback; - worker.postMessage({ - command: 'export', - sampleRate: sampleRate, - }); - }; - - /** - * Checks the time domain data to see if the amplitude of the audio waveform is more than - * the silence threshold. If it is, "noise" has been detected and it resets the start time. - * If the elapsed time reaches the time threshold the silence callback is called. If there is a - * visualizationCallback it invokes the visualization callback with the time domain data. - */ - var analyse = function() { - analyser.fftSize = 2048; - var bufferLength = analyser.fftSize; - var dataArray = new Uint8Array(bufferLength); - var amplitude = silenceDetectionConfig.amplitude; - var time = silenceDetectionConfig.time; - - analyser.getByteTimeDomainData(dataArray); - - if (typeof visualizationCallback === 'function') { - visualizationCallback(dataArray, bufferLength); - } - - for (var i = 0; i < bufferLength; i++) { - // Normalize between -1 and 1. - var curr_value_time = dataArray[i] / 128 - 1.0; - if ( - curr_value_time > amplitude || - curr_value_time < -1 * amplitude - ) { - start = Date.now(); - } - } - var newtime = Date.now(); - var elapsedTime = newtime - start; - if (elapsedTime > time) { - silenceCallback(); - } - }; - - /** - * The onaudioprocess event handler of the ScriptProcessorNode interface. It is the EventHandler to be - * called for the audioprocess event that is dispatched to ScriptProcessorNode node types. - * @param {AudioProcessingEvent} audioProcessingEvent - The audio processing event. - */ - node.onaudioprocess = function(audioProcessingEvent) { - if (!recording) { - return; - } - worker.postMessage({ - command: 'record', - buffer: [audioProcessingEvent.inputBuffer.getChannelData(0)], - }); - analyse(); - }; - - var analyser = source.context.createAnalyser(); - analyser.minDecibels = -90; - analyser.maxDecibels = -10; - analyser.smoothingTimeConstant = 0.85; - - source.connect(analyser); - analyser.connect(node); - node.connect(source.context.destination); - - return { - record: record, - stop: stop, - clear: clear, - exportWAV: exportWAV, - }; - }; - - /** - * Audio recorder object. Handles setting up the audio context, - * accessing the mike, and creating the Recorder object. - */ - exports.audioRecorder = function() { - /** - * Creates an audio context and calls getUserMedia to request the mic (audio). - */ - var requestDevice = function() { - if (typeof audio_context === 'undefined') { - // @ts-ignore - window.AudioContext = - // @ts-ignore - window.AudioContext || window.webkitAudioContext; - audio_context = new AudioContext(); - } - - return navigator.mediaDevices - .getUserMedia({ audio: true }) - .then(function(stream) { - audio_stream = stream; - }); - }; - - var createRecorder = function(silenceDetectionConfig) { - return recorder( - audio_context.createMediaStreamSource(audio_stream), - silenceDetectionConfig - ); - }; - - var audioContext = function() { - return audio_context; - }; - - return { - requestDevice: requestDevice, - createRecorder: createRecorder, - audioContext: audioContext, - }; - }; - })(); - }, - { './worker.js': 6, webworkify: 4 }, - ], - 6: [ - function(require, module, exports) { - module.exports = function(self) { - 'use strict'; - var recLength = 0, - recBuffer = [], - recordSampleRate; - - self.addEventListener('message', function(e) { - switch (e.data.command) { - case 'init': - init(e.data.config); - break; - case 'record': - record(e.data.buffer); - break; - case 'export': - exportBuffer(e.data.sampleRate); - break; - case 'clear': - clear(); - break; - } - }); - - function init(config) { - recordSampleRate = config.sampleRate; - } - - function record(inputBuffer) { - recBuffer.push(inputBuffer[0]); - recLength += inputBuffer[0].length; - } - - function exportBuffer(exportSampleRate) { - var mergedBuffers = mergeBuffers(recBuffer, recLength); - var downsampledBuffer = downsampleBuffer( - mergedBuffers, - exportSampleRate - ); - var encodedWav = encodeWAV(downsampledBuffer); - var audioBlob = new Blob([encodedWav], { - type: 'application/octet-stream', - }); - // @ts-ignore - postMessage(audioBlob); - } - - function clear() { - recLength = 0; - recBuffer = []; - } - - function downsampleBuffer(buffer, exportSampleRate) { - if (exportSampleRate === recordSampleRate) { - return buffer; - } - var sampleRateRatio = recordSampleRate / exportSampleRate; - var newLength = Math.round(buffer.length / sampleRateRatio); - var result = new Float32Array(newLength); - var offsetResult = 0; - var offsetBuffer = 0; - while (offsetResult < result.length) { - var nextOffsetBuffer = Math.round( - (offsetResult + 1) * sampleRateRatio - ); - var accum = 0, - count = 0; - for ( - var i = offsetBuffer; - i < nextOffsetBuffer && i < buffer.length; - i++ - ) { - accum += buffer[i]; - count++; - } - result[offsetResult] = accum / count; - offsetResult++; - offsetBuffer = nextOffsetBuffer; - } - return result; - } - - function mergeBuffers(bufferArray, recLength) { - var result = new Float32Array(recLength); - var offset = 0; - for (var i = 0; i < bufferArray.length; i++) { - result.set(bufferArray[i], offset); - offset += bufferArray[i].length; - } - return result; - } - - function floatTo16BitPCM(output, offset, input) { - for (var i = 0; i < input.length; i++, offset += 2) { - var s = Math.max(-1, Math.min(1, input[i])); - output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); - } - } - - function writeString(view, offset, string) { - for (var i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)); - } - } - - function encodeWAV(samples) { - var buffer = new ArrayBuffer(44 + samples.length * 2); - var view = new DataView(buffer); - - writeString(view, 0, 'RIFF'); - view.setUint32(4, 32 + samples.length * 2, true); - writeString(view, 8, 'WAVE'); - writeString(view, 12, 'fmt '); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, 1, true); - view.setUint32(24, recordSampleRate, true); - view.setUint32(28, recordSampleRate * 2, true); - view.setUint16(32, 2, true); - view.setUint16(34, 16, true); - writeString(view, 36, 'data'); - view.setUint32(40, samples.length * 2, true); - floatTo16BitPCM(view, 44, samples); - - return view; - } - }; - }, - {}, - ], - }, - {}, - [3] -); diff --git a/packages/core/src/AudioControl/control.ts b/packages/core/src/AudioControl/control.ts new file mode 100644 index 00000000000..6aeb3a0cb66 --- /dev/null +++ b/packages/core/src/AudioControl/control.ts @@ -0,0 +1,152 @@ +import { exportBuffer } from './helper'; + +type SilenceDetectionConfig = { + time: number; + amplitude: number; +}; + +export class AudioRecorder { + private options: SilenceDetectionConfig; + private audioContext: AudioContext; + private audioSupported: boolean; + + private analyserNode: AnalyserNode; + private onSilence: Function; + + // input mic stream is stored in a buffer + private streamBuffer: Float32Array[] = []; + private streamBufferLength = 0; + + // recording props + private start: number; + private recording = false; + + constructor(options: SilenceDetectionConfig) { + this.options = options; + this.init(); + } + + private init() { + window.AudioContext = + window.AudioContext || (window as any).webkitAudioContext; + this.audioContext = new AudioContext(); + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then(stream => { + this.audioSupported = true; + this.setupAudioNodes(stream); + }) + .catch(() => { + this.audioSupported = false; + }); + } + + private setupAudioNodes(stream: MediaStream) { + const sourceNode = this.audioContext.createMediaStreamSource(stream); + const processorNode = this.audioContext.createScriptProcessor(4096, 1, 1); + + processorNode.onaudioprocess = audioProcessingEvent => { + if (!this.recording) return; + const stream = audioProcessingEvent.inputBuffer.getChannelData(0); + this.streamBuffer.push(new Float32Array(stream)); // set to a copy of the stream + this.streamBufferLength += stream.length; + this.analyse(); + }; + + const analyserNode = this.audioContext.createAnalyser(); + analyserNode.minDecibels = -90; + analyserNode.maxDecibels = -10; + analyserNode.smoothingTimeConstant = 0.85; + + sourceNode.connect(analyserNode); + analyserNode.connect(processorNode); + processorNode.connect(sourceNode.context.destination); + + this.analyserNode = analyserNode; + } + + public startRecording(onSilence?: Function) { + const silenceHandler = onSilence || function() {}; + this.onSilence = silenceHandler; + if (!this.audioSupported) return; + + const context = this.audioContext; + context.resume().then(() => { + this.start = Date.now(); + this.recording = true; + }); + } + + public stopRecording() { + if (!this.audioSupported) return; + this.recording = false; + } + + public clear() { + this.stopRecording(); + this.streamBufferLength = 0; + this.streamBuffer = []; + } + + public play(buffer: Uint8Array, callback: Function) { + if (!buffer) return; + const myBlob = new Blob([buffer]); + + const fileReader = new FileReader(); + fileReader.onload = () => { + const playbackSource = this.audioContext.createBufferSource(); + this.audioContext.decodeAudioData( + fileReader.result as ArrayBuffer, + buf => { + playbackSource.buffer = buf; + playbackSource.connect(this.audioContext.destination); + playbackSource.onended = function(event) { + if (typeof callback === 'function') { + callback(); + } + }; + playbackSource.start(0); + } + ); + }; + fileReader.readAsArrayBuffer(myBlob); + } + + private analyse() { + const analyser = this.analyserNode; + analyser.fftSize = 2048; + + const bufferLength = analyser.fftSize; + const dataArray = new Uint8Array(bufferLength); + const amplitude = this.options.amplitude; + const time = this.options.time; + + analyser.getByteTimeDomainData(dataArray); + + for (let i = 0; i < bufferLength; i++) { + // Normalize between -1 and 1. + const curr_value_time = dataArray[i] / 128 - 1.0; + if (curr_value_time > amplitude || curr_value_time < -1 * amplitude) { + this.start = Date.now(); + } + } + const newtime = Date.now(); + const elapsedTime = newtime - this.start; + if (elapsedTime > time) { + this.onSilence(); + } + } + + public exportWAV(callback: Function, exportSampleRate: number = 16000) { + if (!this.audioSupported) return; + const recordSampleRate = this.audioContext.sampleRate; + const blob = exportBuffer( + this.streamBuffer, + this.streamBufferLength, + recordSampleRate, + exportSampleRate + ); + callback(blob); + this.clear(); + } +} diff --git a/packages/core/src/AudioControl/helper.ts b/packages/core/src/AudioControl/helper.ts new file mode 100644 index 00000000000..3bb42443f7d --- /dev/null +++ b/packages/core/src/AudioControl/helper.ts @@ -0,0 +1,98 @@ +export const exportBuffer = ( + recBuffer: Float32Array[], + recLength: number, + recordSampleRate: number, + exportSampleRate: number +) => { + const mergedBuffers = mergeBuffers(recBuffer, recLength); + const downsampledBuffer = downsampleBuffer( + mergedBuffers, + recordSampleRate, + exportSampleRate + ); + const encodedWav = encodeWAV(downsampledBuffer, exportSampleRate); + const audioBlob = new Blob([encodedWav], { + type: 'application/octet-stream', + }); + return audioBlob; +}; + +const mergeBuffers = (bufferArray: Float32Array[], recLength: number) => { + const result = new Float32Array(recLength); + let offset = 0; + for (let i = 0; i < bufferArray.length; i++) { + result.set(bufferArray[i], offset); + offset += bufferArray[i].length; + } + return result; +}; + +const downsampleBuffer = ( + buffer: Float32Array, + recordSampleRate: number, + exportSampleRate: number +) => { + if (exportSampleRate === recordSampleRate) { + return buffer; + } + const sampleRateRatio = recordSampleRate / exportSampleRate; + const newLength = Math.round(buffer.length / sampleRateRatio); + const result = new Float32Array(newLength); + let offsetResult = 0; + let offsetBuffer = 0; + while (offsetResult < result.length) { + const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio); + let accum = 0, + count = 0; + for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) { + accum += buffer[i]; + count++; + } + result[offsetResult] = accum / count; + offsetResult++; + offsetBuffer = nextOffsetBuffer; + } + return result; +}; + +const floatTo16BitPCM = ( + output: DataView, + offset: number, + input: Float32Array +) => { + let byteOffset = offset; + for (let i = 0; i < input.length; i++, byteOffset += 2) { + const s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(byteOffset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } +}; + +const writeString = (view: DataView, offset: number, string: string) => { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +}; + +const encodeWAV = (samples: Float32Array, exportSampleRate: number) => { + const buffer = new ArrayBuffer(44 + samples.length * 2); + const view = new DataView(buffer); + writeString(view, 0, 'RIFF'); + view.setUint32(4, 32 + samples.length * 2, true); + writeString(view, 8, 'WAVE'); + + // fmt subchunk + writeString(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, exportSampleRate, true); + view.setUint32(28, exportSampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + + // data subchunk + writeString(view, 36, 'data'); + view.setUint32(40, samples.length * 2, true); + floatTo16BitPCM(view, 44, samples); + return view; +}; diff --git a/packages/core/src/AudioControl/index.ts b/packages/core/src/AudioControl/index.ts new file mode 100644 index 00000000000..ad828d59f33 --- /dev/null +++ b/packages/core/src/AudioControl/index.ts @@ -0,0 +1 @@ +export { AudioRecorder } from './control'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 99b699ccaa9..f08cc4db487 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -29,6 +29,7 @@ export { ServiceWorker } from './ServiceWorker'; export { ICredentials } from './types'; export { StorageHelper, MemoryStorage } from './StorageHelper'; export { Platform, getAmplifyUserAgent } from './Platform'; +export { AudioRecorder } from './AudioControl'; export * from './constants'; export const Constants = {