From 4c78b1f93e00c2832daa7130e92d2a5b5d6bbd8a Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 13 Apr 2015 01:21:04 -0400 Subject: [PATCH] Added opus streaming --- README.md | 7 +- oggopus.js | 211 ++++++++++++++++++---------------------------- recorder.js | 6 +- recorderWorker.js | 26 ++++-- wavepcm.js | 2 +- 5 files changed, 107 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 399f622c..0b07c9dc 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,9 @@ Creates a recorder instance. Instantiating an instance will prompt the user for - **bufferLength** - (*optional*) The length of the buffer that the internal JavaScriptNode uses to capture the audio. Can be tweaked if experiencing performance issues. Defaults to 4096. - **monitorGain** - (*optional*) Sets the gain of the monitoring output. Gain is an a-weighted value between 0 and 1. Defaults to 0 - **numberOfChannels** - (*optional*) The number of channels to record. 1 = mono, 2 = stereo. Defaults to 1. More than two channels has not been tested. -- **recordOpus** - (*optional*) Specifies if recorder should record using the opus encoder. Defaults to true. -- **sampleRate** - (*optional*) Specifies the sample rate to record at. Defaults to device sample rate. If resampling occurs, the audio is filtered with a 6th order butterworth filter and then resampled using a gaussing convolution. The Opus encoder will not work if the value is not 8000, 12000, 16000, 24000 or 48000. +- **recordOpus** - (*optional*) Specifies if recorder should record using the opus encoder. Defaults to true. If set to { stream : true }, then dataAvailable event will fire when each page is ready. Additionaly if stream is true, you can specify a maximum number of buffers per page to reduce latency using { stream: true, maxBuffersPerPage: 10 }. At 44100 Hz, 10 buffers will be ~1 second of latency. +- **sampleRate** - (*optional*) Specifies the sample rate to record at. Defaults to device sample rate. If different than native rate, the audio will be filtered and resampled. If recordOpus is true, this value will default to 48000. +The Opus encoder will not work if the value is not 8000, 12000, 16000, 24000 or 48000. - **workerPath** - (*optional*) Path to recorder.js worker script. Defaults to 'recorderWorker.js' @@ -44,7 +45,7 @@ Creates a recorder instance. Instantiating an instance will prompt the user for rec.requestData() -**requestData** will request the recorded data if not recording. If successful, the event "dataAvailable" will be published with a blob containing the appropriate data as an ogg or wav file depending on config. +**requestData** will request the recorded data if not recording. If successful, the event "dataAvailable" will be published with a blob containing the appropriate data as an ogg or wav file depending on config. requestData will not work if recordOpus stream is enabled, as the data is being streamed and not recorded. rec.resume() diff --git a/oggopus.js b/oggopus.js index d14476fd..d333ee21 100644 --- a/oggopus.js +++ b/oggopus.js @@ -1,39 +1,34 @@ importScripts( 'libopus.js', 'wavepcm.js' ); var OggOpus = function( config ){ - this.numberOfChannels = config.numberOfChannels; this.inputSampleRate = config.inputSampleRate; this.outputSampleRate = config.outputSampleRate; - this.bitDepth = config.bitDepth; - this.encoderApplication = config.encoderApplication || 2049; // 2048 = Voice, 2049 = Full Band Audio, 2051 = Restricted Low Delay + this.onPageComplete = config.onPageComplete || this.onPageComplete; + this.maxBuffersPerPage = config.recordOpus.maxBuffersPerPage || 40; // Limit latency for streaming + this.encoderApplication = config.recordOpus.encoderApplication || 2049; // 2048 = Voice, 2049 = Full Band Audio, 2051 = Restricted Low Delay this.encoderFrameSize = config.encoderFrameSize || 20; // 20ms frame - this.granuleIncrement = 48 * this.encoderFrameSize this.wavepcm = new WavePCM( config ); - this.packets = []; + + this.pageIndex = 0; + this.granulePosition = 0; + this.segmentData = new Uint8Array( 65025 ); + this.segmentDataIndex = 0; + this.segmentTable = new Uint8Array( 255 ); + this.segmentTableIndex = 0; + this.pages = []; + this.fileLength = 0; + this.buffersInPage = 0; this.initChecksumTable(); this.initCodec(); -}; - -OggOpus.prototype.decode = function( packets ) { - var outputSampleLength; - var decodedAudio = new Int16Array( packets.length * this.encoderBufferLength ); - - for ( var i = 0; i < packets.length; i++ ) { - this.decoderBuffer.set( packets[i] ); - outputSampleLength = _opus_decode( this.decoder, this.decoderBufferPointer, packets[i].byteLength, this.decoderOutputPointer, this.decoderOutputMaxLength, 0); - decodedAudio.set( this.decoderOutputBuffer, i*this.encoderBufferLength ); - } - - return new Uint8Array( decodedAudio.buffer ); + this.generateIdPage(); + this.generateCommentPage(); }; OggOpus.prototype.encode = function( samples ) { - var outputPackets = []; var sampleIndex = 0; var lengthToCopy; - var outputPacketLength; while ( sampleIndex < samples.length ) { @@ -43,13 +38,22 @@ OggOpus.prototype.encode = function( samples ) { this.encoderBufferIndex += lengthToCopy; if ( this.encoderBufferIndex === this.encoderBufferLength ) { - outputPacketLength = _opus_encode_float( this.encoder, this.encoderBufferPointer, this.encoderSamplesPerChannelPerPacket, this.encoderOutputPointer, this.encoderOutputMaxLength ); - outputPackets.push( new Uint8Array( this.encoderOutputBuffer.subarray(0, outputPacketLength) ) ); + var packetLength = _opus_encode_float( this.encoder, this.encoderBufferPointer, this.encoderSamplesPerChannelPerPacket, this.encoderOutputPointer, this.encoderOutputMaxLength ); + this.segmentPacket( packetLength ); this.encoderBufferIndex = 0; } } - return outputPackets; + this.buffersInPage++; + if ( this.buffersInPage >= this.maxBuffersPerPage ) { + this.generatePage(); + } +}; + +OggOpus.prototype.encodeFinalFrame = function() { + this.encode( new Float32Array( this.encoderBufferLength - this.encoderBufferIndex ) ); + this.headerType += 4; + this.generatePage(); }; OggOpus.prototype.getChecksum = function( data ){ @@ -60,51 +64,22 @@ OggOpus.prototype.getChecksum = function( data ){ return checksum >>> 0; }; -OggOpus.prototype.getCommentPage = function( pageIndex ){ - var segmentDataBuffer = new ArrayBuffer( 24 ); - var segmentDataView = new DataView( segmentDataBuffer ); - var segmentData = new Uint8Array( segmentDataBuffer ); - var segmentTable = new Uint8Array(1); - - segmentTable[0] = segmentData.length; +OggOpus.prototype.generateCommentPage = function(){ + var segmentDataView = new DataView( this.segmentData.buffer ); segmentDataView.setUint32( 0, 1332770163, false ) // Magic Signature 'Opus' segmentDataView.setUint32( 4, 1415669619, false ) // Magic Signature 'Tags' segmentDataView.setUint32( 8, 8, true ); // Vendor Length segmentDataView.setUint32( 12, 1382376303, false ); // Vendor name 'Reco' segmentDataView.setUint32( 16, 1919182194, false ); // Vendor name 'rder' segmentDataView.setUint32( 20, 0, true ); // User Comment List Length - - return this.getPage( 0, 0, pageIndex, segmentTable, segmentData ); + this.segmentTableIndex = 1; + this.segmentDataIndex = this.segmentTable[0] = 24; + this.headerType = 0; + this.generatePage(); }; -OggOpus.prototype.getFile = function( pageData ){ - var lastPage = pageData[ pageData.length-1 ]; - var oggFile = new Uint8Array( lastPage.fileOffset + lastPage.segmentTable.length + lastPage.segmentData.length + 27 ); - var oggPageIndex = 0; - - oggFile.set( this.getIdPage( oggPageIndex++ ) ); - oggFile.set( this.getCommentPage( oggPageIndex++ ), 47 ); - - for ( var i = 0; i < pageData.length; i++ ) { - oggFile.set( this.getPage( - pageData[i].headerType, - pageData[i].granulePosition, - oggPageIndex++, - pageData[i].segmentTable, - pageData[i].segmentData - ), pageData[i].fileOffset ); - } - - return oggFile; -}; - -OggOpus.prototype.getIdPage = function( pageIndex ){ - var segmentDataBuffer = new ArrayBuffer( 19 ); - var segmentDataView = new DataView( segmentDataBuffer ); - var segmentData = new Uint8Array( segmentDataBuffer ); - var segmentTable = new Uint8Array(1); - - segmentTable[0] = segmentData.length; +OggOpus.prototype.generateIdPage = function(){ + var segmentDataView = new DataView( this.segmentData.buffer ); segmentDataView.setUint32( 0, 1332770163, false ) // Magic Signature 'Opus' segmentDataView.setUint32( 4, 1214603620, false ) // Magic Signature 'Head' segmentDataView.setUint8( 8, 1, true ); // Version @@ -113,19 +88,21 @@ OggOpus.prototype.getIdPage = function( pageIndex ){ segmentDataView.setUint32( 12, this.inputSampleRate, true ); // original sample rate segmentDataView.setUint16( 16, 0, true ); // output gain segmentDataView.setUint8( 18, 0, true ); // channel map 0 = mono or stereo - - return this.getPage( 2, 0, pageIndex, segmentTable, segmentData ); + this.segmentTableIndex = 1; + this.segmentDataIndex = this.segmentTable[0] = 19; + this.headerType = 2; + this.generatePage(); }; -OggOpus.prototype.getPage = function( headerType, granulePosition, pageIndex, segmentTable, segmentData ){ - var numberOfSegments = segmentTable.length; - var pageBuffer = new ArrayBuffer( 27 + numberOfSegments + segmentData.length ); +OggOpus.prototype.generatePage = function(){ + var granulePosition = ( this.lastPositiveGranulePosition === this.granulePosition) ? -1 : this.granulePosition; + var pageBuffer = new ArrayBuffer( 27 + this.segmentTableIndex + this.segmentDataIndex ); var pageBufferView = new DataView( pageBuffer ); var page = new Uint8Array( pageBuffer ); pageBufferView.setUint32( 0, 1332176723, false); // Capture Pattern starts all page headers 'OggS' pageBufferView.setUint8( 4, 0, true ); // Version - pageBufferView.setUint8( 5, headerType, true ); // 1 = continuation, 2 = beginning of stream, 4 = end of stream + pageBufferView.setUint8( 5, this.headerType, true ); // 1 = continuation, 2 = beginning of stream, 4 = end of stream // Number of samples upto and including this page at 48000Hz, into 64 bits pageBufferView.setUint32( 6, granulePosition, true ); @@ -134,13 +111,19 @@ OggOpus.prototype.getPage = function( headerType, granulePosition, pageIndex, se } pageBufferView.setUint32( 14, 0, true ); // Bitstream serial number - pageBufferView.setUint32( 18, pageIndex, true ); // Page sequence number - pageBufferView.setUint8( 26, numberOfSegments, true ); // Number of segments in page. - page.set( segmentTable, 27 ); // Segment Table - page.set( segmentData, 27 + numberOfSegments ); // Segment Data + pageBufferView.setUint32( 18, this.pageIndex++, true ); // Page sequence number + pageBufferView.setUint8( 26, this.segmentTableIndex, true ); // Number of segments in page. + page.set( this.segmentTable.subarray(0, this.segmentTableIndex), 27 ); // Segment Table + page.set( this.segmentData.subarray(0, this.segmentDataIndex), 27 + this.segmentTableIndex ); // Segment Data pageBufferView.setUint32( 22, this.getChecksum( page ), true ); // Checksum - return page; + this.onPageComplete( page ); + this.segmentTableIndex = 0; + this.segmentDataIndex = 0; + this.buffersInPage = 0; + if ( granulePosition > 0 ) { + this.lastPositiveGranulePosition = granulePosition; + } }; OggOpus.prototype.initChecksumTable = function(){ @@ -164,75 +147,43 @@ OggOpus.prototype.initCodec = function() { this.encoderOutputMaxLength = 4000; this.encoderOutputPointer = _malloc( this.encoderOutputMaxLength ); this.encoderOutputBuffer = HEAPU8.subarray( this.encoderOutputPointer, this.encoderOutputPointer + this.encoderOutputMaxLength ); +}; - this.decoder = _opus_decoder_create( this.outputSampleRate, this.numberOfChannels, allocate(4, 'i32', ALLOC_STACK) ); - this.decoderBufferPointer = _malloc( this.encoderOutputMaxLength ); - this.decoderBuffer = HEAPU8.subarray( this.decoderBufferPointer, this.decoderBufferPointer + this.encoderOutputMaxLength ); - this.decoderOutputMaxLength = this.encoderBufferLength * 2; // 2 bytes per sample - this.decoderOutputPointer = _malloc( this.decoderOutputMaxLength ); - this.decoderOutputBuffer = HEAP16.subarray( this.decoderOutputPointer >> 1, (this.decoderOutputPointer + this.decoderOutputMaxLength) >> 1 ); +OggOpus.prototype.onPageComplete = function( page ){ + this.fileLength += page.length; + this.pages.push( page ); }; OggOpus.prototype.recordBuffers = function( buffers ) { - this.packets.push.apply( this.packets, this.encode( this.wavepcm.resampleAndInterleave( buffers ) ) ); + this.encode( this.wavepcm.resampleAndInterleave( buffers ) ); }; -OggOpus.prototype.requestData = function(){ - return this.getFile( this.segmentPackets( this.packets ) ); +OggOpus.prototype.requestData = function() { + var data = new Uint8Array( this.fileLength ); + var offset = 0; + for ( var i = 0; i < this.pages.length; i++ ) { + data.set( this.pages[i], offset ); + offset += this.pages[i].length; + } + return data; }; -OggOpus.prototype.segmentPackets = function( packets ) { - var segmentTable = new Uint8Array( 255 ); - var segmentTableIndex = 0; - var segmentData = new Uint8Array( 65025 ); - var segmentDataIndex = 0; - var granulePosition = 0; - var headerType = 0; - var lastPositiveGranulePosition = 0; - var segmentedPackets = []; - var fileOffset = 99; // size of comment and id page - var pageComplete = function(){ - - segmentedPackets.push({ - segmentTable: new Uint8Array( segmentTable.subarray(0, segmentTableIndex) ), - segmentData: new Uint8Array( segmentData.subarray(0, segmentDataIndex) ), - headerType: headerType, - granulePosition: (lastPositiveGranulePosition === granulePosition) ? -1 : granulePosition, - fileOffset: fileOffset - }); - - fileOffset += 27 + segmentTableIndex + segmentDataIndex; - segmentTableIndex = 0; - segmentDataIndex = 0; - if ( segmentedPackets[segmentedPackets.length-1].granulePosition !== -1 ) { - lastPositiveGranulePosition = granulePosition; - } - }; - - for ( var i = 0; i < packets.length; i++ ) { - var remainingPacketLength = packets[i].length; - var packetIndex = 0; - - while ( remainingPacketLength >= 0 ) { +OggOpus.prototype.segmentPacket = function( packetLength ) { + var packetIndex = 0; - if ( segmentTableIndex === 255 ) { - pageComplete(); - headerType = ( remainingPacketLength >= 255 ) ? 1 : 0; - } + while ( packetLength >= 0 ) { + var segmentLength = Math.min( packetLength, 255 ); + this.segmentTable[ this.segmentTableIndex++ ] = segmentLength; + this.segmentData.set( this.encoderOutputBuffer.subarray( packetIndex, packetIndex + segmentLength ), this.segmentDataIndex ); + this.segmentDataIndex += segmentLength; + packetIndex += segmentLength; + packetLength -= 255; - var dataLength = Math.min( remainingPacketLength, 255 ); - segmentData.set( packets[i].subarray( packetIndex, dataLength ), segmentDataIndex ); - segmentTable[ segmentTableIndex++ ] = dataLength; - packetIndex += dataLength; - segmentDataIndex += dataLength; - remainingPacketLength -= 255; + if ( this.segmentTableIndex === 255 ) { + this.generatePage(); + this.headerType = ( packetLength >= 0 ) ? 1 : 0; } - - granulePosition += this.granuleIncrement; } - headerType += 4; - pageComplete(); - - return segmentedPackets; -}; \ No newline at end of file + this.granulePosition += ( 48 * this.encoderFrameSize ); +}; diff --git a/recorder.js b/recorder.js index 7ae9b760..f065b094 100755 --- a/recorder.js +++ b/recorder.js @@ -8,8 +8,8 @@ var Recorder = function( config ){ } config = config || {}; - config.recordOpus = config.recordOpus === false ? false : true; - config.bitDepth = (config.recordOpus ? 16 : config.bitDepth) || 16; + config.recordOpus = (config.recordOpus === false) ? false : config.recordOpus || true; + config.bitDepth = config.recordOpus ? 16 : config.bitDepth || 16; config.bufferLength = config.bufferLength || 4096; config.monitorGain = config.monitorGain || 0; config.numberOfChannels = config.numberOfChannels || 1; @@ -164,7 +164,7 @@ Recorder.prototype.stop = function(){ this.scriptProcessorNode.disconnect(); this.state = "inactive"; this.eventTarget.dispatchEvent( new Event( 'stop' ) ); - this.requestData(); + this.worker.postMessage({ command: "requestData" }); this.worker.postMessage({ command: "stop" }); } }; diff --git a/recorderWorker.js b/recorderWorker.js index 4d651bc0..a7bc59c9 100644 --- a/recorderWorker.js +++ b/recorderWorker.js @@ -1,28 +1,38 @@ this.onmessage = function( e ){ + var worker = this; switch( e.data.command ){ case 'recordBuffers': - this.recorder.recordBuffers( e.data.buffers ); + worker.recorder.recordBuffers( e.data.buffers ); break; case 'requestData': - var data = this.recorder.requestData(); - this.postMessage( data, [data.buffer] ); + if ( worker.recorder.encodeFinalFrame ) { + worker.recorder.encodeFinalFrame(); + } + if ( !worker.recordOpus.stream ) { + var data = worker.recorder.requestData(); + worker.postMessage( data, [data.buffer] ); + } break; case 'stop': - this.close(); + worker.close(); break; case 'start': - if ( e.data.recordOpus ) { + worker.recordOpus = e.data.recordOpus; + if ( worker.recordOpus ) { importScripts( 'oggopus.js' ); - this.recorder = new OggOpus( e.data ); + if ( worker.recordOpus.stream ) { + e.data.onPageComplete = function( page ){ worker.postMessage( page, [page.buffer] ); }; + } + worker.recorder = new OggOpus( e.data ); } else { importScripts( 'wavepcm.js' ); - this.recorder = new WavePCM( e.data ); + worker.recorder = new WavePCM( e.data ); } break; } -}; \ No newline at end of file +}; diff --git a/wavepcm.js b/wavepcm.js index ad1c3371..d497acdd 100644 --- a/wavepcm.js +++ b/wavepcm.js @@ -137,7 +137,7 @@ WavePCM.prototype.resampleAndInterleave = function( buffers ) { var channelData = buffers[ channel ]; for ( var tap = -1; tap < 2; tap++ ) { - var sampleValue = channelData[ nearestPoint + tap ] || this.cachedSamples[channel][ 1 + tap ]; + var sampleValue = channelData[ nearestPoint + tap ] || this.cachedSamples[channel][ 1 + tap ] || channelData[ nearestPoint ]; outputData[ i * this.numberOfChannels + channel ] += sampleValue * this.magicKernel( resampleValue - nearestPoint - tap ); } }