diff --git a/src/i18n/resources/en.json b/src/i18n/resources/en.json index ab71a62262..a30f05f4bc 100644 --- a/src/i18n/resources/en.json +++ b/src/i18n/resources/en.json @@ -357,6 +357,40 @@ "description": "Apply compression to audio (lowers the volume of the loudest parts of the signal and raises the volume of the softest parts)", "name": "Audio Compressor" }, + "audio-stream": { + "description": "Stream audio as PCM data over HTTP for external applications", + "menu": { + "port": { + "label": "Port" + }, + "quality-latency": { + "label": "Quality & Latency", + "submenu": { + "bit-depth": { + "label": "Bit Depth" + }, + "buffer-size": { + "label": "Buffer Size" + }, + "channels": { + "label": "Channels", + "mono": "Mono", + "stereo": "Stereo" + }, + "sample-rate": { + "label": "Sample Rate" + } + } + } + }, + "name": "Audio Stream", + "prompt": { + "port": { + "label": "Enter the port for the audio stream server:\nStream URL: {{streamUrl}}", + "title": "Audio Stream Port" + } + } + }, "auth-proxy-adapter": { "description": "Support for the use of authentication proxy services", "menu": { diff --git a/src/pear-desktop.ts b/src/pear-desktop.ts index b5ace4e1fe..c07e5b7f92 100644 --- a/src/pear-desktop.ts +++ b/src/pear-desktop.ts @@ -39,3 +39,9 @@ declare module '*.css?inline' { export default css; } + +declare module '*.js?raw' { + const javascript: string; + + export default javascript; +} diff --git a/src/plugins/audio-stream/BroadcastStream.ts b/src/plugins/audio-stream/BroadcastStream.ts new file mode 100644 index 0000000000..c5df99e9d6 --- /dev/null +++ b/src/plugins/audio-stream/BroadcastStream.ts @@ -0,0 +1,34 @@ +export class BroadcastStream { + private subscribers: Set> = + new Set(); + + // A way for readers to get a new stream + subscribe() { + let controller!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(c) { + controller = c; + }, + cancel: () => { + this.subscribers.delete(controller); + }, + }); + + this.subscribers.add(controller); + return stream; + } + + // A way for you to write data to all readers + write(chunk: Uint8Array) { + for (const controller of this.subscribers) { + controller.enqueue(chunk); + } + } + + close() { + for (const controller of this.subscribers) { + controller.close(); + } + this.subscribers.clear(); + } +} diff --git a/src/plugins/audio-stream/StreamProcessor.js b/src/plugins/audio-stream/StreamProcessor.js new file mode 100644 index 0000000000..374f1f1679 --- /dev/null +++ b/src/plugins/audio-stream/StreamProcessor.js @@ -0,0 +1,35 @@ +// audio-processor.js (loaded by audioContext.audioWorklet.addModule) +class RecorderProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const bufferSize = options.bufferSize || 4096; + // Prepare an interleaved stereo buffer [L,R,L,R,...] + this.buffer = new Float32Array(bufferSize * 2); + this.bufferIndex = 0; + } + + process(inputs, outputs) { + const input = inputs[0]; + if (input && input[0]) { + const left = input[0]; + const right = input[1] || left; // if mono input, duplicate for right + for (let i = 0; i < left.length; i++) { + this.buffer[this.bufferIndex++] = left[i]; + this.buffer[this.bufferIndex++] = right[i]; + if (this.bufferIndex >= this.buffer.length) { + // Buffer full: send a copy to the main thread + this.port.postMessage(new Float32Array(this.buffer)); + this.bufferIndex = 0; + } + } + } + // Optionally pass the audio through unchanged + if (outputs[0] && inputs[0]) { + outputs[0][0].set(inputs[0][0]); + if (inputs[0][1]) outputs[0][1].set(inputs[0][1]); + } + return true; + } +} + +registerProcessor('recorder-processor', RecorderProcessor); diff --git a/src/plugins/audio-stream/backend.ts b/src/plugins/audio-stream/backend.ts new file mode 100644 index 0000000000..c9f2d4eada --- /dev/null +++ b/src/plugins/audio-stream/backend.ts @@ -0,0 +1,118 @@ +import { Hono } from 'hono'; +import { streamText } from 'hono/streaming'; +import { serve, type ServerType } from '@hono/node-server'; + +import { createBackend } from '@/utils'; +import { type AudioStreamConfig } from './config'; +import { BroadcastStream } from './BroadcastStream'; + +const META_INT = 16_000; + +let config: AudioStreamConfig; +const broadcast = new BroadcastStream(); + +export const backend = createBackend< + { + app: Hono; + server?: ServerType; + }, + AudioStreamConfig +>({ + app: new Hono().get('/stream', (ctx) => { + const icyMetadata = ctx.req.header('Icy-Metadata'); + if (icyMetadata === '1') { + ctx.header('icy-metaint', META_INT.toString(10)); + ctx.header('icy-name', 'Pear Desktop'); + ctx.header('icy-url', 'https://github.com/pear-devs/pear-desktop'); + ctx.header( + 'icy-audio-info', + `ice-channels=2;ice-samplerate=${config.sampleRate.toString( + 10, + )};ice-bitrate=128`, + ); + ctx.header('icy-pub', '1'); + ctx.header('icy-sr', config.sampleRate.toString(10)); + ctx.header('Content-Type', 'audio/L16'); + ctx.header('Server', 'Pear Desktop'); + } + + return streamText(ctx, async (stream) => { + let readable = broadcast.subscribe(); + if (icyMetadata === '1') { + let bytesUntilMetadata = META_INT; + + readable = readable.pipeThrough( + new TransformStream({ + transform( + chunk: Uint8Array, + controller: TransformStreamDefaultController, + ) { + console.log({ bytesUntilMetadata }); + let offset = 0; + + while (offset < chunk.byteLength) { + if (bytesUntilMetadata === 0) { + const encoder = new TextEncoder(); + + // TODO: add real metadata + const metaBuffer = encoder.encode( + ".StreamTitle='My Cool Stream Title';", + ); + + const padding = (16 - (metaBuffer.byteLength % 16)) % 16; + const metaLength = metaBuffer.byteLength + padding; + const lengthByte = metaLength / 16; + + controller.enqueue(Uint8Array.from([lengthByte])); + + if (metaLength > 0) { + controller.enqueue(Uint8Array.from(metaBuffer)); + } + + bytesUntilMetadata = META_INT; + } + + const chunkRemaining = chunk.byteLength - offset; + const canSend = Math.min(chunkRemaining, bytesUntilMetadata); + controller.enqueue(chunk.subarray(offset, offset + canSend)); + + bytesUntilMetadata -= canSend; + offset += canSend; + } + }, + }), + ); + } + + return await stream.pipe(readable); + }); + }), + + async start({ getConfig, ipc }) { + config = await getConfig(); + + this.server = serve( + { + fetch: this.app.fetch.bind(this.app), + hostname: config.hostname, + port: config.port, + }, + ({ address, port }) => console.log('Listening on', { address, port }), + ); + + ipc.on('audio-stream:pcm-binary', (chunk: Uint8Array) => { + broadcast.write(chunk); + }); + }, + async stop() { + let resolve; + + const promise = new Promise((r) => (resolve = r)); + this.server?.close(resolve); + + await promise; + }, + onConfigChange(newConfig) { + config = newConfig; + }, +}); diff --git a/src/plugins/audio-stream/config.ts b/src/plugins/audio-stream/config.ts new file mode 100644 index 0000000000..2ccac25f48 --- /dev/null +++ b/src/plugins/audio-stream/config.ts @@ -0,0 +1,22 @@ +export interface AudioStreamConfig { + enabled: boolean; + port: number; + hostname: string; + // Audio quality settings for PCM streaming + sampleRate: number; // Audio sample rate (e.g., 44100, 48000, 96000) + bitDepth: number; // Bit depth (16 or 32) + channels: number; // Number of channels (1 = mono, 2 = stereo) + bufferSize: number; // Audio buffer size (1024, 2048, 4096, 8192) - affects latency +} + +export const defaultAudioStreamConfig: AudioStreamConfig = { + enabled: false, + port: 8765, + hostname: '0.0.0.0', + // High quality audio settings for local network + // Using 48kHz/16-bit for stability - can increase to 96kHz/32-bit once working + sampleRate: 48000, // 48kHz - high quality and widely supported + bitDepth: 16, // 16-bit - reliable and well-tested + channels: 2, // Stereo + bufferSize: 2048, // Low latency buffer size +}; diff --git a/src/plugins/audio-stream/index.ts b/src/plugins/audio-stream/index.ts new file mode 100644 index 0000000000..8504216cb8 --- /dev/null +++ b/src/plugins/audio-stream/index.ts @@ -0,0 +1,17 @@ +import { t } from '@/i18n'; +import { createPlugin } from '@/utils'; + +import { defaultAudioStreamConfig } from './config'; +import { backend } from './backend'; +import { onMenu } from './menu'; +import { renderer } from './renderer'; + +export default createPlugin({ + name: () => t('plugins.audio-stream.name'), + description: () => t('plugins.audio-stream.description'), + restartNeeded: false, + config: defaultAudioStreamConfig, + backend, + renderer, + menu: onMenu, +}); diff --git a/src/plugins/audio-stream/menu.ts b/src/plugins/audio-stream/menu.ts new file mode 100644 index 0000000000..8be990eb6b --- /dev/null +++ b/src/plugins/audio-stream/menu.ts @@ -0,0 +1,127 @@ +import prompt from 'custom-electron-prompt'; + +import { t } from '@/i18n'; +import promptOptions from '@/providers/prompt-options'; + +import type { MenuContext } from '@/types/contexts'; +import type { MenuTemplate } from '@/menu'; + +import { + type AudioStreamConfig, + defaultAudioStreamConfig, +} from './config'; + +// Quality and latency presets +const SAMPLE_RATES = [44100, 48000, 96000]; +const BIT_DEPTHS = [16, 32]; +const CHANNELS = [1, 2]; +const BUFFER_SIZES = [1024, 2048, 4096, 8192]; + +export const onMenu = async ({ + getConfig, + setConfig, + window, +}: MenuContext): Promise => { + const config = await getConfig(); + + return [ + { + label: t('plugins.audio-stream.menu.port.label'), + type: 'normal', + async click() { + const config = await getConfig(); + + const currentPort = config.port || defaultAudioStreamConfig.port; + const streamUrl = `http://localhost:${currentPort}/stream`; + + const newPort = + (await prompt( + { + title: t('plugins.audio-stream.prompt.port.title'), + label: t('plugins.audio-stream.prompt.port.label', { + streamUrl, + }), + value: config.port, + type: 'counter', + counterOptions: { minimum: 1, maximum: 65535 }, + width: 450, + ...promptOptions(), + }, + window, + )) ?? + config.port ?? + defaultAudioStreamConfig.port; + + if (newPort !== config.port) { + await setConfig({ ...config, port: newPort }); + } + }, + }, + { + label: t('plugins.audio-stream.menu.quality-latency.label'), + type: 'submenu', + submenu: [ + { + label: t('plugins.audio-stream.menu.quality-latency.submenu.sample-rate.label'), + type: 'submenu', + submenu: SAMPLE_RATES.map((sampleRate) => ({ + label: `${sampleRate} Hz`, + type: 'radio' as const, + checked: config.sampleRate === sampleRate, + async click() { + const currentConfig = await getConfig(); + if (currentConfig.sampleRate !== sampleRate) { + await setConfig({ ...currentConfig, sampleRate }); + } + }, + })), + }, + { + label: t('plugins.audio-stream.menu.quality-latency.submenu.bit-depth.label'), + type: 'submenu', + submenu: BIT_DEPTHS.map((bitDepth) => ({ + label: `${bitDepth}-bit`, + type: 'radio' as const, + checked: config.bitDepth === bitDepth, + async click() { + const currentConfig = await getConfig(); + if (currentConfig.bitDepth !== bitDepth) { + await setConfig({ ...currentConfig, bitDepth }); + } + }, + })), + }, + { + label: t('plugins.audio-stream.menu.quality-latency.submenu.channels.label'), + type: 'submenu', + submenu: CHANNELS.map((channels) => ({ + label: channels === 1 ? t('plugins.audio-stream.menu.quality-latency.submenu.channels.mono') : t('plugins.audio-stream.menu.quality-latency.submenu.channels.stereo'), + type: 'radio' as const, + checked: config.channels === channels, + async click() { + const currentConfig = await getConfig(); + if (currentConfig.channels !== channels) { + await setConfig({ ...currentConfig, channels }); + } + }, + })), + }, + { + label: t('plugins.audio-stream.menu.quality-latency.submenu.buffer-size.label'), + type: 'submenu', + submenu: BUFFER_SIZES.map((bufferSize) => ({ + label: `${bufferSize} samples`, + type: 'radio' as const, + checked: config.bufferSize === bufferSize, + async click() { + const currentConfig = await getConfig(); + if (currentConfig.bufferSize !== bufferSize) { + await setConfig({ ...currentConfig, bufferSize }); + } + }, + })), + }, + ], + }, + ]; +}; diff --git a/src/plugins/audio-stream/renderer.ts b/src/plugins/audio-stream/renderer.ts new file mode 100644 index 0000000000..9d0389ce7a --- /dev/null +++ b/src/plugins/audio-stream/renderer.ts @@ -0,0 +1,311 @@ +import { createRenderer } from '@/utils'; + +import workletCode from './StreamProcessor.js?raw'; + +import type { RendererContext } from '@/types/contexts'; +import type { MusicPlayer } from '@/types/music-player'; + +import type { AudioStreamConfig } from './config'; + +type ProcessingQueueItem = { + buffer: Int16Array | Int32Array; + metadata: { + timestamp: number; + sampleRate: number; + bitDepth: number; + channels: number; + }; +}; + +type RendererProperties = { + audioContext?: AudioContext; + audioSource?: AudioNode; + scriptProcessor?: ScriptProcessorNode; + config?: AudioStreamConfig; + context?: RendererContext; + isStreaming: boolean; + batchBuffer: Int16Array | Int32Array | null; + batchCount: number; + processingQueue: ProcessingQueueItem[]; + isProcessing: boolean; + startStreaming: ( + ipc: RendererContext<{ enabled: boolean }>['ipc'], + audioContext: AudioContext, + audioSource: AudioNode, + ) => void; +}; + +function writeString(view: DataView, offset: number, str: string) { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} + +function createWavHeader( + sampleRate: number, + numChannels: number, + dataLength: number, +) { + const header = new ArrayBuffer(44); + const view = new DataView(header); + writeString(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataLength, true); // file size - 8 + writeString(view, 8, 'WAVE'); + writeString(view, 12, 'fmt '); + view.setUint32(16, 16, true); // subchunk1 size (16 for PCM) + view.setUint16(20, 1, true); // audio format (1 = PCM) + view.setUint16(22, numChannels, true); // number of channels + view.setUint32(24, sampleRate, true); // sample rate + view.setUint32(28, sampleRate * numChannels * 2, true); // byte rate + view.setUint16(32, numChannels * 2, true); // block align + view.setUint16(34, 16, true); // bits per sample + writeString(view, 36, 'data'); + view.setUint32(40, dataLength, true); // data chunk size + return new Uint8Array(header); +} + +export const renderer = createRenderer({ + isStreaming: false, + batchBuffer: null, + batchCount: 0, + processingQueue: [], + isProcessing: false, + + async onPlayerApiReady( + _: MusicPlayer, + context: RendererContext, + ) { + this.context = context; + this.config = await context.getConfig(); + + if (!this.config.enabled) { + return; + } + + // Wait for audio to be ready + document.addEventListener( + 'peard:audio-can-play', + (e) => { + this.startStreaming( + context.ipc, + e.detail.audioContext, + e.detail.audioSource, + ); + }, + { once: true, passive: true }, + ); + }, + + startStreaming( + ipc: RendererContext<{ enabled: boolean }>['ipc'], + audioContext: AudioContext, + audioSource: AudioNode, + bufferSize = 4096, + ) { + if (this.isStreaming || !this.context) { + return; + } + + this.audioContext = audioContext; + this.audioSource = audioSource; + + const sampleRate = audioContext.sampleRate; + + const blob = new Blob([workletCode], { type: 'application/javascript' }); + const blobUrl = URL.createObjectURL(blob); + + try { + audioContext.audioWorklet + .addModule(blobUrl) + .then(() => { + const workletNode = new AudioWorkletNode( + audioContext, + 'recorder-processor', + { + sampleRate: this.config!.sampleRate, + bufferSize: bufferSize, + }, + ); + + workletNode.port.onmessage = (event) => { + // Received a Float32Array of interleaved stereo samples from the worklet + const float32Data = event.data; + + // Convert floats [-1,1] to 16-bit PCM + const int16Buffer = new ArrayBuffer(float32Data.length * 2); + const view = new DataView(int16Buffer); + for (let i = 0; i < float32Data.length; i++) { + const s = Math.max(-1, Math.min(1, float32Data[i])); // clamp + // Scale to 16-bit signed range + view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } + const pcmData = new Uint8Array(int16Buffer); + + // Build WAV header (16-bit, stereo, given sample rate, data length = pcmData.byteLength) + const wavHeader = createWavHeader( + audioContext.sampleRate, + 2, + pcmData.byteLength, + ); + + // Combine header + PCM data into one Uint8Array + const wavChunk = new Uint8Array(wavHeader.length + pcmData.length); + wavChunk.set(wavHeader, 0); + wavChunk.set(pcmData, wavHeader.length); + + ipc.send('audio-stream:pcm-binary', wavChunk); + }; + + audioSource.connect(workletNode); + this.isStreaming = true; + }) + .catch((err) => { + console.error( + '[Audio Stream] Failed to add audio worklet module:', + err, + ); + }); + } catch (err) { + console.error('[Audio Stream] AudioWorklet setup failed:', err); + } + this.isStreaming = true; + + console.log('[Audio Stream] Started PCM streaming:'); + }, + + stop() { + this.isStreaming = false; + + // Clear processing queue to prevent sending stale data + this.processingQueue = []; + this.isProcessing = false; + + // Flush any remaining batched data + if (this.batchBuffer && this.batchBuffer.length > 0 && this.context) { + try { + let buffer: ArrayBuffer; + if (this.batchBuffer.buffer instanceof SharedArrayBuffer) { + buffer = new ArrayBuffer(this.batchBuffer.buffer.byteLength); + new Uint8Array(buffer).set(new Uint8Array(this.batchBuffer.buffer)); + } else { + buffer = this.batchBuffer.buffer; + } + const uint8 = new Uint8Array(buffer); + this.context.ipc.send('audio-stream:pcm-binary', uint8); + } catch { + // Ignore flush errors + } + this.batchBuffer = null; + this.batchCount = 0; + } + + if (this.scriptProcessor) { + try { + this.scriptProcessor.disconnect(); + } catch { + // Ignore disconnect errors + } + this.scriptProcessor = undefined; + } + + this.audioContext = undefined; + this.audioSource = undefined; + }, + + onConfigChange(config: AudioStreamConfig) { + const wasEnabled = this.config?.enabled; + const oldBitDepth = this.config?.bitDepth; + const oldChannels = this.config?.channels; + const oldBufferSize = this.config?.bufferSize; + + // Check if quality/latency settings changed + const qualityChanged = + oldBitDepth !== config.bitDepth || + oldChannels !== config.channels || + oldBufferSize !== config.bufferSize; + + this.config = config; + + if (config.enabled && !wasEnabled) { + // Wait for audio to be ready if not already streaming + if (!this.isStreaming && this.audioContext && this.audioSource) { + // Already have audio context, start immediately + this.startStreaming( + this.context!.ipc, + this.audioContext, + this.audioSource, + ); + } else if (!this.isStreaming) { + // Wait for audio to be ready + document.addEventListener( + 'peard:audio-can-play', + (e) => { + this.startStreaming( + this.context!.ipc, + e.detail.audioContext, + e.detail.audioSource, + ); + }, + { once: true, passive: true }, + ); + } + } else if (!config.enabled && wasEnabled) { + // Stop streaming + this.isStreaming = false; + + if (this.scriptProcessor) { + try { + this.scriptProcessor.disconnect(); + } catch { + // Ignore disconnect errors + } + this.scriptProcessor = undefined; + } + + this.audioContext = undefined; + this.audioSource = undefined; + } else if ( + config.enabled && + wasEnabled && + qualityChanged && + this.isStreaming + ) { + // Quality/latency settings changed while streaming - restart with new settings + if (this.audioContext && this.audioSource) { + // Stop current streaming + this.isStreaming = false; + + // Clear processing queue to prevent sending stale data + this.processingQueue = []; + this.isProcessing = false; + + // Store references before cleanup + const audioContext = this.audioContext; + const audioSource = this.audioSource; + + if (this.scriptProcessor) { + try { + this.scriptProcessor.disconnect(); + } catch (error) { + // Ignore disconnect errors + } + this.scriptProcessor = undefined; + } + + // Use requestAnimationFrame to ensure cleanup is complete before restarting + requestAnimationFrame(() => { + // Double-check we're not streaming and have valid references + if ( + audioContext && + audioSource && + !this.isStreaming && + this.context + ) { + // Restart with new settings - this will send new config to backend + this.startStreaming(this.context.ipc, audioContext, audioSource); + } + }); + } + } + }, +}); diff --git a/src/plugins/audio-stream/test-gui.html b/src/plugins/audio-stream/test-gui.html new file mode 100644 index 0000000000..5f1817a370 --- /dev/null +++ b/src/plugins/audio-stream/test-gui.html @@ -0,0 +1,816 @@ + + + + + + PCM Audio Stream Test GUI + + + +
+

🎵 PCM Audio Stream Test

+

Test the audio streaming plugin with real-time PCM playback

+ +
+
+ + Disconnected +
+ +
+ + +
+ +
+ + +
+
+ + + +
+
📝 Log
+
+
+
+ + + + +