diff --git a/package-lock.json b/package-lock.json index 2c0f218..96b7883 100644 --- a/package-lock.json +++ b/package-lock.json @@ -84,7 +84,7 @@ "use-debounced-effect": "^2.0.1", "util": "^0.12.5", "uuid": "^9.0.0", - "wavesurfer.js": "^7.1.1", + "wavesurfer.js": "file:///Users/rewbs/code/wavesurfer.js", "web-vitals": "^3.4.0" }, "devDependencies": { @@ -109,6 +109,24 @@ "react-git-info": "^2.0.1" } }, + "../wavesurfer.js": { + "version": "7.1.1", + "license": "BSD-3-Clause", + "devDependencies": { + "@rollup/plugin-terser": "^0.4.3", + "@rollup/plugin-typescript": "^11.1.2", + "@typescript-eslint/eslint-plugin": "^5.57.0", + "@typescript-eslint/parser": "^5.57.0", + "cypress": "^12.9.0", + "cypress-image-snapshot": "^4.0.1", + "eslint": "^8.37.0", + "eslint-config-prettier": "^8.8.0", + "eslint-plugin-prettier": "^4.2.1", + "prettier": "^2.8.7", + "rollup": "^3.26.2", + "typescript": "^5.0.4" + } + }, "node_modules/@adobe/css-tools": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.1.0.tgz", @@ -25468,9 +25486,8 @@ } }, "node_modules/wavesurfer.js": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.1.1.tgz", - "integrity": "sha512-nChYa5M4tOGkTP4EtzyHXY3pk/T7bI2ttv6A5wFPS/3+jjQ8I85esqqPn52+ZJyE72r2hpgk863xekgRswAcaw==" + "resolved": "../wavesurfer.js", + "link": true }, "node_modules/wbuf": { "version": "1.7.3", @@ -45235,9 +45252,21 @@ } }, "wavesurfer.js": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.1.1.tgz", - "integrity": "sha512-nChYa5M4tOGkTP4EtzyHXY3pk/T7bI2ttv6A5wFPS/3+jjQ8I85esqqPn52+ZJyE72r2hpgk863xekgRswAcaw==" + "version": "file:../wavesurfer.js", + "requires": { + "@rollup/plugin-terser": "^0.4.3", + "@rollup/plugin-typescript": "^11.1.2", + "@typescript-eslint/eslint-plugin": "^5.57.0", + "@typescript-eslint/parser": "^5.57.0", + "cypress": "^12.9.0", + "cypress-image-snapshot": "^4.0.1", + "eslint": "^8.37.0", + "eslint-config-prettier": "^8.8.0", + "eslint-plugin-prettier": "^4.2.1", + "prettier": "^2.8.7", + "rollup": "^3.26.2", + "typescript": "^5.0.4" + } }, "wbuf": { "version": "1.7.3", diff --git a/package.json b/package.json index 94ecbc2..d7930ec 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "use-debounced-effect": "^2.0.1", "util": "^0.12.5", "uuid": "^9.0.0", - "wavesurfer.js": "^7.1.1", + "wavesurfer.js": "file:///Users/rewbs/code/wavesurfer.js", "web-vitals": "^3.4.0" }, "scripts": { diff --git a/src/Labs.tsx b/src/Labs.tsx index e1e0fb4..cf1a877 100644 --- a/src/Labs.tsx +++ b/src/Labs.tsx @@ -1,129 +1,89 @@ -import { Box, MenuItem, TextField } from "@mui/material"; +import { Box, Button, MenuItem, TextField } from "@mui/material"; import Grid from '@mui/material/Unstable_Grid2'; -import { MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import WaveSurfer from "wavesurfer.js"; -import { SmallTextField } from "./components/SmallTextField"; +import { SupportedColorScheme, experimental_extendTheme as extendTheme, useColorScheme } from "@mui/material/styles"; +import { useEffect, useMemo, useRef, useState } from "react"; +import MinimapPlugin from "wavesurfer.js/dist/plugins/minimap"; +import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram"; import { BiquadFilter } from "./components/BiquadFilter"; +import { SmallTextField } from "./components/SmallTextField"; +import { TimelineOptions, ViewportOptions, WaveSurferPlayer } from "./components/WaveSurferPlayer"; +import { themeFactory } from "./theme"; import { getWavBytes } from "./utils/utils"; -import { saveAs } from 'file-saver'; - -// WaveSurfer hook -const useWavesurfer = (containerRef: MutableRefObject, options: any) => { - const [wavesurfer, setWavesurfer] = useState(null) - - // Initialize wavesurfer when the container mounts - // or any of the props change - useEffect(() => { - if (!containerRef.current) return - - console.log("---> Creating wavesurfer") - const ws = WaveSurfer.create({ - ...options, - container: containerRef.current, - }) - - setWavesurfer(ws) - - return () => { - console.log("<--- Destroying wavesurfer") - ws.destroy() - } - }, [options, containerRef]) - - return wavesurfer -} - -type WaveSurferPlayerProps = { - audioFile: Blob | undefined; -} - -// Create a React component that will render wavesurfer. -// Props are wavesurfer options. - -const staticOptions = {} - -const WaveSurferPlayer = ({ audioFile }: WaveSurferPlayerProps) => { - const containerRef = useRef() - const [isPlaying, setIsPlaying] = useState(false) - const [currentTime, setCurrentTime] = useState(0) - const wavesurfer = useWavesurfer(containerRef, staticOptions) - - // On play button click - const onPlayClick = useCallback(() => { - wavesurfer?.isPlaying() ? wavesurfer?.pause() : wavesurfer?.play() - }, [wavesurfer]) - - // Initialize wavesurfer when the container mounts - // or any of the props change - useEffect(() => { - if (!wavesurfer) return - - setCurrentTime(0) - setIsPlaying(false) - - const subscriptions = [ - wavesurfer.on('play', () => setIsPlaying(true)), - wavesurfer.on('pause', () => setIsPlaying(false)), - wavesurfer.on('timeupdate', (currentTime) => setCurrentTime(currentTime)), - - wavesurfer.on('load', () => console.log('load event')), - wavesurfer.on('decode', () => console.log('decode event')), - wavesurfer.on('ready', () => console.log('ready event')), - - ] - - if (audioFile) { - try { - wavesurfer.loadBlob(audioFile); - wavesurfer.loadBlob(audioFile); - } catch (e: any) { - console.error(`Failed to load '${audioFile}':`, e) - } - } - - return () => { - subscriptions.forEach((unsub) => unsub()) - } - }, [wavesurfer, audioFile]) - - return ( - <> - {/* @ts-ignore */} -
- - - -

Seconds played: {currentTime}

- - ) -} +import colormap from './data/hot-colormap.json'; const Labs = () => { + console.log("Rendering Labs"); const [fps, setFps] = useState(30); const [bpm, setBpm] = useState(120); const [xaxisType, setXaxisType] = useState<"frames" | "seconds" | "beats">("frames"); const [startFrame, setStartFrame] = useState(0); const [endFrame, setEndFrame] = useState(0); + const [startVisibleFrame, setStartVisibleFrame] = useState(0); + const [endVisibleFrame, setEndVisibleFrame] = useState(0); + + const [timelineOptions, setTimelineOptions] = useState({ fps, bpm, xaxisType }); + const [viewport, setViewport] = useState({ startFrame, endFrame }); + const [audioBuffer, setAudioBuffer] = useState(); const [audioFile, setAudioFile] = useState(); const fileInput = useRef("" as any); + const theme = extendTheme(themeFactory()); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { colorScheme, setColorScheme } = useColorScheme(); + const palette = theme.colorSchemes[(colorScheme || 'light') as SupportedColorScheme].palette; + + const wsOptions = useMemo(() => ({ + barWidth: 0, + normalize: true, + fillParent: true, + minPxPerSec: 10, + autoCenter: false, + interact: true, + cursorColor: palette.success.light, + waveColor: [palette.waveformStart.main, palette.waveformEnd.main], + progressColor: [palette.waveformProgressMaskStart.main, palette.waveformProgressMaskEnd.main], + cursorWidth: 1, + plugins: [ + //@ts-ignore + //RegionsPlugin.create(), + SpectrogramPlugin.create({ + labels: true, + height: 75, + colorMap: colormap + }), + //@ts-ignore + MinimapPlugin.create({ + //container: "#minimap", + height: 20, + waveColor: [palette.waveformStart.main, palette.waveformEnd.main], + progressColor: [palette.waveformProgressMaskStart.main, palette.waveformProgressMaskEnd.main], + }) + ], + }), [palette]); + + useEffect(() => { + setViewport({ startFrame, endFrame }); + }, [startFrame, endFrame]) + + useEffect(() => { + setTimelineOptions({ fps, bpm, xaxisType }); + }, [bpm, fps, xaxisType]) + const loadFile = async (event: any) => { try { const selectedFiles = fileInput.current.files; if (!selectedFiles || selectedFiles.length < 1) { return; } - // Prepare audio buffer for analysis. const selectedFile = selectedFiles[0]; const arrayBuffer = await selectedFile.arrayBuffer(); const audioContext = new AudioContext(); const newAudioBuffer = await audioContext.decodeAudioData(arrayBuffer); setAudioBuffer(newAudioBuffer); setAudioFile(selectedFile); + setViewport({ ...viewport }); // Force scroll and zoom back to same position after load. } catch (e: any) { console.error(e); } @@ -146,61 +106,79 @@ const Labs = () => { }, }}> - setFps(parseFloat(e.target.value))} - /> - setBpm(parseFloat(e.target.value))} - /> - setStartFrame(parseInt(e.target.value))} - /> - setEndFrame(parseInt(e.target.value))} - /> + setXaxisType(e.target.value as "frames" | "seconds" | "beats")} - > - Frames - Seconds - Beats - - + select + fullWidth={false} + size="small" + style={{ width: '8em', marginLeft: '5px' }} + label={"Show time as: "} + InputLabelProps={{ shrink: true, }} + InputProps={{ style: { fontSize: '0.75em' } }} + value={xaxisType} + onChange={(e) => setXaxisType(e.target.value as "frames" | "seconds" | "beats")} + > + Frames + Seconds + Beats + + setFps(parseFloat(e.target.value))} + /> + setBpm(parseFloat(e.target.value))} + /> + setStartFrame(parseInt(e.target.value))} + /> + setEndFrame(parseInt(e.target.value))} + /> + + + - {audioBuffer && { - const wavBytes = getWavBytes(updatedAudioBuffer.getChannelData(0).buffer, { - isFloat: true, // floating point or 16-bit integer - numChannels: 1, - sampleRate: updatedAudioBuffer.sampleRate, - }) - const blob = new Blob([wavBytes], { type: 'audio/wav' }) - setAudioFile(blob); - //TODO - there's some kind of memory leak after running this a few times. - }} - /> } + {audioBuffer && { + const wavBytes = getWavBytes(updatedAudioBuffer.getChannelData(0).buffer, { + isFloat: true, // floating point or 16-bit integer + numChannels: 1, + sampleRate: updatedAudioBuffer.sampleRate, + }) + const blob = new Blob([wavBytes], { type: 'audio/wav' }) + setAudioFile(blob); + }} + />} @@ -215,10 +193,15 @@ const Labs = () => { { + setStartVisibleFrame(startFrame); + setEndVisibleFrame(endFrame); + }} /> - {} - - +

Visble frame range: {startVisibleFrame.toFixed(2)} - {endVisibleFrame.toFixed(2)}

 diff --git a/src/components/AudioWaveform.tsx b/src/components/AudioWaveform.tsx index 8533053..ac98169 100644 --- a/src/components/AudioWaveform.tsx +++ b/src/components/AudioWaveform.tsx @@ -3,18 +3,16 @@ import Tooltip from './PatchedToolTip'; import Fade from '@mui/material/Fade'; import Grid from '@mui/material/Unstable_Grid2'; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import WaveSurfer from "wavesurfer.js"; -import TimelinePlugin from "wavesurfer.js/dist/plugins/timeline.js"; - -import RegionsPlugin from 'wavesurfer.js/dist/plugins/regions.js'; -import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram.js"; -import Minimap from "wavesurfer.js/dist/plugins/minimap.js"; +import RegionsPlugin from 'wavesurfer.js/dist/plugins/regions'; +import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram"; +import MinimapPlugin from "wavesurfer.js/dist/plugins/minimap"; +import TimelinePlugin from "wavesurfer.js/dist/plugins/timeline"; import colormap from '../data/hot-colormap.json'; //@ts-ignore import debounce from 'lodash.debounce'; import { frameToBeat, secToFrame, secToBeat, frameToSec, beatToSec } from "../utils/maths"; -import { createAudioBufferCopy } from "../utils/utils"; +import { createAudioBufferCopy, getWavBytes } from "../utils/utils"; import { SmallTextField } from "./SmallTextField"; import { TabPanel } from "./TabPanel"; import { BiquadFilter } from "./BiquadFilter"; @@ -22,13 +20,14 @@ import { useHotkeys } from 'react-hotkeys-hook' import { CssVarsPalette, Palette, SupportedColorScheme, experimental_extendTheme as extendTheme, useColorScheme } from "@mui/material/styles"; import { themeFactory } from "../theme"; import _ from "lodash"; +import WaveSurferPlayer, { TimelineOptions, ViewportOptions } from "./WaveSurferPlayer"; type AudioWaveformProps = { fps: number, bpm: number, xaxisType: "frames" | "seconds" | "beats", - viewport: { startFrame: number, endFrame: number }, + viewport: ViewportOptions, keyframesPositions: number[], gridCursorPos: number, beatMarkerInterval: number, @@ -39,57 +38,28 @@ type AudioWaveformProps = { } -// TODO move to Utils -function calculateNiceStepSize(roughStepSize : number) : number { - const exponent = Math.floor(Math.log10(roughStepSize)); - const normalizedStepSize = roughStepSize / Math.pow(10, exponent); - - let niceStepSize; - if (normalizedStepSize < 1.5) { - niceStepSize = 1; - } else if (normalizedStepSize < 3) { - niceStepSize = 2; - } else if (normalizedStepSize < 7) { - niceStepSize = 5; - } else { - niceStepSize = 10; - } - - return niceStepSize * Math.pow(10, exponent); - } - -// Used by the audio reference view in the Main UI. -// TODO: extract shared component with WavesurferWaveform.tsx +// The parent component on the main view that includes the waveform, the controls, the analysis tabs etc... export function AudioWaveform(props: AudioWaveformProps) { + //console.log("Initialising AudioReference with props: ", props); - //console.log("Initialising Waveform with props: ", props); const analysisBufferSize = 4096; const analysisHopSize = 256; - const wavesurferRef = useRef(); - const [timelinePlugin, setTimelinePlugin] = useState(); - const fileInput = useRef("" as any); const waveformRef = useRef(null); - const [isPlaying, setIsPlaying] = useState(false); - const [playbackPos, setPlaybackPos] = useState(); - const [trackLength, setTrackLength] = useState(0); const [statusMessage, setStatusMessage] = useState(<>); const [lastPxPerSec, setLastPxPerSec] = useState(0); const [lastViewport, setLastViewport] = useState({ startFrame: 0, endFrame: 0 }); - const [prevXaxisType, setPrevXaxisType] = useState<"frames" | "seconds" | "beats" | undefined>(); + const [timelineOptions, setTimelineOptions] = useState({ fps: props.fps, bpm: props.bpm, xaxisType: props.xaxisType }); + const [viewport, setViewport] = useState(props.viewport); const [prevTimelineOptions, setPrevTimelineOptions] = useState(); const [tab, setTab] = useState(1); - // if the user clicks on the waveform or pauses the track, we capture the position - // so that we can repeatedly play back from that position with ctrl+space - const [capturedPos, setCapturedPos] = useState(0); - const [audioBuffer, setAudioBuffer] = useState(); - const [prevAudioBuffer, setPrevAudioBuffer] = useState(); - const [unfilteredAudioBuffer, setUnfilteredAudioBuffer] = useState(); + const [audioFile, setAudioFile] = useState(); + //const [unfilteredAudioBuffer, setUnfilteredAudioBuffer] = useState(); const [isAnalysing, setIsAnalysing] = useState(false); const [isLoaded, setIsLoaded] = useState(false); @@ -117,231 +87,38 @@ export function AudioWaveform(props: AudioWaveformProps) { const palette = theme.colorSchemes[(colorScheme || 'light') as SupportedColorScheme].palette; const [prevPalette, setPrevPalette] = useState(); - // Triggered when user makes viewport changes outside of wavesurfer, and we need to update wavesurfer. - const scrollToPosition = useCallback((startFrame: number) => { - if (!trackLength) { - //console.log('No track loaded'); - return; - } - if (!wavesurferRef.current) { - console.log('WaveSurfer not yet initialised.'); - return; - } - - // Convert the frame position to pixels - const pxPerSec = wavesurferRef.current.options.minPxPerSec; - const startSec = startFrame / props.fps; - const startPx = startSec * pxPerSec; - - // Update the scroll position - // TODO – how to SET scrollposition with wavesurfer 7? - // wavesurferRef.current.drawer.wrapper.scrollLeft = startPx; - }, [trackLength, props.fps]); - - - // Triggered when user scrolls wavesurfer itself - const onScroll = debounce(useCallback(() => { - if (!trackLength) { - //console.log('No track loaded'); - return; - } - - //console.log("Scrolling. Old viewport:", lastViewport); - if (!wavesurferRef.current || !waveformRef.current) { - return; - } - const pxPerSec = wavesurferRef.current.options.minPxPerSec; - const startFrame = Math.round(wavesurferRef.current.getScroll() / pxPerSec * props.fps); - const endFrame = Math.round((wavesurferRef.current.getScroll() + waveformRef.current.clientWidth) / pxPerSec * props.fps); - - // Check both start and end frame because of elastic scroll - // at full right scroll mistakenly triggering a zoom-in. - if (startFrame === lastViewport.startFrame - || endFrame === lastViewport.endFrame) { - //console.log("no scrolling required"); - return; - } - const newViewport = { startFrame, endFrame }; - //console.log("New viewport:", newViewport); - if ((newViewport.startFrame !== lastViewport.startFrame - || newViewport.endFrame !== lastViewport.endFrame) - && newViewport.startFrame < newViewport.endFrame - && newViewport.startFrame >= 0) { - - setLastViewport(newViewport); - - // Communicate the new viewport to the parent component - props.onScroll(newViewport); - //console.log("Scrolled viewport to:", newViewport); - } - - }, [props, lastViewport, trackLength]), 1); - - const debouncedOnCursorMove = useMemo(() => debounce(props.onCursorMove, 1), [props]); - - - const formatTimeCallback = useCallback((sec: number) => { - switch (props.xaxisType) { - case "frames": - return `${(sec * props.fps).toFixed(0)}`; - case "seconds": - return `${sec.toFixed(2)}`; - case "beats": - return `${(sec * props.bpm / 60).toFixed(2)}`; - } - }, [props]); - - - // Rebuild & re-register the timeline plugin when necessary, e.g.: - // - on zoom, because intervals may change - // - on xaxis unit change, because we need to relabel everything - // - on bpm/fps change, because the label position may change - // - on screen resize, because we need to recalculate the number of labels - const registerNewTimeline = useCallback(() => { - const startTime = window.performance.now(); - - if (!wavesurferRef.current) { - return - } - - // Establish how many pixels are currently used to show each second of audio (i.e. zoom level) - const pxPerSec = wavesurferRef.current.options.minPxPerSec; - - // Get pixels available in the waveform viewport - const wsWidth = wavesurferRef.current.getWrapper().clientWidth; - - // Calculate the number of units that can be displayed in the viewport - const secondsInView = wsWidth / pxPerSec; - let unitsInView : number; - switch (props.xaxisType) { - case "seconds": - unitsInView = secondsInView; - break; - case "frames": - unitsInView = secToFrame(secondsInView, props.fps); - break; - case "beats": - unitsInView = secToBeat(secondsInView, props.bpm); - break; - } - - // Max number of ticks to display is 10 per 100px, rounded to the closest 10. - const maxTicks = Math.floor(wsWidth / 100) * 10; - - // Units per tick should be a power of 10, so that we can label the ticks with sensible numbers. - // We also want to avoid having too many ticks, so we round up to the nearest power of 10. - //const unitsPerTick = Math.pow(10, Math.ceil(Math.log10(unitsInView / maxTicks))); - - const roughStepSize = unitsInView / maxTicks; - const niceStepSize = calculateNiceStepSize(roughStepSize); - - let secondsPerTick : number; - switch (props.xaxisType) { - case "seconds": - secondsPerTick = niceStepSize; - break; - case "frames": - secondsPerTick = frameToSec(niceStepSize, props.fps); - break; - case "beats": - secondsPerTick = beatToSec(niceStepSize, props.bpm); - break; - } - - const newTimelineOptions = { - formatTimeCallback: formatTimeCallback, - timeInterval: secondsPerTick, - //primaryLabelInterval: secondsPerTick*2, - //secondaryLabelInterval: secondsPerTick*5, - }; - - // Check whether a new timeline is needed - if (! _.isEqual(newTimelineOptions, prevTimelineOptions)) { - console.log(newTimelineOptions, prevTimelineOptions); - - // Remove the previous timeline - if (timelinePlugin) { - console.log("Destroying plugin") - timelinePlugin.destroy(); - } - const newtimelinePlugin = TimelinePlugin.create(newTimelineOptions) + const wsOptions = useMemo(() => ({ + barWidth: 0, + normalize: true, + fillParent: true, + minPxPerSec: 10, + autoCenter: false, + interact: true, + cursorColor: palette.success.light, + waveColor: [palette.waveformStart.main, palette.waveformEnd.main], + progressColor: [palette.waveformProgressMaskStart.main, palette.waveformProgressMaskEnd.main], + cursorWidth: 1, + plugins: [ //@ts-ignore - wavesurferRef.current.registerPlugin(newtimelinePlugin); - setTimelinePlugin(newtimelinePlugin); - setPrevTimelineOptions(newTimelineOptions); - - } else { - console.log(`Unnecessarily computed new timeline options in ${(window.performance.now() - startTime)/1000}ms`); - } - - }, [formatTimeCallback, props.bpm, props.fps, props.xaxisType, timelinePlugin, prevTimelineOptions]); - - - - useEffect(() => { - - let wavesurfer: WaveSurfer; - - // Recreate wavesurfer iff the audio buffer or color scheme has changed - // TODO - when do we need to expliclty destroy? - // if (audioBuffer !== prevAudioBuffer || palette.mode !== prevPalette?.mode) { - // if (wavesurferRef.current) { - // wavesurferRef.current.destroy(); - // wavesurferRef.current = undefined; - // } - // setPrevAudioBuffer(audioBuffer); - // setPrevPalette(palette); - // } - - if (waveformRef.current && !wavesurferRef.current) { - console.log("Preparing wavesurfer.") - wavesurfer = WaveSurfer.create({ - container: waveformRef.current, - normalize: true, - //scrollParent: true, - fillParent: false, - minPxPerSec: 10, - autoCenter: false, - interact: true, - - cursorColor: palette.success.light, + //RegionsPlugin.create(), + SpectrogramPlugin.create({ + labels: true, + height: 75, + colorMap: colormap + }), + //@ts-ignore + MinimapPlugin.create({ + //container: "#minimap", + height: 20, waveColor: [palette.waveformStart.main, palette.waveformEnd.main], progressColor: [palette.waveformProgressMaskStart.main, palette.waveformProgressMaskEnd.main], - cursorWidth: 1, - plugins: [ - //@ts-ignore - RegionsPlugin.create(), - // SpectrogramPlugin.create({ - // container: "#spectrogram", - // labels: true, - // height: 75, - // colorMap: colormap - // }), - // Minimap.create({ - // container: "#minimap", - // height: 20, - // waveColor: '#ddd', - // progressColor: '#999', - // }) - ], - }); - - wavesurfer.on("ready", () => { - //console.log("WaveSurfer7 is ready.") - }); - - // TODO - I think this should be valid but currently breaks things - // return () => { - // wavesurfer?.destroy(); - // } - - wavesurferRef.current = wavesurfer; - - } else { - console.log("NOT preparing wavesurfer.", wavesurferRef.current) - } + }) + ], + }), [palette]); - }, [audioBuffer, formatTimeCallback, palette, prevAudioBuffer, prevPalette?.mode, registerNewTimeline]); + useEffect(() => { + setTimelineOptions({ fps: props.fps, bpm: props.bpm, xaxisType: props.xaxisType }); + }, [props.fps, props.bpm, props.xaxisType]) // // Update the colours manually on palette change. // // This is necessary because we are not recreating wavesurfer that often @@ -356,22 +133,15 @@ export function AudioWaveform(props: AudioWaveformProps) { // }, [palette]); + // const handleDoubleClick = useCallback((event: any) => { + // const time = wavesurferRef.current?.getCurrentTime(); + // //@ts-ignore + // const newMarkers = [...manualEvents, time].sort((a, b) => a - b) + // //@ts-ignore + // setManualEvents(newMarkers); + // }, [manualEvents]); - - const handleDoubleClick = useCallback((event: any) => { - const time = wavesurferRef.current?.getCurrentTime(); - //@ts-ignore - const newMarkers = [...manualEvents, time].sort((a, b) => a - b) - //@ts-ignore - setManualEvents(newMarkers); - }, [manualEvents]); - - const handleClick = useCallback((event: any) => { - const time = wavesurferRef.current?.getCurrentTime(); - setCapturedPos(time || 0); - }, []); - // const handleMarkerDrop = useCallback((marker: Marker) => { // //console.log("In handleMarkerDrop", marker); // const draggedEventType = deduceMarkerType(marker); @@ -413,35 +183,6 @@ export function AudioWaveform(props: AudioWaveformProps) { // return "unknown"; // }; - const updatePlaybackPos = useCallback(() => { - const lengthFrames = trackLength * props.fps; - const curPosSeconds = wavesurferRef.current?.getCurrentTime() || 0; - const curPosFrames = curPosSeconds * props.fps; - setPlaybackPos(`${curPosSeconds.toFixed(3)}/${trackLength.toFixed(3)}s - (frame: ${curPosFrames.toFixed(0)}/${lengthFrames.toFixed(0)}) - (beat: ${frameToBeat(curPosFrames, props.fps, props.bpm).toFixed(2)}/${frameToBeat(lengthFrames, props.fps, props.bpm).toFixed(2)})`); - - debouncedOnCursorMove(curPosFrames); - - }, [trackLength, debouncedOnCursorMove, props]); - - //Update wavesurfer's seek callback on track length change and fps change - useEffect(() => { - updatePlaybackPos(); - wavesurferRef.current?.on("seeking", updatePlaybackPos); - return () => { - wavesurferRef.current?.un("seeking", updatePlaybackPos); - } - }, [updatePlaybackPos]); - - //Update wavesurfer's scroll callback when the viewport changes - useEffect(() => { - onScroll(); - wavesurferRef.current?.on("scroll", onScroll); - return () => { - wavesurferRef.current?.un("scroll", onScroll); - } - }, [onScroll]); // Update wavesurfer's double-click callback when manual events change // TODO: we have to explicitly register the event handler on the drawer element. @@ -481,62 +222,19 @@ export function AudioWaveform(props: AudioWaveformProps) { - // Update position, zoom and timeline if the viewport has changed. - if (props.viewport) { - - //TODO we could get pxPerSec from the wavesurfer object - check whether everything still works if we do that. - const pxPerSec = (waveformRef.current?.clientWidth ?? 600) / ((props.viewport.endFrame - props.viewport.startFrame) / props.fps); - - // console.log("duration", wavesurferRef.current?.getDuration()); - // console.log("viewport", props.viewport); - // console.log("starting point in seconds", props.viewport.startFrame/props.fps); - // console.log("seconds to display", (props.viewport.endFrame-props.viewport.startFrame)/props.fps); - // console.log("width in px", waveformRef.current?.clientWidth); - // console.log("pxPerSec", pxPerSec); - - ; - - if (lastPxPerSec !== pxPerSec && (wavesurferRef.current?.getDuration()||0)>0) { - setLastPxPerSec(pxPerSec); - wavesurferRef.current?.zoom(pxPerSec); - scrollToPosition(props.viewport.startFrame); - } - if (props.viewport.startFrame !== lastViewport.startFrame || props.viewport.endFrame !== lastViewport.endFrame) { - setLastViewport({ ...props.viewport }) - } - if (props.viewport.startFrame !== lastViewport.startFrame) { - scrollToPosition(props.viewport.startFrame); - } - } - - if (props.xaxisType !== prevXaxisType) { - setPrevXaxisType(props.xaxisType); - } - - registerNewTimeline(); - - - const loadFile = async (event: any) => { try { const selectedFiles = fileInput.current.files; if (!selectedFiles || selectedFiles.length < 1) { - setStatusMessage(Select an audio file above.); return; } - setIsLoaded(true); - - // Prepare audio buffer for analysis. const selectedFile = selectedFiles[0]; const arrayBuffer = await selectedFile.arrayBuffer(); const audioContext = new AudioContext(); const newAudioBuffer = await audioContext.decodeAudioData(arrayBuffer); - setTrackLength(newAudioBuffer.duration); setAudioBuffer(newAudioBuffer); - setUnfilteredAudioBuffer(createAudioBufferCopy(newAudioBuffer)); - - // Load audio file into wavesurfer visualisation - wavesurferRef.current?.loadBlob(selectedFile); + setAudioFile(selectedFile); + setViewport({ ...viewport }); // Force scroll and zoom back to same position after load. //updateMarkers(); event.target.blur(); // Remove focus from the file input so that spacebar doesn't trigger it again (and can be used for immediate playback) @@ -546,22 +244,6 @@ export function AudioWaveform(props: AudioWaveformProps) { } } - function playPause(from: number = -1, pauseIfPlaying = true) { - if (isPlaying && pauseIfPlaying) { - wavesurferRef.current?.pause(); - setIsPlaying(false); - } else { - if (from >= 0) { - wavesurferRef.current?.setTime(from); - } if (!isPlaying) { - setCapturedPos(wavesurferRef.current?.getCurrentTime() || 0); - wavesurferRef.current?.play(); - setIsPlaying(true); - } - } - updatePlaybackPos(); - } - // const updateMarkers = useCallback(() => { // wavesurferRef.current?.markers.clear(); @@ -649,55 +331,6 @@ export function AudioWaveform(props: AudioWaveformProps) { // }, [updateMarkers]); - function timeInterval(pxPerSec: number) { - var retval = 1; - if (pxPerSec >= 25 * 100) { - retval = 0.01; - } else if (pxPerSec >= 25 * 40) { - retval = 0.025; - } else if (pxPerSec >= 25 * 10) { - retval = 0.1; - } else if (pxPerSec >= 25 * 4) { - retval = 0.25; - } else if (pxPerSec >= 25) { - retval = 1; - } else if (pxPerSec * 5 >= 25) { - retval = 5; - } else if (pxPerSec * 15 >= 25) { - retval = 15; - } else { - retval = Math.ceil(0.5 / pxPerSec) * 60; - } - return retval; - } - - // Force timeline to pick up new drawing method and - // to redraw when formatTimeCallback changes - // TODO this is being called way too often. Every click triggers it. - useEffect(() => { - if (wavesurferRef.current) { - // Destroy the existing timeline instance - //wavesurferRef.current.getActivePlugins().find((plugin) => plugin. .name === "timeline")?.destroy(); - - // Re-create the timeline instance with - - - //wavesurferRef.current.initPlugin('timeline'); - // HACK to force the timeline position to update. - // if (props.viewport.startFrame > 0) { - // scrollToPosition(props.viewport.startFrame - 1); - // scrollToPosition(props.viewport.startFrame); - // } - - // Force the wavesurfer to redraw the timeline - if (wavesurferRef.current.getDuration() > 0) { - wavesurferRef.current.zoom(wavesurferRef.current.options.minPxPerSec); - } - - } - }, [formatTimeCallback, props.viewport.startFrame, scrollToPosition, palette, registerNewTimeline]); - - const estimateBPM = (): void => { if (!audioBuffer) { console.log("No buffer to analyse."); @@ -745,7 +378,7 @@ export function AudioWaveform(props: AudioWaveformProps) { const newDetectedEvents: number[] = []; onsetDetectionWorker.onmessage = (e: any) => { - if (onsetRef.current && wavesurferRef.current && e.data.eventS) { + if (onsetRef.current && e.data.eventS) { newDetectedEvents.push(e.data.eventS); onsetRef.current.innerText = `${newDetectedEvents.length} events detected`; } @@ -796,31 +429,19 @@ export function AudioWaveform(props: AudioWaveformProps) { props.onAddKeyframes(frames, infoLabel); } - useHotkeys('space', - () => playPause(), - { preventDefault: true, scopes: ['main'] }, - [playPause]); - - useHotkeys('shift+space', - () => playPause(0, false), - { preventDefault: true, scopes: ['main'] }, - [playPause]); + // useHotkeys('shift+a', + // () => { + // // TODO might need to pass a custom handler down from here to the player + // //const time = wavesurferRef.current?.getCurrentTime(); + // //@ts-ignore + // const newMarkers = [...manualEvents, time].sort((a, b) => a - b) + // //@ts-ignore + // setManualEvents(newMarkers); + // }, + // { preventDefault: true, scopes: ['main'] }, + // [manualEvents]) - useHotkeys('ctrl+space', - () => playPause(capturedPos, false), - { preventDefault: true, scopes: ['main'] }, - [playPause, capturedPos]); - useHotkeys('shift+a', - () => { - const time = wavesurferRef.current?.getCurrentTime(); - //@ts-ignore - const newMarkers = [...manualEvents, time].sort((a, b) => a - b) - //@ts-ignore - setManualEvents(newMarkers); - }, - { preventDefault: true, scopes: ['main'] }, - [manualEvents]) return <> @@ -829,27 +450,30 @@ export function AudioWaveform(props: AudioWaveformProps) { File: e.target.value = null // Ensures onChange fires even if same file is re-selected. + onClick={ e => { + //@ts-ignore + e.target.value = null; + } } type="file" accept=".mp3,.wav,.flac,.flc,.wma,.aac,.ogg,.aiff,.alac" ref={fileInput} - onChange={loadFile} /> + onChange={(e) => { + setIsLoaded(true); + loadFile(e); + }} /> - {unfilteredAudioBuffer && - { - setAudioBuffer(updatedAudio); - //TODO: neither of these work yet: - //wavesurferRef.current?.loadAudio("blob", undefined, [updatedAudio.getChannelData(0)]); - // wavesurferRef.current?.setOptions({ - // peaks: [updatedAudio.getChannelData(0)], - // }); - }} - /> - } + {audioBuffer && { + const wavBytes = getWavBytes(updatedAudioBuffer.getChannelData(0).buffer, { + isFloat: true, // floating point or 16-bit integer + numChannels: 1, + sampleRate: updatedAudioBuffer.sampleRate, + }) + const blob = new Blob([wavBytes], { type: 'audio/wav' }) + setAudioFile(blob); + }} + />} Note: audio data is not saved with Parseq documents. You will need to reload your reference audio file when you reload your Parseq document. @@ -857,23 +481,30 @@ export function AudioWaveform(props: AudioWaveformProps) { - -
-
-
- - - - - {playbackPos} - - + { + const newViewPort = {startFrame: Math.round(startFrame), endFrame: Math.round(endFrame)}; + if (!_.isEqual(viewport, newViewPort)) { + props.onScroll(newViewPort); + } + //setStartVisibleFrame(startFrame); + //setEndVisibleFrame(endFrame); + }} + onready={() => { + setIsLoaded(true); + }} + /> + + setTab(newValue)}> @@ -884,7 +515,7 @@ export function AudioWaveform(props: AudioWaveformProps) { - + @@ -969,7 +600,7 @@ export function AudioWaveform(props: AudioWaveformProps) { disabled={isAnalysing} /> - + diff --git a/src/components/WaveSurferPlayer.tsx b/src/components/WaveSurferPlayer.tsx new file mode 100644 index 0000000..4ff1abc --- /dev/null +++ b/src/components/WaveSurferPlayer.tsx @@ -0,0 +1,291 @@ +import { Box, Button, MenuItem, TextField } from "@mui/material"; +import Grid from '@mui/material/Unstable_Grid2'; +import { MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import WaveSurfer, { WaveSurferEvents, WaveSurferOptions } from "wavesurfer.js"; +import { SmallTextField } from "./SmallTextField"; +import { BiquadFilter } from "./BiquadFilter"; +import { getWavBytes } from "../utils/utils"; +import { CssVarsPalette, Palette, SupportedColorScheme, experimental_extendTheme as extendTheme, useColorScheme } from "@mui/material/styles"; +import { themeFactory } from "../theme"; +import { Viewport } from "./Viewport"; +import TimelinePlugin from "wavesurfer.js/dist/plugins/timeline"; +import MinimapPlugin from "wavesurfer.js/dist/plugins/minimap"; +import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram"; +import { useHotkeys } from 'react-hotkeys-hook' +import { beatToSec, frameToSec, secToBeat, secToFrame, calculateNiceStepSize } from "../utils/maths"; + + +export type TimelineOptions = { + xaxisType: "frames" | "seconds" | "beats"; + bpm: number; + fps: number; +}; + +export type ViewportOptions = { + startFrame: number; + endFrame: number; +}; + +type WaveSurferPlayerProps = { + audioFile: Blob | undefined; + wsOptions: Partial; + timelineOptions: TimelineOptions; + viewport: ViewportOptions; + onscroll: (startX: number, endX: number) => void; + onready: () => void; +} + +const resetHandler = (wavesurferRef: MutableRefObject, event : keyof WaveSurferEvents, eventHandlerRef: MutableRefObject<(() => void) | undefined>, logic : any) => { + if (!wavesurferRef.current) { + return; + } + if (eventHandlerRef.current) { + wavesurferRef.current.un(event, eventHandlerRef.current); + } + eventHandlerRef.current = wavesurferRef.current.on(event, logic); +} + +export const WaveSurferPlayer = ({ audioFile, wsOptions, timelineOptions, viewport, onscroll, onready }: WaveSurferPlayerProps) => { + console.log("Rendering WaveSurferPlayer: ", audioFile?.name || "no audio file"); + + const containerRef = useRef(); + const [isPlaying, setIsPlaying] = useState(false); + const [currentTime, setCurrentTime] = useState(0); + const wavesurferRef = useRef(null); + + const timelinePluginRef = useRef() + const zoomHandlerRef = useRef<(() => void) | undefined>(); + const scrollHandlerRef = useRef<(() => void) | undefined>(); + const readyHandlerRef = useRef<(() => void) | undefined>(); + + const [lastSeekedPos, setLastSeekedPos] = useState(0); + + const [pxPerSec, setPxPerSec] = useState(10); + + + const recreateWavesurfer = useCallback(() => { + if (!containerRef.current) { + return; + } + if (wavesurferRef.current) { + wavesurferRef.current.unAll(); + wavesurferRef.current.destroy(); + } + console.log("---> Creating wavesurfer") + const ws = WaveSurfer.create({ + ...wsOptions, + container: containerRef.current, + }) + + // Static handlers that won't change when props change + ws.on('play', () => setIsPlaying(true)); + ws.on('pause', () => setIsPlaying(false)); + ws.on('seeking', (pos) => setLastSeekedPos(pos)); + ws.on('timeupdate', (currentTime) => setCurrentTime(currentTime)); + ws.on('load', () => console.log('load event')); + ws.on('decode', () => console.log('decode event')); + ws.on('ready', () => console.log('ready event')); + + wavesurferRef.current = ws; + + return () => { + // TODO - I should be unsubscribing & destroying here, + // but that is causing issues – possible leak. + console.log("<--- SHOULD destroy wavesurfer") + } + }, [wsOptions]); + + // Initialise wavesurfer once. + useEffect(() => { + if (!wavesurferRef.current) { + recreateWavesurfer(); + } + }) + + // Re-register scroll handler when timeline options change, because of dependency on fps. + useEffect(() => { + resetHandler(wavesurferRef, 'scroll', scrollHandlerRef, (startX:number, endX:number) => { + onscroll(startX * timelineOptions.fps, endX * timelineOptions.fps) + }); + }, [onscroll, timelineOptions]); + + // Re-register zoom handler . + useEffect(() => { + resetHandler(wavesurferRef, 'zoom', zoomHandlerRef, (newPxPerSec:number) => { + console.log('zoom event (pxPerSec):', newPxPerSec); + setPxPerSec(newPxPerSec); // Will trigger timeline rebuild. + // No callback for zoom because there is no internal zoom control + }); + }, []); + + // Re-register ready handler when prop callback changes + useEffect(() => { + resetHandler(wavesurferRef, 'ready', readyHandlerRef, () => { + onready(); + }); + }, [onready]); + + // If audio the audio file changes, load it in wavesurfer. + useEffect(() => { + if (!wavesurferRef.current) return; + if (audioFile) { + try { + //recreateWavesurfer(); + (wavesurferRef.current as any).timer.stop(); // HACK to workaround https://github.com/katspaugh/wavesurfer.js/issues/3097 + wavesurferRef.current.loadBlob(audioFile); + } catch (e: any) { + console.error(`Failed to load ${audioFile.name}':`, e) + } + } + }, [wavesurferRef, audioFile]); + + + // If viewport changes, force wavesurfer to scroll or zoom + useEffect(() => { + if (!wavesurferRef.current || !wavesurferRef.current.getDuration()) { + return; + } + const startSec = viewport.startFrame / timelineOptions.fps; + const endSec = viewport.endFrame / timelineOptions.fps; + + const desiredPxPerSec = (wavesurferRef.current as any).renderer.scrollContainer.clientWidth / (endSec - startSec); + wavesurferRef.current.zoom(desiredPxPerSec); + (wavesurferRef.current as any).renderer.scrollContainer.scrollLeft = startSec * desiredPxPerSec; + + }, [viewport, timelineOptions]); + + const getTimeIntervals = useCallback((pxPerSec: number, clientWidth: number, inTimelineOptions: TimelineOptions) => { + + const { bpm, fps, xaxisType } = inTimelineOptions; + + // Calculate the number of units that can be displayed in the viewport + const secondsInView = clientWidth / pxPerSec; + let unitsInView: number; + switch (xaxisType) { + case "seconds": + unitsInView = secondsInView; + break; + case "frames": + unitsInView = secToFrame(secondsInView, fps); + break; + case "beats": + unitsInView = secToBeat(secondsInView, bpm); + break; + } + + // Ticks to display is 10 per 100px, rounded to the closest 10. + const ticksInView = Math.floor(clientWidth / 100) * 10; + + // Units per tick should be a nice number, e.g. 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100... + const roughStepSize = unitsInView / ticksInView; + const niceStepSize = calculateNiceStepSize(roughStepSize); + + let secondsPerTick: number; + switch (xaxisType) { + case "seconds": + secondsPerTick = niceStepSize; + break; + case "frames": + secondsPerTick = frameToSec(niceStepSize, fps); + break; + case "beats": + secondsPerTick = beatToSec(niceStepSize, bpm); + break; + } + + console.log("timeline settings", { + clientWidth, + pxPerSec, + unitsInView, + ticksInView, + roughStepSize, + niceStepSize, + secondsPerTick, + }); + + return { + timeInterval: secondsPerTick, + primaryLabelSpacing: 10, + secondaryLabelSpacing: 5, + formatTimeCallback: (sec: number) => { + switch (xaxisType) { + case "frames": + return `${(sec * fps).toFixed(0)}`; + case "seconds": + return `${sec.toFixed(2)}`; + case "beats": + return `${(sec * bpm / 60).toFixed(2)}`; + } + } + }; + + }, []); + + + // Rebuild the timeline if any of the timeline options change. + const rebuildTimeline = useCallback(() => { + if (!wavesurferRef.current) { + return + } + if (timelinePluginRef.current) { + timelinePluginRef.current.destroy(); + } + const options = { + ...getTimeIntervals(pxPerSec, wavesurferRef.current.getWrapper().clientWidth, timelineOptions), + } + //@ts-ignore + const newTimelinePlugin: TimelinePlugin = wavesurferRef.current.registerPlugin(TimelinePlugin.create(options)); + timelinePluginRef.current = newTimelinePlugin; + + }, [getTimeIntervals, timelineOptions, wavesurferRef, pxPerSec]); + + useEffect(() => { + rebuildTimeline(); + }, [rebuildTimeline]) + + // On play button click + const onPlayClick = useCallback(() => { + if (!wavesurferRef.current) return; + wavesurferRef.current.isPlaying() ? wavesurferRef.current.pause() : wavesurferRef.current.play(); + }, [wavesurferRef]) + + function playPause(from: number = -1, pauseIfPlaying = true) { + if (isPlaying && pauseIfPlaying) { + wavesurferRef.current?.pause(); + } else { + if (from >= 0) { + wavesurferRef.current?.setTime(from); + } if (!isPlaying) { + wavesurferRef.current?.play(); + } + } + } + + useHotkeys('space', + () => playPause(), + { preventDefault: true, scopes: ['main'], description: 'Play from cursor position / pause.' }, + [playPause]); + + useHotkeys('shift+space', + () => playPause(0, false), + { preventDefault: true, scopes: ['main'], description: 'Play from start.' }, + [playPause]); + + useHotkeys('ctrl+space', + () => playPause(lastSeekedPos, false), + { preventDefault: true, scopes: ['main'], description: 'Play from the last seek position.' }, + [playPause, lastSeekedPos]); + + return ( + <> + {/* @ts-ignore */} +
+ +

{currentTime.toFixed(3)}s / {secToFrame(currentTime, timelineOptions.fps).toFixed(2)} frames / {secToBeat(currentTime, timelineOptions.bpm).toFixed(2)} beats

+ + ) +} + +export default WaveSurferPlayer; \ No newline at end of file diff --git a/src/components/WavesurferWaveform.tsx b/src/components/WavesurferWaveform.tsx index 3a39164..f95d133 100644 --- a/src/components/WavesurferWaveform.tsx +++ b/src/components/WavesurferWaveform.tsx @@ -109,6 +109,10 @@ const WavesurferAudioWaveform = ({ audioBuffer, initialSelection, onSelectionCha wavesurfer.load("", [audioBuffer.getChannelData(0)]); + + return () => { + waveSurferRef.current?.destroy(); + } } diff --git a/src/utils/maths.ts b/src/utils/maths.ts index 2b9bf0c..fc384f3 100644 --- a/src/utils/maths.ts +++ b/src/utils/maths.ts @@ -70,4 +70,23 @@ export function remapFrameCount(frame:number, keyframeLock: "frames" | "seconds" const lockedPosition = Number(frameToXAxisType(frame, keyframeLock, oldFps, oldBpm)); const newFramePosition = xAxisTypeToFrame(lockedPosition, keyframeLock, newFps, newBpm); return newFramePosition; - } \ No newline at end of file + } + + +export function calculateNiceStepSize(roughStepSize : number) : number { + const exponent = Math.floor(Math.log10(roughStepSize)); + const normalizedStepSize = roughStepSize / Math.pow(10, exponent); + + let niceStepSize; + if (normalizedStepSize < 1.5) { + niceStepSize = 1; + } else if (normalizedStepSize < 3) { + niceStepSize = 2; + } else if (normalizedStepSize < 7) { + niceStepSize = 5; + } else { + niceStepSize = 10; + } + + return niceStepSize * Math.pow(10, exponent); + } diff --git a/src/wdyr.js b/src/wdyr.js index 04c1606..41722f7 100644 --- a/src/wdyr.js +++ b/src/wdyr.js @@ -4,9 +4,9 @@ if (process.env.NODE_ENV === 'development') { const whyDidYouRender = require('@welldone-software/why-did-you-render'); //@ts-ignore whyDidYouRender(React, { - trackAllPureComponents: true, + trackAllPureComponents: false, //include: [/Editable/, /ParseqUI/] //include: [/Labs/, /WaveSurferComponent/] - //include: [] + include: [] }); } \ No newline at end of file