Wavesurfer7 component now in place in both UI. Remaining work require…

…d to port marker logic, fix scrolling, dark/light mode, other bug fixes
rewbs · Aug 20, 2023 · e4ca755 · e4ca755
1 parent a22f00c
commit e4ca755
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 236 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -79,7 +79,7 @@
     "use-debounced-effect": "^2.0.1",
     "util": "^0.12.5",
     "uuid": "^9.0.0",
-    "wavesurfer.js": "file:///Users/rewbs/code/wavesurfer.js",
+    "wavesurfer.js": "7.1.5",
     "web-vitals": "^3.4.0"
   },
   "scripts": {

diff --git a/src/ParseqUI.js b/src/ParseqUI.js
@@ -1529,6 +1529,8 @@ const ParseqUI = (props) => {
           {options && lastFrame && <TimeSeriesUI
             lastFrame={lastFrame}
             fps={options.output_fps}
+            bpm={options.bpm}
+            xaxisType={{xaxisType}}
             allTimeSeries={timeSeries}
             onChange={(allTimeSeries) =>
               setTimeSeries([...allTimeSeries])

diff --git a/src/components/TimeSeriesUI.tsx b/src/components/TimeSeriesUI.tsx
@@ -30,15 +30,21 @@ import { InterpolationType, TimeSeries, TimestampType } from '../parseq-lang/par
 import { themeFactory } from "../theme";
 import { DECIMATION_THRESHOLD } from '../utils/consts';
 import { frameToSec } from '../utils/maths';
-import { channelToRgba, createAudioBufferCopy } from '../utils/utils';
+import { channelToRgba, createAudioBufferCopy, getWavBytes } from '../utils/utils';
 import { SmallTextField } from './SmallTextField';
 import { TabPanel } from './TabPanel';
-import WavesurferAudioWaveform from './WavesurferWaveform';
+
 import { BiquadFilter } from './BiquadFilter';
+import WaveSurferPlayer from './WaveSurferPlayer';
+import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram";
+import RegionsPlugin from 'wavesurfer.js/dist/plugins/regions';
+import colormap from '../data/hot-colormap.json';
 
 type TimeSeriesUIProps = {
   lastFrame: number,
   fps: number,
+  bpm: number,
+  xaxisType: "frames" | "seconds" | "beats",
   allTimeSeries: {
     alias: string;
     ts: TimeSeries;
@@ -70,7 +76,7 @@ const defaultData = {
 
 export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
 
-  const { lastFrame, fps, onChange, afterBlur, afterFocus } = props;
+  const { lastFrame, fps, bpm, xaxisType, onChange, afterBlur, afterFocus } = props;
 
   const [open, setOpen] = useState(false);
   const [allTimeSeries, setAllTimeSeries] = useState(props.allTimeSeries);
@@ -94,13 +100,14 @@ export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
   const [showValuesAtFrames, setShowValuesAtFrames] = useState(false);
 
   const [selectionStartMs, setSelectionStartMs] = useState(0);
-  const [selectionEndMs, setSelectionEndMs] = useState(0);
+  const [selectionEndMs, setSelectionEndMs] = useState(frameToSec(lastFrame, fps)*1000);
 
   const [pitchMethod, setPitchMethod] = useState("default");
   const pitchProgressRef = useRef<HTMLInputElement>(null);
 
-  const [unfilteredAudioBuffer, setUnfilteredAudioBuffer] = useState<AudioBuffer>();
   const [audioBuffer, setAudioBuffer] = useState<AudioBuffer>();
+  const [unfilteredAudioBuffer, setUnfilteredAudioBuffer] = useState<AudioBuffer>();
+  const [audioFile, setAudioFile] = useState<Blob>();
 
   const [status, setStatus] = useState(<></>);
 
@@ -111,6 +118,41 @@ export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
 
   const { disableScope: disableHotkeyScope, enableScope: enableHotkeyScope } = useHotkeysContext();
 
+
+  const regionsPlugin = useMemo(() => {
+    console.log("!!!! creating new regions plugin")
+    const wsRegions = RegionsPlugin.create();
+    wsRegions.on('region-updated', (region: any) => {
+      setSelectionStartMs(region.start*1000);
+      setSelectionEndMs(region.end*1000);
+    });
+    return wsRegions;
+  }, []);
+
+  const wsOptions = useMemo(() => ({
+    barWidth: 0,
+    normalize: true,
+    fillParent: true,
+    minPxPerSec: 10,
+    autoCenter: false,
+    interact: true,
+    cursorColor: palette.success.light,
+    waveColor: [palette.waveformStart.main, palette.waveformEnd.main],
+    progressColor: [palette.waveformProgressMaskStart.main, palette.waveformProgressMaskEnd.main],
+    cursorWidth: 1,
+    plugins: [
+        regionsPlugin,
+        //@ts-ignore        
+        SpectrogramPlugin.create({
+            labels: true,
+            height: 75,
+            colorMap: colormap
+        })
+    ],
+}), [palette.success.light, palette.waveformEnd.main, palette.waveformProgressMaskEnd.main, palette.waveformProgressMaskStart.main, palette.waveformStart.main, regionsPlugin]);
+
+
+
   const handleTimestampTypeChange = (event: any) => {
     setTimestampType(event.target.value);
   };
@@ -221,8 +263,10 @@ export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
       const arrayBuffer = await file.arrayBuffer();
       const audioContext = new AudioContext();
       const newAudioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-      setUnfilteredAudioBuffer(createAudioBufferCopy(newAudioBuffer));
       setAudioBuffer(newAudioBuffer);
+      setUnfilteredAudioBuffer(createAudioBufferCopy(newAudioBuffer))
+      setAudioFile(file);
+
       event.target.blur(); // Remove focus from the file input so that spacebar doesn't trigger it again (and can be used for immediate playback)
     } catch (e: any) {
       console.error(e);
@@ -406,14 +450,36 @@ export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
     </Stack>
   </>), [allTimeSeries, handleDeleteTimeSeries, onChange, afterBlur, afterFocus, palette]);
 
-  const waveSuferWaveform = useMemo(() => audioBuffer &&
-    <WavesurferAudioWaveform
-      audioBuffer={audioBuffer}
-      initialSelection={{ start: 0, end: frameToSec(lastFrame, fps) * 1000 }}
-      onSelectionChange={(start, end) => {
-        setSelectionStartMs(start);
-        setSelectionEndMs(end);
-      }} />, [audioBuffer, fps, lastFrame]);
+  const waveSuferWaveform = useMemo(() => audioFile &&
+    <WaveSurferPlayer
+      audioFile={audioFile}
+      wsOptions={wsOptions}
+      timelineOptions={{fps:fps, bpm:bpm, xaxisType:xaxisType }}
+      viewport={{startFrame:0, endFrame:lastFrame}}
+      hotkeyScopes={['timeseries']}
+      onscroll={(startFrame, endFrame) => {
+        //console.log(regionsPlugin.getRegions());
+      }}
+      onready={() => {
+        regionsPlugin.clearRegions();
+        const durationSec = audioBuffer ?  audioBuffer.length / audioBuffer.sampleRate : 1;
+        regionsPlugin.addRegion({
+          start: selectionStartMs/1000,         
+          end: Math.min(selectionEndMs/1000, durationSec),
+          //loop: false,
+          drag: true,
+          resize: false,
+          color: channelToRgba(palette.primary.mainChannel, 0.3),
+        })
+
+      }}      
+
+      // onSelectionChange={(start, end) => {
+      //   setSelectionStartMs(start);
+      //   setSelectionEndMs(end);
+      // }}
+
+      />, [audioBuffer, audioFile, fps, lastFrame, xaxisType, regionsPlugin, selectionEndMs, selectionStartMs, wsOptions, bpm, palette.primary.mainChannel]);  
 
   return <>
     <p><small>Custom time series are arbitrary sequences of numbers that can be referenced from you Parseq formula. You can import them from audio data, or from a CSV file.</small></p>
@@ -444,10 +510,19 @@ export const TimeSeriesUI = (props: TimeSeriesUIProps) => {
                 }
                 onChange={handleLoadFromAudio} />
             </Box>
-            {audioBuffer && <>
+            {unfilteredAudioBuffer && <>
               <BiquadFilter
-                  unfilteredAudioBuffer={unfilteredAudioBuffer||audioBuffer}
-                  updateAudioBuffer={newAudioBuffer => setAudioBuffer(newAudioBuffer)}
+                unfilteredAudioBuffer={unfilteredAudioBuffer}
+                updateAudioBuffer={(updatedAudioBuffer) => {
+                    const wavBytes = getWavBytes(updatedAudioBuffer.getChannelData(0).buffer, {
+                        isFloat: true,       // floating point or 16-bit integer
+                        numChannels: 1,
+                        sampleRate: updatedAudioBuffer.sampleRate,
+                    })
+                    const blob = new Blob([wavBytes], { type: 'audio/wav' })
+                    setAudioFile(blob);
+                    setAudioBuffer(updatedAudioBuffer);
+                }}
                 />
             </>}
           </Stack>

diff --git a/src/components/WaveSurferPlayer.tsx b/src/components/WaveSurferPlayer.tsx
@@ -1,18 +1,8 @@
-import { Box, Button, MenuItem, TextField } from "@mui/material";
-import Grid from '@mui/material/Unstable_Grid2';
-import { MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { MutableRefObject, useCallback, useEffect, useRef, useState } from "react";
+import { useHotkeys } from 'react-hotkeys-hook';
 import WaveSurfer, { WaveSurferEvents, WaveSurferOptions } from "wavesurfer.js";
-import { SmallTextField } from "./SmallTextField";
-import { BiquadFilter } from "./BiquadFilter";
-import { getWavBytes } from "../utils/utils";
-import { CssVarsPalette, Palette, SupportedColorScheme, experimental_extendTheme as extendTheme, useColorScheme } from "@mui/material/styles";
-import { themeFactory } from "../theme";
-import { Viewport } from "./Viewport";
 import TimelinePlugin from "wavesurfer.js/dist/plugins/timeline";
-import MinimapPlugin from "wavesurfer.js/dist/plugins/minimap";
-import SpectrogramPlugin from "wavesurfer.js/dist/plugins/spectrogram";
-import { useHotkeys } from 'react-hotkeys-hook'
-import { beatToSec, frameToSec, secToBeat, secToFrame, calculateNiceStepSize } from "../utils/maths";
+import { beatToSec, calculateNiceStepSize, frameToSec, secToBeat, secToFrame } from "../utils/maths";
 
 
 export type TimelineOptions = {
@@ -31,6 +21,7 @@ type WaveSurferPlayerProps = {
     wsOptions: Partial<WaveSurferOptions>;
     timelineOptions: TimelineOptions;
     viewport: ViewportOptions;
+    hotkeyScopes?: string[];
     onscroll: (startX: number, endX: number) => void;
     onready: () => void;
 }
@@ -45,8 +36,8 @@ const resetHandler = (wavesurferRef: MutableRefObject<WaveSurfer | null>, event
     eventHandlerRef.current = wavesurferRef.current.on(event, logic);
 }
 
-export const WaveSurferPlayer = ({ audioFile, wsOptions, timelineOptions, viewport, onscroll, onready }: WaveSurferPlayerProps) => {
-    console.log("Rendering WaveSurferPlayer: ", audioFile?.name || "no audio file");
+export const WaveSurferPlayer = ({ audioFile, wsOptions, timelineOptions, viewport, hotkeyScopes, onscroll, onready }: WaveSurferPlayerProps) => {
+    //console.log("Rendering WaveSurferPlayer: ", audioFile?.name || "no audio file");
 
     const containerRef = useRef<HTMLDivElement>();
     const [isPlaying, setIsPlaying] = useState(false);
@@ -263,17 +254,17 @@ export const WaveSurferPlayer = ({ audioFile, wsOptions, timelineOptions, viewpo
 
     useHotkeys('space',
         () => playPause(),
-        { preventDefault: true, scopes: ['main'], description: 'Play from cursor position / pause.' },
+        { preventDefault: true, scopes: hotkeyScopes??['main'], description: 'Play from cursor position / pause.' },
         [playPause]);
 
     useHotkeys('shift+space',
         () => playPause(0, false),
-        { preventDefault: true, scopes: ['main'], description: 'Play from start.' },
+        { preventDefault: true, scopes: hotkeyScopes??['main'], description: 'Play from start.' },
         [playPause]);
 
     useHotkeys('ctrl+space',
         () => playPause(lastSeekedPos, false),
-        { preventDefault: true, scopes: ['main'], description: 'Play from the last seek position.' },
+        { preventDefault: true, scopes: hotkeyScopes??['main'], description: 'Play from the last seek position.' },
         [playPause, lastSeekedPos]);
 
     return (