feat(video): add preliminary changes for video support, re #695

Author: 2xAA

src/application/createWebcodecVideo.js

@@ -0,0 +1,38 @@
+export function createWebcodecVideo(id, modV) {
+  return Promise(async (resolve, reject) => {
+    const url = modV.store.state.videos[id];
+    const video = document.createElement("video");
+    video.setAttribute("crossorigin", "anonymous");
+    video.setAttribute("loop", true);
+    video.onerror(reject);
+    video.muted = true;
+
+    video.onloadedmetadata = async () => {
+      const stream = video.captureStream();
+      const [track] = stream.getVideoTracks();
+
+      // eslint-disable-next-line
+      const processor = new MediaStreamTrackProcessor(track);
+      const frameStream = processor.readable;
+
+      // Transfer the readable stream to the worker.
+      // NOTE: transferring frameStream and reading it in the worker is more
+      // efficient than reading frameStream here and transferring VideoFrames individually.
+      this.$modV.store.dispatch(
+        "videos/assignVideoStream",
+        {
+          id,
+          stream: frameStream,
+          width: video.videoWidth || 256,
+          height: video.videoHeight || 256
+        },
+        [frameStream]
+      );
+
+      resolve();
+    };
+
+    video.setAttribute("src", url);
+    await video.play();
+  });
+}

src/application/index.js

@@ -15,6 +15,7 @@ import PromiseWorker from "promise-worker-transferable";
 import Vue from "vue";
 import { ipcRenderer } from "electron";
 import { app } from "@electron/remote";
+import { createWebcodecVideo } from "./createWebcodecVideo";
 
 let imageBitmap;
 const imageBitmapQueue = [];
@@ -27,6 +28,7 @@ export default class ModV {
   setupBeatDetektor = setupBeatDetektor;
   setupMidi = setupMidi;
   windowHandler = windowHandler;
+  createWebcodecVideo = createWebcodecVideo;
   use = use;
   debug = false;
   features = Vue.observable({
@@ -76,6 +78,10 @@ export default class ModV {
       //   console.log(`⚙️%c ${type}`, "color: red");
       // }
 
+      if (type === "createWebcodecVideo") {
+        this.createWebcodecVideo();
+      }
+
       if (e.data.type === "tick" && this.ready) {
         this.tick(e.data.payload);
         return;

src/application/worker/store/modules/dataTypes.js

@@ -48,7 +48,22 @@ const state = {
         textureDefinition.id = id;
       }
 
-      if (type === "canvas" || type == "group") {
+      if (type === "video") {
+        const { path } = options;
+        let id;
+        try {
+          ({ id } = await store.dispatch("images/createVideoFromPath", {
+            path
+          }));
+        } catch (e) {
+          console.error(e);
+        }
+
+        textureDefinition.location = "videos/video";
+        textureDefinition.id = id;
+      }
+
+      if (type === "canvas" || type === "group") {
         const { id } = options;
         textureDefinition.location = "outputs/auxillaryCanvas";
         textureDefinition.id = id;

src/application/worker/store/modules/videos.js

@@ -0,0 +1,49 @@
+import Vue from "vue";
+import uuidv4 from "uuid/v4";
+
+const state = {};
+
+const getters = {
+  video: state => id => state[id]
+};
+
+const actions = {
+  createVideoFromPath({ commit }, { path: filePath }) {
+    const id = uuidv4();
+    const path = `modv://${filePath}`;
+
+    if (typeof window !== "undefined") {
+      self.postMessage({
+        type: "createWebcodecVideo",
+        id,
+        path
+      });
+    }
+
+    commit("CREATE_VIDEO", { id, path });
+    return { id };
+  },
+
+  assignVideoStream({ commit }, { id, stream, width, height }) {
+    commit("UPDATE_VIDEO", { id, stream, width, height });
+  }
+};
+
+const mutations = {
+  CREATE_VIDEO(state, { id, path }) {
+    Vue.set(state, id, path);
+  },
+
+  UPDATE_VIDEO(state, video) {
+    const { id } = video;
+    state[id] = { ...state[id], ...video };
+  }
+};
+
+export default {
+  namespaced: true,
+  state,
+  getters,
+  actions,
+  mutations
+};

src/background/background.js

@@ -1,5 +1,6 @@
 import { app, protocol } from "electron";
 import { APP_SCHEME } from "./background-constants";
+// import { getMediaManager } from "./media-manager";
 import { openFile } from "./open-file";
 import { createWindow } from "./windows";
 
@@ -39,10 +40,33 @@ app.on("activate", async () => {
   createWindow("mainWindow");
 });
 
+// https://stackoverflow.com/a/66673831
+function fileHandler(req, callback) {
+  // const { mediaDirectoryPath } = getMediaManager();
+  const requestedPath = req.url;
+  // Write some code to resolve path, calculate absolute path etc
+  const check = true; // requestedPath.indexOf(mediaDirectoryPath) > -1;
+
+  if (!check) {
+    callback({
+      // -6 is FILE_NOT_FOUND
+      // https://source.chromium.org/chromium/chromium/src/+/master:net/base/net_error_list.h
+      error: -6
+    });
+    return;
+  }
+
+  callback({
+    path: requestedPath
+  });
+}
+
 // This method will be called when Electron has finished
 // initialization and is ready to create browser windows.
 // Some APIs can only be used after this event occurs.
 app.on("ready", async () => {
+  protocol.registerFileProtocol("modv", fileHandler);
+
   app.commandLine.appendSwitch(
     "disable-backgrounding-occluded-windows",
     "true"