Skip to content

Commit e39a2b3

Browse files
2xAA2xAA
committed
feat(video): add support for webcodec video, re #695 
1 parent 6985cf6 commit e39a2b3

File tree

10 files changed

+307
-32
lines changed

10 files changed

+307
-32
lines changed

‎src/application/createWebcodecVideo.js

+5-6
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
export function createWebcodecVideo(id, modV) {
2-
return Promise(async (resolve, reject) => {
3-
const url = modV.store.state.videos[id];
1+
export function createWebcodecVideo(id, url) {
2+
return new Promise(async (resolve, reject) => {
43
const video = document.createElement("video");
54
video.setAttribute("crossorigin", "anonymous");
65
video.setAttribute("loop", true);
7-
video.onerror(reject);
6+
video.onerror = reject;
87
video.muted = true;
98

109
video.onloadedmetadata = async () => {
@@ -18,7 +17,7 @@ export function createWebcodecVideo(id, modV) {
1817
// Transfer the readable stream to the worker.
1918
// NOTE: transferring frameStream and reading it in the worker is more
2019
// efficient than reading frameStream here and transferring VideoFrames individually.
21-
this.$modV.store.dispatch(
20+
this.store.dispatch(
2221
"videos/assignVideoStream",
2322
{
2423
id,
@@ -29,7 +28,7 @@ export function createWebcodecVideo(id, modV) {
2928
[frameStream]
3029
);
3130

32-
resolve();
31+
resolve({ id, video, stream });
3332
};
3433

3534
video.setAttribute("src", url);

‎src/application/index.js

+15-2
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ export default class ModV {
4545
perceptualSpread: 0,
4646
perceptualSharpness: 0
4747
});
48+
videos = {};
4849

4950
_store = store;
5051
store = {
@@ -64,7 +65,7 @@ export default class ModV {
6465
payload: app.getAppPath()
6566
});
6667

67-
this.$worker.addEventListener("message", e => {
68+
this.$worker.addEventListener("message", async e => {
6869
const message = e.data;
6970
const { type } = message;
7071

@@ -79,7 +80,19 @@ export default class ModV {
7980
// }
8081

8182
if (type === "createWebcodecVideo") {
82-
this.createWebcodecVideo();
83+
const videoContext = await this.createWebcodecVideo(
84+
message.id,
85+
message.path
86+
);
87+
this.videos[videoContext.id] = videoContext;
88+
}
89+
90+
if (type === "removeWebcodecVideo") {
91+
const { video, stream } = this.videos[message.id];
92+
video.src = "";
93+
// eslint-disable-next-line no-for-each/no-for-each
94+
stream.getTracks().forEach(track => track.stop());
95+
delete this.videos[message.id];
8396
}
8497

8598
if (e.data.type === "tick" && this.ready) {

‎src/application/worker/store/modules/dataTypes.js

+10-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ const state = {
5252
const { path } = options;
5353
let id;
5454
try {
55-
({ id } = await store.dispatch("images/createVideoFromPath", {
55+
({ id } = await store.dispatch("videos/createVideoFromPath", {
5656
path
5757
}));
5858
} catch (e) {
@@ -78,6 +78,15 @@ const state = {
7878
}
7979
});
8080
},
81+
async destroy(textureDefinition) {
82+
const { type, id } = textureDefinition;
83+
84+
if (type === "video") {
85+
await store.dispatch("videos/removeVideoById", {
86+
id
87+
});
88+
}
89+
},
8190
get: textureDefinition => {
8291
if (!textureDefinition.location.length) {
8392
return false;

‎src/application/worker/store/modules/modules.js

+16-4
Original file line numberDiff line numberDiff line change
@@ -564,12 +564,24 @@ const actions = {
564564
meta.compositeOperationInputId,
565565
meta.enabledInputId
566566
];
567-
const moduleProperties = Object.values(module.$props).map(prop => ({
568-
id: prop.id,
569-
type: prop.type
570-
}));
567+
const moduleProperties = Object.entries(module.$props).map(
568+
([key, prop]) => ({
569+
key,
570+
id: prop.id,
571+
type: prop.type
572+
})
573+
);
571574
const inputIds = [...moduleProperties, ...metaInputIds.map(id => ({ id }))];
572575

576+
for (let i = 0, len = moduleProperties.length; i < len; i++) {
577+
const { key, type: propType } = moduleProperties[i];
578+
579+
// destroy anything created by datatypes we don't need anymore
580+
if (store.state.dataTypes[propType].destroy) {
581+
store.state.dataTypes[propType].destroy(module.props[key]);
582+
}
583+
}
584+
573585
for (let i = 0, len = inputIds.length; i < len; i++) {
574586
const { id: inputId, type: propType } = inputIds[i];
575587

‎src/application/worker/store/modules/videos.js

+73-6
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
import Vue from "vue";
22
import uuidv4 from "uuid/v4";
3+
import store from "../";
34

45
const state = {};
56

67
const getters = {
7-
video: state => id => state[id]
8+
video: state => id => state[id]?.outputContext?.context.canvas
89
};
910

1011
const actions = {
11-
createVideoFromPath({ commit }, { path: filePath }) {
12+
createVideoFromPath({ rootState, commit }, { path: filePath }) {
1213
const id = uuidv4();
13-
const path = `modv://${filePath}`;
14+
const path = `modv://${rootState.media.path}${filePath}`;
1415

1516
if (typeof window !== "undefined") {
1617
self.postMessage({
@@ -24,19 +25,85 @@ const actions = {
2425
return { id };
2526
},
2627

27-
assignVideoStream({ commit }, { id, stream, width, height }) {
28-
commit("UPDATE_VIDEO", { id, stream, width, height });
28+
async assignVideoStream({ commit }, { id, stream, width, height }) {
29+
const frameReader = stream.getReader();
30+
const outputContext = await store.dispatch("outputs/getAuxillaryOutput", {
31+
name: state[id].path,
32+
options: {
33+
desynchronized: true
34+
},
35+
group: "videos",
36+
reactToResize: false,
37+
width,
38+
height
39+
});
40+
41+
frameReader.read().then(function processFrame({ done, value: frame }) {
42+
const { stream, needsRemoval } = state[id];
43+
if (done) {
44+
return;
45+
}
46+
47+
// NOTE: all paths below must call frame.close(). Otherwise, the GC won't
48+
// be fast enough to recollect VideoFrames, and decoding can stall.
49+
50+
if (needsRemoval) {
51+
// TODO: There might be a more elegant way of closing a stream, or other
52+
// events to listen for - do we need to use frameReader.cancel(); somehow?
53+
frameReader.releaseLock();
54+
stream.cancel();
55+
56+
frame.close();
57+
58+
commit("REMOVE_VIDEO", { id });
59+
60+
if (typeof window !== "undefined") {
61+
self.postMessage({
62+
type: "removeWebcodecVideo",
63+
id
64+
});
65+
}
66+
return;
67+
}
68+
69+
// Processing on 'frame' goes here!
70+
// E.g. this is where encoding via a VideoEncoder could be set up, or
71+
// rendering to an OffscreenCanvas.
72+
73+
outputContext.context.drawImage(frame, 0, 0);
74+
frame.close();
75+
76+
frameReader.read().then(processFrame);
77+
});
78+
79+
commit("UPDATE_VIDEO", {
80+
id,
81+
stream,
82+
width,
83+
height,
84+
frameReader,
85+
outputContext,
86+
needsRemoval: false
87+
});
88+
},
89+
90+
async removeVideoById({ commit }, { id }) {
91+
commit("UPDATE_VIDEO", { id, needsRemoval: true });
2992
}
3093
};
3194

3295
const mutations = {
3396
CREATE_VIDEO(state, { id, path }) {
34-
Vue.set(state, id, path);
97+
Vue.set(state, id, { path });
3598
},
3699

37100
UPDATE_VIDEO(state, video) {
38101
const { id } = video;
39102
state[id] = { ...state[id], ...video };
103+
},
104+
105+
REMOVE_VIDEO(state, { id }) {
106+
delete state[id];
40107
}
41108
};
42109

‎src/background/background.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { app, protocol } from "electron";
22
import { APP_SCHEME } from "./background-constants";
3-
// import { getMediaManager } from "./media-manager";
3+
import { getMediaManager } from "./media-manager";
44
import { openFile } from "./open-file";
55
import { createWindow } from "./windows";
66

@@ -42,10 +42,10 @@ app.on("activate", async () => {
4242

4343
// https://stackoverflow.com/a/66673831
4444
function fileHandler(req, callback) {
45-
// const { mediaDirectoryPath } = getMediaManager();
46-
const requestedPath = req.url;
45+
const { mediaDirectoryPath } = getMediaManager();
46+
const requestedPath = req.url.substr(7);
4747
// Write some code to resolve path, calculate absolute path etc
48-
const check = true; // requestedPath.indexOf(mediaDirectoryPath) > -1;
48+
const check = requestedPath.indexOf(mediaDirectoryPath) > -1;
4949

5050
if (!check) {
5151
callback({

‎src/components/Controls/TextureControl.vue

+60-9
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,12 @@
3636
</div>
3737

3838
<div v-if="type === 'image'">
39-
<select v-model="modelImagePath" @change="setTexture('image')">
39+
<select
40+
v-model="modelImagePath"
41+
@change="setTexture('image')"
42+
:disabled="!images"
43+
>
44+
<option selected value="">No image</option>
4045
<option
4146
v-for="(image, index) in images"
4247
:key="index"
@@ -45,20 +50,47 @@
4550
>
4651
</select>
4752
</div>
53+
54+
<div v-if="type === 'video'">
55+
<select
56+
v-model="modelVideoPath"
57+
@change="setTexture('video')"
58+
:disabled="!videos"
59+
>
60+
<option selected value="">No video</option>
61+
<option
62+
v-for="output in videos"
63+
:key="output.path"
64+
:value="output.path"
65+
>{{ output.name }}</option
66+
>
67+
</select>
68+
69+
<VideoControl
70+
v-if="type === 'video' && modelVideoPath && value.id"
71+
:video-id="value.id"
72+
/>
73+
</div>
4874
</div>
4975
</template>
5076

5177
<script>
5278
import constants from "../../application/constants";
79+
import { VideoControl } from "./VideoControl.vue";
5380
5481
export default {
5582
props: ["value"],
5683
84+
components: {
85+
VideoControl
86+
},
87+
5788
data() {
5889
return {
59-
textureTypes: ["inherit", "group", "canvas", "image"],
90+
textureTypes: ["inherit", "group", "canvas", "image", "video"],
6091
type: "",
6192
modelImagePath: "",
93+
modelVideoPath: "",
6294
modelCanvasId: ""
6395
};
6496
},
@@ -68,8 +100,8 @@ export default {
68100
this.value.type && this.value.type.length
69101
? this.value.type
70102
: this.textureTypes[0];
71-
this.modelImagePath = this.value.options.path || "";
72-
this.modelCanvasId = this.value.options.id || "";
103+
this.modelImagePath = this.value.options?.path || "";
104+
this.modelCanvasId = this.value.options?.id || "";
73105
},
74106
75107
computed: {
@@ -79,8 +111,9 @@ export default {
79111
80112
groupOutputs() {
81113
return Object.values(this.auxillaries).filter(
82-
group =>
83-
group.group === "group" && group.name !== constants.GALLERY_GROUP_NAME
114+
auxillary =>
115+
auxillary.group === "group" &&
116+
auxillary.name !== constants.GALLERY_GROUP_NAME
84117
);
85118
},
86119
@@ -101,9 +134,19 @@ export default {
101134
},
102135
103136
images() {
104-
return this.$modV.store.state.media.media[
105-
this.$modV.store.state.projects.currentProject
106-
].image;
137+
return (
138+
this.$modV.store.state.media.media[
139+
this.$modV.store.state.projects.currentProject
140+
].image ?? false
141+
);
142+
},
143+
144+
videos() {
145+
return (
146+
this.$modV.store.state.media.media[
147+
this.$modV.store.state.projects.currentProject
148+
].video ?? false
149+
);
107150
}
108151
},
109152
@@ -118,6 +161,14 @@ export default {
118161
textureDefinition.options.path = this.modelImagePath;
119162
}
120163
164+
if (type === "video") {
165+
if (!this.modelVideoPath) {
166+
return;
167+
}
168+
169+
textureDefinition.options.path = this.modelVideoPath;
170+
}
171+
121172
if (type === "canvas" || type === "group") {
122173
if (!this.modelCanvasId) {
123174
return;

0 commit comments

Comments
 (0)