From 9a15da649980041b924105652f0044821fe6bf08 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Thu, 1 Dec 2022 18:45:22 +0900 Subject: [PATCH] =?UTF-8?q?ENH:=20=E3=83=A2=E3=83=BC=E3=83=95=E3=82=A3?= =?UTF-8?q?=E3=83=B3=E3=82=B0UI=E3=81=A8=E3=83=97=E3=83=AA=E3=82=BB?= =?UTF-8?q?=E3=83=83=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/background.ts | 19 +++ src/components/AudioInfo.vue | 237 ++++++++++++++++++++++++++++- src/components/CharacterButton.vue | 14 +- src/components/SettingDialog.vue | 24 +++ src/helpers/previewSliderHelper.ts | 1 + src/store/audio.ts | 50 ++++-- src/store/project.ts | 10 ++ src/store/setting.ts | 1 + src/store/type.ts | 8 + src/type/preload.ts | 1 + tests/unit/store/Vuex.spec.ts | 1 + 11 files changed, 340 insertions(+), 26 deletions(-) diff --git a/src/background.ts b/src/background.ts index 696f57b8cc..145f155a60 100644 --- a/src/background.ts +++ b/src/background.ts @@ -430,6 +430,23 @@ const store = new Store({ volumeScale: { type: "number" }, prePhonemeLength: { type: "number" }, postPhonemeLength: { type: "number" }, + morphingInfo: { + type: "object", + properties: { + rate: { type: "number" }, + targetEngineId: { + type: "string", + pattern: + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + }, + targetSpeakerId: { + type: "string", + pattern: + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + }, + targetStyleId: { type: "number" }, + }, + }, }, }, }, @@ -458,10 +475,12 @@ const store = new Store({ type: "boolean", default: false, }, + enableMorphing: { type: "boolean", default: false }, }, default: { enablePreset: false, enableInterrogativeUpspeak: false, + enableMorphing: false, }, }, acceptRetrieveTelemetry: { diff --git a/src/components/AudioInfo.vue b/src/components/AudioInfo.vue index e60a2f7bf9..815c815599 100644 --- a/src/components/AudioInfo.vue +++ b/src/components/AudioInfo.vue @@ -350,6 +350,92 @@ @pan="postPhonemeLengthSlider.qSliderProps.onPan" /> +
+ + モーフィング +
+ +
+
+ {{ + morphingTargetCharacterInfo + ? morphingTargetCharacterInfo.metas.speakerName + : "未設定" + }} +
+
+ ({{ + morphingTargetStyleInfo + ? morphingTargetStyleInfo.styleName + : undefined + }}) +
+
+
+
+ 非対応エンジンです +
+
+ エンジンが異なります +
+
+ 割合 + {{ + morphingRateSlider.state.currentValue.value != undefined + ? morphingRateSlider.state.currentValue.value.toFixed(2) + : undefined + }} + +
+
@@ -358,8 +444,9 @@ import { computed, defineComponent, ref } from "vue"; import { QSelectProps } from "quasar"; import { useStore } from "@/store"; -import { Preset } from "@/type/preload"; +import { MorphingInfo, Preset, Voice } from "@/type/preload"; import { previewSliderHelper } from "@/helpers/previewSliderHelper"; +import CharacterButton from "./CharacterButton.vue"; import PresetManageDialog from "./PresetManageDialog.vue"; import { EngineManifest } from "@/openapi"; @@ -367,6 +454,7 @@ export default defineComponent({ name: "AudioInfo", components: { + CharacterButton, PresetManageDialog, }, @@ -443,6 +531,22 @@ export default defineComponent({ }); }; + const setMorphingRate = (rate: number) => { + const info = audioItem.value.morphingInfo; + if (info == undefined) { + throw new Error("audioItem.value.morphingInfo == undefined"); + } + store.dispatch("COMMAND_SET_MORPHING_INFO", { + audioKey: props.activeAudioKey, + morphingInfo: { + rate, + targetEngineId: info.targetEngineId, + targetSpeakerId: info.targetSpeakerId, + targetStyleId: info.targetStyleId, + }, + }); + }; + const speedScaleSlider = previewSliderHelper({ modelValue: () => query.value?.speedScale ?? null, disable: () => @@ -508,6 +612,88 @@ export default defineComponent({ scrollMinStep: () => 0.01, }); + const showMorphing = computed( + () => store.state.experimentalSetting.enableMorphing + ); + + const supportMorphing = computed( + () => supportedFeatures.value?.synthesisMorphing + ); + + const warnMorphing = computed(() => { + if (audioItem.value.morphingInfo == undefined) return false; + return !store.getters.VALID_MOPHING_INFO(audioItem.value); + }); + + const selectableEngines = store.getters.SELECTABLE_MOPHING_TARGET_ENGINES; + + const selectableCharacters = computed(() => { + const allCharacters = store.getters.GET_ORDERED_ALL_CHARACTER_INFOS; + return allCharacters + .map((character) => { + const targetStyles = character.metas.styles.filter((style) => + selectableEngines.includes(style.engineId) + ); + character.metas.styles = targetStyles; + return character; + }) + .filter((characters) => characters.metas.styles.length >= 1); + }); + + const morphingTargetVoice = computed({ + get() { + const morphingInfo = audioItem.value.morphingInfo; + if (morphingInfo == undefined) return undefined; + return { + engineId: morphingInfo.targetEngineId, + speakerId: morphingInfo.targetSpeakerId, + styleId: morphingInfo.targetStyleId, + }; + }, + set(voice: Voice | undefined) { + const morphingInfo = + voice != undefined + ? { + rate: audioItem.value.morphingInfo?.rate ?? 0.5, + targetEngineId: voice.engineId, + targetSpeakerId: voice.speakerId, + targetStyleId: voice.styleId, + } + : undefined; + store.dispatch("COMMAND_SET_MORPHING_INFO", { + audioKey: props.activeAudioKey, + morphingInfo, + }); + }, + }); + + const morphingTargetCharacterInfo = computed(() => + selectableCharacters.value.find( + (character) => + character.metas.speakerUuid === morphingTargetVoice.value?.speakerId + ) + ); + + const morphingTargetStyleInfo = computed(() => { + const targetVoice = morphingTargetVoice.value; + return morphingTargetCharacterInfo.value?.metas.styles.find( + (style) => + style.engineId === targetVoice?.engineId && + style.styleId === targetVoice.styleId + ); + }); + + const morphingRateSlider = previewSliderHelper({ + modelValue: () => audioItem.value.morphingInfo?.rate ?? null, + disable: () => uiLocked.value, + onChange: setMorphingRate, + max: () => 1, + min: () => 0, + step: () => 0.01, + scrollStep: () => 0.1, + scrollMinStep: () => 0.01, + }); + // プリセット const enablePreset = computed( () => store.state.experimentalSetting.enablePreset @@ -530,13 +716,30 @@ export default defineComponent({ if (audioPresetKey.value == undefined) throw new Error("audioPresetKey is undefined"); // 次のコードが何故かコンパイルエラーになるチェック const preset = presetItems.value[audioPresetKey.value]; - const { name: _, ...presetParts } = preset; + const { name: _, morphingInfo, ...presetParts } = preset; // 入力パラメータと比較 - const keys = Object.keys(presetParts) as (keyof Omit)[]; - return keys.some( - (key) => presetParts[key] !== presetPartsFromParameter.value[key] - ); + const keys = Object.keys(presetParts) as (keyof Omit< + Preset, + "name" | "morphingInfo" + >)[]; + if ( + keys.some( + (key) => presetParts[key] !== presetPartsFromParameter.value[key] + ) + ) + return true; + const morphingInfoFromParameter = + presetPartsFromParameter.value.morphingInfo; + if (morphingInfo && morphingInfoFromParameter) { + const morphingInfoKeys = Object.keys( + morphingInfo + ) as (keyof MorphingInfo)[]; + return morphingInfoKeys.some( + (key) => morphingInfo[key] !== morphingInfoFromParameter[key] + ); + } + return morphingInfo != morphingInfoFromParameter; }); type PresetSelectModelType = { @@ -695,6 +898,18 @@ export default defineComponent({ volumeScale: volumeScaleSlider.state.currentValue.value, prePhonemeLength: prePhonemeLengthSlider.state.currentValue.value, postPhonemeLength: postPhonemeLengthSlider.state.currentValue.value, + morphingInfo: + morphingTargetStyleInfo.value && + morphingTargetCharacterInfo.value && + morphingRateSlider.state.currentValue.value + ? { + rate: morphingRateSlider.state.currentValue.value, + targetEngineId: morphingTargetStyleInfo.value.engineId, + targetSpeakerId: + morphingTargetCharacterInfo.value.metas.speakerUuid, + targetStyleId: morphingTargetStyleInfo.value.styleId, + } + : undefined, }; }); @@ -752,6 +967,7 @@ export default defineComponent({ setAudioVolumeScale, setAudioPrePhonemeLength, setAudioPostPhonemeLength, + setMorphingRate, applyPreset, enablePreset, isRegisteredPreset, @@ -777,6 +993,15 @@ export default defineComponent({ volumeScaleSlider, prePhonemeLengthSlider, postPhonemeLengthSlider, + selectableEngines, + showMorphing, + supportMorphing, + warnMorphing, + selectableCharacters, + morphingTargetVoice, + morphingTargetCharacterInfo, + morphingTargetStyleInfo, + morphingRateSlider, }; }, }); diff --git a/src/components/CharacterButton.vue b/src/components/CharacterButton.vue index fffc6f638c..13f49361e0 100644 --- a/src/components/CharacterButton.vue +++ b/src/components/CharacterButton.vue @@ -24,9 +24,9 @@ transition-show="none" transition-hide="none" > - + - + { const selectedVoice = props.selectedVoice; + if (selectedVoice == undefined) return undefined; const character = props.characterInfos.find( (characterInfo) => characterInfo.metas.speakerUuid === selectedVoice?.speakerId && @@ -261,9 +262,10 @@ export default defineComponent({ (x) => x.speakerUuid === speakerUuid )?.defaultStyleId; - const defaultStyle = characterInfo?.metas.styles.find( - (style) => style.styleId === defaultStyleId - ); + const defaultStyle = + characterInfo?.metas.styles.find( + (style) => style.styleId === defaultStyleId + ) ?? characterInfo?.metas.styles[0]; // FIXME: デフォルトのスタイルIDが見つからない場合stylesの先頭を選択する if (defaultStyle == undefined) throw new Error("defaultStyle == undefined"); diff --git a/src/components/SettingDialog.vue b/src/components/SettingDialog.vue index fcef51b7e9..b4dd2a017e 100644 --- a/src/components/SettingDialog.vue +++ b/src/components/SettingDialog.vue @@ -584,6 +584,30 @@ > + +
モーフィング機能
+
+ + + 2人の話者でモーフィングした音声を合成する + + +
+ + + + diff --git a/src/helpers/previewSliderHelper.ts b/src/helpers/previewSliderHelper.ts index 17f96b072c..5e5ecdf119 100644 --- a/src/helpers/previewSliderHelper.ts +++ b/src/helpers/previewSliderHelper.ts @@ -25,6 +25,7 @@ export type PreviewSliderHelper = { max: Ref; step: Ref; disable: Ref; + modelValue: Ref; "onUpdate:modelValue": (value: number) => void; onChange: (value: number) => void; onWheel: (event: Events["onWheel"]) => void; diff --git a/src/store/audio.ts b/src/store/audio.ts index 5fb5111730..c84b15d9c4 100644 --- a/src/store/audio.ts +++ b/src/store/audio.ts @@ -52,7 +52,7 @@ async function generateUniqueIdAndQuery( audioQuery, audioItem.engineId, audioItem.styleId, - audioItem.morphingInfo, + audioItem.morphingInfo, // FIXME: モーフィング非対応エンジンの場合morphingInfoが異なっていても同じ音声が出力されるが異なるIDが生成されてしまう state.experimentalSetting.enableInterrogativeUpspeak, // このフラグが違うと、同じAudioQueryで違う音声が生成されるので追加 ]) ); @@ -478,11 +478,6 @@ export const audioStore = createPartialStore({ audioItem.query.outputSamplingRate = baseAudioItem.query.outputSamplingRate; audioItem.query.outputStereo = baseAudioItem.query.outputStereo; - } - if ( - baseAudioItem && - audioItem.engineId === baseAudioItem.morphingInfo?.targetEngineId - ) { audioItem.morphingInfo = baseAudioItem.morphingInfo; } return audioItem; @@ -691,6 +686,30 @@ export const audioStore = createPartialStore({ }, }, + SELECTABLE_MOPHING_TARGET_ENGINES: { + getter: (state) => + state.engineIds.filter( + (engineId) => + state.engineManifests[engineId].supportedFeatures?.synthesisMorphing + ), + }, + + VALID_MOPHING_INFO: { + getter: (state, getters) => (audioItem: AudioItem) => { + if ( + !state.experimentalSetting.enableMorphing || + audioItem.morphingInfo == undefined || + audioItem.engineId == undefined + ) + return false; + return ( + getters.SELECTABLE_MOPHING_TARGET_ENGINES.includes( + audioItem.engineId + ) && audioItem.engineId === audioItem.morphingInfo.targetEngineId + ); + }, + }, + SET_AUDIO_QUERY: { mutation( state, @@ -742,11 +761,6 @@ export const audioStore = createPartialStore({ ) { state.audioItems[audioKey].engineId = engineId; state.audioItems[audioKey].styleId = styleId; - if ( - state.audioItems[audioKey].morphingInfo?.targetEngineId !== engineId - ) { - state.audioItems[audioKey].morphingInfo = undefined; - } }, }, @@ -888,7 +902,7 @@ export const audioStore = createPartialStore({ if (presetItem == undefined) return; // Filter name property from presetItem in order to extract audioInfos. - const { name: _, ...presetAudioInfos } = presetItem; + const { name: _, morphingInfo, ...presetAudioInfos } = presetItem; // Type Assertion const audioInfos: Omit< @@ -897,6 +911,8 @@ export const audioStore = createPartialStore({ > = presetAudioInfos; audioItem.query = { ...audioItem.query, ...audioInfos }; + + audioItem.morphingInfo = morphingInfo; }, }, @@ -1061,7 +1077,10 @@ export const audioStore = createPartialStore({ GENERATE_AUDIO_FROM_AUDIO_ITEM: { action: createUILockAction( - async ({ dispatch, state }, { audioItem }: { audioItem: AudioItem }) => { + async ( + { dispatch, getters, state }, + { audioItem }: { audioItem: AudioItem } + ) => { const engineId = audioItem.engineId; if (engineId === undefined) throw new Error(`engineId is not defined for audioItem`); @@ -1079,7 +1098,10 @@ export const audioStore = createPartialStore({ engineId, }) .then((instance) => { - if (audioItem.morphingInfo) { + if ( + audioItem.morphingInfo != undefined && + getters.VALID_MOPHING_INFO(audioItem) + ) { return instance.invoke("synthesisMorphingSynthesisMorphingPost")({ audioQuery: convertAudioQueryFromEditorToEngine( audioQuery, diff --git a/src/store/project.ts b/src/store/project.ts index f11bdfbc2d..bb7830da3a 100755 --- a/src/store/project.ts +++ b/src/store/project.ts @@ -419,6 +419,15 @@ const audioQuerySchema = { }, } as const; +const morphingInfoSchema = { + properties: { + rate: { type: "float32" }, + targetEngineId: { type: "string" }, + targetSpeakerId: { type: "string" }, + targetStyleId: { type: "int32" }, + }, +} as const; + const audioItemSchema = { properties: { text: { type: "string" }, @@ -428,6 +437,7 @@ const audioItemSchema = { styleId: { type: "int32" }, query: audioQuerySchema, presetKey: { type: "string" }, + morphingInfo: morphingInfoSchema, }, } as const; diff --git a/src/store/setting.ts b/src/store/setting.ts index 95bf559591..abbca24bde 100644 --- a/src/store/setting.ts +++ b/src/store/setting.ts @@ -43,6 +43,7 @@ export const settingStoreState: SettingStoreState = { experimentalSetting: { enablePreset: false, enableInterrogativeUpspeak: false, + enableMorphing: false, }, splitTextWhenPaste: "PERIOD_AND_NEW_LINE", splitterPosition: { diff --git a/src/store/type.ts b/src/store/type.ts index c170b43a51..5c32954acb 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -273,6 +273,14 @@ export type AudioStoreTypes = { }; }; + SELECTABLE_MOPHING_TARGET_ENGINES: { + getter: string[]; + }; + + VALID_MOPHING_INFO: { + getter(audioItem: AudioItem): boolean; + }; + SET_AUDIO_QUERY: { mutation: { audioKey: string; audioQuery: AudioQuery }; action(payload: { audioKey: string; audioQuery: AudioQuery }): void; diff --git a/src/type/preload.ts b/src/type/preload.ts index 6f92798cf9..85e2c7c310 100644 --- a/src/type/preload.ts +++ b/src/type/preload.ts @@ -303,6 +303,7 @@ export type ThemeSetting = { export type ExperimentalSetting = { enablePreset: boolean; enableInterrogativeUpspeak: boolean; + enableMorphing: boolean; }; export type SplitterPosition = { diff --git a/tests/unit/store/Vuex.spec.ts b/tests/unit/store/Vuex.spec.ts index bd730a6e19..0b6f477b84 100644 --- a/tests/unit/store/Vuex.spec.ts +++ b/tests/unit/store/Vuex.spec.ts @@ -112,6 +112,7 @@ describe("store/vuex.js test", () => { experimentalSetting: { enablePreset: false, enableInterrogativeUpspeak: false, + enableMorphing: false, }, splitTextWhenPaste: "PERIOD_AND_NEW_LINE", splitterPosition: {