From 660240f1e6f4ff86a62393f964dc46e2ce83d2dd Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 8 Sep 2024 23:02:58 +0200 Subject: [PATCH 1/4] Improve history import performance and fix some bugs --- src/constants.js | 2 ++ src/datastores/handlers/base.js | 6 ++++++ src/datastores/handlers/electron.js | 7 +++++++ src/datastores/handlers/web.js | 4 ++++ src/main/index.js | 9 ++++++++ .../components/data-settings/data-settings.js | 21 ++++++++++++++----- src/renderer/store/modules/history.js | 21 +++++++++++++++++++ src/renderer/store/modules/settings.js | 12 +++++++++++ 8 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/constants.js b/src/constants.js index 490c8d67771a5..a45a71598d527 100644 --- a/src/constants.js +++ b/src/constants.js @@ -51,6 +51,7 @@ const DBActions = { }, HISTORY: { + OVERWRITE: 'db-action-history-overwrite', UPDATE_WATCH_PROGRESS: 'db-action-history-update-watch-progress', UPDATE_PLAYLIST: 'db-action-history-update-playlist', }, @@ -78,6 +79,7 @@ const SyncEvents = { }, HISTORY: { + OVERWRITE: 'sync-history-overwrite', UPDATE_WATCH_PROGRESS: 'sync-history-update-watch-progress', UPDATE_PLAYLIST: 'sync-history-update-playlist', }, diff --git a/src/datastores/handlers/base.js b/src/datastores/handlers/base.js index 4a7db5cbb8c3d..b3ec944b319ed 100644 --- a/src/datastores/handlers/base.js +++ b/src/datastores/handlers/base.js @@ -56,6 +56,12 @@ class History { return db.history.updateAsync({ videoId: record.videoId }, record, { upsert: true }) } + static async overwrite(records) { + await db.history.removeAsync({}, { multi: true }) + + await db.history.insertAsync(records) + } + static updateWatchProgress(videoId, watchProgress) { return db.history.updateAsync({ videoId }, { $set: { watchProgress } }, { upsert: true }) } diff --git a/src/datastores/handlers/electron.js b/src/datastores/handlers/electron.js index cc0b473a3b990..41d4872e45d8e 100644 --- a/src/datastores/handlers/electron.js +++ b/src/datastores/handlers/electron.js @@ -32,6 +32,13 @@ class History { ) } + static overwrite(records) { + return ipcRenderer.invoke( + IpcChannels.DB_HISTORY, + { action: DBActions.HISTORY.OVERWRITE, data: records } + ) + } + static updateWatchProgress(videoId, watchProgress) { return ipcRenderer.invoke( IpcChannels.DB_HISTORY, diff --git a/src/datastores/handlers/web.js b/src/datastores/handlers/web.js index 93ffa3d68c8ff..0fa321bcb4beb 100644 --- a/src/datastores/handlers/web.js +++ b/src/datastores/handlers/web.js @@ -29,6 +29,10 @@ class History { return baseHandlers.history.upsert(record) } + static overwrite(records) { + return baseHandlers.history.overwrite(records) + } + static updateWatchProgress(videoId, watchProgress) { return baseHandlers.history.updateWatchProgress(videoId, watchProgress) } diff --git a/src/main/index.js b/src/main/index.js index 4cd8cd680a5c0..0848a677c2f1d 100644 --- a/src/main/index.js +++ b/src/main/index.js @@ -1075,6 +1075,15 @@ function runApp() { ) return null + case DBActions.HISTORY.OVERWRITE: + await baseHandlers.history.overwrite(data) + syncOtherWindows( + IpcChannels.SYNC_HISTORY, + event, + { event: SyncEvents.HISTORY.OVERWRITE, data } + ) + return null + case DBActions.HISTORY.UPDATE_WATCH_PROGRESS: await baseHandlers.history.updateWatchProgress(data.videoId, data.watchProgress) syncOtherWindows( diff --git a/src/renderer/components/data-settings/data-settings.js b/src/renderer/components/data-settings/data-settings.js index c528fcabd283e..cd4f61cca59ce 100644 --- a/src/renderer/components/data-settings/data-settings.js +++ b/src/renderer/components/data-settings/data-settings.js @@ -50,6 +50,9 @@ export default defineComponent({ allPlaylists: function () { return this.$store.getters.getAllPlaylists }, + historyCacheById: function () { + return this.$store.getters.getHistoryCacheById + }, historyCacheSorted: function () { return this.$store.getters.getHistoryCacheSorted }, @@ -616,7 +619,7 @@ export default defineComponent({ }) }, - importFreeTubeHistory(textDecode) { + async importFreeTubeHistory(textDecode) { textDecode.pop() const requiredKeys = [ @@ -644,6 +647,8 @@ export default defineComponent({ 'paid', ] + const historyItems = new Map(Object.entries(this.historyCacheById)) + textDecode.forEach((history) => { const historyData = JSON.parse(history) // We would technically already be done by the time the data is parsed, @@ -667,14 +672,16 @@ export default defineComponent({ showToast(this.$t('Settings.Data Settings.History object has insufficient data, skipping item')) console.error('Missing Keys: ', missingKeys, historyData) } else { - this.updateHistory(historyObject) + historyItems.set(historyObject.videoId, historyObject) } }) + await this.overwriteHistory(historyItems) + showToast(this.$t('Settings.Data Settings.All watched history has been successfully imported')) }, - importYouTubeHistory(historyData) { + async importYouTubeHistory(historyData) { const filterPredicate = item => item.products.includes('YouTube') && item.titleUrl != null && // removed video doesnt contain url... @@ -722,6 +729,8 @@ export default defineComponent({ 'activityControls', ].concat(Object.keys(keyMapping)) + const historyItems = new Map(Object.entries(this.historyCacheById)) + filteredHistoryData.forEach(element => { const historyObject = {} @@ -750,10 +759,12 @@ export default defineComponent({ historyObject.watchProgress = 1 historyObject.isLive = false - this.updateHistory(historyObject) + historyItems.set(historyObject.videoId, historyObject) } }) + await this.overwriteHistory(historyItems) + showToast(this.$t('Settings.Data Settings.All watched history has been successfully imported')) }, @@ -1069,10 +1080,10 @@ export default defineComponent({ ...mapActions([ 'updateProfile', 'updateShowProgressBar', - 'updateHistory', 'addPlaylist', 'addVideo', 'updatePlaylist', + 'overwriteHistory' ]), ...mapMutations([ diff --git a/src/renderer/store/modules/history.js b/src/renderer/store/modules/history.js index 59d7fa1fc3213..b99b63a1c2b4a 100644 --- a/src/renderer/store/modules/history.js +++ b/src/renderer/store/modules/history.js @@ -45,6 +45,27 @@ const actions = { } }, + /** + * @param {any} param0 + * @param {Map} historyItems + */ + async overwriteHistory({ commit }, historyItems) { + try { + const sortedRecords = Array.from(historyItems.values()) + + // sort before sending saving to the database and passing to other windows + // so that the other windows can use it as is, without having to sort the array themselves + sortedRecords.sort((a, b) => b.timeWatched - a.timeWatched) + + await DBHistoryHandlers.overwrite(sortedRecords) + + commit('setHistoryCacheSorted', sortedRecords) + commit('setHistoryCacheById', Object.fromEntries(historyItems)) + } catch (errMessage) { + console.error(errMessage) + } + }, + async removeFromHistory({ commit }, videoId) { try { await DBHistoryHandlers.delete(videoId) diff --git a/src/renderer/store/modules/settings.js b/src/renderer/store/modules/settings.js index 864890afc9ce5..bb7c1aa7784da 100644 --- a/src/renderer/store/modules/settings.js +++ b/src/renderer/store/modules/settings.js @@ -476,6 +476,18 @@ const customActions = { commit('upsertToHistoryCache', data) break + case SyncEvents.HISTORY.OVERWRITE: { + const byId = {} + data.forEach(video => { + byId[video.videoId] = video + }) + + // It comes pre-sorted, so we don't have to sort it here + commit('setHistoryCacheSorted', data) + commit('setHistoryCacheById', byId) + break + } + case SyncEvents.HISTORY.UPDATE_WATCH_PROGRESS: commit('updateRecordWatchProgressInHistoryCache', data) break From 375516776f59f8b3d62de95b8b42115550ea301c Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:01:47 +0200 Subject: [PATCH 2/4] Make the view count optional --- src/renderer/components/data-settings/data-settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/components/data-settings/data-settings.js b/src/renderer/components/data-settings/data-settings.js index cd4f61cca59ce..c368f542c9987 100644 --- a/src/renderer/components/data-settings/data-settings.js +++ b/src/renderer/components/data-settings/data-settings.js @@ -633,7 +633,6 @@ export default defineComponent({ 'title', 'type', 'videoId', - 'viewCount', 'watchProgress', ] @@ -641,6 +640,7 @@ export default defineComponent({ // `_id` absent if marked as watched manually '_id', 'lastViewedPlaylistId', + 'viewCount', ] const ignoredKeys = [ From bf0565240c7e8127669adc496f958ee579507dd1 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 15 Sep 2024 23:19:06 +0200 Subject: [PATCH 3/4] Add lastViewedPlaylistItemId to the optional keys --- src/renderer/components/data-settings/data-settings.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/renderer/components/data-settings/data-settings.js b/src/renderer/components/data-settings/data-settings.js index c368f542c9987..789b7db32a599 100644 --- a/src/renderer/components/data-settings/data-settings.js +++ b/src/renderer/components/data-settings/data-settings.js @@ -640,6 +640,7 @@ export default defineComponent({ // `_id` absent if marked as watched manually '_id', 'lastViewedPlaylistId', + 'lastViewedPlaylistItemId', 'viewCount', ] From 4d2bd5578bc7b9ef9353031869fb6797c8a82a19 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 15 Sep 2024 23:29:15 +0200 Subject: [PATCH 4/4] I forgot lastViewedPlaylistType --- src/renderer/components/data-settings/data-settings.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/renderer/components/data-settings/data-settings.js b/src/renderer/components/data-settings/data-settings.js index 789b7db32a599..7c950732ebb5e 100644 --- a/src/renderer/components/data-settings/data-settings.js +++ b/src/renderer/components/data-settings/data-settings.js @@ -641,6 +641,7 @@ export default defineComponent({ '_id', 'lastViewedPlaylistId', 'lastViewedPlaylistItemId', + 'lastViewedPlaylistType', 'viewCount', ]