diff --git a/_scripts/build.js b/_scripts/build.js index 3c4c0312b6d4f..035f986c97dbe 100644 --- a/_scripts/build.js +++ b/_scripts/build.js @@ -62,13 +62,6 @@ const config = { './dist/**/*', '!dist/web/*', '!node_modules/**/*', - - // renderer - 'node_modules/{miniget,ytsr}/**/*', - - '!**/README.md', - '!**/*.js.map', - '!**/*.d.ts', ], dmg: { contents: [ diff --git a/_scripts/webpack.renderer.config.js b/_scripts/webpack.renderer.config.js index 723661f8abf87..8b8ad3c6e4e26 100644 --- a/_scripts/webpack.renderer.config.js +++ b/_scripts/webpack.renderer.config.js @@ -32,10 +32,6 @@ const config = { path: path.join(__dirname, '../dist'), filename: '[name].js', }, - // webpack spits out errors while inlining ytsr as - // they dynamically import their package.json file to extract the bug report URL - // the error: "Critical dependency: the request of a dependency is an expression" - externals: ['ytsr'], module: { rules: [ { diff --git a/_scripts/webpack.web.config.js b/_scripts/webpack.web.config.js index ac15675428feb..17681dd7f0e9c 100644 --- a/_scripts/webpack.web.config.js +++ b/_scripts/webpack.web.config.js @@ -24,8 +24,7 @@ const config = { }, externals: { electron: '{}', - 'youtubei.js': '{}', - ytsr: '{}' + 'youtubei.js': '{}' }, module: { rules: [ diff --git a/package.json b/package.json index 6c1f837615f61..ee918b1734573 100644 --- a/package.json +++ b/package.json @@ -83,8 +83,7 @@ "youtubei.js": "^2.7.0", "yt-channel-info": "^3.2.1", "yt-dash-manifest-generator": "1.1.0", - "ytdl-core": "https://github.com/absidue/node-ytdl-core#fix-likes-extraction", - "ytsr": "^3.8.0" + "ytdl-core": "https://github.com/absidue/node-ytdl-core#fix-likes-extraction" }, "devDependencies": { "@babel/core": "^7.20.7", diff --git a/src/renderer/components/ft-list-channel/ft-list-channel.js b/src/renderer/components/ft-list-channel/ft-list-channel.js index aac1e76e08365..31de269b672c8 100644 --- a/src/renderer/components/ft-list-channel/ft-list-channel.js +++ b/src/renderer/components/ft-list-channel/ft-list-channel.js @@ -20,6 +20,7 @@ export default Vue.extend({ channelName: '', subscriberCount: 0, videoCount: '', + handle: null, uploadedTime: '', description: '' } @@ -39,7 +40,7 @@ export default Vue.extend({ } }, mounted: function () { - if (typeof (this.data.avatars) !== 'undefined') { + if (this.data.dataSource === 'local' || typeof (this.data.avatars) !== 'undefined') { this.parseLocalData() } else { this.parseInvidiousData() @@ -47,7 +48,7 @@ export default Vue.extend({ }, methods: { parseLocalData: function () { - this.thumbnail = this.data.bestAvatar.url + this.thumbnail = this.data.thumbnail ?? this.data.bestAvatar.url if (!this.thumbnail.includes('https:')) { this.thumbnail = `https:${this.thumbnail}` @@ -66,6 +67,10 @@ export default Vue.extend({ this.videoCount = Intl.NumberFormat(this.currentLocale).format(this.data.videos) } + if (this.data.handle) { + this.handle = this.data.handle + } + this.description = this.data.descriptionShort }, diff --git a/src/renderer/components/ft-list-channel/ft-list-channel.scss b/src/renderer/components/ft-list-channel/ft-list-channel.scss index af386ff4b9119..9ceb653420faa 100644 --- a/src/renderer/components/ft-list-channel/ft-list-channel.scss +++ b/src/renderer/components/ft-list-channel/ft-list-channel.scss @@ -1 +1,6 @@ @use '../../scss-partials/_ft-list-item'; + +.handle { + color: inherit; + text-decoration: none; +} diff --git a/src/renderer/components/ft-list-channel/ft-list-channel.vue b/src/renderer/components/ft-list-channel/ft-list-channel.vue index 3d29d12e3c5c0..6ac9f8ed835dd 100644 --- a/src/renderer/components/ft-list-channel/ft-list-channel.vue +++ b/src/renderer/components/ft-list-channel/ft-list-channel.vue @@ -31,7 +31,15 @@ > {{ subscriberCount }} subscribers - + + {{ handle }} + {{ videoCount }} videos diff --git a/src/renderer/components/ft-list-playlist/ft-list-playlist.js b/src/renderer/components/ft-list-playlist/ft-list-playlist.js index 308ec4639a4ff..b310135f30e03 100644 --- a/src/renderer/components/ft-list-playlist/ft-list-playlist.js +++ b/src/renderer/components/ft-list-playlist/ft-list-playlist.js @@ -56,7 +56,10 @@ export default Vue.extend({ } }, mounted: function () { - if (typeof (this.data.owner) === 'object') { + // temporary until we've migrated the whole local API to youtubei.js + if (this.data.dataSource === 'local') { + this.parseLocalDataNew() + } else if (typeof (this.data.owner) === 'object') { this.parseLocalData() } else { this.parseInvidiousData() @@ -98,6 +101,17 @@ export default Vue.extend({ this.videoCount = this.data.length }, + // TODO: after the local API is fully switched to YouTube.js + // cleanup the old local API stuff + parseLocalDataNew: function () { + this.title = this.data.title + this.thumbnail = this.data.thumbnail + this.channelName = this.data.channelName + this.channelLink = this.data.channelId + this.playlistLink = this.data.playlistId + this.videoCount = this.data.videoCount + }, + ...mapActions([ 'openInExternalPlayer' ]) diff --git a/src/renderer/components/ft-list-video/ft-list-video.js b/src/renderer/components/ft-list-video/ft-list-video.js index 8111889d7be09..5f80427690003 100644 --- a/src/renderer/components/ft-list-video/ft-list-video.js +++ b/src/renderer/components/ft-list-video/ft-list-video.js @@ -385,7 +385,9 @@ export default Vue.extend({ this.isPremium = this.data.premium || false this.viewCount = this.data.viewCount - if (typeof (this.data.premiereTimestamp) !== 'undefined') { + if (typeof this.data.premiereDate !== 'undefined') { + this.publishedText = this.data.premiereDate.toLocaleString() + } else if (typeof (this.data.premiereTimestamp) !== 'undefined') { this.publishedText = new Date(this.data.premiereTimestamp * 1000).toLocaleString() } else { this.publishedText = this.data.publishedText diff --git a/src/renderer/components/ft-list-video/ft-list-video.vue b/src/renderer/components/ft-list-video/ft-list-video.vue index 78c8110d48aff..d866b5cc2f02d 100644 --- a/src/renderer/components/ft-list-video/ft-list-video.vue +++ b/src/renderer/components/ft-list-video/ft-list-video.vue @@ -27,9 +27,12 @@
- {{ isLive ? $t("Video.Live") : duration }} + {{ isLive ? $t("Video.Live") : (isUpcoming ? $t("Video.Upcoming") : duration) }}
fetch(input, init), cache @@ -74,7 +76,7 @@ export async function getLocalTrending(location, tab, instance) { const results = resultsInstance.videos .filter((video) => video.type === 'Video') - .map(parseLocalListVideo) + .map(parseListVideo) return { results, @@ -82,6 +84,54 @@ export async function getLocalTrending(location, tab, instance) { } } +/** + * @param {string} query + * @param {object} filters + * @param {boolean} safetyMode + */ +export async function getLocalSearchResults(query, filters, safetyMode) { + const innertube = await createInnertube({ safetyMode }) + const response = await innertube.search(query, convertSearchFilters(filters)) + + return handleSearchResponse(response) +} + +/** + * @typedef {import('youtubei.js/dist/src/parser/youtube/Search').default} Search + */ + +/** + * @param {Search} continuationData + */ +export async function getLocalSearchContinuation(continuationData) { + const response = await continuationData.getContinuation() + + return handleSearchResponse(response) +} + +/** + * @param {Search} response + */ +function handleSearchResponse(response) { + if (!response.results) { + return { + results: [], + continuationData: null + } + } + + const results = response.results + .filter((item) => { + return item.type === 'Video' || item.type === 'Channel' || item.type === 'Playlist' + }) + .map((item) => parseListItem(item)) + + return { + results, + continuationData: response.has_continuation ? response : null + } +} + /** * @typedef {import('youtubei.js/dist/src/parser/classes/PlaylistVideo').default} PlaylistVideo */ @@ -106,7 +156,7 @@ export function parseLocalPlaylistVideo(video) { /** * @param {Video} video */ -function parseLocalListVideo(video) { +function parseListVideo(video) { return { type: 'video', videoId: video.id, @@ -117,6 +167,96 @@ function parseLocalListVideo(video) { viewCount: extractNumberFromString(video.view_count.text), publishedText: video.published.text, lengthSeconds: isNaN(video.duration.seconds) ? '' : video.duration.seconds, - liveNow: video.is_live + liveNow: video.is_live, + isUpcoming: video.is_upcoming || video.is_premiere, + premiereDate: video.upcoming } } + +/** + * @typedef {import('youtubei.js/dist/src/parser/helpers').YTNode} YTNode + * @typedef {import('youtubei.js/dist/src/parser/classes/Channel').default} Channel + * @typedef {import('youtubei.js/dist/src/parser/classes/Playlist').default} Playlist + */ + +/** + * @param {YTNode} item + */ +function parseListItem(item) { + switch (item.type) { + case 'Video': + return parseListVideo(item) + case 'Channel': { + /** @type {Channel} */ + const channel = item + + // see upstream TODO: https://github.com/LuanRT/YouTube.js/blob/main/src/parser/classes/Channel.ts#L33 + + // according to https://github.com/iv-org/invidious/issues/3514#issuecomment-1368080392 + // the response can be the new or old one, so we currently need to handle both here + let subscribers + let videos = null + let handle = null + if (channel.subscribers.text.startsWith('@')) { + subscribers = channel.videos.text + handle = channel.subscribers.text + } else { + subscribers = channel.subscribers.text + videos = channel.videos.text + } + + return { + type: 'channel', + dataSource: 'local', + thumbnail: channel.author.best_thumbnail?.url, + name: channel.author.name, + channelID: channel.author.id, + subscribers, + videos, + handle, + descriptionShort: channel.description_snippet.text + } + } + case 'Playlist': { + /** @type {Playlist} */ + const playlist = item + return { + type: 'playlist', + dataSource: 'local', + title: playlist.title, + thumbnail: playlist.thumbnails[0].url, + channelName: playlist.author.name, + channelId: playlist.author.id, + playlistId: playlist.id, + videoCount: extractNumberFromString(playlist.video_count.text) + } + } + } +} + +function convertSearchFilters(filters) { + const convertedFilters = {} + + // some of the fields have different names and + // others have empty strings that we don't want to pass to youtubei.js + + if (filters) { + if (filters.sortBy) { + convertedFilters.sort_by = filters.sortBy + } + + if (filters.time) { + convertedFilters.upload_date = filters.time + } + + if (filters.type) { + convertedFilters.type = filters.type + } + + if (filters.duration) { + convertedFilters.type = filters.duration + } + } + + return convertedFilters +} diff --git a/src/renderer/helpers/utils.js b/src/renderer/helpers/utils.js index 6c9e6dfbfeb25..31f818902afaf 100644 --- a/src/renderer/helpers/utils.js +++ b/src/renderer/helpers/utils.js @@ -481,7 +481,7 @@ export async function getPicturesPath() { export function extractNumberFromString(str) { if (typeof str === 'string') { - return parseInt(str.replace(/\D+/, '')) + return parseInt(str.replaceAll(/\D+/g, '')) } else { return NaN } diff --git a/src/renderer/scss-partials/_ft-list-item.scss b/src/renderer/scss-partials/_ft-list-item.scss index 806fdea65f0b0..2b03737e49dbf 100644 --- a/src/renderer/scss-partials/_ft-list-item.scss +++ b/src/renderer/scss-partials/_ft-list-item.scss @@ -287,6 +287,7 @@ $watched-transition-duration: 0.5s; } .videoWatched, -.live { +.live, +.upcoming { text-transform: uppercase; } diff --git a/src/renderer/store/modules/ytdl.js b/src/renderer/store/modules/ytdl.js index 03299b7451c5e..3a946f9ffc3cb 100644 --- a/src/renderer/store/modules/ytdl.js +++ b/src/renderer/store/modules/ytdl.js @@ -1,15 +1,10 @@ import ytdl from 'ytdl-core' -import ytsr from 'ytsr' import { SocksProxyAgent } from 'socks-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' import { HttpProxyAgent } from 'http-proxy-agent' -import { searchFiltersMatch } from '../../helpers/utils' - -const state = { - isYtSearchRunning: false -} +const state = {} const getters = {} @@ -41,156 +36,6 @@ function createProxyAgent(protocol, hostname, port) { } const actions = { - ytSearch ({ commit, dispatch, rootState }, payload) { - return new Promise((resolve, reject) => { - if (state.isYtSearchRunning) { - resolve(false) - } - - if (typeof payload.options.nextpageRef !== 'undefined') { - const continuation = payload.options.nextpageRef - const nextPageResults = ytsr.continueReq(continuation) - resolve(nextPageResults) - return - } - - const defaultFilters = { - sortBy: 'relevance', - time: '', - type: 'all', - duration: '' - } - - const settings = rootState.settings - - if (settings.useProxy) { - const agent = createProxyAgent(settings.proxyProtocol, settings.proxyHostname, settings.proxyPort) - - payload.options.requestOptions = { agent } - } - - commit('toggleIsYtSearchRunning') - - if (!searchFiltersMatch(defaultFilters, rootState.utils.searchSettings)) { - dispatch('ytSearchGetFilters', payload).then((filter) => { - if (typeof (payload.options.nextpageRef) === 'undefined' && filter !== payload.query) { - payload.options.nextpageRef = filter - } - - const query = filter || payload.query - - ytsr(query, payload.options).then((result) => { - resolve(result) - }).catch((err) => { - console.error(err) - reject(err) - }).finally(() => { - commit('toggleIsYtSearchRunning') - }) - }).catch((err) => { - console.error(err) - commit('toggleIsYtSearchRunning') - reject(err) - }) - } else { - ytsr(payload.query, payload.options).then((result) => { - resolve(result) - }).catch((err) => { - console.error(err) - reject(err) - }).finally(() => { - commit('toggleIsYtSearchRunning') - }) - } - }) - }, - - async ytSearchGetFilters ({ rootState }, payload) { - let options = null - let agent = null - const settings = rootState.settings - - if (settings.useProxy) { - agent = createProxyAgent(settings.proxyProtocol, settings.proxyHostname, settings.proxyPort) - } - - options = { - requestOptions: { agent } - } - - let filter = await ytsr.getFilters(payload.query, options) - let filterUrl = null - let searchSettings = payload.searchSettings - - if (typeof (searchSettings) === 'undefined') { - searchSettings = rootState.utils.searchSettings - } - - if (searchSettings.sortBy !== 'relevance') { - let filterValue - switch (searchSettings.sortBy) { - case 'rating': - filterValue = 'Rating' - break - case 'upload_date': - filterValue = 'Upload date' - break - case 'view_count': - filterValue = 'View count' - break - } - filterUrl = filter.get('Sort by').get(filterValue).url - filter = await ytsr.getFilters(filterUrl, options) - } - - if (searchSettings.duration !== '') { - let filterValue = null - if (searchSettings.duration === 'short') { - filterValue = 'Under 4 minutes' - } else if (searchSettings.duration === 'long') { - filterValue = 'Over 20 minutes' - } - - filterUrl = filter.get('Duration').get(filterValue).url - filter = await ytsr.getFilters(filterUrl, options) - } - - if (searchSettings.time !== '') { - let filterValue = null - - switch (searchSettings.time) { - case 'hour': - filterValue = 'Last hour' - break - case 'today': - filterValue = 'Today' - break - case 'week': - filterValue = 'This week' - break - case 'month': - filterValue = 'This month' - break - case 'year': - filterValue = 'This year' - break - } - - filterUrl = filter.get('Upload date').get(filterValue).url - filter = await ytsr.getFilters(filterUrl, options) - } - - if (searchSettings.type !== 'all') { - const filterValue = searchSettings.type.charAt(0).toUpperCase() + searchSettings.type.slice(1) - filterUrl = filter.get('Type').get(filterValue).url - filter = await ytsr.getFilters(filterUrl, options) - } - - return new Promise((resolve, reject) => { - resolve(filterUrl) - }) - }, - ytGetVideoInformation ({ rootState }, videoId) { return new Promise((resolve, reject) => { let agent = null @@ -212,11 +57,7 @@ const actions = { } } -const mutations = { - toggleIsYtSearchRunning (state) { - state.isYtSearchRunning = !state.isYtSearchRunning - } -} +const mutations = {} export default { state, diff --git a/src/renderer/views/Search/Search.js b/src/renderer/views/Search/Search.js index 0b762d76db501..2ff2468fdd8cc 100644 --- a/src/renderer/views/Search/Search.js +++ b/src/renderer/views/Search/Search.js @@ -3,8 +3,8 @@ import { mapActions } from 'vuex' import FtLoader from '../../components/ft-loader/ft-loader.vue' import FtCard from '../../components/ft-card/ft-card.vue' import FtElementList from '../../components/ft-element-list/ft-element-list.vue' -import { timeToSeconds } from 'youtubei.js/dist/src/utils/Utils' import { copyToClipboard, searchFiltersMatch, showToast } from '../../helpers/utils' +import { getLocalSearchContinuation, getLocalSearchResults } from '../../helpers/api/local' export default Vue.extend({ name: 'Search', @@ -20,7 +20,7 @@ export default Vue.extend({ amountOfResults: 0, query: '', searchPage: 1, - nextPageRef: '', + nextPageRef: null, lastSearchQuery: '', searchSettings: {}, shownResults: [] @@ -65,7 +65,6 @@ export default Vue.extend({ const payload = { query: query, - nextPage: false, options: {}, searchSettings: searchSettings } @@ -87,7 +86,6 @@ export default Vue.extend({ const payload = { query: this.query, - nextPage: false, options: {}, searchSettings: this.searchSettings } @@ -122,86 +120,68 @@ export default Vue.extend({ } }, - performSearchLocal: function (payload) { - if (!payload.nextPage) { - this.isLoading = true - payload.options.pages = 1 - } + performSearchLocal: async function (payload) { + this.isLoading = true - payload.options.safeSearch = this.showFamilyFriendlyOnly + try { + const { results, continuationData } = await getLocalSearchResults(payload.query, payload.searchSettings, this.showFamilyFriendlyOnly) - this.ytSearch(payload).then((result) => { - if (!result) { + if (results.length === 0) { return } this.apiUsed = 'local' - this.amountOfResults = result.results + this.shownResults = results + this.nextPageRef = continuationData - const returnData = result.items.filter((item) => { - if (typeof item !== 'undefined') { - return item.type === 'video' || item.type === 'channel' || item.type === 'playlist' - } + this.isLoading = false - return null - }) + const historyPayload = { + query: payload.query, + data: this.shownResults, + searchSettings: this.searchSettings, + nextPageRef: this.nextPageRef + } - const dataToShow = [] - returnData.forEach((video) => { - if (video.type === 'video') { - const authId = video.author.channelID - const publishDate = video.uploadedAt - let videoDuration = video.duration - const videoId = video.id - if (videoDuration !== null && videoDuration !== '' && videoDuration !== 'LIVE' && videoDuration !== 'UPCOMING' && videoDuration !== 'PREMIERE') { - videoDuration = timeToSeconds(video.duration) - } - dataToShow.push( - { - videoId: videoId, - title: video.title, - type: 'video', - author: video.author.name, - authorId: authId, - authorUrl: video.author.url, - videoThumbnails: video.thumbnail, - description: video.description, - viewCount: video.views, - published: publishDate, - publishedText: publishDate, - lengthSeconds: videoDuration, - liveNow: video.isLive || videoDuration === 'LIVE', - paid: false, - premium: false, - isUpcoming: videoDuration === 'UPCOMING' || videoDuration === 'PREMIERE', - timeText: videoDuration - } - ) - } else { - dataToShow.push(video) - } + this.$store.commit('addToSessionSearchHistory', historyPayload) + } catch (err) { + console.error(err) + const errorMessage = this.$t('Local API Error (Click to copy)') + showToast(`${errorMessage}: ${err}`, 10000, () => { + copyToClipboard(err) }) - - if (payload.nextPage) { - this.shownResults = this.shownResults.concat(dataToShow) + if (this.backendPreference === 'local' && this.backendFallback) { + showToast(this.$t('Falling back to Invidious API')) + this.performSearchInvidious(payload) } else { - this.shownResults = dataToShow + this.isLoading = false } + } + }, - this.nextPageRef = result.continuation - this.isLoading = false + getNextpageLocal: async function (payload) { + try { + const { results, continuationData } = getLocalSearchContinuation(payload.options.nextPageRef) + + if (results.length === 0) { + return + } + + this.apiUsed = 'local' + + this.shownResults = this.shownResults.concat(results) + this.nextPageRef = continuationData const historyPayload = { query: payload.query, data: this.shownResults, searchSettings: this.searchSettings, - nextPageRef: result.continuation, - amountOfResults: result.results + nextPageRef: this.nextPageRef } this.$store.commit('addToSessionSearchHistory', historyPayload) - }).catch((err) => { + } catch (err) { console.error(err) const errorMessage = this.$t('Local API Error (Click to copy)') showToast(`${errorMessage}: ${err}`, 10000, () => { @@ -213,7 +193,7 @@ export default Vue.extend({ } else { this.isLoading = false } - }) + } }, performSearchInvidious: function (payload) { @@ -281,19 +261,18 @@ export default Vue.extend({ nextPage: function () { const payload = { query: this.query, - nextPage: true, searchSettings: this.searchSettings, options: { - nextpageRef: this.nextPageRef + nextPageRef: this.nextPageRef } } if (this.apiUsed === 'local') { - if (this.amountOfResults <= this.shownResults.length) { - showToast(this.$t('Search Filters.There are no more results for this search')) - } else { + if (this.nextPageRef !== null) { showToast(this.$t('Search Filters["Fetching results. Please wait"]')) - this.performSearchLocal(payload) + this.getNextpageLocal(payload) + } else { + showToast(this.$t('Search Filters.There are no more results for this search')) } } else { showToast(this.$t('Search Filters["Fetching results. Please wait"]')) @@ -319,7 +298,6 @@ export default Vue.extend({ }, ...mapActions([ - 'ytSearch', 'invidiousAPICall' ]) } diff --git a/static/locales/en-US.yaml b/static/locales/en-US.yaml index 432a263a5c1e4..602e253089ded 100644 --- a/static/locales/en-US.yaml +++ b/static/locales/en-US.yaml @@ -566,6 +566,7 @@ Video: # As in a Live Video Premieres on: Premieres on Premieres: Premieres + Upcoming: Upcoming Live: Live Live Now: Live Now Live Chat: Live Chat diff --git a/yarn.lock b/yarn.lock index 83ec644e6be9c..3622c61003cee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9286,10 +9286,3 @@ ytdl-core@^3.2.2: m3u8stream "^0.8.6" miniget "^4.2.2" sax "^1.1.3" - -ytsr@^3.8.0: - version "3.8.0" - resolved "https://registry.yarnpkg.com/ytsr/-/ytsr-3.8.0.tgz#49a8e5dc413f41515fc3d79d93ee3e073d10e772" - integrity sha512-R+RfYXvBBMAr2e4OxrQ5SBv5x/Mdhmcj1Q8TH0f2HK5d2jbhHOtK4BdzPvLriA6MDoMwqqX04GD8Rpf9UNtSTg== - dependencies: - miniget "^4.2.2"