From 18951cc0cddd651227e5c97646ee892bd6933663 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Sat, 28 Oct 2023 00:26:07 +0200 Subject: [PATCH 01/27] Merge previous scanner changes --- .gitignore | 1 + Cargo.lock | 284 ++++++++++++++++++ Cargo.toml | 2 + package-lock.json | 37 ++- package.json | 2 +- packages/main/package.json | 1 + .../main/src/controllers/local-library.ts | 17 +- .../library-scanner/library-scanner.test.ts | 10 + packages/main/tsconfig.json | 3 +- packages/main/webpack.config.ts | 7 +- packages/scanner/.gitignore | 5 + packages/scanner/Cargo.lock | 156 ++++++++++ packages/scanner/Cargo.toml | 24 ++ packages/scanner/README.md | 119 ++++++++ packages/scanner/index.d.ts | 16 + packages/scanner/package-lock.json | 25 ++ packages/scanner/package.json | 27 ++ packages/scanner/src/error.rs | 15 + packages/scanner/src/js.rs | 37 +++ packages/scanner/src/lib.rs | 156 ++++++++++ packages/scanner/src/local_track.rs | 13 + 21 files changed, 943 insertions(+), 14 deletions(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 packages/main/src/services/library-scanner/library-scanner.test.ts create mode 100644 packages/scanner/.gitignore create mode 100644 packages/scanner/Cargo.lock create mode 100644 packages/scanner/Cargo.toml create mode 100644 packages/scanner/README.md create mode 100644 packages/scanner/index.d.ts create mode 100644 packages/scanner/package-lock.json create mode 100644 packages/scanner/package.json create mode 100644 packages/scanner/src/error.rs create mode 100644 packages/scanner/src/js.rs create mode 100644 packages/scanner/src/lib.rs create mode 100644 packages/scanner/src/local_track.rs diff --git a/.gitignore b/.gitignore index 271c1c0bb7..c95bd90f3d 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ build-dir #Travis/IDE stuff .travis.yml .vs/ +target/* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..5b8adf3843 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,284 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "bitflags" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "id3" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9389dd9c8c4671b1e4b2878a6329bccb573f9c24a75bc91c641c451ce5436501" +dependencies = [ + "bitflags", + "byteorder", + "flate2", +] + +[[package]] +name = "libc" +version = "0.2.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "libloading" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "neon" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e15415261d880aed48122e917a45e87bb82cf0260bb6db48bbab44b7464373" +dependencies = [ + "neon-build", + "neon-macros", + "neon-runtime", + "semver", + "smallvec", +] + +[[package]] +name = "neon-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bac98a702e71804af3dacfde41edde4a16076a7bbe889ae61e56e18c5b1c811" + +[[package]] +name = "neon-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7288eac8b54af7913c60e0eb0e2a7683020dffa342ab3fd15e28f035ba897cf" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "neon-runtime" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676720fa8bb32c64c3d9f49c47a47289239ec46b4bdb66d0913cc512cb0daca" +dependencies = [ + "cfg-if", + "libloading", + "smallvec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "scanner" +version = "0.1.0" +dependencies = [ + "id3", + "neon", + "uuid", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "uuid" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +dependencies = [ + "getrandom", + "rand", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..aeae537396 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["packages/scanner"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1cd9698d16..c3070177d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4969,6 +4969,10 @@ "resolved": "packages/main", "link": true }, + "node_modules/@nuclear/scanner": { + "resolved": "packages/scanner", + "link": true + }, "node_modules/@nuclear/ui": { "resolved": "packages/ui", "link": true @@ -16815,6 +16819,15 @@ } ] }, + "node_modules/cargo-cp-artifact": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.8.tgz", + "integrity": "sha512-3j4DaoTrsCD1MRkTF2Soacii0Nx7UHCce0EwUf4fHnggwiE4fbmF2AbnfzayR36DF8KGadfh7M/Yfy625kgPlA==", + "dev": true, + "bin": { + "cargo-cp-artifact": "bin/cargo-cp-artifact.js" + } + }, "node_modules/case-sensitive-paths-webpack-plugin": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz", @@ -46373,6 +46386,7 @@ "license": "AGPL-3.0", "dependencies": { "@nuclear/core": "^0.6.30", + "@nuclear/scanner": "^0.6.21", "autobind-decorator": "^2.4.0", "body-parser": "^1.19.0", "concat-stream": "^2.0.0", @@ -46463,6 +46477,14 @@ "uuid": "bin/uuid" } }, + "packages/scanner": { + "version": "0.6.21", + "hasInstallScript": true, + "license": "AGPL-3.0", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } + }, "packages/ui": { "name": "@nuclear/ui", "version": "0.6.30", @@ -50122,7 +50144,7 @@ "react-full-screen": "^1.0.1", "react-hifi": "^2.2.1", "react-hls-player": "^3.0.1", - "react-hot-loader": "^4.12", + "react-hot-loader": "^4.13.1", "react-i18next": "^11.12.0", "react-image": "^2.2.2", "react-list": "^0.8.13", @@ -50211,6 +50233,7 @@ "version": "file:packages/main", "requires": { "@nuclear/core": "^0.6.30", + "@nuclear/scanner": "^0.6.21", "@types/body-parser": "^1.17.1", "@types/concat-stream": "^1.6.0", "@types/cors": "^2.8.6", @@ -50295,6 +50318,12 @@ } } }, + "@nuclear/scanner": { + "version": "file:packages/scanner", + "requires": { + "cargo-cp-artifact": "^0.1" + } + }, "@nuclear/ui": { "version": "file:packages/ui", "requires": { @@ -59609,6 +59638,12 @@ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001547.tgz", "integrity": "sha512-W7CrtIModMAxobGhz8iXmDfuJiiKg1WADMO/9x7/CLNin5cpSbuBjooyoIUVB5eyCc36QuTVlkVa1iB2S5+/eA==" }, + "cargo-cp-artifact": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.8.tgz", + "integrity": "sha512-3j4DaoTrsCD1MRkTF2Soacii0Nx7UHCce0EwUf4fHnggwiE4fbmF2AbnfzayR36DF8KGadfh7M/Yfy625kgPlA==", + "dev": true + }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz", diff --git a/package.json b/package.json index 654976f0c4..21ad36e460 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/nukeop/nuclear#readme", "scripts": { - "postinstall": "electron-builder install-app-deps", + "postinstall": "electron-builder install-app-deps && lerna run build --scope @nuclear/scanner", "start": "lerna run start --stream", "build": "shx rm -rf dist && lerna run build && npm run pack", "test": "lerna run test", diff --git a/packages/main/package.json b/packages/main/package.json index de4c1c78c9..bf52389973 100644 --- a/packages/main/package.json +++ b/packages/main/package.json @@ -21,6 +21,7 @@ "homepage": "https://github.com/nukeop/nuclear#readme", "dependencies": { "@nuclear/core": "^0.6.30", + "@nuclear/scanner": "^0.6.21", "autobind-decorator": "^2.4.0", "body-parser": "^1.19.0", "concat-stream": "^2.0.0", diff --git a/packages/main/src/controllers/local-library.ts b/packages/main/src/controllers/local-library.ts index a4ca27dffa..84b98eb28b 100644 --- a/packages/main/src/controllers/local-library.ts +++ b/packages/main/src/controllers/local-library.ts @@ -1,12 +1,13 @@ import { inject } from 'inversify'; import { IpcMessageEvent } from 'electron'; +import { IpcEvents } from '@nuclear/core'; +import {scanFolders} from '@nuclear/scanner'; import LocalLibrary from '../services/local-library'; import { ipcController, ipcEvent } from '../utils/decorators'; import LocalLibraryDb from '../services/local-library/db'; import Platform from '../services/platform'; import Window from '../services/window'; -import { IpcEvents } from '@nuclear/core'; @ipcController() class LocalIpcCtrl { @@ -48,12 +49,12 @@ class LocalIpcCtrl { */ @ipcEvent(IpcEvents.LOCALFOLDERS_SET) async setLocalFolders(event: IpcMessageEvent, directories: string[]) { - const localFolders = await Promise.all( + await Promise.all( directories .map(folder => this.localLibraryDb.addFolder(this.normalizeFolderPath(folder))) ); - const cache = await this.localLibrary.scanFoldersAndGetMeta(localFolders, (scanProgress, scanTotal) => { + const cache = await scanFolders(directories, ['mp3'], (scanProgress, scanTotal) => { this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); }); @@ -82,12 +83,10 @@ class LocalIpcCtrl { async onRefreshLocalFolders() { try { const folders = await this.localLibraryDb.getLocalFolders(); - const cache = await this.localLibrary.scanFoldersAndGetMeta( - folders, - (scanProgress, scanTotal) => { - this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); - } - ); + + const cache = await scanFolders(folders.map(folder => folder.path), ['mp3'], (scanProgress, scanTotal) => { + this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); + }); this.window.send(IpcEvents.LOCAL_FILES, cache); } catch (err) { diff --git a/packages/main/src/services/library-scanner/library-scanner.test.ts b/packages/main/src/services/library-scanner/library-scanner.test.ts new file mode 100644 index 0000000000..a803a3e732 --- /dev/null +++ b/packages/main/src/services/library-scanner/library-scanner.test.ts @@ -0,0 +1,10 @@ +import {scanFolders} from '@nuclear/scanner'; + +describe('Local library scanner', () => { + it('scans folders', async () => { + const result = await scanFolders([''], ['mp3'], (progress, total, lastScanned) => { + // console.log({progress, total, lastScanned}); + }); + expect(result).toBe({}); + }); +}); diff --git a/packages/main/tsconfig.json b/packages/main/tsconfig.json index fa6acfa1cb..b56c78a0c9 100644 --- a/packages/main/tsconfig.json +++ b/packages/main/tsconfig.json @@ -15,7 +15,8 @@ "src", "typings", "node_modules/@nuclear", - "../core/src" + "../core/src", + "../scanner" ], "exclude": [ "node_modules/@nuclear/*/test", diff --git a/packages/main/webpack.config.ts b/packages/main/webpack.config.ts index 28bb387932..4f3a02cec1 100644 --- a/packages/main/webpack.config.ts +++ b/packages/main/webpack.config.ts @@ -16,6 +16,8 @@ const osMapper: Record = { const MAIN_DIR = path.resolve(__dirname, 'src'); const CORE_DIR = path.resolve(__dirname, '..', '..', 'node_modules', '@nuclear', 'core', 'src'); +const SCANNER_DIR = path.resolve(__dirname, '..', '..', 'node_modules', '@nuclear', 'scanner'); +const SCANNER_DIR_SYMLINKED = path.resolve(__dirname, 'node_modules', '@nuclear', 'scanner'); module.exports = (env: BuildEnv): webpack.Configuration => { if (!env.TARGET) { @@ -28,7 +30,7 @@ module.exports = (env: BuildEnv): webpack.Configuration => { return { entry: './src/main.ts', resolve: { - extensions: ['.ts', '.js', '.json'], + extensions: ['.ts', '.js', '.json', '.node'], alias: { jsbi: path.resolve(__dirname, '..', '..', 'node_modules', 'jsbi', 'dist', 'jsbi-cjs.js') }, @@ -60,7 +62,8 @@ module.exports = (env: BuildEnv): webpack.Configuration => { }, { test: /\.node$/, - use: 'node-loader' + use: 'node-loader', + include: [MAIN_DIR, SCANNER_DIR, SCANNER_DIR_SYMLINKED] } ] }, diff --git a/packages/scanner/.gitignore b/packages/scanner/.gitignore new file mode 100644 index 0000000000..6ca71fb5fc --- /dev/null +++ b/packages/scanner/.gitignore @@ -0,0 +1,5 @@ +target +index.node +**/node_modules +**/.DS_Store +npm-debug.log* diff --git a/packages/scanner/Cargo.lock b/packages/scanner/Cargo.lock new file mode 100644 index 0000000000..ba3ac02eea --- /dev/null +++ b/packages/scanner/Cargo.lock @@ -0,0 +1,156 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "libloading" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "neon" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e15415261d880aed48122e917a45e87bb82cf0260bb6db48bbab44b7464373" +dependencies = [ + "neon-build", + "neon-macros", + "neon-runtime", + "semver", + "smallvec", +] + +[[package]] +name = "neon-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bac98a702e71804af3dacfde41edde4a16076a7bbe889ae61e56e18c5b1c811" + +[[package]] +name = "neon-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7288eac8b54af7913c60e0eb0e2a7683020dffa342ab3fd15e28f035ba897cf" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "neon-runtime" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676720fa8bb32c64c3d9f49c47a47289239ec46b4bdb66d0913cc512cb0daca" +dependencies = [ + "cfg-if", + "libloading", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "scanner" +version = "0.1.0" +dependencies = [ + "neon", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml new file mode 100644 index 0000000000..31c9810799 --- /dev/null +++ b/packages/scanner/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "scanner" +version = "0.1.0" +license = "AGPL-3.0" +edition = "2018" +exclude = ["index.node"] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +id3 = "1.7.0" + +[dependencies.uuid] +version = "1.3.4" +features = [ + "v4", + "fast-rng" +] + +[dependencies.neon] +version = "0.10" +default-features = false +features = ["napi-6"] diff --git a/packages/scanner/README.md b/packages/scanner/README.md new file mode 100644 index 0000000000..d005526e48 --- /dev/null +++ b/packages/scanner/README.md @@ -0,0 +1,119 @@ +# scanner + +This project was bootstrapped by [create-neon](https://www.npmjs.com/package/create-neon). + +## Installing scanner + +Installing scanner requires a [supported version of Node and Rust](https://github.com/neon-bindings/neon#platform-support). + +You can install the project with npm. In the project directory, run: + +```sh +$ npm install +``` + +This fully installs the project, including installing any dependencies and running the build. + +## Building scanner + +If you have already installed the project and only want to run the build, run: + +```sh +$ npm run build +``` + +This command uses the [cargo-cp-artifact](https://github.com/neon-bindings/cargo-cp-artifact) utility to run the Rust build and copy the built library into `./index.node`. + +## Exploring scanner + +After building scanner, you can explore its exports at the Node REPL: + +```sh +$ npm install +$ node +> require('.').hello() +"hello node" +``` + +## Available Scripts + +In the project directory, you can run: + +### `npm install` + +Installs the project, including running `npm run build`. + +### `npm build` + +Builds the Node addon (`index.node`) from source. + +Additional [`cargo build`](https://doc.rust-lang.org/cargo/commands/cargo-build.html) arguments may be passed to `npm build` and `npm build-*` commands. For example, to enable a [cargo feature](https://doc.rust-lang.org/cargo/reference/features.html): + +``` +npm run build -- --feature=beetle +``` + +#### `npm build-debug` + +Alias for `npm build`. + +#### `npm build-release` + +Same as [`npm build`](#npm-build) but, builds the module with the [`release`](https://doc.rust-lang.org/cargo/reference/profiles.html#release) profile. Release builds will compile slower, but run faster. + +### `npm test` + +Runs the unit tests by calling `cargo test`. You can learn more about [adding tests to your Rust code](https://doc.rust-lang.org/book/ch11-01-writing-tests.html) from the [Rust book](https://doc.rust-lang.org/book/). + +## Project Layout + +The directory structure of this project is: + +``` +scanner/ +├── Cargo.toml +├── README.md +├── index.node +├── package.json +├── src/ +| └── lib.rs +└── target/ +``` + +### Cargo.toml + +The Cargo [manifest file](https://doc.rust-lang.org/cargo/reference/manifest.html), which informs the `cargo` command. + +### README.md + +This file. + +### index.node + +The Node addon—i.e., a binary Node module—generated by building the project. This is the main module for this package, as dictated by the `"main"` key in `package.json`. + +Under the hood, a [Node addon](https://nodejs.org/api/addons.html) is a [dynamically-linked shared object](https://en.wikipedia.org/wiki/Library_(computing)#Shared_libraries). The `"build"` script produces this file by copying it from within the `target/` directory, which is where the Rust build produces the shared object. + +### package.json + +The npm [manifest file](https://docs.npmjs.com/cli/v7/configuring-npm/package-json), which informs the `npm` command. + +### src/ + +The directory tree containing the Rust source code for the project. + +### src/lib.rs + +The Rust library's main module. + +### target/ + +Binary artifacts generated by the Rust build. + +## Learn More + +To learn more about Neon, see the [Neon documentation](https://neon-bindings.com). + +To learn more about Rust, see the [Rust documentation](https://www.rust-lang.org). + +To learn more about Node, see the [Node documentation](https://nodejs.org). diff --git a/packages/scanner/index.d.ts b/packages/scanner/index.d.ts new file mode 100644 index 0000000000..c435528467 --- /dev/null +++ b/packages/scanner/index.d.ts @@ -0,0 +1,16 @@ +type LocalTrack = { + uuid: string; + artist?: string; + title?: string; + album?: string; + duration?: number; + position?: number; + year?: string; + + filename: string; + path: string; + local: true; +} + +declare const scanFolders = (folders: string[], supportedFormats: string[], onProgress: (progress: number, total: number, lastScanned?: string) => void) => Promise; +export { scanFolders }; diff --git a/packages/scanner/package-lock.json b/packages/scanner/package-lock.json new file mode 100644 index 0000000000..772be0c1e9 --- /dev/null +++ b/packages/scanner/package-lock.json @@ -0,0 +1,25 @@ +{ + "name": "@nuclear/scanner", + "version": "0.6.21", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@nuclear/scanner", + "version": "0.6.21", + "license": "AGPL-3.0", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } + }, + "node_modules/cargo-cp-artifact": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.8.tgz", + "integrity": "sha512-3j4DaoTrsCD1MRkTF2Soacii0Nx7UHCce0EwUf4fHnggwiE4fbmF2AbnfzayR36DF8KGadfh7M/Yfy625kgPlA==", + "dev": true, + "bin": { + "cargo-cp-artifact": "bin/cargo-cp-artifact.js" + } + } + } +} diff --git a/packages/scanner/package.json b/packages/scanner/package.json new file mode 100644 index 0000000000..79e3939034 --- /dev/null +++ b/packages/scanner/package.json @@ -0,0 +1,27 @@ +{ + "name": "@nuclear/scanner", + "version": "0.6.21", + "description": "Local library scanner", + "main": "index.node", + "types": "index.d.ts", + "scripts": { + "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build-debug": "npm run build --", + "build-release": "npm run build -- --release", + "install": "npm run build-release", + "test": "cargo test" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/nukeop/nuclear.git" + }, + "author": "nukeop ", + "license": "AGPL-3.0", + "bugs": { + "url": "https://github.com/nukeop/nuclear/issues" + }, + "homepage": "https://github.com/nukeop/nuclear#readme", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } +} \ No newline at end of file diff --git a/packages/scanner/src/error.rs b/packages/scanner/src/error.rs new file mode 100644 index 0000000000..30c9d6a73a --- /dev/null +++ b/packages/scanner/src/error.rs @@ -0,0 +1,15 @@ +use std::error::Error; +use std::fmt; + +#[derive(Debug)] +pub struct ScannerError { + pub message: String, +} + +impl Error for ScannerError {} + +impl fmt::Display for ScannerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ScannerError: {}", self.message) + } +} diff --git a/packages/scanner/src/js.rs b/packages/scanner/src/js.rs new file mode 100644 index 0000000000..2f4f70d256 --- /dev/null +++ b/packages/scanner/src/js.rs @@ -0,0 +1,37 @@ +use neon::prelude::*; + +pub fn set_optional_field_str( + cx: &mut FunctionContext, + obj: &mut Handle, + field_name: &str, + value: Option, +) { + match value { + Some(v) => { + let field_value = cx.string(&v); + obj.set(cx, field_name, field_value).unwrap(); + } + None => { + let undefined = cx.undefined(); + obj.set(cx, field_name, undefined).unwrap(); + } + } +} + +pub fn set_optional_field_u32( + cx: &mut FunctionContext, + obj: &mut Handle, + field_name: &str, + value: Option, +) { + match value { + Some(v) => { + let field_value = cx.number(v as f64); + obj.set(cx, field_name, field_value).unwrap(); + } + None => { + let undefined = cx.undefined(); + obj.set(cx, field_name, undefined).unwrap(); + } + } +} diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs new file mode 100644 index 0000000000..fe9376dcdd --- /dev/null +++ b/packages/scanner/src/lib.rs @@ -0,0 +1,156 @@ +mod error; +mod js; +mod local_track; +use error::ScannerError; +use id3::{Tag, TagLike}; +use js::{set_optional_field_str, set_optional_field_u32}; +use neon::prelude::*; +use std::collections::LinkedList; +use uuid::Uuid; + +use local_track::LocalTrack; + +fn visitFile(path: String) -> Result { + let tag = Tag::read_from_path(&path); + + match tag { + Ok(tag) => Ok(LocalTrack { + uuid: Uuid::new_v4().to_string(), + artist: tag.artist().map(|s| s.to_string()), + title: tag.title().map(|s| s.to_string()), + album: tag.album().map(|s| s.to_string()), + duration: tag.duration().unwrap_or(0), + position: tag.track(), + year: tag.year().map(|s| s as u32), + filename: path.split("/").last().map(|s| s.to_string()).unwrap(), + path: path.clone(), + }), + Err(e) => Err(ScannerError { + message: format!("Error reading file: {}", e), + }), + } +} + +fn visitDirectory( + path: String, + supportedFormats: Vec, + dirsToScanQueue: &mut LinkedList, + filesToScanQueue: &mut LinkedList, +) { + // Read the contents of the directory + let dir = std::fs::read_dir(path.clone()).unwrap(); + for entry in dir { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + // Add the directory to the queue + dirsToScanQueue.push_back(path.to_str().unwrap().to_string()); + } else if let Some(extension) = path.extension().and_then(|ext| ext.to_str()) { + // Add the file to the queue, if it's a supported format + if supportedFormats.contains(&extension.to_string()) { + filesToScanQueue.push_back(path.to_str().unwrap().to_string()); + } + } + } +} + +fn scanFolders(mut cx: FunctionContext) -> JsResult { + let folders: Handle = cx.argument(0)?; + let supported_formats: Handle = cx.argument(1)?; + let onProgressCallback: Handle = cx.argument(2)?; + let result: Handle = cx.empty_array(); + + // Copy all the starting folders to a queue, which holds all the folders left to scan + let supported_formats_vec = supported_formats + .to_vec(&mut cx)? + .into_iter() + .map(|format| format.to_string(&mut cx).unwrap().value(&mut cx)) + .collect::>(); + let folders_vec = folders.to_vec(&mut cx)?; + let mut dirs_to_scan_queue: LinkedList = LinkedList::new(); + let mut files_to_scan_queue: LinkedList = LinkedList::new(); + let mut total_files_to_scan_num; + for folder in folders_vec { + let folder_string = folder.to_string(&mut cx)?.value(&mut cx); + dirs_to_scan_queue.push_back(folder_string); + } + + // While there are still folders left to scan + while !dirs_to_scan_queue.is_empty() { + // Get the next folder to scan + let folder = dirs_to_scan_queue.pop_front().unwrap(); + + // Scan the folder + visitDirectory( + folder.clone(), + supported_formats_vec.clone(), + &mut dirs_to_scan_queue, + &mut files_to_scan_queue, + ); + + // Call the progress callback + let this = cx.undefined(); + let args = vec![ + cx.number(0).upcast(), + cx.number(files_to_scan_queue.len() as f64).upcast(), + cx.string(folder.clone()).upcast(), + ]; + onProgressCallback.call(&mut cx, this, args)?; + } + + // All folders have been scanned, now scan the files + total_files_to_scan_num = files_to_scan_queue.len(); + while !files_to_scan_queue.is_empty() { + // Get the next file to scan + let file = files_to_scan_queue.pop_front().unwrap(); + + // Scan the file + let track = visitFile(file.clone()).unwrap(); + let len = result.len(&mut cx); + let mut track_js_object = JsObject::new(&mut cx); + let track_uuid_js_string = cx.string(track.uuid); + track_js_object.set(&mut cx, "uuid", track_uuid_js_string)?; + + set_optional_field_str(&mut cx, &mut track_js_object, "artist", track.artist); + + set_optional_field_str(&mut cx, &mut track_js_object, "title", track.title); + + set_optional_field_str(&mut cx, &mut track_js_object, "album", track.album); + + let track_duration_js_number = cx.number(track.duration); + track_js_object.set(&mut cx, "duration", track_duration_js_number)?; + + set_optional_field_u32(&mut cx, &mut track_js_object, "position", track.position); + + set_optional_field_u32(&mut cx, &mut track_js_object, "year", track.year); + + let track_filename_js_string = cx.string(track.filename); + track_js_object.set(&mut cx, "filename", track_filename_js_string)?; + + let track_path_js_string = cx.string(track.path); + track_js_object.set(&mut cx, "path", track_path_js_string)?; + + let track_local = cx.boolean(true); + track_js_object.set(&mut cx, "local", track_local)?; + + result.set(&mut cx, len, track_js_object)?; + + // Call the progress callback + let this = cx.undefined(); + let args = vec![ + cx.number((total_files_to_scan_num - files_to_scan_queue.len()) as f64) + .upcast(), + cx.number(total_files_to_scan_num as f64).upcast(), + cx.string(file.clone()).upcast(), + ]; + onProgressCallback.call(&mut cx, this, args)?; + } + + Ok(result) +} + +#[neon::main] +fn main(mut cx: ModuleContext) -> NeonResult<()> { + cx.export_function("scanFolders", scanFolders)?; + Ok(()) +} diff --git a/packages/scanner/src/local_track.rs b/packages/scanner/src/local_track.rs new file mode 100644 index 0000000000..5d8aebd932 --- /dev/null +++ b/packages/scanner/src/local_track.rs @@ -0,0 +1,13 @@ +#[derive(Debug, Clone)] +pub struct LocalTrack { + pub uuid: String, + pub artist: Option, + pub title: Option, + pub album: Option, + pub duration: u32, + pub position: Option, + pub year: Option, + + pub filename: String, + pub path: String, +} From 4c88e09ba349c6479c8eaae09b39b84e518fbabe Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 30 Oct 2023 00:58:41 +0100 Subject: [PATCH 02/27] Refactor visit_file function with dependency inversion to allow injecting mocks --- packages/scanner/Cargo.toml | 1 + packages/scanner/src/lib.rs | 116 +++++++++++++++++++++++++++----- packages/scanner/src/scanner.rs | 6 ++ 3 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 packages/scanner/src/scanner.rs diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 31c9810799..6d5daed87c 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["cdylib"] [dependencies] id3 = "1.7.0" +mockall = "0.11.4" [dependencies.uuid] version = "1.3.4" diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index fe9376dcdd..0608771c47 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -1,6 +1,7 @@ mod error; mod js; mod local_track; +mod scanner; use error::ScannerError; use id3::{Tag, TagLike}; use js::{set_optional_field_str, set_optional_field_u32}; @@ -10,8 +11,11 @@ use uuid::Uuid; use local_track::LocalTrack; -fn visitFile(path: String) -> Result { - let tag = Tag::read_from_path(&path); +fn visit_file(path: String, tag_reader: F) -> Result +where + F: FnOnce(&str) -> Result, +{ + let tag = tag_reader(&path); match tag { Ok(tag) => Ok(LocalTrack { @@ -31,11 +35,11 @@ fn visitFile(path: String) -> Result { } } -fn visitDirectory( +fn visit_directory( path: String, - supportedFormats: Vec, - dirsToScanQueue: &mut LinkedList, - filesToScanQueue: &mut LinkedList, + supported_formats: Vec, + dirs_to_scan_queue: &mut LinkedList, + files_to_scan_queue: &mut LinkedList, ) { // Read the contents of the directory let dir = std::fs::read_dir(path.clone()).unwrap(); @@ -44,20 +48,20 @@ fn visitDirectory( let path = entry.path(); if path.is_dir() { // Add the directory to the queue - dirsToScanQueue.push_back(path.to_str().unwrap().to_string()); + dirs_to_scan_queue.push_back(path.to_str().unwrap().to_string()); } else if let Some(extension) = path.extension().and_then(|ext| ext.to_str()) { // Add the file to the queue, if it's a supported format - if supportedFormats.contains(&extension.to_string()) { - filesToScanQueue.push_back(path.to_str().unwrap().to_string()); + if supported_formats.contains(&extension.to_string()) { + files_to_scan_queue.push_back(path.to_str().unwrap().to_string()); } } } } -fn scanFolders(mut cx: FunctionContext) -> JsResult { +fn scan_folders(mut cx: FunctionContext) -> JsResult { let folders: Handle = cx.argument(0)?; let supported_formats: Handle = cx.argument(1)?; - let onProgressCallback: Handle = cx.argument(2)?; + let on_progress_callback: Handle = cx.argument(2)?; let result: Handle = cx.empty_array(); // Copy all the starting folders to a queue, which holds all the folders left to scan @@ -69,7 +73,7 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { let folders_vec = folders.to_vec(&mut cx)?; let mut dirs_to_scan_queue: LinkedList = LinkedList::new(); let mut files_to_scan_queue: LinkedList = LinkedList::new(); - let mut total_files_to_scan_num; + let total_files_to_scan_num; for folder in folders_vec { let folder_string = folder.to_string(&mut cx)?.value(&mut cx); dirs_to_scan_queue.push_back(folder_string); @@ -81,7 +85,7 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { let folder = dirs_to_scan_queue.pop_front().unwrap(); // Scan the folder - visitDirectory( + visit_directory( folder.clone(), supported_formats_vec.clone(), &mut dirs_to_scan_queue, @@ -95,7 +99,7 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { cx.number(files_to_scan_queue.len() as f64).upcast(), cx.string(folder.clone()).upcast(), ]; - onProgressCallback.call(&mut cx, this, args)?; + on_progress_callback.call(&mut cx, this, args)?; } // All folders have been scanned, now scan the files @@ -105,7 +109,24 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { let file = files_to_scan_queue.pop_front().unwrap(); // Scan the file - let track = visitFile(file.clone()).unwrap(); + let track = visit_file(file.clone(), |path| Tag::read_from_path(path)); + + if track.is_err() { + // Call the progress callback + let this = cx.undefined(); + let args = vec![ + cx.number((total_files_to_scan_num - files_to_scan_queue.len()) as f64) + .upcast(), + cx.number(total_files_to_scan_num as f64).upcast(), + cx.string(file.clone()).upcast(), + ]; + on_progress_callback.call(&mut cx, this, args)?; + + continue; + } + + let track = track.unwrap(); + let len = result.len(&mut cx); let mut track_js_object = JsObject::new(&mut cx); let track_uuid_js_string = cx.string(track.uuid); @@ -143,7 +164,7 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { cx.number(total_files_to_scan_num as f64).upcast(), cx.string(file.clone()).upcast(), ]; - onProgressCallback.call(&mut cx, this, args)?; + on_progress_callback.call(&mut cx, this, args)?; } Ok(result) @@ -151,6 +172,67 @@ fn scanFolders(mut cx: FunctionContext) -> JsResult { #[neon::main] fn main(mut cx: ModuleContext) -> NeonResult<()> { - cx.export_function("scanFolders", scanFolders)?; + cx.export_function("scanFolders", scan_folders)?; Ok(()) } + +#[cfg(test)] +mod tests { + use std::io; + + use super::*; + + #[test] + fn test_sanity_check() { + assert_eq!(1, 1); + } + + #[test] + fn test_visit_file_with_valid_file() { + // With mocked tag + let path = String::from("path/to/valid/file.mp3"); + let result = visit_file(path.clone(), |inner_path| { + let mut tag = Tag::new(); + tag.set_artist("Artist"); + tag.set_title("Title"); + tag.set_album("Album"); + tag.set_duration(123); + tag.set_track(1); + tag.set_year(2020); + Ok(tag) + }); + + if let Some(track) = result.ok() { + //check uuid format + assert_eq!( + track.uuid, + Uuid::parse_str(&track.uuid).unwrap().to_string() + ); + assert_eq!(track.artist, Some(String::from("Artist"))); + assert_eq!(track.title, Some(String::from("Title"))); + assert_eq!(track.album, Some(String::from("Album"))); + assert_eq!(track.duration, 123); + assert_eq!(track.position, Some(1)); + assert_eq!(track.year, Some(2020)); + assert_eq!(track.filename, String::from("file.mp3")); + assert_eq!(track.path, path); + } else { + panic!("Result is not ok"); + } + } + + #[test] + fn test_visit_file_with_no_tags() { + // With mocked tag + let path = String::from("path/to/invalid/file.mp3"); + let result = visit_file(path.clone(), |inner_path| { + Err(id3::Error::new(id3::ErrorKind::NoTag, "")) + }); + + if let Some(error) = result.err() { + assert_eq!(error.message, String::from("Error reading file: NoTag")); + } else { + panic!("Result is not err"); + } + } +} diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs new file mode 100644 index 0000000000..880bd2b0b4 --- /dev/null +++ b/packages/scanner/src/scanner.rs @@ -0,0 +1,6 @@ +use id3::{Error, Tag}; +use std::path::Path; + +pub trait TagReader { + fn read_from_path(path: impl AsRef) -> Result; +} From ccb7c5d2e9f00a84f38637feb2cf0de82d2db39a Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 30 Oct 2023 01:09:01 +0100 Subject: [PATCH 03/27] Cleanup code, fix warnings --- packages/scanner/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 0608771c47..5dfd9ec02f 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -178,8 +178,6 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> { #[cfg(test)] mod tests { - use std::io; - use super::*; #[test] @@ -191,7 +189,7 @@ mod tests { fn test_visit_file_with_valid_file() { // With mocked tag let path = String::from("path/to/valid/file.mp3"); - let result = visit_file(path.clone(), |inner_path| { + let result = visit_file(path.clone(), |_inner_path| { let mut tag = Tag::new(); tag.set_artist("Artist"); tag.set_title("Title"); @@ -225,7 +223,7 @@ mod tests { fn test_visit_file_with_no_tags() { // With mocked tag let path = String::from("path/to/invalid/file.mp3"); - let result = visit_file(path.clone(), |inner_path| { + let result = visit_file(path.clone(), |_inner_path| { Err(id3::Error::new(id3::ErrorKind::NoTag, "")) }); From 2ac419185bcc972848cf56b1a0e836b6ce6426ed Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Wed, 1 Nov 2023 01:07:20 +0100 Subject: [PATCH 04/27] Return thumbnails and disc number when scanning --- packages/app/webpack.config.ts | 1 - packages/main/tsconfig.json | 2 +- packages/scanner/index.d.ts | 11 ++- packages/scanner/src/js.rs | 18 ++++ packages/scanner/src/lib.rs | 121 ++------------------------ packages/scanner/src/local_track.rs | 2 + packages/scanner/src/scanner.rs | 129 +++++++++++++++++++++++++++- packages/scanner/tsconfig.json | 25 ++++++ 8 files changed, 188 insertions(+), 121 deletions(-) create mode 100644 packages/scanner/tsconfig.json diff --git a/packages/app/webpack.config.ts b/packages/app/webpack.config.ts index 3ecdd29fde..b2ca44b1d6 100644 --- a/packages/app/webpack.config.ts +++ b/packages/app/webpack.config.ts @@ -241,7 +241,6 @@ module.exports = (env) => { }; if (IS_DEV) { - config.plugins?.push(new webpack.HotModuleReplacementPlugin()); config.devServer = { hot: true, static: { diff --git a/packages/main/tsconfig.json b/packages/main/tsconfig.json index b56c78a0c9..b5896f30e8 100644 --- a/packages/main/tsconfig.json +++ b/packages/main/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../tsconfig.json", "compilerOptions": { "module": "commonjs", - "lib": ["DOM"], + "lib": ["DOM", "ESNext"], "outDir": "dist", "allowJs": true, "strict": false, diff --git a/packages/scanner/index.d.ts b/packages/scanner/index.d.ts index c435528467..135c9471ea 100644 --- a/packages/scanner/index.d.ts +++ b/packages/scanner/index.d.ts @@ -1,9 +1,10 @@ -type LocalTrack = { +export type LocalTrack = { uuid: string; - artist?: string; + artist: string; title?: string; album?: string; duration?: number; + thumbnail?: Buffer; position?: number; year?: string; @@ -12,5 +13,9 @@ type LocalTrack = { local: true; } -declare const scanFolders = (folders: string[], supportedFormats: string[], onProgress: (progress: number, total: number, lastScanned?: string) => void) => Promise; +declare const scanFolders = ( + folders: string[], + supportedFormats: string[], + onProgress: (progress: number, total: number, lastScanned?: string) => void +) => new Promise; export { scanFolders }; diff --git a/packages/scanner/src/js.rs b/packages/scanner/src/js.rs index 2f4f70d256..e929cbdaf4 100644 --- a/packages/scanner/src/js.rs +++ b/packages/scanner/src/js.rs @@ -35,3 +35,21 @@ pub fn set_optional_field_u32( } } } + +pub fn set_optional_field_buffer( + cx: &mut FunctionContext, + obj: &mut Handle, + field_name: &str, + value: Option>, +) { + match value { + Some(v) => { + let field_value = JsBuffer::external(cx, v); + obj.set(cx, field_name, field_value).unwrap(); + } + None => { + let undefined = cx.undefined(); + obj.set(cx, field_name, undefined).unwrap(); + } + } +} diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 5dfd9ec02f..406bec14a2 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -2,61 +2,11 @@ mod error; mod js; mod local_track; mod scanner; -use error::ScannerError; -use id3::{Tag, TagLike}; -use js::{set_optional_field_str, set_optional_field_u32}; +use id3::Tag; +use js::{set_optional_field_buffer, set_optional_field_str, set_optional_field_u32}; use neon::prelude::*; +use scanner::{visit_directory, visit_file}; use std::collections::LinkedList; -use uuid::Uuid; - -use local_track::LocalTrack; - -fn visit_file(path: String, tag_reader: F) -> Result -where - F: FnOnce(&str) -> Result, -{ - let tag = tag_reader(&path); - - match tag { - Ok(tag) => Ok(LocalTrack { - uuid: Uuid::new_v4().to_string(), - artist: tag.artist().map(|s| s.to_string()), - title: tag.title().map(|s| s.to_string()), - album: tag.album().map(|s| s.to_string()), - duration: tag.duration().unwrap_or(0), - position: tag.track(), - year: tag.year().map(|s| s as u32), - filename: path.split("/").last().map(|s| s.to_string()).unwrap(), - path: path.clone(), - }), - Err(e) => Err(ScannerError { - message: format!("Error reading file: {}", e), - }), - } -} - -fn visit_directory( - path: String, - supported_formats: Vec, - dirs_to_scan_queue: &mut LinkedList, - files_to_scan_queue: &mut LinkedList, -) { - // Read the contents of the directory - let dir = std::fs::read_dir(path.clone()).unwrap(); - for entry in dir { - let entry = entry.unwrap(); - let path = entry.path(); - if path.is_dir() { - // Add the directory to the queue - dirs_to_scan_queue.push_back(path.to_str().unwrap().to_string()); - } else if let Some(extension) = path.extension().and_then(|ext| ext.to_str()) { - // Add the file to the queue, if it's a supported format - if supported_formats.contains(&extension.to_string()) { - files_to_scan_queue.push_back(path.to_str().unwrap().to_string()); - } - } - } -} fn scan_folders(mut cx: FunctionContext) -> JsResult { let folders: Handle = cx.argument(0)?; @@ -133,16 +83,16 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { track_js_object.set(&mut cx, "uuid", track_uuid_js_string)?; set_optional_field_str(&mut cx, &mut track_js_object, "artist", track.artist); - set_optional_field_str(&mut cx, &mut track_js_object, "title", track.title); - set_optional_field_str(&mut cx, &mut track_js_object, "album", track.album); let track_duration_js_number = cx.number(track.duration); track_js_object.set(&mut cx, "duration", track_duration_js_number)?; - set_optional_field_u32(&mut cx, &mut track_js_object, "position", track.position); + set_optional_field_buffer(&mut cx, &mut track_js_object, "thumbnail", track.thumbnail); + set_optional_field_u32(&mut cx, &mut track_js_object, "position", track.position); + set_optional_field_u32(&mut cx, &mut track_js_object, "disc", track.disc); set_optional_field_u32(&mut cx, &mut track_js_object, "year", track.year); let track_filename_js_string = cx.string(track.filename); @@ -175,62 +125,3 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> { cx.export_function("scanFolders", scan_folders)?; Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_sanity_check() { - assert_eq!(1, 1); - } - - #[test] - fn test_visit_file_with_valid_file() { - // With mocked tag - let path = String::from("path/to/valid/file.mp3"); - let result = visit_file(path.clone(), |_inner_path| { - let mut tag = Tag::new(); - tag.set_artist("Artist"); - tag.set_title("Title"); - tag.set_album("Album"); - tag.set_duration(123); - tag.set_track(1); - tag.set_year(2020); - Ok(tag) - }); - - if let Some(track) = result.ok() { - //check uuid format - assert_eq!( - track.uuid, - Uuid::parse_str(&track.uuid).unwrap().to_string() - ); - assert_eq!(track.artist, Some(String::from("Artist"))); - assert_eq!(track.title, Some(String::from("Title"))); - assert_eq!(track.album, Some(String::from("Album"))); - assert_eq!(track.duration, 123); - assert_eq!(track.position, Some(1)); - assert_eq!(track.year, Some(2020)); - assert_eq!(track.filename, String::from("file.mp3")); - assert_eq!(track.path, path); - } else { - panic!("Result is not ok"); - } - } - - #[test] - fn test_visit_file_with_no_tags() { - // With mocked tag - let path = String::from("path/to/invalid/file.mp3"); - let result = visit_file(path.clone(), |_inner_path| { - Err(id3::Error::new(id3::ErrorKind::NoTag, "")) - }); - - if let Some(error) = result.err() { - assert_eq!(error.message, String::from("Error reading file: NoTag")); - } else { - panic!("Result is not err"); - } - } -} diff --git a/packages/scanner/src/local_track.rs b/packages/scanner/src/local_track.rs index 5d8aebd932..12da6cf956 100644 --- a/packages/scanner/src/local_track.rs +++ b/packages/scanner/src/local_track.rs @@ -5,6 +5,8 @@ pub struct LocalTrack { pub title: Option, pub album: Option, pub duration: u32, + pub thumbnail: Option>, + pub disc: Option, pub position: Option, pub year: Option, diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 880bd2b0b4..11b77909c7 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -1,6 +1,133 @@ -use id3::{Error, Tag}; +use id3::{Error, Tag, TagLike}; +use std::collections::LinkedList; use std::path::Path; +use uuid::Uuid; + +use crate::error::ScannerError; +use crate::local_track::LocalTrack; pub trait TagReader { fn read_from_path(path: impl AsRef) -> Result; } + +pub fn visit_file(path: String, tag_reader: F) -> Result +where + F: FnOnce(&str) -> Result, +{ + let tag = tag_reader(&path); + + match tag { + Ok(tag) => Ok(LocalTrack { + uuid: Uuid::new_v4().to_string(), + artist: tag.artist().map(|s| s.to_string()), + title: tag.title().map(|s| s.to_string()), + album: tag.album().map(|s| s.to_string()), + duration: tag.duration().unwrap_or(0), + thumbnail: tag + .pictures() + .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) + .map(|p| p.data.clone()), + position: tag.track(), + disc: tag.disc(), + year: tag.year().map(|s| s as u32), + filename: path.split("/").last().map(|s| s.to_string()).unwrap(), + path: path.clone(), + }), + Err(e) => Err(ScannerError { + message: format!("Error reading file: {}", e), + }), + } +} + +pub fn visit_directory( + path: String, + supported_formats: Vec, + dirs_to_scan_queue: &mut LinkedList, + files_to_scan_queue: &mut LinkedList, +) { + // Read the contents of the directory + let dir = std::fs::read_dir(path.clone()).unwrap(); + for entry in dir { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + // Add the directory to the queue + dirs_to_scan_queue.push_back(path.to_str().unwrap().to_string()); + } else if let Some(extension) = path.extension().and_then(|ext| ext.to_str()) { + // Add the file to the queue, if it's a supported format + if supported_formats.contains(&extension.to_string()) { + files_to_scan_queue.push_back(path.to_str().unwrap().to_string()); + } + } + } +} + +#[cfg(test)] +mod tests { + use id3::{ + frame::{Picture, PictureType}, + Content, Frame, + }; + + use super::*; + + #[test] + fn test_visit_file_with_valid_file() { + // With mocked tag + let path = String::from("path/to/valid/file.mp3"); + let result = visit_file(path.clone(), |_inner_path| { + let mut tag = Tag::new(); + tag.set_artist("Artist"); + tag.set_title("Title"); + tag.set_album("Album"); + tag.set_duration(123); + tag.set_track(1); + tag.set_year(2020); + let picture = Picture { + mime_type: String::new(), + picture_type: PictureType::CoverFront, + description: String::new(), + data: vec![1, 2, 3], + }; + tag.add_frame(Frame::with_content( + "APIC", + Content::Picture(picture.clone()), + )); + Ok(tag) + }); + + if let Some(track) = result.ok() { + //check uuid format + assert_eq!( + track.uuid, + Uuid::parse_str(&track.uuid).unwrap().to_string() + ); + assert_eq!(track.artist, Some(String::from("Artist"))); + assert_eq!(track.title, Some(String::from("Title"))); + assert_eq!(track.album, Some(String::from("Album"))); + assert_eq!(track.duration, 123); + assert_eq!(track.position, Some(1)); + assert_eq!(track.year, Some(2020)); + assert_eq!(track.filename, String::from("file.mp3")); + assert_eq!(track.path, path); + assert_eq!(track.thumbnail, Some(vec![1, 2, 3])); + } else { + panic!("Result is not ok"); + } + } + + #[test] + fn test_visit_file_with_no_tags() { + // With mocked tag + let path = String::from("path/to/invalid/file.mp3"); + let result = visit_file(path.clone(), |_inner_path| { + Err(id3::Error::new(id3::ErrorKind::NoTag, "")) + }); + + if let Some(error) = result.err() { + assert_eq!(error.message, String::from("Error reading file: NoTag")); + } else { + panic!("Result is not err"); + } + } +} diff --git a/packages/scanner/tsconfig.json b/packages/scanner/tsconfig.json new file mode 100644 index 0000000000..4b2d100e9f --- /dev/null +++ b/packages/scanner/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "lib": ["ESNext"], + "outDir": "dist", + "allowJs": true, + "strict": false, + "typeRoots": [ + "typings", + "../../node_modules/@types" + ] + }, + "include": [ + "./index.d.ts" + ], + "exclude": [ + "node_modules/@nuclear/*/test", + "node_modules/@nuclear/**/*.test*", + "node_modules/@nuclear/ui/stories", + "../ui/stories", + "../*/test" + ] + } + \ No newline at end of file From db7f94f8431677c65166c74e78140b2489f43941 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Sat, 4 Nov 2023 00:18:59 +0100 Subject: [PATCH 05/27] Generate thumbnails in rust --- Cargo.lock | 512 +++++++++++++++++- .../main/src/controllers/local-library.ts | 4 +- .../main/src/services/local-library/index.ts | 4 + packages/scanner/Cargo.toml | 5 + packages/scanner/index.d.ts | 8 +- packages/scanner/src/lib.rs | 19 +- packages/scanner/src/local_track.rs | 2 +- packages/scanner/src/scanner.rs | 17 +- packages/scanner/src/thumbnails.rs | 59 ++ 9 files changed, 614 insertions(+), 16 deletions(-) create mode 100644 packages/scanner/src/thumbnails.rs diff --git a/Cargo.lock b/Cargo.lock index 5b8adf3843..0bee7dda5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,24 +8,73 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "jobserver", + "libc", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "crc32fast" version = "1.3.2" @@ -35,6 +84,88 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "exr" +version = "1.71.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832a761f35ab3e6664babfbdc6cef35a4860e816ec3916dcfd0882954e98a8a8" +dependencies = [ + "bit_field", + "flume", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fdeflate" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d6dafc854908ff5da46ff3f8f473c6984119a2876a383a860246dd7841a868" +dependencies = [ + "simd-adler32", +] + [[package]] name = "flate2" version = "1.0.26" @@ -45,6 +176,30 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "spin", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "getrandom" version = "0.2.10" @@ -56,17 +211,101 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "half" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +dependencies = [ + "crunchy", +] + [[package]] name = "id3" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9389dd9c8c4671b1e4b2878a6329bccb573f9c24a75bc91c641c451ce5436501" dependencies = [ - "bitflags", + "bitflags 2.3.2", "byteorder", "flate2", ] +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", + "qoi", + "tiff", + "webp", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "jobserver" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +dependencies = [ + "libc", +] + +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +dependencies = [ + "rayon", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.146" @@ -83,6 +322,47 @@ dependencies = [ "winapi", ] +[[package]] +name = "libwebp-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0df0a0f9444d52aee6335cd724d21a2ee3285f646291799a72be518ec8ee3c" +dependencies = [ + "cc", + "glob", +] + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -90,6 +370,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", +] + +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -133,12 +441,91 @@ dependencies = [ "smallvec", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "png" +version = "0.17.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro2" version = "1.0.60" @@ -148,6 +535,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quote" version = "1.0.28" @@ -187,15 +583,73 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "scanner" version = "0.1.0" dependencies = [ "id3", + "image", + "md5", + "mockall", "neon", "uuid", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "0.9.0" @@ -211,12 +665,27 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "syn" version = "1.0.109" @@ -239,6 +708,23 @@ dependencies = [ "syn", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "tiff" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d172b0f4d3fba17ba89811858b9d3d97f928aece846475bbda076ca46736211" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "unicode-ident" version = "1.0.9" @@ -261,6 +747,21 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "webp" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb5d8e7814e92297b0e1c773ce43d290bef6c17452dafd9fc49e5edb5beba71" +dependencies = [ + "libwebp-sys", +] + +[[package]] +name = "weezl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + [[package]] name = "winapi" version = "0.3.9" @@ -282,3 +783,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] diff --git a/packages/main/src/controllers/local-library.ts b/packages/main/src/controllers/local-library.ts index 84b98eb28b..97a3c90f21 100644 --- a/packages/main/src/controllers/local-library.ts +++ b/packages/main/src/controllers/local-library.ts @@ -54,7 +54,7 @@ class LocalIpcCtrl { .map(folder => this.localLibraryDb.addFolder(this.normalizeFolderPath(folder))) ); - const cache = await scanFolders(directories, ['mp3'], (scanProgress, scanTotal) => { + const cache = await scanFolders(directories, ['mp3'], this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); }); @@ -84,7 +84,7 @@ class LocalIpcCtrl { try { const folders = await this.localLibraryDb.getLocalFolders(); - const cache = await scanFolders(folders.map(folder => folder.path), ['mp3'], (scanProgress, scanTotal) => { + const cache = await scanFolders(folders.map(folder => folder.path), ['mp3'], this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); }); diff --git a/packages/main/src/services/local-library/index.ts b/packages/main/src/services/local-library/index.ts index 09ad0d5729..e875e6c05f 100644 --- a/packages/main/src/services/local-library/index.ts +++ b/packages/main/src/services/local-library/index.ts @@ -53,6 +53,10 @@ class LocalLibrary { } } + getThumbnailsDir() { + return this.mediaDir; + } + /** * Format metadata from files to nuclear format */ diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 6d5daed87c..b622f93c15 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -10,8 +10,13 @@ crate-type = ["cdylib"] [dependencies] id3 = "1.7.0" +md5 = "0.7.0" mockall = "0.11.4" +[dependencies.image] +version = "0.24.7" +features = ["webp-encoder"] + [dependencies.uuid] version = "1.3.4" features = [ diff --git a/packages/scanner/index.d.ts b/packages/scanner/index.d.ts index 135c9471ea..96b0ac9158 100644 --- a/packages/scanner/index.d.ts +++ b/packages/scanner/index.d.ts @@ -4,7 +4,7 @@ export type LocalTrack = { title?: string; album?: string; duration?: number; - thumbnail?: Buffer; + thumbnail?: string; position?: number; year?: string; @@ -16,6 +16,10 @@ export type LocalTrack = { declare const scanFolders = ( folders: string[], supportedFormats: string[], + thumbnailsDir: string, onProgress: (progress: number, total: number, lastScanned?: string) => void ) => new Promise; -export { scanFolders }; + +declare const generateThumbnail = (filename: string, thumbnailsDir: string) => new Promise; + +export { scanFolders, generateThumbnail }; diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 406bec14a2..f21afdf4a8 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -2,16 +2,20 @@ mod error; mod js; mod local_track; mod scanner; +mod thumbnails; use id3::Tag; -use js::{set_optional_field_buffer, set_optional_field_str, set_optional_field_u32}; +use js::{set_optional_field_str, set_optional_field_u32}; use neon::prelude::*; use scanner::{visit_directory, visit_file}; use std::collections::LinkedList; +use thumbnails::create_thumbnails_dir; fn scan_folders(mut cx: FunctionContext) -> JsResult { let folders: Handle = cx.argument(0)?; let supported_formats: Handle = cx.argument(1)?; - let on_progress_callback: Handle = cx.argument(2)?; + let thumbnails_dir: Handle = cx.argument(2)?; + let thumbnails_dir_str = thumbnails_dir.value(&mut cx); + let on_progress_callback: Handle = cx.argument(3)?; let result: Handle = cx.empty_array(); // Copy all the starting folders to a queue, which holds all the folders left to scan @@ -52,6 +56,9 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { on_progress_callback.call(&mut cx, this, args)?; } + // First, create a directory for thumbnails + create_thumbnails_dir(thumbnails_dir_str.as_str()); + // All folders have been scanned, now scan the files total_files_to_scan_num = files_to_scan_queue.len(); while !files_to_scan_queue.is_empty() { @@ -59,7 +66,11 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { let file = files_to_scan_queue.pop_front().unwrap(); // Scan the file - let track = visit_file(file.clone(), |path| Tag::read_from_path(path)); + let track = visit_file( + file.clone(), + |path| Tag::read_from_path(path), + thumbnails_dir_str.as_str(), + ); if track.is_err() { // Call the progress callback @@ -89,7 +100,7 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { let track_duration_js_number = cx.number(track.duration); track_js_object.set(&mut cx, "duration", track_duration_js_number)?; - set_optional_field_buffer(&mut cx, &mut track_js_object, "thumbnail", track.thumbnail); + set_optional_field_str(&mut cx, &mut track_js_object, "thumbnail", track.thumbnail); set_optional_field_u32(&mut cx, &mut track_js_object, "position", track.position); set_optional_field_u32(&mut cx, &mut track_js_object, "disc", track.disc); diff --git a/packages/scanner/src/local_track.rs b/packages/scanner/src/local_track.rs index 12da6cf956..4aeec16cfb 100644 --- a/packages/scanner/src/local_track.rs +++ b/packages/scanner/src/local_track.rs @@ -5,7 +5,7 @@ pub struct LocalTrack { pub title: Option, pub album: Option, pub duration: u32, - pub thumbnail: Option>, + pub thumbnail: Option, pub disc: Option, pub position: Option, pub year: Option, diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 11b77909c7..42e54ed0e7 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -5,12 +5,17 @@ use uuid::Uuid; use crate::error::ScannerError; use crate::local_track::LocalTrack; +use crate::thumbnails::generate_thumbnail; pub trait TagReader { fn read_from_path(path: impl AsRef) -> Result; } -pub fn visit_file(path: String, tag_reader: F) -> Result +pub fn visit_file( + path: String, + tag_reader: F, + thumbnails_dir: &str, +) -> Result where F: FnOnce(&str) -> Result, { @@ -23,10 +28,7 @@ where title: tag.title().map(|s| s.to_string()), album: tag.album().map(|s| s.to_string()), duration: tag.duration().unwrap_or(0), - thumbnail: tag - .pictures() - .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) - .map(|p| p.data.clone()), + thumbnail: generate_thumbnail(&path, thumbnails_dir), position: tag.track(), disc: tag.disc(), year: tag.year().map(|s| s as u32), @@ -110,7 +112,10 @@ mod tests { assert_eq!(track.year, Some(2020)); assert_eq!(track.filename, String::from("file.mp3")); assert_eq!(track.path, path); - assert_eq!(track.thumbnail, Some(vec![1, 2, 3])); + assert_eq!( + track.thumbnail, + Some("file://path/to/valid/file.webp".to_string()) + ); } else { panic!("Result is not ok"); } diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs new file mode 100644 index 0000000000..bd7951c30d --- /dev/null +++ b/packages/scanner/src/thumbnails.rs @@ -0,0 +1,59 @@ +use id3::Tag; +use image::{imageops::resize, imageops::FilterType, io::Reader as ImageReader, ImageFormat}; +use md5; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +fn hash_thumb_filename(path: &str) -> String { + let filename = Path::new(path).file_name().unwrap(); + let hash = md5::compute(filename.to_string_lossy().as_bytes()); + format!("{:x}.webp", hash) +} + +pub fn create_thumbnails_dir(thumbnails_dir: &str) { + let thumbnails_dir_path = Path::new(thumbnails_dir); + + if !thumbnails_dir_path.exists() { + std::fs::create_dir(thumbnails_dir_path).unwrap(); + } +} + +fn url_path_from_path(path: &str) -> String { + let path = path.replace("\\", "/"); + let path = path.replace(" ", "%20"); + format!("file://{}", path) +} + +pub fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { + let mut thumbnail_path = PathBuf::from(thumbnails_dir); + + thumbnail_path.push(hash_thumb_filename(filename)); + + let thumbnail_path_str = thumbnail_path.to_str().unwrap(); + + if Path::new(thumbnail_path_str).exists() { + return Some(url_path_from_path(thumbnail_path_str)); + } + + let tag = Tag::read_from_path(filename).unwrap(); + let thumbnail = tag + .pictures() + .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) + .map(|p| p.data.clone()); + + if let Some(thumbnail) = thumbnail { + let img = ImageReader::new(Cursor::new(&thumbnail)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + let img = resize(&img, 192, 192, FilterType::Lanczos3); + img.save_with_format(thumbnail_path_str, ImageFormat::WebP) + .unwrap(); + } else { + return None; + } + + Some(url_path_from_path(thumbnail_path_str)) +} From 00144d41ed6a1ea08684d80f69c0e7fe497e568a Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Thu, 9 Nov 2023 01:26:00 +0100 Subject: [PATCH 06/27] Implement FLAC scanning --- Cargo.lock | 24 ++++ .../main/src/controllers/local-library.ts | 12 +- packages/scanner/Cargo.toml | 1 + packages/scanner/index.d.ts | 3 +- packages/scanner/src/error.rs | 13 ++ packages/scanner/src/lib.rs | 38 ++++-- packages/scanner/src/local_track.rs | 12 +- packages/scanner/src/metadata.rs | 113 ++++++++++++++++++ packages/scanner/src/scanner.rs | 9 +- 9 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 packages/scanner/src/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 0bee7dda5d..6ff33d8b98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,6 +236,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "id3" version = "1.7.0" @@ -342,6 +348,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + [[package]] name = "md5" version = "0.7.0" @@ -363,6 +375,17 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metaflac" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1470d3cc1bb0d692af5eb3afb594330b8ba09fd91c32c4e1c6322172a5ba750" +dependencies = [ + "byteorder", + "hex", + "log", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -639,6 +662,7 @@ dependencies = [ "id3", "image", "md5", + "metaflac", "mockall", "neon", "uuid", diff --git a/packages/main/src/controllers/local-library.ts b/packages/main/src/controllers/local-library.ts index 97a3c90f21..3b5c5eefc9 100644 --- a/packages/main/src/controllers/local-library.ts +++ b/packages/main/src/controllers/local-library.ts @@ -8,13 +8,17 @@ import { ipcController, ipcEvent } from '../utils/decorators'; import LocalLibraryDb from '../services/local-library/db'; import Platform from '../services/platform'; import Window from '../services/window'; +import Config from '../services/config'; +import Logger, { $mainLogger } from '../services/logger'; @ipcController() class LocalIpcCtrl { constructor( + @inject(Config) private config: Config, @inject(LocalLibrary) private localLibrary: LocalLibrary, @inject(LocalLibraryDb) private localLibraryDb: LocalLibraryDb, @inject(Platform) private platform: Platform, + @inject($mainLogger) private logger: Logger, @inject(Window) private window: Window ) {} @@ -54,9 +58,9 @@ class LocalIpcCtrl { .map(folder => this.localLibraryDb.addFolder(this.normalizeFolderPath(folder))) ); - const cache = await scanFolders(directories, ['mp3'], this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { + const cache = await scanFolders(directories, this.config.supportedFormats, this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); - }); + }, () => {}); this.window.send(IpcEvents.LOCAL_FILES, Object.values(cache).reduce((acc, track) => ({ ...acc, @@ -84,9 +88,9 @@ class LocalIpcCtrl { try { const folders = await this.localLibraryDb.getLocalFolders(); - const cache = await scanFolders(folders.map(folder => folder.path), ['mp3'], this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { + const cache = await scanFolders(folders.map(folder => folder.path), this.config.supportedFormats, this.localLibrary.getThumbnailsDir(), (scanProgress, scanTotal) => { this.window.send(IpcEvents.LOCAL_FILES_PROGRESS, {scanProgress, scanTotal}); - }); + }, () => {}); this.window.send(IpcEvents.LOCAL_FILES, cache); } catch (err) { diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index b622f93c15..d06811c93e 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] [dependencies] id3 = "1.7.0" md5 = "0.7.0" +metaflac = "0.2.5" mockall = "0.11.4" [dependencies.image] diff --git a/packages/scanner/index.d.ts b/packages/scanner/index.d.ts index 96b0ac9158..2ddbc0d594 100644 --- a/packages/scanner/index.d.ts +++ b/packages/scanner/index.d.ts @@ -17,7 +17,8 @@ declare const scanFolders = ( folders: string[], supportedFormats: string[], thumbnailsDir: string, - onProgress: (progress: number, total: number, lastScanned?: string) => void + onProgress: (progress: number, total: number, lastScanned?: string) => void, + onError: (track: string, error: string) => void ) => new Promise; declare const generateThumbnail = (filename: string, thumbnailsDir: string) => new Promise; diff --git a/packages/scanner/src/error.rs b/packages/scanner/src/error.rs index 30c9d6a73a..dafbd940cb 100644 --- a/packages/scanner/src/error.rs +++ b/packages/scanner/src/error.rs @@ -13,3 +13,16 @@ impl fmt::Display for ScannerError { write!(f, "ScannerError: {}", self.message) } } + +#[derive(Debug)] +pub struct MetadataError { + pub message: String, +} + +impl Error for MetadataError {} + +impl fmt::Display for MetadataError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MetadataError: {}", self.message) + } +} diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index f21afdf4a8..20993c3587 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -1,6 +1,8 @@ +#![forbid(unsafe_code)] mod error; mod js; mod local_track; +mod metadata; mod scanner; mod thumbnails; use id3::Tag; @@ -16,6 +18,7 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { let thumbnails_dir: Handle = cx.argument(2)?; let thumbnails_dir_str = thumbnails_dir.value(&mut cx); let on_progress_callback: Handle = cx.argument(3)?; + let on_error_callback: Handle = cx.argument(4)?; let result: Handle = cx.empty_array(); // Copy all the starting folders to a queue, which holds all the folders left to scan @@ -83,6 +86,10 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { ]; on_progress_callback.call(&mut cx, this, args)?; + let error = track.err().unwrap(); + let error_string = cx.string(error.message); + let on_error_args = vec![cx.string(file.clone()).upcast(), error_string.upcast()]; + on_error_callback.call(&mut cx, this, on_error_args)?; continue; } @@ -93,18 +100,33 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { let track_uuid_js_string = cx.string(track.uuid); track_js_object.set(&mut cx, "uuid", track_uuid_js_string)?; - set_optional_field_str(&mut cx, &mut track_js_object, "artist", track.artist); - set_optional_field_str(&mut cx, &mut track_js_object, "title", track.title); - set_optional_field_str(&mut cx, &mut track_js_object, "album", track.album); + set_optional_field_str( + &mut cx, + &mut track_js_object, + "artist", + track.metadata.artist, + ); + set_optional_field_str(&mut cx, &mut track_js_object, "title", track.metadata.title); + set_optional_field_str(&mut cx, &mut track_js_object, "album", track.metadata.album); - let track_duration_js_number = cx.number(track.duration); + let track_duration_js_number = cx.number(track.metadata.duration); track_js_object.set(&mut cx, "duration", track_duration_js_number)?; - set_optional_field_str(&mut cx, &mut track_js_object, "thumbnail", track.thumbnail); + set_optional_field_str( + &mut cx, + &mut track_js_object, + "thumbnail", + track.metadata.thumbnail, + ); - set_optional_field_u32(&mut cx, &mut track_js_object, "position", track.position); - set_optional_field_u32(&mut cx, &mut track_js_object, "disc", track.disc); - set_optional_field_u32(&mut cx, &mut track_js_object, "year", track.year); + set_optional_field_u32( + &mut cx, + &mut track_js_object, + "position", + track.metadata.position, + ); + set_optional_field_u32(&mut cx, &mut track_js_object, "disc", track.metadata.disc); + set_optional_field_u32(&mut cx, &mut track_js_object, "year", track.metadata.year); let track_filename_js_string = cx.string(track.filename); track_js_object.set(&mut cx, "filename", track_filename_js_string)?; diff --git a/packages/scanner/src/local_track.rs b/packages/scanner/src/local_track.rs index 4aeec16cfb..c1d974c228 100644 --- a/packages/scanner/src/local_track.rs +++ b/packages/scanner/src/local_track.rs @@ -1,14 +1,10 @@ +use crate::metadata::AudioMetadata; + #[derive(Debug, Clone)] pub struct LocalTrack { pub uuid: String, - pub artist: Option, - pub title: Option, - pub album: Option, - pub duration: u32, - pub thumbnail: Option, - pub disc: Option, - pub position: Option, - pub year: Option, + + pub metadata: AudioMetadata, pub filename: String, pub path: String, diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs new file mode 100644 index 0000000000..d39ed283c6 --- /dev/null +++ b/packages/scanner/src/metadata.rs @@ -0,0 +1,113 @@ +use id3::TagLike; +use metaflac; + +use crate::{error::MetadataError, thumbnails::generate_thumbnail}; + +pub struct AudioMetadata { + pub artist: Option, + pub title: Option, + pub album: Option, + pub duration: u32, + pub disc: Option, + pub position: Option, + pub year: Option, + pub thumbnail: Option, +} + +impl AudioMetadata { + pub fn new() -> Self { + Self { + artist: None, + title: None, + album: None, + duration: 0, + disc: None, + position: None, + year: None, + thumbnail: None, + } + } +} + +pub trait MetadataExtractor { + fn extract_metadata( + &self, + path: &str, + thumbnails_dir: &str, + ) -> Result; +} + +#[derive(Debug, Clone)] + +pub struct Mp3MetadataExtractor; +impl MetadataExtractor for Mp3MetadataExtractor { + fn extract_metadata( + &self, + path: &str, + thumbnails_dir: &str, + ) -> Result { + let tag = id3::Tag::read_from_path(path).unwrap(); + let mut metadata = AudioMetadata::new(); + + metadata.artist = tag.artist().map(|s| s.to_string()); + metadata.title = tag.title().map(|s| s.to_string()); + metadata.album = tag.album().map(|s| s.to_string()); + metadata.duration = tag.duration().unwrap_or(0); + metadata.position = tag.track(); + metadata.disc = tag.disc(); + metadata.year = tag.year().map(|s| s as u32); + metadata.thumbnail = generate_thumbnail(&path, thumbnails_dir); + + Ok(metadata) + } +} + +pub struct FlacMetadataExtractor; +impl FlacMetadataExtractor { + fn extract_string_metadata( + tag: &metaflac::Tag, + key: &str, + fallback_key: Option<&str>, + ) -> Option { + tag.get_vorbis(key) + .and_then(|mut iter| iter.next()) + .map(|s| s.to_string()) + .or_else(|| { + fallback_key.and_then(|key| { + tag.get_vorbis(key) + .and_then(|mut iter| iter.next()) + .map(|s| s.to_string()) + }) + }) + } + + fn extract_numeric_metadata(tag: &metaflac::Tag, key: &str) -> Option { + tag.get_vorbis(key) + .and_then(|mut iter| iter.next()) + .and_then(|s| s.parse::().ok()) + } +} + +impl MetadataExtractor for FlacMetadataExtractor { + fn extract_metadata( + &self, + path: &str, + thumbnails_dir: &str, + ) -> Result { + // Extract metadata from a FLAC file. + let tag = metaflac::Tag::read_from_path(path).unwrap(); + let mut metadata = AudioMetadata::new(); + metadata.artist = Self::extract_string_metadata(&tag, "ARTIST", Some("ALBUMARTIST")); + metadata.title = Self::extract_string_metadata(&tag, "TITLE", None); + metadata.album = Self::extract_string_metadata(&tag, "ALBUM", None); + metadata.duration = Self::extract_numeric_metadata(&tag, "LENGTH").unwrap_or(0); + metadata.position = Self::extract_numeric_metadata(&tag, "TRACKNUMBER"); + metadata.disc = Self::extract_numeric_metadata(&tag, "DISCNUMBER"); + metadata.year = Self::extract_numeric_metadata(&tag, "DATE"); + let thumbnail_content = tag.pictures().next().map(|p| p.data.clone()).unwrap(); + + //TODO: add thumbnail generation + + Ok(metadata) + } +} diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 42e54ed0e7..79098f3e2d 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -3,8 +3,9 @@ use std::collections::LinkedList; use std::path::Path; use uuid::Uuid; -use crate::error::ScannerError; +use crate::error::{MetadataError, ScannerError}; use crate::local_track::LocalTrack; +use crate::metadata::AudioMetadata; use crate::thumbnails::generate_thumbnail; pub trait TagReader { @@ -13,13 +14,13 @@ pub trait TagReader { pub fn visit_file( path: String, - tag_reader: F, + metadata_reader: F, thumbnails_dir: &str, ) -> Result where - F: FnOnce(&str) -> Result, + F: FnOnce(&str) -> Result, { - let tag = tag_reader(&path); + let meta = metadata_reader(&path); match tag { Ok(tag) => Ok(LocalTrack { From d3dd1fb44eed8ff04ea83897be07e0c3dfd5174d Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 10 Nov 2023 23:08:53 +0100 Subject: [PATCH 07/27] Generate FLAC thumbnails --- packages/scanner/src/error.rs | 13 +++ packages/scanner/src/metadata.rs | 11 ++- packages/scanner/src/thumbnails.rs | 127 +++++++++++++++++++++-------- 3 files changed, 116 insertions(+), 35 deletions(-) diff --git a/packages/scanner/src/error.rs b/packages/scanner/src/error.rs index dafbd940cb..ed462b4fd9 100644 --- a/packages/scanner/src/error.rs +++ b/packages/scanner/src/error.rs @@ -26,3 +26,16 @@ impl fmt::Display for MetadataError { write!(f, "MetadataError: {}", self.message) } } + +#[derive(Debug)] +pub struct ThumbnailError { + pub message: String, +} + +impl Error for ThumbnailError {} + +impl fmt::Display for ThumbnailError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ThumbnailError: {}", self.message) + } +} diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index d39ed283c6..db4bbf3e62 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,7 +1,12 @@ use id3::TagLike; use metaflac; +use neon::meta; -use crate::{error::MetadataError, thumbnails::generate_thumbnail}; +use crate::{ + error::MetadataError, + thumbnails::ThumbnailGenerator, + thumbnails::{FlacThumbnailGenerator, Mp3ThumbnailGenerator}, +}; pub struct AudioMetadata { pub artist: Option, @@ -56,7 +61,7 @@ impl MetadataExtractor for Mp3MetadataExtractor { metadata.position = tag.track(); metadata.disc = tag.disc(); metadata.year = tag.year().map(|s| s as u32); - metadata.thumbnail = generate_thumbnail(&path, thumbnails_dir); + metadata.thumbnail = Mp3ThumbnailGenerator::generate_thumbnail(&path, thumbnails_dir); Ok(metadata) } @@ -106,7 +111,7 @@ impl MetadataExtractor for FlacMetadataExtractor { metadata.year = Self::extract_numeric_metadata(&tag, "DATE"); let thumbnail_content = tag.pictures().next().map(|p| p.data.clone()).unwrap(); - //TODO: add thumbnail generation + metadata.thumbnail = FlacThumbnailGenerator::generate_thumbnail(&path, thumbnails_dir); Ok(metadata) } diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index bd7951c30d..95b133346f 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -1,20 +1,34 @@ use id3::Tag; use image::{imageops::resize, imageops::FilterType, io::Reader as ImageReader, ImageFormat}; use md5; -use std::io::Cursor; +use metaflac; +use std::io::{self, Cursor}; use std::path::{Path, PathBuf}; -fn hash_thumb_filename(path: &str) -> String { - let filename = Path::new(path).file_name().unwrap(); +use crate::error::ThumbnailError; + +pub trait ThumbnailGenerator { + fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option; +} + +fn hash_thumb_filename(path: &str) -> Result { + let filename = Path::new(path).file_name().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid path: {}", path), + ) + })?; let hash = md5::compute(filename.to_string_lossy().as_bytes()); - format!("{:x}.webp", hash) + Ok(format!("{:x}.webp", hash)) } -pub fn create_thumbnails_dir(thumbnails_dir: &str) { +pub fn create_thumbnails_dir(thumbnails_dir: &str) -> io::Result<()> { let thumbnails_dir_path = Path::new(thumbnails_dir); if !thumbnails_dir_path.exists() { - std::fs::create_dir(thumbnails_dir_path).unwrap(); + std::fs::create_dir(thumbnails_dir_path) + } else { + Ok(()) } } @@ -24,36 +38,85 @@ fn url_path_from_path(path: &str) -> String { format!("file://{}", path) } -pub fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { - let mut thumbnail_path = PathBuf::from(thumbnails_dir); +pub struct Mp3ThumbnailGenerator; +impl ThumbnailGenerator for Mp3ThumbnailGenerator { + fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { + let mut thumbnail_path = PathBuf::from(thumbnails_dir); - thumbnail_path.push(hash_thumb_filename(filename)); + let thumbnail_filename = hash_thumb_filename(filename); - let thumbnail_path_str = thumbnail_path.to_str().unwrap(); + let thumbnail_filename = match thumbnail_filename { + Ok(filename) => filename, + Err(e) => return None, + }; + thumbnail_path.push(thumbnail_filename); - if Path::new(thumbnail_path_str).exists() { - return Some(url_path_from_path(thumbnail_path_str)); - } + let thumbnail_path_str = thumbnail_path.to_str()?; - let tag = Tag::read_from_path(filename).unwrap(); - let thumbnail = tag - .pictures() - .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) - .map(|p| p.data.clone()); - - if let Some(thumbnail) = thumbnail { - let img = ImageReader::new(Cursor::new(&thumbnail)) - .with_guessed_format() - .unwrap() - .decode() - .unwrap(); - - let img = resize(&img, 192, 192, FilterType::Lanczos3); - img.save_with_format(thumbnail_path_str, ImageFormat::WebP) - .unwrap(); - } else { - return None; + if thumbnail_path.exists() { + return Some(url_path_from_path(thumbnail_path_str)); + } + + let tag = Tag::read_from_path(filename).unwrap(); + let thumbnail = tag + .pictures() + .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) + .map(|p| p.data.clone()); + + if let Some(thumbnail) = thumbnail { + let img = ImageReader::new(Cursor::new(&thumbnail)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + let img = resize(&img, 192, 192, FilterType::Lanczos3); + img.save_with_format(thumbnail_path_str, ImageFormat::WebP) + .unwrap(); + } else { + return None; + } + + Some(url_path_from_path(thumbnail_path_str)) } +} + +pub struct FlacThumbnailGenerator; +impl ThumbnailGenerator for FlacThumbnailGenerator { + fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { + let mut thumbnail_path = PathBuf::from(thumbnails_dir); + + let thumbnail_filename = hash_thumb_filename(filename); + + let thumbnail_filename = match thumbnail_filename { + Ok(filename) => filename, + Err(e) => return None, + }; + thumbnail_path.push(thumbnail_filename); + + let thumbnail_path_str = thumbnail_path.to_str()?; - Some(url_path_from_path(thumbnail_path_str)) + if thumbnail_path.exists() { + return Some(url_path_from_path(thumbnail_path_str)); + } + + let tag = metaflac::Tag::read_from_path(filename).unwrap(); + let thumbnail = tag.pictures().next().map(|p| p.data.clone()); + + if let Some(thumbnail) = thumbnail { + let img = ImageReader::new(Cursor::new(&thumbnail)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + let img = resize(&img, 192, 192, FilterType::Lanczos3); + img.save_with_format(thumbnail_path_str, ImageFormat::WebP) + .unwrap(); + } else { + return None; + } + + Some(url_path_from_path(thumbnail_path_str)) + } } From b48cc12d2a85f0182da99d532c0f794a3e78bfb7 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 10 Nov 2023 23:45:21 +0100 Subject: [PATCH 08/27] Refactor thumbnail generators --- packages/scanner/src/thumbnails.rs | 159 +++++++++++++---------------- 1 file changed, 71 insertions(+), 88 deletions(-) diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index 95b133346f..3859c12ba1 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -1,14 +1,80 @@ use id3::Tag; -use image::{imageops::resize, imageops::FilterType, io::Reader as ImageReader, ImageFormat}; +use image::{ + imageops::resize, imageops::FilterType, io::Reader as ImageReader, DynamicImage, ImageFormat, +}; +use image::{ImageBuffer, ImageError, ImageResult, Rgba}; use md5; -use metaflac; +use metaflac::Tag as FlacTag; use std::io::{self, Cursor}; use std::path::{Path, PathBuf}; -use crate::error::ThumbnailError; - pub trait ThumbnailGenerator { fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option; + fn read_image_data(filename: &str) -> Option>; +} + +pub struct Mp3ThumbnailGenerator; +impl ThumbnailGenerator for Mp3ThumbnailGenerator { + fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { + generate_thumbnail_common::(filename, thumbnails_dir) + } + + fn read_image_data(filename: &str) -> Option> { + let tag = Tag::read_from_path(filename).ok()?; + tag.pictures() + .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) + .map(|p| p.data.clone()) + } +} + +pub struct FlacThumbnailGenerator; +impl ThumbnailGenerator for FlacThumbnailGenerator { + fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { + generate_thumbnail_common::(filename, thumbnails_dir) + } + + fn read_image_data(filename: &str) -> Option> { + let tag = FlacTag::read_from_path(filename).ok()?; + tag.pictures().next().map(|p| p.data.clone()) + } +} + +fn generate_thumbnail_common( + filename: &str, + thumbnails_dir: &str, +) -> Option { + let thumbnail_path = create_and_get_thumbnail_path(filename, thumbnails_dir)?; + + if Path::new(&thumbnail_path).exists() { + Some(url_path_from_path(&thumbnail_path)) + } else if let Some(thumbnail_data) = T::read_image_data(filename) { + resize_and_save_thumbnail(&thumbnail_data, Path::new(&thumbnail_path)).ok()?; + Some(url_path_from_path(&thumbnail_path)) + } else { + None + } +} + +fn resize_and_save_thumbnail(data: &[u8], path: &Path) -> ImageResult<()> { + let img = get_resized_image(data)?; + img.save_with_format(path, ImageFormat::WebP) +} + +fn get_resized_image(data: &[u8]) -> Result, Vec>, ImageError> { + ImageReader::new(Cursor::new(data)) + .with_guessed_format()? + .decode() + .map(|img| resize(&img, 192, 192, FilterType::Lanczos3)) +} + +fn create_and_get_thumbnail_path(filename: &str, thumbnails_dir: &str) -> Option { + if let Ok(thumbnail_filename) = hash_thumb_filename(filename) { + let mut thumbnail_path = PathBuf::from(thumbnails_dir); + thumbnail_path.push(thumbnail_filename); + thumbnail_path.to_str().map(str::to_owned) + } else { + None + } } fn hash_thumb_filename(path: &str) -> Result { @@ -22,7 +88,7 @@ fn hash_thumb_filename(path: &str) -> Result { Ok(format!("{:x}.webp", hash)) } -pub fn create_thumbnails_dir(thumbnails_dir: &str) -> io::Result<()> { +fn create_thumbnails_dir(thumbnails_dir: &str) -> io::Result<()> { let thumbnails_dir_path = Path::new(thumbnails_dir); if !thumbnails_dir_path.exists() { @@ -37,86 +103,3 @@ fn url_path_from_path(path: &str) -> String { let path = path.replace(" ", "%20"); format!("file://{}", path) } - -pub struct Mp3ThumbnailGenerator; -impl ThumbnailGenerator for Mp3ThumbnailGenerator { - fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { - let mut thumbnail_path = PathBuf::from(thumbnails_dir); - - let thumbnail_filename = hash_thumb_filename(filename); - - let thumbnail_filename = match thumbnail_filename { - Ok(filename) => filename, - Err(e) => return None, - }; - thumbnail_path.push(thumbnail_filename); - - let thumbnail_path_str = thumbnail_path.to_str()?; - - if thumbnail_path.exists() { - return Some(url_path_from_path(thumbnail_path_str)); - } - - let tag = Tag::read_from_path(filename).unwrap(); - let thumbnail = tag - .pictures() - .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) - .map(|p| p.data.clone()); - - if let Some(thumbnail) = thumbnail { - let img = ImageReader::new(Cursor::new(&thumbnail)) - .with_guessed_format() - .unwrap() - .decode() - .unwrap(); - - let img = resize(&img, 192, 192, FilterType::Lanczos3); - img.save_with_format(thumbnail_path_str, ImageFormat::WebP) - .unwrap(); - } else { - return None; - } - - Some(url_path_from_path(thumbnail_path_str)) - } -} - -pub struct FlacThumbnailGenerator; -impl ThumbnailGenerator for FlacThumbnailGenerator { - fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { - let mut thumbnail_path = PathBuf::from(thumbnails_dir); - - let thumbnail_filename = hash_thumb_filename(filename); - - let thumbnail_filename = match thumbnail_filename { - Ok(filename) => filename, - Err(e) => return None, - }; - thumbnail_path.push(thumbnail_filename); - - let thumbnail_path_str = thumbnail_path.to_str()?; - - if thumbnail_path.exists() { - return Some(url_path_from_path(thumbnail_path_str)); - } - - let tag = metaflac::Tag::read_from_path(filename).unwrap(); - let thumbnail = tag.pictures().next().map(|p| p.data.clone()); - - if let Some(thumbnail) = thumbnail { - let img = ImageReader::new(Cursor::new(&thumbnail)) - .with_guessed_format() - .unwrap() - .decode() - .unwrap(); - - let img = resize(&img, 192, 192, FilterType::Lanczos3); - img.save_with_format(thumbnail_path_str, ImageFormat::WebP) - .unwrap(); - } else { - return None; - } - - Some(url_path_from_path(thumbnail_path_str)) - } -} From 41c85db0a659a0599a6fe98ea6780e6ff332b64b Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Sat, 11 Nov 2023 01:31:05 +0100 Subject: [PATCH 09/27] Dependency inversion for visit_file --- packages/scanner/Cargo.toml | 1 + packages/scanner/src/error.rs | 16 +++ packages/scanner/src/lib.rs | 5 +- packages/scanner/src/metadata.rs | 4 +- packages/scanner/src/scanner.rs | 152 ++++++++++++++--------------- packages/scanner/src/thumbnails.rs | 2 +- 6 files changed, 98 insertions(+), 82 deletions(-) diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index d06811c93e..7ce515edd4 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -9,6 +9,7 @@ exclude = ["index.node"] crate-type = ["cdylib"] [dependencies] +derive_builder = "0.12.0" id3 = "1.7.0" md5 = "0.7.0" metaflac = "0.2.5" diff --git a/packages/scanner/src/error.rs b/packages/scanner/src/error.rs index ed462b4fd9..683af2bac3 100644 --- a/packages/scanner/src/error.rs +++ b/packages/scanner/src/error.rs @@ -8,6 +8,14 @@ pub struct ScannerError { impl Error for ScannerError {} +impl ScannerError { + pub fn new(message: &str) -> ScannerError { + ScannerError { + message: message.to_string(), + } + } +} + impl fmt::Display for ScannerError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ScannerError: {}", self.message) @@ -19,6 +27,14 @@ pub struct MetadataError { pub message: String, } +impl MetadataError { + pub fn new(message: &str) -> MetadataError { + MetadataError { + message: message.to_string(), + } + } +} + impl Error for MetadataError {} impl fmt::Display for MetadataError { diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 20993c3587..31671b8a46 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -1,4 +1,5 @@ #![forbid(unsafe_code)] + mod error; mod js; mod local_track; @@ -8,7 +9,7 @@ mod thumbnails; use id3::Tag; use js::{set_optional_field_str, set_optional_field_u32}; use neon::prelude::*; -use scanner::{visit_directory, visit_file}; +use scanner::{extractor_from_path, visit_directory, visit_file}; use std::collections::LinkedList; use thumbnails::create_thumbnails_dir; @@ -71,7 +72,7 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { // Scan the file let track = visit_file( file.clone(), - |path| Tag::read_from_path(path), + extractor_from_path, thumbnails_dir_str.as_str(), ); diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index db4bbf3e62..a09da77ee4 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,6 +1,6 @@ +use derive_builder::Builder; use id3::TagLike; use metaflac; -use neon::meta; use crate::{ error::MetadataError, @@ -8,6 +8,8 @@ use crate::{ thumbnails::{FlacThumbnailGenerator, Mp3ThumbnailGenerator}, }; +#[derive(Default, Debug, Clone, Builder)] +#[builder(setter(strip_option))] pub struct AudioMetadata { pub artist: Option, pub title: Option, diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 79098f3e2d..458472f207 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -1,38 +1,48 @@ -use id3::{Error, Tag, TagLike}; +use id3::{Error, Tag}; use std::collections::LinkedList; +use std::ffi::OsStr; use std::path::Path; use uuid::Uuid; use crate::error::{MetadataError, ScannerError}; use crate::local_track::LocalTrack; -use crate::metadata::AudioMetadata; -use crate::thumbnails::generate_thumbnail; +use crate::metadata::{ + AudioMetadata, AudioMetadataBuilder, FlacMetadataExtractor, MetadataExtractor, + Mp3MetadataExtractor, +}; pub trait TagReader { fn read_from_path(path: impl AsRef) -> Result; } +fn get_extension(path: &str) -> Option<&str> { + Path::new(path).extension().and_then(OsStr::to_str) +} + +pub fn extractor_from_path(path: &str) -> Option> { + match get_extension(path) { + Some("mp3") => Some(Box::new(Mp3MetadataExtractor)), + Some("flac") => Some(Box::new(FlacMetadataExtractor)), + _ => None, + } +} + pub fn visit_file( path: String, - metadata_reader: F, + extractor_provider: F, thumbnails_dir: &str, ) -> Result where - F: FnOnce(&str) -> Result, + F: Fn(&str) -> Option>, { - let meta = metadata_reader(&path); + let extractor: Box = extractor_provider(&path) + .ok_or_else(|| ScannerError::new(&format!("Unsupported file format: {}", path)))?; + let metadata = extractor.extract_metadata(&path, thumbnails_dir); - match tag { - Ok(tag) => Ok(LocalTrack { + match metadata { + Ok(metadata) => Ok(LocalTrack { uuid: Uuid::new_v4().to_string(), - artist: tag.artist().map(|s| s.to_string()), - title: tag.title().map(|s| s.to_string()), - album: tag.album().map(|s| s.to_string()), - duration: tag.duration().unwrap_or(0), - thumbnail: generate_thumbnail(&path, thumbnails_dir), - position: tag.track(), - disc: tag.disc(), - year: tag.year().map(|s| s as u32), + metadata: metadata, filename: path.split("/").last().map(|s| s.to_string()).unwrap(), path: path.clone(), }), @@ -67,73 +77,59 @@ pub fn visit_directory( #[cfg(test)] mod tests { - use id3::{ - frame::{Picture, PictureType}, - Content, Frame, - }; - use super::*; - #[test] - fn test_visit_file_with_valid_file() { - // With mocked tag - let path = String::from("path/to/valid/file.mp3"); - let result = visit_file(path.clone(), |_inner_path| { - let mut tag = Tag::new(); - tag.set_artist("Artist"); - tag.set_title("Title"); - tag.set_album("Album"); - tag.set_duration(123); - tag.set_track(1); - tag.set_year(2020); - let picture = Picture { - mime_type: String::new(), - picture_type: PictureType::CoverFront, - description: String::new(), - data: vec![1, 2, 3], - }; - tag.add_frame(Frame::with_content( - "APIC", - Content::Picture(picture.clone()), - )); - Ok(tag) - }); - - if let Some(track) = result.ok() { - //check uuid format - assert_eq!( - track.uuid, - Uuid::parse_str(&track.uuid).unwrap().to_string() - ); - assert_eq!(track.artist, Some(String::from("Artist"))); - assert_eq!(track.title, Some(String::from("Title"))); - assert_eq!(track.album, Some(String::from("Album"))); - assert_eq!(track.duration, 123); - assert_eq!(track.position, Some(1)); - assert_eq!(track.year, Some(2020)); - assert_eq!(track.filename, String::from("file.mp3")); - assert_eq!(track.path, path); - assert_eq!( - track.thumbnail, - Some("file://path/to/valid/file.webp".to_string()) - ); - } else { - panic!("Result is not ok"); + #[derive(Debug, Clone, Default)] + struct TestMetadataExtractor { + pub test_metadata: AudioMetadata, + } + impl TestMetadataExtractor { + pub fn new() -> TestMetadataExtractor { + TestMetadataExtractor { + test_metadata: AudioMetadata::new(), + } + } + } + impl MetadataExtractor for TestMetadataExtractor { + fn extract_metadata( + &self, + _path: &str, + _thumbnails_dir: &str, + ) -> Result { + Ok(AudioMetadataBuilder::default() + .artist("Test Artist".to_string()) + .title("Test Title".to_string()) + .album("Test Album".to_string()) + .duration(10) + .position(1) + .disc(1) + .year(2020) + .thumbnail("http://localhost:8080/thumbnails/0b/0b0b0b0b0b0b0b0b.webp".to_string()) + .build() + .unwrap()) } } - #[test] - fn test_visit_file_with_no_tags() { - // With mocked tag - let path = String::from("path/to/invalid/file.mp3"); - let result = visit_file(path.clone(), |_inner_path| { - Err(id3::Error::new(id3::ErrorKind::NoTag, "")) - }); + pub fn test_extractor_from_path(_path: &str) -> Option> { + Some(Box::new(TestMetadataExtractor::new())) + } - if let Some(error) = result.err() { - assert_eq!(error.message, String::from("Error reading file: NoTag")); - } else { - panic!("Result is not err"); - } + #[test] + fn test_visit_file() { + let path = "tests/test.mp3".to_string(); + let thumbnails_dir = "tests/thumbnails".to_string(); + let local_track = visit_file(path, test_extractor_from_path, &thumbnails_dir).unwrap(); + assert_eq!(local_track.filename, "test.mp3"); + assert_eq!(local_track.metadata.artist, Some("Test Artist".to_string())); + assert_eq!(local_track.metadata.title, Some("Test Title".to_string())); + assert_eq!(local_track.metadata.album, Some("Test Album".to_string())); + assert_eq!(local_track.metadata.duration, 10); + assert_eq!(local_track.metadata.position, Some(1)); + assert_eq!(local_track.metadata.disc, Some(1)); + assert_eq!(local_track.metadata.year, Some(2020)); + assert_eq!( + local_track.metadata.thumbnail, + Some("http://localhost:8080/thumbnails/0b/0b0b0b0b0b0b0b0b.webp".to_string()) + ); } } diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index 3859c12ba1..bbc18fc12b 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -88,7 +88,7 @@ fn hash_thumb_filename(path: &str) -> Result { Ok(format!("{:x}.webp", hash)) } -fn create_thumbnails_dir(thumbnails_dir: &str) -> io::Result<()> { +pub fn create_thumbnails_dir(thumbnails_dir: &str) -> io::Result<()> { let thumbnails_dir_path = Path::new(thumbnails_dir); if !thumbnails_dir_path.exists() { From ababceae2d3929183ec30ed3dbf3ee8c82cd82fc Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Sat, 11 Nov 2023 01:33:07 +0100 Subject: [PATCH 10/27] Fix FLAC thumbnail generator --- packages/scanner/src/thumbnails.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index bbc18fc12b..4766a1b8a7 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -21,7 +21,9 @@ impl ThumbnailGenerator for Mp3ThumbnailGenerator { fn read_image_data(filename: &str) -> Option> { let tag = Tag::read_from_path(filename).ok()?; - tag.pictures() + let mut pictures = tag.pictures(); + + pictures .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) .map(|p| p.data.clone()) } @@ -35,7 +37,8 @@ impl ThumbnailGenerator for FlacThumbnailGenerator { fn read_image_data(filename: &str) -> Option> { let tag = FlacTag::read_from_path(filename).ok()?; - tag.pictures().next().map(|p| p.data.clone()) + let mut pictures = tag.pictures(); + pictures.next().map(|p| p.data.clone()) } } From 37d35f846de757efc5d3f83b2c6cc988a4932832 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Sun, 12 Nov 2023 03:07:03 +0100 Subject: [PATCH 11/27] Refactor scanning code --- packages/scanner/src/error.rs | 4 +- packages/scanner/src/js.rs | 18 ++++ packages/scanner/src/lib.rs | 173 +++++++++++++++---------------- packages/scanner/src/metadata.rs | 8 +- packages/scanner/src/scanner.rs | 31 +++--- 5 files changed, 128 insertions(+), 106 deletions(-) diff --git a/packages/scanner/src/error.rs b/packages/scanner/src/error.rs index 683af2bac3..1bec6b2f50 100644 --- a/packages/scanner/src/error.rs +++ b/packages/scanner/src/error.rs @@ -4,14 +4,16 @@ use std::fmt; #[derive(Debug)] pub struct ScannerError { pub message: String, + pub path: String, } impl Error for ScannerError {} impl ScannerError { - pub fn new(message: &str) -> ScannerError { + pub fn new(message: &str, path: &str) -> ScannerError { ScannerError { message: message.to_string(), + path: path.to_string(), } } } diff --git a/packages/scanner/src/js.rs b/packages/scanner/src/js.rs index e929cbdaf4..ef3918ee1f 100644 --- a/packages/scanner/src/js.rs +++ b/packages/scanner/src/js.rs @@ -1,5 +1,7 @@ use neon::prelude::*; +use crate::metadata::AudioMetadata; + pub fn set_optional_field_str( cx: &mut FunctionContext, obj: &mut Handle, @@ -53,3 +55,19 @@ pub fn set_optional_field_buffer( } } } + +pub fn set_properties_from_metadata( + cx: &mut FunctionContext, + obj: &mut Handle, + metadata: &AudioMetadata, +) { + set_optional_field_str(cx, obj, "artist", metadata.artist.clone()); + set_optional_field_str(cx, obj, "title", metadata.title.clone()); + set_optional_field_str(cx, obj, "album", metadata.album.clone()); + + set_optional_field_u32(cx, obj, "duration", metadata.duration); + set_optional_field_str(cx, obj, "thumbnail", metadata.thumbnail.clone()); + set_optional_field_u32(cx, obj, "position", metadata.position); + set_optional_field_u32(cx, obj, "disc", metadata.disc); + set_optional_field_u32(cx, obj, "year", metadata.year); +} diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 31671b8a46..097eb98fba 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -6,13 +6,44 @@ mod local_track; mod metadata; mod scanner; mod thumbnails; +use error::ScannerError; use id3::Tag; -use js::{set_optional_field_str, set_optional_field_u32}; +use js::set_properties_from_metadata; +use local_track::LocalTrack; use neon::prelude::*; use scanner::{extractor_from_path, visit_directory, visit_file}; use std::collections::LinkedList; use thumbnails::create_thumbnails_dir; +fn handle_progress<'a>( + cx: &mut FunctionContext<'a>, + total_files_to_scan_num: usize, + index: usize, + filename: String, + on_progress_callback: Handle, +) -> JsResult<'a, JsValue> { + let js_this = cx.undefined(); + let args = vec![ + cx.number(index as f64).upcast(), + cx.number(total_files_to_scan_num as f64).upcast(), + cx.string(filename.clone()).upcast(), + ]; + on_progress_callback.call(cx, js_this, args) +} + +fn handle_error<'a>( + cx: &mut FunctionContext<'a>, + error: &ScannerError, + on_error_callback: Handle, +) -> JsResult<'a, JsValue> { + let js_this = cx.undefined(); + let on_error_args = vec![ + cx.string(error.path.clone()).upcast(), + cx.string(error.message.clone()).upcast(), + ]; + on_error_callback.call(cx, js_this, on_error_args) +} + fn scan_folders(mut cx: FunctionContext) -> JsResult { let folders: Handle = cx.argument(0)?; let supported_formats: Handle = cx.argument(1)?; @@ -61,95 +92,63 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { } // First, create a directory for thumbnails - create_thumbnails_dir(thumbnails_dir_str.as_str()); + create_thumbnails_dir(thumbnails_dir_str.as_str()).unwrap(); - // All folders have been scanned, now scan the files + // All directories have been scanned, now scan the files total_files_to_scan_num = files_to_scan_queue.len(); - while !files_to_scan_queue.is_empty() { - // Get the next file to scan - let file = files_to_scan_queue.pop_front().unwrap(); - - // Scan the file - let track = visit_file( - file.clone(), - extractor_from_path, - thumbnails_dir_str.as_str(), - ); - - if track.is_err() { - // Call the progress callback - let this = cx.undefined(); - let args = vec![ - cx.number((total_files_to_scan_num - files_to_scan_queue.len()) as f64) - .upcast(), - cx.number(total_files_to_scan_num as f64).upcast(), - cx.string(file.clone()).upcast(), - ]; - on_progress_callback.call(&mut cx, this, args)?; - - let error = track.err().unwrap(); - let error_string = cx.string(error.message); - let on_error_args = vec![cx.string(file.clone()).upcast(), error_string.upcast()]; - on_error_callback.call(&mut cx, this, on_error_args)?; - continue; + let scanned_local_tracks: Vec> = files_to_scan_queue + .iter() + .enumerate() + .map(|(index, file)| { + let result = visit_file( + file.clone(), + extractor_from_path, + thumbnails_dir_str.as_str(), + ); + + // Send progress back to JS + handle_progress( + &mut cx, + total_files_to_scan_num, + index, + file.clone(), + on_progress_callback.clone(), + ) + .unwrap(); + return result; + }) + .collect(); + + scanned_local_tracks.iter().for_each(|track| match track { + Ok(track) => { + let len = result.len(&mut cx); + let mut track_js_object = JsObject::new(&mut cx); + let track_uuid_js_string = cx.string(track.uuid.clone()); + track_js_object + .set(&mut cx, "uuid", track_uuid_js_string) + .unwrap(); + + set_properties_from_metadata(&mut cx, &mut track_js_object, &track.metadata); + + let track_filename_js_string = cx.string(track.filename.clone()); + track_js_object + .set(&mut cx, "filename", track_filename_js_string) + .unwrap(); + + let track_path_js_string = cx.string(track.path.clone()); + track_js_object + .set(&mut cx, "path", track_path_js_string) + .unwrap(); + + let track_local = cx.boolean(true); + track_js_object.set(&mut cx, "local", track_local).unwrap(); + + result.set(&mut cx, len, track_js_object).unwrap(); } - - let track = track.unwrap(); - - let len = result.len(&mut cx); - let mut track_js_object = JsObject::new(&mut cx); - let track_uuid_js_string = cx.string(track.uuid); - track_js_object.set(&mut cx, "uuid", track_uuid_js_string)?; - - set_optional_field_str( - &mut cx, - &mut track_js_object, - "artist", - track.metadata.artist, - ); - set_optional_field_str(&mut cx, &mut track_js_object, "title", track.metadata.title); - set_optional_field_str(&mut cx, &mut track_js_object, "album", track.metadata.album); - - let track_duration_js_number = cx.number(track.metadata.duration); - track_js_object.set(&mut cx, "duration", track_duration_js_number)?; - - set_optional_field_str( - &mut cx, - &mut track_js_object, - "thumbnail", - track.metadata.thumbnail, - ); - - set_optional_field_u32( - &mut cx, - &mut track_js_object, - "position", - track.metadata.position, - ); - set_optional_field_u32(&mut cx, &mut track_js_object, "disc", track.metadata.disc); - set_optional_field_u32(&mut cx, &mut track_js_object, "year", track.metadata.year); - - let track_filename_js_string = cx.string(track.filename); - track_js_object.set(&mut cx, "filename", track_filename_js_string)?; - - let track_path_js_string = cx.string(track.path); - track_js_object.set(&mut cx, "path", track_path_js_string)?; - - let track_local = cx.boolean(true); - track_js_object.set(&mut cx, "local", track_local)?; - - result.set(&mut cx, len, track_js_object)?; - - // Call the progress callback - let this = cx.undefined(); - let args = vec![ - cx.number((total_files_to_scan_num - files_to_scan_queue.len()) as f64) - .upcast(), - cx.number(total_files_to_scan_num as f64).upcast(), - cx.string(file.clone()).upcast(), - ]; - on_progress_callback.call(&mut cx, this, args)?; - } + Err(error) => { + handle_error(&mut cx, error, on_error_callback.clone()).unwrap(); + } + }); Ok(result) } diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index a09da77ee4..d2b697e2d5 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -14,7 +14,7 @@ pub struct AudioMetadata { pub artist: Option, pub title: Option, pub album: Option, - pub duration: u32, + pub duration: Option, pub disc: Option, pub position: Option, pub year: Option, @@ -27,7 +27,7 @@ impl AudioMetadata { artist: None, title: None, album: None, - duration: 0, + duration: None, disc: None, position: None, year: None, @@ -59,7 +59,7 @@ impl MetadataExtractor for Mp3MetadataExtractor { metadata.artist = tag.artist().map(|s| s.to_string()); metadata.title = tag.title().map(|s| s.to_string()); metadata.album = tag.album().map(|s| s.to_string()); - metadata.duration = tag.duration().unwrap_or(0); + metadata.duration = tag.duration(); metadata.position = tag.track(); metadata.disc = tag.disc(); metadata.year = tag.year().map(|s| s as u32); @@ -107,7 +107,7 @@ impl MetadataExtractor for FlacMetadataExtractor { metadata.artist = Self::extract_string_metadata(&tag, "ARTIST", Some("ALBUMARTIST")); metadata.title = Self::extract_string_metadata(&tag, "TITLE", None); metadata.album = Self::extract_string_metadata(&tag, "ALBUM", None); - metadata.duration = Self::extract_numeric_metadata(&tag, "LENGTH").unwrap_or(0); + metadata.duration = Self::extract_numeric_metadata(&tag, "LENGTH"); metadata.position = Self::extract_numeric_metadata(&tag, "TRACKNUMBER"); metadata.disc = Self::extract_numeric_metadata(&tag, "DISCNUMBER"); metadata.year = Self::extract_numeric_metadata(&tag, "DATE"); diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 458472f207..146aa74a4f 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -36,7 +36,7 @@ where F: Fn(&str) -> Option>, { let extractor: Box = extractor_provider(&path) - .ok_or_else(|| ScannerError::new(&format!("Unsupported file format: {}", path)))?; + .ok_or_else(|| ScannerError::new(&format!("Unsupported file format: {}", path), &path))?; let metadata = extractor.extract_metadata(&path, thumbnails_dir); match metadata { @@ -46,9 +46,10 @@ where filename: path.split("/").last().map(|s| s.to_string()).unwrap(), path: path.clone(), }), - Err(e) => Err(ScannerError { - message: format!("Error reading file: {}", e), - }), + Err(e) => Err(ScannerError::new( + &format!("Error reading file: {}", e), + &path, + )), } } @@ -84,9 +85,9 @@ mod tests { pub test_metadata: AudioMetadata, } impl TestMetadataExtractor { - pub fn new() -> TestMetadataExtractor { + pub fn new(metadata: AudioMetadata) -> TestMetadataExtractor { TestMetadataExtractor { - test_metadata: AudioMetadata::new(), + test_metadata: metadata, } } } @@ -96,7 +97,13 @@ mod tests { _path: &str, _thumbnails_dir: &str, ) -> Result { - Ok(AudioMetadataBuilder::default() + return Ok(self.test_metadata.clone()); + } + } + + pub fn test_extractor_from_path(_path: &str) -> Option> { + Some(Box::new(TestMetadataExtractor::new( + AudioMetadataBuilder::default() .artist("Test Artist".to_string()) .title("Test Title".to_string()) .album("Test Album".to_string()) @@ -106,12 +113,8 @@ mod tests { .year(2020) .thumbnail("http://localhost:8080/thumbnails/0b/0b0b0b0b0b0b0b0b.webp".to_string()) .build() - .unwrap()) - } - } - - pub fn test_extractor_from_path(_path: &str) -> Option> { - Some(Box::new(TestMetadataExtractor::new())) + .unwrap(), + ))) } #[test] @@ -123,7 +126,7 @@ mod tests { assert_eq!(local_track.metadata.artist, Some("Test Artist".to_string())); assert_eq!(local_track.metadata.title, Some("Test Title".to_string())); assert_eq!(local_track.metadata.album, Some("Test Album".to_string())); - assert_eq!(local_track.metadata.duration, 10); + assert_eq!(local_track.metadata.duration, Some(10)); assert_eq!(local_track.metadata.position, Some(1)); assert_eq!(local_track.metadata.disc, Some(1)); assert_eq!(local_track.metadata.year, Some(2020)); From 468cb3f7db8a6e6cc403bb86be62ad10ebf5b667 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 13 Nov 2023 01:22:10 +0100 Subject: [PATCH 12/27] Create a simple profiler for Rust code --- Cargo.lock | 85 ++++++++++++++++++++++++++++++ packages/core/package.json | 4 +- packages/i18n/package.json | 4 +- packages/main/package.json | 2 +- packages/scanner/Cargo.toml | 8 +-- packages/scanner/package.json | 1 + packages/scanner/src/lib.rs | 1 + packages/scanner/src/profiling.rs | 43 +++++++++++++++ packages/scanner/src/thumbnails.rs | 21 ++++++-- packages/ui/package.json | 4 +- 10 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 packages/scanner/src/profiling.rs diff --git a/Cargo.lock b/Cargo.lock index 6ff33d8b98..df8867139e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,72 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "difflib" version = "0.4.0" @@ -194,6 +260,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fragile" version = "2.0.0" @@ -253,6 +325,12 @@ dependencies = [ "flate2", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "image" version = "0.24.7" @@ -659,6 +737,7 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" name = "scanner" version = "0.1.0" dependencies = [ + "derive_builder", "id3", "image", "md5", @@ -710,6 +789,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.109" diff --git a/packages/core/package.json b/packages/core/package.json index 45f6fac6e6..af52a4d2e1 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -5,7 +5,7 @@ "type": "git", "url": "git+https://github.com/nukeop/nuclear.git" }, - "author": "nukeop ", + "author": "nukeop <12746779+nukeop@users.noreply.github.com>", "license": "AGPL-3.0", "bugs": { "url": "https://github.com/nukeop/nuclear/issues" @@ -58,4 +58,4 @@ "ts-node": "^10.7.0", "typescript": "^4.2.4" } -} +} \ No newline at end of file diff --git a/packages/i18n/package.json b/packages/i18n/package.json index d812da34f8..d64e63fb22 100644 --- a/packages/i18n/package.json +++ b/packages/i18n/package.json @@ -5,7 +5,7 @@ "type": "git", "url": "git+https://github.com/nukeop/nuclear.git" }, - "author": "nukeop ", + "author": "nukeop <12746779+nukeop@users.noreply.github.com>", "license": "AGPL-3.0", "bugs": { "url": "https://github.com/nukeop/nuclear/issues" @@ -25,4 +25,4 @@ "ts-node": "^10.7.0", "typescript": "^4.2.4" } -} +} \ No newline at end of file diff --git a/packages/main/package.json b/packages/main/package.json index bf52389973..815aa4970a 100644 --- a/packages/main/package.json +++ b/packages/main/package.json @@ -13,7 +13,7 @@ "type": "git", "url": "git+https://github.com/nukeop/nuclear.git" }, - "author": "nukeop ", + "author": "nukeop <12746779+nukeop@users.noreply.github.com>", "license": "AGPL-3.0", "bugs": { "url": "https://github.com/nukeop/nuclear/issues" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 7ce515edd4..057000cdb5 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -8,6 +8,9 @@ exclude = ["index.node"] [lib] crate-type = ["cdylib"] +[features] +profiling = [] + [dependencies] derive_builder = "0.12.0" id3 = "1.7.0" @@ -21,10 +24,7 @@ features = ["webp-encoder"] [dependencies.uuid] version = "1.3.4" -features = [ - "v4", - "fast-rng" -] +features = ["v4", "fast-rng"] [dependencies.neon] version = "0.10" diff --git a/packages/scanner/package.json b/packages/scanner/package.json index 79e3939034..a78cec0a6d 100644 --- a/packages/scanner/package.json +++ b/packages/scanner/package.json @@ -6,6 +6,7 @@ "types": "index.d.ts", "scripts": { "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build-profiling": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics --features profiling", "build-debug": "npm run build --", "build-release": "npm run build -- --release", "install": "npm run build-release", diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 097eb98fba..d79e5d503c 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -4,6 +4,7 @@ mod error; mod js; mod local_track; mod metadata; +mod profiling; mod scanner; mod thumbnails; use error::ScannerError; diff --git a/packages/scanner/src/profiling.rs b/packages/scanner/src/profiling.rs new file mode 100644 index 0000000000..5ad7a2cbe8 --- /dev/null +++ b/packages/scanner/src/profiling.rs @@ -0,0 +1,43 @@ +#[cfg(feature = "profiling")] +pub struct Profiler { + label: String, + start: std::time::Instant, +} + +#[cfg(feature = "profiling")] +impl Profiler { + pub fn start(label: &str) -> Self { + Profiler { + label: label.to_owned(), + start: std::time::Instant::now(), + } + } + + pub fn end(self) { + let duration = self.start.elapsed(); + println!("{} - Elapsed time: {:?}", self.label, duration); + } +} + +#[cfg(feature = "profiling")] +impl Drop for Profiler { + fn drop(&mut self) { + let duration = self.start.elapsed(); + println!("{} - Elapsed time: {:?}", self.label, duration); + } +} + +// Provide no-op implementations when profiling feature is not enabled +#[cfg(not(feature = "profiling"))] +pub struct Profiler; + +#[cfg(not(feature = "profiling"))] +impl Profiler { + pub fn start(_label: &str) -> Self { + Profiler {} + } + + pub fn end(self) { + // Do nothing + } +} diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index 4766a1b8a7..92ac07cabd 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -8,6 +8,8 @@ use metaflac::Tag as FlacTag; use std::io::{self, Cursor}; use std::path::{Path, PathBuf}; +use crate::profiling::Profiler; + pub trait ThumbnailGenerator { fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option; fn read_image_data(filename: &str) -> Option>; @@ -59,15 +61,26 @@ fn generate_thumbnail_common( } fn resize_and_save_thumbnail(data: &[u8], path: &Path) -> ImageResult<()> { + let _profiler = Profiler::start("resize_and_save_thumbnail"); let img = get_resized_image(data)?; img.save_with_format(path, ImageFormat::WebP) } fn get_resized_image(data: &[u8]) -> Result, Vec>, ImageError> { - ImageReader::new(Cursor::new(data)) - .with_guessed_format()? - .decode() - .map(|img| resize(&img, 192, 192, FilterType::Lanczos3)) + let img = ImageReader::new(Cursor::new(data)); + + let guess_format_profiler = Profiler::start("guess_format"); + let format = img.with_guessed_format()?; + guess_format_profiler.end(); + + let decode_start = Profiler::start("decode"); + let decoded = format.decode()?; + decode_start.end(); + + let resize_start = Profiler::start("resize"); + let resized = resize(&decoded, 256, 256, FilterType::CatmullRom); + resize_start.end(); + return Ok(resized); } fn create_and_get_thumbnail_path(filename: &str, thumbnails_dir: &str) -> Option { diff --git a/packages/ui/package.json b/packages/ui/package.json index 0e2e92c60f..f65dd6973d 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -2,7 +2,7 @@ "name": "@nuclear/ui", "version": "0.6.30", "description": "Nuclear - UI components", - "author": "nukeop ", + "author": "nukeop <12746779+nukeop@users.noreply.github.com>", "homepage": "https://github.com/nukeop/nuclear/tree/master/packages/ui#readme", "license": "AGPL-3.0", "files": [ @@ -93,4 +93,4 @@ "styled-components": "^4.4.1", "yup": "^0.32.9" } -} +} \ No newline at end of file From 0f725b28eaba5688c16aad86174766d414520d9f Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 13 Nov 2023 01:55:10 +0100 Subject: [PATCH 13/27] Generate only one thumbnail per album --- packages/scanner/package.json | 2 +- packages/scanner/src/lib.rs | 6 +++++- packages/scanner/src/metadata.rs | 20 ++++++++++++++++---- packages/scanner/src/scanner.rs | 15 ++++++++++++--- packages/scanner/src/thumbnails.rs | 30 +++++++++++++++++++++--------- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/packages/scanner/package.json b/packages/scanner/package.json index a78cec0a6d..8b0fd2af8f 100644 --- a/packages/scanner/package.json +++ b/packages/scanner/package.json @@ -6,7 +6,7 @@ "types": "index.d.ts", "scripts": { "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", - "build-profiling": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics --features profiling", + "build-profiling": "npm run build -- --features profiling --release", "build-debug": "npm run build --", "build-release": "npm run build -- --release", "install": "npm run build-release", diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index d79e5d503c..9067b80b68 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -13,7 +13,7 @@ use js::set_properties_from_metadata; use local_track::LocalTrack; use neon::prelude::*; use scanner::{extractor_from_path, visit_directory, visit_file}; -use std::collections::LinkedList; +use std::collections::{HashSet, LinkedList}; use thumbnails::create_thumbnails_dir; fn handle_progress<'a>( @@ -97,6 +97,9 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { // All directories have been scanned, now scan the files total_files_to_scan_num = files_to_scan_queue.len(); + + let mut created_thumbnails_hashset: HashSet = HashSet::new(); + let scanned_local_tracks: Vec> = files_to_scan_queue .iter() .enumerate() @@ -105,6 +108,7 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { file.clone(), extractor_from_path, thumbnails_dir_str.as_str(), + &mut created_thumbnails_hashset, ); // Send progress back to JS diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index d2b697e2d5..b622f6d265 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use derive_builder::Builder; use id3::TagLike; use metaflac; @@ -41,6 +43,7 @@ pub trait MetadataExtractor { &self, path: &str, thumbnails_dir: &str, + created_thumbnails_hashset: &mut HashSet, ) -> Result; } @@ -52,6 +55,7 @@ impl MetadataExtractor for Mp3MetadataExtractor { &self, path: &str, thumbnails_dir: &str, + created_thumbnails_hashset: &mut HashSet, ) -> Result { let tag = id3::Tag::read_from_path(path).unwrap(); let mut metadata = AudioMetadata::new(); @@ -63,7 +67,12 @@ impl MetadataExtractor for Mp3MetadataExtractor { metadata.position = tag.track(); metadata.disc = tag.disc(); metadata.year = tag.year().map(|s| s as u32); - metadata.thumbnail = Mp3ThumbnailGenerator::generate_thumbnail(&path, thumbnails_dir); + + metadata.thumbnail = Mp3ThumbnailGenerator::generate_thumbnail( + &path, + metadata.album.as_deref(), + thumbnails_dir, + ); Ok(metadata) } @@ -100,6 +109,7 @@ impl MetadataExtractor for FlacMetadataExtractor { &self, path: &str, thumbnails_dir: &str, + created_thumbnails_hashset: &mut HashSet, ) -> Result { // Extract metadata from a FLAC file. let tag = metaflac::Tag::read_from_path(path).unwrap(); @@ -111,9 +121,11 @@ impl MetadataExtractor for FlacMetadataExtractor { metadata.position = Self::extract_numeric_metadata(&tag, "TRACKNUMBER"); metadata.disc = Self::extract_numeric_metadata(&tag, "DISCNUMBER"); metadata.year = Self::extract_numeric_metadata(&tag, "DATE"); - let thumbnail_content = tag.pictures().next().map(|p| p.data.clone()).unwrap(); - - metadata.thumbnail = FlacThumbnailGenerator::generate_thumbnail(&path, thumbnails_dir); + metadata.thumbnail = FlacThumbnailGenerator::generate_thumbnail( + &path, + metadata.album.as_deref(), + thumbnails_dir, + ); Ok(metadata) } diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 146aa74a4f..3c1ca06b19 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -1,5 +1,5 @@ use id3::{Error, Tag}; -use std::collections::LinkedList; +use std::collections::{HashSet, LinkedList}; use std::ffi::OsStr; use std::path::Path; use uuid::Uuid; @@ -31,13 +31,14 @@ pub fn visit_file( path: String, extractor_provider: F, thumbnails_dir: &str, + created_thumbnails_hashset: &mut HashSet, ) -> Result where F: Fn(&str) -> Option>, { let extractor: Box = extractor_provider(&path) .ok_or_else(|| ScannerError::new(&format!("Unsupported file format: {}", path), &path))?; - let metadata = extractor.extract_metadata(&path, thumbnails_dir); + let metadata = extractor.extract_metadata(&path, thumbnails_dir, created_thumbnails_hashset); match metadata { Ok(metadata) => Ok(LocalTrack { @@ -96,6 +97,7 @@ mod tests { &self, _path: &str, _thumbnails_dir: &str, + _created_thumbnails_hashset: &mut HashSet, ) -> Result { return Ok(self.test_metadata.clone()); } @@ -121,7 +123,14 @@ mod tests { fn test_visit_file() { let path = "tests/test.mp3".to_string(); let thumbnails_dir = "tests/thumbnails".to_string(); - let local_track = visit_file(path, test_extractor_from_path, &thumbnails_dir).unwrap(); + let mut created_thumbnails_hashset: HashSet = HashSet::new(); + let local_track = visit_file( + path, + test_extractor_from_path, + &thumbnails_dir, + &mut created_thumbnails_hashset, + ) + .unwrap(); assert_eq!(local_track.filename, "test.mp3"); assert_eq!(local_track.metadata.artist, Some("Test Artist".to_string())); assert_eq!(local_track.metadata.title, Some("Test Title".to_string())); diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index 92ac07cabd..6f870a94eb 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -1,7 +1,5 @@ use id3::Tag; -use image::{ - imageops::resize, imageops::FilterType, io::Reader as ImageReader, DynamicImage, ImageFormat, -}; +use image::{imageops::resize, imageops::FilterType, io::Reader as ImageReader, ImageFormat}; use image::{ImageBuffer, ImageError, ImageResult, Rgba}; use md5; use metaflac::Tag as FlacTag; @@ -11,14 +9,22 @@ use std::path::{Path, PathBuf}; use crate::profiling::Profiler; pub trait ThumbnailGenerator { - fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option; + fn generate_thumbnail( + filename: &str, + album: Option<&str>, + thumbnails_dir: &str, + ) -> Option; fn read_image_data(filename: &str) -> Option>; } pub struct Mp3ThumbnailGenerator; impl ThumbnailGenerator for Mp3ThumbnailGenerator { - fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { - generate_thumbnail_common::(filename, thumbnails_dir) + fn generate_thumbnail( + filename: &str, + album: Option<&str>, + thumbnails_dir: &str, + ) -> Option { + generate_thumbnail_common::(filename, album, thumbnails_dir) } fn read_image_data(filename: &str) -> Option> { @@ -33,8 +39,12 @@ impl ThumbnailGenerator for Mp3ThumbnailGenerator { pub struct FlacThumbnailGenerator; impl ThumbnailGenerator for FlacThumbnailGenerator { - fn generate_thumbnail(filename: &str, thumbnails_dir: &str) -> Option { - generate_thumbnail_common::(filename, thumbnails_dir) + fn generate_thumbnail( + filename: &str, + album: Option<&str>, + thumbnails_dir: &str, + ) -> Option { + generate_thumbnail_common::(filename, album, thumbnails_dir) } fn read_image_data(filename: &str) -> Option> { @@ -46,9 +56,11 @@ impl ThumbnailGenerator for FlacThumbnailGenerator { fn generate_thumbnail_common( filename: &str, + album: Option<&str>, thumbnails_dir: &str, ) -> Option { - let thumbnail_path = create_and_get_thumbnail_path(filename, thumbnails_dir)?; + let filename_for_thumbnail = album.unwrap_or(filename); + let thumbnail_path = create_and_get_thumbnail_path(filename_for_thumbnail, thumbnails_dir)?; if Path::new(&thumbnail_path).exists() { Some(url_path_from_path(&thumbnail_path)) From af45f278d9482ec4cb938f20d6441c08b9186b46 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Tue, 14 Nov 2023 00:44:54 +0100 Subject: [PATCH 14/27] Add support for MP4 --- Cargo.lock | 53 +++++++++++++++++++ packages/scanner/Cargo.toml | 2 + packages/scanner/src/lib.rs | 7 +-- packages/scanner/src/metadata.rs | 85 +++++++++++++++++++++++++++--- packages/scanner/src/scanner.rs | 12 +---- packages/scanner/src/thumbnails.rs | 16 ++++++ 6 files changed, 154 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df8867139e..7941d9a432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,6 +390,17 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" +[[package]] +name = "lewton" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030" +dependencies = [ + "byteorder", + "ogg", + "tinyvec", +] + [[package]] name = "libc" version = "0.2.146" @@ -501,6 +512,22 @@ dependencies = [ "syn", ] +[[package]] +name = "mp4ameta" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb23d62e8eb5299a3f79657c70ea9269eac8f6239a76952689bcd06a74057e81" +dependencies = [ + "lazy_static", + "mp4ameta_proc", +] + +[[package]] +name = "mp4ameta_proc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07dcca13d1740c0a665f77104803360da0bdb3323ecce2e93fa2c959a6d52806" + [[package]] name = "neon" version = "0.10.1" @@ -578,6 +605,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "ogg" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e" +dependencies = [ + "byteorder", +] + [[package]] name = "png" version = "0.17.10" @@ -740,9 +776,11 @@ dependencies = [ "derive_builder", "id3", "image", + "lewton", "md5", "metaflac", "mockall", + "mp4ameta", "neon", "uuid", ] @@ -834,6 +872,21 @@ dependencies = [ "weezl", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "unicode-ident" version = "1.0.9" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 057000cdb5..db4cec305b 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -14,9 +14,11 @@ profiling = [] [dependencies] derive_builder = "0.12.0" id3 = "1.7.0" +lewton = "0.10.2" md5 = "0.7.0" metaflac = "0.2.5" mockall = "0.11.4" +mp4ameta = "0.11.0" [dependencies.image] version = "0.24.7" diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 9067b80b68..3b357ceb11 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -8,12 +8,12 @@ mod profiling; mod scanner; mod thumbnails; use error::ScannerError; -use id3::Tag; + use js::set_properties_from_metadata; use local_track::LocalTrack; use neon::prelude::*; use scanner::{extractor_from_path, visit_directory, visit_file}; -use std::collections::{HashSet, LinkedList}; +use std::collections::LinkedList; use thumbnails::create_thumbnails_dir; fn handle_progress<'a>( @@ -98,8 +98,6 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { // All directories have been scanned, now scan the files total_files_to_scan_num = files_to_scan_queue.len(); - let mut created_thumbnails_hashset: HashSet = HashSet::new(); - let scanned_local_tracks: Vec> = files_to_scan_queue .iter() .enumerate() @@ -108,7 +106,6 @@ fn scan_folders(mut cx: FunctionContext) -> JsResult { file.clone(), extractor_from_path, thumbnails_dir_str.as_str(), - &mut created_thumbnails_hashset, ); // Send progress back to JS diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index b622f6d265..8d28d7aa36 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,13 +1,14 @@ -use std::collections::HashSet; +use std::fs::File; use derive_builder::Builder; use id3::TagLike; +use lewton::inside_ogg::OggStreamReader; use metaflac; use crate::{ error::MetadataError, - thumbnails::ThumbnailGenerator, thumbnails::{FlacThumbnailGenerator, Mp3ThumbnailGenerator}, + thumbnails::{Mp4ThumbnailGenerator, ThumbnailGenerator}, }; #[derive(Default, Debug, Clone, Builder)] @@ -43,7 +44,6 @@ pub trait MetadataExtractor { &self, path: &str, thumbnails_dir: &str, - created_thumbnails_hashset: &mut HashSet, ) -> Result; } @@ -55,7 +55,6 @@ impl MetadataExtractor for Mp3MetadataExtractor { &self, path: &str, thumbnails_dir: &str, - created_thumbnails_hashset: &mut HashSet, ) -> Result { let tag = id3::Tag::read_from_path(path).unwrap(); let mut metadata = AudioMetadata::new(); @@ -109,9 +108,7 @@ impl MetadataExtractor for FlacMetadataExtractor { &self, path: &str, thumbnails_dir: &str, - created_thumbnails_hashset: &mut HashSet, ) -> Result { - // Extract metadata from a FLAC file. let tag = metaflac::Tag::read_from_path(path).unwrap(); let mut metadata = AudioMetadata::new(); metadata.artist = Self::extract_string_metadata(&tag, "ARTIST", Some("ALBUMARTIST")); @@ -130,3 +127,79 @@ impl MetadataExtractor for FlacMetadataExtractor { Ok(metadata) } } + +// pub struct OggMetadataExtractor; + +// impl OggMetadataExtractor { +// fn extract_vorbis_comment(metadata: &mut AudioMetadata, comments: &[(String, String)]) { +// for (key, value) in comments { +// match key.as_str() { +// "ARTIST" | "PERFORMER" => metadata.artist = Some(value.clone()), +// "TITLE" => metadata.title = Some(value.clone()), +// "ALBUM" => metadata.album = Some(value.clone()), +// "TRACKNUMBER" => metadata.position = value.parse().ok(), +// "DISCNUMBER" => metadata.disc = value.parse().ok(), +// "DATE" => metadata.year = value.parse().ok(), +// _ => {} +// } +// } +// } +// } + +// impl MetadataExtractor for OggMetadataExtractor { +// fn extract_metadata( +// &self, +// path: &str, +// _thumbnails_dir: &str, // Thumbnail generation is not handled here +// ) -> Result { +// let file = File::open(path)?; +// let mut ogg_reader = OggStreamReader::new(file)?; + +// let mut metadata = AudioMetadata::new(); + +// // Check if there are Vorbis comments, assumes Vorbis codec +// if let Some(comments) = ogg_reader.comment_hdrs() { +// Self::extract_vorbis_comment(&mut metadata, &comments.user_comments); +// } + +// metadata.duration = ogg_reader.ident_hdr().map(|ident| { +// let total_samples = ogg_reader.len() as u32; +// let sample_rate = ident.audio_sample_rate; +// total_samples / sample_rate +// }); + +// Ok(metadata) +// } +// } + +pub struct Mp4MetadataExtractor; + +impl Mp4MetadataExtractor {} + +impl MetadataExtractor for Mp4MetadataExtractor { + fn extract_metadata( + &self, + path: &str, + thumbnails_dir: &str, + ) -> Result { + let tag = mp4ameta::Tag::read_from_path(path).unwrap(); + + let mut metadata = AudioMetadata::new(); + + metadata.artist = tag.artist().map(|s| s.to_string()); + metadata.title = tag.title().map(|s| s.to_string()); + metadata.album = tag.album().map(|s| s.to_string()); + metadata.duration = tag.duration().map(|d| d.as_secs() as u32); + metadata.position = tag.track_number().map(|n| n as u32); + metadata.disc = tag.disc_number().map(|n| n as u32); + metadata.year = tag.year().map(|y: &str| y.parse().unwrap()); + + metadata.thumbnail = Mp4ThumbnailGenerator::generate_thumbnail( + &path, + metadata.album.as_deref(), + thumbnails_dir, + ); + + Ok(metadata) + } +} diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 3c1ca06b19..044e48def1 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -31,14 +31,13 @@ pub fn visit_file( path: String, extractor_provider: F, thumbnails_dir: &str, - created_thumbnails_hashset: &mut HashSet, ) -> Result where F: Fn(&str) -> Option>, { let extractor: Box = extractor_provider(&path) .ok_or_else(|| ScannerError::new(&format!("Unsupported file format: {}", path), &path))?; - let metadata = extractor.extract_metadata(&path, thumbnails_dir, created_thumbnails_hashset); + let metadata = extractor.extract_metadata(&path, thumbnails_dir); match metadata { Ok(metadata) => Ok(LocalTrack { @@ -97,7 +96,6 @@ mod tests { &self, _path: &str, _thumbnails_dir: &str, - _created_thumbnails_hashset: &mut HashSet, ) -> Result { return Ok(self.test_metadata.clone()); } @@ -124,13 +122,7 @@ mod tests { let path = "tests/test.mp3".to_string(); let thumbnails_dir = "tests/thumbnails".to_string(); let mut created_thumbnails_hashset: HashSet = HashSet::new(); - let local_track = visit_file( - path, - test_extractor_from_path, - &thumbnails_dir, - &mut created_thumbnails_hashset, - ) - .unwrap(); + let local_track = visit_file(path, test_extractor_from_path, &thumbnails_dir).unwrap(); assert_eq!(local_track.filename, "test.mp3"); assert_eq!(local_track.metadata.artist, Some("Test Artist".to_string())); assert_eq!(local_track.metadata.title, Some("Test Title".to_string())); diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index 6f870a94eb..e623c004fa 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -37,6 +37,22 @@ impl ThumbnailGenerator for Mp3ThumbnailGenerator { } } +pub struct Mp4ThumbnailGenerator; +impl ThumbnailGenerator for Mp4ThumbnailGenerator { + fn generate_thumbnail( + filename: &str, + album: Option<&str>, + thumbnails_dir: &str, + ) -> Option { + generate_thumbnail_common::(filename, album, thumbnails_dir) + } + + fn read_image_data(filename: &str) -> Option> { + let mut tag = mp4ameta::Tag::read_from_path(filename).ok()?; + tag.artwork().map(|a| a.data.to_vec()) + } +} + pub struct FlacThumbnailGenerator; impl ThumbnailGenerator for FlacThumbnailGenerator { fn generate_thumbnail( From 27f37aba34f3cce00eea023aebe0ec34953274a9 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Thu, 16 Nov 2023 20:59:11 +0100 Subject: [PATCH 15/27] Fix for FLAC duration --- packages/scanner/src/metadata.rs | 4 +++- packages/scanner/src/scanner.rs | 4 +++- packages/scanner/src/thumbnails.rs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index 8d28d7aa36..22825c2d58 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -114,7 +114,9 @@ impl MetadataExtractor for FlacMetadataExtractor { metadata.artist = Self::extract_string_metadata(&tag, "ARTIST", Some("ALBUMARTIST")); metadata.title = Self::extract_string_metadata(&tag, "TITLE", None); metadata.album = Self::extract_string_metadata(&tag, "ALBUM", None); - metadata.duration = Self::extract_numeric_metadata(&tag, "LENGTH"); + let total_samples = tag.get_streaminfo().unwrap().total_samples; + let sample_rate = tag.get_streaminfo().unwrap().sample_rate; + metadata.duration = Some(total_samples as u32 / (sample_rate as u32)); metadata.position = Self::extract_numeric_metadata(&tag, "TRACKNUMBER"); metadata.disc = Self::extract_numeric_metadata(&tag, "DISCNUMBER"); metadata.year = Self::extract_numeric_metadata(&tag, "DATE"); diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 044e48def1..a10db4e88f 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -8,7 +8,7 @@ use crate::error::{MetadataError, ScannerError}; use crate::local_track::LocalTrack; use crate::metadata::{ AudioMetadata, AudioMetadataBuilder, FlacMetadataExtractor, MetadataExtractor, - Mp3MetadataExtractor, + Mp3MetadataExtractor, Mp4MetadataExtractor, }; pub trait TagReader { @@ -23,6 +23,8 @@ pub fn extractor_from_path(path: &str) -> Option> { match get_extension(path) { Some("mp3") => Some(Box::new(Mp3MetadataExtractor)), Some("flac") => Some(Box::new(FlacMetadataExtractor)), + Some("mp4") => Some(Box::new(Mp4MetadataExtractor)), + Some("m4a") => Some(Box::new(Mp4MetadataExtractor)), _ => None, } } diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index e623c004fa..c752cb1646 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -48,7 +48,7 @@ impl ThumbnailGenerator for Mp4ThumbnailGenerator { } fn read_image_data(filename: &str) -> Option> { - let mut tag = mp4ameta::Tag::read_from_path(filename).ok()?; + let tag = mp4ameta::Tag::read_from_path(filename).ok()?; tag.artwork().map(|a| a.data.to_vec()) } } From 3a758fe2d5d94475c65165ce51ff609df4f45227 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 17 Nov 2023 00:16:18 +0100 Subject: [PATCH 16/27] Fix measuring duration for MP3 --- Cargo.lock | 63 ++++++++++++++++++++++++++------ packages/scanner/Cargo.toml | 1 + packages/scanner/src/metadata.rs | 7 +++- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7941d9a432..1712f335f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 1.0.109", ] [[package]] @@ -155,7 +155,7 @@ checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -176,7 +176,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -186,7 +186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.109", ] [[package]] @@ -509,7 +509,16 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "mp3-duration" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348bdc7300502f0801e5b57c448815713cd843b744ef9bda252a2698fdf90a0f" +dependencies = [ + "thiserror", ] [[package]] @@ -554,7 +563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7288eac8b54af7913c60e0eb0e2a7683020dffa342ab3fd15e28f035ba897cf" dependencies = [ "quote", - "syn", + "syn 1.0.109", "syn-mid", ] @@ -665,9 +674,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -683,9 +692,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -780,6 +789,7 @@ dependencies = [ "md5", "metaflac", "mockall", + "mp3-duration", "mp4ameta", "neon", "uuid", @@ -844,6 +854,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn-mid" version = "0.5.3" @@ -852,7 +873,7 @@ checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -861,6 +882,26 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "tiff" version = "0.9.0" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index db4cec305b..5a9a239130 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -18,6 +18,7 @@ lewton = "0.10.2" md5 = "0.7.0" metaflac = "0.2.5" mockall = "0.11.4" +mp3-duration = "0.1.10" mp4ameta = "0.11.0" [dependencies.image] diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index 22825c2d58..83222fe0c7 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -62,7 +62,12 @@ impl MetadataExtractor for Mp3MetadataExtractor { metadata.artist = tag.artist().map(|s| s.to_string()); metadata.title = tag.title().map(|s| s.to_string()); metadata.album = tag.album().map(|s| s.to_string()); - metadata.duration = tag.duration(); + let duration = mp3_duration::from_path(&path).map(|duration| duration.as_secs() as u32); + + match duration { + Ok(duration) => metadata.duration = Some(duration), + Err(_) => metadata.duration = None, + } metadata.position = tag.track(); metadata.disc = tag.disc(); metadata.year = tag.year().map(|s| s as u32); From 607f5fed452cf5b33210128ac46870e68962bd79 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 1 Dec 2023 01:54:07 +0100 Subject: [PATCH 17/27] Ogg metadata --- Cargo.lock | 224 ++++++++++++++++++++++++++----- packages/scanner/Cargo.toml | 5 +- packages/scanner/src/metadata.rs | 126 ++++++++++------- packages/scanner/src/scanner.rs | 3 +- 4 files changed, 275 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1712f335f8..a1f318d9ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "autocfg" version = "1.1.0" @@ -207,6 +213,15 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "exr" version = "1.71.0" @@ -390,17 +405,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" -[[package]] -name = "lewton" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030" -dependencies = [ - "byteorder", - "ogg", - "tinyvec", -] - [[package]] name = "libc" version = "0.2.146" @@ -614,15 +618,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "ogg" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e" -dependencies = [ - "byteorder", -] - [[package]] name = "png" version = "0.17.10" @@ -785,13 +780,13 @@ dependencies = [ "derive_builder", "id3", "image", - "lewton", "md5", "metaflac", "mockall", "mp3-duration", "mp4ameta", "neon", + "symphonia", "uuid", ] @@ -843,6 +838,178 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "symphonia" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e48dba70095f265fdb269b99619b95d04c89e619538138383e63310b14d941" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-wav", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f23b0482a7cb18fcdf9981ab0b78df800ef0080187d294650023c462439058d" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f31d7fece546f1e6973011a9eceae948133bbd18fd3d52f6073b1e38ae6368a" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bdd75b25ce4b84b12a4bd20bfea2460c2dbd7fc1d227ef5533504d3168109d" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870e7dc1865d818c7b6318879d060553a73a3b2a3b8443dff90910f10ac41150" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47f1fbd220a06a641c8ce2ddad10f5ef6ee5cc0c54d9044d25d43b0d3119deaa" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3953397e3506aa01350c4205817e4f95b58d476877a42f0458d07b665749e203" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c73eb88fee79705268cc7b742c7bc93a7b76e092ab751d0833866970754142" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf14bae5cf352032416bc64151e5d6242d29d33cbf3238513b44d4427a1efb" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c61dfc851ad25d4043d8c231d8617e8f7cd02a6cc0edad21ade21848d58895" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bf1a00ccd11452d44048a0368828040f778ae650418dbd9d8765b7ee2574c8d" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-wav" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da76614728fa27c003bdcdfbac51396bd8fcbf94c95fe8e62f1d2bac58ef03a4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c3e1937e31d0e068bbe829f66b2f2bfaa28d056365279e0ef897172c3320c0" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a450ca645b80d69aff8b35576cbfdc7f20940b29998202aab910045714c951f8" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" @@ -913,21 +1080,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "unicode-ident" version = "1.0.9" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 5a9a239130..025ea306c8 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -14,13 +14,16 @@ profiling = [] [dependencies] derive_builder = "0.12.0" id3 = "1.7.0" -lewton = "0.10.2" md5 = "0.7.0" metaflac = "0.2.5" mockall = "0.11.4" mp3-duration = "0.1.10" mp4ameta = "0.11.0" +[dependencies.symphonia] +version = "0.5.3" +features = ["aac", "mp3", "flac", "ogg", "isomp4", "wav"] + [dependencies.image] version = "0.24.7" features = ["webp-encoder"] diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index 83222fe0c7..438ae09383 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,9 +1,14 @@ -use std::fs::File; +use std::{fs::File, io::BufReader}; use derive_builder::Builder; use id3::TagLike; -use lewton::inside_ogg::OggStreamReader; use metaflac; +use symphonia::core::{ + formats::FormatOptions, + io::MediaSourceStream, + meta::{MetadataOptions, MetadataRevision, StandardTagKey}, + probe::Hint, +}; use crate::{ error::MetadataError, @@ -135,49 +140,80 @@ impl MetadataExtractor for FlacMetadataExtractor { } } -// pub struct OggMetadataExtractor; - -// impl OggMetadataExtractor { -// fn extract_vorbis_comment(metadata: &mut AudioMetadata, comments: &[(String, String)]) { -// for (key, value) in comments { -// match key.as_str() { -// "ARTIST" | "PERFORMER" => metadata.artist = Some(value.clone()), -// "TITLE" => metadata.title = Some(value.clone()), -// "ALBUM" => metadata.album = Some(value.clone()), -// "TRACKNUMBER" => metadata.position = value.parse().ok(), -// "DISCNUMBER" => metadata.disc = value.parse().ok(), -// "DATE" => metadata.year = value.parse().ok(), -// _ => {} -// } -// } -// } -// } - -// impl MetadataExtractor for OggMetadataExtractor { -// fn extract_metadata( -// &self, -// path: &str, -// _thumbnails_dir: &str, // Thumbnail generation is not handled here -// ) -> Result { -// let file = File::open(path)?; -// let mut ogg_reader = OggStreamReader::new(file)?; - -// let mut metadata = AudioMetadata::new(); - -// // Check if there are Vorbis comments, assumes Vorbis codec -// if let Some(comments) = ogg_reader.comment_hdrs() { -// Self::extract_vorbis_comment(&mut metadata, &comments.user_comments); -// } - -// metadata.duration = ogg_reader.ident_hdr().map(|ident| { -// let total_samples = ogg_reader.len() as u32; -// let sample_rate = ident.audio_sample_rate; -// total_samples / sample_rate -// }); - -// Ok(metadata) -// } -// } +pub struct OggMetadataExtractor; + +impl OggMetadataExtractor {} + +impl MetadataExtractor for OggMetadataExtractor { + fn extract_metadata( + &self, + path: &str, + _thumbnails_dir: &str, // Thumbnail generation is not handled here + ) -> Result { + let file = File::open(path); + if file.is_err() { + return Err(MetadataError::new( + format!("Could not open file {}", path).as_str(), + )); + } + + let file = File::open(path); + + if let Err(_) = file { + return Err(MetadataError::new( + format!("Could not open file {}", path).as_str(), + )); + } + + let mss = MediaSourceStream::new(Box::new(file.unwrap()), Default::default()); + + let meta_opts: MetadataOptions = Default::default(); + let fmt_opts: FormatOptions = Default::default(); + + let mut hint = Hint::new(); + hint.with_extension("ogg"); + + let mut probed = symphonia::default::get_probe() + .format(&hint, mss, &fmt_opts, &meta_opts) + .expect("unsupported format"); + + if let Some(metadata_rev) = probed.format.metadata().current() { + let tags = metadata_rev.tags(); + } + + let mut metadata = AudioMetadata::new(); + + if let Some(mut meta) = probed.format.metadata().current() { + for tag in meta.tags().iter() { + if tag.is_known() { + match tag.std_key { + Some(StandardTagKey::TrackTitle) => { + metadata.title = Some(tag.value.to_string()); + } + Some(StandardTagKey::Artist) => { + metadata.artist = Some(tag.value.to_string()); + } + Some(StandardTagKey::Album) => { + metadata.album = Some(tag.value.to_string()); + } + Some(StandardTagKey::TrackNumber) => { + metadata.position = Some(tag.value.to_string().parse::().unwrap()); + } + Some(StandardTagKey::DiscNumber) => { + metadata.disc = Some(tag.value.to_string().parse::().unwrap()); + } + Some(StandardTagKey::Date) => { + println!("Year: {:?}", tag.value); + metadata.year = Some(tag.value.to_string().parse::().unwrap()); + } + _ => {} + } + } + } + } + Ok(metadata) + } +} pub struct Mp4MetadataExtractor; diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index a10db4e88f..508562f381 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -8,7 +8,7 @@ use crate::error::{MetadataError, ScannerError}; use crate::local_track::LocalTrack; use crate::metadata::{ AudioMetadata, AudioMetadataBuilder, FlacMetadataExtractor, MetadataExtractor, - Mp3MetadataExtractor, Mp4MetadataExtractor, + Mp3MetadataExtractor, Mp4MetadataExtractor, OggMetadataExtractor, }; pub trait TagReader { @@ -22,6 +22,7 @@ fn get_extension(path: &str) -> Option<&str> { pub fn extractor_from_path(path: &str) -> Option> { match get_extension(path) { Some("mp3") => Some(Box::new(Mp3MetadataExtractor)), + Some("ogg") => Some(Box::new(OggMetadataExtractor)), Some("flac") => Some(Box::new(FlacMetadataExtractor)), Some("mp4") => Some(Box::new(Mp4MetadataExtractor)), Some("m4a") => Some(Box::new(Mp4MetadataExtractor)), From d1feab654b27a882625d8b8144f698d1a9bc2e12 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 4 Mar 2024 00:33:53 +0100 Subject: [PATCH 18/27] Decode duration from ogg files --- packages/scanner/src/js.rs | 2 +- packages/scanner/src/metadata.rs | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/scanner/src/js.rs b/packages/scanner/src/js.rs index ef3918ee1f..3ad62539d5 100644 --- a/packages/scanner/src/js.rs +++ b/packages/scanner/src/js.rs @@ -69,5 +69,5 @@ pub fn set_properties_from_metadata( set_optional_field_str(cx, obj, "thumbnail", metadata.thumbnail.clone()); set_optional_field_u32(cx, obj, "position", metadata.position); set_optional_field_u32(cx, obj, "disc", metadata.disc); - set_optional_field_u32(cx, obj, "year", metadata.year); + set_optional_field_str(cx, obj, "year", metadata.year.clone()); } diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index 438ae09383..082fc1fa78 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -4,6 +4,7 @@ use derive_builder::Builder; use id3::TagLike; use metaflac; use symphonia::core::{ + codecs::DecoderOptions, formats::FormatOptions, io::MediaSourceStream, meta::{MetadataOptions, MetadataRevision, StandardTagKey}, @@ -25,7 +26,7 @@ pub struct AudioMetadata { pub duration: Option, pub disc: Option, pub position: Option, - pub year: Option, + pub year: Option, pub thumbnail: Option, } @@ -75,7 +76,7 @@ impl MetadataExtractor for Mp3MetadataExtractor { } metadata.position = tag.track(); metadata.disc = tag.disc(); - metadata.year = tag.year().map(|s| s as u32); + metadata.year = tag.year().map(|y| y.to_string()); metadata.thumbnail = Mp3ThumbnailGenerator::generate_thumbnail( &path, @@ -176,16 +177,21 @@ impl MetadataExtractor for OggMetadataExtractor { let mut probed = symphonia::default::get_probe() .format(&hint, mss, &fmt_opts, &meta_opts) .expect("unsupported format"); + let mut metadata = AudioMetadata::new(); - if let Some(metadata_rev) = probed.format.metadata().current() { - let tags = metadata_rev.tags(); - } + let track = probed.format.default_track().unwrap(); + let time_base = track.codec_params.time_base.unwrap(); + let duration = track + .codec_params + .n_frames + .map(|frames| track.codec_params.start_ts + frames); - let mut metadata = AudioMetadata::new(); + metadata.duration = duration.map(|d| ((d as u32) * time_base.numer / time_base.denom)); - if let Some(mut meta) = probed.format.metadata().current() { + if let Some(meta) = probed.format.metadata().current() { for tag in meta.tags().iter() { if tag.is_known() { + println!("{:?}", tag.std_key); match tag.std_key { Some(StandardTagKey::TrackTitle) => { metadata.title = Some(tag.value.to_string()); @@ -203,8 +209,7 @@ impl MetadataExtractor for OggMetadataExtractor { metadata.disc = Some(tag.value.to_string().parse::().unwrap()); } Some(StandardTagKey::Date) => { - println!("Year: {:?}", tag.value); - metadata.year = Some(tag.value.to_string().parse::().unwrap()); + metadata.year = Some(tag.value.to_string()); } _ => {} } From b0136a938f339dff1fe91066ae8180d8ee658674 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 4 Mar 2024 01:35:40 +0100 Subject: [PATCH 19/27] Disable unsupported m4a format --- package.json | 4 ---- packages/scanner/src/scanner.rs | 1 - 2 files changed, 5 deletions(-) diff --git a/package.json b/package.json index 21ad36e460..ef4f9d5cc1 100644 --- a/package.json +++ b/package.json @@ -129,10 +129,6 @@ "ext": "wav", "mimeType": "audio/x-wav" }, - { - "ext": "m4a", - "mimeType": "audio/m4a" - }, { "ext": "weba", "mimeType": "audio/weba" diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index 508562f381..f37daadf30 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -25,7 +25,6 @@ pub fn extractor_from_path(path: &str) -> Option> { Some("ogg") => Some(Box::new(OggMetadataExtractor)), Some("flac") => Some(Box::new(FlacMetadataExtractor)), Some("mp4") => Some(Box::new(Mp4MetadataExtractor)), - Some("m4a") => Some(Box::new(Mp4MetadataExtractor)), _ => None, } } From ae45d93bf149024bbbd579f885bcc5bad3e05ed0 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Mon, 4 Mar 2024 01:41:06 +0100 Subject: [PATCH 20/27] Return any image data when generating mp3 covers --- packages/scanner/src/thumbnails.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/scanner/src/thumbnails.rs b/packages/scanner/src/thumbnails.rs index c752cb1646..999d1d7e17 100644 --- a/packages/scanner/src/thumbnails.rs +++ b/packages/scanner/src/thumbnails.rs @@ -31,9 +31,7 @@ impl ThumbnailGenerator for Mp3ThumbnailGenerator { let tag = Tag::read_from_path(filename).ok()?; let mut pictures = tag.pictures(); - pictures - .find(|p| p.picture_type == id3::frame::PictureType::CoverFront) - .map(|p| p.data.clone()) + pictures.next().map(|p| p.data.clone()) } } From c61a4cc6ca27491d8ba1d01c9c257e95a3a329c6 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Wed, 6 Mar 2024 01:35:53 +0100 Subject: [PATCH 21/27] Deprecate no longer supported formats --- package.json | 16 ---------------- packages/scanner/src/metadata.rs | 6 ++---- packages/scanner/src/scanner.rs | 15 ++++++++------- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index ef4f9d5cc1..a124eae70b 100644 --- a/package.json +++ b/package.json @@ -117,29 +117,13 @@ "ext": "opus", "mimeType": "audio/ogg" }, - { - "ext": "aac", - "mimeType": "audio/aac" - }, { "ext": "flac", "mimeType": "audio/flac" }, - { - "ext": "wav", - "mimeType": "audio/x-wav" - }, - { - "ext": "weba", - "mimeType": "audio/weba" - }, { "ext": "mp4", "mimeType": "audio/mp4" - }, - { - "ext": "webm", - "mimeType": "audio/webm" } ], "linux": { diff --git a/packages/scanner/src/metadata.rs b/packages/scanner/src/metadata.rs index 082fc1fa78..697027c24f 100644 --- a/packages/scanner/src/metadata.rs +++ b/packages/scanner/src/metadata.rs @@ -1,13 +1,12 @@ -use std::{fs::File, io::BufReader}; +use std::fs::File; use derive_builder::Builder; use id3::TagLike; use metaflac; use symphonia::core::{ - codecs::DecoderOptions, formats::FormatOptions, io::MediaSourceStream, - meta::{MetadataOptions, MetadataRevision, StandardTagKey}, + meta::{MetadataOptions, StandardTagKey}, probe::Hint, }; @@ -191,7 +190,6 @@ impl MetadataExtractor for OggMetadataExtractor { if let Some(meta) = probed.format.metadata().current() { for tag in meta.tags().iter() { if tag.is_known() { - println!("{:?}", tag.std_key); match tag.std_key { Some(StandardTagKey::TrackTitle) => { metadata.title = Some(tag.value.to_string()); diff --git a/packages/scanner/src/scanner.rs b/packages/scanner/src/scanner.rs index f37daadf30..c90193920a 100644 --- a/packages/scanner/src/scanner.rs +++ b/packages/scanner/src/scanner.rs @@ -1,5 +1,5 @@ use id3::{Error, Tag}; -use std::collections::{HashSet, LinkedList}; +use std::collections::LinkedList; use std::ffi::OsStr; use std::path::Path; use uuid::Uuid; @@ -7,8 +7,8 @@ use uuid::Uuid; use crate::error::{MetadataError, ScannerError}; use crate::local_track::LocalTrack; use crate::metadata::{ - AudioMetadata, AudioMetadataBuilder, FlacMetadataExtractor, MetadataExtractor, - Mp3MetadataExtractor, Mp4MetadataExtractor, OggMetadataExtractor, + AudioMetadata, FlacMetadataExtractor, MetadataExtractor, Mp3MetadataExtractor, + Mp4MetadataExtractor, OggMetadataExtractor, }; pub trait TagReader { @@ -80,6 +80,8 @@ pub fn visit_directory( #[cfg(test)] mod tests { + use crate::metadata::AudioMetadataBuilder; + use super::*; #[derive(Debug, Clone, Default)] @@ -112,7 +114,7 @@ mod tests { .duration(10) .position(1) .disc(1) - .year(2020) + .year("2020".to_string()) .thumbnail("http://localhost:8080/thumbnails/0b/0b0b0b0b0b0b0b0b.webp".to_string()) .build() .unwrap(), @@ -122,8 +124,7 @@ mod tests { #[test] fn test_visit_file() { let path = "tests/test.mp3".to_string(); - let thumbnails_dir = "tests/thumbnails".to_string(); - let mut created_thumbnails_hashset: HashSet = HashSet::new(); + let thumbnails_dir: String = "tests/thumbnails".to_string(); let local_track = visit_file(path, test_extractor_from_path, &thumbnails_dir).unwrap(); assert_eq!(local_track.filename, "test.mp3"); assert_eq!(local_track.metadata.artist, Some("Test Artist".to_string())); @@ -132,7 +133,7 @@ mod tests { assert_eq!(local_track.metadata.duration, Some(10)); assert_eq!(local_track.metadata.position, Some(1)); assert_eq!(local_track.metadata.disc, Some(1)); - assert_eq!(local_track.metadata.year, Some(2020)); + assert_eq!(local_track.metadata.year, Some("2020".to_string())); assert_eq!( local_track.metadata.thumbnail, Some("http://localhost:8080/thumbnails/0b/0b0b0b0b0b0b0b0b.webp".to_string()) From a26a38272f27a700028ea266cfe545e86dc1f16c Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 8 Mar 2024 02:41:48 +0100 Subject: [PATCH 22/27] Add UUID to queue items and fix sorting criteria --- packages/app/app/actions/queue.ts | 2 ++ .../app/app/components/LibraryView/index.js | 17 ++++++++++++++++- .../app/app/components/LibraryView/utils.js | 4 ++-- packages/app/app/components/PlayQueue/index.tsx | 2 +- .../ui/lib/components/AlbumPreview/index.tsx | 4 ++-- packages/ui/lib/types/index.ts | 2 +- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/app/app/actions/queue.ts b/packages/app/app/actions/queue.ts index f6db772737..fb7bb68bd1 100644 --- a/packages/app/app/actions/queue.ts +++ b/packages/app/app/actions/queue.ts @@ -1,6 +1,7 @@ import logger from 'electron-timber'; import _, { isEmpty, isString } from 'lodash'; import { createStandardAction } from 'typesafe-actions'; +import { v4 } from 'uuid'; import { rest, StreamProvider } from '@nuclear/core'; import { getTrackArtist } from '@nuclear/ui'; @@ -37,6 +38,7 @@ const localTrackToQueueItem = (track: LocalTrack, local: LocalLibraryState): Que return toQueueItem({ ...rest, + uuid: v4(), streams: [resolvedStream] }); }; diff --git a/packages/app/app/components/LibraryView/index.js b/packages/app/app/components/LibraryView/index.js index 90698207d4..3d1c87e796 100644 --- a/packages/app/app/components/LibraryView/index.js +++ b/packages/app/app/components/LibraryView/index.js @@ -15,7 +15,22 @@ import LibraryAlbumGrid from './LibraryAlbumGrid'; import LibraryHeader from './LibraryHeader'; import { sortTracks } from './utils'; -const LibraryView = ({ tracksMap, filter, expandedFolders, streamProviders, pending, scanProgress, scanTotal, localFolders, sortBy, direction, listType, actions, queueActions, playerActions }) => { +const LibraryView = ({ + tracksMap, + filter, + expandedFolders, + streamProviders, + pending, + scanProgress, + scanTotal, + localFolders, + sortBy, + direction, + listType, + actions, + queueActions, + playerActions +}) => { const localStreamProviders = useMemo(() => _.filter(streamProviders, { sourceName: 'Local' }), [streamProviders]); const unfilteredTracks = useMemo(() => _.values(tracksMap), [tracksMap]); diff --git a/packages/app/app/components/LibraryView/utils.js b/packages/app/app/components/LibraryView/utils.js index 7919ac4483..a2661fb1b3 100644 --- a/packages/app/app/components/LibraryView/utils.js +++ b/packages/app/app/components/LibraryView/utils.js @@ -3,11 +3,11 @@ import _ from 'lodash'; export function sortTracks(tracks, criteria) { switch (criteria) { case 'artist': - return _.sortBy(tracks, ['artist.name', 'album', 'pos']); + return _.sortBy(tracks, ['artist.name', 'album', 'position']); case 'name': return _.sortBy(tracks, ['name', 'artist.name']); case 'album': - return _.sortBy(tracks, ['album', 'pos']); + return _.sortBy(tracks, ['album', 'position']); default: return tracks; } diff --git a/packages/app/app/components/PlayQueue/index.tsx b/packages/app/app/components/PlayQueue/index.tsx index 3ed4f5af0d..ff81c090a4 100644 --- a/packages/app/app/components/PlayQueue/index.tsx +++ b/packages/app/app/components/PlayQueue/index.tsx @@ -232,7 +232,7 @@ const PlayQueue: React.FC = ({ {({ height, width }) => = (props) => { } key={index} thumb={thumb} - title={track.name} - artist={_.get(album, 'artist.name')} + title={track.title} + artist={album.artist} > Date: Fri, 8 Mar 2024 02:55:35 +0100 Subject: [PATCH 23/27] Update local library tests to use a uuid regex --- .../FavoritesContainer/FavoritesContainer.tracks.test.tsx | 4 ++-- .../LibraryViewContainer/LibraryViewContainer.test.tsx | 6 +++--- packages/app/test/testUtils.tsx | 2 ++ packages/core/src/rest/Nuclear/Configuration.ts | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/app/app/containers/FavoritesContainer/FavoritesContainer.tracks.test.tsx b/packages/app/app/containers/FavoritesContainer/FavoritesContainer.tracks.test.tsx index 9863a94b2c..d6826f6007 100644 --- a/packages/app/app/containers/FavoritesContainer/FavoritesContainer.tracks.test.tsx +++ b/packages/app/app/containers/FavoritesContainer/FavoritesContainer.tracks.test.tsx @@ -5,7 +5,7 @@ import { createMemoryHistory } from 'history'; import _, { pick } from 'lodash'; import { buildStoreState } from '../../../test/storeBuilders'; -import { AnyProps, configureMockStore, setupI18Next, TestRouterProvider, TestStoreProvider } from '../../../test/testUtils'; +import { AnyProps, configureMockStore, setupI18Next, TestRouterProvider, TestStoreProvider, uuidRegex } from '../../../test/testUtils'; import MainContentContainer from '../MainContentContainer'; import PlayerBarContainer from '../PlayerBarContainer'; import { PlaybackStatus } from '@nuclear/core'; @@ -250,7 +250,7 @@ describe('Favorite tracks view container', () => { await waitFor(() => component.getAllByTestId('play-now')[0].click()); const state = store.getState(); expect(state.queue.queueItems[0]).toEqual(expect.objectContaining({ - uuid: 'local-track-1', + uuid: expect.stringMatching(uuidRegex), artist: 'test artist 1', name: 'test track 1', local: true, diff --git a/packages/app/app/containers/LibraryViewContainer/LibraryViewContainer.test.tsx b/packages/app/app/containers/LibraryViewContainer/LibraryViewContainer.test.tsx index 1279ee27be..12f2f7e615 100644 --- a/packages/app/app/containers/LibraryViewContainer/LibraryViewContainer.test.tsx +++ b/packages/app/app/containers/LibraryViewContainer/LibraryViewContainer.test.tsx @@ -1,7 +1,7 @@ import { waitFor } from '@testing-library/react'; import { buildStoreState } from '../../../test/storeBuilders'; -import { mountedComponentFactory, setupI18Next } from '../../../test/testUtils'; +import { mountedComponentFactory, setupI18Next, uuidRegex } from '../../../test/testUtils'; describe('Library view container', () => { beforeAll(() => { @@ -66,7 +66,7 @@ describe('Library view container', () => { expect(state.queue.queueItems).toStrictEqual([ expect.objectContaining({ - uuid: 'local-track-1', + uuid: expect.stringMatching(uuidRegex), artist: 'local artist 1', name: 'local track 1', duration: 300, @@ -121,7 +121,7 @@ describe('Library view container', () => { expect(state.queue.queueItems).toStrictEqual([ expect.objectContaining({ - uuid: 'local-track-1', + uuid: expect.stringMatching(uuidRegex), artist: 'local artist 1', name: 'local track 1', duration: 250, diff --git a/packages/app/test/testUtils.tsx b/packages/app/test/testUtils.tsx index 210ca6bdbd..4a22fb68cb 100644 --- a/packages/app/test/testUtils.tsx +++ b/packages/app/test/testUtils.tsx @@ -24,6 +24,8 @@ export type AnyProps = { [k: string]: any; } +export const uuidRegex = /\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/; + type TestRouteProviderProps = { children: React.ReactNode; history: ReturnType; diff --git a/packages/core/src/rest/Nuclear/Configuration.ts b/packages/core/src/rest/Nuclear/Configuration.ts index a1ad33353d..ab951b6be0 100644 --- a/packages/core/src/rest/Nuclear/Configuration.ts +++ b/packages/core/src/rest/Nuclear/Configuration.ts @@ -42,7 +42,7 @@ export class NuclearConfigurationService extends NuclearSupabaseService { .from('params') .select(); - return dbParams?.data.reduce((acc, curr) => { + return dbParams?.data?.reduce((acc, curr) => { acc[curr.key] = curr.value; return acc; }, {} as Parameters); From 89bcaded14f5abe8a021554df3508787a795ee59 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 8 Mar 2024 03:02:27 +0100 Subject: [PATCH 24/27] Disable local library scanner test --- .../main/src/services/library-scanner/library-scanner.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/main/src/services/library-scanner/library-scanner.test.ts b/packages/main/src/services/library-scanner/library-scanner.test.ts index a803a3e732..64e4563e81 100644 --- a/packages/main/src/services/library-scanner/library-scanner.test.ts +++ b/packages/main/src/services/library-scanner/library-scanner.test.ts @@ -1,6 +1,7 @@ import {scanFolders} from '@nuclear/scanner'; -describe('Local library scanner', () => { +// Enable when testing the local library scanner rust package +xdescribe('Local library scanner', () => { it('scans folders', async () => { const result = await scanFolders([''], ['mp3'], (progress, total, lastScanned) => { // console.log({progress, total, lastScanned}); From 789c02b5016924e0dbe9d182443f7a48d315216c Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 8 Mar 2024 22:08:27 +0100 Subject: [PATCH 25/27] Commented out local library scanner test code --- .../src/services/library-scanner/library-scanner.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/main/src/services/library-scanner/library-scanner.test.ts b/packages/main/src/services/library-scanner/library-scanner.test.ts index 64e4563e81..8bbb445bda 100644 --- a/packages/main/src/services/library-scanner/library-scanner.test.ts +++ b/packages/main/src/services/library-scanner/library-scanner.test.ts @@ -3,9 +3,10 @@ import {scanFolders} from '@nuclear/scanner'; // Enable when testing the local library scanner rust package xdescribe('Local library scanner', () => { it('scans folders', async () => { - const result = await scanFolders([''], ['mp3'], (progress, total, lastScanned) => { - // console.log({progress, total, lastScanned}); - }); + // const result = await scanFolders([''], ['mp3'], (progress, total, lastScanned) => { + // // console.log({progress, total, lastScanned}); + // }); + const result = {}; expect(result).toBe({}); }); }); From 319e19cea5455696701b57611aee8b32e0d7d7e0 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 8 Mar 2024 22:42:08 +0100 Subject: [PATCH 26/27] Add setup tools installation for macOS --- .github/workflows/CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b9e22b84d3..cb3b719891 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,6 +47,10 @@ jobs: run: | echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "C:\Program Files\Git\mingw64\libexec\git-core" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: install setup tools + if: runner.os == 'macOS' + run: | + sudo -H pip install setuptools - run: npm ci - run: npm run lint - run: npm test From 9e8a22c74a2a65572098d986e11574ac33d96064 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 8 Mar 2024 22:45:16 +0100 Subject: [PATCH 27/27] Add setup tools installation for macOS --- .github/workflows/CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index cb3b719891..410d028ca0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -96,6 +96,10 @@ jobs: run: | echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "C:\Program Files\Git\mingw64\libexec\git-core" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: install setup tools + if: runner.os == 'macOS' + run: | + sudo -H pip install setuptools - run: npm ci - run: npm run ${{ matrix.platform.cmd }} shell: bash