diff --git a/.ameba.yml b/.ameba.yml index e61b3b3e..163c936a 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -7,3 +7,8 @@ Lint/UnusedArgument: - src/routes/* Metrics/CyclomaticComplexity: Enabled: false +Layout/LineLength: + Enabled: true + MaxLength: 80 + Excluded: + - src/routes/api.cr diff --git a/Makefile b/Makefile index d9485c50..6c5eebfa 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,6 @@ test: check: crystal tool format --check ./bin/ameba - ./dev/linewidth.sh arm32v7: crystal build src/mango.cr --release --progress --error-trace --cross-compile --target='arm-linux-gnueabihf' -o mango-arm32v7 diff --git a/README.md b/README.md index 9fe8ef68..bf57b7f7 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.19.1 + Mango - Manga Server and Web Reader. Version 0.20.0 Usage: @@ -82,7 +82,6 @@ library_path: ~/mango/library db_path: ~/mango/mango.db scan_interval_minutes: 5 thumbnail_generation_interval_hours: 24 -db_optimization_interval_hours: 24 log_level: info upload_path: ~/mango/uploads plugin_path: ~/mango/plugins diff --git a/dev/linewidth.sh b/dev/linewidth.sh deleted file mode 100755 index 6e83fbc3..00000000 --- a/dev/linewidth.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -[ ! -z "$(grep '.\{80\}' --exclude-dir=lib --include="*.cr" -nr --color=always . | grep -v "routes/api.cr" | tee /dev/tty)" ] \ - && echo "The above lines exceed the 80 characters limit" \ - || exit 0 diff --git a/migration/ids_signature.7.cr b/migration/ids_signature.7.cr new file mode 100644 index 00000000..48da8e7e --- /dev/null +++ b/migration/ids_signature.7.cr @@ -0,0 +1,50 @@ +class IDSignature < MG::Base + def up : String + <<-SQL + ALTER TABLE ids ADD COLUMN signature TEXT; + SQL + end + + def down : String + <<-SQL + -- remove signature column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL + ); + + INSERT INTO ids + SELECT path, id + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + SQL + end +end diff --git a/migration/relative_path.8.cr b/migration/relative_path.8.cr new file mode 100644 index 00000000..ec58ee6b --- /dev/null +++ b/migration/relative_path.8.cr @@ -0,0 +1,33 @@ +class RelativePath < MG::Base + def up : String + base = Config.current.library_path + # Escape single quotes in case the path contains them, and remove the + # trailing slash (this is a mistake, fixed in DB version 10) + base = base.gsub("'", "''").rstrip "/" + + <<-SQL + -- update the path column in ids to relative paths + UPDATE ids + SET path = REPLACE(path, '#{base}', ''); + + -- update the path column in titles to relative paths + UPDATE titles + SET path = REPLACE(path, '#{base}', ''); + SQL + end + + def down : String + base = Config.current.library_path + base = base.gsub("'", "''").rstrip "/" + + <<-SQL + -- update the path column in ids to absolute paths + UPDATE ids + SET path = '#{base}' || path; + + -- update the path column in titles to absolute paths + UPDATE titles + SET path = '#{base}' || path; + SQL + end +end diff --git a/migration/relative_path_fix.10.cr b/migration/relative_path_fix.10.cr new file mode 100644 index 00000000..26df8766 --- /dev/null +++ b/migration/relative_path_fix.10.cr @@ -0,0 +1,31 @@ +# In DB version 8, we replaced the absolute paths in DB with relative paths, +# but we mistakenly left the starting slashes. This migration removes them. +class RelativePathFix < MG::Base + def up : String + <<-SQL + -- remove leading slashes from the paths in ids + UPDATE ids + SET path = SUBSTR(path, 2, LENGTH(path) - 1) + WHERE path LIKE '/%'; + + -- remove leading slashes from the paths in titles + UPDATE titles + SET path = SUBSTR(path, 2, LENGTH(path) - 1) + WHERE path LIKE '/%'; + SQL + end + + def down : String + <<-SQL + -- add leading slashes to paths in ids + UPDATE ids + SET path = '/' || path + WHERE path NOT LIKE '/%'; + + -- add leading slashes to paths in titles + UPDATE titles + SET path = '/' || path + WHERE path NOT LIKE '/%'; + SQL + end +end diff --git a/migration/unavailable.9.cr b/migration/unavailable.9.cr new file mode 100644 index 00000000..172a2f57 --- /dev/null +++ b/migration/unavailable.9.cr @@ -0,0 +1,94 @@ +class UnavailableIDs < MG::Base + def up : String + <<-SQL + -- add unavailable column to ids + ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + + -- add unavailable column to titles + ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + SQL + end + + def down : String + <<-SQL + -- remove unavailable column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO ids + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + + -- remove unavailable column from titles + ALTER TABLE titles RENAME TO tmp; + + CREATE TABLE titles ( + id TEXT NOT NULL, + path TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO titles + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX titles_id_idx on titles (id); + CREATE UNIQUE INDEX titles_path_idx on titles (path); + + -- recreate the foreign key constraint on tags + ALTER TABLE tags RENAME TO tmp; + + CREATE TABLE tags ( + id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE (id, tag), + FOREIGN KEY (id) REFERENCES titles (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO tags + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE INDEX tags_id_idx ON tags (id); + CREATE INDEX tags_tag_idx ON tags (tag); + SQL + end +end diff --git a/public/css/mango.less b/public/css/mango.less index f6e89517..1ac84df9 100644 --- a/public/css/mango.less +++ b/public/css/mango.less @@ -66,7 +66,6 @@ // Dark theme .uk-light { - .uk-navbar-dropdown, .uk-modal-header, .uk-modal-body, .uk-modal-footer { @@ -75,6 +74,7 @@ .uk-navbar-dropdown, .uk-dropdown { color: #ccc; + background: #333; } .uk-nav-header, .uk-description-list > dt { diff --git a/public/js/download.js b/public/js/download.js index aca45eb9..b8d76fe6 100644 --- a/public/js/download.js +++ b/public/js/download.js @@ -156,8 +156,8 @@ const search = () => { langs.unshift('All'); group_names.unshift('All'); - $('select#lang-select').append(langs.map(e => ``).join('')); - $('select#group-select').append(group_names.map(e => ``).join('')); + $('select#lang-select').html(langs.map(e => ``).join('')); + $('select#group-select').html(group_names.map(e => ``).join('')); $('#filter-form').removeAttr('hidden'); @@ -241,7 +241,7 @@ const buildTable = () => { if (v === 'All') return; if (k === 'group') { chapters = chapters.filter(c => { - unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g)); + const unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g)); return unescaped_groups.indexOf(v) >= 0; }); return; diff --git a/public/js/missing-items.js b/public/js/missing-items.js new file mode 100644 index 00000000..0babb24a --- /dev/null +++ b/public/js/missing-items.js @@ -0,0 +1,60 @@ +const component = () => { + return { + empty: true, + titles: [], + entries: [], + loading: true, + + load() { + this.loading = true; + this.request('GET', `${base_url}api/admin/titles/missing`, data => { + this.titles = data.titles; + this.request('GET', `${base_url}api/admin/entries/missing`, data => { + this.entries = data.entries; + this.loading = false; + this.empty = this.entries.length === 0 && this.titles.length === 0; + }); + }); + }, + rm(event) { + const rawID = event.currentTarget.closest('tr').id; + const [type, id] = rawID.split('-'); + const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`; + this.request('DELETE', url, () => { + this.load(); + }); + }, + rmAll() { + UIkit.modal.confirm('Are you sure? All metadata associated with these items, including their tags and thumbnails, will be deleted from the database.', { + labels: { + ok: 'Yes, delete them', + cancel: 'Cancel' + } + }).then(() => { + this.request('DELETE', `${base_url}api/admin/titles/missing`, () => { + this.request('DELETE', `${base_url}api/admin/entries/missing`, () => { + this.load(); + }); + }); + }); + }, + request(method, url, cb) { + console.log(url); + $.ajax({ + type: method, + url: url, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`); + return; + } + if (cb) cb(data); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + } + }; +}; diff --git a/shard.yml b/shard.yml index a999ab0f..d7505a5a 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.19.1 +version: 0.20.0 authors: - Alex Ling diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 94c326c7..3ee4aac3 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -35,6 +35,23 @@ describe "compare_numerically" do end end +describe "is_supported_file" do + it "returns true when the filename has a supported extension" do + filename = "manga.cbz" + is_supported_file(filename).should eq true + end + + it "returns true when the filename does not have a supported extension" do + filename = "info.json" + is_supported_file(filename).should eq false + end + + it "is case insensitive" do + filename = "manga.ZiP" + is_supported_file(filename).should eq true + end +end + describe "chapter_sort" do it "sorts correctly" do ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"] diff --git a/src/config.cr b/src/config.cr index 845fc940..3ac5af28 100644 --- a/src/config.cr +++ b/src/config.cr @@ -13,7 +13,6 @@ class Config property db_path : String = File.expand_path "~/mango/mango.db", home: true property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property db_optimization_interval_hours : Int32 = 24 property log_level : String = "info" property upload_path : String = File.expand_path "~/mango/uploads", home: true diff --git a/src/library/entry.cr b/src/library/entry.cr index 3e270cf4..7a4e7533 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -11,13 +11,13 @@ class Entry @title = File.basename @zip_path, File.extname @zip_path @encoded_title = URI.encode @title @size = (File.size @zip_path).humanize_bytes - id = storage.get_id @zip_path, false + id = storage.get_entry_id @zip_path, File.signature(@zip_path) if id.nil? id = random_str - storage.insert_id({ - path: @zip_path, - id: id, - is_title: false, + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, }) end @id = id diff --git a/src/library/library.cr b/src/library/library.cr index a3db6401..83c0b0a0 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,16 +42,6 @@ class Library end end end - - db_interval = Config.current.db_optimization_interval_hours - unless db_interval < 1 - spawn do - loop do - Storage.default.optimize - sleep db_interval.hours - end - end - end end def titles @@ -119,6 +109,7 @@ class Library storage.close Logger.debug "Scan completed" + Storage.default.mark_unavailable end def get_continue_reading_entries(username) @@ -241,7 +232,7 @@ class Library e.generate_thumbnail # Sleep after each generation to minimize the impact on disk IO # and CPU - sleep 0.5.seconds + sleep 1.seconds end @thumbnails_count += 1 end diff --git a/src/library/title.cr b/src/library/title.cr index 5fef9912..4c439e72 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -3,19 +3,20 @@ require "../archive" class Title getter dir : String, parent_id : String, title_ids : Array(String), entries : Array(Entry), title : String, id : String, - encoded_title : String, mtime : Time + encoded_title : String, mtime : Time, signature : UInt64 @entry_display_name_cache : Hash(String, String)? def initialize(@dir : String, @parent_id) storage = Storage.default - id = storage.get_id @dir, true + @signature = Dir.signature dir + id = storage.get_title_id dir, signature if id.nil? id = random_str - storage.insert_id({ - path: @dir, - id: id, - is_title: true, + storage.insert_title_id({ + path: dir, + id: id, + signature: signature.to_s, }) end @id = id @@ -35,7 +36,7 @@ class Title @title_ids << title.id next end - if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path + if is_supported_file path entry = Entry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end @@ -61,6 +62,7 @@ class Title {% for str in ["dir", "title", "id"] %} json.field {{str}}, @{{str.id}} {% end %} + json.field "signature" { json.number @signature } json.field "display_name", display_name json.field "cover_url", cover_url json.field "mtime" { json.number @mtime.to_unix } diff --git a/src/mango.cr b/src/mango.cr index 138c5a9f..fcade44a 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -8,7 +8,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.19.1" +MANGO_VERSION = "0.20.0" # From http://www.network-science.de/ascii/ BANNER = %{ @@ -63,7 +63,12 @@ class CLI < Clim Plugin::Downloader.default spawn do - Server.new.start + begin + Server.new.start + rescue e + Logger.fatal e + Process.exit 1 + end end MainFiber.start_and_block diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 4445b0e4..fd63ec82 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -1,6 +1,9 @@ struct AdminRouter def initialize get "/admin" do |env| + storage = Storage.default + missing_count = storage.missing_titles.size + + storage.missing_entries.size layout "admin" end @@ -66,5 +69,9 @@ struct AdminRouter mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end + + get "/admin/missing" do |env| + layout "missing-items" + end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d60710b..7fc8b46a 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -166,6 +166,21 @@ struct APIRouter "error" => "string?", } + Koa.object "missing", { + "path" => "string", + "id" => "string", + "signature" => "string", + } + + Koa.array "missingAry", "$missing" + + Koa.object "missingResult", { + "success" => "boolean", + "error" => "string?", + "entries" => "$missingAry?", + "titles" => "$missingAry?", + } + Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" @@ -777,6 +792,120 @@ struct APIRouter end end + Koa.describe "Lists all missing titles" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/titles/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "titles" => Storage.default.missing_titles, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Lists all missing entries" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/entries/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "entries" => Storage.default.missing_entries, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes all missing titles" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/titles/missing" do |env| + begin + Storage.default.delete_missing_title + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes all missing entries" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/entries/missing" do |env| + begin + Storage.default.delete_missing_entry + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing title identified by `tid`", <<-MD + Does nothing if the given `tid` is not found or if the title is not missing. + MD + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/titles/missing/:tid" do |env| + begin + tid = env.params.url["tid"] + Storage.default.delete_missing_title tid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing entry identified by `eid`", <<-MD + Does nothing if the given `eid` is not found or if the entry is not missing. + MD + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/entries/missing/:eid" do |env| + begin + eid = env.params.url["eid"] + Storage.default.delete_missing_entry eid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/routes/main.cr b/src/routes/main.cr index 70b13349..65048005 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -154,6 +154,7 @@ struct MainRouter end get "/api" do |env| + base_url = Config.current.base_url render "src/views/api.html.ecr" end end diff --git a/src/storage.cr b/src/storage.cr index dcc337c1..2bc6daa1 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -15,14 +15,16 @@ def verify_password(hash, pw) end class Storage - @@insert_ids = [] of IDTuple + @@insert_entry_ids = [] of IDTuple + @@insert_title_ids = [] of IDTuple @path : String @db : DB::Database? - alias IDTuple = NamedTuple(path: String, + alias IDTuple = NamedTuple( + path: String, id: String, - is_title: Bool) + signature: String?) use_default @@ -230,24 +232,96 @@ class Storage end end - def get_id(path, is_title) + def get_title_id(path, signature) id = nil + path = Path.new(path).relative_to(Config.current.library_path).to_s MainFiber.run do get_db do |db| - if is_title - id = db.query_one? "select id from titles where path = (?)", path, - as: String - else - id = db.query_one? "select id from ids where path = (?)", path, - as: String + # First attempt to find the matching title in DB using BOTH path + # and signature + id = db.query_one? "select id from titles where path = (?) and " \ + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String + + should_update = id.nil? + # If it fails, try to match using the path only. This could happen + # for example when a new entry is added to the title + id ||= db.query_one? "select id from titles where path = (?)", path, + as: String + + # If it still fails, we will have to rely on the signature values. + # This could happen when the user moved or renamed the title, or + # a title containing the title + unless id + # If there are multiple rows with the same signature (this could + # happen simply by bad luck, or when the user copied a title), + # pick the row that has the most similar path to the give path + rows = [] of Tuple(String, String) + db.query "select id, path from titles where signature = (?)", + signature.to_s do |rs| + rs.each do + rows << {rs.read(String), rs.read(String)} + end + end + row = rows.max_by?(&.[1].components_similarity(path)) + id = row[0] if row + end + + # At this point, `id` would still be nil if there's no row matching + # either the path or the signature + + # If we did identify a matching title, save the path and signature + # values back to the DB + if id && should_update + db.exec "update titles set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id end end end id end - def insert_id(tp : IDTuple) - @@insert_ids << tp + # See the comments in `#get_title_id` to see how this method works. + def get_entry_id(path, signature) + id = nil + path = Path.new(path).relative_to(Config.current.library_path).to_s + MainFiber.run do + get_db do |db| + id = db.query_one? "select id from ids where path = (?) and " \ + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String + + should_update = id.nil? + id ||= db.query_one? "select id from ids where path = (?)", path, + as: String + + unless id + rows = [] of Tuple(String, String) + db.query "select id, path from ids where signature = (?)", + signature.to_s do |rs| + rs.each do + rows << {rs.read(String), rs.read(String)} + end + end + row = rows.max_by?(&.[1].components_similarity(path)) + id = row[0] if row + end + + if id && should_update + db.exec "update ids set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id + end + end + end + id + end + + def insert_entry_id(tp) + @@insert_entry_ids << tp + end + + def insert_title_id(tp) + @@insert_title_ids << tp end def bulk_insert_ids @@ -255,17 +329,24 @@ class Storage get_db do |db| db.transaction do |tran| conn = tran.connection - @@insert_ids.each do |tp| - if tp[:is_title] - conn.exec "insert into titles values (?, ?, null)", tp[:id], - tp[:path] - else - conn.exec "insert into ids values (?, ?)", tp[:path], tp[:id] - end + @@insert_title_ids.each do |tp| + path = Path.new(tp[:path]) + .relative_to(Config.current.library_path).to_s + conn.exec "insert into titles (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s + end + @@insert_entry_ids.each do |tp| + path = Path.new(tp[:path]) + .relative_to(Config.current.library_path).to_s + conn.exec "insert into ids (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s end end end - @@insert_ids.clear + @@insert_entry_ids.clear + @@insert_title_ids.clear end end @@ -322,7 +403,8 @@ class Storage tags = [] of String MainFiber.run do get_db do |db| - db.query "select distinct tag from tags" do |rs| + db.query "select distinct tag from tags natural join titles " \ + "where unavailable = 0" do |rs| rs.each do tags << rs.read String end @@ -354,42 +436,92 @@ class Storage end end - def optimize + def mark_unavailable MainFiber.run do - Logger.info "Starting DB optimization" get_db do |db| - # Delete dangling entry IDs + # Detect dangling entry IDs trash_ids = [] of String - db.query "select path, id from ids" do |rs| + db.query "select path, id from ids where unavailable = 0" do |rs| rs.each do path = rs.read String - trash_ids << rs.read String unless File.exists? path + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_ids << rs.read String unless File.exists? fullpath end end - db.exec "delete from ids where id in " \ + unless trash_ids.empty? + Logger.debug "Marking #{trash_ids.size} entries as unavailable" + end + db.exec "update ids set unavailable = 1 where id in " \ "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \ - if trash_ids.size > 0 - # Delete dangling title IDs + # Detect dangling title IDs trash_titles = [] of String - db.query "select path, id from titles" do |rs| + db.query "select path, id from titles where unavailable = 0" do |rs| rs.each do path = rs.read String - trash_titles << rs.read String unless Dir.exists? path + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_titles << rs.read String unless Dir.exists? fullpath end end - db.exec "delete from titles where id in " \ + unless trash_titles.empty? + Logger.debug "Marking #{trash_titles.size} titles as unavailable" + end + db.exec "update titles set unavailable = 1 where id in " \ "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_titles.size} dangling title IDs deleted" \ - if trash_titles.size > 0 end - Logger.info "DB optimization finished" end end + private def get_missing(tablename) + ary = [] of IDTuple + MainFiber.run do + get_db do |db| + db.query "select id, path, signature from #{tablename} " \ + "where unavailable = 1" do |rs| + rs.each do + ary << { + id: rs.read(String), + path: rs.read(String), + signature: rs.read(String?), + } + end + end + end + end + ary + end + + private def delete_missing(tablename, id : String? = nil) + MainFiber.run do + get_db do |db| + if id + db.exec "delete from #{tablename} where id = (?) " \ + "and unavailable = 1", id + else + db.exec "delete from #{tablename} where unavailable = 1" + end + end + end + end + + def missing_entries + get_missing "ids" + end + + def missing_titles + get_missing "titles" + end + + def delete_missing_entry(id = nil) + delete_missing "ids", id + end + + def delete_missing_title(id = nil) + delete_missing "titles", id + end + def close MainFiber.run do unless @db.nil? diff --git a/src/util/signature.cr b/src/util/signature.cr new file mode 100644 index 00000000..d1a0040c --- /dev/null +++ b/src/util/signature.cr @@ -0,0 +1,51 @@ +require "./util" + +class File + abstract struct Info + def inode : UInt64 + @stat.st_ino.to_u64 + end + end + + # Returns the signature of the file at filename. + # When it is not a supported file, returns 0. Otherwise, uses the inode + # number as its signature. On most file systems, the inode number is + # preserved even when the file is renamed, moved or edited. + # Some cases that would cause the inode number to change: + # - Reboot/remount on some file systems + # - Replaced with a copied file + # - Moved to a different device + # Since we are also using the relative paths to match ids, we won't lose + # information as long as the above changes do not happen together with + # a file/folder rename, with no library scan in between. + def self.signature(filename) : UInt64 + if is_supported_file filename + File.info(filename).inode + else + 0u64 + end + end +end + +class Dir + # Returns the signature of the directory at dirname. See the comments for + # `File.signature` for more information. + def self.signature(dirname) : UInt64 + signatures = [File.info(dirname).inode] + self.open dirname do |dir| + dir.entries.each do |fn| + next if fn.starts_with? "." + path = File.join dirname, fn + if File.directory? path + signatures << Dir.signature path + else + _sig = File.signature path + # Only add its signature value to `signatures` when it is a + # supported file + signatures << _sig if _sig > 0 + end + end + end + Digest::CRC32.checksum(signatures.sort.join).to_u64 + end +end diff --git a/src/util/util.cr b/src/util/util.cr index d7c0412b..873a226f 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -2,6 +2,7 @@ IMGS_PER_PAGE = 5 ENTRIES_IN_HOME_SECTIONS = 8 UPLOAD_URL_PREFIX = "/uploads" STATIC_DIRS = ["/css", "/js", "/img", "/favicon.ico"] +SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"] def random_str UUID.random.to_s.gsub "-", "" @@ -31,6 +32,10 @@ def register_mime_types end end +def is_supported_file(path) + SUPPORTED_FILE_EXTNAMES.includes? File.extname(path).downcase +end + struct Int def or(other : Int) if self == 0 @@ -92,3 +97,18 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) ary end + +class String + # Returns the similarity (in [0, 1]) of two paths. + # For the two paths, separate them into arrays of components, count the + # number of matching components backwards, and divide the count by the + # number of components of the shorter path. + def components_similarity(other : String) : Float64 + s, l = [self, other] + .map { |str| Path.new(str).parts } + .sort_by &.size + + match = s.reverse.zip(l.reverse).count { |a, b| a == b } + match / s.size + end +end diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index 587aaa72..fb64d3ea 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -1,5 +1,13 @@