From 18e8e88c668d8b5b8b14d4332e073b049c7a94e8 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Thu, 14 Jan 2021 08:23:39 +0000 Subject: [PATCH 01/24] Initial work on title signature --- src/library/title.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index 5fef9912..024f532e 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -3,7 +3,7 @@ require "../archive" class Title getter dir : String, parent_id : String, title_ids : Array(String), entries : Array(Entry), title : String, id : String, - encoded_title : String, mtime : Time + encoded_title : String, mtime : Time, signature : UInt64 = 0 @entry_display_name_cache : Hash(String, String)? @@ -25,6 +25,8 @@ class Title @entries = [] of Entry @mtime = File.info(dir).modification_time + signatures = [] of UInt64 + Dir.entries(dir).each do |fn| next if fn.starts_with? "." path = File.join dir, fn @@ -33,14 +35,18 @@ class Title next if title.entries.size == 0 && title.titles.size == 0 Library.default.title_hash[title.id] = title @title_ids << title.id + signatures << title.signature next end if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path entry = Entry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg + signatures << File.size entry.zip_path end end + @signature = Digest::CRC32.checksum(signatures.sort.join "").to_u64 + mtimes = [@mtime] mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } mtimes += @entries.map { |e| e.mtime } @@ -61,6 +67,7 @@ class Title {% for str in ["dir", "title", "id"] %} json.field {{str}}, @{{str.id}} {% end %} + json.field "signature" { json.number @signature } json.field "display_name", display_name json.field "cover_url", cover_url json.field "mtime" { json.number @mtime.to_unix } From 377c4c65548f1fb27b504d5ea85f12c540c6b52a Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 18 Jan 2021 06:44:10 +0000 Subject: [PATCH 02/24] Stop the process when the server fails to start --- src/mango.cr | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mango.cr b/src/mango.cr index 0d57f635..82f6178e 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -63,7 +63,12 @@ class CLI < Clim Plugin::Downloader.default spawn do - Server.new.start + begin + Server.new.start + rescue e + Logger.fatal e + Process.exit 1 + end end MainFiber.start_and_block From 667d390be4c16b4f7bad35f45616407dd4f63b4e Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 19 Jan 2021 08:43:45 +0000 Subject: [PATCH 03/24] Signature matching --- migration/ids_signature.7.cr | 50 +++++++++++++++++ migration/relative_path.8.cr | 31 +++++++++++ src/library/entry.cr | 9 +-- src/library/library.cr | 11 +--- src/library/title.cr | 18 +++--- src/storage.cr | 103 ++++++++++++++++++++++++++++++----- src/util/signature.cr | 50 +++++++++++++++++ src/util/util.cr | 15 +++++ 8 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 migration/ids_signature.7.cr create mode 100644 migration/relative_path.8.cr create mode 100644 src/util/signature.cr diff --git a/migration/ids_signature.7.cr b/migration/ids_signature.7.cr new file mode 100644 index 00000000..48da8e7e --- /dev/null +++ b/migration/ids_signature.7.cr @@ -0,0 +1,50 @@ +class IDSignature < MG::Base + def up : String + <<-SQL + ALTER TABLE ids ADD COLUMN signature TEXT; + SQL + end + + def down : String + <<-SQL + -- remove signature column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL + ); + + INSERT INTO ids + SELECT path, id + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + SQL + end +end diff --git a/migration/relative_path.8.cr b/migration/relative_path.8.cr new file mode 100644 index 00000000..cf22080e --- /dev/null +++ b/migration/relative_path.8.cr @@ -0,0 +1,31 @@ +class RelativePath < MG::Base + def up : String + base = Config.current.library_path + base = base[...-1] if base.ends_with? "/" + + <<-SQL + -- update the path column in ids to relative paths + UPDATE ids + SET path = REPLACE(path, '#{base}', ''); + + -- update the path column in titles to relative paths + UPDATE titles + SET path = REPLACE(path, '#{base}', ''); + SQL + end + + def down : String + base = Config.current.library_path + base = base[...-1] if base.ends_with? "/" + + <<-SQL + -- update the path column in ids to absolute paths + UPDATE ids + SET path = '#{base}' || path; + + -- update the path column in titles to absolute paths + UPDATE titles + SET path = '#{base}' || path; + SQL + end +end diff --git a/src/library/entry.cr b/src/library/entry.cr index 3e270cf4..e3599a8a 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -11,13 +11,14 @@ class Entry @title = File.basename @zip_path, File.extname @zip_path @encoded_title = URI.encode @title @size = (File.size @zip_path).humanize_bytes - id = storage.get_id @zip_path, false + id = storage.get_entry_id @zip_path, File.signature(@zip_path) if id.nil? id = random_str storage.insert_id({ - path: @zip_path, - id: id, - is_title: false, + path: @zip_path, + id: id, + title_signature: nil, + entry_signature: File.signature(@zip_path).to_s, }) end @id = id diff --git a/src/library/library.cr b/src/library/library.cr index a3db6401..1dc426ad 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,16 +42,6 @@ class Library end end end - - db_interval = Config.current.db_optimization_interval_hours - unless db_interval < 1 - spawn do - loop do - Storage.default.optimize - sleep db_interval.hours - end - end - end end def titles @@ -119,6 +109,7 @@ class Library storage.close Logger.debug "Scan completed" + Storage.default.optimize end def get_continue_reading_entries(username) diff --git a/src/library/title.cr b/src/library/title.cr index 024f532e..69441c9c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -3,19 +3,21 @@ require "../archive" class Title getter dir : String, parent_id : String, title_ids : Array(String), entries : Array(Entry), title : String, id : String, - encoded_title : String, mtime : Time, signature : UInt64 = 0 + encoded_title : String, mtime : Time, signature : UInt64 @entry_display_name_cache : Hash(String, String)? def initialize(@dir : String, @parent_id) storage = Storage.default - id = storage.get_id @dir, true + @signature = Dir.signature dir + id = storage.get_title_id dir, signature if id.nil? id = random_str storage.insert_id({ - path: @dir, - id: id, - is_title: true, + path: dir, + id: id, + title_signature: signature.to_s, + entry_signature: nil, }) end @id = id @@ -25,8 +27,6 @@ class Title @entries = [] of Entry @mtime = File.info(dir).modification_time - signatures = [] of UInt64 - Dir.entries(dir).each do |fn| next if fn.starts_with? "." path = File.join dir, fn @@ -35,18 +35,14 @@ class Title next if title.entries.size == 0 && title.titles.size == 0 Library.default.title_hash[title.id] = title @title_ids << title.id - signatures << title.signature next end if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path entry = Entry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg - signatures << File.size entry.zip_path end end - @signature = Digest::CRC32.checksum(signatures.sort.join "").to_u64 - mtimes = [@mtime] mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } mtimes += @entries.map { |e| e.mtime } diff --git a/src/storage.cr b/src/storage.cr index dcc337c1..d437387e 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -20,9 +20,11 @@ class Storage @path : String @db : DB::Database? - alias IDTuple = NamedTuple(path: String, + alias IDTuple = NamedTuple( + path: String, id: String, - is_title: Bool) + entry_signature: String?, + title_signature: String?) use_default @@ -230,16 +232,82 @@ class Storage end end - def get_id(path, is_title) + def get_title_id(path, signature) id = nil + path = Path.new(path).relative_to(Config.current.library_path).to_s MainFiber.run do get_db do |db| - if is_title - id = db.query_one? "select id from titles where path = (?)", path, - as: String - else - id = db.query_one? "select id from ids where path = (?)", path, - as: String + # First attempt to find the matching title in DB using BOTH path + # and signature + id = db.query_one? "select id from titles where path = (?) and " \ + "signature = (?)", path, signature.to_s, as: String + + should_update = id.nil? + # If it fails, try to match using the path only. This could happen + # for example when a new entry is added to the title + id ||= db.query_one? "select id from titles where path = (?)", path, + as: String + + # If it still fails, we will have to rely on the signature values. + # This could happen when the user moved or renamed the title, or + # a title containing the title + unless id + # If there are multiple rows with the same signature (this could + # happen simply by bad luck, or when the user copied a title), + # pick the row that has the most similar path to the give path + rows = [] of Tuple(String, String) + db.query "select id, path from titles where signature = (?)", + signature.to_s do |rs| + rs.each do + rows << {rs.read(String), rs.read(String)} + end + end + row = rows.max_by?(&.[1].components_similarity(path)) + id = row[0] if row + end + + # At this point, `id` would still be nil if there's no row matching + # either the path or the signature + + # If we did identify a matching title, save the path and signature + # values back to the DB + if id && should_update + db.exec "update titles set path = (?), signature = (?) " \ + "where id = (?)", path, signature.to_s, id + end + end + end + id + end + + # See the comments in `#get_title_id` to see how this method works. + def get_entry_id(path, signature) + id = nil + path = Path.new(path).relative_to(Config.current.library_path).to_s + MainFiber.run do + get_db do |db| + id = db.query_one? "select id from ids where path = (?) and " \ + "signature = (?)", path, signature.to_s, as: String + + should_update = id.nil? + id ||= db.query_one? "select id from ids where path = (?)", path, + as: String + + unless id + rows = [] of Tuple(String, String) + db.query "select id, path from ids where signature = (?)", + signature.to_s do |rs| + rs.each do + rows << {rs.read(String), rs.read(String)} + end + end + row = rows.max_by?(&.[1].components_similarity(path)) + id = row[0] if row + end + + if id && should_update + db.exec "update ids set path = (?), signature = (?) " \ + "where id = (?)", path, signature.to_s, id end end end @@ -256,11 +324,14 @@ class Storage db.transaction do |tran| conn = tran.connection @@insert_ids.each do |tp| - if tp[:is_title] - conn.exec "insert into titles values (?, ?, null)", tp[:id], - tp[:path] + path = Path.new(tp[:path]) + .relative_to(Config.current.library_path).to_s + if tp[:title_signature] + conn.exec "insert into titles values (?, ?, ?)", tp[:id], + path, tp[:title_signature].to_s else - conn.exec "insert into ids values (?, ?)", tp[:path], tp[:id] + conn.exec "insert into ids values (?, ?, ?)", path, tp[:id], + tp[:entry_signature].to_s end end end @@ -363,7 +434,8 @@ class Storage db.query "select path, id from ids" do |rs| rs.each do path = rs.read String - trash_ids << rs.read String unless File.exists? path + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_ids << rs.read String unless File.exists? fullpath end end @@ -377,7 +449,8 @@ class Storage db.query "select path, id from titles" do |rs| rs.each do path = rs.read String - trash_titles << rs.read String unless Dir.exists? path + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_titles << rs.read String unless Dir.exists? fullpath end end diff --git a/src/util/signature.cr b/src/util/signature.cr new file mode 100644 index 00000000..0db6b211 --- /dev/null +++ b/src/util/signature.cr @@ -0,0 +1,50 @@ +class File + abstract struct Info + def inode + @stat.st_ino + end + end + + # Returns the signature of the file at filename. + # When it is not a supported file, returns 0. Otherwise, calculate the + # signature by combining its inode value, file size and mtime. This + # ensures that moving (unless to another device) and renaming the file + # preserves the signature, while copying or editing the file changes it. + def self.signature(filename) : UInt64 + return 0u64 unless %w(.zip .rar .cbz .cbr).includes? File.extname filename + info = File.info filename + signatures = [ + info.inode, + File.size(filename), + info.modification_time.to_unix, + ] + Digest::CRC32.checksum(signatures.sort.join).to_u64 + end +end + +class Dir + # Returns the signature of the directory at dirname. + # The signature is calculated by combining its mtime and the signatures of + # all directories and files in it. This ensures that moving (unless to + # another device) and renaming the directory preserves the signature, + # while copying or editing its content changes it. + def self.signature(dirname) : UInt64 + signatures = [] of (UInt64 | Int64) + signatures << File.info(dirname).modification_time.to_unix + self.open dirname do |dir| + dir.entries.each do |fn| + next if fn.starts_with? "." + path = File.join dirname, fn + if File.directory? path + signatures << Dir.signature path + else + _sig = File.signature path + # Only add its signature value to `signatures` when it is a + # supported file + signatures << _sig if _sig > 0 + end + end + end + Digest::CRC32.checksum(signatures.sort.join).to_u64 + end +end diff --git a/src/util/util.cr b/src/util/util.cr index d7c0412b..f174c39f 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -92,3 +92,18 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) ary end + +class String + # Returns the similarity (in [0, 1]) of two paths. + # For the two paths, separate them into arrays of components, count the + # number of matching components backwards, and divide the count by the + # number of components of the shorter path. + def components_similarity(other : String) : Float64 + s, l = [self, other] + .map { |str| Path.new(str).parts } + .sort_by &.size + + match = s.reverse.zip(l.reverse).count { |a, b| a == b } + match / s.size + end +end From c7be0e0e7c738152db647df577104ffed2e3ffbd Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 19 Jan 2021 09:08:31 +0000 Subject: [PATCH 04/24] Separate `insert_id` into titles and entries --- src/library/entry.cr | 9 ++++----- src/library/title.cr | 9 ++++----- src/storage.cr | 41 ++++++++++++++++++++++++++--------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index e3599a8a..7a4e7533 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -14,11 +14,10 @@ class Entry id = storage.get_entry_id @zip_path, File.signature(@zip_path) if id.nil? id = random_str - storage.insert_id({ - path: @zip_path, - id: id, - title_signature: nil, - entry_signature: File.signature(@zip_path).to_s, + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, }) end @id = id diff --git a/src/library/title.cr b/src/library/title.cr index 69441c9c..001e9af2 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -13,11 +13,10 @@ class Title id = storage.get_title_id dir, signature if id.nil? id = random_str - storage.insert_id({ - path: dir, - id: id, - title_signature: signature.to_s, - entry_signature: nil, + storage.insert_title_id({ + path: dir, + id: id, + signature: signature.to_s, }) end @id = id diff --git a/src/storage.cr b/src/storage.cr index d437387e..97f6a666 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -15,16 +15,21 @@ def verify_password(hash, pw) end class Storage - @@insert_ids = [] of IDTuple + @@insert_entry_ids = [] of EntryID + @@insert_title_ids = [] of TitleID @path : String @db : DB::Database? - alias IDTuple = NamedTuple( + alias EntryID = NamedTuple( path: String, id: String, - entry_signature: String?, - title_signature: String?) + signature: String?) + + alias TitleID = NamedTuple( + path: String, + id: String, + signature: String?) use_default @@ -314,8 +319,12 @@ class Storage id end - def insert_id(tp : IDTuple) - @@insert_ids << tp + def insert_entry_id(tp) + @@insert_entry_ids << tp + end + + def insert_title_id(tp) + @@insert_title_ids << tp end def bulk_insert_ids @@ -323,20 +332,22 @@ class Storage get_db do |db| db.transaction do |tran| conn = tran.connection - @@insert_ids.each do |tp| + @@insert_title_ids.each do |tp| path = Path.new(tp[:path]) .relative_to(Config.current.library_path).to_s - if tp[:title_signature] - conn.exec "insert into titles values (?, ?, ?)", tp[:id], - path, tp[:title_signature].to_s - else - conn.exec "insert into ids values (?, ?, ?)", path, tp[:id], - tp[:entry_signature].to_s - end + conn.exec "insert into titles values (?, ?, ?)", tp[:id], + path, tp[:signature].to_s + end + @@insert_entry_ids.each do |tp| + path = Path.new(tp[:path]) + .relative_to(Config.current.library_path).to_s + conn.exec "insert into ids values (?, ?, ?)", path, tp[:id], + tp[:signature].to_s end end end - @@insert_ids.clear + @@insert_entry_ids.clear + @@insert_title_ids.clear end end From 781de97c680f90902a82279b3c6615ad02a9ccf5 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 19 Jan 2021 15:06:27 +0000 Subject: [PATCH 05/24] Make thumbnail generation slower This reduces the IO stress --- src/library/library.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index 1dc426ad..4f62c589 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -232,7 +232,7 @@ class Library e.generate_thumbnail # Sleep after each generation to minimize the impact on disk IO # and CPU - sleep 0.5.seconds + sleep 1.seconds end @thumbnails_count += 1 end From 54cd15d5425bb4c0b7b845ee346544df3ded0ef5 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 19 Jan 2021 15:09:38 +0000 Subject: [PATCH 06/24] Mark items unavailable and retire DB optimization This prepares us for the moving metadata to DB in the future --- migration/unavailable.9.cr | 94 +++++++++++++++++++++++++++ public/js/missing-items.js | 46 +++++++++++++ src/config.cr | 1 - src/library/library.cr | 2 +- src/routes/admin.cr | 7 ++ src/routes/api.cr | 89 +++++++++++++++++++++++++ src/storage.cr | 108 ++++++++++++++++++++++--------- src/views/admin.html.ecr | 8 +++ src/views/missing-items.html.ecr | 39 +++++++++++ 9 files changed, 360 insertions(+), 34 deletions(-) create mode 100644 migration/unavailable.9.cr create mode 100644 public/js/missing-items.js create mode 100644 src/views/missing-items.html.ecr diff --git a/migration/unavailable.9.cr b/migration/unavailable.9.cr new file mode 100644 index 00000000..172a2f57 --- /dev/null +++ b/migration/unavailable.9.cr @@ -0,0 +1,94 @@ +class UnavailableIDs < MG::Base + def up : String + <<-SQL + -- add unavailable column to ids + ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + + -- add unavailable column to titles + ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + SQL + end + + def down : String + <<-SQL + -- remove unavailable column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO ids + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + + -- remove unavailable column from titles + ALTER TABLE titles RENAME TO tmp; + + CREATE TABLE titles ( + id TEXT NOT NULL, + path TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO titles + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX titles_id_idx on titles (id); + CREATE UNIQUE INDEX titles_path_idx on titles (path); + + -- recreate the foreign key constraint on tags + ALTER TABLE tags RENAME TO tmp; + + CREATE TABLE tags ( + id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE (id, tag), + FOREIGN KEY (id) REFERENCES titles (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO tags + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE INDEX tags_id_idx ON tags (id); + CREATE INDEX tags_tag_idx ON tags (tag); + SQL + end +end diff --git a/public/js/missing-items.js b/public/js/missing-items.js new file mode 100644 index 00000000..0d7fa086 --- /dev/null +++ b/public/js/missing-items.js @@ -0,0 +1,46 @@ +const component = () => { + return { + empty: true, + titles: [], + entries: [], + loading: true, + + load() { + this.loading = true; + this.request('GET', `${base_url}api/admin/titles/missing`, data => { + this.titles = data.titles; + this.request('GET', `${base_url}api/admin/entries/missing`, data => { + this.entries = data.entries; + this.loading = false; + this.empty = this.entries.length === 0 && this.titles.length === 0; + }); + }); + }, + rm(event) { + const rawID = event.currentTarget.closest('tr').id; + const [type, id] = rawID.split('-'); + const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`; + this.request('DELETE', url, () => { + this.load(); + }); + }, + request(method, url, cb) { + console.log(url); + $.ajax({ + type: method, + url: url, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`); + return; + } + if (cb) cb(data); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + } + }; +}; diff --git a/src/config.cr b/src/config.cr index 845fc940..3ac5af28 100644 --- a/src/config.cr +++ b/src/config.cr @@ -13,7 +13,6 @@ class Config property db_path : String = File.expand_path "~/mango/mango.db", home: true property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property db_optimization_interval_hours : Int32 = 24 property log_level : String = "info" property upload_path : String = File.expand_path "~/mango/uploads", home: true diff --git a/src/library/library.cr b/src/library/library.cr index 4f62c589..83c0b0a0 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -109,7 +109,7 @@ class Library storage.close Logger.debug "Scan completed" - Storage.default.optimize + Storage.default.mark_unavailable end def get_continue_reading_entries(username) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 4445b0e4..fd63ec82 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -1,6 +1,9 @@ struct AdminRouter def initialize get "/admin" do |env| + storage = Storage.default + missing_count = storage.missing_titles.size + + storage.missing_entries.size layout "admin" end @@ -66,5 +69,9 @@ struct AdminRouter mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end + + get "/admin/missing" do |env| + layout "missing-items" + end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d60710b..cc61ba06 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -166,6 +166,21 @@ struct APIRouter "error" => "string?", } + Koa.object "missing", { + "path" => "string", + "id" => "string", + "signature" => "string", + } + + Koa.array "missingAry", "$missing" + + Koa.object "missingResult", { + "success" => "boolean", + "error" => "string?", + "entries" => "$missingAry?", + "titles" => "$missingAry?", + } + Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" @@ -777,6 +792,80 @@ struct APIRouter end end + Koa.describe "Lists all missing titles" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/titles/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "titles" => Storage.default.missing_titles, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Lists all missing entries" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/entries/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "entries" => Storage.default.missing_entries, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing title with `tid`" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/titles/missing/:tid" do |env| + begin + tid = env.params.url["tid"] + Storage.default.delete_missing_title tid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing entry with `eid`" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/entries/missing/:eid" do |env| + begin + eid = env.params.url["eid"] + Storage.default.delete_missing_entry eid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/storage.cr b/src/storage.cr index 97f6a666..548c9fc5 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -15,18 +15,13 @@ def verify_password(hash, pw) end class Storage - @@insert_entry_ids = [] of EntryID - @@insert_title_ids = [] of TitleID + @@insert_entry_ids = [] of IDTuple + @@insert_title_ids = [] of IDTuple @path : String @db : DB::Database? - alias EntryID = NamedTuple( - path: String, - id: String, - signature: String?) - - alias TitleID = NamedTuple( + alias IDTuple = NamedTuple( path: String, id: String, signature: String?) @@ -245,7 +240,8 @@ class Storage # First attempt to find the matching title in DB using BOTH path # and signature id = db.query_one? "select id from titles where path = (?) and " \ - "signature = (?)", path, signature.to_s, as: String + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String should_update = id.nil? # If it fails, try to match using the path only. This could happen @@ -277,8 +273,8 @@ class Storage # If we did identify a matching title, save the path and signature # values back to the DB if id && should_update - db.exec "update titles set path = (?), signature = (?) " \ - "where id = (?)", path, signature.to_s, id + db.exec "update titles set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id end end end @@ -292,7 +288,8 @@ class Storage MainFiber.run do get_db do |db| id = db.query_one? "select id from ids where path = (?) and " \ - "signature = (?)", path, signature.to_s, as: String + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String should_update = id.nil? id ||= db.query_one? "select id from ids where path = (?)", path, @@ -311,8 +308,8 @@ class Storage end if id && should_update - db.exec "update ids set path = (?), signature = (?) " \ - "where id = (?)", path, signature.to_s, id + db.exec "update ids set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id end end end @@ -335,14 +332,16 @@ class Storage @@insert_title_ids.each do |tp| path = Path.new(tp[:path]) .relative_to(Config.current.library_path).to_s - conn.exec "insert into titles values (?, ?, ?)", tp[:id], - path, tp[:signature].to_s + conn.exec "insert into titles (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s end @@insert_entry_ids.each do |tp| path = Path.new(tp[:path]) .relative_to(Config.current.library_path).to_s - conn.exec "insert into ids values (?, ?, ?)", path, tp[:id], - tp[:signature].to_s + conn.exec "insert into ids (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s end end end @@ -404,7 +403,8 @@ class Storage tags = [] of String MainFiber.run do get_db do |db| - db.query "select distinct tag from tags" do |rs| + db.query "select distinct tag from tags natural join titles " \ + "where unavailable = 0" do |rs| rs.each do tags << rs.read String end @@ -436,13 +436,12 @@ class Storage end end - def optimize + def mark_unavailable MainFiber.run do - Logger.info "Starting DB optimization" get_db do |db| - # Delete dangling entry IDs + # Detect dangling entry IDs trash_ids = [] of String - db.query "select path, id from ids" do |rs| + db.query "select path, id from ids where unavailable = 0" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -450,14 +449,15 @@ class Storage end end - db.exec "delete from ids where id in " \ + unless trash_ids.empty? + Logger.debug "Marking #{trash_ids.size} entries as unavailable" + end + db.exec "update ids set unavailable = 1 where id in " \ "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \ - if trash_ids.size > 0 - # Delete dangling title IDs + # Detect dangling title IDs trash_titles = [] of String - db.query "select path, id from titles" do |rs| + db.query "select path, id from titles where unavailable = 0" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -465,15 +465,59 @@ class Storage end end - db.exec "delete from titles where id in " \ + unless trash_titles.empty? + Logger.debug "Marking #{trash_titles.size} titles as unavailable" + end + db.exec "update titles set unavailable = 1 where id in " \ "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_titles.size} dangling title IDs deleted" \ - if trash_titles.size > 0 end - Logger.info "DB optimization finished" end end + private def get_missing(tablename) + ary = [] of IDTuple + MainFiber.run do + get_db do |db| + db.query "select id, path, signature from #{tablename} " \ + "where unavailable = 1" do |rs| + rs.each do + ary << { + id: rs.read(String), + path: rs.read(String), + signature: rs.read(String?), + } + end + end + end + end + ary + end + + private def delete_missing(tablename, id) + MainFiber.run do + get_db do |db| + db.exec "delete from #{tablename} where id = (?) and unavailable = 1", + id + end + end + end + + def missing_entries + get_missing "ids" + end + + def missing_titles + get_missing "titles" + end + + def delete_missing_entry(id) + delete_missing "ids", id + end + + def delete_missing_title(id) + delete_missing "titles", id + end + def close MainFiber.run do unless @db.nil? diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index 587aaa72..34cd51ae 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -1,5 +1,13 @@