diff --git a/.ameba.yml b/.ameba.yml
index e61b3b3e..163c936a 100644
--- a/.ameba.yml
+++ b/.ameba.yml
@@ -7,3 +7,8 @@ Lint/UnusedArgument:
- src/routes/*
Metrics/CyclomaticComplexity:
Enabled: false
+Layout/LineLength:
+ Enabled: true
+ MaxLength: 80
+ Excluded:
+ - src/routes/api.cr
diff --git a/Makefile b/Makefile
index d9485c50..6c5eebfa 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,6 @@ test:
check:
crystal tool format --check
./bin/ameba
- ./dev/linewidth.sh
arm32v7:
crystal build src/mango.cr --release --progress --error-trace --cross-compile --target='arm-linux-gnueabihf' -o mango-arm32v7
diff --git a/README.md b/README.md
index 9fe8ef68..bf57b7f7 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
- Mango - Manga Server and Web Reader. Version 0.19.1
+ Mango - Manga Server and Web Reader. Version 0.20.0
Usage:
@@ -82,7 +82,6 @@ library_path: ~/mango/library
db_path: ~/mango/mango.db
scan_interval_minutes: 5
thumbnail_generation_interval_hours: 24
-db_optimization_interval_hours: 24
log_level: info
upload_path: ~/mango/uploads
plugin_path: ~/mango/plugins
diff --git a/dev/linewidth.sh b/dev/linewidth.sh
deleted file mode 100755
index 6e83fbc3..00000000
--- a/dev/linewidth.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-[ ! -z "$(grep '.\{80\}' --exclude-dir=lib --include="*.cr" -nr --color=always . | grep -v "routes/api.cr" | tee /dev/tty)" ] \
- && echo "The above lines exceed the 80 characters limit" \
- || exit 0
diff --git a/migration/ids_signature.7.cr b/migration/ids_signature.7.cr
new file mode 100644
index 00000000..48da8e7e
--- /dev/null
+++ b/migration/ids_signature.7.cr
@@ -0,0 +1,50 @@
+class IDSignature < MG::Base
+ def up : String
+ <<-SQL
+ ALTER TABLE ids ADD COLUMN signature TEXT;
+ SQL
+ end
+
+ def down : String
+ <<-SQL
+ -- remove signature column from ids
+ ALTER TABLE ids RENAME TO tmp;
+
+ CREATE TABLE ids (
+ path TEXT NOT NULL,
+ id TEXT NOT NULL
+ );
+
+ INSERT INTO ids
+ SELECT path, id
+ FROM tmp;
+
+ DROP TABLE tmp;
+
+ -- recreate the indices
+ CREATE UNIQUE INDEX path_idx ON ids (path);
+ CREATE UNIQUE INDEX id_idx ON ids (id);
+
+ -- recreate the foreign key constraint on thumbnails
+ ALTER TABLE thumbnails RENAME TO tmp;
+
+ CREATE TABLE thumbnails (
+ id TEXT NOT NULL,
+ data BLOB NOT NULL,
+ filename TEXT NOT NULL,
+ mime TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ FOREIGN KEY (id) REFERENCES ids (id)
+ ON UPDATE CASCADE
+ ON DELETE CASCADE
+ );
+
+ INSERT INTO thumbnails
+ SELECT * FROM tmp;
+
+ DROP TABLE tmp;
+
+ CREATE UNIQUE INDEX tn_index ON thumbnails (id);
+ SQL
+ end
+end
diff --git a/migration/relative_path.8.cr b/migration/relative_path.8.cr
new file mode 100644
index 00000000..ec58ee6b
--- /dev/null
+++ b/migration/relative_path.8.cr
@@ -0,0 +1,33 @@
+class RelativePath < MG::Base
+ def up : String
+ base = Config.current.library_path
+ # Escape single quotes in case the path contains them, and remove the
+ # trailing slash (this is a mistake, fixed in DB version 10)
+ base = base.gsub("'", "''").rstrip "/"
+
+ <<-SQL
+ -- update the path column in ids to relative paths
+ UPDATE ids
+ SET path = REPLACE(path, '#{base}', '');
+
+ -- update the path column in titles to relative paths
+ UPDATE titles
+ SET path = REPLACE(path, '#{base}', '');
+ SQL
+ end
+
+ def down : String
+ base = Config.current.library_path
+ base = base.gsub("'", "''").rstrip "/"
+
+ <<-SQL
+ -- update the path column in ids to absolute paths
+ UPDATE ids
+ SET path = '#{base}' || path;
+
+ -- update the path column in titles to absolute paths
+ UPDATE titles
+ SET path = '#{base}' || path;
+ SQL
+ end
+end
diff --git a/migration/relative_path_fix.10.cr b/migration/relative_path_fix.10.cr
new file mode 100644
index 00000000..26df8766
--- /dev/null
+++ b/migration/relative_path_fix.10.cr
@@ -0,0 +1,31 @@
+# In DB version 8, we replaced the absolute paths in DB with relative paths,
+# but we mistakenly left the starting slashes. This migration removes them.
+class RelativePathFix < MG::Base
+ def up : String
+ <<-SQL
+ -- remove leading slashes from the paths in ids
+ UPDATE ids
+ SET path = SUBSTR(path, 2, LENGTH(path) - 1)
+ WHERE path LIKE '/%';
+
+ -- remove leading slashes from the paths in titles
+ UPDATE titles
+ SET path = SUBSTR(path, 2, LENGTH(path) - 1)
+ WHERE path LIKE '/%';
+ SQL
+ end
+
+ def down : String
+ <<-SQL
+ -- add leading slashes to paths in ids
+ UPDATE ids
+ SET path = '/' || path
+ WHERE path NOT LIKE '/%';
+
+ -- add leading slashes to paths in titles
+ UPDATE titles
+ SET path = '/' || path
+ WHERE path NOT LIKE '/%';
+ SQL
+ end
+end
diff --git a/migration/unavailable.9.cr b/migration/unavailable.9.cr
new file mode 100644
index 00000000..172a2f57
--- /dev/null
+++ b/migration/unavailable.9.cr
@@ -0,0 +1,94 @@
+class UnavailableIDs < MG::Base
+ def up : String
+ <<-SQL
+ -- add unavailable column to ids
+ ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+
+ -- add unavailable column to titles
+ ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+ SQL
+ end
+
+ def down : String
+ <<-SQL
+ -- remove unavailable column from ids
+ ALTER TABLE ids RENAME TO tmp;
+
+ CREATE TABLE ids (
+ path TEXT NOT NULL,
+ id TEXT NOT NULL,
+ signature TEXT
+ );
+
+ INSERT INTO ids
+ SELECT path, id, signature
+ FROM tmp;
+
+ DROP TABLE tmp;
+
+ -- recreate the indices
+ CREATE UNIQUE INDEX path_idx ON ids (path);
+ CREATE UNIQUE INDEX id_idx ON ids (id);
+
+ -- recreate the foreign key constraint on thumbnails
+ ALTER TABLE thumbnails RENAME TO tmp;
+
+ CREATE TABLE thumbnails (
+ id TEXT NOT NULL,
+ data BLOB NOT NULL,
+ filename TEXT NOT NULL,
+ mime TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ FOREIGN KEY (id) REFERENCES ids (id)
+ ON UPDATE CASCADE
+ ON DELETE CASCADE
+ );
+
+ INSERT INTO thumbnails
+ SELECT * FROM tmp;
+
+ DROP TABLE tmp;
+
+ CREATE UNIQUE INDEX tn_index ON thumbnails (id);
+
+ -- remove unavailable column from titles
+ ALTER TABLE titles RENAME TO tmp;
+
+ CREATE TABLE titles (
+ id TEXT NOT NULL,
+ path TEXT NOT NULL,
+ signature TEXT
+ );
+
+ INSERT INTO titles
+ SELECT path, id, signature
+ FROM tmp;
+
+ DROP TABLE tmp;
+
+ -- recreate the indices
+ CREATE UNIQUE INDEX titles_id_idx on titles (id);
+ CREATE UNIQUE INDEX titles_path_idx on titles (path);
+
+ -- recreate the foreign key constraint on tags
+ ALTER TABLE tags RENAME TO tmp;
+
+ CREATE TABLE tags (
+ id TEXT NOT NULL,
+ tag TEXT NOT NULL,
+ UNIQUE (id, tag),
+ FOREIGN KEY (id) REFERENCES titles (id)
+ ON UPDATE CASCADE
+ ON DELETE CASCADE
+ );
+
+ INSERT INTO tags
+ SELECT * FROM tmp;
+
+ DROP TABLE tmp;
+
+ CREATE INDEX tags_id_idx ON tags (id);
+ CREATE INDEX tags_tag_idx ON tags (tag);
+ SQL
+ end
+end
diff --git a/public/css/mango.less b/public/css/mango.less
index f6e89517..1ac84df9 100644
--- a/public/css/mango.less
+++ b/public/css/mango.less
@@ -66,7 +66,6 @@
// Dark theme
.uk-light {
- .uk-navbar-dropdown,
.uk-modal-header,
.uk-modal-body,
.uk-modal-footer {
@@ -75,6 +74,7 @@
.uk-navbar-dropdown,
.uk-dropdown {
color: #ccc;
+ background: #333;
}
.uk-nav-header,
.uk-description-list > dt {
diff --git a/public/js/download.js b/public/js/download.js
index aca45eb9..b8d76fe6 100644
--- a/public/js/download.js
+++ b/public/js/download.js
@@ -156,8 +156,8 @@ const search = () => {
langs.unshift('All');
group_names.unshift('All');
- $('select#lang-select').append(langs.map(e => ``).join(''));
- $('select#group-select').append(group_names.map(e => ``).join(''));
+ $('select#lang-select').html(langs.map(e => ``).join(''));
+ $('select#group-select').html(group_names.map(e => ``).join(''));
$('#filter-form').removeAttr('hidden');
@@ -241,7 +241,7 @@ const buildTable = () => {
if (v === 'All') return;
if (k === 'group') {
chapters = chapters.filter(c => {
- unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g));
+ const unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g));
return unescaped_groups.indexOf(v) >= 0;
});
return;
diff --git a/public/js/missing-items.js b/public/js/missing-items.js
new file mode 100644
index 00000000..0babb24a
--- /dev/null
+++ b/public/js/missing-items.js
@@ -0,0 +1,60 @@
+const component = () => {
+ return {
+ empty: true,
+ titles: [],
+ entries: [],
+ loading: true,
+
+ load() {
+ this.loading = true;
+ this.request('GET', `${base_url}api/admin/titles/missing`, data => {
+ this.titles = data.titles;
+ this.request('GET', `${base_url}api/admin/entries/missing`, data => {
+ this.entries = data.entries;
+ this.loading = false;
+ this.empty = this.entries.length === 0 && this.titles.length === 0;
+ });
+ });
+ },
+ rm(event) {
+ const rawID = event.currentTarget.closest('tr').id;
+ const [type, id] = rawID.split('-');
+ const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`;
+ this.request('DELETE', url, () => {
+ this.load();
+ });
+ },
+ rmAll() {
+ UIkit.modal.confirm('Are you sure? All metadata associated with these items, including their tags and thumbnails, will be deleted from the database.', {
+ labels: {
+ ok: 'Yes, delete them',
+ cancel: 'Cancel'
+ }
+ }).then(() => {
+ this.request('DELETE', `${base_url}api/admin/titles/missing`, () => {
+ this.request('DELETE', `${base_url}api/admin/entries/missing`, () => {
+ this.load();
+ });
+ });
+ });
+ },
+ request(method, url, cb) {
+ console.log(url);
+ $.ajax({
+ type: method,
+ url: url,
+ contentType: 'application/json'
+ })
+ .done(data => {
+ if (data.error) {
+ alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`);
+ return;
+ }
+ if (cb) cb(data);
+ })
+ .fail((jqXHR, status) => {
+ alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
+ });
+ }
+ };
+};
diff --git a/shard.yml b/shard.yml
index a999ab0f..d7505a5a 100644
--- a/shard.yml
+++ b/shard.yml
@@ -1,5 +1,5 @@
name: mango
-version: 0.19.1
+version: 0.20.0
authors:
- Alex Ling
diff --git a/spec/util_spec.cr b/spec/util_spec.cr
index 94c326c7..3ee4aac3 100644
--- a/spec/util_spec.cr
+++ b/spec/util_spec.cr
@@ -35,6 +35,23 @@ describe "compare_numerically" do
end
end
+describe "is_supported_file" do
+ it "returns true when the filename has a supported extension" do
+ filename = "manga.cbz"
+ is_supported_file(filename).should eq true
+ end
+
+ it "returns true when the filename does not have a supported extension" do
+ filename = "info.json"
+ is_supported_file(filename).should eq false
+ end
+
+ it "is case insensitive" do
+ filename = "manga.ZiP"
+ is_supported_file(filename).should eq true
+ end
+end
+
describe "chapter_sort" do
it "sorts correctly" do
ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"]
diff --git a/src/config.cr b/src/config.cr
index 845fc940..3ac5af28 100644
--- a/src/config.cr
+++ b/src/config.cr
@@ -13,7 +13,6 @@ class Config
property db_path : String = File.expand_path "~/mango/mango.db", home: true
property scan_interval_minutes : Int32 = 5
property thumbnail_generation_interval_hours : Int32 = 24
- property db_optimization_interval_hours : Int32 = 24
property log_level : String = "info"
property upload_path : String = File.expand_path "~/mango/uploads",
home: true
diff --git a/src/library/entry.cr b/src/library/entry.cr
index 3e270cf4..7a4e7533 100644
--- a/src/library/entry.cr
+++ b/src/library/entry.cr
@@ -11,13 +11,13 @@ class Entry
@title = File.basename @zip_path, File.extname @zip_path
@encoded_title = URI.encode @title
@size = (File.size @zip_path).humanize_bytes
- id = storage.get_id @zip_path, false
+ id = storage.get_entry_id @zip_path, File.signature(@zip_path)
if id.nil?
id = random_str
- storage.insert_id({
- path: @zip_path,
- id: id,
- is_title: false,
+ storage.insert_entry_id({
+ path: @zip_path,
+ id: id,
+ signature: File.signature(@zip_path).to_s,
})
end
@id = id
diff --git a/src/library/library.cr b/src/library/library.cr
index a3db6401..83c0b0a0 100644
--- a/src/library/library.cr
+++ b/src/library/library.cr
@@ -42,16 +42,6 @@ class Library
end
end
end
-
- db_interval = Config.current.db_optimization_interval_hours
- unless db_interval < 1
- spawn do
- loop do
- Storage.default.optimize
- sleep db_interval.hours
- end
- end
- end
end
def titles
@@ -119,6 +109,7 @@ class Library
storage.close
Logger.debug "Scan completed"
+ Storage.default.mark_unavailable
end
def get_continue_reading_entries(username)
@@ -241,7 +232,7 @@ class Library
e.generate_thumbnail
# Sleep after each generation to minimize the impact on disk IO
# and CPU
- sleep 0.5.seconds
+ sleep 1.seconds
end
@thumbnails_count += 1
end
diff --git a/src/library/title.cr b/src/library/title.cr
index 5fef9912..4c439e72 100644
--- a/src/library/title.cr
+++ b/src/library/title.cr
@@ -3,19 +3,20 @@ require "../archive"
class Title
getter dir : String, parent_id : String, title_ids : Array(String),
entries : Array(Entry), title : String, id : String,
- encoded_title : String, mtime : Time
+ encoded_title : String, mtime : Time, signature : UInt64
@entry_display_name_cache : Hash(String, String)?
def initialize(@dir : String, @parent_id)
storage = Storage.default
- id = storage.get_id @dir, true
+ @signature = Dir.signature dir
+ id = storage.get_title_id dir, signature
if id.nil?
id = random_str
- storage.insert_id({
- path: @dir,
- id: id,
- is_title: true,
+ storage.insert_title_id({
+ path: dir,
+ id: id,
+ signature: signature.to_s,
})
end
@id = id
@@ -35,7 +36,7 @@ class Title
@title_ids << title.id
next
end
- if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path
+ if is_supported_file path
entry = Entry.new path, self
@entries << entry if entry.pages > 0 || entry.err_msg
end
@@ -61,6 +62,7 @@ class Title
{% for str in ["dir", "title", "id"] %}
json.field {{str}}, @{{str.id}}
{% end %}
+ json.field "signature" { json.number @signature }
json.field "display_name", display_name
json.field "cover_url", cover_url
json.field "mtime" { json.number @mtime.to_unix }
diff --git a/src/mango.cr b/src/mango.cr
index 138c5a9f..fcade44a 100644
--- a/src/mango.cr
+++ b/src/mango.cr
@@ -8,7 +8,7 @@ require "option_parser"
require "clim"
require "tallboy"
-MANGO_VERSION = "0.19.1"
+MANGO_VERSION = "0.20.0"
# From http://www.network-science.de/ascii/
BANNER = %{
@@ -63,7 +63,12 @@ class CLI < Clim
Plugin::Downloader.default
spawn do
- Server.new.start
+ begin
+ Server.new.start
+ rescue e
+ Logger.fatal e
+ Process.exit 1
+ end
end
MainFiber.start_and_block
diff --git a/src/routes/admin.cr b/src/routes/admin.cr
index 4445b0e4..fd63ec82 100644
--- a/src/routes/admin.cr
+++ b/src/routes/admin.cr
@@ -1,6 +1,9 @@
struct AdminRouter
def initialize
get "/admin" do |env|
+ storage = Storage.default
+ missing_count = storage.missing_titles.size +
+ storage.missing_entries.size
layout "admin"
end
@@ -66,5 +69,9 @@ struct AdminRouter
mangadex_base_url = Config.current.mangadex["base_url"]
layout "download-manager"
end
+
+ get "/admin/missing" do |env|
+ layout "missing-items"
+ end
end
end
diff --git a/src/routes/api.cr b/src/routes/api.cr
index 7d60710b..7fc8b46a 100644
--- a/src/routes/api.cr
+++ b/src/routes/api.cr
@@ -166,6 +166,21 @@ struct APIRouter
"error" => "string?",
}
+ Koa.object "missing", {
+ "path" => "string",
+ "id" => "string",
+ "signature" => "string",
+ }
+
+ Koa.array "missingAry", "$missing"
+
+ Koa.object "missingResult", {
+ "success" => "boolean",
+ "error" => "string?",
+ "entries" => "$missingAry?",
+ "titles" => "$missingAry?",
+ }
+
Koa.describe "Returns a page in a manga entry"
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
@@ -777,6 +792,120 @@ struct APIRouter
end
end
+ Koa.describe "Lists all missing titles"
+ Koa.response 200, ref: "$missingResult"
+ Koa.tag "admin"
+ get "/api/admin/titles/missing" do |env|
+ begin
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ "titles" => Storage.default.missing_titles,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
+ Koa.describe "Lists all missing entries"
+ Koa.response 200, ref: "$missingResult"
+ Koa.tag "admin"
+ get "/api/admin/entries/missing" do |env|
+ begin
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ "entries" => Storage.default.missing_entries,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
+ Koa.describe "Deletes all missing titles"
+ Koa.response 200, ref: "$result"
+ Koa.tag "admin"
+ delete "/api/admin/titles/missing" do |env|
+ begin
+ Storage.default.delete_missing_title
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
+ Koa.describe "Deletes all missing entries"
+ Koa.response 200, ref: "$result"
+ Koa.tag "admin"
+ delete "/api/admin/entries/missing" do |env|
+ begin
+ Storage.default.delete_missing_entry
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
+ Koa.describe "Deletes a missing title identified by `tid`", <<-MD
+ Does nothing if the given `tid` is not found or if the title is not missing.
+ MD
+ Koa.response 200, ref: "$result"
+ Koa.tag "admin"
+ delete "/api/admin/titles/missing/:tid" do |env|
+ begin
+ tid = env.params.url["tid"]
+ Storage.default.delete_missing_title tid
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
+ Koa.describe "Deletes a missing entry identified by `eid`", <<-MD
+ Does nothing if the given `eid` is not found or if the entry is not missing.
+ MD
+ Koa.response 200, ref: "$result"
+ Koa.tag "admin"
+ delete "/api/admin/entries/missing/:eid" do |env|
+ begin
+ eid = env.params.url["eid"]
+ Storage.default.delete_missing_entry eid
+ send_json env, {
+ "success" => true,
+ "error" => nil,
+ }.to_json
+ rescue e
+ send_json env, {
+ "success" => false,
+ "error" => e.message,
+ }.to_json
+ end
+ end
+
doc = Koa.generate
@@api_json = doc.to_json if doc
diff --git a/src/routes/main.cr b/src/routes/main.cr
index 70b13349..65048005 100644
--- a/src/routes/main.cr
+++ b/src/routes/main.cr
@@ -154,6 +154,7 @@ struct MainRouter
end
get "/api" do |env|
+ base_url = Config.current.base_url
render "src/views/api.html.ecr"
end
end
diff --git a/src/storage.cr b/src/storage.cr
index dcc337c1..2bc6daa1 100644
--- a/src/storage.cr
+++ b/src/storage.cr
@@ -15,14 +15,16 @@ def verify_password(hash, pw)
end
class Storage
- @@insert_ids = [] of IDTuple
+ @@insert_entry_ids = [] of IDTuple
+ @@insert_title_ids = [] of IDTuple
@path : String
@db : DB::Database?
- alias IDTuple = NamedTuple(path: String,
+ alias IDTuple = NamedTuple(
+ path: String,
id: String,
- is_title: Bool)
+ signature: String?)
use_default
@@ -230,24 +232,96 @@ class Storage
end
end
- def get_id(path, is_title)
+ def get_title_id(path, signature)
id = nil
+ path = Path.new(path).relative_to(Config.current.library_path).to_s
MainFiber.run do
get_db do |db|
- if is_title
- id = db.query_one? "select id from titles where path = (?)", path,
- as: String
- else
- id = db.query_one? "select id from ids where path = (?)", path,
- as: String
+ # First attempt to find the matching title in DB using BOTH path
+ # and signature
+ id = db.query_one? "select id from titles where path = (?) and " \
+ "signature = (?) and unavailable = 0",
+ path, signature.to_s, as: String
+
+ should_update = id.nil?
+ # If it fails, try to match using the path only. This could happen
+ # for example when a new entry is added to the title
+ id ||= db.query_one? "select id from titles where path = (?)", path,
+ as: String
+
+ # If it still fails, we will have to rely on the signature values.
+ # This could happen when the user moved or renamed the title, or
+ # a title containing the title
+ unless id
+ # If there are multiple rows with the same signature (this could
+ # happen simply by bad luck, or when the user copied a title),
+ # pick the row that has the most similar path to the give path
+ rows = [] of Tuple(String, String)
+ db.query "select id, path from titles where signature = (?)",
+ signature.to_s do |rs|
+ rs.each do
+ rows << {rs.read(String), rs.read(String)}
+ end
+ end
+ row = rows.max_by?(&.[1].components_similarity(path))
+ id = row[0] if row
+ end
+
+ # At this point, `id` would still be nil if there's no row matching
+ # either the path or the signature
+
+ # If we did identify a matching title, save the path and signature
+ # values back to the DB
+ if id && should_update
+ db.exec "update titles set path = (?), signature = (?), " \
+ "unavailable = 0 where id = (?)", path, signature.to_s, id
end
end
end
id
end
- def insert_id(tp : IDTuple)
- @@insert_ids << tp
+ # See the comments in `#get_title_id` to see how this method works.
+ def get_entry_id(path, signature)
+ id = nil
+ path = Path.new(path).relative_to(Config.current.library_path).to_s
+ MainFiber.run do
+ get_db do |db|
+ id = db.query_one? "select id from ids where path = (?) and " \
+ "signature = (?) and unavailable = 0",
+ path, signature.to_s, as: String
+
+ should_update = id.nil?
+ id ||= db.query_one? "select id from ids where path = (?)", path,
+ as: String
+
+ unless id
+ rows = [] of Tuple(String, String)
+ db.query "select id, path from ids where signature = (?)",
+ signature.to_s do |rs|
+ rs.each do
+ rows << {rs.read(String), rs.read(String)}
+ end
+ end
+ row = rows.max_by?(&.[1].components_similarity(path))
+ id = row[0] if row
+ end
+
+ if id && should_update
+ db.exec "update ids set path = (?), signature = (?), " \
+ "unavailable = 0 where id = (?)", path, signature.to_s, id
+ end
+ end
+ end
+ id
+ end
+
+ def insert_entry_id(tp)
+ @@insert_entry_ids << tp
+ end
+
+ def insert_title_id(tp)
+ @@insert_title_ids << tp
end
def bulk_insert_ids
@@ -255,17 +329,24 @@ class Storage
get_db do |db|
db.transaction do |tran|
conn = tran.connection
- @@insert_ids.each do |tp|
- if tp[:is_title]
- conn.exec "insert into titles values (?, ?, null)", tp[:id],
- tp[:path]
- else
- conn.exec "insert into ids values (?, ?)", tp[:path], tp[:id]
- end
+ @@insert_title_ids.each do |tp|
+ path = Path.new(tp[:path])
+ .relative_to(Config.current.library_path).to_s
+ conn.exec "insert into titles (id, path, signature, " \
+ "unavailable) values (?, ?, ?, 0)",
+ tp[:id], path, tp[:signature].to_s
+ end
+ @@insert_entry_ids.each do |tp|
+ path = Path.new(tp[:path])
+ .relative_to(Config.current.library_path).to_s
+ conn.exec "insert into ids (id, path, signature, " \
+ "unavailable) values (?, ?, ?, 0)",
+ tp[:id], path, tp[:signature].to_s
end
end
end
- @@insert_ids.clear
+ @@insert_entry_ids.clear
+ @@insert_title_ids.clear
end
end
@@ -322,7 +403,8 @@ class Storage
tags = [] of String
MainFiber.run do
get_db do |db|
- db.query "select distinct tag from tags" do |rs|
+ db.query "select distinct tag from tags natural join titles " \
+ "where unavailable = 0" do |rs|
rs.each do
tags << rs.read String
end
@@ -354,42 +436,92 @@ class Storage
end
end
- def optimize
+ def mark_unavailable
MainFiber.run do
- Logger.info "Starting DB optimization"
get_db do |db|
- # Delete dangling entry IDs
+ # Detect dangling entry IDs
trash_ids = [] of String
- db.query "select path, id from ids" do |rs|
+ db.query "select path, id from ids where unavailable = 0" do |rs|
rs.each do
path = rs.read String
- trash_ids << rs.read String unless File.exists? path
+ fullpath = Path.new(path).expand(Config.current.library_path).to_s
+ trash_ids << rs.read String unless File.exists? fullpath
end
end
- db.exec "delete from ids where id in " \
+ unless trash_ids.empty?
+ Logger.debug "Marking #{trash_ids.size} entries as unavailable"
+ end
+ db.exec "update ids set unavailable = 1 where id in " \
"(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
- Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \
- if trash_ids.size > 0
- # Delete dangling title IDs
+ # Detect dangling title IDs
trash_titles = [] of String
- db.query "select path, id from titles" do |rs|
+ db.query "select path, id from titles where unavailable = 0" do |rs|
rs.each do
path = rs.read String
- trash_titles << rs.read String unless Dir.exists? path
+ fullpath = Path.new(path).expand(Config.current.library_path).to_s
+ trash_titles << rs.read String unless Dir.exists? fullpath
end
end
- db.exec "delete from titles where id in " \
+ unless trash_titles.empty?
+ Logger.debug "Marking #{trash_titles.size} titles as unavailable"
+ end
+ db.exec "update titles set unavailable = 1 where id in " \
"(#{trash_titles.map { |i| "'#{i}'" }.join ","})"
- Logger.debug "#{trash_titles.size} dangling title IDs deleted" \
- if trash_titles.size > 0
end
- Logger.info "DB optimization finished"
end
end
+ private def get_missing(tablename)
+ ary = [] of IDTuple
+ MainFiber.run do
+ get_db do |db|
+ db.query "select id, path, signature from #{tablename} " \
+ "where unavailable = 1" do |rs|
+ rs.each do
+ ary << {
+ id: rs.read(String),
+ path: rs.read(String),
+ signature: rs.read(String?),
+ }
+ end
+ end
+ end
+ end
+ ary
+ end
+
+ private def delete_missing(tablename, id : String? = nil)
+ MainFiber.run do
+ get_db do |db|
+ if id
+ db.exec "delete from #{tablename} where id = (?) " \
+ "and unavailable = 1", id
+ else
+ db.exec "delete from #{tablename} where unavailable = 1"
+ end
+ end
+ end
+ end
+
+ def missing_entries
+ get_missing "ids"
+ end
+
+ def missing_titles
+ get_missing "titles"
+ end
+
+ def delete_missing_entry(id = nil)
+ delete_missing "ids", id
+ end
+
+ def delete_missing_title(id = nil)
+ delete_missing "titles", id
+ end
+
def close
MainFiber.run do
unless @db.nil?
diff --git a/src/util/signature.cr b/src/util/signature.cr
new file mode 100644
index 00000000..d1a0040c
--- /dev/null
+++ b/src/util/signature.cr
@@ -0,0 +1,51 @@
+require "./util"
+
+class File
+ abstract struct Info
+ def inode : UInt64
+ @stat.st_ino.to_u64
+ end
+ end
+
+ # Returns the signature of the file at filename.
+ # When it is not a supported file, returns 0. Otherwise, uses the inode
+ # number as its signature. On most file systems, the inode number is
+ # preserved even when the file is renamed, moved or edited.
+ # Some cases that would cause the inode number to change:
+ # - Reboot/remount on some file systems
+ # - Replaced with a copied file
+ # - Moved to a different device
+ # Since we are also using the relative paths to match ids, we won't lose
+ # information as long as the above changes do not happen together with
+ # a file/folder rename, with no library scan in between.
+ def self.signature(filename) : UInt64
+ if is_supported_file filename
+ File.info(filename).inode
+ else
+ 0u64
+ end
+ end
+end
+
+class Dir
+ # Returns the signature of the directory at dirname. See the comments for
+ # `File.signature` for more information.
+ def self.signature(dirname) : UInt64
+ signatures = [File.info(dirname).inode]
+ self.open dirname do |dir|
+ dir.entries.each do |fn|
+ next if fn.starts_with? "."
+ path = File.join dirname, fn
+ if File.directory? path
+ signatures << Dir.signature path
+ else
+ _sig = File.signature path
+ # Only add its signature value to `signatures` when it is a
+ # supported file
+ signatures << _sig if _sig > 0
+ end
+ end
+ end
+ Digest::CRC32.checksum(signatures.sort.join).to_u64
+ end
+end
diff --git a/src/util/util.cr b/src/util/util.cr
index d7c0412b..873a226f 100644
--- a/src/util/util.cr
+++ b/src/util/util.cr
@@ -2,6 +2,7 @@ IMGS_PER_PAGE = 5
ENTRIES_IN_HOME_SECTIONS = 8
UPLOAD_URL_PREFIX = "/uploads"
STATIC_DIRS = ["/css", "/js", "/img", "/favicon.ico"]
+SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"]
def random_str
UUID.random.to_s.gsub "-", ""
@@ -31,6 +32,10 @@ def register_mime_types
end
end
+def is_supported_file(path)
+ SUPPORTED_FILE_EXTNAMES.includes? File.extname(path).downcase
+end
+
struct Int
def or(other : Int)
if self == 0
@@ -92,3 +97,18 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String)
ary
end
+
+class String
+ # Returns the similarity (in [0, 1]) of two paths.
+ # For the two paths, separate them into arrays of components, count the
+ # number of matching components backwards, and divide the count by the
+ # number of components of the shorter path.
+ def components_similarity(other : String) : Float64
+ s, l = [self, other]
+ .map { |str| Path.new(str).parts }
+ .sort_by &.size
+
+ match = s.reverse.zip(l.reverse).count { |a, b| a == b }
+ match / s.size
+ end
+end
diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr
index 587aaa72..fb64d3ea 100644
--- a/src/views/admin.html.ecr
+++ b/src/views/admin.html.ecr
@@ -1,5 +1,13 @@