Merge pull request #156 from hkalexling/dev

v0.20.0

.ameba.yml

@@ -7,3 +7,8 @@ Lint/UnusedArgument:
     - src/routes/*
 Metrics/CyclomaticComplexity:
   Enabled: false
+Layout/LineLength:
+  Enabled: true
+  MaxLength: 80
+  Excluded:
+    - src/routes/api.cr

Makefile

@@ -29,7 +29,6 @@ test:
 check:
 	crystal tool format --check
 	./bin/ameba
-	./dev/linewidth.sh
 
 arm32v7:
 	crystal build src/mango.cr --release --progress --error-trace --cross-compile --target='arm-linux-gnueabihf' -o mango-arm32v7

README.md

@@ -52,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
 ### CLI
 
 ```
-  Mango - Manga Server and Web Reader. Version 0.19.1
+  Mango - Manga Server and Web Reader. Version 0.20.0
 
   Usage:
 
@@ -82,7 +82,6 @@ library_path: ~/mango/library
 db_path: ~/mango/mango.db
 scan_interval_minutes: 5
 thumbnail_generation_interval_hours: 24
-db_optimization_interval_hours: 24
 log_level: info
 upload_path: ~/mango/uploads
 plugin_path: ~/mango/plugins

dev/linewidth.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-[ ! -z "$(grep '.\{80\}' --exclude-dir=lib --include="*.cr" -nr --color=always . | grep -v "routes/api.cr" | tee /dev/tty)" ] \
-	&& echo "The above lines exceed the 80 characters limit" \
-	|| exit 0

migration/ids_signature.7.cr

@@ -0,0 +1,50 @@
+class IDSignature < MG::Base
+  def up : String
+    <<-SQL
+    ALTER TABLE ids ADD COLUMN signature TEXT;
+    SQL
+  end
+
+  def down : String
+    <<-SQL
+    -- remove signature column from ids
+    ALTER TABLE ids RENAME TO tmp;
+
+    CREATE TABLE ids (
+      path TEXT NOT NULL,
+      id TEXT NOT NULL
+    );
+
+    INSERT INTO ids
+    SELECT path, id
+    FROM tmp;
+
+    DROP TABLE tmp;
+
+    -- recreate the indices
+    CREATE UNIQUE INDEX path_idx ON ids (path);
+    CREATE UNIQUE INDEX id_idx ON ids (id);
+
+    -- recreate the foreign key constraint on thumbnails
+    ALTER TABLE thumbnails RENAME TO tmp;
+
+    CREATE TABLE thumbnails (
+      id TEXT NOT NULL,
+      data BLOB NOT NULL,
+      filename TEXT NOT NULL,
+      mime TEXT NOT NULL,
+      size INTEGER NOT NULL,
+      FOREIGN KEY (id) REFERENCES ids (id)
+        ON UPDATE CASCADE
+        ON DELETE CASCADE
+    );
+
+    INSERT INTO thumbnails
+    SELECT * FROM tmp;
+
+    DROP TABLE tmp;
+
+    CREATE UNIQUE INDEX tn_index ON thumbnails (id);
+    SQL
+  end
+end

migration/relative_path.8.cr

@@ -0,0 +1,33 @@
+class RelativePath < MG::Base
+  def up : String
+    base = Config.current.library_path
+    # Escape single quotes in case the path contains them, and remove the
+    #   trailing slash (this is a mistake, fixed in DB version 10)
+    base = base.gsub("'", "''").rstrip "/"
+
+    <<-SQL
+    -- update the path column in ids to relative paths
+    UPDATE ids
+    SET path = REPLACE(path, '#{base}', '');
+
+    -- update the path column in titles to relative paths
+    UPDATE titles
+    SET path = REPLACE(path, '#{base}', '');
+    SQL
+  end
+
+  def down : String
+    base = Config.current.library_path
+    base = base.gsub("'", "''").rstrip "/"
+
+    <<-SQL
+    -- update the path column in ids to absolute paths
+    UPDATE ids
+    SET path = '#{base}' || path;
+
+    -- update the path column in titles to absolute paths
+    UPDATE titles
+    SET path = '#{base}' || path;
+    SQL
+  end
+end

migration/relative_path_fix.10.cr

@@ -0,0 +1,31 @@
+# In DB version 8, we replaced the absolute paths in DB with relative paths,
+#   but we mistakenly left the starting slashes. This migration removes them.
+class RelativePathFix < MG::Base
+  def up : String
+    <<-SQL
+    -- remove leading slashes from the paths in ids
+    UPDATE ids
+    SET path = SUBSTR(path, 2, LENGTH(path) - 1)
+    WHERE path LIKE '/%';
+
+    -- remove leading slashes from the paths in titles
+    UPDATE titles
+    SET path = SUBSTR(path, 2, LENGTH(path) - 1)
+    WHERE path LIKE '/%';
+    SQL
+  end
+
+  def down : String
+    <<-SQL
+    -- add leading slashes to paths in ids
+    UPDATE ids
+    SET path = '/' || path
+    WHERE path NOT LIKE '/%';
+
+    -- add leading slashes to paths in titles
+    UPDATE titles
+    SET path = '/' || path
+    WHERE path NOT LIKE '/%';
+    SQL
+  end
+end

migration/unavailable.9.cr

@@ -0,0 +1,94 @@
+class UnavailableIDs < MG::Base
+  def up : String
+    <<-SQL
+    -- add unavailable column to ids
+    ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+
+    -- add unavailable column to titles
+    ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+    SQL
+  end
+
+  def down : String
+    <<-SQL
+    -- remove unavailable column from ids
+    ALTER TABLE ids RENAME TO tmp;
+
+    CREATE TABLE ids (
+      path TEXT NOT NULL,
+      id TEXT NOT NULL,
+      signature TEXT
+    );
+
+    INSERT INTO ids
+    SELECT path, id, signature
+    FROM tmp;
+
+    DROP TABLE tmp;
+
+    -- recreate the indices
+    CREATE UNIQUE INDEX path_idx ON ids (path);
+    CREATE UNIQUE INDEX id_idx ON ids (id);
+
+    -- recreate the foreign key constraint on thumbnails
+    ALTER TABLE thumbnails RENAME TO tmp;
+
+    CREATE TABLE thumbnails (
+      id TEXT NOT NULL,
+      data BLOB NOT NULL,
+      filename TEXT NOT NULL,
+      mime TEXT NOT NULL,
+      size INTEGER NOT NULL,
+      FOREIGN KEY (id) REFERENCES ids (id)
+        ON UPDATE CASCADE
+        ON DELETE CASCADE
+    );
+
+    INSERT INTO thumbnails
+    SELECT * FROM tmp;
+
+    DROP TABLE tmp;
+
+    CREATE UNIQUE INDEX tn_index ON thumbnails (id);
+
+    -- remove unavailable column from titles
+    ALTER TABLE titles RENAME TO tmp;
+
+    CREATE TABLE titles (
+      id TEXT NOT NULL,
+      path TEXT NOT NULL,
+      signature TEXT
+    );
+
+    INSERT INTO titles
+    SELECT path, id, signature
+    FROM tmp;
+
+    DROP TABLE tmp;
+
+    -- recreate the indices
+    CREATE UNIQUE INDEX titles_id_idx on titles (id);
+    CREATE UNIQUE INDEX titles_path_idx on titles (path);
+
+    -- recreate the foreign key constraint on tags
+    ALTER TABLE tags RENAME TO tmp;
+
+    CREATE TABLE tags (
+      id TEXT NOT NULL,
+      tag TEXT NOT NULL,
+      UNIQUE (id, tag),
+      FOREIGN KEY (id) REFERENCES titles (id)
+        ON UPDATE CASCADE
+        ON DELETE CASCADE
+    );
+
+    INSERT INTO tags
+    SELECT * FROM tmp;
+
+    DROP TABLE tmp;
+
+    CREATE INDEX tags_id_idx ON tags (id);
+    CREATE INDEX tags_tag_idx ON tags (tag);
+    SQL
+  end
+end

public/css/mango.less

@@ -66,7 +66,6 @@
 
 // Dark theme
 .uk-light {
-	.uk-navbar-dropdown,
 	.uk-modal-header,
 	.uk-modal-body,
 	.uk-modal-footer {
@@ -75,6 +74,7 @@
 	.uk-navbar-dropdown,
 	.uk-dropdown {
 		color: #ccc;
+		background: #333;
 	}
 	.uk-nav-header,
 	.uk-description-list > dt {

public/js/download.js

@@ -156,8 +156,8 @@ const search = () => {
 			langs.unshift('All');
 			group_names.unshift('All');
 
-			$('select#lang-select').append(langs.map(e => `<option>${e}</option>`).join(''));
+			$('select#lang-select').html(langs.map(e => `<option>${e}</option>`).join(''));
-			$('select#group-select').append(group_names.map(e => `<option>${e}</option>`).join(''));
+			$('select#group-select').html(group_names.map(e => `<option>${e}</option>`).join(''));
 
 			$('#filter-form').removeAttr('hidden');
 
@@ -241,7 +241,7 @@ const buildTable = () => {
 		if (v === 'All') return;
 		if (k === 'group') {
 			chapters = chapters.filter(c => {
-				unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g));
+				const unescaped_groups = Object.entries(c.groups).map(([g, id]) => unescapeHTML(g));
 				return unescaped_groups.indexOf(v) >= 0;
 			});
 			return;

public/js/missing-items.js

@@ -0,0 +1,60 @@
+const component = () => {
+	return {
+		empty: true,
+		titles: [],
+		entries: [],
+		loading: true,
+
+		load() {
+			this.loading = true;
+			this.request('GET', `${base_url}api/admin/titles/missing`, data => {
+				this.titles = data.titles;
+				this.request('GET', `${base_url}api/admin/entries/missing`, data => {
+					this.entries = data.entries;
+					this.loading = false;
+					this.empty = this.entries.length === 0 && this.titles.length === 0;
+				});
+			});
+		},
+		rm(event) {
+			const rawID = event.currentTarget.closest('tr').id;
+			const [type, id] = rawID.split('-');
+			const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`;
+			this.request('DELETE', url, () => {
+				this.load();
+			});
+		},
+		rmAll() {
+			UIkit.modal.confirm('Are you sure? All metadata associated with these items, including their tags and thumbnails, will be deleted from the database.', {
+				labels: {
+					ok: 'Yes, delete them',
+					cancel: 'Cancel'
+				}
+			}).then(() => {
+				this.request('DELETE', `${base_url}api/admin/titles/missing`, () => {
+					this.request('DELETE', `${base_url}api/admin/entries/missing`, () => {
+						this.load();
+					});
+				});
+			});
+		},
+		request(method, url, cb) {
+			console.log(url);
+			$.ajax({
+					type: method,
+					url: url,
+					contentType: 'application/json'
+				})
+				.done(data => {
+					if (data.error) {
+						alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`);
+						return;
+					}
+					if (cb) cb(data);
+				})
+				.fail((jqXHR, status) => {
+					alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
+				});
+		}
+	};
+};

shard.yml

@@ -1,5 +1,5 @@
 name: mango
-version: 0.19.1
+version: 0.20.0
 
 authors:
   - Alex Ling <hkalexling@gmail.com>

spec/util_spec.cr

@@ -35,6 +35,23 @@ describe "compare_numerically" do
   end
 end
 
+describe "is_supported_file" do
+  it "returns true when the filename has a supported extension" do
+    filename = "manga.cbz"
+    is_supported_file(filename).should eq true
+  end
+
+  it "returns true when the filename does not have a supported extension" do
+    filename = "info.json"
+    is_supported_file(filename).should eq false
+  end
+
+  it "is case insensitive" do
+    filename = "manga.ZiP"
+    is_supported_file(filename).should eq true
+  end
+end
+
 describe "chapter_sort" do
   it "sorts correctly" do
     ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"]

src/config.cr

@@ -13,7 +13,6 @@ class Config
   property db_path : String = File.expand_path "~/mango/mango.db", home: true
   property scan_interval_minutes : Int32 = 5
   property thumbnail_generation_interval_hours : Int32 = 24
-  property db_optimization_interval_hours : Int32 = 24
   property log_level : String = "info"
   property upload_path : String = File.expand_path "~/mango/uploads",
     home: true

src/library/entry.cr

@@ -11,13 +11,13 @@ class Entry
     @title = File.basename @zip_path, File.extname @zip_path
     @encoded_title = URI.encode @title
     @size = (File.size @zip_path).humanize_bytes
-    id = storage.get_id @zip_path, false
+    id = storage.get_entry_id @zip_path, File.signature(@zip_path)
     if id.nil?
       id = random_str
-      storage.insert_id({
+      storage.insert_entry_id({
-        path:     @zip_path,
+        path:      @zip_path,
-        id:       id,
+        id:        id,
-        is_title: false,
+        signature: File.signature(@zip_path).to_s,
       })
     end
     @id = id

src/library/library.cr

@@ -42,16 +42,6 @@ class Library
         end
       end
     end
-
-    db_interval = Config.current.db_optimization_interval_hours
-    unless db_interval < 1
-      spawn do
-        loop do
-          Storage.default.optimize
-          sleep db_interval.hours
-        end
-      end
-    end
   end
 
   def titles
@@ -119,6 +109,7 @@ class Library
     storage.close
 
     Logger.debug "Scan completed"
+    Storage.default.mark_unavailable
   end
 
   def get_continue_reading_entries(username)
@@ -241,7 +232,7 @@ class Library
         e.generate_thumbnail
         # Sleep after each generation to minimize the impact on disk IO
         #   and CPU
-        sleep 0.5.seconds
+        sleep 1.seconds
       end
       @thumbnails_count += 1
     end

src/library/title.cr

@@ -3,19 +3,20 @@ require "../archive"
 class Title
   getter dir : String, parent_id : String, title_ids : Array(String),
     entries : Array(Entry), title : String, id : String,
-    encoded_title : String, mtime : Time
+    encoded_title : String, mtime : Time, signature : UInt64
 
   @entry_display_name_cache : Hash(String, String)?
 
   def initialize(@dir : String, @parent_id)
     storage = Storage.default
-    id = storage.get_id @dir, true
+    @signature = Dir.signature dir
+    id = storage.get_title_id dir, signature
     if id.nil?
       id = random_str
-      storage.insert_id({
+      storage.insert_title_id({
-        path:     @dir,
+        path:      dir,
-        id:       id,
+        id:        id,
-        is_title: true,
+        signature: signature.to_s,
       })
     end
     @id = id
@@ -35,7 +36,7 @@ class Title
         @title_ids << title.id
         next
       end
-      if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path
+      if is_supported_file path
         entry = Entry.new path, self
         @entries << entry if entry.pages > 0 || entry.err_msg
       end
@@ -61,6 +62,7 @@ class Title
       {% for str in ["dir", "title", "id"] %}
         json.field {{str}}, @{{str.id}}
       {% end %}
+      json.field "signature" { json.number @signature }
       json.field "display_name", display_name
       json.field "cover_url", cover_url
       json.field "mtime" { json.number @mtime.to_unix }

src/mango.cr

@@ -8,7 +8,7 @@ require "option_parser"
 require "clim"
 require "tallboy"
 
-MANGO_VERSION = "0.19.1"
+MANGO_VERSION = "0.20.0"
 
 # From http://www.network-science.de/ascii/
 BANNER = %{
@@ -63,7 +63,12 @@ class CLI < Clim
       Plugin::Downloader.default
 
       spawn do
-        Server.new.start
+        begin
+          Server.new.start
+        rescue e
+          Logger.fatal e
+          Process.exit 1
+        end
       end
 
       MainFiber.start_and_block

src/routes/admin.cr

@@ -1,6 +1,9 @@
 struct AdminRouter
   def initialize
     get "/admin" do |env|
+      storage = Storage.default
+      missing_count = storage.missing_titles.size +
+                      storage.missing_entries.size
       layout "admin"
     end
 
@@ -66,5 +69,9 @@ struct AdminRouter
       mangadex_base_url = Config.current.mangadex["base_url"]
       layout "download-manager"
     end
+
+    get "/admin/missing" do |env|
+      layout "missing-items"
+    end
   end
 end

src/routes/api.cr

@@ -166,6 +166,21 @@ struct APIRouter
       "error"   => "string?",
     }
 
+    Koa.object "missing", {
+      "path"      => "string",
+      "id"        => "string",
+      "signature" => "string",
+    }
+
+    Koa.array "missingAry", "$missing"
+
+    Koa.object "missingResult", {
+      "success" => "boolean",
+      "error"   => "string?",
+      "entries" => "$missingAry?",
+      "titles"  => "$missingAry?",
+    }
+
     Koa.describe "Returns a page in a manga entry"
     Koa.path "tid", desc: "Title ID"
     Koa.path "eid", desc: "Entry ID"
@@ -777,6 +792,120 @@ struct APIRouter
       end
     end
 
+    Koa.describe "Lists all missing titles"
+    Koa.response 200, ref: "$missingResult"
+    Koa.tag "admin"
+    get "/api/admin/titles/missing" do |env|
+      begin
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+          "titles"  => Storage.default.missing_titles,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Lists all missing entries"
+    Koa.response 200, ref: "$missingResult"
+    Koa.tag "admin"
+    get "/api/admin/entries/missing" do |env|
+      begin
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+          "entries" => Storage.default.missing_entries,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes all missing titles"
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/titles/missing" do |env|
+      begin
+        Storage.default.delete_missing_title
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes all missing entries"
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/entries/missing" do |env|
+      begin
+        Storage.default.delete_missing_entry
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes a missing title identified by `tid`", <<-MD
+    Does nothing if the given `tid` is not found or if the title is not missing.
+    MD
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/titles/missing/:tid" do |env|
+      begin
+        tid = env.params.url["tid"]
+        Storage.default.delete_missing_title tid
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes a missing entry identified by `eid`", <<-MD
+    Does nothing if the given `eid` is not found or if the entry is not missing.
+    MD
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/entries/missing/:eid" do |env|
+      begin
+        eid = env.params.url["eid"]
+        Storage.default.delete_missing_entry eid
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
     doc = Koa.generate
     @@api_json = doc.to_json if doc
 

src/routes/main.cr

@@ -154,6 +154,7 @@ struct MainRouter
     end
 
     get "/api" do |env|
+      base_url = Config.current.base_url
       render "src/views/api.html.ecr"
     end
   end

src/storage.cr

@@ -15,14 +15,16 @@ def verify_password(hash, pw)
 end
 
 class Storage
-  @@insert_ids = [] of IDTuple
+  @@insert_entry_ids = [] of IDTuple
+  @@insert_title_ids = [] of IDTuple
 
   @path : String
   @db : DB::Database?
 
-  alias IDTuple = NamedTuple(path: String,
+  alias IDTuple = NamedTuple(
+    path: String,
     id: String,
-    is_title: Bool)
+    signature: String?)
 
   use_default
 
@@ -230,24 +232,96 @@ class Storage
     end
   end
 
-  def get_id(path, is_title)
+  def get_title_id(path, signature)
     id = nil
+    path = Path.new(path).relative_to(Config.current.library_path).to_s
     MainFiber.run do
       get_db do |db|
-        if is_title
+        # First attempt to find the matching title in DB using BOTH path
-          id = db.query_one? "select id from titles where path = (?)", path,
+        #   and signature
-            as: String
+        id = db.query_one? "select id from titles where path = (?) and " \
-        else
+                           "signature = (?) and unavailable = 0",
-          id = db.query_one? "select id from ids where path = (?)", path,
+          path, signature.to_s, as: String
-            as: String
+
+        should_update = id.nil?
+        # If it fails, try to match using the path only. This could happen
+        #   for example when a new entry is added to the title
+        id ||= db.query_one? "select id from titles where path = (?)", path,
+          as: String
+
+        # If it still fails, we will have to rely on the signature values.
+        #   This could happen when the user moved or renamed the title, or
+        #   a title containing the title
+        unless id
+          # If there are multiple rows with the same signature (this could
+          #   happen simply by bad luck, or when the user copied a title),
+          #   pick the row that has the most similar path to the give path
+          rows = [] of Tuple(String, String)
+          db.query "select id, path from titles where signature = (?)",
+            signature.to_s do |rs|
+            rs.each do
+              rows << {rs.read(String), rs.read(String)}
+            end
+          end
+          row = rows.max_by?(&.[1].components_similarity(path))
+          id = row[0] if row
+        end
+
+        # At this point, `id` would still be nil if there's no row matching
+        #   either the path or the signature
+
+        # If we did identify a matching title, save the path and signature
+        #   values back to the DB
+        if id && should_update
+          db.exec "update titles set path = (?), signature = (?), " \
+                  "unavailable = 0 where id = (?)", path, signature.to_s, id
         end
       end
     end
     id
   end
 
-  def insert_id(tp : IDTuple)
+  # See the comments in `#get_title_id` to see how this method works.
-    @@insert_ids << tp
+  def get_entry_id(path, signature)
+    id = nil
+    path = Path.new(path).relative_to(Config.current.library_path).to_s
+    MainFiber.run do
+      get_db do |db|
+        id = db.query_one? "select id from ids where path = (?) and " \
+                           "signature = (?) and unavailable = 0",
+          path, signature.to_s, as: String
+
+        should_update = id.nil?
+        id ||= db.query_one? "select id from ids where path = (?)", path,
+          as: String
+
+        unless id
+          rows = [] of Tuple(String, String)
+          db.query "select id, path from ids where signature = (?)",
+            signature.to_s do |rs|
+            rs.each do
+              rows << {rs.read(String), rs.read(String)}
+            end
+          end
+          row = rows.max_by?(&.[1].components_similarity(path))
+          id = row[0] if row
+        end
+
+        if id && should_update
+          db.exec "update ids set path = (?), signature = (?), " \
+                  "unavailable = 0 where id = (?)", path, signature.to_s, id
+        end
+      end
+    end
+    id
+  end
+
+  def insert_entry_id(tp)
+    @@insert_entry_ids << tp
+  end
+
+  def insert_title_id(tp)
+    @@insert_title_ids << tp
   end
 
   def bulk_insert_ids
@@ -255,17 +329,24 @@ class Storage
       get_db do |db|
         db.transaction do |tran|
           conn = tran.connection
-          @@insert_ids.each do |tp|
+          @@insert_title_ids.each do |tp|
-            if tp[:is_title]
+            path = Path.new(tp[:path])
-              conn.exec "insert into titles values (?, ?, null)", tp[:id],
+              .relative_to(Config.current.library_path).to_s
-                tp[:path]
+            conn.exec "insert into titles (id, path, signature, " \
-            else
+                      "unavailable) values (?, ?, ?, 0)",
-              conn.exec "insert into ids values (?, ?)", tp[:path], tp[:id]
+              tp[:id], path, tp[:signature].to_s
-            end
+          end
+          @@insert_entry_ids.each do |tp|
+            path = Path.new(tp[:path])
+              .relative_to(Config.current.library_path).to_s
+            conn.exec "insert into ids (id, path, signature, " \
+                      "unavailable) values (?, ?, ?, 0)",
+              tp[:id], path, tp[:signature].to_s
           end
         end
       end
-      @@insert_ids.clear
+      @@insert_entry_ids.clear
+      @@insert_title_ids.clear
     end
   end
 
@@ -322,7 +403,8 @@ class Storage
     tags = [] of String
     MainFiber.run do
       get_db do |db|
-        db.query "select distinct tag from tags" do |rs|
+        db.query "select distinct tag from tags natural join titles " \
+                 "where unavailable = 0" do |rs|
           rs.each do
             tags << rs.read String
           end
@@ -354,42 +436,92 @@ class Storage
     end
   end
 
-  def optimize
+  def mark_unavailable
     MainFiber.run do
-      Logger.info "Starting DB optimization"
       get_db do |db|
-        # Delete dangling entry IDs
+        # Detect dangling entry IDs
         trash_ids = [] of String
-        db.query "select path, id from ids" do |rs|
+        db.query "select path, id from ids where unavailable = 0" do |rs|
           rs.each do
             path = rs.read String
-            trash_ids << rs.read String unless File.exists? path
+            fullpath = Path.new(path).expand(Config.current.library_path).to_s
+            trash_ids << rs.read String unless File.exists? fullpath
           end
         end
 
-        db.exec "delete from ids where id in " \
+        unless trash_ids.empty?
+          Logger.debug "Marking #{trash_ids.size} entries as unavailable"
+        end
+        db.exec "update ids set unavailable = 1 where id in " \
                 "(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
-        Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \
-           if trash_ids.size > 0
 
-        # Delete dangling title IDs
+        # Detect dangling title IDs
         trash_titles = [] of String
-        db.query "select path, id from titles" do |rs|
+        db.query "select path, id from titles where unavailable  = 0" do |rs|
           rs.each do
             path = rs.read String
-            trash_titles << rs.read String unless Dir.exists? path
+            fullpath = Path.new(path).expand(Config.current.library_path).to_s
+            trash_titles << rs.read String unless Dir.exists? fullpath
           end
         end
 
-        db.exec "delete from titles where id in " \
+        unless trash_titles.empty?
+          Logger.debug "Marking #{trash_titles.size} titles as unavailable"
+        end
+        db.exec "update titles set unavailable = 1 where id in " \
                 "(#{trash_titles.map { |i| "'#{i}'" }.join ","})"
-        Logger.debug "#{trash_titles.size} dangling title IDs deleted" \
-           if trash_titles.size > 0
       end
-      Logger.info "DB optimization finished"
     end
   end
 
+  private def get_missing(tablename)
+    ary = [] of IDTuple
+    MainFiber.run do
+      get_db do |db|
+        db.query "select id, path, signature from #{tablename} " \
+                 "where unavailable = 1" do |rs|
+          rs.each do
+            ary << {
+              id:        rs.read(String),
+              path:      rs.read(String),
+              signature: rs.read(String?),
+            }
+          end
+        end
+      end
+    end
+    ary
+  end
+
+  private def delete_missing(tablename, id : String? = nil)
+    MainFiber.run do
+      get_db do |db|
+        if id
+          db.exec "delete from #{tablename} where id = (?) " \
+                  "and unavailable = 1", id
+        else
+          db.exec "delete from #{tablename} where unavailable = 1"
+        end
+      end
+    end
+  end
+
+  def missing_entries
+    get_missing "ids"
+  end
+
+  def missing_titles
+    get_missing "titles"
+  end
+
+  def delete_missing_entry(id = nil)
+    delete_missing "ids", id
+  end
+
+  def delete_missing_title(id = nil)
+    delete_missing "titles", id
+  end
+
   def close
     MainFiber.run do
       unless @db.nil?

src/util/signature.cr

@@ -0,0 +1,51 @@
+require "./util"
+
+class File
+  abstract struct Info
+    def inode : UInt64
+      @stat.st_ino.to_u64
+    end
+  end
+
+  # Returns the signature of the file at filename.
+  # When it is not a supported file, returns 0. Otherwise, uses the inode
+  #   number as its signature. On most file systems, the inode number is
+  #   preserved even when the file is renamed, moved or edited.
+  # Some cases that would cause the inode number to change:
+  #   - Reboot/remount on some file systems
+  #   - Replaced with a copied file
+  #   - Moved to a different device
+  # Since we are also using the relative paths to match ids, we won't lose
+  #   information as long as the above changes do not happen together with
+  #   a file/folder rename, with no library scan in between.
+  def self.signature(filename) : UInt64
+    if is_supported_file filename
+      File.info(filename).inode
+    else
+      0u64
+    end
+  end
+end
+
+class Dir
+  # Returns the signature of the directory at dirname. See the comments for
+  #   `File.signature` for more information.
+  def self.signature(dirname) : UInt64
+    signatures = [File.info(dirname).inode]
+    self.open dirname do |dir|
+      dir.entries.each do |fn|
+        next if fn.starts_with? "."
+        path = File.join dirname, fn
+        if File.directory? path
+          signatures << Dir.signature path
+        else
+          _sig = File.signature path
+          # Only add its signature value to `signatures` when it is a
+          #   supported file
+          signatures << _sig if _sig > 0
+        end
+      end
+    end
+    Digest::CRC32.checksum(signatures.sort.join).to_u64
+  end
+end

src/util/util.cr

@@ -2,6 +2,7 @@ IMGS_PER_PAGE            = 5
 ENTRIES_IN_HOME_SECTIONS = 8
 UPLOAD_URL_PREFIX        = "/uploads"
 STATIC_DIRS              = ["/css", "/js", "/img", "/favicon.ico"]
+SUPPORTED_FILE_EXTNAMES  = [".zip", ".cbz", ".rar", ".cbr"]
 
 def random_str
   UUID.random.to_s.gsub "-", ""
@@ -31,6 +32,10 @@ def register_mime_types
   end
 end
 
+def is_supported_file(path)
+  SUPPORTED_FILE_EXTNAMES.includes? File.extname(path).downcase
+end
+
 struct Int
   def or(other : Int)
     if self == 0
@@ -92,3 +97,18 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String)
 
   ary
 end
+
+class String
+  # Returns the similarity (in [0, 1]) of two paths.
+  # For the two paths, separate them into arrays of components, count the
+  #   number of matching components backwards, and divide the count by the
+  #   number of components of the shorter path.
+  def components_similarity(other : String) : Float64
+    s, l = [self, other]
+      .map { |str| Path.new(str).parts }
+      .sort_by &.size
+
+    match = s.reverse.zip(l.reverse).count { |a, b| a == b }
+    match / s.size
+  end
+end

src/views/admin.html.ecr

@@ -1,5 +1,13 @@
 <ul class="uk-list uk-list-large uk-list-divider" x-data="component()" x-init="init()">
   <li><a class="uk-link-reset" href="<%= base_url %>admin/user">User Management</a></li>
+  <li>
+    <a class="uk-link-reset" href="<%= base_url %>admin/missing">Missing Items</a>
+    <% if missing_count > 0 %>
+      <div class="uk-align-right">
+        <span class="uk-badge"><%= missing_count %></span>
+      </div>
+    <% end %>
+  </li>
   <li>
     <a class="uk-link-reset" @click="scan()">
       <span :style="`${scanning ? 'color:grey' : ''}`">Scan Library Files</span>
@@ -19,7 +27,7 @@
   </li>
   <li>
     <span>Theme</span>
-    <select id="theme-select" class="uk-select uk-align-right uk-width-1-3@m uk-width-1-2" :val="themeSetting" @change="themeChanged($event)">
+    <select id="theme-select" class="uk-select uk-align-right uk-width-1-3@m uk-width-1-2" :value="themeSetting" @change="themeChanged($event)">
       <option>Dark</option>
       <option>Light</option>
       <option>System</option>

src/views/api.html.ecr

@@ -8,7 +8,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1">
   </head>
   <body>
-    <redoc spec-url="/openapi.json"></redoc>
+    <redoc spec-url="<%= base_url %>openapi.json"></redoc>
     <script src="https://cdn.jsdelivr.net/npm/redoc/bundles/redoc.standalone.js"></script>
   </body>
 </html>

src/views/missing-items.html.ecr

@@ -0,0 +1,40 @@
+<div x-data="component()" x-init="load()" x-cloak x-show="!loading">
+  <p x-show="empty" class="uk-text-lead uk-text-center">No missing items found.</p>
+  <div x-show="!empty">
+    <p>The following items were present in your library, but now we can't find them anymore. If you deleted them mistakenly, try to recover the files or folders, put them back to where they were, and rescan the library. Otherwise, you can safely delete them and the associated metadata using the buttons below to free up database space.</p>
+    <button class="uk-button uk-button-danger" @click="rmAll()">Delete All</button>
+    <table class="uk-table uk-table-striped uk-overflow-auto">
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Relative Path</th>
+          <th>ID</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody>
+        <template x-for="title in titles" :key="title">
+          <tr :id="`title-${title.id}`">
+            <td>Title</td>
+            <td x-text="title.path"></td>
+            <td x-text="title.id"></td>
+            <td><a @click="rm($event)" uk-icon="trash"></a></td>
+          </tr>
+        </template>
+        <template x-for="entry in entries" :key="entry">
+          <tr :id="`entry-${entry.id}`">
+            <td>Entry</td>
+            <td x-text="entry.path"></td>
+            <td x-text="entry.id"></td>
+            <td><a @click="rm($event)" uk-icon="trash"></a></td>
+          </tr>
+        </template>
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<% content_for "script" do %>
+  <script src="<%= base_url %>js/alert.js"></script>
+  <script src="<%= base_url %>js/missing-items.js"></script>
+<% end %>

src/views/title.html.ecr

@@ -125,7 +125,7 @@
 <% content_for "script" do %>
   <%= render_component "dots-scripts" %>
   <link href="https://cdn.jsdelivr.net/npm/select2@4.1.0-beta.1/dist/css/select2.min.css" rel="stylesheet" />
-  <link href="/css/tags.css" rel="stylesheet" />
+  <link href="<%= base_url %>css/tags.css" rel="stylesheet" />
   <script src="https://cdn.jsdelivr.net/npm/select2@4.1.0-beta.1/dist/js/select2.min.js"></script>
   <script src="<%= base_url %>js/alert.js"></script>
   <script src="<%= base_url %>js/title.js"></script>