Mark items unavailable and retire DB optimization

This prepares us for the moving metadata to DB in the future

migration/unavailable.9.cr

@@ -0,0 +1,94 @@
+class UnavailableIDs < MG::Base
+  def up : String
+    <<-SQL
+    -- add unavailable column to ids
+    ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+
+    -- add unavailable column to titles
+    ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0;
+    SQL
+  end
+
+  def down : String
+    <<-SQL
+    -- remove unavailable column from ids
+    ALTER TABLE ids RENAME TO tmp;
+
+    CREATE TABLE ids (
+      path TEXT NOT NULL,
+      id TEXT NOT NULL,
+      signature TEXT
+    );
+
+    INSERT INTO ids
+    SELECT path, id, signature
+    FROM tmp;
+
+    DROP TABLE tmp;
+
+    -- recreate the indices
+    CREATE UNIQUE INDEX path_idx ON ids (path);
+    CREATE UNIQUE INDEX id_idx ON ids (id);
+
+    -- recreate the foreign key constraint on thumbnails
+    ALTER TABLE thumbnails RENAME TO tmp;
+
+    CREATE TABLE thumbnails (
+      id TEXT NOT NULL,
+      data BLOB NOT NULL,
+      filename TEXT NOT NULL,
+      mime TEXT NOT NULL,
+      size INTEGER NOT NULL,
+      FOREIGN KEY (id) REFERENCES ids (id)
+        ON UPDATE CASCADE
+        ON DELETE CASCADE
+    );
+
+    INSERT INTO thumbnails
+    SELECT * FROM tmp;
+
+    DROP TABLE tmp;
+
+    CREATE UNIQUE INDEX tn_index ON thumbnails (id);
+
+    -- remove unavailable column from titles
+    ALTER TABLE titles RENAME TO tmp;
+
+    CREATE TABLE titles (
+      id TEXT NOT NULL,
+      path TEXT NOT NULL,
+      signature TEXT
+    );
+
+    INSERT INTO titles
+    SELECT path, id, signature
+    FROM tmp;
+
+    DROP TABLE tmp;
+
+    -- recreate the indices
+    CREATE UNIQUE INDEX titles_id_idx on titles (id);
+    CREATE UNIQUE INDEX titles_path_idx on titles (path);
+
+    -- recreate the foreign key constraint on tags
+    ALTER TABLE tags RENAME TO tmp;
+
+    CREATE TABLE tags (
+      id TEXT NOT NULL,
+      tag TEXT NOT NULL,
+      UNIQUE (id, tag),
+      FOREIGN KEY (id) REFERENCES titles (id)
+        ON UPDATE CASCADE
+        ON DELETE CASCADE
+    );
+
+    INSERT INTO tags
+    SELECT * FROM tmp;
+
+    DROP TABLE tmp;
+
+    CREATE INDEX tags_id_idx ON tags (id);
+    CREATE INDEX tags_tag_idx ON tags (tag);
+    SQL
+  end
+end

public/js/missing-items.js

@@ -0,0 +1,46 @@
+const component = () => {
+	return {
+		empty: true,
+		titles: [],
+		entries: [],
+		loading: true,
+
+		load() {
+			this.loading = true;
+			this.request('GET', `${base_url}api/admin/titles/missing`, data => {
+				this.titles = data.titles;
+				this.request('GET', `${base_url}api/admin/entries/missing`, data => {
+					this.entries = data.entries;
+					this.loading = false;
+					this.empty = this.entries.length === 0 && this.titles.length === 0;
+				});
+			});
+		},
+		rm(event) {
+			const rawID = event.currentTarget.closest('tr').id;
+			const [type, id] = rawID.split('-');
+			const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`;
+			this.request('DELETE', url, () => {
+				this.load();
+			});
+		},
+		request(method, url, cb) {
+			console.log(url);
+			$.ajax({
+					type: method,
+					url: url,
+					contentType: 'application/json'
+				})
+				.done(data => {
+					if (data.error) {
+						alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`);
+						return;
+					}
+					if (cb) cb(data);
+				})
+				.fail((jqXHR, status) => {
+					alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
+				});
+		}
+	};
+};

src/config.cr

@@ -13,7 +13,6 @@ class Config
   property db_path : String = File.expand_path "~/mango/mango.db", home: true
   property scan_interval_minutes : Int32 = 5
   property thumbnail_generation_interval_hours : Int32 = 24
-  property db_optimization_interval_hours : Int32 = 24
   property log_level : String = "info"
   property upload_path : String = File.expand_path "~/mango/uploads",
     home: true

src/library/library.cr

@@ -109,7 +109,7 @@ class Library
     storage.close
 
     Logger.debug "Scan completed"
-    Storage.default.optimize
+    Storage.default.mark_unavailable
   end
 
   def get_continue_reading_entries(username)

src/routes/admin.cr

@@ -1,6 +1,9 @@
 struct AdminRouter
   def initialize
     get "/admin" do |env|
+      storage = Storage.default
+      missing_count = storage.missing_titles.size +
+                      storage.missing_entries.size
       layout "admin"
     end
 
@@ -66,5 +69,9 @@ struct AdminRouter
       mangadex_base_url = Config.current.mangadex["base_url"]
       layout "download-manager"
     end
+
+    get "/admin/missing" do |env|
+      layout "missing-items"
+    end
   end
 end

src/routes/api.cr

@@ -166,6 +166,21 @@ struct APIRouter
       "error"   => "string?",
     }
 
+    Koa.object "missing", {
+      "path"      => "string",
+      "id"        => "string",
+      "signature" => "string",
+    }
+
+    Koa.array "missingAry", "$missing"
+
+    Koa.object "missingResult", {
+      "success" => "boolean",
+      "error"   => "string?",
+      "entries" => "$missingAry?",
+      "titles"  => "$missingAry?",
+    }
+
     Koa.describe "Returns a page in a manga entry"
     Koa.path "tid", desc: "Title ID"
     Koa.path "eid", desc: "Entry ID"
@@ -777,6 +792,80 @@ struct APIRouter
       end
     end
 
+    Koa.describe "Lists all missing titles"
+    Koa.response 200, ref: "$missingResult"
+    Koa.tag "admin"
+    get "/api/admin/titles/missing" do |env|
+      begin
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+          "titles"  => Storage.default.missing_titles,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Lists all missing entries"
+    Koa.response 200, ref: "$missingResult"
+    Koa.tag "admin"
+    get "/api/admin/entries/missing" do |env|
+      begin
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+          "entries" => Storage.default.missing_entries,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes a missing title with `tid`"
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/titles/missing/:tid" do |env|
+      begin
+        tid = env.params.url["tid"]
+        Storage.default.delete_missing_title tid
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    Koa.describe "Deletes a missing entry with `eid`"
+    Koa.response 200, ref: "$result"
+    Koa.tag "admin"
+    delete "/api/admin/entries/missing/:eid" do |env|
+      begin
+        eid = env.params.url["eid"]
+        Storage.default.delete_missing_entry eid
+        send_json env, {
+          "success" => true,
+          "error"   => nil,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
     doc = Koa.generate
     @@api_json = doc.to_json if doc
 

src/storage.cr

@@ -15,18 +15,13 @@ def verify_password(hash, pw)
 end
 
 class Storage
-  @@insert_entry_ids = [] of EntryID
+  @@insert_entry_ids = [] of IDTuple
-  @@insert_title_ids = [] of TitleID
+  @@insert_title_ids = [] of IDTuple
 
   @path : String
   @db : DB::Database?
 
-  alias EntryID = NamedTuple(
+  alias IDTuple = NamedTuple(
-    path: String,
-    id: String,
-    signature: String?)
-
-  alias TitleID = NamedTuple(
     path: String,
     id: String,
     signature: String?)
@@ -245,7 +240,8 @@ class Storage
         # First attempt to find the matching title in DB using BOTH path
         #   and signature
         id = db.query_one? "select id from titles where path = (?) and " \
-                           "signature = (?)", path, signature.to_s, as: String
+                           "signature = (?) and unavailable = 0",
+          path, signature.to_s, as: String
 
         should_update = id.nil?
         # If it fails, try to match using the path only. This could happen
@@ -277,8 +273,8 @@ class Storage
         # If we did identify a matching title, save the path and signature
         #   values back to the DB
         if id && should_update
-          db.exec "update titles set path = (?), signature = (?) " \
+          db.exec "update titles set path = (?), signature = (?), " \
-                  "where id = (?)", path, signature.to_s, id
+                  "unavailable = 0 where id = (?)", path, signature.to_s, id
         end
       end
     end
@@ -292,7 +288,8 @@ class Storage
     MainFiber.run do
       get_db do |db|
         id = db.query_one? "select id from ids where path = (?) and " \
-                           "signature = (?)", path, signature.to_s, as: String
+                           "signature = (?) and unavailable = 0",
+          path, signature.to_s, as: String
 
         should_update = id.nil?
         id ||= db.query_one? "select id from ids where path = (?)", path,
@@ -311,8 +308,8 @@ class Storage
         end
 
         if id && should_update
-          db.exec "update ids set path = (?), signature = (?) " \
+          db.exec "update ids set path = (?), signature = (?), " \
-                  "where id = (?)", path, signature.to_s, id
+                  "unavailable = 0 where id = (?)", path, signature.to_s, id
         end
       end
     end
@@ -335,14 +332,16 @@ class Storage
           @@insert_title_ids.each do |tp|
             path = Path.new(tp[:path])
               .relative_to(Config.current.library_path).to_s
-            conn.exec "insert into titles values (?, ?, ?)", tp[:id],
+            conn.exec "insert into titles (id, path, signature, " \
-              path, tp[:signature].to_s
+                      "unavailable) values (?, ?, ?, 0)",
+              tp[:id], path, tp[:signature].to_s
           end
           @@insert_entry_ids.each do |tp|
             path = Path.new(tp[:path])
               .relative_to(Config.current.library_path).to_s
-            conn.exec "insert into ids values (?, ?, ?)", path, tp[:id],
+            conn.exec "insert into ids (id, path, signature, " \
-              tp[:signature].to_s
+                      "unavailable) values (?, ?, ?, 0)",
+              tp[:id], path, tp[:signature].to_s
           end
         end
       end
@@ -404,7 +403,8 @@ class Storage
     tags = [] of String
     MainFiber.run do
       get_db do |db|
-        db.query "select distinct tag from tags" do |rs|
+        db.query "select distinct tag from tags natural join titles " \
+                 "where unavailable = 0" do |rs|
           rs.each do
             tags << rs.read String
           end
@@ -436,13 +436,12 @@ class Storage
     end
   end
 
-  def optimize
+  def mark_unavailable
     MainFiber.run do
-      Logger.info "Starting DB optimization"
       get_db do |db|
-        # Delete dangling entry IDs
+        # Detect dangling entry IDs
         trash_ids = [] of String
-        db.query "select path, id from ids" do |rs|
+        db.query "select path, id from ids where unavailable = 0" do |rs|
           rs.each do
             path = rs.read String
             fullpath = Path.new(path).expand(Config.current.library_path).to_s
@@ -450,14 +449,15 @@ class Storage
           end
         end
 
-        db.exec "delete from ids where id in " \
+        unless trash_ids.empty?
+          Logger.debug "Marking #{trash_ids.size} entries as unavailable"
+        end
+        db.exec "update ids set unavailable = 1 where id in " \
                 "(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
-        Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \
-           if trash_ids.size > 0
 
-        # Delete dangling title IDs
+        # Detect dangling title IDs
         trash_titles = [] of String
-        db.query "select path, id from titles" do |rs|
+        db.query "select path, id from titles where unavailable  = 0" do |rs|
           rs.each do
             path = rs.read String
             fullpath = Path.new(path).expand(Config.current.library_path).to_s
@@ -465,15 +465,59 @@ class Storage
           end
         end
 
-        db.exec "delete from titles where id in " \
+        unless trash_titles.empty?
+          Logger.debug "Marking #{trash_titles.size} titles as unavailable"
+        end
+        db.exec "update titles set unavailable = 1 where id in " \
                 "(#{trash_titles.map { |i| "'#{i}'" }.join ","})"
-        Logger.debug "#{trash_titles.size} dangling title IDs deleted" \
-           if trash_titles.size > 0
       end
-      Logger.info "DB optimization finished"
     end
   end
 
+  private def get_missing(tablename)
+    ary = [] of IDTuple
+    MainFiber.run do
+      get_db do |db|
+        db.query "select id, path, signature from #{tablename} " \
+                 "where unavailable = 1" do |rs|
+          rs.each do
+            ary << {
+              id:        rs.read(String),
+              path:      rs.read(String),
+              signature: rs.read(String?),
+            }
+          end
+        end
+      end
+    end
+    ary
+  end
+
+  private def delete_missing(tablename, id)
+    MainFiber.run do
+      get_db do |db|
+        db.exec "delete from #{tablename} where id = (?) and unavailable = 1",
+          id
+      end
+    end
+  end
+
+  def missing_entries
+    get_missing "ids"
+  end
+
+  def missing_titles
+    get_missing "titles"
+  end
+
+  def delete_missing_entry(id)
+    delete_missing "ids", id
+  end
+
+  def delete_missing_title(id)
+    delete_missing "titles", id
+  end
+
   def close
     MainFiber.run do
       unless @db.nil?

src/views/admin.html.ecr

@@ -1,5 +1,13 @@
 <ul class="uk-list uk-list-large uk-list-divider" x-data="component()" x-init="init()">
   <li><a class="uk-link-reset" href="<%= base_url %>admin/user">User Management</a></li>
+  <li>
+    <a class="uk-link-reset" href="<%= base_url %>admin/missing">Missing Items</a>
+    <% if missing_count > 0 %>
+      <div class="uk-align-right">
+        <span class="uk-badge"><%= missing_count %></span>
+      </div>
+    <% end %>
+  </li>
   <li>
     <a class="uk-link-reset" @click="scan()">
       <span :style="`${scanning ? 'color:grey' : ''}`">Scan Library Files</span>

src/views/missing-items.html.ecr

@@ -0,0 +1,39 @@
+<div x-data="component()" x-init="load()" x-cloak x-show="!loading">
+  <p x-show="empty" class="uk-text-lead uk-text-center">No missing items found.</p>
+  <div x-show="!empty">
+    <p>The following items were present in your library, but now we can't find them anymore. If you deleted them mistakenly, try to recover the files or folders, put them back to where they were, and rescan the library. Otherwise, you can safely delete them and the associated metadata using the buttons below to free up database space.</p>
+    <table class="uk-table uk-table-striped uk-overflow-auto">
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Relative Path</th>
+          <th>ID</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody>
+        <template x-for="title in titles" :key="title">
+          <tr :id="`title-${title.id}`">
+            <td>Title</td>
+            <td x-text="title.path"></td>
+            <td x-text="title.id"></td>
+            <td><a @click="rm($event)" uk-icon="trash"></a></td>
+          </tr>
+        </template>
+        <template x-for="entry in entries" :key="entry">
+          <tr :id="`entry-${entry.id}`">
+            <td>Entry</td>
+            <td x-text="entry.path"></td>
+            <td x-text="entry.id"></td>
+            <td><a @click="rm($event)" uk-icon="trash"></a></td>
+          </tr>
+        </template>
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<% content_for "script" do %>
+  <script src="<%= base_url %>js/alert.js"></script>
+  <script src="<%= base_url %>js/missing-items.js"></script>
+<% end %>