diff --git a/migration/unavailable.9.cr b/migration/unavailable.9.cr new file mode 100644 index 0000000..172a2f5 --- /dev/null +++ b/migration/unavailable.9.cr @@ -0,0 +1,94 @@ +class UnavailableIDs < MG::Base + def up : String + <<-SQL + -- add unavailable column to ids + ALTER TABLE ids ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + + -- add unavailable column to titles + ALTER TABLE titles ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0; + SQL + end + + def down : String + <<-SQL + -- remove unavailable column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO ids + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + + -- remove unavailable column from titles + ALTER TABLE titles RENAME TO tmp; + + CREATE TABLE titles ( + id TEXT NOT NULL, + path TEXT NOT NULL, + signature TEXT + ); + + INSERT INTO titles + SELECT path, id, signature + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX titles_id_idx on titles (id); + CREATE UNIQUE INDEX titles_path_idx on titles (path); + + -- recreate the foreign key constraint on tags + ALTER TABLE tags RENAME TO tmp; + + CREATE TABLE tags ( + id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE (id, tag), + FOREIGN KEY (id) REFERENCES titles (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO tags + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE INDEX tags_id_idx ON tags (id); + CREATE INDEX tags_tag_idx ON tags (tag); + SQL + end +end diff --git a/public/js/missing-items.js b/public/js/missing-items.js new file mode 100644 index 0000000..0d7fa08 --- /dev/null +++ b/public/js/missing-items.js @@ -0,0 +1,46 @@ +const component = () => { + return { + empty: true, + titles: [], + entries: [], + loading: true, + + load() { + this.loading = true; + this.request('GET', `${base_url}api/admin/titles/missing`, data => { + this.titles = data.titles; + this.request('GET', `${base_url}api/admin/entries/missing`, data => { + this.entries = data.entries; + this.loading = false; + this.empty = this.entries.length === 0 && this.titles.length === 0; + }); + }); + }, + rm(event) { + const rawID = event.currentTarget.closest('tr').id; + const [type, id] = rawID.split('-'); + const url = `${base_url}api/admin/${type === 'title' ? 'titles' : 'entries'}/missing/${id}`; + this.request('DELETE', url, () => { + this.load(); + }); + }, + request(method, url, cb) { + console.log(url); + $.ajax({ + type: method, + url: url, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to ${method} ${url}. Error: ${data.error}`); + return; + } + if (cb) cb(data); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to ${method} ${url}. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + } + }; +}; diff --git a/src/config.cr b/src/config.cr index 845fc94..3ac5af2 100644 --- a/src/config.cr +++ b/src/config.cr @@ -13,7 +13,6 @@ class Config property db_path : String = File.expand_path "~/mango/mango.db", home: true property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property db_optimization_interval_hours : Int32 = 24 property log_level : String = "info" property upload_path : String = File.expand_path "~/mango/uploads", home: true diff --git a/src/library/library.cr b/src/library/library.cr index 4f62c58..83c0b0a 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -109,7 +109,7 @@ class Library storage.close Logger.debug "Scan completed" - Storage.default.optimize + Storage.default.mark_unavailable end def get_continue_reading_entries(username) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 4445b0e..fd63ec8 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -1,6 +1,9 @@ struct AdminRouter def initialize get "/admin" do |env| + storage = Storage.default + missing_count = storage.missing_titles.size + + storage.missing_entries.size layout "admin" end @@ -66,5 +69,9 @@ struct AdminRouter mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end + + get "/admin/missing" do |env| + layout "missing-items" + end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d60710..cc61ba0 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -166,6 +166,21 @@ struct APIRouter "error" => "string?", } + Koa.object "missing", { + "path" => "string", + "id" => "string", + "signature" => "string", + } + + Koa.array "missingAry", "$missing" + + Koa.object "missingResult", { + "success" => "boolean", + "error" => "string?", + "entries" => "$missingAry?", + "titles" => "$missingAry?", + } + Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" @@ -777,6 +792,80 @@ struct APIRouter end end + Koa.describe "Lists all missing titles" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/titles/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "titles" => Storage.default.missing_titles, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Lists all missing entries" + Koa.response 200, ref: "$missingResult" + Koa.tag "admin" + get "/api/admin/entries/missing" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "entries" => Storage.default.missing_entries, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing title with `tid`" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/titles/missing/:tid" do |env| + begin + tid = env.params.url["tid"] + Storage.default.delete_missing_title tid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a missing entry with `eid`" + Koa.response 200, ref: "$result" + Koa.tag "admin" + delete "/api/admin/entries/missing/:eid" do |env| + begin + eid = env.params.url["eid"] + Storage.default.delete_missing_entry eid + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/storage.cr b/src/storage.cr index 97f6a66..548c9fc 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -15,18 +15,13 @@ def verify_password(hash, pw) end class Storage - @@insert_entry_ids = [] of EntryID - @@insert_title_ids = [] of TitleID + @@insert_entry_ids = [] of IDTuple + @@insert_title_ids = [] of IDTuple @path : String @db : DB::Database? - alias EntryID = NamedTuple( - path: String, - id: String, - signature: String?) - - alias TitleID = NamedTuple( + alias IDTuple = NamedTuple( path: String, id: String, signature: String?) @@ -245,7 +240,8 @@ class Storage # First attempt to find the matching title in DB using BOTH path # and signature id = db.query_one? "select id from titles where path = (?) and " \ - "signature = (?)", path, signature.to_s, as: String + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String should_update = id.nil? # If it fails, try to match using the path only. This could happen @@ -277,8 +273,8 @@ class Storage # If we did identify a matching title, save the path and signature # values back to the DB if id && should_update - db.exec "update titles set path = (?), signature = (?) " \ - "where id = (?)", path, signature.to_s, id + db.exec "update titles set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id end end end @@ -292,7 +288,8 @@ class Storage MainFiber.run do get_db do |db| id = db.query_one? "select id from ids where path = (?) and " \ - "signature = (?)", path, signature.to_s, as: String + "signature = (?) and unavailable = 0", + path, signature.to_s, as: String should_update = id.nil? id ||= db.query_one? "select id from ids where path = (?)", path, @@ -311,8 +308,8 @@ class Storage end if id && should_update - db.exec "update ids set path = (?), signature = (?) " \ - "where id = (?)", path, signature.to_s, id + db.exec "update ids set path = (?), signature = (?), " \ + "unavailable = 0 where id = (?)", path, signature.to_s, id end end end @@ -335,14 +332,16 @@ class Storage @@insert_title_ids.each do |tp| path = Path.new(tp[:path]) .relative_to(Config.current.library_path).to_s - conn.exec "insert into titles values (?, ?, ?)", tp[:id], - path, tp[:signature].to_s + conn.exec "insert into titles (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s end @@insert_entry_ids.each do |tp| path = Path.new(tp[:path]) .relative_to(Config.current.library_path).to_s - conn.exec "insert into ids values (?, ?, ?)", path, tp[:id], - tp[:signature].to_s + conn.exec "insert into ids (id, path, signature, " \ + "unavailable) values (?, ?, ?, 0)", + tp[:id], path, tp[:signature].to_s end end end @@ -404,7 +403,8 @@ class Storage tags = [] of String MainFiber.run do get_db do |db| - db.query "select distinct tag from tags" do |rs| + db.query "select distinct tag from tags natural join titles " \ + "where unavailable = 0" do |rs| rs.each do tags << rs.read String end @@ -436,13 +436,12 @@ class Storage end end - def optimize + def mark_unavailable MainFiber.run do - Logger.info "Starting DB optimization" get_db do |db| - # Delete dangling entry IDs + # Detect dangling entry IDs trash_ids = [] of String - db.query "select path, id from ids" do |rs| + db.query "select path, id from ids where unavailable = 0" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -450,14 +449,15 @@ class Storage end end - db.exec "delete from ids where id in " \ + unless trash_ids.empty? + Logger.debug "Marking #{trash_ids.size} entries as unavailable" + end + db.exec "update ids set unavailable = 1 where id in " \ "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_ids.size} dangling entry IDs deleted" \ - if trash_ids.size > 0 - # Delete dangling title IDs + # Detect dangling title IDs trash_titles = [] of String - db.query "select path, id from titles" do |rs| + db.query "select path, id from titles where unavailable = 0" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -465,15 +465,59 @@ class Storage end end - db.exec "delete from titles where id in " \ + unless trash_titles.empty? + Logger.debug "Marking #{trash_titles.size} titles as unavailable" + end + db.exec "update titles set unavailable = 1 where id in " \ "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" - Logger.debug "#{trash_titles.size} dangling title IDs deleted" \ - if trash_titles.size > 0 end - Logger.info "DB optimization finished" end end + private def get_missing(tablename) + ary = [] of IDTuple + MainFiber.run do + get_db do |db| + db.query "select id, path, signature from #{tablename} " \ + "where unavailable = 1" do |rs| + rs.each do + ary << { + id: rs.read(String), + path: rs.read(String), + signature: rs.read(String?), + } + end + end + end + end + ary + end + + private def delete_missing(tablename, id) + MainFiber.run do + get_db do |db| + db.exec "delete from #{tablename} where id = (?) and unavailable = 1", + id + end + end + end + + def missing_entries + get_missing "ids" + end + + def missing_titles + get_missing "titles" + end + + def delete_missing_entry(id) + delete_missing "ids", id + end + + def delete_missing_title(id) + delete_missing "titles", id + end + def close MainFiber.run do unless @db.nil? diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index 587aaa7..34cd51a 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -1,5 +1,13 @@