diff --git a/README.md b/README.md index b4d3d3a..e89a9a7 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.24.0 + Mango - Manga Server and Web Reader. Version 0.25.0 Usage: @@ -80,6 +80,7 @@ base_url: / session_secret: mango-session-secret library_path: ~/mango/library db_path: ~/mango/mango.db +queue_db_path: ~/mango/queue.db scan_interval_minutes: 5 thumbnail_generation_interval_hours: 24 log_level: info @@ -87,23 +88,15 @@ upload_path: ~/mango/uploads plugin_path: ~/mango/plugins download_timeout_seconds: 30 library_cache_path: ~/mango/library.yml.gz -cache_enabled: false +cache_enabled: true cache_size_mbs: 50 cache_log_enabled: true disable_login: false default_username: "" auth_proxy_header_name: "" -mangadex: - base_url: https://mangadex.org - api_url: https://api.mangadex.org/v2 - download_wait_seconds: 5 - download_retries: 4 - download_queue_db_path: ~/mango/queue.db - chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' - manga_rename_rule: '{title}' ``` -- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks +- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging - You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work. - By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`. diff --git a/migration/sort_title.12.cr b/migration/sort_title.12.cr new file mode 100644 index 0000000..9853182 --- /dev/null +++ b/migration/sort_title.12.cr @@ -0,0 +1,94 @@ +class SortTitle < MG::Base + def up : String + <<-SQL + -- add sort_title column to ids and titles + ALTER TABLE ids ADD COLUMN sort_title TEXT; + ALTER TABLE titles ADD COLUMN sort_title TEXT; + SQL + end + + def down : String + <<-SQL + -- remove sort_title column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL, + signature TEXT, + unavailable INTEGER NOT NULL DEFAULT 0 + ); + + INSERT INTO ids + SELECT path, id, signature, unavailable + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + + -- remove sort_title column from titles + ALTER TABLE titles RENAME TO tmp; + + CREATE TABLE titles ( + id TEXT NOT NULL, + path TEXT NOT NULL, + signature TEXT, + unavailable INTEGER NOT NULL DEFAULT 0 + ); + + INSERT INTO titles + SELECT id, path, signature, unavailable + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX titles_id_idx on titles (id); + CREATE UNIQUE INDEX titles_path_idx on titles (path); + + -- recreate the foreign key constraint on tags + ALTER TABLE tags RENAME TO tmp; + + CREATE TABLE tags ( + id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE (id, tag), + FOREIGN KEY (id) REFERENCES titles (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO tags + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE INDEX tags_id_idx ON tags (id); + CREATE INDEX tags_tag_idx ON tags (tag); + SQL + end +end diff --git a/public/js/download-manager.js b/public/js/download-manager.js index 0393dd3..1183ce5 100644 --- a/public/js/download-manager.js +++ b/public/js/download-manager.js @@ -55,7 +55,7 @@ const component = () => { jobAction(action, event) { let url = `${base_url}api/admin/mangadex/queue/${action}`; if (event) { - const id = event.currentTarget.closest('tr').id.split('-')[1]; + const id = event.currentTarget.closest('tr').id.split('-').slice(1).join('-'); url = `${url}?${$.param({ id: id })}`; diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index a335e03..11c047c 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -68,7 +68,12 @@ const buildTable = (chapters) => { $('table').append(thead); const rows = chapters.map(ch => { - const tds = Object.values(ch).map(v => `${v}`).join(''); + const tds = Object.values(ch).map(v => { + const maxLength = 40; + const shouldShrink = v && v.length > maxLength; + const content = shouldShrink ? `${v.substring(0, maxLength)}...
${v}
` : v; + return `${content}` + }).join(''); return `${tds}`; }); const tbody = `${rows}`; diff --git a/public/js/reader.js b/public/js/reader.js index 9b17276..2cb3a66 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -13,6 +13,7 @@ const readerComponent = () => { selectedIndex: 0, // 0: not selected; 1: the first page margin: 30, preloadLookahead: 3, + enableRightToLeft: false, /** * Initialize the component by fetching the page dimensions @@ -64,6 +65,13 @@ const readerComponent = () => { const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; + + const savedRightToLeft = localStorage.getItem('enableRightToLeft'); + if (savedRightToLeft === null) { + this.enableRightToLeft = false; + } else { + this.enableRightToLeft = (savedRightToLeft === 'true'); + } }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -114,9 +122,9 @@ const readerComponent = () => { if (this.mode === 'continuous') return; if (event.key === 'ArrowLeft' || event.key === 'k') - this.flipPage(false); + this.flipPage(false ^ this.enableRightToLeft); if (event.key === 'ArrowRight' || event.key === 'j') - this.flipPage(true); + this.flipPage(true ^ this.enableRightToLeft); }, /** * Flips to the next or the previous page @@ -136,7 +144,7 @@ const readerComponent = () => { this.toPage(newIdx); if (this.enableFlipAnimation) { - if (isNext) + if (isNext ^ this.enableRightToLeft) this.flipAnimation = 'right'; else this.flipAnimation = 'left'; @@ -320,5 +328,9 @@ const readerComponent = () => { enableFlipAnimationChanged() { localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation); }, + + enableRightToLeftChanged() { + localStorage.setItem('enableRightToLeft', this.enableRightToLeft); + }, }; } diff --git a/public/js/title.js b/public/js/title.js index 1aca6d6..5d0e49f 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -60,6 +60,11 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi UIkit.modal($('#modal')).show(); } +UIkit.util.on(document, 'hidden', '#modal', () => { + $('#read-btn').off('click'); + $('#unread-btn').off('click'); +}); + const updateProgress = (tid, eid, page) => { let url = `${base_url}api/progress/${tid}/${page}` const query = $.param({ @@ -90,8 +95,6 @@ const renameSubmit = (name, eid) => { const upload = $('.upload-field'); const titleId = upload.attr('data-title-id'); - console.log(name); - if (name.length === 0) { alert('danger', 'The display name should not be empty'); return; @@ -122,15 +125,47 @@ const renameSubmit = (name, eid) => { }); }; +const renameSortNameSubmit = (name, eid) => { + const upload = $('.upload-field'); + const titleId = upload.attr('data-title-id'); + + const params = {}; + if (eid) params.eid = eid; + if (name) params.name = name; + const query = $.param(params); + let url = `${base_url}api/admin/sort_title/${titleId}?${query}`; + + $.ajax({ + type: 'PUT', + url, + contentType: 'application/json', + dataType: 'json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to update sort title. Error: ${data.error}`); + return; + } + location.reload(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to update sort title. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); +}; + const edit = (eid) => { const cover = $('#edit-modal #cover'); let url = cover.attr('data-title-cover'); let displayName = $('h2.uk-title > span').text(); + let fileTitle = $('h2.uk-title').attr('data-file-title'); + let sortTitle = $('h2.uk-title').attr('data-sort-title'); if (eid) { const item = $(`#${eid}`); url = item.find('img').attr('data-src'); displayName = item.find('.uk-card-title').attr('data-title'); + fileTitle = item.find('.uk-card-title').attr('data-file-title'); + sortTitle = item.find('.uk-card-title').attr('data-sort-title'); $('#title-progress-control').attr('hidden', ''); } else { $('#title-progress-control').removeAttr('hidden'); @@ -140,14 +175,26 @@ const edit = (eid) => { const displayNameField = $('#display-name-field'); displayNameField.attr('value', displayName); - console.log(displayNameField); + displayNameField.attr('placeholder', fileTitle); displayNameField.keyup(event => { if (event.keyCode === 13) { - renameSubmit(displayNameField.val(), eid); + renameSubmit(displayNameField.val() || fileTitle, eid); } }); displayNameField.siblings('a.uk-form-icon').click(() => { - renameSubmit(displayNameField.val(), eid); + renameSubmit(displayNameField.val() || fileTitle, eid); + }); + + const sortTitleField = $('#sort-title-field'); + sortTitleField.val(sortTitle); + sortTitleField.attr('placeholder', fileTitle); + sortTitleField.keyup(event => { + if (event.keyCode === 13) { + renameSortNameSubmit(sortTitleField.val(), eid); + } + }); + sortTitleField.siblings('a.uk-form-icon').click(() => { + renameSortNameSubmit(sortTitleField.val(), eid); }); setupUpload(eid); @@ -155,6 +202,16 @@ const edit = (eid) => { UIkit.modal($('#edit-modal')).show(); }; +UIkit.util.on(document, 'hidden', '#edit-modal', () => { + const displayNameField = $('#display-name-field'); + displayNameField.off('keyup'); + displayNameField.off('click'); + + const sortTitleField = $('#sort-title-field'); + sortTitleField.off('keyup'); + sortTitleField.off('click'); +}); + const setupUpload = (eid) => { const upload = $('.upload-field'); const bar = $('#upload-progress').get(0); @@ -166,7 +223,6 @@ const setupUpload = (eid) => { queryObj['eid'] = eid; const query = $.param(queryObj); const url = `${base_url}api/admin/upload/cover?${query}`; - console.log(url); UIkit.upload('.upload-field', { url: url, name: 'file', diff --git a/shard.yml b/shard.yml index 0054a23..44a0924 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.24.0 +version: 0.25.0 authors: - Alex Ling diff --git a/src/config.cr b/src/config.cr index aa818c3..b5b77db 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,43 +4,27 @@ class Config include YAML::Serializable @[YAML::Field(ignore: true)] - property path : String = "" - property host : String = "0.0.0.0" + property path = "" + property host = "0.0.0.0" property port : Int32 = 9000 - property base_url : String = "/" - property session_secret : String = "mango-session-secret" - property library_path : String = File.expand_path "~/mango/library", - home: true - property library_cache_path = File.expand_path "~/mango/library.yml.gz", - home: true - property db_path : String = File.expand_path "~/mango/mango.db", home: true + property base_url = "/" + property session_secret = "mango-session-secret" + property library_path = "~/mango/library" + property library_cache_path = "~/mango/library.yml.gz" + property db_path = "~/mango/mango.db" + property queue_db_path = "~/mango/queue.db" property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property log_level : String = "info" - property upload_path : String = File.expand_path "~/mango/uploads", - home: true - property plugin_path : String = File.expand_path "~/mango/plugins", - home: true + property log_level = "info" + property upload_path = "~/mango/uploads" + property plugin_path = "~/mango/plugins" property download_timeout_seconds : Int32 = 30 - property cache_enabled = false + property cache_enabled = true property cache_size_mbs = 50 property cache_log_enabled = true property disable_login = false property default_username = "" property auth_proxy_header_name = "" - property mangadex = Hash(String, String | Int32).new - - @[YAML::Field(ignore: true)] - @mangadex_defaults = { - "base_url" => "https://mangadex.org", - "api_url" => "https://api.mangadex.org/v2", - "download_wait_seconds" => 5, - "download_retries" => 4, - "download_queue_db_path" => File.expand_path("~/mango/queue.db", - home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", - } @@singlet : Config? @@ -58,7 +42,7 @@ class Config if File.exists? cfg_path config = self.from_yaml File.read cfg_path config.path = path - config.fill_defaults + config.expand_paths config.preprocess return config end @@ -66,7 +50,7 @@ class Config "Dumping the default config there." default = self.allocate default.path = path - default.fill_defaults + default.expand_paths cfg_dir = File.dirname cfg_path unless Dir.exists? cfg_dir Dir.mkdir_p cfg_dir @@ -76,13 +60,9 @@ class Config default end - def fill_defaults - {% for hash_name in ["mangadex"] %} - @{{hash_name.id}}_defaults.map do |k, v| - if @{{hash_name.id}}[k]?.nil? - @{{hash_name.id}}[k] = v - end - end + def expand_paths + {% for p in %w(library library_cache db queue_db upload plugin) %} + @{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true {% end %} end @@ -97,24 +77,5 @@ class Config raise "Login is disabled, but default username is not set. " \ "Please set a default username" end - - # `Logger.default` is not available yet - Log.setup :debug - unless mangadex["api_url"] =~ /\/v2/ - Log.warn { "It looks like you are using the deprecated MangaDex API " \ - "v1 in your config file. Please update it to " \ - "https://api.mangadex.org/v2 to suppress this warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - if mangadex["api_url"] =~ /\/api\/v2/ - Log.warn { "It looks like you are using the outdated MangaDex API " \ - "url (mangadex.org/api/v2) in your config file. Please " \ - "update it to https://api.mangadex.org/v2 to suppress this " \ - "warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - - mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/" - mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/" end end diff --git a/src/handlers/auth_handler.cr b/src/handlers/auth_handler.cr index 692fa8a..bf79dc3 100644 --- a/src/handlers/auth_handler.cr +++ b/src/handlers/auth_handler.cr @@ -54,8 +54,9 @@ class AuthHandler < Kemal::Handler end def call(env) - # Skip all authentication if requesting /login, /logout, or a static file - if request_path_startswith(env, ["/login", "/logout"]) || + # Skip all authentication if requesting /login, /logout, /api/login, + # or a static file + if request_path_startswith(env, ["/login", "/logout", "/api/login"]) || requesting_static_file env return call_next(env) end diff --git a/src/library/cache.cr b/src/library/cache.cr index d0d3f01..10e4f60 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -1,6 +1,7 @@ require "digest" require "./entry" +require "./title" require "./types" # Base class for an entry in the LRU cache. @@ -81,6 +82,31 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end end +class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title)) + def self.to_save_t(value : Array(Title)) + value.map &.id + end + + def self.to_return_t(value : Array(String)) + value.map { |title_id| Library.default.title_hash[title_id].not_nil! } + end + + def instance_size + instance_sizeof(SortedTitlesCacheEntry) + # sizeof itself + instance_sizeof(String) + @key.bytesize + # allocated memory for @key + @value.size * (instance_sizeof(String) + sizeof(String)) + + @value.sum(&.bytesize) # elements in Array(String) + end + + def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?) + titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s + user_context = opt && opt.method == SortMethod::Progress ? username : "" + sig = Digest::SHA1.hexdigest (titles_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) + "#{sig}:sorted_titles" + end +end + class String def instance_size instance_sizeof(String) + bytesize @@ -101,14 +127,18 @@ struct Tuple(*T) end end -alias CacheableType = Array(Entry) | String | Tuple(String, Int32) +alias CacheableType = Array(Entry) | Array(Title) | String | + Tuple(String, Int32) alias CacheEntryType = SortedEntriesCacheEntry | + SortedTitlesCacheEntry | CacheEntry(String, String) | CacheEntry(Tuple(String, Int32), Tuple(String, Int32)) def generate_cache_entry(key : String, value : CacheableType) if value.is_a? Array(Entry) SortedEntriesCacheEntry.new key, value + elsif value.is_a? Array(Title) + SortedTitlesCacheEntry.new key, value else CacheEntry(typeof(value), typeof(value)).new key, value end diff --git a/src/library/entry.cr b/src/library/entry.cr index 43fbb23..ceaa531 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -8,6 +8,9 @@ class Entry size : String, pages : Int32, id : String, encoded_path : String, encoded_title : String, mtime : Time, err_msg : String? + @[YAML::Field(ignore: true)] + @sort_title : String? + def initialize(@zip_path, @book) storage = Storage.default @encoded_path = URI.encode @zip_path @@ -56,6 +59,7 @@ class Entry json.field {{str}}, @{{str.id}} {% end %} json.field "title_id", @book.id + json.field "sort_title", sort_title json.field "pages" { json.number @pages } unless slim json.field "display_name", @book.display_name @title @@ -66,6 +70,35 @@ class Entry end end + def sort_title + sort_title_cached = @sort_title + return sort_title_cached if sort_title_cached + sort_title = @book.entry_sort_title_db id + if sort_title + @sort_title = sort_title + return sort_title + end + @sort_title = @title + @title + end + + def set_sort_title(sort_title : String | Nil, username : String) + Storage.default.set_entry_sort_title id, sort_title + if sort_title == "" || sort_title.nil? + @sort_title = nil + else + @sort_title = sort_title + end + + @book.entry_sort_title_cache = nil + @book.remove_sorted_entries_cache [SortMethod::Auto, SortMethod::Title], + username + end + + def sort_title_db + @book.entry_sort_title_db @id + end + def display_name @book.display_name @title end @@ -177,11 +210,7 @@ class Entry @book.parents.each do |parent| LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end - [false, true].each do |ascend| - sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, - username, @book.entries, SortOptions.new(SortMethod::Progress, ascend) - LRUCache.invalidate sorted_entries_cache_key - end + @book.remove_sorted_caches [SortMethod::Progress], username TitleInfo.new @book.dir do |info| if info.progress[username]?.nil? diff --git a/src/library/library.cr b/src/library/library.cr index 93eac0c..210912f 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,9 +1,38 @@ class Library + struct ThumbnailContext + property current : Int32, total : Int32 + + def initialize + @current = 0 + @total = 0 + end + + def progress + if total == 0 + 0 + else + current / total + end + end + + def reset + @current = 0 + @total = 0 + end + + def increment + @current += 1 + end + end + include YAML::Serializable getter dir : String, title_ids : Array(String), title_hash : Hash(String, Title) + @[YAML::Field(ignore: true)] + getter thumbnail_ctx = ThumbnailContext.new + use_default def save_instance @@ -24,7 +53,23 @@ class Library begin Compress::Gzip::Reader.open path do |content| - @@default = Library.from_yaml content + loaded = Library.from_yaml content + # We will have to do a full restart in these cases. Otherwise having + # two instances of the library will cause some weirdness. + if loaded.dir != Config.current.library_path + Logger.fatal "Cached library dir #{loaded.dir} does not match " \ + "current library dir #{Config.current.library_path}. " \ + "Deleting cache" + delete_cache_and_exit path + end + if loaded.title_ids.size > 0 && + Storage.default.count_titles == 0 + Logger.fatal "The library cache is inconsistent with the DB. " \ + "Deleting cache" + delete_cache_and_exit path + end + @@default = loaded + Logger.debug "Library cache loaded" end Library.default.register_jobs rescue e @@ -39,9 +84,6 @@ class Library @title_ids = [] of String @title_hash = {} of String => Title - @entries_count = 0 - @thumbnails_count = 0 - register_jobs end @@ -136,8 +178,12 @@ class Library deleted_entry_ids: [] of String, } + library_paths = (Dir.entries @dir) + .select { |fn| !fn.starts_with? "." } + .map { |fn| File.join @dir, fn } @title_ids.select! do |title_id| title = @title_hash[title_id] + next false unless library_paths.includes? title.dir existence = title.examine examine_context unless existence examine_context["deleted_title_ids"].concat [title_id] + @@ -152,14 +198,12 @@ class Library end cache = examine_context["cached_contents_signature"] - (Dir.entries @dir) - .select { |fn| !fn.starts_with? "." } - .map { |fn| File.join @dir, fn } + library_paths .select { |path| !(remained_title_dirs.includes? path) } .select { |path| File.directory? path } .map { |path| Title.new path, "", cache } .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort! { |a, b| a.title <=> b.title } + .sort! { |a, b| a.sort_title <=> b.sort_title } .each do |title| @title_hash[title.id] = title @title_ids << title.id @@ -260,34 +304,29 @@ class Library .shuffle! end - def thumbnail_generation_progress - return 0 if @entries_count == 0 - @thumbnails_count / @entries_count - end - def generate_thumbnails - if @thumbnails_count > 0 + if thumbnail_ctx.current > 0 Logger.debug "Thumbnail generation in progress" return end Logger.info "Starting thumbnail generation" entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg - @entries_count = entries.size - @thumbnails_count = 0 + thumbnail_ctx.total = entries.size + thumbnail_ctx.current = 0 # Report generation progress regularly spawn do loop do - unless @thumbnails_count == 0 + unless thumbnail_ctx.current == 0 Logger.debug "Thumbnail generation progress: " \ - "#{(thumbnail_generation_progress * 100).round 1}%" + "#{(thumbnail_ctx.progress * 100).round 1}%" end # Generation is completed. We reset the count to 0 to allow subsequent # calls to the function, and break from the loop to stop the progress # report fiber - if thumbnail_generation_progress.to_i == 1 - @thumbnails_count = 0 + if thumbnail_ctx.progress.to_i == 1 + thumbnail_ctx.reset break end sleep 10.seconds @@ -301,7 +340,7 @@ class Library # and CPU sleep 1.seconds end - @thumbnails_count += 1 + thumbnail_ctx.increment end Logger.info "Thumbnail generation finished" end diff --git a/src/library/title.cr b/src/library/title.cr index 9b797f4..539f114 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -8,8 +8,13 @@ class Title entries : Array(Entry), title : String, id : String, encoded_title : String, mtime : Time, signature : UInt64, entry_cover_url_cache : Hash(String, String)? - setter entry_cover_url_cache : Hash(String, String)? + setter entry_cover_url_cache : Hash(String, String)?, + entry_sort_title_cache : Hash(String, String | Nil)? + @[YAML::Field(ignore: true)] + @sort_title : String? + @[YAML::Field(ignore: true)] + @entry_sort_title_cache : Hash(String, String | Nil)? @[YAML::Field(ignore: true)] @entry_display_name_cache : Hash(String, String)? @[YAML::Field(ignore: true)] @@ -66,7 +71,7 @@ class Title end sorter = ChapterSorter.new @entries.map &.title @entries.sort! do |a, b| - sorter.compare a.title, b.title + sorter.compare a.sort_title, b.sort_title end end @@ -102,7 +107,11 @@ class Title previous_titles_size = @title_ids.size @title_ids.select! do |title_id| - title = Library.default.get_title! title_id + title = Library.default.get_title title_id + unless title # for if data consistency broken + context["deleted_title_ids"].concat [title_id] + next false + end existence = title.examine context unless existence context["deleted_title_ids"].concat [title_id] + @@ -137,6 +146,18 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id is_titles_added = true + + # We think they are removed, but they are here! + # Cancel reserved jobs + revival_title_ids = [title.id] + title.deep_titles.map &.id + context["deleted_title_ids"].select! do |deleted_title_id| + !(revival_title_ids.includes? deleted_title_id) + end + revival_entry_ids = title.deep_entries.map &.id + context["deleted_entry_ids"].select! do |deleted_entry_id| + !(revival_entry_ids.includes? deleted_entry_id) + end + next end if is_supported_file path @@ -145,6 +166,9 @@ class Title if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true + context["deleted_entry_ids"].select! do |deleted_entry_id| + entry.id != deleted_entry_id + end end end end @@ -161,13 +185,18 @@ class Title end end if is_entries_added || previous_entries_size != @entries.size - sorter = ChapterSorter.new @entries.map &.title + sorter = ChapterSorter.new @entries.map &.sort_title @entries.sort! do |a, b| - sorter.compare a.title, b.title + sorter.compare a.sort_title, b.sort_title end end - true + if @title_ids.size > 0 || @entries.size > 0 + true + else + context["deleted_title_ids"].concat [@id] + false + end end alias SortContext = NamedTuple(username: String, opt: SortOptions) @@ -180,6 +209,7 @@ class Title json.field {{str}}, @{{str.id}} {% end %} json.field "signature" { json.number @signature } + json.field "sort_title", sort_title unless slim json.field "display_name", display_name json.field "cover_url", cover_url @@ -226,6 +256,15 @@ class Title @title_ids.map { |tid| Library.default.get_title! tid } end + def sorted_titles(username, opt : SortOptions? = nil) + if opt.nil? + opt = SortOptions.from_info_json @dir, username + end + + # Helper function from src/util/util.cr + sort_titles titles, opt.not_nil!, username + end + # Get all entries, including entries in nested titles def deep_entries return @entries if title_ids.empty? @@ -262,6 +301,48 @@ class Title ary.join " and " end + def sort_title + sort_title_cached = @sort_title + return sort_title_cached if sort_title_cached + sort_title = Storage.default.get_title_sort_title id + if sort_title + @sort_title = sort_title + return sort_title + end + @sort_title = @title + @title + end + + def set_sort_title(sort_title : String | Nil, username : String) + Storage.default.set_title_sort_title id, sort_title + if sort_title == "" || sort_title.nil? + @sort_title = nil + else + @sort_title = sort_title + end + + if parents.size > 0 + target = parents[-1].titles + else + target = Library.default.titles + end + remove_sorted_titles_cache target, + [SortMethod::Auto, SortMethod::Title], username + end + + def sort_title_db + Storage.default.get_title_sort_title id + end + + def entry_sort_title_db(entry_id) + unless @entry_sort_title_cache + @entry_sort_title_cache = + Storage.default.get_entries_sort_title @entries.map &.id + end + + @entry_sort_title_cache.not_nil![entry_id]? + end + def tags Storage.default.get_title_tags @id end @@ -448,28 +529,30 @@ class Title case opt.not_nil!.method when .title? - ary = @entries.sort { |a, b| compare_numerically a.title, b.title } + ary = @entries.sort do |a, b| + compare_numerically a.sort_title, b.sort_title + end when .time_modified? ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \ - compare_numerically a.title, b.title } + compare_numerically a.sort_title, b.sort_title } when .time_added? ary = @entries.sort { |a, b| (a.date_added <=> b.date_added).or \ - compare_numerically a.title, b.title } + compare_numerically a.sort_title, b.sort_title } when .progress? percentage_ary = load_percentage_for_all_entries username, opt, true ary = @entries.zip(percentage_ary) .sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \ - compare_numerically a_tp[0].title, b_tp[0].title } + compare_numerically a_tp[0].sort_title, b_tp[0].sort_title } .map &.[0] else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - sorter = ChapterSorter.new @entries.map &.title + sorter = ChapterSorter.new @entries.map &.sort_title ary = @entries.sort do |a, b| - sorter.compare(a.title, b.title).or \ - compare_numerically a.title, b.title + sorter.compare(a.sort_title, b.sort_title).or \ + compare_numerically a.sort_title, b.sort_title end end @@ -536,17 +619,32 @@ class Title zip + titles.flat_map &.deep_entries_with_date_added end + def remove_sorted_entries_cache(sort_methods : Array(SortMethod), + username : String) + [false, true].each do |ascend| + sort_methods.each do |sort_method| + sorted_entries_cache_key = + SortedEntriesCacheEntry.gen_key @id, username, @entries, + SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_entries_cache_key + end + end + end + + def remove_sorted_caches(sort_methods : Array(SortMethod), username : String) + remove_sorted_entries_cache sort_methods, username + parents.each do |parent| + remove_sorted_titles_cache parent.titles, sort_methods, username + end + remove_sorted_titles_cache Library.default.titles, sort_methods, username + end + def bulk_progress(action, ids : Array(String), username) LRUCache.invalidate "#{@id}:#{username}:progress_sum" parents.each do |parent| LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end - [false, true].each do |ascend| - sorted_entries_cache_key = - SortedEntriesCacheEntry.gen_key @id, username, @entries, - SortOptions.new(SortMethod::Progress, ascend) - LRUCache.invalidate sorted_entries_cache_key - end + remove_sorted_caches [SortMethod::Progress], username selected_entries = ids .map { |id| diff --git a/src/mango.cr b/src/mango.cr index 8716d04..3cdafc0 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.24.0" +MANGO_VERSION = "0.25.0" # From http://www.network-science.de/ascii/ BANNER = %{ diff --git a/src/queue.cr b/src/queue.cr index 381441b..01cef38 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -112,7 +112,7 @@ class Queue use_default def initialize(db_path : String? = nil) - @path = db_path || Config.current.mangadex["download_queue_db_path"].to_s + @path = db_path || Config.current.queue_db_path.to_s dir = File.dirname @path unless Dir.exists? dir Logger.info "The queue DB directory #{dir} does not exist. " \ diff --git a/src/routes/admin.cr b/src/routes/admin.cr index fd63ec8..a63bc0e 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -66,7 +66,6 @@ struct AdminRouter end get "/admin/downloads" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end diff --git a/src/routes/api.cr b/src/routes/api.cr index ed9bb29..413c318 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -23,7 +23,7 @@ struct APIRouter # Authentication - All endpoints require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access. + All endpoints except `/api/login` require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access. # Terminologies @@ -56,6 +56,29 @@ struct APIRouter "error" => String?, } + Koa.describe "Authenticates a user", <<-MD + After successful login, the cookie `mango-sessid-#{Config.current.port}` will contain a valid session ID that can be used for subsequent requests + MD + Koa.body schema: { + "username" => String, + "password" => String, + } + Koa.tag "users" + post "/api/login" do |env| + begin + username = env.params.json["username"].as String + password = env.params.json["password"].as String + token = Storage.default.verify_user(username, password).not_nil! + + env.session.string "token", token + "Authenticated" + rescue e + Logger.error e + env.response.status_code = 403 + e.message + end + end + Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" @@ -217,7 +240,7 @@ struct APIRouter } get "/api/admin/thumbnail_progress" do |env| send_json env, { - "progress" => Library.default.thumbnail_generation_progress, + "progress" => Library.default.thumbnail_ctx.progress, }.to_json end @@ -348,6 +371,38 @@ struct APIRouter end end + Koa.describe "Sets the sort title of a title or an entry", <<-MD + When `eid` is provided, apply the sort title to the entry. Otherwise, apply the sort title to the title identified by `tid`. + MD + Koa.tags ["admin", "library"] + Koa.path "tid", desc: "Title ID" + Koa.query "eid", desc: "Entry ID", required: false + Koa.query "name", desc: "The new sort title" + Koa.response 200, schema: "result" + put "/api/admin/sort_title/:tid" do |env| + username = get_username env + begin + title = (Library.default.get_title env.params.url["tid"]) + .not_nil! + name = env.params.query["name"]? + entry = env.params.query["eid"]? + if entry.nil? + title.set_sort_title name, username + else + eobj = title.get_entry entry + eobj.set_sort_title name, username unless eobj.nil? + end + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + else + send_json env, {"success" => true}.to_json + end + end + ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 diff --git a/src/routes/main.cr b/src/routes/main.cr index 4aa7da6..ea2f0d8 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -61,9 +61,15 @@ struct MainRouter sort_opt = SortOptions.from_info_json title.dir, username get_and_save_sort_opt title.dir + sorted_titles = title.sorted_titles username, sort_opt entries = title.sorted_entries username, sort_opt percentage = title.load_percentage_for_all_entries username, sort_opt title_percentage = title.titles.map &.load_percentage username + title_percentage_map = {} of String => Float64 + title_percentage.each_with_index do |tp, i| + t = title.titles[i] + title_percentage_map[t.id] = tp + end layout "title" rescue e diff --git a/src/storage.cr b/src/storage.cr index 32f446a..5622241 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -342,6 +342,67 @@ class Storage end end + def get_title_sort_title(title_id : String) + sort_title = nil + MainFiber.run do + get_db do |db| + sort_title = + db.query_one? "Select sort_title from titles where id = (?)", + title_id, as: String | Nil + end + end + sort_title + end + + def set_title_sort_title(title_id : String, sort_title : String | Nil) + sort_title = nil if sort_title == "" + MainFiber.run do + get_db do |db| + db.exec "update titles set sort_title = (?) where id = (?)", + sort_title, title_id + end + end + end + + def get_entry_sort_title(entry_id : String) + sort_title = nil + MainFiber.run do + get_db do |db| + sort_title = + db.query_one? "Select sort_title from ids where id = (?)", + entry_id, as: String | Nil + end + end + sort_title + end + + def get_entries_sort_title(ids : Array(String)) + results = Hash(String, String | Nil).new + MainFiber.run do + get_db do |db| + db.query "select id, sort_title from ids where id in " \ + "(#{ids.join "," { |id| "'#{id}'" }})" do |rs| + rs.each do + id = rs.read String + sort_title = rs.read String | Nil + results[id] = sort_title + end + end + end + end + results + end + + def set_entry_sort_title(entry_id : String, sort_title : String | Nil) + sort_title = nil if sort_title == "" + MainFiber.run do + get_db do |db| + db.exec "update ids set sort_title = (?) where id = (?)", + sort_title, entry_id + end + end + end + def save_thumbnail(id : String, img : Image) MainFiber.run do get_db do |db| @@ -558,6 +619,20 @@ class Storage {token, expires} end + def count_titles : Int32 + count = 0 + MainFiber.run do + get_db do |db| + db.query "select count(*) from titles" do |rs| + rs.each do + count = rs.read Int32 + end + end + end + end + count + end + def close MainFiber.run do unless @db.nil? diff --git a/src/util/util.cr b/src/util/util.cr index 9f5ffee..e7b1b1a 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -87,30 +87,49 @@ def env_is_true?(key : String) : Bool end def sort_titles(titles : Array(Title), opt : SortOptions, username : String) - ary = titles + cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt + cached_titles = LRUCache.get cache_key + return cached_titles if cached_titles.is_a? Array(Title) case opt.method when .time_modified? - ary.sort! { |a, b| (a.mtime <=> b.mtime).or \ - compare_numerically a.title, b.title } + ary = titles.sort { |a, b| (a.mtime <=> b.mtime).or \ + compare_numerically a.sort_title, b.sort_title } when .progress? - ary.sort! do |a, b| + ary = titles.sort do |a, b| (a.load_percentage(username) <=> b.load_percentage(username)).or \ - compare_numerically a.title, b.title + compare_numerically a.sort_title, b.sort_title + end + when .title? + ary = titles.sort do |a, b| + compare_numerically a.sort_title, b.sort_title end else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - ary.sort! { |a, b| compare_numerically a.title, b.title } + ary = titles.sort { |a, b| compare_numerically a.sort_title, b.sort_title } end ary.reverse! unless opt.not_nil!.ascend + LRUCache.set generate_cache_entry cache_key, ary ary end +def remove_sorted_titles_cache(titles : Array(Title), + sort_methods : Array(SortMethod), + username : String) + [false, true].each do |ascend| + sort_methods.each do |sort_method| + sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username, + titles, SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_titles_cache_key + end + end +end + class String # Returns the similarity (in [0, 1]) of two paths. # For the two paths, separate them into arrays of components, count the @@ -144,3 +163,12 @@ def sanitize_filename(str : String) : String .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "") sanitized.size > 0 ? sanitized : random_str end + +def delete_cache_and_exit(path : String) + File.delete path + Logger.fatal "Invalid library cache deleted. Mango needs to " \ + "perform a full reset to recover from this. " \ + "Pleae restart Mango. This is NOT a bug." + Logger.fatal "Exiting" + exit 1 +end diff --git a/src/views/components/card.html.ecr b/src/views/components/card.html.ecr index b85d39e..5549499 100644 --- a/src/views/components/card.html.ecr +++ b/src/views/components/card.html.ecr @@ -61,7 +61,9 @@ <% if page == "home" && item.is_a? Entry %> <%= "uk-margin-remove-bottom" %> <% end %> - " data-title="<%= HTML.escape(item.display_name) %>"><%= HTML.escape(item.display_name) %> + " data-title="<%= HTML.escape(item.display_name) %>" + data-file-title="<%= HTML.escape(item.title || "") %>" + data-sort-title="<%= HTML.escape(item.sort_title_db || "") %>"><%= HTML.escape(item.display_name) %> <% if page == "home" && item.is_a? Entry %> <%= HTML.escape(item.book.display_name) %> diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index c264177..a8394ff 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -24,16 +24,10 @@ - - diff --git a/src/views/layout.html.ecr b/src/views/layout.html.ecr index 70c5a51..c32bfb5 100644 --- a/src/views/layout.html.ecr +++ b/src/views/layout.html.ecr @@ -32,10 +32,10 @@
-
+
-
+
  • Home
    • @@ -57,7 +57,7 @@ <% end %>
-
+
  • Logout
    • diff --git a/src/views/library.html.ecr b/src/views/library.html.ecr index 39e9856..a5e8b59 100644 --- a/src/views/library.html.ecr +++ b/src/views/library.html.ecr @@ -10,6 +10,7 @@
    <% hash = { "auto" => "Auto", + "title" => "Name", "time_modified" => "Date Modified", "progress" => "Progress" } %> diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 0e46ed2..395de41 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -55,8 +55,8 @@ object-fit: contain; `" /> -
    -
    +
    +
@@ -114,6 +114,12 @@
+
+ +
+ +
+
diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index 78edf98..6880347 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -18,7 +18,8 @@
-

<%= title.display_name %> +

"> + <%= title.display_name %>   <% if is_admin %> @@ -59,8 +60,8 @@

- <% title.titles.each_with_index do |item, i| %> - <% progress = title_percentage[i] %> + <% sorted_titles.each do |item| %> + <% progress = title_percentage_map[item.id] %> <%= render_component "card" %> <% end %>
@@ -89,6 +90,13 @@ +
+ +
+ + +
+