diff --git a/README.md b/README.md index 413ae8b..5883b54 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Mango is a self-hosted manga server and reader. Its features include - Supported formats: `.cbz`, `.zip`, `.cbr` and `.rar` - Supports nested folders in library - Automatically stores reading progress +- Thumbnail generation - Built-in [MangaDex](https://mangadex.org/) downloader - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites - The web reader is responsive and works well on mobile, so there is no need for a mobile app @@ -51,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.14.0 + Mango - Manga Server and Web Reader. Version 0.15.0 Usage: @@ -80,6 +81,8 @@ session_secret: mango-session-secret library_path: ~/mango/library db_path: ~/mango/mango.db scan_interval_minutes: 5 +thumbnail_generation_interval_hours: 24 +db_optimization_interval_hours: 24 log_level: info upload_path: ~/mango/uploads plugin_path: ~/mango/plugins @@ -89,12 +92,12 @@ mangadex: api_url: https://mangadex.org/api download_wait_seconds: 5 download_retries: 4 - download_queue_db_path: ~/mango/queue.db + download_queue_db_path: /home/alex_ling/mango/queue.db chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' manga_rename_rule: '{title}' ``` -- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan +- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging ### Library Structure diff --git a/public/js/admin.js b/public/js/admin.js index 5c9051d..4a5246a 100644 --- a/public/js/admin.js +++ b/public/js/admin.js @@ -1,40 +1,90 @@ -let scanning = false; - -const scan = () => { - scanning = true; - $('#scan-status > div').removeAttr('hidden'); - $('#scan-status > span').attr('hidden', ''); - const color = $('#scan').css('color'); - $('#scan').css('color', 'gray'); - $.post(base_url + 'api/admin/scan', (data) => { - const ms = data.milliseconds; - const titles = data.titles; - $('#scan-status > span').text('Scanned ' + titles + ' titles in ' + ms + 'ms'); - $('#scan-status > span').removeAttr('hidden'); - $('#scan').css('color', color); - $('#scan-status > div').attr('hidden', ''); - scanning = false; - }); -} - -String.prototype.capitalize = function() { - return this.charAt(0).toUpperCase() + this.slice(1); -} - $(() => { - $('li').click((e) => { - const url = $(e.currentTarget).attr('data-url'); - if (url) { - $(location).attr('href', url); - } - }); - const setting = loadThemeSetting(); - $('#theme-select').val(setting.capitalize()); - + $('#theme-select').val(capitalize(setting)); $('#theme-select').change((e) => { const newSetting = $(e.currentTarget).val().toLowerCase(); saveThemeSetting(newSetting); setTheme(); }); + + getProgress(); + setInterval(getProgress, 5000); }); + +/** + * Capitalize String + * + * @function capitalize + * @param {string} str - The string to be capitalized + * @return {string} The capitalized string + */ +const capitalize = (str) => { + return str.charAt(0).toUpperCase() + str.slice(1); +}; + +/** + * Set an alpine.js property + * + * @function setProp + * @param {string} key - Key of the data property + * @param {*} prop - The data property + */ +const setProp = (key, prop) => { + $('#root').get(0).__x.$data[key] = prop; +}; + +/** + * Get an alpine.js property + * + * @function getProp + * @param {string} key - Key of the data property + * @return {*} The data property + */ +const getProp = (key) => { + return $('#root').get(0).__x.$data[key]; +}; + +/** + * Get the thumbnail generation progress from the API + * + * @function getProgress + */ +const getProgress = () => { + $.get(`${base_url}api/admin/thumbnail_progress`) + .then(data => { + setProp('progress', data.progress); + const generating = data.progress > 0 + setProp('generating', generating); + }); +}; + +/** + * Trigger the thumbnail generation + * + * @function generateThumbnails + */ +const generateThumbnails = () => { + setProp('generating', true); + setProp('progress', 0.0); + $.post(`${base_url}api/admin/generate_thumbnails`) + .then(getProgress); +}; + +/** + * Trigger the scan + * + * @function scan + */ +const scan = () => { + setProp('scanning', true); + setProp('scanMs', -1); + setProp('scanTitles', 0); + $.post(`${base_url}api/admin/scan`) + .then(data => { + setProp('scanMs', data.milliseconds); + setProp('scanTitles', data.titles); + }) + .always(() => { + setProp('scanning', false); + }); +} diff --git a/public/js/dots.js b/public/js/dots.js index 75a6aa6..266c846 100644 --- a/public/js/dots.js +++ b/public/js/dots.js @@ -1,17 +1,26 @@ -const truncate = () => { - $('.uk-card-title').each((i, e) => { - $(e).dotdotdot({ - truncate: 'letter', - watch: true, - callback: (truncated) => { - if (truncated) { - $(e).attr('uk-tooltip', $(e).attr('data-title')); - } else { - $(e).removeAttr('uk-tooltip'); - } +/** + * Truncate a .uk-card-title element + * + * @function truncate + * @param {object} e - The title element to truncate + */ +const truncate = (e) => { + $(e).dotdotdot({ + truncate: 'letter', + watch: true, + callback: (truncated) => { + if (truncated) { + $(e).attr('uk-tooltip', $(e).attr('data-title')); + } else { + $(e).removeAttr('uk-tooltip'); } - }); + } }); }; -truncate(); +$('.uk-card-title').each((i, e) => { + // Truncate the title when it first enters the view + $(e).one('inview', () => { + truncate(e); + }); +}); diff --git a/shard.lock b/shard.lock index 529edf5..f2604c9 100644 --- a/shard.lock +++ b/shard.lock @@ -34,7 +34,7 @@ shards: image_size: github: hkalexling/image_size.cr - version: 0.2.0 + version: 0.4.0 kemal: github: kemalcr/kemal diff --git a/shard.yml b/shard.yml index 02438e5..d187ca4 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.14.0 +version: 0.15.0 authors: - Alex Ling diff --git a/src/config.cr b/src/config.cr index 60c2e40..1c80542 100644 --- a/src/config.cr +++ b/src/config.cr @@ -11,8 +11,9 @@ class Config property library_path : String = File.expand_path "~/mango/library", home: true property db_path : String = File.expand_path "~/mango/mango.db", home: true - @[YAML::Field(key: "scan_interval_minutes")] - property scan_interval : Int32 = 5 + property scan_interval_minutes : Int32 = 5 + property thumbnail_generation_interval_hours : Int32 = 24 + property db_optimization_interval_hours : Int32 = 24 property log_level : String = "info" property upload_path : String = File.expand_path "~/mango/uploads", home: true diff --git a/src/library/entry.cr b/src/library/entry.cr index 589794a..b96681e 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -69,7 +69,7 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg - url = "#{Config.current.base_url}api/page/#{@book.id}/#{@id}/1" + url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}" TitleInfo.new @book.dir do |info| info_url = info.entry_cover_url[@title]? unless info_url.nil? || info_url.empty? @@ -207,4 +207,29 @@ class Entry def started?(username) load_progress(username) > 0 end + + def generate_thumbnail : Image? + return if @err_msg + + img = read_page(1).not_nil! + begin + size = ImageSize.get img.data + if size.height > size.width + thumbnail = ImageSize.resize img.data, width: 200 + else + thumbnail = ImageSize.resize img.data, height: 300 + end + img.data = thumbnail + Storage.default.save_thumbnail @id, img + rescue e + Logger.warn "Failed to generate thumbnail for entry " \ + "#{@book.title}/#{@title}. #{e}" + end + + img + end + + def get_thumbnail : Image? + Storage.default.get_thumbnail @id + end end diff --git a/src/library/library.cr b/src/library/library.cr index f145342..35d9e89 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,5 +1,5 @@ class Library - property dir : String, title_ids : Array(String), scan_interval : Int32, + property dir : String, title_ids : Array(String), title_hash : Hash(String, Title) use_default @@ -8,20 +8,48 @@ class Library register_mime_types @dir = Config.current.library_path - @scan_interval = Config.current.scan_interval # explicitly initialize @titles to bypass the compiler check. it will # be filled with actual Titles in the `scan` call below @title_ids = [] of String @title_hash = {} of String => Title - return scan if @scan_interval < 1 - spawn do - loop do - start = Time.local - scan - ms = (Time.local - start).total_milliseconds - Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" - sleep @scan_interval * 60 + @entries_count = 0 + @thumbnails_count = 0 + + scan_interval = Config.current.scan_interval_minutes + if scan_interval < 1 + scan + else + spawn do + loop do + start = Time.local + scan + ms = (Time.local - start).total_milliseconds + Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" + sleep scan_interval.minutes + end + end + end + + thumbnail_interval = Config.current.thumbnail_generation_interval_hours + unless thumbnail_interval < 1 + spawn do + loop do + # Wait for scan to complete (in most cases) + sleep 1.minutes + generate_thumbnails + sleep thumbnail_interval.hours + end + end + end + + db_interval = Config.current.db_optimization_interval_hours + unless db_interval < 1 + spawn do + loop do + Storage.default.optimize + sleep db_interval.hours + end end end end @@ -194,4 +222,50 @@ class Library .sample(ENTRIES_IN_HOME_SECTIONS) .shuffle end + + def thumbnail_generation_progress + return 0 if @entries_count == 0 + @thumbnails_count / @entries_count + end + + def generate_thumbnails + if @thumbnails_count > 0 + Logger.debug "Thumbnail generation in progress" + return + end + + Logger.info "Starting thumbnail generation" + entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg + @entries_count = entries.size + @thumbnails_count = 0 + + # Report generation progress regularly + spawn do + loop do + unless @thumbnails_count == 0 + Logger.debug "Thumbnail generation progress: " \ + "#{(thumbnail_generation_progress * 100).round 1}%" + end + # Generation is completed. We reset the count to 0 to allow subsequent + # calls to the function, and break from the loop to stop the progress + # report fiber + if thumbnail_generation_progress.to_i == 1 + @thumbnails_count = 0 + break + end + sleep 10.seconds + end + end + + entries.each do |e| + unless e.get_thumbnail + e.generate_thumbnail + # Sleep after each generation to minimize the impact on disk IO + # and CPU + sleep 0.5.seconds + end + @thumbnails_count += 1 + end + Logger.info "Thumbnail generation finished" + end end diff --git a/src/library/types.cr b/src/library/types.cr index b51671b..4e83135 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -57,6 +57,16 @@ struct Image def initialize(@data, @mime, @filename, @size) end + + def self.from_db(res : DB::ResultSet) + img = Image.allocate + res.read String + img.data = res.read Bytes + img.filename = res.read String + img.mime = res.read String + img.size = res.read Int32 + img + end end class TitleInfo diff --git a/src/mango.cr b/src/mango.cr index 2f9ddd8..16a334e 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "./plugin/*" -MANGO_VERSION = "0.14.0" +MANGO_VERSION = "0.15.0" # From http://www.network-science.de/ascii/ BANNER = %{ diff --git a/src/routes/api.cr b/src/routes/api.cr index a131a97..ce2caf5 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -26,6 +26,28 @@ class APIRouter < Router end end + get "/api/cover/:tid/:eid" do |env| + begin + tid = env.params.url["tid"] + eid = env.params.url["eid"] + + title = @context.library.get_title tid + raise "Title ID `#{tid}` not found" if title.nil? + entry = title.get_entry eid + raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? + + img = entry.get_thumbnail || entry.read_page 1 + raise "Failed to get cover of `#{title.title}/#{entry.title}`" \ + if img.nil? + + send_img env, img + rescue e + @context.error e + env.response.status_code = 500 + e.message + end + end + get "/api/book/:tid" do |env| begin tid = env.params.url["tid"] @@ -54,6 +76,18 @@ class APIRouter < Router }.to_json end + get "/api/admin/thumbnail_progress" do |env| + send_json env, { + "progress" => Library.default.thumbnail_generation_progress, + }.to_json + end + + post "/api/admin/generate_thumbnails" do |env| + spawn do + Library.default.generate_thumbnails + end + end + post "/api/admin/user/delete/:username" do |env| begin username = env.params.url["username"] diff --git a/src/storage.cr b/src/storage.cr index afbba8d..592da0e 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -35,9 +35,11 @@ class Storage MainFiber.run do DB.open "sqlite3://#{@path}" do |db| begin - # We create the `ids` table first. even if the uses has an - # early version installed and has the `user` table only, - # we will still be able to create `ids` + db.exec "create table thumbnails " \ + "(id text, data blob, filename text, " \ + "mime text, size integer)" + db.exec "create unique index tn_index on thumbnails (id)" + db.exec "create table ids" \ "(path text, id text, is_title integer)" db.exec "create unique index path_idx on ids (path)" @@ -243,6 +245,58 @@ class Storage end end + def save_thumbnail(id : String, img : Image) + MainFiber.run do + get_db do |db| + db.exec "insert into thumbnails values (?, ?, ?, ?, ?)", id, img.data, + img.filename, img.mime, img.size + end + end + end + + def get_thumbnail(id : String) : Image? + img = nil + MainFiber.run do + get_db do |db| + db.query_one? "select * from thumbnails where id = (?)", id do |res| + img = Image.from_db res + end + end + end + img + end + + def optimize + MainFiber.run do + Logger.info "Starting DB optimization" + get_db do |db| + trash_ids = [] of String + db.query "select path, id from ids" do |rs| + rs.each do + path = rs.read String + trash_ids << rs.read String unless File.exists? path + end + end + + # Delete dangling IDs + db.exec "delete from ids where id in " \ + "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + Logger.debug "#{trash_ids.size} dangling IDs deleted" \ + if trash_ids.size > 0 + + # Delete dangling thumbnails + trash_thumbnails_count = db.query_one "select count(*) from " \ + "thumbnails where id not in " \ + "(select id from ids)", as: Int32 + if trash_thumbnails_count > 0 + db.exec "delete from thumbnails where id not in (select id from ids)" + Logger.info "#{trash_thumbnails_count} dangling thumbnails deleted" + end + end + Logger.debug "DB optimization finished" + end + end + def close MainFiber.run do unless @db.nil? diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index 456dff4..a0959bf 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -1,11 +1,17 @@ -