diff --git a/.all-contributorsrc b/.all-contributorsrc index ca614a0..8f287e9 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -113,6 +113,24 @@ "contributions": [ "infra" ] + }, + { + "login": "BradleyDS2", + "name": "BradleyDS2", + "avatar_url": "https://avatars.githubusercontent.com/u/2174921?v=4", + "profile": "https://github.com/BradleyDS2", + "contributions": [ + "doc" + ] + }, + { + "login": "nduja", + "name": "Robbo", + "avatar_url": "https://avatars.githubusercontent.com/u/69299134?v=4", + "profile": "https://github.com/nduja", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 1a010dd..53cea05 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Mango is a self-hosted manga server and reader. Its features include - Supports nested folders in library - Automatically stores reading progress - Thumbnail generation -- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites +- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from third-party sites - The web reader is responsive and works well on mobile, so there is no need for a mobile app - All the static files are embedded in the binary, so the deployment process is easy and painless @@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.24.0 + Mango - Manga Server and Web Reader. Version 0.25.0 Usage: @@ -80,6 +80,7 @@ base_url: / session_secret: mango-session-secret library_path: ~/mango/library db_path: ~/mango/mango.db +queue_db_path: ~/mango/queue.db scan_interval_minutes: 5 thumbnail_generation_interval_hours: 24 log_level: info @@ -93,17 +94,9 @@ cache_log_enabled: true disable_login: false default_username: "" auth_proxy_header_name: "" -mangadex: - base_url: https://mangadex.org - api_url: https://api.mangadex.org/v2 - download_wait_seconds: 5 - download_retries: 4 - download_queue_db_path: ~/mango/queue.db - chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' - manga_rename_rule: '{title}' ``` -- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks +- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging - You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work. - By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`. @@ -179,6 +172,8 @@ Please check the [development guideline](https://github.com/hkalexling/Mango/wik
Simon
💻
David Knaack
🚇
i use arch btw
🚇 +
BradleyDS2
📖 +
Robbo
💻 diff --git a/shard.yml b/shard.yml index 0054a23..44a0924 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.24.0 +version: 0.25.0 authors: - Alex Ling diff --git a/src/config.cr b/src/config.cr index 41a4cf1..807a74c 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,44 +4,28 @@ class Config include YAML::Serializable @[YAML::Field(ignore: true)] - property path : String = "" - property host : String = "0.0.0.0" + property path = "" + property host = "0.0.0.0" property port : Int32 = 9000 - property base_url : String = "/" - property session_secret : String = "mango-session-secret" - property library_path : String = File.expand_path "~/mango/library", - home: true - property library_cache_path = File.expand_path "~/mango/library.yml.gz", - home: true - property db_path : String = File.expand_path "~/mango/mango.db", home: true + property base_url = "/" + property session_secret = "mango-session-secret" + property library_path = "~/mango/library" + property library_cache_path = "~/mango/library.yml.gz" + property db_path = "~/mango/mango.db" + property queue_db_path = "~/mango/queue.db" property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property log_level : String = "info" - property upload_path : String = File.expand_path "~/mango/uploads", - home: true - property plugin_path : String = File.expand_path "~/mango/plugins", - home: true + property log_level = "info" + property upload_path = "~/mango/uploads" + property plugin_path = "~/mango/plugins" property download_timeout_seconds : Int32 = 30 - property cache_enabled = false + property cache_enabled = true property cache_size_mbs = 50 property cache_log_enabled = true property disable_login = false property default_username = "" property auth_proxy_header_name = "" property plugin_update_interval_hours : Int32 = 24 - property mangadex = Hash(String, String | Int32).new - - @[YAML::Field(ignore: true)] - @mangadex_defaults = { - "base_url" => "https://mangadex.org", - "api_url" => "https://api.mangadex.org/v2", - "download_wait_seconds" => 5, - "download_retries" => 4, - "download_queue_db_path" => File.expand_path("~/mango/queue.db", - home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", - } @@singlet : Config? @@ -59,7 +43,7 @@ class Config if File.exists? cfg_path config = self.from_yaml File.read cfg_path config.path = path - config.fill_defaults + config.expand_paths config.preprocess return config end @@ -67,7 +51,7 @@ class Config "Dumping the default config there." default = self.allocate default.path = path - default.fill_defaults + default.expand_paths cfg_dir = File.dirname cfg_path unless Dir.exists? cfg_dir Dir.mkdir_p cfg_dir @@ -77,13 +61,9 @@ class Config default end - def fill_defaults - {% for hash_name in ["mangadex"] %} - @{{hash_name.id}}_defaults.map do |k, v| - if @{{hash_name.id}}[k]?.nil? - @{{hash_name.id}}[k] = v - end - end + def expand_paths + {% for p in %w(library library_cache db queue_db upload plugin) %} + @{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true {% end %} end @@ -98,24 +78,5 @@ class Config raise "Login is disabled, but default username is not set. " \ "Please set a default username" end - - # `Logger.default` is not available yet - Log.setup :debug - unless mangadex["api_url"] =~ /\/v2/ - Log.warn { "It looks like you are using the deprecated MangaDex API " \ - "v1 in your config file. Please update it to " \ - "https://api.mangadex.org/v2 to suppress this warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - if mangadex["api_url"] =~ /\/api\/v2/ - Log.warn { "It looks like you are using the outdated MangaDex API " \ - "url (mangadex.org/api/v2) in your config file. Please " \ - "update it to https://api.mangadex.org/v2 to suppress this " \ - "warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - - mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/" - mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/" end end diff --git a/src/library/library.cr b/src/library/library.cr index a2dc28d..210912f 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,9 +1,38 @@ class Library + struct ThumbnailContext + property current : Int32, total : Int32 + + def initialize + @current = 0 + @total = 0 + end + + def progress + if total == 0 + 0 + else + current / total + end + end + + def reset + @current = 0 + @total = 0 + end + + def increment + @current += 1 + end + end + include YAML::Serializable getter dir : String, title_ids : Array(String), title_hash : Hash(String, Title) + @[YAML::Field(ignore: true)] + getter thumbnail_ctx = ThumbnailContext.new + use_default def save_instance @@ -24,7 +53,23 @@ class Library begin Compress::Gzip::Reader.open path do |content| - @@default = Library.from_yaml content + loaded = Library.from_yaml content + # We will have to do a full restart in these cases. Otherwise having + # two instances of the library will cause some weirdness. + if loaded.dir != Config.current.library_path + Logger.fatal "Cached library dir #{loaded.dir} does not match " \ + "current library dir #{Config.current.library_path}. " \ + "Deleting cache" + delete_cache_and_exit path + end + if loaded.title_ids.size > 0 && + Storage.default.count_titles == 0 + Logger.fatal "The library cache is inconsistent with the DB. " \ + "Deleting cache" + delete_cache_and_exit path + end + @@default = loaded + Logger.debug "Library cache loaded" end Library.default.register_jobs rescue e @@ -39,9 +84,6 @@ class Library @title_ids = [] of String @title_hash = {} of String => Title - @entries_count = 0 - @thumbnails_count = 0 - register_jobs end @@ -262,34 +304,29 @@ class Library .shuffle! end - def thumbnail_generation_progress - return 0 if @entries_count == 0 - @thumbnails_count / @entries_count - end - def generate_thumbnails - if @thumbnails_count > 0 + if thumbnail_ctx.current > 0 Logger.debug "Thumbnail generation in progress" return end Logger.info "Starting thumbnail generation" entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg - @entries_count = entries.size - @thumbnails_count = 0 + thumbnail_ctx.total = entries.size + thumbnail_ctx.current = 0 # Report generation progress regularly spawn do loop do - unless @thumbnails_count == 0 + unless thumbnail_ctx.current == 0 Logger.debug "Thumbnail generation progress: " \ - "#{(thumbnail_generation_progress * 100).round 1}%" + "#{(thumbnail_ctx.progress * 100).round 1}%" end # Generation is completed. We reset the count to 0 to allow subsequent # calls to the function, and break from the loop to stop the progress # report fiber - if thumbnail_generation_progress.to_i == 1 - @thumbnails_count = 0 + if thumbnail_ctx.progress.to_i == 1 + thumbnail_ctx.reset break end sleep 10.seconds @@ -303,7 +340,7 @@ class Library # and CPU sleep 1.seconds end - @thumbnails_count += 1 + thumbnail_ctx.increment end Logger.info "Thumbnail generation finished" end diff --git a/src/mango.cr b/src/mango.cr index b19799f..14603b9 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.24.0" +MANGO_VERSION = "0.25.0" # From http://www.network-science.de/ascii/ BANNER = %{ diff --git a/src/queue.cr b/src/queue.cr index dc15d36..82fdeda 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -118,7 +118,7 @@ class Queue use_default def initialize(db_path : String? = nil) - @path = db_path || Config.current.mangadex["download_queue_db_path"].to_s + @path = db_path || Config.current.queue_db_path.to_s dir = File.dirname @path unless Dir.exists? dir Logger.info "The queue DB directory #{dir} does not exist. " \ diff --git a/src/routes/admin.cr b/src/routes/admin.cr index c78df86..c3692c9 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -66,7 +66,6 @@ struct AdminRouter end get "/admin/downloads" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end diff --git a/src/routes/api.cr b/src/routes/api.cr index ffed75f..3b33eca 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -240,7 +240,7 @@ struct APIRouter } get "/api/admin/thumbnail_progress" do |env| send_json env, { - "progress" => Library.default.thumbnail_generation_progress, + "progress" => Library.default.thumbnail_ctx.progress, }.to_json end diff --git a/src/storage.cr b/src/storage.cr index 4c8cfe7..5622241 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -619,6 +619,20 @@ class Storage {token, expires} end + def count_titles : Int32 + count = 0 + MainFiber.run do + get_db do |db| + db.query "select count(*) from titles" do |rs| + rs.each do + count = rs.read Int32 + end + end + end + end + count + end + def close MainFiber.run do unless @db.nil? diff --git a/src/util/util.cr b/src/util/util.cr index 11d1a13..e7b1b1a 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -163,3 +163,12 @@ def sanitize_filename(str : String) : String .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "") sanitized.size > 0 ? sanitized : random_str end + +def delete_cache_and_exit(path : String) + File.delete path + Logger.fatal "Invalid library cache deleted. Mango needs to " \ + "perform a full reset to recover from this. " \ + "Pleae restart Mango. This is NOT a bug." + Logger.fatal "Exiting" + exit 1 +end diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index c264177..a8394ff 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -24,16 +24,10 @@ - -