Merge branch 'dev' into feature/plugin-v2

.all-contributorsrc

@@ -113,6 +113,24 @@
       "contributions": [
         "infra"
       ]
+    },
+    {
+      "login": "BradleyDS2",
+      "name": "BradleyDS2",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2174921?v=4",
+      "profile": "https://github.com/BradleyDS2",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "nduja",
+      "name": "Robbo",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69299134?v=4",
+      "profile": "https://github.com/nduja",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -13,7 +13,7 @@ Mango is a self-hosted manga server and reader. Its features include
 - Supports nested folders in library
 - Automatically stores reading progress
 - Thumbnail generation
-- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
+- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from third-party sites
 - The web reader is responsive and works well on mobile, so there is no need for a mobile app
 - All the static files are embedded in the binary, so the deployment process is easy and painless
 
@@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
 ### CLI
 
 ```
-  Mango - Manga Server and Web Reader. Version 0.24.0
+  Mango - Manga Server and Web Reader. Version 0.25.0
 
   Usage:
 
@@ -80,6 +80,7 @@ base_url: /
 session_secret: mango-session-secret
 library_path: ~/mango/library
 db_path: ~/mango/mango.db
+queue_db_path: ~/mango/queue.db
 scan_interval_minutes: 5
 thumbnail_generation_interval_hours: 24
 log_level: info
@@ -93,17 +94,9 @@ cache_log_enabled: true
 disable_login: false
 default_username: ""
 auth_proxy_header_name: ""
-mangadex:
-  base_url: https://mangadex.org
-  api_url: https://api.mangadex.org/v2
-  download_wait_seconds: 5
-  download_retries: 4
-  download_queue_db_path: ~/mango/queue.db
-  chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
-  manga_rename_rule: '{title}'
 ```
 
-- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
+- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
 - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
 - You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work.
 - By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`.
@@ -179,6 +172,8 @@ Please check the [development guideline](https://github.com/hkalexling/Mango/wik
     <td align="center"><a href="http://h45h74x.eu.org"><img src="https://avatars1.githubusercontent.com/u/27204033?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Simon</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=h45h74x" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/davidkna"><img src="https://avatars.githubusercontent.com/u/835177?v=4?s=100" width="100px;" alt=""/><br /><sub><b>David Knaack</b></sub></a><br /><a href="#infra-davidkna" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
     <td align="center"><a href="https://lncn.dev"><img src="https://avatars.githubusercontent.com/u/41193328?v=4?s=100" width="100px;" alt=""/><br /><sub><b>i use arch btw</b></sub></a><br /><a href="#infra-lincolnthedev" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    <td align="center"><a href="https://github.com/BradleyDS2"><img src="https://avatars.githubusercontent.com/u/2174921?v=4?s=100" width="100px;" alt=""/><br /><sub><b>BradleyDS2</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=BradleyDS2" title="Documentation">📖</a></td>
+    <td align="center"><a href="https://github.com/nduja"><img src="https://avatars.githubusercontent.com/u/69299134?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Robbo</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=nduja" title="Code">💻</a></td>
   </tr>
 </table>
 

shard.yml

@@ -1,5 +1,5 @@
 name: mango
-version: 0.24.0
+version: 0.25.0
 
 authors:
   - Alex Ling <hkalexling@gmail.com>

src/config.cr

@@ -4,44 +4,28 @@ class Config
   include YAML::Serializable
 
   @[YAML::Field(ignore: true)]
-  property path : String = ""
+  property path = ""
-  property host : String = "0.0.0.0"
+  property host = "0.0.0.0"
   property port : Int32 = 9000
-  property base_url : String = "/"
+  property base_url = "/"
-  property session_secret : String = "mango-session-secret"
+  property session_secret = "mango-session-secret"
-  property library_path : String = File.expand_path "~/mango/library",
+  property library_path = "~/mango/library"
-    home: true
+  property library_cache_path = "~/mango/library.yml.gz"
-  property library_cache_path = File.expand_path "~/mango/library.yml.gz",
+  property db_path = "~/mango/mango.db"
-    home: true
+  property queue_db_path = "~/mango/queue.db"
-  property db_path : String = File.expand_path "~/mango/mango.db", home: true
   property scan_interval_minutes : Int32 = 5
   property thumbnail_generation_interval_hours : Int32 = 24
-  property log_level : String = "info"
+  property log_level = "info"
-  property upload_path : String = File.expand_path "~/mango/uploads",
+  property upload_path = "~/mango/uploads"
-    home: true
+  property plugin_path = "~/mango/plugins"
-  property plugin_path : String = File.expand_path "~/mango/plugins",
-    home: true
   property download_timeout_seconds : Int32 = 30
-  property cache_enabled = false
+  property cache_enabled = true
   property cache_size_mbs = 50
   property cache_log_enabled = true
   property disable_login = false
   property default_username = ""
   property auth_proxy_header_name = ""
   property plugin_update_interval_hours : Int32 = 24
-  property mangadex = Hash(String, String | Int32).new
-
-  @[YAML::Field(ignore: true)]
-  @mangadex_defaults = {
-    "base_url"               => "https://mangadex.org",
-    "api_url"                => "https://api.mangadex.org/v2",
-    "download_wait_seconds"  => 5,
-    "download_retries"       => 4,
-    "download_queue_db_path" => File.expand_path("~/mango/queue.db",
-      home: true),
-    "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}",
-    "manga_rename_rule"   => "{title}",
-  }
 
   @@singlet : Config?
 
@@ -59,7 +43,7 @@ class Config
     if File.exists? cfg_path
       config = self.from_yaml File.read cfg_path
       config.path = path
-      config.fill_defaults
+      config.expand_paths
       config.preprocess
       return config
     end
@@ -67,7 +51,7 @@ class Config
          "Dumping the default config there."
     default = self.allocate
     default.path = path
-    default.fill_defaults
+    default.expand_paths
     cfg_dir = File.dirname cfg_path
     unless Dir.exists? cfg_dir
       Dir.mkdir_p cfg_dir
@@ -77,13 +61,9 @@ class Config
     default
   end
 
-  def fill_defaults
+  def expand_paths
-    {% for hash_name in ["mangadex"] %}
+    {% for p in %w(library library_cache db queue_db upload plugin) %}
-      @{{hash_name.id}}_defaults.map do |k, v|
+      @{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true
-        if @{{hash_name.id}}[k]?.nil?
-          @{{hash_name.id}}[k] = v
-        end
-      end
     {% end %}
   end
 
@@ -98,24 +78,5 @@ class Config
       raise "Login is disabled, but default username is not set. " \
             "Please set a default username"
     end
-
-    # `Logger.default` is not available yet
-    Log.setup :debug
-    unless mangadex["api_url"] =~ /\/v2/
-      Log.warn { "It looks like you are using the deprecated MangaDex API " \
-                 "v1 in your config file. Please update it to " \
-                 "https://api.mangadex.org/v2 to suppress this warning." }
-      mangadex["api_url"] = "https://api.mangadex.org/v2"
-    end
-    if mangadex["api_url"] =~ /\/api\/v2/
-      Log.warn { "It looks like you are using the outdated MangaDex API " \
-                 "url (mangadex.org/api/v2) in your config file. Please " \
-                 "update it to https://api.mangadex.org/v2 to suppress this " \
-                 "warning." }
-      mangadex["api_url"] = "https://api.mangadex.org/v2"
-    end
-
-    mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/"
-    mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/"
   end
 end

src/library/library.cr

@@ -1,9 +1,38 @@
 class Library
+  struct ThumbnailContext
+    property current : Int32, total : Int32
+
+    def initialize
+      @current = 0
+      @total = 0
+    end
+
+    def progress
+      if total == 0
+        0
+      else
+        current / total
+      end
+    end
+
+    def reset
+      @current = 0
+      @total = 0
+    end
+
+    def increment
+      @current += 1
+    end
+  end
+
   include YAML::Serializable
 
   getter dir : String, title_ids : Array(String),
     title_hash : Hash(String, Title)
 
+  @[YAML::Field(ignore: true)]
+  getter thumbnail_ctx = ThumbnailContext.new
+
   use_default
 
   def save_instance
@@ -24,7 +53,23 @@ class Library
 
     begin
       Compress::Gzip::Reader.open path do |content|
-        @@default = Library.from_yaml content
+        loaded = Library.from_yaml content
+        # We will have to do a full restart in these cases. Otherwise having
+        #   two instances of the library will cause some weirdness.
+        if loaded.dir != Config.current.library_path
+          Logger.fatal "Cached library dir #{loaded.dir} does not match " \
+                       "current library dir #{Config.current.library_path}. " \
+                       "Deleting cache"
+          delete_cache_and_exit path
+        end
+        if loaded.title_ids.size > 0 &&
+           Storage.default.count_titles == 0
+          Logger.fatal "The library cache is inconsistent with the DB. " \
+                       "Deleting cache"
+          delete_cache_and_exit path
+        end
+        @@default = loaded
+        Logger.debug "Library cache loaded"
       end
       Library.default.register_jobs
     rescue e
@@ -39,9 +84,6 @@ class Library
     @title_ids = [] of String
     @title_hash = {} of String => Title
 
-    @entries_count = 0
-    @thumbnails_count = 0
-
     register_jobs
   end
 
@@ -262,34 +304,29 @@ class Library
       .shuffle!
   end
 
-  def thumbnail_generation_progress
-    return 0 if @entries_count == 0
-    @thumbnails_count / @entries_count
-  end
-
   def generate_thumbnails
-    if @thumbnails_count > 0
+    if thumbnail_ctx.current > 0
       Logger.debug "Thumbnail generation in progress"
       return
     end
 
     Logger.info "Starting thumbnail generation"
     entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg
-    @entries_count = entries.size
+    thumbnail_ctx.total = entries.size
-    @thumbnails_count = 0
+    thumbnail_ctx.current = 0
 
     # Report generation progress regularly
     spawn do
       loop do
-        unless @thumbnails_count == 0
+        unless thumbnail_ctx.current == 0
           Logger.debug "Thumbnail generation progress: " \
-                       "#{(thumbnail_generation_progress * 100).round 1}%"
+                       "#{(thumbnail_ctx.progress * 100).round 1}%"
         end
         # Generation is completed. We reset the count to 0 to allow subsequent
         #   calls to the function, and break from the loop to stop the progress
         #   report fiber
-        if thumbnail_generation_progress.to_i == 1
+        if thumbnail_ctx.progress.to_i == 1
-          @thumbnails_count = 0
+          thumbnail_ctx.reset
           break
         end
         sleep 10.seconds
@@ -303,7 +340,7 @@ class Library
         #   and CPU
         sleep 1.seconds
       end
-      @thumbnails_count += 1
+      thumbnail_ctx.increment
     end
     Logger.info "Thumbnail generation finished"
   end

src/mango.cr

@@ -7,7 +7,7 @@ require "option_parser"
 require "clim"
 require "tallboy"
 
-MANGO_VERSION = "0.24.0"
+MANGO_VERSION = "0.25.0"
 
 # From http://www.network-science.de/ascii/
 BANNER = %{

src/queue.cr

@@ -118,7 +118,7 @@ class Queue
   use_default
 
   def initialize(db_path : String? = nil)
-    @path = db_path || Config.current.mangadex["download_queue_db_path"].to_s
+    @path = db_path || Config.current.queue_db_path.to_s
     dir = File.dirname @path
     unless Dir.exists? dir
       Logger.info "The queue DB directory #{dir} does not exist. " \

src/routes/admin.cr

@@ -66,7 +66,6 @@ struct AdminRouter
     end
 
     get "/admin/downloads" do |env|
-      mangadex_base_url = Config.current.mangadex["base_url"]
       layout "download-manager"
     end
 

src/routes/api.cr

@@ -240,7 +240,7 @@ struct APIRouter
     }
     get "/api/admin/thumbnail_progress" do |env|
       send_json env, {
-        "progress" => Library.default.thumbnail_generation_progress,
+        "progress" => Library.default.thumbnail_ctx.progress,
       }.to_json
     end
 

src/storage.cr

@@ -619,6 +619,20 @@ class Storage
     {token, expires}
   end
 
+  def count_titles : Int32
+    count = 0
+    MainFiber.run do
+      get_db do |db|
+        db.query "select count(*) from titles" do |rs|
+          rs.each do
+            count = rs.read Int32
+          end
+        end
+      end
+    end
+    count
+  end
+
   def close
     MainFiber.run do
       unless @db.nil?

src/util/util.cr

@@ -163,3 +163,12 @@ def sanitize_filename(str : String) : String
     .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "")
   sanitized.size > 0 ? sanitized : random_str
 end
+
+def delete_cache_and_exit(path : String)
+  File.delete path
+  Logger.fatal "Invalid library cache deleted. Mango needs to " \
+               "perform a full reset to recover from this. " \
+               "Pleae restart Mango. This is NOT a bug."
+  Logger.fatal "Exiting"
+  exit 1
+end

src/views/download-manager.html.ecr

@@ -24,16 +24,10 @@
           <template x-if="job.plugin_id">
             <td x-text="job.title"></td>
           </template>
-          <template x-if="!job.plugin_id">
-            <td><a :href="`<%= mangadex_base_url %>/chapter/${job.id}`" x-text="job.title"></td>
-          </template>
 
           <template x-if="job.plugin_id">
             <td x-text="job.manga_title"></td>
           </template>
-          <template x-if="!job.plugin_id">
-            <td><a :href="`<%= mangadex_base_url %>/manga/${job.manga_id}`" x-text="job.manga_title"></td>
-          </template>
 
           <td x-text="`${job.success_count}/${job.pages}`"></td>
           <td x-text="`${moment(job.time).fromNow()}`"></td>