Upgrade to MangaDex API v2

public/js/download.js

@@ -27,7 +27,7 @@ const download = () => {
 		$('#download-btn').attr('hidden', '');
 		$('#download-spinner').removeAttr('hidden');
 		const ids = selected.map((i, e) => {
-			return $(e).find('td').first().text();
+			return parseInt($(e).find('td').first().text());
 		}).get();
 		const chapters = globalChapters.filter(c => ids.indexOf(c.id) >= 0);
 		console.log(ids);
@@ -114,8 +114,7 @@ const search = () => {
 				return;
 			}
 
-			const cover = baseURL + data.cover_url;
+			$('#cover').attr("src", data.mainCover);
-			$('#cover').attr("src", cover);
 			$('#title').text("Title: " + data.title);
 			$('#artist').text("Artist: " + data.artist);
 			$('#author').text("Author: " + data.author);
@@ -285,7 +284,7 @@ const buildTable = () => {
 						<td>${group_str}</td>
 						<td>${chp.volume}</td>
 						<td>${chp.chapter}</td>
-						<td>${moment.unix(chp.time).fromNow()}</td>
+						<td>${moment.unix(chp.timestamp).fromNow()}</td>
 					</tr>`;
 	}).join('');
 	const tbody = `<tbody id="selectable">${inner}</tbody>`;

shard.lock

@@ -52,9 +52,13 @@ shards:
     git: https://github.com/hkalexling/koa.git
     version: 0.5.0
 
+  mangadex:
+    git: https://github.com/hkalexling/mangadex.git
+    version: 0.4.0+git.commit.0c2eb69f46d8e2d0ecca3b5ed088dca36a1b5308
+
   mg:
     git: https://github.com/hkalexling/mg.git
-    version: 0.2.0+git.commit.171c46489d991a8353818e00fc6a3c4e0809ded9
+    version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3
 
   myhtml:
     git: https://github.com/kostya/myhtml.git

shard.yml

@@ -43,3 +43,5 @@ dependencies:
     github: epoch/tallboy
   mg:
    github: hkalexling/mg
+  mangadex:
+   github: hkalexling/mangadex

src/assets/lang_codes.csv

@@ -1,41 +0,0 @@
-Arabic,sa
-Bengali,bd
-Bulgarian,bg
-Burmese,mm
-Catalan,ct
-Chinese (Simp),cn
-Chinese (Trad),hk
-Czech,cz
-Danish,dk
-Dutch,nl
-English,gb
-Filipino,ph
-Finnish,fi
-French,fr
-German,de
-Greek,gr
-Hebrew,il
-Hindi,in
-Hungarian,hu
-Indonesian,id
-Italian,it
-Japanese,jp
-Korean,kr
-Lithuanian,lt
-Malay,my
-Mongolian,mn
-Other,
-Persian,ir
-Polish,pl
-Portuguese (Br),br
-Portuguese (Pt),pt
-Romanian,ro
-Russian,ru
-Serbo-Croatian,rs
-Spanish (Es),es
-Spanish (LATAM),mx
-Swedish,se
-Thai,th
-Turkish,tr
-Ukrainian,ua
-Vietnames,vn

src/config.cr

@@ -27,7 +27,7 @@ class Config
   @[YAML::Field(ignore: true)]
   @mangadex_defaults = {
     "base_url"               => "https://mangadex.org",
-    "api_url"                => "https://mangadex.org/api",
+    "api_url"                => "https://mangadex.org/api/v2",
     "download_wait_seconds"  => 5,
     "download_retries"       => 4,
     "download_queue_db_path" => File.expand_path("~/mango/queue.db",
@@ -91,5 +91,14 @@ class Config
       raise "Login is disabled, but default username is not set. " \
             "Please set a default username"
     end
+    unless mangadex["api_url"] =~ /\/v2/
+      # `Logger.default` is not available yet
+      Log.setup :debug
+      Log.warn { "It looks like you are using the deprecated MangaDex API " \
+                 "v1 in your config file. Please update it to either " \
+                 "https://mangadex.org/api/v2 or " \
+                 "https://api.mangadex.org/v2 to suppress this warning." }
+      mangadex["api_url"] = "https://mangadex.org/api/v2"
+    end
   end
 end

src/mangadex/api.cr

@@ -1,217 +0,0 @@
-require "json"
-require "csv"
-require "../rename"
-
-macro string_properties(names)
-  {% for name in names %}
-    property {{name.id}} = ""
-  {% end %}
-end
-
-macro parse_strings_from_json(names)
-  {% for name in names %}
-    @{{name.id}} = obj[{{name}}].as_s
-  {% end %}
-end
-
-macro properties_to_hash(names)
-  {
-    {% for name in names %}
-      "{{name.id}}" => @{{name.id}}.to_s,
-    {% end %}
-  }
-end
-
-module MangaDex
-  class Chapter
-    string_properties ["lang_code", "title", "volume", "chapter"]
-    property manga : Manga
-    property time = Time.local
-    property id : String
-    property full_title = ""
-    property language = ""
-    property pages = [] of {String, String} # filename, url
-    property groups = [] of {Int32, String} # group_id, group_name
-
-    def initialize(@id, json_obj : JSON::Any, @manga,
-                   lang : Hash(String, String))
-      self.parse_json json_obj, lang
-    end
-
-    def to_info_json
-      JSON.build do |json|
-        json.object do
-          {% for name in ["id", "title", "volume", "chapter",
-                          "language", "full_title"] %}
-          json.field {{name}}, @{{name.id}}
-        {% end %}
-          json.field "time", @time.to_unix.to_s
-          json.field "manga_title", @manga.title
-          json.field "manga_id", @manga.id
-          json.field "groups" do
-            json.object do
-              @groups.each do |gid, gname|
-                json.field gname, gid
-              end
-            end
-          end
-        end
-      end
-    end
-
-    def parse_json(obj, lang)
-      parse_strings_from_json ["lang_code", "title", "volume",
-                               "chapter"]
-      language = lang[@lang_code]?
-      @language = language if language
-      @time = Time.unix obj["timestamp"].as_i
-      suffixes = ["", "_2", "_3"]
-      suffixes.each do |s|
-        gid = obj["group_id#{s}"].as_i
-        next if gid == 0
-        gname = obj["group_name#{s}"].as_s
-        @groups << {gid, gname}
-      end
-
-      rename_rule = Rename::Rule.new \
-        Config.current.mangadex["chapter_rename_rule"].to_s
-      @full_title = rename rename_rule
-    rescue e
-      raise "failed to parse json: #{e}"
-    end
-
-    def rename(rule : Rename::Rule)
-      hash = properties_to_hash ["id", "title", "volume", "chapter",
-                                 "lang_code", "language", "pages"]
-      hash["groups"] = @groups.map { |g| g[1] }.join ","
-      rule.render hash
-    end
-  end
-
-  class Manga
-    string_properties ["cover_url", "description", "title", "author", "artist"]
-    property chapters = [] of Chapter
-    property id : String
-
-    def initialize(@id, json_obj : JSON::Any)
-      self.parse_json json_obj
-    end
-
-    def to_info_json(with_chapters = true)
-      JSON.build do |json|
-        json.object do
-          {% for name in ["id", "title", "description", "author", "artist",
-                          "cover_url"] %}
-            json.field {{name}}, @{{name.id}}
-          {% end %}
-          if with_chapters
-            json.field "chapters" do
-              json.array do
-                @chapters.each do |c|
-                  json.raw c.to_info_json
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-
-    def parse_json(obj)
-      parse_strings_from_json ["cover_url", "description", "title", "author",
-                               "artist"]
-    rescue e
-      raise "failed to parse json: #{e}"
-    end
-
-    def rename(rule : Rename::Rule)
-      rule.render properties_to_hash ["id", "title", "author", "artist"]
-    end
-  end
-
-  class API
-    use_default
-
-    def initialize
-      @base_url = Config.current.mangadex["api_url"].to_s ||
-                  "https://mangadex.org/api/"
-      @lang = {} of String => String
-      CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
-        @lang[row[1]] = row[0]
-      end
-    end
-
-    def get(url)
-      headers = HTTP::Headers{
-        "User-agent" => "Mangadex.cr",
-      }
-      res = HTTP::Client.get url, headers
-      raise "Failed to get #{url}. [#{res.status_code}] " \
-            "#{res.status_message}" if !res.success?
-      JSON.parse res.body
-    end
-
-    def get_manga(id)
-      obj = self.get File.join @base_url, "manga/#{id}"
-      if obj["status"]? != "OK"
-        raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
-      end
-      begin
-        manga = Manga.new id, obj["manga"]
-        obj["chapter"].as_h.map do |k, v|
-          chapter = Chapter.new k, v, manga, @lang
-          manga.chapters << chapter
-        end
-        manga
-      rescue
-        raise "Failed to parse JSON"
-      end
-    end
-
-    def get_chapter(chapter : Chapter)
-      obj = self.get File.join @base_url, "chapter/#{chapter.id}"
-      if obj["status"]? == "external"
-        raise "This chapter is hosted on an external site " \
-              "#{obj["external"]?}, and Mango does not support " \
-              "external chapters."
-      end
-      if obj["status"]? != "OK"
-        raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
-      end
-      begin
-        server = obj["server"].as_s
-        hash = obj["hash"].as_s
-        chapter.pages = obj["page_array"].as_a.map do |fn|
-          {
-            fn.as_s,
-            "#{server}#{hash}/#{fn.as_s}",
-          }
-        end
-      rescue
-        raise "Failed to parse JSON"
-      end
-    end
-
-    def get_chapter(id : String)
-      obj = self.get File.join @base_url, "chapter/#{id}"
-      if obj["status"]? == "external"
-        raise "This chapter is hosted on an external site " \
-              "#{obj["external"]?}, and Mango does not support " \
-              "external chapters."
-      end
-      if obj["status"]? != "OK"
-        raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
-      end
-      manga_id = ""
-      begin
-        manga_id = obj["manga_id"].as_i.to_s
-      rescue
-        raise "Failed to parse JSON"
-      end
-      manga = self.get_manga manga_id
-      chapter = manga.chapters.find { |c| c.id == id }.not_nil!
-      self.get_chapter chapter
-      chapter
-    end
-  end
-end

src/mangadex/downloader.cr

@@ -1,5 +1,7 @@
-require "./api"
+require "mangadex"
 require "compress/zip"
+require "../rename"
+require "./ext"
 
 module MangaDex
   class PageJob
@@ -21,7 +23,7 @@ module MangaDex
     use_default
 
     def initialize
-      @api = API.default
+      @client = Client.from_config
       super
     end
 
@@ -46,7 +48,7 @@ module MangaDex
       @downloading = true
       @queue.set_status Queue::JobStatus::Downloading, job
       begin
-        chapter = @api.get_chapter(job.id)
+        chapter = @client.chapter job.id
       rescue e
         Logger.error e
         @queue.set_status Queue::JobStatus::Error, job
@@ -73,8 +75,8 @@ module MangaDex
       # Create a buffered channel. It works as an FIFO queue
       channel = Channel(PageJob).new chapter.pages.size
       spawn do
-        chapter.pages.each_with_index do |tuple, i|
+        chapter.pages.each_with_index do |url, i|
-          fn, url = tuple
+          fn = Path.new(URI.parse(url).path).basename
           ext = File.extname fn
           fn = "#{i.to_s.rjust len, '0'}#{ext}"
           page_job = PageJob.new url, fn, writer, @retries

src/mangadex/ext.cr

@@ -0,0 +1,60 @@
+private macro properties_to_hash(names)
+  {
+    {% for name in names %}
+      "{{name.id}}" => {{name.id}}.to_s,
+    {% end %}
+  }
+end
+
+# Monkey-patch the structures in the `mangadex` shard to suit our needs
+module MangaDex
+  struct Client
+    @@group_cache = {} of String => Group
+
+    def self.from_config : Client
+      self.new base_url: Config.current.mangadex["base_url"].to_s,
+        api_url: Config.current.mangadex["api_url"].to_s
+    end
+  end
+
+  struct Manga
+    def rename(rule : Rename::Rule)
+      rule.render properties_to_hash %w(id title author artist)
+    end
+
+    def to_info_json
+      hash = JSON.parse(to_json).as_h
+      _chapters = chapters.map do |c|
+        JSON.parse c.to_info_json
+      end
+      hash["chapters"] = JSON::Any.new _chapters
+      hash.to_json
+    end
+  end
+
+  struct Chapter
+    def rename(rule : Rename::Rule)
+      hash = properties_to_hash %w(id title volume chapter lang_code language)
+      hash["groups"] = groups.map(&.name).join ","
+      rule.render hash
+    end
+
+    def full_title
+      rule = Rename::Rule.new \
+        Config.current.mangadex["chapter_rename_rule"].to_s
+      rename rule
+    end
+
+    def to_info_json
+      hash = JSON.parse(to_json).as_h
+      hash["language"] = JSON::Any.new language
+      _groups = {} of String => JSON::Any
+      groups.each do |g|
+        _groups[g.name] = JSON::Any.new g.id
+      end
+      hash["groups"] = JSON::Any.new _groups
+      hash["full_title"] = JSON::Any.new full_title
+      hash.to_json
+    end
+  end
+end

src/routes/api.cr

@@ -414,8 +414,7 @@ struct APIRouter
     get "/api/admin/mangadex/manga/:id" do |env|
       begin
         id = env.params.url["id"]
-        api = MangaDex::API.default
+        manga = MangaDex::Client.from_config.manga id
-        manga = api.get_manga id
         send_json env, manga.to_info_json
       rescue e
         Logger.error e
@@ -434,12 +433,12 @@ struct APIRouter
         chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h }
         jobs = chapters.map { |chapter|
           Queue::Job.new(
-            chapter["id"].as_s,
+            chapter["id"].as_i64.to_s,
-            chapter["manga_id"].as_s,
+            chapter["mangaId"].as_i64.to_s,
             chapter["full_title"].as_s,
-            chapter["manga_title"].as_s,
+            chapter["mangaTitle"].as_s,
             Queue::JobStatus::Pending,
-            Time.unix chapter["time"].as_s.to_i
+            Time.unix chapter["timestamp"].as_i64
           )
         }
         inserted_count = Queue.default.push jobs