diff --git a/public/js/download.js b/public/js/download.js index e3d653d..60aac64 100644 --- a/public/js/download.js +++ b/public/js/download.js @@ -27,7 +27,7 @@ const download = () => { $('#download-btn').attr('hidden', ''); $('#download-spinner').removeAttr('hidden'); const ids = selected.map((i, e) => { - return $(e).find('td').first().text(); + return parseInt($(e).find('td').first().text()); }).get(); const chapters = globalChapters.filter(c => ids.indexOf(c.id) >= 0); console.log(ids); @@ -114,8 +114,7 @@ const search = () => { return; } - const cover = baseURL + data.cover_url; - $('#cover').attr("src", cover); + $('#cover').attr("src", data.mainCover); $('#title').text("Title: " + data.title); $('#artist').text("Artist: " + data.artist); $('#author').text("Author: " + data.author); @@ -285,7 +284,7 @@ const buildTable = () => { ${group_str} ${chp.volume} ${chp.chapter} - ${moment.unix(chp.time).fromNow()} + ${moment.unix(chp.timestamp).fromNow()} `; }).join(''); const tbody = `${inner}`; diff --git a/shard.lock b/shard.lock index 99d3c5a..bd355b2 100644 --- a/shard.lock +++ b/shard.lock @@ -52,9 +52,13 @@ shards: git: https://github.com/hkalexling/koa.git version: 0.5.0 + mangadex: + git: https://github.com/hkalexling/mangadex.git + version: 0.4.0+git.commit.0c2eb69f46d8e2d0ecca3b5ed088dca36a1b5308 + mg: git: https://github.com/hkalexling/mg.git - version: 0.2.0+git.commit.171c46489d991a8353818e00fc6a3c4e0809ded9 + version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3 myhtml: git: https://github.com/kostya/myhtml.git diff --git a/shard.yml b/shard.yml index d7505a5..97bbc5b 100644 --- a/shard.yml +++ b/shard.yml @@ -43,3 +43,5 @@ dependencies: github: epoch/tallboy mg: github: hkalexling/mg + mangadex: + github: hkalexling/mangadex diff --git a/src/assets/lang_codes.csv b/src/assets/lang_codes.csv deleted file mode 100644 index 035e880..0000000 --- a/src/assets/lang_codes.csv +++ /dev/null @@ -1,41 +0,0 @@ -Arabic,sa -Bengali,bd -Bulgarian,bg -Burmese,mm -Catalan,ct -Chinese (Simp),cn -Chinese (Trad),hk -Czech,cz -Danish,dk -Dutch,nl -English,gb -Filipino,ph -Finnish,fi -French,fr -German,de -Greek,gr -Hebrew,il -Hindi,in -Hungarian,hu -Indonesian,id -Italian,it -Japanese,jp -Korean,kr -Lithuanian,lt -Malay,my -Mongolian,mn -Other, -Persian,ir -Polish,pl -Portuguese (Br),br -Portuguese (Pt),pt -Romanian,ro -Russian,ru -Serbo-Croatian,rs -Spanish (Es),es -Spanish (LATAM),mx -Swedish,se -Thai,th -Turkish,tr -Ukrainian,ua -Vietnames,vn diff --git a/src/config.cr b/src/config.cr index 3ac5af2..f8746fb 100644 --- a/src/config.cr +++ b/src/config.cr @@ -27,7 +27,7 @@ class Config @[YAML::Field(ignore: true)] @mangadex_defaults = { "base_url" => "https://mangadex.org", - "api_url" => "https://mangadex.org/api", + "api_url" => "https://mangadex.org/api/v2", "download_wait_seconds" => 5, "download_retries" => 4, "download_queue_db_path" => File.expand_path("~/mango/queue.db", @@ -91,5 +91,14 @@ class Config raise "Login is disabled, but default username is not set. " \ "Please set a default username" end + unless mangadex["api_url"] =~ /\/v2/ + # `Logger.default` is not available yet + Log.setup :debug + Log.warn { "It looks like you are using the deprecated MangaDex API " \ + "v1 in your config file. Please update it to either " \ + "https://mangadex.org/api/v2 or " \ + "https://api.mangadex.org/v2 to suppress this warning." } + mangadex["api_url"] = "https://mangadex.org/api/v2" + end end end diff --git a/src/mangadex/api.cr b/src/mangadex/api.cr deleted file mode 100644 index b521a27..0000000 --- a/src/mangadex/api.cr +++ /dev/null @@ -1,217 +0,0 @@ -require "json" -require "csv" -require "../rename" - -macro string_properties(names) - {% for name in names %} - property {{name.id}} = "" - {% end %} -end - -macro parse_strings_from_json(names) - {% for name in names %} - @{{name.id}} = obj[{{name}}].as_s - {% end %} -end - -macro properties_to_hash(names) - { - {% for name in names %} - "{{name.id}}" => @{{name.id}}.to_s, - {% end %} - } -end - -module MangaDex - class Chapter - string_properties ["lang_code", "title", "volume", "chapter"] - property manga : Manga - property time = Time.local - property id : String - property full_title = "" - property language = "" - property pages = [] of {String, String} # filename, url - property groups = [] of {Int32, String} # group_id, group_name - - def initialize(@id, json_obj : JSON::Any, @manga, - lang : Hash(String, String)) - self.parse_json json_obj, lang - end - - def to_info_json - JSON.build do |json| - json.object do - {% for name in ["id", "title", "volume", "chapter", - "language", "full_title"] %} - json.field {{name}}, @{{name.id}} - {% end %} - json.field "time", @time.to_unix.to_s - json.field "manga_title", @manga.title - json.field "manga_id", @manga.id - json.field "groups" do - json.object do - @groups.each do |gid, gname| - json.field gname, gid - end - end - end - end - end - end - - def parse_json(obj, lang) - parse_strings_from_json ["lang_code", "title", "volume", - "chapter"] - language = lang[@lang_code]? - @language = language if language - @time = Time.unix obj["timestamp"].as_i - suffixes = ["", "_2", "_3"] - suffixes.each do |s| - gid = obj["group_id#{s}"].as_i - next if gid == 0 - gname = obj["group_name#{s}"].as_s - @groups << {gid, gname} - end - - rename_rule = Rename::Rule.new \ - Config.current.mangadex["chapter_rename_rule"].to_s - @full_title = rename rename_rule - rescue e - raise "failed to parse json: #{e}" - end - - def rename(rule : Rename::Rule) - hash = properties_to_hash ["id", "title", "volume", "chapter", - "lang_code", "language", "pages"] - hash["groups"] = @groups.map { |g| g[1] }.join "," - rule.render hash - end - end - - class Manga - string_properties ["cover_url", "description", "title", "author", "artist"] - property chapters = [] of Chapter - property id : String - - def initialize(@id, json_obj : JSON::Any) - self.parse_json json_obj - end - - def to_info_json(with_chapters = true) - JSON.build do |json| - json.object do - {% for name in ["id", "title", "description", "author", "artist", - "cover_url"] %} - json.field {{name}}, @{{name.id}} - {% end %} - if with_chapters - json.field "chapters" do - json.array do - @chapters.each do |c| - json.raw c.to_info_json - end - end - end - end - end - end - end - - def parse_json(obj) - parse_strings_from_json ["cover_url", "description", "title", "author", - "artist"] - rescue e - raise "failed to parse json: #{e}" - end - - def rename(rule : Rename::Rule) - rule.render properties_to_hash ["id", "title", "author", "artist"] - end - end - - class API - use_default - - def initialize - @base_url = Config.current.mangadex["api_url"].to_s || - "https://mangadex.org/api/" - @lang = {} of String => String - CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row| - @lang[row[1]] = row[0] - end - end - - def get(url) - headers = HTTP::Headers{ - "User-agent" => "Mangadex.cr", - } - res = HTTP::Client.get url, headers - raise "Failed to get #{url}. [#{res.status_code}] " \ - "#{res.status_message}" if !res.success? - JSON.parse res.body - end - - def get_manga(id) - obj = self.get File.join @base_url, "manga/#{id}" - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" - end - begin - manga = Manga.new id, obj["manga"] - obj["chapter"].as_h.map do |k, v| - chapter = Chapter.new k, v, manga, @lang - manga.chapters << chapter - end - manga - rescue - raise "Failed to parse JSON" - end - end - - def get_chapter(chapter : Chapter) - obj = self.get File.join @base_url, "chapter/#{chapter.id}" - if obj["status"]? == "external" - raise "This chapter is hosted on an external site " \ - "#{obj["external"]?}, and Mango does not support " \ - "external chapters." - end - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" - end - begin - server = obj["server"].as_s - hash = obj["hash"].as_s - chapter.pages = obj["page_array"].as_a.map do |fn| - { - fn.as_s, - "#{server}#{hash}/#{fn.as_s}", - } - end - rescue - raise "Failed to parse JSON" - end - end - - def get_chapter(id : String) - obj = self.get File.join @base_url, "chapter/#{id}" - if obj["status"]? == "external" - raise "This chapter is hosted on an external site " \ - "#{obj["external"]?}, and Mango does not support " \ - "external chapters." - end - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" - end - manga_id = "" - begin - manga_id = obj["manga_id"].as_i.to_s - rescue - raise "Failed to parse JSON" - end - manga = self.get_manga manga_id - chapter = manga.chapters.find { |c| c.id == id }.not_nil! - self.get_chapter chapter - chapter - end - end -end diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr index e2babb6..c0b50c7 100644 --- a/src/mangadex/downloader.cr +++ b/src/mangadex/downloader.cr @@ -1,5 +1,7 @@ -require "./api" +require "mangadex" require "compress/zip" +require "../rename" +require "./ext" module MangaDex class PageJob @@ -21,7 +23,7 @@ module MangaDex use_default def initialize - @api = API.default + @client = Client.from_config super end @@ -46,7 +48,7 @@ module MangaDex @downloading = true @queue.set_status Queue::JobStatus::Downloading, job begin - chapter = @api.get_chapter(job.id) + chapter = @client.chapter job.id rescue e Logger.error e @queue.set_status Queue::JobStatus::Error, job @@ -73,8 +75,8 @@ module MangaDex # Create a buffered channel. It works as an FIFO queue channel = Channel(PageJob).new chapter.pages.size spawn do - chapter.pages.each_with_index do |tuple, i| - fn, url = tuple + chapter.pages.each_with_index do |url, i| + fn = Path.new(URI.parse(url).path).basename ext = File.extname fn fn = "#{i.to_s.rjust len, '0'}#{ext}" page_job = PageJob.new url, fn, writer, @retries diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr new file mode 100644 index 0000000..dfb302c --- /dev/null +++ b/src/mangadex/ext.cr @@ -0,0 +1,60 @@ +private macro properties_to_hash(names) + { + {% for name in names %} + "{{name.id}}" => {{name.id}}.to_s, + {% end %} + } +end + +# Monkey-patch the structures in the `mangadex` shard to suit our needs +module MangaDex + struct Client + @@group_cache = {} of String => Group + + def self.from_config : Client + self.new base_url: Config.current.mangadex["base_url"].to_s, + api_url: Config.current.mangadex["api_url"].to_s + end + end + + struct Manga + def rename(rule : Rename::Rule) + rule.render properties_to_hash %w(id title author artist) + end + + def to_info_json + hash = JSON.parse(to_json).as_h + _chapters = chapters.map do |c| + JSON.parse c.to_info_json + end + hash["chapters"] = JSON::Any.new _chapters + hash.to_json + end + end + + struct Chapter + def rename(rule : Rename::Rule) + hash = properties_to_hash %w(id title volume chapter lang_code language) + hash["groups"] = groups.map(&.name).join "," + rule.render hash + end + + def full_title + rule = Rename::Rule.new \ + Config.current.mangadex["chapter_rename_rule"].to_s + rename rule + end + + def to_info_json + hash = JSON.parse(to_json).as_h + hash["language"] = JSON::Any.new language + _groups = {} of String => JSON::Any + groups.each do |g| + _groups[g.name] = JSON::Any.new g.id + end + hash["groups"] = JSON::Any.new _groups + hash["full_title"] = JSON::Any.new full_title + hash.to_json + end + end +end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7fc8b46..ff6a087 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -414,8 +414,7 @@ struct APIRouter get "/api/admin/mangadex/manga/:id" do |env| begin id = env.params.url["id"] - api = MangaDex::API.default - manga = api.get_manga id + manga = MangaDex::Client.from_config.manga id send_json env, manga.to_info_json rescue e Logger.error e @@ -434,12 +433,12 @@ struct APIRouter chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h } jobs = chapters.map { |chapter| Queue::Job.new( - chapter["id"].as_s, - chapter["manga_id"].as_s, + chapter["id"].as_i64.to_s, + chapter["mangaId"].as_i64.to_s, chapter["full_title"].as_s, - chapter["manga_title"].as_s, + chapter["mangaTitle"].as_s, Queue::JobStatus::Pending, - Time.unix chapter["time"].as_s.to_i + Time.unix chapter["timestamp"].as_i64 ) } inserted_count = Queue.default.push jobs