Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2025-08-02 10:55:30 -04:00
Code Issues Projects Releases Wiki Activity

Upgrade to MangaDex API v2

Browse Source
This commit is contained in:
Alex Ling 2021-01-30 17:08:04 +00:00
parent ca8e9a164e
commit 70d418d1a1
9 changed files with 92 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
public/js/download.js
View File

@ -27,7 +27,7 @@ const download = () => {
$('#download-btn').attr('hidden', '');
$('#download-spinner').removeAttr('hidden');
const ids = selected.map((i, e) => {
return $(e).find('td').first().text();
return parseInt($(e).find('td').first().text());
}).get();
const chapters = globalChapters.filter(c => ids.indexOf(c.id) >= 0);
console.log(ids);
@ -114,8 +114,7 @@ const search = () => {
return;
}
const cover = baseURL + data.cover_url;
$('#cover').attr("src", cover);
$('#cover').attr("src", data.mainCover);
$('#title').text("Title: " + data.title);
$('#artist').text("Artist: " + data.artist);
$('#author').text("Author: " + data.author);
@ -285,7 +284,7 @@ const buildTable = () => {
<td>${group_str}</td>
<td>${chp.volume}</td>
<td>${chp.chapter}</td>
<td>${moment.unix(chp.time).fromNow()}</td>
<td>${moment.unix(chp.timestamp).fromNow()}</td>
</tr>`;
}).join('');
const tbody = `<tbody id="selectable">${inner}</tbody>`;

6
shard.lock
View File

@ -52,9 +52,13 @@ shards:
git: https://github.com/hkalexling/koa.git
version: 0.5.0
mangadex:
git: https://github.com/hkalexling/mangadex.git
version: 0.4.0+git.commit.0c2eb69f46d8e2d0ecca3b5ed088dca36a1b5308
mg:
git: https://github.com/hkalexling/mg.git
version: 0.2.0+git.commit.171c46489d991a8353818e00fc6a3c4e0809ded9
version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3
myhtml:
git: https://github.com/kostya/myhtml.git

2
shard.yml
View File

@ -43,3 +43,5 @@ dependencies:
github: epoch/tallboy
mg:
github: hkalexling/mg
mangadex:
github: hkalexling/mangadex

41
src/assets/lang_codes.csv
View File

@ -1,41 +0,0 @@
Arabic,sa
Bengali,bd
Bulgarian,bg
Burmese,mm
Catalan,ct
Chinese (Simp),cn
Chinese (Trad),hk
Czech,cz
Danish,dk
Dutch,nl
English,gb
Filipino,ph
Finnish,fi
French,fr
German,de
Greek,gr
Hebrew,il
Hindi,in
Hungarian,hu
Indonesian,id
Italian,it
Japanese,jp
Korean,kr
Lithuanian,lt
Malay,my
Mongolian,mn
Other,
Persian,ir
Polish,pl
Portuguese (Br),br
Portuguese (Pt),pt
Romanian,ro
Russian,ru
Serbo-Croatian,rs
Spanish (Es),es
Spanish (LATAM),mx
Swedish,se
Thai,th
Turkish,tr
Ukrainian,ua
Vietnames,vn
1 Arabic sa
2 Bengali bd
3 Bulgarian bg
4 Burmese mm
5 Catalan ct
6 Chinese (Simp) cn
7 Chinese (Trad) hk
8 Czech cz
9 Danish dk
10 Dutch nl
11 English gb
12 Filipino ph
13 Finnish fi
14 French fr
15 German de
16 Greek gr
17 Hebrew il
18 Hindi in
19 Hungarian hu
20 Indonesian id
21 Italian it
22 Japanese jp
23 Korean kr
24 Lithuanian lt
25 Malay my
26 Mongolian mn
27 Other
28 Persian ir
29 Polish pl
30 Portuguese (Br) br
31 Portuguese (Pt) pt
32 Romanian ro
33 Russian ru
34 Serbo-Croatian rs
35 Spanish (Es) es
36 Spanish (LATAM) mx
37 Swedish se
38 Thai th
39 Turkish tr
40 Ukrainian ua
41 Vietnames vn

11
src/config.cr
View File

@ -27,7 +27,7 @@ class Config
@[YAML::Field(ignore: true)]
@mangadex_defaults = {
"base_url" => "https://mangadex.org",
"api_url" => "https://mangadex.org/api",
"api_url" => "https://mangadex.org/api/v2",
"download_wait_seconds" => 5,
"download_retries" => 4,
"download_queue_db_path" => File.expand_path("~/mango/queue.db",
@ -91,5 +91,14 @@ class Config
raise "Login is disabled, but default username is not set. " \
"Please set a default username"
end
unless mangadex["api_url"] =~ /\/v2/
# `Logger.default` is not available yet
Log.setup :debug
Log.warn { "It looks like you are using the deprecated MangaDex API " \
"v1 in your config file. Please update it to either " \
"https://mangadex.org/api/v2 or " \
"https://api.mangadex.org/v2 to suppress this warning." }
mangadex["api_url"] = "https://mangadex.org/api/v2"
end
end
end

217
src/mangadex/api.cr
View File

@ -1,217 +0,0 @@
require "json"
require "csv"
require "../rename"
macro string_properties(names)
{% for name in names %}
property {{name.id}} = ""
{% end %}
end
macro parse_strings_from_json(names)
{% for name in names %}
@{{name.id}} = obj[{{name}}].as_s
{% end %}
end
macro properties_to_hash(names)
{
{% for name in names %}
"{{name.id}}" => @{{name.id}}.to_s,
{% end %}
}
end
module MangaDex
class Chapter
string_properties ["lang_code", "title", "volume", "chapter"]
property manga : Manga
property time = Time.local
property id : String
property full_title = ""
property language = ""
property pages = [] of {String, String} # filename, url
property groups = [] of {Int32, String} # group_id, group_name
def initialize(@id, json_obj : JSON::Any, @manga,
lang : Hash(String, String))
self.parse_json json_obj, lang
end
def to_info_json
JSON.build do |json|
json.object do
{% for name in ["id", "title", "volume", "chapter",
"language", "full_title"] %}
json.field {{name}}, @{{name.id}}
{% end %}
json.field "time", @time.to_unix.to_s
json.field "manga_title", @manga.title
json.field "manga_id", @manga.id
json.field "groups" do
json.object do
@groups.each do |gid, gname|
json.field gname, gid
end
end
end
end
end
end
def parse_json(obj, lang)
parse_strings_from_json ["lang_code", "title", "volume",
"chapter"]
language = lang[@lang_code]?
@language = language if language
@time = Time.unix obj["timestamp"].as_i
suffixes = ["", "_2", "_3"]
suffixes.each do |s|
gid = obj["group_id#{s}"].as_i
next if gid == 0
gname = obj["group_name#{s}"].as_s
@groups << {gid, gname}
end
rename_rule = Rename::Rule.new \
Config.current.mangadex["chapter_rename_rule"].to_s
@full_title = rename rename_rule
rescue e
raise "failed to parse json: #{e}"
end
def rename(rule : Rename::Rule)
hash = properties_to_hash ["id", "title", "volume", "chapter",
"lang_code", "language", "pages"]
hash["groups"] = @groups.map { |g| g[1] }.join ","
rule.render hash
end
end
class Manga
string_properties ["cover_url", "description", "title", "author", "artist"]
property chapters = [] of Chapter
property id : String
def initialize(@id, json_obj : JSON::Any)
self.parse_json json_obj
end
def to_info_json(with_chapters = true)
JSON.build do |json|
json.object do
{% for name in ["id", "title", "description", "author", "artist",
"cover_url"] %}
json.field {{name}}, @{{name.id}}
{% end %}
if with_chapters
json.field "chapters" do
json.array do
@chapters.each do |c|
json.raw c.to_info_json
end
end
end
end
end
end
end
def parse_json(obj)
parse_strings_from_json ["cover_url", "description", "title", "author",
"artist"]
rescue e
raise "failed to parse json: #{e}"
end
def rename(rule : Rename::Rule)
rule.render properties_to_hash ["id", "title", "author", "artist"]
end
end
class API
use_default
def initialize
@base_url = Config.current.mangadex["api_url"].to_s ||
"https://mangadex.org/api/"
@lang = {} of String => String
CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
@lang[row[1]] = row[0]
end
end
def get(url)
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
res = HTTP::Client.get url, headers
raise "Failed to get #{url}. [#{res.status_code}] " \
"#{res.status_message}" if !res.success?
JSON.parse res.body
end
def get_manga(id)
obj = self.get File.join @base_url, "manga/#{id}"
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
manga = Manga.new id, obj["manga"]
obj["chapter"].as_h.map do |k, v|
chapter = Chapter.new k, v, manga, @lang
manga.chapters << chapter
end
manga
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(chapter : Chapter)
obj = self.get File.join @base_url, "chapter/#{chapter.id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
server = obj["server"].as_s
hash = obj["hash"].as_s
chapter.pages = obj["page_array"].as_a.map do |fn|
{
fn.as_s,
"#{server}#{hash}/#{fn.as_s}",
}
end
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(id : String)
obj = self.get File.join @base_url, "chapter/#{id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
manga_id = ""
begin
manga_id = obj["manga_id"].as_i.to_s
rescue
raise "Failed to parse JSON"
end
manga = self.get_manga manga_id
chapter = manga.chapters.find { |c| c.id == id }.not_nil!
self.get_chapter chapter
chapter
end
end
end

12
src/mangadex/downloader.cr
View File

@ -1,5 +1,7 @@
require "./api"
require "mangadex"
require "compress/zip"
require "../rename"
require "./ext"
module MangaDex
class PageJob
@ -21,7 +23,7 @@ module MangaDex
use_default
def initialize
@api = API.default
@client = Client.from_config
super
end
@ -46,7 +48,7 @@ module MangaDex
@downloading = true
@queue.set_status Queue::JobStatus::Downloading, job
begin
chapter = @api.get_chapter(job.id)
chapter = @client.chapter job.id
rescue e
Logger.error e
@queue.set_status Queue::JobStatus::Error, job
@ -73,8 +75,8 @@ module MangaDex
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new chapter.pages.size
spawn do
chapter.pages.each_with_index do |tuple, i|
fn, url = tuple
chapter.pages.each_with_index do |url, i|
fn = Path.new(URI.parse(url).path).basename
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries

60
src/mangadex/ext.cr Normal file
View File

@ -0,0 +1,60 @@
private macro properties_to_hash(names)
{
{% for name in names %}
"{{name.id}}" => {{name.id}}.to_s,
{% end %}
}
end
# Monkey-patch the structures in the `mangadex` shard to suit our needs
module MangaDex
struct Client
@@group_cache = {} of String => Group
def self.from_config : Client
self.new base_url: Config.current.mangadex["base_url"].to_s,
api_url: Config.current.mangadex["api_url"].to_s
end
end
struct Manga
def rename(rule : Rename::Rule)
rule.render properties_to_hash %w(id title author artist)
end
def to_info_json
hash = JSON.parse(to_json).as_h
_chapters = chapters.map do |c|
JSON.parse c.to_info_json
end
hash["chapters"] = JSON::Any.new _chapters
hash.to_json
end
end
struct Chapter
def rename(rule : Rename::Rule)
hash = properties_to_hash %w(id title volume chapter lang_code language)
hash["groups"] = groups.map(&.name).join ","
rule.render hash
end
def full_title
rule = Rename::Rule.new \
Config.current.mangadex["chapter_rename_rule"].to_s
rename rule
end
def to_info_json
hash = JSON.parse(to_json).as_h
hash["language"] = JSON::Any.new language
_groups = {} of String => JSON::Any
groups.each do |g|
_groups[g.name] = JSON::Any.new g.id
end
hash["groups"] = JSON::Any.new _groups
hash["full_title"] = JSON::Any.new full_title
hash.to_json
end
end
end

11
src/routes/api.cr
View File

@ -414,8 +414,7 @@ struct APIRouter
get "/api/admin/mangadex/manga/:id" do |env|
begin
id = env.params.url["id"]
api = MangaDex::API.default
manga = api.get_manga id
manga = MangaDex::Client.from_config.manga id
send_json env, manga.to_info_json
rescue e
Logger.error e
@ -434,12 +433,12 @@ struct APIRouter
chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h }
jobs = chapters.map { |chapter|
Queue::Job.new(
chapter["id"].as_s,
chapter["manga_id"].as_s,
chapter["id"].as_i64.to_s,
chapter["mangaId"].as_i64.to_s,
chapter["full_title"].as_s,
chapter["manga_title"].as_s,
chapter["mangaTitle"].as_s,
Queue::JobStatus::Pending,
Time.unix chapter["time"].as_s.to_i
Time.unix chapter["timestamp"].as_i64
)
}
inserted_count = Queue.default.push jobs