Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2026-04-25 00:00:52 -04:00
Code Issues Projects Releases Wiki Activity

Project-wise code formatting

Browse Source
This commit is contained in:
Alex Ling
2020-04-08 03:38:02 +00:00
parent 3866c81588
commit 8b184ed48d
23 changed files with 1864 additions and 1870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mangadex/api.cr
+181 -183
View File
@@ -2,202 +2,200 @@ require "http/client"
require "json"
require "csv"
macro string_properties (names)
{% for name in names %}
property {{name.id}} = ""
{% end %}
macro string_properties(names)
{% for name in names %}
property {{name.id}} = ""
{% end %}
end
macro parse_strings_from_json (names)
{% for name in names %}
@{{name.id}} = obj[{{name}}].as_s
{% end %}
macro parse_strings_from_json(names)
{% for name in names %}
@{{name.id}} = obj[{{name}}].as_s
{% end %}
end
module MangaDex
class Chapter
string_properties ["lang_code", "title", "volume", "chapter"]
property manga : Manga
property time = Time.local
property id : String
property full_title = ""
property language = ""
property pages = [] of {String, String} # filename, url
property groups = [] of {Int32, String} # group_id, group_name
class Chapter
string_properties ["lang_code", "title", "volume", "chapter"]
property manga : Manga
property time = Time.local
property id : String
property full_title = ""
property language = ""
property pages = [] of {String, String} # filename, url
property groups = [] of {Int32, String} # group_id, group_name
def initialize(@id, json_obj : JSON::Any, @manga, lang :
Hash(String, String))
self.parse_json json_obj, lang
end
def initialize(@id, json_obj : JSON::Any, @manga,
lang : Hash(String, String))
self.parse_json json_obj, lang
end
def to_info_json
JSON.build do |json|
json.object do
{% for name in ["id", "title", "volume", "chapter",
"language", "full_title"] %}
json.field {{name}}, @{{name.id}}
{% end %}
json.field "time", @time.to_unix.to_s
json.field "manga_title", @manga.title
json.field "manga_id", @manga.id
json.field "groups" do
json.object do
@groups.each do |gid, gname|
json.field gname, gid
end
end
end
end
end
end
def to_info_json
JSON.build do |json|
json.object do
{% for name in ["id", "title", "volume", "chapter",
"language", "full_title"] %}
json.field {{name}}, @{{name.id}}
{% end %}
json.field "time", @time.to_unix.to_s
json.field "manga_title", @manga.title
json.field "manga_id", @manga.id
json.field "groups" do
json.object do
@groups.each do |gid, gname|
json.field gname, gid
end
end
end
end
end
end
def parse_json(obj, lang)
begin
parse_strings_from_json ["lang_code", "title", "volume",
"chapter"]
language = lang[@lang_code]?
@language = language if language
@time = Time.unix obj["timestamp"].as_i
suffixes = ["", "_2", "_3"]
suffixes.each do |s|
gid = obj["group_id#{s}"].as_i
next if gid == 0
gname = obj["group_name#{s}"].as_s
@groups << {gid, gname}
end
@full_title = @title
unless @chapter.empty?
@full_title = "Ch.#{@chapter} " + @full_title
end
unless @volume.empty?
@full_title = "Vol.#{@volume} " + @full_title
end
rescue e
raise "failed to parse json: #{e}"
end
end
end
class Manga
string_properties ["cover_url", "description", "title", "author",
"artist"]
property chapters = [] of Chapter
property id : String
def parse_json(obj, lang)
begin
parse_strings_from_json ["lang_code", "title", "volume",
"chapter"]
language = lang[@lang_code]?
@language = language if language
@time = Time.unix obj["timestamp"].as_i
suffixes = ["", "_2", "_3"]
suffixes.each do |s|
gid = obj["group_id#{s}"].as_i
next if gid == 0
gname = obj["group_name#{s}"].as_s
@groups << {gid, gname}
end
@full_title = @title
unless @chapter.empty?
@full_title = "Ch.#{@chapter} " + @full_title
end
unless @volume.empty?
@full_title = "Vol.#{@volume} " + @full_title
end
rescue e
raise "failed to parse json: #{e}"
end
end
end
def initialize(@id, json_obj : JSON::Any)
self.parse_json json_obj
end
class Manga
string_properties ["cover_url", "description", "title", "author", "artist"]
property chapters = [] of Chapter
property id : String
def to_info_json(with_chapters = true)
JSON.build do |json|
json.object do
{% for name in ["id", "title", "description",
"author", "artist", "cover_url"] %}
json.field {{name}}, @{{name.id}}
{% end %}
if with_chapters
json.field "chapters" do
json.array do
@chapters.each do |c|
json.raw c.to_info_json
end
end
end
end
end
end
end
def initialize(@id, json_obj : JSON::Any)
self.parse_json json_obj
end
def parse_json(obj)
begin
parse_strings_from_json ["cover_url", "description", "title",
"author", "artist"]
rescue e
raise "failed to parse json: #{e}"
end
end
end
class API
def initialize(@base_url = "https://mangadex.org/api/")
@lang = {} of String => String
CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
@lang[row[1]] = row[0]
end
end
def to_info_json(with_chapters = true)
JSON.build do |json|
json.object do
{% for name in ["id", "title", "description", "author", "artist",
"cover_url"] %}
json.field {{name}}, @{{name.id}}
{% end %}
if with_chapters
json.field "chapters" do
json.array do
@chapters.each do |c|
json.raw c.to_info_json
end
end
end
end
end
end
end
def get(url)
headers = HTTP::Headers {
"User-agent" => "Mangadex.cr"
}
res = HTTP::Client.get url, headers
raise "Failed to get #{url}. [#{res.status_code}] "\
"#{res.status_message}" if !res.success?
JSON.parse res.body
end
def parse_json(obj)
begin
parse_strings_from_json ["cover_url", "description", "title", "author",
"artist"]
rescue e
raise "failed to parse json: #{e}"
end
end
end
def get_manga(id)
obj = self.get File.join @base_url, "manga/#{id}"
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
begin
manga = Manga.new id, obj["manga"]
obj["chapter"].as_h.map do |k, v|
chapter = Chapter.new k, v, manga, @lang
manga.chapters << chapter
end
return manga
rescue
raise "Failed to parse JSON"
end
end
class API
def initialize(@base_url = "https://mangadex.org/api/")
@lang = {} of String => String
CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
@lang[row[1]] = row[0]
end
end
def get_chapter(chapter : Chapter)
obj = self.get File.join @base_url, "chapter/#{chapter.id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
begin
server = obj["server"].as_s
hash = obj["hash"].as_s
chapter.pages = obj["page_array"].as_a.map do |fn|
{
fn.as_s,
"#{server}#{hash}/#{fn.as_s}"
}
end
rescue
raise "Failed to parse JSON"
end
end
def get(url)
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
res = HTTP::Client.get url, headers
raise "Failed to get #{url}. [#{res.status_code}] " \
"#{res.status_message}" if !res.success?
JSON.parse res.body
end
def get_chapter(id : String)
obj = self.get File.join @base_url, "chapter/#{id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
manga_id = ""
begin
manga_id = obj["manga_id"].as_i.to_s
rescue
raise "Failed to parse JSON"
end
manga = self.get_manga manga_id
chapter = manga.chapters.find {|c| c.id == id}.not_nil!
self.get_chapter chapter
return chapter
end
end
def get_manga(id)
obj = self.get File.join @base_url, "manga/#{id}"
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
manga = Manga.new id, obj["manga"]
obj["chapter"].as_h.map do |k, v|
chapter = Chapter.new k, v, manga, @lang
manga.chapters << chapter
end
return manga
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(chapter : Chapter)
obj = self.get File.join @base_url, "chapter/#{chapter.id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
server = obj["server"].as_s
hash = obj["hash"].as_s
chapter.pages = obj["page_array"].as_a.map do |fn|
{
fn.as_s,
"#{server}#{hash}/#{fn.as_s}",
}
end
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(id : String)
obj = self.get File.join @base_url, "chapter/#{id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
manga_id = ""
begin
manga_id = obj["manga_id"].as_i.to_s
rescue
raise "Failed to parse JSON"
end
manga = self.get_manga manga_id
chapter = manga.chapters.find { |c| c.id == id }.not_nil!
self.get_chapter chapter
return chapter
end
end
end
src/mangadex/downloader.cr
+334 -333
View File
@@ -2,373 +2,374 @@ require "./api"
require "sqlite3"
module MangaDex
class PageJob
property success = false
property url : String
property filename : String
property writer : Zip::Writer
property tries_remaning : Int32
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
class PageJob
property success = false
property url : String
property filename : String
property writer : Zip::Writer
property tries_remaning : Int32
enum JobStatus
Pending # 0
Downloading # 1
Error # 2
Completed # 3
MissingPages # 4
end
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
struct Job
property id : String
property manga_id : String
property title : String
property manga_title : String
property status : JobStatus
property status_message : String = ""
property pages : Int32 = 0
property success_count : Int32 = 0
property fail_count : Int32 = 0
property time : Time
enum JobStatus
Pending # 0
Downloading # 1
Error # 2
Completed # 3
MissingPages # 4
end
def parse_query_result(res : DB::ResultSet)
@id = res.read String
@manga_id = res.read String
@title = res.read String
@manga_title = res.read String
status = res.read Int32
@status_message = res.read String
@pages = res.read Int32
@success_count = res.read Int32
@fail_count = res.read Int32
time = res.read Int64
@status = JobStatus.new status
@time = Time.unix_ms time
end
struct Job
property id : String
property manga_id : String
property title : String
property manga_title : String
property status : JobStatus
property status_message : String = ""
property pages : Int32 = 0
property success_count : Int32 = 0
property fail_count : Int32 = 0
property time : Time
# Raises if the result set does not contain the correct set of columns
def self.from_query_result(res : DB::ResultSet)
job = Job.allocate
job.parse_query_result res
return job
end
def parse_query_result(res : DB::ResultSet)
@id = res.read String
@manga_id = res.read String
@title = res.read String
@manga_title = res.read String
status = res.read Int32
@status_message = res.read String
@pages = res.read Int32
@success_count = res.read Int32
@fail_count = res.read Int32
time = res.read Int64
@status = JobStatus.new status
@time = Time.unix_ms time
end
def initialize(@id, @manga_id, @title, @manga_title, @status, @time)
end
# Raises if the result set does not contain the correct set of columns
def self.from_query_result(res : DB::ResultSet)
job = Job.allocate
job.parse_query_result res
return job
end
def to_json(json)
json.object do
{% for name in ["id", "manga_id", "title", "manga_title",
"status_message"] %}
json.field {{name}}, @{{name.id}}
{% end %}
{% for name in ["pages", "success_count", "fail_count"] %}
json.field {{name}} do
json.number @{{name.id}}
end
{% end %}
json.field "status", @status.to_s
json.field "time" do
json.number @time.to_unix_ms
end
end
end
end
def initialize(@id, @manga_id, @title, @manga_title, @status, @time)
end
class Queue
property downloader : Downloader?
def to_json(json)
json.object do
{% for name in ["id", "manga_id", "title", "manga_title",
"status_message"] %}
json.field {{name}}, @{{name.id}}
{% end %}
{% for name in ["pages", "success_count", "fail_count"] %}
json.field {{name}} do
json.number @{{name.id}}
end
{% end %}
json.field "status", @status.to_s
json.field "time" do
json.number @time.to_unix_ms
end
end
end
end
def initialize(@path : String, @logger : Logger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The queue DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
@logger.error "Error when checking tables in DB: #{e}"
raise e
end
end
end
class Queue
property downloader : Downloader?
# Returns the earliest job in queue or nil if the job cannot be parsed.
# Returns nil if queue is empty
def pop
job = nil
DB.open "sqlite3://#{@path}" do |db|
begin
db.query_one "select * from queue where status = 0 "\
"or status = 1 order by time limit 1" do |res|
job = Job.from_query_result res
end
rescue
end
end
return job
end
def initialize(@path : String, @logger : Logger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The queue DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
@logger.error "Error when checking tables in DB: #{e}"
raise e
end
end
end
# Push an array of jobs into the queue, and return the number of jobs
# inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job))
start_count = self.count
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values "\
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end
self.count - start_count
end
# Returns the earliest job in queue or nil if the job cannot be parsed.
# Returns nil if queue is empty
def pop
job = nil
DB.open "sqlite3://#{@path}" do |db|
begin
db.query_one "select * from queue where status = 0 " \
"or status = 1 order by time limit 1" do |res|
job = Job.from_query_result res
end
rescue
end
end
return job
end
def reset(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end
# Push an array of jobs into the queue, and return the number of jobs
# inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job))
start_count = self.count
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values " \
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end
self.count - start_count
end
def reset (job : Job)
self.reset job.id
end
def reset(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end
# Reset all failed tasks (missing pages and error)
def reset
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end
def reset(job : Job)
self.reset job.id
end
def delete(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end
# Reset all failed tasks (missing pages and error)
def reset
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end
def delete(job : Job)
self.delete job.id
end
def delete(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end
def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end
def delete(job : Job)
self.delete job.id
end
def count_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue where "\
"status = (?)", status.to_i, as: Int32
end
end
def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end
def count
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue", as: Int32
end
end
def count_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue where " \
"status = (?)", status.to_i, as: Int32
end
end
def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end
def count
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue", as: Int32
end
end
def get_all
jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time", do |rs|
Job.from_query_result rs
end
end
return jobs
end
def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end
def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end
def get_all
jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time" do |rs|
Job.from_query_result rs
end
end
return jobs
end
def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end
def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end
def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end
def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end
def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end
def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end
def pause
@downloader.not_nil!.stopped = true
end
def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end
def resume
@downloader.not_nil!.stopped = false
end
def pause
@downloader.not_nil!.stopped = true
end
def paused?
@downloader.not_nil!.stopped
end
end
def resume
@downloader.not_nil!.stopped = false
end
class Downloader
property stopped = false
@downloading = false
def paused?
@downloader.not_nil!.stopped
end
end
def initialize(@queue : Queue, @api : API, @library_path : String,
@wait_seconds : Int32, @retries : Int32,
@logger : Logger)
@queue.downloader = self
class Downloader
property stopped = false
@downloading = false
spawn do
loop do
sleep 1.second
next if @stopped || @downloading
begin
job = @queue.pop
next if job.nil?
download job
rescue e
@logger.error e
end
end
end
end
def initialize(@queue : Queue, @api : API, @library_path : String,
@wait_seconds : Int32, @retries : Int32,
@logger : Logger)
@queue.downloader = self
private def download(job : Job)
@downloading = true
@queue.set_status JobStatus::Downloading, job
begin
chapter = @api.get_chapter(job.id)
rescue e
@logger.error e
@queue.set_status JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages chapter.pages.size, job
lib_dir = @library_path
manga_dir = File.join lib_dir, chapter.manga.title
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz"
spawn do
loop do
sleep 1.second
next if @stopped || @downloading
begin
job = @queue.pop
next if job.nil?
download job
rescue e
@logger.error e
end
end
end
end
# Find the number of digits needed to store the number of pages
len = Math.log10(chapter.pages.size).to_i + 1
private def download(job : Job)
@downloading = true
@queue.set_status JobStatus::Downloading, job
begin
chapter = @api.get_chapter(job.id)
rescue e
@logger.error e
@queue.set_status JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages chapter.pages.size, job
lib_dir = @library_path
manga_dir = File.join lib_dir, chapter.manga.title
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz"
writer = Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new chapter.pages.size
spawn do
chapter.pages.each_with_index do |tuple, i|
fn, url = tuple
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
@logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
@logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
# Find the number of digits needed to store the number of pages
len = Math.log10(chapter.pages.size).to_i + 1
channel.send page_job
end
end
writer = Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new chapter.pages.size
spawn do
chapter.pages.each_with_index do |tuple, i|
fn, url = tuple
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
@logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
@logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
spawn do
page_jobs = [] of PageJob
chapter.pages.size.times do
page_job = channel.receive
@logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
@logger.error msg
end
end
fail_count = page_jobs.select{|j| !j.success}.size
@logger.debug "Download completed. "\
"#{fail_count}/#{page_jobs.size} failed"
writer.close
@logger.debug "cbz File created at #{zip_path}"
if fail_count == 0
@queue.set_status JobStatus::Completed, job
else
@queue.set_status JobStatus::MissingPages, job
end
@downloading = false
end
end
channel.send page_job
end
end
private def download_page(job : PageJob)
@logger.debug "downloading #{job.url}"
headers = HTTP::Headers {
"User-agent" => "Mangadex.cr"
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
@logger.error e
job.success = false
end
end
end
spawn do
page_jobs = [] of PageJob
chapter.pages.size.times do
page_job = channel.receive
@logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
@logger.error msg
end
end
fail_count = page_jobs.select { |j| !j.success }.size
@logger.debug "Download completed. " \
"#{fail_count}/#{page_jobs.size} failed"
writer.close
@logger.debug "cbz File created at #{zip_path}"
if fail_count == 0
@queue.set_status JobStatus::Completed, job
else
@queue.set_status JobStatus::MissingPages, job
end
@downloading = false
end
end
private def download_page(job : PageJob)
@logger.debug "downloading #{job.url}"
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
@logger.error e
job.success = false
end
end
end
end