Add and use MLogger in MangaDex::Queue and MangaDex::Downloader

This commit is contained in:
Alex Ling 2020-03-03 02:05:15 +00:00
parent 589483cd75
commit f140ffa4b2
2 changed files with 23 additions and 17 deletions

View File

@ -84,10 +84,10 @@ module MangaDex
class Queue class Queue
property downloader : Downloader? property downloader : Downloader?
def initialize(@path : String) def initialize(@path : String, @logger : MLogger)
dir = File.dirname path dir = File.dirname path
unless Dir.exists? dir unless Dir.exists? dir
puts "The queue DB directory #{dir} does not exist. " \ @logger.info "The queue DB directory #{dir} does not exist. " \
"Attepmting to create it" "Attepmting to create it"
Dir.mkdir_p dir Dir.mkdir_p dir
end end
@ -105,7 +105,7 @@ module MangaDex
db.exec "create index if not exists status_idx " \ db.exec "create index if not exists status_idx " \
"on queue (status)" "on queue (status)"
rescue e rescue e
puts "Error when checking tables in DB: #{e}" @logger.error "Error when checking tables in DB: #{e}"
raise e raise e
end end
end end
@ -264,7 +264,8 @@ module MangaDex
property stopped = false property stopped = false
def initialize(@queue : Queue, @api : API, @library_path : String, def initialize(@queue : Queue, @api : API, @library_path : String,
@wait_seconds : Int32, @retries : Int32) @wait_seconds : Int32, @retries : Int32,
@logger : MLogger)
@queue.downloader = self @queue.downloader = self
spawn do spawn do
@ -276,7 +277,7 @@ module MangaDex
next if job.nil? next if job.nil?
download job download job
rescue e rescue e
puts e @logger.error e
end end
end end
end end
@ -288,7 +289,7 @@ module MangaDex
begin begin
chapter = @api.get_chapter(job.id) chapter = @api.get_chapter(job.id)
rescue e rescue e
puts e @logger.error e
@queue.set_status JobStatus::Error, job @queue.set_status JobStatus::Error, job
unless e.message.nil? unless e.message.nil?
@queue.add_message e.message.not_nil!, job @queue.add_message e.message.not_nil!, job
@ -316,14 +317,15 @@ module MangaDex
ext = File.extname fn ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}" fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries page_job = PageJob.new url, fn, writer, @retries
puts "Downloading #{url}" @logger.debug "Downloading #{url}"
loop do loop do
sleep @wait_seconds.seconds sleep @wait_seconds.seconds
download_page page_job download_page page_job
break if page_job.success || break if page_job.success ||
page_job.tries_remaning <= 0 page_job.tries_remaning <= 0
page_job.tries_remaning -= 1 page_job.tries_remaning -= 1
puts "Retrying... Remaining retries: "\ @logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}" "#{page_job.tries_remaning}"
end end
@ -335,22 +337,23 @@ module MangaDex
page_jobs = [] of PageJob page_jobs = [] of PageJob
chapter.pages.size.times do chapter.pages.size.times do
page_job = channel.receive page_job = channel.receive
puts "[#{page_job.success ? "success" : "failed"}] " \ @logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}" "#{page_job.url}"
page_jobs << page_job page_jobs << page_job
if page_job.success if page_job.success
@queue.add_success job @queue.add_success job
else else
@queue.add_fail job @queue.add_fail job
@queue.add_message \ msg = "Failed to download page #{page_job.url}"
"Failed to download page #{page_job.url}", job @queue.add_message msg, job
@logger.error msg
end end
end end
fail_count = page_jobs.select{|j| !j.success}.size fail_count = page_jobs.select{|j| !j.success}.size
puts "Download completed. "\ @logger.debug "Download completed. "\
"#{fail_count}/#{page_jobs.size} failed" "#{fail_count}/#{page_jobs.size} failed"
writer.close writer.close
puts "cbz File created at #{zip_path}" @logger.debug "cbz File created at #{zip_path}"
if fail_count == 0 if fail_count == 0
@queue.set_status JobStatus::Completed, job @queue.set_status JobStatus::Completed, job
else else
@ -361,7 +364,7 @@ module MangaDex
end end
private def download_page(job : PageJob) private def download_page(job : PageJob)
puts "downloading #{job.url}" @logger.debug "downloading #{job.url}"
headers = HTTP::Headers { headers = HTTP::Headers {
"User-agent" => "Mangadex.cr" "User-agent" => "Mangadex.cr"
} }
@ -375,7 +378,7 @@ module MangaDex
end end
job.success = true job.success = true
rescue e rescue e
puts e @logger.error e
job.success = false job.success = false
end end
end end

View File

@ -28,9 +28,12 @@ config = Config.load config_path
logger = MLogger.new config logger = MLogger.new config
storage = Storage.new config.db_path, logger storage = Storage.new config.db_path, logger
library = Library.new config.library_path, config.scan_interval, logger, storage library = Library.new config.library_path, config.scan_interval, logger, storage
queue = MangaDex::Queue.new config.mangadex["download_queue_db_path"].to_s queue = MangaDex::Queue.new config.mangadex["download_queue_db_path"].to_s,
logger
api = MangaDex::API.new config.mangadex["api_url"].to_s api = MangaDex::API.new config.mangadex["api_url"].to_s
downloader = MangaDex::Downloader.new queue, api, config.library_path, config.mangadex["download_wait_seconds"].to_i, config.mangadex["download_retries"].to_i downloader = MangaDex::Downloader.new queue, api, config.library_path,
config.mangadex["download_wait_seconds"].to_i,
config.mangadex["download_retries"].to_i, logger
context = Context.new config, logger, library, storage, queue context = Context.new config, logger, library, storage, queue