diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr index a507efd..ef000ec 100644 --- a/src/mangadex/downloader.cr +++ b/src/mangadex/downloader.cr @@ -2,7 +2,7 @@ require "./api" require "sqlite3" module MangaDex - struct PageJob + class PageJob property success = false property url : String property filename : String @@ -24,16 +24,20 @@ module MangaDex property title : String property manga_title : String property status : JobStatus - property log : String + property pages : Int32 = 0 + property success_count : Int32 = 0 + property fail_count : Int32 = 0 property time : Time - def load_query_result(res : DB::ResultSet) + def parse_query_result(res : DB::ResultSet) begin @id = res.read String @manga_id = res.read String @title = res.read String @manga_title = res.read String status = res.read Int32 - @log = res.read String + @pages = res.read Int32 + @success_count = res.read Int32 + @fail_count = res.read Int32 time = res.read Int64 @status = JobStatus.new status @time = Time.unix_ms time @@ -45,18 +49,21 @@ module MangaDex end def self.from_query_result(res : DB::ResultSet) job = Job.allocate - success = job.load_query_result res + success = job.parse_query_result res return success ? job : nil end - def initialize(@id, @manga_id, @title, @manga_title, @status, @log, - @time) + def initialize(@id, @manga_id, @title, @manga_title, @status, @time) end def to_json(json) json.object do - {% for name in ["id", "manga_id", "title", "manga_title", - "log"] %} + {% for name in ["id", "manga_id", "title", "manga_title"] %} json.field {{name}}, @{{name.id}} {% end %} + {% for name in ["pages", "success_count", "fail_count"] %} + json.field {{name}} do + json.number @{{name.id}} + end + {% end %} json.field "status", @status.to_s json.field "time" do json.number @time.to_unix_ms @@ -74,12 +81,17 @@ module MangaDex end DB.open "sqlite3://#{@path}" do |db| begin - db.exec "create table if not exists queue" \ - "(id text, manga_id text, title text, manga_title "\ - "text, status integer, log text, time integer)" - db.exec "create unique index if not exists id_idx on queue (id)" - db.exec "create index if not exists manga_id_idx on queue (manga_id)" - db.exec "create index if not exists status_idx on queue (status)" + db.exec "create table if not exists queue " \ + "(id text, manga_id text, title text, manga_title " \ + "text, status integer, pages integer, " \ + "success_count integer, fail_count integer, " \ + "time integer)" + db.exec "create unique index if not exists id_idx " \ + "on queue (id)" + db.exec "create index if not exists manga_id_idx " \ + "on queue (manga_id)" + db.exec "create index if not exists status_idx " \ + "on queue (status)" rescue e puts "Error when checking tables in DB: #{e}" raise e @@ -108,9 +120,10 @@ module MangaDex DB.open "sqlite3://#{@path}" do |db| jobs.each do |job| db.exec "insert or ignore into queue values "\ - "(?, ?, ?, ?, ?, ?, ?)", + "(?, ?, ?, ?, ?, ?, ?, ?, ?)", job.id, job.manga_id, job.title, job.manga_title, - job.status.to_i, job.log, job.time.to_unix_ms + job.status.to_i, job.pages, job.success_count, + job.fail_count, job.time.to_unix_ms end end self.count - start_count @@ -123,7 +136,7 @@ module MangaDex def get(job : Job) DB.open "sqlite3://#{@path}" do |db| db.query_one "select * from queue where id = (?)", id do |res| - job.load_query_result res + job.parse_query_result res end end end @@ -143,12 +156,6 @@ module MangaDex return db.query_one "select count(*) from queue", as: Int32 end end - def log(msg : String, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set log = log || (?) || (?) where "\ - "id = (?)", msg, "\n", job.id - end - end def set_status(status : JobStatus, job : Job) DB.open "sqlite3://#{@path}" do |db| db.exec "update queue set status = (?) where id = (?)", @@ -164,6 +171,24 @@ module MangaDex end return jobs end + def add_success(job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set success_count = success_count + 1 " \ + "where id = (?)", job.id + end + end + def add_fail(job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set fail_count = fail_count + 1 " \ + "where id = (?)", job.id + end + end + def set_pages(pages : Int32, job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set pages = (?), success_count = 0, fail_count = 0 where id = (?)", + pages, job.id + end + end end class Downloader @@ -179,6 +204,8 @@ module MangaDex job = @queue.pop next if job.nil? download job + rescue e + puts e end end end @@ -194,14 +221,21 @@ module MangaDex private def download(job : Job) self.stop @queue.set_status JobStatus::Downloading, job - chapter = @api.get_chapter(job.id) + begin + chapter = @api.get_chapter(job.id) + rescue e + puts e + @queue.set_status JobStatus::Error, job + self.resume + return + end + @queue.set_pages chapter.pages.size, job lib_dir = @library_path manga_dir = File.join lib_dir, chapter.manga.title unless File.exists? manga_dir Dir.mkdir_p manga_dir end zip_path = File.join manga_dir, "#{job.title}.cbz" - @queue.log "Downloading to #{zip_path}", job # Find the number of digits needed to store the number of pages len = Math.log10(chapter.pages.size).to_i + 1 @@ -216,7 +250,6 @@ module MangaDex fn = "#{i.to_s.rjust len, '0'}#{ext}" page_job = PageJob.new url, fn, writer, @retries puts "Downloading #{url}" - @queue.log "Downloading #{url}", job loop do sleep @wait_seconds.seconds download_page page_job @@ -225,7 +258,6 @@ module MangaDex page_job.tries_remaning -= 1 puts "Retrying... Remaining retries: "\ "#{page_job.tries_remaning}" - @queue.log "Retrying. Remaining retries: #{page_job.tries_remaning}", job end channel.send page_job @@ -236,16 +268,17 @@ module MangaDex page_jobs = [] of PageJob chapter.pages.size.times do page_job = channel.receive - log_str = "[#{page_job.success ? "success" : "failed"}] #{page_job.url}" - puts log_str - @queue.log log_str, job + puts "[#{page_job.success ? "success" : "failed"}] #{page_job.url}" page_jobs << page_job + if page_job.success + @queue.add_success job + else + @queue.add_fail job + end end fail_count = page_jobs.select{|j| !j.success}.size - log_str = "Download completed. "\ + puts "Download completed. "\ "#{fail_count}/#{page_jobs.size} failed" - puts log_str - @queue.log log_str, job writer.close puts "cbz File created at #{zip_path}" if fail_count == 0 @@ -258,12 +291,16 @@ module MangaDex end private def download_page(job : PageJob) + puts "downloading #{job.url}" headers = HTTP::Headers { "User-agent" => "Mangadex.cr" } begin HTTP::Client.get job.url, headers do |res| - return if !res.success? + unless res.success? + raise "Failed to download page #{job.url}. " \ + "[#{res.status_code}] #{res.status_message}" + end job.writer.add job.filename, res.body_io end job.success = true diff --git a/src/routes/api.cr b/src/routes/api.cr index 44b73d5..8c06dcb 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -113,7 +113,6 @@ class APIRouter < Router chapter["full_title"].as_s, chapter["manga_title"].as_s, MangaDex::JobStatus::Pending, - "", Time.utc ) }