Remove @log from MangaDex::Job and save the @success_count and

@fail_count instead

src/mangadex/downloader.cr

@@ -2,7 +2,7 @@ require "./api"
 require "sqlite3"
 
 module MangaDex
-	struct PageJob
+	class PageJob
 		property success = false
 		property url : String
 		property filename : String
@@ -24,16 +24,20 @@ module MangaDex
 		property title : String
 		property manga_title : String
 		property status : JobStatus
-		property log : String
+		property pages : Int32 = 0
+		property success_count : Int32 = 0
+		property fail_count : Int32 = 0
 		property time : Time
-		def load_query_result(res : DB::ResultSet)
+		def parse_query_result(res : DB::ResultSet)
 			begin
 				@id = res.read String
 				@manga_id = res.read String
 				@title = res.read String
 				@manga_title = res.read String
 				status = res.read Int32
-				@log = res.read String
+				@pages = res.read Int32
+				@success_count = res.read Int32
+				@fail_count = res.read Int32
 				time = res.read Int64
 				@status = JobStatus.new status
 				@time = Time.unix_ms time
@@ -45,18 +49,21 @@ module MangaDex
 		end
 		def self.from_query_result(res : DB::ResultSet)
 			job = Job.allocate
-			success = job.load_query_result res
+			success = job.parse_query_result res
 			return success ? job : nil
 		end
-		def initialize(@id, @manga_id, @title, @manga_title, @status, @log,
+		def initialize(@id, @manga_id, @title, @manga_title, @status, @time)
-					   @time)
 		end
 		def to_json(json)
 			json.object do
-				{% for name in ["id", "manga_id", "title", "manga_title",
+				{% for name in ["id", "manga_id", "title", "manga_title"] %}
-					"log"] %}
 					json.field {{name}}, @{{name.id}}
 				{% end %}
+				{% for name in ["pages", "success_count", "fail_count"] %}
+					json.field {{name}} do
+						json.number @{{name.id}}
+					end
+				{% end %}
 				json.field "status", @status.to_s
 				json.field "time" do
 					json.number @time.to_unix_ms
@@ -74,12 +81,17 @@ module MangaDex
 			end
 			DB.open "sqlite3://#{@path}" do |db|
 				begin
-					db.exec "create table if not exists queue" \
+					db.exec "create table if not exists queue " \
-						"(id text, manga_id text, title text, manga_title "\
+						"(id text, manga_id text, title text, manga_title " \
-						"text, status integer, log text, time integer)"
+						"text, status integer, pages integer, " \
-					db.exec "create unique index if not exists id_idx on queue (id)"
+						"success_count integer, fail_count integer, " \
-					db.exec "create index if not exists manga_id_idx on queue (manga_id)"
+						"time integer)"
-					db.exec "create index if not exists status_idx on queue (status)"
+					db.exec "create unique index if not exists id_idx " \
+						"on queue (id)"
+					db.exec "create index if not exists manga_id_idx " \
+						"on queue (manga_id)"
+					db.exec "create index if not exists status_idx " \
+						"on queue (status)"
 				rescue e
 					puts "Error when checking tables in DB: #{e}"
 					raise e
@@ -108,9 +120,10 @@ module MangaDex
 			DB.open "sqlite3://#{@path}" do |db|
 				jobs.each do |job|
 					db.exec "insert or ignore into queue values "\
-						"(?, ?, ?, ?, ?, ?, ?)",
+						"(?, ?, ?, ?, ?, ?, ?, ?, ?)",
 						job.id, job.manga_id, job.title, job.manga_title,
-						job.status.to_i, job.log, job.time.to_unix_ms
+						job.status.to_i, job.pages, job.success_count,
+						job.fail_count, job.time.to_unix_ms
 				end
 			end
 			self.count - start_count
@@ -123,7 +136,7 @@ module MangaDex
 		def get(job : Job)
 			DB.open "sqlite3://#{@path}" do |db|
 				db.query_one "select * from queue where id = (?)", id do |res|
-					job.load_query_result res
+					job.parse_query_result res
 				end
 			end
 		end
@@ -143,12 +156,6 @@ module MangaDex
 				return db.query_one "select count(*) from queue", as: Int32
 			end
 		end
-		def log(msg : String, job : Job)
-			DB.open "sqlite3://#{@path}" do |db|
-				db.exec "update queue set log = log || (?) || (?) where "\
-					"id = (?)", msg, "\n", job.id
-			end
-		end
 		def set_status(status : JobStatus, job : Job)
 			DB.open "sqlite3://#{@path}" do |db|
 				db.exec "update queue set status = (?) where id = (?)",
@@ -164,6 +171,24 @@ module MangaDex
 			end
 			return jobs
 		end
+		def add_success(job : Job)
+			DB.open "sqlite3://#{@path}" do |db|
+				db.exec "update queue set success_count = success_count + 1 " \
+					"where id = (?)", job.id
+			end
+		end
+		def add_fail(job : Job)
+			DB.open "sqlite3://#{@path}" do |db|
+				db.exec "update queue set fail_count = fail_count + 1 " \
+					"where id = (?)", job.id
+			end
+		end
+		def set_pages(pages : Int32, job : Job)
+			DB.open "sqlite3://#{@path}" do |db|
+				db.exec "update queue set pages = (?), success_count = 0, fail_count = 0 where id = (?)",
+					pages, job.id
+			end
+		end
 	end
 
 	class Downloader
@@ -179,6 +204,8 @@ module MangaDex
 						job = @queue.pop
 						next if job.nil?
 						download job
+					rescue e
+						puts e
 					end
 				end
 			end
@@ -194,14 +221,21 @@ module MangaDex
 		private def download(job : Job)
 			self.stop
 			@queue.set_status JobStatus::Downloading, job
+			begin
 				chapter = @api.get_chapter(job.id)
+			rescue e
+				puts e
+				@queue.set_status JobStatus::Error, job
+				self.resume
+				return
+			end
+			@queue.set_pages chapter.pages.size, job
 			lib_dir = @library_path
 			manga_dir = File.join lib_dir, chapter.manga.title
 			unless File.exists? manga_dir
 				Dir.mkdir_p manga_dir
 			end
 			zip_path = File.join manga_dir, "#{job.title}.cbz"
-			@queue.log "Downloading to #{zip_path}", job
 
 			# Find the number of digits needed to store the number of pages
 			len = Math.log10(chapter.pages.size).to_i + 1
@@ -216,7 +250,6 @@ module MangaDex
 					fn = "#{i.to_s.rjust len, '0'}#{ext}"
 					page_job = PageJob.new url, fn, writer, @retries
 					puts "Downloading #{url}"
-					@queue.log "Downloading #{url}", job
 					loop do
 						sleep @wait_seconds.seconds
 						download_page page_job
@@ -225,7 +258,6 @@ module MangaDex
 						page_job.tries_remaning -= 1
 						puts "Retrying... Remaining retries: "\
 							"#{page_job.tries_remaning}"
-						@queue.log "Retrying. Remaining retries: #{page_job.tries_remaning}", job
 					end
 
 					channel.send page_job
@@ -236,16 +268,17 @@ module MangaDex
 				page_jobs = [] of PageJob
 				chapter.pages.size.times do
 					page_job = channel.receive
-					log_str = "[#{page_job.success ? "success" : "failed"}] #{page_job.url}"
+					puts "[#{page_job.success ? "success" : "failed"}] #{page_job.url}"
-					puts log_str
-					@queue.log log_str, job
 					page_jobs << page_job
+					if page_job.success
+						@queue.add_success job
+					else
+						@queue.add_fail job
+					end
 				end
 				fail_count = page_jobs.select{|j| !j.success}.size
-				log_str = "Download completed. "\
+				puts "Download completed. "\
 					"#{fail_count}/#{page_jobs.size} failed"
-				puts log_str
-				@queue.log log_str, job
 				writer.close
 				puts "cbz File created at #{zip_path}"
 				if fail_count == 0
@@ -258,12 +291,16 @@ module MangaDex
 		end
 
 		private def download_page(job : PageJob)
+			puts "downloading #{job.url}"
 			headers = HTTP::Headers {
 				"User-agent" => "Mangadex.cr"
 			}
 			begin
 				HTTP::Client.get job.url, headers do |res|
-					return if !res.success?
+					unless res.success?
+						raise "Failed to download page #{job.url}. " \
+							"[#{res.status_code}] #{res.status_message}"
+					end
 					job.writer.add job.filename, res.body_io
 				end
 				job.success = true

src/routes/api.cr

@@ -113,7 +113,6 @@ class APIRouter < Router
 						chapter["full_title"].as_s,
 						chapter["manga_title"].as_s,
 						MangaDex::JobStatus::Pending,
-						"",
 						Time.utc
 					)
 				}