diff --git a/shard.lock b/shard.lock index c84ac06..86fc39f 100644 --- a/shard.lock +++ b/shard.lock @@ -4,6 +4,10 @@ shards: github: crystal-ameba/ameba version: 0.12.0 + archive: + github: hkalexling/archive.cr + commit: 64223b64d79afafcec67a127b92f21f6625ce5ce + baked_file_system: github: schovi/baked_file_system version: 0.9.8 diff --git a/shard.yml b/shard.yml index 15e8fd3..5eb5561 100644 --- a/shard.yml +++ b/shard.yml @@ -19,6 +19,8 @@ dependencies: github: crystal-lang/crystal-sqlite3 baked_file_system: github: schovi/baked_file_system + archive: + github: hkalexling/archive.cr development_dependencies: ameba: diff --git a/src/archive.cr b/src/archive.cr new file mode 100644 index 0000000..afdf143 --- /dev/null +++ b/src/archive.cr @@ -0,0 +1,53 @@ +require "zip" +require "archive" + +# A unified class to handle all supported archive formats. It uses the ::Zip +# module in crystal standard library if the target file is a zip archive. +# Otherwise it uses `archive.cr`. +class ArchiveFile + def initialize(@filename : String) + if [".cbz", ".zip"].includes? File.extname filename + @archive_file = Zip::File.new filename + else + @archive_file = Archive::File.new filename + end + end + + def self.open(filename : String, &) + s = self.new filename + yield s + s.close + end + + def close + if @archive_file.is_a? Zip::File + @archive_file.as(Zip::File).close + end + end + + # Lists all file entries + def entries + ary = [] of Zip::File::Entry | Archive::Entry + @archive_file.entries.map do |e| + if (e.is_a? Zip::File::Entry && e.file?) || + (e.is_a? Archive::Entry && e.info.file?) + ary.push e + end + end + ary + end + + def read_entry(e : Zip::File::Entry | Archive::Entry) : Bytes? + if e.is_a? Zip::File::Entry + data = nil + e.open do |io| + slice = Bytes.new e.uncompressed_size + bytes_read = io.read_fully? slice + data = slice if bytes_read + end + data + else + e.read + end + end +end diff --git a/src/library.cr b/src/library.cr index 0af92ac..2520f6c 100644 --- a/src/library.cr +++ b/src/library.cr @@ -1,8 +1,8 @@ -require "zip" require "mime" require "json" require "uri" require "./util" +require "./archive" struct Image property data : Bytes @@ -25,7 +25,7 @@ class Entry @title = File.basename path, File.extname path @encoded_title = URI.encode @title @size = (File.size path).humanize_bytes - file = Zip::File.new path + file = ArchiveFile.new path @pages = file.entries.count do |e| ["image/jpeg", "image/png"].includes? \ MIME.from_filename? e.filename @@ -68,7 +68,8 @@ class Entry end def read_page(page_num) - Zip::File.open @zip_path do |file| + img = nil + ArchiveFile.open @zip_path do |file| page = file.entries .select { |e| ["image/jpeg", "image/png"].includes? \ @@ -78,16 +79,13 @@ class Entry compare_alphanumerically a.filename, b.filename } .[page_num - 1] - page.open do |io| - slice = Bytes.new page.uncompressed_size - bytes_read = io.read_fully? slice - unless bytes_read - return nil - end - return Image.new slice, MIME.from_filename(page.filename), - page.filename, bytes_read + data = file.read_entry page + if data + img = Image.new data, MIME.from_filename(page.filename), page.filename, + data.size end end + img end end @@ -115,12 +113,12 @@ class Title @title_ids << title.id next end - if [".zip", ".cbz"].includes? File.extname path - zip_exception = validate_zip path - unless zip_exception.nil? - Logger.warn "File #{path} is corrupted or is not a valid zip " \ - "archive. Ignoring it." - Logger.debug "Zip error: #{zip_exception}" + if [".zip", ".cbz", ".rar", ".cbr"].includes? File.extname path + archive_exception = validate_archive path + unless archive_exception.nil? + Logger.warn "File #{path} is corrupted or is not a valid archive. " \ + "Ignoring it." + Logger.debug "Archive error: #{archive_exception}" next end entry = Entry.new path, self, @id, storage diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr index 53c6776..6a62e59 100644 --- a/src/mangadex/downloader.cr +++ b/src/mangadex/downloader.cr @@ -371,7 +371,7 @@ module MangaDex writer.close Logger.debug "cbz File created at #{zip_path}" - zip_exception = validate_zip zip_path + zip_exception = validate_archive zip_path if !zip_exception.nil? @queue.add_message "The downloaded archive is corrupted. " \ "Error: #{zip_exception}", job diff --git a/src/util.cr b/src/util.cr index bb6cd0f..cfdd2ea 100644 --- a/src/util.cr +++ b/src/util.cr @@ -85,12 +85,8 @@ def compare_alphanumerically(a : String, b : String) compare_alphanumerically split_by_alphanumeric(a), split_by_alphanumeric(b) end -# When downloading from MangaDex, the zip/cbz file would not be valid -# before the download is completed. If we scan the zip file, -# Entry.new would throw, so we use this method to check before -# constructing Entry -def validate_zip(path : String) : Exception? - file = Zip::File.new path +def validate_archive(path : String) : Exception? + file = ArchiveFile.new path file.close return rescue e