From 8c7ced87f1c3d13bcb8800a5ff0c16730b9a844b Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Thu, 12 Mar 2020 20:37:03 +0000 Subject: [PATCH 1/9] Add nested library support (WIP) --- src/library.cr | 49 +++++++++++++++++++++++++++++++-------------- src/routes/main.cr | 2 ++ src/views/index.ecr | 2 ++ src/views/title.ecr | 18 +++++++++++++++++ 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/library.cr b/src/library.cr index ca6ac8b..3e8a9f4 100644 --- a/src/library.cr +++ b/src/library.cr @@ -60,25 +60,40 @@ class Entry end class Title - JSON.mapping dir: String, entries: Array(Entry), title: String, - id: String, encoded_title: String, mtime: Time, logger: MLogger + JSON.mapping dir: String, titles: Array(Title), entries: Array(Entry), + title: String, id: String, encoded_title: String, mtime: Time, + logger: MLogger def initialize(dir : String, storage, @logger : MLogger) @dir = dir @id = storage.get_id @dir, true @title = File.basename dir @encoded_title = URI.encode @title - @entries = (Dir.entries dir) - .select { |path| [".zip", ".cbz"].includes? File.extname path } - .map { |path| File.join dir, path } - .select { |path| valid_zip path } - .map { |path| - Entry.new path, @title, @id, storage - } - .select { |e| e.pages > 0 } - .sort { |a, b| a.title <=> b.title } + @titles = [] of Title + @entries = [] of Entry + + Dir.entries(dir).each do |fn| + next if fn.starts_with? "." + path = File.join dir, fn + if File.directory? path + title = Title.new path, storage, @logger + next if title.entries.size == 0 && title.titles.size == 0 + @titles << title + next + end + if [".zip", ".cbz"].includes? File.extname path + next if !valid_zip path + entry = Entry.new path, @title, @id, storage + @entries << entry if entry.pages > 0 + end + end + + @titles.sort! { |a,b| a.title <=> b.title } + @entries.sort! { |a,b| a.title <=> b.title } + mtimes = [File.info(dir).modification_time] mtimes += @entries.map{|e| e.mtime} + mtimes += @titles.map{|e| e.mtime} @mtime = mtimes.max end # When downloading from MangaDex, the zip/cbz file would not be valid @@ -192,7 +207,9 @@ class Library end end def get_title(tid) - @titles.find { |t| t.id == tid } + # top level + title = @titles.find { |t| t.id == tid } + return title if !title.nil? end def scan unless Dir.exists? @dir @@ -201,9 +218,11 @@ class Library Dir.mkdir_p @dir end @titles = (Dir.entries @dir) - .select { |path| File.directory? File.join @dir, path } - .map { |path| Title.new File.join(@dir, path), @storage, @logger } - .select { |title| !title.entries.empty? } + .select { |fn| !fn.starts_with? "." } + .map { |fn| File.join @dir, fn } + .select { |path| File.directory? path } + .map { |path| Title.new path, @storage, @logger } + .select { |title| !(title.entries.empty? && title.titles.empty?) } .sort { |a, b| a.title <=> b.title } @logger.debug "Scan completed" end diff --git a/src/routes/main.cr b/src/routes/main.cr index eeab285..ac4007c 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -46,6 +46,8 @@ class MainRouter < Router username = get_username env percentage = title.entries.map { |e| title.load_percetage username, e.title } + titles_percentage = title.titles.map { |t| + title.load_percetage username, t.title } layout "title" rescue e @context.error e diff --git a/src/views/index.ecr b/src/views/index.ecr index 1aee880..15cb974 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -25,9 +25,11 @@
+ <%- if t.entries.size > 0 -%>
+ <%- end -%>
<%= (percentage[i] * 100).round(1) %>%

<%= t.title %>

diff --git a/src/views/title.ecr b/src/views/title.ecr index 86386c7..724abba 100644 --- a/src/views/title.ecr +++ b/src/views/title.ecr @@ -23,6 +23,24 @@
+ <%- title.titles.each_with_index do |t, i| -%> +
+ +
+ <%- if t.entries.size > 0 -%> +
+ +
+ <%- end -%> +
+
<%= (titles_percentage[i] * 100).round(1) %>%
+

<%= t.title %>

+

<%= t.entries.size %> entries

+
+
+ +
+ <%- end -%> <%- title.entries.each_with_index do |e, i| -%>
From 82fb45b242650bd05f660ba3d7cc7a5c78f3a5dc Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 14 Mar 2020 23:58:49 +0000 Subject: [PATCH 2/9] Use json builder in `src/library.cr` instead of json mapping --- src/library.cr | 56 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/src/library.cr b/src/library.cr index 3e8a9f4..00a9a0e 100644 --- a/src/library.cr +++ b/src/library.cr @@ -14,10 +14,10 @@ struct Image end class Entry - JSON.mapping zip_path: String, book_title: String, title: String, - size: String, pages: Int32, cover_url: String, id: String, - title_id: String, encoded_path: String, encoded_title: String, - mtime: Time + property zip_path : String, book_title : String, title : String, + size : String, pages : Int32, cover_url : String, id : String, + title_id : String, encoded_path : String, encoded_title : String, + mtime : Time def initialize(path, @book_title, @title_id, storage) @zip_path = path @@ -37,6 +37,19 @@ class Entry @cover_url = "/api/page/#{@title_id}/#{@id}/1" @mtime = File.info(@zip_path).modification_time end + + def to_json(json : JSON::Builder) + json.object do + {% for str in ["zip_path", "book_title", "title", "size", + "cover_url", "id", "title_id", "encoded_path", + "encoded_title"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "pages" {json.number @pages} + json.field "mtime" {json.number @mtime.to_unix} + end + end + def read_page(page_num) Zip::File.open @zip_path do |file| page = file.entries @@ -60,9 +73,10 @@ class Entry end class Title - JSON.mapping dir: String, titles: Array(Title), entries: Array(Entry), - title: String, id: String, encoded_title: String, mtime: Time, - logger: MLogger + property dir : String, titles : Array(Title), entries : Array(Entry), + title : String, id : String, encoded_title : String, mtime : Time, + logger : MLogger + def initialize(dir : String, storage, @logger : MLogger) @dir = dir @@ -96,6 +110,22 @@ class Title mtimes += @titles.map{|e| e.mtime} @mtime = mtimes.max end + + def to_json(json : JSON::Builder) + json.object do + {% for str in ["dir", "title", "id", "encoded_title"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "mtime" {json.number @mtime.to_unix} + json.field "titles" do + json.raw @titles.to_json + end + json.field "entries" do + json.raw @entries.to_json + end + end + end + # When downloading from MangaDex, the zip/cbz file would not be valid # before the download is completed. If we scan the zip file, # Entry.new would throw, so we use this method to check before @@ -187,8 +217,8 @@ class TitleInfo end class Library - JSON.mapping dir: String, titles: Array(Title), scan_interval: Int32, - logger: MLogger, storage: Storage + property dir : String, titles : Array(Title), scan_interval : Int32, + logger : MLogger, storage : Storage def initialize(@dir, @scan_interval, @logger, @storage) # explicitly initialize @titles to bypass the compiler check. it will @@ -206,6 +236,14 @@ class Library end end end + def to_json(json : JSON::Builder) + json.object do + json.field "dir", @dir + json.field "titles" do + json.raw @titles.to_json + end + end + end def get_title(tid) # top level title = @titles.find { |t| t.id == tid } From e68678f2fb75fac18157d53a601a5fec160cd180 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 14 Mar 2020 23:59:46 +0000 Subject: [PATCH 3/9] Remove unnecessary `JSON::Field` calls --- src/library.cr | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/library.cr b/src/library.cr index 00a9a0e..e714a51 100644 --- a/src/library.cr +++ b/src/library.cr @@ -192,10 +192,7 @@ class TitleInfo # { user1: { entry1: 10, entry2: 0 } } include JSON::Serializable - @[JSON::Field(key: "comment")] property comment = "Generated by Mango. DO NOT EDIT!" - - @[JSON::Field(key: "progress")] property progress : Hash(String, Hash(String, Int32)) def initialize(title_dir) From 7e22cc5f57b35e2eea7442a2228fa2bdc4d57720 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 01:03:49 +0000 Subject: [PATCH 4/9] Fix bug in API `/api/book/:tid` that causes 500 --- src/routes/api.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.cr b/src/routes/api.cr index c8ec60d..cefb1b6 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -26,7 +26,7 @@ class APIRouter < Router end end - get "/api/book/:title" do |env| + get "/api/book/:tid" do |env| begin tid = env.params.url["tid"] title = @context.library.get_title tid From 6407cea7bf69422db3e7725e63569e7545755891 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 01:05:37 +0000 Subject: [PATCH 5/9] Refactor `src/library.cr` to reduce memory usage - Store the `Title` objects in `Library@title_hash` - The `Title` objects only stores IDs to other titles --- src/library.cr | 57 +++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/library.cr b/src/library.cr index e714a51..4cb3390 100644 --- a/src/library.cr +++ b/src/library.cr @@ -73,26 +73,25 @@ class Entry end class Title - property dir : String, titles : Array(Title), entries : Array(Entry), - title : String, id : String, encoded_title : String, mtime : Time, - logger : MLogger + property dir : String, title_ids : Array(String), entries : Array(Entry), + title : String, id : String, encoded_title : String, mtime : Time - - def initialize(dir : String, storage, @logger : MLogger) + def initialize(dir : String, storage, @logger : MLogger, @library : Library) @dir = dir @id = storage.get_id @dir, true @title = File.basename dir @encoded_title = URI.encode @title - @titles = [] of Title + @title_ids = [] of String @entries = [] of Entry Dir.entries(dir).each do |fn| next if fn.starts_with? "." path = File.join dir, fn if File.directory? path - title = Title.new path, storage, @logger + title = Title.new path, storage, @logger, library next if title.entries.size == 0 && title.titles.size == 0 - @titles << title + @library.title_hash[title.id] = title + @title_ids << title.id next end if [".zip", ".cbz"].includes? File.extname path @@ -102,12 +101,14 @@ class Title end end - @titles.sort! { |a,b| a.title <=> b.title } + @title_ids.sort! { |a,b| + @library.title_hash[a].title <=> @library.title_hash[b].title + } @entries.sort! { |a,b| a.title <=> b.title } mtimes = [File.info(dir).modification_time] + mtimes += @title_ids.map{|e| @library.title_hash[e].mtime} mtimes += @entries.map{|e| e.mtime} - mtimes += @titles.map{|e| e.mtime} @mtime = mtimes.max end @@ -118,7 +119,7 @@ class Title {% end %} json.field "mtime" {json.number @mtime.to_unix} json.field "titles" do - json.raw @titles.to_json + json.raw self.titles.to_json end json.field "entries" do json.raw @entries.to_json @@ -126,6 +127,10 @@ class Title end end + def titles + @title_ids.map {|tid| @library.get_title! tid} + end + # When downloading from MangaDex, the zip/cbz file would not be valid # before the download is completed. If we scan the zip file, # Entry.new would throw, so we use this method to check before @@ -214,13 +219,14 @@ class TitleInfo end class Library - property dir : String, titles : Array(Title), scan_interval : Int32, - logger : MLogger, storage : Storage + property dir : String, title_ids : Array(String), scan_interval : Int32, + logger : MLogger, storage : Storage, title_hash : Hash(String, Title) def initialize(@dir, @scan_interval, @logger, @storage) # explicitly initialize @titles to bypass the compiler check. it will # be filled with actual Titles in the `scan` call below - @titles = [] of Title + @title_ids = [] of String + @title_hash = {} of String => Title return scan if @scan_interval < 1 spawn do @@ -228,23 +234,27 @@ class Library start = Time.local scan ms = (Time.local - start).total_milliseconds - @logger.info "Scanned #{@titles.size} titles in #{ms}ms" + @logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" sleep @scan_interval * 60 end end end + def titles + @title_ids.map {|tid| self.get_title!(tid) } + end def to_json(json : JSON::Builder) json.object do json.field "dir", @dir json.field "titles" do - json.raw @titles.to_json + json.raw self.titles.to_json end end end def get_title(tid) - # top level - title = @titles.find { |t| t.id == tid } - return title if !title.nil? + @title_hash[tid]? + end + def get_title!(tid) + @title_hash[tid] end def scan unless Dir.exists? @dir @@ -252,13 +262,18 @@ class Library "Attempting to create it" Dir.mkdir_p @dir end - @titles = (Dir.entries @dir) + @title_ids.clear + (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } .select { |path| File.directory? path } - .map { |path| Title.new path, @storage, @logger } + .map { |path| Title.new path, @storage, @logger, self } .select { |title| !(title.entries.empty? && title.titles.empty?) } .sort { |a, b| a.title <=> b.title } + .each do |title| + @title_hash[title.id] = title + @title_ids << title.id + end @logger.debug "Scan completed" end end From 3e4226695549fb3ccf51b49e12479b4ae7f802e9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 01:31:14 +0000 Subject: [PATCH 6/9] List the parent title objects in `Title.to_json` --- src/library.cr | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/library.cr b/src/library.cr index 4cb3390..a5e22ed 100644 --- a/src/library.cr +++ b/src/library.cr @@ -73,10 +73,12 @@ class Entry end class Title - property dir : String, title_ids : Array(String), entries : Array(Entry), - title : String, id : String, encoded_title : String, mtime : Time + property dir : String, parent_id : String, title_ids : Array(String), + entries : Array(Entry), title : String, id : String, + encoded_title : String, mtime : Time - def initialize(dir : String, storage, @logger : MLogger, @library : Library) + def initialize(dir : String, @parent_id, storage, + @logger : MLogger, @library : Library) @dir = dir @id = storage.get_id @dir, true @title = File.basename dir @@ -88,7 +90,7 @@ class Title next if fn.starts_with? "." path = File.join dir, fn if File.directory? path - title = Title.new path, storage, @logger, library + title = Title.new path, @id, storage, @logger, library next if title.entries.size == 0 && title.titles.size == 0 @library.title_hash[title.id] = title @title_ids << title.id @@ -124,6 +126,16 @@ class Title json.field "entries" do json.raw @entries.to_json end + json.field "parents" do + json.array do + self.parents.each do |title| + json.object do + json.field "title", title.title + json.field "id", title.id + end + end + end + end end end @@ -131,6 +143,17 @@ class Title @title_ids.map {|tid| @library.get_title! tid} end + def parents + ary = [] of Title + tid = @parent_id + while !tid.empty? + title = @library.get_title! tid + ary << title + tid = title.parent_id + end + ary + end + # When downloading from MangaDex, the zip/cbz file would not be valid # before the download is completed. If we scan the zip file, # Entry.new would throw, so we use this method to check before @@ -267,7 +290,7 @@ class Library .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } .select { |path| File.directory? path } - .map { |path| Title.new path, @storage, @logger, self } + .map { |path| Title.new path, "", @storage, @logger, self } .select { |title| !(title.entries.empty? && title.titles.empty?) } .sort { |a, b| a.title <=> b.title } .each do |title| From 5a500364fc108845ffef544069e1905222d7b04b Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 01:45:10 +0000 Subject: [PATCH 7/9] Show a list of parent directories on the title page --- src/views/title.ecr | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/views/title.ecr b/src/views/title.ecr index 724abba..49281ec 100644 --- a/src/views/title.ecr +++ b/src/views/title.ecr @@ -1,4 +1,11 @@

<%= title.title %>

+

<%= title.entries.size %> entries found

From e65d701e0a10cdf4b465b8bd998091a042922310 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 02:08:20 +0000 Subject: [PATCH 8/9] Show sum of entries and titles count when displaying the number of entries --- src/library.cr | 4 ++++ src/views/index.ecr | 2 +- src/views/title.ecr | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/library.cr b/src/library.cr index a5e22ed..d1d4fdd 100644 --- a/src/library.cr +++ b/src/library.cr @@ -154,6 +154,10 @@ class Title ary end + def size + @entries.size + @title_ids.size + end + # When downloading from MangaDex, the zip/cbz file would not be valid # before the download is completed. If we scan the zip file, # Entry.new would throw, so we use this method to check before diff --git a/src/views/index.ecr b/src/views/index.ecr index 15cb974..c90aaf9 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -33,7 +33,7 @@
<%= (percentage[i] * 100).round(1) %>%

<%= t.title %>

-

<%= t.entries.size %> entries

+

<%= t.size %> entries

diff --git a/src/views/title.ecr b/src/views/title.ecr index 49281ec..d25df90 100644 --- a/src/views/title.ecr +++ b/src/views/title.ecr @@ -6,7 +6,7 @@ <%- end -%>
  • <%= title.title %>
    • -

    <%= title.entries.size %> entries found

    +

    <%= title.size %> entries found

    @@ -42,7 +42,7 @@
    <%= (titles_percentage[i] * 100).round(1) %>%

    <%= t.title %>

    -

    <%= t.entries.size %> entries

    +

    <%= t.size %> entries

    From 9fe32b5011946f90dda5eabf7a6ee2db7de8ddb3 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 15 Mar 2020 02:10:22 +0000 Subject: [PATCH 9/9] When a title contains no entry as immediate child, display mango logo and remove progress badge --- src/views/index.ecr | 8 ++++++-- src/views/title.ecr | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/views/index.ecr b/src/views/index.ecr index c90aaf9..bc3c591 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -25,13 +25,17 @@
    - <%- if t.entries.size > 0 -%>
    + <%- if t.entries.size > 0 -%> -
    + <%- else -%> + <%- end -%> +
    + <%- if t.entries.size > 0 -%>
    <%= (percentage[i] * 100).round(1) %>%
    + <%- end -%>

    <%= t.title %>

    <%= t.size %> entries

    diff --git a/src/views/title.ecr b/src/views/title.ecr index d25df90..31d72eb 100644 --- a/src/views/title.ecr +++ b/src/views/title.ecr @@ -34,13 +34,17 @@
    - <%- if t.entries.size > 0 -%>
    + <%- if t.entries.size > 0 -%> -
    + <%- else -%> + <%- end -%> +
    + <%- if t.entries.size > 0 -%>
    <%= (titles_percentage[i] * 100).round(1) %>%
    + <%- end -%>

    <%= t.title %>

    <%= t.size %> entries