Refactor src/library.cr to reduce memory usage

- Store the `Title` objects in `Library@title_hash`
- The `Title` objects only stores IDs to other titles
This commit is contained in:
Alex Ling 2020-03-15 01:05:37 +00:00
parent 7e22cc5f57
commit 6407cea7bf

View File

@ -73,26 +73,25 @@ class Entry
end end
class Title class Title
property dir : String, titles : Array(Title), entries : Array(Entry), property dir : String, title_ids : Array(String), entries : Array(Entry),
title : String, id : String, encoded_title : String, mtime : Time, title : String, id : String, encoded_title : String, mtime : Time
logger : MLogger
def initialize(dir : String, storage, @logger : MLogger, @library : Library)
def initialize(dir : String, storage, @logger : MLogger)
@dir = dir @dir = dir
@id = storage.get_id @dir, true @id = storage.get_id @dir, true
@title = File.basename dir @title = File.basename dir
@encoded_title = URI.encode @title @encoded_title = URI.encode @title
@titles = [] of Title @title_ids = [] of String
@entries = [] of Entry @entries = [] of Entry
Dir.entries(dir).each do |fn| Dir.entries(dir).each do |fn|
next if fn.starts_with? "." next if fn.starts_with? "."
path = File.join dir, fn path = File.join dir, fn
if File.directory? path if File.directory? path
title = Title.new path, storage, @logger title = Title.new path, storage, @logger, library
next if title.entries.size == 0 && title.titles.size == 0 next if title.entries.size == 0 && title.titles.size == 0
@titles << title @library.title_hash[title.id] = title
@title_ids << title.id
next next
end end
if [".zip", ".cbz"].includes? File.extname path if [".zip", ".cbz"].includes? File.extname path
@ -102,12 +101,14 @@ class Title
end end
end end
@titles.sort! { |a,b| a.title <=> b.title } @title_ids.sort! { |a,b|
@library.title_hash[a].title <=> @library.title_hash[b].title
}
@entries.sort! { |a,b| a.title <=> b.title } @entries.sort! { |a,b| a.title <=> b.title }
mtimes = [File.info(dir).modification_time] mtimes = [File.info(dir).modification_time]
mtimes += @title_ids.map{|e| @library.title_hash[e].mtime}
mtimes += @entries.map{|e| e.mtime} mtimes += @entries.map{|e| e.mtime}
mtimes += @titles.map{|e| e.mtime}
@mtime = mtimes.max @mtime = mtimes.max
end end
@ -118,7 +119,7 @@ class Title
{% end %} {% end %}
json.field "mtime" {json.number @mtime.to_unix} json.field "mtime" {json.number @mtime.to_unix}
json.field "titles" do json.field "titles" do
json.raw @titles.to_json json.raw self.titles.to_json
end end
json.field "entries" do json.field "entries" do
json.raw @entries.to_json json.raw @entries.to_json
@ -126,6 +127,10 @@ class Title
end end
end end
def titles
@title_ids.map {|tid| @library.get_title! tid}
end
# When downloading from MangaDex, the zip/cbz file would not be valid # When downloading from MangaDex, the zip/cbz file would not be valid
# before the download is completed. If we scan the zip file, # before the download is completed. If we scan the zip file,
# Entry.new would throw, so we use this method to check before # Entry.new would throw, so we use this method to check before
@ -214,13 +219,14 @@ class TitleInfo
end end
class Library class Library
property dir : String, titles : Array(Title), scan_interval : Int32, property dir : String, title_ids : Array(String), scan_interval : Int32,
logger : MLogger, storage : Storage logger : MLogger, storage : Storage, title_hash : Hash(String, Title)
def initialize(@dir, @scan_interval, @logger, @storage) def initialize(@dir, @scan_interval, @logger, @storage)
# explicitly initialize @titles to bypass the compiler check. it will # explicitly initialize @titles to bypass the compiler check. it will
# be filled with actual Titles in the `scan` call below # be filled with actual Titles in the `scan` call below
@titles = [] of Title @title_ids = [] of String
@title_hash = {} of String => Title
return scan if @scan_interval < 1 return scan if @scan_interval < 1
spawn do spawn do
@ -228,23 +234,27 @@ class Library
start = Time.local start = Time.local
scan scan
ms = (Time.local - start).total_milliseconds ms = (Time.local - start).total_milliseconds
@logger.info "Scanned #{@titles.size} titles in #{ms}ms" @logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep @scan_interval * 60 sleep @scan_interval * 60
end end
end end
end end
def titles
@title_ids.map {|tid| self.get_title!(tid) }
end
def to_json(json : JSON::Builder) def to_json(json : JSON::Builder)
json.object do json.object do
json.field "dir", @dir json.field "dir", @dir
json.field "titles" do json.field "titles" do
json.raw @titles.to_json json.raw self.titles.to_json
end end
end end
end end
def get_title(tid) def get_title(tid)
# top level @title_hash[tid]?
title = @titles.find { |t| t.id == tid } end
return title if !title.nil? def get_title!(tid)
@title_hash[tid]
end end
def scan def scan
unless Dir.exists? @dir unless Dir.exists? @dir
@ -252,13 +262,18 @@ class Library
"Attempting to create it" "Attempting to create it"
Dir.mkdir_p @dir Dir.mkdir_p @dir
end end
@titles = (Dir.entries @dir) @title_ids.clear
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." } .select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn } .map { |fn| File.join @dir, fn }
.select { |path| File.directory? path } .select { |path| File.directory? path }
.map { |path| Title.new path, @storage, @logger } .map { |path| Title.new path, @storage, @logger, self }
.select { |title| !(title.entries.empty? && title.titles.empty?) } .select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort { |a, b| a.title <=> b.title } .sort { |a, b| a.title <=> b.title }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id
end
@logger.debug "Scan completed" @logger.debug "Scan completed"
end end
end end