From de410f42b8cd67769aa1778d89837cdbde3966a9 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 01:54:19 +0900 Subject: [PATCH] Replace InfoCache to LRUCache --- src/library/cache.cr | 197 ++++++++--------------------------------- src/library/entry.cr | 12 ++- src/library/library.cr | 4 - src/library/title.cr | 28 +++--- src/library/types.cr | 7 +- src/util/web.cr | 3 +- 6 files changed, 61 insertions(+), 190 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index b399632..f772c66 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -1,164 +1,7 @@ require "digest" require "./entry" - -class InfoCache - alias ProgressCache = Tuple(String, Int32) - - def self.clear - clear_cover_url - clear_progress_cache - clear_sort_opt - end - - def self.clean - clean_cover_url - clean_progress_cache - clean_sort_opt - end - - # item id => cover_url - @@cached_cover_url = {} of String => String - @@cached_cover_url_previous = {} of String => String # item id => cover_url - - def self.set_cover_url(id : String, cover_url : String) - @@cached_cover_url[id] = cover_url - end - - def self.get_cover_url(id : String) - @@cached_cover_url[id]? - end - - def self.invalidate_cover_url(id : String) - @@cached_cover_url.delete id - end - - def self.move_cover_url(id : String) - if @@cached_cover_url_previous[id]? - @@cached_cover_url[id] = @@cached_cover_url_previous[id] - end - end - - private def self.clear_cover_url - @@cached_cover_url_previous = @@cached_cover_url - @@cached_cover_url = {} of String => String - end - - private def self.clean_cover_url - @@cached_cover_url_previous = {} of String => String - end - - # book.id:username => {signature, sum} - @@progress_cache = {} of String => ProgressCache - # book.id => username => {signature, sum} - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - - def self.set_progress_cache(book_id : String, username : String, - entries : Array(Entry), sum : Int32) - progress_cache_id = "#{book_id}:#{username}" - progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s - @@progress_cache[progress_cache_id] = {progress_cache_sig, sum} - Logger.debug "Progress Cached #{progress_cache_id}" - end - - def self.get_progress_cache(book_id : String, username : String, - entries : Array(Entry)) - progress_cache_id = "#{book_id}:#{username}" - progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s - cached = @@progress_cache[progress_cache_id]? - if cached && cached[0] == progress_cache_sig - Logger.debug "Progress Cache Hit! #{progress_cache_id}" - return cached[1] - end - end - - def self.invalidate_progress_cache(book_id : String, username : String) - progress_cache_id = "#{book_id}:#{username}" - if @@progress_cache[progress_cache_id]? - @@progress_cache.delete progress_cache_id - Logger.debug "Progress Invalidate Cache #{progress_cache_id}" - end - end - - def self.move_progress_cache(book_id : String) - if @@progress_cache_previous[book_id]? - @@progress_cache_previous[book_id].each do |username, cached| - id = "#{book_id}:#{username}" - unless @@progress_cache[id]? - # It would be invalidated when entries changed - @@progress_cache[id] = cached - end - end - end - end - - private def self.clear_progress_cache - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - @@progress_cache.each do |id, cached| - splitted = id.split(':', 2) - book_id = splitted[0] - username = splitted[1] - unless @@progress_cache_previous[book_id]? - @@progress_cache_previous[book_id] = {} of String => ProgressCache - end - - @@progress_cache_previous[book_id][username] = cached - end - @@progress_cache = {} of String => ProgressCache - end - - private def self.clean_progress_cache - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - end - - # book.dir:username => SortOptions - @@cached_sort_opt = {} of String => SortOptions - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - - def self.set_sort_opt(dir : String, username : String, sort_opt : SortOptions) - id = "#{dir}:#{username}" - @@cached_sort_opt[id] = sort_opt - end - - def self.get_sort_opt(dir : String, username : String) - id = "#{dir}:#{username}" - @@cached_sort_opt[id]? - end - - def self.invalidate_sort_opt(dir : String, username : String) - id = "#{dir}:#{username}" - @@cached_sort_opt.delete id - end - - def self.move_sort_opt(dir : String) - if @@cached_sort_opt_previous[dir]? - @@cached_sort_opt_previous[dir].each do |username, cached| - id = "#{dir}:#{username}" - unless @@cached_sort_opt[id]? - @@cached_sort_opt[id] = cached - end - end - end - end - - private def self.clear_sort_opt - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - @@cached_sort_opt.each do |id, cached| - splitted = id.split(':', 2) - book_dir = splitted[0] - username = splitted[1] - unless @@cached_sort_opt_previous[book_dir]? - @@cached_sort_opt_previous[book_dir] = {} of String => SortOptions - end - @@cached_sort_opt_previous[book_dir][username] = cached - end - @@cached_sort_opt = {} of String => SortOptions - end - - private def self.clean_sort_opt - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - end -end +require "./types" private class CacheEntry(SaveT, ReturnT) getter key : String, atime : Time @@ -227,11 +70,45 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end end -alias CacheEntryType = SortedEntriesCacheEntry +class SortOptionsCacheEntry < CacheEntry(Tuple(String, Bool), SortOptions) + def self.to_save_t(value : SortOptions) + value.to_tuple + end -def generate_cache_entry(key : String, value : Array(Entry) | Int32 | String) + def self.to_return_t(value : Tuple(String, Bool)) + SortOptions.from_tuple value + end + + def instance_size + instance_sizeof(SortOptionsCacheEntry) + + @value[0].instance_size + end +end + +class String + def instance_size + instance_sizeof(String) + bytesize + end +end + +struct Tuple(*T) + def instance_size + sizeof(T) # iterate T and add instance_size of that + end +end + +alias CacheableType = Array(Entry) | String | Tuple(String, Int32) | + SortOptions +alias CacheEntryType = SortedEntriesCacheEntry | + SortOptionsCacheEntry | + CacheEntry(String, String) | + CacheEntry(Tuple(String, Int32), Tuple(String, Int32)) + +def generate_cache_entry(key : String, value : CacheableType) if value.is_a? Array(Entry) SortedEntriesCacheEntry.new key, value + elsif value.is_a? SortOptions + SortOptionsCacheEntry.new key, value else CacheEntry(typeof(value), typeof(value)).new key, value end diff --git a/src/library/entry.cr b/src/library/entry.cr index 176efe1..2c1ae34 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -44,8 +44,6 @@ class Entry MIME.from_filename? e.filename end file.close - - InfoCache.move_cover_url @id end def to_slim_json : String @@ -83,8 +81,8 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg - cached_cover_url = InfoCache.get_cover_url @id - return cached_cover_url if cached_cover_url + cached_cover_url = LRUCache.get "#{@id}:cover_url" + return cached_cover_url if cached_cover_url.is_a? String unless @book.entry_cover_url_cache TitleInfo.new @book.dir do |info| @@ -100,7 +98,7 @@ class Entry url = File.join Config.current.base_url, info_url end end - InfoCache.set_cover_url @id, url + LRUCache.set generate_cache_entry "#{@id}:cover_url", url url end @@ -183,9 +181,9 @@ class Entry # For backward backward compatibility with v0.1.0, we save entry titles # instead of IDs in info.json def save_progress(username, page) - InfoCache.invalidate_progress_cache @book.id, username + LRUCache.invalidate "#{@book.id}:#{username}:progress_sum" @book.parents.each do |parent| - InfoCache.invalidate_progress_cache parent.id, username + LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end [false, true].each do |ascend| sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, diff --git a/src/library/library.cr b/src/library/library.cr index 21e5c8b..9351e60 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -106,8 +106,6 @@ class Library storage = Storage.new auto_close: false - InfoCache.clear - (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } @@ -121,8 +119,6 @@ class Library @title_ids << title.id end - InfoCache.clean - storage.bulk_insert_ids storage.close diff --git a/src/library/title.cr b/src/library/title.cr index a5a6850..cdf78fd 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -60,10 +60,6 @@ class Title @entries.sort! do |a, b| sorter.compare a.title, b.title end - - InfoCache.move_cover_url @id - InfoCache.move_progress_cache @id - InfoCache.move_sort_opt @dir end def to_slim_json : String @@ -234,8 +230,8 @@ class Title end def cover_url - cached_cover_url = InfoCache.get_cover_url @id - return cached_cover_url if cached_cover_url + cached_cover_url = LRUCache.get "#{@id}:cover_url" + return cached_cover_url if cached_cover_url.is_a? String url = "#{Config.current.base_url}img/icon.png" readable_entries = @entries.select &.err_msg.nil? @@ -248,12 +244,12 @@ class Title url = File.join Config.current.base_url, info_url end end - InfoCache.set_cover_url @id, url + LRUCache.set generate_cache_entry "#{@id}:cover_url", url url end def set_cover_url(url : String) - InfoCache.invalidate_cover_url @id + LRUCache.invalidate "#{@id}:cover_url" TitleInfo.new @dir do |info| info.cover_url = url info.save @@ -262,7 +258,7 @@ class Title def set_cover_url(entry_name : String, url : String) selected_entry = @entries.find { |entry| entry.display_name == entry_name } - InfoCache.invalidate_cover_url selected_entry.id if selected_entry + LRUCache.invalidate "#{selected_entry.id}:cover_url" if selected_entry TitleInfo.new @dir do |info| info.entry_cover_url[entry_name] = url info.save @@ -284,12 +280,14 @@ class Title end def deep_read_page_count(username) : Int32 - # CACHE HERE - cached_sum = InfoCache.get_progress_cache @id, username, @entries - return cached_sum unless cached_sum.nil? + key = "#{@id}:#{username}:progress_sum" + sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + cached_sum = LRUCache.get key + return cached_sum[1] if cached_sum.is_a? Tuple(String, Int32) && + cached_sum[0] == sig sum = load_progress_for_all_entries(username).sum + titles.flat_map(&.deep_read_page_count username).sum - InfoCache.set_progress_cache @id, username, @entries, sum + LRUCache.set generate_cache_entry key, {sig, sum} sum end @@ -445,9 +443,9 @@ class Title end def bulk_progress(action, ids : Array(String), username) - InfoCache.invalidate_progress_cache @id, username + LRUCache.invalidate "#{@id}:#{username}:progress_sum" parents.each do |parent| - InfoCache.invalidate_progress_cache parent.id, username + LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end [false, true].each do |ascend| sorted_entries_cache_key = diff --git a/src/library/types.cr b/src/library/types.cr index 094cb64..891ee08 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -35,15 +35,16 @@ class SortOptions end def self.from_info_json(dir, username) - cached_opt = InfoCache.get_sort_opt dir, username - return cached_opt if cached_opt + key = "#{dir}:#{username}:sort_opt" + cached_opt = LRUCache.get key + return cached_opt if cached_opt.is_a? SortOptions opt = SortOptions.new TitleInfo.new dir do |info| if info.sort_by.has_key? username opt = SortOptions.from_tuple info.sort_by[username] end end - InfoCache.set_sort_opt dir, username, opt + LRUCache.set generate_cache_entry key, opt opt end diff --git a/src/util/web.cr b/src/util/web.cr index 5e873ca..9b967da 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -120,7 +120,8 @@ macro get_and_save_sort_opt(dir) sort_opt = SortOptions.new sort_method, is_ascending - InfoCache.set_sort_opt {{dir}}, username, sort_opt + key = "#{{{dir}}}:#{username}:sort_opt" + LRUCache.set generate_cache_entry key, sort_opt TitleInfo.new {{dir}} do |info| info.sort_by[username] = sort_opt.to_tuple info.save