mirror of
https://github.com/hkalexling/Mango.git
synced 2025-08-02 10:55:30 -04:00
Merge pull request #222 from Leeingnyo/feature/enhance-loading-library
Improve loading pages (library, titles)
This commit is contained in:
commit
da8a485087
@ -20,6 +20,9 @@ class Config
|
||||
property plugin_path : String = File.expand_path "~/mango/plugins",
|
||||
home: true
|
||||
property download_timeout_seconds : Int32 = 30
|
||||
property cache_enabled = false
|
||||
property cache_size_mbs = 50
|
||||
property cache_log_enabled = true
|
||||
property disable_login = false
|
||||
property default_username = ""
|
||||
property auth_proxy_header_name = ""
|
||||
|
188
src/library/cache.cr
Normal file
188
src/library/cache.cr
Normal file
@ -0,0 +1,188 @@
|
||||
require "digest"
|
||||
|
||||
require "./entry"
|
||||
require "./types"
|
||||
|
||||
# Base class for an entry in the LRU cache.
|
||||
# There are two ways to use it:
|
||||
# 1. Use it as it is by instantiating with the appropriate `SaveT` and
|
||||
# `ReturnT`. Note that in this case, `SaveT` and `ReturnT` must be the
|
||||
# same type. That is, the input value will be stored as it is without
|
||||
# any transformation.
|
||||
# 2. You can also subclass it and provide custom implementations for
|
||||
# `to_save_t` and `to_return_t`. This allows you to transform and store
|
||||
# the input value to a different type. See `SortedEntriesCacheEntry` as
|
||||
# an example.
|
||||
private class CacheEntry(SaveT, ReturnT)
|
||||
getter key : String, atime : Time
|
||||
|
||||
@value : SaveT
|
||||
|
||||
def initialize(@key : String, value : ReturnT)
|
||||
@atime = @ctime = Time.utc
|
||||
@value = self.class.to_save_t value
|
||||
end
|
||||
|
||||
def value
|
||||
@atime = Time.utc
|
||||
self.class.to_return_t @value
|
||||
end
|
||||
|
||||
def self.to_save_t(value : ReturnT)
|
||||
value
|
||||
end
|
||||
|
||||
def self.to_return_t(value : SaveT)
|
||||
value
|
||||
end
|
||||
|
||||
def instance_size
|
||||
instance_sizeof(CacheEntry(SaveT, ReturnT)) + # sizeof itself
|
||||
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
|
||||
@value.instance_size
|
||||
end
|
||||
end
|
||||
|
||||
class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry))
|
||||
def self.to_save_t(value : Array(Entry))
|
||||
value.map &.id
|
||||
end
|
||||
|
||||
def self.to_return_t(value : Array(String))
|
||||
ids_to_entries value
|
||||
end
|
||||
|
||||
private def self.ids_to_entries(ids : Array(String))
|
||||
e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} }
|
||||
entries = [] of Entry
|
||||
begin
|
||||
ids.each do |id|
|
||||
entries << e_map[id]
|
||||
end
|
||||
return entries if ids.size == entries.size
|
||||
rescue
|
||||
end
|
||||
end
|
||||
|
||||
def instance_size
|
||||
instance_sizeof(SortedEntriesCacheEntry) + # sizeof itself
|
||||
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
|
||||
@value.size * (instance_sizeof(String) + sizeof(String)) +
|
||||
@value.sum(&.bytesize) # elements in Array(String)
|
||||
end
|
||||
|
||||
def self.gen_key(book_id : String, username : String,
|
||||
entries : Array(Entry), opt : SortOptions?)
|
||||
entries_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s
|
||||
user_context = opt && opt.method == SortMethod::Progress ? username : ""
|
||||
sig = Digest::SHA1.hexdigest (book_id + entries_sig + user_context +
|
||||
(opt ? opt.to_tuple.to_s : "nil"))
|
||||
"#{sig}:sorted_entries"
|
||||
end
|
||||
end
|
||||
|
||||
class String
|
||||
def instance_size
|
||||
instance_sizeof(String) + bytesize
|
||||
end
|
||||
end
|
||||
|
||||
struct Tuple(*T)
|
||||
def instance_size
|
||||
sizeof(T) + # total size of non-reference types
|
||||
self.sum do |e|
|
||||
next 0 unless e.is_a? Reference
|
||||
if e.responds_to? :instance_size
|
||||
e.instance_size
|
||||
else
|
||||
instance_sizeof(typeof(e))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
alias CacheableType = Array(Entry) | String | Tuple(String, Int32)
|
||||
alias CacheEntryType = SortedEntriesCacheEntry |
|
||||
CacheEntry(String, String) |
|
||||
CacheEntry(Tuple(String, Int32), Tuple(String, Int32))
|
||||
|
||||
def generate_cache_entry(key : String, value : CacheableType)
|
||||
if value.is_a? Array(Entry)
|
||||
SortedEntriesCacheEntry.new key, value
|
||||
else
|
||||
CacheEntry(typeof(value), typeof(value)).new key, value
|
||||
end
|
||||
end
|
||||
|
||||
# LRU Cache
|
||||
class LRUCache
|
||||
@@limit : Int128 = Int128.new 0
|
||||
@@should_log = true
|
||||
# key => entry
|
||||
@@cache = {} of String => CacheEntryType
|
||||
|
||||
def self.enabled
|
||||
Config.current.cache_enabled
|
||||
end
|
||||
|
||||
def self.init
|
||||
cache_size = Config.current.cache_size_mbs
|
||||
@@limit = Int128.new cache_size * 1024 * 1024 if enabled
|
||||
@@should_log = Config.current.cache_log_enabled
|
||||
end
|
||||
|
||||
def self.get(key : String)
|
||||
return unless enabled
|
||||
entry = @@cache[key]?
|
||||
if @@should_log
|
||||
Logger.debug "LRUCache #{entry.nil? ? "miss" : "hit"} #{key}"
|
||||
end
|
||||
return entry.value unless entry.nil?
|
||||
end
|
||||
|
||||
def self.set(cache_entry : CacheEntryType)
|
||||
return unless enabled
|
||||
key = cache_entry.key
|
||||
@@cache[key] = cache_entry
|
||||
Logger.debug "LRUCache cached #{key}" if @@should_log
|
||||
remove_least_recent_access
|
||||
end
|
||||
|
||||
def self.invalidate(key : String)
|
||||
return unless enabled
|
||||
@@cache.delete key
|
||||
end
|
||||
|
||||
def self.print
|
||||
return unless @@should_log
|
||||
sum = @@cache.sum { |_, entry| entry.instance_size }
|
||||
Logger.debug "---- LRU Cache ----"
|
||||
Logger.debug "Size: #{sum} Bytes"
|
||||
Logger.debug "List:"
|
||||
@@cache.each do |k, v|
|
||||
Logger.debug "#{k} | #{v.atime} | #{v.instance_size}"
|
||||
end
|
||||
Logger.debug "-------------------"
|
||||
end
|
||||
|
||||
private def self.is_cache_full
|
||||
sum = @@cache.sum { |_, entry| entry.instance_size }
|
||||
sum > @@limit
|
||||
end
|
||||
|
||||
private def self.remove_least_recent_access
|
||||
if @@should_log && is_cache_full
|
||||
Logger.debug "Removing entries from LRUCache"
|
||||
end
|
||||
while is_cache_full && @@cache.size > 0
|
||||
min_tuple = @@cache.min_by { |_, entry| entry.atime }
|
||||
min_key = min_tuple[0]
|
||||
min_entry = min_tuple[1]
|
||||
|
||||
Logger.debug " \
|
||||
Target: #{min_key}, \
|
||||
Last Access Time: #{min_entry.atime}" if @@should_log
|
||||
invalidate min_key
|
||||
end
|
||||
end
|
||||
end
|
@ -81,9 +81,17 @@ class Entry
|
||||
|
||||
def cover_url
|
||||
return "#{Config.current.base_url}img/icon.png" if @err_msg
|
||||
|
||||
unless @book.entry_cover_url_cache
|
||||
TitleInfo.new @book.dir do |info|
|
||||
@book.entry_cover_url_cache = info.entry_cover_url
|
||||
end
|
||||
end
|
||||
entry_cover_url = @book.entry_cover_url_cache
|
||||
|
||||
url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}"
|
||||
TitleInfo.new @book.dir do |info|
|
||||
info_url = info.entry_cover_url[@title]?
|
||||
if entry_cover_url
|
||||
info_url = entry_cover_url[@title]?
|
||||
unless info_url.nil? || info_url.empty?
|
||||
url = File.join Config.current.base_url, info_url
|
||||
end
|
||||
@ -170,6 +178,16 @@ class Entry
|
||||
# For backward backward compatibility with v0.1.0, we save entry titles
|
||||
# instead of IDs in info.json
|
||||
def save_progress(username, page)
|
||||
LRUCache.invalidate "#{@book.id}:#{username}:progress_sum"
|
||||
@book.parents.each do |parent|
|
||||
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
|
||||
end
|
||||
[false, true].each do |ascend|
|
||||
sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id,
|
||||
username, @book.entries, SortOptions.new(SortMethod::Progress, ascend)
|
||||
LRUCache.invalidate sorted_entries_cache_key
|
||||
end
|
||||
|
||||
TitleInfo.new @book.dir do |info|
|
||||
if info.progress[username]?.nil?
|
||||
info.progress[username] = {@title => page}
|
||||
|
@ -51,11 +51,6 @@ class Library
|
||||
def sorted_titles(username, opt : SortOptions? = nil)
|
||||
if opt.nil?
|
||||
opt = SortOptions.from_info_json @dir, username
|
||||
else
|
||||
TitleInfo.new @dir do |info|
|
||||
info.sort_by[username] = opt.to_tuple
|
||||
info.save
|
||||
end
|
||||
end
|
||||
|
||||
# Helper function from src/util/util.cr
|
||||
@ -66,6 +61,10 @@ class Library
|
||||
titles + titles.flat_map &.deep_titles
|
||||
end
|
||||
|
||||
def deep_entries
|
||||
titles.flat_map &.deep_entries
|
||||
end
|
||||
|
||||
def to_slim_json : String
|
||||
JSON.build do |json|
|
||||
json.object do
|
||||
|
@ -1,11 +1,17 @@
|
||||
require "digest"
|
||||
require "../archive"
|
||||
|
||||
class Title
|
||||
getter dir : String, parent_id : String, title_ids : Array(String),
|
||||
entries : Array(Entry), title : String, id : String,
|
||||
encoded_title : String, mtime : Time, signature : UInt64
|
||||
encoded_title : String, mtime : Time, signature : UInt64,
|
||||
entry_cover_url_cache : Hash(String, String)?
|
||||
setter entry_cover_url_cache : Hash(String, String)?
|
||||
|
||||
@entry_display_name_cache : Hash(String, String)?
|
||||
@entry_cover_url_cache : Hash(String, String)?
|
||||
@cached_display_name : String?
|
||||
@cached_cover_url : String?
|
||||
|
||||
def initialize(@dir : String, @parent_id)
|
||||
storage = Storage.default
|
||||
@ -177,11 +183,15 @@ class Title
|
||||
end
|
||||
|
||||
def display_name
|
||||
cached_display_name = @cached_display_name
|
||||
return cached_display_name unless cached_display_name.nil?
|
||||
|
||||
dn = @title
|
||||
TitleInfo.new @dir do |info|
|
||||
info_dn = info.display_name
|
||||
dn = info_dn unless info_dn.empty?
|
||||
end
|
||||
@cached_display_name = dn
|
||||
dn
|
||||
end
|
||||
|
||||
@ -205,6 +215,7 @@ class Title
|
||||
end
|
||||
|
||||
def set_display_name(dn)
|
||||
@cached_display_name = dn
|
||||
TitleInfo.new @dir do |info|
|
||||
info.display_name = dn
|
||||
info.save
|
||||
@ -214,11 +225,15 @@ class Title
|
||||
def set_display_name(entry_name : String, dn)
|
||||
TitleInfo.new @dir do |info|
|
||||
info.entry_display_name[entry_name] = dn
|
||||
@entry_display_name_cache = info.entry_display_name
|
||||
info.save
|
||||
end
|
||||
end
|
||||
|
||||
def cover_url
|
||||
cached_cover_url = @cached_cover_url
|
||||
return cached_cover_url unless cached_cover_url.nil?
|
||||
|
||||
url = "#{Config.current.base_url}img/icon.png"
|
||||
readable_entries = @entries.select &.err_msg.nil?
|
||||
if readable_entries.size > 0
|
||||
@ -230,10 +245,12 @@ class Title
|
||||
url = File.join Config.current.base_url, info_url
|
||||
end
|
||||
end
|
||||
@cached_cover_url = url
|
||||
url
|
||||
end
|
||||
|
||||
def set_cover_url(url : String)
|
||||
@cached_cover_url = url
|
||||
TitleInfo.new @dir do |info|
|
||||
info.cover_url = url
|
||||
info.save
|
||||
@ -243,6 +260,7 @@ class Title
|
||||
def set_cover_url(entry_name : String, url : String)
|
||||
TitleInfo.new @dir do |info|
|
||||
info.entry_cover_url[entry_name] = url
|
||||
@entry_cover_url_cache = info.entry_cover_url
|
||||
info.save
|
||||
end
|
||||
end
|
||||
@ -262,8 +280,15 @@ class Title
|
||||
end
|
||||
|
||||
def deep_read_page_count(username) : Int32
|
||||
load_progress_for_all_entries(username).sum +
|
||||
titles.flat_map(&.deep_read_page_count username).sum
|
||||
key = "#{@id}:#{username}:progress_sum"
|
||||
sig = Digest::SHA1.hexdigest (entries.map &.id).to_s
|
||||
cached_sum = LRUCache.get key
|
||||
return cached_sum[1] if cached_sum.is_a? Tuple(String, Int32) &&
|
||||
cached_sum[0] == sig
|
||||
sum = load_progress_for_all_entries(username).sum +
|
||||
titles.flat_map(&.deep_read_page_count username).sum
|
||||
LRUCache.set generate_cache_entry key, {sig, sum}
|
||||
sum
|
||||
end
|
||||
|
||||
def deep_total_page_count : Int32
|
||||
@ -317,13 +342,12 @@ class Title
|
||||
# use the default (auto, ascending)
|
||||
# When `opt` is not nil, it saves the options to info.json
|
||||
def sorted_entries(username, opt : SortOptions? = nil)
|
||||
cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, opt
|
||||
cached_entries = LRUCache.get cache_key
|
||||
return cached_entries if cached_entries.is_a? Array(Entry)
|
||||
|
||||
if opt.nil?
|
||||
opt = SortOptions.from_info_json @dir, username
|
||||
else
|
||||
TitleInfo.new @dir do |info|
|
||||
info.sort_by[username] = opt.to_tuple
|
||||
info.save
|
||||
end
|
||||
end
|
||||
|
||||
case opt.not_nil!.method
|
||||
@ -355,6 +379,7 @@ class Title
|
||||
|
||||
ary.reverse! unless opt.not_nil!.ascend
|
||||
|
||||
LRUCache.set generate_cache_entry cache_key, ary
|
||||
ary
|
||||
end
|
||||
|
||||
@ -416,6 +441,17 @@ class Title
|
||||
end
|
||||
|
||||
def bulk_progress(action, ids : Array(String), username)
|
||||
LRUCache.invalidate "#{@id}:#{username}:progress_sum"
|
||||
parents.each do |parent|
|
||||
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
|
||||
end
|
||||
[false, true].each do |ascend|
|
||||
sorted_entries_cache_key =
|
||||
SortedEntriesCacheEntry.gen_key @id, username, @entries,
|
||||
SortOptions.new(SortMethod::Progress, ascend)
|
||||
LRUCache.invalidate sorted_entries_cache_key
|
||||
end
|
||||
|
||||
selected_entries = ids
|
||||
.map { |id|
|
||||
@entries.find &.id.==(id)
|
||||
|
@ -88,6 +88,18 @@ class TitleInfo
|
||||
@@mutex_hash = {} of String => Mutex
|
||||
|
||||
def self.new(dir, &)
|
||||
key = "#{dir}:info.json"
|
||||
info = LRUCache.get key
|
||||
if info.is_a? String
|
||||
begin
|
||||
instance = TitleInfo.from_json info
|
||||
instance.dir = dir
|
||||
yield instance
|
||||
return
|
||||
rescue
|
||||
end
|
||||
end
|
||||
|
||||
if @@mutex_hash[dir]?
|
||||
mutex = @@mutex_hash[dir]
|
||||
else
|
||||
@ -101,6 +113,7 @@ class TitleInfo
|
||||
instance = TitleInfo.from_json File.read json_path
|
||||
end
|
||||
instance.dir = dir
|
||||
LRUCache.set generate_cache_entry key, instance.to_json
|
||||
yield instance
|
||||
end
|
||||
end
|
||||
@ -108,5 +121,7 @@ class TitleInfo
|
||||
def save
|
||||
json_path = File.join @dir, "info.json"
|
||||
File.write json_path, self.to_pretty_json
|
||||
key = "#{@dir}:info.json"
|
||||
LRUCache.set generate_cache_entry key, self.to_json
|
||||
end
|
||||
end
|
||||
|
@ -55,6 +55,7 @@ class CLI < Clim
|
||||
Config.load(opts.config).set_current
|
||||
|
||||
# Initialize main components
|
||||
LRUCache.init
|
||||
Storage.default
|
||||
Queue.default
|
||||
Library.default
|
||||
|
@ -41,7 +41,7 @@ struct MainRouter
|
||||
username = get_username env
|
||||
|
||||
sort_opt = SortOptions.from_info_json Library.default.dir, username
|
||||
get_sort_opt
|
||||
get_and_save_sort_opt Library.default.dir
|
||||
|
||||
titles = Library.default.sorted_titles username, sort_opt
|
||||
percentage = titles.map &.load_percentage username
|
||||
@ -59,12 +59,12 @@ struct MainRouter
|
||||
username = get_username env
|
||||
|
||||
sort_opt = SortOptions.from_info_json title.dir, username
|
||||
get_sort_opt
|
||||
get_and_save_sort_opt title.dir
|
||||
|
||||
entries = title.sorted_entries username, sort_opt
|
||||
|
||||
percentage = title.load_percentage_for_all_entries username, sort_opt
|
||||
title_percentage = title.titles.map &.load_percentage username
|
||||
|
||||
layout "title"
|
||||
rescue e
|
||||
Logger.error e
|
||||
|
@ -107,6 +107,26 @@ macro get_sort_opt
|
||||
end
|
||||
end
|
||||
|
||||
macro get_and_save_sort_opt(dir)
|
||||
sort_method = env.params.query["sort"]?
|
||||
|
||||
if sort_method
|
||||
is_ascending = true
|
||||
|
||||
ascend = env.params.query["ascend"]?
|
||||
if ascend && ascend.to_i? == 0
|
||||
is_ascending = false
|
||||
end
|
||||
|
||||
sort_opt = SortOptions.new sort_method, is_ascending
|
||||
|
||||
TitleInfo.new {{dir}} do |info|
|
||||
info.sort_by[username] = sort_opt.to_tuple
|
||||
info.save
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module HTTP
|
||||
class Client
|
||||
private def self.exec(uri : URI, tls : TLSContext = nil)
|
||||
|
Loading…
x
Reference in New Issue
Block a user