Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2026-04-03 00:03:14 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'dev' into feature/plugin-v2

Browse Source
This commit is contained in:
Alex Ling
2022-01-21 13:20:23 +00:00
parent 748386f0af b2329a79b4
commit 5fac8c6a60
16 changed files with 507 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/library/cache.cr
View File

@@ -1,6 +1,7 @@
require "digest"
require "./entry"
require "./title"
require "./types"
# Base class for an entry in the LRU cache.
@@ -81,6 +82,31 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry))
end
end
class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title))
def self.to_save_t(value : Array(Title))
value.map &.id
end
def self.to_return_t(value : Array(String))
value.map { |title_id| Library.default.title_hash[title_id].not_nil! }
end
def instance_size
instance_sizeof(SortedTitlesCacheEntry) + # sizeof itself
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
@value.size * (instance_sizeof(String) + sizeof(String)) +
@value.sum(&.bytesize) # elements in Array(String)
end
def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?)
titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s
user_context = opt && opt.method == SortMethod::Progress ? username : ""
sig = Digest::SHA1.hexdigest (titles_sig + user_context +
(opt ? opt.to_tuple.to_s : "nil"))
"#{sig}:sorted_titles"
end
end
class String
def instance_size
instance_sizeof(String) + bytesize
@@ -101,14 +127,18 @@ struct Tuple(*T)
end
end
alias CacheableType = Array(Entry) | String | Tuple(String, Int32)
alias CacheableType = Array(Entry) | Array(Title) | String |
Tuple(String, Int32)
alias CacheEntryType = SortedEntriesCacheEntry |
SortedTitlesCacheEntry |
CacheEntry(String, String) |
CacheEntry(Tuple(String, Int32), Tuple(String, Int32))
def generate_cache_entry(key : String, value : CacheableType)
if value.is_a? Array(Entry)
SortedEntriesCacheEntry.new key, value
elsif value.is_a? Array(Title)
SortedTitlesCacheEntry.new key, value
else
CacheEntry(typeof(value), typeof(value)).new key, value
end

39
src/library/entry.cr
View File

@@ -8,6 +8,9 @@ class Entry
size : String, pages : Int32, id : String, encoded_path : String,
encoded_title : String, mtime : Time, err_msg : String?
@[YAML::Field(ignore: true)]
@sort_title : String?
def initialize(@zip_path, @book)
storage = Storage.default
@encoded_path = URI.encode @zip_path
@@ -57,6 +60,7 @@ class Entry
{% end %}
json.field "title_id", @book.id
json.field "title_title", @book.title
json.field "sort_title", sort_title
json.field "pages" { json.number @pages }
unless slim
json.field "display_name", @book.display_name @title
@@ -67,6 +71,35 @@ class Entry
end
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = @book.entry_sort_title_db id
if sort_title
@sort_title = sort_title
return sort_title
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_entry_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
@book.entry_sort_title_cache = nil
@book.remove_sorted_entries_cache [SortMethod::Auto, SortMethod::Title],
username
end
def sort_title_db
@book.entry_sort_title_db @id
end
def display_name
@book.display_name @title
end
@@ -178,11 +211,7 @@ class Entry
@book.parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
[false, true].each do |ascend|
sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id,
username, @book.entries, SortOptions.new(SortMethod::Progress, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
@book.remove_sorted_caches [SortMethod::Progress], username
TitleInfo.new @book.dir do |info|
if info.progress[username]?.nil?

10
src/library/library.cr
View File

@@ -136,8 +136,12 @@ class Library
deleted_entry_ids: [] of String,
}
library_paths = (Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
@title_ids.select! do |title_id|
title = @title_hash[title_id]
next false unless library_paths.includes? title.dir
existence = title.examine examine_context
unless existence
examine_context["deleted_title_ids"].concat [title_id] +
@@ -152,14 +156,12 @@ class Library
end
cache = examine_context["cached_contents_signature"]
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
library_paths
.select { |path| !(remained_title_dirs.includes? path) }
.select { |path| File.directory? path }
.map { |path| Title.new path, "", cache }
.select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort! { |a, b| a.title <=> b.title }
.sort! { |a, b| a.sort_title <=> b.sort_title }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id

136
src/library/title.cr
View File

@@ -8,8 +8,13 @@ class Title
entries : Array(Entry), title : String, id : String,
encoded_title : String, mtime : Time, signature : UInt64,
entry_cover_url_cache : Hash(String, String)?
setter entry_cover_url_cache : Hash(String, String)?
setter entry_cover_url_cache : Hash(String, String)?,
entry_sort_title_cache : Hash(String, String | Nil)?
@[YAML::Field(ignore: true)]
@sort_title : String?
@[YAML::Field(ignore: true)]
@entry_sort_title_cache : Hash(String, String | Nil)?
@[YAML::Field(ignore: true)]
@entry_display_name_cache : Hash(String, String)?
@[YAML::Field(ignore: true)]
@@ -66,7 +71,7 @@ class Title
end
sorter = ChapterSorter.new @entries.map &.title
@entries.sort! do |a, b|
sorter.compare a.title, b.title
sorter.compare a.sort_title, b.sort_title
end
end
@@ -102,7 +107,11 @@ class Title
previous_titles_size = @title_ids.size
@title_ids.select! do |title_id|
title = Library.default.get_title! title_id
title = Library.default.get_title title_id
unless title # for if data consistency broken
context["deleted_title_ids"].concat [title_id]
next false
end
existence = title.examine context
unless existence
context["deleted_title_ids"].concat [title_id] +
@@ -137,6 +146,18 @@ class Title
Library.default.title_hash[title.id] = title
@title_ids << title.id
is_titles_added = true
# We think they are removed, but they are here!
# Cancel reserved jobs
revival_title_ids = [title.id] + title.deep_titles.map &.id
context["deleted_title_ids"].select! do |deleted_title_id|
!(revival_title_ids.includes? deleted_title_id)
end
revival_entry_ids = title.deep_entries.map &.id
context["deleted_entry_ids"].select! do |deleted_entry_id|
!(revival_entry_ids.includes? deleted_entry_id)
end
next
end
if is_supported_file path
@@ -145,6 +166,9 @@ class Title
if entry.pages > 0 || entry.err_msg
@entries << entry
is_entries_added = true
context["deleted_entry_ids"].select! do |deleted_entry_id|
entry.id != deleted_entry_id
end
end
end
end
@@ -161,13 +185,18 @@ class Title
end
end
if is_entries_added || previous_entries_size != @entries.size
sorter = ChapterSorter.new @entries.map &.title
sorter = ChapterSorter.new @entries.map &.sort_title
@entries.sort! do |a, b|
sorter.compare a.title, b.title
sorter.compare a.sort_title, b.sort_title
end
end
true
if @title_ids.size > 0 || @entries.size > 0
true
else
context["deleted_title_ids"].concat [@id]
false
end
end
alias SortContext = NamedTuple(username: String, opt: SortOptions)
@@ -180,6 +209,7 @@ class Title
json.field {{str}}, @{{str.id}}
{% end %}
json.field "signature" { json.number @signature }
json.field "sort_title", sort_title
unless slim
json.field "display_name", display_name
json.field "cover_url", cover_url
@@ -226,6 +256,15 @@ class Title
@title_ids.map { |tid| Library.default.get_title! tid }
end
def sorted_titles(username, opt : SortOptions? = nil)
if opt.nil?
opt = SortOptions.from_info_json @dir, username
end
# Helper function from src/util/util.cr
sort_titles titles, opt.not_nil!, username
end
# Get all entries, including entries in nested titles
def deep_entries
return @entries if title_ids.empty?
@@ -262,6 +301,48 @@ class Title
ary.join " and "
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = Storage.default.get_title_sort_title id
if sort_title
@sort_title = sort_title
return sort_title
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_title_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
if parents.size > 0
target = parents[-1].titles
else
target = Library.default.titles
end
remove_sorted_titles_cache target,
[SortMethod::Auto, SortMethod::Title], username
end
def sort_title_db
Storage.default.get_title_sort_title id
end
def entry_sort_title_db(entry_id)
unless @entry_sort_title_cache
@entry_sort_title_cache =
Storage.default.get_entries_sort_title @entries.map &.id
end
@entry_sort_title_cache.not_nil![entry_id]?
end
def tags
Storage.default.get_title_tags @id
end
@@ -448,28 +529,30 @@ class Title
case opt.not_nil!.method
when .title?
ary = @entries.sort { |a, b| compare_numerically a.title, b.title }
ary = @entries.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
end
when .time_modified?
ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.title, b.title }
compare_numerically a.sort_title, b.sort_title }
when .time_added?
ary = @entries.sort { |a, b| (a.date_added <=> b.date_added).or \
compare_numerically a.title, b.title }
compare_numerically a.sort_title, b.sort_title }
when .progress?
percentage_ary = load_percentage_for_all_entries username, opt, true
ary = @entries.zip(percentage_ary)
.sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \
compare_numerically a_tp[0].title, b_tp[0].title }
compare_numerically a_tp[0].sort_title, b_tp[0].sort_title }
.map &.[0]
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
sorter = ChapterSorter.new @entries.map &.title
sorter = ChapterSorter.new @entries.map &.sort_title
ary = @entries.sort do |a, b|
sorter.compare(a.title, b.title).or \
compare_numerically a.title, b.title
sorter.compare(a.sort_title, b.sort_title).or \
compare_numerically a.sort_title, b.sort_title
end
end
@@ -536,17 +619,32 @@ class Title
zip + titles.flat_map &.deep_entries_with_date_added
end
def remove_sorted_entries_cache(sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_entries_cache_key =
SortedEntriesCacheEntry.gen_key @id, username, @entries,
SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
end
end
def remove_sorted_caches(sort_methods : Array(SortMethod), username : String)
remove_sorted_entries_cache sort_methods, username
parents.each do |parent|
remove_sorted_titles_cache parent.titles, sort_methods, username
end
remove_sorted_titles_cache Library.default.titles, sort_methods, username
end
def bulk_progress(action, ids : Array(String), username)
LRUCache.invalidate "#{@id}:#{username}:progress_sum"
parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
[false, true].each do |ascend|
sorted_entries_cache_key =
SortedEntriesCacheEntry.gen_key @id, username, @entries,
SortOptions.new(SortMethod::Progress, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
remove_sorted_caches [SortMethod::Progress], username
selected_entries = ids
.map { |id|

32
src/routes/api.cr
View File

@@ -371,6 +371,38 @@ struct APIRouter
end
end
Koa.describe "Sets the sort title of a title or an entry", <<-MD
When `eid` is provided, apply the sort title to the entry. Otherwise, apply the sort title to the title identified by `tid`.
MD
Koa.tags ["admin", "library"]
Koa.path "tid", desc: "Title ID"
Koa.query "eid", desc: "Entry ID", required: false
Koa.query "name", desc: "The new sort title"
Koa.response 200, schema: "result"
put "/api/admin/sort_title/:tid" do |env|
username = get_username env
begin
title = (Library.default.get_title env.params.url["tid"])
.not_nil!
name = env.params.query["name"]?
entry = env.params.query["eid"]?
if entry.nil?
title.set_sort_title name, username
else
eobj = title.get_entry entry
eobj.set_sort_title name, username unless eobj.nil?
end
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
ws "/api/admin/mangadex/queue" do |socket, env|
interval_raw = env.params.query["interval"]?
interval = (interval_raw.to_i? if interval_raw) || 5

6
src/routes/main.cr
View File

@@ -61,9 +61,15 @@ struct MainRouter
sort_opt = SortOptions.from_info_json title.dir, username
get_and_save_sort_opt title.dir
sorted_titles = title.sorted_titles username, sort_opt
entries = title.sorted_entries username, sort_opt
percentage = title.load_percentage_for_all_entries username, sort_opt
title_percentage = title.titles.map &.load_percentage username
title_percentage_map = {} of String => Float64
title_percentage.each_with_index do |tp, i|
t = title.titles[i]
title_percentage_map[t.id] = tp
end
layout "title"
rescue e

61
src/storage.cr
View File

@@ -342,6 +342,67 @@ class Storage
end
end
def get_title_sort_title(title_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from titles where id = (?)",
title_id, as: String | Nil
end
end
sort_title
end
def set_title_sort_title(title_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update titles set sort_title = (?) where id = (?)",
sort_title, title_id
end
end
end
def get_entry_sort_title(entry_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from ids where id = (?)",
entry_id, as: String | Nil
end
end
sort_title
end
def get_entries_sort_title(ids : Array(String))
results = Hash(String, String | Nil).new
MainFiber.run do
get_db do |db|
db.query "select id, sort_title from ids where id in " \
"(#{ids.join "," { |id| "'#{id}'" }})" do |rs|
rs.each do
id = rs.read String
sort_title = rs.read String | Nil
results[id] = sort_title
end
end
end
end
results
end
def set_entry_sort_title(entry_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update ids set sort_title = (?) where id = (?)",
sort_title, entry_id
end
end
end
def save_thumbnail(id : String, img : Image)
MainFiber.run do
get_db do |db|

31
src/util/util.cr
View File

@@ -87,30 +87,49 @@ def env_is_true?(key : String) : Bool
end
def sort_titles(titles : Array(Title), opt : SortOptions, username : String)
ary = titles
cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt
cached_titles = LRUCache.get cache_key
return cached_titles if cached_titles.is_a? Array(Title)
case opt.method
when .time_modified?
ary.sort! { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.title, b.title }
ary = titles.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.sort_title, b.sort_title }
when .progress?
ary.sort! do |a, b|
ary = titles.sort do |a, b|
(a.load_percentage(username) <=> b.load_percentage(username)).or \
compare_numerically a.title, b.title
compare_numerically a.sort_title, b.sort_title
end
when .title?
ary = titles.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
end
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
ary.sort! { |a, b| compare_numerically a.title, b.title }
ary = titles.sort { |a, b| compare_numerically a.sort_title, b.sort_title }
end
ary.reverse! unless opt.not_nil!.ascend
LRUCache.set generate_cache_entry cache_key, ary
ary
end
def remove_sorted_titles_cache(titles : Array(Title),
sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username,
titles, SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_titles_cache_key
end
end
end
class String
# Returns the similarity (in [0, 1]) of two paths.
# For the two paths, separate them into arrays of components, count the

4
src/views/components/card.html.ecr
View File

@@ -61,7 +61,9 @@
<% if page == "home" && item.is_a? Entry %>
<%= "uk-margin-remove-bottom" %>
<% end %>
" data-title="<%= HTML.escape(item.display_name) %>"><%= HTML.escape(item.display_name) %>
" data-title="<%= HTML.escape(item.display_name) %>"
data-file-title="<%= HTML.escape(item.title || "") %>"
data-sort-title="<%= HTML.escape(item.sort_title_db || "") %>"><%= HTML.escape(item.display_name) %>
</h3>
<% if page == "home" && item.is_a? Entry %>
<a class="uk-card-title break-word uk-margin-remove-top uk-text-meta uk-display-inline-block no-modal" data-title="<%= HTML.escape(item.book.display_name) %>" href="<%= base_url %>book/<%= item.book.id %>"><%= HTML.escape(item.book.display_name) %></a>

1
src/views/library.html.ecr
View File

@@ -10,6 +10,7 @@
<div class="uk-margin-bottom uk-width-1-4@s">
<% hash = {
"auto" => "Auto",
"title" => "Name",
"time_modified" => "Date Modified",
"progress" => "Progress"
} %>

10
src/views/reader.html.ecr
View File

@@ -55,8 +55,8 @@
object-fit: contain;
`" />
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true)"></div>
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false ^ enableRightToLeft)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true ^ enableRightToLeft)"></div>
</div>
</div>
@@ -114,6 +114,12 @@
</div>
</div>
<div class="uk-margin uk-form-horizontal" x-show="mode !== 'continuous'">
<label class="uk-form-label" for="enable-right-to-left">Right to Left</label>
<div class="uk-form-controls">
<input id="enable-right-to-left" class="uk-checkbox" type="checkbox" x-model="enableRightToLeft" @change="enableRightToLeftChanged()">
</div>
</div>
<hr class="uk-divider-icon">
<div class="uk-margin">

14
src/views/title.html.ecr
View File

@@ -18,7 +18,8 @@
</div>
</div>
</div>
<h2 class=uk-title><span><%= title.display_name %></span>
<h2 class=uk-title data-file-title="<%= HTML.escape(title.title) %>" data-sort-title="<%= HTML.escape(title.sort_title_db || "") %>">
<span><%= title.display_name %></span>
&nbsp;
<% if is_admin %>
<a onclick="edit()" class="uk-icon-button" uk-icon="icon:pencil"></a>
@@ -59,8 +60,8 @@
</div>
<div class="uk-child-width-1-4@m uk-child-width-1-2" uk-grid>
<% title.titles.each_with_index do |item, i| %>
<% progress = title_percentage[i] %>
<% sorted_titles.each do |item| %>
<% progress = title_percentage_map[item.id] %>
<%= render_component "card" %>
<% end %>
</div>
@@ -89,6 +90,13 @@
<input class="uk-input" type="text" name="display-name" id="display-name-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label" for="sort-title">Sort Title</label>
<div class="uk-inline">
<a class="uk-form-icon uk-form-icon-flip" uk-icon="icon:check"></a>
<input class="uk-input" type="text" name="sort-title" id="sort-title-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label">Cover Image</label>
<div class="uk-grid">