Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2025-08-02 10:55:30 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #271 from hkalexling/rc/0.25.0

Browse Source 
v0.25.0
This commit is contained in:
Alex Ling 2022-02-12 12:55:35 +08:00 committed by GitHub
commit a101526672
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 645 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
README.md
View File

@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
Mango - Manga Server and Web Reader. Version 0.24.0
Mango - Manga Server and Web Reader. Version 0.25.0
Usage:
@ -80,6 +80,7 @@ base_url: /
session_secret: mango-session-secret
library_path: ~/mango/library
db_path: ~/mango/mango.db
queue_db_path: ~/mango/queue.db
scan_interval_minutes: 5
thumbnail_generation_interval_hours: 24
log_level: info
@ -87,23 +88,15 @@ upload_path: ~/mango/uploads
plugin_path: ~/mango/plugins
download_timeout_seconds: 30
library_cache_path: ~/mango/library.yml.gz
cache_enabled: false
cache_enabled: true
cache_size_mbs: 50
cache_log_enabled: true
disable_login: false
default_username: ""
auth_proxy_header_name: ""
mangadex:
base_url: https://mangadex.org
api_url: https://api.mangadex.org/v2
download_wait_seconds: 5
download_retries: 4
download_queue_db_path: ~/mango/queue.db
chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
manga_rename_rule: '{title}'
```
- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
- `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
- You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work.
- By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`.

94
migration/sort_title.12.cr Normal file
View File

@ -0,0 +1,94 @@
class SortTitle < MG::Base
def up : String
<<-SQL
-- add sort_title column to ids and titles
ALTER TABLE ids ADD COLUMN sort_title TEXT;
ALTER TABLE titles ADD COLUMN sort_title TEXT;
SQL
end
def down : String
<<-SQL
-- remove sort_title column from ids
ALTER TABLE ids RENAME TO tmp;
CREATE TABLE ids (
path TEXT NOT NULL,
id TEXT NOT NULL,
signature TEXT,
unavailable INTEGER NOT NULL DEFAULT 0
);
INSERT INTO ids
SELECT path, id, signature, unavailable
FROM tmp;
DROP TABLE tmp;
-- recreate the indices
CREATE UNIQUE INDEX path_idx ON ids (path);
CREATE UNIQUE INDEX id_idx ON ids (id);
-- recreate the foreign key constraint on thumbnails
ALTER TABLE thumbnails RENAME TO tmp;
CREATE TABLE thumbnails (
id TEXT NOT NULL,
data BLOB NOT NULL,
filename TEXT NOT NULL,
mime TEXT NOT NULL,
size INTEGER NOT NULL,
FOREIGN KEY (id) REFERENCES ids (id)
ON UPDATE CASCADE
ON DELETE CASCADE
);
INSERT INTO thumbnails
SELECT * FROM tmp;
DROP TABLE tmp;
CREATE UNIQUE INDEX tn_index ON thumbnails (id);
-- remove sort_title column from titles
ALTER TABLE titles RENAME TO tmp;
CREATE TABLE titles (
id TEXT NOT NULL,
path TEXT NOT NULL,
signature TEXT,
unavailable INTEGER NOT NULL DEFAULT 0
);
INSERT INTO titles
SELECT id, path, signature, unavailable
FROM tmp;
DROP TABLE tmp;
-- recreate the indices
CREATE UNIQUE INDEX titles_id_idx on titles (id);
CREATE UNIQUE INDEX titles_path_idx on titles (path);
-- recreate the foreign key constraint on tags
ALTER TABLE tags RENAME TO tmp;
CREATE TABLE tags (
id TEXT NOT NULL,
tag TEXT NOT NULL,
UNIQUE (id, tag),
FOREIGN KEY (id) REFERENCES titles (id)
ON UPDATE CASCADE
ON DELETE CASCADE
);
INSERT INTO tags
SELECT * FROM tmp;
DROP TABLE tmp;
CREATE INDEX tags_id_idx ON tags (id);
CREATE INDEX tags_tag_idx ON tags (tag);
SQL
end
end

2
public/js/download-manager.js
View File

@ -55,7 +55,7 @@ const component = () => {
jobAction(action, event) {
let url = `${base_url}api/admin/mangadex/queue/${action}`;
if (event) {
const id = event.currentTarget.closest('tr').id.split('-')[1];
const id = event.currentTarget.closest('tr').id.split('-').slice(1).join('-');
url = `${url}?${$.param({
id: id
})}`;

7
public/js/plugin-download.js
View File

@ -68,7 +68,12 @@ const buildTable = (chapters) => {
$('table').append(thead);
const rows = chapters.map(ch => {
const tds = Object.values(ch).map(v => `<td>${v}</td>`).join('');
const tds = Object.values(ch).map(v => {
const maxLength = 40;
const shouldShrink = v && v.length > maxLength;
const content = shouldShrink ? `<span title="${v}">${v.substring(0, maxLength)}...</span><div uk-dropdown><span>${v}</span></div>` : v;
return `<td>${content}</td>`
}).join('');
return `<tr data-id="${ch.id}" data-title="${ch.title}">${tds}</tr>`;
});
const tbody = `<tbody id="selectable">${rows}</tbody>`;

18
public/js/reader.js
View File

@ -13,6 +13,7 @@ const readerComponent = () => {
selectedIndex: 0, // 0: not selected; 1: the first page
margin: 30,
preloadLookahead: 3,
enableRightToLeft: false,
/**
* Initialize the component by fetching the page dimensions
@ -64,6 +65,13 @@ const readerComponent = () => {
const savedFlipAnimation = localStorage.getItem('enableFlipAnimation');
this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true';
const savedRightToLeft = localStorage.getItem('enableRightToLeft');
if (savedRightToLeft === null) {
this.enableRightToLeft = false;
} else {
this.enableRightToLeft = (savedRightToLeft === 'true');
}
})
.catch(e => {
const errMsg = `Failed to get the page dimensions. ${e}`;
@ -114,9 +122,9 @@ const readerComponent = () => {
if (this.mode === 'continuous') return;
if (event.key === 'ArrowLeft' || event.key === 'k')
this.flipPage(false);
this.flipPage(false ^ this.enableRightToLeft);
if (event.key === 'ArrowRight' || event.key === 'j')
this.flipPage(true);
this.flipPage(true ^ this.enableRightToLeft);
},
/**
* Flips to the next or the previous page
@ -136,7 +144,7 @@ const readerComponent = () => {
this.toPage(newIdx);
if (this.enableFlipAnimation) {
if (isNext)
if (isNext ^ this.enableRightToLeft)
this.flipAnimation = 'right';
else
this.flipAnimation = 'left';
@ -320,5 +328,9 @@ const readerComponent = () => {
enableFlipAnimationChanged() {
localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation);
},
enableRightToLeftChanged() {
localStorage.setItem('enableRightToLeft', this.enableRightToLeft);
},
};
}

68
public/js/title.js
View File

@ -60,6 +60,11 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi
UIkit.modal($('#modal')).show();
}
UIkit.util.on(document, 'hidden', '#modal', () => {
$('#read-btn').off('click');
$('#unread-btn').off('click');
});
const updateProgress = (tid, eid, page) => {
let url = `${base_url}api/progress/${tid}/${page}`
const query = $.param({
@ -90,8 +95,6 @@ const renameSubmit = (name, eid) => {
const upload = $('.upload-field');
const titleId = upload.attr('data-title-id');
console.log(name);
if (name.length === 0) {
alert('danger', 'The display name should not be empty');
return;
@ -122,15 +125,47 @@ const renameSubmit = (name, eid) => {
});
};
const renameSortNameSubmit = (name, eid) => {
const upload = $('.upload-field');
const titleId = upload.attr('data-title-id');
const params = {};
if (eid) params.eid = eid;
if (name) params.name = name;
const query = $.param(params);
let url = `${base_url}api/admin/sort_title/${titleId}?${query}`;
$.ajax({
type: 'PUT',
url,
contentType: 'application/json',
dataType: 'json'
})
.done(data => {
if (data.error) {
alert('danger', `Failed to update sort title. Error: ${data.error}`);
return;
}
location.reload();
})
.fail((jqXHR, status) => {
alert('danger', `Failed to update sort title. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
});
};
const edit = (eid) => {
const cover = $('#edit-modal #cover');
let url = cover.attr('data-title-cover');
let displayName = $('h2.uk-title > span').text();
let fileTitle = $('h2.uk-title').attr('data-file-title');
let sortTitle = $('h2.uk-title').attr('data-sort-title');
if (eid) {
const item = $(`#${eid}`);
url = item.find('img').attr('data-src');
displayName = item.find('.uk-card-title').attr('data-title');
fileTitle = item.find('.uk-card-title').attr('data-file-title');
sortTitle = item.find('.uk-card-title').attr('data-sort-title');
$('#title-progress-control').attr('hidden', '');
} else {
$('#title-progress-control').removeAttr('hidden');
@ -140,14 +175,26 @@ const edit = (eid) => {
const displayNameField = $('#display-name-field');
displayNameField.attr('value', displayName);
console.log(displayNameField);
displayNameField.attr('placeholder', fileTitle);
displayNameField.keyup(event => {
if (event.keyCode === 13) {
renameSubmit(displayNameField.val(), eid);
renameSubmit(displayNameField.val() || fileTitle, eid);
}
});
displayNameField.siblings('a.uk-form-icon').click(() => {
renameSubmit(displayNameField.val(), eid);
renameSubmit(displayNameField.val() || fileTitle, eid);
});
const sortTitleField = $('#sort-title-field');
sortTitleField.val(sortTitle);
sortTitleField.attr('placeholder', fileTitle);
sortTitleField.keyup(event => {
if (event.keyCode === 13) {
renameSortNameSubmit(sortTitleField.val(), eid);
}
});
sortTitleField.siblings('a.uk-form-icon').click(() => {
renameSortNameSubmit(sortTitleField.val(), eid);
});
setupUpload(eid);
@ -155,6 +202,16 @@ const edit = (eid) => {
UIkit.modal($('#edit-modal')).show();
};
UIkit.util.on(document, 'hidden', '#edit-modal', () => {
const displayNameField = $('#display-name-field');
displayNameField.off('keyup');
displayNameField.off('click');
const sortTitleField = $('#sort-title-field');
sortTitleField.off('keyup');
sortTitleField.off('click');
});
const setupUpload = (eid) => {
const upload = $('.upload-field');
const bar = $('#upload-progress').get(0);
@ -166,7 +223,6 @@ const setupUpload = (eid) => {
queryObj['eid'] = eid;
const query = $.param(queryObj);
const url = `${base_url}api/admin/upload/cover?${query}`;
console.log(url);
UIkit.upload('.upload-field', {
url: url,
name: 'file',

2
shard.yml
View File

@ -1,5 +1,5 @@
name: mango
version: 0.24.0
version: 0.25.0
authors:
- Alex Ling <hkalexling@gmail.com>

73
src/config.cr
View File

@ -4,43 +4,27 @@ class Config
include YAML::Serializable
@[YAML::Field(ignore: true)]
property path : String = ""
property host : String = "0.0.0.0"
property path = ""
property host = "0.0.0.0"
property port : Int32 = 9000
property base_url : String = "/"
property session_secret : String = "mango-session-secret"
property library_path : String = File.expand_path "~/mango/library",
home: true
property library_cache_path = File.expand_path "~/mango/library.yml.gz",
home: true
property db_path : String = File.expand_path "~/mango/mango.db", home: true
property base_url = "/"
property session_secret = "mango-session-secret"
property library_path = "~/mango/library"
property library_cache_path = "~/mango/library.yml.gz"
property db_path = "~/mango/mango.db"
property queue_db_path = "~/mango/queue.db"
property scan_interval_minutes : Int32 = 5
property thumbnail_generation_interval_hours : Int32 = 24
property log_level : String = "info"
property upload_path : String = File.expand_path "~/mango/uploads",
home: true
property plugin_path : String = File.expand_path "~/mango/plugins",
home: true
property log_level = "info"
property upload_path = "~/mango/uploads"
property plugin_path = "~/mango/plugins"
property download_timeout_seconds : Int32 = 30
property cache_enabled = false
property cache_enabled = true
property cache_size_mbs = 50
property cache_log_enabled = true
property disable_login = false
property default_username = ""
property auth_proxy_header_name = ""
property mangadex = Hash(String, String | Int32).new
@[YAML::Field(ignore: true)]
@mangadex_defaults = {
"base_url" => "https://mangadex.org",
"api_url" => "https://api.mangadex.org/v2",
"download_wait_seconds" => 5,
"download_retries" => 4,
"download_queue_db_path" => File.expand_path("~/mango/queue.db",
home: true),
"chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}",
"manga_rename_rule" => "{title}",
}
@@singlet : Config?
@ -58,7 +42,7 @@ class Config
if File.exists? cfg_path
config = self.from_yaml File.read cfg_path
config.path = path
config.fill_defaults
config.expand_paths
config.preprocess
return config
end
@ -66,7 +50,7 @@ class Config
"Dumping the default config there."
default = self.allocate
default.path = path
default.fill_defaults
default.expand_paths
cfg_dir = File.dirname cfg_path
unless Dir.exists? cfg_dir
Dir.mkdir_p cfg_dir
@ -76,13 +60,9 @@ class Config
default
end
def fill_defaults
{% for hash_name in ["mangadex"] %}
@{{hash_name.id}}_defaults.map do |k, v|
if @{{hash_name.id}}[k]?.nil?
@{{hash_name.id}}[k] = v
end
end
def expand_paths
{% for p in %w(library library_cache db queue_db upload plugin) %}
@{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true
{% end %}
end
@ -97,24 +77,5 @@ class Config
raise "Login is disabled, but default username is not set. " \
"Please set a default username"
end
# `Logger.default` is not available yet
Log.setup :debug
unless mangadex["api_url"] =~ /\/v2/
Log.warn { "It looks like you are using the deprecated MangaDex API " \
"v1 in your config file. Please update it to " \
"https://api.mangadex.org/v2 to suppress this warning." }
mangadex["api_url"] = "https://api.mangadex.org/v2"
end
if mangadex["api_url"] =~ /\/api\/v2/
Log.warn { "It looks like you are using the outdated MangaDex API " \
"url (mangadex.org/api/v2) in your config file. Please " \
"update it to https://api.mangadex.org/v2 to suppress this " \
"warning." }
mangadex["api_url"] = "https://api.mangadex.org/v2"
end
mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/"
mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/"
end
end

5
src/handlers/auth_handler.cr
View File

@ -54,8 +54,9 @@ class AuthHandler < Kemal::Handler
end
def call(env)
# Skip all authentication if requesting /login, /logout, or a static file
if request_path_startswith(env, ["/login", "/logout"]) ||
# Skip all authentication if requesting /login, /logout, /api/login,
# or a static file
if request_path_startswith(env, ["/login", "/logout", "/api/login"]) ||
requesting_static_file env
return call_next(env)
end

32
src/library/cache.cr
View File

@ -1,6 +1,7 @@
require "digest"
require "./entry"
require "./title"
require "./types"
# Base class for an entry in the LRU cache.
@ -81,6 +82,31 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry))
end
end
class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title))
def self.to_save_t(value : Array(Title))
value.map &.id
end
def self.to_return_t(value : Array(String))
value.map { |title_id| Library.default.title_hash[title_id].not_nil! }
end
def instance_size
instance_sizeof(SortedTitlesCacheEntry) + # sizeof itself
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
@value.size * (instance_sizeof(String) + sizeof(String)) +
@value.sum(&.bytesize) # elements in Array(String)
end
def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?)
titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s
user_context = opt && opt.method == SortMethod::Progress ? username : ""
sig = Digest::SHA1.hexdigest (titles_sig + user_context +
(opt ? opt.to_tuple.to_s : "nil"))
"#{sig}:sorted_titles"
end
end
class String
def instance_size
instance_sizeof(String) + bytesize
@ -101,14 +127,18 @@ struct Tuple(*T)
end
end
alias CacheableType = Array(Entry) | String | Tuple(String, Int32)
alias CacheableType = Array(Entry) | Array(Title) | String |
Tuple(String, Int32)
alias CacheEntryType = SortedEntriesCacheEntry |
SortedTitlesCacheEntry |
CacheEntry(String, String) |
CacheEntry(Tuple(String, Int32), Tuple(String, Int32))
def generate_cache_entry(key : String, value : CacheableType)
if value.is_a? Array(Entry)
SortedEntriesCacheEntry.new key, value
elsif value.is_a? Array(Title)
SortedTitlesCacheEntry.new key, value
else
CacheEntry(typeof(value), typeof(value)).new key, value
end

39
src/library/entry.cr
View File

@ -8,6 +8,9 @@ class Entry
size : String, pages : Int32, id : String, encoded_path : String,
encoded_title : String, mtime : Time, err_msg : String?
@[YAML::Field(ignore: true)]
@sort_title : String?
def initialize(@zip_path, @book)
storage = Storage.default
@encoded_path = URI.encode @zip_path
@ -56,6 +59,7 @@ class Entry
json.field {{str}}, @{{str.id}}
{% end %}
json.field "title_id", @book.id
json.field "sort_title", sort_title
json.field "pages" { json.number @pages }
unless slim
json.field "display_name", @book.display_name @title
@ -66,6 +70,35 @@ class Entry
end
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = @book.entry_sort_title_db id
if sort_title
@sort_title = sort_title
return sort_title
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_entry_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
@book.entry_sort_title_cache = nil
@book.remove_sorted_entries_cache [SortMethod::Auto, SortMethod::Title],
username
end
def sort_title_db
@book.entry_sort_title_db @id
end
def display_name
@book.display_name @title
end
@ -177,11 +210,7 @@ class Entry
@book.parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
[false, true].each do |ascend|
sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id,
username, @book.entries, SortOptions.new(SortMethod::Progress, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
@book.remove_sorted_caches [SortMethod::Progress], username
TitleInfo.new @book.dir do |info|
if info.progress[username]?.nil?

81
src/library/library.cr
View File

@ -1,9 +1,38 @@
class Library
struct ThumbnailContext
property current : Int32, total : Int32
def initialize
@current = 0
@total = 0
end
def progress
if total == 0
0
else
current / total
end
end
def reset
@current = 0
@total = 0
end
def increment
@current += 1
end
end
include YAML::Serializable
getter dir : String, title_ids : Array(String),
title_hash : Hash(String, Title)
@[YAML::Field(ignore: true)]
getter thumbnail_ctx = ThumbnailContext.new
use_default
def save_instance
@ -24,7 +53,23 @@ class Library
begin
Compress::Gzip::Reader.open path do |content|
@@default = Library.from_yaml content
loaded = Library.from_yaml content
# We will have to do a full restart in these cases. Otherwise having
# two instances of the library will cause some weirdness.
if loaded.dir != Config.current.library_path
Logger.fatal "Cached library dir #{loaded.dir} does not match " \
"current library dir #{Config.current.library_path}. " \
"Deleting cache"
delete_cache_and_exit path
end
if loaded.title_ids.size > 0 &&
Storage.default.count_titles == 0
Logger.fatal "The library cache is inconsistent with the DB. " \
"Deleting cache"
delete_cache_and_exit path
end
@@default = loaded
Logger.debug "Library cache loaded"
end
Library.default.register_jobs
rescue e
@ -39,9 +84,6 @@ class Library
@title_ids = [] of String
@title_hash = {} of String => Title
@entries_count = 0
@thumbnails_count = 0
register_jobs
end
@ -136,8 +178,12 @@ class Library
deleted_entry_ids: [] of String,
}
library_paths = (Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
@title_ids.select! do |title_id|
title = @title_hash[title_id]
next false unless library_paths.includes? title.dir
existence = title.examine examine_context
unless existence
examine_context["deleted_title_ids"].concat [title_id] +
@ -152,14 +198,12 @@ class Library
end
cache = examine_context["cached_contents_signature"]
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
library_paths
.select { |path| !(remained_title_dirs.includes? path) }
.select { |path| File.directory? path }
.map { |path| Title.new path, "", cache }
.select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort! { |a, b| a.title <=> b.title }
.sort! { |a, b| a.sort_title <=> b.sort_title }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id
@ -260,34 +304,29 @@ class Library
.shuffle!
end
def thumbnail_generation_progress
return 0 if @entries_count == 0
@thumbnails_count / @entries_count
end
def generate_thumbnails
if @thumbnails_count > 0
if thumbnail_ctx.current > 0
Logger.debug "Thumbnail generation in progress"
return
end
Logger.info "Starting thumbnail generation"
entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg
@entries_count = entries.size
@thumbnails_count = 0
thumbnail_ctx.total = entries.size
thumbnail_ctx.current = 0
# Report generation progress regularly
spawn do
loop do
unless @thumbnails_count == 0
unless thumbnail_ctx.current == 0
Logger.debug "Thumbnail generation progress: " \
"#{(thumbnail_generation_progress * 100).round 1}%"
"#{(thumbnail_ctx.progress * 100).round 1}%"
end
# Generation is completed. We reset the count to 0 to allow subsequent
# calls to the function, and break from the loop to stop the progress
# report fiber
if thumbnail_generation_progress.to_i == 1
@thumbnails_count = 0
if thumbnail_ctx.progress.to_i == 1
thumbnail_ctx.reset
break
end
sleep 10.seconds
@ -301,7 +340,7 @@ class Library
# and CPU
sleep 1.seconds
end
@thumbnails_count += 1
thumbnail_ctx.increment
end
Logger.info "Thumbnail generation finished"
end

136
src/library/title.cr
View File

@ -8,8 +8,13 @@ class Title
entries : Array(Entry), title : String, id : String,
encoded_title : String, mtime : Time, signature : UInt64,
entry_cover_url_cache : Hash(String, String)?
setter entry_cover_url_cache : Hash(String, String)?
setter entry_cover_url_cache : Hash(String, String)?,
entry_sort_title_cache : Hash(String, String | Nil)?
@[YAML::Field(ignore: true)]
@sort_title : String?
@[YAML::Field(ignore: true)]
@entry_sort_title_cache : Hash(String, String | Nil)?
@[YAML::Field(ignore: true)]
@entry_display_name_cache : Hash(String, String)?
@[YAML::Field(ignore: true)]
@ -66,7 +71,7 @@ class Title
end
sorter = ChapterSorter.new @entries.map &.title
@entries.sort! do |a, b|
sorter.compare a.title, b.title
sorter.compare a.sort_title, b.sort_title
end
end
@ -102,7 +107,11 @@ class Title
previous_titles_size = @title_ids.size
@title_ids.select! do |title_id|
title = Library.default.get_title! title_id
title = Library.default.get_title title_id
unless title # for if data consistency broken
context["deleted_title_ids"].concat [title_id]
next false
end
existence = title.examine context
unless existence
context["deleted_title_ids"].concat [title_id] +
@ -137,6 +146,18 @@ class Title
Library.default.title_hash[title.id] = title
@title_ids << title.id
is_titles_added = true
# We think they are removed, but they are here!
# Cancel reserved jobs
revival_title_ids = [title.id] + title.deep_titles.map &.id
context["deleted_title_ids"].select! do |deleted_title_id|
!(revival_title_ids.includes? deleted_title_id)
end
revival_entry_ids = title.deep_entries.map &.id
context["deleted_entry_ids"].select! do |deleted_entry_id|
!(revival_entry_ids.includes? deleted_entry_id)
end
next
end
if is_supported_file path
@ -145,6 +166,9 @@ class Title
if entry.pages > 0 || entry.err_msg
@entries << entry
is_entries_added = true
context["deleted_entry_ids"].select! do |deleted_entry_id|
entry.id != deleted_entry_id
end
end
end
end
@ -161,13 +185,18 @@ class Title
end
end
if is_entries_added || previous_entries_size != @entries.size
sorter = ChapterSorter.new @entries.map &.title
sorter = ChapterSorter.new @entries.map &.sort_title
@entries.sort! do |a, b|
sorter.compare a.title, b.title
sorter.compare a.sort_title, b.sort_title
end
end
true
if @title_ids.size > 0 || @entries.size > 0
true
else
context["deleted_title_ids"].concat [@id]
false
end
end
alias SortContext = NamedTuple(username: String, opt: SortOptions)
@ -180,6 +209,7 @@ class Title
json.field {{str}}, @{{str.id}}
{% end %}
json.field "signature" { json.number @signature }
json.field "sort_title", sort_title
unless slim
json.field "display_name", display_name
json.field "cover_url", cover_url
@ -226,6 +256,15 @@ class Title
@title_ids.map { |tid| Library.default.get_title! tid }
end
def sorted_titles(username, opt : SortOptions? = nil)
if opt.nil?
opt = SortOptions.from_info_json @dir, username
end
# Helper function from src/util/util.cr
sort_titles titles, opt.not_nil!, username
end
# Get all entries, including entries in nested titles
def deep_entries
return @entries if title_ids.empty?
@ -262,6 +301,48 @@ class Title
ary.join " and "
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = Storage.default.get_title_sort_title id
if sort_title
@sort_title = sort_title
return sort_title
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_title_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
if parents.size > 0
target = parents[-1].titles
else
target = Library.default.titles
end
remove_sorted_titles_cache target,
[SortMethod::Auto, SortMethod::Title], username
end
def sort_title_db
Storage.default.get_title_sort_title id
end
def entry_sort_title_db(entry_id)
unless @entry_sort_title_cache
@entry_sort_title_cache =
Storage.default.get_entries_sort_title @entries.map &.id
end
@entry_sort_title_cache.not_nil![entry_id]?
end
def tags
Storage.default.get_title_tags @id
end
@ -448,28 +529,30 @@ class Title
case opt.not_nil!.method
when .title?
ary = @entries.sort { |a, b| compare_numerically a.title, b.title }
ary = @entries.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
end
when .time_modified?
ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.title, b.title }
compare_numerically a.sort_title, b.sort_title }
when .time_added?
ary = @entries.sort { |a, b| (a.date_added <=> b.date_added).or \
compare_numerically a.title, b.title }
compare_numerically a.sort_title, b.sort_title }
when .progress?
percentage_ary = load_percentage_for_all_entries username, opt, true
ary = @entries.zip(percentage_ary)
.sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \
compare_numerically a_tp[0].title, b_tp[0].title }
compare_numerically a_tp[0].sort_title, b_tp[0].sort_title }
.map &.[0]
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
sorter = ChapterSorter.new @entries.map &.title
sorter = ChapterSorter.new @entries.map &.sort_title
ary = @entries.sort do |a, b|
sorter.compare(a.title, b.title).or \
compare_numerically a.title, b.title
sorter.compare(a.sort_title, b.sort_title).or \
compare_numerically a.sort_title, b.sort_title
end
end
@ -536,17 +619,32 @@ class Title
zip + titles.flat_map &.deep_entries_with_date_added
end
def remove_sorted_entries_cache(sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_entries_cache_key =
SortedEntriesCacheEntry.gen_key @id, username, @entries,
SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
end
end
def remove_sorted_caches(sort_methods : Array(SortMethod), username : String)
remove_sorted_entries_cache sort_methods, username
parents.each do |parent|
remove_sorted_titles_cache parent.titles, sort_methods, username
end
remove_sorted_titles_cache Library.default.titles, sort_methods, username
end
def bulk_progress(action, ids : Array(String), username)
LRUCache.invalidate "#{@id}:#{username}:progress_sum"
parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
[false, true].each do |ascend|
sorted_entries_cache_key =
SortedEntriesCacheEntry.gen_key @id, username, @entries,
SortOptions.new(SortMethod::Progress, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
remove_sorted_caches [SortMethod::Progress], username
selected_entries = ids
.map { |id|

2
src/mango.cr
View File

@ -7,7 +7,7 @@ require "option_parser"
require "clim"
require "tallboy"
MANGO_VERSION = "0.24.0"
MANGO_VERSION = "0.25.0"
# From http://www.network-science.de/ascii/
BANNER = %{

2
src/queue.cr
View File

@ -112,7 +112,7 @@ class Queue
use_default
def initialize(db_path : String? = nil)
@path = db_path || Config.current.mangadex["download_queue_db_path"].to_s
@path = db_path || Config.current.queue_db_path.to_s
dir = File.dirname @path
unless Dir.exists? dir
Logger.info "The queue DB directory #{dir} does not exist. " \

1
src/routes/admin.cr
View File

@ -66,7 +66,6 @@ struct AdminRouter
end
get "/admin/downloads" do |env|
mangadex_base_url = Config.current.mangadex["base_url"]
layout "download-manager"
end

59
src/routes/api.cr
View File

@ -23,7 +23,7 @@ struct APIRouter
# Authentication
All endpoints require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access.
All endpoints except `/api/login` require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access.
# Terminologies
@ -56,6 +56,29 @@ struct APIRouter
"error" => String?,
}
Koa.describe "Authenticates a user", <<-MD
After successful login, the cookie `mango-sessid-#{Config.current.port}` will contain a valid session ID that can be used for subsequent requests
MD
Koa.body schema: {
"username" => String,
"password" => String,
}
Koa.tag "users"
post "/api/login" do |env|
begin
username = env.params.json["username"].as String
password = env.params.json["password"].as String
token = Storage.default.verify_user(username, password).not_nil!
env.session.string "token", token
"Authenticated"
rescue e
Logger.error e
env.response.status_code = 403
e.message
end
end
Koa.describe "Returns a page in a manga entry"
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
@ -217,7 +240,7 @@ struct APIRouter
}
get "/api/admin/thumbnail_progress" do |env|
send_json env, {
"progress" => Library.default.thumbnail_generation_progress,
"progress" => Library.default.thumbnail_ctx.progress,
}.to_json
end
@ -348,6 +371,38 @@ struct APIRouter
end
end
Koa.describe "Sets the sort title of a title or an entry", <<-MD
When `eid` is provided, apply the sort title to the entry. Otherwise, apply the sort title to the title identified by `tid`.
MD
Koa.tags ["admin", "library"]
Koa.path "tid", desc: "Title ID"
Koa.query "eid", desc: "Entry ID", required: false
Koa.query "name", desc: "The new sort title"
Koa.response 200, schema: "result"
put "/api/admin/sort_title/:tid" do |env|
username = get_username env
begin
title = (Library.default.get_title env.params.url["tid"])
.not_nil!
name = env.params.query["name"]?
entry = env.params.query["eid"]?
if entry.nil?
title.set_sort_title name, username
else
eobj = title.get_entry entry
eobj.set_sort_title name, username unless eobj.nil?
end
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
ws "/api/admin/mangadex/queue" do |socket, env|
interval_raw = env.params.query["interval"]?
interval = (interval_raw.to_i? if interval_raw) || 5

6
src/routes/main.cr
View File

@ -61,9 +61,15 @@ struct MainRouter
sort_opt = SortOptions.from_info_json title.dir, username
get_and_save_sort_opt title.dir
sorted_titles = title.sorted_titles username, sort_opt
entries = title.sorted_entries username, sort_opt
percentage = title.load_percentage_for_all_entries username, sort_opt
title_percentage = title.titles.map &.load_percentage username
title_percentage_map = {} of String => Float64
title_percentage.each_with_index do |tp, i|
t = title.titles[i]
title_percentage_map[t.id] = tp
end
layout "title"
rescue e

75
src/storage.cr
View File

@ -342,6 +342,67 @@ class Storage
end
end
def get_title_sort_title(title_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from titles where id = (?)",
title_id, as: String | Nil
end
end
sort_title
end
def set_title_sort_title(title_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update titles set sort_title = (?) where id = (?)",
sort_title, title_id
end
end
end
def get_entry_sort_title(entry_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from ids where id = (?)",
entry_id, as: String | Nil
end
end
sort_title
end
def get_entries_sort_title(ids : Array(String))
results = Hash(String, String | Nil).new
MainFiber.run do
get_db do |db|
db.query "select id, sort_title from ids where id in " \
"(#{ids.join "," { |id| "'#{id}'" }})" do |rs|
rs.each do
id = rs.read String
sort_title = rs.read String | Nil
results[id] = sort_title
end
end
end
end
results
end
def set_entry_sort_title(entry_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update ids set sort_title = (?) where id = (?)",
sort_title, entry_id
end
end
end
def save_thumbnail(id : String, img : Image)
MainFiber.run do
get_db do |db|
@ -558,6 +619,20 @@ class Storage
{token, expires}
end
def count_titles : Int32
count = 0
MainFiber.run do
get_db do |db|
db.query "select count(*) from titles" do |rs|
rs.each do
count = rs.read Int32
end
end
end
end
count
end
def close
MainFiber.run do
unless @db.nil?

40
src/util/util.cr
View File

@ -87,30 +87,49 @@ def env_is_true?(key : String) : Bool
end
def sort_titles(titles : Array(Title), opt : SortOptions, username : String)
ary = titles
cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt
cached_titles = LRUCache.get cache_key
return cached_titles if cached_titles.is_a? Array(Title)
case opt.method
when .time_modified?
ary.sort! { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.title, b.title }
ary = titles.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.sort_title, b.sort_title }
when .progress?
ary.sort! do |a, b|
ary = titles.sort do |a, b|
(a.load_percentage(username) <=> b.load_percentage(username)).or \
compare_numerically a.title, b.title
compare_numerically a.sort_title, b.sort_title
end
when .title?
ary = titles.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
end
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
ary.sort! { |a, b| compare_numerically a.title, b.title }
ary = titles.sort { |a, b| compare_numerically a.sort_title, b.sort_title }
end
ary.reverse! unless opt.not_nil!.ascend
LRUCache.set generate_cache_entry cache_key, ary
ary
end
def remove_sorted_titles_cache(titles : Array(Title),
sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username,
titles, SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_titles_cache_key
end
end
end
class String
# Returns the similarity (in [0, 1]) of two paths.
# For the two paths, separate them into arrays of components, count the
@ -144,3 +163,12 @@ def sanitize_filename(str : String) : String
.gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "")
sanitized.size > 0 ? sanitized : random_str
end
def delete_cache_and_exit(path : String)
File.delete path
Logger.fatal "Invalid library cache deleted. Mango needs to " \
"perform a full reset to recover from this. " \
"Pleae restart Mango. This is NOT a bug."
Logger.fatal "Exiting"
exit 1
end

4
src/views/components/card.html.ecr
View File

@ -61,7 +61,9 @@
<% if page == "home" && item.is_a? Entry %>
<%= "uk-margin-remove-bottom" %>
<% end %>
" data-title="<%= HTML.escape(item.display_name) %>"><%= HTML.escape(item.display_name) %>
" data-title="<%= HTML.escape(item.display_name) %>"
data-file-title="<%= HTML.escape(item.title || "") %>"
data-sort-title="<%= HTML.escape(item.sort_title_db || "") %>"><%= HTML.escape(item.display_name) %>
</h3>
<% if page == "home" && item.is_a? Entry %>
<a class="uk-card-title break-word uk-margin-remove-top uk-text-meta uk-display-inline-block no-modal" data-title="<%= HTML.escape(item.book.display_name) %>" href="<%= base_url %>book/<%= item.book.id %>"><%= HTML.escape(item.book.display_name) %></a>

6
src/views/download-manager.html.ecr
View File

@ -24,16 +24,10 @@
<template x-if="job.plugin_id">
<td x-text="job.title"></td>
</template>
<template x-if="!job.plugin_id">
<td><a :href="`<%= mangadex_base_url %>/chapter/${job.id}`" x-text="job.title"></td>
</template>
<template x-if="job.plugin_id">
<td x-text="job.manga_title"></td>
</template>
<template x-if="!job.plugin_id">
<td><a :href="`<%= mangadex_base_url %>/manga/${job.manga_id}`" x-text="job.manga_title"></td>
</template>
<td x-text="`${job.success_count}/${job.pages}`"></td>
<td x-text="`${moment(job.time).fromNow()}`"></td>

6
src/views/layout.html.ecr
View File

@ -32,10 +32,10 @@
</div>
<div class="uk-position-top">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div class="uk-navbar-left uk-hidden@s">
<div class="uk-navbar-left uk-hidden@m">
<div class="uk-navbar-toggle" uk-navbar-toggle-icon="uk-navbar-toggle-icon" uk-toggle="target: #mobile-nav"></div>
</div>
<div class="uk-navbar-left uk-visible@s">
<div class="uk-navbar-left uk-visible@m">
<a class="uk-navbar-item uk-logo" href="<%= base_url %>"><img src="<%= base_url %>img/icon.png" style="width:90px;height:90px;"></a>
<ul class="uk-navbar-nav">
<li><a href="<%= base_url %>">Home</a></li>
@ -57,7 +57,7 @@
<% end %>
</ul>
</div>
<div class="uk-navbar-right uk-visible@s">
<div class="uk-navbar-right uk-visible@m">
<ul class="uk-navbar-nav">
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>

1
src/views/library.html.ecr
View File

@ -10,6 +10,7 @@
<div class="uk-margin-bottom uk-width-1-4@s">
<% hash = {
"auto" => "Auto",
"title" => "Name",
"time_modified" => "Date Modified",
"progress" => "Progress"
} %>

10
src/views/reader.html.ecr
View File

@ -55,8 +55,8 @@
object-fit: contain;
`" />
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true)"></div>
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false ^ enableRightToLeft)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true ^ enableRightToLeft)"></div>
</div>
</div>
@ -114,6 +114,12 @@
</div>
</div>
<div class="uk-margin uk-form-horizontal" x-show="mode !== 'continuous'">
<label class="uk-form-label" for="enable-right-to-left">Right to Left</label>
<div class="uk-form-controls">
<input id="enable-right-to-left" class="uk-checkbox" type="checkbox" x-model="enableRightToLeft" @change="enableRightToLeftChanged()">
</div>
</div>
<hr class="uk-divider-icon">
<div class="uk-margin">

14
src/views/title.html.ecr
View File

@ -18,7 +18,8 @@
</div>
</div>
</div>
<h2 class=uk-title><span><%= title.display_name %></span>
<h2 class=uk-title data-file-title="<%= HTML.escape(title.title) %>" data-sort-title="<%= HTML.escape(title.sort_title_db || "") %>">
<span><%= title.display_name %></span>
&nbsp;
<% if is_admin %>
<a onclick="edit()" class="uk-icon-button" uk-icon="icon:pencil"></a>
@ -59,8 +60,8 @@
</div>
<div class="uk-child-width-1-4@m uk-child-width-1-2" uk-grid>
<% title.titles.each_with_index do |item, i| %>
<% progress = title_percentage[i] %>
<% sorted_titles.each do |item| %>
<% progress = title_percentage_map[item.id] %>
<%= render_component "card" %>
<% end %>
</div>
@ -89,6 +90,13 @@
<input class="uk-input" type="text" name="display-name" id="display-name-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label" for="sort-title">Sort Title</label>
<div class="uk-inline">
<a class="uk-form-icon uk-form-icon-flip" uk-icon="icon:check"></a>
<input class="uk-input" type="text" name="sort-title" id="sort-title-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label">Cover Image</label>
<div class="uk-grid">