Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2025-08-02 10:55:30 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
Alex Ling 2020-10-25 07:57:26 +00:00
commit bf68e32ac8
17 changed files with 348 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -12,6 +12,7 @@ Mango is a self-hosted manga server and reader. Its features include
- Supported formats: `.cbz`, `.zip`, `.cbr` and `.rar`
- Supports nested folders in library
- Automatically stores reading progress
- Thumbnail generation
- Built-in [MangaDex](https://mangadex.org/) downloader
- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
- The web reader is responsive and works well on mobile, so there is no need for a mobile app
@ -51,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
Mango - Manga Server and Web Reader. Version 0.14.0
Mango - Manga Server and Web Reader. Version 0.15.0
Usage:
@ -80,6 +81,8 @@ session_secret: mango-session-secret
library_path: ~/mango/library
db_path: ~/mango/mango.db
scan_interval_minutes: 5
thumbnail_generation_interval_hours: 24
db_optimization_interval_hours: 24
log_level: info
upload_path: ~/mango/uploads
plugin_path: ~/mango/plugins
@ -89,12 +92,12 @@ mangadex:
api_url: https://mangadex.org/api
download_wait_seconds: 5
download_retries: 4
download_queue_db_path: ~/mango/queue.db
download_queue_db_path: /home/alex_ling/mango/queue.db
chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
manga_rename_rule: '{title}'
```
- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan
- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
- `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
### Library Structure

114
public/js/admin.js
View File

@ -1,40 +1,90 @@
let scanning = false;
const scan = () => {
scanning = true;
$('#scan-status > div').removeAttr('hidden');
$('#scan-status > span').attr('hidden', '');
const color = $('#scan').css('color');
$('#scan').css('color', 'gray');
$.post(base_url + 'api/admin/scan', (data) => {
const ms = data.milliseconds;
const titles = data.titles;
$('#scan-status > span').text('Scanned ' + titles + ' titles in ' + ms + 'ms');
$('#scan-status > span').removeAttr('hidden');
$('#scan').css('color', color);
$('#scan-status > div').attr('hidden', '');
scanning = false;
});
}
String.prototype.capitalize = function() {
return this.charAt(0).toUpperCase() + this.slice(1);
}
$(() => {
$('li').click((e) => {
const url = $(e.currentTarget).attr('data-url');
if (url) {
$(location).attr('href', url);
}
});
const setting = loadThemeSetting();
$('#theme-select').val(setting.capitalize());
$('#theme-select').val(capitalize(setting));
$('#theme-select').change((e) => {
const newSetting = $(e.currentTarget).val().toLowerCase();
saveThemeSetting(newSetting);
setTheme();
});
getProgress();
setInterval(getProgress, 5000);
});
/**
* Capitalize String
*
* @function capitalize
* @param {string} str - The string to be capitalized
* @return {string} The capitalized string
*/
const capitalize = (str) => {
return str.charAt(0).toUpperCase() + str.slice(1);
};
/**
* Set an alpine.js property
*
* @function setProp
* @param {string} key - Key of the data property
* @param {*} prop - The data property
*/
const setProp = (key, prop) => {
$('#root').get(0).__x.$data[key] = prop;
};
/**
* Get an alpine.js property
*
* @function getProp
* @param {string} key - Key of the data property
* @return {*} The data property
*/
const getProp = (key) => {
return $('#root').get(0).__x.$data[key];
};
/**
* Get the thumbnail generation progress from the API
*
* @function getProgress
*/
const getProgress = () => {
$.get(`${base_url}api/admin/thumbnail_progress`)
.then(data => {
setProp('progress', data.progress);
const generating = data.progress > 0
setProp('generating', generating);
});
};
/**
* Trigger the thumbnail generation
*
* @function generateThumbnails
*/
const generateThumbnails = () => {
setProp('generating', true);
setProp('progress', 0.0);
$.post(`${base_url}api/admin/generate_thumbnails`)
.then(getProgress);
};
/**
* Trigger the scan
*
* @function scan
*/
const scan = () => {
setProp('scanning', true);
setProp('scanMs', -1);
setProp('scanTitles', 0);
$.post(`${base_url}api/admin/scan`)
.then(data => {
setProp('scanMs', data.milliseconds);
setProp('scanTitles', data.titles);
})
.always(() => {
setProp('scanning', false);
});
}

35
public/js/dots.js
View File

@ -1,17 +1,26 @@
const truncate = () => {
$('.uk-card-title').each((i, e) => {
$(e).dotdotdot({
truncate: 'letter',
watch: true,
callback: (truncated) => {
if (truncated) {
$(e).attr('uk-tooltip', $(e).attr('data-title'));
} else {
$(e).removeAttr('uk-tooltip');
}
/**
* Truncate a .uk-card-title element
*
* @function truncate
* @param {object} e - The title element to truncate
*/
const truncate = (e) => {
$(e).dotdotdot({
truncate: 'letter',
watch: true,
callback: (truncated) => {
if (truncated) {
$(e).attr('uk-tooltip', $(e).attr('data-title'));
} else {
$(e).removeAttr('uk-tooltip');
}
});
}
});
};
truncate();
$('.uk-card-title').each((i, e) => {
// Truncate the title when it first enters the view
$(e).one('inview', () => {
truncate(e);
});
});

2
shard.lock
View File

@ -34,7 +34,7 @@ shards:
image_size:
github: hkalexling/image_size.cr
version: 0.2.0
version: 0.4.0
kemal:
github: kemalcr/kemal

2
shard.yml
View File

@ -1,5 +1,5 @@
name: mango
version: 0.14.0
version: 0.15.0
authors:
- Alex Ling <hkalexling@gmail.com>

5
src/config.cr
View File

@ -11,8 +11,9 @@ class Config
property library_path : String = File.expand_path "~/mango/library",
home: true
property db_path : String = File.expand_path "~/mango/mango.db", home: true
@[YAML::Field(key: "scan_interval_minutes")]
property scan_interval : Int32 = 5
property scan_interval_minutes : Int32 = 5
property thumbnail_generation_interval_hours : Int32 = 24
property db_optimization_interval_hours : Int32 = 24
property log_level : String = "info"
property upload_path : String = File.expand_path "~/mango/uploads",
home: true

27
src/library/entry.cr
View File

@ -69,7 +69,7 @@ class Entry
def cover_url
return "#{Config.current.base_url}img/icon.png" if @err_msg
url = "#{Config.current.base_url}api/page/#{@book.id}/#{@id}/1"
url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}"
TitleInfo.new @book.dir do |info|
info_url = info.entry_cover_url[@title]?
unless info_url.nil? || info_url.empty?
@ -207,4 +207,29 @@ class Entry
def started?(username)
load_progress(username) > 0
end
def generate_thumbnail : Image?
return if @err_msg
img = read_page(1).not_nil!
begin
size = ImageSize.get img.data
if size.height > size.width
thumbnail = ImageSize.resize img.data, width: 200
else
thumbnail = ImageSize.resize img.data, height: 300
end
img.data = thumbnail
Storage.default.save_thumbnail @id, img
rescue e
Logger.warn "Failed to generate thumbnail for entry " \
"#{@book.title}/#{@title}. #{e}"
end
img
end
def get_thumbnail : Image?
Storage.default.get_thumbnail @id
end
end

94
src/library/library.cr
View File

@ -1,5 +1,5 @@
class Library
property dir : String, title_ids : Array(String), scan_interval : Int32,
property dir : String, title_ids : Array(String),
title_hash : Hash(String, Title)
use_default
@ -8,20 +8,48 @@ class Library
register_mime_types
@dir = Config.current.library_path
@scan_interval = Config.current.scan_interval
# explicitly initialize @titles to bypass the compiler check. it will
# be filled with actual Titles in the `scan` call below
@title_ids = [] of String
@title_hash = {} of String => Title
return scan if @scan_interval < 1
spawn do
loop do
start = Time.local
scan
ms = (Time.local - start).total_milliseconds
Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep @scan_interval * 60
@entries_count = 0
@thumbnails_count = 0
scan_interval = Config.current.scan_interval_minutes
if scan_interval < 1
scan
else
spawn do
loop do
start = Time.local
scan
ms = (Time.local - start).total_milliseconds
Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep scan_interval.minutes
end
end
end
thumbnail_interval = Config.current.thumbnail_generation_interval_hours
unless thumbnail_interval < 1
spawn do
loop do
# Wait for scan to complete (in most cases)
sleep 1.minutes
generate_thumbnails
sleep thumbnail_interval.hours
end
end
end
db_interval = Config.current.db_optimization_interval_hours
unless db_interval < 1
spawn do
loop do
Storage.default.optimize
sleep db_interval.hours
end
end
end
end
@ -194,4 +222,50 @@ class Library
.sample(ENTRIES_IN_HOME_SECTIONS)
.shuffle
end
def thumbnail_generation_progress
return 0 if @entries_count == 0
@thumbnails_count / @entries_count
end
def generate_thumbnails
if @thumbnails_count > 0
Logger.debug "Thumbnail generation in progress"
return
end
Logger.info "Starting thumbnail generation"
entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg
@entries_count = entries.size
@thumbnails_count = 0
# Report generation progress regularly
spawn do
loop do
unless @thumbnails_count == 0
Logger.debug "Thumbnail generation progress: " \
"#{(thumbnail_generation_progress * 100).round 1}%"
end
# Generation is completed. We reset the count to 0 to allow subsequent
# calls to the function, and break from the loop to stop the progress
# report fiber
if thumbnail_generation_progress.to_i == 1
@thumbnails_count = 0
break
end
sleep 10.seconds
end
end
entries.each do |e|
unless e.get_thumbnail
e.generate_thumbnail
# Sleep after each generation to minimize the impact on disk IO
# and CPU
sleep 0.5.seconds
end
@thumbnails_count += 1
end
Logger.info "Thumbnail generation finished"
end
end

10
src/library/types.cr
View File

@ -57,6 +57,16 @@ struct Image
def initialize(@data, @mime, @filename, @size)
end
def self.from_db(res : DB::ResultSet)
img = Image.allocate
res.read String
img.data = res.read Bytes
img.filename = res.read String
img.mime = res.read String
img.size = res.read Int32
img
end
end
class TitleInfo

2
src/mango.cr
View File

@ -7,7 +7,7 @@ require "option_parser"
require "clim"
require "./plugin/*"
MANGO_VERSION = "0.14.0"
MANGO_VERSION = "0.15.0"
# From http://www.network-science.de/ascii/
BANNER = %{

34
src/routes/api.cr
View File

@ -26,6 +26,28 @@ class APIRouter < Router
end
end
get "/api/cover/:tid/:eid" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
title = @context.library.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
entry = title.get_entry eid
raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil?
img = entry.get_thumbnail || entry.read_page 1
raise "Failed to get cover of `#{title.title}/#{entry.title}`" \
if img.nil?
send_img env, img
rescue e
@context.error e
env.response.status_code = 500
e.message
end
end
get "/api/book/:tid" do |env|
begin
tid = env.params.url["tid"]
@ -54,6 +76,18 @@ class APIRouter < Router
}.to_json
end
get "/api/admin/thumbnail_progress" do |env|
send_json env, {
"progress" => Library.default.thumbnail_generation_progress,
}.to_json
end
post "/api/admin/generate_thumbnails" do |env|
spawn do
Library.default.generate_thumbnails
end
end
post "/api/admin/user/delete/:username" do |env|
begin
username = env.params.url["username"]

60
src/storage.cr
View File

@ -35,9 +35,11 @@ class Storage
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
begin
# We create the `ids` table first. even if the uses has an
# early version installed and has the `user` table only,
# we will still be able to create `ids`
db.exec "create table thumbnails " \
"(id text, data blob, filename text, " \
"mime text, size integer)"
db.exec "create unique index tn_index on thumbnails (id)"
db.exec "create table ids" \
"(path text, id text, is_title integer)"
db.exec "create unique index path_idx on ids (path)"
@ -243,6 +245,58 @@ class Storage
end
end
def save_thumbnail(id : String, img : Image)
MainFiber.run do
get_db do |db|
db.exec "insert into thumbnails values (?, ?, ?, ?, ?)", id, img.data,
img.filename, img.mime, img.size
end
end
end
def get_thumbnail(id : String) : Image?
img = nil
MainFiber.run do
get_db do |db|
db.query_one? "select * from thumbnails where id = (?)", id do |res|
img = Image.from_db res
end
end
end
img
end
def optimize
MainFiber.run do
Logger.info "Starting DB optimization"
get_db do |db|
trash_ids = [] of String
db.query "select path, id from ids" do |rs|
rs.each do
path = rs.read String
trash_ids << rs.read String unless File.exists? path
end
end
# Delete dangling IDs
db.exec "delete from ids where id in " \
"(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
Logger.debug "#{trash_ids.size} dangling IDs deleted" \
if trash_ids.size > 0
# Delete dangling thumbnails
trash_thumbnails_count = db.query_one "select count(*) from " \
"thumbnails where id not in " \
"(select id from ids)", as: Int32
if trash_thumbnails_count > 0
db.exec "delete from thumbnails where id not in (select id from ids)"
Logger.info "#{trash_thumbnails_count} dangling thumbnails deleted"
end
end
Logger.debug "DB optimization finished"
end
end
def close
MainFiber.run do
unless @db.nil?

22
src/views/admin.html.ecr
View File

@ -1,11 +1,17 @@
<ul class="uk-list uk-list-large uk-list-divider">
<li data-url="<%= base_url %>admin/user">User Managerment</li>
<li onclick="if(!scanning){scan()}">
<span id="scan">Scan Library Files</span>
<span id="scan-status" class="uk-align-right">
<div uk-spinner hidden></div>
<span hidden></span>
</span>
<ul class="uk-list uk-list-large uk-list-divider" id="root" x-data="{progress : 1.0, generating : false, scanTitles: 0, scanMs: -1, scanning : false}">
<li @click="location.href = '<%= base_url %>admin/user'">User Managerment</li>
<li :class="{'nopointer' : scanning}" @click="scan()">
<span :style="`${scanning ? 'color:grey' : ''}`">Scan Library Files</span>
<div class="uk-align-right">
<div uk-spinner x-show="scanning"></div>
<span x-show="!scanning && scanMs > 0" x-text="`Scan ${scanTitles} titles in ${scanMs}ms`"></span>
</div>
</li>
<li :class="{'nopointer' : generating}" @click="generateThumbnails()">
<span :style="`${generating ? 'color:grey' : ''}`">Generate Thumbnails</span>
<div class="uk-align-right">
<span x-show="generating && progress > 0" x-text="`${(progress * 100).toFixed(2)}%`"></span>
</div>
</li>
<li class="nopointer">
<span>Theme</span>

3
src/views/components/dots-scripts.html.ecr Normal file
View File

@ -0,0 +1,3 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/protonet-jquery.inview/1.1.2/jquery.inview.min.js"></script>
<script src="<%= base_url %>js/dots.js"></script>

5
src/views/home.html.ecr
View File

@ -11,7 +11,7 @@
<dd>Update <code>config.yml</code> located at: <code><%= Config.current.path %></code></dd>
<dt style="font-weight: 500;">Can't see your files yet?</dt>
<dd>
You must wait <%= Config.current.scan_interval %> minutes for the library scan to complete
You must wait <%= Config.current.scan_interval_minutes %> minutes for the library scan to complete
<% if is_admin %>
, or manually re-scan from <a href="<%= base_url %>admin">Admin</a>
<% end %>.
@ -77,8 +77,7 @@
<%- end -%>
<% content_for "script" do %>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
<script src="<%= base_url %>js/dots.js"></script>
<%= render_component "dots-scripts" %>
<script src="<%= base_url %>js/alert.js"></script>
<script src="<%= base_url %>js/title.js"></script>
<% end %>

3
src/views/library.html.ecr
View File

@ -24,8 +24,7 @@
</div>
<% content_for "script" do %>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
<script src="<%= base_url %>js/dots.js"></script>
<%= render_component "dots-scripts" %>
<script src="<%= base_url %>js/search.js"></script>
<script src="<%= base_url %>js/sort-items.js"></script>
<% end %>

3
src/views/title.html.ecr
View File

@ -117,8 +117,7 @@
</div>
<% content_for "script" do %>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
<script src="<%= base_url %>js/dots.js"></script>
<%= render_component "dots-scripts" %>
<script src="<%= base_url %>js/alert.js"></script>
<script src="<%= base_url %>js/title.js"></script>
<script src="<%= base_url %>js/search.js"></script>