Merge branch 'dev'

This commit is contained in:
Alex Ling 2020-04-22 14:36:36 +00:00
commit a354d811d9
41 changed files with 2407 additions and 1882 deletions

9
.ameba.yml Normal file
View File

@ -0,0 +1,9 @@
Lint/UselessAssign:
Excluded:
- src/routes/*
- src/server.cr
Lint/UnusedArgument:
Excluded:
- src/routes/*
Metrics/CyclomaticComplexity:
Enabled: false

View File

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
container:
image: crystallang/crystal:0.32.1-alpine
image: crystallang/crystal:0.34.0-alpine
steps:
- uses: actions/checkout@v2
@ -20,5 +20,7 @@ jobs:
run: apk add --no-cache yarn yaml sqlite-static
- name: Build
run: make
- name: Linter
run: make check
- name: Run tests
run: make test

View File

@ -1,4 +1,4 @@
FROM crystallang/crystal:0.32.1-alpine AS builder
FROM crystallang/crystal:0.34.0-alpine AS builder
WORKDIR /Mango

View File

@ -1,4 +1,4 @@
PREFIX=/usr/local
PREFIX ?= /usr/local
INSTALL_DIR=$(PREFIX)/bin
all: uglify | build
@ -22,6 +22,10 @@ run:
test:
crystal spec
check:
crystal tool format --check
./bin/ameba
install:
cp mango $(INSTALL_DIR)/mango

View File

@ -50,7 +50,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
Mango e-manga server/reader. Version 0.2.0
Mango e-manga server/reader. Version 0.3.0
-v, --version Show version
-h, --help Show help
@ -64,8 +64,9 @@ The default config file location is `~/.config/mango/config.yml`. It might be di
```yaml
---
port: 9000
library_path: ~/mango/library
db_path: ~/mango/mango.db
library_path: /home/alex_ling/mango/library
upload_path: /home/alex_ling/mango/uploads
db_path: /home/alex_ling/mango/mango.db
scan_interval_minutes: 5
log_level: info
mangadex:
@ -73,7 +74,7 @@ mangadex:
api_url: https://mangadex.org/api
download_wait_seconds: 5
download_retries: 4
download_queue_db_path: ~/mango/queue.db
download_queue_db_path: /home/alex_ling/mango/queue.db
```
- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan

View File

@ -56,3 +56,18 @@
td > .uk-dropdown {
white-space: pre-line;
}
#edit-modal .uk-grid > div {
height: 300px;
}
#edit-modal #cover {
height: 100%;
width: 100%;
object-fit: cover;
}
#edit-modal #cover-upload {
height: 100%;
box-sizing: border-box;
}
#edit-modal .uk-modal-body .uk-inline {
width: 100%;
}

View File

@ -15,7 +15,10 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi
if (percentage === 100) {
$('#read-btn').attr('hidden', '');
}
$('#modal-title').text(entry);
$('#modal-title').find('span').text(entry);
$('#modal-title').next().attr('data-id', titleID);
$('#modal-title').next().attr('data-entry-id', entryID);
$('#modal-title').next().find('.title-rename-field').val(entry);
$('#path-text').text(zipPath);
$('#pages-text').text(pages + ' pages');
@ -29,11 +32,18 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi
updateProgress(titleID, entryID, 0);
});
$('.uk-modal-title.break-word > a').attr('onclick', `edit("${entryID}")`);
UIkit.modal($('#modal')).show();
styleModal();
}
function updateProgress(titleID, entryID, page) {
$.post('/api/progress/' + titleID + '/' + entryID + '/' + page, function(data) {
const updateProgress = (tid, eid, page) => {
let url = `/api/progress/${tid}/${page}`
const query = $.param({entry: eid});
if (eid)
url += `?${query}`;
$.post(url, (data) => {
if (data.success) {
location.reload();
}
@ -42,4 +52,108 @@ function updateProgress(titleID, entryID, page) {
alert('danger', error);
}
});
}
};
const renameSubmit = (name, eid) => {
const upload = $('.upload-field');
const titleId = upload.attr('data-title-id');
console.log(name);
if (name.length === 0) {
alert('danger', 'The display name should not be empty');
return;
}
const query = $.param({ entry: eid });
let url = `/api/admin/display_name/${titleId}/${name}`;
if (eid)
url += `?${query}`;
$.ajax({
type: 'POST',
url: url,
contentType: "application/json",
dataType: 'json'
})
.done(data => {
if (data.error) {
alert('danger', `Failed to update display name. Error: ${data.error}`);
return;
}
location.reload();
})
.fail((jqXHR, status) => {
alert('danger', `Failed to update display name. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
});
};
const edit = (eid) => {
const cover = $('#edit-modal #cover');
let url = cover.attr('data-title-cover');
let displayName = $('h2.uk-title > span').text();
if (eid) {
const item = $(`#${eid}`);
url = item.find('img').attr('data-src');
displayName = item.find('.uk-card-title').attr('data-title');
$('#title-progress-control').attr('hidden', '');
}
else {
$('#title-progress-control').removeAttr('hidden');
}
cover.attr('data-src', url);
const displayNameField = $('#display-name-field');
displayNameField.attr('value', displayName);
displayNameField.keyup(event => {
if (event.keyCode === 13) {
renameSubmit(displayNameField.val(), eid);
}
});
displayNameField.siblings('a.uk-form-icon').click(() => {
renameSubmit(displayNameField.val(), eid);
});
setupUpload(eid);
UIkit.modal($('#edit-modal')).show();
styleModal();
};
const setupUpload = (eid) => {
const upload = $('.upload-field');
const bar = $('#upload-progress').get(0);
const titleId = upload.attr('data-title-id');
const queryObj = {title: titleId};
if (eid)
queryObj['entry'] = eid;
const query = $.param(queryObj);
const url = `/api/admin/upload/cover?${query}`;
console.log(url);
UIkit.upload('.upload-field', {
url: url,
name: 'file',
error: (e) => {
alert('danger', `Failed to upload cover image: ${e.toString()}`);
},
loadStart: (e) => {
$(bar).removeAttr('hidden');
bar.max = e.total;
bar.value = e.loaded;
},
progress: (e) => {
bar.max = e.total;
bar.value = e.loaded;
},
loadEnd: (e) => {
bar.max = e.total;
bar.value = e.loaded;
},
completeAll: () => {
$(bar).attr('hidden', '');
location.reload();
}
});
};

View File

@ -1,5 +1,9 @@
version: 1.0
shards:
ameba:
github: crystal-ameba/ameba
version: 0.12.0
baked_file_system:
github: schovi/baked_file_system
version: 0.9.8

View File

@ -1,5 +1,5 @@
name: mango
version: 0.2.5
version: 0.3.0
authors:
- Alex Ling <hkalexling@gmail.com>
@ -8,7 +8,7 @@ targets:
mango:
main: src/mango.cr
crystal: 0.32.1
crystal: 0.34.0
license: MIT
@ -19,3 +19,7 @@ dependencies:
github: crystal-lang/crystal-sqlite3
baked_file_system:
github: schovi/baked_file_system
development_dependencies:
ameba:
github: crystal-ameba/ameba

View File

@ -1,14 +1,14 @@
require "./spec_helper"
describe Config do
it "creates config if it does not exist" do
with_default_config do |config, logger, path|
File.exists?(path).should be_true
end
end
it "creates config if it does not exist" do
with_default_config do |_, _, path|
File.exists?(path).should be_true
end
end
it "correctly loads config" do
config = Config.load "spec/asset/test-config.yml"
config.port.should eq 3000
end
it "correctly loads config" do
config = Config.load "spec/asset/test-config.yml"
config.port.should eq 3000
end
end

View File

@ -3,103 +3,102 @@ require "./spec_helper"
include MangaDex
describe Queue do
it "creates DB at given path" do
with_queue do |queue, path|
File.exists?(path).should be_true
end
end
it "creates DB at given path" do
with_queue do |_, path|
File.exists?(path).should be_true
end
end
it "pops nil when empty" do
with_queue do |queue|
queue.pop.should be_nil
end
end
it "pops nil when empty" do
with_queue do |queue|
queue.pop.should be_nil
end
end
it "inserts multiple jobs" do
with_queue do |queue|
j1 = Job.new "1", "1", "title", "manga_title", JobStatus::Error,
Time.utc
j2 = Job.new "2", "2", "title", "manga_title", JobStatus::Completed,
Time.utc
j3 = Job.new "3", "3", "title", "manga_title", JobStatus::Pending,
Time.utc
j4 = Job.new "4", "4", "title", "manga_title",
JobStatus::Downloading, Time.utc
count = queue.push [j1, j2, j3, j4]
count.should eq 4
end
end
it "inserts multiple jobs" do
with_queue do |queue|
j1 = Job.new "1", "1", "title", "manga_title", JobStatus::Error,
Time.utc
j2 = Job.new "2", "2", "title", "manga_title", JobStatus::Completed,
Time.utc
j3 = Job.new "3", "3", "title", "manga_title", JobStatus::Pending,
Time.utc
j4 = Job.new "4", "4", "title", "manga_title",
JobStatus::Downloading, Time.utc
count = queue.push [j1, j2, j3, j4]
count.should eq 4
end
end
it "pops pending job" do
with_queue do |queue|
job = queue.pop
job.should_not be_nil
job.not_nil!.id.should eq "3"
end
end
it "pops pending job" do
with_queue do |queue|
job = queue.pop
job.should_not be_nil
job.not_nil!.id.should eq "3"
end
end
it "correctly counts jobs" do
with_queue do |queue|
queue.count.should eq 4
end
end
it "correctly counts jobs" do
with_queue do |queue|
queue.count.should eq 4
end
end
it "deletes job" do
with_queue do |queue|
queue.delete "4"
queue.count.should eq 3
end
end
it "deletes job" do
with_queue do |queue|
queue.delete "4"
queue.count.should eq 3
end
end
it "sets status" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.set_status JobStatus::Downloading, job
job = queue.pop
job.should_not be_nil
job.not_nil!.status.should eq JobStatus::Downloading
end
end
it "sets status" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.set_status JobStatus::Downloading, job
job = queue.pop
job.should_not be_nil
job.not_nil!.status.should eq JobStatus::Downloading
end
end
it "sets number of pages" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.set_pages 100, job
job = queue.pop
job.should_not be_nil
job.not_nil!.pages.should eq 100
end
end
it "sets number of pages" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.set_pages 100, job
job = queue.pop
job.should_not be_nil
job.not_nil!.pages.should eq 100
end
end
it "adds fail/success counts" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.add_success job
queue.add_success job
queue.add_fail job
job = queue.pop
job.should_not be_nil
job.not_nil!.success_count.should eq 2
job.not_nil!.fail_count.should eq 1
end
end
it "adds fail/success counts" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.add_success job
queue.add_success job
queue.add_fail job
job = queue.pop
job.should_not be_nil
job.not_nil!.success_count.should eq 2
job.not_nil!.fail_count.should eq 1
end
end
it "appends status message" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.add_message "hello", job
queue.add_message "world", job
job = queue.pop
job.should_not be_nil
job.not_nil!.status_message.should eq "\nhello\nworld"
end
end
it "appends status message" do
with_queue do |queue|
job = queue.pop.not_nil!
queue.add_message "hello", job
queue.add_message "world", job
job = queue.pop
job.should_not be_nil
job.not_nil!.status_message.should eq "\nhello\nworld"
end
end
it "cleans up" do
with_queue do
true
end
State.reset
end
it "cleans up" do
with_queue do
true
end
State.reset
end
end

View File

@ -3,63 +3,63 @@ require "../src/context"
require "../src/server"
class State
@@hash = {} of String => String
@@hash = {} of String => String
def self.get(key)
@@hash[key]?
end
def self.get(key)
@@hash[key]?
end
def self.get!(key)
@@hash[key]
end
def self.get!(key)
@@hash[key]
end
def self.set(key, value)
return if value.nil?
@@hash[key] = value
end
def self.set(key, value)
return if value.nil?
@@hash[key] = value
end
def self.reset
@@hash.clear
end
def self.reset
@@hash.clear
end
end
def get_tempfile(name)
path = State.get name
if path.nil? || !File.exists? path
file = File.tempfile name
State.set name, file.path
return file
else
return File.new path
end
path = State.get name
if path.nil? || !File.exists? path
file = File.tempfile name
State.set name, file.path
file
else
File.new path
end
end
def with_default_config
temp_config = get_tempfile "mango-test-config"
config = Config.load temp_config.path
logger = MLogger.new config
yield config, logger, temp_config.path
temp_config.delete
temp_config = get_tempfile "mango-test-config"
config = Config.load temp_config.path
logger = Logger.new config.log_level
yield config, logger, temp_config.path
temp_config.delete
end
def with_storage
with_default_config do |config, logger|
temp_db = get_tempfile "mango-test-db"
storage = Storage.new temp_db.path, logger
clear = yield storage, temp_db.path
if clear == true
temp_db.delete
end
end
with_default_config do |_, logger|
temp_db = get_tempfile "mango-test-db"
storage = Storage.new temp_db.path, logger
clear = yield storage, temp_db.path
if clear == true
temp_db.delete
end
end
end
def with_queue
with_default_config do |config, logger|
temp_queue_db = get_tempfile "mango-test-queue-db"
queue = MangaDex::Queue.new temp_queue_db.path, logger
clear = yield queue, temp_queue_db.path
if clear == true
temp_queue_db.delete
end
end
with_default_config do |_, logger|
temp_queue_db = get_tempfile "mango-test-queue-db"
queue = MangaDex::Queue.new temp_queue_db.path, logger
clear = yield queue, temp_queue_db.path
if clear == true
temp_queue_db.delete
end
end
end

View File

@ -1,91 +1,91 @@
require "./spec_helper"
describe Storage do
it "creates DB at given path" do
with_storage do |storage, path|
File.exists?(path).should be_true
end
end
it "creates DB at given path" do
with_storage do |_, path|
File.exists?(path).should be_true
end
end
it "deletes user" do
with_storage do |storage|
storage.delete_user "admin"
end
end
it "deletes user" do
with_storage do |storage|
storage.delete_user "admin"
end
end
it "creates new user" do
with_storage do |storage|
storage.new_user "user", "123456", false
storage.new_user "admin", "123456", true
end
end
it "creates new user" do
with_storage do |storage|
storage.new_user "user", "123456", false
storage.new_user "admin", "123456", true
end
end
it "verifies username/password combination" do
with_storage do |storage|
user_token = storage.verify_user "user", "123456"
admin_token = storage.verify_user "admin", "123456"
user_token.should_not be_nil
admin_token.should_not be_nil
State.set "user_token", user_token
State.set "admin_token", admin_token
end
end
it "verifies username/password combination" do
with_storage do |storage|
user_token = storage.verify_user "user", "123456"
admin_token = storage.verify_user "admin", "123456"
user_token.should_not be_nil
admin_token.should_not be_nil
State.set "user_token", user_token
State.set "admin_token", admin_token
end
end
it "rejects duplicate username" do
with_storage do |storage|
expect_raises SQLite3::Exception,
"UNIQUE constraint failed: users.username" do
storage.new_user "admin", "123456", true
end
end
end
it "rejects duplicate username" do
with_storage do |storage|
expect_raises SQLite3::Exception,
"UNIQUE constraint failed: users.username" do
storage.new_user "admin", "123456", true
end
end
end
it "verifies token" do
with_storage do |storage|
user_token = State.get! "user_token"
user = storage.verify_token user_token
user.should eq "user"
end
end
it "verifies token" do
with_storage do |storage|
user_token = State.get! "user_token"
user = storage.verify_token user_token
user.should eq "user"
end
end
it "verfies admin token" do
with_storage do |storage|
admin_token = State.get! "admin_token"
storage.verify_admin(admin_token).should be_true
end
end
it "verfies admin token" do
with_storage do |storage|
admin_token = State.get! "admin_token"
storage.verify_admin(admin_token).should be_true
end
end
it "rejects non-admin token" do
with_storage do |storage|
user_token = State.get! "user_token"
storage.verify_admin(user_token).should be_false
end
end
it "rejects non-admin token" do
with_storage do |storage|
user_token = State.get! "user_token"
storage.verify_admin(user_token).should be_false
end
end
it "updates user" do
with_storage do |storage|
storage.update_user "admin", "admin", "654321", true
token = storage.verify_user "admin", "654321"
admin_token = State.get! "admin_token"
token.should eq admin_token
end
end
it "updates user" do
with_storage do |storage|
storage.update_user "admin", "admin", "654321", true
token = storage.verify_user "admin", "654321"
admin_token = State.get! "admin_token"
token.should eq admin_token
end
end
it "logs user out" do
with_storage do |storage|
user_token = State.get! "user_token"
admin_token = State.get! "admin_token"
storage.logout user_token
storage.logout admin_token
storage.verify_token(user_token).should be_nil
storage.verify_token(admin_token).should be_nil
end
end
it "logs user out" do
with_storage do |storage|
user_token = State.get! "user_token"
admin_token = State.get! "admin_token"
storage.logout user_token
storage.logout admin_token
storage.verify_token(user_token).should be_nil
storage.verify_token(admin_token).should be_nil
end
end
it "cleans up" do
with_storage do
true
end
State.reset
end
it "cleans up" do
with_storage do
true
end
State.reset
end
end

View File

@ -1,36 +1,36 @@
require "./spec_helper"
describe "compare_alphanumerically" do
it "sorts filenames with leading zeros correctly" do
ary = ["010.jpg", "001.jpg", "002.png"]
ary.sort! {|a, b|
compare_alphanumerically a, b
}
ary.should eq ["001.jpg", "002.png", "010.jpg"]
end
it "sorts filenames with leading zeros correctly" do
ary = ["010.jpg", "001.jpg", "002.png"]
ary.sort! { |a, b|
compare_alphanumerically a, b
}
ary.should eq ["001.jpg", "002.png", "010.jpg"]
end
it "sorts filenames without leading zeros correctly" do
ary = ["10.jpg", "1.jpg", "0.png", "0100.jpg"]
ary.sort! {|a, b|
compare_alphanumerically a, b
}
ary.should eq ["0.png", "1.jpg", "10.jpg", "0100.jpg"]
end
it "sorts filenames without leading zeros correctly" do
ary = ["10.jpg", "1.jpg", "0.png", "0100.jpg"]
ary.sort! { |a, b|
compare_alphanumerically a, b
}
ary.should eq ["0.png", "1.jpg", "10.jpg", "0100.jpg"]
end
# https://ux.stackexchange.com/a/95441
it "sorts like the stack exchange post" do
ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2",
"text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"]
ary.reverse.sort {|a, b|
compare_alphanumerically a, b
}.should eq ary
end
# https://ux.stackexchange.com/a/95441
it "sorts like the stack exchange post" do
ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2",
"text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"]
ary.reverse.sort { |a, b|
compare_alphanumerically a, b
}.should eq ary
end
# https://github.com/hkalexling/Mango/issues/22
it "handles numbers larger than Int32" do
ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"]
ary.reverse.sort {|a, b|
compare_alphanumerically a, b
}.should eq ary
end
# https://github.com/hkalexling/Mango/issues/22
it "handles numbers larger than Int32" do
ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"]
ary.reverse.sort { |a, b|
compare_alphanumerically a, b
}.should eq ary
end
end

View File

@ -1,26 +0,0 @@
require "kemal"
require "./storage"
require "./util"
class AuthHandler < Kemal::Handler
def initialize(@storage : Storage)
end
def call(env)
return call_next(env) \
if request_path_startswith env, ["/login", "/logout"]
cookie = env.request.cookies.find { |c| c.name == "token" }
if cookie.nil? || ! @storage.verify_token cookie.value
return env.redirect "/login"
end
if request_path_startswith env, ["/admin", "/api/admin", "/download"]
unless @storage.verify_admin cookie.value
env.response.status_code = 403
end
end
call_next env
end
end

View File

@ -1,60 +1,61 @@
require "yaml"
class Config
include YAML::Serializable
include YAML::Serializable
property port : Int32 = 9000
property library_path : String = \
File.expand_path "~/mango/library", home: true
property db_path : String = \
File.expand_path "~/mango/mango.db", home: true
@[YAML::Field(key: "scan_interval_minutes")]
property scan_interval : Int32 = 5
property log_level : String = "info"
property mangadex = Hash(String, String|Int32).new
property port : Int32 = 9000
property library_path : String = File.expand_path "~/mango/library",
home: true
property db_path : String = File.expand_path "~/mango/mango.db", home: true
@[YAML::Field(key: "scan_interval_minutes")]
property scan_interval : Int32 = 5
property log_level : String = "info"
property upload_path : String = File.expand_path "~/mango/uploads",
home: true
property mangadex = Hash(String, String | Int32).new
@[YAML::Field(ignore: true)]
@mangadex_defaults = {
"base_url" => "https://mangadex.org",
"api_url" => "https://mangadex.org/api",
"download_wait_seconds" => 5,
"download_retries" => 4,
"download_queue_db_path" => File.expand_path "~/mango/queue.db",
home: true
}
@[YAML::Field(ignore: true)]
@mangadex_defaults = {
"base_url" => "https://mangadex.org",
"api_url" => "https://mangadex.org/api",
"download_wait_seconds" => 5,
"download_retries" => 4,
"download_queue_db_path" => File.expand_path("~/mango/queue.db",
home: true),
}
def self.load(path : String?)
path = "~/.config/mango/config.yml" if path.nil?
cfg_path = File.expand_path path, home: true
if File.exists? cfg_path
config = self.from_yaml File.read cfg_path
config.fill_defaults
return config
end
puts "The config file #{cfg_path} does not exist." \
" Do you want mango to dump the default config there? [Y/n]"
input = gets
if input && input.downcase == "n"
abort "Aborting..."
end
default = self.allocate
default.fill_defaults
cfg_dir = File.dirname cfg_path
unless Dir.exists? cfg_dir
Dir.mkdir_p cfg_dir
end
File.write cfg_path, default.to_yaml
puts "The config file has been created at #{cfg_path}."
default
end
def self.load(path : String?)
path = "~/.config/mango/config.yml" if path.nil?
cfg_path = File.expand_path path, home: true
if File.exists? cfg_path
config = self.from_yaml File.read cfg_path
config.fill_defaults
return config
end
puts "The config file #{cfg_path} does not exist." \
" Do you want mango to dump the default config there? [Y/n]"
input = gets
if input && input.downcase == "n"
abort "Aborting..."
end
default = self.allocate
default.fill_defaults
cfg_dir = File.dirname cfg_path
unless Dir.exists? cfg_dir
Dir.mkdir_p cfg_dir
end
File.write cfg_path, default.to_yaml
puts "The config file has been created at #{cfg_path}."
default
end
def fill_defaults
{% for hash_name in ["mangadex"] %}
@{{hash_name.id}}_defaults.map do |k, v|
if @{{hash_name.id}}[k]?.nil?
@{{hash_name.id}}[k] = v
end
end
{% end %}
end
def fill_defaults
{% for hash_name in ["mangadex"] %}
@{{hash_name.id}}_defaults.map do |k, v|
if @{{hash_name.id}}[k]?.nil?
@{{hash_name.id}}[k] = v
end
end
{% end %}
end
end

View File

@ -4,18 +4,18 @@ require "./storage"
require "./logger"
class Context
property config : Config
property library : Library
property storage : Storage
property logger : MLogger
property queue : MangaDex::Queue
property config : Config
property library : Library
property storage : Storage
property logger : Logger
property queue : MangaDex::Queue
def initialize(@config, @logger, @library, @storage, @queue)
end
def initialize(@config, @logger, @library, @storage, @queue)
end
{% for lvl in LEVELS %}
def {{lvl.id}}(msg)
@logger.{{lvl.id}} msg
end
{% end %}
{% for lvl in Logger::LEVELS %}
def {{lvl.id}}(msg)
@logger.{{lvl.id}} msg
end
{% end %}
end

View File

@ -0,0 +1,25 @@
require "kemal"
require "../storage"
require "../util"
class AuthHandler < Kemal::Handler
def initialize(@storage : Storage)
end
def call(env)
return call_next(env) if request_path_startswith env, ["/login", "/logout"]
cookie = env.request.cookies.find { |c| c.name == "token" }
if cookie.nil? || !@storage.verify_token cookie.value
return env.redirect "/login"
end
if request_path_startswith env, ["/admin", "/api/admin", "/download"]
unless @storage.verify_admin cookie.value
env.response.status_code = 403
end
end
call_next env
end
end

View File

@ -0,0 +1,26 @@
require "kemal"
require "../logger"
class LogHandler < Kemal::BaseLogHandler
def initialize(@logger : Logger)
end
def call(env)
elapsed_time = Time.measure { call_next env }
elapsed_text = elapsed_text elapsed_time
msg = "#{env.response.status_code} #{env.request.method}" \
" #{env.request.resource} #{elapsed_text}"
@logger.debug msg
env
end
def write(msg)
@logger.debug msg
end
private def elapsed_text(elapsed)
millis = elapsed.total_milliseconds
return "#{millis.round(2)}ms" if millis >= 1
"#{(millis * 1000).round(2)}µs"
end
end

View File

@ -0,0 +1,32 @@
require "baked_file_system"
require "kemal"
require "../util"
class FS
extend BakedFileSystem
{% if flag?(:release) %}
{% if read_file? "#{__DIR__}/../../dist/favicon.ico" %}
{% puts "baking ../../dist" %}
bake_folder "../../dist"
{% else %}
{% puts "baking ../../public" %}
bake_folder "../../public"
{% end %}
{% end %}
end
class StaticHandler < Kemal::Handler
@dirs = ["/css", "/js", "/img", "/favicon.ico"]
def call(env)
if request_path_startswith env, @dirs
file = FS.get? env.request.path
return call_next env if file.nil?
slice = Bytes.new file.size
file.read slice
return send_file env, slice, file.mime_type
end
call_next env
end
end

View File

@ -0,0 +1,24 @@
require "kemal"
require "../util"
class UploadHandler < Kemal::Handler
def initialize(@upload_dir : String)
end
def call(env)
unless request_path_startswith(env, [UPLOAD_URL_PREFIX]) &&
env.request.method == "GET"
return call_next env
end
ary = env.request.path.split(File::SEPARATOR).select { |part| !part.empty? }
ary[0] = @upload_dir
path = File.join ary
if File.exists? path
send_file env, path
else
env.response.status_code = 404
end
end
end

View File

@ -5,309 +5,427 @@ require "uri"
require "./util"
struct Image
property data : Bytes
property mime : String
property filename : String
property size : Int32
property data : Bytes
property mime : String
property filename : String
property size : Int32
def initialize(@data, @mime, @filename, @size)
end
def initialize(@data, @mime, @filename, @size)
end
end
class Entry
property zip_path : String, book_title : String, title : String,
size : String, pages : Int32, cover_url : String, id : String,
title_id : String, encoded_path : String, encoded_title : String,
mtime : Time
property zip_path : String, book : Title, title : String,
size : String, pages : Int32, id : String, title_id : String,
encoded_path : String, encoded_title : String, mtime : Time
def initialize(path, @book_title, @title_id, storage)
@zip_path = path
@encoded_path = URI.encode path
@title = File.basename path, File.extname path
@encoded_title = URI.encode @title
@size = (File.size path).humanize_bytes
file = Zip::File.new path
@pages = file.entries
.select { |e|
["image/jpeg", "image/png"].includes? \
MIME.from_filename? e.filename
}
.size
file.close
@id = storage.get_id @zip_path, false
@cover_url = "/api/page/#{@title_id}/#{@id}/1"
@mtime = File.info(@zip_path).modification_time
end
def initialize(path, @book, @title_id, storage)
@zip_path = path
@encoded_path = URI.encode path
@title = File.basename path, File.extname path
@encoded_title = URI.encode @title
@size = (File.size path).humanize_bytes
file = Zip::File.new path
@pages = file.entries.count do |e|
["image/jpeg", "image/png"].includes? \
MIME.from_filename? e.filename
end
file.close
@id = storage.get_id @zip_path, false
@mtime = File.info(@zip_path).modification_time
end
def to_json(json : JSON::Builder)
json.object do
{% for str in ["zip_path", "book_title", "title", "size",
"cover_url", "id", "title_id", "encoded_path",
"encoded_title"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "pages" {json.number @pages}
json.field "mtime" {json.number @mtime.to_unix}
end
end
def to_json(json : JSON::Builder)
json.object do
{% for str in ["zip_path", "title", "size", "id", "title_id",
"encoded_path", "encoded_title"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "display_name", @book.display_name @title
json.field "cover_url", cover_url
json.field "pages" { json.number @pages }
json.field "mtime" { json.number @mtime.to_unix }
end
end
def read_page(page_num)
Zip::File.open @zip_path do |file|
page = file.entries
.select { |e|
["image/jpeg", "image/png"].includes? \
MIME.from_filename? e.filename
}
.sort { |a, b|
compare_alphanumerically a.filename, b.filename
}
.[page_num - 1]
page.open do |io|
slice = Bytes.new page.uncompressed_size
bytes_read = io.read_fully? slice
unless bytes_read
return nil
end
return Image.new slice, MIME.from_filename(page.filename),\
page.filename, bytes_read
end
end
end
def display_name
@book.display_name @title
end
def encoded_display_name
URI.encode display_name
end
def cover_url
url = "/api/page/#{@title_id}/#{@id}/1"
TitleInfo.new @book.dir do |info|
info_url = info.entry_cover_url[@title]?
unless info_url.nil? || info_url.empty?
url = info_url
end
end
url
end
def read_page(page_num)
Zip::File.open @zip_path do |file|
page = file.entries
.select { |e|
["image/jpeg", "image/png"].includes? \
MIME.from_filename? e.filename
}
.sort { |a, b|
compare_alphanumerically a.filename, b.filename
}
.[page_num - 1]
page.open do |io|
slice = Bytes.new page.uncompressed_size
bytes_read = io.read_fully? slice
unless bytes_read
return nil
end
return Image.new slice, MIME.from_filename(page.filename),
page.filename, bytes_read
end
end
end
end
class Title
property dir : String, parent_id : String, title_ids : Array(String),
entries : Array(Entry), title : String, id : String,
encoded_title : String, mtime : Time
property dir : String, parent_id : String, title_ids : Array(String),
entries : Array(Entry), title : String, id : String,
encoded_title : String, mtime : Time
def initialize(dir : String, @parent_id, storage,
@logger : MLogger, @library : Library)
@dir = dir
@id = storage.get_id @dir, true
@title = File.basename dir
@encoded_title = URI.encode @title
@title_ids = [] of String
@entries = [] of Entry
def initialize(@dir : String, @parent_id, storage,
@logger : Logger, @library : Library)
@id = storage.get_id @dir, true
@title = File.basename dir
@encoded_title = URI.encode @title
@title_ids = [] of String
@entries = [] of Entry
@mtime = File.info(dir).modification_time
Dir.entries(dir).each do |fn|
next if fn.starts_with? "."
path = File.join dir, fn
if File.directory? path
title = Title.new path, @id, storage, @logger, library
next if title.entries.size == 0 && title.titles.size == 0
@library.title_hash[title.id] = title
@title_ids << title.id
next
end
if [".zip", ".cbz"].includes? File.extname path
next if !valid_zip path
entry = Entry.new path, @title, @id, storage
@entries << entry if entry.pages > 0
end
end
Dir.entries(dir).each do |fn|
next if fn.starts_with? "."
path = File.join dir, fn
if File.directory? path
title = Title.new path, @id, storage, @logger, library
next if title.entries.size == 0 && title.titles.size == 0
@library.title_hash[title.id] = title
@title_ids << title.id
next
end
if [".zip", ".cbz"].includes? File.extname path
zip_exception = validate_zip path
unless zip_exception.nil?
@logger.warn "File #{path} is corrupted or is not a valid zip " \
"archive. Ignoring it."
@logger.debug "Zip error: #{zip_exception}"
next
end
entry = Entry.new path, self, @id, storage
@entries << entry if entry.pages > 0
end
end
@title_ids.sort! do |a, b|
compare_alphanumerically @library.title_hash[a].title,
@library.title_hash[b].title
end
@entries.sort! do |a, b|
compare_alphanumerically a.title, b.title
end
mtimes = [@mtime]
mtimes += @title_ids.map { |e| @library.title_hash[e].mtime }
mtimes += @entries.map { |e| e.mtime }
@mtime = mtimes.max
mtimes = [File.info(dir).modification_time]
mtimes += @title_ids.map{|e| @library.title_hash[e].mtime}
mtimes += @entries.map{|e| e.mtime}
@mtime = mtimes.max
end
@title_ids.sort! do |a, b|
compare_alphanumerically @library.title_hash[a].title,
@library.title_hash[b].title
end
@entries.sort! do |a, b|
compare_alphanumerically a.title, b.title
end
end
def to_json(json : JSON::Builder)
json.object do
{% for str in ["dir", "title", "id", "encoded_title"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "mtime" {json.number @mtime.to_unix}
json.field "titles" do
json.raw self.titles.to_json
end
json.field "entries" do
json.raw @entries.to_json
end
json.field "parents" do
json.array do
self.parents.each do |title|
json.object do
json.field "title", title.title
json.field "id", title.id
end
end
end
end
end
end
def to_json(json : JSON::Builder)
json.object do
{% for str in ["dir", "title", "id", "encoded_title"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "display_name", display_name
json.field "cover_url", cover_url
json.field "mtime" { json.number @mtime.to_unix }
json.field "titles" do
json.raw self.titles.to_json
end
json.field "entries" do
json.raw @entries.to_json
end
json.field "parents" do
json.array do
self.parents.each do |title|
json.object do
json.field "title", title.title
json.field "id", title.id
end
end
end
end
end
end
def titles
@title_ids.map {|tid| @library.get_title! tid}
end
def titles
@title_ids.map { |tid| @library.get_title! tid }
end
def parents
ary = [] of Title
tid = @parent_id
while !tid.empty?
title = @library.get_title! tid
ary << title
tid = title.parent_id
end
ary
end
def parents
ary = [] of Title
tid = @parent_id
while !tid.empty?
title = @library.get_title! tid
ary << title
tid = title.parent_id
end
ary
end
def size
@entries.size + @title_ids.size
end
def size
@entries.size + @title_ids.size
end
# When downloading from MangaDex, the zip/cbz file would not be valid
# before the download is completed. If we scan the zip file,
# Entry.new would throw, so we use this method to check before
# constructing Entry
private def valid_zip(path : String)
begin
file = Zip::File.new path
file.close
return true
rescue
@logger.warn "File #{path} is corrupted or is not a valid zip "\
"archive. Ignoring it."
return false
end
end
def get_entry(eid)
@entries.find { |e| e.id == eid }
end
# For backward backward compatibility with v0.1.0, we save entry titles
# instead of IDs in info.json
def save_progress(username, entry, page)
info = TitleInfo.new @dir
if info.progress[username]?.nil?
info.progress[username] = {entry => page}
info.save @dir
return
end
info.progress[username][entry] = page
info.save @dir
end
def load_progress(username, entry)
info = TitleInfo.new @dir
if info.progress[username]?.nil?
return 0
end
if info.progress[username][entry]?.nil?
return 0
end
info.progress[username][entry]
end
def load_percetage(username, entry)
info = TitleInfo.new @dir
page = load_progress username, entry
entry_obj = @entries.find{|e| e.title == entry}
return 0.0 if entry_obj.nil?
page / entry_obj.pages
end
def load_percetage(username)
return 0.0 if @entries.empty?
read_pages = total_pages = 0
@entries.each do |e|
read_pages += load_progress username, e.title
total_pages += e.pages
end
read_pages / total_pages
end
def next_entry(current_entry_obj)
idx = @entries.index current_entry_obj
return nil if idx.nil? || idx == @entries.size - 1
@entries[idx + 1]
end
def get_entry(eid)
@entries.find { |e| e.id == eid }
end
def display_name
dn = @title
TitleInfo.new @dir do |info|
info_dn = info.display_name
dn = info_dn unless info_dn.empty?
end
dn
end
def encoded_display_name
URI.encode display_name
end
def display_name(entry_name)
dn = entry_name
TitleInfo.new @dir do |info|
info_dn = info.entry_display_name[entry_name]?
unless info_dn.nil? || info_dn.empty?
dn = info_dn
end
end
dn
end
def set_display_name(dn)
TitleInfo.new @dir do |info|
info.display_name = dn
info.save
end
end
def set_display_name(entry_name : String, dn)
TitleInfo.new @dir do |info|
info.entry_display_name[entry_name] = dn
info.save
end
end
def cover_url
url = "img/icon.png"
if @entries.size > 0
url = @entries[0].cover_url
end
TitleInfo.new @dir do |info|
info_url = info.cover_url
unless info_url.nil? || info_url.empty?
url = info_url
end
end
url
end
def set_cover_url(url : String)
TitleInfo.new @dir do |info|
info.cover_url = url
info.save
end
end
def set_cover_url(entry_name : String, url : String)
TitleInfo.new @dir do |info|
info.entry_cover_url[entry_name] = url
info.save
end
end
# Set the reading progress of all entries and nested libraries to 100%
def read_all(username)
@entries.each do |e|
save_progress username, e.title, e.pages
end
titles.each do |t|
t.read_all username
end
end
# Set the reading progress of all entries and nested libraries to 0%
def unread_all(username)
@entries.each do |e|
save_progress username, e.title, 0
end
titles.each do |t|
t.unread_all username
end
end
# For backward backward compatibility with v0.1.0, we save entry titles
# instead of IDs in info.json
def save_progress(username, entry, page)
TitleInfo.new @dir do |info|
if info.progress[username]?.nil?
info.progress[username] = {entry => page}
else
info.progress[username][entry] = page
end
info.save
end
end
def load_progress(username, entry)
progress = 0
TitleInfo.new @dir do |info|
unless info.progress[username]?.nil? ||
info.progress[username][entry]?.nil?
progress = info.progress[username][entry]
end
end
progress
end
def load_percetage(username, entry)
page = load_progress username, entry
entry_obj = @entries.find { |e| e.title == entry }
return 0.0 if entry_obj.nil?
page / entry_obj.pages
end
def load_percetage(username)
return 0.0 if @entries.empty?
read_pages = total_pages = 0
@entries.each do |e|
read_pages += load_progress username, e.title
total_pages += e.pages
end
read_pages / total_pages
end
def next_entry(current_entry_obj)
idx = @entries.index current_entry_obj
return nil if idx.nil? || idx == @entries.size - 1
@entries[idx + 1]
end
end
class TitleInfo
# { user1: { entry1: 10, entry2: 0 } }
include JSON::Serializable
include JSON::Serializable
property comment = "Generated by Mango. DO NOT EDIT!"
property progress : Hash(String, Hash(String, Int32))
property comment = "Generated by Mango. DO NOT EDIT!"
property progress = {} of String => Hash(String, Int32)
property display_name = ""
property entry_display_name = {} of String => String
property cover_url = ""
property entry_cover_url = {} of String => String
def initialize(title_dir)
info = nil
@[JSON::Field(ignore: true)]
property dir : String = ""
json_path = File.join title_dir, "info.json"
if File.exists? json_path
info = TitleInfo.from_json File.read json_path
else
info = TitleInfo.from_json "{\"progress\": {}}"
end
@@mutex_hash = {} of String => Mutex
@progress = info.progress.clone
end
def save(title_dir)
json_path = File.join title_dir, "info.json"
File.write json_path, self.to_pretty_json
end
def self.new(dir, &)
if @@mutex_hash[dir]?
mutex = @@mutex_hash[dir]
else
mutex = Mutex.new
@@mutex_hash[dir] = mutex
end
mutex.synchronize do
instance = TitleInfo.allocate
json_path = File.join dir, "info.json"
if File.exists? json_path
instance = TitleInfo.from_json File.read json_path
end
instance.dir = dir
yield instance
end
end
def save
json_path = File.join @dir, "info.json"
File.write json_path, self.to_pretty_json
end
end
class Library
property dir : String, title_ids : Array(String), scan_interval : Int32,
logger : MLogger, storage : Storage, title_hash : Hash(String, Title)
property dir : String, title_ids : Array(String), scan_interval : Int32,
logger : Logger, storage : Storage, title_hash : Hash(String, Title)
def initialize(@dir, @scan_interval, @logger, @storage)
# explicitly initialize @titles to bypass the compiler check. it will
# be filled with actual Titles in the `scan` call below
@title_ids = [] of String
@title_hash = {} of String => Title
def initialize(@dir, @scan_interval, @logger, @storage)
# explicitly initialize @titles to bypass the compiler check. it will
# be filled with actual Titles in the `scan` call below
@title_ids = [] of String
@title_hash = {} of String => Title
return scan if @scan_interval < 1
spawn do
loop do
start = Time.local
scan
ms = (Time.local - start).total_milliseconds
@logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep @scan_interval * 60
end
end
end
def titles
@title_ids.map {|tid| self.get_title!(tid) }
end
def to_json(json : JSON::Builder)
json.object do
json.field "dir", @dir
json.field "titles" do
json.raw self.titles.to_json
end
end
end
def get_title(tid)
@title_hash[tid]?
end
def get_title!(tid)
@title_hash[tid]
end
def scan
unless Dir.exists? @dir
@logger.info "The library directory #{@dir} does not exist. " \
"Attempting to create it"
Dir.mkdir_p @dir
end
@title_ids.clear
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
.select { |path| File.directory? path }
.map { |path| Title.new path, "", @storage, @logger, self }
.select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort { |a, b| a.title <=> b.title }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id
end
@logger.debug "Scan completed"
end
return scan if @scan_interval < 1
spawn do
loop do
start = Time.local
scan
ms = (Time.local - start).total_milliseconds
@logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep @scan_interval * 60
end
end
end
def titles
@title_ids.map { |tid| self.get_title!(tid) }
end
def to_json(json : JSON::Builder)
json.object do
json.field "dir", @dir
json.field "titles" do
json.raw self.titles.to_json
end
end
end
def get_title(tid)
@title_hash[tid]?
end
def get_title!(tid)
@title_hash[tid]
end
def scan
unless Dir.exists? @dir
@logger.info "The library directory #{@dir} does not exist. " \
"Attempting to create it"
Dir.mkdir_p @dir
end
@title_ids.clear
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
.select { |path| File.directory? path }
.map { |path| Title.new path, "", @storage, @logger, self }
.select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort { |a, b| a.title <=> b.title }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id
end
@logger.debug "Scan completed"
end
end

View File

@ -1,26 +0,0 @@
require "kemal"
require "./logger"
class LogHandler < Kemal::BaseLogHandler
def initialize(@logger : MLogger)
end
def call(env)
elapsed_time = Time.measure { call_next env }
elapsed_text = elapsed_text elapsed_time
msg = "#{env.response.status_code} #{env.request.method}" \
" #{env.request.resource} #{elapsed_text}"
@logger.debug(msg)
env
end
def write(msg)
@logger.debug(msg)
end
private def elapsed_text(elapsed)
millis = elapsed.total_milliseconds
return "#{millis.round(2)}ms" if millis >= 1
"#{(millis * 1000).round(2)}µs"
end
end

View File

@ -1,59 +1,58 @@
require "./config"
require "logger"
require "log"
require "colorize"
LEVELS = ["debug", "error", "fatal", "info", "warn"]
COLORS = [:light_cyan, :light_red, :red, :light_yellow, :light_magenta]
class Logger
LEVELS = ["debug", "error", "fatal", "info", "warn"]
SEVERITY_IDS = [0, 4, 5, 2, 3]
COLORS = [:light_cyan, :light_red, :red, :light_yellow, :light_magenta]
class MLogger
def initialize(config : Config)
@logger = Logger.new STDOUT
@@severity : Log::Severity = :info
@log_off = false
log_level = config.log_level
if log_level == "off"
@log_off = true
return
end
def initialize(level : String)
{% begin %}
case level.downcase
when "off"
@@severity = :none
{% for lvl, i in LEVELS %}
when {{lvl}}
@@severity = Log::Severity.new SEVERITY_IDS[{{i}}]
{% end %}
else
raise "Unknown log level #{level}"
end
{% end %}
{% begin %}
case log_level
{% for lvl in LEVELS %}
when {{lvl}}
@logger.level = Logger::{{lvl.upcase.id}}
{% end %}
else
raise "Unknown log level #{log_level}"
end
{% end %}
@log = Log.for("")
@logger.formatter = Logger::Formatter.new do \
|severity, datetime, progname, message, io|
@backend = Log::IOBackend.new
@backend.formatter = ->(entry : Log::Entry, io : IO) do
color = :default
{% begin %}
case entry.severity.label.to_s().downcase
{% for lvl, i in LEVELS %}
when {{lvl}}, "#{{{lvl}}}ing"
color = COLORS[{{i}}]
{% end %}
else
end
{% end %}
color = :default
{% begin %}
case severity.to_s().downcase
{% for lvl, i in LEVELS %}
when {{lvl}}
color = COLORS[{{i}}]
{% end %}
end
{% end %}
io << "[#{entry.severity.label}]".ljust(10).colorize(color)
io << entry.timestamp.to_s("%Y/%m/%d %H:%M:%S") << " | "
io << entry.message
end
io << "[#{severity}]".ljust(8).colorize(color)
io << datetime.to_s("%Y/%m/%d %H:%M:%S") << " | "
io << message
end
end
Log.builder.bind "*", @@severity, @backend
end
{% for lvl in LEVELS %}
def {{lvl.id}}(msg)
return if @log_off
@logger.{{lvl.id}} msg
end
{% end %}
# Ignores @@severity and always log msg
def log(msg)
@backend.write Log::Entry.new "", Log::Severity::None, msg, nil
end
def to_json(json : JSON::Builder)
json.string self
end
{% for lvl in LEVELS %}
def {{lvl.id}}(msg)
@log.{{lvl.id}} { msg }
end
{% end %}
end

View File

@ -2,202 +2,196 @@ require "http/client"
require "json"
require "csv"
macro string_properties (names)
{% for name in names %}
property {{name.id}} = ""
{% end %}
macro string_properties(names)
{% for name in names %}
property {{name.id}} = ""
{% end %}
end
macro parse_strings_from_json (names)
{% for name in names %}
@{{name.id}} = obj[{{name}}].as_s
{% end %}
macro parse_strings_from_json(names)
{% for name in names %}
@{{name.id}} = obj[{{name}}].as_s
{% end %}
end
module MangaDex
class Chapter
string_properties ["lang_code", "title", "volume", "chapter"]
property manga : Manga
property time = Time.local
property id : String
property full_title = ""
property language = ""
property pages = [] of {String, String} # filename, url
property groups = [] of {Int32, String} # group_id, group_name
class Chapter
string_properties ["lang_code", "title", "volume", "chapter"]
property manga : Manga
property time = Time.local
property id : String
property full_title = ""
property language = ""
property pages = [] of {String, String} # filename, url
property groups = [] of {Int32, String} # group_id, group_name
def initialize(@id, json_obj : JSON::Any, @manga, lang :
Hash(String, String))
self.parse_json json_obj, lang
end
def initialize(@id, json_obj : JSON::Any, @manga,
lang : Hash(String, String))
self.parse_json json_obj, lang
end
def to_info_json
JSON.build do |json|
json.object do
{% for name in ["id", "title", "volume", "chapter",
"language", "full_title"] %}
json.field {{name}}, @{{name.id}}
{% end %}
json.field "time", @time.to_unix.to_s
json.field "manga_title", @manga.title
json.field "manga_id", @manga.id
json.field "groups" do
json.object do
@groups.each do |gid, gname|
json.field gname, gid
end
end
end
end
end
end
def to_info_json
JSON.build do |json|
json.object do
{% for name in ["id", "title", "volume", "chapter",
"language", "full_title"] %}
json.field {{name}}, @{{name.id}}
{% end %}
json.field "time", @time.to_unix.to_s
json.field "manga_title", @manga.title
json.field "manga_id", @manga.id
json.field "groups" do
json.object do
@groups.each do |gid, gname|
json.field gname, gid
end
end
end
end
end
end
def parse_json(obj, lang)
begin
parse_strings_from_json ["lang_code", "title", "volume",
"chapter"]
language = lang[@lang_code]?
@language = language if language
@time = Time.unix obj["timestamp"].as_i
suffixes = ["", "_2", "_3"]
suffixes.each do |s|
gid = obj["group_id#{s}"].as_i
next if gid == 0
gname = obj["group_name#{s}"].as_s
@groups << {gid, gname}
end
@full_title = @title
unless @chapter.empty?
@full_title = "Ch.#{@chapter} " + @full_title
end
unless @volume.empty?
@full_title = "Vol.#{@volume} " + @full_title
end
rescue e
raise "failed to parse json: #{e}"
end
end
end
class Manga
string_properties ["cover_url", "description", "title", "author",
"artist"]
property chapters = [] of Chapter
property id : String
def parse_json(obj, lang)
parse_strings_from_json ["lang_code", "title", "volume",
"chapter"]
language = lang[@lang_code]?
@language = language if language
@time = Time.unix obj["timestamp"].as_i
suffixes = ["", "_2", "_3"]
suffixes.each do |s|
gid = obj["group_id#{s}"].as_i
next if gid == 0
gname = obj["group_name#{s}"].as_s
@groups << {gid, gname}
end
@full_title = @title
unless @chapter.empty?
@full_title = "Ch.#{@chapter} " + @full_title
end
unless @volume.empty?
@full_title = "Vol.#{@volume} " + @full_title
end
rescue e
raise "failed to parse json: #{e}"
end
end
def initialize(@id, json_obj : JSON::Any)
self.parse_json json_obj
end
class Manga
string_properties ["cover_url", "description", "title", "author", "artist"]
property chapters = [] of Chapter
property id : String
def to_info_json(with_chapters = true)
JSON.build do |json|
json.object do
{% for name in ["id", "title", "description",
"author", "artist", "cover_url"] %}
json.field {{name}}, @{{name.id}}
{% end %}
if with_chapters
json.field "chapters" do
json.array do
@chapters.each do |c|
json.raw c.to_info_json
end
end
end
end
end
end
end
def initialize(@id, json_obj : JSON::Any)
self.parse_json json_obj
end
def parse_json(obj)
begin
parse_strings_from_json ["cover_url", "description", "title",
"author", "artist"]
rescue e
raise "failed to parse json: #{e}"
end
end
end
class API
def initialize(@base_url = "https://mangadex.org/api/")
@lang = {} of String => String
CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
@lang[row[1]] = row[0]
end
end
def to_info_json(with_chapters = true)
JSON.build do |json|
json.object do
{% for name in ["id", "title", "description", "author", "artist",
"cover_url"] %}
json.field {{name}}, @{{name.id}}
{% end %}
if with_chapters
json.field "chapters" do
json.array do
@chapters.each do |c|
json.raw c.to_info_json
end
end
end
end
end
end
end
def get(url)
headers = HTTP::Headers {
"User-agent" => "Mangadex.cr"
}
res = HTTP::Client.get url, headers
raise "Failed to get #{url}. [#{res.status_code}] "\
"#{res.status_message}" if !res.success?
JSON.parse res.body
end
def parse_json(obj)
parse_strings_from_json ["cover_url", "description", "title", "author",
"artist"]
rescue e
raise "failed to parse json: #{e}"
end
end
def get_manga(id)
obj = self.get File.join @base_url, "manga/#{id}"
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
begin
manga = Manga.new id, obj["manga"]
obj["chapter"].as_h.map do |k, v|
chapter = Chapter.new k, v, manga, @lang
manga.chapters << chapter
end
return manga
rescue
raise "Failed to parse JSON"
end
end
class API
def initialize(@base_url = "https://mangadex.org/api/")
@lang = {} of String => String
CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row|
@lang[row[1]] = row[0]
end
end
def get_chapter(chapter : Chapter)
obj = self.get File.join @base_url, "chapter/#{chapter.id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
begin
server = obj["server"].as_s
hash = obj["hash"].as_s
chapter.pages = obj["page_array"].as_a.map do |fn|
{
fn.as_s,
"#{server}#{hash}/#{fn.as_s}"
}
end
rescue
raise "Failed to parse JSON"
end
end
def get(url)
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
res = HTTP::Client.get url, headers
raise "Failed to get #{url}. [#{res.status_code}] " \
"#{res.status_message}" if !res.success?
JSON.parse res.body
end
def get_chapter(id : String)
obj = self.get File.join @base_url, "chapter/#{id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. " \
"Got `#{obj["status"]?}`"
end
manga_id = ""
begin
manga_id = obj["manga_id"].as_i.to_s
rescue
raise "Failed to parse JSON"
end
manga = self.get_manga manga_id
chapter = manga.chapters.find {|c| c.id == id}.not_nil!
self.get_chapter chapter
return chapter
end
end
def get_manga(id)
obj = self.get File.join @base_url, "manga/#{id}"
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
manga = Manga.new id, obj["manga"]
obj["chapter"].as_h.map do |k, v|
chapter = Chapter.new k, v, manga, @lang
manga.chapters << chapter
end
manga
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(chapter : Chapter)
obj = self.get File.join @base_url, "chapter/#{chapter.id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
begin
server = obj["server"].as_s
hash = obj["hash"].as_s
chapter.pages = obj["page_array"].as_a.map do |fn|
{
fn.as_s,
"#{server}#{hash}/#{fn.as_s}",
}
end
rescue
raise "Failed to parse JSON"
end
end
def get_chapter(id : String)
obj = self.get File.join @base_url, "chapter/#{id}"
if obj["status"]? == "external"
raise "This chapter is hosted on an external site " \
"#{obj["external"]?}, and Mango does not support " \
"external chapters."
end
if obj["status"]? != "OK"
raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`"
end
manga_id = ""
begin
manga_id = obj["manga_id"].as_i.to_s
rescue
raise "Failed to parse JSON"
end
manga = self.get_manga manga_id
chapter = manga.chapters.find { |c| c.id == id }.not_nil!
self.get_chapter chapter
chapter
end
end
end

View File

@ -2,373 +2,384 @@ require "./api"
require "sqlite3"
module MangaDex
class PageJob
property success = false
property url : String
property filename : String
property writer : Zip::Writer
property tries_remaning : Int32
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
class PageJob
property success = false
property url : String
property filename : String
property writer : Zip::Writer
property tries_remaning : Int32
enum JobStatus
Pending # 0
Downloading # 1
Error # 2
Completed # 3
MissingPages # 4
end
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
struct Job
property id : String
property manga_id : String
property title : String
property manga_title : String
property status : JobStatus
property status_message : String = ""
property pages : Int32 = 0
property success_count : Int32 = 0
property fail_count : Int32 = 0
property time : Time
enum JobStatus
Pending # 0
Downloading # 1
Error # 2
Completed # 3
MissingPages # 4
end
def parse_query_result(res : DB::ResultSet)
@id = res.read String
@manga_id = res.read String
@title = res.read String
@manga_title = res.read String
status = res.read Int32
@status_message = res.read String
@pages = res.read Int32
@success_count = res.read Int32
@fail_count = res.read Int32
time = res.read Int64
@status = JobStatus.new status
@time = Time.unix_ms time
end
struct Job
property id : String
property manga_id : String
property title : String
property manga_title : String
property status : JobStatus
property status_message : String = ""
property pages : Int32 = 0
property success_count : Int32 = 0
property fail_count : Int32 = 0
property time : Time
# Raises if the result set does not contain the correct set of columns
def self.from_query_result(res : DB::ResultSet)
job = Job.allocate
job.parse_query_result res
return job
end
def parse_query_result(res : DB::ResultSet)
@id = res.read String
@manga_id = res.read String
@title = res.read String
@manga_title = res.read String
status = res.read Int32
@status_message = res.read String
@pages = res.read Int32
@success_count = res.read Int32
@fail_count = res.read Int32
time = res.read Int64
@status = JobStatus.new status
@time = Time.unix_ms time
end
def initialize(@id, @manga_id, @title, @manga_title, @status, @time)
end
# Raises if the result set does not contain the correct set of columns
def self.from_query_result(res : DB::ResultSet)
job = Job.allocate
job.parse_query_result res
job
end
def to_json(json)
json.object do
{% for name in ["id", "manga_id", "title", "manga_title",
"status_message"] %}
json.field {{name}}, @{{name.id}}
{% end %}
{% for name in ["pages", "success_count", "fail_count"] %}
json.field {{name}} do
json.number @{{name.id}}
end
{% end %}
json.field "status", @status.to_s
json.field "time" do
json.number @time.to_unix_ms
end
end
end
end
def initialize(@id, @manga_id, @title, @manga_title, @status, @time)
end
class Queue
property downloader : Downloader?
def to_json(json)
json.object do
{% for name in ["id", "manga_id", "title", "manga_title",
"status_message"] %}
json.field {{name}}, @{{name.id}}
{% end %}
{% for name in ["pages", "success_count", "fail_count"] %}
json.field {{name}} do
json.number @{{name.id}}
end
{% end %}
json.field "status", @status.to_s
json.field "time" do
json.number @time.to_unix_ms
end
end
end
end
def initialize(@path : String, @logger : MLogger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The queue DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
@logger.error "Error when checking tables in DB: #{e}"
raise e
end
end
end
class Queue
property downloader : Downloader?
# Returns the earliest job in queue or nil if the job cannot be parsed.
# Returns nil if queue is empty
def pop
job = nil
DB.open "sqlite3://#{@path}" do |db|
begin
db.query_one "select * from queue where status = 0 "\
"or status = 1 order by time limit 1" do |res|
job = Job.from_query_result res
end
rescue
end
end
return job
end
def initialize(@path : String, @logger : Logger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The queue DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
@logger.error "Error when checking tables in DB: #{e}"
raise e
end
end
end
# Push an array of jobs into the queue, and return the number of jobs
# inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job))
start_count = self.count
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values "\
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end
self.count - start_count
end
# Returns the earliest job in queue or nil if the job cannot be parsed.
# Returns nil if queue is empty
def pop
job = nil
DB.open "sqlite3://#{@path}" do |db|
begin
db.query_one "select * from queue where status = 0 " \
"or status = 1 order by time limit 1" do |res|
job = Job.from_query_result res
end
rescue
end
end
job
end
def reset(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end
# Push an array of jobs into the queue, and return the number of jobs
# inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job))
start_count = self.count
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values " \
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end
self.count - start_count
end
def reset (job : Job)
self.reset job.id
end
def reset(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end
# Reset all failed tasks (missing pages and error)
def reset
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end
def reset(job : Job)
self.reset job.id
end
def delete(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end
# Reset all failed tasks (missing pages and error)
def reset
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end
def delete(job : Job)
self.delete job.id
end
def delete(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end
def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end
def delete(job : Job)
self.delete job.id
end
def count_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue where "\
"status = (?)", status.to_i, as: Int32
end
end
def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end
def count
DB.open "sqlite3://#{@path}" do |db|
return db.query_one "select count(*) from queue", as: Int32
end
end
def count_status(status : JobStatus)
num = 0
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue where " \
"status = (?)", status.to_i, as: Int32
end
num
end
def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end
def count
num = 0
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue", as: Int32
end
num
end
def get_all
jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time", do |rs|
Job.from_query_result rs
end
end
return jobs
end
def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end
def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end
def get_all
jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time" do |rs|
Job.from_query_result rs
end
end
jobs
end
def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end
def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end
def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end
def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end
def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end
def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end
def pause
@downloader.not_nil!.stopped = true
end
def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end
def resume
@downloader.not_nil!.stopped = false
end
def pause
@downloader.not_nil!.stopped = true
end
def paused?
@downloader.not_nil!.stopped
end
end
def resume
@downloader.not_nil!.stopped = false
end
class Downloader
property stopped = false
@downloading = false
def paused?
@downloader.not_nil!.stopped
end
end
def initialize(@queue : Queue, @api : API, @library_path : String,
@wait_seconds : Int32, @retries : Int32,
@logger : MLogger)
@queue.downloader = self
class Downloader
property stopped = false
@downloading = false
spawn do
loop do
sleep 1.second
next if @stopped || @downloading
begin
job = @queue.pop
next if job.nil?
download job
rescue e
@logger.error e
end
end
end
end
def initialize(@queue : Queue, @api : API, @library_path : String,
@wait_seconds : Int32, @retries : Int32,
@logger : Logger)
@queue.downloader = self
private def download(job : Job)
@downloading = true
@queue.set_status JobStatus::Downloading, job
begin
chapter = @api.get_chapter(job.id)
rescue e
@logger.error e
@queue.set_status JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages chapter.pages.size, job
lib_dir = @library_path
manga_dir = File.join lib_dir, chapter.manga.title
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz"
spawn do
loop do
sleep 1.second
next if @stopped || @downloading
begin
job = @queue.pop
next if job.nil?
download job
rescue e
@logger.error e
end
end
end
end
# Find the number of digits needed to store the number of pages
len = Math.log10(chapter.pages.size).to_i + 1
private def download(job : Job)
@downloading = true
@queue.set_status JobStatus::Downloading, job
begin
chapter = @api.get_chapter(job.id)
rescue e
@logger.error e
@queue.set_status JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages chapter.pages.size, job
lib_dir = @library_path
manga_dir = File.join lib_dir, chapter.manga.title
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz"
writer = Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new chapter.pages.size
spawn do
chapter.pages.each_with_index do |tuple, i|
fn, url = tuple
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
@logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
@logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
# Find the number of digits needed to store the number of pages
len = Math.log10(chapter.pages.size).to_i + 1
channel.send page_job
end
end
writer = Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new chapter.pages.size
spawn do
chapter.pages.each_with_index do |tuple, i|
fn, url = tuple
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
@logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
@logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
spawn do
page_jobs = [] of PageJob
chapter.pages.size.times do
page_job = channel.receive
@logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
@logger.error msg
end
end
fail_count = page_jobs.select{|j| !j.success}.size
@logger.debug "Download completed. "\
"#{fail_count}/#{page_jobs.size} failed"
writer.close
@logger.debug "cbz File created at #{zip_path}"
if fail_count == 0
@queue.set_status JobStatus::Completed, job
else
@queue.set_status JobStatus::MissingPages, job
end
@downloading = false
end
end
channel.send page_job
end
end
private def download_page(job : PageJob)
@logger.debug "downloading #{job.url}"
headers = HTTP::Headers {
"User-agent" => "Mangadex.cr"
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
@logger.error e
job.success = false
end
end
end
spawn do
page_jobs = [] of PageJob
chapter.pages.size.times do
page_job = channel.receive
@logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
@logger.error msg
end
end
fail_count = page_jobs.count { |j| !j.success }
@logger.debug "Download completed. " \
"#{fail_count}/#{page_jobs.size} failed"
writer.close
@logger.debug "cbz File created at #{zip_path}"
zip_exception = validate_zip zip_path
if !zip_exception.nil?
@queue.add_message "The downloaded archive is corrupted. " \
"Error: #{zip_exception}", job
@queue.set_status JobStatus::Error, job
elsif fail_count > 0
@queue.set_status JobStatus::MissingPages, job
else
@queue.set_status JobStatus::Completed, job
end
@downloading = false
end
end
private def download_page(job : PageJob)
@logger.debug "downloading #{job.url}"
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
@logger.error e
job.success = false
end
end
end
end

View File

@ -3,37 +3,37 @@ require "./context"
require "./mangadex/*"
require "option_parser"
VERSION = "0.2.5"
VERSION = "0.3.0"
config_path = nil
parser = OptionParser.parse do |parser|
parser.banner = "Mango e-manga server/reader. Version #{VERSION}\n"
OptionParser.parse do |parser|
parser.banner = "Mango e-manga server/reader. Version #{VERSION}\n"
parser.on "-v", "--version", "Show version" do
puts "Version #{VERSION}"
exit
end
parser.on "-h", "--help", "Show help" do
puts parser
exit
end
parser.on "-c PATH", "--config=PATH", "Path to the config file. " \
"Default is `~/.config/mango/config.yml`" do |path|
config_path = path
end
parser.on "-v", "--version", "Show version" do
puts "Version #{VERSION}"
exit
end
parser.on "-h", "--help", "Show help" do
puts parser
exit
end
parser.on "-c PATH", "--config=PATH",
"Path to the config file. Default is `~/.config/mango/config.yml`" do |path|
config_path = path
end
end
config = Config.load config_path
logger = MLogger.new config
logger = Logger.new config.log_level
storage = Storage.new config.db_path, logger
library = Library.new config.library_path, config.scan_interval, logger, storage
queue = MangaDex::Queue.new config.mangadex["download_queue_db_path"].to_s,
logger
api = MangaDex::API.new config.mangadex["api_url"].to_s
downloader = MangaDex::Downloader.new queue, api, config.library_path,
config.mangadex["download_wait_seconds"].to_i,
config.mangadex["download_retries"].to_i, logger
logger
api = MangaDex::API.new config.mangadex["api_url"].to_s
MangaDex::Downloader.new queue, api, config.library_path,
config.mangadex["download_wait_seconds"].to_i,
config.mangadex["download_retries"].to_i, logger
context = Context.new config, logger, library, storage, queue

View File

@ -1,108 +1,103 @@
require "./router"
class AdminRouter < Router
def setup
get "/admin" do |env|
layout "admin"
end
def setup
get "/admin" do |env|
layout "admin"
end
get "/admin/user" do |env|
users = @context.storage.list_users
username = get_username env
layout "user"
end
get "/admin/user" do |env|
users = @context.storage.list_users
username = get_username env
layout "user"
end
get "/admin/user/edit" do |env|
username = env.params.query["username"]?
admin = env.params.query["admin"]?
if admin
admin = admin == "true"
end
error = env.params.query["error"]?
current_user = get_username env
new_user = username.nil? && admin.nil?
layout "user-edit"
end
get "/admin/user/edit" do |env|
username = env.params.query["username"]?
admin = env.params.query["admin"]?
if admin
admin = admin == "true"
end
error = env.params.query["error"]?
current_user = get_username env
new_user = username.nil? && admin.nil?
layout "user-edit"
end
post "/admin/user/edit" do |env|
# creating new user
begin
username = env.params.body["username"]
password = env.params.body["password"]
# if `admin` is unchecked, the body hash
# would not contain `admin`
admin = !env.params.body["admin"]?.nil?
post "/admin/user/edit" do |env|
# creating new user
username = env.params.body["username"]
password = env.params.body["password"]
# if `admin` is unchecked, the body hash
# would not contain `admin`
admin = !env.params.body["admin"]?.nil?
if username.size < 3
raise "Username should contain at least 3 characters"
end
if (username =~ /^[A-Za-z0-9_]+$/).nil?
raise "Username should contain alphanumeric characters "\
"and underscores only"
end
if password.size < 6
raise "Password should contain at least 6 characters"
end
if (password =~ /^[[:ascii:]]+$/).nil?
raise "password should contain ASCII characters only"
end
if username.size < 3
raise "Username should contain at least 3 characters"
end
if (username =~ /^[A-Za-z0-9_]+$/).nil?
raise "Username should contain alphanumeric characters " \
"and underscores only"
end
if password.size < 6
raise "Password should contain at least 6 characters"
end
if (password =~ /^[[:ascii:]]+$/).nil?
raise "password should contain ASCII characters only"
end
@context.storage.new_user username, password, admin
@context.storage.new_user username, password, admin
env.redirect "/admin/user"
rescue e
@context.error e
redirect_url = URI.new \
path: "/admin/user/edit",\
query: hash_to_query({"error" => e.message})
env.redirect redirect_url.to_s
end
end
env.redirect "/admin/user"
rescue e
@context.error e
redirect_url = URI.new \
path: "/admin/user/edit",
query: hash_to_query({"error" => e.message})
env.redirect redirect_url.to_s
end
post "/admin/user/edit/:original_username" do |env|
# editing existing user
begin
username = env.params.body["username"]
password = env.params.body["password"]
# if `admin` is unchecked, the body
# hash would not contain `admin`
admin = !env.params.body["admin"]?.nil?
original_username = env.params.url["original_username"]
post "/admin/user/edit/:original_username" do |env|
# editing existing user
username = env.params.body["username"]
password = env.params.body["password"]
# if `admin` is unchecked, the body hash would not contain `admin`
admin = !env.params.body["admin"]?.nil?
original_username = env.params.url["original_username"]
if username.size < 3
raise "Username should contain at least 3 characters"
end
if (username =~ /^[A-Za-z0-9_]+$/).nil?
raise "Username should contain alphanumeric characters "\
"and underscores only"
end
if username.size < 3
raise "Username should contain at least 3 characters"
end
if (username =~ /^[A-Za-z0-9_]+$/).nil?
raise "Username should contain alphanumeric characters " \
"and underscores only"
end
if password.size != 0
if password.size < 6
raise "Password should contain at least 6 characters"
end
if (password =~ /^[[:ascii:]]+$/).nil?
raise "password should contain ASCII characters only"
end
end
if password.size != 0
if password.size < 6
raise "Password should contain at least 6 characters"
end
if (password =~ /^[[:ascii:]]+$/).nil?
raise "password should contain ASCII characters only"
end
end
@context.storage.update_user \
original_username, username, password, admin
@context.storage.update_user \
original_username, username, password, admin
env.redirect "/admin/user"
rescue e
@context.error e
redirect_url = URI.new \
path: "/admin/user/edit",\
query: hash_to_query({"username" => original_username, \
"admin" => admin, "error" => e.message})
env.redirect redirect_url.to_s
end
end
env.redirect "/admin/user"
rescue e
@context.error e
redirect_url = URI.new \
path: "/admin/user/edit",
query: hash_to_query({"username" => original_username, \
"admin" => admin, "error" => e.message})
env.redirect redirect_url.to_s
end
get "/admin/downloads" do |env|
base_url = @context.config.mangadex["base_url"];
layout "download-manager"
end
end
get "/admin/downloads" do |env|
base_url = @context.config.mangadex["base_url"]
layout "download-manager"
end
end
end

View File

@ -1,180 +1,263 @@
require "./router"
require "../mangadex/*"
require "../upload"
class APIRouter < Router
def setup
get "/api/page/:tid/:eid/:page" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
page = env.params.url["page"].to_i
def setup
get "/api/page/:tid/:eid/:page" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
page = env.params.url["page"].to_i
title = @context.library.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
entry = title.get_entry eid
raise "Entry ID `#{eid}` of `#{title.title}` not found" if \
entry.nil?
img = entry.read_page page
raise "Failed to load page #{page} of " \
"`#{title.title}/#{entry.title}`" if img.nil?
title = @context.library.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
entry = title.get_entry eid
raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil?
img = entry.read_page page
raise "Failed to load page #{page} of " \
"`#{title.title}/#{entry.title}`" if img.nil?
send_img env, img
rescue e
@context.error e
env.response.status_code = 500
e.message
end
end
send_img env, img
rescue e
@context.error e
env.response.status_code = 500
e.message
end
end
get "/api/book/:tid" do |env|
begin
tid = env.params.url["tid"]
title = @context.library.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
get "/api/book/:tid" do |env|
begin
tid = env.params.url["tid"]
title = @context.library.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
send_json env, title.to_json
rescue e
@context.error e
env.response.status_code = 500
e.message
end
end
send_json env, title.to_json
rescue e
@context.error e
env.response.status_code = 500
e.message
end
end
get "/api/book" do |env|
send_json env, @context.library.to_json
end
get "/api/book" do |env|
send_json env, @context.library.to_json
end
post "/api/admin/scan" do |env|
start = Time.utc
@context.library.scan
ms = (Time.utc - start).total_milliseconds
send_json env, {
"milliseconds" => ms,
"titles" => @context.library.titles.size
}.to_json
end
post "/api/admin/scan" do |env|
start = Time.utc
@context.library.scan
ms = (Time.utc - start).total_milliseconds
send_json env, {
"milliseconds" => ms,
"titles" => @context.library.titles.size,
}.to_json
end
post "/api/admin/user/delete/:username" do |env|
begin
username = env.params.url["username"]
@context.storage.delete_user username
rescue e
@context.error e
send_json env, {
"success" => false,
"error" => e.message
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
post "/api/admin/user/delete/:username" do |env|
begin
username = env.params.url["username"]
@context.storage.delete_user username
rescue e
@context.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
post "/api/progress/:title/:entry/:page" do |env|
begin
username = get_username env
title = (@context.library.get_title env.params.url["title"])
.not_nil!
entry = (title.get_entry env.params.url["entry"]).not_nil!
page = env.params.url["page"].to_i
post "/api/progress/:title/:page" do |env|
begin
username = get_username env
title = (@context.library.get_title env.params.url["title"])
.not_nil!
page = env.params.url["page"].to_i
entry_id = env.params.query["entry"]?
raise "incorrect page value" if page < 0 || page > entry.pages
title.save_progress username, entry.title, page
rescue e
@context.error e
send_json env, {
"success" => false,
"error" => e.message
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
if !entry_id.nil?
entry = title.get_entry(entry_id).not_nil!
raise "incorrect page value" if page < 0 || page > entry.pages
title.save_progress username, entry.title, page
elsif page == 0
title.unread_all username
else
title.read_all username
end
rescue e
@context.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
get "/api/admin/mangadex/manga/:id" do |env|
begin
id = env.params.url["id"]
api = MangaDex::API.new \
@context.config.mangadex["api_url"].to_s
manga = api.get_manga id
send_json env, manga.to_info_json
rescue e
@context.error e
send_json env, {"error" => e.message}.to_json
end
end
post "/api/admin/display_name/:title/:name" do |env|
begin
title = (@context.library.get_title env.params.url["title"])
.not_nil!
name = env.params.url["name"]
entry = env.params.query["entry"]?
if entry.nil?
title.set_display_name name
else
eobj = title.get_entry entry
title.set_display_name eobj.not_nil!.title, name
end
rescue e
@context.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
post "/api/admin/mangadex/download" do |env|
begin
chapters = env.params.json["chapters"].as(Array).map{|c| c.as_h}
jobs = chapters.map {|chapter|
MangaDex::Job.new(
chapter["id"].as_s,
chapter["manga_id"].as_s,
chapter["full_title"].as_s,
chapter["manga_title"].as_s,
MangaDex::JobStatus::Pending,
Time.unix chapter["time"].as_s.to_i
)
}
inserted_count = @context.queue.push jobs
send_json env, {
"success": inserted_count,
"fail": jobs.size - inserted_count
}.to_json
rescue e
@context.error e
send_json env, {"error" => e.message}.to_json
end
end
get "/api/admin/mangadex/manga/:id" do |env|
begin
id = env.params.url["id"]
api = MangaDex::API.new @context.config.mangadex["api_url"].to_s
manga = api.get_manga id
send_json env, manga.to_info_json
rescue e
@context.error e
send_json env, {"error" => e.message}.to_json
end
end
get "/api/admin/mangadex/queue" do |env|
begin
jobs = @context.queue.get_all
send_json env, {
"jobs" => jobs,
"paused" => @context.queue.paused?,
"success" => true
}.to_json
rescue e
send_json env, {
"success" => false,
"error" => e.message
}.to_json
end
end
post "/api/admin/mangadex/download" do |env|
begin
chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h }
jobs = chapters.map { |chapter|
MangaDex::Job.new(
chapter["id"].as_s,
chapter["manga_id"].as_s,
chapter["full_title"].as_s,
chapter["manga_title"].as_s,
MangaDex::JobStatus::Pending,
Time.unix chapter["time"].as_s.to_i
)
}
inserted_count = @context.queue.push jobs
send_json env, {
"success": inserted_count,
"fail": jobs.size - inserted_count,
}.to_json
rescue e
@context.error e
send_json env, {"error" => e.message}.to_json
end
end
post "/api/admin/mangadex/queue/:action" do |env|
begin
action = env.params.url["action"]
id = env.params.query["id"]?
case action
when "delete"
if id.nil?
@context.queue.delete_status MangaDex::JobStatus::Completed
else
@context.queue.delete id
end
when "retry"
if id.nil?
@context.queue.reset
else
@context.queue.reset id
end
when "pause"
@context.queue.pause
when "resume"
@context.queue.resume
else
raise "Unknown queue action #{action}"
end
get "/api/admin/mangadex/queue" do |env|
begin
jobs = @context.queue.get_all
send_json env, {
"jobs" => jobs,
"paused" => @context.queue.paused?,
"success" => true,
}.to_json
rescue e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
send_json env, {"success" => true}.to_json
rescue e
send_json env, {
"success" => false,
"error" => e.message
}.to_json
end
end
end
post "/api/admin/mangadex/queue/:action" do |env|
begin
action = env.params.url["action"]
id = env.params.query["id"]?
case action
when "delete"
if id.nil?
@context.queue.delete_status MangaDex::JobStatus::Completed
else
@context.queue.delete id
end
when "retry"
if id.nil?
@context.queue.reset
else
@context.queue.reset id
end
when "pause"
@context.queue.pause
when "resume"
@context.queue.resume
else
raise "Unknown queue action #{action}"
end
send_json env, {"success" => true}.to_json
rescue e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
post "/api/admin/upload/:target" do |env|
begin
target = env.params.url["target"]
HTTP::FormData.parse env.request do |part|
next if part.name != "file"
filename = part.filename
if filename.nil?
raise "No file uploaded"
end
case target
when "cover"
title_id = env.params.query["title"]
entry_id = env.params.query["entry"]?
title = @context.library.get_title(title_id).not_nil!
unless ["image/jpeg", "image/png"].includes? \
MIME.from_filename? filename
raise "The uploaded image must be either JPEG or PNG"
end
ext = File.extname filename
upload = Upload.new @context.config.upload_path, @context.logger
url = upload.path_to_url upload.save "img", ext, part.body
if url.nil?
raise "Failed to generate a public URL for the uploaded file"
end
if entry_id.nil?
title.set_cover_url url
else
entry_name = title.get_entry(entry_id).not_nil!.title
title.set_cover_url entry_name, url
end
else
raise "Unkown upload target #{target}"
end
send_json env, {"success" => true}.to_json
env.response.close
end
raise "No part with name `file` found"
rescue e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
end
end

View File

@ -1,63 +1,66 @@
require "./router"
class MainRouter < Router
def setup
get "/login" do |env|
render "src/views/login.ecr"
end
def setup
get "/login" do |env|
render "src/views/login.ecr"
end
get "/logout" do |env|
begin
cookie = env.request.cookies
.find { |c| c.name == "token" }.not_nil!
@context.storage.logout cookie.value
rescue e
@context.error "Error when attempting to log out: #{e}"
ensure
env.redirect "/login"
end
end
get "/logout" do |env|
begin
cookie = env.request.cookies.find { |c| c.name == "token" }.not_nil!
@context.storage.logout cookie.value
rescue e
@context.error "Error when attempting to log out: #{e}"
ensure
env.redirect "/login"
end
end
post "/login" do |env|
begin
username = env.params.body["username"]
password = env.params.body["password"]
token = @context.storage.verify_user(username, password)
.not_nil!
post "/login" do |env|
begin
username = env.params.body["username"]
password = env.params.body["password"]
token = @context.storage.verify_user(username, password).not_nil!
cookie = HTTP::Cookie.new "token", token
cookie.expires = Time.local.shift years: 1
env.response.cookies << cookie
env.redirect "/"
rescue
env.redirect "/login"
end
end
cookie = HTTP::Cookie.new "token", token
cookie.expires = Time.local.shift years: 1
env.response.cookies << cookie
env.redirect "/"
rescue
env.redirect "/login"
end
end
get "/" do |env|
titles = @context.library.titles
username = get_username env
percentage = titles.map &.load_percetage username
layout "index"
end
get "/" do |env|
begin
titles = @context.library.titles
username = get_username env
percentage = titles.map &.load_percetage username
layout "index"
rescue e
@context.error e
env.response.status_code = 500
end
end
get "/book/:title" do |env|
begin
title = (@context.library.get_title env.params.url["title"])
.not_nil!
username = get_username env
percentage = title.entries.map { |e|
title.load_percetage username, e.title }
layout "title"
rescue e
@context.error e
env.response.status_code = 404
end
end
get "/book/:title" do |env|
begin
title = (@context.library.get_title env.params.url["title"]).not_nil!
username = get_username env
percentage = title.entries.map { |e|
title.load_percetage username, e.title
}
layout "title"
rescue e
@context.error e
env.response.status_code = 404
end
end
get "/download" do |env|
base_url = @context.config.mangadex["base_url"];
layout "download"
end
end
get "/download" do |env|
base_url = @context.config.mangadex["base_url"]
layout "download"
end
end
end

View File

@ -1,58 +1,61 @@
require "./router"
class ReaderRouter < Router
def setup
get "/reader/:title/:entry" do |env|
begin
title = (@context.library.get_title env.params.url["title"])
.not_nil!
entry = (title.get_entry env.params.url["entry"]).not_nil!
def setup
get "/reader/:title/:entry" do |env|
begin
title = (@context.library.get_title env.params.url["title"]).not_nil!
entry = (title.get_entry env.params.url["entry"]).not_nil!
# load progress
username = get_username env
page = title.load_progress username, entry.title
# we go back 2 * `IMGS_PER_PAGE` pages. the infinite scroll
# library perloads a few pages in advance, and the user
# might not have actually read them
page = [page - 2 * IMGS_PER_PAGE, 1].max
# load progress
username = get_username env
page = title.load_progress username, entry.title
# we go back 2 * `IMGS_PER_PAGE` pages. the infinite scroll
# library perloads a few pages in advance, and the user
# might not have actually read them
page = [page - 2 * IMGS_PER_PAGE, 1].max
env.redirect "/reader/#{title.id}/#{entry.id}/#{page}"
rescue e
@context.error e
env.response.status_code = 404
end
end
env.redirect "/reader/#{title.id}/#{entry.id}/#{page}"
rescue e
@context.error e
env.response.status_code = 404
end
end
get "/reader/:title/:entry/:page" do |env|
begin
title = (@context.library.get_title env.params.url["title"])
.not_nil!
entry = (title.get_entry env.params.url["entry"]).not_nil!
page = env.params.url["page"].to_i
raise "" if page > entry.pages || page <= 0
get "/reader/:title/:entry/:page" do |env|
begin
title = (@context.library.get_title env.params.url["title"]).not_nil!
entry = (title.get_entry env.params.url["entry"]).not_nil!
page = env.params.url["page"].to_i
raise "" if page > entry.pages || page <= 0
# save progress
username = get_username env
title.save_progress username, entry.title, page
# save progress
username = get_username env
title.save_progress username, entry.title, page
pages = (page...[entry.pages + 1, page + IMGS_PER_PAGE].min)
urls = pages.map { |idx|
"/api/page/#{title.id}/#{entry.id}/#{idx}" }
reader_urls = pages.map { |idx|
"/reader/#{title.id}/#{entry.id}/#{idx}" }
next_page = page + IMGS_PER_PAGE
next_url = next_page > entry.pages ? nil :
"/reader/#{title.id}/#{entry.id}/#{next_page}"
exit_url = "/book/#{title.id}"
next_entry = title.next_entry entry
next_entry_url = next_entry.nil? ? nil : \
"/reader/#{title.id}/#{next_entry.id}"
pages = (page...[entry.pages + 1, page + IMGS_PER_PAGE].min)
urls = pages.map { |idx|
"/api/page/#{title.id}/#{entry.id}/#{idx}"
}
reader_urls = pages.map { |idx|
"/reader/#{title.id}/#{entry.id}/#{idx}"
}
next_page = page + IMGS_PER_PAGE
next_url = next_entry_url = nil
exit_url = "/book/#{title.id}"
next_entry = title.next_entry entry
unless next_page > entry.pages
next_url = "/reader/#{title.id}/#{entry.id}/#{next_page}"
end
unless next_entry.nil?
next_entry_url = "/reader/#{title.id}/#{next_entry.id}"
end
render "src/views/reader.ecr"
rescue e
@context.error e
env.response.status_code = 404
end
end
end
render "src/views/reader.ecr"
rescue e
@context.error e
env.response.status_code = 404
end
end
end
end

View File

@ -1,6 +1,6 @@
require "../context"
class Router
def initialize(@context : Context)
end
def initialize(@context : Context)
end
end

View File

@ -1,50 +1,47 @@
require "kemal"
require "./context"
require "./auth_handler"
require "./static_handler"
require "./log_handler"
require "./handlers/*"
require "./util"
require "./routes/*"
class Server
def initialize(@context : Context)
def initialize(@context : Context)
error 403 do |env|
message = "HTTP 403: You are not authorized to visit #{env.request.path}"
layout "message"
end
error 404 do |env|
message = "HTTP 404: Mango cannot find the page #{env.request.path}"
layout "message"
end
error 500 do |env|
message = "HTTP 500: Internal server error. Please try again later."
layout "message"
end
error 403 do |env|
message = "HTTP 403: You are not authorized to visit " \
"#{env.request.path}"
layout "message"
end
error 404 do |env|
message = "HTTP 404: Mango cannot find the page #{env.request.path}"
layout "message"
end
error 500 do |env|
message = "HTTP 500: Internal server error. Please try again later."
layout "message"
end
MainRouter.new(@context).setup
AdminRouter.new(@context).setup
ReaderRouter.new(@context).setup
APIRouter.new(@context).setup
MainRouter.new(@context).setup
AdminRouter.new(@context).setup
ReaderRouter.new(@context).setup
APIRouter.new(@context).setup
Kemal.config.logging = false
add_handler LogHandler.new @context.logger
add_handler AuthHandler.new @context.storage
add_handler UploadHandler.new @context.config.upload_path
{% if flag?(:release) %}
# when building for relase, embed the static files in binary
@context.debug "We are in release mode. Using embedded static files."
serve_static false
add_handler StaticHandler.new
{% end %}
end
Kemal.config.logging = false
add_handler LogHandler.new @context.logger
add_handler AuthHandler.new @context.storage
{% if flag?(:release) %}
# when building for relase, embed the static files in binary
@context.debug "We are in release mode. Using embedded static files."
serve_static false
add_handler StaticHandler.new
{% end %}
end
def start
@context.debug "Starting Kemal server"
{% if flag?(:release) %}
Kemal.config.env = "production"
{% end %}
Kemal.config.port = @context.config.port
Kemal.run
end
def start
@context.debug "Starting Kemal server"
{% if flag?(:release) %}
Kemal.config.env = "production"
{% end %}
Kemal.config.port = @context.config.port
Kemal.run
end
end

View File

@ -1,32 +0,0 @@
require "baked_file_system"
require "kemal"
require "./util"
class FS
extend BakedFileSystem
{% if flag?(:release) %}
{% if read_file? "#{__DIR__}/../dist/favicon.ico" %}
{% puts "baking ../dist" %}
bake_folder "../dist"
{% else %}
{% puts "baking ../public" %}
bake_folder "../public"
{% end %}
{% end %}
end
class StaticHandler < Kemal::Handler
@dirs = ["/css", "/js", "/img", "/favicon.ico"]
def call(env)
if request_path_startswith env, @dirs
file = FS.get? env.request.path
return call_next env if file.nil?
slice = Bytes.new file.size
file.read slice
return send_file env, slice, file.mime_type
end
call_next env
end
end

View File

@ -2,176 +2,171 @@ require "sqlite3"
require "crypto/bcrypt"
require "uuid"
require "base64"
require "./util"
def hash_password(pw)
Crypto::Bcrypt::Password.create(pw).to_s
Crypto::Bcrypt::Password.create(pw).to_s
end
def verify_password(hash, pw)
(Crypto::Bcrypt::Password.new hash).verify pw
end
def random_str
UUID.random.to_s.gsub "-", ""
(Crypto::Bcrypt::Password.new hash).verify pw
end
class Storage
def initialize(@path : String, @logger : MLogger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{path}" do |db|
begin
# We create the `ids` table first. even if the uses has an
# early version installed and has the `user` table only,
# we will still be able to create `ids`
db.exec "create table ids" \
"(path text, id text, is_title integer)"
db.exec "create unique index path_idx on ids (path)"
db.exec "create unique index id_idx on ids (id)"
def initialize(@path : String, @logger : Logger)
dir = File.dirname path
unless Dir.exists? dir
@logger.info "The DB directory #{dir} does not exist. " \
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{path}" do |db|
begin
# We create the `ids` table first. even if the uses has an
# early version installed and has the `user` table only,
# we will still be able to create `ids`
db.exec "create table ids" \
"(path text, id text, is_title integer)"
db.exec "create unique index path_idx on ids (path)"
db.exec "create unique index id_idx on ids (id)"
db.exec "create table users" \
"(username text, password text, token text, admin integer)"
rescue e
unless e.message.not_nil!.ends_with? "already exists"
@logger.fatal "Error when checking tables in DB: #{e}"
raise e
end
else
@logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
random_pw = random_str
hash = hash_password random_pw
db.exec "insert into users values (?, ?, ?, ?)",
"admin", hash, nil, 1
puts "Initial user created. You can log in with " \
"#{{"username" => "admin", "password" => random_pw}}"
end
end
end
db.exec "create table users" \
"(username text, password text, token text, admin integer)"
rescue e
unless e.message.not_nil!.ends_with? "already exists"
@logger.fatal "Error when checking tables in DB: #{e}"
raise e
end
else
@logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
random_pw = random_str
hash = hash_password random_pw
db.exec "insert into users values (?, ?, ?, ?)",
"admin", hash, nil, 1
@logger.log "Initial user created. You can log in with " \
"#{{"username" => "admin", "password" => random_pw}}"
end
end
end
def verify_user(username, password)
DB.open "sqlite3://#{@path}" do |db|
begin
hash, token = db.query_one "select password, token from "\
"users where username = (?)", \
username, as: {String, String?}
unless verify_password hash, password
@logger.debug "Password does not match the hash"
return nil
end
@logger.debug "User #{username} verified"
return token if token
token = random_str
@logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
return token
rescue e
@logger.error "Error when verifying user #{username}: #{e}"
return nil
end
end
end
def verify_user(username, password)
DB.open "sqlite3://#{@path}" do |db|
begin
hash, token = db.query_one "select password, token from " \
"users where username = (?)",
username, as: {String, String?}
unless verify_password hash, password
@logger.debug "Password does not match the hash"
return nil
end
@logger.debug "User #{username} verified"
return token if token
token = random_str
@logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
return token
rescue e
@logger.error "Error when verifying user #{username}: #{e}"
return nil
end
end
end
def verify_token(token)
DB.open "sqlite3://#{@path}" do |db|
begin
username = db.query_one "select username from users where " \
"token = (?)", token, as: String
return username
rescue e
@logger.debug "Unable to verify token"
return nil
end
end
end
def verify_token(token)
username = nil
DB.open "sqlite3://#{@path}" do |db|
begin
username = db.query_one "select username from users where " \
"token = (?)", token, as: String
rescue e
@logger.debug "Unable to verify token"
end
end
username
end
def verify_admin(token)
DB.open "sqlite3://#{@path}" do |db|
begin
return db.query_one "select admin from users where " \
"token = (?)", token, as: Bool
rescue e
@logger.debug "Unable to verify user as admin"
return false
end
end
end
def verify_admin(token)
is_admin = false
DB.open "sqlite3://#{@path}" do |db|
begin
is_admin = db.query_one "select admin from users where " \
"token = (?)", token, as: Bool
rescue e
@logger.debug "Unable to verify user as admin"
end
end
is_admin
end
def list_users
results = Array(Tuple(String, Bool)).new
DB.open "sqlite3://#{@path}" do |db|
db.query "select username, admin from users" do |rs|
rs.each do
results << {rs.read(String), rs.read(Bool)}
end
end
end
results
end
def list_users
results = Array(Tuple(String, Bool)).new
DB.open "sqlite3://#{@path}" do |db|
db.query "select username, admin from users" do |rs|
rs.each do
results << {rs.read(String), rs.read(Bool)}
end
end
end
results
end
def new_user(username, password, admin)
admin = (admin ? 1 : 0)
DB.open "sqlite3://#{@path}" do |db|
hash = hash_password password
db.exec "insert into users values (?, ?, ?, ?)",
username, hash, nil, admin
end
end
def new_user(username, password, admin)
admin = (admin ? 1 : 0)
DB.open "sqlite3://#{@path}" do |db|
hash = hash_password password
db.exec "insert into users values (?, ?, ?, ?)",
username, hash, nil, admin
end
end
def update_user(original_username, username, password, admin)
admin = (admin ? 1 : 0)
DB.open "sqlite3://#{@path}" do |db|
if password.size == 0
db.exec "update users set username = (?), admin = (?) "\
"where username = (?)",\
username, admin, original_username
else
hash = hash_password password
db.exec "update users set username = (?), admin = (?),"\
"password = (?) where username = (?)",\
username, admin, hash, original_username
end
end
end
def update_user(original_username, username, password, admin)
admin = (admin ? 1 : 0)
DB.open "sqlite3://#{@path}" do |db|
if password.size == 0
db.exec "update users set username = (?), admin = (?) " \
"where username = (?)",
username, admin, original_username
else
hash = hash_password password
db.exec "update users set username = (?), admin = (?)," \
"password = (?) where username = (?)",
username, admin, hash, original_username
end
end
end
def delete_user(username)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from users where username = (?)", username
end
end
def delete_user(username)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from users where username = (?)", username
end
end
def logout(token)
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "update users set token = (?) where token = (?)", \
nil, token
rescue
end
end
end
def logout(token)
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "update users set token = (?) where token = (?)", nil, token
rescue
end
end
end
def get_id(path, is_title)
DB.open "sqlite3://#{@path}" do |db|
begin
id = db.query_one "select id from ids where path = (?)",
path, as: {String}
return id
rescue
id = random_str
db.exec "insert into ids values (?, ?, ?)", path, id,
is_title ? 1 : 0
return id
end
end
end
def get_id(path, is_title)
id = random_str
DB.open "sqlite3://#{@path}" do |db|
begin
id = db.query_one "select id from ids where path = (?)", path,
as: {String}
rescue
db.exec "insert into ids values (?, ?, ?)", path, id, is_title ? 1 : 0
end
end
id
end
def to_json(json : JSON::Builder)
json.string self
end
def to_json(json : JSON::Builder)
json.string self
end
end

60
src/upload.cr Normal file
View File

@ -0,0 +1,60 @@
require "./util"
class Upload
def initialize(@dir : String, @logger : Logger)
unless Dir.exists? @dir
@logger.info "The uploads directory #{@dir} does not exist. " \
"Attempting to create it"
Dir.mkdir_p @dir
end
end
# Writes IO to a file with random filename in the uploads directory and
# returns the full path of created file
# e.g., save("image", ".png", <io>)
# ==> "~/mango/uploads/image/<random string>.png"
def save(sub_dir : String, ext : String, io : IO)
full_dir = File.join @dir, sub_dir
filename = random_str + ext
file_path = File.join full_dir, filename
unless Dir.exists? full_dir
@logger.debug "creating directory #{full_dir}"
Dir.mkdir_p full_dir
end
File.open file_path, "w" do |f|
IO.copy io, f
end
file_path
end
# Converts path to a file in the uploads directory to the URL path for
# accessing the file.
def path_to_url(path : String)
dir_mathed = false
ary = [] of String
# We fill it with parts until it equals to @upload_dir
dir_ary = [] of String
Path.new(path).each_part do |part|
if dir_mathed
ary << part
else
dir_ary << part
if File.same? @dir, File.join dir_ary
dir_mathed = true
end
end
end
if ary.empty?
@logger.warn "File #{path} is not in the upload directory #{@dir}"
return
end
ary.unshift UPLOAD_URL_PREFIX
File.join(ary).to_s
end
end

View File

@ -1,73 +1,101 @@
require "big"
IMGS_PER_PAGE = 5
IMGS_PER_PAGE = 5
UPLOAD_URL_PREFIX = "/uploads"
macro layout(name)
render "src/views/#{{{name}}}.ecr", "src/views/layout.ecr"
begin
cookie = env.request.cookies.find { |c| c.name == "token" }
is_admin = false
unless cookie.nil?
is_admin = @context.storage.verify_admin cookie.value
end
render "src/views/#{{{name}}}.ecr", "src/views/layout.ecr"
rescue e
message = e.to_s
@context.error message
render "src/views/message.ecr", "src/views/layout.ecr"
end
end
macro send_img(env, img)
send_file {{env}}, {{img}}.data, {{img}}.mime
send_file {{env}}, {{img}}.data, {{img}}.mime
end
macro get_username(env)
# if the request gets here, it has gone through the auth handler, and
# we can be sure that a valid token exists, so we can use not_nil! here
cookie = {{env}}.request.cookies.find { |c| c.name == "token" }.not_nil!
(@context.storage.verify_token cookie.value).not_nil!
# if the request gets here, it has gone through the auth handler, and
# we can be sure that a valid token exists, so we can use not_nil! here
cookie = {{env}}.request.cookies.find { |c| c.name == "token" }.not_nil!
(@context.storage.verify_token cookie.value).not_nil!
end
macro send_json(env, json)
{{env}}.response.content_type = "application/json"
{{json}}
def send_json(env, json)
env.response.content_type = "application/json"
env.response.print json
end
def hash_to_query(hash)
hash.map { |k, v| "#{k}=#{v}" }.join("&")
hash.map { |k, v| "#{k}=#{v}" }.join("&")
end
def request_path_startswith(env, ary)
ary.each do |prefix|
if env.request.path.starts_with? prefix
return true
end
end
return false
ary.each do |prefix|
if env.request.path.starts_with? prefix
return true
end
end
false
end
def is_numeric(str)
/^\d+/.match(str) != nil
/^\d+/.match(str) != nil
end
def split_by_alphanumeric(str)
arr = [] of String
str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match|
arr += match.captures.select{|s| s != ""}
end
arr
arr = [] of String
str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match|
arr += match.captures.select { |s| s != "" }
end
arr
end
def compare_alphanumerically(c, d)
is_c_bigger = c.size <=> d.size
if c.size > d.size
d += [nil] * (c.size - d.size)
elsif c.size < d.size
c += [nil] * (d.size - c.size)
end
c.zip(d) do |a, b|
return -1 if a.nil?
return 1 if b.nil?
if is_numeric(a) && is_numeric(b)
compare = a.to_big_i <=> b.to_big_i
return compare if compare != 0
else
compare = a <=> b
return compare if compare != 0
end
end
is_c_bigger
is_c_bigger = c.size <=> d.size
if c.size > d.size
d += [nil] * (c.size - d.size)
elsif c.size < d.size
c += [nil] * (d.size - c.size)
end
c.zip(d) do |a, b|
return -1 if a.nil?
return 1 if b.nil?
if is_numeric(a) && is_numeric(b)
compare = a.to_big_i <=> b.to_big_i
return compare if compare != 0
else
compare = a <=> b
return compare if compare != 0
end
end
is_c_bigger
end
def compare_alphanumerically(a : String, b : String)
compare_alphanumerically split_by_alphanumeric(a), split_by_alphanumeric(b)
compare_alphanumerically split_by_alphanumeric(a), split_by_alphanumeric(b)
end
# When downloading from MangaDex, the zip/cbz file would not be valid
# before the download is completed. If we scan the zip file,
# Entry.new would throw, so we use this method to check before
# constructing Entry
def validate_zip(path : String) : Exception?
file = Zip::File.new path
file.close
return
rescue e
e
end
def random_str
UUID.random.to_s.gsub "-", ""
end

View File

@ -26,17 +26,13 @@
<a class="acard" href="/book/<%= t.id %>">
<div class="uk-card uk-card-default">
<div class="uk-card-media-top">
<%- if t.entries.size > 0 -%>
<img data-src="<%= t.entries[0].cover_url %>" data-width data-height alt="" uk-img>
<%- else -%>
<img data-src="/img/icon.png" data-width data-height alt="" uk-img>
<%- end -%>
<img data-src="<%= t.cover_url %>" data-width data-height alt="" uk-img>
</div>
<div class="uk-card-body">
<%- if t.entries.size > 0 -%>
<div class="uk-card-badge uk-label"><%= (percentage[i] * 100).round(1) %>%</div>
<%- end -%>
<h3 class="uk-card-title break-word" data-title="<%= t.title.gsub("\"", "&quot;") %>"><%= t.title %></h3>
<h3 class="uk-card-title break-word" data-title="<%= t.display_name.gsub("\"", "&quot;") %>"><%= t.display_name %></h3>
<p><%= t.size %> entries</p>
</div>
</div>

View File

@ -21,8 +21,10 @@
<div class="uk-offcanvas-bar uk-flex uk-flex-column">
<ul class="uk-nav uk-nav-primary uk-nav-center uk-margin-auto-vertical">
<li><a href="/">Home</a></li>
<li><a href="/admin">Admin</a></li>
<li><a href="/download">Download</a></li>
<% if is_admin %>
<li><a href="/admin">Admin</a></li>
<li><a href="/download">Download</a></li>
<% end %>
<hr uk-divider>
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="/logout">Logout</a></li>
@ -40,8 +42,10 @@
<a class="uk-navbar-item uk-logo" href="/"><img src="/img/icon.png"></a>
<ul class="uk-navbar-nav">
<li><a href="/">Home</a></li>
<li><a href="/admin">Admin</a></li>
<li><a href="/download">Download</a></li>
<% if is_admin %>
<li><a href="/admin">Admin</a></li>
<li><a href="/download">Download</a></li>
<% end %>
</ul>
</div>
<div class="uk-navbar-right uk-visible@s">

View File

@ -8,9 +8,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/uikit@3.3.1/dist/css/uikit.min.css" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
<script src="/js/theme.js"></script>
</head>
<body>
<script src="/js/theme.js"></script>
<div class="uk-section uk-flex uk-flex-middle uk-animation-fade" uk-height-viewport="">
<div class="uk-width-1-1">
<div class="uk-container">
@ -33,6 +34,9 @@
</div>
</div>
</div>
<script>
setTheme(getTheme());
</script>
<script src="https://cdn.jsdelivr.net/npm/uikit@3.3.1/dist/js/uikit.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/uikit@3.3.1/dist/js/uikit-icons.min.js"></script>
</body>

View File

@ -1,10 +1,17 @@
<h2 class=uk-title><%= title.title %></h2>
<div>
<h2 class=uk-title><span><%= title.display_name %></span>
&nbsp;
<% if is_admin %>
<a onclick="edit()" class="uk-icon-button" uk-icon="icon:pencil"></a>
<% end %>
</h2>
</div>
<ul class="uk-breadcrumb">
<li><a href="/">Library</a></li>
<%- title.parents.each do |t| -%>
<li><a href="/book/<%= t.id %>"><%= t.title %></a></li>
<li><a href="/book/<%= t.id %>"><%= t.display_name %></a></li>
<%- end -%>
<li class="uk-disabled"><a><%= title.title %></a></li>
<li class="uk-disabled"><a><%= title.display_name %></a></li>
</ul>
<p class="uk-text-meta"><%= title.size %> entries found</p>
<div class="uk-grid-small" uk-grid>
@ -35,14 +42,10 @@
<a class="acard" href="/book/<%= t.id %>">
<div class="uk-card uk-card-default">
<div class="uk-card-media-top">
<%- if t.entries.size > 0 -%>
<img data-src="<%= t.entries[0].cover_url %>" data-width data-height alt="" uk-img>
<%- else -%>
<img data-src="/img/icon.png" data-width data-height alt="" uk-img>
<%- end -%>
<img data-src="<%= t.cover_url %>" data-width data-height alt="" uk-img>
</div>
<div class="uk-card-body">
<h3 class="uk-card-title break-word" data-title="<%= t.title.gsub("\"", "&quot;") %>"><%= t.title %></h3>
<h3 class="uk-card-title break-word" data-title="<%= t.display_name.gsub("\"", "&quot;") %>"><%= t.display_name %></h3>
<p><%= t.size %> entries</p>
</div>
</div>
@ -50,15 +53,15 @@
</div>
<%- end -%>
<%- title.entries.each_with_index do |e, i| -%>
<div class="item" data-mtime="<%= e.mtime.to_unix %>" data-progress="<%= percentage[i] %>">
<div class="item" data-mtime="<%= e.mtime.to_unix %>" data-progress="<%= percentage[i] %>" id="<%= e.id %>">
<a class="acard">
<div class="uk-card uk-card-default" onclick="showModal(&quot;<%= e.encoded_path %>&quot;, '<%= e.pages %>', <%= (percentage[i] * 100).round(1) %>, &quot;<%= title.encoded_title %>&quot;, &quot;<%= e.encoded_title %>&quot;, '<%= e.title_id %>', '<%= e.id %>')">
<div class="uk-card uk-card-default" onclick="showModal(&quot;<%= e.encoded_path %>&quot;, '<%= e.pages %>', <%= (percentage[i] * 100).round(1) %>, &quot;<%= title.encoded_display_name %>&quot;, &quot;<%= e.encoded_display_name %>&quot;, '<%= e.title_id %>', '<%= e.id %>')">
<div class="uk-card-media-top">
<img data-src="<%= e.cover_url %>" alt="" data-width data-height uk-img>
</div>
<div class="uk-card-body">
<div class="uk-card-badge uk-label"><%= (percentage[i] * 100).round(1) %>%</div>
<h3 class="uk-card-title break-word" data-title="<%= e.title.gsub("\"", "&quot;") %>"><%= e.title %></h3>
<h3 class="uk-card-title break-word" data-title="<%= e.display_name.gsub("\"", "&quot;") %>"><%= e.display_name %></h3>
<p><%= e.pages %> pages</p>
</div>
</div>
@ -71,7 +74,14 @@
<div class="uk-modal-dialog uk-margin-auto-vertical">
<button class="uk-modal-close-default" type="button" uk-close></button>
<div class="uk-modal-header">
<h3 class="uk-modal-title break-word" id="modal-title"></h3>
<div>
<h3 class="uk-modal-title break-word" id="modal-title"><span></span>
&nbsp;
<% if is_admin %>
<a class="uk-icon-button" uk-icon="icon:pencil"></a>
<% end %>
</h3>
</div>
<p class="uk-text-meta uk-margin-remove-bottom break-word" id="path-text"></p>
<p class="uk-text-meta uk-margin-remove-top" id="pages-text"></p>
</div>
@ -90,6 +100,55 @@
</div>
</div>
<div id="edit-modal" class="uk-flex-top" uk-modal>
<div class="uk-modal-dialog uk-margin-auto-vertical">
<button class="uk-modal-close-default" type="button" uk-close></button>
<div class="uk-modal-header">
<div>
<h3 class="uk-modal-title break-word" id="modal-title">Edit</h3>
</div>
</div>
<div class="uk-modal-body">
<div class="uk-margin">
<label class="uk-form-label" for="display-name">Display Name</label>
<div class="uk-inline">
<a class="uk-form-icon uk-form-icon-flip" uk-icon="icon:check"></a>
<input class="uk-input" type="text" name="display-name" id="display-name-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label">Cover Image</label>
<div class="uk-grid">
<div class="uk-width-1-2@s">
<img id="cover" data-title-cover="<%= title.cover_url %>" alt="" data-width data-height uk-img>
</div>
<div class="uk-width-1-2@s">
<div id="cover-upload" class="upload-field uk-placeholder uk-text-center uk-flex uk-flex-middle" data-title-id="<%= title.id %>">
<div>
<span uk-icon="icon: cloud-upload"></span>
<span class="uk-text-middle">Upload a cover image by dropping it here or</span>
<div uk-form-custom>
<input type="file" accept="image/jpeg, image/png">
<span class="uk-link">selecting one</span>
</div>
</div>
</div>
</div>
</div>
<progress id="upload-progress" class="uk-progress" value="0" max="100" hidden></progress>
</div>
<div id="title-progress-control" hidden>
<label class="uk-form-label">Progress</label>
<p class="uk-margin-remove-vertical">
<button id="read-btn" class="uk-button uk-button-default" onclick="updateProgress('<%= title.id %>', null, 1)">Mark all as read (100%)</button>
<button id="unread-btn" class="uk-button uk-button-default" onclick="updateProgress('<%= title.id %>', null, 0)">Mark all as unread (0%)</button>
</p>
</div>
</div>
</div>
</div>
<% content_for "script" do %>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
<script src="/js/dots.js"></script>