Merge branch 'dev' into feature/plugin-v2

This commit is contained in:
Alex Ling 2021-09-04 02:25:19 +00:00
commit ae1c36263b
22 changed files with 291 additions and 988 deletions

View File

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
container:
image: crystallang/crystal:0.36.1-alpine
image: crystallang/crystal:1.0.0-alpine
steps:
- uses: actions/checkout@v2

View File

@ -1,9 +1,9 @@
FROM crystallang/crystal:0.36.1-alpine AS builder
FROM crystallang/crystal:1.0.0-alpine AS builder
WORKDIR /Mango
COPY . .
RUN apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev
RUN apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev
RUN make static || make static
FROM library/alpine

View File

@ -2,10 +2,10 @@ FROM arm32v7/ubuntu:18.04
RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd ..
RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd ..
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd ..
RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd ..
COPY mango-arm32v7.o .

View File

@ -2,10 +2,10 @@ FROM arm64v8/ubuntu:18.04
RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd ..
RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd ..
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd ..
RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd ..
COPY mango-arm64v8.o .

View File

@ -2,7 +2,7 @@
# Mango
[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Discord](https://img.shields.io/discord/855633663425118228?label=discord)](http://discord.com/invite/ezKtacCp9Q)
Mango is a self-hosted manga server and reader. Its features include
@ -13,7 +13,6 @@ Mango is a self-hosted manga server and reader. Its features include
- Supports nested folders in library
- Automatically stores reading progress
- Thumbnail generation
- Built-in [MangaDex](https://mangadex.org/) downloader
- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
- The web reader is responsive and works well on mobile, so there is no need for a mobile app
- All the static files are embedded in the binary, so the deployment process is easy and painless
@ -52,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
Mango - Manga Server and Web Reader. Version 0.22.0
Mango - Manga Server and Web Reader. Version 0.23.0
Usage:
@ -87,7 +86,6 @@ log_level: info
upload_path: ~/mango/uploads
plugin_path: ~/mango/plugins
download_timeout_seconds: 30
page_margin: 30
disable_login: false
default_username: ""
auth_proxy_header_name: ""

View File

@ -1,285 +0,0 @@
const downloadComponent = () => {
return {
chaptersLimit: 1000,
loading: false,
addingToDownload: false,
searchAvailable: false,
searchInput: '',
data: {},
chapters: [],
mangaAry: undefined, // undefined: not searching; []: searched but no result
candidateManga: {},
langChoice: 'All',
groupChoice: 'All',
chapterRange: '',
volumeRange: '',
get languages() {
const set = new Set();
if (this.data.chapters) {
this.data.chapters.forEach(chp => {
set.add(chp.language);
});
}
const ary = [...set].sort();
ary.unshift('All');
return ary;
},
get groups() {
const set = new Set();
if (this.data.chapters) {
this.data.chapters.forEach(chp => {
Object.keys(chp.groups).forEach(g => {
set.add(g);
});
});
}
const ary = [...set].sort();
ary.unshift('All');
return ary;
},
init() {
const tableObserver = new MutationObserver(() => {
console.log('table mutated');
$("#selectable").selectable({
filter: 'tr'
});
});
tableObserver.observe($('table').get(0), {
childList: true,
subtree: true
});
$.getJSON(`${base_url}api/admin/mangadex/expires`)
.done((data) => {
if (data.error) {
alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error);
return;
}
if (data.expires && data.expires > Math.floor(Date.now() / 1000))
this.searchAvailable = true;
})
.fail((jqXHR, status) => {
alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
})
},
filtersUpdated() {
if (!this.data.chapters)
this.chapters = [];
const filters = {
chapter: this.parseRange(this.chapterRange),
volume: this.parseRange(this.volumeRange),
lang: this.langChoice,
group: this.groupChoice
};
console.log('filters:', filters);
let _chapters = this.data.chapters.slice();
Object.entries(filters).forEach(([k, v]) => {
if (v === 'All') return;
if (k === 'group') {
_chapters = _chapters.filter(c => {
const unescaped_groups = Object.entries(c.groups).map(([g, id]) => this.unescapeHTML(g));
return unescaped_groups.indexOf(v) >= 0;
});
return;
}
if (k === 'lang') {
_chapters = _chapters.filter(c => c.language === v);
return;
}
const lb = parseFloat(v[0]);
const ub = parseFloat(v[1]);
if (isNaN(lb) && isNaN(ub)) return;
_chapters = _chapters.filter(c => {
const val = parseFloat(c[k]);
if (isNaN(val)) return false;
if (isNaN(lb))
return val <= ub;
else if (isNaN(ub))
return val >= lb;
else
return val >= lb && val <= ub;
});
});
console.log('filtered chapters:', _chapters);
this.chapters = _chapters;
},
search() {
if (this.loading || this.searchInput === '') return;
this.data = {};
this.mangaAry = undefined;
var int_id = -1;
try {
const path = new URL(this.searchInput).pathname;
const match = /\/(?:title|manga)\/([0-9]+)/.exec(path);
int_id = parseInt(match[1]);
} catch (e) {
int_id = parseInt(this.searchInput);
}
if (!isNaN(int_id) && int_id > 0) {
// The input is a positive integer. We treat it as an ID.
this.loading = true;
$.getJSON(`${base_url}api/admin/mangadex/manga/${int_id}`)
.done((data) => {
if (data.error) {
alert('danger', 'Failed to get manga info. Error: ' + data.error);
return;
}
this.data = data;
this.chapters = data.chapters;
this.mangaAry = undefined;
})
.fail((jqXHR, status) => {
alert('danger', `Failed to get manga info. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
})
.always(() => {
this.loading = false;
});
} else {
if (!this.searchAvailable) {
alert('danger', 'Please make sure you are using a valid manga ID or manga URL from Mangadex. If you are trying to search MangaDex with a search term, please log in to MangaDex first by going to "Admin -> Connect to MangaDex".');
return;
}
// Search as a search term
this.loading = true;
$.getJSON(`${base_url}api/admin/mangadex/search?${$.param({
query: this.searchInput
})}`)
.done((data) => {
if (data.error) {
alert('danger', `Failed to search MangaDex. Error: ${data.error}`);
return;
}
this.mangaAry = data.manga;
this.data = {};
})
.fail((jqXHR, status) => {
alert('danger', `Failed to search MangaDex. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
})
.always(() => {
this.loading = false;
});
}
},
parseRange(str) {
const regex = /^[\t ]*(?:(?:(<|<=|>|>=)[\t ]*([0-9]+))|(?:([0-9]+))|(?:([0-9]+)[\t ]*-[\t ]*([0-9]+))|(?:[\t ]*))[\t ]*$/m;
const matches = str.match(regex);
var num;
if (!matches) {
return [null, null];
} else if (typeof matches[1] !== 'undefined' && typeof matches[2] !== 'undefined') {
// e.g., <= 30
num = parseInt(matches[2]);
if (isNaN(num)) {
return [null, null];
}
switch (matches[1]) {
case '<':
return [null, num - 1];
case '<=':
return [null, num];
case '>':
return [num + 1, null];
case '>=':
return [num, null];
}
} else if (typeof matches[3] !== 'undefined') {
// a single number
num = parseInt(matches[3]);
if (isNaN(num)) {
return [null, null];
}
return [num, num];
} else if (typeof matches[4] !== 'undefined' && typeof matches[5] !== 'undefined') {
// e.g., 10 - 23
num = parseInt(matches[4]);
const n2 = parseInt(matches[5]);
if (isNaN(num) || isNaN(n2) || num > n2) {
return [null, null];
}
return [num, n2];
} else {
// empty or space only
return [null, null];
}
},
unescapeHTML(str) {
var elt = document.createElement("span");
elt.innerHTML = str;
return elt.innerText;
},
selectAll() {
$('tbody > tr').each((i, e) => {
$(e).addClass('ui-selected');
});
},
clearSelection() {
$('tbody > tr').each((i, e) => {
$(e).removeClass('ui-selected');
});
},
download() {
const selected = $('tbody > tr.ui-selected');
if (selected.length === 0) return;
UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => {
const ids = selected.map((i, e) => {
return parseInt($(e).find('td').first().text());
}).get();
const chapters = this.chapters.filter(c => ids.indexOf(c.id) >= 0);
console.log(ids);
this.addingToDownload = true;
$.ajax({
type: 'POST',
url: `${base_url}api/admin/mangadex/download`,
data: JSON.stringify({
chapters: chapters
}),
contentType: "application/json",
dataType: 'json'
})
.done(data => {
console.log(data);
if (data.error) {
alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`);
return;
}
const successCount = parseInt(data.success);
const failCount = parseInt(data.fail);
alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the <a href="${base_url}admin/downloads">download manager page</a>.`);
})
.fail((jqXHR, status) => {
alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
})
.always(() => {
this.addingToDownload = false;
});
});
},
chooseManga(manga) {
this.candidateManga = manga;
UIkit.modal($('#modal').get(0)).show();
},
confirmManga(id) {
UIkit.modal($('#modal').get(0)).hide();
this.searchInput = id;
this.search();
}
};
};

View File

@ -1,61 +0,0 @@
const component = () => {
return {
username: '',
password: '',
expires: undefined,
loading: true,
loggingIn: false,
init() {
this.loading = true;
$.ajax({
type: 'GET',
url: `${base_url}api/admin/mangadex/expires`,
contentType: "application/json",
})
.done(data => {
console.log(data);
if (data.error) {
alert('danger', `Failed to retrieve MangaDex token status. Error: ${data.error}`);
return;
}
this.expires = data.expires;
this.loading = false;
})
.fail((jqXHR, status) => {
alert('danger', `Failed to retrieve MangaDex token status. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
});
},
login() {
if (!(this.username && this.password)) return;
this.loggingIn = true;
$.ajax({
type: 'POST',
url: `${base_url}api/admin/mangadex/login`,
contentType: "application/json",
dataType: 'json',
data: JSON.stringify({
username: this.username,
password: this.password
})
})
.done(data => {
console.log(data);
if (data.error) {
alert('danger', `Failed to log in. Error: ${data.error}`);
return;
}
this.expires = data.expires;
})
.fail((jqXHR, status) => {
alert('danger', `Failed to log in. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
})
.always(() => {
this.loggingIn = false;
});
},
get expired() {
return this.expires && moment().diff(moment.unix(this.expires)) > 0;
}
};
};

View File

@ -6,11 +6,13 @@ const readerComponent = () => {
alertClass: 'uk-alert-primary',
items: [],
curItem: {},
enableFlipAnimation: true,
flipAnimation: null,
longPages: false,
lastSavedPage: page,
selectedIndex: 0, // 0: not selected; 1: the first page
margin: 30,
preloadLookahead: 3,
/**
* Initialize the component by fetching the page dimensions
@ -52,6 +54,16 @@ const readerComponent = () => {
if (savedMargin) {
this.margin = savedMargin;
}
// Preload Images
this.preloadLookahead = +(localStorage.getItem('preloadLookahead') ?? 3);
const limit = Math.min(page + this.preloadLookahead, this.items.length + 1);
for (let idx = page + 1; idx <= limit; idx++) {
this.preloadImage(this.items[idx - 1].url);
}
const savedFlipAnimation = localStorage.getItem('enableFlipAnimation');
this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true';
})
.catch(e => {
const errMsg = `Failed to get the page dimensions. ${e}`;
@ -60,6 +72,12 @@ const readerComponent = () => {
this.msg = errMsg;
})
},
/**
* Preload an image, which is expected to be cached
*/
preloadImage(url) {
(new Image()).src = url;
},
/**
* Handles the `change` event for the page selector
*/
@ -111,12 +129,18 @@ const readerComponent = () => {
if (newIdx <= 0 || newIdx > this.items.length) return;
if (newIdx + this.preloadLookahead < this.items.length + 1) {
this.preloadImage(this.items[newIdx + this.preloadLookahead - 1].url);
}
this.toPage(newIdx);
if (this.enableFlipAnimation) {
if (isNext)
this.flipAnimation = 'right';
else
this.flipAnimation = 'left';
}
setTimeout(() => {
this.flipAnimation = null;
@ -287,6 +311,14 @@ const readerComponent = () => {
marginChanged() {
localStorage.setItem('margin', this.margin);
this.toPage(this.selectedIndex);
}
},
preloadLookaheadChanged() {
localStorage.setItem('preloadLookahead', this.preloadLookahead);
},
enableFlipAnimationChanged() {
localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation);
},
};
}

View File

@ -2,31 +2,31 @@ version: 2.0
shards:
ameba:
git: https://github.com/crystal-ameba/ameba.git
version: 0.14.0
version: 0.14.3
archive:
git: https://github.com/hkalexling/archive.cr.git
version: 0.4.0
version: 0.5.0
baked_file_system:
git: https://github.com/schovi/baked_file_system.git
version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b
version: 0.10.0
clim:
git: https://github.com/at-grandpa/clim.git
version: 0.12.0
version: 0.17.1
db:
git: https://github.com/crystal-lang/crystal-db.git
version: 0.9.0
version: 0.10.1
duktape:
git: https://github.com/jessedoyle/duktape.cr.git
version: 0.20.0
version: 1.0.0
exception_page:
git: https://github.com/crystal-loot/exception_page.git
version: 0.1.4
version: 0.1.5
http_proxy:
git: https://github.com/mamantoha/http_proxy.git
@ -34,49 +34,45 @@ shards:
image_size:
git: https://github.com/hkalexling/image_size.cr.git
version: 0.4.0
version: 0.5.0
kemal:
git: https://github.com/kemalcr/kemal.git
version: 0.27.0
version: 1.0.0
kemal-session:
git: https://github.com/kemalcr/kemal-session.git
version: 0.13.0
version: 1.0.0
kilt:
git: https://github.com/jeromegn/kilt.git
version: 0.4.0
version: 0.4.1
koa:
git: https://github.com/hkalexling/koa.git
version: 0.7.0
mangadex:
git: https://github.com/hkalexling/mangadex.git
version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6
version: 0.8.0
mg:
git: https://github.com/hkalexling/mg.git
version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3
version: 0.5.0+git.commit.697e46e27cde8c3969346e228e372db2455a6264
myhtml:
git: https://github.com/kostya/myhtml.git
version: 1.5.1
version: 1.5.8
open_api:
git: https://github.com/jreinert/open_api.cr.git
version: 1.2.1+git.commit.95e4df2ca10b1fe88b8b35c62a18b06a10267b6c
git: https://github.com/hkalexling/open_api.cr.git
version: 1.2.1+git.commit.1d3c55dd5534c6b0af18964d031858a08515553a
radix:
git: https://github.com/luislavena/radix.git
version: 0.3.9
version: 0.4.1
sqlite3:
git: https://github.com/crystal-lang/crystal-sqlite3.git
version: 0.16.0
version: 0.18.0
tallboy:
git: https://github.com/epoch/tallboy.git
version: 0.9.3
version: 0.9.3+git.commit.9be1510bb0391c95e92f1b288f3afb429a73caa6

View File

@ -1,5 +1,5 @@
name: mango
version: 0.22.0
version: 0.23.0
authors:
- Alex Ling <hkalexling@gmail.com>
@ -8,7 +8,7 @@ targets:
mango:
main: src/mango.cr
crystal: 0.36.1
crystal: 1.0.0
license: MIT
@ -21,7 +21,6 @@ dependencies:
github: crystal-lang/crystal-sqlite3
baked_file_system:
github: schovi/baked_file_system
version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b
archive:
github: hkalexling/archive.cr
ameba:
@ -30,7 +29,6 @@ dependencies:
github: at-grandpa/clim
duktape:
github: jessedoyle/duktape.cr
version: ~> 0.20.0
myhtml:
github: kostya/myhtml
http_proxy:
@ -41,7 +39,6 @@ dependencies:
github: hkalexling/koa
tallboy:
github: epoch/tallboy
branch: master
mg:
github: hkalexling/mg
mangadex:
github: hkalexling/mangadex

View File

@ -34,7 +34,11 @@ class Logger
end
@backend.formatter = Log::Formatter.new &format_proc
Log.setup @@severity, @backend
Log.setup do |c|
c.bind "*", @@severity, @backend
c.bind "db.*", :error, @backend
end
end
def self.get_severity(level = "") : Log::Severity

View File

@ -1,172 +0,0 @@
require "mangadex"
require "compress/zip"
require "../rename"
require "./ext"
module MangaDex
class PageJob
property success = false
property url : String
property filename : String
property writer : Compress::Zip::Writer
property tries_remaning : Int32
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
class Downloader < Queue::Downloader
@wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"]
.to_i32
@retries : Int32 = Config.current.mangadex["download_retries"].to_i32
use_default
def initialize
@client = Client.from_config
super
end
def pop : Queue::Job?
job = nil
MainFiber.run do
DB.open "sqlite3://#{@queue.path}" do |db|
begin
db.query_one "select * from queue where id not like '%-%' " \
"and (status = 0 or status = 1) " \
"order by time limit 1" do |res|
job = Queue::Job.from_query_result res
end
rescue
end
end
end
job
end
private def download(job : Queue::Job)
@downloading = true
@queue.set_status Queue::JobStatus::Downloading, job
begin
chapter = @client.chapter job.id
# We must put the `.pages` call in a rescue block to handle external
# chapters.
pages = chapter.pages
rescue e
Logger.error e
@queue.set_status Queue::JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages pages.size, job
lib_dir = @library_path
rename_rule = Rename::Rule.new \
Config.current.mangadex["manga_rename_rule"].to_s
manga_dir = File.join lib_dir, chapter.manga.rename rename_rule
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz.part"
# Find the number of digits needed to store the number of pages
len = Math.log10(pages.size).to_i + 1
writer = Compress::Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new pages.size
spawn do
pages.each_with_index do |url, i|
fn = Path.new(URI.parse(url).path).basename
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
Logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
Logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
channel.send page_job
break unless @queue.exists? job
end
end
spawn do
page_jobs = [] of PageJob
pages.size.times do
page_job = channel.receive
break unless @queue.exists? job
Logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
Logger.error msg
end
end
unless @queue.exists? job
Logger.debug "Download cancelled"
@downloading = false
next
end
fail_count = page_jobs.count { |j| !j.success }
Logger.debug "Download completed. " \
"#{fail_count}/#{page_jobs.size} failed"
writer.close
filename = File.join File.dirname(zip_path), File.basename(zip_path,
".part")
File.rename zip_path, filename
Logger.debug "cbz File created at #{filename}"
zip_exception = validate_archive filename
if !zip_exception.nil?
@queue.add_message "The downloaded archive is corrupted. " \
"Error: #{zip_exception}", job
@queue.set_status Queue::JobStatus::Error, job
elsif fail_count > 0
@queue.set_status Queue::JobStatus::MissingPages, job
else
@queue.set_status Queue::JobStatus::Completed, job
end
@downloading = false
end
end
private def download_page(job : PageJob)
Logger.debug "downloading #{job.url}"
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
Logger.error e
job.success = false
end
end
end
end

View File

@ -1,60 +0,0 @@
private macro properties_to_hash(names)
{
{% for name in names %}
"{{name.id}}" => {{name.id}}.to_s,
{% end %}
}
end
# Monkey-patch the structures in the `mangadex` shard to suit our needs
module MangaDex
struct Client
@@group_cache = {} of String => Group
def self.from_config : Client
self.new base_url: Config.current.mangadex["base_url"].to_s,
api_url: Config.current.mangadex["api_url"].to_s
end
end
struct Manga
def rename(rule : Rename::Rule)
rule.render properties_to_hash %w(id title author artist)
end
def to_info_json
hash = JSON.parse(to_json).as_h
_chapters = chapters.map do |c|
JSON.parse c.to_info_json
end
hash["chapters"] = JSON::Any.new _chapters
hash.to_json
end
end
struct Chapter
def rename(rule : Rename::Rule)
hash = properties_to_hash %w(id title volume chapter lang_code language)
hash["groups"] = groups.join(",", &.name)
rule.render hash
end
def full_title
rule = Rename::Rule.new \
Config.current.mangadex["chapter_rename_rule"].to_s
rename rule
end
def to_info_json
hash = JSON.parse(to_json).as_h
hash["language"] = JSON::Any.new language
_groups = {} of String => JSON::Any
groups.each do |g|
_groups[g.name] = JSON::Any.new g.id
end
hash["groups"] = JSON::Any.new _groups
hash["full_title"] = JSON::Any.new full_title
hash.to_json
end
end
end

View File

@ -2,13 +2,12 @@ require "./config"
require "./queue"
require "./server"
require "./main_fiber"
require "./mangadex/*"
require "./plugin/*"
require "option_parser"
require "clim"
require "tallboy"
MANGO_VERSION = "0.22.0"
MANGO_VERSION = "0.23.0"
# From http://www.network-science.de/ascii/
BANNER = %{
@ -59,7 +58,6 @@ class CLI < Clim
Storage.default
Queue.default
Library.default
MangaDex::Downloader.default
Plugin::Downloader.default
spawn do

View File

@ -73,9 +73,5 @@ struct AdminRouter
get "/admin/missing" do |env|
layout "missing-items"
end
get "/admin/mangadex" do |env|
layout "mangadex"
end
end
end

View File

@ -1,6 +1,6 @@
require "../mangadex/*"
require "../upload"
require "koa"
require "digest"
struct APIRouter
@@api_json : String?
@ -56,31 +56,20 @@ struct APIRouter
"error" => String?,
}
Koa.schema("mdChapter", {
"id" => Int64,
"group" => {} of String => String,
}.merge(s %w(title volume chapter language full_title time
manga_title manga_id)),
desc: "A MangaDex chapter")
Koa.schema "mdManga", {
"id" => Int64,
"chapters" => ["mdChapter"],
}.merge(s %w(title description author artist cover_url)),
desc: "A MangaDex manga"
Koa.describe "Returns a page in a manga entry"
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
Koa.path "page", schema: Int32, desc: "The page number to return (starts from 1)"
Koa.response 200, schema: Bytes, media_type: "image/*"
Koa.response 500, "Page not found or not readable"
Koa.response 304, "Page not modified (only available when `If-None-Match` is set)"
Koa.tag "reader"
get "/api/page/:tid/:eid/:page" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
page = env.params.url["page"].to_i
prev_e_tag = env.request.headers["If-None-Match"]?
title = Library.default.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
@ -90,7 +79,15 @@ struct APIRouter
raise "Failed to load page #{page} of " \
"`#{title.title}/#{entry.title}`" if img.nil?
e_tag = Digest::SHA1.hexdigest img.data
if prev_e_tag == e_tag
env.response.status_code = 304
""
else
env.response.headers["ETag"] = e_tag
env.response.headers["Cache-Control"] = "public, max-age=86400"
send_img env, img
end
rescue e
Logger.error e
env.response.status_code = 500
@ -102,12 +99,14 @@ struct APIRouter
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
Koa.response 200, schema: Bytes, media_type: "image/*"
Koa.response 304, "Page not modified (only available when `If-None-Match` is set)"
Koa.response 500, "Page not found or not readable"
Koa.tag "library"
get "/api/cover/:tid/:eid" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
prev_e_tag = env.request.headers["If-None-Match"]?
title = Library.default.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
@ -118,7 +117,14 @@ struct APIRouter
raise "Failed to get cover of `#{title.title}/#{entry.title}`" \
if img.nil?
e_tag = Digest::SHA1.hexdigest img.data
if prev_e_tag == e_tag
env.response.status_code = 304
""
else
env.response.headers["ETag"] = e_tag
send_img env, img
end
rescue e
Logger.error e
env.response.status_code = 500
@ -323,58 +329,6 @@ struct APIRouter
end
end
Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD
On error, returns a JSON that contains the error message in the `error` field.
MD
Koa.tags ["admin", "mangadex"]
Koa.path "id", desc: "A MangaDex manga ID"
Koa.response 200, schema: "mdManga"
get "/api/admin/mangadex/manga/:id" do |env|
begin
id = env.params.url["id"]
manga = MangaDex::Client.from_config.manga id
send_json env, manga.to_info_json
rescue e
Logger.error e
send_json env, {"error" => e.message}.to_json
end
end
Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD
On error, returns a JSON that contains the error message in the `error` field.
MD
Koa.tags ["admin", "mangadex", "downloader"]
Koa.body schema: {
"chapters" => ["mdChapter"],
}
Koa.response 200, schema: {
"success" => Int32,
"fail" => Int32,
}
post "/api/admin/mangadex/download" do |env|
begin
chapters = env.params.json["chapters"].as(Array).map &.as_h
jobs = chapters.map { |chapter|
Queue::Job.new(
chapter["id"].as_i64.to_s,
chapter["mangaId"].as_i64.to_s,
chapter["full_title"].as_s,
chapter["mangaTitle"].as_s,
Queue::JobStatus::Pending,
Time.unix chapter["timestamp"].as_i64
)
}
inserted_count = Queue.default.push jobs
send_json env, {
"success": inserted_count,
"fail": jobs.size - inserted_count,
}.to_json
rescue e
Logger.error e
send_json env, {"error" => e.message}.to_json
end
end
ws "/api/admin/mangadex/queue" do |socket, env|
interval_raw = env.params.query["interval"]?
interval = (interval_raw.to_i? if interval_raw) || 5
@ -755,21 +709,32 @@ struct APIRouter
"height" => Int32,
}],
}
Koa.response 304, "Not modified (only available when `If-None-Match` is set)"
get "/api/dimensions/:tid/:eid" do |env|
begin
tid = env.params.url["tid"]
eid = env.params.url["eid"]
prev_e_tag = env.request.headers["If-None-Match"]?
title = Library.default.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
entry = title.get_entry eid
raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil?
file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s)
e_tag = "W/#{file_hash}"
if e_tag == prev_e_tag
env.response.status_code = 304
""
else
sizes = entry.page_dimensions
env.response.headers["ETag"] = e_tag
env.response.headers["Cache-Control"] = "public, max-age=86400"
send_json env, {
"success" => true,
"dimensions" => sizes,
}.to_json
end
rescue e
Logger.error e
send_json env, {
@ -1028,115 +993,6 @@ struct APIRouter
end
end
Koa.describe "Logs the current user into their MangaDex account", <<-MD
If successful, returns the expiration date (as a unix timestamp) of the newly created token.
MD
Koa.body schema: {
"username" => String,
"password" => String,
}
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"expires" => Int64?,
}
Koa.tags ["admin", "mangadex", "users"]
post "/api/admin/mangadex/login" do |env|
begin
username = env.params.json["username"].as String
password = env.params.json["password"].as String
mango_username = get_username env
client = MangaDex::Client.from_config
client.auth username, password
Storage.default.save_md_token mango_username, client.token.not_nil!,
client.token_expires
send_json env, {
"success" => true,
"error" => nil,
"expires" => client.token_expires.to_unix,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists"
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"expires" => Int64?,
}
Koa.tags ["admin", "mangadex", "users"]
get "/api/admin/mangadex/expires" do |env|
begin
username = get_username env
_, expires = Storage.default.get_md_token username
send_json env, {
"success" => true,
"error" => nil,
"expires" => expires.try &.to_unix,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
Koa.describe "Searches MangaDex for manga matching `query`", <<-MD
Returns an empty list if the current user hasn't logged in to MangaDex.
MD
Koa.query "query"
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"manga?" => [{
"id" => Int64,
"title" => String,
"description" => String,
"mainCover" => String,
}],
}
Koa.tags ["admin", "mangadex"]
get "/api/admin/mangadex/search" do |env|
begin
username = get_username env
token, expires = Storage.default.get_md_token username
unless expires && token
raise "No token found for user #{username}"
end
client = MangaDex::Client.from_config
client.token = token
client.token_expires = expires
query = env.params.query["query"]
send_json env, {
"success" => true,
"error" => nil,
"manga" => client.partial_search query,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
doc = Koa.generate
@@api_json = doc.to_json if doc

View File

@ -72,11 +72,6 @@ struct MainRouter
end
end
get "/download" do |env|
mangadex_base_url = Config.current.mangadex["base_url"]
layout "download"
end
get "/download/plugins" do |env|
begin
layout "plugin-download"

View File

@ -33,7 +33,6 @@
<option>System</option>
</select>
</li>
<li><a class="uk-link-reset" href="<%= base_url %>admin/mangadex">Connect to MangaDex</a></li>
</ul>
<hr class="uk-divider-icon">

View File

@ -5,8 +5,7 @@
<button class="uk-button uk-button-default" @click="load()" :disabled="loading">Refresh Queue</button>
<button class="uk-button uk-button-default" x-show="paused !== undefined" x-text="paused ? 'Resume Download' : 'Pause Download'" @click="toggle()" :disabled="toggling"></button>
</div>
<div class="uk-overflow-auto">
<table class="uk-table uk-table-striped">
<table class="uk-table uk-table-striped uk-overflow-auto">
<thead>
<tr>
<th>Chapter</th>
@ -50,11 +49,10 @@
</td>
<td x-text="`${job.plugin_id || ''}`"></td>
<td>
<a @click="jobAction('delete', $event)" uk-icon="trash"></a>
<a @click="jobAction('delete', $event)" uk-icon="trash" uk-tooltip="Delete"></a>
<template x-if="job.status_message.length > 0">
<a @click="jobAction('retry', $event)" uk-icon="refresh"></a>
<a @click="jobAction('retry', $event)" uk-icon="refresh" uk-tooltip="Retry"></a>
</template>
</td>
</tr>

View File

@ -17,7 +17,6 @@
<li class="uk-parent">
<a href="#">Download</a>
<ul class="uk-nav-sub">
<li><a href="<%= base_url %>download">MangaDex</a></li>
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>
</ul>
@ -49,7 +48,6 @@
<div class="uk-navbar-dropdown">
<ul class="uk-nav uk-navbar-dropdown-nav">
<li class="uk-nav-header">Source</li>
<li><a href="<%= base_url %>download">MangaDex</a></li>
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li class="uk-nav-divider"></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>

View File

@ -3,8 +3,7 @@
<div x-show="!empty">
<p>The following items were present in your library, but now we can't find them anymore. If you deleted them mistakenly, try to recover the files or folders, put them back to where they were, and rescan the library. Otherwise, you can safely delete them and the associated metadata using the buttons below to free up database space.</p>
<button class="uk-button uk-button-danger" @click="rmAll()">Delete All</button>
<div class="uk-overflow-auto">
<table class="uk-table uk-table-striped">
<table class="uk-table uk-table-striped uk-overflow-auto">
<thead>
<tr>
<th>Type</th>
@ -34,7 +33,6 @@
</table>
</div>
</div>
</div>
<% content_for "script" do %>
<script src="<%= base_url %>js/alert.js"></script>

View File

@ -21,7 +21,7 @@
<div
:class="{'uk-container': true, 'uk-container-small': mode === 'continuous', 'uk-container-expand': mode !== 'continuous'}">
<div x-show="!loading && mode === 'continuous'" x-cloak>
<template x-for="item in items">
<template x-if="!loading && mode === 'continuous'" x-for="item in items">
<img
uk-img
:class="{'uk-align-center': true, 'spine': item.width < 50}"
@ -50,6 +50,9 @@
width:${mode === 'width' ? '100vw' : 'auto'};
height:${mode === 'height' ? '100vh' : 'auto'};
margin-bottom:0;
max-width:100%;
max-height:100%;
object-fit: contain;
`" />
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false)"></div>
@ -98,6 +101,19 @@
</div>
</div>
<div class="uk-margin uk-form-horizontal" x-show="mode !== 'continuous'">
<label class="uk-form-label" for="enable-flip-animation">Enable Flip Animation</label>
<div class="uk-form-controls">
<input id="enable-flip-animation" class="uk-checkbox" type="checkbox" x-model="enableFlipAnimation" @change="enableFlipAnimationChanged()">
</div>
</div>
<div class="uk-margin uk-form-horizontal" x-show="mode !== 'continuous'">
<label class="uk-form-label" for="preload-lookahead" x-text="`Preload Image: ${preloadLookahead} page(s)`"></label>
<div class="uk-form-controls">
<input id="preload-lookahead" class="uk-range" type="range" min="0" max="5" step="1" x-model.number="preloadLookahead" @change="preloadLookaheadChanged()">
</div>
</div>
<hr class="uk-divider-icon">
<div class="uk-margin">