diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bbf1395..8c74353 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: - image: crystallang/crystal:0.36.1-alpine + image: crystallang/crystal:1.0.0-alpine steps: - uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 6e5f3d8..5b24dbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM crystallang/crystal:0.36.1-alpine AS builder +FROM crystallang/crystal:1.0.0-alpine AS builder WORKDIR /Mango COPY . . -RUN apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev +RUN apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev RUN make static || make static FROM library/alpine diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 5a31a26..8f58631 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -2,10 +2,10 @@ FROM arm32v7/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. -RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. -RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. -RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd .. +RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd .. +RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd .. +RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd .. COPY mango-arm32v7.o . diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index 3cd5939..29045cf 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -2,10 +2,10 @@ FROM arm64v8/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. -RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. -RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. -RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd .. +RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd .. +RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd .. +RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd .. COPY mango-arm64v8.o . diff --git a/README.md b/README.md index 3f1a274..3ca1113 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Mango -[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Discord](https://img.shields.io/discord/855633663425118228?label=discord)](http://discord.com/invite/ezKtacCp9Q) Mango is a self-hosted manga server and reader. Its features include @@ -13,7 +13,6 @@ Mango is a self-hosted manga server and reader. Its features include - Supports nested folders in library - Automatically stores reading progress - Thumbnail generation -- Built-in [MangaDex](https://mangadex.org/) downloader - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites - The web reader is responsive and works well on mobile, so there is no need for a mobile app - All the static files are embedded in the binary, so the deployment process is easy and painless @@ -52,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.22.0 + Mango - Manga Server and Web Reader. Version 0.23.0 Usage: @@ -87,7 +86,6 @@ log_level: info upload_path: ~/mango/uploads plugin_path: ~/mango/plugins download_timeout_seconds: 30 -page_margin: 30 disable_login: false default_username: "" auth_proxy_header_name: "" diff --git a/public/js/download.js b/public/js/download.js deleted file mode 100644 index 4d8504b..0000000 --- a/public/js/download.js +++ /dev/null @@ -1,285 +0,0 @@ -const downloadComponent = () => { - return { - chaptersLimit: 1000, - loading: false, - addingToDownload: false, - searchAvailable: false, - searchInput: '', - data: {}, - chapters: [], - mangaAry: undefined, // undefined: not searching; []: searched but no result - candidateManga: {}, - langChoice: 'All', - groupChoice: 'All', - chapterRange: '', - volumeRange: '', - - get languages() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - set.add(chp.language); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - get groups() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - Object.keys(chp.groups).forEach(g => { - set.add(g); - }); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - init() { - const tableObserver = new MutationObserver(() => { - console.log('table mutated'); - $("#selectable").selectable({ - filter: 'tr' - }); - }); - tableObserver.observe($('table').get(0), { - childList: true, - subtree: true - }); - - $.getJSON(`${base_url}api/admin/mangadex/expires`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); - return; - } - if (data.expires && data.expires > Math.floor(Date.now() / 1000)) - this.searchAvailable = true; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - }, - - filtersUpdated() { - if (!this.data.chapters) - this.chapters = []; - const filters = { - chapter: this.parseRange(this.chapterRange), - volume: this.parseRange(this.volumeRange), - lang: this.langChoice, - group: this.groupChoice - }; - console.log('filters:', filters); - let _chapters = this.data.chapters.slice(); - Object.entries(filters).forEach(([k, v]) => { - if (v === 'All') return; - if (k === 'group') { - _chapters = _chapters.filter(c => { - const unescaped_groups = Object.entries(c.groups).map(([g, id]) => this.unescapeHTML(g)); - return unescaped_groups.indexOf(v) >= 0; - }); - return; - } - if (k === 'lang') { - _chapters = _chapters.filter(c => c.language === v); - return; - } - const lb = parseFloat(v[0]); - const ub = parseFloat(v[1]); - if (isNaN(lb) && isNaN(ub)) return; - _chapters = _chapters.filter(c => { - const val = parseFloat(c[k]); - if (isNaN(val)) return false; - if (isNaN(lb)) - return val <= ub; - else if (isNaN(ub)) - return val >= lb; - else - return val >= lb && val <= ub; - }); - }); - console.log('filtered chapters:', _chapters); - this.chapters = _chapters; - }, - - search() { - if (this.loading || this.searchInput === '') return; - this.data = {}; - this.mangaAry = undefined; - - var int_id = -1; - try { - const path = new URL(this.searchInput).pathname; - const match = /\/(?:title|manga)\/([0-9]+)/.exec(path); - int_id = parseInt(match[1]); - } catch (e) { - int_id = parseInt(this.searchInput); - } - - if (!isNaN(int_id) && int_id > 0) { - // The input is a positive integer. We treat it as an ID. - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/manga/${int_id}`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to get manga info. Error: ' + data.error); - return; - } - - this.data = data; - this.chapters = data.chapters; - this.mangaAry = undefined; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to get manga info. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } else { - if (!this.searchAvailable) { - alert('danger', 'Please make sure you are using a valid manga ID or manga URL from Mangadex. If you are trying to search MangaDex with a search term, please log in to MangaDex first by going to "Admin -> Connect to MangaDex".'); - return; - } - - // Search as a search term - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/search?${$.param({ - query: this.searchInput - })}`) - .done((data) => { - if (data.error) { - alert('danger', `Failed to search MangaDex. Error: ${data.error}`); - return; - } - - this.mangaAry = data.manga; - this.data = {}; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to search MangaDex. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } - }, - - parseRange(str) { - const regex = /^[\t ]*(?:(?:(<|<=|>|>=)[\t ]*([0-9]+))|(?:([0-9]+))|(?:([0-9]+)[\t ]*-[\t ]*([0-9]+))|(?:[\t ]*))[\t ]*$/m; - const matches = str.match(regex); - var num; - - if (!matches) { - return [null, null]; - } else if (typeof matches[1] !== 'undefined' && typeof matches[2] !== 'undefined') { - // e.g., <= 30 - num = parseInt(matches[2]); - if (isNaN(num)) { - return [null, null]; - } - switch (matches[1]) { - case '<': - return [null, num - 1]; - case '<=': - return [null, num]; - case '>': - return [num + 1, null]; - case '>=': - return [num, null]; - } - } else if (typeof matches[3] !== 'undefined') { - // a single number - num = parseInt(matches[3]); - if (isNaN(num)) { - return [null, null]; - } - return [num, num]; - } else if (typeof matches[4] !== 'undefined' && typeof matches[5] !== 'undefined') { - // e.g., 10 - 23 - num = parseInt(matches[4]); - const n2 = parseInt(matches[5]); - if (isNaN(num) || isNaN(n2) || num > n2) { - return [null, null]; - } - return [num, n2]; - } else { - // empty or space only - return [null, null]; - } - }, - - unescapeHTML(str) { - var elt = document.createElement("span"); - elt.innerHTML = str; - return elt.innerText; - }, - - selectAll() { - $('tbody > tr').each((i, e) => { - $(e).addClass('ui-selected'); - }); - }, - - clearSelection() { - $('tbody > tr').each((i, e) => { - $(e).removeClass('ui-selected'); - }); - }, - - download() { - const selected = $('tbody > tr.ui-selected'); - if (selected.length === 0) return; - UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => { - const ids = selected.map((i, e) => { - return parseInt($(e).find('td').first().text()); - }).get(); - const chapters = this.chapters.filter(c => ids.indexOf(c.id) >= 0); - console.log(ids); - this.addingToDownload = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/download`, - data: JSON.stringify({ - chapters: chapters - }), - contentType: "application/json", - dataType: 'json' - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`); - return; - } - const successCount = parseInt(data.success); - const failCount = parseInt(data.fail); - alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.addingToDownload = false; - }); - }); - }, - - chooseManga(manga) { - this.candidateManga = manga; - UIkit.modal($('#modal').get(0)).show(); - }, - - confirmManga(id) { - UIkit.modal($('#modal').get(0)).hide(); - this.searchInput = id; - this.search(); - } - }; -}; diff --git a/public/js/mangadex.js b/public/js/mangadex.js deleted file mode 100644 index 3271c4b..0000000 --- a/public/js/mangadex.js +++ /dev/null @@ -1,61 +0,0 @@ -const component = () => { - return { - username: '', - password: '', - expires: undefined, - loading: true, - loggingIn: false, - - init() { - this.loading = true; - $.ajax({ - type: 'GET', - url: `${base_url}api/admin/mangadex/expires`, - contentType: "application/json", - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to retrieve MangaDex token status. Error: ${data.error}`); - return; - } - this.expires = data.expires; - this.loading = false; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to retrieve MangaDex token status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }); - }, - login() { - if (!(this.username && this.password)) return; - this.loggingIn = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/login`, - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({ - username: this.username, - password: this.password - }) - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to log in. Error: ${data.error}`); - return; - } - this.expires = data.expires; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to log in. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loggingIn = false; - }); - }, - get expired() { - return this.expires && moment().diff(moment.unix(this.expires)) > 0; - } - }; -}; diff --git a/public/js/reader.js b/public/js/reader.js index f227972..9b17276 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -6,11 +6,13 @@ const readerComponent = () => { alertClass: 'uk-alert-primary', items: [], curItem: {}, + enableFlipAnimation: true, flipAnimation: null, longPages: false, lastSavedPage: page, selectedIndex: 0, // 0: not selected; 1: the first page margin: 30, + preloadLookahead: 3, /** * Initialize the component by fetching the page dimensions @@ -52,6 +54,16 @@ const readerComponent = () => { if (savedMargin) { this.margin = savedMargin; } + + // Preload Images + this.preloadLookahead = +(localStorage.getItem('preloadLookahead') ?? 3); + const limit = Math.min(page + this.preloadLookahead, this.items.length + 1); + for (let idx = page + 1; idx <= limit; idx++) { + this.preloadImage(this.items[idx - 1].url); + } + + const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); + this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -60,6 +72,12 @@ const readerComponent = () => { this.msg = errMsg; }) }, + /** + * Preload an image, which is expected to be cached + */ + preloadImage(url) { + (new Image()).src = url; + }, /** * Handles the `change` event for the page selector */ @@ -111,12 +129,18 @@ const readerComponent = () => { if (newIdx <= 0 || newIdx > this.items.length) return; + if (newIdx + this.preloadLookahead < this.items.length + 1) { + this.preloadImage(this.items[newIdx + this.preloadLookahead - 1].url); + } + this.toPage(newIdx); - if (isNext) - this.flipAnimation = 'right'; - else - this.flipAnimation = 'left'; + if (this.enableFlipAnimation) { + if (isNext) + this.flipAnimation = 'right'; + else + this.flipAnimation = 'left'; + } setTimeout(() => { this.flipAnimation = null; @@ -287,6 +311,14 @@ const readerComponent = () => { marginChanged() { localStorage.setItem('margin', this.margin); this.toPage(this.selectedIndex); - } + }, + + preloadLookaheadChanged() { + localStorage.setItem('preloadLookahead', this.preloadLookahead); + }, + + enableFlipAnimationChanged() { + localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation); + }, }; } diff --git a/shard.lock b/shard.lock index c22f839..df6fc4c 100644 --- a/shard.lock +++ b/shard.lock @@ -2,31 +2,31 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 0.14.0 + version: 0.14.3 archive: git: https://github.com/hkalexling/archive.cr.git - version: 0.4.0 + version: 0.5.0 baked_file_system: git: https://github.com/schovi/baked_file_system.git - version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b + version: 0.10.0 clim: git: https://github.com/at-grandpa/clim.git - version: 0.12.0 + version: 0.17.1 db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.9.0 + version: 0.10.1 duktape: git: https://github.com/jessedoyle/duktape.cr.git - version: 0.20.0 + version: 1.0.0 exception_page: git: https://github.com/crystal-loot/exception_page.git - version: 0.1.4 + version: 0.1.5 http_proxy: git: https://github.com/mamantoha/http_proxy.git @@ -34,49 +34,45 @@ shards: image_size: git: https://github.com/hkalexling/image_size.cr.git - version: 0.4.0 + version: 0.5.0 kemal: git: https://github.com/kemalcr/kemal.git - version: 0.27.0 + version: 1.0.0 kemal-session: git: https://github.com/kemalcr/kemal-session.git - version: 0.13.0 + version: 1.0.0 kilt: git: https://github.com/jeromegn/kilt.git - version: 0.4.0 + version: 0.4.1 koa: git: https://github.com/hkalexling/koa.git - version: 0.7.0 - - mangadex: - git: https://github.com/hkalexling/mangadex.git - version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6 + version: 0.8.0 mg: git: https://github.com/hkalexling/mg.git - version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3 + version: 0.5.0+git.commit.697e46e27cde8c3969346e228e372db2455a6264 myhtml: git: https://github.com/kostya/myhtml.git - version: 1.5.1 + version: 1.5.8 open_api: - git: https://github.com/jreinert/open_api.cr.git - version: 1.2.1+git.commit.95e4df2ca10b1fe88b8b35c62a18b06a10267b6c + git: https://github.com/hkalexling/open_api.cr.git + version: 1.2.1+git.commit.1d3c55dd5534c6b0af18964d031858a08515553a radix: git: https://github.com/luislavena/radix.git - version: 0.3.9 + version: 0.4.1 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git - version: 0.16.0 + version: 0.18.0 tallboy: git: https://github.com/epoch/tallboy.git - version: 0.9.3 + version: 0.9.3+git.commit.9be1510bb0391c95e92f1b288f3afb429a73caa6 diff --git a/shard.yml b/shard.yml index 3991a13..1da553d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.22.0 +version: 0.23.0 authors: - Alex Ling @@ -8,7 +8,7 @@ targets: mango: main: src/mango.cr -crystal: 0.36.1 +crystal: 1.0.0 license: MIT @@ -21,7 +21,6 @@ dependencies: github: crystal-lang/crystal-sqlite3 baked_file_system: github: schovi/baked_file_system - version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b archive: github: hkalexling/archive.cr ameba: @@ -30,7 +29,6 @@ dependencies: github: at-grandpa/clim duktape: github: jessedoyle/duktape.cr - version: ~> 0.20.0 myhtml: github: kostya/myhtml http_proxy: @@ -41,7 +39,6 @@ dependencies: github: hkalexling/koa tallboy: github: epoch/tallboy + branch: master mg: github: hkalexling/mg - mangadex: - github: hkalexling/mangadex diff --git a/src/logger.cr b/src/logger.cr index d434cfa..040e5aa 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -34,7 +34,11 @@ class Logger end @backend.formatter = Log::Formatter.new &format_proc - Log.setup @@severity, @backend + + Log.setup do |c| + c.bind "*", @@severity, @backend + c.bind "db.*", :error, @backend + end end def self.get_severity(level = "") : Log::Severity diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr deleted file mode 100644 index e677a71..0000000 --- a/src/mangadex/downloader.cr +++ /dev/null @@ -1,172 +0,0 @@ -require "mangadex" -require "compress/zip" -require "../rename" -require "./ext" - -module MangaDex - class PageJob - property success = false - property url : String - property filename : String - property writer : Compress::Zip::Writer - property tries_remaning : Int32 - - def initialize(@url, @filename, @writer, @tries_remaning) - end - end - - class Downloader < Queue::Downloader - @wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"] - .to_i32 - @retries : Int32 = Config.current.mangadex["download_retries"].to_i32 - - use_default - - def initialize - @client = Client.from_config - super - end - - def pop : Queue::Job? - job = nil - MainFiber.run do - DB.open "sqlite3://#{@queue.path}" do |db| - begin - db.query_one "select * from queue where id not like '%-%' " \ - "and (status = 0 or status = 1) " \ - "order by time limit 1" do |res| - job = Queue::Job.from_query_result res - end - rescue - end - end - end - job - end - - private def download(job : Queue::Job) - @downloading = true - @queue.set_status Queue::JobStatus::Downloading, job - begin - chapter = @client.chapter job.id - # We must put the `.pages` call in a rescue block to handle external - # chapters. - pages = chapter.pages - rescue e - Logger.error e - @queue.set_status Queue::JobStatus::Error, job - unless e.message.nil? - @queue.add_message e.message.not_nil!, job - end - @downloading = false - return - end - @queue.set_pages pages.size, job - lib_dir = @library_path - rename_rule = Rename::Rule.new \ - Config.current.mangadex["manga_rename_rule"].to_s - manga_dir = File.join lib_dir, chapter.manga.rename rename_rule - unless File.exists? manga_dir - Dir.mkdir_p manga_dir - end - zip_path = File.join manga_dir, "#{job.title}.cbz.part" - - # Find the number of digits needed to store the number of pages - len = Math.log10(pages.size).to_i + 1 - - writer = Compress::Zip::Writer.new zip_path - # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new pages.size - spawn do - pages.each_with_index do |url, i| - fn = Path.new(URI.parse(url).path).basename - ext = File.extname fn - fn = "#{i.to_s.rjust len, '0'}#{ext}" - page_job = PageJob.new url, fn, writer, @retries - Logger.debug "Downloading #{url}" - loop do - sleep @wait_seconds.seconds - download_page page_job - break if page_job.success || - page_job.tries_remaning <= 0 - page_job.tries_remaning -= 1 - Logger.warn "Failed to download page #{url}. " \ - "Retrying... Remaining retries: " \ - "#{page_job.tries_remaning}" - end - - channel.send page_job - break unless @queue.exists? job - end - end - - spawn do - page_jobs = [] of PageJob - pages.size.times do - page_job = channel.receive - - break unless @queue.exists? job - - Logger.debug "[#{page_job.success ? "success" : "failed"}] " \ - "#{page_job.url}" - page_jobs << page_job - if page_job.success - @queue.add_success job - else - @queue.add_fail job - msg = "Failed to download page #{page_job.url}" - @queue.add_message msg, job - Logger.error msg - end - end - - unless @queue.exists? job - Logger.debug "Download cancelled" - @downloading = false - next - end - - fail_count = page_jobs.count { |j| !j.success } - Logger.debug "Download completed. " \ - "#{fail_count}/#{page_jobs.size} failed" - writer.close - filename = File.join File.dirname(zip_path), File.basename(zip_path, - ".part") - File.rename zip_path, filename - Logger.debug "cbz File created at #{filename}" - - zip_exception = validate_archive filename - if !zip_exception.nil? - @queue.add_message "The downloaded archive is corrupted. " \ - "Error: #{zip_exception}", job - @queue.set_status Queue::JobStatus::Error, job - elsif fail_count > 0 - @queue.set_status Queue::JobStatus::MissingPages, job - else - @queue.set_status Queue::JobStatus::Completed, job - end - @downloading = false - end - end - - private def download_page(job : PageJob) - Logger.debug "downloading #{job.url}" - headers = HTTP::Headers{ - "User-agent" => "Mangadex.cr", - } - begin - HTTP::Client.get job.url, headers do |res| - unless res.success? - raise "Failed to download page #{job.url}. " \ - "[#{res.status_code}] #{res.status_message}" - end - job.writer.add job.filename, res.body_io - end - job.success = true - rescue e - Logger.error e - job.success = false - end - end - end -end diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr deleted file mode 100644 index deb09c8..0000000 --- a/src/mangadex/ext.cr +++ /dev/null @@ -1,60 +0,0 @@ -private macro properties_to_hash(names) - { - {% for name in names %} - "{{name.id}}" => {{name.id}}.to_s, - {% end %} - } -end - -# Monkey-patch the structures in the `mangadex` shard to suit our needs -module MangaDex - struct Client - @@group_cache = {} of String => Group - - def self.from_config : Client - self.new base_url: Config.current.mangadex["base_url"].to_s, - api_url: Config.current.mangadex["api_url"].to_s - end - end - - struct Manga - def rename(rule : Rename::Rule) - rule.render properties_to_hash %w(id title author artist) - end - - def to_info_json - hash = JSON.parse(to_json).as_h - _chapters = chapters.map do |c| - JSON.parse c.to_info_json - end - hash["chapters"] = JSON::Any.new _chapters - hash.to_json - end - end - - struct Chapter - def rename(rule : Rename::Rule) - hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.join(",", &.name) - rule.render hash - end - - def full_title - rule = Rename::Rule.new \ - Config.current.mangadex["chapter_rename_rule"].to_s - rename rule - end - - def to_info_json - hash = JSON.parse(to_json).as_h - hash["language"] = JSON::Any.new language - _groups = {} of String => JSON::Any - groups.each do |g| - _groups[g.name] = JSON::Any.new g.id - end - hash["groups"] = JSON::Any.new _groups - hash["full_title"] = JSON::Any.new full_title - hash.to_json - end - end -end diff --git a/src/mango.cr b/src/mango.cr index 768058f..e8d32a3 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -2,13 +2,12 @@ require "./config" require "./queue" require "./server" require "./main_fiber" -require "./mangadex/*" require "./plugin/*" require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.22.0" +MANGO_VERSION = "0.23.0" # From http://www.network-science.de/ascii/ BANNER = %{ @@ -59,7 +58,6 @@ class CLI < Clim Storage.default Queue.default Library.default - MangaDex::Downloader.default Plugin::Downloader.default spawn do diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 616d452..fd63ec8 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -73,9 +73,5 @@ struct AdminRouter get "/admin/missing" do |env| layout "missing-items" end - - get "/admin/mangadex" do |env| - layout "mangadex" - end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 4a403e9..85df516 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1,6 +1,6 @@ -require "../mangadex/*" require "../upload" require "koa" +require "digest" struct APIRouter @@api_json : String? @@ -56,31 +56,20 @@ struct APIRouter "error" => String?, } - Koa.schema("mdChapter", { - "id" => Int64, - "group" => {} of String => String, - }.merge(s %w(title volume chapter language full_title time - manga_title manga_id)), - desc: "A MangaDex chapter") - - Koa.schema "mdManga", { - "id" => Int64, - "chapters" => ["mdChapter"], - }.merge(s %w(title description author artist cover_url)), - desc: "A MangaDex manga" - Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.path "page", schema: Int32, desc: "The page number to return (starts from 1)" Koa.response 200, schema: Bytes, media_type: "image/*" Koa.response 500, "Page not found or not readable" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.tag "reader" get "/api/page/:tid/:eid/:page" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] page = env.params.url["page"].to_i + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -90,7 +79,15 @@ struct APIRouter raise "Failed to load page #{page} of " \ "`#{title.title}/#{entry.title}`" if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -102,12 +99,14 @@ struct APIRouter Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.response 200, schema: Bytes, media_type: "image/*" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.response 500, "Page not found or not readable" Koa.tag "library" get "/api/cover/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -118,7 +117,14 @@ struct APIRouter raise "Failed to get cover of `#{title.title}/#{entry.title}`" \ if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -323,58 +329,6 @@ struct APIRouter end end - Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex"] - Koa.path "id", desc: "A MangaDex manga ID" - Koa.response 200, schema: "mdManga" - get "/api/admin/mangadex/manga/:id" do |env| - begin - id = env.params.url["id"] - manga = MangaDex::Client.from_config.manga id - send_json env, manga.to_info_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - - Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex", "downloader"] - Koa.body schema: { - "chapters" => ["mdChapter"], - } - Koa.response 200, schema: { - "success" => Int32, - "fail" => Int32, - } - post "/api/admin/mangadex/download" do |env| - begin - chapters = env.params.json["chapters"].as(Array).map &.as_h - jobs = chapters.map { |chapter| - Queue::Job.new( - chapter["id"].as_i64.to_s, - chapter["mangaId"].as_i64.to_s, - chapter["full_title"].as_s, - chapter["mangaTitle"].as_s, - Queue::JobStatus::Pending, - Time.unix chapter["timestamp"].as_i64 - ) - } - inserted_count = Queue.default.push jobs - send_json env, { - "success": inserted_count, - "fail": jobs.size - inserted_count, - }.to_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 @@ -755,21 +709,32 @@ struct APIRouter "height" => Int32, }], } + Koa.response 304, "Not modified (only available when `If-None-Match` is set)" get "/api/dimensions/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - sizes = entry.page_dimensions - send_json env, { - "success" => true, - "dimensions" => sizes, - }.to_json + file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s) + e_tag = "W/#{file_hash}" + if e_tag == prev_e_tag + env.response.status_code = 304 + "" + else + sizes = entry.page_dimensions + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_json env, { + "success" => true, + "dimensions" => sizes, + }.to_json + end rescue e Logger.error e send_json env, { @@ -1028,115 +993,6 @@ struct APIRouter end end - Koa.describe "Logs the current user into their MangaDex account", <<-MD - If successful, returns the expiration date (as a unix timestamp) of the newly created token. - MD - Koa.body schema: { - "username" => String, - "password" => String, - } - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - post "/api/admin/mangadex/login" do |env| - begin - username = env.params.json["username"].as String - password = env.params.json["password"].as String - mango_username = get_username env - - client = MangaDex::Client.from_config - client.auth username, password - - Storage.default.save_md_token mango_username, client.token.not_nil!, - client.token_expires - - send_json env, { - "success" => true, - "error" => nil, - "expires" => client.token_expires.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - get "/api/admin/mangadex/expires" do |env| - begin - username = get_username env - _, expires = Storage.default.get_md_token username - - send_json env, { - "success" => true, - "error" => nil, - "expires" => expires.try &.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Searches MangaDex for manga matching `query`", <<-MD - Returns an empty list if the current user hasn't logged in to MangaDex. - MD - Koa.query "query" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "manga?" => [{ - "id" => Int64, - "title" => String, - "description" => String, - "mainCover" => String, - }], - } - Koa.tags ["admin", "mangadex"] - get "/api/admin/mangadex/search" do |env| - begin - username = get_username env - token, expires = Storage.default.get_md_token username - - unless expires && token - raise "No token found for user #{username}" - end - - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires - - query = env.params.query["query"] - - send_json env, { - "success" => true, - "error" => nil, - "manga" => client.partial_search query, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/routes/main.cr b/src/routes/main.cr index 80bc16d..5b00f67 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -72,11 +72,6 @@ struct MainRouter end end - get "/download" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] - layout "download" - end - get "/download/plugins" do |env| begin layout "plugin-download" diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index a6e9b31..fb64d3e 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -33,7 +33,6 @@ -
  • Connect to MangaDex
    • diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 73a5445..c264177 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -5,63 +5,61 @@ -
    - - - - - - - - - - +
    ChapterMangaProgressTimeStatusPluginActions
    + + + + + + + + + + + + + + +
    ChapterMangaProgressTimeStatusPluginActions
    +
    <% content_for "script" do %> diff --git a/src/views/layout.html.ecr b/src/views/layout.html.ecr index ff4853c..70c5a51 100644 --- a/src/views/layout.html.ecr +++ b/src/views/layout.html.ecr @@ -1,89 +1,87 @@ - <%= render_component "head" %> + <%= render_component "head" %> - -
    -
    -
    -
    - -
    -
    -
    + +
    +
    +
    +
    + +
    -
    -
    -
    -
    -
    -
    - - -
    -
    - -
    -
    +
    +
    +
    +
    +
    +
    -
    -
    -
    -
    -
    - <%= content %> -
    - +
    + +
    + + <% end %> +
    - - <%= render_component "uikit" %> - <%= yield_content "script" %> - +
    + +
    +
    +
    +
    +
    +
    +
    +
    + <%= content %> +
    + +
    +
    +
    + + <%= render_component "uikit" %> + <%= yield_content "script" %> + diff --git a/src/views/missing-items.html.ecr b/src/views/missing-items.html.ecr index 024960b..334e185 100644 --- a/src/views/missing-items.html.ecr +++ b/src/views/missing-items.html.ecr @@ -3,36 +3,34 @@

    The following items were present in your library, but now we can't find them anymore. If you deleted them mistakenly, try to recover the files or folders, put them back to where they were, and rescan the library. Otherwise, you can safely delete them and the associated metadata using the buttons below to free up database space.

    -
    - - - - - - - +
    TypeRelative PathIDActions
    + + + + + + + + + + + + +
    TypeRelative PathIDActions
    diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 6b7bb11..0e46ed2 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -21,7 +21,7 @@
    -