diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1967b06..8c74353 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,12 +12,12 @@ jobs: runs-on: ubuntu-latest container: - image: crystallang/crystal:0.35.1-alpine + image: crystallang/crystal:1.0.0-alpine steps: - uses: actions/checkout@v2 - name: Install dependencies - run: apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev + run: apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev - name: Build run: make static || make static - name: Linter diff --git a/Dockerfile b/Dockerfile index f203c6c..5b24dbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM crystallang/crystal:0.35.1-alpine AS builder +FROM crystallang/crystal:1.0.0-alpine AS builder WORKDIR /Mango COPY . . -RUN apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev +RUN apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev RUN make static || make static FROM library/alpine diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 65abb90..8f58631 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -2,10 +2,10 @@ FROM arm32v7/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. -RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. -RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. -RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd .. +RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd .. +RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd .. +RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd .. COPY mango-arm32v7.o . diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index d9b4186..29045cf 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -2,10 +2,10 @@ FROM arm64v8/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. -RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. -RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. -RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 1.0.0 && make deps && cd .. +RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.8 && make && cd .. +RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v1.0.0 && make && cd .. +RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.5.0 && make && cd .. COPY mango-arm64v8.o . diff --git a/README.md b/README.md index e1545d6..3ca1113 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Mango -[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Patreon](https://img.shields.io/badge/support-patreon-brightgreen?link=https://www.patreon.com/hkalexling)](https://www.patreon.com/hkalexling) ![Build](https://github.com/hkalexling/Mango/workflows/Build/badge.svg) [![Gitter](https://badges.gitter.im/mango-cr/mango.svg)](https://gitter.im/mango-cr/mango?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Discord](https://img.shields.io/discord/855633663425118228?label=discord)](http://discord.com/invite/ezKtacCp9Q) Mango is a self-hosted manga server and reader. Its features include @@ -13,7 +13,6 @@ Mango is a self-hosted manga server and reader. Its features include - Supports nested folders in library - Automatically stores reading progress - Thumbnail generation -- Built-in [MangaDex](https://mangadex.org/) downloader - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites - The web reader is responsive and works well on mobile, so there is no need for a mobile app - All the static files are embedded in the binary, so the deployment process is easy and painless @@ -52,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.21.0 + Mango - Manga Server and Web Reader. Version 0.23.0 Usage: @@ -87,7 +86,6 @@ log_level: info upload_path: ~/mango/uploads plugin_path: ~/mango/plugins download_timeout_seconds: 30 -page_margin: 30 disable_login: false default_username: "" auth_proxy_header_name: "" @@ -99,6 +97,7 @@ mangadex: download_queue_db_path: ~/mango/queue.db chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' manga_rename_rule: '{title}' + subscription_update_interval_hours: 24 ``` - `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks diff --git a/public/js/download.js b/public/js/download.js deleted file mode 100644 index 74fab76..0000000 --- a/public/js/download.js +++ /dev/null @@ -1,287 +0,0 @@ -const downloadComponent = () => { - return { - chaptersLimit: 1000, - loading: false, - addingToDownload: false, - searchAvailable: false, - searchInput: '', - data: {}, - chapters: [], - mangaAry: undefined, // undefined: not searching; []: searched but no result - candidateManga: {}, - langChoice: 'All', - groupChoice: 'All', - chapterRange: '', - volumeRange: '', - - get languages() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - set.add(chp.language); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - get groups() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - Object.keys(chp.groups).forEach(g => { - set.add(g); - }); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - init() { - const tableObserver = new MutationObserver(() => { - console.log('table mutated'); - $("#selectable").selectable({ - filter: 'tr' - }); - }); - tableObserver.observe($('table').get(0), { - childList: true, - subtree: true - }); - - $.getJSON(`${base_url}api/admin/mangadex/expires`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); - return; - } - if (data.expires && data.expires > Math.floor(Date.now() / 1000)) - this.searchAvailable = true; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - }, - - filtersUpdated() { - if (!this.data.chapters) - this.chapters = []; - const filters = { - chapter: this.parseRange(this.chapterRange), - volume: this.parseRange(this.volumeRange), - lang: this.langChoice, - group: this.groupChoice - }; - console.log('filters:', filters); - let _chapters = this.data.chapters.slice(); - Object.entries(filters).forEach(([k, v]) => { - if (v === 'All') return; - if (k === 'group') { - _chapters = _chapters.filter(c => { - const unescaped_groups = Object.entries(c.groups).map(([g, id]) => this.unescapeHTML(g)); - return unescaped_groups.indexOf(v) >= 0; - }); - return; - } - if (k === 'lang') { - _chapters = _chapters.filter(c => c.language === v); - return; - } - const lb = parseFloat(v[0]); - const ub = parseFloat(v[1]); - if (isNaN(lb) && isNaN(ub)) return; - _chapters = _chapters.filter(c => { - const val = parseFloat(c[k]); - if (isNaN(val)) return false; - if (isNaN(lb)) - return val <= ub; - else if (isNaN(ub)) - return val >= lb; - else - return val >= lb && val <= ub; - }); - }); - console.log('filtered chapters:', _chapters); - this.chapters = _chapters; - }, - - search() { - if (this.loading || this.searchInput === '') return; - this.data = {}; - this.mangaAry = undefined; - - var int_id = -1; - try { - const path = new URL(this.searchInput).pathname; - const match = /\/(?:title|manga)\/([0-9]+)/.exec(path); - int_id = parseInt(match[1]); - } catch (e) { - int_id = parseInt(this.searchInput); - } - - if (!isNaN(int_id) && int_id > 0) { - // The input is a positive integer. We treat it as an ID. - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/manga/${int_id}`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to get manga info. Error: ' + data.error); - return; - } - - this.data = data; - this.chapters = data.chapters; - this.mangaAry = undefined; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to get manga info. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } else { - if (!this.searchAvailable) { - alert('danger', 'Please make sure you are using a valid manga ID or manga URL from Mangadex. If you are trying to search MangaDex with a search term, please log in to MangaDex first by going to "Admin -> Connect to MangaDex".'); - return; - } - - // Search as a search term - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/search?${$.param({ - query: this.searchInput - })}`) - .done((data) => { - if (data.error) { - alert('danger', `Failed to search MangaDex. Error: ${data.error}`); - return; - } - - this.mangaAry = data.manga; - this.data = {}; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to search MangaDex. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } - }, - - parseRange(str) { - const regex = /^[\t ]*(?:(?:(<|<=|>|>=)[\t ]*([0-9]+))|(?:([0-9]+))|(?:([0-9]+)[\t ]*-[\t ]*([0-9]+))|(?:[\t ]*))[\t ]*$/m; - const matches = str.match(regex); - var num; - - if (!matches) { - return [null, null]; - } else if (typeof matches[1] !== 'undefined' && typeof matches[2] !== 'undefined') { - // e.g., <= 30 - num = parseInt(matches[2]); - if (isNaN(num)) { - return [null, null]; - } - switch (matches[1]) { - case '<': - return [null, num - 1]; - case '<=': - return [null, num]; - case '>': - return [num + 1, null]; - case '>=': - return [num, null]; - } - } else if (typeof matches[3] !== 'undefined') { - // a single number - num = parseInt(matches[3]); - if (isNaN(num)) { - return [null, null]; - } - return [num, num]; - } else if (typeof matches[4] !== 'undefined' && typeof matches[5] !== 'undefined') { - // e.g., 10 - 23 - num = parseInt(matches[4]); - const n2 = parseInt(matches[5]); - if (isNaN(num) || isNaN(n2) || num > n2) { - return [null, null]; - } - return [num, n2]; - } else { - // empty or space only - return [null, null]; - } - }, - - unescapeHTML(str) { - var elt = document.createElement("span"); - elt.innerHTML = str; - return elt.innerText; - }, - - selectAll() { - $('tbody > tr').each((i, e) => { - $(e).addClass('ui-selected'); - }); - }, - - clearSelection() { - $('tbody > tr').each((i, e) => { - $(e).removeClass('ui-selected'); - }); - }, - - download() { - const selected = $('tbody > tr.ui-selected'); - if (selected.length === 0) return; - UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => { - const ids = selected.map((i, e) => { - return parseInt($(e).find('td').first().text()); - }).get(); - const chapters = this.chapters.filter(c => ids.indexOf(c.id) >= 0); - console.log(ids); - this.addingToDownload = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/download`, - data: JSON.stringify({ - chapters: chapters - }), - contentType: "application/json", - dataType: 'json' - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`); - return; - } - const successCount = parseInt(data.success); - const failCount = parseInt(data.fail); - UIkit.modal.confirm(`${successCount} of ${successCount + failCount} chapters added to the download queue. Proceed to the download manager?`).then(() => { - window.location.href = base_url + 'admin/downloads'; - }); - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.addingToDownload = false; - }); - }); - }, - - chooseManga(manga) { - this.candidateManga = manga; - UIkit.modal($('#modal').get(0)).show(); - }, - - confirmManga(id) { - UIkit.modal($('#modal').get(0)).hide(); - this.searchInput = id; - this.search(); - } - }; -}; diff --git a/public/js/mangadex.js b/public/js/mangadex.js deleted file mode 100644 index 3271c4b..0000000 --- a/public/js/mangadex.js +++ /dev/null @@ -1,61 +0,0 @@ -const component = () => { - return { - username: '', - password: '', - expires: undefined, - loading: true, - loggingIn: false, - - init() { - this.loading = true; - $.ajax({ - type: 'GET', - url: `${base_url}api/admin/mangadex/expires`, - contentType: "application/json", - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to retrieve MangaDex token status. Error: ${data.error}`); - return; - } - this.expires = data.expires; - this.loading = false; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to retrieve MangaDex token status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }); - }, - login() { - if (!(this.username && this.password)) return; - this.loggingIn = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/login`, - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({ - username: this.username, - password: this.password - }) - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to log in. Error: ${data.error}`); - return; - } - this.expires = data.expires; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to log in. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loggingIn = false; - }); - }, - get expired() { - return this.expires && moment().diff(moment.unix(this.expires)) > 0; - } - }; -}; diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index e1fba96..a335e03 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -126,9 +126,7 @@ const download = () => { } const successCount = parseInt(data.success); const failCount = parseInt(data.fail); - UIkit.modal.confirm(`${successCount} of ${successCount + failCount} chapters added to the download queue. Proceed to the download manager?`).then(() => { - window.location.href = base_url + 'admin/downloads'; - }); + alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); }) .fail((jqXHR, status) => { alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); diff --git a/public/js/reader.js b/public/js/reader.js index 1aa1bd1..9b17276 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -6,10 +6,13 @@ const readerComponent = () => { alertClass: 'uk-alert-primary', items: [], curItem: {}, + enableFlipAnimation: true, flipAnimation: null, longPages: false, lastSavedPage: page, selectedIndex: 0, // 0: not selected; 1: the first page + margin: 30, + preloadLookahead: 3, /** * Initialize the component by fetching the page dimensions @@ -27,7 +30,6 @@ const readerComponent = () => { url: `${base_url}api/page/${tid}/${eid}/${i+1}`, width: d.width, height: d.height, - style: `margin-top: ${data.margin}px; margin-bottom: ${data.margin}px;` }; }); @@ -47,6 +49,21 @@ const readerComponent = () => { const mode = this.mode; this.updateMode(this.mode, page, nextTick); $('#mode-select').val(mode); + + const savedMargin = localStorage.getItem('margin'); + if (savedMargin) { + this.margin = savedMargin; + } + + // Preload Images + this.preloadLookahead = +(localStorage.getItem('preloadLookahead') ?? 3); + const limit = Math.min(page + this.preloadLookahead, this.items.length + 1); + for (let idx = page + 1; idx <= limit; idx++) { + this.preloadImage(this.items[idx - 1].url); + } + + const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); + this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -55,6 +72,12 @@ const readerComponent = () => { this.msg = errMsg; }) }, + /** + * Preload an image, which is expected to be cached + */ + preloadImage(url) { + (new Image()).src = url; + }, /** * Handles the `change` event for the page selector */ @@ -106,12 +129,18 @@ const readerComponent = () => { if (newIdx <= 0 || newIdx > this.items.length) return; + if (newIdx + this.preloadLookahead < this.items.length + 1) { + this.preloadImage(this.items[newIdx + this.preloadLookahead - 1].url); + } + this.toPage(newIdx); - if (isNext) - this.flipAnimation = 'right'; - else - this.flipAnimation = 'left'; + if (this.enableFlipAnimation) { + if (isNext) + this.flipAnimation = 'right'; + else + this.flipAnimation = 'left'; + } setTimeout(() => { this.flipAnimation = null; @@ -277,6 +306,19 @@ const readerComponent = () => { entryChanged() { const id = $('#entry-select').val(); this.redirect(`${base_url}reader/${tid}/${id}`); - } + }, + + marginChanged() { + localStorage.setItem('margin', this.margin); + this.toPage(this.selectedIndex); + }, + + preloadLookaheadChanged() { + localStorage.setItem('preloadLookahead', this.preloadLookahead); + }, + + enableFlipAnimationChanged() { + localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation); + }, }; } diff --git a/public/js/subscription.js b/public/js/subscription.js new file mode 100644 index 0000000..ed2cb17 --- /dev/null +++ b/public/js/subscription.js @@ -0,0 +1,82 @@ +const component = () => { + return { + available: undefined, + subscriptions: [], + + init() { + $.getJSON(`${base_url}api/admin/mangadex/expires`) + .done((data) => { + if (data.error) { + alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); + return; + } + this.available = Boolean(data.expires && data.expires > Math.floor(Date.now() / 1000)); + + if (this.available) this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + getSubscriptions() { + $.getJSON(`${base_url}api/admin/mangadex/subscriptions`) + .done(data => { + if (data.error) { + alert('danger', 'Failed to get subscriptions. Error: ' + data.error); + return; + } + this.subscriptions = data.subscriptions; + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to get subscriptions. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + rm(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'DELETE', + url: `${base_url}api/admin/mangadex/subscriptions/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to delete subscription. Error: ${data.error}`); + } + this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to delete subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + check(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions/check/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to check subscription. Error: ${data.error}`); + return; + } + alert('success', 'Mango is now checking the subscription for updates. This might take a while, but you can safely leave the page.'); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + formatRange(min, max) { + if (!isNaN(min) && isNaN(max)) return `≥ ${min}`; + if (isNaN(min) && !isNaN(max)) return `≤ ${max}`; + if (isNaN(min) && isNaN(max)) return 'All'; + + if (min === max) return `= ${min}`; + return `${min} - ${max}`; + } + }; +}; diff --git a/shard.lock b/shard.lock index 254f3a5..df6fc4c 100644 --- a/shard.lock +++ b/shard.lock @@ -2,81 +2,77 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 0.12.1 + version: 0.14.3 archive: git: https://github.com/hkalexling/archive.cr.git - version: 0.4.0 + version: 0.5.0 baked_file_system: git: https://github.com/schovi/baked_file_system.git - version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b + version: 0.10.0 clim: git: https://github.com/at-grandpa/clim.git - version: 0.12.0 + version: 0.17.1 db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.9.0 + version: 0.10.1 duktape: git: https://github.com/jessedoyle/duktape.cr.git - version: 0.20.0 + version: 1.0.0 exception_page: git: https://github.com/crystal-loot/exception_page.git - version: 0.1.4 + version: 0.1.5 http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.7.1 + version: 0.8.0 image_size: git: https://github.com/hkalexling/image_size.cr.git - version: 0.4.0 + version: 0.5.0 kemal: git: https://github.com/kemalcr/kemal.git - version: 0.27.0 + version: 1.0.0 kemal-session: git: https://github.com/kemalcr/kemal-session.git - version: 0.12.1 + version: 1.0.0 kilt: git: https://github.com/jeromegn/kilt.git - version: 0.4.0 + version: 0.4.1 koa: git: https://github.com/hkalexling/koa.git - version: 0.7.0 - - mangadex: - git: https://github.com/hkalexling/mangadex.git - version: 0.8.0+git.commit.24e6fb51afd043721139355854e305b43bf98c43 + version: 0.8.0 mg: git: https://github.com/hkalexling/mg.git - version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3 + version: 0.5.0+git.commit.697e46e27cde8c3969346e228e372db2455a6264 myhtml: git: https://github.com/kostya/myhtml.git - version: 1.5.1 + version: 1.5.8 open_api: - git: https://github.com/jreinert/open_api.cr.git - version: 1.2.1+git.commit.95e4df2ca10b1fe88b8b35c62a18b06a10267b6c + git: https://github.com/hkalexling/open_api.cr.git + version: 1.2.1+git.commit.1d3c55dd5534c6b0af18964d031858a08515553a radix: git: https://github.com/luislavena/radix.git - version: 0.3.9 + version: 0.4.1 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git - version: 0.16.0 + version: 0.18.0 tallboy: git: https://github.com/epoch/tallboy.git - version: 0.9.3 + version: 0.9.3+git.commit.9be1510bb0391c95e92f1b288f3afb429a73caa6 diff --git a/shard.yml b/shard.yml index 390ddc6..1da553d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.21.0 +version: 0.23.0 authors: - Alex Ling @@ -8,7 +8,7 @@ targets: mango: main: src/mango.cr -crystal: 0.35.1 +crystal: 1.0.0 license: MIT @@ -21,7 +21,6 @@ dependencies: github: crystal-lang/crystal-sqlite3 baked_file_system: github: schovi/baked_file_system - version: 0.9.8+git.commit.fb3091b546797fbec3c25dc0e1e2cff60bb9033b archive: github: hkalexling/archive.cr ameba: @@ -30,7 +29,6 @@ dependencies: github: at-grandpa/clim duktape: github: jessedoyle/duktape.cr - version: ~> 0.20.0 myhtml: github: kostya/myhtml http_proxy: @@ -41,7 +39,6 @@ dependencies: github: hkalexling/koa tallboy: github: epoch/tallboy + branch: master mg: github: hkalexling/mg - mangadex: - github: hkalexling/mangadex diff --git a/spec/storage_spec.cr b/spec/storage_spec.cr index 44bfb5a..ad10de5 100644 --- a/spec/storage_spec.cr +++ b/spec/storage_spec.cr @@ -8,9 +8,7 @@ describe Storage do end it "deletes user" do - with_storage do |storage| - storage.delete_user "admin" - end + with_storage &.delete_user "admin" end it "creates new user" do diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 3ee4aac..27d97c2 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -21,7 +21,7 @@ describe "compare_numerically" do it "sorts like the stack exchange post" do ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2", "text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -29,7 +29,7 @@ describe "compare_numerically" do # https://github.com/hkalexling/Mango/issues/22 it "handles numbers larger than Int32" do ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -56,7 +56,7 @@ describe "chapter_sort" do it "sorts correctly" do ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"] sorter = ChapterSorter.new ary - ary.reverse.sort do |a, b| + ary.reverse.sort! do |a, b| sorter.compare a, b end.should eq ary end diff --git a/src/config.cr b/src/config.cr index a05c3a7..332a159 100644 --- a/src/config.cr +++ b/src/config.cr @@ -20,7 +20,6 @@ class Config property plugin_path : String = File.expand_path "~/mango/plugins", home: true property download_timeout_seconds : Int32 = 30 - property page_margin : Int32 = 30 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/entry.cr b/src/library/entry.cr index cb45895..92f4def 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -46,6 +46,18 @@ class Entry file.close end + def to_slim_json : String + JSON.build do |json| + json.object do + {% for str in ["zip_path", "title", "size", "id"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "title_id", @book.id + json.field "pages" { json.number @pages } + end + end + end + def to_json(json : JSON::Builder) json.object do {% for str in ["zip_path", "title", "size", "id"] %} @@ -86,7 +98,7 @@ class Entry SUPPORTED_IMG_TYPES.includes? \ MIME.from_filename? e.filename } - .sort { |a, b| + .sort! { |a, b| compare_numerically a.filename, b.filename } yield file, entries diff --git a/src/library/library.cr b/src/library/library.cr index 7e97e85..a5a4a80 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -63,7 +63,22 @@ class Library end def deep_titles - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles + end + + def to_slim_json : String + JSON.build do |json| + json.object do + json.field "dir", @dir + json.field "titles" do + json.array do + self.titles.each do |title| + json.raw title.to_slim_json + end + end + end + end + end end def to_json(json : JSON::Builder) @@ -98,7 +113,7 @@ class Library .select { |path| File.directory? path } .map { |path| Title.new path, "" } .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort { |a, b| a.title <=> b.title } + .sort! { |a, b| a.title <=> b.title } .tap { |_| @title_ids.clear } .each do |title| @title_hash[title.id] = title @@ -114,7 +129,7 @@ class Library def get_continue_reading_entries(username) cr_entries = deep_titles - .map { |t| t.get_last_read_entry username } + .map(&.get_last_read_entry username) # Select elements with type `Entry` from the array and ignore all `Nil`s .select(Entry)[0...ENTRIES_IN_HOME_SECTIONS] .map { |e| @@ -150,14 +165,14 @@ class Library recently_added = [] of RA last_date_added = nil - titles.map { |t| t.deep_entries_with_date_added }.flatten - .select { |e| e[:date_added] > 1.month.ago } - .sort { |a, b| b[:date_added] <=> a[:date_added] } + titles.flat_map(&.deep_entries_with_date_added) + .select(&.[:date_added].> 1.month.ago) + .sort! { |a, b| b[:date_added] <=> a[:date_added] } .each do |e| break if recently_added.size > 12 last = recently_added.last? if last && e[:entry].book.id == last[:entry].book.id && - (e[:date_added] - last_date_added.not_nil!).duration < 1.day + (e[:date_added] - last_date_added.not_nil!).abs < 1.day # A NamedTuple is immutable, so we have to cast it to a Hash first last_hash = last.to_h count = last_hash[:grouped_count].as(Int32) @@ -188,9 +203,9 @@ class Library # If we use `deep_titles`, the start reading section might include `Vol. 2` # when the user hasn't started `Vol. 1` yet titles - .select { |t| t.load_percentage(username) == 0 } + .select(&.load_percentage(username).== 0) .sample(ENTRIES_IN_HOME_SECTIONS) - .shuffle + .shuffle! end def thumbnail_generation_progress @@ -205,7 +220,7 @@ class Library end Logger.info "Starting thumbnail generation" - entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg + entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg @entries_count = entries.size @thumbnails_count = 0 diff --git a/src/library/title.cr b/src/library/title.cr index 4c439e7..61c9813 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -44,19 +44,54 @@ class Title mtimes = [@mtime] mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } - mtimes += @entries.map { |e| e.mtime } + mtimes += @entries.map &.mtime @mtime = mtimes.max @title_ids.sort! do |a, b| compare_numerically Library.default.title_hash[a].title, Library.default.title_hash[b].title end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title @entries.sort! do |a, b| sorter.compare a.title, b.title end end + def to_slim_json : String + JSON.build do |json| + json.object do + {% for str in ["dir", "title", "id"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "signature" { json.number @signature } + json.field "titles" do + json.array do + self.titles.each do |title| + json.raw title.to_slim_json + end + end + end + json.field "entries" do + json.array do + @entries.each do |entry| + json.raw entry.to_slim_json + end + end + end + json.field "parents" do + json.array do + self.parents.each do |title| + json.object do + json.field "title", title.title + json.field "id", title.id + end + end + end + end + end + end + end + def to_json(json : JSON::Builder) json.object do {% for str in ["dir", "title", "id"] %} @@ -92,12 +127,12 @@ class Title # Get all entries, including entries in nested titles def deep_entries return @entries if title_ids.empty? - @entries + titles.map { |t| t.deep_entries }.flatten + @entries + titles.flat_map &.deep_entries end def deep_titles return [] of Title if titles.empty? - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles end def parents @@ -138,7 +173,7 @@ class Title end def get_entry(eid) - @entries.find { |e| e.id == eid } + @entries.find &.id.== eid end def display_name @@ -217,29 +252,23 @@ class Title @entries.each do |e| e.save_progress username, e.pages end - titles.each do |t| - t.read_all username - end + titles.each &.read_all username end # Set the reading progress of all entries and nested libraries to 0% def unread_all(username) - @entries.each do |e| - e.save_progress username, 0 - end - titles.each do |t| - t.unread_all username - end + @entries.each &.save_progress(username, 0) + titles.each &.unread_all username end def deep_read_page_count(username) : Int32 load_progress_for_all_entries(username).sum + - titles.map { |t| t.deep_read_page_count username }.flatten.sum + titles.flat_map(&.deep_read_page_count username).sum end def deep_total_page_count : Int32 - entries.map { |e| e.pages }.sum + - titles.map { |t| t.deep_total_page_count }.flatten.sum + entries.sum(&.pages) + + titles.flat_map(&.deep_total_page_count).sum end def load_percentage(username) @@ -311,13 +340,13 @@ class Title ary = @entries.zip(percentage_ary) .sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \ compare_numerically a_tp[0].title, b_tp[0].title } - .map { |tp| tp[0] } + .map &.[0] else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title ary = @entries.sort do |a, b| sorter.compare(a.title, b.title).or \ compare_numerically a.title, b.title @@ -383,13 +412,13 @@ class Title {entry: e, date_added: da_ary[i]} end return zip if title_ids.empty? - zip + titles.map { |t| t.deep_entries_with_date_added }.flatten + zip + titles.flat_map &.deep_entries_with_date_added end def bulk_progress(action, ids : Array(String), username) selected_entries = ids .map { |id| - @entries.find { |e| e.id == id } + @entries.find &.id.==(id) } .select(Entry) diff --git a/src/logger.cr b/src/logger.cr index d434cfa..040e5aa 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -34,7 +34,11 @@ class Logger end @backend.formatter = Log::Formatter.new &format_proc - Log.setup @@severity, @backend + + Log.setup do |c| + c.bind "*", @@severity, @backend + c.bind "db.*", :error, @backend + end end def self.get_severity(level = "") : Log::Severity diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr deleted file mode 100644 index c0b50c7..0000000 --- a/src/mangadex/downloader.cr +++ /dev/null @@ -1,169 +0,0 @@ -require "mangadex" -require "compress/zip" -require "../rename" -require "./ext" - -module MangaDex - class PageJob - property success = false - property url : String - property filename : String - property writer : Compress::Zip::Writer - property tries_remaning : Int32 - - def initialize(@url, @filename, @writer, @tries_remaning) - end - end - - class Downloader < Queue::Downloader - @wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"] - .to_i32 - @retries : Int32 = Config.current.mangadex["download_retries"].to_i32 - - use_default - - def initialize - @client = Client.from_config - super - end - - def pop : Queue::Job? - job = nil - MainFiber.run do - DB.open "sqlite3://#{@queue.path}" do |db| - begin - db.query_one "select * from queue where id not like '%-%' " \ - "and (status = 0 or status = 1) " \ - "order by time limit 1" do |res| - job = Queue::Job.from_query_result res - end - rescue - end - end - end - job - end - - private def download(job : Queue::Job) - @downloading = true - @queue.set_status Queue::JobStatus::Downloading, job - begin - chapter = @client.chapter job.id - rescue e - Logger.error e - @queue.set_status Queue::JobStatus::Error, job - unless e.message.nil? - @queue.add_message e.message.not_nil!, job - end - @downloading = false - return - end - @queue.set_pages chapter.pages.size, job - lib_dir = @library_path - rename_rule = Rename::Rule.new \ - Config.current.mangadex["manga_rename_rule"].to_s - manga_dir = File.join lib_dir, chapter.manga.rename rename_rule - unless File.exists? manga_dir - Dir.mkdir_p manga_dir - end - zip_path = File.join manga_dir, "#{job.title}.cbz.part" - - # Find the number of digits needed to store the number of pages - len = Math.log10(chapter.pages.size).to_i + 1 - - writer = Compress::Zip::Writer.new zip_path - # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new chapter.pages.size - spawn do - chapter.pages.each_with_index do |url, i| - fn = Path.new(URI.parse(url).path).basename - ext = File.extname fn - fn = "#{i.to_s.rjust len, '0'}#{ext}" - page_job = PageJob.new url, fn, writer, @retries - Logger.debug "Downloading #{url}" - loop do - sleep @wait_seconds.seconds - download_page page_job - break if page_job.success || - page_job.tries_remaning <= 0 - page_job.tries_remaning -= 1 - Logger.warn "Failed to download page #{url}. " \ - "Retrying... Remaining retries: " \ - "#{page_job.tries_remaning}" - end - - channel.send page_job - break unless @queue.exists? job - end - end - - spawn do - page_jobs = [] of PageJob - chapter.pages.size.times do - page_job = channel.receive - - break unless @queue.exists? job - - Logger.debug "[#{page_job.success ? "success" : "failed"}] " \ - "#{page_job.url}" - page_jobs << page_job - if page_job.success - @queue.add_success job - else - @queue.add_fail job - msg = "Failed to download page #{page_job.url}" - @queue.add_message msg, job - Logger.error msg - end - end - - unless @queue.exists? job - Logger.debug "Download cancelled" - @downloading = false - next - end - - fail_count = page_jobs.count { |j| !j.success } - Logger.debug "Download completed. " \ - "#{fail_count}/#{page_jobs.size} failed" - writer.close - filename = File.join File.dirname(zip_path), File.basename(zip_path, - ".part") - File.rename zip_path, filename - Logger.debug "cbz File created at #{filename}" - - zip_exception = validate_archive filename - if !zip_exception.nil? - @queue.add_message "The downloaded archive is corrupted. " \ - "Error: #{zip_exception}", job - @queue.set_status Queue::JobStatus::Error, job - elsif fail_count > 0 - @queue.set_status Queue::JobStatus::MissingPages, job - else - @queue.set_status Queue::JobStatus::Completed, job - end - @downloading = false - end - end - - private def download_page(job : PageJob) - Logger.debug "downloading #{job.url}" - headers = HTTP::Headers{ - "User-agent" => "Mangadex.cr", - } - begin - HTTP::Client.get job.url, headers do |res| - unless res.success? - raise "Failed to download page #{job.url}. " \ - "[#{res.status_code}] #{res.status_message}" - end - job.writer.add job.filename, res.body_io - end - job.success = true - rescue e - Logger.error e - job.success = false - end - end - end -end diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr deleted file mode 100644 index dfb302c..0000000 --- a/src/mangadex/ext.cr +++ /dev/null @@ -1,60 +0,0 @@ -private macro properties_to_hash(names) - { - {% for name in names %} - "{{name.id}}" => {{name.id}}.to_s, - {% end %} - } -end - -# Monkey-patch the structures in the `mangadex` shard to suit our needs -module MangaDex - struct Client - @@group_cache = {} of String => Group - - def self.from_config : Client - self.new base_url: Config.current.mangadex["base_url"].to_s, - api_url: Config.current.mangadex["api_url"].to_s - end - end - - struct Manga - def rename(rule : Rename::Rule) - rule.render properties_to_hash %w(id title author artist) - end - - def to_info_json - hash = JSON.parse(to_json).as_h - _chapters = chapters.map do |c| - JSON.parse c.to_info_json - end - hash["chapters"] = JSON::Any.new _chapters - hash.to_json - end - end - - struct Chapter - def rename(rule : Rename::Rule) - hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.map(&.name).join "," - rule.render hash - end - - def full_title - rule = Rename::Rule.new \ - Config.current.mangadex["chapter_rename_rule"].to_s - rename rule - end - - def to_info_json - hash = JSON.parse(to_json).as_h - hash["language"] = JSON::Any.new language - _groups = {} of String => JSON::Any - groups.each do |g| - _groups[g.name] = JSON::Any.new g.id - end - hash["groups"] = JSON::Any.new _groups - hash["full_title"] = JSON::Any.new full_title - hash.to_json - end - end -end diff --git a/src/mango.cr b/src/mango.cr index b5fd168..e8d32a3 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -2,13 +2,12 @@ require "./config" require "./queue" require "./server" require "./main_fiber" -require "./mangadex/*" require "./plugin/*" require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.21.0" +MANGO_VERSION = "0.23.0" # From http://www.network-science.de/ascii/ BANNER = %{ @@ -59,7 +58,6 @@ class CLI < Clim Storage.default Queue.default Library.default - MangaDex::Downloader.default Plugin::Downloader.default spawn do diff --git a/src/plugin/plugin.cr b/src/plugin/plugin.cr index baa77d1..6bedea1 100644 --- a/src/plugin/plugin.cr +++ b/src/plugin/plugin.cr @@ -117,7 +117,7 @@ class Plugin def initialize(id : String) Plugin.build_info_ary - @info = @@info_ary.find { |i| i.id == id } + @info = @@info_ary.find &.id.== id if @info.nil? raise Error.new "Plugin with ID #{id} not found" end diff --git a/src/queue.cr b/src/queue.cr index c9f805c..381441b 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -303,12 +303,12 @@ class Queue end def pause - @downloaders.each { |d| d.stopped = true } + @downloaders.each &.stopped=(true) @paused = true end def resume - @downloaders.each { |d| d.stopped = false } + @downloaders.each &.stopped=(false) @paused = false end diff --git a/src/rename.cr b/src/rename.cr index 1fc7693..3e00fdc 100644 --- a/src/rename.cr +++ b/src/rename.cr @@ -35,15 +35,15 @@ module Rename class Group < Base(Pattern | String) def render(hash : VHash) - return "" if @ary.select(&.is_a? Pattern) + return "" if @ary.select(Pattern) .any? &.as(Pattern).render(hash).empty? - @ary.map do |e| + @ary.join do |e| if e.is_a? Pattern e.render hash else e end - end.join + end end end @@ -129,13 +129,13 @@ module Rename end def render(hash : VHash) - str = @ary.map do |e| + str = @ary.join do |e| if e.is_a? String e else e.render hash end - end.join.strip + end.strip post_process str end diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 616d452..fd63ec8 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -73,9 +73,5 @@ struct AdminRouter get "/admin/missing" do |env| layout "missing-items" end - - get "/admin/mangadex" do |env| - layout "mangadex" - end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d2af12..b1fb3b3 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1,6 +1,6 @@ -require "../mangadex/*" require "../upload" require "koa" +require "digest" struct APIRouter @@api_json : String? @@ -56,31 +56,20 @@ struct APIRouter "error" => String?, } - Koa.schema("mdChapter", { - "id" => Int64, - "group" => {} of String => String, - }.merge(s %w(title volume chapter language full_title time - manga_title manga_id)), - desc: "A MangaDex chapter") - - Koa.schema "mdManga", { - "id" => Int64, - "chapters" => ["mdChapter"], - }.merge(s %w(title description author artist cover_url)), - desc: "A MangaDex manga" - Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.path "page", schema: Int32, desc: "The page number to return (starts from 1)" Koa.response 200, schema: Bytes, media_type: "image/*" Koa.response 500, "Page not found or not readable" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.tag "reader" get "/api/page/:tid/:eid/:page" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] page = env.params.url["page"].to_i + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -90,7 +79,15 @@ struct APIRouter raise "Failed to load page #{page} of " \ "`#{title.title}/#{entry.title}`" if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -102,12 +99,14 @@ struct APIRouter Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.response 200, schema: Bytes, media_type: "image/*" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.response 500, "Page not found or not readable" Koa.tag "library" get "/api/cover/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -118,7 +117,14 @@ struct APIRouter raise "Failed to get cover of `#{title.title}/#{entry.title}`" \ if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -126,8 +132,11 @@ struct APIRouter end end - Koa.describe "Returns the book with title `tid`" + Koa.describe "Returns the book with title `tid`", <<-MD + Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + MD Koa.path "tid", desc: "Title ID" + Koa.query "slim" Koa.response 200, schema: "title" Koa.response 404, "Title not found" Koa.tag "library" @@ -137,7 +146,11 @@ struct APIRouter title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? - send_json env, title.to_json + if env.params.query["slim"]? + send_json env, title.to_slim_json + else + send_json env, title.to_json + end rescue e Logger.error e env.response.status_code = 404 @@ -145,14 +158,21 @@ struct APIRouter end end - Koa.describe "Returns the entire library with all titles and entries" + Koa.describe "Returns the entire library with all titles and entries", <<-MD + Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + MD + Koa.query "slim" Koa.response 200, schema: { "dir" => String, "titles" => ["title"], } Koa.tag "library" get "/api/library" do |env| - send_json env, Library.default.to_json + if env.params.query["slim"]? + send_json env, Library.default.to_slim_json + else + send_json env, Library.default.to_json + end end Koa.describe "Triggers a library scan" @@ -309,64 +329,12 @@ struct APIRouter end end - Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex"] - Koa.path "id", desc: "A MangaDex manga ID" - Koa.response 200, schema: "mdManga" - get "/api/admin/mangadex/manga/:id" do |env| - begin - id = env.params.url["id"] - manga = MangaDex::Client.from_config.manga id - send_json env, manga.to_info_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - - Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex", "downloader"] - Koa.body schema: { - "chapters" => ["mdChapter"], - } - Koa.response 200, schema: { - "success" => Int32, - "fail" => Int32, - } - post "/api/admin/mangadex/download" do |env| - begin - chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h } - jobs = chapters.map { |chapter| - Queue::Job.new( - chapter["id"].as_i64.to_s, - chapter["mangaId"].as_i64.to_s, - chapter["full_title"].as_s, - chapter["mangaTitle"].as_s, - Queue::JobStatus::Pending, - Time.unix chapter["timestamp"].as_i64 - ) - } - inserted_count = Queue.default.push jobs - send_json env, { - "success": inserted_count, - "fail": jobs.size - inserted_count, - }.to_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 loop do socket.send({ - "jobs" => Queue.default.get_all, + "jobs" => Queue.default.get_all.reverse, "paused" => Queue.default.paused?, }.to_json) sleep interval.seconds @@ -390,13 +358,13 @@ struct APIRouter } get "/api/admin/mangadex/queue" do |env| begin - jobs = Queue.default.get_all send_json env, { - "jobs" => jobs, + "jobs" => Queue.default.get_all.reverse, "paused" => Queue.default.paused?, "success" => true, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -444,6 +412,7 @@ struct APIRouter send_json env, {"success" => true}.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -516,6 +485,7 @@ struct APIRouter raise "No part with name `file` found" rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -551,6 +521,7 @@ struct APIRouter "title" => title, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -594,6 +565,7 @@ struct APIRouter "fail": jobs.size - inserted_count, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -612,25 +584,35 @@ struct APIRouter "width" => Int32, "height" => Int32, }], - "margin" => Int32?, } + Koa.response 304, "Not modified (only available when `If-None-Match` is set)" get "/api/dimensions/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - sizes = entry.page_dimensions - send_json env, { - "success" => true, - "dimensions" => sizes, - "margin" => Config.current.page_margin, - }.to_json + file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s) + e_tag = "W/#{file_hash}" + if e_tag == prev_e_tag + env.response.status_code = 304 + "" + else + sizes = entry.page_dimensions + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_json env, { + "success" => true, + "dimensions" => sizes, + }.to_json + end rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -770,6 +752,7 @@ struct APIRouter "titles" => Storage.default.missing_titles, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -796,6 +779,7 @@ struct APIRouter "entries" => Storage.default.missing_entries, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -814,6 +798,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -832,6 +817,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -853,6 +839,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -873,114 +860,6 @@ struct APIRouter "success" => true, "error" => nil, }.to_json - rescue e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Logs the current user into their MangaDex account", <<-MD - If successful, returns the expiration date (as a unix timestamp) of the newly created token. - MD - Koa.body schema: { - "username" => String, - "password" => String, - } - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - post "/api/admin/mangadex/login" do |env| - begin - username = env.params.json["username"].as String - password = env.params.json["password"].as String - mango_username = get_username env - - client = MangaDex::Client.from_config - client.auth username, password - - Storage.default.save_md_token mango_username, client.token.not_nil!, - client.token_expires - - send_json env, { - "success" => true, - "error" => nil, - "expires" => client.token_expires.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - get "/api/admin/mangadex/expires" do |env| - begin - username = get_username env - _, expires = Storage.default.get_md_token username - - send_json env, { - "success" => true, - "error" => nil, - "expires" => expires.try &.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Searches MangaDex for manga matching `query`", <<-MD - Returns an empty list if the current user hasn't logged in to MangaDex. - MD - Koa.query "query" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "manga?" => [{ - "id" => Int64, - "title" => String, - "description" => String, - "mainCover" => String, - }], - } - Koa.tags ["admin", "mangadex"] - get "/api/admin/mangadex/search" do |env| - begin - username = get_username env - token, expires = Storage.default.get_md_token username - - unless expires && token - raise "No token found for user #{username}" - end - - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires - - query = env.params.query["query"] - - send_json env, { - "success" => true, - "error" => nil, - "manga" => client.partial_search query, - }.to_json rescue e Logger.error e send_json env, { diff --git a/src/routes/main.cr b/src/routes/main.cr index 6504800..57917bb 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -30,7 +30,8 @@ struct MainRouter else redirect env, "/" end - rescue + rescue e + Logger.error e redirect env, "/login" end end @@ -71,11 +72,6 @@ struct MainRouter end end - get "/download" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] - layout "download" - end - get "/download/plugins" do |env| begin id = env.params.query["plugin"]? @@ -103,7 +99,7 @@ struct MainRouter recently_added = Library.default.get_recently_added_entries username start_reading = Library.default.get_start_reading_titles username titles = Library.default.titles - new_user = !titles.any? { |t| t.load_percentage(username) > 0 } + new_user = !titles.any? &.load_percentage(username).> 0 empty_library = titles.size == 0 layout "home" rescue e diff --git a/src/storage.cr b/src/storage.cr index 971bba3..39116b9 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -445,7 +445,7 @@ class Storage Logger.debug "Marking #{trash_ids.size} entries as unavailable" end db.exec "update ids set unavailable = 1 where id in " \ - "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + "(#{trash_ids.join "," { |i| "'#{i}'" }})" # Detect dangling title IDs trash_titles = [] of String @@ -461,7 +461,7 @@ class Storage Logger.debug "Marking #{trash_titles.size} titles as unavailable" end db.exec "update titles set unavailable = 1 where id in " \ - "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" + "(#{trash_titles.join "," { |i| "'#{i}'" }})" end end end diff --git a/src/subscription.cr b/src/subscription.cr new file mode 100644 index 0000000..e391360 --- /dev/null +++ b/src/subscription.cr @@ -0,0 +1,83 @@ +require "db" +require "json" + +struct Subscription + include DB::Serializable + include JSON::Serializable + + getter id : Int64 = 0 + getter username : String + getter manga_id : Int64 + property language : String? + property group_id : Int64? + property min_volume : Int64? + property max_volume : Int64? + property min_chapter : Int64? + property max_chapter : Int64? + @[DB::Field(key: "last_checked")] + @[JSON::Field(key: "last_checked")] + @raw_last_checked : Int64 + @[DB::Field(key: "created_at")] + @[JSON::Field(key: "created_at")] + @raw_created_at : Int64 + + def last_checked : Time + Time.unix @raw_last_checked + end + + def created_at : Time + Time.unix @raw_created_at + end + + def initialize(@manga_id, @username) + @raw_created_at = Time.utc.to_unix + @raw_last_checked = Time.utc.to_unix + end + + private def in_range?(value : String, lowerbound : Int64?, + upperbound : Int64?) : Bool + lb = lowerbound.try &.to_f64 + ub = upperbound.try &.to_f64 + + return true if lb.nil? && ub.nil? + + v = value.to_f64? + return false unless v + + if lb.nil? + v <= ub.not_nil! + elsif ub.nil? + v >= lb.not_nil! + else + v >= lb.not_nil! && v <= ub.not_nil! + end + end + + def match?(chapter : MangaDex::Chapter) : Bool + if chapter.manga_id != manga_id || + (language && chapter.language != language) || + (group_id && !chapter.groups.map(&.id).includes? group_id) + return false + end + + in_range?(chapter.volume, min_volume, max_volume) && + in_range?(chapter.chapter, min_chapter, max_chapter) + end + + def check_for_updates : Int32 + Logger.debug "Checking updates for subscription with ID #{id}" + jobs = [] of Queue::Job + get_client(username).user.updates_after last_checked do |chapter| + next unless match? chapter + jobs << chapter.to_job + end + Storage.default.update_subscription_last_checked id + count = Queue.default.push jobs + Logger.debug "#{count}/#{jobs.size} of updates added to queue" + count + rescue e + Logger.error "Error occurred when checking updates for " \ + "subscription with ID #{id}. #{e}" + 0 + end +end diff --git a/src/util/chapter_sort.cr b/src/util/chapter_sort.cr index 44dfb4e..eed7a3e 100644 --- a/src/util/chapter_sort.cr +++ b/src/util/chapter_sort.cr @@ -73,7 +73,7 @@ class ChapterSorter .select do |key| keys[key].count >= str_ary.size / 2 end - .sort do |a_key, b_key| + .sort! do |a_key, b_key| a = keys[a_key] b = keys[b_key] # Sort keys by the number of times they appear diff --git a/src/util/numeric_sort.cr b/src/util/numeric_sort.cr index 7365a9f..c455b47 100644 --- a/src/util/numeric_sort.cr +++ b/src/util/numeric_sort.cr @@ -11,7 +11,7 @@ end def split_by_alphanumeric(str) arr = [] of String str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match| - arr += match.captures.select { |s| s != "" } + arr += match.captures.select &.!= "" end arr end diff --git a/src/util/util.cr b/src/util/util.cr index 8903f5e..c4e168a 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -114,7 +114,7 @@ class String def components_similarity(other : String) : Float64 s, l = [self, other] .map { |str| Path.new(str).parts } - .sort_by &.size + .sort_by! &.size match = s.reverse.zip(l.reverse).count { |a, b| a == b } match / s.size diff --git a/src/util/web.cr b/src/util/web.cr index 67227c7..12459e5 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -72,7 +72,7 @@ def redirect(env, path) end def hash_to_query(hash) - hash.map { |k, v| "#{k}=#{v}" }.join("&") + hash.join "&" { |k, v| "#{k}=#{v}" } end def request_path_startswith(env, ary) diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index a6e9b31..fb64d3e 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -33,7 +33,6 @@ -
  • Connect to MangaDex
    • diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 2b6e434..c264177 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -49,11 +49,10 @@ - - + @@ -61,6 +60,7 @@ + <% content_for "script" do %> <%= render_component "moment" %> diff --git a/src/views/layout.html.ecr b/src/views/layout.html.ecr index ff4853c..70c5a51 100644 --- a/src/views/layout.html.ecr +++ b/src/views/layout.html.ecr @@ -1,89 +1,87 @@ - <%= render_component "head" %> + <%= render_component "head" %> - -
    -
    -
    -
    - -
    -
    -
    + +
    +
    +
    +
    + +
    -
    -
    -
    -
    -
    -
    - - -
    -
    - -
    -
    +
    +
    +
    +
    +
    +
    -
    -
    -
    -
    -
    - <%= content %> -
    - +
    + +
    + + <% end %> +
    - - <%= render_component "uikit" %> - <%= yield_content "script" %> - +
    + +
    +
    +
    +
    +
    +
    +
    +
    + <%= content %> +
    + +
    +
    +
    + + <%= render_component "uikit" %> + <%= yield_content "script" %> + diff --git a/src/views/plugin-download.html.ecr b/src/views/plugin-download.html.ecr index 692e22f..ece56b6 100644 --- a/src/views/plugin-download.html.ecr +++ b/src/views/plugin-download.html.ecr @@ -56,8 +56,10 @@

    Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

    - -
    +
    + +
    +
    <% end %> diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 0b4ee4d..0e46ed2 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -21,15 +21,15 @@
    -