diff --git a/migration/subscription.12.cr b/migration/subscription.12.cr deleted file mode 100644 index 3810755..0000000 --- a/migration/subscription.12.cr +++ /dev/null @@ -1,31 +0,0 @@ -class CreateSubscription < MG::Base - def up : String - # We allow multiple subscriptions for the same manga. - # This can be useful for example when you want to download from multiple - # groups. - <<-SQL - CREATE TABLE subscription ( - id INTEGER PRIMARY KEY, - manga_id INTEGER NOT NULL, - language TEXT, - group_id INTEGER, - min_volume INTEGER, - max_volume INTEGER, - min_chapter INTEGER, - max_chapter INTEGER, - last_checked INTEGER NOT NULL, - created_at INTEGER NOT NULL, - username TEXT NOT NULL, - FOREIGN KEY (username) REFERENCES users (username) - ON UPDATE CASCADE - ON DELETE CASCADE - ); - SQL - end - - def down : String - <<-SQL - DROP TABLE subscription; - SQL - end -end diff --git a/public/js/download.js b/public/js/download.js deleted file mode 100644 index 31a803e..0000000 --- a/public/js/download.js +++ /dev/null @@ -1,379 +0,0 @@ -const downloadComponent = () => { - return { - chaptersLimit: 1000, - loading: false, - addingToDownload: false, - searchAvailable: false, - searchInput: '', - data: {}, - chapters: [], - mangaAry: undefined, // undefined: not searching; []: searched but no result - candidateManga: {}, - langChoice: 'All', - groupChoice: 'All', - chapterRange: '', - volumeRange: '', - - get languages() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - set.add(chp.language); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - get groups() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - Object.keys(chp.groups).forEach(g => { - set.add(g); - }); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - init() { - const tableObserver = new MutationObserver(() => { - console.log('table mutated'); - $("#selectable").selectable({ - filter: 'tr' - }); - }); - tableObserver.observe($('table').get(0), { - childList: true, - subtree: true - }); - - $.getJSON(`${base_url}api/admin/mangadex/expires`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); - return; - } - if (data.expires && data.expires > Math.floor(Date.now() / 1000)) - this.searchAvailable = true; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - }, - - filtersUpdated() { - if (!this.data.chapters) - this.chapters = []; - const filters = { - chapter: this.parseRange(this.chapterRange), - volume: this.parseRange(this.volumeRange), - lang: this.langChoice, - group: this.groupChoice - }; - console.log('filters:', filters); - let _chapters = this.data.chapters.slice(); - Object.entries(filters).forEach(([k, v]) => { - if (v === 'All') return; - if (k === 'group') { - _chapters = _chapters.filter(c => { - const unescaped_groups = Object.entries(c.groups).map(([g, id]) => this.unescapeHTML(g)); - return unescaped_groups.indexOf(v) >= 0; - }); - return; - } - if (k === 'lang') { - _chapters = _chapters.filter(c => c.language === v); - return; - } - const lb = parseFloat(v[0]); - const ub = parseFloat(v[1]); - if (isNaN(lb) && isNaN(ub)) return; - _chapters = _chapters.filter(c => { - const val = parseFloat(c[k]); - if (isNaN(val)) return false; - if (isNaN(lb)) - return val <= ub; - else if (isNaN(ub)) - return val >= lb; - else - return val >= lb && val <= ub; - }); - }); - console.log('filtered chapters:', _chapters); - this.chapters = _chapters; - }, - - search() { - if (this.loading || this.searchInput === '') return; - this.data = {}; - this.mangaAry = undefined; - - var int_id = -1; - try { - const path = new URL(this.searchInput).pathname; - const match = /\/(?:title|manga)\/([0-9]+)/.exec(path); - int_id = parseInt(match[1]); - } catch (e) { - int_id = parseInt(this.searchInput); - } - - if (!isNaN(int_id) && int_id > 0) { - // The input is a positive integer. We treat it as an ID. - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/manga/${int_id}`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to get manga info. Error: ' + data.error); - return; - } - - this.data = data; - this.chapters = data.chapters; - this.mangaAry = undefined; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to get manga info. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } else { - if (!this.searchAvailable) { - alert('danger', 'Please make sure you are using a valid manga ID or manga URL from Mangadex. If you are trying to search MangaDex with a search term, please log in to MangaDex first by going to "Admin -> Connect to MangaDex".'); - return; - } - - // Search as a search term - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/search?${$.param({ - query: this.searchInput - })}`) - .done((data) => { - if (data.error) { - alert('danger', `Failed to search MangaDex. Error: ${data.error}`); - return; - } - - this.mangaAry = data.manga; - this.data = {}; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to search MangaDex. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } - }, - - parseRange(str) { - const regex = /^[\t ]*(?:(?:(<|<=|>|>=)[\t ]*([0-9]+))|(?:([0-9]+))|(?:([0-9]+)[\t ]*-[\t ]*([0-9]+))|(?:[\t ]*))[\t ]*$/m; - const matches = str.match(regex); - var num; - - if (!matches) { - return [null, null]; - } else if (typeof matches[1] !== 'undefined' && typeof matches[2] !== 'undefined') { - // e.g., <= 30 - num = parseInt(matches[2]); - if (isNaN(num)) { - return [null, null]; - } - switch (matches[1]) { - case '<': - return [null, num - 1]; - case '<=': - return [null, num]; - case '>': - return [num + 1, null]; - case '>=': - return [num, null]; - } - } else if (typeof matches[3] !== 'undefined') { - // a single number - num = parseInt(matches[3]); - if (isNaN(num)) { - return [null, null]; - } - return [num, num]; - } else if (typeof matches[4] !== 'undefined' && typeof matches[5] !== 'undefined') { - // e.g., 10 - 23 - num = parseInt(matches[4]); - const n2 = parseInt(matches[5]); - if (isNaN(num) || isNaN(n2) || num > n2) { - return [null, null]; - } - return [num, n2]; - } else { - // empty or space only - return [null, null]; - } - }, - - unescapeHTML(str) { - var elt = document.createElement("span"); - elt.innerHTML = str; - return elt.innerText; - }, - - selectAll() { - $('tbody > tr').each((i, e) => { - $(e).addClass('ui-selected'); - }); - }, - - clearSelection() { - $('tbody > tr').each((i, e) => { - $(e).removeClass('ui-selected'); - }); - }, - - download() { - const selected = $('tbody > tr.ui-selected'); - if (selected.length === 0) return; - UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => { - const ids = selected.map((i, e) => { - return parseInt($(e).find('td').first().text()); - }).get(); - const chapters = this.chapters.filter(c => ids.indexOf(c.id) >= 0); - console.log(ids); - this.addingToDownload = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/download`, - data: JSON.stringify({ - chapters: chapters - }), - contentType: "application/json", - dataType: 'json' - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`); - return; - } - const successCount = parseInt(data.success); - const failCount = parseInt(data.fail); - alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.addingToDownload = false; - }); - }); - }, - - chooseManga(manga) { - this.candidateManga = manga; - UIkit.modal($('#modal').get(0)).show(); - }, - - confirmManga(id) { - UIkit.modal($('#modal').get(0)).hide(); - this.searchInput = id; - this.search(); - }, - - subscribe(langConfirmed = false, groupConfirmed = false) { - const filters = { - manga: this.data.id, - language: this.langChoice === 'All' ? null : this.langChoice, - group: this.groupChoice === 'All' ? null : this.groupChoice, - volume: this.volumeRange === '' ? null : this.volumeRange, - chapter: this.chapterRange === '' ? null : this.chapterRange - }; - - // Get group ID - if (filters.group) { - this.data.chapters.forEach(chp => { - const gid = chp.groups[filters.group]; - if (gid) { - filters.groupId = gid; - return; - } - }); - } - - // Parse range values - if (filters.volume) { - [filters.volumeMin, filters.volumeMax] = this.parseRange(filters.volume); - } - if (filters.chapter) { - [filters.chapterMin, filters.chapterMax] = this.parseRange(filters.chapter); - } - - if (!filters.language && !langConfirmed) { - UIkit.modal.confirm('You didn\'t specify a language in the filtering rules. This might cause Mango to download chapters that are not in your preferred language. Are you sure you want to continue?', { - labels: { - ok: 'Yes', - cancel: 'Cancel' - } - }).then(() => { - this.subscribe(true, groupConfirmed); - }); - return; - } - - if (!filters.group && !groupConfirmed) { - UIkit.modal.confirm('You didn\'t specify a group in the filtering rules. This might cause Mango to download multiple versions of the same chapter. Are you sure you want to continue?', { - labels: { - ok: 'Yes', - cancel: 'Cancel' - } - }).then(() => { - this.subscribe(langConfirmed, true); - }); - return; - } - - const mangaURL = `${mangadex_base_url}/manga/${filters.manga}`; - - console.log(filters); - UIkit.modal.confirm(`All FUTURE chapters matching the following filters will be downloaded:
- - - IMPORTANT: Please make sure you are following the manga on MangaDex, otherwise Mango won't be able to receive any updates. To follow it, visit ${mangaURL} and click "Follow". - `, { - labels: { - ok: 'Confirm', - cancel: 'Cancel' - } - }).then(() => { - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/subscriptions`, - data: JSON.stringify({ - subscription: filters - }), - contentType: "application/json", - dataType: 'json' - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to subscribe. Error: ${data.error}`); - return; - } - alert('success', `You've successfully subscribed to this manga! You can view and manage your subscriptions on the subscription manager page.`); - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to subscribe. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }); - }); - } - }; -}; diff --git a/public/js/mangadex.js b/public/js/mangadex.js deleted file mode 100644 index 3271c4b..0000000 --- a/public/js/mangadex.js +++ /dev/null @@ -1,61 +0,0 @@ -const component = () => { - return { - username: '', - password: '', - expires: undefined, - loading: true, - loggingIn: false, - - init() { - this.loading = true; - $.ajax({ - type: 'GET', - url: `${base_url}api/admin/mangadex/expires`, - contentType: "application/json", - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to retrieve MangaDex token status. Error: ${data.error}`); - return; - } - this.expires = data.expires; - this.loading = false; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to retrieve MangaDex token status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }); - }, - login() { - if (!(this.username && this.password)) return; - this.loggingIn = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/login`, - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({ - username: this.username, - password: this.password - }) - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to log in. Error: ${data.error}`); - return; - } - this.expires = data.expires; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to log in. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loggingIn = false; - }); - }, - get expired() { - return this.expires && moment().diff(moment.unix(this.expires)) > 0; - } - }; -}; diff --git a/public/js/reader.js b/public/js/reader.js index f227972..9b17276 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -6,11 +6,13 @@ const readerComponent = () => { alertClass: 'uk-alert-primary', items: [], curItem: {}, + enableFlipAnimation: true, flipAnimation: null, longPages: false, lastSavedPage: page, selectedIndex: 0, // 0: not selected; 1: the first page margin: 30, + preloadLookahead: 3, /** * Initialize the component by fetching the page dimensions @@ -52,6 +54,16 @@ const readerComponent = () => { if (savedMargin) { this.margin = savedMargin; } + + // Preload Images + this.preloadLookahead = +(localStorage.getItem('preloadLookahead') ?? 3); + const limit = Math.min(page + this.preloadLookahead, this.items.length + 1); + for (let idx = page + 1; idx <= limit; idx++) { + this.preloadImage(this.items[idx - 1].url); + } + + const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); + this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -60,6 +72,12 @@ const readerComponent = () => { this.msg = errMsg; }) }, + /** + * Preload an image, which is expected to be cached + */ + preloadImage(url) { + (new Image()).src = url; + }, /** * Handles the `change` event for the page selector */ @@ -111,12 +129,18 @@ const readerComponent = () => { if (newIdx <= 0 || newIdx > this.items.length) return; + if (newIdx + this.preloadLookahead < this.items.length + 1) { + this.preloadImage(this.items[newIdx + this.preloadLookahead - 1].url); + } + this.toPage(newIdx); - if (isNext) - this.flipAnimation = 'right'; - else - this.flipAnimation = 'left'; + if (this.enableFlipAnimation) { + if (isNext) + this.flipAnimation = 'right'; + else + this.flipAnimation = 'left'; + } setTimeout(() => { this.flipAnimation = null; @@ -287,6 +311,14 @@ const readerComponent = () => { marginChanged() { localStorage.setItem('margin', this.margin); this.toPage(this.selectedIndex); - } + }, + + preloadLookaheadChanged() { + localStorage.setItem('preloadLookahead', this.preloadLookahead); + }, + + enableFlipAnimationChanged() { + localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation); + }, }; } diff --git a/shard.lock b/shard.lock index 157b6b3..28cb12a 100644 --- a/shard.lock +++ b/shard.lock @@ -52,10 +52,6 @@ shards: git: https://github.com/hkalexling/koa.git version: 0.7.0 - mangadex: - git: https://github.com/hkalexling/mangadex.git - version: 0.11.0+git.commit.f5b0d64fbb138879fb9228b6e9ff34ec97c3e824 - mg: git: https://github.com/hkalexling/mg.git version: 0.5.0+git.commit.697e46e27cde8c3969346e228e372db2455a6264 diff --git a/shard.yml b/shard.yml index 1b7f7b3..0598146 100644 --- a/shard.yml +++ b/shard.yml @@ -41,5 +41,3 @@ dependencies: github: epoch/tallboy mg: github: hkalexling/mg - mangadex: - github: hkalexling/mangadex diff --git a/src/config.cr b/src/config.cr index 6de78a7..332a159 100644 --- a/src/config.cr +++ b/src/config.cr @@ -33,10 +33,8 @@ class Config "download_retries" => 4, "download_queue_db_path" => File.expand_path("~/mango/queue.db", home: true), - "chapter_rename_rule" => "[Vol.{volume} ]" \ - "[Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", - "subscription_update_interval_hours" => 24, + "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", + "manga_rename_rule" => "{title}", } @@singlet : Config? diff --git a/src/library/library.cr b/src/library/library.cr index e359847..a5a4a80 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,25 +42,6 @@ class Library end end end - - subscription_interval = Config.current - .mangadex["subscription_update_interval_hours"].as Int32 - unless subscription_interval < 1 - spawn do - loop do - subscriptions = Storage.default.subscriptions - Logger.info "Checking MangaDex for updates on " \ - "#{subscriptions.size} subscriptions" - added_count = 0 - subscriptions.each do |sub| - added_count += sub.check_for_updates - end - Logger.info "Subscription update completed. Added #{added_count} " \ - "chapters to the download queue" - sleep subscription_interval.hours - end - end - end end def titles diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr deleted file mode 100644 index e677a71..0000000 --- a/src/mangadex/downloader.cr +++ /dev/null @@ -1,172 +0,0 @@ -require "mangadex" -require "compress/zip" -require "../rename" -require "./ext" - -module MangaDex - class PageJob - property success = false - property url : String - property filename : String - property writer : Compress::Zip::Writer - property tries_remaning : Int32 - - def initialize(@url, @filename, @writer, @tries_remaning) - end - end - - class Downloader < Queue::Downloader - @wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"] - .to_i32 - @retries : Int32 = Config.current.mangadex["download_retries"].to_i32 - - use_default - - def initialize - @client = Client.from_config - super - end - - def pop : Queue::Job? - job = nil - MainFiber.run do - DB.open "sqlite3://#{@queue.path}" do |db| - begin - db.query_one "select * from queue where id not like '%-%' " \ - "and (status = 0 or status = 1) " \ - "order by time limit 1" do |res| - job = Queue::Job.from_query_result res - end - rescue - end - end - end - job - end - - private def download(job : Queue::Job) - @downloading = true - @queue.set_status Queue::JobStatus::Downloading, job - begin - chapter = @client.chapter job.id - # We must put the `.pages` call in a rescue block to handle external - # chapters. - pages = chapter.pages - rescue e - Logger.error e - @queue.set_status Queue::JobStatus::Error, job - unless e.message.nil? - @queue.add_message e.message.not_nil!, job - end - @downloading = false - return - end - @queue.set_pages pages.size, job - lib_dir = @library_path - rename_rule = Rename::Rule.new \ - Config.current.mangadex["manga_rename_rule"].to_s - manga_dir = File.join lib_dir, chapter.manga.rename rename_rule - unless File.exists? manga_dir - Dir.mkdir_p manga_dir - end - zip_path = File.join manga_dir, "#{job.title}.cbz.part" - - # Find the number of digits needed to store the number of pages - len = Math.log10(pages.size).to_i + 1 - - writer = Compress::Zip::Writer.new zip_path - # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new pages.size - spawn do - pages.each_with_index do |url, i| - fn = Path.new(URI.parse(url).path).basename - ext = File.extname fn - fn = "#{i.to_s.rjust len, '0'}#{ext}" - page_job = PageJob.new url, fn, writer, @retries - Logger.debug "Downloading #{url}" - loop do - sleep @wait_seconds.seconds - download_page page_job - break if page_job.success || - page_job.tries_remaning <= 0 - page_job.tries_remaning -= 1 - Logger.warn "Failed to download page #{url}. " \ - "Retrying... Remaining retries: " \ - "#{page_job.tries_remaning}" - end - - channel.send page_job - break unless @queue.exists? job - end - end - - spawn do - page_jobs = [] of PageJob - pages.size.times do - page_job = channel.receive - - break unless @queue.exists? job - - Logger.debug "[#{page_job.success ? "success" : "failed"}] " \ - "#{page_job.url}" - page_jobs << page_job - if page_job.success - @queue.add_success job - else - @queue.add_fail job - msg = "Failed to download page #{page_job.url}" - @queue.add_message msg, job - Logger.error msg - end - end - - unless @queue.exists? job - Logger.debug "Download cancelled" - @downloading = false - next - end - - fail_count = page_jobs.count { |j| !j.success } - Logger.debug "Download completed. " \ - "#{fail_count}/#{page_jobs.size} failed" - writer.close - filename = File.join File.dirname(zip_path), File.basename(zip_path, - ".part") - File.rename zip_path, filename - Logger.debug "cbz File created at #{filename}" - - zip_exception = validate_archive filename - if !zip_exception.nil? - @queue.add_message "The downloaded archive is corrupted. " \ - "Error: #{zip_exception}", job - @queue.set_status Queue::JobStatus::Error, job - elsif fail_count > 0 - @queue.set_status Queue::JobStatus::MissingPages, job - else - @queue.set_status Queue::JobStatus::Completed, job - end - @downloading = false - end - end - - private def download_page(job : PageJob) - Logger.debug "downloading #{job.url}" - headers = HTTP::Headers{ - "User-agent" => "Mangadex.cr", - } - begin - HTTP::Client.get job.url, headers do |res| - unless res.success? - raise "Failed to download page #{job.url}. " \ - "[#{res.status_code}] #{res.status_message}" - end - job.writer.add job.filename, res.body_io - end - job.success = true - rescue e - Logger.error e - job.success = false - end - end - end -end diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr deleted file mode 100644 index e919d97..0000000 --- a/src/mangadex/ext.cr +++ /dev/null @@ -1,94 +0,0 @@ -private macro properties_to_hash(names) - { - {% for name in names %} - "{{name.id}}" => {{name.id}}.to_s, - {% end %} - } -end - -# Monkey-patch the structures in the `mangadex` shard to suit our needs -module MangaDex - struct Client - @@group_cache = {} of String => Group - - def self.from_config : Client - self.new base_url: Config.current.mangadex["base_url"].to_s, - api_url: Config.current.mangadex["api_url"].to_s - end - end - - struct Manga - def rename(rule : Rename::Rule) - rule.render properties_to_hash %w(id title author artist) - end - - def to_info_json - hash = JSON.parse(to_json).as_h - _chapters = chapters.map do |c| - JSON.parse c.to_info_json - end - hash["chapters"] = JSON::Any.new _chapters - hash.to_json - end - end - - struct Chapter - def rename(rule : Rename::Rule) - hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.join(",", &.name) - rule.render hash - end - - def full_title - rule = Rename::Rule.new \ - Config.current.mangadex["chapter_rename_rule"].to_s - rename rule - end - - def to_info_json - hash = JSON.parse(to_json).as_h - hash["language"] = JSON::Any.new language - _groups = {} of String => JSON::Any - groups.each do |g| - _groups[g.name] = JSON::Any.new g.id - end - hash["groups"] = JSON::Any.new _groups - hash["full_title"] = JSON::Any.new full_title - hash.to_json - end - - # We don't need to rename the manga title here. It will be renamed in - # src/mangadex/downloader.cr - def to_job : Queue::Job - Queue::Job.new( - id.to_s, - manga_id.to_s, - full_title, - manga_title, - Queue::JobStatus::Pending, - Time.unix timestamp - ) - end - end - - struct User - def updates_after(time : Time, &block : Chapter ->) - page = 1 - stopped = false - until stopped - chapters = followed_updates(page: page).chapters - return if chapters.empty? - chapters.each do |c| - if time > Time.unix c.timestamp - stopped = true - break - end - yield c - end - page += 1 - # Let's not DDOS MangaDex :) - sleep 5.seconds - end - end - end -end diff --git a/src/mango.cr b/src/mango.cr index 768058f..0912009 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -2,7 +2,6 @@ require "./config" require "./queue" require "./server" require "./main_fiber" -require "./mangadex/*" require "./plugin/*" require "option_parser" require "clim" @@ -59,7 +58,6 @@ class CLI < Clim Storage.default Queue.default Library.default - MangaDex::Downloader.default Plugin::Downloader.default spawn do diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 616d452..fd63ec8 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -73,9 +73,5 @@ struct AdminRouter get "/admin/missing" do |env| layout "missing-items" end - - get "/admin/mangadex" do |env| - layout "mangadex" - end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index a66210a..b1fb3b3 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1,6 +1,6 @@ -require "../mangadex/*" require "../upload" require "koa" +require "digest" struct APIRouter @@api_json : String? @@ -56,31 +56,20 @@ struct APIRouter "error" => String?, } - Koa.schema("mdChapter", { - "id" => Int64, - "group" => {} of String => String, - }.merge(s %w(title volume chapter language full_title time - manga_title manga_id)), - desc: "A MangaDex chapter") - - Koa.schema "mdManga", { - "id" => Int64, - "chapters" => ["mdChapter"], - }.merge(s %w(title description author artist cover_url)), - desc: "A MangaDex manga" - Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.path "page", schema: Int32, desc: "The page number to return (starts from 1)" Koa.response 200, schema: Bytes, media_type: "image/*" Koa.response 500, "Page not found or not readable" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.tag "reader" get "/api/page/:tid/:eid/:page" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] page = env.params.url["page"].to_i + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -90,7 +79,15 @@ struct APIRouter raise "Failed to load page #{page} of " \ "`#{title.title}/#{entry.title}`" if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -102,12 +99,14 @@ struct APIRouter Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" Koa.response 200, schema: Bytes, media_type: "image/*" + Koa.response 304, "Page not modified (only available when `If-None-Match` is set)" Koa.response 500, "Page not found or not readable" Koa.tag "library" get "/api/cover/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -118,7 +117,14 @@ struct APIRouter raise "Failed to get cover of `#{title.title}/#{entry.title}`" \ if img.nil? - send_img env, img + e_tag = Digest::SHA1.hexdigest img.data + if prev_e_tag == e_tag + env.response.status_code = 304 + "" + else + env.response.headers["ETag"] = e_tag + send_img env, img + end rescue e Logger.error e env.response.status_code = 500 @@ -323,58 +329,6 @@ struct APIRouter end end - Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex"] - Koa.path "id", desc: "A MangaDex manga ID" - Koa.response 200, schema: "mdManga" - get "/api/admin/mangadex/manga/:id" do |env| - begin - id = env.params.url["id"] - manga = MangaDex::Client.from_config.manga id - send_json env, manga.to_info_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - - Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex", "downloader"] - Koa.body schema: { - "chapters" => ["mdChapter"], - } - Koa.response 200, schema: { - "success" => Int32, - "fail" => Int32, - } - post "/api/admin/mangadex/download" do |env| - begin - chapters = env.params.json["chapters"].as(Array).map &.as_h - jobs = chapters.map { |chapter| - Queue::Job.new( - chapter["id"].as_i64.to_s, - chapter["mangaId"].as_i64.to_s, - chapter["full_title"].as_s, - chapter["mangaTitle"].as_s, - Queue::JobStatus::Pending, - Time.unix chapter["timestamp"].as_i64 - ) - } - inserted_count = Queue.default.push jobs - send_json env, { - "success": inserted_count, - "fail": jobs.size - inserted_count, - }.to_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 @@ -631,21 +585,32 @@ struct APIRouter "height" => Int32, }], } + Koa.response 304, "Not modified (only available when `If-None-Match` is set)" get "/api/dimensions/:tid/:eid" do |env| begin tid = env.params.url["tid"] eid = env.params.url["eid"] + prev_e_tag = env.request.headers["If-None-Match"]? title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - sizes = entry.page_dimensions - send_json env, { - "success" => true, - "dimensions" => sizes, - }.to_json + file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s) + e_tag = "W/#{file_hash}" + if e_tag == prev_e_tag + env.response.status_code = 304 + "" + else + sizes = entry.page_dimensions + env.response.headers["ETag"] = e_tag + env.response.headers["Cache-Control"] = "public, max-age=86400" + send_json env, { + "success" => true, + "dimensions" => sizes, + }.to_json + end rescue e Logger.error e send_json env, { @@ -904,239 +869,6 @@ struct APIRouter end end - Koa.describe "Logs the current user into their MangaDex account", <<-MD - If successful, returns the expiration date (as a unix timestamp) of the newly created token. - MD - Koa.body schema: { - "username" => String, - "password" => String, - } - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - post "/api/admin/mangadex/login" do |env| - begin - username = env.params.json["username"].as String - password = env.params.json["password"].as String - mango_username = get_username env - - client = MangaDex::Client.from_config - client.auth username, password - - Storage.default.save_md_token mango_username, client.token.not_nil!, - client.token_expires - - send_json env, { - "success" => true, - "error" => nil, - "expires" => client.token_expires.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - get "/api/admin/mangadex/expires" do |env| - begin - username = get_username env - _, expires = Storage.default.get_md_token username - - send_json env, { - "success" => true, - "error" => nil, - "expires" => expires.try &.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Searches MangaDex for manga matching `query`", <<-MD - Returns an empty list if the current user hasn't logged in to MangaDex. - MD - Koa.query "query" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "manga?" => [{ - "id" => Int64, - "title" => String, - "description" => String, - "mainCover" => String, - }], - } - Koa.tags ["admin", "mangadex"] - get "/api/admin/mangadex/search" do |env| - begin - query = env.params.query["query"] - - send_json env, { - "success" => true, - "error" => nil, - "manga" => get_client(env).partial_search query, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Lists all MangaDex subscriptions" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "subscriptions?" => [{ - "id" => Int64, - "username" => String, - "manga_id" => Int64, - "language" => String?, - "group_id" => Int64?, - "min_volume" => Int64?, - "max_volume" => Int64?, - "min_chapter" => Int64?, - "max_chapter" => Int64?, - "last_checked" => Int64, - "created_at" => Int64, - }], - } - Koa.tags ["admin", "mangadex", "subscriptions"] - get "/api/admin/mangadex/subscriptions" do |env| - begin - send_json env, { - "success" => true, - "error" => nil, - "subscriptions" => Storage.default.subscriptions, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Creates a new MangaDex subscription" - Koa.body schema: { - "subscription" => { - "manga" => Int64, - "language" => String?, - "groupId" => Int64?, - "volumeMin" => Int64?, - "volumeMax" => Int64?, - "chapterMin" => Int64?, - "chapterMax" => Int64?, - }, - } - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - } - Koa.tags ["admin", "mangadex", "subscriptions"] - post "/api/admin/mangadex/subscriptions" do |env| - begin - json = env.params.json["subscription"].as Hash(String, JSON::Any) - sub = Subscription.new json["manga"].as_i64, get_username env - sub.language = json["language"]?.try &.as_s? - sub.group_id = json["groupId"]?.try &.as_i64? - sub.min_volume = json["volumeMin"]?.try &.as_i64? - sub.max_volume = json["volumeMax"]?.try &.as_i64? - sub.min_chapter = json["chapterMin"]?.try &.as_i64? - sub.max_chapter = json["chapterMax"]?.try &.as_i64? - - Storage.default.save_subscription sub - - send_json env, { - "success" => true, - "error" => nil, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Deletes a MangaDex subscription identified by `id`", <<-MD - Does nothing if the subscription was not created by the current user. - MD - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - } - Koa.tags ["admin", "mangadex", "subscriptions"] - delete "/api/admin/mangadex/subscriptions/:id" do |env| - begin - id = env.params.url["id"].to_i64 - Storage.default.delete_subscription id, get_username env - send_json env, { - "success" => true, - "error" => nil, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Triggers an update for a MangaDex subscription identified by `id`", <<-MD - Does nothing if the subscription was not created by the current user. - MD - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - } - Koa.tags ["admin", "mangadex", "subscriptions"] - post "/api/admin/mangadex/subscriptions/check/:id" do |env| - begin - id = env.params.url["id"].to_i64 - username = get_username env - sub = Storage.default.get_subscription id, username - unless sub - raise "Subscription with id #{id} not found under user #{username}" - end - spawn do - sub.check_for_updates - end - send_json env, { - "success" => true, - "error" => nil, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/routes/main.cr b/src/routes/main.cr index 54e3fba..57917bb 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -72,11 +72,6 @@ struct MainRouter end end - get "/download" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] - layout "download" - end - get "/download/plugins" do |env| begin id = env.params.query["plugin"]? @@ -96,12 +91,6 @@ struct MainRouter end end - get "/download/subscription" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] - username = get_username env - layout "subscription" - end - get "/" do |env| begin username = get_username env diff --git a/src/storage.cr b/src/storage.cr index 164ce40..39116b9 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -5,7 +5,6 @@ require "base64" require "./util/*" require "mg" require "../migration/*" -require "./subscription" def hash_password(pw) Crypto::Bcrypt::Password.create(pw).to_s @@ -15,9 +14,6 @@ def verify_password(hash, pw) (Crypto::Bcrypt::Password.new hash).verify pw end -SUB_ATTR = %w(manga_id language group_id min_volume max_volume min_chapter - max_chapter username) - class Storage @@insert_entry_ids = [] of IDTuple @@insert_title_ids = [] of IDTuple @@ -549,70 +545,6 @@ class Storage {token, expires} end - def save_subscription(sub : Subscription) - MainFiber.run do - get_db do |db| - {% begin %} - db.exec "insert into subscription (#{SUB_ATTR.join ","}, " \ - "last_checked, created_at) values " \ - "(#{Array.new(SUB_ATTR.size + 2, "?").join ","})", - {% for type in SUB_ATTR %} - sub.{{type.id}}, - {% end %} - sub.last_checked.to_unix, sub.created_at.to_unix - {% end %} - end - end - end - - def subscriptions : Array(Subscription) - subs = [] of Subscription - MainFiber.run do - get_db do |db| - db.query "select * from subscription" do |rs| - subs += Subscription.from_rs rs - end - end - end - subs - end - - def delete_subscription(id : Int64, username : String) - MainFiber.run do - get_db do |db| - db.exec "delete from subscription where id = (?) and username = (?)", - id, username - end - end - end - - def get_subscription(id : Int64, username : String) : Subscription? - sub = nil - MainFiber.run do - get_db do |db| - db.query "select * from subscription where id = (?) and " \ - "username = (?) limit 1", id, username do |rs| - sub = Subscription.from_rs(rs).first? - end - end - end - sub - end - - def update_subscription_last_checked(id : Int64? = nil) - MainFiber.run do - get_db do |db| - if id - db.exec "update subscription set last_checked = (?) where id = (?)", - Time.utc.to_unix, id - else - db.exec "update subscription set last_checked = (?)", - Time.utc.to_unix - end - end - end - end - def close MainFiber.run do unless @db.nil? diff --git a/src/util/web.cr b/src/util/web.cr index efa269d..12459e5 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -107,25 +107,6 @@ macro get_sort_opt end end -# Returns an authorized client -def get_client(username : String) : MangaDex::Client - token, expires = Storage.default.get_md_token username - - unless expires && token - raise "No token found for user #{username}" - end - - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires - - client -end - -def get_client(env) : MangaDex::Client - get_client get_username env -end - module HTTP class Client private def self.exec(uri : URI, tls : TLSContext = nil) diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index a6e9b31..fb64d3e 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -33,7 +33,6 @@ -
  • Connect to MangaDex
    • diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 2075e01..c264177 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -5,63 +5,61 @@ -
    - - - - - - - - - - +
    ChapterMangaProgressTimeStatusPluginActions
    + + + + + + + + + + + + + + +
    ChapterMangaProgressTimeStatusPluginActions
    +
    <% content_for "script" do %> diff --git a/src/views/download.html.ecr b/src/views/download.html.ecr index 983cae0..0ea8527 100644 --- a/src/views/download.html.ecr +++ b/src/views/download.html.ecr @@ -1,170 +1,162 @@

    Download from MangaDex

    -
    -
    - -
    -
    -
    - -
    +
    +
    +
    +
    +
    + +
    +
    - -
    -
    -
    - -
    -
    -

    Title:

    -

    -

    -
    -
    -

    - Filter Chapters - -

    -

    -
    - -
    - -
    -
    - -
    - -
    - -
    -
    - -
    - -
    - -
    -
    - -
    - -
    - -
    -
    -
    +
    +
    +
    + +
    +
    +

    Title:

    +

    +

    +
    +
    +

    Filter Chapters

    +

    +
    + +
    + +
    -
    - - - -
    -
    -

    Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

    + +
    + +
    -

    -
    - - - - - - - - - - - - -
    IDTitleLanguageGroupVolumeChapterTimestamp