diff --git a/public/js/download.js b/public/js/download.js deleted file mode 100644 index 4d8504b..0000000 --- a/public/js/download.js +++ /dev/null @@ -1,285 +0,0 @@ -const downloadComponent = () => { - return { - chaptersLimit: 1000, - loading: false, - addingToDownload: false, - searchAvailable: false, - searchInput: '', - data: {}, - chapters: [], - mangaAry: undefined, // undefined: not searching; []: searched but no result - candidateManga: {}, - langChoice: 'All', - groupChoice: 'All', - chapterRange: '', - volumeRange: '', - - get languages() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - set.add(chp.language); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - get groups() { - const set = new Set(); - if (this.data.chapters) { - this.data.chapters.forEach(chp => { - Object.keys(chp.groups).forEach(g => { - set.add(g); - }); - }); - } - const ary = [...set].sort(); - ary.unshift('All'); - return ary; - }, - - init() { - const tableObserver = new MutationObserver(() => { - console.log('table mutated'); - $("#selectable").selectable({ - filter: 'tr' - }); - }); - tableObserver.observe($('table').get(0), { - childList: true, - subtree: true - }); - - $.getJSON(`${base_url}api/admin/mangadex/expires`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); - return; - } - if (data.expires && data.expires > Math.floor(Date.now() / 1000)) - this.searchAvailable = true; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - }, - - filtersUpdated() { - if (!this.data.chapters) - this.chapters = []; - const filters = { - chapter: this.parseRange(this.chapterRange), - volume: this.parseRange(this.volumeRange), - lang: this.langChoice, - group: this.groupChoice - }; - console.log('filters:', filters); - let _chapters = this.data.chapters.slice(); - Object.entries(filters).forEach(([k, v]) => { - if (v === 'All') return; - if (k === 'group') { - _chapters = _chapters.filter(c => { - const unescaped_groups = Object.entries(c.groups).map(([g, id]) => this.unescapeHTML(g)); - return unescaped_groups.indexOf(v) >= 0; - }); - return; - } - if (k === 'lang') { - _chapters = _chapters.filter(c => c.language === v); - return; - } - const lb = parseFloat(v[0]); - const ub = parseFloat(v[1]); - if (isNaN(lb) && isNaN(ub)) return; - _chapters = _chapters.filter(c => { - const val = parseFloat(c[k]); - if (isNaN(val)) return false; - if (isNaN(lb)) - return val <= ub; - else if (isNaN(ub)) - return val >= lb; - else - return val >= lb && val <= ub; - }); - }); - console.log('filtered chapters:', _chapters); - this.chapters = _chapters; - }, - - search() { - if (this.loading || this.searchInput === '') return; - this.data = {}; - this.mangaAry = undefined; - - var int_id = -1; - try { - const path = new URL(this.searchInput).pathname; - const match = /\/(?:title|manga)\/([0-9]+)/.exec(path); - int_id = parseInt(match[1]); - } catch (e) { - int_id = parseInt(this.searchInput); - } - - if (!isNaN(int_id) && int_id > 0) { - // The input is a positive integer. We treat it as an ID. - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/manga/${int_id}`) - .done((data) => { - if (data.error) { - alert('danger', 'Failed to get manga info. Error: ' + data.error); - return; - } - - this.data = data; - this.chapters = data.chapters; - this.mangaAry = undefined; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to get manga info. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } else { - if (!this.searchAvailable) { - alert('danger', 'Please make sure you are using a valid manga ID or manga URL from Mangadex. If you are trying to search MangaDex with a search term, please log in to MangaDex first by going to "Admin -> Connect to MangaDex".'); - return; - } - - // Search as a search term - this.loading = true; - $.getJSON(`${base_url}api/admin/mangadex/search?${$.param({ - query: this.searchInput - })}`) - .done((data) => { - if (data.error) { - alert('danger', `Failed to search MangaDex. Error: ${data.error}`); - return; - } - - this.mangaAry = data.manga; - this.data = {}; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to search MangaDex. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loading = false; - }); - } - }, - - parseRange(str) { - const regex = /^[\t ]*(?:(?:(<|<=|>|>=)[\t ]*([0-9]+))|(?:([0-9]+))|(?:([0-9]+)[\t ]*-[\t ]*([0-9]+))|(?:[\t ]*))[\t ]*$/m; - const matches = str.match(regex); - var num; - - if (!matches) { - return [null, null]; - } else if (typeof matches[1] !== 'undefined' && typeof matches[2] !== 'undefined') { - // e.g., <= 30 - num = parseInt(matches[2]); - if (isNaN(num)) { - return [null, null]; - } - switch (matches[1]) { - case '<': - return [null, num - 1]; - case '<=': - return [null, num]; - case '>': - return [num + 1, null]; - case '>=': - return [num, null]; - } - } else if (typeof matches[3] !== 'undefined') { - // a single number - num = parseInt(matches[3]); - if (isNaN(num)) { - return [null, null]; - } - return [num, num]; - } else if (typeof matches[4] !== 'undefined' && typeof matches[5] !== 'undefined') { - // e.g., 10 - 23 - num = parseInt(matches[4]); - const n2 = parseInt(matches[5]); - if (isNaN(num) || isNaN(n2) || num > n2) { - return [null, null]; - } - return [num, n2]; - } else { - // empty or space only - return [null, null]; - } - }, - - unescapeHTML(str) { - var elt = document.createElement("span"); - elt.innerHTML = str; - return elt.innerText; - }, - - selectAll() { - $('tbody > tr').each((i, e) => { - $(e).addClass('ui-selected'); - }); - }, - - clearSelection() { - $('tbody > tr').each((i, e) => { - $(e).removeClass('ui-selected'); - }); - }, - - download() { - const selected = $('tbody > tr.ui-selected'); - if (selected.length === 0) return; - UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => { - const ids = selected.map((i, e) => { - return parseInt($(e).find('td').first().text()); - }).get(); - const chapters = this.chapters.filter(c => ids.indexOf(c.id) >= 0); - console.log(ids); - this.addingToDownload = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/download`, - data: JSON.stringify({ - chapters: chapters - }), - contentType: "application/json", - dataType: 'json' - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`); - return; - } - const successCount = parseInt(data.success); - const failCount = parseInt(data.fail); - alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.addingToDownload = false; - }); - }); - }, - - chooseManga(manga) { - this.candidateManga = manga; - UIkit.modal($('#modal').get(0)).show(); - }, - - confirmManga(id) { - UIkit.modal($('#modal').get(0)).hide(); - this.searchInput = id; - this.search(); - } - }; -}; diff --git a/public/js/mangadex.js b/public/js/mangadex.js deleted file mode 100644 index 3271c4b..0000000 --- a/public/js/mangadex.js +++ /dev/null @@ -1,61 +0,0 @@ -const component = () => { - return { - username: '', - password: '', - expires: undefined, - loading: true, - loggingIn: false, - - init() { - this.loading = true; - $.ajax({ - type: 'GET', - url: `${base_url}api/admin/mangadex/expires`, - contentType: "application/json", - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to retrieve MangaDex token status. Error: ${data.error}`); - return; - } - this.expires = data.expires; - this.loading = false; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to retrieve MangaDex token status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }); - }, - login() { - if (!(this.username && this.password)) return; - this.loggingIn = true; - $.ajax({ - type: 'POST', - url: `${base_url}api/admin/mangadex/login`, - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({ - username: this.username, - password: this.password - }) - }) - .done(data => { - console.log(data); - if (data.error) { - alert('danger', `Failed to log in. Error: ${data.error}`); - return; - } - this.expires = data.expires; - }) - .fail((jqXHR, status) => { - alert('danger', `Failed to log in. Error: [${jqXHR.status}] ${jqXHR.statusText}`); - }) - .always(() => { - this.loggingIn = false; - }); - }, - get expired() { - return this.expires && moment().diff(moment.unix(this.expires)) > 0; - } - }; -}; diff --git a/shard.lock b/shard.lock index c22f839..a8e9e6d 100644 --- a/shard.lock +++ b/shard.lock @@ -52,10 +52,6 @@ shards: git: https://github.com/hkalexling/koa.git version: 0.7.0 - mangadex: - git: https://github.com/hkalexling/mangadex.git - version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6 - mg: git: https://github.com/hkalexling/mg.git version: 0.3.0+git.commit.a19417abf03eece80039f89569926cff1ce3a1a3 diff --git a/shard.yml b/shard.yml index 3991a13..d66f521 100644 --- a/shard.yml +++ b/shard.yml @@ -43,5 +43,3 @@ dependencies: github: epoch/tallboy mg: github: hkalexling/mg - mangadex: - github: hkalexling/mangadex diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr deleted file mode 100644 index e677a71..0000000 --- a/src/mangadex/downloader.cr +++ /dev/null @@ -1,172 +0,0 @@ -require "mangadex" -require "compress/zip" -require "../rename" -require "./ext" - -module MangaDex - class PageJob - property success = false - property url : String - property filename : String - property writer : Compress::Zip::Writer - property tries_remaning : Int32 - - def initialize(@url, @filename, @writer, @tries_remaning) - end - end - - class Downloader < Queue::Downloader - @wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"] - .to_i32 - @retries : Int32 = Config.current.mangadex["download_retries"].to_i32 - - use_default - - def initialize - @client = Client.from_config - super - end - - def pop : Queue::Job? - job = nil - MainFiber.run do - DB.open "sqlite3://#{@queue.path}" do |db| - begin - db.query_one "select * from queue where id not like '%-%' " \ - "and (status = 0 or status = 1) " \ - "order by time limit 1" do |res| - job = Queue::Job.from_query_result res - end - rescue - end - end - end - job - end - - private def download(job : Queue::Job) - @downloading = true - @queue.set_status Queue::JobStatus::Downloading, job - begin - chapter = @client.chapter job.id - # We must put the `.pages` call in a rescue block to handle external - # chapters. - pages = chapter.pages - rescue e - Logger.error e - @queue.set_status Queue::JobStatus::Error, job - unless e.message.nil? - @queue.add_message e.message.not_nil!, job - end - @downloading = false - return - end - @queue.set_pages pages.size, job - lib_dir = @library_path - rename_rule = Rename::Rule.new \ - Config.current.mangadex["manga_rename_rule"].to_s - manga_dir = File.join lib_dir, chapter.manga.rename rename_rule - unless File.exists? manga_dir - Dir.mkdir_p manga_dir - end - zip_path = File.join manga_dir, "#{job.title}.cbz.part" - - # Find the number of digits needed to store the number of pages - len = Math.log10(pages.size).to_i + 1 - - writer = Compress::Zip::Writer.new zip_path - # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new pages.size - spawn do - pages.each_with_index do |url, i| - fn = Path.new(URI.parse(url).path).basename - ext = File.extname fn - fn = "#{i.to_s.rjust len, '0'}#{ext}" - page_job = PageJob.new url, fn, writer, @retries - Logger.debug "Downloading #{url}" - loop do - sleep @wait_seconds.seconds - download_page page_job - break if page_job.success || - page_job.tries_remaning <= 0 - page_job.tries_remaning -= 1 - Logger.warn "Failed to download page #{url}. " \ - "Retrying... Remaining retries: " \ - "#{page_job.tries_remaning}" - end - - channel.send page_job - break unless @queue.exists? job - end - end - - spawn do - page_jobs = [] of PageJob - pages.size.times do - page_job = channel.receive - - break unless @queue.exists? job - - Logger.debug "[#{page_job.success ? "success" : "failed"}] " \ - "#{page_job.url}" - page_jobs << page_job - if page_job.success - @queue.add_success job - else - @queue.add_fail job - msg = "Failed to download page #{page_job.url}" - @queue.add_message msg, job - Logger.error msg - end - end - - unless @queue.exists? job - Logger.debug "Download cancelled" - @downloading = false - next - end - - fail_count = page_jobs.count { |j| !j.success } - Logger.debug "Download completed. " \ - "#{fail_count}/#{page_jobs.size} failed" - writer.close - filename = File.join File.dirname(zip_path), File.basename(zip_path, - ".part") - File.rename zip_path, filename - Logger.debug "cbz File created at #{filename}" - - zip_exception = validate_archive filename - if !zip_exception.nil? - @queue.add_message "The downloaded archive is corrupted. " \ - "Error: #{zip_exception}", job - @queue.set_status Queue::JobStatus::Error, job - elsif fail_count > 0 - @queue.set_status Queue::JobStatus::MissingPages, job - else - @queue.set_status Queue::JobStatus::Completed, job - end - @downloading = false - end - end - - private def download_page(job : PageJob) - Logger.debug "downloading #{job.url}" - headers = HTTP::Headers{ - "User-agent" => "Mangadex.cr", - } - begin - HTTP::Client.get job.url, headers do |res| - unless res.success? - raise "Failed to download page #{job.url}. " \ - "[#{res.status_code}] #{res.status_message}" - end - job.writer.add job.filename, res.body_io - end - job.success = true - rescue e - Logger.error e - job.success = false - end - end - end -end diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr deleted file mode 100644 index deb09c8..0000000 --- a/src/mangadex/ext.cr +++ /dev/null @@ -1,60 +0,0 @@ -private macro properties_to_hash(names) - { - {% for name in names %} - "{{name.id}}" => {{name.id}}.to_s, - {% end %} - } -end - -# Monkey-patch the structures in the `mangadex` shard to suit our needs -module MangaDex - struct Client - @@group_cache = {} of String => Group - - def self.from_config : Client - self.new base_url: Config.current.mangadex["base_url"].to_s, - api_url: Config.current.mangadex["api_url"].to_s - end - end - - struct Manga - def rename(rule : Rename::Rule) - rule.render properties_to_hash %w(id title author artist) - end - - def to_info_json - hash = JSON.parse(to_json).as_h - _chapters = chapters.map do |c| - JSON.parse c.to_info_json - end - hash["chapters"] = JSON::Any.new _chapters - hash.to_json - end - end - - struct Chapter - def rename(rule : Rename::Rule) - hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.join(",", &.name) - rule.render hash - end - - def full_title - rule = Rename::Rule.new \ - Config.current.mangadex["chapter_rename_rule"].to_s - rename rule - end - - def to_info_json - hash = JSON.parse(to_json).as_h - hash["language"] = JSON::Any.new language - _groups = {} of String => JSON::Any - groups.each do |g| - _groups[g.name] = JSON::Any.new g.id - end - hash["groups"] = JSON::Any.new _groups - hash["full_title"] = JSON::Any.new full_title - hash.to_json - end - end -end diff --git a/src/mango.cr b/src/mango.cr index 768058f..0912009 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -2,7 +2,6 @@ require "./config" require "./queue" require "./server" require "./main_fiber" -require "./mangadex/*" require "./plugin/*" require "option_parser" require "clim" @@ -59,7 +58,6 @@ class CLI < Clim Storage.default Queue.default Library.default - MangaDex::Downloader.default Plugin::Downloader.default spawn do diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 616d452..fd63ec8 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -73,9 +73,5 @@ struct AdminRouter get "/admin/missing" do |env| layout "missing-items" end - - get "/admin/mangadex" do |env| - layout "mangadex" - end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index c84af20..8ae011f 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1,4 +1,3 @@ -require "../mangadex/*" require "../upload" require "koa" @@ -56,19 +55,6 @@ struct APIRouter "error" => String?, } - Koa.schema("mdChapter", { - "id" => Int64, - "group" => {} of String => String, - }.merge(s %w(title volume chapter language full_title time - manga_title manga_id)), - desc: "A MangaDex chapter") - - Koa.schema "mdManga", { - "id" => Int64, - "chapters" => ["mdChapter"], - }.merge(s %w(title description author artist cover_url)), - desc: "A MangaDex manga" - Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" @@ -323,58 +309,6 @@ struct APIRouter end end - Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex"] - Koa.path "id", desc: "A MangaDex manga ID" - Koa.response 200, schema: "mdManga" - get "/api/admin/mangadex/manga/:id" do |env| - begin - id = env.params.url["id"] - manga = MangaDex::Client.from_config.manga id - send_json env, manga.to_info_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - - Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD - On error, returns a JSON that contains the error message in the `error` field. - MD - Koa.tags ["admin", "mangadex", "downloader"] - Koa.body schema: { - "chapters" => ["mdChapter"], - } - Koa.response 200, schema: { - "success" => Int32, - "fail" => Int32, - } - post "/api/admin/mangadex/download" do |env| - begin - chapters = env.params.json["chapters"].as(Array).map &.as_h - jobs = chapters.map { |chapter| - Queue::Job.new( - chapter["id"].as_i64.to_s, - chapter["mangaId"].as_i64.to_s, - chapter["full_title"].as_s, - chapter["mangaTitle"].as_s, - Queue::JobStatus::Pending, - Time.unix chapter["timestamp"].as_i64 - ) - } - inserted_count = Queue.default.push jobs - send_json env, { - "success": inserted_count, - "fail": jobs.size - inserted_count, - }.to_json - rescue e - Logger.error e - send_json env, {"error" => e.message}.to_json - end - end - ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 @@ -904,115 +838,6 @@ struct APIRouter end end - Koa.describe "Logs the current user into their MangaDex account", <<-MD - If successful, returns the expiration date (as a unix timestamp) of the newly created token. - MD - Koa.body schema: { - "username" => String, - "password" => String, - } - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - post "/api/admin/mangadex/login" do |env| - begin - username = env.params.json["username"].as String - password = env.params.json["password"].as String - mango_username = get_username env - - client = MangaDex::Client.from_config - client.auth username, password - - Storage.default.save_md_token mango_username, client.token.not_nil!, - client.token_expires - - send_json env, { - "success" => true, - "error" => nil, - "expires" => client.token_expires.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "expires" => Int64?, - } - Koa.tags ["admin", "mangadex", "users"] - get "/api/admin/mangadex/expires" do |env| - begin - username = get_username env - _, expires = Storage.default.get_md_token username - - send_json env, { - "success" => true, - "error" => nil, - "expires" => expires.try &.to_unix, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - - Koa.describe "Searches MangaDex for manga matching `query`", <<-MD - Returns an empty list if the current user hasn't logged in to MangaDex. - MD - Koa.query "query" - Koa.response 200, schema: { - "success" => Bool, - "error" => String?, - "manga?" => [{ - "id" => Int64, - "title" => String, - "description" => String, - "mainCover" => String, - }], - } - Koa.tags ["admin", "mangadex"] - get "/api/admin/mangadex/search" do |env| - begin - username = get_username env - token, expires = Storage.default.get_md_token username - - unless expires && token - raise "No token found for user #{username}" - end - - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires - - query = env.params.query["query"] - - send_json env, { - "success" => true, - "error" => nil, - "manga" => client.partial_search query, - }.to_json - rescue e - Logger.error e - send_json env, { - "success" => false, - "error" => e.message, - }.to_json - end - end - doc = Koa.generate @@api_json = doc.to_json if doc diff --git a/src/routes/main.cr b/src/routes/main.cr index 6c1a611..57917bb 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -72,11 +72,6 @@ struct MainRouter end end - get "/download" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] - layout "download" - end - get "/download/plugins" do |env| begin id = env.params.query["plugin"]? diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index a6e9b31..fb64d3e 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -33,7 +33,6 @@ -
  • Connect to MangaDex
    • diff --git a/src/views/layout.html.ecr b/src/views/layout.html.ecr index ff4853c..70c5a51 100644 --- a/src/views/layout.html.ecr +++ b/src/views/layout.html.ecr @@ -1,89 +1,87 @@ - <%= render_component "head" %> + <%= render_component "head" %> - -
    -
    -
    -
    - -
    -
    -
    + +
    +
    +
    +
    + +
    -
    -
    -
    -
    -
    -
    - - -
    -
    - -
    -
    +
    +
    +
    +
    +
    +
    -
    -
    -
    -
    -
    - <%= content %> -
    - +
    + +
    + + <% end %> +
    - - <%= render_component "uikit" %> - <%= yield_content "script" %> - +
    + +
    +
    +
    +
    +
    +
    +
    +
    + <%= content %> +
    + +
    +
    +
    + + <%= render_component "uikit" %> + <%= yield_content "script" %> +