diff --git a/migration/subscription.12.cr b/migration/subscription.12.cr new file mode 100644 index 0000000..3810755 --- /dev/null +++ b/migration/subscription.12.cr @@ -0,0 +1,31 @@ +class CreateSubscription < MG::Base + def up : String + # We allow multiple subscriptions for the same manga. + # This can be useful for example when you want to download from multiple + # groups. + <<-SQL + CREATE TABLE subscription ( + id INTEGER PRIMARY KEY, + manga_id INTEGER NOT NULL, + language TEXT, + group_id INTEGER, + min_volume INTEGER, + max_volume INTEGER, + min_chapter INTEGER, + max_chapter INTEGER, + last_checked INTEGER NOT NULL, + created_at INTEGER NOT NULL, + username TEXT NOT NULL, + FOREIGN KEY (username) REFERENCES users (username) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + SQL + end + + def down : String + <<-SQL + DROP TABLE subscription; + SQL + end +end diff --git a/public/js/download.js b/public/js/download.js index 74fab76..957767d 100644 --- a/public/js/download.js +++ b/public/js/download.js @@ -282,6 +282,99 @@ const downloadComponent = () => { UIkit.modal($('#modal').get(0)).hide(); this.searchInput = id; this.search(); + }, + + subscribe(langConfirmed = false, groupConfirmed = false) { + const filters = { + manga: this.data.id, + language: this.langChoice === 'All' ? null : this.langChoice, + group: this.groupChoice === 'All' ? null : this.groupChoice, + volume: this.volumeRange === '' ? null : this.volumeRange, + chapter: this.chapterRange === '' ? null : this.chapterRange + }; + + // Get group ID + if (filters.group) { + this.data.chapters.forEach(chp => { + const gid = chp.groups[filters.group]; + if (gid) { + filters.groupId = gid; + return; + } + }); + } + + // Parse range values + if (filters.volume) { + [filters.volumeMin, filters.volumeMax] = this.parseRange(filters.volume); + } + if (filters.chapter) { + [filters.chapterMin, filters.chapterMax] = this.parseRange(filters.chapter); + } + + if (!filters.language && !langConfirmed) { + UIkit.modal.confirm('You didn\'t specify a language in the filtering rules. This might cause Mango to download chapters that are not in your preferred language. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(true, groupConfirmed); + }); + return; + } + + if (!filters.group && !groupConfirmed) { + UIkit.modal.confirm('You didn\'t specify a group in the filtering rules. This might cause Mango to download multiple versions of the same chapter. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(langConfirmed, true); + }); + return; + } + + const mangaURL = `${mangadex_base_url}/manga/${filters.manga}`; + + console.log(filters); + UIkit.modal.confirm(`All FUTURE chapters matching the following filters will be downloaded:
+ + + IMPORTANT: Please make sure you are following the manga on MangaDex, otherwise Mango won't be able to receive any updates. To follow it, visit ${mangaURL} and click "Follow". + `, { + labels: { + ok: 'Confirm', + cancel: 'Cancel' + } + }).then(() => { + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions`, + data: JSON.stringify({ + subscription: filters + }), + contentType: "application/json", + dataType: 'json' + }) + .done(data => { + console.log(data); + if (data.error) { + alert('danger', `Failed to subscribe. Error: ${data.error}`); + return; + } + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to subscribe. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }); } }; }; diff --git a/public/js/subscription.js b/public/js/subscription.js new file mode 100644 index 0000000..e6e4909 --- /dev/null +++ b/public/js/subscription.js @@ -0,0 +1,73 @@ +const component = () => { + return { + available: undefined, + subscriptions: [], + + init() { + $.getJSON(`${base_url}api/admin/mangadex/expires`) + .done((data) => { + if (data.error) { + alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); + return; + } + this.available = Boolean(data.expires && data.expires > Math.floor(Date.now() / 1000)); + + if (this.available) this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + getSubscriptions() { + $.getJSON(`${base_url}api/admin/mangadex/subscriptions`) + .done(data => { + if (data.error) { + alert('danger', 'Failed to get subscriptions. Error: ' + data.error); + return; + } + this.subscriptions = data.subscriptions; + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to get subscriptions. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + rm(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'DELETE', + url: `${base_url}api/admin/mangadex/subscriptions/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to delete subscription. Error: ${data.error}`); + } + this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to delete subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + check(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions/check/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to check subscription. Error: ${data.error}`); + return; + } + alert('success', 'Mango is now checking the subscription for updates. This might take a while, but you can safely leave the page.'); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + } + }; +}; diff --git a/shard.lock b/shard.lock index c22f839..13ce55d 100644 --- a/shard.lock +++ b/shard.lock @@ -54,7 +54,7 @@ shards: mangadex: git: https://github.com/hkalexling/mangadex.git - version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6 + version: 0.11.0+git.commit.f5b0d64fbb138879fb9228b6e9ff34ec97c3e824 mg: git: https://github.com/hkalexling/mg.git diff --git a/src/config.cr b/src/config.cr index 332a159..6de78a7 100644 --- a/src/config.cr +++ b/src/config.cr @@ -33,8 +33,10 @@ class Config "download_retries" => 4, "download_queue_db_path" => File.expand_path("~/mango/queue.db", home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", + "chapter_rename_rule" => "[Vol.{volume} ]" \ + "[Ch.{chapter} ]{title|id}", + "manga_rename_rule" => "{title}", + "subscription_update_interval_hours" => 24, } @@singlet : Config? diff --git a/src/library/library.cr b/src/library/library.cr index 48bd619..8fad451 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,6 +42,25 @@ class Library end end end + + subscription_interval = Config.current + .mangadex["subscription_update_interval_hours"].as Int32 + unless subscription_interval < 1 + spawn do + loop do + subscriptions = Storage.default.subscriptions + Logger.info "Checking MangaDex for updates on " \ + "#{subscriptions.size} subscriptions" + added_count = 0 + subscriptions.each do |sub| + added_count += sub.check_for_updates + end + Logger.info "Subscription update completed. Added #{added_count} " \ + "chapters to the download queue" + sleep subscription_interval.hours + end + end + end end def titles diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr index deb09c8..e919d97 100644 --- a/src/mangadex/ext.cr +++ b/src/mangadex/ext.cr @@ -56,5 +56,39 @@ module MangaDex hash["full_title"] = JSON::Any.new full_title hash.to_json end + + # We don't need to rename the manga title here. It will be renamed in + # src/mangadex/downloader.cr + def to_job : Queue::Job + Queue::Job.new( + id.to_s, + manga_id.to_s, + full_title, + manga_title, + Queue::JobStatus::Pending, + Time.unix timestamp + ) + end + end + + struct User + def updates_after(time : Time, &block : Chapter ->) + page = 1 + stopped = false + until stopped + chapters = followed_updates(page: page).chapters + return if chapters.empty? + chapters.each do |c| + if time > Time.unix c.timestamp + stopped = true + break + end + yield c + end + page += 1 + # Let's not DDOS MangaDex :) + sleep 5.seconds + end + end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index be7fc1d..8a43b9d 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -961,23 +961,95 @@ struct APIRouter Koa.tags ["admin", "mangadex"] get "/api/admin/mangadex/search" do |env| begin - username = get_username env - token, expires = Storage.default.get_md_token username - - unless expires && token - raise "No token found for user #{username}" - end - - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires - query = env.params.query["query"] send_json env, { "success" => true, "error" => nil, - "manga" => client.partial_search query, + "manga" => get_client(env).partial_search query, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + get "/api/admin/mangadex/subscriptions" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "subscriptions" => Storage.default.subscriptions, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + post "/api/admin/mangadex/subscriptions" do |env| + begin + json = env.params.json["subscription"].as Hash(String, JSON::Any) + sub = Subscription.new json["manga"].as_i64, get_username env + sub.language = json["language"]?.try &.as_s? + sub.group_id = json["groupId"]?.try &.as_i64? + sub.min_volume = json["volumeMin"]?.try &.as_i64? + sub.max_volume = json["volumeMax"]?.try &.as_i64? + sub.min_chapter = json["chapterMin"]?.try &.as_i64? + sub.max_chapter = json["chapterMax"]?.try &.as_i64? + + Storage.default.save_subscription sub + + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + delete "/api/admin/mangadex/subscriptions/:id" do |env| + begin + id = env.params.url["id"].to_i64 + Storage.default.delete_subscription id, get_username env + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + post "/api/admin/mangadex/subscriptions/check/:id" do |env| + begin + id = env.params.url["id"].to_i64 + username = get_username env + sub = Storage.default.get_subscription id, username + unless sub + raise "Subscription with id #{id} not found under user #{username}" + end + spawn do + sub.check_for_updates + end + send_json env, { + "success" => true, + "error" => nil, }.to_json rescue e Logger.error e diff --git a/src/routes/main.cr b/src/routes/main.cr index 2497c2f..2993db7 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -95,6 +95,12 @@ struct MainRouter end end + get "/download/subscription" do |env| + mangadex_base_url = Config.current.mangadex["base_url"] + username = get_username env + layout "subscription" + end + get "/" do |env| begin username = get_username env diff --git a/src/storage.cr b/src/storage.cr index 39116b9..164ce40 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -5,6 +5,7 @@ require "base64" require "./util/*" require "mg" require "../migration/*" +require "./subscription" def hash_password(pw) Crypto::Bcrypt::Password.create(pw).to_s @@ -14,6 +15,9 @@ def verify_password(hash, pw) (Crypto::Bcrypt::Password.new hash).verify pw end +SUB_ATTR = %w(manga_id language group_id min_volume max_volume min_chapter + max_chapter username) + class Storage @@insert_entry_ids = [] of IDTuple @@insert_title_ids = [] of IDTuple @@ -545,6 +549,70 @@ class Storage {token, expires} end + def save_subscription(sub : Subscription) + MainFiber.run do + get_db do |db| + {% begin %} + db.exec "insert into subscription (#{SUB_ATTR.join ","}, " \ + "last_checked, created_at) values " \ + "(#{Array.new(SUB_ATTR.size + 2, "?").join ","})", + {% for type in SUB_ATTR %} + sub.{{type.id}}, + {% end %} + sub.last_checked.to_unix, sub.created_at.to_unix + {% end %} + end + end + end + + def subscriptions : Array(Subscription) + subs = [] of Subscription + MainFiber.run do + get_db do |db| + db.query "select * from subscription" do |rs| + subs += Subscription.from_rs rs + end + end + end + subs + end + + def delete_subscription(id : Int64, username : String) + MainFiber.run do + get_db do |db| + db.exec "delete from subscription where id = (?) and username = (?)", + id, username + end + end + end + + def get_subscription(id : Int64, username : String) : Subscription? + sub = nil + MainFiber.run do + get_db do |db| + db.query "select * from subscription where id = (?) and " \ + "username = (?) limit 1", id, username do |rs| + sub = Subscription.from_rs(rs).first? + end + end + end + sub + end + + def update_subscription_last_checked(id : Int64? = nil) + MainFiber.run do + get_db do |db| + if id + db.exec "update subscription set last_checked = (?) where id = (?)", + Time.utc.to_unix, id + else + db.exec "update subscription set last_checked = (?)", + Time.utc.to_unix + end + end + end + end + def close MainFiber.run do unless @db.nil? diff --git a/src/subscription.cr b/src/subscription.cr new file mode 100644 index 0000000..1c0764a --- /dev/null +++ b/src/subscription.cr @@ -0,0 +1,83 @@ +require "db" +require "json" + +struct Subscription + include DB::Serializable + include JSON::Serializable + + getter id : Int64 = 0 + getter username : String + getter manga_id : Int64 + property language : String? + property group_id : Int64? + property min_volume : Int64? + property max_volume : Int64? + property min_chapter : Int64? + property max_chapter : Int64? + @[DB::Field(key: "last_checked")] + @[JSON::Field(key: "last_checked")] + @raw_last_checked : Int64 + @[DB::Field(key: "created_at")] + @[JSON::Field(key: "created_at")] + @raw_created_at : Int64 + + def last_checked : Time + Time.unix @raw_last_checked + end + + def created_at : Time + Time.unix @raw_created_at + end + + def initialize(@manga_id, @username) + @raw_created_at = Time.utc.to_unix + @raw_last_checked = Time.utc.to_unix + end + + def in_range?(value : String, lowerbound : Int64?, + upperbound : Int64?) : Bool + lb = lowerbound.try &.to_f64 + ub = upperbound.try &.to_f64 + + return true if lb.nil? && ub.nil? + + v = value.to_f64? + return false unless v + + if lb.nil? + v <= ub.not_nil! + elsif ub.nil? + v >= lb.not_nil! + else + v >= lb.not_nil! && v <= ub.not_nil! + end + end + + def match?(chapter : MangaDex::Chapter) : Bool + if chapter.manga_id != manga_id || + (language && chapter.language != language) || + (group_id && !chapter.groups.map(&.id).includes? group_id) + return false + end + + in_range?(chapter.volume, min_volume, max_volume) && + in_range?(chapter.chapter, min_chapter, max_chapter) + end + + def check_for_updates : Int32 + Logger.debug "Checking updates for subscription with ID #{id}" + jobs = [] of Queue::Job + get_client(username).user.updates_after last_checked do |chapter| + next unless match? chapter + jobs << chapter.to_job + end + Storage.default.update_subscription_last_checked id + count = Queue.default.push jobs + Logger.debug "#{count}/#{jobs.size} of updates added to queue" + count + rescue e + Logger.error "Error occurred when checking updates for " \ + "subscription with ID #{id}. #{e}" + 0 + end +end diff --git a/src/util/web.cr b/src/util/web.cr index 12459e5..efa269d 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -107,6 +107,25 @@ macro get_sort_opt end end +# Returns an authorized client +def get_client(username : String) : MangaDex::Client + token, expires = Storage.default.get_md_token username + + unless expires && token + raise "No token found for user #{username}" + end + + client = MangaDex::Client.from_config + client.token = token + client.token_expires = expires + + client +end + +def get_client(env) : MangaDex::Client + get_client get_username env +end + module HTTP class Client private def self.exec(uri : URI, tls : TLSContext = nil) diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 2b6e434..73a5445 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -5,61 +5,63 @@ - - - - - - - - - - - - - - - -
ChapterMangaProgressTimeStatusPluginActions
+ + + + + + <% content_for "script" do %> diff --git a/src/views/download.html.ecr b/src/views/download.html.ecr index 0ea8527..983cae0 100644 --- a/src/views/download.html.ecr +++ b/src/views/download.html.ecr @@ -1,162 +1,170 @@

Download from MangaDex

-
-
- -
-
-
- -
-
- - - -
-
-
- -
-
-

Title:

-

-

-
-
-

Filter Chapters

-

-
- -
- -
+
+
+
- -
- -
- -
+
+
+
- -
- -
- -
-
- -
- -
- -
-
-
-
-
- - - -
-
-

Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

-
-

- - - - - - - - - - - - +
IDTitleLanguageGroupVolumeChapterTimestamp