From 031df3a2bc8a6f03e1138695dff56efe3be91840 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 22 Jan 2022 13:11:58 +0000 Subject: [PATCH] Finish plugin updater --- public/js/plugin-download.js | 19 ++++----- src/config.cr | 1 + src/mango.cr | 1 + src/plugin/plugin.cr | 4 ++ src/plugin/subscriptions.cr | 38 ++++++++++++------ src/plugin/updater.cr | 74 ++++++++++++++++++++++++++++++++++++ src/routes/api.cr | 5 ++- 7 files changed, 120 insertions(+), 22 deletions(-) create mode 100644 src/plugin/updater.cr diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index 75a3313..ef18d57 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -100,14 +100,14 @@ const component = () => { this.mangaTitle = data.title; } - data.chapters.forEach((c) => { - c.array = ["hello", "world", "haha", "wtf"] - .sort(() => 0.5 - Math.random()) - .slice(0, 2); - c.date = [612892800000, "1625068800000"].sort( - () => 0.5 - Math.random() - )[0]; - }); + //data.chapters.forEach((c) => { + //c.array = ["hello", "world", "haha", "wtf"] + //.sort(() => 0.5 - Math.random()) + //.slice(0, 2); + //c.date = [612892800000, "1625068800000"].sort( + //() => 0.5 - Math.random() + //)[0]; + //}); this.allChapters = data.chapters; this.chapters = data.chapters; @@ -387,7 +387,8 @@ const component = () => { filters: this.filterSettings, plugin: this.pid, name: this.subscriptionName.trim(), - manga: this.mid, + manga: this.mangaTitle, + manga_id: this.mid, }), headers: { "Content-Type": "application/json", diff --git a/src/config.cr b/src/config.cr index aa818c3..41a4cf1 100644 --- a/src/config.cr +++ b/src/config.cr @@ -28,6 +28,7 @@ class Config property disable_login = false property default_username = "" property auth_proxy_header_name = "" + property plugin_update_interval_hours : Int32 = 24 property mangadex = Hash(String, String | Int32).new @[YAML::Field(ignore: true)] diff --git a/src/mango.cr b/src/mango.cr index 8716d04..c3e86cd 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -61,6 +61,7 @@ class CLI < Clim Library.load_instance Library.default Plugin::Downloader.default + Plugin::Updater.new spawn do begin diff --git a/src/plugin/plugin.cr b/src/plugin/plugin.cr index e743915..7f44ccc 100644 --- a/src/plugin/plugin.cr +++ b/src/plugin/plugin.cr @@ -143,6 +143,10 @@ class Plugin SubscriptionList.new(info.dir).ary end + def list_subscriptions_raw + SubscriptionList.new(info.dir) + end + def unsubscribe(id : String) list = SubscriptionList.new info.dir list.reject! &.id.== id diff --git a/src/plugin/subscriptions.cr b/src/plugin/subscriptions.cr index 5a19027..f60e11c 100644 --- a/src/plugin/subscriptions.cr +++ b/src/plugin/subscriptions.cr @@ -50,35 +50,40 @@ struct Filter end def match_chapter(obj : JSON::Any) : Bool + return true if value.nil? || value.to_s.empty? raw_value = obj[key] case type when FilterType::String - raw_value.as_s.lower == value.as_s.lower - when FilterType::NumMin - when FilterType::DateMin - BigFloat.new(raw_value) >= BigFloat.new(value) - when FilterType::NumMax - when FilterType::DateMax - BigFloat.new(raw_value) <= BigFloat.new(value) + raw_value.as_s.downcase == value.to_s.downcase + when FilterType::NumMin, FilterType::DateMin + raw_value.as_bf >= BigFloat.new value.not_nil!.to_f32 + when FilterType::NumMax, FilterType::DateMax + raw_value.as_bf <= BigFloat.new value.not_nil!.to_f32 when FilterType::Array - raw_value.as_s.lower.split(",") - .map(&.strip).include? value.as_s.lower.strip + return true if value == "all" + raw_value.as_s.downcase.split(",") + .map(&.strip).includes? value.to_s.downcase.strip + else + false end end end -struct Subscription +# We use class instead of struct so we can update `last_checked` from +# `SubscriptionList` +class Subscription include JSON::Serializable property id : String property plugin_id : String property manga_id : String + property manga_title : String property name : String property created_at : Int64 property last_checked : Int64 property filters = [] of Filter - def initialize(@plugin_id, @manga_id, @name) + def initialize(@plugin_id, @manga_id, @manga_title, @name) @id = UUID.random.to_s @created_at = Time.utc.to_unix @last_checked = Time.utc.to_unix @@ -108,3 +113,14 @@ struct SubscriptionList File.write @path, @ary.to_pretty_json end end + +struct JSON::Any + def as_bf : BigFloat + i64_or_f32 = begin + as_i64 + rescue + as_f32 + end + BigFloat.new i64_or_f32 + end +end diff --git a/src/plugin/updater.cr b/src/plugin/updater.cr new file mode 100644 index 0000000..4b69826 --- /dev/null +++ b/src/plugin/updater.cr @@ -0,0 +1,74 @@ +require "../config" + +class Plugin + class Updater + def initialize + interval = Config.current.plugin_update_interval_hours + return if interval <= 0 + spawn do + loop do + Plugin.list.map(&.["id"]).each do |pid| + check_updates pid + end + sleep interval.hours + end + end + end + + def check_updates(plugin_id : String) + Logger.debug "Checking plugin #{plugin_id} for updates" + + plugin = Plugin.new plugin_id + if plugin.info.version == 1 + Logger.debug "Plugin #{plugin_id} is targeting API version 1. " \ + "Skipping update check" + return + end + + subscriptions = plugin.list_subscriptions_raw + subscriptions.each do |sub| + check_subscription plugin, sub + end + subscriptions.save + rescue e + Logger.error "Error checking plugin #{plugin_id} for updates: " \ + "#{e.message}" + end + + def check_subscription(plugin : Plugin, sub : Subscription) + Logger.debug "Checking subscription #{sub.name} for updates" + matches = plugin.new_chapters(sub.manga_id, sub.last_checked) + .as_a.select do |chapter| + sub.match_chapter chapter + end + if matches.empty? + Logger.debug "No new chapters found." + return + end + Logger.debug "Found #{matches.size} new chapters. " \ + "Pushing to download queue" + jobs = matches.map { |ch| + Queue::Job.new( + "#{plugin.info.id}-#{ch["id"]}", + "", # manga_id + ch["title"].as_s, + sub.manga_title, + Queue::JobStatus::Pending, + Time.utc + ) + } + inserted_count = Queue.default.push jobs + Logger.info "#{inserted_count}/#{matches.size} new chapters added " \ + "to the download queue. Plugin ID #{plugin.info.id}, " \ + "subscription name #{sub.name}" + if inserted_count != matches.size + Logger.error "Failed to add #{matches.size - inserted_count} " \ + "chapters to download queue" + end + sub.last_checked = Time.utc.to_unix + rescue e + Logger.error "Error when checking updates for subscription " \ + "#{sub.name}: #{e.message}" + end + end +end diff --git a/src/routes/api.cr b/src/routes/api.cr index 5d589b5..4489f85 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -661,13 +661,14 @@ struct APIRouter post "/api/admin/plugin/subscriptions" do |env| begin plugin_id = env.params.json["plugin"].as String - manga_id = env.params.json["manga"].as String + manga_title = env.params.json["manga"].as String + manga_id = env.params.json["manga_id"].as String filters = env.params.json["filters"].as(Array(JSON::Any)).map do |f| Filter.from_json f.to_json end name = env.params.json["name"].as String - sub = Subscription.new plugin_id, manga_id, name + sub = Subscription.new plugin_id, manga_id, manga_title, name sub.filters = filters plugin = Plugin.new plugin_id