Finish plugin functionalities

public/js/download-manager.js

@@ -119,11 +119,12 @@ const load = () => {
 				const dropdown = obj.status_message.length > 0 ? `<div uk-dropdown>${obj.status_message}</div>` : '';
 				const retryBtn = obj.status_message.length > 0 ? `<a onclick="refresh('${obj.id}')" uk-icon="refresh"></a>` : '';
 				return `<tr id="chapter-${obj.id}">
-				<td><a href="${baseURL}/chapter/${obj.id}">${obj.title}</a></td>
+				<td>${obj.plugin_name ? obj.title : `<a href="${baseURL}/chapter/${obj.id}">${obj.title}</a>`}</td>
-				<td><a href="${baseURL}/manga/${obj.manga_id}">${obj.manga_title}</a></td>
+				<td>${obj.plugin_name ? obj.manga_title : `<a href="${baseURL}/manga/${obj.manga_id}">${obj.manga_title}</a>`}</td>
 				<td>${obj.success_count}/${obj.pages}</td>
 				<td>${moment(obj.time).fromNow()}</td>
 				<td>${statusSpan} ${dropdown}</td>
+				<td>${obj.plugin_name || ""}</td>
 				<td>
 					<a onclick="remove('${obj.id}')" uk-icon="trash"></a>
 					${retryBtn}

public/js/plugin-download.js

@@ -4,8 +4,18 @@ $(() => {
 			search();
 		}
 	});
+	$('#plugin-select').change(() => {
+		const title = $('#plugin-select').val();
+		const url = `${location.protocol}//${location.host}${location.pathname}`;
+		const newURL = `${url}?${$.param({
+			plugin: encodeURIComponent(title)
+		})}`;
+		window.location.href = newURL;
+	});
+	$('#plugin-select').val(plugin);
 });
 
+let mangaTitle = "";
 let searching = false;
 const search = () => {
 	if (searching)
@@ -28,9 +38,92 @@ const search = () => {
 				alert('danger', `Search failed. Error: ${data.error}`);
 				return;
 			}
+			mangaTitle = data.title;
+			$('#title-text').text(data.title);
+			buildTable(data.chapters);
 		})
 		.fail((jqXHR, status) => {
 			alert('danger', `Search failed. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
 		})
 		.always(() => {});
 };
+
+const buildTable = (chapters) => {
+	$('#table').attr('hidden', '');
+	$('table').empty();
+
+	const keys = Object.keys(chapters[0]).map(k => `<th>${k}</th>`).join('');
+	const thead = `<thead><tr>${keys}</tr></thead>`;
+	$('table').append(thead);
+
+	const rows = chapters.map(ch => {
+		const tds = Object.values(ch).map(v => `<td>${v}</td>`).join('');
+		return `<tr data-id="${ch.id}" data-title="${ch.title}">${tds}</tr>`;
+	});
+	const tbody = `<tbody id="selectable">${rows}</tbody>`;
+	$('table').append(tbody);
+
+	$('#selectable').selectable({
+		filter: 'tr'
+	});
+
+	$('#table').removeAttr('hidden');
+};
+
+const selectAll = () => {
+	$('tbody > tr').each((i, e) => {
+		$(e).addClass('ui-selected');
+	});
+};
+
+const unselect = () => {
+	$('tbody > tr').each((i, e) => {
+		$(e).removeClass('ui-selected');
+	});
+};
+
+const download = () => {
+	const selected = $('tbody > tr.ui-selected');
+	if (selected.length === 0) return;
+	UIkit.modal.confirm(`Download ${selected.length} selected chapters?`).then(() => {
+		$('#download-btn').attr('hidden', '');
+		$('#download-spinner').removeAttr('hidden');
+		const chapters = selected.map((i, e) => {
+			return {
+				id: $(e).attr('data-id'),
+				title: $(e).attr('data-title')
+			}
+		}).get();
+		console.log(chapters);
+		$.ajax({
+				type: 'POST',
+				url: base_url + 'api/admin/plugin/download',
+				data: JSON.stringify({
+					plugin: plugin,
+					chapters: chapters,
+					title: mangaTitle
+				}),
+				contentType: "application/json",
+				dataType: 'json'
+			})
+			.done(data => {
+				console.log(data);
+				if (data.error) {
+					alert('danger', `Failed to add chapters to the download queue. Error: ${data.error}`);
+					return;
+				}
+				const successCount = parseInt(data.success);
+				const failCount = parseInt(data.fail);
+				UIkit.modal.confirm(`${successCount} of ${successCount + failCount} chapters added to the download queue. Proceed to the download manager?`).then(() => {
+					window.location.href = base_url + 'admin/downloads';
+				});
+			})
+			.fail((jqXHR, status) => {
+				alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
+			})
+			.always(() => {
+				$('#download-spinner').attr('hidden', '');
+				$('#download-btn').removeAttr('hidden');
+			});
+	});
+};

src/plugin/downloader.cr

@@ -35,20 +35,22 @@ class Plugin
           raise "Job does not have plugin name specificed"
         end
 
-        plugin = Plugin.new job.plugin_name.not_nil!
+        plugin = Plugin.new_from_id job.plugin_name.not_nil!
-        info = plugin.select_chapter job.id
+        info = plugin.select_chapter job.plugin_chapter_id.not_nil!
 
-        title = process_filename info["title"].as_s
         pages = info["pages"].as_i
 
+        manga_title = process_filename job.manga_title
+        chapter_title = process_filename info["title"].as_s
+
         @queue.set_pages pages, job
         lib_dir = @library_path
-        manga_dir = File.join lib_dir, title
+        manga_dir = File.join lib_dir, manga_title
         unless File.exists? manga_dir
           Dir.mkdir_p manga_dir
         end
 
-        zip_path = File.join manga_dir, "#{job.title}.cbz.part"
+        zip_path = File.join manga_dir, "#{chapter_title}.cbz.part"
         writer = Zip::Writer.new zip_path
       rescue e
         @queue.set_status Queue::JobStatus::Error, job
@@ -76,7 +78,7 @@ class Plugin
         tries = 4
 
         loop do
-          sleep plugin.wait_seconds.seconds
+          sleep plugin.info.wait_seconds.seconds
           Logger.debug "downloading #{url}"
           tries -= 1
 

src/plugin/plugin.cr

@@ -1,6 +1,7 @@
 require "duktape/runtime"
 require "myhtml"
 require "http"
+require "xml"
 
 class Plugin
   class Error < ::Exception
@@ -15,70 +16,164 @@ class Plugin
   class SyntaxError < Error
   end
 
-  {% for name in ["id", "title", "author", "version", "placeholder"] %}
+  struct Info
-    getter {{name.id}} = ""
+    {% for name in ["id", "title", "author", "version", "placeholder"] %}
-  {% end %}
+      getter {{name.id}} = ""
-  getter wait_seconds : UInt64 = 0
+    {% end %}
-  getter filename
+    getter wait_seconds : UInt64 = 0
+    getter dir : String
 
-  def self.list
+    def initialize(@dir)
-    dir = Config.current.plugin_path
+      info_path = File.join @dir, "info.json"
-    Dir.mkdir_p dir unless Dir.exists? dir
 
-    Dir.children(dir)
+      unless File.exists? info_path
-      .select do |f|
+        raise MetadataError.new "File `info.json` not found in the " \
-        fp = File.join dir, f
+                                "plugin directory #{dir}"
-        File.file?(fp) && File.extname(fp) == ".js"
       end
-      .map do |f|
+
-        File.basename f, ".js"
+      json = JSON.parse File.read info_path
+
+      begin
+        {% for name in ["id", "title", "author", "version", "placeholder"] %}
+          @{{name.id}} = json[{{name}}].as_s
+        {% end %}
+        @wait_seconds = json["wait_seconds"].as_i.to_u64
+
+        unless @id.alphanumeric_underscore?
+          raise "Plugin ID can only contain alphanumeric characters and " \
+                "underscores"
+        end
+      rescue e
+        raise MetadataError.new "Failed to retrieve metadata from plugin " \
+                                "at #{@dir}. Error: #{e.message}"
       end
+    end
   end
 
-  def initialize(@filename : String)
+  struct State
+    @hash = {} of String => String
+
+    def initialize(@path : String)
+      unless File.exists? @path
+        save
+      end
+
+      json = JSON.parse File.read @path
+      json.as_h.each do |k, v|
+        @hash[k] = v.as_s
+      end
+    end
+
+    def []?(key)
+      @hash[key]?
+    end
+
+    def []=(key, val : String)
+      @hash[key] = val
+    end
+
+    def save
+      File.write @path, @hash.to_pretty_json
+    end
+  end
+
+  @@info_ary = [] of Info
+  @info : Info?
+
+  getter js_path = ""
+  getter state_path = ""
+
+  def self.build_info_ary
+    return unless @@info_ary.empty?
+
     dir = Config.current.plugin_path
     Dir.mkdir_p dir unless Dir.exists? dir
 
-    @path = File.join dir, "#{filename}.js"
+    Dir.each_child dir do |f|
-    unless File.exists? @path
+      path = File.join dir, f
-      raise Error.new "Plugin script not found at #{@path}"
+      next unless File.directory? path
+
+      begin
+        @@info_ary << Info.new path
+      rescue e : MetadataError
+        Logger.warn e
+      end
+    end
+  end
+
+  def self.list
+    self.build_info_ary
+
+    @@info_ary.map &.title
+  end
+
+  def info
+    @info.not_nil!
+  end
+
+  def self.new_from_id(id : String)
+    self.build_info_ary
+
+    info = @@info_ary.find { |i| i.id == id }
+    raise Error.new "Plugin with id #{id} not found" unless info
+    self.new info.title
+  end
+
+  def initialize(title : String)
+    Plugin.build_info_ary
+
+    @info = @@info_ary.find { |i| i.title == title }
+    if @info.nil?
+      raise Error.new "Plugin with title #{title} not found"
+    end
+
+    @js_path = File.join info.dir, "main.js"
+    @state_path = File.join info.dir, "state.json"
+
+    unless File.exists? @js_path
+      raise Error.new "Plugin script not found at #{@js_path}"
     end
 
     @rt = Duktape::Runtime.new do |sbx|
       sbx.push_global_object
 
-      sbx.del_prop_string -1, "print"
+      sbx.push_pointer @state_path.as(Void*)
-      sbx.del_prop_string -1, "alert"
+      path = sbx.require_pointer(-1).as String
-      sbx.del_prop_string -1, "console"
+      sbx.pop
+      sbx.push_string path
+      sbx.put_prop_string -2, "state_path"
 
       def_helper_functions sbx
     end
 
-    eval File.read @path
+    eval File.read @js_path
+  end
 
-    begin
+  macro check_fields(ary)
-      data = eval_json "metadata"
+    {% for field in ary %}
-      {% for name in ["id", "title", "author", "version", "placeholder"] %}
+      unless json[{{field}}]?
-        @{{name.id}} = data[{{name}}].as_s
+        raise "Field `{{field.id}}` is missing from the function outputs"
-      {% end %}
+      end
-      @wait_seconds = data["wait_seconds"].as_i.to_u64
+    {% end %}
-    rescue e
-      raise MetadataError.new "Failed to retrieve metadata from plugin " \
-                              "at #{@path}. Error: #{e.message}"
-    end
   end
 
   def search(query : String)
     json = eval_json "search('#{query}')"
     begin
-      ary = json.as_a
+      check_fields ["title", "chapters"]
+
+      ary = json["chapters"].as_a
       ary.each do |obj|
         id = obj["id"]?
         raise "Field `id` missing from `search` outputs" if id.nil?
 
-        unless id.to_s.chars.all? &.number?
+        unless id.to_s.alphanumeric_underscore?
-          raise "The `id` values must be numeric" unless id
+          raise "The `id` field can only contain alphanumeric characters and " \
+                "underscores"
         end
+
+        title = obj["title"]?
+        raise "Field `title` missing from `search` outputs" if title.nil?
       end
     rescue e
       raise Error.new e.message
@@ -89,12 +184,11 @@ class Plugin
   def select_chapter(id : String)
     json = eval_json "selectChapter('#{id}')"
     begin
-      {% for field in ["title", "pages"] %}
+      check_fields ["title", "pages"]
-        unless json[{{field}}]?
+
-          raise "Field `{{field.id}}` is missing from the " \
+      if json["title"].to_s.empty?
-                "`selectChapter` outputs"
+        raise "The `title` field of the chapter can not be empty"
-        end
+      end
-      {% end %}
     rescue e
       raise Error.new e.message
     end
@@ -105,12 +199,7 @@ class Plugin
     json = eval_json "nextPage()"
     return if json.size == 0
     begin
-      {% for field in ["filename", "url"] %}
+      check_fields ["filename", "url"]
-        unless json[{{field}}]?
-          raise "Field `{{field.id}}` is missing from the " \
-                "`nextPage` outputs"
-        end
-      {% end %}
     rescue e
       raise Error.new e.message
     end
@@ -173,16 +262,28 @@ class Plugin
       env = Duktape::Sandbox.new ptr
       html = env.require_string 0
 
-      myhtml = Myhtml::Parser.new html
+      str = XML.parse(html).inner_text
-      root = myhtml.root
-
-      str = ""
-      str = root.inner_text if root
 
       env.push_string str
       env.call_success
     end
-    sbx.put_prop_string -2, "innerText"
+    sbx.put_prop_string -2, "text"
+
+    sbx.push_proc 2 do |ptr|
+      env = Duktape::Sandbox.new ptr
+      html = env.require_string 0
+      name = env.require_string 1
+
+      begin
+        attr = XML.parse(html).first_element_child.not_nil![name]
+        env.push_string attr
+      rescue
+        env.push_undefined
+      end
+
+      env.call_success
+    end
+    sbx.put_prop_string -2, "attribute"
 
     sbx.push_proc 1 do |ptr|
       env = Duktape::Sandbox.new ptr
@@ -193,6 +294,32 @@ class Plugin
     end
     sbx.put_prop_string -2, "raise"
 
+    sbx.push_proc LibDUK::VARARGS do |ptr|
+      env = Duktape::Sandbox.new ptr
+      key = env.require_string 0
+
+      env.get_global_string "state_path"
+      state_path = env.require_string -1
+      env.pop
+      state = State.new state_path
+
+      if env.get_top == 2
+        val = env.require_string 1
+        state[key] = val
+        state.save
+      else
+        val = state[key]?
+        if val
+          env.push_string val
+        else
+          env.push_undefined
+        end
+      end
+
+      env.call_success
+    end
+    sbx.put_prop_string -2, "state"
+
     sbx.put_prop_string -2, "mango"
   end
 end

src/queue.cr

@@ -51,6 +51,7 @@ class Queue
     property fail_count : Int32 = 0
     property time : Time
     property plugin_name : String?
+    property plugin_chapter_id : String?
 
     def parse_query_result(res : DB::ResultSet)
       @id = res.read String
@@ -69,7 +70,7 @@ class Queue
       ary = @id.split("-")
       if ary.size == 2
         @plugin_name = ary[0]
-        @id = ary[1]
+        @plugin_chapter_id = ary[1]
       end
     end
 
@@ -99,6 +100,7 @@ class Queue
         json.field "time" do
           json.number @time.to_unix_ms
         end
+        json.field "plugin_name", @plugin_name if @plugin_name
       end
     end
   end

src/routes/api.cr

@@ -265,11 +265,43 @@ class APIRouter < Router
         query = env.params.json["query"].as String
         plugin = Plugin.new env.params.json["plugin"].as String
 
-        chapters = plugin.search query
+        json = plugin.search query
+        chapters = json["chapters"]
+        title = json["title"]
 
         send_json env, {
           "success"  => true,
           "chapters" => chapters,
+          "title"    => title,
+        }.to_json
+      rescue e
+        send_json env, {
+          "success" => false,
+          "error"   => e.message,
+        }.to_json
+      end
+    end
+
+    post "/api/admin/plugin/download" do |env|
+      begin
+        plugin = Plugin.new env.params.json["plugin"].as String
+        chapters = env.params.json["chapters"].as Array(JSON::Any)
+        manga_title = env.params.json["title"].as String
+
+        jobs = chapters.map { |ch|
+          Queue::Job.new(
+            "#{plugin.info.id}-#{ch["id"]}",
+            "", # manga_id
+            ch["title"].as_s,
+            manga_title,
+            Queue::JobStatus::Pending,
+            Time.utc
+          )
+        }
+        inserted_count = @context.queue.push jobs
+        send_json env, {
+          "success": inserted_count,
+          "fail":    jobs.size - inserted_count,
         }.to_json
       rescue e
         send_json env, {

src/routes/main.cr

@@ -79,9 +79,21 @@ class MainRouter < Router
     end
 
     get "/download/plugins" do |env|
-      plugins = Plugin.list
+      begin
-      plugin = Plugin.new plugins[0]
+        title = env.params.query["plugin"]?
-      layout "plugin-download"
+        plugins = Plugin.list
+
+        if title
+          plugin = Plugin.new title
+        else
+          plugin = Plugin.new plugins[0]
+        end
+
+        layout "plugin-download"
+      rescue e
+        @context.error e
+        env.response.status_code = 500
+      end
     end
 
     get "/" do |env|

src/util/util.cr

@@ -54,3 +54,9 @@ macro use_default
     @@default.not_nil!
   end
 end
+
+class String
+  def alphanumeric_underscore?
+    self.chars.all? { |c| c.alphanumeric? || c == '_' }
+  end
+end

src/views/download-manager.html.ecr

@@ -17,6 +17,7 @@
       <th>Progress</th>
       <th>Time</th>
       <th>Status</th>
+      <th>Plugin</th>
       <th>Actions</th>
     </tr>
   </thead>

src/views/plugin-download.html.ecr

@@ -13,7 +13,7 @@
       <div class="uk-margin">
         <label class="uk-form-label" for="search-input">&nbsp;</label>
         <div class="uk-form-controls">
-          <input id="search-input" class="uk-input" type="text" placeholder="<%= plugin.placeholder %>">
+          <input id="search-input" class="uk-input" type="text" placeholder="<%= plugin.info.placeholder %>">
         </div>
       </div>
     </div>
@@ -30,13 +30,29 @@
       </div>
     </div>
   </div>
+
+
+  <div id="table" class="uk-margin-large-top" hidden>
+    <h3 id="title-text"></h3>
+
+    <div class="uk-margin">
+      <button class="uk-button uk-button-default" onclick="selectAll()">Select All</button>
+      <button class="uk-button uk-button-default" onclick="unselect()">Clear Selections</button>
+      <button class="uk-button uk-button-primary" id="download-btn" onclick="download()">Download Selected</button>
+      <div id="download-spinner" uk-spinner class="uk-margin-left" hidden></div>
+    </div>
+    <p class="uk-text-meta">Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.</p>
+    <table class="uk-table uk-table-striped uk-overflow-auto">
+    </table>
+  </div>
 <% end %>
 
 
 <% content_for "script" do %>
   <script>
-    var plugin = "<%= plugin.filename %>";
+    var plugin = "<%= plugin.info.title %>";
   </script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
   <script src="<%= base_url %>js/alert.js"></script>
   <script src="<%= base_url %>js/plugin-download.js"></script>
 <% end %>