ARM support (#25, #78)

Dockerfile.arm32v7

@@ -0,0 +1,13 @@
+FROM arm32v7/ubuntu:18.04
+
+RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev
+
+RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.32.1 && make deps && cd ..
+RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && make && cd ..
+RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && make && cd ..
+
+COPY mango.o .
+
+RUN cc 'mango.o' -o 'mango'  -rdynamic  -lxml2 /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/arm-linux-gnueabihf/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib
+
+CMD ["./mango"]

src/main_fiber.cr

@@ -0,0 +1,29 @@
+# On ARM, connecting to the SQLite DB from a spawned fiber would crash
+#   https://github.com/crystal-lang/crystal-sqlite3/issues/30
+# This is a temporary workaround that forces the relevant code to run in the
+#   main fiber
+
+class MainFiber
+  @@channel = Channel(-> Nil).new
+  @@done = Channel(Bool).new
+
+  def self.start_and_block
+    loop do
+      if proc = @@channel.receive
+        begin
+          proc.call
+        ensure
+          @@done.send true
+        end
+      end
+      Fiber.yield
+    end
+  end
+
+  def self.run(&block : -> Nil)
+    @@channel.send block
+    until @@done.receive
+      Fiber.yield
+    end
+  end
+end

src/mango.cr

@@ -1,6 +1,7 @@
 require "./config"
 require "./queue"
 require "./server"
+require "./main_fiber"
 require "./mangadex/*"
 require "option_parser"
 require "clim"
@@ -54,8 +55,7 @@ class CLI < Clim
 
       # empty ARGV so it won't be passed to Kemal
       ARGV.clear
-      server = Server.new
+      Server.new.start
-      server.start
     end
 
     sub "admin" do
@@ -123,4 +123,8 @@ class CLI < Clim
   end
 end
 
-CLI.start(ARGV)
+spawn do
+  CLI.start(ARGV)
+end
+
+MainFiber.start_and_block

src/queue.cr

@@ -119,6 +119,7 @@ class Queue
                   "Attepmting to create it"
       Dir.mkdir_p dir
     end
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         begin
           db.exec "create table if not exists queue " \
@@ -138,11 +139,13 @@ class Queue
         end
       end
     end
+  end
 
   # Push an array of jobs into the queue, and return the number of jobs
   #   inserted. Any job already exists in the queue will be ignored.
   def push(jobs : Array(Job))
     start_count = self.count
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         jobs.each do |job|
           db.exec "insert or ignore into queue values " \
@@ -152,16 +155,19 @@ class Queue
             job.success_count, job.fail_count, job.time.to_unix_ms
         end
       end
+    end
     self.count - start_count
   end
 
   def reset(id : String)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set status = 0, status_message = '', " \
                 "pages = 0, success_count = 0, fail_count = 0 " \
                 "where id = (?)", id
       end
     end
+  end
 
   def reset(job : Job)
     self.reset job.id
@@ -169,91 +175,113 @@ class Queue
 
   # Reset all failed tasks (missing pages and error)
   def reset
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set status = 0, status_message = '', " \
                 "pages = 0, success_count = 0, fail_count = 0 " \
                 "where status = 2 or status = 4"
       end
     end
+  end
 
   def delete(id : String)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "delete from queue where id = (?)", id
       end
     end
+  end
 
   def delete(job : Job)
     self.delete job.id
   end
 
   def delete_status(status : JobStatus)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "delete from queue where status = (?)", status.to_i
       end
     end
+  end
 
   def count_status(status : JobStatus)
     num = 0
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         num = db.query_one "select count(*) from queue where " \
                            "status = (?)", status.to_i, as: Int32
       end
+    end
     num
   end
 
   def count
     num = 0
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         num = db.query_one "select count(*) from queue", as: Int32
       end
+    end
     num
   end
 
   def set_status(status : JobStatus, job : Job)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set status = (?) where id = (?)",
           status.to_i, job.id
       end
     end
+  end
 
   def get_all
     jobs = [] of Job
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         jobs = db.query_all "select * from queue order by time" do |rs|
           Job.from_query_result rs
         end
       end
+    end
     jobs
   end
 
   def add_success(job : Job)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set success_count = success_count + 1 " \
                 "where id = (?)", job.id
       end
     end
+  end
 
   def add_fail(job : Job)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set fail_count = fail_count + 1 " \
                 "where id = (?)", job.id
       end
     end
+  end
 
   def set_pages(pages : Int32, job : Job)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set pages = (?), success_count = 0, " \
                 "fail_count = 0 where id = (?)", pages, job.id
       end
     end
+  end
 
   def add_message(msg : String, job : Job)
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         db.exec "update queue set status_message = " \
                 "status_message || (?) || (?) where id = (?)",
           "\n", msg, job.id
       end
     end
+  end
 
   def <<(downloader : Downloader)
     @downloaders << downloader

src/storage.cr

@@ -32,6 +32,7 @@ class Storage
                   "Attepmting to create it"
       Dir.mkdir_p dir
     end
+    MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         begin
           # We create the `ids` table first. even if the uses has an
@@ -66,6 +67,7 @@ class Storage
         @db = DB.open "sqlite3://#{@path}"
       end
     end
+  end
 
   macro init_admin
     random_pw = random_str
@@ -87,6 +89,8 @@ class Storage
   end
 
   def verify_user(username, password)
+    out_token = nil
+    MainFiber.run do
       get_db do |db|
         begin
           hash, token = db.query_one "select password, token from " \
@@ -94,24 +98,29 @@ class Storage
             username, as: {String, String?}
           unless verify_password hash, password
             Logger.debug "Password does not match the hash"
-          return nil
+            next
           end
           Logger.debug "User #{username} verified"
-        return token if token
+          if token
+            out_token = token
+            next
+          end
           token = random_str
           Logger.debug "Updating token for #{username}"
           db.exec "update users set token = (?) where username = (?)",
             token, username
-        return token
+          out_token = token
         rescue e
           Logger.error "Error when verifying user #{username}: #{e}"
-        return nil
         end
       end
     end
+    out_token
+  end
 
   def verify_token(token)
     username = nil
+    MainFiber.run do
       get_db do |db|
         begin
           username = db.query_one "select username from users where " \
@@ -120,11 +129,13 @@ class Storage
           Logger.debug "Unable to verify token"
         end
       end
+    end
     username
   end
 
   def verify_admin(token)
     is_admin = false
+    MainFiber.run do
       get_db do |db|
         begin
           is_admin = db.query_one "select admin from users where " \
@@ -133,11 +144,13 @@ class Storage
           Logger.debug "Unable to verify user as admin"
         end
       end
+    end
     is_admin
   end
 
   def list_users
     results = Array(Tuple(String, Bool)).new
+    MainFiber.run do
       get_db do |db|
         db.query "select username, admin from users" do |rs|
           rs.each do
@@ -145,6 +158,7 @@ class Storage
           end
         end
       end
+    end
     results
   end
 
@@ -152,17 +166,20 @@ class Storage
     validate_username username
     validate_password password
     admin = (admin ? 1 : 0)
+    MainFiber.run do
       get_db do |db|
         hash = hash_password password
         db.exec "insert into users values (?, ?, ?, ?)",
           username, hash, nil, admin
       end
     end
+  end
 
   def update_user(original_username, username, password, admin)
     admin = (admin ? 1 : 0)
     validate_username username
     validate_password password unless password.empty?
+    MainFiber.run do
       get_db do |db|
         if password.empty?
           db.exec "update users set username = (?), admin = (?) " \
@@ -176,14 +193,18 @@ class Storage
         end
       end
     end
+  end
 
   def delete_user(username)
+    MainFiber.run do
       get_db do |db|
         db.exec "delete from users where username = (?)", username
       end
     end
+  end
 
   def logout(token)
+    MainFiber.run do
       get_db do |db|
         begin
           db.exec "update users set token = (?) where token = (?)", nil, token
@@ -191,13 +212,16 @@ class Storage
         end
       end
     end
+  end
 
   def get_id(path, is_title)
     id = nil
+    MainFiber.run do
       get_db do |db|
         id = db.query_one? "select id from ids where path = (?)", path,
           as: {String}
       end
+    end
     id
   end
 
@@ -206,6 +230,7 @@ class Storage
   end
 
   def bulk_insert_ids
+    MainFiber.run do
       get_db do |db|
         db.transaction do |tx|
           @insert_ids.each do |tp|
@@ -216,12 +241,15 @@ class Storage
       end
       @insert_ids.clear
     end
+  end
 
   def close
+    MainFiber.run do
       unless @db.nil?
         @db.not_nil!.close
       end
     end
+  end
 
   def to_json(json : JSON::Builder)
     json.string self