diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 new file mode 100644 index 0000000..b8107db --- /dev/null +++ b/Dockerfile.arm32v7 @@ -0,0 +1,13 @@ +FROM arm32v7/ubuntu:18.04 + +RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev + +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.32.1 && make deps && cd .. +RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && make && cd .. +RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && make && cd .. + +COPY mango.o . + +RUN cc 'mango.o' -o 'mango' -rdynamic -lxml2 /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/arm-linux-gnueabihf/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib + +CMD ["./mango"] diff --git a/src/main_fiber.cr b/src/main_fiber.cr new file mode 100644 index 0000000..0598657 --- /dev/null +++ b/src/main_fiber.cr @@ -0,0 +1,29 @@ +# On ARM, connecting to the SQLite DB from a spawned fiber would crash +# https://github.com/crystal-lang/crystal-sqlite3/issues/30 +# This is a temporary workaround that forces the relevant code to run in the +# main fiber + +class MainFiber + @@channel = Channel(-> Nil).new + @@done = Channel(Bool).new + + def self.start_and_block + loop do + if proc = @@channel.receive + begin + proc.call + ensure + @@done.send true + end + end + Fiber.yield + end + end + + def self.run(&block : -> Nil) + @@channel.send block + until @@done.receive + Fiber.yield + end + end +end diff --git a/src/mango.cr b/src/mango.cr index 7b8f960..b32dbbb 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -1,6 +1,7 @@ require "./config" require "./queue" require "./server" +require "./main_fiber" require "./mangadex/*" require "option_parser" require "clim" @@ -54,8 +55,7 @@ class CLI < Clim # empty ARGV so it won't be passed to Kemal ARGV.clear - server = Server.new - server.start + Server.new.start end sub "admin" do @@ -123,4 +123,8 @@ class CLI < Clim end end -CLI.start(ARGV) +spawn do + CLI.start(ARGV) +end + +MainFiber.start_and_block diff --git a/src/queue.cr b/src/queue.cr index e69ac66..0281b61 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -119,22 +119,24 @@ class Queue "Attepmting to create it" Dir.mkdir_p dir end - DB.open "sqlite3://#{@path}" do |db| - begin - db.exec "create table if not exists queue " \ - "(id text, manga_id text, title text, manga_title " \ - "text, status integer, status_message text, " \ - "pages integer, success_count integer, " \ - "fail_count integer, time integer)" - db.exec "create unique index if not exists id_idx " \ - "on queue (id)" - db.exec "create index if not exists manga_id_idx " \ - "on queue (manga_id)" - db.exec "create index if not exists status_idx " \ - "on queue (status)" - rescue e - Logger.error "Error when checking tables in DB: #{e}" - raise e + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + begin + db.exec "create table if not exists queue " \ + "(id text, manga_id text, title text, manga_title " \ + "text, status integer, status_message text, " \ + "pages integer, success_count integer, " \ + "fail_count integer, time integer)" + db.exec "create unique index if not exists id_idx " \ + "on queue (id)" + db.exec "create index if not exists manga_id_idx " \ + "on queue (manga_id)" + db.exec "create index if not exists status_idx " \ + "on queue (status)" + rescue e + Logger.error "Error when checking tables in DB: #{e}" + raise e + end end end end @@ -143,23 +145,27 @@ class Queue # inserted. Any job already exists in the queue will be ignored. def push(jobs : Array(Job)) start_count = self.count - DB.open "sqlite3://#{@path}" do |db| - jobs.each do |job| - db.exec "insert or ignore into queue values " \ - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - job.id, job.manga_id, job.title, job.manga_title, - job.status.to_i, job.status_message, job.pages, - job.success_count, job.fail_count, job.time.to_unix_ms + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + jobs.each do |job| + db.exec "insert or ignore into queue values " \ + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + job.id, job.manga_id, job.title, job.manga_title, + job.status.to_i, job.status_message, job.pages, + job.success_count, job.fail_count, job.time.to_unix_ms + end end end self.count - start_count end def reset(id : String) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = 0, status_message = '', " \ - "pages = 0, success_count = 0, fail_count = 0 " \ - "where id = (?)", id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = 0, status_message = '', " \ + "pages = 0, success_count = 0, fail_count = 0 " \ + "where id = (?)", id + end end end @@ -169,16 +175,20 @@ class Queue # Reset all failed tasks (missing pages and error) def reset - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = 0, status_message = '', " \ - "pages = 0, success_count = 0, fail_count = 0 " \ - "where status = 2 or status = 4" + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = 0, status_message = '', " \ + "pages = 0, success_count = 0, fail_count = 0 " \ + "where status = 2 or status = 4" + end end end def delete(id : String) - DB.open "sqlite3://#{@path}" do |db| - db.exec "delete from queue where id = (?)", id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "delete from queue where id = (?)", id + end end end @@ -187,71 +197,89 @@ class Queue end def delete_status(status : JobStatus) - DB.open "sqlite3://#{@path}" do |db| - db.exec "delete from queue where status = (?)", status.to_i + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "delete from queue where status = (?)", status.to_i + end end end def count_status(status : JobStatus) num = 0 - DB.open "sqlite3://#{@path}" do |db| - num = db.query_one "select count(*) from queue where " \ - "status = (?)", status.to_i, as: Int32 + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + num = db.query_one "select count(*) from queue where " \ + "status = (?)", status.to_i, as: Int32 + end end num end def count num = 0 - DB.open "sqlite3://#{@path}" do |db| - num = db.query_one "select count(*) from queue", as: Int32 + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + num = db.query_one "select count(*) from queue", as: Int32 + end end num end def set_status(status : JobStatus, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = (?) where id = (?)", - status.to_i, job.id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = (?) where id = (?)", + status.to_i, job.id + end end end def get_all jobs = [] of Job - DB.open "sqlite3://#{@path}" do |db| - jobs = db.query_all "select * from queue order by time" do |rs| - Job.from_query_result rs + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + jobs = db.query_all "select * from queue order by time" do |rs| + Job.from_query_result rs + end end end jobs end def add_success(job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set success_count = success_count + 1 " \ - "where id = (?)", job.id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set success_count = success_count + 1 " \ + "where id = (?)", job.id + end end end def add_fail(job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set fail_count = fail_count + 1 " \ - "where id = (?)", job.id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set fail_count = fail_count + 1 " \ + "where id = (?)", job.id + end end end def set_pages(pages : Int32, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set pages = (?), success_count = 0, " \ - "fail_count = 0 where id = (?)", pages, job.id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set pages = (?), success_count = 0, " \ + "fail_count = 0 where id = (?)", pages, job.id + end end end def add_message(msg : String, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status_message = " \ - "status_message || (?) || (?) where id = (?)", - "\n", msg, job.id + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status_message = " \ + "status_message || (?) || (?) where id = (?)", + "\n", msg, job.id + end end end diff --git a/src/storage.cr b/src/storage.cr index 2c42719..afbba8d 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -32,38 +32,40 @@ class Storage "Attepmting to create it" Dir.mkdir_p dir end - DB.open "sqlite3://#{@path}" do |db| - begin - # We create the `ids` table first. even if the uses has an - # early version installed and has the `user` table only, - # we will still be able to create `ids` - db.exec "create table ids" \ - "(path text, id text, is_title integer)" - db.exec "create unique index path_idx on ids (path)" - db.exec "create unique index id_idx on ids (id)" + MainFiber.run do + DB.open "sqlite3://#{@path}" do |db| + begin + # We create the `ids` table first. even if the uses has an + # early version installed and has the `user` table only, + # we will still be able to create `ids` + db.exec "create table ids" \ + "(path text, id text, is_title integer)" + db.exec "create unique index path_idx on ids (path)" + db.exec "create unique index id_idx on ids (id)" - db.exec "create table users" \ - "(username text, password text, token text, admin integer)" - rescue e - unless e.message.not_nil!.ends_with? "already exists" - Logger.fatal "Error when checking tables in DB: #{e}" - raise e + db.exec "create table users" \ + "(username text, password text, token text, admin integer)" + rescue e + unless e.message.not_nil!.ends_with? "already exists" + Logger.fatal "Error when checking tables in DB: #{e}" + raise e + end + + # If the DB is initialized through CLI but no user is added, we need + # to create the admin user when first starting the app + user_count = db.query_one "select count(*) from users", as: Int32 + init_admin if init_user && user_count == 0 + else + Logger.debug "Creating DB file at #{@path}" + db.exec "create unique index username_idx on users (username)" + db.exec "create unique index token_idx on users (token)" + + init_admin if init_user end - - # If the DB is initialized through CLI but no user is added, we need - # to create the admin user when first starting the app - user_count = db.query_one "select count(*) from users", as: Int32 - init_admin if init_user && user_count == 0 - else - Logger.debug "Creating DB file at #{@path}" - db.exec "create unique index username_idx on users (username)" - db.exec "create unique index token_idx on users (token)" - - init_admin if init_user end - end - unless @auto_close - @db = DB.open "sqlite3://#{@path}" + unless @auto_close + @db = DB.open "sqlite3://#{@path}" + end end end @@ -87,37 +89,45 @@ class Storage end def verify_user(username, password) - get_db do |db| - begin - hash, token = db.query_one "select password, token from " \ - "users where username = (?)", - username, as: {String, String?} - unless verify_password hash, password - Logger.debug "Password does not match the hash" - return nil + out_token = nil + MainFiber.run do + get_db do |db| + begin + hash, token = db.query_one "select password, token from " \ + "users where username = (?)", + username, as: {String, String?} + unless verify_password hash, password + Logger.debug "Password does not match the hash" + next + end + Logger.debug "User #{username} verified" + if token + out_token = token + next + end + token = random_str + Logger.debug "Updating token for #{username}" + db.exec "update users set token = (?) where username = (?)", + token, username + out_token = token + rescue e + Logger.error "Error when verifying user #{username}: #{e}" end - Logger.debug "User #{username} verified" - return token if token - token = random_str - Logger.debug "Updating token for #{username}" - db.exec "update users set token = (?) where username = (?)", - token, username - return token - rescue e - Logger.error "Error when verifying user #{username}: #{e}" - return nil end end + out_token end def verify_token(token) username = nil - get_db do |db| - begin - username = db.query_one "select username from users where " \ - "token = (?)", token, as: String - rescue e - Logger.debug "Unable to verify token" + MainFiber.run do + get_db do |db| + begin + username = db.query_one "select username from users where " \ + "token = (?)", token, as: String + rescue e + Logger.debug "Unable to verify token" + end end end username @@ -125,12 +135,14 @@ class Storage def verify_admin(token) is_admin = false - get_db do |db| - begin - is_admin = db.query_one "select admin from users where " \ - "token = (?)", token, as: Bool - rescue e - Logger.debug "Unable to verify user as admin" + MainFiber.run do + get_db do |db| + begin + is_admin = db.query_one "select admin from users where " \ + "token = (?)", token, as: Bool + rescue e + Logger.debug "Unable to verify user as admin" + end end end is_admin @@ -138,10 +150,12 @@ class Storage def list_users results = Array(Tuple(String, Bool)).new - get_db do |db| - db.query "select username, admin from users" do |rs| - rs.each do - results << {rs.read(String), rs.read(Bool)} + MainFiber.run do + get_db do |db| + db.query "select username, admin from users" do |rs| + rs.each do + results << {rs.read(String), rs.read(Bool)} + end end end end @@ -152,10 +166,12 @@ class Storage validate_username username validate_password password admin = (admin ? 1 : 0) - get_db do |db| - hash = hash_password password - db.exec "insert into users values (?, ?, ?, ?)", - username, hash, nil, admin + MainFiber.run do + get_db do |db| + hash = hash_password password + db.exec "insert into users values (?, ?, ?, ?)", + username, hash, nil, admin + end end end @@ -163,40 +179,48 @@ class Storage admin = (admin ? 1 : 0) validate_username username validate_password password unless password.empty? - get_db do |db| - if password.empty? - db.exec "update users set username = (?), admin = (?) " \ - "where username = (?)", - username, admin, original_username - else - hash = hash_password password - db.exec "update users set username = (?), admin = (?)," \ - "password = (?) where username = (?)", - username, admin, hash, original_username + MainFiber.run do + get_db do |db| + if password.empty? + db.exec "update users set username = (?), admin = (?) " \ + "where username = (?)", + username, admin, original_username + else + hash = hash_password password + db.exec "update users set username = (?), admin = (?)," \ + "password = (?) where username = (?)", + username, admin, hash, original_username + end end end end def delete_user(username) - get_db do |db| - db.exec "delete from users where username = (?)", username + MainFiber.run do + get_db do |db| + db.exec "delete from users where username = (?)", username + end end end def logout(token) - get_db do |db| - begin - db.exec "update users set token = (?) where token = (?)", nil, token - rescue + MainFiber.run do + get_db do |db| + begin + db.exec "update users set token = (?) where token = (?)", nil, token + rescue + end end end end def get_id(path, is_title) id = nil - get_db do |db| - id = db.query_one? "select id from ids where path = (?)", path, - as: {String} + MainFiber.run do + get_db do |db| + id = db.query_one? "select id from ids where path = (?)", path, + as: {String} + end end id end @@ -206,20 +230,24 @@ class Storage end def bulk_insert_ids - get_db do |db| - db.transaction do |tx| - @insert_ids.each do |tp| - tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path], - tp[:id], tp[:is_title] ? 1 : 0 + MainFiber.run do + get_db do |db| + db.transaction do |tx| + @insert_ids.each do |tp| + tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path], + tp[:id], tp[:is_title] ? 1 : 0 + end end end + @insert_ids.clear end - @insert_ids.clear end def close - unless @db.nil? - @db.not_nil!.close + MainFiber.run do + unless @db.nil? + @db.not_nil!.close + end end end