Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2025-08-03 03:15:31 -04:00
Code Issues Projects Releases Wiki Activity

ARM support (#25, #78)

Browse Source
This commit is contained in:
Alex Ling 2020-08-04 11:00:09 +00:00
parent 87dea01917
commit 8a83c0df4e
5 changed files with 258 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Dockerfile.arm32v7 Normal file
View File

@ -0,0 +1,13 @@
FROM arm32v7/ubuntu:18.04
RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.32.1 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && make && cd ..
COPY mango.o .
RUN cc 'mango.o' -o 'mango' -rdynamic -lxml2 /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/arm-linux-gnueabihf/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib
CMD ["./mango"]

29
src/main_fiber.cr Normal file
View File

@ -0,0 +1,29 @@
# On ARM, connecting to the SQLite DB from a spawned fiber would crash
# https://github.com/crystal-lang/crystal-sqlite3/issues/30
# This is a temporary workaround that forces the relevant code to run in the
# main fiber
class MainFiber
@@channel = Channel(-> Nil).new
@@done = Channel(Bool).new
def self.start_and_block
loop do
if proc = @@channel.receive
begin
proc.call
ensure
@@done.send true
end
end
Fiber.yield
end
end
def self.run(&block : -> Nil)
@@channel.send block
until @@done.receive
Fiber.yield
end
end
end

10
src/mango.cr
View File

@ -1,6 +1,7 @@
require "./config" require "./config"
require "./queue" require "./queue"
require "./server" require "./server"
require "./main_fiber"
require "./mangadex/*" require "./mangadex/*"
require "option_parser" require "option_parser"
require "clim" require "clim"
@ -54,8 +55,7 @@ class CLI < Clim
# empty ARGV so it won't be passed to Kemal # empty ARGV so it won't be passed to Kemal
ARGV.clear ARGV.clear
server = Server.new Server.new.start
server.start
end end
sub "admin" do sub "admin" do
@ -123,4 +123,8 @@ class CLI < Clim
end end
end end
CLI.start(ARGV) spawn do
CLI.start(ARGV)
end
MainFiber.start_and_block

146
src/queue.cr
View File

@ -119,22 +119,24 @@ class Queue
"Attepmting to create it" "Attepmting to create it"
Dir.mkdir_p dir Dir.mkdir_p dir
end end
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
begin DB.open "sqlite3://#{@path}" do |db|
db.exec "create table if not exists queue " \ begin
"(id text, manga_id text, title text, manga_title " \ db.exec "create table if not exists queue " \
"text, status integer, status_message text, " \ "(id text, manga_id text, title text, manga_title " \
"pages integer, success_count integer, " \ "text, status integer, status_message text, " \
"fail_count integer, time integer)" "pages integer, success_count integer, " \
db.exec "create unique index if not exists id_idx " \ "fail_count integer, time integer)"
"on queue (id)" db.exec "create unique index if not exists id_idx " \
db.exec "create index if not exists manga_id_idx " \ "on queue (id)"
"on queue (manga_id)" db.exec "create index if not exists manga_id_idx " \
db.exec "create index if not exists status_idx " \ "on queue (manga_id)"
"on queue (status)" db.exec "create index if not exists status_idx " \
rescue e "on queue (status)"
Logger.error "Error when checking tables in DB: #{e}" rescue e
raise e Logger.error "Error when checking tables in DB: #{e}"
raise e
end
end end
end end
end end
@ -143,23 +145,27 @@ class Queue
# inserted. Any job already exists in the queue will be ignored. # inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job)) def push(jobs : Array(Job))
start_count = self.count start_count = self.count
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
jobs.each do |job| DB.open "sqlite3://#{@path}" do |db|
db.exec "insert or ignore into queue values " \ jobs.each do |job|
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", db.exec "insert or ignore into queue values " \
job.id, job.manga_id, job.title, job.manga_title, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.status.to_i, job.status_message, job.pages, job.id, job.manga_id, job.title, job.manga_title,
job.success_count, job.fail_count, job.time.to_unix_ms job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end end
end end
self.count - start_count self.count - start_count
end end
def reset(id : String) def reset(id : String)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set status = 0, status_message = '', " \ DB.open "sqlite3://#{@path}" do |db|
"pages = 0, success_count = 0, fail_count = 0 " \ db.exec "update queue set status = 0, status_message = '', " \
"where id = (?)", id "pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end end
end end
@ -169,16 +175,20 @@ class Queue
# Reset all failed tasks (missing pages and error) # Reset all failed tasks (missing pages and error)
def reset def reset
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set status = 0, status_message = '', " \ DB.open "sqlite3://#{@path}" do |db|
"pages = 0, success_count = 0, fail_count = 0 " \ db.exec "update queue set status = 0, status_message = '', " \
"where status = 2 or status = 4" "pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end end
end end
def delete(id : String) def delete(id : String)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "delete from queue where id = (?)", id DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end end
end end
@ -187,71 +197,89 @@ class Queue
end end
def delete_status(status : JobStatus) def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "delete from queue where status = (?)", status.to_i DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end end
end end
def count_status(status : JobStatus) def count_status(status : JobStatus)
num = 0 num = 0
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
num = db.query_one "select count(*) from queue where " \ DB.open "sqlite3://#{@path}" do |db|
"status = (?)", status.to_i, as: Int32 num = db.query_one "select count(*) from queue where " \
"status = (?)", status.to_i, as: Int32
end
end end
num num
end end
def count def count
num = 0 num = 0
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
num = db.query_one "select count(*) from queue", as: Int32 DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue", as: Int32
end
end end
num num
end end
def set_status(status : JobStatus, job : Job) def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set status = (?) where id = (?)", DB.open "sqlite3://#{@path}" do |db|
status.to_i, job.id db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end end
end end
def get_all def get_all
jobs = [] of Job jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
jobs = db.query_all "select * from queue order by time" do |rs| DB.open "sqlite3://#{@path}" do |db|
Job.from_query_result rs jobs = db.query_all "select * from queue order by time" do |rs|
Job.from_query_result rs
end
end end
end end
jobs jobs
end end
def add_success(job : Job) def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set success_count = success_count + 1 " \ DB.open "sqlite3://#{@path}" do |db|
"where id = (?)", job.id db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end end
end end
def add_fail(job : Job) def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set fail_count = fail_count + 1 " \ DB.open "sqlite3://#{@path}" do |db|
"where id = (?)", job.id db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end end
end end
def set_pages(pages : Int32, job : Job) def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set pages = (?), success_count = 0, " \ DB.open "sqlite3://#{@path}" do |db|
"fail_count = 0 where id = (?)", pages, job.id db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end end
end end
def add_message(msg : String, job : Job) def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
db.exec "update queue set status_message = " \ DB.open "sqlite3://#{@path}" do |db|
"status_message || (?) || (?) where id = (?)", db.exec "update queue set status_message = " \
"\n", msg, job.id "status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end end
end end

216
src/storage.cr
View File

@ -32,38 +32,40 @@ class Storage
"Attepmting to create it" "Attepmting to create it"
Dir.mkdir_p dir Dir.mkdir_p dir
end end
DB.open "sqlite3://#{@path}" do |db| MainFiber.run do
begin DB.open "sqlite3://#{@path}" do |db|
# We create the `ids` table first. even if the uses has an begin
# early version installed and has the `user` table only, # We create the `ids` table first. even if the uses has an
# we will still be able to create `ids` # early version installed and has the `user` table only,
db.exec "create table ids" \ # we will still be able to create `ids`
"(path text, id text, is_title integer)" db.exec "create table ids" \
db.exec "create unique index path_idx on ids (path)" "(path text, id text, is_title integer)"
db.exec "create unique index id_idx on ids (id)" db.exec "create unique index path_idx on ids (path)"
db.exec "create unique index id_idx on ids (id)"
db.exec "create table users" \ db.exec "create table users" \
"(username text, password text, token text, admin integer)" "(username text, password text, token text, admin integer)"
rescue e rescue e
unless e.message.not_nil!.ends_with? "already exists" unless e.message.not_nil!.ends_with? "already exists"
Logger.fatal "Error when checking tables in DB: #{e}" Logger.fatal "Error when checking tables in DB: #{e}"
raise e raise e
end
# If the DB is initialized through CLI but no user is added, we need
# to create the admin user when first starting the app
user_count = db.query_one "select count(*) from users", as: Int32
init_admin if init_user && user_count == 0
else
Logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
init_admin if init_user
end end
# If the DB is initialized through CLI but no user is added, we need
# to create the admin user when first starting the app
user_count = db.query_one "select count(*) from users", as: Int32
init_admin if init_user && user_count == 0
else
Logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
init_admin if init_user
end end
end unless @auto_close
unless @auto_close @db = DB.open "sqlite3://#{@path}"
@db = DB.open "sqlite3://#{@path}" end
end end
end end
@ -87,37 +89,45 @@ class Storage
end end
def verify_user(username, password) def verify_user(username, password)
get_db do |db| out_token = nil
begin MainFiber.run do
hash, token = db.query_one "select password, token from " \ get_db do |db|
"users where username = (?)", begin
username, as: {String, String?} hash, token = db.query_one "select password, token from " \
unless verify_password hash, password "users where username = (?)",
Logger.debug "Password does not match the hash" username, as: {String, String?}
return nil unless verify_password hash, password
Logger.debug "Password does not match the hash"
next
end
Logger.debug "User #{username} verified"
if token
out_token = token
next
end
token = random_str
Logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
out_token = token
rescue e
Logger.error "Error when verifying user #{username}: #{e}"
end end
Logger.debug "User #{username} verified"
return token if token
token = random_str
Logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
return token
rescue e
Logger.error "Error when verifying user #{username}: #{e}"
return nil
end end
end end
out_token
end end
def verify_token(token) def verify_token(token)
username = nil username = nil
get_db do |db| MainFiber.run do
begin get_db do |db|
username = db.query_one "select username from users where " \ begin
"token = (?)", token, as: String username = db.query_one "select username from users where " \
rescue e "token = (?)", token, as: String
Logger.debug "Unable to verify token" rescue e
Logger.debug "Unable to verify token"
end
end end
end end
username username
@ -125,12 +135,14 @@ class Storage
def verify_admin(token) def verify_admin(token)
is_admin = false is_admin = false
get_db do |db| MainFiber.run do
begin get_db do |db|
is_admin = db.query_one "select admin from users where " \ begin
"token = (?)", token, as: Bool is_admin = db.query_one "select admin from users where " \
rescue e "token = (?)", token, as: Bool
Logger.debug "Unable to verify user as admin" rescue e
Logger.debug "Unable to verify user as admin"
end
end end
end end
is_admin is_admin
@ -138,10 +150,12 @@ class Storage
def list_users def list_users
results = Array(Tuple(String, Bool)).new results = Array(Tuple(String, Bool)).new
get_db do |db| MainFiber.run do
db.query "select username, admin from users" do |rs| get_db do |db|
rs.each do db.query "select username, admin from users" do |rs|
results << {rs.read(String), rs.read(Bool)} rs.each do
results << {rs.read(String), rs.read(Bool)}
end
end end
end end
end end
@ -152,10 +166,12 @@ class Storage
validate_username username validate_username username
validate_password password validate_password password
admin = (admin ? 1 : 0) admin = (admin ? 1 : 0)
get_db do |db| MainFiber.run do
hash = hash_password password get_db do |db|
db.exec "insert into users values (?, ?, ?, ?)", hash = hash_password password
username, hash, nil, admin db.exec "insert into users values (?, ?, ?, ?)",
username, hash, nil, admin
end
end end
end end
@ -163,40 +179,48 @@ class Storage
admin = (admin ? 1 : 0) admin = (admin ? 1 : 0)
validate_username username validate_username username
validate_password password unless password.empty? validate_password password unless password.empty?
get_db do |db| MainFiber.run do
if password.empty? get_db do |db|
db.exec "update users set username = (?), admin = (?) " \ if password.empty?
"where username = (?)", db.exec "update users set username = (?), admin = (?) " \
username, admin, original_username "where username = (?)",
else username, admin, original_username
hash = hash_password password else
db.exec "update users set username = (?), admin = (?)," \ hash = hash_password password
"password = (?) where username = (?)", db.exec "update users set username = (?), admin = (?)," \
username, admin, hash, original_username "password = (?) where username = (?)",
username, admin, hash, original_username
end
end end
end end
end end
def delete_user(username) def delete_user(username)
get_db do |db| MainFiber.run do
db.exec "delete from users where username = (?)", username get_db do |db|
db.exec "delete from users where username = (?)", username
end
end end
end end
def logout(token) def logout(token)
get_db do |db| MainFiber.run do
begin get_db do |db|
db.exec "update users set token = (?) where token = (?)", nil, token begin
rescue db.exec "update users set token = (?) where token = (?)", nil, token
rescue
end
end end
end end
end end
def get_id(path, is_title) def get_id(path, is_title)
id = nil id = nil
get_db do |db| MainFiber.run do
id = db.query_one? "select id from ids where path = (?)", path, get_db do |db|
as: {String} id = db.query_one? "select id from ids where path = (?)", path,
as: {String}
end
end end
id id
end end
@ -206,20 +230,24 @@ class Storage
end end
def bulk_insert_ids def bulk_insert_ids
get_db do |db| MainFiber.run do
db.transaction do |tx| get_db do |db|
@insert_ids.each do |tp| db.transaction do |tx|
tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path], @insert_ids.each do |tp|
tp[:id], tp[:is_title] ? 1 : 0 tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path],
tp[:id], tp[:is_title] ? 1 : 0
end
end end
end end
@insert_ids.clear
end end
@insert_ids.clear
end end
def close def close
unless @db.nil? MainFiber.run do
@db.not_nil!.close unless @db.nil?
@db.not_nil!.close
end
end end
end end