Backups/Mango
1
0
Fork 0
mirror of https://github.com/hkalexling/Mango.git synced 2025-08-02 10:55:30 -04:00
Code Issues Projects Releases Wiki Activity

ARM support (#25, #78)

Browse Source
This commit is contained in:
Alex Ling 2020-08-04 11:00:09 +00:00
parent 87dea01917
commit 8a83c0df4e
5 changed files with 258 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Dockerfile.arm32v7 Normal file
View File

@ -0,0 +1,13 @@
FROM arm32v7/ubuntu:18.04
RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev
RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.32.1 && make deps && cd ..
RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && make && cd ..
RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && make && cd ..
COPY mango.o .
RUN cc 'mango.o' -o 'mango' -rdynamic -lxml2 /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/arm-linux-gnueabihf/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib
CMD ["./mango"]

29
src/main_fiber.cr Normal file
View File

@ -0,0 +1,29 @@
# On ARM, connecting to the SQLite DB from a spawned fiber would crash
# https://github.com/crystal-lang/crystal-sqlite3/issues/30
# This is a temporary workaround that forces the relevant code to run in the
# main fiber
class MainFiber
@@channel = Channel(-> Nil).new
@@done = Channel(Bool).new
def self.start_and_block
loop do
if proc = @@channel.receive
begin
proc.call
ensure
@@done.send true
end
end
Fiber.yield
end
end
def self.run(&block : -> Nil)
@@channel.send block
until @@done.receive
Fiber.yield
end
end
end

10
src/mango.cr
View File

@ -1,6 +1,7 @@
require "./config"
require "./queue"
require "./server"
require "./main_fiber"
require "./mangadex/*"
require "option_parser"
require "clim"
@ -54,8 +55,7 @@ class CLI < Clim
# empty ARGV so it won't be passed to Kemal
ARGV.clear
server = Server.new
server.start
Server.new.start
end
sub "admin" do
@ -123,4 +123,8 @@ class CLI < Clim
end
end
CLI.start(ARGV)
spawn do
CLI.start(ARGV)
end
MainFiber.start_and_block

146
src/queue.cr
View File

@ -119,22 +119,24 @@ class Queue
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
Logger.error "Error when checking tables in DB: #{e}"
raise e
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
begin
db.exec "create table if not exists queue " \
"(id text, manga_id text, title text, manga_title " \
"text, status integer, status_message text, " \
"pages integer, success_count integer, " \
"fail_count integer, time integer)"
db.exec "create unique index if not exists id_idx " \
"on queue (id)"
db.exec "create index if not exists manga_id_idx " \
"on queue (manga_id)"
db.exec "create index if not exists status_idx " \
"on queue (status)"
rescue e
Logger.error "Error when checking tables in DB: #{e}"
raise e
end
end
end
end
@ -143,23 +145,27 @@ class Queue
# inserted. Any job already exists in the queue will be ignored.
def push(jobs : Array(Job))
start_count = self.count
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values " \
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
jobs.each do |job|
db.exec "insert or ignore into queue values " \
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
job.id, job.manga_id, job.title, job.manga_title,
job.status.to_i, job.status_message, job.pages,
job.success_count, job.fail_count, job.time.to_unix_ms
end
end
end
self.count - start_count
end
def reset(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where id = (?)", id
end
end
end
@ -169,16 +175,20 @@ class Queue
# Reset all failed tasks (missing pages and error)
def reset
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = 0, status_message = '', " \
"pages = 0, success_count = 0, fail_count = 0 " \
"where status = 2 or status = 4"
end
end
end
def delete(id : String)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where id = (?)", id
end
end
end
@ -187,71 +197,89 @@ class Queue
end
def delete_status(status : JobStatus)
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "delete from queue where status = (?)", status.to_i
end
end
end
def count_status(status : JobStatus)
num = 0
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue where " \
"status = (?)", status.to_i, as: Int32
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue where " \
"status = (?)", status.to_i, as: Int32
end
end
num
end
def count
num = 0
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue", as: Int32
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
num = db.query_one "select count(*) from queue", as: Int32
end
end
num
end
def set_status(status : JobStatus, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status = (?) where id = (?)",
status.to_i, job.id
end
end
end
def get_all
jobs = [] of Job
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time" do |rs|
Job.from_query_result rs
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
jobs = db.query_all "select * from queue order by time" do |rs|
Job.from_query_result rs
end
end
end
jobs
end
def add_success(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set success_count = success_count + 1 " \
"where id = (?)", job.id
end
end
end
def add_fail(job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set fail_count = fail_count + 1 " \
"where id = (?)", job.id
end
end
end
def set_pages(pages : Int32, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set pages = (?), success_count = 0, " \
"fail_count = 0 where id = (?)", pages, job.id
end
end
end
def add_message(msg : String, job : Job)
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
db.exec "update queue set status_message = " \
"status_message || (?) || (?) where id = (?)",
"\n", msg, job.id
end
end
end

216
src/storage.cr
View File

@ -32,38 +32,40 @@ class Storage
"Attepmting to create it"
Dir.mkdir_p dir
end
DB.open "sqlite3://#{@path}" do |db|
begin
# We create the `ids` table first. even if the uses has an
# early version installed and has the `user` table only,
# we will still be able to create `ids`
db.exec "create table ids" \
"(path text, id text, is_title integer)"
db.exec "create unique index path_idx on ids (path)"
db.exec "create unique index id_idx on ids (id)"
MainFiber.run do
DB.open "sqlite3://#{@path}" do |db|
begin
# We create the `ids` table first. even if the uses has an
# early version installed and has the `user` table only,
# we will still be able to create `ids`
db.exec "create table ids" \
"(path text, id text, is_title integer)"
db.exec "create unique index path_idx on ids (path)"
db.exec "create unique index id_idx on ids (id)"
db.exec "create table users" \
"(username text, password text, token text, admin integer)"
rescue e
unless e.message.not_nil!.ends_with? "already exists"
Logger.fatal "Error when checking tables in DB: #{e}"
raise e
db.exec "create table users" \
"(username text, password text, token text, admin integer)"
rescue e
unless e.message.not_nil!.ends_with? "already exists"
Logger.fatal "Error when checking tables in DB: #{e}"
raise e
end
# If the DB is initialized through CLI but no user is added, we need
# to create the admin user when first starting the app
user_count = db.query_one "select count(*) from users", as: Int32
init_admin if init_user && user_count == 0
else
Logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
init_admin if init_user
end
# If the DB is initialized through CLI but no user is added, we need
# to create the admin user when first starting the app
user_count = db.query_one "select count(*) from users", as: Int32
init_admin if init_user && user_count == 0
else
Logger.debug "Creating DB file at #{@path}"
db.exec "create unique index username_idx on users (username)"
db.exec "create unique index token_idx on users (token)"
init_admin if init_user
end
end
unless @auto_close
@db = DB.open "sqlite3://#{@path}"
unless @auto_close
@db = DB.open "sqlite3://#{@path}"
end
end
end
@ -87,37 +89,45 @@ class Storage
end
def verify_user(username, password)
get_db do |db|
begin
hash, token = db.query_one "select password, token from " \
"users where username = (?)",
username, as: {String, String?}
unless verify_password hash, password
Logger.debug "Password does not match the hash"
return nil
out_token = nil
MainFiber.run do
get_db do |db|
begin
hash, token = db.query_one "select password, token from " \
"users where username = (?)",
username, as: {String, String?}
unless verify_password hash, password
Logger.debug "Password does not match the hash"
next
end
Logger.debug "User #{username} verified"
if token
out_token = token
next
end
token = random_str
Logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
out_token = token
rescue e
Logger.error "Error when verifying user #{username}: #{e}"
end
Logger.debug "User #{username} verified"
return token if token
token = random_str
Logger.debug "Updating token for #{username}"
db.exec "update users set token = (?) where username = (?)",
token, username
return token
rescue e
Logger.error "Error when verifying user #{username}: #{e}"
return nil
end
end
out_token
end
def verify_token(token)
username = nil
get_db do |db|
begin
username = db.query_one "select username from users where " \
"token = (?)", token, as: String
rescue e
Logger.debug "Unable to verify token"
MainFiber.run do
get_db do |db|
begin
username = db.query_one "select username from users where " \
"token = (?)", token, as: String
rescue e
Logger.debug "Unable to verify token"
end
end
end
username
@ -125,12 +135,14 @@ class Storage
def verify_admin(token)
is_admin = false
get_db do |db|
begin
is_admin = db.query_one "select admin from users where " \
"token = (?)", token, as: Bool
rescue e
Logger.debug "Unable to verify user as admin"
MainFiber.run do
get_db do |db|
begin
is_admin = db.query_one "select admin from users where " \
"token = (?)", token, as: Bool
rescue e
Logger.debug "Unable to verify user as admin"
end
end
end
is_admin
@ -138,10 +150,12 @@ class Storage
def list_users
results = Array(Tuple(String, Bool)).new
get_db do |db|
db.query "select username, admin from users" do |rs|
rs.each do
results << {rs.read(String), rs.read(Bool)}
MainFiber.run do
get_db do |db|
db.query "select username, admin from users" do |rs|
rs.each do
results << {rs.read(String), rs.read(Bool)}
end
end
end
end
@ -152,10 +166,12 @@ class Storage
validate_username username
validate_password password
admin = (admin ? 1 : 0)
get_db do |db|
hash = hash_password password
db.exec "insert into users values (?, ?, ?, ?)",
username, hash, nil, admin
MainFiber.run do
get_db do |db|
hash = hash_password password
db.exec "insert into users values (?, ?, ?, ?)",
username, hash, nil, admin
end
end
end
@ -163,40 +179,48 @@ class Storage
admin = (admin ? 1 : 0)
validate_username username
validate_password password unless password.empty?
get_db do |db|
if password.empty?
db.exec "update users set username = (?), admin = (?) " \
"where username = (?)",
username, admin, original_username
else
hash = hash_password password
db.exec "update users set username = (?), admin = (?)," \
"password = (?) where username = (?)",
username, admin, hash, original_username
MainFiber.run do
get_db do |db|
if password.empty?
db.exec "update users set username = (?), admin = (?) " \
"where username = (?)",
username, admin, original_username
else
hash = hash_password password
db.exec "update users set username = (?), admin = (?)," \
"password = (?) where username = (?)",
username, admin, hash, original_username
end
end
end
end
def delete_user(username)
get_db do |db|
db.exec "delete from users where username = (?)", username
MainFiber.run do
get_db do |db|
db.exec "delete from users where username = (?)", username
end
end
end
def logout(token)
get_db do |db|
begin
db.exec "update users set token = (?) where token = (?)", nil, token
rescue
MainFiber.run do
get_db do |db|
begin
db.exec "update users set token = (?) where token = (?)", nil, token
rescue
end
end
end
end
def get_id(path, is_title)
id = nil
get_db do |db|
id = db.query_one? "select id from ids where path = (?)", path,
as: {String}
MainFiber.run do
get_db do |db|
id = db.query_one? "select id from ids where path = (?)", path,
as: {String}
end
end
id
end
@ -206,20 +230,24 @@ class Storage
end
def bulk_insert_ids
get_db do |db|
db.transaction do |tx|
@insert_ids.each do |tp|
tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path],
tp[:id], tp[:is_title] ? 1 : 0
MainFiber.run do
get_db do |db|
db.transaction do |tx|
@insert_ids.each do |tp|
tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path],
tp[:id], tp[:is_title] ? 1 : 0
end
end
end
@insert_ids.clear
end
@insert_ids.clear
end
def close
unless @db.nil?
@db.not_nil!.close
MainFiber.run do
unless @db.nil?
@db.not_nil!.close
end
end
end