diff --git a/.ameba.yml b/.ameba.yml new file mode 100644 index 0000000..e61b3b3 --- /dev/null +++ b/.ameba.yml @@ -0,0 +1,9 @@ +Lint/UselessAssign: + Excluded: + - src/routes/* + - src/server.cr +Lint/UnusedArgument: + Excluded: + - src/routes/* +Metrics/CyclomaticComplexity: + Enabled: false diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15b9838..ac18e6e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: - image: crystallang/crystal:0.32.1-alpine + image: crystallang/crystal:0.34.0-alpine steps: - uses: actions/checkout@v2 @@ -20,5 +20,7 @@ jobs: run: apk add --no-cache yarn yaml sqlite-static - name: Build run: make + - name: Linter + run: make check - name: Run tests run: make test diff --git a/Dockerfile b/Dockerfile index d230549..d763ddf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:0.32.1-alpine AS builder +FROM crystallang/crystal:0.34.0-alpine AS builder WORKDIR /Mango diff --git a/Makefile b/Makefile index 199745b..c31eb29 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PREFIX=/usr/local +PREFIX ?= /usr/local INSTALL_DIR=$(PREFIX)/bin all: uglify | build @@ -22,6 +22,10 @@ run: test: crystal spec +check: + crystal tool format --check + ./bin/ameba + install: cp mango $(INSTALL_DIR)/mango diff --git a/README.md b/README.md index aa8f2a2..bcfadbe 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` -Mango e-manga server/reader. Version 0.2.0 +Mango e-manga server/reader. Version 0.3.0 -v, --version Show version -h, --help Show help @@ -64,8 +64,9 @@ The default config file location is `~/.config/mango/config.yml`. It might be di ```yaml --- port: 9000 -library_path: ~/mango/library -db_path: ~/mango/mango.db +library_path: /home/alex_ling/mango/library +upload_path: /home/alex_ling/mango/uploads +db_path: /home/alex_ling/mango/mango.db scan_interval_minutes: 5 log_level: info mangadex: @@ -73,7 +74,7 @@ mangadex: api_url: https://mangadex.org/api download_wait_seconds: 5 download_retries: 4 - download_queue_db_path: ~/mango/queue.db + download_queue_db_path: /home/alex_ling/mango/queue.db ``` - `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan diff --git a/public/css/mango.css b/public/css/mango.css index 8b9df2a..ee20a98 100644 --- a/public/css/mango.css +++ b/public/css/mango.css @@ -56,3 +56,18 @@ td > .uk-dropdown { white-space: pre-line; } +#edit-modal .uk-grid > div { + height: 300px; +} +#edit-modal #cover { + height: 100%; + width: 100%; + object-fit: cover; +} +#edit-modal #cover-upload { + height: 100%; + box-sizing: border-box; +} +#edit-modal .uk-modal-body .uk-inline { + width: 100%; +} diff --git a/public/js/title.js b/public/js/title.js index a719fb7..4febf2c 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -15,7 +15,10 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi if (percentage === 100) { $('#read-btn').attr('hidden', ''); } - $('#modal-title').text(entry); + $('#modal-title').find('span').text(entry); + $('#modal-title').next().attr('data-id', titleID); + $('#modal-title').next().attr('data-entry-id', entryID); + $('#modal-title').next().find('.title-rename-field').val(entry); $('#path-text').text(zipPath); $('#pages-text').text(pages + ' pages'); @@ -29,11 +32,18 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi updateProgress(titleID, entryID, 0); }); + $('.uk-modal-title.break-word > a').attr('onclick', `edit("${entryID}")`); + UIkit.modal($('#modal')).show(); styleModal(); } -function updateProgress(titleID, entryID, page) { - $.post('/api/progress/' + titleID + '/' + entryID + '/' + page, function(data) { + +const updateProgress = (tid, eid, page) => { + let url = `/api/progress/${tid}/${page}` + const query = $.param({entry: eid}); + if (eid) + url += `?${query}`; + $.post(url, (data) => { if (data.success) { location.reload(); } @@ -42,4 +52,108 @@ function updateProgress(titleID, entryID, page) { alert('danger', error); } }); -} +}; + +const renameSubmit = (name, eid) => { + const upload = $('.upload-field'); + const titleId = upload.attr('data-title-id'); + + console.log(name); + + if (name.length === 0) { + alert('danger', 'The display name should not be empty'); + return; + } + + const query = $.param({ entry: eid }); + let url = `/api/admin/display_name/${titleId}/${name}`; + if (eid) + url += `?${query}`; + + $.ajax({ + type: 'POST', + url: url, + contentType: "application/json", + dataType: 'json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to update display name. Error: ${data.error}`); + return; + } + location.reload(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to update display name. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); +}; + +const edit = (eid) => { + const cover = $('#edit-modal #cover'); + let url = cover.attr('data-title-cover'); + let displayName = $('h2.uk-title > span').text(); + + if (eid) { + const item = $(`#${eid}`); + url = item.find('img').attr('data-src'); + displayName = item.find('.uk-card-title').attr('data-title'); + $('#title-progress-control').attr('hidden', ''); + } + else { + $('#title-progress-control').removeAttr('hidden'); + } + + cover.attr('data-src', url); + + const displayNameField = $('#display-name-field'); + displayNameField.attr('value', displayName); + displayNameField.keyup(event => { + if (event.keyCode === 13) { + renameSubmit(displayNameField.val(), eid); + } + }); + displayNameField.siblings('a.uk-form-icon').click(() => { + renameSubmit(displayNameField.val(), eid); + }); + + setupUpload(eid); + + UIkit.modal($('#edit-modal')).show(); + styleModal(); +}; + +const setupUpload = (eid) => { + const upload = $('.upload-field'); + const bar = $('#upload-progress').get(0); + const titleId = upload.attr('data-title-id'); + const queryObj = {title: titleId}; + if (eid) + queryObj['entry'] = eid; + const query = $.param(queryObj); + const url = `/api/admin/upload/cover?${query}`; + console.log(url); + UIkit.upload('.upload-field', { + url: url, + name: 'file', + error: (e) => { + alert('danger', `Failed to upload cover image: ${e.toString()}`); + }, + loadStart: (e) => { + $(bar).removeAttr('hidden'); + bar.max = e.total; + bar.value = e.loaded; + }, + progress: (e) => { + bar.max = e.total; + bar.value = e.loaded; + }, + loadEnd: (e) => { + bar.max = e.total; + bar.value = e.loaded; + }, + completeAll: () => { + $(bar).attr('hidden', ''); + location.reload(); + } + }); +}; diff --git a/shard.lock b/shard.lock index 5f28d68..c84ac06 100644 --- a/shard.lock +++ b/shard.lock @@ -1,5 +1,9 @@ version: 1.0 shards: + ameba: + github: crystal-ameba/ameba + version: 0.12.0 + baked_file_system: github: schovi/baked_file_system version: 0.9.8 diff --git a/shard.yml b/shard.yml index 8d5788a..c5bae1d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.2.5 +version: 0.3.0 authors: - Alex Ling @@ -8,7 +8,7 @@ targets: mango: main: src/mango.cr -crystal: 0.32.1 +crystal: 0.34.0 license: MIT @@ -19,3 +19,7 @@ dependencies: github: crystal-lang/crystal-sqlite3 baked_file_system: github: schovi/baked_file_system + +development_dependencies: + ameba: + github: crystal-ameba/ameba diff --git a/spec/config_spec.cr b/spec/config_spec.cr index c4f8700..bacf8f9 100644 --- a/spec/config_spec.cr +++ b/spec/config_spec.cr @@ -1,14 +1,14 @@ require "./spec_helper" describe Config do - it "creates config if it does not exist" do - with_default_config do |config, logger, path| - File.exists?(path).should be_true - end - end + it "creates config if it does not exist" do + with_default_config do |_, _, path| + File.exists?(path).should be_true + end + end - it "correctly loads config" do - config = Config.load "spec/asset/test-config.yml" - config.port.should eq 3000 - end + it "correctly loads config" do + config = Config.load "spec/asset/test-config.yml" + config.port.should eq 3000 + end end diff --git a/spec/mangadex_spec.cr b/spec/mangadex_spec.cr index 21e5853..04aa3cb 100644 --- a/spec/mangadex_spec.cr +++ b/spec/mangadex_spec.cr @@ -3,103 +3,102 @@ require "./spec_helper" include MangaDex describe Queue do - it "creates DB at given path" do - with_queue do |queue, path| - File.exists?(path).should be_true - end - end + it "creates DB at given path" do + with_queue do |_, path| + File.exists?(path).should be_true + end + end - it "pops nil when empty" do - with_queue do |queue| - queue.pop.should be_nil - end - end + it "pops nil when empty" do + with_queue do |queue| + queue.pop.should be_nil + end + end - it "inserts multiple jobs" do - with_queue do |queue| - j1 = Job.new "1", "1", "title", "manga_title", JobStatus::Error, - Time.utc - j2 = Job.new "2", "2", "title", "manga_title", JobStatus::Completed, - Time.utc - j3 = Job.new "3", "3", "title", "manga_title", JobStatus::Pending, - Time.utc - j4 = Job.new "4", "4", "title", "manga_title", - JobStatus::Downloading, Time.utc - count = queue.push [j1, j2, j3, j4] - count.should eq 4 - end - end + it "inserts multiple jobs" do + with_queue do |queue| + j1 = Job.new "1", "1", "title", "manga_title", JobStatus::Error, + Time.utc + j2 = Job.new "2", "2", "title", "manga_title", JobStatus::Completed, + Time.utc + j3 = Job.new "3", "3", "title", "manga_title", JobStatus::Pending, + Time.utc + j4 = Job.new "4", "4", "title", "manga_title", + JobStatus::Downloading, Time.utc + count = queue.push [j1, j2, j3, j4] + count.should eq 4 + end + end - it "pops pending job" do - with_queue do |queue| - job = queue.pop - job.should_not be_nil - job.not_nil!.id.should eq "3" - end - end + it "pops pending job" do + with_queue do |queue| + job = queue.pop + job.should_not be_nil + job.not_nil!.id.should eq "3" + end + end - it "correctly counts jobs" do - with_queue do |queue| - queue.count.should eq 4 - end - end + it "correctly counts jobs" do + with_queue do |queue| + queue.count.should eq 4 + end + end - it "deletes job" do - with_queue do |queue| - queue.delete "4" - queue.count.should eq 3 - end - end + it "deletes job" do + with_queue do |queue| + queue.delete "4" + queue.count.should eq 3 + end + end - it "sets status" do - with_queue do |queue| - job = queue.pop.not_nil! - queue.set_status JobStatus::Downloading, job - job = queue.pop - job.should_not be_nil - job.not_nil!.status.should eq JobStatus::Downloading - end - end + it "sets status" do + with_queue do |queue| + job = queue.pop.not_nil! + queue.set_status JobStatus::Downloading, job + job = queue.pop + job.should_not be_nil + job.not_nil!.status.should eq JobStatus::Downloading + end + end - it "sets number of pages" do - with_queue do |queue| - job = queue.pop.not_nil! - queue.set_pages 100, job - job = queue.pop - job.should_not be_nil - job.not_nil!.pages.should eq 100 - end - end + it "sets number of pages" do + with_queue do |queue| + job = queue.pop.not_nil! + queue.set_pages 100, job + job = queue.pop + job.should_not be_nil + job.not_nil!.pages.should eq 100 + end + end - it "adds fail/success counts" do - with_queue do |queue| - job = queue.pop.not_nil! - queue.add_success job - queue.add_success job - queue.add_fail job - job = queue.pop - job.should_not be_nil - job.not_nil!.success_count.should eq 2 - job.not_nil!.fail_count.should eq 1 - end - end + it "adds fail/success counts" do + with_queue do |queue| + job = queue.pop.not_nil! + queue.add_success job + queue.add_success job + queue.add_fail job + job = queue.pop + job.should_not be_nil + job.not_nil!.success_count.should eq 2 + job.not_nil!.fail_count.should eq 1 + end + end - it "appends status message" do - with_queue do |queue| - job = queue.pop.not_nil! - queue.add_message "hello", job - queue.add_message "world", job - job = queue.pop - job.should_not be_nil - job.not_nil!.status_message.should eq "\nhello\nworld" - end - end + it "appends status message" do + with_queue do |queue| + job = queue.pop.not_nil! + queue.add_message "hello", job + queue.add_message "world", job + job = queue.pop + job.should_not be_nil + job.not_nil!.status_message.should eq "\nhello\nworld" + end + end - it "cleans up" do - with_queue do - true - end - State.reset - end + it "cleans up" do + with_queue do + true + end + State.reset + end end - diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index fa4409b..1d0d5f3 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -3,63 +3,63 @@ require "../src/context" require "../src/server" class State - @@hash = {} of String => String + @@hash = {} of String => String - def self.get(key) - @@hash[key]? - end + def self.get(key) + @@hash[key]? + end - def self.get!(key) - @@hash[key] - end + def self.get!(key) + @@hash[key] + end - def self.set(key, value) - return if value.nil? - @@hash[key] = value - end + def self.set(key, value) + return if value.nil? + @@hash[key] = value + end - def self.reset - @@hash.clear - end + def self.reset + @@hash.clear + end end def get_tempfile(name) - path = State.get name - if path.nil? || !File.exists? path - file = File.tempfile name - State.set name, file.path - return file - else - return File.new path - end + path = State.get name + if path.nil? || !File.exists? path + file = File.tempfile name + State.set name, file.path + file + else + File.new path + end end def with_default_config - temp_config = get_tempfile "mango-test-config" - config = Config.load temp_config.path - logger = MLogger.new config - yield config, logger, temp_config.path - temp_config.delete + temp_config = get_tempfile "mango-test-config" + config = Config.load temp_config.path + logger = Logger.new config.log_level + yield config, logger, temp_config.path + temp_config.delete end def with_storage - with_default_config do |config, logger| - temp_db = get_tempfile "mango-test-db" - storage = Storage.new temp_db.path, logger - clear = yield storage, temp_db.path - if clear == true - temp_db.delete - end - end + with_default_config do |_, logger| + temp_db = get_tempfile "mango-test-db" + storage = Storage.new temp_db.path, logger + clear = yield storage, temp_db.path + if clear == true + temp_db.delete + end + end end def with_queue - with_default_config do |config, logger| - temp_queue_db = get_tempfile "mango-test-queue-db" - queue = MangaDex::Queue.new temp_queue_db.path, logger - clear = yield queue, temp_queue_db.path - if clear == true - temp_queue_db.delete - end - end + with_default_config do |_, logger| + temp_queue_db = get_tempfile "mango-test-queue-db" + queue = MangaDex::Queue.new temp_queue_db.path, logger + clear = yield queue, temp_queue_db.path + if clear == true + temp_queue_db.delete + end + end end diff --git a/spec/storage_spec.cr b/spec/storage_spec.cr index 8b80003..44bfb5a 100644 --- a/spec/storage_spec.cr +++ b/spec/storage_spec.cr @@ -1,91 +1,91 @@ require "./spec_helper" describe Storage do - it "creates DB at given path" do - with_storage do |storage, path| - File.exists?(path).should be_true - end - end + it "creates DB at given path" do + with_storage do |_, path| + File.exists?(path).should be_true + end + end - it "deletes user" do - with_storage do |storage| - storage.delete_user "admin" - end - end + it "deletes user" do + with_storage do |storage| + storage.delete_user "admin" + end + end - it "creates new user" do - with_storage do |storage| - storage.new_user "user", "123456", false - storage.new_user "admin", "123456", true - end - end + it "creates new user" do + with_storage do |storage| + storage.new_user "user", "123456", false + storage.new_user "admin", "123456", true + end + end - it "verifies username/password combination" do - with_storage do |storage| - user_token = storage.verify_user "user", "123456" - admin_token = storage.verify_user "admin", "123456" - user_token.should_not be_nil - admin_token.should_not be_nil - State.set "user_token", user_token - State.set "admin_token", admin_token - end - end + it "verifies username/password combination" do + with_storage do |storage| + user_token = storage.verify_user "user", "123456" + admin_token = storage.verify_user "admin", "123456" + user_token.should_not be_nil + admin_token.should_not be_nil + State.set "user_token", user_token + State.set "admin_token", admin_token + end + end - it "rejects duplicate username" do - with_storage do |storage| - expect_raises SQLite3::Exception, - "UNIQUE constraint failed: users.username" do - storage.new_user "admin", "123456", true - end - end - end + it "rejects duplicate username" do + with_storage do |storage| + expect_raises SQLite3::Exception, + "UNIQUE constraint failed: users.username" do + storage.new_user "admin", "123456", true + end + end + end - it "verifies token" do - with_storage do |storage| - user_token = State.get! "user_token" - user = storage.verify_token user_token - user.should eq "user" - end - end + it "verifies token" do + with_storage do |storage| + user_token = State.get! "user_token" + user = storage.verify_token user_token + user.should eq "user" + end + end - it "verfies admin token" do - with_storage do |storage| - admin_token = State.get! "admin_token" - storage.verify_admin(admin_token).should be_true - end - end + it "verfies admin token" do + with_storage do |storage| + admin_token = State.get! "admin_token" + storage.verify_admin(admin_token).should be_true + end + end - it "rejects non-admin token" do - with_storage do |storage| - user_token = State.get! "user_token" - storage.verify_admin(user_token).should be_false - end - end + it "rejects non-admin token" do + with_storage do |storage| + user_token = State.get! "user_token" + storage.verify_admin(user_token).should be_false + end + end - it "updates user" do - with_storage do |storage| - storage.update_user "admin", "admin", "654321", true - token = storage.verify_user "admin", "654321" - admin_token = State.get! "admin_token" - token.should eq admin_token - end - end + it "updates user" do + with_storage do |storage| + storage.update_user "admin", "admin", "654321", true + token = storage.verify_user "admin", "654321" + admin_token = State.get! "admin_token" + token.should eq admin_token + end + end - it "logs user out" do - with_storage do |storage| - user_token = State.get! "user_token" - admin_token = State.get! "admin_token" - storage.logout user_token - storage.logout admin_token - storage.verify_token(user_token).should be_nil - storage.verify_token(admin_token).should be_nil - end - end + it "logs user out" do + with_storage do |storage| + user_token = State.get! "user_token" + admin_token = State.get! "admin_token" + storage.logout user_token + storage.logout admin_token + storage.verify_token(user_token).should be_nil + storage.verify_token(admin_token).should be_nil + end + end - it "cleans up" do - with_storage do - true - end - State.reset - end + it "cleans up" do + with_storage do + true + end + State.reset + end end diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 47e26d6..da2bb66 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -1,36 +1,36 @@ require "./spec_helper" describe "compare_alphanumerically" do - it "sorts filenames with leading zeros correctly" do - ary = ["010.jpg", "001.jpg", "002.png"] - ary.sort! {|a, b| - compare_alphanumerically a, b - } - ary.should eq ["001.jpg", "002.png", "010.jpg"] - end + it "sorts filenames with leading zeros correctly" do + ary = ["010.jpg", "001.jpg", "002.png"] + ary.sort! { |a, b| + compare_alphanumerically a, b + } + ary.should eq ["001.jpg", "002.png", "010.jpg"] + end - it "sorts filenames without leading zeros correctly" do - ary = ["10.jpg", "1.jpg", "0.png", "0100.jpg"] - ary.sort! {|a, b| - compare_alphanumerically a, b - } - ary.should eq ["0.png", "1.jpg", "10.jpg", "0100.jpg"] - end + it "sorts filenames without leading zeros correctly" do + ary = ["10.jpg", "1.jpg", "0.png", "0100.jpg"] + ary.sort! { |a, b| + compare_alphanumerically a, b + } + ary.should eq ["0.png", "1.jpg", "10.jpg", "0100.jpg"] + end - # https://ux.stackexchange.com/a/95441 - it "sorts like the stack exchange post" do - ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2", - "text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"] - ary.reverse.sort {|a, b| - compare_alphanumerically a, b - }.should eq ary - end + # https://ux.stackexchange.com/a/95441 + it "sorts like the stack exchange post" do + ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2", + "text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"] + ary.reverse.sort { |a, b| + compare_alphanumerically a, b + }.should eq ary + end - # https://github.com/hkalexling/Mango/issues/22 - it "handles numbers larger than Int32" do - ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"] - ary.reverse.sort {|a, b| - compare_alphanumerically a, b - }.should eq ary - end + # https://github.com/hkalexling/Mango/issues/22 + it "handles numbers larger than Int32" do + ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"] + ary.reverse.sort { |a, b| + compare_alphanumerically a, b + }.should eq ary + end end diff --git a/src/auth_handler.cr b/src/auth_handler.cr deleted file mode 100644 index b624b22..0000000 --- a/src/auth_handler.cr +++ /dev/null @@ -1,26 +0,0 @@ -require "kemal" -require "./storage" -require "./util" - -class AuthHandler < Kemal::Handler - def initialize(@storage : Storage) - end - - def call(env) - return call_next(env) \ - if request_path_startswith env, ["/login", "/logout"] - - cookie = env.request.cookies.find { |c| c.name == "token" } - if cookie.nil? || ! @storage.verify_token cookie.value - return env.redirect "/login" - end - - if request_path_startswith env, ["/admin", "/api/admin", "/download"] - unless @storage.verify_admin cookie.value - env.response.status_code = 403 - end - end - - call_next env - end -end diff --git a/src/config.cr b/src/config.cr index 18be18b..5e57edf 100644 --- a/src/config.cr +++ b/src/config.cr @@ -1,60 +1,61 @@ require "yaml" class Config - include YAML::Serializable + include YAML::Serializable - property port : Int32 = 9000 - property library_path : String = \ - File.expand_path "~/mango/library", home: true - property db_path : String = \ - File.expand_path "~/mango/mango.db", home: true - @[YAML::Field(key: "scan_interval_minutes")] - property scan_interval : Int32 = 5 - property log_level : String = "info" - property mangadex = Hash(String, String|Int32).new + property port : Int32 = 9000 + property library_path : String = File.expand_path "~/mango/library", + home: true + property db_path : String = File.expand_path "~/mango/mango.db", home: true + @[YAML::Field(key: "scan_interval_minutes")] + property scan_interval : Int32 = 5 + property log_level : String = "info" + property upload_path : String = File.expand_path "~/mango/uploads", + home: true + property mangadex = Hash(String, String | Int32).new - @[YAML::Field(ignore: true)] - @mangadex_defaults = { - "base_url" => "https://mangadex.org", - "api_url" => "https://mangadex.org/api", - "download_wait_seconds" => 5, - "download_retries" => 4, - "download_queue_db_path" => File.expand_path "~/mango/queue.db", - home: true - } + @[YAML::Field(ignore: true)] + @mangadex_defaults = { + "base_url" => "https://mangadex.org", + "api_url" => "https://mangadex.org/api", + "download_wait_seconds" => 5, + "download_retries" => 4, + "download_queue_db_path" => File.expand_path("~/mango/queue.db", + home: true), + } - def self.load(path : String?) - path = "~/.config/mango/config.yml" if path.nil? - cfg_path = File.expand_path path, home: true - if File.exists? cfg_path - config = self.from_yaml File.read cfg_path - config.fill_defaults - return config - end - puts "The config file #{cfg_path} does not exist." \ - " Do you want mango to dump the default config there? [Y/n]" - input = gets - if input && input.downcase == "n" - abort "Aborting..." - end - default = self.allocate - default.fill_defaults - cfg_dir = File.dirname cfg_path - unless Dir.exists? cfg_dir - Dir.mkdir_p cfg_dir - end - File.write cfg_path, default.to_yaml - puts "The config file has been created at #{cfg_path}." - default - end + def self.load(path : String?) + path = "~/.config/mango/config.yml" if path.nil? + cfg_path = File.expand_path path, home: true + if File.exists? cfg_path + config = self.from_yaml File.read cfg_path + config.fill_defaults + return config + end + puts "The config file #{cfg_path} does not exist." \ + " Do you want mango to dump the default config there? [Y/n]" + input = gets + if input && input.downcase == "n" + abort "Aborting..." + end + default = self.allocate + default.fill_defaults + cfg_dir = File.dirname cfg_path + unless Dir.exists? cfg_dir + Dir.mkdir_p cfg_dir + end + File.write cfg_path, default.to_yaml + puts "The config file has been created at #{cfg_path}." + default + end - def fill_defaults - {% for hash_name in ["mangadex"] %} - @{{hash_name.id}}_defaults.map do |k, v| - if @{{hash_name.id}}[k]?.nil? - @{{hash_name.id}}[k] = v - end - end - {% end %} - end + def fill_defaults + {% for hash_name in ["mangadex"] %} + @{{hash_name.id}}_defaults.map do |k, v| + if @{{hash_name.id}}[k]?.nil? + @{{hash_name.id}}[k] = v + end + end + {% end %} + end end diff --git a/src/context.cr b/src/context.cr index a6dfec4..a4e4d4b 100644 --- a/src/context.cr +++ b/src/context.cr @@ -4,18 +4,18 @@ require "./storage" require "./logger" class Context - property config : Config - property library : Library - property storage : Storage - property logger : MLogger - property queue : MangaDex::Queue + property config : Config + property library : Library + property storage : Storage + property logger : Logger + property queue : MangaDex::Queue - def initialize(@config, @logger, @library, @storage, @queue) - end + def initialize(@config, @logger, @library, @storage, @queue) + end - {% for lvl in LEVELS %} - def {{lvl.id}}(msg) - @logger.{{lvl.id}} msg - end - {% end %} + {% for lvl in Logger::LEVELS %} + def {{lvl.id}}(msg) + @logger.{{lvl.id}} msg + end + {% end %} end diff --git a/src/handlers/auth_handler.cr b/src/handlers/auth_handler.cr new file mode 100644 index 0000000..7ec0138 --- /dev/null +++ b/src/handlers/auth_handler.cr @@ -0,0 +1,25 @@ +require "kemal" +require "../storage" +require "../util" + +class AuthHandler < Kemal::Handler + def initialize(@storage : Storage) + end + + def call(env) + return call_next(env) if request_path_startswith env, ["/login", "/logout"] + + cookie = env.request.cookies.find { |c| c.name == "token" } + if cookie.nil? || !@storage.verify_token cookie.value + return env.redirect "/login" + end + + if request_path_startswith env, ["/admin", "/api/admin", "/download"] + unless @storage.verify_admin cookie.value + env.response.status_code = 403 + end + end + + call_next env + end +end diff --git a/src/handlers/log_handler.cr b/src/handlers/log_handler.cr new file mode 100644 index 0000000..105c9f1 --- /dev/null +++ b/src/handlers/log_handler.cr @@ -0,0 +1,26 @@ +require "kemal" +require "../logger" + +class LogHandler < Kemal::BaseLogHandler + def initialize(@logger : Logger) + end + + def call(env) + elapsed_time = Time.measure { call_next env } + elapsed_text = elapsed_text elapsed_time + msg = "#{env.response.status_code} #{env.request.method}" \ + " #{env.request.resource} #{elapsed_text}" + @logger.debug msg + env + end + + def write(msg) + @logger.debug msg + end + + private def elapsed_text(elapsed) + millis = elapsed.total_milliseconds + return "#{millis.round(2)}ms" if millis >= 1 + "#{(millis * 1000).round(2)}µs" + end +end diff --git a/src/handlers/static_handler.cr b/src/handlers/static_handler.cr new file mode 100644 index 0000000..0287445 --- /dev/null +++ b/src/handlers/static_handler.cr @@ -0,0 +1,32 @@ +require "baked_file_system" +require "kemal" +require "../util" + +class FS + extend BakedFileSystem + {% if flag?(:release) %} + {% if read_file? "#{__DIR__}/../../dist/favicon.ico" %} + {% puts "baking ../../dist" %} + bake_folder "../../dist" + {% else %} + {% puts "baking ../../public" %} + bake_folder "../../public" + {% end %} + {% end %} +end + +class StaticHandler < Kemal::Handler + @dirs = ["/css", "/js", "/img", "/favicon.ico"] + + def call(env) + if request_path_startswith env, @dirs + file = FS.get? env.request.path + return call_next env if file.nil? + + slice = Bytes.new file.size + file.read slice + return send_file env, slice, file.mime_type + end + call_next env + end +end diff --git a/src/handlers/upload_handler.cr b/src/handlers/upload_handler.cr new file mode 100644 index 0000000..a6db8b4 --- /dev/null +++ b/src/handlers/upload_handler.cr @@ -0,0 +1,24 @@ +require "kemal" +require "../util" + +class UploadHandler < Kemal::Handler + def initialize(@upload_dir : String) + end + + def call(env) + unless request_path_startswith(env, [UPLOAD_URL_PREFIX]) && + env.request.method == "GET" + return call_next env + end + + ary = env.request.path.split(File::SEPARATOR).select { |part| !part.empty? } + ary[0] = @upload_dir + path = File.join ary + + if File.exists? path + send_file env, path + else + env.response.status_code = 404 + end + end +end diff --git a/src/library.cr b/src/library.cr index 2ce3ce1..058b04a 100644 --- a/src/library.cr +++ b/src/library.cr @@ -5,309 +5,427 @@ require "uri" require "./util" struct Image - property data : Bytes - property mime : String - property filename : String - property size : Int32 + property data : Bytes + property mime : String + property filename : String + property size : Int32 - def initialize(@data, @mime, @filename, @size) - end + def initialize(@data, @mime, @filename, @size) + end end class Entry - property zip_path : String, book_title : String, title : String, - size : String, pages : Int32, cover_url : String, id : String, - title_id : String, encoded_path : String, encoded_title : String, - mtime : Time + property zip_path : String, book : Title, title : String, + size : String, pages : Int32, id : String, title_id : String, + encoded_path : String, encoded_title : String, mtime : Time - def initialize(path, @book_title, @title_id, storage) - @zip_path = path - @encoded_path = URI.encode path - @title = File.basename path, File.extname path - @encoded_title = URI.encode @title - @size = (File.size path).humanize_bytes - file = Zip::File.new path - @pages = file.entries - .select { |e| - ["image/jpeg", "image/png"].includes? \ - MIME.from_filename? e.filename - } - .size - file.close - @id = storage.get_id @zip_path, false - @cover_url = "/api/page/#{@title_id}/#{@id}/1" - @mtime = File.info(@zip_path).modification_time - end + def initialize(path, @book, @title_id, storage) + @zip_path = path + @encoded_path = URI.encode path + @title = File.basename path, File.extname path + @encoded_title = URI.encode @title + @size = (File.size path).humanize_bytes + file = Zip::File.new path + @pages = file.entries.count do |e| + ["image/jpeg", "image/png"].includes? \ + MIME.from_filename? e.filename + end + file.close + @id = storage.get_id @zip_path, false + @mtime = File.info(@zip_path).modification_time + end - def to_json(json : JSON::Builder) - json.object do - {% for str in ["zip_path", "book_title", "title", "size", - "cover_url", "id", "title_id", "encoded_path", - "encoded_title"] %} - json.field {{str}}, @{{str.id}} - {% end %} - json.field "pages" {json.number @pages} - json.field "mtime" {json.number @mtime.to_unix} - end - end + def to_json(json : JSON::Builder) + json.object do + {% for str in ["zip_path", "title", "size", "id", "title_id", + "encoded_path", "encoded_title"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "display_name", @book.display_name @title + json.field "cover_url", cover_url + json.field "pages" { json.number @pages } + json.field "mtime" { json.number @mtime.to_unix } + end + end - def read_page(page_num) - Zip::File.open @zip_path do |file| - page = file.entries - .select { |e| - ["image/jpeg", "image/png"].includes? \ - MIME.from_filename? e.filename - } - .sort { |a, b| - compare_alphanumerically a.filename, b.filename - } - .[page_num - 1] - page.open do |io| - slice = Bytes.new page.uncompressed_size - bytes_read = io.read_fully? slice - unless bytes_read - return nil - end - return Image.new slice, MIME.from_filename(page.filename),\ - page.filename, bytes_read - end - end - end + def display_name + @book.display_name @title + end + + def encoded_display_name + URI.encode display_name + end + + def cover_url + url = "/api/page/#{@title_id}/#{@id}/1" + TitleInfo.new @book.dir do |info| + info_url = info.entry_cover_url[@title]? + unless info_url.nil? || info_url.empty? + url = info_url + end + end + url + end + + def read_page(page_num) + Zip::File.open @zip_path do |file| + page = file.entries + .select { |e| + ["image/jpeg", "image/png"].includes? \ + MIME.from_filename? e.filename + } + .sort { |a, b| + compare_alphanumerically a.filename, b.filename + } + .[page_num - 1] + page.open do |io| + slice = Bytes.new page.uncompressed_size + bytes_read = io.read_fully? slice + unless bytes_read + return nil + end + return Image.new slice, MIME.from_filename(page.filename), + page.filename, bytes_read + end + end + end end class Title - property dir : String, parent_id : String, title_ids : Array(String), - entries : Array(Entry), title : String, id : String, - encoded_title : String, mtime : Time + property dir : String, parent_id : String, title_ids : Array(String), + entries : Array(Entry), title : String, id : String, + encoded_title : String, mtime : Time - def initialize(dir : String, @parent_id, storage, - @logger : MLogger, @library : Library) - @dir = dir - @id = storage.get_id @dir, true - @title = File.basename dir - @encoded_title = URI.encode @title - @title_ids = [] of String - @entries = [] of Entry + def initialize(@dir : String, @parent_id, storage, + @logger : Logger, @library : Library) + @id = storage.get_id @dir, true + @title = File.basename dir + @encoded_title = URI.encode @title + @title_ids = [] of String + @entries = [] of Entry + @mtime = File.info(dir).modification_time - Dir.entries(dir).each do |fn| - next if fn.starts_with? "." - path = File.join dir, fn - if File.directory? path - title = Title.new path, @id, storage, @logger, library - next if title.entries.size == 0 && title.titles.size == 0 - @library.title_hash[title.id] = title - @title_ids << title.id - next - end - if [".zip", ".cbz"].includes? File.extname path - next if !valid_zip path - entry = Entry.new path, @title, @id, storage - @entries << entry if entry.pages > 0 - end - end + Dir.entries(dir).each do |fn| + next if fn.starts_with? "." + path = File.join dir, fn + if File.directory? path + title = Title.new path, @id, storage, @logger, library + next if title.entries.size == 0 && title.titles.size == 0 + @library.title_hash[title.id] = title + @title_ids << title.id + next + end + if [".zip", ".cbz"].includes? File.extname path + zip_exception = validate_zip path + unless zip_exception.nil? + @logger.warn "File #{path} is corrupted or is not a valid zip " \ + "archive. Ignoring it." + @logger.debug "Zip error: #{zip_exception}" + next + end + entry = Entry.new path, self, @id, storage + @entries << entry if entry.pages > 0 + end + end - @title_ids.sort! do |a, b| - compare_alphanumerically @library.title_hash[a].title, - @library.title_hash[b].title - end - @entries.sort! do |a, b| - compare_alphanumerically a.title, b.title - end + mtimes = [@mtime] + mtimes += @title_ids.map { |e| @library.title_hash[e].mtime } + mtimes += @entries.map { |e| e.mtime } + @mtime = mtimes.max - mtimes = [File.info(dir).modification_time] - mtimes += @title_ids.map{|e| @library.title_hash[e].mtime} - mtimes += @entries.map{|e| e.mtime} - @mtime = mtimes.max - end + @title_ids.sort! do |a, b| + compare_alphanumerically @library.title_hash[a].title, + @library.title_hash[b].title + end + @entries.sort! do |a, b| + compare_alphanumerically a.title, b.title + end + end - def to_json(json : JSON::Builder) - json.object do - {% for str in ["dir", "title", "id", "encoded_title"] %} - json.field {{str}}, @{{str.id}} - {% end %} - json.field "mtime" {json.number @mtime.to_unix} - json.field "titles" do - json.raw self.titles.to_json - end - json.field "entries" do - json.raw @entries.to_json - end - json.field "parents" do - json.array do - self.parents.each do |title| - json.object do - json.field "title", title.title - json.field "id", title.id - end - end - end - end - end - end + def to_json(json : JSON::Builder) + json.object do + {% for str in ["dir", "title", "id", "encoded_title"] %} + json.field {{str}}, @{{str.id}} + {% end %} + json.field "display_name", display_name + json.field "cover_url", cover_url + json.field "mtime" { json.number @mtime.to_unix } + json.field "titles" do + json.raw self.titles.to_json + end + json.field "entries" do + json.raw @entries.to_json + end + json.field "parents" do + json.array do + self.parents.each do |title| + json.object do + json.field "title", title.title + json.field "id", title.id + end + end + end + end + end + end - def titles - @title_ids.map {|tid| @library.get_title! tid} - end + def titles + @title_ids.map { |tid| @library.get_title! tid } + end - def parents - ary = [] of Title - tid = @parent_id - while !tid.empty? - title = @library.get_title! tid - ary << title - tid = title.parent_id - end - ary - end + def parents + ary = [] of Title + tid = @parent_id + while !tid.empty? + title = @library.get_title! tid + ary << title + tid = title.parent_id + end + ary + end - def size - @entries.size + @title_ids.size - end + def size + @entries.size + @title_ids.size + end - # When downloading from MangaDex, the zip/cbz file would not be valid - # before the download is completed. If we scan the zip file, - # Entry.new would throw, so we use this method to check before - # constructing Entry - private def valid_zip(path : String) - begin - file = Zip::File.new path - file.close - return true - rescue - @logger.warn "File #{path} is corrupted or is not a valid zip "\ - "archive. Ignoring it." - return false - end - end - def get_entry(eid) - @entries.find { |e| e.id == eid } - end - # For backward backward compatibility with v0.1.0, we save entry titles - # instead of IDs in info.json - def save_progress(username, entry, page) - info = TitleInfo.new @dir - if info.progress[username]?.nil? - info.progress[username] = {entry => page} - info.save @dir - return - end - info.progress[username][entry] = page - info.save @dir - end - def load_progress(username, entry) - info = TitleInfo.new @dir - if info.progress[username]?.nil? - return 0 - end - if info.progress[username][entry]?.nil? - return 0 - end - info.progress[username][entry] - end - def load_percetage(username, entry) - info = TitleInfo.new @dir - page = load_progress username, entry - entry_obj = @entries.find{|e| e.title == entry} - return 0.0 if entry_obj.nil? - page / entry_obj.pages - end - def load_percetage(username) - return 0.0 if @entries.empty? - read_pages = total_pages = 0 - @entries.each do |e| - read_pages += load_progress username, e.title - total_pages += e.pages - end - read_pages / total_pages - end - def next_entry(current_entry_obj) - idx = @entries.index current_entry_obj - return nil if idx.nil? || idx == @entries.size - 1 - @entries[idx + 1] - end + def get_entry(eid) + @entries.find { |e| e.id == eid } + end + + def display_name + dn = @title + TitleInfo.new @dir do |info| + info_dn = info.display_name + dn = info_dn unless info_dn.empty? + end + dn + end + + def encoded_display_name + URI.encode display_name + end + + def display_name(entry_name) + dn = entry_name + TitleInfo.new @dir do |info| + info_dn = info.entry_display_name[entry_name]? + unless info_dn.nil? || info_dn.empty? + dn = info_dn + end + end + dn + end + + def set_display_name(dn) + TitleInfo.new @dir do |info| + info.display_name = dn + info.save + end + end + + def set_display_name(entry_name : String, dn) + TitleInfo.new @dir do |info| + info.entry_display_name[entry_name] = dn + info.save + end + end + + def cover_url + url = "img/icon.png" + if @entries.size > 0 + url = @entries[0].cover_url + end + TitleInfo.new @dir do |info| + info_url = info.cover_url + unless info_url.nil? || info_url.empty? + url = info_url + end + end + url + end + + def set_cover_url(url : String) + TitleInfo.new @dir do |info| + info.cover_url = url + info.save + end + end + + def set_cover_url(entry_name : String, url : String) + TitleInfo.new @dir do |info| + info.entry_cover_url[entry_name] = url + info.save + end + end + + # Set the reading progress of all entries and nested libraries to 100% + def read_all(username) + @entries.each do |e| + save_progress username, e.title, e.pages + end + titles.each do |t| + t.read_all username + end + end + + # Set the reading progress of all entries and nested libraries to 0% + def unread_all(username) + @entries.each do |e| + save_progress username, e.title, 0 + end + titles.each do |t| + t.unread_all username + end + end + + # For backward backward compatibility with v0.1.0, we save entry titles + # instead of IDs in info.json + def save_progress(username, entry, page) + TitleInfo.new @dir do |info| + if info.progress[username]?.nil? + info.progress[username] = {entry => page} + else + info.progress[username][entry] = page + end + info.save + end + end + + def load_progress(username, entry) + progress = 0 + TitleInfo.new @dir do |info| + unless info.progress[username]?.nil? || + info.progress[username][entry]?.nil? + progress = info.progress[username][entry] + end + end + progress + end + + def load_percetage(username, entry) + page = load_progress username, entry + entry_obj = @entries.find { |e| e.title == entry } + return 0.0 if entry_obj.nil? + page / entry_obj.pages + end + + def load_percetage(username) + return 0.0 if @entries.empty? + read_pages = total_pages = 0 + @entries.each do |e| + read_pages += load_progress username, e.title + total_pages += e.pages + end + read_pages / total_pages + end + + def next_entry(current_entry_obj) + idx = @entries.index current_entry_obj + return nil if idx.nil? || idx == @entries.size - 1 + @entries[idx + 1] + end end class TitleInfo - # { user1: { entry1: 10, entry2: 0 } } - include JSON::Serializable + include JSON::Serializable - property comment = "Generated by Mango. DO NOT EDIT!" - property progress : Hash(String, Hash(String, Int32)) + property comment = "Generated by Mango. DO NOT EDIT!" + property progress = {} of String => Hash(String, Int32) + property display_name = "" + property entry_display_name = {} of String => String + property cover_url = "" + property entry_cover_url = {} of String => String - def initialize(title_dir) - info = nil + @[JSON::Field(ignore: true)] + property dir : String = "" - json_path = File.join title_dir, "info.json" - if File.exists? json_path - info = TitleInfo.from_json File.read json_path - else - info = TitleInfo.from_json "{\"progress\": {}}" - end + @@mutex_hash = {} of String => Mutex - @progress = info.progress.clone - end - def save(title_dir) - json_path = File.join title_dir, "info.json" - File.write json_path, self.to_pretty_json - end + def self.new(dir, &) + if @@mutex_hash[dir]? + mutex = @@mutex_hash[dir] + else + mutex = Mutex.new + @@mutex_hash[dir] = mutex + end + mutex.synchronize do + instance = TitleInfo.allocate + json_path = File.join dir, "info.json" + if File.exists? json_path + instance = TitleInfo.from_json File.read json_path + end + instance.dir = dir + yield instance + end + end + + def save + json_path = File.join @dir, "info.json" + File.write json_path, self.to_pretty_json + end end class Library - property dir : String, title_ids : Array(String), scan_interval : Int32, - logger : MLogger, storage : Storage, title_hash : Hash(String, Title) + property dir : String, title_ids : Array(String), scan_interval : Int32, + logger : Logger, storage : Storage, title_hash : Hash(String, Title) - def initialize(@dir, @scan_interval, @logger, @storage) - # explicitly initialize @titles to bypass the compiler check. it will - # be filled with actual Titles in the `scan` call below - @title_ids = [] of String - @title_hash = {} of String => Title + def initialize(@dir, @scan_interval, @logger, @storage) + # explicitly initialize @titles to bypass the compiler check. it will + # be filled with actual Titles in the `scan` call below + @title_ids = [] of String + @title_hash = {} of String => Title - return scan if @scan_interval < 1 - spawn do - loop do - start = Time.local - scan - ms = (Time.local - start).total_milliseconds - @logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" - sleep @scan_interval * 60 - end - end - end - def titles - @title_ids.map {|tid| self.get_title!(tid) } - end - def to_json(json : JSON::Builder) - json.object do - json.field "dir", @dir - json.field "titles" do - json.raw self.titles.to_json - end - end - end - def get_title(tid) - @title_hash[tid]? - end - def get_title!(tid) - @title_hash[tid] - end - def scan - unless Dir.exists? @dir - @logger.info "The library directory #{@dir} does not exist. " \ - "Attempting to create it" - Dir.mkdir_p @dir - end - @title_ids.clear - (Dir.entries @dir) - .select { |fn| !fn.starts_with? "." } - .map { |fn| File.join @dir, fn } - .select { |path| File.directory? path } - .map { |path| Title.new path, "", @storage, @logger, self } - .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort { |a, b| a.title <=> b.title } - .each do |title| - @title_hash[title.id] = title - @title_ids << title.id - end - @logger.debug "Scan completed" - end + return scan if @scan_interval < 1 + spawn do + loop do + start = Time.local + scan + ms = (Time.local - start).total_milliseconds + @logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" + sleep @scan_interval * 60 + end + end + end + + def titles + @title_ids.map { |tid| self.get_title!(tid) } + end + + def to_json(json : JSON::Builder) + json.object do + json.field "dir", @dir + json.field "titles" do + json.raw self.titles.to_json + end + end + end + + def get_title(tid) + @title_hash[tid]? + end + + def get_title!(tid) + @title_hash[tid] + end + + def scan + unless Dir.exists? @dir + @logger.info "The library directory #{@dir} does not exist. " \ + "Attempting to create it" + Dir.mkdir_p @dir + end + @title_ids.clear + (Dir.entries @dir) + .select { |fn| !fn.starts_with? "." } + .map { |fn| File.join @dir, fn } + .select { |path| File.directory? path } + .map { |path| Title.new path, "", @storage, @logger, self } + .select { |title| !(title.entries.empty? && title.titles.empty?) } + .sort { |a, b| a.title <=> b.title } + .each do |title| + @title_hash[title.id] = title + @title_ids << title.id + end + @logger.debug "Scan completed" + end end diff --git a/src/log_handler.cr b/src/log_handler.cr deleted file mode 100644 index 4990c7f..0000000 --- a/src/log_handler.cr +++ /dev/null @@ -1,26 +0,0 @@ -require "kemal" -require "./logger" - -class LogHandler < Kemal::BaseLogHandler - def initialize(@logger : MLogger) - end - - def call(env) - elapsed_time = Time.measure { call_next env } - elapsed_text = elapsed_text elapsed_time - msg = "#{env.response.status_code} #{env.request.method}" \ - " #{env.request.resource} #{elapsed_text}" - @logger.debug(msg) - env - end - - def write(msg) - @logger.debug(msg) - end - - private def elapsed_text(elapsed) - millis = elapsed.total_milliseconds - return "#{millis.round(2)}ms" if millis >= 1 - "#{(millis * 1000).round(2)}µs" - end -end diff --git a/src/logger.cr b/src/logger.cr index 1e6d446..6a7536a 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -1,59 +1,58 @@ -require "./config" -require "logger" +require "log" require "colorize" -LEVELS = ["debug", "error", "fatal", "info", "warn"] -COLORS = [:light_cyan, :light_red, :red, :light_yellow, :light_magenta] +class Logger + LEVELS = ["debug", "error", "fatal", "info", "warn"] + SEVERITY_IDS = [0, 4, 5, 2, 3] + COLORS = [:light_cyan, :light_red, :red, :light_yellow, :light_magenta] -class MLogger - def initialize(config : Config) - @logger = Logger.new STDOUT + @@severity : Log::Severity = :info - @log_off = false - log_level = config.log_level - if log_level == "off" - @log_off = true - return - end + def initialize(level : String) + {% begin %} + case level.downcase + when "off" + @@severity = :none + {% for lvl, i in LEVELS %} + when {{lvl}} + @@severity = Log::Severity.new SEVERITY_IDS[{{i}}] + {% end %} + else + raise "Unknown log level #{level}" + end + {% end %} - {% begin %} - case log_level - {% for lvl in LEVELS %} - when {{lvl}} - @logger.level = Logger::{{lvl.upcase.id}} - {% end %} - else - raise "Unknown log level #{log_level}" - end - {% end %} + @log = Log.for("") - @logger.formatter = Logger::Formatter.new do \ - |severity, datetime, progname, message, io| + @backend = Log::IOBackend.new + @backend.formatter = ->(entry : Log::Entry, io : IO) do + color = :default + {% begin %} + case entry.severity.label.to_s().downcase + {% for lvl, i in LEVELS %} + when {{lvl}}, "#{{{lvl}}}ing" + color = COLORS[{{i}}] + {% end %} + else + end + {% end %} - color = :default - {% begin %} - case severity.to_s().downcase - {% for lvl, i in LEVELS %} - when {{lvl}} - color = COLORS[{{i}}] - {% end %} - end - {% end %} + io << "[#{entry.severity.label}]".ljust(10).colorize(color) + io << entry.timestamp.to_s("%Y/%m/%d %H:%M:%S") << " | " + io << entry.message + end - io << "[#{severity}]".ljust(8).colorize(color) - io << datetime.to_s("%Y/%m/%d %H:%M:%S") << " | " - io << message - end - end + Log.builder.bind "*", @@severity, @backend + end - {% for lvl in LEVELS %} - def {{lvl.id}}(msg) - return if @log_off - @logger.{{lvl.id}} msg - end - {% end %} + # Ignores @@severity and always log msg + def log(msg) + @backend.write Log::Entry.new "", Log::Severity::None, msg, nil + end - def to_json(json : JSON::Builder) - json.string self - end + {% for lvl in LEVELS %} + def {{lvl.id}}(msg) + @log.{{lvl.id}} { msg } + end + {% end %} end diff --git a/src/mangadex/api.cr b/src/mangadex/api.cr index 0cf181c..38c4ae0 100644 --- a/src/mangadex/api.cr +++ b/src/mangadex/api.cr @@ -2,202 +2,196 @@ require "http/client" require "json" require "csv" -macro string_properties (names) - {% for name in names %} - property {{name.id}} = "" - {% end %} +macro string_properties(names) + {% for name in names %} + property {{name.id}} = "" + {% end %} end -macro parse_strings_from_json (names) - {% for name in names %} - @{{name.id}} = obj[{{name}}].as_s - {% end %} +macro parse_strings_from_json(names) + {% for name in names %} + @{{name.id}} = obj[{{name}}].as_s + {% end %} end module MangaDex - class Chapter - string_properties ["lang_code", "title", "volume", "chapter"] - property manga : Manga - property time = Time.local - property id : String - property full_title = "" - property language = "" - property pages = [] of {String, String} # filename, url - property groups = [] of {Int32, String} # group_id, group_name + class Chapter + string_properties ["lang_code", "title", "volume", "chapter"] + property manga : Manga + property time = Time.local + property id : String + property full_title = "" + property language = "" + property pages = [] of {String, String} # filename, url + property groups = [] of {Int32, String} # group_id, group_name - def initialize(@id, json_obj : JSON::Any, @manga, lang : - Hash(String, String)) - self.parse_json json_obj, lang - end + def initialize(@id, json_obj : JSON::Any, @manga, + lang : Hash(String, String)) + self.parse_json json_obj, lang + end - def to_info_json - JSON.build do |json| - json.object do - {% for name in ["id", "title", "volume", "chapter", - "language", "full_title"] %} - json.field {{name}}, @{{name.id}} - {% end %} - json.field "time", @time.to_unix.to_s - json.field "manga_title", @manga.title - json.field "manga_id", @manga.id - json.field "groups" do - json.object do - @groups.each do |gid, gname| - json.field gname, gid - end - end - end - end - end - end + def to_info_json + JSON.build do |json| + json.object do + {% for name in ["id", "title", "volume", "chapter", + "language", "full_title"] %} + json.field {{name}}, @{{name.id}} + {% end %} + json.field "time", @time.to_unix.to_s + json.field "manga_title", @manga.title + json.field "manga_id", @manga.id + json.field "groups" do + json.object do + @groups.each do |gid, gname| + json.field gname, gid + end + end + end + end + end + end - def parse_json(obj, lang) - begin - parse_strings_from_json ["lang_code", "title", "volume", - "chapter"] - language = lang[@lang_code]? - @language = language if language - @time = Time.unix obj["timestamp"].as_i - suffixes = ["", "_2", "_3"] - suffixes.each do |s| - gid = obj["group_id#{s}"].as_i - next if gid == 0 - gname = obj["group_name#{s}"].as_s - @groups << {gid, gname} - end - @full_title = @title - unless @chapter.empty? - @full_title = "Ch.#{@chapter} " + @full_title - end - unless @volume.empty? - @full_title = "Vol.#{@volume} " + @full_title - end - rescue e - raise "failed to parse json: #{e}" - end - end - end - class Manga - string_properties ["cover_url", "description", "title", "author", - "artist"] - property chapters = [] of Chapter - property id : String + def parse_json(obj, lang) + parse_strings_from_json ["lang_code", "title", "volume", + "chapter"] + language = lang[@lang_code]? + @language = language if language + @time = Time.unix obj["timestamp"].as_i + suffixes = ["", "_2", "_3"] + suffixes.each do |s| + gid = obj["group_id#{s}"].as_i + next if gid == 0 + gname = obj["group_name#{s}"].as_s + @groups << {gid, gname} + end + @full_title = @title + unless @chapter.empty? + @full_title = "Ch.#{@chapter} " + @full_title + end + unless @volume.empty? + @full_title = "Vol.#{@volume} " + @full_title + end + rescue e + raise "failed to parse json: #{e}" + end + end - def initialize(@id, json_obj : JSON::Any) - self.parse_json json_obj - end + class Manga + string_properties ["cover_url", "description", "title", "author", "artist"] + property chapters = [] of Chapter + property id : String - def to_info_json(with_chapters = true) - JSON.build do |json| - json.object do - {% for name in ["id", "title", "description", - "author", "artist", "cover_url"] %} - json.field {{name}}, @{{name.id}} - {% end %} - if with_chapters - json.field "chapters" do - json.array do - @chapters.each do |c| - json.raw c.to_info_json - end - end - end - end - end - end - end + def initialize(@id, json_obj : JSON::Any) + self.parse_json json_obj + end - def parse_json(obj) - begin - parse_strings_from_json ["cover_url", "description", "title", - "author", "artist"] - rescue e - raise "failed to parse json: #{e}" - end - end - end - class API - def initialize(@base_url = "https://mangadex.org/api/") - @lang = {} of String => String - CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row| - @lang[row[1]] = row[0] - end - end + def to_info_json(with_chapters = true) + JSON.build do |json| + json.object do + {% for name in ["id", "title", "description", "author", "artist", + "cover_url"] %} + json.field {{name}}, @{{name.id}} + {% end %} + if with_chapters + json.field "chapters" do + json.array do + @chapters.each do |c| + json.raw c.to_info_json + end + end + end + end + end + end + end - def get(url) - headers = HTTP::Headers { - "User-agent" => "Mangadex.cr" - } - res = HTTP::Client.get url, headers - raise "Failed to get #{url}. [#{res.status_code}] "\ - "#{res.status_message}" if !res.success? - JSON.parse res.body - end + def parse_json(obj) + parse_strings_from_json ["cover_url", "description", "title", "author", + "artist"] + rescue e + raise "failed to parse json: #{e}" + end + end - def get_manga(id) - obj = self.get File.join @base_url, "manga/#{id}" - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. " \ - "Got `#{obj["status"]?}`" - end - begin - manga = Manga.new id, obj["manga"] - obj["chapter"].as_h.map do |k, v| - chapter = Chapter.new k, v, manga, @lang - manga.chapters << chapter - end - return manga - rescue - raise "Failed to parse JSON" - end - end + class API + def initialize(@base_url = "https://mangadex.org/api/") + @lang = {} of String => String + CSV.each_row {{read_file "src/assets/lang_codes.csv"}} do |row| + @lang[row[1]] = row[0] + end + end - def get_chapter(chapter : Chapter) - obj = self.get File.join @base_url, "chapter/#{chapter.id}" - if obj["status"]? == "external" - raise "This chapter is hosted on an external site " \ - "#{obj["external"]?}, and Mango does not support " \ - "external chapters." - end - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. " \ - "Got `#{obj["status"]?}`" - end - begin - server = obj["server"].as_s - hash = obj["hash"].as_s - chapter.pages = obj["page_array"].as_a.map do |fn| - { - fn.as_s, - "#{server}#{hash}/#{fn.as_s}" - } - end - rescue - raise "Failed to parse JSON" - end - end + def get(url) + headers = HTTP::Headers{ + "User-agent" => "Mangadex.cr", + } + res = HTTP::Client.get url, headers + raise "Failed to get #{url}. [#{res.status_code}] " \ + "#{res.status_message}" if !res.success? + JSON.parse res.body + end - def get_chapter(id : String) - obj = self.get File.join @base_url, "chapter/#{id}" - if obj["status"]? == "external" - raise "This chapter is hosted on an external site " \ - "#{obj["external"]?}, and Mango does not support " \ - "external chapters." - end - if obj["status"]? != "OK" - raise "Expecting `OK` in the `status` field. " \ - "Got `#{obj["status"]?}`" - end - manga_id = "" - begin - manga_id = obj["manga_id"].as_i.to_s - rescue - raise "Failed to parse JSON" - end - manga = self.get_manga manga_id - chapter = manga.chapters.find {|c| c.id == id}.not_nil! - self.get_chapter chapter - return chapter - end - end + def get_manga(id) + obj = self.get File.join @base_url, "manga/#{id}" + if obj["status"]? != "OK" + raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" + end + begin + manga = Manga.new id, obj["manga"] + obj["chapter"].as_h.map do |k, v| + chapter = Chapter.new k, v, manga, @lang + manga.chapters << chapter + end + manga + rescue + raise "Failed to parse JSON" + end + end + + def get_chapter(chapter : Chapter) + obj = self.get File.join @base_url, "chapter/#{chapter.id}" + if obj["status"]? == "external" + raise "This chapter is hosted on an external site " \ + "#{obj["external"]?}, and Mango does not support " \ + "external chapters." + end + if obj["status"]? != "OK" + raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" + end + begin + server = obj["server"].as_s + hash = obj["hash"].as_s + chapter.pages = obj["page_array"].as_a.map do |fn| + { + fn.as_s, + "#{server}#{hash}/#{fn.as_s}", + } + end + rescue + raise "Failed to parse JSON" + end + end + + def get_chapter(id : String) + obj = self.get File.join @base_url, "chapter/#{id}" + if obj["status"]? == "external" + raise "This chapter is hosted on an external site " \ + "#{obj["external"]?}, and Mango does not support " \ + "external chapters." + end + if obj["status"]? != "OK" + raise "Expecting `OK` in the `status` field. Got `#{obj["status"]?}`" + end + manga_id = "" + begin + manga_id = obj["manga_id"].as_i.to_s + rescue + raise "Failed to parse JSON" + end + manga = self.get_manga manga_id + chapter = manga.chapters.find { |c| c.id == id }.not_nil! + self.get_chapter chapter + chapter + end + end end diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr index 30043c1..2561f6b 100644 --- a/src/mangadex/downloader.cr +++ b/src/mangadex/downloader.cr @@ -2,373 +2,384 @@ require "./api" require "sqlite3" module MangaDex - class PageJob - property success = false - property url : String - property filename : String - property writer : Zip::Writer - property tries_remaning : Int32 - def initialize(@url, @filename, @writer, @tries_remaning) - end - end + class PageJob + property success = false + property url : String + property filename : String + property writer : Zip::Writer + property tries_remaning : Int32 - enum JobStatus - Pending # 0 - Downloading # 1 - Error # 2 - Completed # 3 - MissingPages # 4 - end + def initialize(@url, @filename, @writer, @tries_remaning) + end + end - struct Job - property id : String - property manga_id : String - property title : String - property manga_title : String - property status : JobStatus - property status_message : String = "" - property pages : Int32 = 0 - property success_count : Int32 = 0 - property fail_count : Int32 = 0 - property time : Time + enum JobStatus + Pending # 0 + Downloading # 1 + Error # 2 + Completed # 3 + MissingPages # 4 + end - def parse_query_result(res : DB::ResultSet) - @id = res.read String - @manga_id = res.read String - @title = res.read String - @manga_title = res.read String - status = res.read Int32 - @status_message = res.read String - @pages = res.read Int32 - @success_count = res.read Int32 - @fail_count = res.read Int32 - time = res.read Int64 - @status = JobStatus.new status - @time = Time.unix_ms time - end + struct Job + property id : String + property manga_id : String + property title : String + property manga_title : String + property status : JobStatus + property status_message : String = "" + property pages : Int32 = 0 + property success_count : Int32 = 0 + property fail_count : Int32 = 0 + property time : Time - # Raises if the result set does not contain the correct set of columns - def self.from_query_result(res : DB::ResultSet) - job = Job.allocate - job.parse_query_result res - return job - end + def parse_query_result(res : DB::ResultSet) + @id = res.read String + @manga_id = res.read String + @title = res.read String + @manga_title = res.read String + status = res.read Int32 + @status_message = res.read String + @pages = res.read Int32 + @success_count = res.read Int32 + @fail_count = res.read Int32 + time = res.read Int64 + @status = JobStatus.new status + @time = Time.unix_ms time + end - def initialize(@id, @manga_id, @title, @manga_title, @status, @time) - end + # Raises if the result set does not contain the correct set of columns + def self.from_query_result(res : DB::ResultSet) + job = Job.allocate + job.parse_query_result res + job + end - def to_json(json) - json.object do - {% for name in ["id", "manga_id", "title", "manga_title", - "status_message"] %} - json.field {{name}}, @{{name.id}} - {% end %} - {% for name in ["pages", "success_count", "fail_count"] %} - json.field {{name}} do - json.number @{{name.id}} - end - {% end %} - json.field "status", @status.to_s - json.field "time" do - json.number @time.to_unix_ms - end - end - end - end + def initialize(@id, @manga_id, @title, @manga_title, @status, @time) + end - class Queue - property downloader : Downloader? + def to_json(json) + json.object do + {% for name in ["id", "manga_id", "title", "manga_title", + "status_message"] %} + json.field {{name}}, @{{name.id}} + {% end %} + {% for name in ["pages", "success_count", "fail_count"] %} + json.field {{name}} do + json.number @{{name.id}} + end + {% end %} + json.field "status", @status.to_s + json.field "time" do + json.number @time.to_unix_ms + end + end + end + end - def initialize(@path : String, @logger : MLogger) - dir = File.dirname path - unless Dir.exists? dir - @logger.info "The queue DB directory #{dir} does not exist. " \ - "Attepmting to create it" - Dir.mkdir_p dir - end - DB.open "sqlite3://#{@path}" do |db| - begin - db.exec "create table if not exists queue " \ - "(id text, manga_id text, title text, manga_title " \ - "text, status integer, status_message text, " \ - "pages integer, success_count integer, " \ - "fail_count integer, time integer)" - db.exec "create unique index if not exists id_idx " \ - "on queue (id)" - db.exec "create index if not exists manga_id_idx " \ - "on queue (manga_id)" - db.exec "create index if not exists status_idx " \ - "on queue (status)" - rescue e - @logger.error "Error when checking tables in DB: #{e}" - raise e - end - end - end + class Queue + property downloader : Downloader? - # Returns the earliest job in queue or nil if the job cannot be parsed. - # Returns nil if queue is empty - def pop - job = nil - DB.open "sqlite3://#{@path}" do |db| - begin - db.query_one "select * from queue where status = 0 "\ - "or status = 1 order by time limit 1" do |res| - job = Job.from_query_result res - end - rescue - end - end - return job - end + def initialize(@path : String, @logger : Logger) + dir = File.dirname path + unless Dir.exists? dir + @logger.info "The queue DB directory #{dir} does not exist. " \ + "Attepmting to create it" + Dir.mkdir_p dir + end + DB.open "sqlite3://#{@path}" do |db| + begin + db.exec "create table if not exists queue " \ + "(id text, manga_id text, title text, manga_title " \ + "text, status integer, status_message text, " \ + "pages integer, success_count integer, " \ + "fail_count integer, time integer)" + db.exec "create unique index if not exists id_idx " \ + "on queue (id)" + db.exec "create index if not exists manga_id_idx " \ + "on queue (manga_id)" + db.exec "create index if not exists status_idx " \ + "on queue (status)" + rescue e + @logger.error "Error when checking tables in DB: #{e}" + raise e + end + end + end - # Push an array of jobs into the queue, and return the number of jobs - # inserted. Any job already exists in the queue will be ignored. - def push(jobs : Array(Job)) - start_count = self.count - DB.open "sqlite3://#{@path}" do |db| - jobs.each do |job| - db.exec "insert or ignore into queue values "\ - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - job.id, job.manga_id, job.title, job.manga_title, - job.status.to_i, job.status_message, job.pages, - job.success_count, job.fail_count, job.time.to_unix_ms - end - end - self.count - start_count - end + # Returns the earliest job in queue or nil if the job cannot be parsed. + # Returns nil if queue is empty + def pop + job = nil + DB.open "sqlite3://#{@path}" do |db| + begin + db.query_one "select * from queue where status = 0 " \ + "or status = 1 order by time limit 1" do |res| + job = Job.from_query_result res + end + rescue + end + end + job + end - def reset(id : String) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = 0, status_message = '', " \ - "pages = 0, success_count = 0, fail_count = 0 " \ - "where id = (?)", id - end - end + # Push an array of jobs into the queue, and return the number of jobs + # inserted. Any job already exists in the queue will be ignored. + def push(jobs : Array(Job)) + start_count = self.count + DB.open "sqlite3://#{@path}" do |db| + jobs.each do |job| + db.exec "insert or ignore into queue values " \ + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + job.id, job.manga_id, job.title, job.manga_title, + job.status.to_i, job.status_message, job.pages, + job.success_count, job.fail_count, job.time.to_unix_ms + end + end + self.count - start_count + end - def reset (job : Job) - self.reset job.id - end + def reset(id : String) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = 0, status_message = '', " \ + "pages = 0, success_count = 0, fail_count = 0 " \ + "where id = (?)", id + end + end - # Reset all failed tasks (missing pages and error) - def reset - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = 0, status_message = '', " \ - "pages = 0, success_count = 0, fail_count = 0 " \ - "where status = 2 or status = 4" - end - end + def reset(job : Job) + self.reset job.id + end - def delete(id : String) - DB.open "sqlite3://#{@path}" do |db| - db.exec "delete from queue where id = (?)", id - end - end + # Reset all failed tasks (missing pages and error) + def reset + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = 0, status_message = '', " \ + "pages = 0, success_count = 0, fail_count = 0 " \ + "where status = 2 or status = 4" + end + end - def delete(job : Job) - self.delete job.id - end + def delete(id : String) + DB.open "sqlite3://#{@path}" do |db| + db.exec "delete from queue where id = (?)", id + end + end - def delete_status(status : JobStatus) - DB.open "sqlite3://#{@path}" do |db| - db.exec "delete from queue where status = (?)", status.to_i - end - end + def delete(job : Job) + self.delete job.id + end - def count_status(status : JobStatus) - DB.open "sqlite3://#{@path}" do |db| - return db.query_one "select count(*) from queue where "\ - "status = (?)", status.to_i, as: Int32 - end - end + def delete_status(status : JobStatus) + DB.open "sqlite3://#{@path}" do |db| + db.exec "delete from queue where status = (?)", status.to_i + end + end - def count - DB.open "sqlite3://#{@path}" do |db| - return db.query_one "select count(*) from queue", as: Int32 - end - end + def count_status(status : JobStatus) + num = 0 + DB.open "sqlite3://#{@path}" do |db| + num = db.query_one "select count(*) from queue where " \ + "status = (?)", status.to_i, as: Int32 + end + num + end - def set_status(status : JobStatus, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status = (?) where id = (?)", - status.to_i, job.id - end - end + def count + num = 0 + DB.open "sqlite3://#{@path}" do |db| + num = db.query_one "select count(*) from queue", as: Int32 + end + num + end - def get_all - jobs = [] of Job - DB.open "sqlite3://#{@path}" do |db| - jobs = db.query_all "select * from queue order by time", do |rs| - Job.from_query_result rs - end - end - return jobs - end + def set_status(status : JobStatus, job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status = (?) where id = (?)", + status.to_i, job.id + end + end - def add_success(job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set success_count = success_count + 1 " \ - "where id = (?)", job.id - end - end + def get_all + jobs = [] of Job + DB.open "sqlite3://#{@path}" do |db| + jobs = db.query_all "select * from queue order by time" do |rs| + Job.from_query_result rs + end + end + jobs + end - def add_fail(job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set fail_count = fail_count + 1 " \ - "where id = (?)", job.id - end - end + def add_success(job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set success_count = success_count + 1 " \ + "where id = (?)", job.id + end + end - def set_pages(pages : Int32, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set pages = (?), success_count = 0, " \ - "fail_count = 0 where id = (?)", pages, job.id - end - end + def add_fail(job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set fail_count = fail_count + 1 " \ + "where id = (?)", job.id + end + end - def add_message(msg : String, job : Job) - DB.open "sqlite3://#{@path}" do |db| - db.exec "update queue set status_message = " \ - "status_message || (?) || (?) where id = (?)", - "\n", msg, job.id - end - end + def set_pages(pages : Int32, job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set pages = (?), success_count = 0, " \ + "fail_count = 0 where id = (?)", pages, job.id + end + end - def pause - @downloader.not_nil!.stopped = true - end + def add_message(msg : String, job : Job) + DB.open "sqlite3://#{@path}" do |db| + db.exec "update queue set status_message = " \ + "status_message || (?) || (?) where id = (?)", + "\n", msg, job.id + end + end - def resume - @downloader.not_nil!.stopped = false - end + def pause + @downloader.not_nil!.stopped = true + end - def paused? - @downloader.not_nil!.stopped - end - end + def resume + @downloader.not_nil!.stopped = false + end - class Downloader - property stopped = false - @downloading = false + def paused? + @downloader.not_nil!.stopped + end + end - def initialize(@queue : Queue, @api : API, @library_path : String, - @wait_seconds : Int32, @retries : Int32, - @logger : MLogger) - @queue.downloader = self + class Downloader + property stopped = false + @downloading = false - spawn do - loop do - sleep 1.second - next if @stopped || @downloading - begin - job = @queue.pop - next if job.nil? - download job - rescue e - @logger.error e - end - end - end - end + def initialize(@queue : Queue, @api : API, @library_path : String, + @wait_seconds : Int32, @retries : Int32, + @logger : Logger) + @queue.downloader = self - private def download(job : Job) - @downloading = true - @queue.set_status JobStatus::Downloading, job - begin - chapter = @api.get_chapter(job.id) - rescue e - @logger.error e - @queue.set_status JobStatus::Error, job - unless e.message.nil? - @queue.add_message e.message.not_nil!, job - end - @downloading = false - return - end - @queue.set_pages chapter.pages.size, job - lib_dir = @library_path - manga_dir = File.join lib_dir, chapter.manga.title - unless File.exists? manga_dir - Dir.mkdir_p manga_dir - end - zip_path = File.join manga_dir, "#{job.title}.cbz" + spawn do + loop do + sleep 1.second + next if @stopped || @downloading + begin + job = @queue.pop + next if job.nil? + download job + rescue e + @logger.error e + end + end + end + end - # Find the number of digits needed to store the number of pages - len = Math.log10(chapter.pages.size).to_i + 1 + private def download(job : Job) + @downloading = true + @queue.set_status JobStatus::Downloading, job + begin + chapter = @api.get_chapter(job.id) + rescue e + @logger.error e + @queue.set_status JobStatus::Error, job + unless e.message.nil? + @queue.add_message e.message.not_nil!, job + end + @downloading = false + return + end + @queue.set_pages chapter.pages.size, job + lib_dir = @library_path + manga_dir = File.join lib_dir, chapter.manga.title + unless File.exists? manga_dir + Dir.mkdir_p manga_dir + end + zip_path = File.join manga_dir, "#{job.title}.cbz" - writer = Zip::Writer.new zip_path - # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new chapter.pages.size - spawn do - chapter.pages.each_with_index do |tuple, i| - fn, url = tuple - ext = File.extname fn - fn = "#{i.to_s.rjust len, '0'}#{ext}" - page_job = PageJob.new url, fn, writer, @retries - @logger.debug "Downloading #{url}" - loop do - sleep @wait_seconds.seconds - download_page page_job - break if page_job.success || - page_job.tries_remaning <= 0 - page_job.tries_remaning -= 1 - @logger.warn "Failed to download page #{url}. " \ - "Retrying... Remaining retries: " \ - "#{page_job.tries_remaning}" - end + # Find the number of digits needed to store the number of pages + len = Math.log10(chapter.pages.size).to_i + 1 - channel.send page_job - end - end + writer = Zip::Writer.new zip_path + # Create a buffered channel. It works as an FIFO queue + channel = Channel(PageJob).new chapter.pages.size + spawn do + chapter.pages.each_with_index do |tuple, i| + fn, url = tuple + ext = File.extname fn + fn = "#{i.to_s.rjust len, '0'}#{ext}" + page_job = PageJob.new url, fn, writer, @retries + @logger.debug "Downloading #{url}" + loop do + sleep @wait_seconds.seconds + download_page page_job + break if page_job.success || + page_job.tries_remaning <= 0 + page_job.tries_remaning -= 1 + @logger.warn "Failed to download page #{url}. " \ + "Retrying... Remaining retries: " \ + "#{page_job.tries_remaning}" + end - spawn do - page_jobs = [] of PageJob - chapter.pages.size.times do - page_job = channel.receive - @logger.debug "[#{page_job.success ? "success" : "failed"}] " \ - "#{page_job.url}" - page_jobs << page_job - if page_job.success - @queue.add_success job - else - @queue.add_fail job - msg = "Failed to download page #{page_job.url}" - @queue.add_message msg, job - @logger.error msg - end - end - fail_count = page_jobs.select{|j| !j.success}.size - @logger.debug "Download completed. "\ - "#{fail_count}/#{page_jobs.size} failed" - writer.close - @logger.debug "cbz File created at #{zip_path}" - if fail_count == 0 - @queue.set_status JobStatus::Completed, job - else - @queue.set_status JobStatus::MissingPages, job - end - @downloading = false - end - end + channel.send page_job + end + end - private def download_page(job : PageJob) - @logger.debug "downloading #{job.url}" - headers = HTTP::Headers { - "User-agent" => "Mangadex.cr" - } - begin - HTTP::Client.get job.url, headers do |res| - unless res.success? - raise "Failed to download page #{job.url}. " \ - "[#{res.status_code}] #{res.status_message}" - end - job.writer.add job.filename, res.body_io - end - job.success = true - rescue e - @logger.error e - job.success = false - end - end - end + spawn do + page_jobs = [] of PageJob + chapter.pages.size.times do + page_job = channel.receive + @logger.debug "[#{page_job.success ? "success" : "failed"}] " \ + "#{page_job.url}" + page_jobs << page_job + if page_job.success + @queue.add_success job + else + @queue.add_fail job + msg = "Failed to download page #{page_job.url}" + @queue.add_message msg, job + @logger.error msg + end + end + fail_count = page_jobs.count { |j| !j.success } + @logger.debug "Download completed. " \ + "#{fail_count}/#{page_jobs.size} failed" + writer.close + @logger.debug "cbz File created at #{zip_path}" + + zip_exception = validate_zip zip_path + if !zip_exception.nil? + @queue.add_message "The downloaded archive is corrupted. " \ + "Error: #{zip_exception}", job + @queue.set_status JobStatus::Error, job + elsif fail_count > 0 + @queue.set_status JobStatus::MissingPages, job + else + @queue.set_status JobStatus::Completed, job + end + @downloading = false + end + end + + private def download_page(job : PageJob) + @logger.debug "downloading #{job.url}" + headers = HTTP::Headers{ + "User-agent" => "Mangadex.cr", + } + begin + HTTP::Client.get job.url, headers do |res| + unless res.success? + raise "Failed to download page #{job.url}. " \ + "[#{res.status_code}] #{res.status_message}" + end + job.writer.add job.filename, res.body_io + end + job.success = true + rescue e + @logger.error e + job.success = false + end + end + end end diff --git a/src/mango.cr b/src/mango.cr index b443099..8266471 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -3,37 +3,37 @@ require "./context" require "./mangadex/*" require "option_parser" -VERSION = "0.2.5" +VERSION = "0.3.0" config_path = nil -parser = OptionParser.parse do |parser| - parser.banner = "Mango e-manga server/reader. Version #{VERSION}\n" +OptionParser.parse do |parser| + parser.banner = "Mango e-manga server/reader. Version #{VERSION}\n" - parser.on "-v", "--version", "Show version" do - puts "Version #{VERSION}" - exit - end - parser.on "-h", "--help", "Show help" do - puts parser - exit - end - parser.on "-c PATH", "--config=PATH", "Path to the config file. " \ - "Default is `~/.config/mango/config.yml`" do |path| - config_path = path - end + parser.on "-v", "--version", "Show version" do + puts "Version #{VERSION}" + exit + end + parser.on "-h", "--help", "Show help" do + puts parser + exit + end + parser.on "-c PATH", "--config=PATH", + "Path to the config file. Default is `~/.config/mango/config.yml`" do |path| + config_path = path + end end config = Config.load config_path -logger = MLogger.new config +logger = Logger.new config.log_level storage = Storage.new config.db_path, logger library = Library.new config.library_path, config.scan_interval, logger, storage queue = MangaDex::Queue.new config.mangadex["download_queue_db_path"].to_s, - logger -api = MangaDex::API.new config.mangadex["api_url"].to_s -downloader = MangaDex::Downloader.new queue, api, config.library_path, - config.mangadex["download_wait_seconds"].to_i, - config.mangadex["download_retries"].to_i, logger + logger +api = MangaDex::API.new config.mangadex["api_url"].to_s +MangaDex::Downloader.new queue, api, config.library_path, + config.mangadex["download_wait_seconds"].to_i, + config.mangadex["download_retries"].to_i, logger context = Context.new config, logger, library, storage, queue diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 02ca4c1..37cf656 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -1,108 +1,103 @@ require "./router" class AdminRouter < Router - def setup - get "/admin" do |env| - layout "admin" - end + def setup + get "/admin" do |env| + layout "admin" + end - get "/admin/user" do |env| - users = @context.storage.list_users - username = get_username env - layout "user" - end + get "/admin/user" do |env| + users = @context.storage.list_users + username = get_username env + layout "user" + end - get "/admin/user/edit" do |env| - username = env.params.query["username"]? - admin = env.params.query["admin"]? - if admin - admin = admin == "true" - end - error = env.params.query["error"]? - current_user = get_username env - new_user = username.nil? && admin.nil? - layout "user-edit" - end + get "/admin/user/edit" do |env| + username = env.params.query["username"]? + admin = env.params.query["admin"]? + if admin + admin = admin == "true" + end + error = env.params.query["error"]? + current_user = get_username env + new_user = username.nil? && admin.nil? + layout "user-edit" + end - post "/admin/user/edit" do |env| - # creating new user - begin - username = env.params.body["username"] - password = env.params.body["password"] - # if `admin` is unchecked, the body hash - # would not contain `admin` - admin = !env.params.body["admin"]?.nil? + post "/admin/user/edit" do |env| + # creating new user + username = env.params.body["username"] + password = env.params.body["password"] + # if `admin` is unchecked, the body hash + # would not contain `admin` + admin = !env.params.body["admin"]?.nil? - if username.size < 3 - raise "Username should contain at least 3 characters" - end - if (username =~ /^[A-Za-z0-9_]+$/).nil? - raise "Username should contain alphanumeric characters "\ - "and underscores only" - end - if password.size < 6 - raise "Password should contain at least 6 characters" - end - if (password =~ /^[[:ascii:]]+$/).nil? - raise "password should contain ASCII characters only" - end + if username.size < 3 + raise "Username should contain at least 3 characters" + end + if (username =~ /^[A-Za-z0-9_]+$/).nil? + raise "Username should contain alphanumeric characters " \ + "and underscores only" + end + if password.size < 6 + raise "Password should contain at least 6 characters" + end + if (password =~ /^[[:ascii:]]+$/).nil? + raise "password should contain ASCII characters only" + end - @context.storage.new_user username, password, admin + @context.storage.new_user username, password, admin - env.redirect "/admin/user" - rescue e - @context.error e - redirect_url = URI.new \ - path: "/admin/user/edit",\ - query: hash_to_query({"error" => e.message}) - env.redirect redirect_url.to_s - end - end + env.redirect "/admin/user" + rescue e + @context.error e + redirect_url = URI.new \ + path: "/admin/user/edit", + query: hash_to_query({"error" => e.message}) + env.redirect redirect_url.to_s + end - post "/admin/user/edit/:original_username" do |env| - # editing existing user - begin - username = env.params.body["username"] - password = env.params.body["password"] - # if `admin` is unchecked, the body - # hash would not contain `admin` - admin = !env.params.body["admin"]?.nil? - original_username = env.params.url["original_username"] + post "/admin/user/edit/:original_username" do |env| + # editing existing user + username = env.params.body["username"] + password = env.params.body["password"] + # if `admin` is unchecked, the body hash would not contain `admin` + admin = !env.params.body["admin"]?.nil? + original_username = env.params.url["original_username"] - if username.size < 3 - raise "Username should contain at least 3 characters" - end - if (username =~ /^[A-Za-z0-9_]+$/).nil? - raise "Username should contain alphanumeric characters "\ - "and underscores only" - end + if username.size < 3 + raise "Username should contain at least 3 characters" + end + if (username =~ /^[A-Za-z0-9_]+$/).nil? + raise "Username should contain alphanumeric characters " \ + "and underscores only" + end - if password.size != 0 - if password.size < 6 - raise "Password should contain at least 6 characters" - end - if (password =~ /^[[:ascii:]]+$/).nil? - raise "password should contain ASCII characters only" - end - end + if password.size != 0 + if password.size < 6 + raise "Password should contain at least 6 characters" + end + if (password =~ /^[[:ascii:]]+$/).nil? + raise "password should contain ASCII characters only" + end + end - @context.storage.update_user \ - original_username, username, password, admin + @context.storage.update_user \ + original_username, username, password, admin - env.redirect "/admin/user" - rescue e - @context.error e - redirect_url = URI.new \ - path: "/admin/user/edit",\ - query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) - env.redirect redirect_url.to_s - end - end + env.redirect "/admin/user" + rescue e + @context.error e + redirect_url = URI.new \ + path: "/admin/user/edit", + query: hash_to_query({"username" => original_username, \ + "admin" => admin, "error" => e.message}) + env.redirect redirect_url.to_s + end - get "/admin/downloads" do |env| - base_url = @context.config.mangadex["base_url"]; - layout "download-manager" - end - end + get "/admin/downloads" do |env| + base_url = @context.config.mangadex["base_url"] + layout "download-manager" + end + end end diff --git a/src/routes/api.cr b/src/routes/api.cr index cefb1b6..e888239 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1,180 +1,263 @@ require "./router" require "../mangadex/*" +require "../upload" class APIRouter < Router - def setup - get "/api/page/:tid/:eid/:page" do |env| - begin - tid = env.params.url["tid"] - eid = env.params.url["eid"] - page = env.params.url["page"].to_i + def setup + get "/api/page/:tid/:eid/:page" do |env| + begin + tid = env.params.url["tid"] + eid = env.params.url["eid"] + page = env.params.url["page"].to_i - title = @context.library.get_title tid - raise "Title ID `#{tid}` not found" if title.nil? - entry = title.get_entry eid - raise "Entry ID `#{eid}` of `#{title.title}` not found" if \ - entry.nil? - img = entry.read_page page - raise "Failed to load page #{page} of " \ - "`#{title.title}/#{entry.title}`" if img.nil? + title = @context.library.get_title tid + raise "Title ID `#{tid}` not found" if title.nil? + entry = title.get_entry eid + raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? + img = entry.read_page page + raise "Failed to load page #{page} of " \ + "`#{title.title}/#{entry.title}`" if img.nil? - send_img env, img - rescue e - @context.error e - env.response.status_code = 500 - e.message - end - end + send_img env, img + rescue e + @context.error e + env.response.status_code = 500 + e.message + end + end - get "/api/book/:tid" do |env| - begin - tid = env.params.url["tid"] - title = @context.library.get_title tid - raise "Title ID `#{tid}` not found" if title.nil? + get "/api/book/:tid" do |env| + begin + tid = env.params.url["tid"] + title = @context.library.get_title tid + raise "Title ID `#{tid}` not found" if title.nil? - send_json env, title.to_json - rescue e - @context.error e - env.response.status_code = 500 - e.message - end - end + send_json env, title.to_json + rescue e + @context.error e + env.response.status_code = 500 + e.message + end + end - get "/api/book" do |env| - send_json env, @context.library.to_json - end + get "/api/book" do |env| + send_json env, @context.library.to_json + end - post "/api/admin/scan" do |env| - start = Time.utc - @context.library.scan - ms = (Time.utc - start).total_milliseconds - send_json env, { - "milliseconds" => ms, - "titles" => @context.library.titles.size - }.to_json - end + post "/api/admin/scan" do |env| + start = Time.utc + @context.library.scan + ms = (Time.utc - start).total_milliseconds + send_json env, { + "milliseconds" => ms, + "titles" => @context.library.titles.size, + }.to_json + end - post "/api/admin/user/delete/:username" do |env| - begin - username = env.params.url["username"] - @context.storage.delete_user username - rescue e - @context.error e - send_json env, { - "success" => false, - "error" => e.message - }.to_json - else - send_json env, {"success" => true}.to_json - end - end + post "/api/admin/user/delete/:username" do |env| + begin + username = env.params.url["username"] + @context.storage.delete_user username + rescue e + @context.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + else + send_json env, {"success" => true}.to_json + end + end - post "/api/progress/:title/:entry/:page" do |env| - begin - username = get_username env - title = (@context.library.get_title env.params.url["title"]) - .not_nil! - entry = (title.get_entry env.params.url["entry"]).not_nil! - page = env.params.url["page"].to_i + post "/api/progress/:title/:page" do |env| + begin + username = get_username env + title = (@context.library.get_title env.params.url["title"]) + .not_nil! + page = env.params.url["page"].to_i + entry_id = env.params.query["entry"]? - raise "incorrect page value" if page < 0 || page > entry.pages - title.save_progress username, entry.title, page - rescue e - @context.error e - send_json env, { - "success" => false, - "error" => e.message - }.to_json - else - send_json env, {"success" => true}.to_json - end - end + if !entry_id.nil? + entry = title.get_entry(entry_id).not_nil! + raise "incorrect page value" if page < 0 || page > entry.pages + title.save_progress username, entry.title, page + elsif page == 0 + title.unread_all username + else + title.read_all username + end + rescue e + @context.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + else + send_json env, {"success" => true}.to_json + end + end - get "/api/admin/mangadex/manga/:id" do |env| - begin - id = env.params.url["id"] - api = MangaDex::API.new \ - @context.config.mangadex["api_url"].to_s - manga = api.get_manga id - send_json env, manga.to_info_json - rescue e - @context.error e - send_json env, {"error" => e.message}.to_json - end - end + post "/api/admin/display_name/:title/:name" do |env| + begin + title = (@context.library.get_title env.params.url["title"]) + .not_nil! + name = env.params.url["name"] + entry = env.params.query["entry"]? + if entry.nil? + title.set_display_name name + else + eobj = title.get_entry entry + title.set_display_name eobj.not_nil!.title, name + end + rescue e + @context.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + else + send_json env, {"success" => true}.to_json + end + end - post "/api/admin/mangadex/download" do |env| - begin - chapters = env.params.json["chapters"].as(Array).map{|c| c.as_h} - jobs = chapters.map {|chapter| - MangaDex::Job.new( - chapter["id"].as_s, - chapter["manga_id"].as_s, - chapter["full_title"].as_s, - chapter["manga_title"].as_s, - MangaDex::JobStatus::Pending, - Time.unix chapter["time"].as_s.to_i - ) - } - inserted_count = @context.queue.push jobs - send_json env, { - "success": inserted_count, - "fail": jobs.size - inserted_count - }.to_json - rescue e - @context.error e - send_json env, {"error" => e.message}.to_json - end - end + get "/api/admin/mangadex/manga/:id" do |env| + begin + id = env.params.url["id"] + api = MangaDex::API.new @context.config.mangadex["api_url"].to_s + manga = api.get_manga id + send_json env, manga.to_info_json + rescue e + @context.error e + send_json env, {"error" => e.message}.to_json + end + end - get "/api/admin/mangadex/queue" do |env| - begin - jobs = @context.queue.get_all - send_json env, { - "jobs" => jobs, - "paused" => @context.queue.paused?, - "success" => true - }.to_json - rescue e - send_json env, { - "success" => false, - "error" => e.message - }.to_json - end - end + post "/api/admin/mangadex/download" do |env| + begin + chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h } + jobs = chapters.map { |chapter| + MangaDex::Job.new( + chapter["id"].as_s, + chapter["manga_id"].as_s, + chapter["full_title"].as_s, + chapter["manga_title"].as_s, + MangaDex::JobStatus::Pending, + Time.unix chapter["time"].as_s.to_i + ) + } + inserted_count = @context.queue.push jobs + send_json env, { + "success": inserted_count, + "fail": jobs.size - inserted_count, + }.to_json + rescue e + @context.error e + send_json env, {"error" => e.message}.to_json + end + end - post "/api/admin/mangadex/queue/:action" do |env| - begin - action = env.params.url["action"] - id = env.params.query["id"]? - case action - when "delete" - if id.nil? - @context.queue.delete_status MangaDex::JobStatus::Completed - else - @context.queue.delete id - end - when "retry" - if id.nil? - @context.queue.reset - else - @context.queue.reset id - end - when "pause" - @context.queue.pause - when "resume" - @context.queue.resume - else - raise "Unknown queue action #{action}" - end + get "/api/admin/mangadex/queue" do |env| + begin + jobs = @context.queue.get_all + send_json env, { + "jobs" => jobs, + "paused" => @context.queue.paused?, + "success" => true, + }.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end - send_json env, {"success" => true}.to_json - rescue e - send_json env, { - "success" => false, - "error" => e.message - }.to_json - end - end - end + post "/api/admin/mangadex/queue/:action" do |env| + begin + action = env.params.url["action"] + id = env.params.query["id"]? + case action + when "delete" + if id.nil? + @context.queue.delete_status MangaDex::JobStatus::Completed + else + @context.queue.delete id + end + when "retry" + if id.nil? + @context.queue.reset + else + @context.queue.reset id + end + when "pause" + @context.queue.pause + when "resume" + @context.queue.resume + else + raise "Unknown queue action #{action}" + end + + send_json env, {"success" => true}.to_json + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + post "/api/admin/upload/:target" do |env| + begin + target = env.params.url["target"] + + HTTP::FormData.parse env.request do |part| + next if part.name != "file" + + filename = part.filename + if filename.nil? + raise "No file uploaded" + end + + case target + when "cover" + title_id = env.params.query["title"] + entry_id = env.params.query["entry"]? + title = @context.library.get_title(title_id).not_nil! + + unless ["image/jpeg", "image/png"].includes? \ + MIME.from_filename? filename + raise "The uploaded image must be either JPEG or PNG" + end + + ext = File.extname filename + upload = Upload.new @context.config.upload_path, @context.logger + url = upload.path_to_url upload.save "img", ext, part.body + + if url.nil? + raise "Failed to generate a public URL for the uploaded file" + end + + if entry_id.nil? + title.set_cover_url url + else + entry_name = title.get_entry(entry_id).not_nil!.title + title.set_cover_url entry_name, url + end + else + raise "Unkown upload target #{target}" + end + + send_json env, {"success" => true}.to_json + env.response.close + end + + raise "No part with name `file` found" + rescue e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + end end diff --git a/src/routes/main.cr b/src/routes/main.cr index ae3d586..3e21fc1 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -1,63 +1,66 @@ require "./router" class MainRouter < Router - def setup - get "/login" do |env| - render "src/views/login.ecr" - end + def setup + get "/login" do |env| + render "src/views/login.ecr" + end - get "/logout" do |env| - begin - cookie = env.request.cookies - .find { |c| c.name == "token" }.not_nil! - @context.storage.logout cookie.value - rescue e - @context.error "Error when attempting to log out: #{e}" - ensure - env.redirect "/login" - end - end + get "/logout" do |env| + begin + cookie = env.request.cookies.find { |c| c.name == "token" }.not_nil! + @context.storage.logout cookie.value + rescue e + @context.error "Error when attempting to log out: #{e}" + ensure + env.redirect "/login" + end + end - post "/login" do |env| - begin - username = env.params.body["username"] - password = env.params.body["password"] - token = @context.storage.verify_user(username, password) - .not_nil! + post "/login" do |env| + begin + username = env.params.body["username"] + password = env.params.body["password"] + token = @context.storage.verify_user(username, password).not_nil! - cookie = HTTP::Cookie.new "token", token - cookie.expires = Time.local.shift years: 1 - env.response.cookies << cookie - env.redirect "/" - rescue - env.redirect "/login" - end - end + cookie = HTTP::Cookie.new "token", token + cookie.expires = Time.local.shift years: 1 + env.response.cookies << cookie + env.redirect "/" + rescue + env.redirect "/login" + end + end - get "/" do |env| - titles = @context.library.titles - username = get_username env - percentage = titles.map &.load_percetage username - layout "index" - end + get "/" do |env| + begin + titles = @context.library.titles + username = get_username env + percentage = titles.map &.load_percetage username + layout "index" + rescue e + @context.error e + env.response.status_code = 500 + end + end - get "/book/:title" do |env| - begin - title = (@context.library.get_title env.params.url["title"]) - .not_nil! - username = get_username env - percentage = title.entries.map { |e| - title.load_percetage username, e.title } - layout "title" - rescue e - @context.error e - env.response.status_code = 404 - end - end + get "/book/:title" do |env| + begin + title = (@context.library.get_title env.params.url["title"]).not_nil! + username = get_username env + percentage = title.entries.map { |e| + title.load_percetage username, e.title + } + layout "title" + rescue e + @context.error e + env.response.status_code = 404 + end + end - get "/download" do |env| - base_url = @context.config.mangadex["base_url"]; - layout "download" - end - end + get "/download" do |env| + base_url = @context.config.mangadex["base_url"] + layout "download" + end + end end diff --git a/src/routes/reader.cr b/src/routes/reader.cr index 266396da..49d9a27 100644 --- a/src/routes/reader.cr +++ b/src/routes/reader.cr @@ -1,58 +1,61 @@ require "./router" class ReaderRouter < Router - def setup - get "/reader/:title/:entry" do |env| - begin - title = (@context.library.get_title env.params.url["title"]) - .not_nil! - entry = (title.get_entry env.params.url["entry"]).not_nil! + def setup + get "/reader/:title/:entry" do |env| + begin + title = (@context.library.get_title env.params.url["title"]).not_nil! + entry = (title.get_entry env.params.url["entry"]).not_nil! - # load progress - username = get_username env - page = title.load_progress username, entry.title - # we go back 2 * `IMGS_PER_PAGE` pages. the infinite scroll - # library perloads a few pages in advance, and the user - # might not have actually read them - page = [page - 2 * IMGS_PER_PAGE, 1].max + # load progress + username = get_username env + page = title.load_progress username, entry.title + # we go back 2 * `IMGS_PER_PAGE` pages. the infinite scroll + # library perloads a few pages in advance, and the user + # might not have actually read them + page = [page - 2 * IMGS_PER_PAGE, 1].max - env.redirect "/reader/#{title.id}/#{entry.id}/#{page}" - rescue e - @context.error e - env.response.status_code = 404 - end - end + env.redirect "/reader/#{title.id}/#{entry.id}/#{page}" + rescue e + @context.error e + env.response.status_code = 404 + end + end - get "/reader/:title/:entry/:page" do |env| - begin - title = (@context.library.get_title env.params.url["title"]) - .not_nil! - entry = (title.get_entry env.params.url["entry"]).not_nil! - page = env.params.url["page"].to_i - raise "" if page > entry.pages || page <= 0 + get "/reader/:title/:entry/:page" do |env| + begin + title = (@context.library.get_title env.params.url["title"]).not_nil! + entry = (title.get_entry env.params.url["entry"]).not_nil! + page = env.params.url["page"].to_i + raise "" if page > entry.pages || page <= 0 - # save progress - username = get_username env - title.save_progress username, entry.title, page + # save progress + username = get_username env + title.save_progress username, entry.title, page - pages = (page...[entry.pages + 1, page + IMGS_PER_PAGE].min) - urls = pages.map { |idx| - "/api/page/#{title.id}/#{entry.id}/#{idx}" } - reader_urls = pages.map { |idx| - "/reader/#{title.id}/#{entry.id}/#{idx}" } - next_page = page + IMGS_PER_PAGE - next_url = next_page > entry.pages ? nil : - "/reader/#{title.id}/#{entry.id}/#{next_page}" - exit_url = "/book/#{title.id}" - next_entry = title.next_entry entry - next_entry_url = next_entry.nil? ? nil : \ - "/reader/#{title.id}/#{next_entry.id}" + pages = (page...[entry.pages + 1, page + IMGS_PER_PAGE].min) + urls = pages.map { |idx| + "/api/page/#{title.id}/#{entry.id}/#{idx}" + } + reader_urls = pages.map { |idx| + "/reader/#{title.id}/#{entry.id}/#{idx}" + } + next_page = page + IMGS_PER_PAGE + next_url = next_entry_url = nil + exit_url = "/book/#{title.id}" + next_entry = title.next_entry entry + unless next_page > entry.pages + next_url = "/reader/#{title.id}/#{entry.id}/#{next_page}" + end + unless next_entry.nil? + next_entry_url = "/reader/#{title.id}/#{next_entry.id}" + end - render "src/views/reader.ecr" - rescue e - @context.error e - env.response.status_code = 404 - end - end - end + render "src/views/reader.ecr" + rescue e + @context.error e + env.response.status_code = 404 + end + end + end end diff --git a/src/routes/router.cr b/src/routes/router.cr index 16389fb..97fb698 100644 --- a/src/routes/router.cr +++ b/src/routes/router.cr @@ -1,6 +1,6 @@ require "../context" class Router - def initialize(@context : Context) - end + def initialize(@context : Context) + end end diff --git a/src/server.cr b/src/server.cr index e31e10a..fef8412 100644 --- a/src/server.cr +++ b/src/server.cr @@ -1,50 +1,47 @@ require "kemal" require "./context" -require "./auth_handler" -require "./static_handler" -require "./log_handler" +require "./handlers/*" require "./util" require "./routes/*" class Server - def initialize(@context : Context) + def initialize(@context : Context) + error 403 do |env| + message = "HTTP 403: You are not authorized to visit #{env.request.path}" + layout "message" + end + error 404 do |env| + message = "HTTP 404: Mango cannot find the page #{env.request.path}" + layout "message" + end + error 500 do |env| + message = "HTTP 500: Internal server error. Please try again later." + layout "message" + end - error 403 do |env| - message = "HTTP 403: You are not authorized to visit " \ - "#{env.request.path}" - layout "message" - end - error 404 do |env| - message = "HTTP 404: Mango cannot find the page #{env.request.path}" - layout "message" - end - error 500 do |env| - message = "HTTP 500: Internal server error. Please try again later." - layout "message" - end + MainRouter.new(@context).setup + AdminRouter.new(@context).setup + ReaderRouter.new(@context).setup + APIRouter.new(@context).setup - MainRouter.new(@context).setup - AdminRouter.new(@context).setup - ReaderRouter.new(@context).setup - APIRouter.new(@context).setup + Kemal.config.logging = false + add_handler LogHandler.new @context.logger + add_handler AuthHandler.new @context.storage + add_handler UploadHandler.new @context.config.upload_path + {% if flag?(:release) %} + # when building for relase, embed the static files in binary + @context.debug "We are in release mode. Using embedded static files." + serve_static false + add_handler StaticHandler.new + {% end %} + end - Kemal.config.logging = false - add_handler LogHandler.new @context.logger - add_handler AuthHandler.new @context.storage - {% if flag?(:release) %} - # when building for relase, embed the static files in binary - @context.debug "We are in release mode. Using embedded static files." - serve_static false - add_handler StaticHandler.new - {% end %} - end - - def start - @context.debug "Starting Kemal server" - {% if flag?(:release) %} - Kemal.config.env = "production" - {% end %} - Kemal.config.port = @context.config.port - Kemal.run - end + def start + @context.debug "Starting Kemal server" + {% if flag?(:release) %} + Kemal.config.env = "production" + {% end %} + Kemal.config.port = @context.config.port + Kemal.run + end end diff --git a/src/static_handler.cr b/src/static_handler.cr deleted file mode 100644 index f39cd46..0000000 --- a/src/static_handler.cr +++ /dev/null @@ -1,32 +0,0 @@ -require "baked_file_system" -require "kemal" -require "./util" - -class FS - extend BakedFileSystem - {% if flag?(:release) %} - {% if read_file? "#{__DIR__}/../dist/favicon.ico" %} - {% puts "baking ../dist" %} - bake_folder "../dist" - {% else %} - {% puts "baking ../public" %} - bake_folder "../public" - {% end %} - {% end %} -end - -class StaticHandler < Kemal::Handler - @dirs = ["/css", "/js", "/img", "/favicon.ico"] - - def call(env) - if request_path_startswith env, @dirs - file = FS.get? env.request.path - return call_next env if file.nil? - - slice = Bytes.new file.size - file.read slice - return send_file env, slice, file.mime_type - end - call_next env - end -end diff --git a/src/storage.cr b/src/storage.cr index 8d20e15..382709f 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -2,176 +2,171 @@ require "sqlite3" require "crypto/bcrypt" require "uuid" require "base64" +require "./util" def hash_password(pw) - Crypto::Bcrypt::Password.create(pw).to_s + Crypto::Bcrypt::Password.create(pw).to_s end def verify_password(hash, pw) - (Crypto::Bcrypt::Password.new hash).verify pw -end - -def random_str - UUID.random.to_s.gsub "-", "" + (Crypto::Bcrypt::Password.new hash).verify pw end class Storage - def initialize(@path : String, @logger : MLogger) - dir = File.dirname path - unless Dir.exists? dir - @logger.info "The DB directory #{dir} does not exist. " \ - "Attepmting to create it" - Dir.mkdir_p dir - end - DB.open "sqlite3://#{path}" do |db| - begin - # We create the `ids` table first. even if the uses has an - # early version installed and has the `user` table only, - # we will still be able to create `ids` - db.exec "create table ids" \ - "(path text, id text, is_title integer)" - db.exec "create unique index path_idx on ids (path)" - db.exec "create unique index id_idx on ids (id)" + def initialize(@path : String, @logger : Logger) + dir = File.dirname path + unless Dir.exists? dir + @logger.info "The DB directory #{dir} does not exist. " \ + "Attepmting to create it" + Dir.mkdir_p dir + end + DB.open "sqlite3://#{path}" do |db| + begin + # We create the `ids` table first. even if the uses has an + # early version installed and has the `user` table only, + # we will still be able to create `ids` + db.exec "create table ids" \ + "(path text, id text, is_title integer)" + db.exec "create unique index path_idx on ids (path)" + db.exec "create unique index id_idx on ids (id)" - db.exec "create table users" \ - "(username text, password text, token text, admin integer)" - rescue e - unless e.message.not_nil!.ends_with? "already exists" - @logger.fatal "Error when checking tables in DB: #{e}" - raise e - end - else - @logger.debug "Creating DB file at #{@path}" - db.exec "create unique index username_idx on users (username)" - db.exec "create unique index token_idx on users (token)" - random_pw = random_str - hash = hash_password random_pw - db.exec "insert into users values (?, ?, ?, ?)", - "admin", hash, nil, 1 - puts "Initial user created. You can log in with " \ - "#{{"username" => "admin", "password" => random_pw}}" - end - end - end + db.exec "create table users" \ + "(username text, password text, token text, admin integer)" + rescue e + unless e.message.not_nil!.ends_with? "already exists" + @logger.fatal "Error when checking tables in DB: #{e}" + raise e + end + else + @logger.debug "Creating DB file at #{@path}" + db.exec "create unique index username_idx on users (username)" + db.exec "create unique index token_idx on users (token)" + random_pw = random_str + hash = hash_password random_pw + db.exec "insert into users values (?, ?, ?, ?)", + "admin", hash, nil, 1 + @logger.log "Initial user created. You can log in with " \ + "#{{"username" => "admin", "password" => random_pw}}" + end + end + end - def verify_user(username, password) - DB.open "sqlite3://#{@path}" do |db| - begin - hash, token = db.query_one "select password, token from "\ - "users where username = (?)", \ - username, as: {String, String?} - unless verify_password hash, password - @logger.debug "Password does not match the hash" - return nil - end - @logger.debug "User #{username} verified" - return token if token - token = random_str - @logger.debug "Updating token for #{username}" - db.exec "update users set token = (?) where username = (?)", - token, username - return token - rescue e - @logger.error "Error when verifying user #{username}: #{e}" - return nil - end - end - end + def verify_user(username, password) + DB.open "sqlite3://#{@path}" do |db| + begin + hash, token = db.query_one "select password, token from " \ + "users where username = (?)", + username, as: {String, String?} + unless verify_password hash, password + @logger.debug "Password does not match the hash" + return nil + end + @logger.debug "User #{username} verified" + return token if token + token = random_str + @logger.debug "Updating token for #{username}" + db.exec "update users set token = (?) where username = (?)", + token, username + return token + rescue e + @logger.error "Error when verifying user #{username}: #{e}" + return nil + end + end + end - def verify_token(token) - DB.open "sqlite3://#{@path}" do |db| - begin - username = db.query_one "select username from users where " \ - "token = (?)", token, as: String - return username - rescue e - @logger.debug "Unable to verify token" - return nil - end - end - end + def verify_token(token) + username = nil + DB.open "sqlite3://#{@path}" do |db| + begin + username = db.query_one "select username from users where " \ + "token = (?)", token, as: String + rescue e + @logger.debug "Unable to verify token" + end + end + username + end - def verify_admin(token) - DB.open "sqlite3://#{@path}" do |db| - begin - return db.query_one "select admin from users where " \ - "token = (?)", token, as: Bool - rescue e - @logger.debug "Unable to verify user as admin" - return false - end - end - end + def verify_admin(token) + is_admin = false + DB.open "sqlite3://#{@path}" do |db| + begin + is_admin = db.query_one "select admin from users where " \ + "token = (?)", token, as: Bool + rescue e + @logger.debug "Unable to verify user as admin" + end + end + is_admin + end - def list_users - results = Array(Tuple(String, Bool)).new - DB.open "sqlite3://#{@path}" do |db| - db.query "select username, admin from users" do |rs| - rs.each do - results << {rs.read(String), rs.read(Bool)} - end - end - end - results - end + def list_users + results = Array(Tuple(String, Bool)).new + DB.open "sqlite3://#{@path}" do |db| + db.query "select username, admin from users" do |rs| + rs.each do + results << {rs.read(String), rs.read(Bool)} + end + end + end + results + end - def new_user(username, password, admin) - admin = (admin ? 1 : 0) - DB.open "sqlite3://#{@path}" do |db| - hash = hash_password password - db.exec "insert into users values (?, ?, ?, ?)", - username, hash, nil, admin - end - end + def new_user(username, password, admin) + admin = (admin ? 1 : 0) + DB.open "sqlite3://#{@path}" do |db| + hash = hash_password password + db.exec "insert into users values (?, ?, ?, ?)", + username, hash, nil, admin + end + end - def update_user(original_username, username, password, admin) - admin = (admin ? 1 : 0) - DB.open "sqlite3://#{@path}" do |db| - if password.size == 0 - db.exec "update users set username = (?), admin = (?) "\ - "where username = (?)",\ - username, admin, original_username - else - hash = hash_password password - db.exec "update users set username = (?), admin = (?),"\ - "password = (?) where username = (?)",\ - username, admin, hash, original_username - end - end - end + def update_user(original_username, username, password, admin) + admin = (admin ? 1 : 0) + DB.open "sqlite3://#{@path}" do |db| + if password.size == 0 + db.exec "update users set username = (?), admin = (?) " \ + "where username = (?)", + username, admin, original_username + else + hash = hash_password password + db.exec "update users set username = (?), admin = (?)," \ + "password = (?) where username = (?)", + username, admin, hash, original_username + end + end + end - def delete_user(username) - DB.open "sqlite3://#{@path}" do |db| - db.exec "delete from users where username = (?)", username - end - end + def delete_user(username) + DB.open "sqlite3://#{@path}" do |db| + db.exec "delete from users where username = (?)", username + end + end - def logout(token) - DB.open "sqlite3://#{@path}" do |db| - begin - db.exec "update users set token = (?) where token = (?)", \ - nil, token - rescue - end - end - end + def logout(token) + DB.open "sqlite3://#{@path}" do |db| + begin + db.exec "update users set token = (?) where token = (?)", nil, token + rescue + end + end + end - def get_id(path, is_title) - DB.open "sqlite3://#{@path}" do |db| - begin - id = db.query_one "select id from ids where path = (?)", - path, as: {String} - return id - rescue - id = random_str - db.exec "insert into ids values (?, ?, ?)", path, id, - is_title ? 1 : 0 - return id - end - end - end + def get_id(path, is_title) + id = random_str + DB.open "sqlite3://#{@path}" do |db| + begin + id = db.query_one "select id from ids where path = (?)", path, + as: {String} + rescue + db.exec "insert into ids values (?, ?, ?)", path, id, is_title ? 1 : 0 + end + end + id + end - def to_json(json : JSON::Builder) - json.string self - end + def to_json(json : JSON::Builder) + json.string self + end end diff --git a/src/upload.cr b/src/upload.cr new file mode 100644 index 0000000..c6d27be --- /dev/null +++ b/src/upload.cr @@ -0,0 +1,60 @@ +require "./util" + +class Upload + def initialize(@dir : String, @logger : Logger) + unless Dir.exists? @dir + @logger.info "The uploads directory #{@dir} does not exist. " \ + "Attempting to create it" + Dir.mkdir_p @dir + end + end + + # Writes IO to a file with random filename in the uploads directory and + # returns the full path of created file + # e.g., save("image", ".png", ) + # ==> "~/mango/uploads/image/.png" + def save(sub_dir : String, ext : String, io : IO) + full_dir = File.join @dir, sub_dir + filename = random_str + ext + file_path = File.join full_dir, filename + + unless Dir.exists? full_dir + @logger.debug "creating directory #{full_dir}" + Dir.mkdir_p full_dir + end + + File.open file_path, "w" do |f| + IO.copy io, f + end + + file_path + end + + # Converts path to a file in the uploads directory to the URL path for + # accessing the file. + def path_to_url(path : String) + dir_mathed = false + ary = [] of String + # We fill it with parts until it equals to @upload_dir + dir_ary = [] of String + + Path.new(path).each_part do |part| + if dir_mathed + ary << part + else + dir_ary << part + if File.same? @dir, File.join dir_ary + dir_mathed = true + end + end + end + + if ary.empty? + @logger.warn "File #{path} is not in the upload directory #{@dir}" + return + end + + ary.unshift UPLOAD_URL_PREFIX + File.join(ary).to_s + end +end diff --git a/src/util.cr b/src/util.cr index ca7b08d..bc97887 100644 --- a/src/util.cr +++ b/src/util.cr @@ -1,73 +1,101 @@ require "big" -IMGS_PER_PAGE = 5 +IMGS_PER_PAGE = 5 +UPLOAD_URL_PREFIX = "/uploads" macro layout(name) - render "src/views/#{{{name}}}.ecr", "src/views/layout.ecr" + begin + cookie = env.request.cookies.find { |c| c.name == "token" } + is_admin = false + unless cookie.nil? + is_admin = @context.storage.verify_admin cookie.value + end + render "src/views/#{{{name}}}.ecr", "src/views/layout.ecr" + rescue e + message = e.to_s + @context.error message + render "src/views/message.ecr", "src/views/layout.ecr" + end end macro send_img(env, img) - send_file {{env}}, {{img}}.data, {{img}}.mime + send_file {{env}}, {{img}}.data, {{img}}.mime end macro get_username(env) - # if the request gets here, it has gone through the auth handler, and - # we can be sure that a valid token exists, so we can use not_nil! here - cookie = {{env}}.request.cookies.find { |c| c.name == "token" }.not_nil! - (@context.storage.verify_token cookie.value).not_nil! + # if the request gets here, it has gone through the auth handler, and + # we can be sure that a valid token exists, so we can use not_nil! here + cookie = {{env}}.request.cookies.find { |c| c.name == "token" }.not_nil! + (@context.storage.verify_token cookie.value).not_nil! end -macro send_json(env, json) - {{env}}.response.content_type = "application/json" - {{json}} +def send_json(env, json) + env.response.content_type = "application/json" + env.response.print json end def hash_to_query(hash) - hash.map { |k, v| "#{k}=#{v}" }.join("&") + hash.map { |k, v| "#{k}=#{v}" }.join("&") end def request_path_startswith(env, ary) - ary.each do |prefix| - if env.request.path.starts_with? prefix - return true - end - end - return false + ary.each do |prefix| + if env.request.path.starts_with? prefix + return true + end + end + false end def is_numeric(str) - /^\d+/.match(str) != nil + /^\d+/.match(str) != nil end def split_by_alphanumeric(str) - arr = [] of String - str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match| - arr += match.captures.select{|s| s != ""} - end - arr + arr = [] of String + str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match| + arr += match.captures.select { |s| s != "" } + end + arr end def compare_alphanumerically(c, d) - is_c_bigger = c.size <=> d.size - if c.size > d.size - d += [nil] * (c.size - d.size) - elsif c.size < d.size - c += [nil] * (d.size - c.size) - end - c.zip(d) do |a, b| - return -1 if a.nil? - return 1 if b.nil? - if is_numeric(a) && is_numeric(b) - compare = a.to_big_i <=> b.to_big_i - return compare if compare != 0 - else - compare = a <=> b - return compare if compare != 0 - end - end - is_c_bigger + is_c_bigger = c.size <=> d.size + if c.size > d.size + d += [nil] * (c.size - d.size) + elsif c.size < d.size + c += [nil] * (d.size - c.size) + end + c.zip(d) do |a, b| + return -1 if a.nil? + return 1 if b.nil? + if is_numeric(a) && is_numeric(b) + compare = a.to_big_i <=> b.to_big_i + return compare if compare != 0 + else + compare = a <=> b + return compare if compare != 0 + end + end + is_c_bigger end def compare_alphanumerically(a : String, b : String) - compare_alphanumerically split_by_alphanumeric(a), split_by_alphanumeric(b) + compare_alphanumerically split_by_alphanumeric(a), split_by_alphanumeric(b) +end + +# When downloading from MangaDex, the zip/cbz file would not be valid +# before the download is completed. If we scan the zip file, +# Entry.new would throw, so we use this method to check before +# constructing Entry +def validate_zip(path : String) : Exception? + file = Zip::File.new path + file.close + return +rescue e + e +end + +def random_str + UUID.random.to_s.gsub "-", "" end diff --git a/src/views/index.ecr b/src/views/index.ecr index b7a9727..63508bb 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -26,17 +26,13 @@
- <%- if t.entries.size > 0 -%> - - <%- else -%> - - <%- end -%> +
<%- if t.entries.size > 0 -%>
<%= (percentage[i] * 100).round(1) %>%
<%- end -%> -

"><%= t.title %>

+

"><%= t.display_name %>

<%= t.size %> entries

diff --git a/src/views/layout.ecr b/src/views/layout.ecr index 68d0498..cbd6ead 100644 --- a/src/views/layout.ecr +++ b/src/views/layout.ecr @@ -21,8 +21,10 @@
diff --git a/src/views/login.ecr b/src/views/login.ecr index ff56165..80afeb6 100644 --- a/src/views/login.ecr +++ b/src/views/login.ecr @@ -8,9 +8,10 @@ + + -
@@ -33,6 +34,9 @@
+ diff --git a/src/views/title.ecr b/src/views/title.ecr index ef502df..c6ade3a 100644 --- a/src/views/title.ecr +++ b/src/views/title.ecr @@ -1,10 +1,17 @@ -

<%= title.title %>

+
+

<%= title.display_name %> +   + <% if is_admin %> + + <% end %> +

+

<%= title.size %> entries found

@@ -35,14 +42,10 @@
- <%- if t.entries.size > 0 -%> - - <%- else -%> - - <%- end -%> +
-

"><%= t.title %>

+

"><%= t.display_name %>

<%= t.size %> entries

@@ -50,15 +53,15 @@
<%- end -%> <%- title.entries.each_with_index do |e, i| -%> -
+
-
+
<%= (percentage[i] * 100).round(1) %>%
-

"><%= e.title %>

+

"><%= e.display_name %>

<%= e.pages %> pages

@@ -71,7 +74,14 @@
- +

@@ -90,6 +100,55 @@
+
+
+ +
+
+ +
+
+
+
+ +
+ + +
+
+
+ +
+
+ +
+
+
+
+ + Upload a cover image by dropping it here or +
+ + selecting one +
+
+
+ +
+
+ +
+ +
+
+
+ <% content_for "script" do %>