Finish thumbnail generation and DB optimization

(#93)

src/library/entry.cr

@@ -213,11 +213,17 @@ class Entry
 
     img = read_page(1).not_nil!
     begin
-      thumbnail = ImageSize.resize img.data, height: 300
+      size = ImageSize.get img.data
+      if size.height > size.width
+        thumbnail = ImageSize.resize img.data, width: 200
+      else
+        thumbnail = ImageSize.resize img.data, height: 300
+      end
       img.data = thumbnail
       Storage.default.save_thumbnail @id, img
     rescue e
-      Logger.warn "Failed to generate thumbnail for entry #{@id}. #{e}"
+      Logger.warn "Failed to generate thumbnail for entry " \
+                  "#{@book.title}/#{@title}. #{e}"
     end
 
     img

src/library/library.cr

@@ -1,7 +1,6 @@
 class Library
-  property dir : String, title_ids : Array(String), scan_interval : Int32,
+  property dir : String, title_ids : Array(String),
-    title_hash : Hash(String, Title), entries_count = 0,
+    title_hash : Hash(String, Title), entries_count = 0, thumbnails_count = 0
-    thumbnails_count = 0
 
   use_default
 
@@ -9,20 +8,45 @@ class Library
     register_mime_types
 
     @dir = Config.current.library_path
-    @scan_interval = Config.current.scan_interval
     # explicitly initialize @titles to bypass the compiler check. it will
     #   be filled with actual Titles in the `scan` call below
     @title_ids = [] of String
     @title_hash = {} of String => Title
 
-    return scan if @scan_interval < 1
+    scan_interval = Config.current.scan_interval_minutes
-    spawn do
+    if scan_interval < 1
-      loop do
+      scan
-        start = Time.local
+    else
-        scan
+      spawn do
-        ms = (Time.local - start).total_milliseconds
+        loop do
-        Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
+          start = Time.local
-        sleep @scan_interval * 60
+          scan
+          ms = (Time.local - start).total_milliseconds
+          Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
+          sleep scan_interval.minutes
+        end
+      end
+    end
+
+    thumbnail_interval = Config.current.thumbnail_generation_interval_hours
+    unless thumbnail_interval < 1
+      spawn do
+        loop do
+          # Wait for scan to complete (in most cases)
+          sleep 1.minutes
+          generate_thumbnails
+          sleep thumbnail_interval.hours
+        end
+      end
+    end
+
+    db_interval = Config.current.db_optimization_interval_hours
+    unless db_interval < 1
+      spawn do
+        loop do
+          Storage.default.optimize
+          sleep db_interval.hours
+        end
       end
     end
   end
@@ -195,4 +219,38 @@ class Library
       .sample(ENTRIES_IN_HOME_SECTIONS)
       .shuffle
   end
+
+  def thumbnail_generation_progress
+    @thumbnails_count / @entries_count
+  end
+
+  def generate_thumbnails
+    Logger.info "Starting thumbnail generation"
+    entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg
+    @entries_count = entries.size
+    @thumbnails_count = 0
+
+    # Report generation progress regularly
+    spawn do
+      loop do
+        break if thumbnail_generation_progress.to_i == 1
+        Logger.debug "Thumbnail generation progress: " \
+                     "#{(thumbnail_generation_progress * 100).round 1}%"
+        sleep 30.seconds
+      end
+    end
+
+    entries.each do |e|
+      unless e.get_thumbnail
+        e.generate_thumbnail
+        # Sleep after each generation to minimize the impact on disk IO
+        #   and CPU
+        sleep 0.5.seconds
+      end
+      @thumbnails_count += 1
+    end
+    Logger.info "Thumbnail generation finished. " \
+                "#{@thumbnails_count}/#{@entries_count} " \
+                "thumbnails generated"
+  end
 end

src/storage.cr

@@ -266,6 +266,29 @@ class Storage
     img
   end
 
+  def optimize
+    MainFiber.run do
+      Logger.info "Starting DB optimization"
+      get_db do |db|
+        trash_ids = [] of String
+        db.query "select path, id from ids" do |rs|
+          rs.each do
+            path = rs.read String
+            trash_ids << rs.read String unless File.exists? path
+          end
+        end
+
+        # Delete dangling IDs
+        db.exec "delete from ids where id in " \
+                "(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
+
+        # Delete dangling thumbnails
+        db.exec "delete from thumbnails where id not in (select id from ids)"
+      end
+      Logger.info "DB optimization finished"
+    end
+  end
+
   def close
     MainFiber.run do
       unless @db.nil?