Improve scan performance (#79)

src/library.cr

@@ -33,7 +33,16 @@ class Entry
         MIME.from_filename? e.filename
     end
     file.close
-    @id = storage.get_id @zip_path, false
+    id = storage.get_id @zip_path, false
+    if id.nil?
+      id = random_str
+      storage.insert_id({
+        path:     @zip_path,
+        id:       id,
+        is_title: false,
+      })
+    end
+    @id = id
     @mtime = File.info(@zip_path).modification_time
   end
 
@@ -178,7 +187,16 @@ class Title
 
   def initialize(@dir : String, @parent_id, storage,
                  @library : Library)
-    @id = storage.get_id @dir, true
+    id = storage.get_id @dir, true
+    if id.nil?
+      id = random_str
+      storage.insert_id({
+        path:     @dir,
+        id:       id,
+        is_title: true,
+      })
+    end
+    @id = id
     @title = File.basename dir
     @encoded_title = URI.encode @title
     @title_ids = [] of String
@@ -525,7 +543,7 @@ end
 
 class Library
   property dir : String, title_ids : Array(String), scan_interval : Int32,
-    storage : Storage, title_hash : Hash(String, Title)
+    title_hash : Hash(String, Title)
 
   def self.default : self
     unless @@default
@@ -535,7 +553,6 @@ class Library
   end
 
   def initialize
-    @storage = Storage.default
     register_mime_types
 
     @dir = Config.current.library_path
@@ -589,17 +606,24 @@ class Library
       Dir.mkdir_p @dir
     end
     @title_ids.clear
+
+    storage = Storage.new auto_close: false
+
     (Dir.entries @dir)
       .select { |fn| !fn.starts_with? "." }
       .map { |fn| File.join @dir, fn }
       .select { |path| File.directory? path }
-      .map { |path| Title.new path, "", @storage, self }
+      .map { |path| Title.new path, "", storage, self }
       .select { |title| !(title.entries.empty? && title.titles.empty?) }
       .sort { |a, b| a.title <=> b.title }
       .each do |title|
         @title_hash[title.id] = title
         @title_ids << title.id
       end
+
+    storage.bulk_insert_ids
+    storage.close
+
     Logger.debug "Scan completed"
   end
 

src/storage.cr

@@ -14,6 +14,12 @@ end
 
 class Storage
   @path : String
+  @db : DB::Database?
+  @insert_ids = [] of IDTuple
+
+  alias IDTuple = NamedTuple(path: String,
+    id: String,
+    is_title: Bool)
 
   def self.default : self
     unless @@default
@@ -22,7 +28,8 @@ class Storage
     @@default.not_nil!
   end
 
-  def initialize(db_path : String? = nil, init_user = true)
+  def initialize(db_path : String? = nil, init_user = true, *,
+                 @auto_close = true)
     @path = db_path || Config.current.db_path
     dir = File.dirname @path
     unless Dir.exists? dir
@@ -60,6 +67,9 @@ class Storage
         init_admin if init_user
       end
     end
+    unless @auto_close
+      @db = DB.open "sqlite3://#{@path}"
+    end
   end
 
   macro init_admin
@@ -71,8 +81,18 @@ class Storage
                "#{{"username" => "admin", "password" => random_pw}}"
   end
 
+  private def get_db(&block : DB::Database ->)
+    if @db.nil?
+      DB.open "sqlite3://#{@path}" do |db|
+        yield db
+      end
+    else
+      yield @db.not_nil!
+    end
+  end
+
   def verify_user(username, password)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         hash, token = db.query_one "select password, token from " \
                                    "users where username = (?)",
@@ -97,7 +117,7 @@ class Storage
 
   def verify_token(token)
     username = nil
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         username = db.query_one "select username from users where " \
                                 "token = (?)", token, as: String
@@ -110,7 +130,7 @@ class Storage
 
   def verify_admin(token)
     is_admin = false
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         is_admin = db.query_one "select admin from users where " \
                                 "token = (?)", token, as: Bool
@@ -123,7 +143,7 @@ class Storage
 
   def list_users
     results = Array(Tuple(String, Bool)).new
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       db.query "select username, admin from users" do |rs|
         rs.each do
           results << {rs.read(String), rs.read(Bool)}
@@ -137,7 +157,7 @@ class Storage
     validate_username username
     validate_password password
     admin = (admin ? 1 : 0)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       hash = hash_password password
       db.exec "insert into users values (?, ?, ?, ?)",
         username, hash, nil, admin
@@ -148,7 +168,7 @@ class Storage
     admin = (admin ? 1 : 0)
     validate_username username
     validate_password password unless password.empty?
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       if password.empty?
         db.exec "update users set username = (?), admin = (?) " \
                 "where username = (?)",
@@ -163,13 +183,13 @@ class Storage
   end
 
   def delete_user(username)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       db.exec "delete from users where username = (?)", username
     end
   end
 
   def logout(token)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         db.exec "update users set token = (?) where token = (?)", nil, token
       rescue
@@ -178,18 +198,36 @@ class Storage
   end
 
   def get_id(path, is_title)
-    id = random_str
+    id = nil
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
-      begin
+      id = db.query_one? "select id from ids where path = (?)", path,
-        id = db.query_one "select id from ids where path = (?)", path,
+        as: {String}
-          as: {String}
-      rescue
-        db.exec "insert into ids values (?, ?, ?)", path, id, is_title ? 1 : 0
-      end
     end
     id
   end
 
+  def insert_id(tp : IDTuple)
+    @insert_ids << tp
+  end
+
+  def bulk_insert_ids
+    get_db do |db|
+      db.transaction do |tx|
+        @insert_ids.each do |tp|
+          tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path],
+            tp[:id], tp[:is_title] ? 1 : 0
+        end
+      end
+    end
+    @insert_ids.clear
+  end
+
+  def close
+    unless @db.nil?
+      @db.not_nil!.close
+    end
+  end
+
   def to_json(json : JSON::Builder)
     json.string self
   end