refactor: use OPFS sqlite for search

src/electron/electron/core.cljs

@@ -1,6 +1,5 @@
 (ns electron.core
   (:require [electron.handler :as handler]
-            [electron.search :as search]
             [electron.db :as db]
             [electron.updater :refer [init-updater] :as updater]
             [electron.utils :refer [*win mac? linux? dev? get-win-from-sender
@@ -255,11 +254,8 @@
 
            (js-utils/disableXFrameOptions win)
 
-           (search/ensure-search-dir!)
            (db/ensure-graphs-dir!)
 
-           (search/open-dbs!)
-
            (git/auto-commit-current-graph!)
 
            (vreset! *setup-fn
@@ -309,7 +305,6 @@
   (if-not (.requestSingleInstanceLock app)
     (do
       (db/close!)
-      (search/close!)
       (.quit app))
     (let [privileges {:standard        true
                       :secure          true
@@ -339,7 +334,6 @@
                                      (try
                                        (fs-watcher/close-watcher!)
                                        (db/close!)
-                                       (search/close!)
                                        (catch :default e
                                          (logger/error "window-all-closed" e)))
                                      (.quit app)))

src/electron/electron/handler.cljs

@@ -23,7 +23,6 @@
             [electron.git :as git]
             [electron.logger :as logger]
             [electron.plugin :as plugin]
-            [electron.search :as search]
             [electron.db :as db]
             [electron.server :as server]
             [electron.shell :as shell]
@@ -333,33 +332,6 @@
   (async/put! state/persistent-dbs-chan true)
   true)
 
-;; Search related IPCs
-(defmethod handle :search-blocks [_window [_ repo q opts]]
-  (search/search-blocks repo q opts))
-
-(defmethod handle :rebuild-indice [_window [_ repo block-data]]
-  (search/truncate-blocks-table! repo)
-  ;; unneeded serialization
-  (search/upsert-blocks! repo (bean/->js block-data))
-  [])
-
-(defmethod handle :transact-blocks [_window [_ repo data]]
-  (let [{:keys [blocks-to-remove-set blocks-to-add]} data]
-    ;; Order matters! Same id will delete then upsert sometimes.
-    (when (seq blocks-to-remove-set)
-      (search/delete-blocks! repo blocks-to-remove-set))
-    (when (seq blocks-to-add)
-      ;; unneeded serialization
-      (search/upsert-blocks! repo (bean/->js blocks-to-add)))))
-
-(defmethod handle :truncate-indice [_window [_ repo]]
-  (search/truncate-blocks-table! repo))
-
-(defmethod handle :remove-db [_window [_ repo]]
-  (search/delete-db! repo))
-;; ^^^^
-;; Search related IPCs End
-
 ;; DB related IPCs start
 
 (defmethod handle :db-export [_window [_ repo data]]
@@ -384,9 +356,7 @@
 
 (defmethod handle :clearCache [window _]
   (logger/info ::clear-cache)
-  (search/close!)
-  (clear-cache! window)
-  (search/ensure-search-dir!))
+  (clear-cache! window))
 
 (defmethod handle :openDialog [^js _window _messages]
   (open-dir-dialog))

src/electron/electron/search.cljs

@@ -1,249 +0,0 @@
-(ns electron.search
-  "Provides block level index"
-  (:require ["path" :as node-path]
-            ["fs-extra" :as fs]
-            ["better-sqlite3" :as sqlite3]
-            [clojure.string :as string]
-            ["electron" :refer [app]]
-            [electron.logger :as logger]
-            [medley.core :as medley]
-            [electron.utils :as utils]))
-
-(defonce databases (atom nil))
-
-(defn close!
-  []
-  (when @databases
-    (doseq [[_ database] @databases]
-      (.close database))
-    (reset! databases nil)))
-
-(defn sanitize-db-name
-  [db-name]
-  (-> db-name
-      (string/replace "/" "_")
-      (string/replace "\\" "_")
-      (string/replace ":" "_"))) ;; windows
-
-(defn get-db
-  [repo]
-  (get @databases (sanitize-db-name repo)))
-
-(declare delete-db!)
-
-(defn prepare
-  [^object db sql db-name]
-  (when db
-    (try
-      (.prepare db sql)
-      (catch :default e
-        (logger/error (str "SQLite prepare failed: " e ": " db-name))
-        ;; case 1: vtable constructor failed: blocks_fts https://github.com/logseq/logseq/issues/7467
-        (delete-db! db-name)
-        (utils/send-to-renderer "rebuildSearchIndice" {})
-        (throw e)))))
-
-(defn add-blocks-fts-triggers!
-  "Table bindings of blocks tables and the blocks FTS virtual tables"
-  [db db-name]
-  (let [triggers [;; add
-                  "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
-                  BEGIN
-                      DELETE from blocks_fts where rowid = old.id;
-                  END;"
-                  ;; insert
-                  "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks
-                  BEGIN
-                      INSERT INTO blocks_fts (rowid, uuid, content, page)
-                      VALUES (new.id, new.uuid, new.content, new.page);
-                  END;"
-                  ;; update
-                  "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks
-                  BEGIN
-                      DELETE from blocks_fts where rowid = old.id;
-                      INSERT INTO blocks_fts (rowid, uuid, content, page)
-                      VALUES (new.id, new.uuid, new.content, new.page);
-                  END;"]]
-    (doseq [trigger triggers]
-      (let [stmt (prepare db trigger db-name)]
-        (.run ^object stmt)))))
-
-(defn create-blocks-table!
-  [db db-name]
-  (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks (
-                        id INTEGER PRIMARY KEY,
-                        uuid TEXT NOT NULL,
-                        content TEXT NOT NULL,
-                        page INTEGER)"
-                      db-name)]
-    (.run ^object stmt)))
-
-(defn create-blocks-fts-table!
-  [db db-name]
-  (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)" db-name)]
-    (.run ^object stmt)))
-
-(defn get-search-dir
-  []
-  (let [path (.getPath ^object app "userData")]
-    (node-path/join path "search")))
-
-(defn ensure-search-dir!
-  []
-  (fs/ensureDirSync (get-search-dir)))
-
-(defn get-db-full-path
-  [db-name]
-  (let [db-name (sanitize-db-name db-name)
-        search-dir (get-search-dir)]
-    [db-name (node-path/join search-dir db-name)]))
-
-(defn get-db-path
-  "Search cache paths"
-  [db-name]
-  (let [db-name (sanitize-db-name db-name)
-        search-dir (get-search-dir)]
-    [db-name (node-path/join search-dir db-name)]))
-
-(defn open-db!
-  "Open a SQLite db for search index"
-  [db-name]
-  (let [[db-sanitized-name db-full-path] (get-db-full-path db-name)]
-    (try (let [db (sqlite3 db-full-path nil)]
-           (create-blocks-table! db db-name)
-           (create-blocks-fts-table! db db-name)
-           (add-blocks-fts-triggers! db db-name)
-           (swap! databases assoc db-sanitized-name db))
-         (catch :default e
-           (logger/error (str e ": " db-name))
-           (try
-             (fs/unlinkSync db-full-path)
-             (catch :default e
-               (logger/error "cannot unlink search db:" e)
-               (utils/send-to-renderer "notification"
-                                       {:type    "error"
-                                        :payload (str "Search index error, please manually delete “" db-full-path "”: \n" e)})))))))
-
-(defn open-dbs!
-  []
-  (let [search-dir (get-search-dir)
-        dbs (fs/readdirSync search-dir)
-        dbs (remove (fn [file-name] (.startsWith file-name ".")) dbs)]
-    (when (seq dbs)
-      (doseq [db-name dbs]
-        (open-db! db-name)))))
-
-(defn- clj-list->sql
-  "Turn clojure list into SQL list
-   '(1 2 3 4)
-   ->
-   \"('1','2','3','4')\""
-  [ids]
-  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
-                (string/join ", ")) ")"))
-
-(defn- get-or-open-db [repo]
-  (or (get-db repo)
-      (do
-        (open-db! repo)
-        (get-db repo))))
-
-(defn upsert-blocks!
-  [repo blocks]
-  (when-let [db (get-or-open-db repo)]
-    ;; TODO: what if a CONFLICT on uuid
-    ;; Should update all values on id conflict
-    (let [insert (prepare db "INSERT INTO blocks (id, uuid, content, page) VALUES (@id, @uuid, @content, @page) ON CONFLICT (id) DO UPDATE SET (uuid, content, page) = (@uuid, @content, @page)" repo)
-          insert-many (.transaction ^object db
-                                    (fn [blocks]
-                                      (doseq [block blocks]
-                                        (.run ^object insert block))))]
-      (insert-many blocks))))
-
-(defn delete-blocks!
-  [repo ids]
-  (when-let [db (get-db repo)]
-    (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
-          stmt (prepare db sql repo)]
-      (.run ^object stmt))))
-
-(defn- search-blocks-aux
-  [repo database sql input page limit]
-  (try
-    (let [stmt (prepare database sql repo)]
-      (js->clj
-       (if page
-         (.all ^object stmt (int page) input limit)
-         (.all ^object stmt  input limit))
-       :keywordize-keys true))
-    (catch :default e
-      (logger/error "Search blocks failed: " (str e)))))
-
-(defn- get-match-inputs
-  [q]
-  (let [match-input (-> q
-                        (string/replace " and " " AND ")
-                        (string/replace " & " " AND ")
-                        (string/replace " or " " OR ")
-                        (string/replace " | " " OR ")
-                        (string/replace " not " " NOT "))]
-    (if (not= q match-input)
-      [(string/replace match-input "," "")]
-      [q
-       (str "\"" match-input "\"")])))
-
-(defn distinct-by
-  [f col]
-  (medley/distinct-by f (seq col)))
-
-(defn search-blocks
-  ":page - the page to specifically search on"
-  [repo q {:keys [limit page]}]
-  (when-let [database (get-db repo)]
-    (when-not (string/blank? q)
-      (let [match-inputs (get-match-inputs q)
-            non-match-input (str "%" (string/replace q #"\s+" "%") "%")
-            limit  (or limit 20)
-            select "select rowid, uuid, content, page from blocks_fts where "
-            pg-sql (if page "page = ? and" "")
-            match-sql (str select
-                           pg-sql
-                           " content match ? order by rank limit ?")
-            non-match-sql (str select
-                               pg-sql
-                               " content like ? limit ?")
-            matched-result (->>
-                            (map
-                             (fn [match-input]
-                               (search-blocks-aux repo database match-sql match-input page limit))
-                             match-inputs)
-                            (apply concat))]
-        (->>
-         (concat matched-result
-                 (search-blocks-aux repo database non-match-sql non-match-input page limit))
-         (distinct-by :rowid)
-         (take limit)
-         (vec))))))
-
-(defn truncate-blocks-table!
-  [repo]
-  (when-let [database (get-db repo)]
-    (let [stmt (prepare database "delete from blocks;" repo)
-          _ (.run ^object stmt)
-          stmt (prepare database "delete from blocks_fts;" repo)]
-      (.run ^object stmt))))
-
-(defn query
-  [repo sql]
-  (when-let [database (get-db repo)]
-    (let [stmt (prepare database sql repo)]
-      (.all ^object stmt))))
-
-(defn delete-db!
-  [repo]
-  (when-let [database (get-db repo)]
-    (.close database)
-    (let [[db-name db-full-path] (get-db-path repo)]
-      (logger/info "Delete search indice: " db-full-path)
-      (fs/unlinkSync db-full-path)
-      (swap! databases dissoc db-name))))

src/main/frontend/db_worker.cljs

@@ -11,16 +11,23 @@
             ["@logseq/sqlite-wasm" :default sqlite3InitModule]
             ["comlink" :as Comlink]
             [clojure.string :as string]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            [frontend.worker.search :as search]))
 
 (defonce *sqlite (atom nil))
+;; repo -> {:db conn :search conn}
 (defonce *sqlite-conns (atom nil))
+;; repo -> conn
 (defonce *datascript-conns (atom nil))
+;; repo -> pool
 (defonce *opfs-pools (atom nil))
 
 (defn- get-sqlite-conn
-  [repo]
-  (get @*sqlite-conns repo))
+  [repo & {:keys [search?]
+           :or {search? false}
+           :as _opts}]
+  (let [k (if search? :search :db)]
+    (get-in @*sqlite-conns [repo k])))
 
 (defn get-datascript-conn
   [repo]
@@ -34,7 +41,7 @@
   [graph]
   (or (get-opfs-pool graph)
       (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" graph)
-                                                             :initialCapacity 10})]
+                                                             :initialCapacity 20})]
         (swap! *opfs-pools assoc graph pool)
         pool)))
 
@@ -105,31 +112,39 @@
       (-restore [_ addr]
         (restore-data-from-addr repo addr)))))
 
+(defn- clean-db!
+  [repo db search]
+  (when (or db search)
+    (swap! *sqlite-conns dissoc repo)
+    (swap! *datascript-conns dissoc repo)
+    (.close ^Object db)
+    (.close ^Object search)))
+
 (defn- close-other-dbs!
   [repo]
-  (doseq [[r db] @*sqlite-conns]
+  (doseq [[r {:keys [db search]}] @*sqlite-conns]
     (when-not (= repo r)
-      (swap! *datascript-conns dissoc r)
-      (swap! *sqlite-conns dissoc r)
       (swap! *opfs-pools dissoc r)
-      (.close ^Object db))))
+      (clean-db! r db search))))
 
 (defn- close-db!
   [repo]
-  (when-let [db (@*sqlite-conns repo)]
-    (swap! *sqlite-conns dissoc repo)
-    (swap! *datascript-conns dissoc repo)
-    (.close ^Object db)))
+  (let [{:keys [db search]} (@*sqlite-conns repo)]
+    (clean-db! repo db search)))
 
 (defn- create-or-open-db!
   [repo]
   (when-not (get-sqlite-conn repo)
     (p/let [pool (<get-opfs-pool repo)
-            db (new (.-OpfsSAHPoolDb pool) (get-repo-path repo))
+            path (get-repo-path repo)
+            db (new (.-OpfsSAHPoolDb pool) path)
+            search-db (new (.-OpfsSAHPoolDb pool) (str "search-" path))
             storage (new-sqlite-storage repo {})]
-      (swap! *sqlite-conns assoc repo db)
+      (swap! *sqlite-conns assoc repo {:db db
+                                       :search search-db})
       (.exec db "PRAGMA locking_mode=exclusive")
       (.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
+      (search/create-tables-and-triggers! search-db)
       (let [conn (or (d/restore-conn storage)
                      (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
         (swap! *datascript-conns assoc repo conn)
@@ -185,6 +200,10 @@
               _ (.wipeFiles pool)]
         (.removeVfs ^js pool)))))
 
+(defn- get-search-db
+  [repo]
+  (get-sqlite-conn repo {:search? true}))
+
 #_:clj-kondo/ignore
 (defclass SQLiteDB
   (extends js/Object)
@@ -261,7 +280,32 @@
    [this repo data]
    (when-not (string/blank? repo)
      (p/let [pool (<get-opfs-pool repo)]
-       (<import-db repo data)))))
+       (<import-db repo data))))
+
+  ;; Search
+  (search-blocks
+   [this repo q option]
+   (p/let [db (get-search-db repo)
+           result (search/search-blocks db q (bean/->clj option))]
+     (bean/->js result)))
+
+  (search-upsert-blocks
+   [this repo blocks]
+   (p/let [db (get-search-db repo)]
+     (search/upsert-blocks! db blocks)
+     nil))
+
+  (search-delete-blocks
+   [this repo ids]
+   (p/let [db (get-search-db repo)]
+     (search/delete-blocks! db ids)
+     nil))
+
+  (search-truncate-tables
+   [this repo]
+   (p/let [db (get-search-db repo)]
+     (search/truncate-table! db)
+     nil)))
 
 (defn init
   "web worker entry"

src/main/frontend/search/agency.cljs

@@ -2,16 +2,12 @@
   "Agent entry for search engine impls"
   (:require [frontend.search.protocol :as protocol]
             [frontend.search.browser :as search-browser]
-            [frontend.search.node :as search-node]
             [frontend.search.plugin :as search-plugin]
-            [frontend.state :as state]
-            [frontend.util :as util]))
+            [frontend.state :as state]))
 
 (defn get-registered-engines
   [repo]
-  [(if (util/electron?)
-     (search-node/->Node repo)
-     (search-browser/->Browser repo))
+  [(search-browser/->Browser repo)
    (when state/lsp-enabled?
      (for [s (state/get-all-plugin-services-with-type :search)]
        (search-plugin/->Plugin s repo)))])

src/main/frontend/search/browser.cljs

@@ -1,57 +1,39 @@
 (ns frontend.search.browser
   "Browser implementation of search protocol"
   (:require [cljs-bean.core :as bean]
-            [frontend.search.db :as search-db :refer [indices]]
             [frontend.search.protocol :as protocol]
-            [goog.object :as gobj]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.persist-db.browser :as browser]
+            [frontend.state :as state]))
 
-;; fuse.js
-
-(defn search-blocks
-  [repo q {:keys [limit page]
-            :or {limit 20}}]
-  (let [indice (or (get-in @indices [repo :blocks])
-                   (search-db/make-blocks-indice! repo))
-        result
-        (if page
-          (.search indice
-                   (clj->js {:$and [{"page" page} {"content" q}]})
-                   (clj->js {:limit limit}))
-          (.search indice q (clj->js {:limit limit})))
-        result (bean/->clj result)]
-    (->>
-     (map
-       (fn [{:keys [item matches]}]
-         (let [{:keys [content uuid page]} item]
-           {:block/uuid uuid
-            :block/content content
-            :block/page page
-            :search/matches matches}))
-       result)
-     (remove nil?))))
+(defonce *sqlite browser/*sqlite)
 
 (defrecord Browser [repo]
   protocol/Engine
   (query [_this q option]
-    (p/promise (search-blocks repo q option)))
+    (if-let [^js sqlite @*sqlite]
+      (p/let [result (.search-blocks sqlite (state/get-current-repo) q (bean/->js option))
+              result (bean/->clj result)]
+        (keep (fn [{:keys [content uuid page]}]
+              (when-not (> (count content) (state/block-content-max-length repo))
+                {:block/uuid uuid
+                 :block/content content
+                 :block/page page})) result))
+      (p/resolved nil)))
   (rebuild-blocks-indice! [_this]
-    (let [indice (search-db/make-blocks-indice! repo)]
-      (p/promise indice)))
+    (p/resolved nil))
   (transact-blocks! [_this {:keys [blocks-to-remove-set
                                    blocks-to-add]}]
-    (swap! search-db/indices update-in [repo :blocks]
-           (fn [indice]
-             (when indice
-               (doseq [block-id blocks-to-remove-set]
-                 (.remove indice
-                          (fn [block]
-                            (= block-id (gobj/get block "id")))))
-               (when (seq blocks-to-add)
-                 (doseq [block blocks-to-add]
-                   (.add indice (bean/->js block)))))
-             indice)))
+    (if-let [^js sqlite @*sqlite]
+      (let [repo (state/get-current-repo)]
+        (p/let [_ (when (seq blocks-to-remove-set)
+                    (.search-delete-blocks sqlite repo (bean/->js blocks-to-remove-set)))]
+          (when (seq blocks-to-add)
+            (.search-upsert-blocks sqlite repo (bean/->js blocks-to-add)))))
+      (p/resolved nil)))
   (truncate-blocks! [_this]
-    (swap! indices assoc-in [repo :blocks] nil))
+    (if-let [^js sqlite @*sqlite]
+      (.search-truncate-tables sqlite (state/get-current-repo))
+      (p/resolved nil)))
   (remove-db! [_this]
-    nil))
+    (p/resolved nil)))

src/main/frontend/search/db.cljs

@@ -81,28 +81,6 @@
                                   (get-db-properties-str properties)))))]
           m')))))
 
-(defn build-blocks-indice
-  ;; TODO: Remove repo effects fns further up the call stack. db fns need standardization on taking connection
-  #_:clj-kondo/ignore
-  [repo]
-  (->> (db/get-all-block-contents)
-       (map block->index)
-       (remove nil?)
-       (bean/->js)))
-
-(defn make-blocks-indice!
-  ([repo] (make-blocks-indice! repo (build-blocks-indice repo)))
-  ([repo blocks]
-   (let [indice (fuse. blocks
-                       (clj->js {:keys ["uuid" "content" "page"]
-                                 :shouldSort true
-                                 :tokenize true
-                                 :minMatchCharLength 1
-                                 :distance 1000
-                                 :threshold 0.35}))]
-     (swap! indices assoc-in [repo :blocks] indice)
-     indice)))
-
 (defn original-page-name->index
   [p]
   (when p

src/main/frontend/search/node.cljs

@@ -1,28 +0,0 @@
-(ns frontend.search.node
-  "NodeJS implementation of search protocol"
-  (:require [cljs-bean.core :as bean]
-            [electron.ipc :as ipc]
-            [frontend.search.db :as search-db]
-            [frontend.search.protocol :as protocol]
-            [promesa.core :as p]
-            [frontend.state :as state]))
-
-(defrecord Node [repo]
-  protocol/Engine
-  (query [_this q opts]
-    (p/let [result (ipc/ipc "search-blocks" repo q opts)
-            result (bean/->clj result)]
-      (keep (fn [{:keys [content uuid page]}]
-              (when-not (> (count content) (state/block-content-max-length repo))
-                {:block/uuid uuid
-                 :block/content content
-                 :block/page page})) result)))
-  (rebuild-blocks-indice! [_this]
-    (let [blocks-indice (search-db/build-blocks-indice repo)]
-      (ipc/ipc "rebuild-indice" repo blocks-indice)))
-  (transact-blocks! [_this data]
-    (ipc/ipc "transact-blocks" repo (bean/->js data)))
-  (truncate-blocks! [_this]
-    (ipc/ipc "truncate-indice" repo))
-  (remove-db! [_this]
-    (ipc/ipc "remove-db" repo)))

src/main/frontend/worker/search.cljs

@@ -0,0 +1,153 @@
+(ns frontend.worker.search
+  "SQLite search"
+  (:require [clojure.string :as string]
+            [promesa.core :as p]
+            [medley.core :as medley]
+            [cljs-bean.core :as bean]))
+
+;; TODO: remove id as :db/id can change
+(defn- add-blocks-fts-triggers!
+  "Table bindings of blocks tables and the blocks FTS virtual tables"
+  [db]
+  (let [triggers [;; add
+                  "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                  END;"
+                  ;; insert
+                  "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks
+                  BEGIN
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"
+                  ;; update
+                  "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"]]
+    (doseq [trigger triggers]
+      (.exec db trigger))))
+
+(defn- create-blocks-table!
+  [db]
+  (.exec db "CREATE TABLE IF NOT EXISTS blocks (
+                        id INTEGER PRIMARY KEY,
+                        uuid TEXT NOT NULL,
+                        content TEXT NOT NULL,
+                        page INTEGER)"))
+
+(defn- create-blocks-fts-table!
+  [db]
+  (.exec db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)"))
+
+(defn create-tables-and-triggers!
+  "Open a SQLite db for search index"
+  [db]
+  (try
+    (create-blocks-table! db)
+    (create-blocks-fts-table! db)
+    (add-blocks-fts-triggers! db)
+    (catch :default e
+      (prn "Failed to create tables and triggers")
+      (js/console.error e)
+      ;; FIXME:
+      ;; (try
+      ;;   ;; unlink db
+      ;;   (catch :default e
+      ;;     (js/console.error "cannot unlink search db:" e)))
+      )))
+
+(defn- clj-list->sql
+  "Turn clojure list into SQL list
+   '(1 2 3 4)
+   ->
+   \"('1','2','3','4')\""
+  [ids]
+  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
+                (string/join ", ")) ")"))
+
+(defn upsert-blocks!
+  [^Object db blocks]
+  (.transaction db (fn [tx]
+                     (doseq [item blocks]
+                       (.exec tx #js {:sql "INSERT INTO blocks (id, uuid, content, page) VALUES ($id, $uuid, $content, $page) ON CONFLICT (id) DO UPDATE SET (uuid, content, page) = ($uuid, $content, $page)"
+                                      :bind #js {:$id (.-id item)
+                                                 :$uuid (.-uuid item)
+                                                 :$content (.-content item)
+                                                 :$page (.-page item)}})))))
+
+(defn delete-blocks!
+  [db ids]
+  (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))]
+    (.exec db sql)))
+
+(defn- search-blocks-aux
+  [db sql input page limit]
+  (try
+    (p/let [result (if page
+                     (.exec db #js {:sql sql
+                                    :bind #js [input page limit]
+                                    :rowMode "array"})
+                     (.exec db #js {:sql sql
+                                    :bind #js [input limit]
+                                    :rowMode "array"}))]
+      (bean/->clj result))
+    (catch :default e
+      (prn :debug "Search blocks failed: ")
+      (js/console.error e))))
+
+(defn- get-match-inputs
+  [q]
+  (let [match-input (-> q
+                        (string/replace " and " " AND ")
+                        (string/replace " & " " AND ")
+                        (string/replace " or " " OR ")
+                        (string/replace " | " " OR ")
+                        (string/replace " not " " NOT "))]
+    (if (not= q match-input)
+      [(string/replace match-input "," "")]
+      [q
+       (str "\"" match-input "\"")])))
+
+(defn distinct-by
+  [f col]
+  (medley/distinct-by f (seq col)))
+
+(defn search-blocks
+  ":page - the page to specifically search on"
+  [db q {:keys [limit page]}]
+  (when-not (string/blank? q)
+    (p/let [match-inputs (get-match-inputs q)
+            non-match-input (str "%" (string/replace q #"\s+" "%") "%")
+            limit  (or limit 20)
+            select "select rowid, uuid, content, page from blocks_fts where "
+            pg-sql (if page "page = ? and" "")
+            match-sql (str select
+                           pg-sql
+                           " content match ? order by rank limit ?")
+            non-match-sql (str select
+                               pg-sql
+                               " content like ? limit ?")
+            results (p/all (map
+                            (fn [match-input]
+                              (search-blocks-aux db match-sql match-input page limit))
+                            match-inputs))
+            matched-result (apply concat results)
+            non-match-result (search-blocks-aux db non-match-sql non-match-input page limit)
+            all-result (->> (concat matched-result non-match-result)
+                            (map (fn [[id uuid content page]]
+                                   {:id id
+                                    :uuid uuid
+                                    :content content
+                                    :page page})))]
+      (->>
+       all-result
+       (distinct-by :uuid)
+       (take limit)))))
+
+(defn truncate-table!
+  [db]
+  (.exec db "delete from blocks")
+  (.exec db "delete from blocks_fts"))