From 9b6d3f243b46a2dc516479780e9108f453253cdc Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 12 Dec 2023 03:03:34 +0800 Subject: [PATCH] refactor: use OPFS sqlite for search --- src/electron/electron/core.cljs | 6 - src/electron/electron/handler.cljs | 32 +--- src/electron/electron/search.cljs | 249 -------------------------- src/main/frontend/db_worker.cljs | 74 ++++++-- src/main/frontend/search/agency.cljs | 8 +- src/main/frontend/search/browser.cljs | 68 +++---- src/main/frontend/search/db.cljs | 22 --- src/main/frontend/search/node.cljs | 28 --- src/main/frontend/worker/search.cljs | 153 ++++++++++++++++ 9 files changed, 240 insertions(+), 400 deletions(-) delete mode 100644 src/electron/electron/search.cljs delete mode 100644 src/main/frontend/search/node.cljs create mode 100644 src/main/frontend/worker/search.cljs diff --git a/src/electron/electron/core.cljs b/src/electron/electron/core.cljs index 09913b0df..1c111f893 100644 --- a/src/electron/electron/core.cljs +++ b/src/electron/electron/core.cljs @@ -1,6 +1,5 @@ (ns electron.core (:require [electron.handler :as handler] - [electron.search :as search] [electron.db :as db] [electron.updater :refer [init-updater] :as updater] [electron.utils :refer [*win mac? linux? dev? get-win-from-sender @@ -255,11 +254,8 @@ (js-utils/disableXFrameOptions win) - (search/ensure-search-dir!) (db/ensure-graphs-dir!) - (search/open-dbs!) - (git/auto-commit-current-graph!) (vreset! *setup-fn @@ -309,7 +305,6 @@ (if-not (.requestSingleInstanceLock app) (do (db/close!) - (search/close!) (.quit app)) (let [privileges {:standard true :secure true @@ -339,7 +334,6 @@ (try (fs-watcher/close-watcher!) (db/close!) - (search/close!) (catch :default e (logger/error "window-all-closed" e))) (.quit app))) diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index a2e520f1a..7cf94b44c 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -23,7 +23,6 @@ [electron.git :as git] [electron.logger :as logger] [electron.plugin :as plugin] - [electron.search :as search] [electron.db :as db] [electron.server :as server] [electron.shell :as shell] @@ -333,33 +332,6 @@ (async/put! state/persistent-dbs-chan true) true) -;; Search related IPCs -(defmethod handle :search-blocks [_window [_ repo q opts]] - (search/search-blocks repo q opts)) - -(defmethod handle :rebuild-indice [_window [_ repo block-data]] - (search/truncate-blocks-table! repo) - ;; unneeded serialization - (search/upsert-blocks! repo (bean/->js block-data)) - []) - -(defmethod handle :transact-blocks [_window [_ repo data]] - (let [{:keys [blocks-to-remove-set blocks-to-add]} data] - ;; Order matters! Same id will delete then upsert sometimes. - (when (seq blocks-to-remove-set) - (search/delete-blocks! repo blocks-to-remove-set)) - (when (seq blocks-to-add) - ;; unneeded serialization - (search/upsert-blocks! repo (bean/->js blocks-to-add))))) - -(defmethod handle :truncate-indice [_window [_ repo]] - (search/truncate-blocks-table! repo)) - -(defmethod handle :remove-db [_window [_ repo]] - (search/delete-db! repo)) -;; ^^^^ -;; Search related IPCs End - ;; DB related IPCs start (defmethod handle :db-export [_window [_ repo data]] @@ -384,9 +356,7 @@ (defmethod handle :clearCache [window _] (logger/info ::clear-cache) - (search/close!) - (clear-cache! window) - (search/ensure-search-dir!)) + (clear-cache! window)) (defmethod handle :openDialog [^js _window _messages] (open-dir-dialog)) diff --git a/src/electron/electron/search.cljs b/src/electron/electron/search.cljs deleted file mode 100644 index c7881bc0f..000000000 --- a/src/electron/electron/search.cljs +++ /dev/null @@ -1,249 +0,0 @@ -(ns electron.search - "Provides block level index" - (:require ["path" :as node-path] - ["fs-extra" :as fs] - ["better-sqlite3" :as sqlite3] - [clojure.string :as string] - ["electron" :refer [app]] - [electron.logger :as logger] - [medley.core :as medley] - [electron.utils :as utils])) - -(defonce databases (atom nil)) - -(defn close! - [] - (when @databases - (doseq [[_ database] @databases] - (.close database)) - (reset! databases nil))) - -(defn sanitize-db-name - [db-name] - (-> db-name - (string/replace "/" "_") - (string/replace "\\" "_") - (string/replace ":" "_"))) ;; windows - -(defn get-db - [repo] - (get @databases (sanitize-db-name repo))) - -(declare delete-db!) - -(defn prepare - [^object db sql db-name] - (when db - (try - (.prepare db sql) - (catch :default e - (logger/error (str "SQLite prepare failed: " e ": " db-name)) - ;; case 1: vtable constructor failed: blocks_fts https://github.com/logseq/logseq/issues/7467 - (delete-db! db-name) - (utils/send-to-renderer "rebuildSearchIndice" {}) - (throw e))))) - -(defn add-blocks-fts-triggers! - "Table bindings of blocks tables and the blocks FTS virtual tables" - [db db-name] - (let [triggers [;; add - "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks - BEGIN - DELETE from blocks_fts where rowid = old.id; - END;" - ;; insert - "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks - BEGIN - INSERT INTO blocks_fts (rowid, uuid, content, page) - VALUES (new.id, new.uuid, new.content, new.page); - END;" - ;; update - "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks - BEGIN - DELETE from blocks_fts where rowid = old.id; - INSERT INTO blocks_fts (rowid, uuid, content, page) - VALUES (new.id, new.uuid, new.content, new.page); - END;"]] - (doseq [trigger triggers] - (let [stmt (prepare db trigger db-name)] - (.run ^object stmt))))) - -(defn create-blocks-table! - [db db-name] - (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks ( - id INTEGER PRIMARY KEY, - uuid TEXT NOT NULL, - content TEXT NOT NULL, - page INTEGER)" - db-name)] - (.run ^object stmt))) - -(defn create-blocks-fts-table! - [db db-name] - (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)" db-name)] - (.run ^object stmt))) - -(defn get-search-dir - [] - (let [path (.getPath ^object app "userData")] - (node-path/join path "search"))) - -(defn ensure-search-dir! - [] - (fs/ensureDirSync (get-search-dir))) - -(defn get-db-full-path - [db-name] - (let [db-name (sanitize-db-name db-name) - search-dir (get-search-dir)] - [db-name (node-path/join search-dir db-name)])) - -(defn get-db-path - "Search cache paths" - [db-name] - (let [db-name (sanitize-db-name db-name) - search-dir (get-search-dir)] - [db-name (node-path/join search-dir db-name)])) - -(defn open-db! - "Open a SQLite db for search index" - [db-name] - (let [[db-sanitized-name db-full-path] (get-db-full-path db-name)] - (try (let [db (sqlite3 db-full-path nil)] - (create-blocks-table! db db-name) - (create-blocks-fts-table! db db-name) - (add-blocks-fts-triggers! db db-name) - (swap! databases assoc db-sanitized-name db)) - (catch :default e - (logger/error (str e ": " db-name)) - (try - (fs/unlinkSync db-full-path) - (catch :default e - (logger/error "cannot unlink search db:" e) - (utils/send-to-renderer "notification" - {:type "error" - :payload (str "Search index error, please manually delete “" db-full-path "”: \n" e)}))))))) - -(defn open-dbs! - [] - (let [search-dir (get-search-dir) - dbs (fs/readdirSync search-dir) - dbs (remove (fn [file-name] (.startsWith file-name ".")) dbs)] - (when (seq dbs) - (doseq [db-name dbs] - (open-db! db-name))))) - -(defn- clj-list->sql - "Turn clojure list into SQL list - '(1 2 3 4) - -> - \"('1','2','3','4')\"" - [ids] - (str "(" (->> (map (fn [id] (str "'" id "'")) ids) - (string/join ", ")) ")")) - -(defn- get-or-open-db [repo] - (or (get-db repo) - (do - (open-db! repo) - (get-db repo)))) - -(defn upsert-blocks! - [repo blocks] - (when-let [db (get-or-open-db repo)] - ;; TODO: what if a CONFLICT on uuid - ;; Should update all values on id conflict - (let [insert (prepare db "INSERT INTO blocks (id, uuid, content, page) VALUES (@id, @uuid, @content, @page) ON CONFLICT (id) DO UPDATE SET (uuid, content, page) = (@uuid, @content, @page)" repo) - insert-many (.transaction ^object db - (fn [blocks] - (doseq [block blocks] - (.run ^object insert block))))] - (insert-many blocks)))) - -(defn delete-blocks! - [repo ids] - (when-let [db (get-db repo)] - (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids)) - stmt (prepare db sql repo)] - (.run ^object stmt)))) - -(defn- search-blocks-aux - [repo database sql input page limit] - (try - (let [stmt (prepare database sql repo)] - (js->clj - (if page - (.all ^object stmt (int page) input limit) - (.all ^object stmt input limit)) - :keywordize-keys true)) - (catch :default e - (logger/error "Search blocks failed: " (str e))))) - -(defn- get-match-inputs - [q] - (let [match-input (-> q - (string/replace " and " " AND ") - (string/replace " & " " AND ") - (string/replace " or " " OR ") - (string/replace " | " " OR ") - (string/replace " not " " NOT "))] - (if (not= q match-input) - [(string/replace match-input "," "")] - [q - (str "\"" match-input "\"")]))) - -(defn distinct-by - [f col] - (medley/distinct-by f (seq col))) - -(defn search-blocks - ":page - the page to specifically search on" - [repo q {:keys [limit page]}] - (when-let [database (get-db repo)] - (when-not (string/blank? q) - (let [match-inputs (get-match-inputs q) - non-match-input (str "%" (string/replace q #"\s+" "%") "%") - limit (or limit 20) - select "select rowid, uuid, content, page from blocks_fts where " - pg-sql (if page "page = ? and" "") - match-sql (str select - pg-sql - " content match ? order by rank limit ?") - non-match-sql (str select - pg-sql - " content like ? limit ?") - matched-result (->> - (map - (fn [match-input] - (search-blocks-aux repo database match-sql match-input page limit)) - match-inputs) - (apply concat))] - (->> - (concat matched-result - (search-blocks-aux repo database non-match-sql non-match-input page limit)) - (distinct-by :rowid) - (take limit) - (vec)))))) - -(defn truncate-blocks-table! - [repo] - (when-let [database (get-db repo)] - (let [stmt (prepare database "delete from blocks;" repo) - _ (.run ^object stmt) - stmt (prepare database "delete from blocks_fts;" repo)] - (.run ^object stmt)))) - -(defn query - [repo sql] - (when-let [database (get-db repo)] - (let [stmt (prepare database sql repo)] - (.all ^object stmt)))) - -(defn delete-db! - [repo] - (when-let [database (get-db repo)] - (.close database) - (let [[db-name db-full-path] (get-db-path repo)] - (logger/info "Delete search indice: " db-full-path) - (fs/unlinkSync db-full-path) - (swap! databases dissoc db-name)))) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index 799e522b6..99e3bc480 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -11,16 +11,23 @@ ["@logseq/sqlite-wasm" :default sqlite3InitModule] ["comlink" :as Comlink] [clojure.string :as string] - [cljs-bean.core :as bean])) + [cljs-bean.core :as bean] + [frontend.worker.search :as search])) (defonce *sqlite (atom nil)) +;; repo -> {:db conn :search conn} (defonce *sqlite-conns (atom nil)) +;; repo -> conn (defonce *datascript-conns (atom nil)) +;; repo -> pool (defonce *opfs-pools (atom nil)) (defn- get-sqlite-conn - [repo] - (get @*sqlite-conns repo)) + [repo & {:keys [search?] + :or {search? false} + :as _opts}] + (let [k (if search? :search :db)] + (get-in @*sqlite-conns [repo k]))) (defn get-datascript-conn [repo] @@ -34,7 +41,7 @@ [graph] (or (get-opfs-pool graph) (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" graph) - :initialCapacity 10})] + :initialCapacity 20})] (swap! *opfs-pools assoc graph pool) pool))) @@ -105,31 +112,39 @@ (-restore [_ addr] (restore-data-from-addr repo addr))))) +(defn- clean-db! + [repo db search] + (when (or db search) + (swap! *sqlite-conns dissoc repo) + (swap! *datascript-conns dissoc repo) + (.close ^Object db) + (.close ^Object search))) + (defn- close-other-dbs! [repo] - (doseq [[r db] @*sqlite-conns] + (doseq [[r {:keys [db search]}] @*sqlite-conns] (when-not (= repo r) - (swap! *datascript-conns dissoc r) - (swap! *sqlite-conns dissoc r) (swap! *opfs-pools dissoc r) - (.close ^Object db)))) + (clean-db! r db search)))) (defn- close-db! [repo] - (when-let [db (@*sqlite-conns repo)] - (swap! *sqlite-conns dissoc repo) - (swap! *datascript-conns dissoc repo) - (.close ^Object db))) + (let [{:keys [db search]} (@*sqlite-conns repo)] + (clean-db! repo db search))) (defn- create-or-open-db! [repo] (when-not (get-sqlite-conn repo) (p/let [pool (clj option))] + (bean/->js result))) + + (search-upsert-blocks + [this repo blocks] + (p/let [db (get-search-db repo)] + (search/upsert-blocks! db blocks) + nil)) + + (search-delete-blocks + [this repo ids] + (p/let [db (get-search-db repo)] + (search/delete-blocks! db ids) + nil)) + + (search-truncate-tables + [this repo] + (p/let [db (get-search-db repo)] + (search/truncate-table! db) + nil))) (defn init "web worker entry" diff --git a/src/main/frontend/search/agency.cljs b/src/main/frontend/search/agency.cljs index f193bbccb..31b2eb5b6 100644 --- a/src/main/frontend/search/agency.cljs +++ b/src/main/frontend/search/agency.cljs @@ -2,16 +2,12 @@ "Agent entry for search engine impls" (:require [frontend.search.protocol :as protocol] [frontend.search.browser :as search-browser] - [frontend.search.node :as search-node] [frontend.search.plugin :as search-plugin] - [frontend.state :as state] - [frontend.util :as util])) + [frontend.state :as state])) (defn get-registered-engines [repo] - [(if (util/electron?) - (search-node/->Node repo) - (search-browser/->Browser repo)) + [(search-browser/->Browser repo) (when state/lsp-enabled? (for [s (state/get-all-plugin-services-with-type :search)] (search-plugin/->Plugin s repo)))]) diff --git a/src/main/frontend/search/browser.cljs b/src/main/frontend/search/browser.cljs index 372ff4099..271830b0c 100644 --- a/src/main/frontend/search/browser.cljs +++ b/src/main/frontend/search/browser.cljs @@ -1,57 +1,39 @@ (ns frontend.search.browser "Browser implementation of search protocol" (:require [cljs-bean.core :as bean] - [frontend.search.db :as search-db :refer [indices]] [frontend.search.protocol :as protocol] - [goog.object :as gobj] - [promesa.core :as p])) + [promesa.core :as p] + [frontend.persist-db.browser :as browser] + [frontend.state :as state])) -;; fuse.js - -(defn search-blocks - [repo q {:keys [limit page] - :or {limit 20}}] - (let [indice (or (get-in @indices [repo :blocks]) - (search-db/make-blocks-indice! repo)) - result - (if page - (.search indice - (clj->js {:$and [{"page" page} {"content" q}]}) - (clj->js {:limit limit})) - (.search indice q (clj->js {:limit limit}))) - result (bean/->clj result)] - (->> - (map - (fn [{:keys [item matches]}] - (let [{:keys [content uuid page]} item] - {:block/uuid uuid - :block/content content - :block/page page - :search/matches matches})) - result) - (remove nil?)))) +(defonce *sqlite browser/*sqlite) (defrecord Browser [repo] protocol/Engine (query [_this q option] - (p/promise (search-blocks repo q option))) + (if-let [^js sqlite @*sqlite] + (p/let [result (.search-blocks sqlite (state/get-current-repo) q (bean/->js option)) + result (bean/->clj result)] + (keep (fn [{:keys [content uuid page]}] + (when-not (> (count content) (state/block-content-max-length repo)) + {:block/uuid uuid + :block/content content + :block/page page})) result)) + (p/resolved nil))) (rebuild-blocks-indice! [_this] - (let [indice (search-db/make-blocks-indice! repo)] - (p/promise indice))) + (p/resolved nil)) (transact-blocks! [_this {:keys [blocks-to-remove-set blocks-to-add]}] - (swap! search-db/indices update-in [repo :blocks] - (fn [indice] - (when indice - (doseq [block-id blocks-to-remove-set] - (.remove indice - (fn [block] - (= block-id (gobj/get block "id"))))) - (when (seq blocks-to-add) - (doseq [block blocks-to-add] - (.add indice (bean/->js block))))) - indice))) + (if-let [^js sqlite @*sqlite] + (let [repo (state/get-current-repo)] + (p/let [_ (when (seq blocks-to-remove-set) + (.search-delete-blocks sqlite repo (bean/->js blocks-to-remove-set)))] + (when (seq blocks-to-add) + (.search-upsert-blocks sqlite repo (bean/->js blocks-to-add))))) + (p/resolved nil))) (truncate-blocks! [_this] - (swap! indices assoc-in [repo :blocks] nil)) + (if-let [^js sqlite @*sqlite] + (.search-truncate-tables sqlite (state/get-current-repo)) + (p/resolved nil))) (remove-db! [_this] - nil)) + (p/resolved nil))) diff --git a/src/main/frontend/search/db.cljs b/src/main/frontend/search/db.cljs index 4dc05ff78..d378ae9f6 100644 --- a/src/main/frontend/search/db.cljs +++ b/src/main/frontend/search/db.cljs @@ -81,28 +81,6 @@ (get-db-properties-str properties)))))] m'))))) -(defn build-blocks-indice - ;; TODO: Remove repo effects fns further up the call stack. db fns need standardization on taking connection - #_:clj-kondo/ignore - [repo] - (->> (db/get-all-block-contents) - (map block->index) - (remove nil?) - (bean/->js))) - -(defn make-blocks-indice! - ([repo] (make-blocks-indice! repo (build-blocks-indice repo))) - ([repo blocks] - (let [indice (fuse. blocks - (clj->js {:keys ["uuid" "content" "page"] - :shouldSort true - :tokenize true - :minMatchCharLength 1 - :distance 1000 - :threshold 0.35}))] - (swap! indices assoc-in [repo :blocks] indice) - indice))) - (defn original-page-name->index [p] (when p diff --git a/src/main/frontend/search/node.cljs b/src/main/frontend/search/node.cljs deleted file mode 100644 index fdc550083..000000000 --- a/src/main/frontend/search/node.cljs +++ /dev/null @@ -1,28 +0,0 @@ -(ns frontend.search.node - "NodeJS implementation of search protocol" - (:require [cljs-bean.core :as bean] - [electron.ipc :as ipc] - [frontend.search.db :as search-db] - [frontend.search.protocol :as protocol] - [promesa.core :as p] - [frontend.state :as state])) - -(defrecord Node [repo] - protocol/Engine - (query [_this q opts] - (p/let [result (ipc/ipc "search-blocks" repo q opts) - result (bean/->clj result)] - (keep (fn [{:keys [content uuid page]}] - (when-not (> (count content) (state/block-content-max-length repo)) - {:block/uuid uuid - :block/content content - :block/page page})) result))) - (rebuild-blocks-indice! [_this] - (let [blocks-indice (search-db/build-blocks-indice repo)] - (ipc/ipc "rebuild-indice" repo blocks-indice))) - (transact-blocks! [_this data] - (ipc/ipc "transact-blocks" repo (bean/->js data))) - (truncate-blocks! [_this] - (ipc/ipc "truncate-indice" repo)) - (remove-db! [_this] - (ipc/ipc "remove-db" repo))) diff --git a/src/main/frontend/worker/search.cljs b/src/main/frontend/worker/search.cljs new file mode 100644 index 000000000..51e46e966 --- /dev/null +++ b/src/main/frontend/worker/search.cljs @@ -0,0 +1,153 @@ +(ns frontend.worker.search + "SQLite search" + (:require [clojure.string :as string] + [promesa.core :as p] + [medley.core :as medley] + [cljs-bean.core :as bean])) + +;; TODO: remove id as :db/id can change +(defn- add-blocks-fts-triggers! + "Table bindings of blocks tables and the blocks FTS virtual tables" + [db] + (let [triggers [;; add + "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks + BEGIN + DELETE from blocks_fts where rowid = old.id; + END;" + ;; insert + "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks + BEGIN + INSERT INTO blocks_fts (rowid, uuid, content, page) + VALUES (new.id, new.uuid, new.content, new.page); + END;" + ;; update + "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks + BEGIN + DELETE from blocks_fts where rowid = old.id; + INSERT INTO blocks_fts (rowid, uuid, content, page) + VALUES (new.id, new.uuid, new.content, new.page); + END;"]] + (doseq [trigger triggers] + (.exec db trigger)))) + +(defn- create-blocks-table! + [db] + (.exec db "CREATE TABLE IF NOT EXISTS blocks ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + content TEXT NOT NULL, + page INTEGER)")) + +(defn- create-blocks-fts-table! + [db] + (.exec db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)")) + +(defn create-tables-and-triggers! + "Open a SQLite db for search index" + [db] + (try + (create-blocks-table! db) + (create-blocks-fts-table! db) + (add-blocks-fts-triggers! db) + (catch :default e + (prn "Failed to create tables and triggers") + (js/console.error e) + ;; FIXME: + ;; (try + ;; ;; unlink db + ;; (catch :default e + ;; (js/console.error "cannot unlink search db:" e))) + ))) + +(defn- clj-list->sql + "Turn clojure list into SQL list + '(1 2 3 4) + -> + \"('1','2','3','4')\"" + [ids] + (str "(" (->> (map (fn [id] (str "'" id "'")) ids) + (string/join ", ")) ")")) + +(defn upsert-blocks! + [^Object db blocks] + (.transaction db (fn [tx] + (doseq [item blocks] + (.exec tx #js {:sql "INSERT INTO blocks (id, uuid, content, page) VALUES ($id, $uuid, $content, $page) ON CONFLICT (id) DO UPDATE SET (uuid, content, page) = ($uuid, $content, $page)" + :bind #js {:$id (.-id item) + :$uuid (.-uuid item) + :$content (.-content item) + :$page (.-page item)}}))))) + +(defn delete-blocks! + [db ids] + (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))] + (.exec db sql))) + +(defn- search-blocks-aux + [db sql input page limit] + (try + (p/let [result (if page + (.exec db #js {:sql sql + :bind #js [input page limit] + :rowMode "array"}) + (.exec db #js {:sql sql + :bind #js [input limit] + :rowMode "array"}))] + (bean/->clj result)) + (catch :default e + (prn :debug "Search blocks failed: ") + (js/console.error e)))) + +(defn- get-match-inputs + [q] + (let [match-input (-> q + (string/replace " and " " AND ") + (string/replace " & " " AND ") + (string/replace " or " " OR ") + (string/replace " | " " OR ") + (string/replace " not " " NOT "))] + (if (not= q match-input) + [(string/replace match-input "," "")] + [q + (str "\"" match-input "\"")]))) + +(defn distinct-by + [f col] + (medley/distinct-by f (seq col))) + +(defn search-blocks + ":page - the page to specifically search on" + [db q {:keys [limit page]}] + (when-not (string/blank? q) + (p/let [match-inputs (get-match-inputs q) + non-match-input (str "%" (string/replace q #"\s+" "%") "%") + limit (or limit 20) + select "select rowid, uuid, content, page from blocks_fts where " + pg-sql (if page "page = ? and" "") + match-sql (str select + pg-sql + " content match ? order by rank limit ?") + non-match-sql (str select + pg-sql + " content like ? limit ?") + results (p/all (map + (fn [match-input] + (search-blocks-aux db match-sql match-input page limit)) + match-inputs)) + matched-result (apply concat results) + non-match-result (search-blocks-aux db non-match-sql non-match-input page limit) + all-result (->> (concat matched-result non-match-result) + (map (fn [[id uuid content page]] + {:id id + :uuid uuid + :content content + :page page})))] + (->> + all-result + (distinct-by :uuid) + (take limit))))) + +(defn truncate-table! + [db] + (.exec db "delete from blocks") + (.exec db "delete from blocks_fts"))