mirror of https://github.com/logseq/logseq
feat(search): single database done
parent
2d71389fe5
commit
2fdfd0dcb1
|
@ -1,5 +1,6 @@
|
|||
(ns electron.core
|
||||
(:require [electron.handler :as handler]
|
||||
[electron.search :as search]
|
||||
[electron.updater :refer [init-updater]]
|
||||
[electron.utils :refer [mac? win32? prod? dev? logger open]]
|
||||
[clojure.string :as string]
|
||||
|
@ -148,7 +149,9 @@
|
|||
(defn main
|
||||
[]
|
||||
(if-not (.requestSingleInstanceLock app)
|
||||
(.quit app)
|
||||
(do
|
||||
(search/close!)
|
||||
(.quit app))
|
||||
(do
|
||||
(.on app "second-instance"
|
||||
(fn [_event _commandLine _workingDirectory]
|
||||
|
@ -156,7 +159,9 @@
|
|||
(when (.isMinimized ^object win)
|
||||
(.restore win))
|
||||
(.focus win))))
|
||||
(.on app "window-all-closed" (fn [] (.quit app)))
|
||||
(.on app "window-all-closed" (fn []
|
||||
(search/close!)
|
||||
(.quit app)))
|
||||
(.on app "ready"
|
||||
(fn []
|
||||
(let [^js win (create-main-window)
|
||||
|
@ -164,6 +169,8 @@
|
|||
*quitting? (atom false)]
|
||||
(.. logger (info (str "Logseq App(" (.getVersion app) ") Starting... ")))
|
||||
|
||||
(search/open-db!)
|
||||
|
||||
(vreset! *setup-fn
|
||||
(fn []
|
||||
(let [t0 (setup-updater! win)
|
||||
|
|
|
@ -89,7 +89,6 @@
|
|||
(remove nil?))]
|
||||
(vec (cons {:path (fix-win-path! path)} result))))
|
||||
|
||||
;; TODO: Is it going to be slow if it's a huge directory
|
||||
(defmethod handle :openDir [^js window _messages]
|
||||
(let [result (.showOpenDialogSync dialog (bean/->js
|
||||
{:properties ["openDirectory" "createDirectory" "promptToCreate"]}))
|
||||
|
@ -106,16 +105,23 @@
|
|||
(async/put! state/persistent-dbs-chan true )
|
||||
true)
|
||||
|
||||
(defmethod handle :search [window [_ q]]
|
||||
(search/search-blocks q))
|
||||
(defmethod handle :search-blocks [window [_ repo q limit]]
|
||||
(search/search-blocks q limit))
|
||||
|
||||
(defmethod handle :upsert-blocks [window [_ blocks]]
|
||||
(search/upsert-blocks! blocks))
|
||||
(defmethod handle :rebuild-blocks-indice [window [_ repo data]]
|
||||
(search/truncate-blocks-table!)
|
||||
;; unneeded serialization
|
||||
(search/upsert-blocks! (bean/->js data)))
|
||||
|
||||
(defmethod handle :delete-blocks [window [_ block-ids]]
|
||||
(search/delete-blocks! block-ids))
|
||||
(defmethod handle :transact-blocks [window [_ repo data]]
|
||||
(let [{:keys [blocks-to-remove-set blocks-to-add]} data]
|
||||
(when (seq blocks-to-remove-set)
|
||||
(search/delete-blocks! blocks-to-remove-set))
|
||||
(when (seq blocks-to-add)
|
||||
;; unneeded serialization
|
||||
(search/upsert-blocks! (bean/->js blocks-to-add)))))
|
||||
|
||||
(defmethod handle :truncate-blocks [window _]
|
||||
(defmethod handle :truncate-blocks [window [_ repo]]
|
||||
(search/truncate-blocks-table!))
|
||||
|
||||
(defn- get-file-ext
|
||||
|
|
|
@ -8,6 +8,12 @@
|
|||
|
||||
(defonce database (atom nil))
|
||||
|
||||
(defn close!
|
||||
[]
|
||||
(when @database
|
||||
(.close @database)
|
||||
(reset! database nil)))
|
||||
|
||||
(defn prepare
|
||||
[^object db sql]
|
||||
(.prepare db sql))
|
||||
|
@ -38,14 +44,15 @@
|
|||
|
||||
(defn create-blocks-table!
|
||||
[db]
|
||||
(let [stmt (prepare db "CREATE TABLE blocks (
|
||||
id TEXT PRIMARY KEY,
|
||||
text TEXT NOT NULL)")]
|
||||
(let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
uuid TEXT NOT NULL,
|
||||
content TEXT NOT NULL)")]
|
||||
(.run ^object stmt)))
|
||||
|
||||
(defn create-blocks-fts-table!
|
||||
[db]
|
||||
(let [stmt (prepare db "CREATE VIRTUAL TABLE blocks_fts USING fts5(id, text)")]
|
||||
(let [stmt (prepare db "CREATE VIRTUAL TABLE blocks_fts USING fts5(id, uuid, content)")]
|
||||
(.run ^object stmt)))
|
||||
|
||||
(defn open-db!
|
||||
|
@ -66,14 +73,16 @@
|
|||
(reset! database db)
|
||||
db))
|
||||
|
||||
(defonce debug-blocks (atom nil))
|
||||
(defn upsert-blocks!
|
||||
[blocks]
|
||||
(reset! debug-blocks blocks)
|
||||
(when-let [db @database]
|
||||
(let [insert (prepare db "INSERT INTO blocks (id, text) VALUES (@id, @text) ON CONFLICT (id) DO UPDATE SET text = @text")
|
||||
;; TODO: what if a CONFLICT on uuid
|
||||
(let [insert (prepare db "INSERT INTO blocks (id, uuid, content) VALUES (@id, @uuid, @content) ON CONFLICT (id) DO UPDATE SET content = @content")
|
||||
insert-many (.transaction ^object db
|
||||
(fn [blocks]
|
||||
(doseq [block blocks]
|
||||
(prn {:block block})
|
||||
(.run ^object insert block))))]
|
||||
(insert-many blocks))))
|
||||
|
||||
|
@ -83,7 +92,6 @@
|
|||
(let [sql (utils/format "DELETE from blocks WHERE id IN (%s)"
|
||||
(->> (map (fn [id] (str "'" id "'")) ids)
|
||||
(string/join ", ")))
|
||||
_ (prn {:sql sql})
|
||||
stmt (prepare db sql)]
|
||||
(.run ^object stmt))))
|
||||
|
||||
|
@ -97,15 +105,15 @@
|
|||
;; [q]
|
||||
;; (when-not (string/blank? q)
|
||||
;; (let [stmt (prepare @database
|
||||
;; "select id, text from blocks_fts where text match ? ORDER BY rank")]
|
||||
;; "select id, uuid, content from blocks_fts where content match ? ORDER BY rank")]
|
||||
;; (js->clj (.all ^object stmt q) :keywordize-keys true))))
|
||||
|
||||
(defn search-blocks
|
||||
[q]
|
||||
[q limit]
|
||||
(when-not (string/blank? q)
|
||||
(let [stmt (prepare @database
|
||||
"select id, text from blocks_fts where text like ?")]
|
||||
(js->clj (.all ^object stmt (str "%" q "%")) :keywordize-keys true))))
|
||||
"select id, uuid, content from blocks where content like ? limit ?")]
|
||||
(js->clj (.all ^object stmt (str "%" q "%") limit) :keywordize-keys true))))
|
||||
|
||||
(defn truncate-blocks-table!
|
||||
[]
|
||||
|
@ -129,17 +137,24 @@
|
|||
(open-db!)
|
||||
|
||||
(add-blocks! (clj->js [{:id "a"
|
||||
:text "hello world"}
|
||||
:uuid ""
|
||||
:content "hello world"}
|
||||
{:id "b"
|
||||
:text "foo bar"}]))
|
||||
:uuid ""
|
||||
:content "foo bar"}]))
|
||||
|
||||
(time
|
||||
(let [blocks (for [i (range 10000)]
|
||||
{:id (str i)
|
||||
:text (rand-nth ["hello" "world" "nice"])})]
|
||||
:uuid ""
|
||||
:content (rand-nth ["hello" "world" "nice"])})]
|
||||
(add-blocks! (clj->js blocks))))
|
||||
|
||||
(get-all-blocks)
|
||||
|
||||
(search-blocks "hello")
|
||||
|
||||
(def block {:id 16, :uuid "5f713e91-8a3c-4b04-a33a-c39482428e2d", :content "Hello, I'm a block!"})
|
||||
|
||||
(add-blocks! (clj->js [block]))
|
||||
)
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
[goog.object :as gobj]
|
||||
[goog.dom :as gdom]
|
||||
[clojure.string :as string]
|
||||
[promesa.core :as p]
|
||||
[frontend.commands :as commands
|
||||
:refer [*show-commands
|
||||
*matched-commands
|
||||
|
@ -89,6 +90,30 @@
|
|||
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a page"]
|
||||
:class "black"}))))))
|
||||
|
||||
(rum/defcs block-search-auto-complete < rum/reactive
|
||||
{:init (fn [state]
|
||||
(assoc state ::result (atom nil)))
|
||||
:did-update (fn [state]
|
||||
(let [result (::result state)
|
||||
[edit-block _ _ q] (:rum/args state)]
|
||||
(p/let [matched-blocks (when-not (string/blank? q)
|
||||
(editor-handler/get-matched-blocks q (:block/uuid edit-block)))]
|
||||
(reset! result matched-blocks)))
|
||||
state)}
|
||||
[state edit-block input id q format content]
|
||||
(let [result (rum/react (get state ::result))
|
||||
chosen-handler (editor-handler/block-on-chosen-handler input id q format)
|
||||
non-exist-block-handler (editor-handler/block-non-exist-handler input)]
|
||||
(when result
|
||||
(ui/auto-complete
|
||||
result
|
||||
{:on-chosen chosen-handler
|
||||
:on-enter non-exist-block-handler
|
||||
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a block"]
|
||||
:item-render (fn [{:block/keys [content]}]
|
||||
content)
|
||||
:class "black"}))))
|
||||
|
||||
(rum/defcs block-search < rum/reactive
|
||||
{:will-unmount (fn [state]
|
||||
(reset! editor-handler/*selected-text nil)
|
||||
|
@ -106,19 +131,8 @@
|
|||
@editor-handler/*selected-text
|
||||
(when (> (count edit-content) current-pos)
|
||||
(subs edit-content pos current-pos)))]
|
||||
(p/let [matched-blocks (when-not (string/blank? q)
|
||||
(editor-handler/get-matched-blocks q (:block/uuid edit-block)))]
|
||||
(when input
|
||||
(let [chosen-handler (editor-handler/block-on-chosen-handler input id q format)
|
||||
non-exist-block-handler (editor-handler/block-non-exist-handler input)]
|
||||
(ui/auto-complete
|
||||
matched-blocks
|
||||
{:on-chosen chosen-handler
|
||||
:on-enter non-exist-block-handler
|
||||
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a block"]
|
||||
:item-render (fn [{:block/keys [content]}]
|
||||
(subs content 0 64))
|
||||
:class "black"})))))))
|
||||
(when input
|
||||
(block-search-auto-complete edit-block input id q format)))))
|
||||
|
||||
(rum/defc template-search < rum/reactive
|
||||
{:will-unmount (fn [state] (reset! editor-handler/*selected-text nil) state)}
|
||||
|
|
|
@ -1756,7 +1756,7 @@
|
|||
99)
|
||||
(map (comp str :block/uuid))))
|
||||
current-and-parents (set/union #{(str (:block/uuid current-block))} block-parents)]
|
||||
(p/let [result (search/block-search (state/get-current-repo) q {:limit 10})]
|
||||
(p/let [result (search/block-search (state/get-current-repo) q {:limit 20})]
|
||||
(remove
|
||||
(fn [h]
|
||||
(contains? current-and-parents (:block/uuid h)))
|
||||
|
|
|
@ -21,12 +21,16 @@
|
|||
:uuid (str uuid)
|
||||
:content result}))
|
||||
|
||||
(defn build-blocks-indice
|
||||
[repo]
|
||||
(->> (db/get-all-block-contents)
|
||||
(map block->index)
|
||||
(remove nil?)
|
||||
(bean/->js)))
|
||||
|
||||
(defn make-blocks-indice!
|
||||
[repo]
|
||||
(let [blocks (->> (db/get-all-block-contents)
|
||||
(map block->index)
|
||||
(remove nil?)
|
||||
(bean/->js))
|
||||
(let [blocks (build-blocks-indice repo)
|
||||
indice (fuse. blocks
|
||||
(clj->js {:keys ["uuid" "content"]
|
||||
:shouldSort true
|
||||
|
|
|
@ -1,12 +1,26 @@
|
|||
(ns frontend.search.node
|
||||
(:require [frontend.search.protocol :as protocol]
|
||||
[frontend.util :as util]))
|
||||
|
||||
;; sqlite3
|
||||
[frontend.util :as util]
|
||||
[electron.ipc :as ipc]
|
||||
[cljs-bean.core :as bean]
|
||||
[frontend.search.db :as search-db]
|
||||
[frontend.db :as db]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defrecord Node [repo]
|
||||
protocol/Engine
|
||||
(query [this q option])
|
||||
(rebuild-blocks-indice! [this])
|
||||
(transact-blocks! [this data])
|
||||
(truncate-blocks! [this]))
|
||||
(query [this q {:keys [limit]
|
||||
:or {limit 20}}]
|
||||
(p/let [result (ipc/ipc "search-blocks" repo q limit)
|
||||
result (bean/->clj result)]
|
||||
(map (fn [{:keys [content id uuid]}]
|
||||
{:block/uuid uuid
|
||||
:block/content content
|
||||
:block/page (:block/page (db/entity id))}) result)))
|
||||
(rebuild-blocks-indice! [this]
|
||||
(let [indice (search-db/build-blocks-indice repo)]
|
||||
(ipc/ipc "rebuild-blocks-indice" repo indice)))
|
||||
(transact-blocks! [this data]
|
||||
(ipc/ipc "transact-blocks" repo (bean/->js data)))
|
||||
(truncate-blocks! [this]
|
||||
(ipc/ipc "truncate-blocks" repo)))
|
||||
|
|
Loading…
Reference in New Issue