greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(search): single database done

pull/1631/head
Tienson Qin 2021-04-03 21:12:35 +08:00
parent 2d71389fe5
commit 2fdfd0dcb1
7 changed files with 109 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/electron/electron/core.cljs
View File

@ -1,5 +1,6 @@
(ns electron.core
(:require [electron.handler :as handler]
[electron.search :as search]
[electron.updater :refer [init-updater]]
[electron.utils :refer [mac? win32? prod? dev? logger open]]
[clojure.string :as string]
@ -148,7 +149,9 @@
(defn main
[]
(if-not (.requestSingleInstanceLock app)
(.quit app)
(do
(search/close!)
(.quit app))
(do
(.on app "second-instance"
(fn [_event _commandLine _workingDirectory]
@ -156,7 +159,9 @@
(when (.isMinimized ^object win)
(.restore win))
(.focus win))))
(.on app "window-all-closed" (fn [] (.quit app)))
(.on app "window-all-closed" (fn []
(search/close!)
(.quit app)))
(.on app "ready"
(fn []
(let [^js win (create-main-window)
@ -164,6 +169,8 @@
*quitting? (atom false)]
(.. logger (info (str "Logseq App(" (.getVersion app) ") Starting... ")))
(search/open-db!)
(vreset! *setup-fn
(fn []
(let [t0 (setup-updater! win)

22
src/electron/electron/handler.cljs
View File

@ -89,7 +89,6 @@
(remove nil?))]
(vec (cons {:path (fix-win-path! path)} result))))
;; TODO: Is it going to be slow if it's a huge directory
(defmethod handle :openDir [^js window _messages]
(let [result (.showOpenDialogSync dialog (bean/->js
{:properties ["openDirectory" "createDirectory" "promptToCreate"]}))
@ -106,16 +105,23 @@
(async/put! state/persistent-dbs-chan true )
true)
(defmethod handle :search [window [_ q]]
(search/search-blocks q))
(defmethod handle :search-blocks [window [_ repo q limit]]
(search/search-blocks q limit))
(defmethod handle :upsert-blocks [window [_ blocks]]
(search/upsert-blocks! blocks))
(defmethod handle :rebuild-blocks-indice [window [_ repo data]]
(search/truncate-blocks-table!)
;; unneeded serialization
(search/upsert-blocks! (bean/->js data)))
(defmethod handle :delete-blocks [window [_ block-ids]]
(search/delete-blocks! block-ids))
(defmethod handle :transact-blocks [window [_ repo data]]
(let [{:keys [blocks-to-remove-set blocks-to-add]} data]
(when (seq blocks-to-remove-set)
(search/delete-blocks! blocks-to-remove-set))
(when (seq blocks-to-add)
;; unneeded serialization
(search/upsert-blocks! (bean/->js blocks-to-add)))))
(defmethod handle :truncate-blocks [window _]
(defmethod handle :truncate-blocks [window [_ repo]]
(search/truncate-blocks-table!))
(defn- get-file-ext

43
src/electron/electron/search.cljs
View File

@ -8,6 +8,12 @@
(defonce database (atom nil))
(defn close!
[]
(when @database
(.close @database)
(reset! database nil)))
(defn prepare
[^object db sql]
(.prepare db sql))
@ -38,14 +44,15 @@
(defn create-blocks-table!
[db]
(let [stmt (prepare db "CREATE TABLE blocks (
id TEXT PRIMARY KEY,
text TEXT NOT NULL)")]
(let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks (
id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL,
content TEXT NOT NULL)")]
(.run ^object stmt)))
(defn create-blocks-fts-table!
[db]
(let [stmt (prepare db "CREATE VIRTUAL TABLE blocks_fts USING fts5(id, text)")]
(let [stmt (prepare db "CREATE VIRTUAL TABLE blocks_fts USING fts5(id, uuid, content)")]
(.run ^object stmt)))
(defn open-db!
@ -66,14 +73,16 @@
(reset! database db)
db))
(defonce debug-blocks (atom nil))
(defn upsert-blocks!
[blocks]
(reset! debug-blocks blocks)
(when-let [db @database]
(let [insert (prepare db "INSERT INTO blocks (id, text) VALUES (@id, @text) ON CONFLICT (id) DO UPDATE SET text = @text")
;; TODO: what if a CONFLICT on uuid
(let [insert (prepare db "INSERT INTO blocks (id, uuid, content) VALUES (@id, @uuid, @content) ON CONFLICT (id) DO UPDATE SET content = @content")
insert-many (.transaction ^object db
(fn [blocks]
(doseq [block blocks]
(prn {:block block})
(.run ^object insert block))))]
(insert-many blocks))))
@ -83,7 +92,6 @@
(let [sql (utils/format "DELETE from blocks WHERE id IN (%s)"
(->> (map (fn [id] (str "'" id "'")) ids)
(string/join ", ")))
_ (prn {:sql sql})
stmt (prepare db sql)]
(.run ^object stmt))))
@ -97,15 +105,15 @@
;; [q]
;; (when-not (string/blank? q)
;; (let [stmt (prepare @database
;; "select id, text from blocks_fts where text match ? ORDER BY rank")]
;; "select id, uuid, content from blocks_fts where content match ? ORDER BY rank")]
;; (js->clj (.all ^object stmt q) :keywordize-keys true))))
(defn search-blocks
[q]
[q limit]
(when-not (string/blank? q)
(let [stmt (prepare @database
"select id, text from blocks_fts where text like ?")]
(js->clj (.all ^object stmt (str "%" q "%")) :keywordize-keys true))))
"select id, uuid, content from blocks where content like ? limit ?")]
(js->clj (.all ^object stmt (str "%" q "%") limit) :keywordize-keys true))))
(defn truncate-blocks-table!
[]
@ -129,17 +137,24 @@
(open-db!)
(add-blocks! (clj->js [{:id "a"
:text "hello world"}
:uuid ""
:content "hello world"}
{:id "b"
:text "foo bar"}]))
:uuid ""
:content "foo bar"}]))
(time
(let [blocks (for [i (range 10000)]
{:id (str i)
:text (rand-nth ["hello" "world" "nice"])})]
:uuid ""
:content (rand-nth ["hello" "world" "nice"])})]
(add-blocks! (clj->js blocks))))
(get-all-blocks)
(search-blocks "hello")
(def block {:id 16, :uuid "5f713e91-8a3c-4b04-a33a-c39482428e2d", :content "Hello, I'm a block!"})
(add-blocks! (clj->js [block]))
)

40
src/main/frontend/components/editor.cljs
View File

@ -16,6 +16,7 @@
[goog.object :as gobj]
[goog.dom :as gdom]
[clojure.string :as string]
[promesa.core :as p]
[frontend.commands :as commands
:refer [*show-commands
*matched-commands
@ -89,6 +90,30 @@
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a page"]
:class "black"}))))))
(rum/defcs block-search-auto-complete < rum/reactive
{:init (fn [state]
(assoc state ::result (atom nil)))
:did-update (fn [state]
(let [result (::result state)
[edit-block _ _ q] (:rum/args state)]
(p/let [matched-blocks (when-not (string/blank? q)
(editor-handler/get-matched-blocks q (:block/uuid edit-block)))]
(reset! result matched-blocks)))
state)}
[state edit-block input id q format content]
(let [result (rum/react (get state ::result))
chosen-handler (editor-handler/block-on-chosen-handler input id q format)
non-exist-block-handler (editor-handler/block-non-exist-handler input)]
(when result
(ui/auto-complete
result
{:on-chosen chosen-handler
:on-enter non-exist-block-handler
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a block"]
:item-render (fn [{:block/keys [content]}]
content)
:class "black"}))))
(rum/defcs block-search < rum/reactive
{:will-unmount (fn [state]
(reset! editor-handler/*selected-text nil)
@ -106,19 +131,8 @@
@editor-handler/*selected-text
(when (> (count edit-content) current-pos)
(subs edit-content pos current-pos)))]
(p/let [matched-blocks (when-not (string/blank? q)
(editor-handler/get-matched-blocks q (:block/uuid edit-block)))]
(when input
(let [chosen-handler (editor-handler/block-on-chosen-handler input id q format)
non-exist-block-handler (editor-handler/block-non-exist-handler input)]
(ui/auto-complete
matched-blocks
{:on-chosen chosen-handler
:on-enter non-exist-block-handler
:empty-div [:div.text-gray-500.pl-4.pr-4 "Search for a block"]
:item-render (fn [{:block/keys [content]}]
(subs content 0 64))
:class "black"})))))))
(when input
(block-search-auto-complete edit-block input id q format)))))
(rum/defc template-search < rum/reactive
{:will-unmount (fn [state] (reset! editor-handler/*selected-text nil) state)}

2
src/main/frontend/handler/editor.cljs
View File

@ -1756,7 +1756,7 @@
99)
(map (comp str :block/uuid))))
current-and-parents (set/union #{(str (:block/uuid current-block))} block-parents)]
(p/let [result (search/block-search (state/get-current-repo) q {:limit 10})]
(p/let [result (search/block-search (state/get-current-repo) q {:limit 20})]
(remove
(fn [h]
(contains? current-and-parents (:block/uuid h)))

12
src/main/frontend/search/db.cljs
View File

@ -21,12 +21,16 @@
:uuid (str uuid)
:content result}))
(defn build-blocks-indice
[repo]
(->> (db/get-all-block-contents)
(map block->index)
(remove nil?)
(bean/->js)))
(defn make-blocks-indice!
[repo]
(let [blocks (->> (db/get-all-block-contents)
(map block->index)
(remove nil?)
(bean/->js))
(let [blocks (build-blocks-indice repo)
indice (fuse. blocks
(clj->js {:keys ["uuid" "content"]
:shouldSort true

28
src/main/frontend/search/node.cljs
View File

@ -1,12 +1,26 @@
(ns frontend.search.node
(:require [frontend.search.protocol :as protocol]
[frontend.util :as util]))
;; sqlite3
[frontend.util :as util]
[electron.ipc :as ipc]
[cljs-bean.core :as bean]
[frontend.search.db :as search-db]
[frontend.db :as db]
[promesa.core :as p]))
(defrecord Node [repo]
protocol/Engine
(query [this q option])
(rebuild-blocks-indice! [this])
(transact-blocks! [this data])
(truncate-blocks! [this]))
(query [this q {:keys [limit]
:or {limit 20}}]
(p/let [result (ipc/ipc "search-blocks" repo q limit)
result (bean/->clj result)]
(map (fn [{:keys [content id uuid]}]
{:block/uuid uuid
:block/content content
:block/page (:block/page (db/entity id))}) result)))
(rebuild-blocks-indice! [this]
(let [indice (search-db/build-blocks-indice repo)]
(ipc/ipc "rebuild-blocks-indice" repo indice)))
(transact-blocks! [this data]
(ipc/ipc "transact-blocks" repo (bean/->js data)))
(truncate-blocks! [this]
(ipc/ipc "truncate-blocks" repo)))