mirror of https://github.com/logseq/logseq
refactor: blocks search returns all blocks including pages
Fuzzy search support both pages and objects (blocks have tags).pull/11433/head
Tienson Qin
parent
36fed3e212
commit
ed1ee94d88
|
|
@ -257,9 +257,7 @@
|
|||
property i.e. what the user sees. For page types the content is the page name"
|
||||
[ent]
|
||||
(or (:block/title ent)
|
||||
(if-some [content (:property.value/content ent)]
|
||||
content
|
||||
(:block/title ent))))
|
||||
(:property.value/content ent)))
|
||||
|
||||
(defn ref->property-value-content
|
||||
"Given a ref from a pulled query e.g. `{:db/id X}`, gets a readable name for
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
[goog.functions :as gfun]
|
||||
[goog.object :as gobj]
|
||||
[logseq.shui.ui :as shui]
|
||||
[logseq.db :as ldb]
|
||||
[promesa.core :as p]
|
||||
[rum.core :as rum]
|
||||
[frontend.mixins :as mixins]
|
||||
|
|
@ -46,8 +47,7 @@
|
|||
(def GROUP-LIMIT 5)
|
||||
|
||||
(def search-actions
|
||||
[{:filter {:group :pages :mode "search"} :text "Search only pages" :info "Add filter to search" :icon-theme :gray :icon "page"}
|
||||
{:filter {:group :current-page} :text "Search only current page" :info "Add filter to search" :icon-theme :gray :icon "page"}
|
||||
[{:filter {:group :current-page} :text "Search only current page" :info "Add filter to search" :icon-theme :gray :icon "page"}
|
||||
{:filter {:group :blocks} :text "Search only blocks" :info "Add filter to search" :icon-theme :gray :icon "block"}
|
||||
{:filter {:group :commands} :text "Search only commands" :info "Add filter to search" :icon-theme :gray :icon "command"}
|
||||
{:filter {:group :whiteboards} :text "Search only whiteboards" :info "Add filter to search" :icon-theme :gray :icon "whiteboard"}
|
||||
|
|
@ -61,7 +61,6 @@
|
|||
{:commands {:status :success :show :less :items nil}
|
||||
:favorites {:status :success :show :less :items nil}
|
||||
:current-page {:status :success :show :less :items nil}
|
||||
:pages {:status :success :show :less :items nil}
|
||||
:blocks {:status :success :show :less :items nil}
|
||||
:files {:status :success :show :less :items nil}
|
||||
:themes {:status :success :show :less :items nil}
|
||||
|
|
@ -108,17 +107,13 @@
|
|||
page-exists? (when-not (string/blank? input)
|
||||
(db/get-page (string/trim input)))
|
||||
include-slash? (or (string/includes? input "/")
|
||||
(string/starts-with? input "/"))
|
||||
(string/starts-with? input "/"))
|
||||
order* (cond
|
||||
(= search-mode :graph)
|
||||
[["Pages" :pages (visible-items :pages)]]
|
||||
[]
|
||||
|
||||
include-slash?
|
||||
[
|
||||
(when page-exists?
|
||||
["Pages" :pages (visible-items :pages)])
|
||||
|
||||
(when-not page-exists?
|
||||
[(when-not page-exists?
|
||||
["Create" :create (create-items input)])
|
||||
|
||||
["Current page" :current-page (visible-items :current-page)]
|
||||
|
|
@ -132,21 +127,19 @@
|
|||
[(if (= filter-group :current-page) "Current page" (name filter-group))
|
||||
filter-group
|
||||
(visible-items filter-group)]
|
||||
(when (= filter-group :pages)
|
||||
(when-not page-exists?
|
||||
["Create" :create (create-items input)]))]
|
||||
(when-not page-exists?
|
||||
["Create" :create (create-items input)])]
|
||||
|
||||
:else
|
||||
(->>
|
||||
[["Pages" :pages (visible-items :pages)]
|
||||
(when-not page-exists?
|
||||
["Create" :create (create-items input)])
|
||||
["Commands" :commands (visible-items :commands)]
|
||||
["Current page" :current-page (visible-items :current-page)]
|
||||
["Blocks" :blocks (visible-items :blocks)]
|
||||
["Files" :files (visible-items :files)]
|
||||
["Filters" :filters (visible-items :filters)]]
|
||||
(remove nil?)))
|
||||
[(when-not page-exists?
|
||||
["Create" :create (create-items input)])
|
||||
["Current page" :current-page (visible-items :current-page)]
|
||||
["Blocks" :blocks (visible-items :blocks)]
|
||||
["Commands" :commands (visible-items :commands)]
|
||||
["Files" :files (visible-items :files)]
|
||||
["Filters" :filters (visible-items :filters)]]
|
||||
(remove nil?)))
|
||||
order (remove nil? order*)]
|
||||
(for [[group-name group-key group-items] order]
|
||||
[group-name
|
||||
|
|
@ -207,27 +200,6 @@
|
|||
(hash-map :status :success :items)
|
||||
(swap! !results update group merge)))))
|
||||
|
||||
;; The pages search action uses an existing handler
|
||||
(defmethod load-results :pages [group state]
|
||||
(let [!input (::input state)
|
||||
!results (::results state)
|
||||
repo (state/get-current-repo)]
|
||||
(swap! !results assoc-in [group :status] :loading)
|
||||
(p/let [pages (search/page-search @!input)
|
||||
items (->> pages
|
||||
(map
|
||||
(fn [page]
|
||||
(let [entity (db/entity [:block/uuid (uuid (:id page))])
|
||||
whiteboard? (contains? (:block/type entity) "whiteboard")
|
||||
source-page (model/get-alias-source-page repo (:db/id entity))]
|
||||
(hash-map :icon (if whiteboard? "whiteboard" "page")
|
||||
:icon-theme :gray
|
||||
:text (:title page)
|
||||
:source-page (if source-page
|
||||
(:block/title source-page)
|
||||
(:title page)))))))]
|
||||
(swap! !results update group merge {:status :success :items items}))))
|
||||
|
||||
(defmethod load-results :whiteboards [group state]
|
||||
(let [!input (::input state)
|
||||
!results (::results state)]
|
||||
|
|
@ -258,14 +230,36 @@
|
|||
hiccups-add [(when-not (string/blank? b-cut)
|
||||
[:span b-cut])
|
||||
(when-not (string/blank? hl-cut)
|
||||
(let [hl-cut' (string/trimr hl-cut)]
|
||||
[:mark.p-0.rounded-none hl-cut']))]
|
||||
[:mark.p-0.rounded-none hl-cut])]
|
||||
hiccups-add (remove nil? hiccups-add)
|
||||
new-result (concat result hiccups-add)]
|
||||
(if-not (string/blank? e-cut)
|
||||
(recur e-cut new-result)
|
||||
new-result)))]))
|
||||
|
||||
(defn- page-item
|
||||
[repo page]
|
||||
(let [entity (db/entity [:block/uuid (:block/uuid page)])
|
||||
whiteboard? (contains? (:block/type entity) "whiteboard")
|
||||
source-page (model/get-alias-source-page repo (:db/id entity))]
|
||||
(hash-map :icon (if whiteboard? "whiteboard" "page")
|
||||
:icon-theme :gray
|
||||
:text (:block/title page)
|
||||
:source-page (if source-page
|
||||
(:block/title source-page)
|
||||
(:block/title page)))))
|
||||
|
||||
(defn- block-item
|
||||
[repo block current-page !input]
|
||||
(let [id (:block/uuid block)]
|
||||
{:icon "block"
|
||||
:icon-theme :gray
|
||||
:text (highlight-content-query (:block/title block) @!input)
|
||||
:header (block/breadcrumb {:search? true} repo id {})
|
||||
:current-page? (when-let [page-id (:block/page block)]
|
||||
(= page-id (:block/uuid current-page)))
|
||||
:source-block block}))
|
||||
|
||||
;; The blocks search action uses an existing handler
|
||||
(defmethod load-results :blocks [group state]
|
||||
(let [!input (::input state)
|
||||
|
|
@ -278,15 +272,10 @@
|
|||
(swap! !results assoc-in [:current-page :status] :loading)
|
||||
(p/let [blocks (search/block-search repo @!input opts)
|
||||
blocks (remove nil? blocks)
|
||||
items (map (fn [block]
|
||||
(let [id (:block/uuid block)]
|
||||
{:icon "block"
|
||||
:icon-theme :gray
|
||||
:text (highlight-content-query (:block/title block) @!input)
|
||||
:header (block/breadcrumb {:search? true} repo id {})
|
||||
:current-page? (when-let [page-id (:block/page block)]
|
||||
(= page-id (:block/uuid current-page)))
|
||||
:source-block block})) blocks)
|
||||
items (keep (fn [block]
|
||||
(if (:page? block)
|
||||
(page-item repo block)
|
||||
(block-item repo block current-page !input))) blocks)
|
||||
items-on-other-pages (remove :current-page? items)
|
||||
items-on-current-page (filter :current-page? items)]
|
||||
(swap! !results update group merge {:status :success :items items-on-other-pages})
|
||||
|
|
@ -387,7 +376,6 @@
|
|||
(do
|
||||
(load-results :commands state)
|
||||
(load-results :blocks state)
|
||||
(load-results :pages state)
|
||||
(load-results :filters state)
|
||||
(load-results :files state)
|
||||
;; (load-results :recents state)
|
||||
|
|
@ -825,7 +813,7 @@
|
|||
backspace? (= (util/ekey e) "Backspace")
|
||||
filter-group (:group @(::filter state))
|
||||
slash? (= (util/ekey e) "/")
|
||||
namespace-pages (when (and slash? (contains? #{:pages :whiteboards} filter-group))
|
||||
namespace-pages (when (and slash? (contains? #{:whiteboards} filter-group))
|
||||
(search/page-search (str value "/")))
|
||||
namespace-page-matched? (some #(string/includes? % "/") namespace-pages)]
|
||||
(when (and filter-group
|
||||
|
|
@ -1010,7 +998,7 @@
|
|||
(or (= group-filter group-key)
|
||||
(and (= group-filter :blocks)
|
||||
(= group-key :current-page))
|
||||
(and (contains? #{:pages :create} group-filter)
|
||||
(and (contains? #{:create} group-filter)
|
||||
(= group-key :create))))))
|
||||
results-ordered)]
|
||||
(if (seq items)
|
||||
|
|
|
|||
|
|
@ -483,8 +483,9 @@
|
|||
;; Search
|
||||
(search-blocks
|
||||
[this repo q option]
|
||||
(p/let [db (get-search-db repo)
|
||||
result (search/search-blocks db q (bean/->clj option))]
|
||||
(p/let [search-db (get-search-db repo)
|
||||
conn (worker-state/get-datascript-conn repo)
|
||||
result (search/search-blocks repo conn search-db q (bean/->clj option))]
|
||||
(bean/->js result)))
|
||||
|
||||
(search-upsert-blocks
|
||||
|
|
@ -508,18 +509,11 @@
|
|||
(search-build-blocks-indice
|
||||
[this repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(search/build-blocks-indice @conn)))
|
||||
(search/build-blocks-indice repo @conn)))
|
||||
|
||||
(search-build-pages-indice
|
||||
[this repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(search/build-page-indice repo @conn)
|
||||
nil))
|
||||
|
||||
(page-search
|
||||
[this repo q options]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(search/page-search repo @conn q (bean/->clj options))))
|
||||
nil)
|
||||
|
||||
(apply-outliner-ops
|
||||
[this repo ops-str opts-str]
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@
|
|||
|
||||
(defn search-handler
|
||||
[q filters]
|
||||
(let [{:keys [pages? blocks?]} (js->clj filters {:keywordize-keys true})
|
||||
(let [{:keys [blocks?]} (js->clj filters {:keywordize-keys true})
|
||||
repo (state/get-current-repo)
|
||||
limit 100]
|
||||
(p/let [blocks (when blocks? (search/block-search repo q {:limit limit}))
|
||||
|
|
@ -77,9 +77,8 @@
|
|||
(-> b
|
||||
(update :block/uuid str)
|
||||
(update :block/title #(->> (text-util/cut-by % "$pfts_2lqh>$" "$<pfts_2lqh$")
|
||||
(apply str))))) blocks)
|
||||
pages (when pages? (search/page-search q))]
|
||||
(clj->js {:pages pages :blocks blocks}))))
|
||||
(apply str))))) blocks)]
|
||||
(clj->js {:blocks blocks}))))
|
||||
|
||||
(defn save-asset-handler
|
||||
[file]
|
||||
|
|
|
|||
|
|
@ -1654,14 +1654,6 @@
|
|||
pages)
|
||||
(map :title))))
|
||||
|
||||
(comment
|
||||
(defn get-matched-classes
|
||||
"Return matched class names"
|
||||
[q]
|
||||
(let [classes (->> (db-model/get-all-classes (state/get-current-repo))
|
||||
(map first))]
|
||||
(search/fuzzy-search classes q {:limit 100}))))
|
||||
|
||||
(defn get-matched-blocks
|
||||
[q block-id]
|
||||
;; remove current block
|
||||
|
|
|
|||
|
|
@ -33,14 +33,12 @@
|
|||
page-db-id)
|
||||
opts (if page-db-id (assoc opts :page (str page-db-id)) opts)]
|
||||
(p/let [blocks (search/block-search repo q opts)
|
||||
pages (search/page-search q)
|
||||
files (search/file-search q)]
|
||||
(let [result (merge
|
||||
{:blocks blocks
|
||||
:has-more? (= limit (count blocks))}
|
||||
(when-not page-db-id
|
||||
{:pages pages
|
||||
:files files}))
|
||||
{:files files}))
|
||||
search-key (if more? :search/more-result :search/result)]
|
||||
(swap! state/state assoc search-key result)
|
||||
result))))))
|
||||
|
|
|
|||
|
|
@ -34,10 +34,8 @@
|
|||
(defn page-search
|
||||
([q]
|
||||
(page-search q {}))
|
||||
([q options]
|
||||
(when-let [^js sqlite @search-browser/*sqlite]
|
||||
(p/let [result (.page-search sqlite (state/get-current-repo) q (clj->js (merge {:limit 100} options)))]
|
||||
(bean/->clj result)))))
|
||||
([q option]
|
||||
(when-not (string/blank? q) (block-search (state/get-current-repo) q option))))
|
||||
|
||||
(defn file-search
|
||||
([q]
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@
|
|||
[frontend.worker.util :as worker-util]
|
||||
[logseq.db.sqlite.util :as sqlite-util]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.property :as db-property]))
|
||||
[logseq.db :as ldb]))
|
||||
|
||||
;; TODO: use sqlite for fuzzy search
|
||||
(defonce indices (atom nil))
|
||||
;; maybe https://github.com/nalgeon/sqlean/blob/main/docs/fuzzy.md?
|
||||
(defonce fuzzy-search-indices (atom {}))
|
||||
|
||||
(defn- add-blocks-fts-triggers!
|
||||
"Table bindings of blocks tables and the blocks FTS virtual tables"
|
||||
|
|
@ -125,7 +125,11 @@
|
|||
:rowMode "array"}))
|
||||
blocks (bean/->clj result)]
|
||||
(map (fn [block]
|
||||
(update block 3 get-snippet-result)) blocks))
|
||||
(let [[id page title snippet] (update block 3 get-snippet-result)]
|
||||
{:id id
|
||||
:page page
|
||||
:title title
|
||||
:snippet snippet})) blocks))
|
||||
(catch :default e
|
||||
(prn :debug "Search blocks failed: ")
|
||||
(js/console.error e))))
|
||||
|
|
@ -142,9 +146,116 @@
|
|||
(string/replace match-input "," "")
|
||||
(str "\"" match-input "\"*"))))
|
||||
|
||||
(defn exact-matched?
|
||||
"Check if two strings points toward same search result"
|
||||
[q match]
|
||||
(when (and (string? q) (string? match))
|
||||
(boolean
|
||||
(reduce
|
||||
(fn [coll char]
|
||||
(let [coll' (drop-while #(not= char %) coll)]
|
||||
(if (seq coll')
|
||||
(rest coll')
|
||||
(reduced false))))
|
||||
(seq (worker-util/search-normalize match true))
|
||||
(seq (worker-util/search-normalize q true))))))
|
||||
|
||||
(defn- hidden-page?
|
||||
[page]
|
||||
(when page
|
||||
(if (string? page)
|
||||
(string/starts-with? page "$$$")
|
||||
(contains? (set (:block/type page)) "hidden"))))
|
||||
|
||||
(defn- page-or-object?
|
||||
[entity]
|
||||
(and (or (ldb/page? entity) (seq (:block/tags entity)))
|
||||
(not (hidden-page? entity))))
|
||||
|
||||
(defn get-all-fuzzy-supported-blocks
|
||||
"Only pages and objects are supported now."
|
||||
[db]
|
||||
(let [page-ids (->> (d/datoms db :avet :block/name)
|
||||
(map :e))
|
||||
object-ids (->> (d/datoms db :avet :block/tags)
|
||||
(map :e))
|
||||
blocks (->> (distinct (concat page-ids object-ids))
|
||||
(map #(d/entity db %)))]
|
||||
(remove hidden-page? blocks)))
|
||||
|
||||
(defn- sanitize
|
||||
[content]
|
||||
(some-> content
|
||||
(worker-util/search-normalize true)))
|
||||
|
||||
(defn block->index
|
||||
"Convert a block to the index for searching"
|
||||
[{:block/keys [uuid page title format] :as block}]
|
||||
(when-not (or
|
||||
(ldb/closed-value? block)
|
||||
(and (string? title) (> (count title) 10000))
|
||||
(string/blank? title)) ; empty page or block
|
||||
;; Should properties be included in the search indice?
|
||||
;; It could slow down the search indexing, also it can be confusing
|
||||
;; if the showing properties are not useful to users.
|
||||
;; (let [content (if (and db-based? (seq (:block/properties block)))
|
||||
;; (str content (when (not= content "") "\n") (get-db-properties-str db properties))
|
||||
;; content)])
|
||||
(when uuid
|
||||
{:id (str uuid)
|
||||
:page (str (or (:block/uuid page) uuid))
|
||||
:title (sanitize title)
|
||||
:built-in? (ldb/built-in? block)
|
||||
:format format})))
|
||||
|
||||
(defn build-fuzzy-search-indice
|
||||
"Build a block title indice from scratch.
|
||||
Incremental page title indice is implemented in frontend.search.sync-search-indice!"
|
||||
[repo db]
|
||||
(let [blocks (->> (get-all-fuzzy-supported-blocks db)
|
||||
(map block->index)
|
||||
(bean/->js))
|
||||
indice (fuse. blocks
|
||||
(clj->js {:keys ["title"]
|
||||
:shouldSort true
|
||||
:tokenize true
|
||||
:distance 1024
|
||||
:threshold 0.5 ;; search for 50% match from the start
|
||||
:minMatchCharLength 1}))]
|
||||
(swap! fuzzy-search-indices assoc-in repo indice)
|
||||
indice))
|
||||
|
||||
(defn fuzzy-search
|
||||
"Return a list of blocks (pages && tagged blocks) that match the query. Takes the following
|
||||
options:
|
||||
* :limit - Number of result to limit search results. Defaults to 100
|
||||
* :built-in? - Whether to return built-in pages for db graphs. Defaults to true"
|
||||
[repo db q {:keys [limit built-in?]
|
||||
:or {limit 100
|
||||
built-in? true}}]
|
||||
(when repo
|
||||
(let [q (worker-util/search-normalize q true)
|
||||
q (fuzzy/clean-str q)
|
||||
q (if (= \# (first q)) (subs q 1) q)]
|
||||
(when-not (string/blank? q)
|
||||
(let [indice (or (get @fuzzy-search-indices repo)
|
||||
(build-fuzzy-search-indice repo db))
|
||||
result (cond->>
|
||||
(->> (.search indice q (clj->js {:limit limit}))
|
||||
(bean/->clj))
|
||||
|
||||
(and (sqlite-util/db-based-graph? repo) (= false built-in?))
|
||||
(remove #(get-in % [:item :built-in?])))]
|
||||
(->> (map :item result)
|
||||
(filter (fn [{:keys [title]}]
|
||||
(exact-matched? q title)))))))))
|
||||
|
||||
(defn search-blocks
|
||||
":page - the page to specifically search on"
|
||||
[db q {:keys [limit page]}]
|
||||
"Options:
|
||||
* :page - the page to specifically search on
|
||||
* :limit - Number of result to limit search results. Defaults to 100
|
||||
* :built-in? - Whether to return built-in pages for db graphs. Defaults to true"
|
||||
[repo conn search-db q {:keys [limit page] :as option}]
|
||||
(when-not (string/blank? q)
|
||||
(p/let [match-input (get-match-input q)
|
||||
limit (or limit 100)
|
||||
|
|
@ -152,33 +263,27 @@
|
|||
;; the 2nd column in blocks_fts (content)
|
||||
;; pfts_2lqh is a key for retrieval
|
||||
;; highlight and snippet only works for some matching with high rank
|
||||
snippet-aux "snippet(blocks_fts, 1, ' $pfts_2lqh>$ ', ' $<pfts_2lqh$ ', '...', 32)"
|
||||
snippet-aux "snippet(blocks_fts, 1, '$pfts_2lqh>$', '$<pfts_2lqh$', '...', 32)"
|
||||
select (str "select id, page, title, " snippet-aux " from blocks_fts where ")
|
||||
pg-sql (if page "page = ? and" "")
|
||||
match-sql (str select
|
||||
pg-sql
|
||||
" title match ? order by rank limit ?")
|
||||
matched-result (search-blocks-aux db match-sql match-input page limit)
|
||||
all-result (->> matched-result
|
||||
matched-result (search-blocks-aux search-db match-sql match-input page limit)
|
||||
fuzzy-result (when-not page (fuzzy-search repo @conn q option))
|
||||
all-result (->> (concat fuzzy-result matched-result)
|
||||
(map (fn [result]
|
||||
(let [[id page _title snippet] result]
|
||||
(let [{:keys [id page title snippet]} result]
|
||||
{:uuid id
|
||||
:title snippet
|
||||
:title (or snippet title)
|
||||
:page page}))))]
|
||||
(->>
|
||||
all-result
|
||||
(common-util/distinct-by :uuid)))))
|
||||
(common-util/distinct-by :uuid all-result))))
|
||||
|
||||
(defn truncate-table!
|
||||
[db]
|
||||
(.exec db "delete from blocks")
|
||||
(.exec db "delete from blocks_fts"))
|
||||
|
||||
(defn- sanitize
|
||||
[content]
|
||||
(some-> content
|
||||
(worker-util/search-normalize true)))
|
||||
|
||||
(comment
|
||||
(defn- property-value-when-closed
|
||||
"Returns property value if the given entity is type 'closed value' or nil"
|
||||
|
|
@ -217,26 +322,6 @@
|
|||
(string/join "; " values))))))
|
||||
(string/join ", "))))
|
||||
|
||||
(defn block->index
|
||||
"Convert a block to the index for searching"
|
||||
[{:block/keys [uuid page title format] :as block}]
|
||||
(when-not (or
|
||||
(ldb/closed-value? block)
|
||||
(and (string? title) (> (count title) 10000))
|
||||
(string/blank? title)) ; empty page or block
|
||||
;; Should properties be included in the search indice?
|
||||
;; It could slow down the search indexing, also it can be confusing
|
||||
;; if the showing properties are not useful to users.
|
||||
;; (let [content (if (and db-based? (seq (:block/properties block)))
|
||||
;; (str content (when (not= content "") "\n") (get-db-properties-str db properties))
|
||||
;; content)])
|
||||
(when uuid
|
||||
{:id (str uuid)
|
||||
:page (str (or (:block/uuid page) uuid))
|
||||
:title (sanitize title)
|
||||
:format format})))
|
||||
|
||||
|
||||
(defn get-all-block-contents
|
||||
[db]
|
||||
(when db
|
||||
|
|
@ -245,51 +330,12 @@
|
|||
(keep #(d/entity db [:block/uuid %])))))
|
||||
|
||||
(defn build-blocks-indice
|
||||
[db]
|
||||
[repo db]
|
||||
(build-fuzzy-search-indice repo db)
|
||||
(->> (get-all-block-contents db)
|
||||
(keep block->index)
|
||||
(bean/->js)))
|
||||
|
||||
(defn original-page-name->index
|
||||
[p]
|
||||
(when p
|
||||
{:id (str (:block/uuid p))
|
||||
:name (:block/name p)
|
||||
:built-in? (boolean (db-property/property-value-content (:logseq.property/built-in? p)))
|
||||
:title (:block/title p)}))
|
||||
|
||||
(defn- hidden-page?
|
||||
[page]
|
||||
(when page
|
||||
(if (string? page)
|
||||
(string/starts-with? page "$$$")
|
||||
(contains? (set (:block/type page)) "hidden"))))
|
||||
|
||||
(defn get-all-pages
|
||||
[db]
|
||||
(let [page-datoms (d/datoms db :avet :block/name)
|
||||
pages (map (fn [d] (d/entity db (:e d))) page-datoms)]
|
||||
(remove (fn [p] (hidden-page? (:block/name p))) pages)))
|
||||
|
||||
(defn build-page-indice
|
||||
"Build a page title indice from scratch.
|
||||
Incremental page title indice is implemented in frontend.search.sync-search-indice!
|
||||
Rename from the page indice since 10.25.2022, since this is only used for page title search.
|
||||
From now on, page indice is talking about page content search."
|
||||
[repo db]
|
||||
(let [pages (->> (get-all-pages db)
|
||||
(map original-page-name->index)
|
||||
(bean/->js))
|
||||
indice (fuse. pages
|
||||
(clj->js {:keys ["title"]
|
||||
:shouldSort true
|
||||
:tokenize true
|
||||
:distance 1024
|
||||
:threshold 0.5 ;; search for 50% match from the start
|
||||
:minMatchCharLength 1}))]
|
||||
(swap! indices assoc-in [repo :pages] indice)
|
||||
indice))
|
||||
|
||||
(defn- get-blocks-from-datoms-impl
|
||||
[repo {:keys [db-after db-before]} datoms]
|
||||
(when (seq datoms)
|
||||
|
|
@ -313,7 +359,7 @@
|
|||
(keep #(d/entity db-after %) blocks-to-add-set')
|
||||
(remove hidden-page?))})))
|
||||
|
||||
(defn- get-direct-blocks-and-pages
|
||||
(defn- get-affected-blocks
|
||||
[repo tx-report]
|
||||
(let [data (:tx-data tx-report)
|
||||
datoms (filter
|
||||
|
|
@ -326,19 +372,19 @@
|
|||
|
||||
(defn sync-search-indice
|
||||
[repo tx-report]
|
||||
(let [{:keys [blocks-to-add blocks-to-remove]} (get-direct-blocks-and-pages repo tx-report)]
|
||||
(let [{:keys [blocks-to-add blocks-to-remove]} (get-affected-blocks repo tx-report)]
|
||||
;; update page title indice
|
||||
(let [pages-to-add (filter :block/name blocks-to-add)
|
||||
pages-to-remove (filter :block/name blocks-to-remove)]
|
||||
(when (or (seq pages-to-add) (seq pages-to-remove))
|
||||
(swap! indices update-in [repo :pages]
|
||||
(let [fuzzy-blocks-to-add (filter page-or-object? blocks-to-add)
|
||||
fuzzy-blocks-to-remove (filter page-or-object? blocks-to-remove)]
|
||||
(when (or (seq fuzzy-blocks-to-add) (seq fuzzy-blocks-to-remove))
|
||||
(swap! fuzzy-search-indices update-in repo
|
||||
(fn [indice]
|
||||
(when indice
|
||||
(doseq [page-entity pages-to-remove]
|
||||
(doseq [page-entity fuzzy-blocks-to-remove]
|
||||
(.remove indice (fn [page] (= (str (:block/uuid page-entity)) (gobj/get page "id")))))
|
||||
(doseq [page pages-to-add]
|
||||
(doseq [page fuzzy-blocks-to-add]
|
||||
(.remove indice (fn [p] (= (str (:block/uuid page)) (gobj/get p "id"))))
|
||||
(.add indice (bean/->js (original-page-name->index page))))
|
||||
(.add indice (bean/->js (block->index page))))
|
||||
indice)))))
|
||||
|
||||
;; update block indice
|
||||
|
|
@ -347,48 +393,3 @@
|
|||
blocks-to-remove (set (map (comp str :block/uuid) blocks-to-remove))]
|
||||
{:blocks-to-remove-set blocks-to-remove
|
||||
:blocks-to-add blocks-to-add}))))
|
||||
|
||||
(defn exact-matched?
|
||||
"Check if two strings points toward same search result"
|
||||
[q match]
|
||||
(when (and (string? q) (string? match))
|
||||
(boolean
|
||||
(reduce
|
||||
(fn [coll char]
|
||||
(let [coll' (drop-while #(not= char %) coll)]
|
||||
(if (seq coll')
|
||||
(rest coll')
|
||||
(reduced false))))
|
||||
(seq (worker-util/search-normalize match true))
|
||||
(seq (worker-util/search-normalize q true))))))
|
||||
|
||||
(defn page-search
|
||||
"Return a list of page names that match the query. Takes the following
|
||||
options:
|
||||
* :limit - Number of pages to limit search results. Defaults to 100
|
||||
* :built-in? - Whether to return built-in pages for db graphs. Defaults to true"
|
||||
[repo db q {:keys [limit built-in?]
|
||||
:or {limit 100
|
||||
built-in? true}}]
|
||||
(when repo
|
||||
(let [q (worker-util/search-normalize q true)
|
||||
q (fuzzy/clean-str q)
|
||||
q (if (= \# (first q)) (subs q 1) q)]
|
||||
(when-not (string/blank? q)
|
||||
(let [indice (or (get-in @indices [repo :pages])
|
||||
(build-page-indice repo db))
|
||||
result (cond->>
|
||||
(->> (.search indice q (clj->js {:limit limit}))
|
||||
(bean/->clj))
|
||||
|
||||
(and (sqlite-util/db-based-graph? repo) (= false built-in?))
|
||||
(remove #(get-in % [:item :built-in?])))]
|
||||
(->> result
|
||||
(keep
|
||||
(fn [{:keys [item]}]
|
||||
{:id (:id item)
|
||||
:title (:title item)}))
|
||||
(distinct)
|
||||
(filter (fn [{:keys [title]}]
|
||||
(exact-matched? q title)))
|
||||
bean/->js))))))
|
||||
|
|
|
|||
Loading…
Reference in New Issue