greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move all search related impl to worker 

This commit also introduced a new ns `frontend.db.async` for
async queries.
pull/10770/head
Tienson Qin 2023-12-25 14:32:28 +08:00
parent e6a464e64f
commit b134954e2c
32 changed files with 967 additions and 756 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
shadow-cljs.edn
View File

@ -118,4 +118,3 @@
:devtools {:before-load frontend.core/stop
:after-load frontend.core/start
:preloads [devtools.preload]}}}}

15
src/main/frontend/components/cmdk.cljs
View File

@ -754,13 +754,14 @@
(on-blur input)))
:on-composition-end (fn [e] (handle-input-change state e))
:on-key-down (fn [e]
(let [value (.-value @input-ref)
last-char (last value)
backspace? (= (util/ekey e) "Backspace")
filter-group (:group @(::filter state))
slash? (= (util/ekey e) "/")
namespace-page-matched? (when (and slash? (contains? #{:pages :whiteboards} filter-group))
(some #(string/includes? % "/") (search/page-search (str value "/"))))]
(p/let [value (.-value @input-ref)
last-char (last value)
backspace? (= (util/ekey e) "Backspace")
filter-group (:group @(::filter state))
slash? (= (util/ekey e) "/")
namespace-pages (when (and slash? (contains? #{:pages :whiteboards} filter-group))
(search/page-search (str value "/")))
namespace-page-matched? (some #(string/includes? % "/") namespace-pages)]
(when (and filter-group
(or (and slash? (not namespace-page-matched?))
(and backspace? (= last-char "/"))

22
src/main/frontend/components/content.cljs
View File

@ -31,7 +31,8 @@
[frontend.db.rtc.debug-ui :as rtc-debug-ui]
[cljs.core.async :as async]
[cljs.pprint :as pp]
[cljs-time.coerce :as tc]))
[cljs-time.coerce :as tc]
[promesa.core :as p]))
;; TODO i18n support
@ -156,15 +157,16 @@
:on-click (fn []
(let [title (string/trim @input)]
(when (not (string/blank? title))
(if (page-handler/template-exists? title)
(notification/show!
[:p (t :context-menu/template-exists-warning)]
:error)
(do
(property-handler/set-block-property! repo block-id :template title)
(when (false? template-including-parent?)
(property-handler/set-block-property! repo block-id :template-including-parent false))
(state/hide-custom-context-menu!)))))))]
(p/let [exists? (page-handler/<template-exists? title)]
(if exists?
(notification/show!
[:p (t :context-menu/template-exists-warning)]
:error)
(do
(property-handler/set-block-property! repo block-id :template title)
(when (false? template-including-parent?)
(property-handler/set-block-property! repo block-id :template-including-parent false))
(state/hide-custom-context-menu!))))))))]
[:hr.menu-separator]])
(ui/menu-link
{:key "Make a Template"

190
src/main/frontend/components/editor.cljs
View File

@ -121,12 +121,73 @@
:other-attrs {:block/link (:db/id (db/entity [:block/name page-name]))}}))))
(page-handler/on-chosen-handler input id q pos format)))
(rum/defcs page-search < rum/reactive
(rum/defc page-search-aux
[id format embed? db-tag? create-page? q current-pos edit-content input pos]
(let [[matched-pages set-matched-pages!] (rum/use-state nil)]
(rum/use-effect! (fn []
(when-not (string/blank? q)
(p/let [result (editor-handler/<get-matched-pages q)]
(set-matched-pages! result))))
[q])
(let [matched-pages (cond
(contains? (set (map util/page-name-sanity-lc matched-pages))
(util/page-name-sanity-lc (string/trim q))) ;; if there's a page name fully matched
(sort-by (fn [m]
[(count m) m])
matched-pages)
(string/blank? q)
nil
(empty? matched-pages)
(when-not (db/page-exists? q)
(if db-tag?
(concat [(str (t :new-page) " " q)
(str (t :new-class) " " q)]
matched-pages)
(cons q matched-pages)))
;; reorder, shortest and starts-with first.
:else
(let [matched-pages (remove nil? matched-pages)
matched-pages (sort-by
(fn [m]
[(not (gstring/caseInsensitiveStartsWith m q)) (count m) m])
matched-pages)]
(if (gstring/caseInsensitiveStartsWith (first matched-pages) q)
(cons (first matched-pages)
(cons q (rest matched-pages)))
(cons q matched-pages))))]
[:div
(when (and db-tag?
;; Don't display in heading
(not (some->> edit-content (re-find #"^\s*#"))))
[:div.flex.flex-row.items-center.px-4.py-1.text-sm.opacity-70.gap-2
"Turn this block into a page:"
(ui/toggle create-page?
(fn [_e]
(swap! (:editor/create-page? @state/state) not))
true)])
(ui/auto-complete
matched-pages
{:on-chosen (page-on-chosen-handler embed? input id q pos format)
:on-enter (fn []
(page-handler/page-not-exists-handler input id q current-pos))
:item-render (fn [page-name _chosen?]
[:div.flex
(when (db-model/whiteboard-page? page-name) [:span.mr-1 (ui/icon "whiteboard" {:extension? true})])
(search-handler/highlight-exact-query page-name q)])
:empty-placeholder [:div.text-gray-500.text-sm.px-4.py-2 (if db-tag?
"Search for a page or a class"
"Search for a page")]
:class "black"})])))
(rum/defc page-search < rum/reactive
{:will-unmount (fn [state]
(reset! commands/*current-command nil)
state)}
"Page or tag searching popup"
[state id format]
[id format]
(let [action (state/sub :editor/action)
db? (config/db-based-graph? (state/get-current-repo))
embed? (and db? (= @commands/*current-command "Page embed"))
@ -145,63 +206,8 @@
(gp-util/safe-subs edit-content pos current-pos))
(when (> (count edit-content) current-pos)
(gp-util/safe-subs edit-content pos current-pos))
"")
;; FIXME: display refed pages recentedly or frequencyly used
matched-pages (when-not (string/blank? q)
(editor-handler/get-matched-pages q))
matched-pages (cond
(contains? (set (map util/page-name-sanity-lc matched-pages))
(util/page-name-sanity-lc (string/trim q))) ;; if there's a page name fully matched
(sort-by (fn [m]
[(count m) m])
matched-pages)
(string/blank? q)
nil
(empty? matched-pages)
(when-not (db/page-exists? q)
(if db-tag?
(concat [(str (t :new-page) " " q)
(str (t :new-class) " " q)]
matched-pages)
(cons q matched-pages)))
;; reorder, shortest and starts-with first.
:else
(let [matched-pages (remove nil? matched-pages)
matched-pages (sort-by
(fn [m]
[(not (gstring/caseInsensitiveStartsWith m q)) (count m) m])
matched-pages)]
(if (gstring/caseInsensitiveStartsWith (first matched-pages) q)
(cons (first matched-pages)
(cons q (rest matched-pages)))
(cons q matched-pages))))]
[:div
(when (and db-tag?
;; Don't display in heading
(not (some->> edit-content (re-find #"^\s*#"))))
[:div.flex.flex-row.items-center.px-4.py-1.text-sm.opacity-70.gap-2
"Turn this block into a page:"
(ui/toggle create-page?
(fn [_e]
(swap! (:editor/create-page? @state/state) not))
true)])
(ui/auto-complete
matched-pages
{:on-chosen (page-on-chosen-handler embed? input id q pos format)
:on-enter (fn []
(page-handler/page-not-exists-handler input id q current-pos))
:item-render (fn [page-name _chosen?]
[:div.flex
(when (db-model/whiteboard-page? page-name) [:span.mr-1 (ui/icon "whiteboard" {:extension? true})])
(search-handler/highlight-exact-query page-name q)])
:empty-placeholder [:div.text-gray-500.text-sm.px-4.py-2 (if db-tag?
"Search for a page or a class"
"Search for a page")]
:class "black"})]))))))
"")]
(page-search-aux id format embed? db-tag? create-page? q current-pos edit-content input pos)))))))
(defn- search-blocks!
[state result]
@ -231,6 +237,7 @@
(state/clear-edit!))))
(editor-handler/block-on-chosen-handler id q format selected-text)))
;; TODO: use rum/use-effect instead
(rum/defcs block-search-auto-complete < rum/reactive
{:init (fn [state]
(let [result (atom nil)]
@ -283,6 +290,22 @@
(when input
(block-search-auto-complete edit-block input id q format selected-text)))))
(rum/defc template-search-aux
[id q]
(let [[matched-templates set-matched-templates!] (rum/use-state nil)]
(rum/use-effect! (fn []
(p/let [result (editor-handler/<get-matched-templates q)]
(set-matched-templates! result)))
[q])
(ui/auto-complete
matched-templates
{:on-chosen (editor-handler/template-on-chosen-handler id)
:on-enter (fn [_state] (state/clear-editor-action!))
:empty-placeholder [:div.text-gray-500.px-4.py-2.text-sm "Search for a template"]
:item-render (fn [[template _block-db-id]]
template)
:class "black"})))
(rum/defc template-search < rum/reactive
[id _format]
(let [pos (state/get-editor-last-pos)
@ -293,37 +316,32 @@
q (or
(when (>= (count edit-content) current-pos)
(subs edit-content pos current-pos))
"")
matched-templates (editor-handler/get-matched-templates q)
non-exist-handler (fn [_state]
(state/clear-editor-action!))]
(ui/auto-complete
matched-templates
{:on-chosen (editor-handler/template-on-chosen-handler id)
:on-enter non-exist-handler
:empty-placeholder [:div.text-gray-500.px-4.py-2.text-sm "Search for a template"]
:item-render (fn [[template _block-db-id]]
template)
:class "black"})))))
"")]
(template-search-aux id q)))))
(rum/defc property-search < rum/reactive
(rum/defc property-search
[id]
(let [input (gdom/getElement id)]
(let [input (gdom/getElement id)
[matched-properties set-matched-properties!] (rum/use-state nil)]
(when input
(let [q (or (:searching-property (editor-handler/get-searching-property input))
"")
matched-properties (editor-handler/get-matched-properties q)
q-property (string/replace (string/lower-case q) #"\s+" "-")
non-exist-handler (fn [_state]
((editor-handler/property-on-chosen-handler id q-property) nil))]
(ui/auto-complete
matched-properties
{:on-chosen (editor-handler/property-on-chosen-handler id q-property)
:on-enter non-exist-handler
:empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
:header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
:item-render (fn [property] property)
:class "black"})))))
"")]
(rum/use-effect!
(fn []
(p/let [matched-properties (editor-handler/<get-matched-properties q)]
(set-matched-properties! matched-properties)))
[q])
(let [q-property (string/replace (string/lower-case q) #"\s+" "-")
non-exist-handler (fn [_state]
((editor-handler/property-on-chosen-handler id q-property) nil))]
(ui/auto-complete
matched-properties
{:on-chosen (editor-handler/property-on-chosen-handler id q-property)
:on-enter non-exist-handler
:empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
:header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
:item-render (fn [property] property)
:class "black"}))))))
(rum/defc property-value-search < rum/reactive
[id]

59
src/main/frontend/components/property.cljs
View File

@ -29,7 +29,8 @@
[frontend.components.dnd :as dnd]
[dommy.core :as dom]
[frontend.components.property.closed-value :as closed-value]
[frontend.components.property.util :as components-pu]))
[frontend.components.property.util :as components-pu]
[promesa.core :as p]))
(def icon closed-value/icon)
@ -345,6 +346,27 @@
(do (notification/show! "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['." :error)
(pv/exit-edit-property))))))
(rum/defc property-select
[exclude-properties on-chosen input-opts]
(let [[properties set-properties!] (rum/use-state nil)]
(rum/use-effect!
(fn []
(p/let [properties (search/get-all-properties)]
(set-properties! (remove exclude-properties properties))))
[])
[:div.ls-property-add.flex.flex-row.items-center
[:span.bullet-container.cursor [:span.bullet]]
[:div.ls-property-key {:style {:padding-left 6
:height "1.5em"}} ; TODO: ugly
(select/select {:items (map (fn [x] {:value x}) properties)
:dropdown? true
:close-modal? false
:show-new-when-not-exact-match? true
:exact-match-exclude-items exclude-properties
:input-default-placeholder "Add property"
:on-chosen on-chosen
:input-opts input-opts})]]))
(rum/defcs property-input < rum/reactive
(rum/local false ::show-new-property-config?)
shortcut/disable-all-shortcuts
@ -364,9 +386,7 @@
#{}
[:tags :alias])
exclude-properties* (set/union entity-properties existing-tag-alias)
exclude-properties (set/union exclude-properties* (set (map string/lower-case exclude-properties*)))
properties (->> (search/get-all-properties)
(remove exclude-properties))]
exclude-properties (set/union exclude-properties* (set (map string/lower-case exclude-properties*)))]
[:div.ls-property-input.flex.flex-1.flex-row.items-center.flex-wrap.gap-1
(if in-block-container? {:style {:padding-left 22}} {})
(if @*property-key
@ -395,26 +415,17 @@
"origin-top-right.absolute.left-0.rounded-md.shadow-lg.mt-2")})
(pv/property-value entity property @*property-value (assoc opts :editing? true))))]])
[:div.ls-property-add.flex.flex-row.items-center
[:span.bullet-container.cursor [:span.bullet]]
[:div.ls-property-key {:style {:padding-left 6
:height "1.5em"}} ; TODO: ugly
(select/select {:items (map (fn [x] {:value x}) properties)
:dropdown? true
:close-modal? false
:show-new-when-not-exact-match? true
:exact-match-exclude-items exclude-properties
:input-default-placeholder "Add property"
:on-chosen (fn [{:keys [value]}]
(reset! *property-key value)
(add-property-from-dropdown entity value (assoc opts :*show-new-property-config? *show-new-property-config?)))
:input-opts {:on-blur (fn [] (pv/exit-edit-property))
:on-key-down
(fn [e]
(case (util/ekey e)
"Escape"
(pv/exit-edit-property)
nil))}})]])]))
(let [on-chosen (fn [{:keys [value]}]
(reset! *property-key value)
(add-property-from-dropdown entity value (assoc opts :*show-new-property-config? *show-new-property-config?)))
input-opts {:on-blur (fn [] (pv/exit-edit-property))
:on-key-down
(fn [e]
(case (util/ekey e)
"Escape"
(pv/exit-edit-property)
nil))}]
(property-select exclude-properties on-chosen input-opts)))]))
(defonce *last-new-property-input-id (atom nil))
(rum/defcs new-property < rum/reactive

54
src/main/frontend/components/query/builder.cljs
View File

@ -1,9 +1,9 @@
(ns frontend.components.query.builder
"DSL query builder."
(:require [frontend.config :as config]
[frontend.date :as date]
(:require [frontend.date :as date]
[frontend.ui :as ui]
[frontend.db :as db]
[frontend.db.async :as db-async]
[frontend.db.model :as db-model]
[frontend.db.query-dsl :as query-dsl]
[frontend.handler.editor :as editor-handler]
@ -17,7 +17,8 @@
[rum.core :as rum]
[clojure.string :as string]
[logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.util.page-ref :as page-ref]))
[logseq.graph-parser.util.page-ref :as page-ref]
[promesa.core :as p]))
(rum/defc page-block-selector
[*find]
@ -118,6 +119,36 @@
(append-tree! tree opts loc clause)
(reset! *between-dates {}))))))])
(rum/defc property-select
[*mode *property]
(let [[properties set-properties!] (rum/use-state nil)]
(rum/use-effect!
(fn []
(p/let [properties (search/get-all-properties)]
(set-properties! properties)))
[])
(select properties
(fn [{:keys [value]}]
(reset! *mode "property-value")
(reset! *property (keyword value))))))
(rum/defc property-value-select
[repo *property *find *tree opts loc]
(let [[values set-values!] (rum/use-state nil)]
(rum/use-effect!
(fn []
(p/let [result (db-async/<get-property-values repo @*property)]
(set-values! result)))
[@*property])
(let [values (cons "Select all" values)]
(select values
(fn [{:keys [value]}]
(let [x (if (= value "Select all")
[(if (= @*find :page) :page-property :property) @*property]
[(if (= @*find :page) :page-property :property) @*property value])]
(reset! *property nil)
(append-tree! *tree opts loc x)))))))
(defn- query-filter-picker
[state *find *tree loc clause opts]
(let [*mode (::mode state)
@ -140,23 +171,10 @@
(append-tree! *tree opts loc [:page-tags value]))))
"property"
(let [properties (search/get-all-properties)]
(select properties
(fn [{:keys [value]}]
(reset! *mode "property-value")
(reset! *property (keyword value)))))
(property-select *mode *property)
"property-value"
(let [values (cons "Select all" (if (config/db-based-graph? repo)
(db-model/get-db-property-values repo @*property)
(db-model/get-property-values @*property)))]
(select values
(fn [{:keys [value]}]
(let [x (if (= value "Select all")
[(if (= @*find :page) :page-property :property) @*property]
[(if (= @*find :page) :page-property :property) @*property value])]
(reset! *property nil)
(append-tree! *tree opts loc x)))))
(property-value-select repo *property *find *tree opts loc)
"sample"
(select (range 1 101)

6
src/main/frontend/db.cljs
View File

@ -30,8 +30,8 @@
[frontend.db.model
delete-blocks get-pre-block
delete-files delete-pages-by-files get-all-block-contents get-all-tagged-pages get-single-block-contents
get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
delete-files delete-pages-by-files get-all-tagged-pages
get-block-and-children get-block-by-uuid get-block-children sort-by-left
get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
get-block-immediate-children get-block-page
get-custom-css get-date-scheduled-or-deadlines
@ -40,7 +40,7 @@
get-latest-journals get-page get-page-alias get-page-alias-names
get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-page-unlinked-references
get-all-pages get-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
journal-page? page-alias-set sub-block
set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page
set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition

100
src/main/frontend/db/async.cljs Normal file
View File

@ -0,0 +1,100 @@
(ns frontend.db.async
"Async queries"
(:require [promesa.core :as p]
[frontend.state :as state]
[frontend.config :as config]
[clojure.string :as string]
[logseq.graph-parser.util.page-ref :as page-ref]
[frontend.util :as util]
[frontend.db.utils :as db-utils]
[frontend.db.async.util :as db-async-util]
[frontend.db.file-based.async :as file-async]))
(def <q db-async-util/<q)
(defn <get-files
[graph]
(p/let [result (<q
graph
'[:find ?path ?modified-at
:where
[?file :file/path ?path]
[(get-else $ ?file :file/last-modified-at 0) ?modified-at]])]
(->> result seq reverse)))
(defn <get-all-templates
[graph]
(p/let [result (<q graph
'[:find ?t ?b
:where
[?b :block/properties ?p]
[(get ?p :template) ?t]])]
(into {} result)))
(defn <db-based-get-all-properties
":block/type could be one of [property, class]."
[graph]
(<q graph
'[:find [?n ...]
:where
[?e :block/type "property"]
[?e :block/original-name ?n]]))
(defn <get-all-properties
"Returns a seq of property name strings"
[]
(when-let [graph (state/get-current-repo)]
(if (config/db-based-graph? graph)
(<db-based-get-all-properties graph)
(file-async/<file-based-get-all-properties graph))))
(comment
(defn <get-pages
[graph]
(p/let [result (<q graph
'[:find [?page-original-name ...]
:where
[?page :block/name ?page-name]
[(get-else $ ?page :block/original-name ?page-name) ?page-original-name]])]
(remove db-model/hidden-page? result))))
(defn <get-db-based-property-values
[graph property]
(let [property-name (if (keyword? property)
(name property)
(util/page-name-sanity-lc property))]
(p/let [result (<q graph
'[:find ?prop-type ?v
:in $ ?prop-name
:where
[?b :block/properties ?bp]
[?prop-b :block/name ?prop-name]
[?prop-b :block/uuid ?prop-uuid]
[?prop-b :block/schema ?prop-schema]
[(get ?prop-schema :type) ?prop-type]
[(get ?bp ?prop-uuid) ?v]]
property-name)]
(->> result
(map (fn [[prop-type v]] [prop-type (if (coll? v) v [v])]))
(mapcat (fn [[prop-type vals]]
(case prop-type
:default
;; Remove multi-block properties as there isn't a supported approach to query them yet
(map str (remove uuid? vals))
(:page :date)
(map #(page-ref/->page-ref (:block/original-name (db-utils/entity graph [:block/uuid %])))
vals)
:number
vals
;; Checkboxes returned as strings as builder doesn't display boolean values correctly
(map str vals))))
;; Remove blanks as they match on everything
(remove string/blank?)
(distinct)
(sort)))))
(defn <get-property-values
[graph property]
(if (config/db-based-graph? graph)
(<get-db-based-property-values graph property)
(file-async/<get-file-based-property-values graph property)))

12
src/main/frontend/db/async/util.cljs Normal file
View File

@ -0,0 +1,12 @@
(ns frontend.db.async.util
"Async util helper"
(:require [frontend.persist-db.browser :as db-browser]
[cljs-bean.core :as bean]
[promesa.core :as p]))
(defn <q
[graph & inputs]
(assert (not-any? fn? inputs) "Async query inptus can't include fns because fn can't be serialized")
(when-let [sqlite @db-browser/*sqlite]
(p/let [result (.q sqlite graph (pr-str inputs))]
(bean/->clj result))))

57
src/main/frontend/db/file_based/async.cljs Normal file
View File

@ -0,0 +1,57 @@
(ns frontend.db.file-based.async
"File based async queries"
(:require [promesa.core :as p]
[frontend.db.async.util :as db-async-util]
[clojure.string :as string]
[logseq.graph-parser.util.page-ref :as page-ref]))
(def <q db-async-util/<q)
(defn <file-based-get-all-properties
[graph]
(p/let [properties (<q graph
'[:find [?p ...]
:where
[_ :block/properties ?p]])
properties (remove (fn [m] (empty? m)) properties)]
(->> (map keys properties)
(apply concat)
distinct
sort
(map name))))
(defn- property-value-for-refs-and-text
"Given a property value's refs and full text, determines the value to
autocomplete"
[[refs text]]
(if (or (not (coll? refs)) (= 1 (count refs)))
text
(map #(cond
(string/includes? text (page-ref/->page-ref %))
(page-ref/->page-ref %)
(string/includes? text (str "#" %))
(str "#" %)
:else
%)
refs)))
(defn <get-file-based-property-values
[graph property]
(p/let [result (<q graph
'[:find ?property-val ?text-property-val
:in $ ?property
:where
[?b :block/properties ?p]
[?b :block/properties-text-values ?p2]
[(get ?p ?property) ?property-val]
[(get ?p2 ?property) ?text-property-val]]
property)]
(->>
result
(map property-value-for-refs-and-text)
(map (fn [x] (if (coll? x) x [x])))
(apply concat)
(map str)
(remove string/blank?)
distinct
sort)))

146
src/main/frontend/db/model.cljs
View File

@ -17,7 +17,6 @@
[logseq.db.frontend.rules :as rules]
[logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.text :as text]
[logseq.graph-parser.util.page-ref :as page-ref]
[logseq.graph-parser.util.db :as db-util]
[logseq.graph-parser.util :as gp-util]
[logseq.outliner.pipeline :as outliner-pipeline]
@ -1186,130 +1185,6 @@ independent of format as format specific heading characters are stripped"
[page-name]
(:block/journal? (db-utils/entity [:block/name page-name])))
;; This is a file graph only feature
(defn get-all-templates
[]
(let [pred (fn [_db properties]
(some? (:template properties)))]
(->> (d/q
'[:find ?b ?p
:in $ ?pred
:where
[?b :block/properties ?p]
[(?pred $ ?p)]]
(conn/get-db)
pred)
(map (fn [[e m]]
[(get m :template) e]))
(into {}))))
(defn file-based-get-all-properties
[]
(let [db (conn/get-db)
properties (d/q
'[:find [?p ...]
:where
[_ :block/properties ?p]]
db)
properties (remove (fn [m] (empty? m)) properties)]
(->> (map keys properties)
(apply concat)
distinct
sort)))
(defn db-based-get-all-properties
":block/type could be one of [property, class]."
[]
(let [db (conn/get-db)
ids (->> (d/datoms db :aevt :block/schema)
(map :e))]
(->> ids
(map db-utils/entity)
(filter #(contains? (:block/type %) "property"))
(map :block/original-name))))
(defn get-all-properties
"Returns a seq of property name strings"
[]
(if (config/db-based-graph? (state/get-current-repo))
(db-based-get-all-properties)
(map name (file-based-get-all-properties))))
(defn- property-value-for-refs-and-text
"Given a property value's refs and full text, determines the value to
autocomplete"
[[refs text]]
(if (or (not (coll? refs)) (= 1 (count refs)))
text
(map #(cond
(string/includes? text (page-ref/->page-ref %))
(page-ref/->page-ref %)
(string/includes? text (str "#" %))
(str "#" %)
:else
%)
refs)))
(defn get-property-values
[property]
(let [pred (fn [_db properties text-properties]
[(get properties property)
(get text-properties property)])]
(->>
(d/q
'[:find ?property-val ?text-property-val
:in $ ?pred
:where
[?b :block/properties ?p]
[?b :block/properties-text-values ?p2]
[(?pred $ ?p ?p2) [?property-val ?text-property-val]]]
(conn/get-db)
pred)
(map property-value-for-refs-and-text)
(map (fn [x] (if (coll? x) x [x])))
(apply concat)
(map str)
(remove string/blank?)
(distinct)
(sort))))
(defn get-db-property-values
"Returns all property values of a given property for use in a simple query.
Property values that are references are displayed as page references"
[repo property]
(let [property-name (if (keyword? property)
(name property)
(util/page-name-sanity-lc property))]
(->> (d/q
'[:find ?prop-type ?v
:in $ ?prop-name
:where
[?b :block/properties ?bp]
[?prop-b :block/name ?prop-name]
[?prop-b :block/uuid ?prop-uuid]
[?prop-b :block/schema ?prop-schema]
[(get ?prop-schema :type) ?prop-type]
[(get ?bp ?prop-uuid) ?v]]
(conn/get-db repo)
property-name)
(map (fn [[prop-type v]] [prop-type (if (coll? v) v [v])]))
(mapcat (fn [[prop-type vals]]
(case prop-type
:default
;; Remove multi-block properties as there isn't a supported approach to query them yet
(map str (remove uuid? vals))
(:page :date)
(map #(page-ref/->page-ref (:block/original-name (db-utils/entity repo [:block/uuid %])))
vals)
:number
vals
;; Checkboxes returned as strings as builder doesn't display boolean values correctly
(map str vals))))
;; Remove blanks as they match on everything
(remove string/blank?)
(distinct)
(sort))))
(defn get-block-property-values
"Get blocks which have this property."
[property-uuid]
@ -1362,27 +1237,6 @@ independent of format as format specific heading characters are stripped"
[?refed-b :block/uuid ?refed-uuid]
[?referee-b :block/refs ?refed-b]] db)))
;; block/uuid and block/content
(defn get-single-block-contents [id]
(let [e (db-utils/entity [:block/uuid id])]
(when-not (and (nil? (:block/name e))
(string/blank? (:block/content e))) ; empty block
{:db/id (:db/id e)
:block/name (:block/name e)
:block/uuid id
:block/page (:db/id (:block/page e))
:block/content (:block/content e)
:block/format (:block/format e)
:block/properties (:block/properties e)})))
(defn get-all-block-contents
[]
(when-let [db (conn/get-db)]
(->> (d/datoms db :avet :block/uuid)
(map :v)
(map get-single-block-contents)
(remove nil?))))
(defn delete-blocks
[repo-url files _delete-page?]
(when (seq files)

28
src/main/frontend/db_worker.cljs
View File

@ -254,14 +254,21 @@
(when-let [conn (get-datascript-conn repo)]
(:max-tx @conn)))
(q [_this repo inputs-str]
"Datascript q"
(when-let [conn (get-datascript-conn repo)]
(let [inputs (edn/read-string inputs-str)]
(let [result (apply d/q (first inputs) @conn (rest inputs))]
(bean/->js result)))))
(transact
[_this repo tx-data tx-meta]
(when-let [conn (get-datascript-conn repo)]
(try
(let [tx-data (edn/read-string tx-data)
tx-meta (edn/read-string tx-meta)]
(d/transact! conn tx-data tx-meta)
nil)
tx-meta (edn/read-string tx-meta)
tx-report (d/transact! conn tx-data tx-meta)]
(search/sync-search-indice repo tx-report))
(catch :default e
(prn :debug :error)
(js/console.error e)))))
@ -323,6 +330,21 @@
(search/truncate-table! db)
nil))
(search-build-blocks-indice
[this repo]
(when-let [conn (get-datascript-conn repo)]
(search/build-blocks-indice repo @conn)))
(search-build-pages-indice
[this repo]
(when-let [conn (get-datascript-conn repo)]
(search/build-blocks-indice repo @conn)))
(page-search
[this repo q limit]
(when-let [conn (get-datascript-conn repo)]
(search/page-search repo @conn q limit)))
(dangerousRemoveAllDbs
[this repo]
(p/let [dbs (.listDB this)]

16
src/main/frontend/handler/editor.cljs
View File

@ -1644,14 +1644,14 @@
(when (>= pos 0)
(text-util/wrapped-by? value pos before end)))))
(defn get-matched-pages
(defn <get-matched-pages
"Return matched page names"
[q]
(let [block (state/get-edit-block)
editing-page (and block
(when-let [page-id (:db/id (:block/page block))]
(:block/name (db/entity page-id))))
pages (search/page-search q)]
(p/let [block (state/get-edit-block)
editing-page (and block
(when-let [page-id (:db/id (:block/page block))]
(:block/name (db/entity page-id))))
pages (search/page-search q)]
(if editing-page
;; To prevent self references
(remove (fn [p] (= (util/page-name-sanity-lc p) editing-page)) pages)
@ -1680,11 +1680,11 @@
(contains? current-and-parents (:block/uuid h)))
result))))
(defn get-matched-templates
(defn <get-matched-templates
[q]
(search/template-search q))
(defn get-matched-properties
(defn <get-matched-properties
[q]
(search/property-search q))

3
src/main/frontend/handler/events.cljs
View File

@ -808,6 +808,9 @@
{:id :new-db-graph
:label "graph-setup"}))
(defmethod handle :search/transact-data [[_ repo data]]
(search/transact-blocks! repo data))
(defmethod handle :class/configure [[_ page]]
(state/set-modal!
#(vector :<>

6
src/main/frontend/handler/page.cljs
View File

@ -6,6 +6,7 @@
[frontend.config :as config]
[frontend.date :as date]
[frontend.db :as db]
[frontend.db.async :as db-async]
[frontend.db.model :as model]
[frontend.fs :as fs]
[frontend.handler.common :as common-handler]
@ -130,10 +131,11 @@
(def rebuild-slash-commands-list!
(debounce init-commands! 1500))
(defn template-exists?
(defn <template-exists?
[title]
(when title
(let [templates (keys (db/get-all-templates))]
(p/let [result (db-async/<get-all-templates (state/get-current-repo))
templates (keys result)]
(when (seq templates)
(let [templates (map string/lower-case templates)]
(contains? (set templates) (string/lower-case title)))))))

8
src/main/frontend/handler/search.cljs
View File

@ -32,13 +32,15 @@
(:db/id (db/entity repo [:block/name (util/page-name-sanity-lc page-db-id)]))
page-db-id)
opts (if page-db-id (assoc opts :page (str page-db-id)) opts)]
(p/let [blocks (search/block-search repo q opts)]
(p/let [blocks (search/block-search repo q opts)
pages (search/page-search q)
files (search/file-search q)]
(let [result (merge
{:blocks blocks
:has-more? (= limit (count blocks))}
(when-not page-db-id
{:pages (search/page-search q)
:files (search/file-search q)}))
{:pages pages
:files files}))
search-key (if more? :search/more-result :search/result)]
(swap! state/state assoc search-key result)
result))))))

5
src/main/frontend/modules/outliner/datascript.cljs
View File

@ -7,7 +7,6 @@
[frontend.config :as config]
[logseq.graph-parser.util :as gp-util]
[lambdaisland.glogi :as log]
[frontend.search :as search]
[clojure.string :as string]
[frontend.util :as util]
[logseq.graph-parser.util.block-ref :as block-ref]
@ -61,9 +60,7 @@
(when (or (:outliner/transact? tx-meta)
(:outliner-op tx-meta)
(:whiteboard/transact? tx-meta))
(undo-redo/listen-db-changes! tx-report))
(search/sync-search-indice! repo tx-report)))
(undo-redo/listen-db-changes! tx-report))))
(defn- remove-nil-from-transaction
[txs]

18
src/main/frontend/persist_db/browser.cljs
View File

@ -10,7 +10,8 @@
[frontend.handler.notification :as notification]
[cljs-bean.core :as bean]
[frontend.state :as state]
[electron.ipc :as ipc]))
[electron.ipc :as ipc]
[frontend.handler.file-based.property.util :as property-util]))
(defonce *sqlite (atom nil))
@ -83,7 +84,20 @@
(p/do!
(ipc/ipc :db-transact repo tx-data' tx-meta')
(if sqlite
(.transact sqlite repo tx-data' tx-meta')
(p/let [result (.transact sqlite repo tx-data' tx-meta')
result' (bean/->clj result)
file-based? (config/local-file-based-graph? repo)
data (cond-> result'
file-based?
;; remove built-in properties from content
(update :blocks-to-add
(fn [blocks]
(map #(update % :content
(fn [content]
(property-util/remove-built-in-properties (get % :format :markdown) content)))
blocks))))]
(state/pub-event! [:search/transact-data repo data])
nil)
(notification/show! "Latest change was not saved! Please restart the application." :error))
nil)))

283
src/main/frontend/search.cljs
View File

@ -1,92 +1,28 @@
(ns frontend.search
"Provides search functionality for a number of features including Cmd-K
search. Most of these fns depend on the search protocol"
(:require [cljs-bean.core :as bean]
[clojure.string :as string]
[logseq.graph-parser.config :as gp-config]
[frontend.db :as db]
[frontend.db.model :as db-model]
(:require [clojure.string :as string]
[frontend.search.agency :as search-agency]
[frontend.search.db :as search-db :refer [indices]]
[frontend.search.protocol :as protocol]
[frontend.state :as state]
[frontend.util :as util]
[goog.object :as gobj]
[promesa.core :as p]
[datascript.core :as d]
[frontend.handler.file-based.property.util :as property-util]
[frontend.search.browser :as search-browser]
[frontend.search.fuzzy :as fuzzy]
[logseq.graph-parser.config :as gp-config]
[frontend.db.async :as db-async]
[frontend.config :as config]
[logseq.db.frontend.property :as db-property]))
[logseq.db.frontend.property :as db-property]
[frontend.handler.file-based.property.util :as property-util]
[frontend.db.model :as db-model]
[cljs-bean.core :as bean]))
(def fuzzy-search fuzzy/fuzzy-search)
(defn get-engine
[repo]
(search-agency/->Agency repo))
;; Copied from https://gist.github.com/vaughnd/5099299
(defn str-len-distance
;; normalized multiplier 0-1
;; measures length distance between strings.
;; 1 = same length
[s1 s2]
(let [c1 (count s1)
c2 (count s2)
maxed (max c1 c2)
mined (min c1 c2)]
(double (- 1
(/ (- maxed mined)
maxed)))))
(def MAX-STRING-LENGTH 1000.0)
(defn clean-str
[s]
(string/replace (string/lower-case s) #"[\[ \\/_\]\(\)]+" ""))
(defn char-array
[s]
(bean/->js (seq s)))
(defn score
[oquery ostr]
(let [query (clean-str oquery)
str (clean-str ostr)]
(loop [q (seq (char-array query))
s (seq (char-array str))
mult 1
idx MAX-STRING-LENGTH
score 0]
(cond
;; add str-len-distance to score, so strings with matches in same position get sorted by length
;; boost score if we have an exact match including punctuation
(empty? q) (+ score
(str-len-distance query str)
(if (<= 0 (.indexOf ostr oquery)) MAX-STRING-LENGTH 0))
(empty? s) 0
:else (if (= (first q) (first s))
(recur (rest q)
(rest s)
(inc mult) ;; increase the multiplier as more query chars are matched
(dec idx) ;; decrease idx so score gets lowered the further into the string we match
(+ mult score)) ;; score for this match is current multiplier * idx
(recur q
(rest s)
1 ;; when there is no match, reset multiplier to one
(dec idx)
score))))))
(defn fuzzy-search
[data query & {:keys [limit extract-fn]
:or {limit 20}}]
(let [query (util/search-normalize query (state/enable-search-remove-accents?))]
(->> (take limit
(sort-by :score (comp - compare)
(filter #(< 0 (:score %))
(for [item data]
(let [s (str (if extract-fn (extract-fn item) item))]
{:data item
:score (score query (util/search-normalize s (state/enable-search-remove-accents?)))})))))
(map :data))))
(defn block-search
[repo q option]
(when-let [engine (get-engine repo)]
@ -94,176 +30,80 @@
(when-not (string/blank? q)
(protocol/query engine q option)))))
(defn- transact-blocks!
[repo data]
(when-let [engine (get-engine repo)]
(protocol/transact-blocks! engine data)))
(defn exact-matched?
"Check if two strings points toward same search result"
[q match]
(when (and (string? q) (string? match))
(boolean
(reduce
(fn [coll char]
(let [coll' (drop-while #(not= char %) coll)]
(if (seq coll')
(rest coll')
(reduced false))))
(seq (util/search-normalize match (state/enable-search-remove-accents?)))
(seq (util/search-normalize q (state/enable-search-remove-accents?)))))))
(defn page-search
"Return a list of page names that match the query"
([q]
(page-search q 100))
([q limit]
(when-let [repo (state/get-current-repo)]
(let [q (util/search-normalize q (state/enable-search-remove-accents?))
q (clean-str q)
q (if (= \# (first q)) (subs q 1) q)]
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :pages])
(search-db/make-pages-title-indice!))
result (->> (.search indice q (clj->js {:limit limit}))
(bean/->clj))]
(->> result
(util/distinct-by (fn [i] (string/trim (get-in i [:item :name]))))
(map
(fn [{:keys [item]}]
(:original-name item)))
(remove nil?)
(map string/trim)
(distinct)
(filter (fn [original-name]
(exact-matched? q original-name))))))))))
(when-let [^js sqlite @search-browser/*sqlite]
(p/let [result (.page-search sqlite (state/get-current-repo) q limit)]
(bean/->clj result)))))
(defn file-search
([q]
(file-search q 3))
([q limit]
(let [q (clean-str q)]
(when-not (string/blank? q)
(let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
files (->> (db/get-files (state/get-current-repo))
(map first)
(remove (fn [file]
(mldoc-exts (util/get-file-ext file)))))]
(when (seq files)
(fuzzy-search files q :limit limit)))))))
(when-let [repo (state/get-current-repo)]
(let [q (fuzzy/clean-str q)]
(when-not (string/blank? q)
(p/let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
result (db-async/<get-files repo)
files (->> result
(map first)
(remove (fn [file]
(mldoc-exts (util/get-file-ext file)))))]
(when (seq files)
(fuzzy/fuzzy-search files q :limit limit))))))))
(defn template-search
([q]
(template-search q 100))
([q limit]
(when q
(let [q (clean-str q)
templates (db/get-all-templates)]
(when (seq templates)
(let [result (fuzzy-search (keys templates) q :limit limit)]
(vec (select-keys templates result))))))))
(when-let [repo (state/get-current-repo)]
(when q
(p/let [q (fuzzy/clean-str q)
templates (db-async/<get-all-templates repo)]
(when (seq templates)
(let [result (fuzzy/fuzzy-search (keys templates) q {:limit limit})]
(vec (select-keys templates result)))))))))
(defn get-all-properties
[]
(let [hidden-props (if (config/db-based-graph? (state/get-current-repo))
(set (map #(or (get-in db-property/built-in-properties [% :original-name])
(name %))
db-property/hidden-built-in-properties))
(set (map name (property-util/hidden-properties))))]
(remove hidden-props (db-model/get-all-properties))))
(when-let [repo (state/get-current-repo)]
(let [hidden-props (if (config/db-based-graph? repo)
(set (map #(or (get-in db-property/built-in-properties [% :original-name])
(name %))
db-property/hidden-built-in-properties))
(set (map name (property-util/hidden-properties))))]
(p/let [properties (db-async/<get-all-properties)]
(remove hidden-props properties)))))
(defn property-search
([q]
(property-search q 100))
([q limit]
(when q
(let [q (clean-str q)
properties (get-all-properties)]
(p/let [q (fuzzy/clean-str q)
properties (get-all-properties)]
(when (seq properties)
(if (string/blank? q)
properties
(let [result (fuzzy-search properties q :limit limit)]
(let [result (fuzzy/fuzzy-search properties q :limit limit)]
(vec result))))))))
;; file-based graph only
(defn property-value-search
([property q]
(property-value-search property q 100))
([property q limit]
(when q
(let [q (clean-str q)
result (db-model/get-property-values (keyword property))]
(when (seq result)
(if (string/blank? q)
result
(let [result (fuzzy-search result q :limit limit)]
(vec result))))))))
(defn- get-blocks-from-datoms-impl
[{:keys [db-after db-before]} datoms]
(when (seq datoms)
(let [blocks-to-add-set (->> (filter :added datoms)
(map :e)
(set))
blocks-to-remove-set (->> (remove :added datoms)
(filter #(= :block/uuid (:a %)))
(map :e)
(set))
blocks-to-add-set' (if (and (config/db-based-graph? (state/get-current-repo)) (seq blocks-to-add-set))
(->> blocks-to-add-set
(mapcat (fn [id] (map :db/id (:block/_refs (db/entity id)))))
(concat blocks-to-add-set)
set)
blocks-to-add-set)]
{:blocks-to-remove (->>
(map #(d/entity db-before %) blocks-to-remove-set)
(remove nil?)
(remove db-model/hidden-page?))
:blocks-to-add (->>
(map #(d/entity db-after %) blocks-to-add-set')
(remove nil?)
(remove db-model/hidden-page?))})))
(defn- get-direct-blocks-and-pages
[tx-report]
(let [data (:tx-data tx-report)
datoms (filter
(fn [datom]
;; Capture any direct change on page display title, page ref or block content
(contains? #{:block/uuid :block/name :block/original-name :block/content :block/properties :block/schema} (:a datom)))
data)]
(when (seq datoms)
(get-blocks-from-datoms-impl tx-report datoms))))
;; TODO merge with logic in `invoke-hooks` when feature and test is sufficient
(defn sync-search-indice!
[repo tx-report]
(let [{:keys [blocks-to-add blocks-to-remove]} (get-direct-blocks-and-pages tx-report)]
;; TODO: remove this once we have fuzzy search support on SQLite
;; update page title indice
(let [pages-to-add (filter :block/name blocks-to-add)
pages-to-remove (filter :block/name blocks-to-remove)]
(when (or (seq pages-to-add) (seq pages-to-remove))
(swap! search-db/indices update-in [repo :pages]
(fn [indice]
(when indice
(doseq [page-entity pages-to-remove]
(.remove indice
(fn [page]
(= (:block/name page-entity)
(util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
(doseq [page pages-to-add]
(.add indice (bean/->js (search-db/original-page-name->index
(or (:block/original-name page)
(:block/name page))))))
indice)))))
;; update block indice
(when (or (seq blocks-to-add) (seq blocks-to-remove))
(let [blocks-to-add (remove nil? (map search-db/block->index blocks-to-add))
blocks-to-remove (set (map (comp str :block/uuid) blocks-to-remove))]
(transact-blocks! repo
{:blocks-to-remove-set blocks-to-remove
:blocks-to-add blocks-to-add})))))
(when-let [repo (state/get-current-repo)]
(when q
(let [q (fuzzy/clean-str q)
result (db-async/<get-property-values repo (keyword property))]
(when (seq result)
(if (string/blank? q)
result
(let [result (fuzzy/fuzzy-search result q :limit limit)]
(vec result)))))))))
(defn rebuild-indices!
([]
@ -271,20 +111,21 @@
([repo]
(when repo
(when-let [engine (get-engine repo)]
(let [page-titles (search-db/make-pages-title-indice!)]
(p/let [_ (protocol/rebuild-blocks-indice! engine)]
(let [result {:pages page-titles ;; TODO: rename key to :page-titles
}]
(swap! indices assoc repo result)
indices)))))))
(p/do!
(protocol/rebuild-pages-indice! engine)
(protocol/rebuild-blocks-indice! engine))))))
(defn reset-indice!
[repo]
(when-let [engine (get-engine repo)]
(protocol/truncate-blocks! engine))
(swap! indices assoc-in [repo :pages] nil))
(protocol/truncate-blocks! engine)))
(defn remove-db!
[repo]
(when-let [engine (get-engine repo)]
(protocol/remove-db! engine)))
(defn transact-blocks!
[repo data]
(when-let [engine (get-engine repo)]
(protocol/transact-blocks! engine data)))

6
src/main/frontend/search/agency.cljs
View File

@ -31,6 +31,12 @@
(protocol/rebuild-blocks-indice! e))
(protocol/rebuild-blocks-indice! e1)))
(rebuild-pages-indice! [_this]
(let [[e1 e2] (get-registered-engines repo)]
(doseq [e e2]
(protocol/rebuild-pages-indice! e))
(protocol/rebuild-pages-indice! e1)))
(transact-blocks! [_this data]
(doseq [e (get-flatten-registered-engines repo)]
(protocol/transact-blocks! e data)))

21
src/main/frontend/search/browser.cljs
View File

@ -5,7 +5,8 @@
[promesa.core :as p]
[frontend.persist-db.browser :as browser]
[frontend.state :as state]
[frontend.search.db :as search-db]))
[frontend.config :as config]
[frontend.handler.file-based.property.util :as property-util]))
(defonce *sqlite browser/*sqlite)
@ -20,10 +21,24 @@
:block/content content
:block/page (uuid page)}) result))
(p/resolved nil)))
(rebuild-pages-indice! [_this]
(if-let [^js sqlite @*sqlite]
(.search-build-pages-indice sqlite repo)
(p/resolved nil)))
(rebuild-blocks-indice! [this]
(if-let [^js sqlite @*sqlite]
(p/let [_ (protocol/truncate-blocks! this)
blocks (search-db/build-blocks-indice)
(p/let [repo (state/get-current-repo)
file-based? (config/local-file-based-graph? repo)
_ (protocol/truncate-blocks! this)
result (.search-build-blocks-indice sqlite repo)
blocks (cond->> (bean/->clj result)
file-based?
;; remove built-in properties from content
(map #(update % :content
(fn [content]
(property-util/remove-built-in-properties (get % :format :markdown) content))))
true
bean/->js)
_ (when (seq blocks)
(.search-upsert-blocks sqlite repo blocks))])
(p/resolved nil)))

118
src/main/frontend/search/db.cljs
View File

@ -1,118 +0,0 @@
(ns ^:no-doc frontend.search.db
(:require [cljs-bean.core :as bean]
[clojure.string :as string]
[frontend.db :as db]
[frontend.db.model :as model]
[frontend.handler.db-based.property.util :as db-pu]
[frontend.state :as state]
[frontend.config :as config]
[frontend.util :as util]
["fuse.js" :as fuse]
[frontend.handler.file-based.property.util :as property-util]))
;; Notice: When breaking changes happen, bump version in src/electron/electron/search.cljs
(defonce indices (atom nil))
(defn- max-len
[]
(state/block-content-max-length (state/get-current-repo)))
(defn- sanitize
[content]
(some-> content
(util/search-normalize (state/enable-search-remove-accents?))))
(defn- get-db-properties-str
"Similar to db-pu/readable-properties but with a focus on making property values searchable"
[properties]
(->> properties
(map
(fn [[k v]]
(let [values
(->> (if (set? v) v #{v})
(map (fn [val]
(if (uuid? val)
(let [e (db/entity [:block/uuid val])
value (or
;; closed value
(db-pu/property-value-when-closed e)
;; page
(:block/original-name e)
;; block generated by template
(and
(get-in e [:block/metadata :created-from-template])
(:block/content e))
;; first child
(let [parent-id (:db/id e)]
(:block/content (model/get-by-parent-&-left (db/get-db) parent-id parent-id))))]
value)
val)))
(remove string/blank?))]
(when (seq values)
(str (:block/original-name (db/entity [:block/uuid k]))
": "
(string/join "; " values))))))
(remove nil?)
(string/join ";; ")))
(defn block->index
"Convert a block to the index for searching"
[{:block/keys [name uuid page content properties format]
:or {format :markdown}
:as block}]
(let [repo (state/get-current-repo)
page? (some? name)
block? (nil? name)
db-based? (config/db-based-graph? repo)]
(when-not (or
(and page? name (model/whiteboard-page? name))
(and block? (> (count content) (max-len)))
(and (empty? properties)
(or (and block? (string/blank? content))
(and db-based? page?)))) ; empty page or block
(let [content (if block?
(if db-based? content (property-util/remove-built-in-properties format content))
;; File based page content
(if db-based?
"" ; empty page content
(some-> (:block/file (db/entity (:db/id block))) :file/content)))
content' (if (and db-based? (seq properties))
(str content (when (not= content "") "\n") (get-db-properties-str properties))
content)]
(when-not (string/blank? content')
{:id (str uuid)
:page (str (:block/uuid page))
:content (sanitize content')})))))
(defn original-page-name->index
[p]
(when p
{:name (util/search-normalize p (state/enable-search-remove-accents?))
:original-name p}))
(defn make-pages-title-indice!
"Build a page title indice from scratch.
Incremental page title indice is implemented in frontend.search.sync-search-indice!
Rename from the page indice since 10.25.2022, since this is only used for page title search.
From now on, page indice is talking about page content search."
[]
(when-let [repo (state/get-current-repo)]
(let [pages (->> (db/get-pages (state/get-current-repo))
(remove string/blank?)
(map original-page-name->index)
(bean/->js))
indice (fuse. pages
(clj->js {:keys ["name"]
:shouldSort true
:tokenize true
:minMatchCharLength 1}))]
(swap! indices assoc-in [repo :pages] indice)
indice)))
(defn build-blocks-indice
[]
(->> (db/get-all-block-contents)
(map block->index)
(remove nil?)
(bean/->js)))

70
src/main/frontend/search/fuzzy.cljs Normal file
View File

@ -0,0 +1,70 @@
(ns frontend.search.fuzzy
"fuzzy search"
(:require [clojure.string :as string]
[cljs-bean.core :as bean]
[frontend.worker.util :as util]))
(def MAX-STRING-LENGTH 1000.0)
(defn clean-str
[s]
(string/replace (string/lower-case s) #"[\[ \\/_\]\(\)]+" ""))
(defn char-array
[s]
(bean/->js (seq s)))
;; Copied from https://gist.github.com/vaughnd/5099299
(defn str-len-distance
;; normalized multiplier 0-1
;; measures length distance between strings.
;; 1 = same length
[s1 s2]
(let [c1 (count s1)
c2 (count s2)
maxed (max c1 c2)
mined (min c1 c2)]
(double (- 1
(/ (- maxed mined)
maxed)))))
(defn score
[oquery ostr]
(let [query (clean-str oquery)
str (clean-str ostr)]
(loop [q (seq (char-array query))
s (seq (char-array str))
mult 1
idx MAX-STRING-LENGTH
score 0]
(cond
;; add str-len-distance to score, so strings with matches in same position get sorted by length
;; boost score if we have an exact match including punctuation
(empty? q) (+ score
(str-len-distance query str)
(if (<= 0 (.indexOf ostr oquery)) MAX-STRING-LENGTH 0))
(empty? s) 0
:else (if (= (first q) (first s))
(recur (rest q)
(rest s)
(inc mult) ;; increase the multiplier as more query chars are matched
(dec idx) ;; decrease idx so score gets lowered the further into the string we match
(+ mult score)) ;; score for this match is current multiplier * idx
(recur q
(rest s)
1 ;; when there is no match, reset multiplier to one
(dec idx)
score))))))
(defn fuzzy-search
[data query & {:keys [limit extract-fn]
:or {limit 20}}]
(let [query (util/search-normalize query true)]
(->> (take limit
(sort-by :score (comp - compare)
(filter #(< 0 (:score %))
(for [item data]
(let [s (str (if extract-fn (extract-fn item) item))]
{:data item
:score (score query (util/search-normalize s true))})))))
(map :data))))

4
src/main/frontend/search/plugin.cljs
View File

@ -23,12 +23,14 @@
(query [_this q opts]
(call-service! service "search:query" (merge {:q q} opts) true))
(rebuild-blocks-indice! [_this]
;; Not pushing all data for performance temporarily
;;(let [blocks (search-db/build-blocks-indice repo)])
(call-service! service "search:rebuildBlocksIndice" {}))
(rebuild-pages-indice! [_this]
(call-service! service "search:rebuildPagesIndice" {}))
(transact-blocks! [_this data]
(let [{:keys [blocks-to-remove-set blocks-to-add]} data]
(call-service! service "search:transactBlocks"

1
src/main/frontend/search/protocol.cljs
View File

@ -3,6 +3,7 @@
(defprotocol Engine
(query [this q option])
(rebuild-blocks-indice! [this]) ;; TODO: rename to rebuild-indice!
(rebuild-pages-indice! [this]) ;; TODO: rename to rebuild-indice!
(transact-blocks! [this data])
(truncate-blocks! [this]) ;; TODO: rename to truncate-indice!
(remove-db! [this]))

4
src/main/frontend/state.cljs
View File

@ -972,7 +972,9 @@ Similar to re-frame subscriptions"
(gobj/get "id")))
(when-let [elem js/document.activeElement]
(when (util/input? elem)
(gobj/get elem "id")))))
(let [id (gobj/get elem "id")]
(when (string/starts-with? id "edit-block-")
id))))))
(defn get-input
[]

39
src/main/frontend/util.cljc
View File

@ -8,7 +8,6 @@
["@capacitor/status-bar" :refer [^js StatusBar Style]]
["@capgo/capacitor-navigation-bar" :refer [^js NavigationBar]]
["grapheme-splitter" :as GraphemeSplitter]
["remove-accents" :as removeAccents]
["sanitize-filename" :as sanitizeFilename]
["check-password-strength" :refer [passwordStrength]]
["path-complete-extname" :as pathCompleteExtname]
@ -28,8 +27,8 @@
[rum.core :as rum]
[clojure.core.async :as async]
[cljs.core.async.impl.channels :refer [ManyToManyChannel]]
[medley.core :as medley]
[frontend.pubsub :as pubsub]))
[frontend.pubsub :as pubsub]
[frontend.worker.util :as worker-util]))
#?(:cljs (:import [goog.async Debouncer]))
(:require
[clojure.pprint]
@ -76,23 +75,11 @@
(string/join "/" parts))
#?(:cljs
(defn safe-re-find
{:malli/schema [:=> [:cat :any :string] [:or :nil :string [:vector [:maybe :string]]]]}
[pattern s]
(when-not (string? s)
;; TODO: sentry
(js/console.trace))
(when (string? s)
(re-find pattern s))))
(def safe-re-find worker-util/safe-re-find))
#?(:cljs
(do
(def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
(defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
(defn uuid-string?
{:malli/schema [:=> [:cat :string] :boolean]}
[s]
(boolean (safe-re-find exactly-uuid-pattern s)))
(def uuid-string? worker-util/uuid-string?)
(defn check-password-strength
{:malli/schema [:=> [:cat :string] [:maybe
[:map
@ -594,9 +581,7 @@
#?(:cljs
(defn distinct-by
[f col]
(medley/distinct-by f (seq col))))
(def distinct-by worker-util/distinct-by))
#?(:cljs
(defn distinct-by-last-wins
@ -1034,14 +1019,7 @@
(some-> string str (js/encodeURIComponent) (.replace "+" "%20"))))
#?(:cljs
(defn search-normalize
"Normalize string for searching (loose)"
[s remove-accents?]
(when s
(let [normalize-str (.normalize (string/lower-case s) "NFKC")]
(if remove-accents?
(removeAccents normalize-str)
normalize-str)))))
(def search-normalize worker-util/search-normalize))
#?(:cljs
(def page-name-sanity-lc
@ -1049,10 +1027,7 @@
gp-util/page-name-sanity-lc))
#?(:cljs
(defn safe-page-name-sanity-lc
[s]
(if (string? s)
(page-name-sanity-lc s) s)))
(def safe-page-name-sanity-lc worker-util/safe-page-name-sanity-lc))
(defn get-page-original-name
[page]

302
src/main/frontend/worker/search.cljs
View File

@ -1,9 +1,23 @@
(ns frontend.worker.search
"SQLite search"
"Full-text and fuzzy search"
(:require [clojure.string :as string]
[promesa.core :as p]
[medley.core :as medley]
[cljs-bean.core :as bean]))
[cljs-bean.core :as bean]
["fuse.js" :as fuse]
[goog.object :as gobj]
[datascript.core :as d]
[frontend.search.fuzzy :as fuzzy]
[frontend.worker.util :as util]))
(defonce db-version-prefix "logseq_db_")
(defn db-based-graph?
[s]
(boolean
(and (string? s)
(string/starts-with? s db-version-prefix))))
;; TODO: use sqlite for fuzzy search
(defonce indices (atom nil))
(defn- add-blocks-fts-triggers!
"Table bindings of blocks tables and the blocks FTS virtual tables"
@ -125,10 +139,6 @@
(string/replace match-input "," "")
(str "\"" match-input "\""))))
(defn distinct-by
[f col]
(medley/distinct-by f (seq col)))
(defn search-blocks
":page - the page to specifically search on"
[db q {:keys [limit page]}]
@ -158,10 +168,286 @@
:page page})))]
(->>
all-result
(distinct-by :uuid)
(util/distinct-by :uuid)
(take limit)))))
(defn truncate-table!
[db]
(.exec db "delete from blocks")
(.exec db "delete from blocks_fts"))
(defn- sanitize
[content]
(some-> content
(util/search-normalize true)))
(defn- property-value-when-closed
"Returns property value if the given entity is type 'closed value' or nil"
[ent]
(when (contains? (:block/type ent) "closed value")
(get-in ent [:block/schema :value])))
(defn get-by-parent-&-left
[db parent-id left-id]
(when (and parent-id left-id)
(let [lefts (:block/_left (d/entity db left-id))]
(some (fn [node] (when (and (= parent-id (:db/id (:block/parent node)))
(not= parent-id (:db/id node)))
node)) lefts))))
(defn- get-db-properties-str
"Similar to db-pu/readable-properties but with a focus on making property values searchable"
[db properties]
(->> properties
(map
(fn [[k v]]
(let [values
(->> (if (set? v) v #{v})
(map (fn [val]
(if (uuid? val)
(let [e (d/entity db [:block/uuid val])
value (or
;; closed value
(property-value-when-closed e)
;; page
(:block/original-name e)
;; block generated by template
(and
(get-in e [:block/metadata :created-from-template])
(:block/content e))
;; first child
(let [parent-id (:db/id e)]
(:block/content (get-by-parent-&-left db parent-id parent-id))))]
value)
val)))
(remove string/blank?))]
(when (seq values)
(str (:block/original-name (d/entity db [:block/uuid k]))
": "
(string/join "; " values))))))
(remove nil?)
(string/join ";; ")))
(defn whiteboard-page?
"Given a page name or a page object, check if it is a whiteboard page"
[db page]
(cond
(string? page)
(let [page (d/entity db [:block/name page])]
(or
(= (:block/type page) "whiteboard")
(contains? (set (:block/type page)) "whiteboard")))
(seq page)
(contains? (set (:block/type page)) "whiteboard")
:else false))
(defn block->index
"Convert a block to the index for searching"
[repo db {:block/keys [name uuid page content properties format]
:as block}]
(let [page? (some? name)
block? (nil? name)
db-based? (db-based-graph? repo)]
(when-not (or
(and page? name (whiteboard-page? db name))
(and block? (> (count content) 10000))
(and (empty? properties)
(or (and block? (string/blank? content))
(and db-based? page?)))) ; empty page or block
(let [content (if block?
content
;; File based page content
(if db-based?
"" ; empty page content
(some-> (:block/file (d/entity db (:db/id block))) :file/content)))
content' (if (and db-based? (seq properties))
(str content (when (not= content "") "\n") (get-db-properties-str db properties))
content)]
(when-not (string/blank? content')
{:id (str uuid)
:page (str (:block/uuid page))
:content (sanitize content')
:format format})))))
(defn get-single-block-contents [db id]
(let [e (d/entity db [:block/uuid id])]
(when-not (and (nil? (:block/name e))
(string/blank? (:block/content e))) ; empty block
{:db/id (:db/id e)
:block/name (:block/name e)
:block/uuid id
:block/page (:db/id (:block/page e))
:block/content (:block/content e)
:block/format (:block/format e)
:block/properties (:block/properties e)})))
(defn get-all-block-contents
[db]
(when db
(->> (d/datoms db :avet :block/uuid)
(map :v)
(map #(get-single-block-contents db %))
(remove nil?))))
(defn build-blocks-indice
[repo db]
(->> (get-all-block-contents db)
(map #(block->index repo db %))
(remove nil?)
(bean/->js)))
(defn original-page-name->index
[p]
(when p
{:name (util/search-normalize p true)
:original-name p}))
(defn- safe-subs
([s start]
(let [c (count s)]
(safe-subs s start c)))
([s start end]
(let [c (count s)]
(subs s (min c start) (min c end)))))
(defn- hidden-page?
[page]
(when page
(if (string? page)
(and (string/starts-with? page "$$$")
(util/uuid-string? (safe-subs page 3)))
(contains? (set (:block/type page)) "hidden"))))
(defn get-all-pages
[db]
(->>
(d/q
'[:find [?page-original-name ...]
:where
[?page :block/name ?page-name]
[(get-else $ ?page :block/original-name ?page-name) ?page-original-name]]
db)
(remove hidden-page?)))
(defn build-page-indice
"Build a page title indice from scratch.
Incremental page title indice is implemented in frontend.search.sync-search-indice!
Rename from the page indice since 10.25.2022, since this is only used for page title search.
From now on, page indice is talking about page content search."
[repo db]
(let [pages (->> (get-all-pages db)
(remove string/blank?)
(map original-page-name->index)
(bean/->js))
indice (fuse. pages
(clj->js {:keys ["name"]
:shouldSort true
:tokenize true
:minMatchCharLength 1}))]
(swap! indices assoc-in [repo :pages] indice)
indice))
(defn- get-blocks-from-datoms-impl
[repo {:keys [db-after db-before]} datoms]
(when (seq datoms)
(let [blocks-to-add-set (->> (filter :added datoms)
(map :e)
(set))
blocks-to-remove-set (->> (remove :added datoms)
(filter #(= :block/uuid (:a %)))
(map :e)
(set))
blocks-to-add-set' (if (and (db-based-graph? repo) (seq blocks-to-add-set))
(->> blocks-to-add-set
(mapcat (fn [id] (map :db/id (:block/_refs (d/entity db-after id)))))
(concat blocks-to-add-set)
set)
blocks-to-add-set)]
{:blocks-to-remove (->>
(map #(d/entity db-before %) blocks-to-remove-set)
(remove nil?)
(remove hidden-page?))
:blocks-to-add (->>
(map #(d/entity db-after %) blocks-to-add-set')
(remove nil?)
(remove hidden-page?))})))
(defn- get-direct-blocks-and-pages
[repo tx-report]
(let [data (:tx-data tx-report)
datoms (filter
(fn [datom]
;; Capture any direct change on page display title, page ref or block content
(contains? #{:block/uuid :block/name :block/original-name :block/content :block/properties :block/schema} (:a datom)))
data)]
(when (seq datoms)
(get-blocks-from-datoms-impl repo tx-report datoms))))
(defn sync-search-indice
[repo tx-report]
(let [{:keys [blocks-to-add blocks-to-remove]} (get-direct-blocks-and-pages repo tx-report)]
;; update page title indice
(let [pages-to-add (filter :block/name blocks-to-add)
pages-to-remove (filter :block/name blocks-to-remove)]
(when (or (seq pages-to-add) (seq pages-to-remove))
(swap! indices update-in [repo :pages]
(fn [indice]
(when indice
(doseq [page-entity pages-to-remove]
(.remove indice
(fn [page]
(= (:block/name page-entity)
(util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
(doseq [page pages-to-add]
(.add indice (bean/->js (original-page-name->index
(or (:block/original-name page)
(:block/name page))))))
indice)))))
;; update block indice
(when (or (seq blocks-to-add) (seq blocks-to-remove))
(let [blocks-to-add (remove nil? (map #(block->index repo (:db-after tx-report) %) blocks-to-add))
blocks-to-remove (set (map (comp str :block/uuid) blocks-to-remove))]
(bean/->js
{:blocks-to-remove-set blocks-to-remove
:blocks-to-add blocks-to-add})))))
(defn exact-matched?
"Check if two strings points toward same search result"
[q match]
(when (and (string? q) (string? match))
(boolean
(reduce
(fn [coll char]
(let [coll' (drop-while #(not= char %) coll)]
(if (seq coll')
(rest coll')
(reduced false))))
(seq (util/search-normalize match true))
(seq (util/search-normalize q true))))))
(defn page-search
"Return a list of page names that match the query"
[repo db q limit]
(when repo
(let [q (util/search-normalize q true)
q (fuzzy/clean-str q)
q (if (= \# (first q)) (subs q 1) q)]
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :pages])
(build-page-indice repo db))
result (->> (.search indice q (clj->js {:limit limit}))
(bean/->clj))]
(->> result
(util/distinct-by (fn [i] (string/trim (get-in i [:item :name]))))
(map
(fn [{:keys [item]}]
(:original-name item)))
(remove nil?)
(map string/trim)
(distinct)
(filter (fn [original-name]
(exact-matched? q original-name)))
bean/->js))))))

45
src/main/frontend/worker/util.cljs Normal file
View File

@ -0,0 +1,45 @@
(ns frontend.worker.util
"Worker utils"
(:require [clojure.string :as string]
["remove-accents" :as removeAccents]
[medley.core :as medley]
[logseq.graph-parser.util :as gp-util]))
(defn search-normalize
"Normalize string for searching (loose)"
[s remove-accents?]
(when s
(let [normalize-str (.normalize (string/lower-case s) "NFKC")]
(if remove-accents?
(removeAccents normalize-str)
normalize-str))))
(defn safe-re-find
{:malli/schema [:=> [:cat :any :string] [:or :nil :string [:vector [:maybe :string]]]]}
[pattern s]
(when-not (string? s)
;; TODO: sentry
(js/console.trace))
(when (string? s)
(re-find pattern s)))
(def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
(defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
(defn uuid-string?
{:malli/schema [:=> [:cat :string] :boolean]}
[s]
(boolean (safe-re-find exactly-uuid-pattern s)))
(def page-name-sanity-lc
"Delegate to gp-util to loosely couple app usages to graph-parser"
gp-util/page-name-sanity-lc)
(defn safe-page-name-sanity-lc
[s]
(if (string? s)
(page-name-sanity-lc s) s))
(defn distinct-by
[f col]
(medley/distinct-by f (seq col)))

27
src/main/logseq/api.cljs
View File

@ -15,6 +15,7 @@
[frontend.handler.recent :as recent-handler]
[frontend.handler.route :as route-handler]
[frontend.db :as db]
[frontend.db.async :as db-async]
[frontend.db.model :as db-model]
[frontend.db.query-dsl :as query-dsl]
[frontend.db.utils :as db-utils]
@ -137,11 +138,12 @@
(def ^:export get_current_graph_templates
(fn []
(when (state/get-current-repo)
(some-> (db-model/get-all-templates)
(update-vals db/pull)
(sdk-utils/normalize-keyword-for-json)
(bean/->js)))))
(when-let [repo (state/get-current-repo)]
(let [templates (db-async/<get-all-templates repo)]
(some-> templates
(update-vals db/pull)
(sdk-utils/normalize-keyword-for-json)
(bean/->js))))))
(def ^:export get_current_graph
(fn []
@ -967,20 +969,21 @@
(defn ^:export insert_template
[target-uuid template-name]
(when-let [target (and (page-handler/template-exists? template-name)
(db-model/get-block-by-uuid target-uuid))]
(editor-handler/insert-template! nil template-name {:target target}) nil))
(p/let [exists? (page-handler/<template-exists? template-name)]
(when exists?
(when-let [target (db-model/get-block-by-uuid target-uuid)]
(editor-handler/insert-template! nil template-name {:target target}) nil))))
(defn ^:export exist_template
[name]
(page-handler/template-exists? name))
(page-handler/<template-exists? name))
(defn ^:export create_template
[target-uuid template-name ^js opts]
(when (and template-name (db-model/get-block-by-uuid target-uuid))
(let [{:keys [overwrite]} (bean/->clj opts)
exist? (page-handler/template-exists? template-name)
repo (state/get-current-repo)]
(p/let [{:keys [overwrite]} (bean/->clj opts)
exist? (page-handler/<template-exists? template-name)
repo (state/get-current-repo)]
(if (or (not exist?) (true? overwrite))
(do (when-let [old-target (and exist? (db-model/get-template-by-name template-name))]
(property-handler/remove-block-property! repo (:block/uuid old-target) :template))

33
src/test/frontend/db/db_based_model_test.cljs
View File

@ -22,11 +22,6 @@
(use-fixtures :each start-and-destroy-db)
(deftest get-all-properties-test
(db-property-handler/set-block-property! repo fbid "property-1" "value" {})
(db-property-handler/set-block-property! repo fbid "property-2" "1" {})
(is (= '("property-1" "property-2") (model/get-all-properties))))
(deftest get-block-property-values-test
(db-property-handler/set-block-property! repo fbid "property-1" "value 1" {})
(db-property-handler/set-block-property! repo sbid "property-1" "value 2" {})
@ -34,21 +29,21 @@
(is (= (map second (model/get-block-property-values (:block/uuid property)))
["value 1" "value 2"]))))
(deftest get-db-property-values-test
(db-property-handler/set-block-property! repo fbid "property-1" "1" {})
(db-property-handler/set-block-property! repo sbid "property-1" "2" {})
(is (= [1 2] (model/get-db-property-values repo "property-1"))))
;; (deftest get-db-property-values-test
;; (db-property-handler/set-block-property! repo fbid "property-1" "1" {})
;; (db-property-handler/set-block-property! repo sbid "property-1" "2" {})
;; (is (= [1 2] (model/get-db-property-values repo "property-1"))))
(deftest get-db-property-values-test-with-pages
(let [opts {:redirect? false :create-first-block? false}
_ (page-handler/create! "page1" opts)
_ (page-handler/create! "page2" opts)
p1id (:block/uuid (db/entity [:block/name "page1"]))
p2id (:block/uuid (db/entity [:block/name "page2"]))]
(db-property-handler/upsert-property! repo "property-1" {:type :page} {})
(db-property-handler/set-block-property! repo fbid "property-1" p1id {})
(db-property-handler/set-block-property! repo sbid "property-1" p2id {})
(is (= '("[[page1]]" "[[page2]]") (model/get-db-property-values repo "property-1")))))
;; (deftest get-db-property-values-test-with-pages
;; (let [opts {:redirect? false :create-first-block? false}
;; _ (page-handler/create! "page1" opts)
;; _ (page-handler/create! "page2" opts)
;; p1id (:block/uuid (db/entity [:block/name "page1"]))
;; p2id (:block/uuid (db/entity [:block/name "page2"]))]
;; (db-property-handler/upsert-property! repo "property-1" {:type :page} {})
;; (db-property-handler/set-block-property! repo fbid "property-1" p1id {})
;; (db-property-handler/set-block-property! repo sbid "property-1" p2id {})
;; (is (= '("[[page1]]" "[[page2]]") (model/get-db-property-values repo "property-1")))))
(deftest get-all-classes-test
(let [opts {:redirect? false :create-first-block? false :class? true}

24
src/test/frontend/db/model_test.cljs
View File

@ -163,27 +163,3 @@ foo:: bar"}])
(is (= ["child 1" "child 2" "child 3"]
(map :block/content
(model/get-block-immediate-children test-helper/test-db (:block/uuid parent)))))))
(deftest get-property-values
(load-test-files [{:file/path "pages/Feature.md"
:file/content "type:: [[Class]]"}
{:file/path "pages/Class.md"
:file/content "type:: https://schema.org/Class\npublic:: true"}
{:file/path "pages/DatePicker.md"
:file/content "type:: #Feature, #Command"}
{:file/path "pages/Whiteboard___Tool___Eraser.md"
:file/content "type:: [[Tool]], [[Whiteboard/Object]]"}])
(let [type-values (set (model/get-property-values :type))
public-values (set (model/get-property-values :public))]
(is (contains? type-values "[[Class]]")
"Property value from single page-ref is wrapped in square brackets")
(is (= #{} (set/difference #{"[[Tool]]" "[[Whiteboard/Object]]"} type-values))
"Property values from multiple page-refs are wrapped in square brackets")
(is (= #{} (set/difference #{"#Feature" "#Command"} type-values))
"Property values from multiple tags have hashtags")
(is (contains? type-values "https://schema.org/Class")
"Property value text is not modified")
(is (contains? public-values "true")
"Property value that is not text is not modified")))