mirror of https://github.com/logseq/logseq
Split out parse-property for use in dsl-query and tests
- Add tests for all *property dsl queries with and without new config option - Add tests for property persistence - Add tests for property relationshipspull/6336/head
parent
405183db09
commit
0719163d30
|
@ -157,12 +157,15 @@
|
||||||
distinct)
|
distinct)
|
||||||
[]))
|
[]))
|
||||||
|
|
||||||
|
;; Regex first checks page-refs
|
||||||
|
;; to handle multi-word
|
||||||
|
;; page-refs e.g. #[[foo bar]]
|
||||||
|
(def ^:private page-ref-or-tag-re
|
||||||
|
(re-pattern (str "#?" (page-ref/->page-ref-re-str "(.*?)") "|#(\\S+)")))
|
||||||
|
|
||||||
(defn- extract-page-refs-and-tags [string]
|
(defn- extract-page-refs-and-tags [string]
|
||||||
(let [refs (map #(or (second %) (get % 2))
|
(let [refs (map #(or (second %) (get % 2))
|
||||||
;; Regex first checks page-refs
|
(re-seq page-ref-or-tag-re string))]
|
||||||
;; to handle multi-word
|
|
||||||
;; page-refs e.g. #[[foo bar]]
|
|
||||||
(re-seq #"#?\[\[(.*?)\]\]|#(\S+)" string))]
|
|
||||||
(or (seq refs) string)))
|
(or (seq refs) string)))
|
||||||
|
|
||||||
(defn- get-page-ref-names-from-properties
|
(defn- get-page-ref-names-from-properties
|
||||||
|
|
|
@ -124,9 +124,7 @@
|
||||||
(let [k (keyword (string/lower-case k))
|
(let [k (keyword (string/lower-case k))
|
||||||
v (if (contains? #{:title :description :filters :macro} k)
|
v (if (contains? #{:title :description :filters :macro} k)
|
||||||
v
|
v
|
||||||
;; user config ignored as we always want to parse links
|
(parse-property k v config-state))]
|
||||||
;; out of page properties
|
|
||||||
(parse-property k v (dissoc config-state :property-values-allow-links-and-text?)))]
|
|
||||||
[k v]))))
|
[k v]))))
|
||||||
properties (into (linked/map) properties)
|
properties (into (linked/map) properties)
|
||||||
macro-properties (filter (fn [x] (= :macro (first x))) properties)
|
macro-properties (filter (fn [x] (= :macro (first x))) properties)
|
||||||
|
@ -156,7 +154,8 @@
|
||||||
(remove string/blank?)))
|
(remove string/blank?)))
|
||||||
tags (:tags properties)
|
tags (:tags properties)
|
||||||
tags (->> (->vec-concat tags filetags)
|
tags (->> (->vec-concat tags filetags)
|
||||||
(remove string/blank?))
|
(remove string/blank?)
|
||||||
|
vec)
|
||||||
properties (assoc properties :tags tags :alias alias)
|
properties (assoc properties :tags tags :alias alias)
|
||||||
properties (-> properties
|
properties (-> properties
|
||||||
(update :filetags (constantly filetags)))
|
(update :filetags (constantly filetags)))
|
||||||
|
|
|
@ -8,6 +8,13 @@
|
||||||
|
|
||||||
(def colons "Property delimiter for markdown mode" "::")
|
(def colons "Property delimiter for markdown mode" "::")
|
||||||
|
|
||||||
|
(defn ->block-content
|
||||||
|
"Creates a block content string from properties map"
|
||||||
|
[properties]
|
||||||
|
(->> properties
|
||||||
|
(map #(str (name (key %)) (str colons " ") (val %)))
|
||||||
|
(string/join "\n")))
|
||||||
|
|
||||||
(defn properties-ast?
|
(defn properties-ast?
|
||||||
[block]
|
[block]
|
||||||
(and
|
(and
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
[clojure.set :as set]
|
[clojure.set :as set]
|
||||||
[logseq.graph-parser.mldoc :as gp-mldoc]
|
[logseq.graph-parser.mldoc :as gp-mldoc]
|
||||||
[logseq.graph-parser.util :as gp-util]
|
[logseq.graph-parser.util :as gp-util]
|
||||||
|
[logseq.graph-parser.property :as gp-property]
|
||||||
[logseq.graph-parser.util.page-ref :as page-ref :refer [right-brackets]]))
|
[logseq.graph-parser.util.page-ref :as page-ref :refer [right-brackets]]))
|
||||||
|
|
||||||
(defn get-file-basename
|
(defn get-file-basename
|
||||||
|
@ -199,7 +200,26 @@
|
||||||
(defonce non-parsing-properties
|
(defonce non-parsing-properties
|
||||||
(atom #{"background-color" "background_color"}))
|
(atom #{"background-color" "background_color"}))
|
||||||
|
|
||||||
|
(defn parse-property-value
|
||||||
|
"Property agnostic value parsing used to save properties to db and to
|
||||||
|
construct simple queries from user input"
|
||||||
|
[v]
|
||||||
|
(cond
|
||||||
|
(= v "true")
|
||||||
|
true
|
||||||
|
|
||||||
|
(= v "false")
|
||||||
|
false
|
||||||
|
|
||||||
|
(gp-util/safe-re-find #"^\d+$" v)
|
||||||
|
(parse-long v)
|
||||||
|
|
||||||
|
:else
|
||||||
|
(split-page-refs-without-brackets v)))
|
||||||
|
|
||||||
(defn parse-property
|
(defn parse-property
|
||||||
|
"Property value parsing that takes into account built-in properties, format
|
||||||
|
and user config"
|
||||||
([k v config-state]
|
([k v config-state]
|
||||||
(parse-property :markdown k v config-state))
|
(parse-property :markdown k v config-state))
|
||||||
([format k v config-state]
|
([format k v config-state]
|
||||||
|
@ -212,14 +232,6 @@
|
||||||
(get config-state :ignored-page-references-keywords)) k)
|
(get config-state :ignored-page-references-keywords)) k)
|
||||||
v
|
v
|
||||||
|
|
||||||
(= v "true")
|
|
||||||
true
|
|
||||||
(= v "false")
|
|
||||||
false
|
|
||||||
|
|
||||||
(and (not= k "alias") (gp-util/safe-re-find #"^\d+$" v))
|
|
||||||
(parse-long v)
|
|
||||||
|
|
||||||
(gp-util/wrapped-by-quotes? v) ; wrapped in ""
|
(gp-util/wrapped-by-quotes? v) ; wrapped in ""
|
||||||
v
|
v
|
||||||
|
|
||||||
|
@ -229,9 +241,11 @@
|
||||||
(gp-mldoc/link? format v)
|
(gp-mldoc/link? format v)
|
||||||
v
|
v
|
||||||
|
|
||||||
(and (:property-values-allow-links-and-text? config-state)
|
(contains? gp-property/editable-linkable-built-in-properties (keyword k))
|
||||||
(not (contains? gp-property/editable-linkable-built-in-properties k)))
|
(split-page-refs-without-brackets v)
|
||||||
|
|
||||||
|
(:property-values-allow-links-and-text? config-state)
|
||||||
v
|
v
|
||||||
|
|
||||||
:else
|
:else
|
||||||
(split-page-refs-without-brackets v)))))
|
(parse-property-value v)))))
|
||||||
|
|
|
@ -27,6 +27,11 @@ a logseq page-ref e.g. [[page name]]"
|
||||||
[page-name]
|
[page-name]
|
||||||
(str left-brackets page-name right-brackets))
|
(str left-brackets page-name right-brackets))
|
||||||
|
|
||||||
|
(defn ->page-ref-re-str
|
||||||
|
"Create a page ref regex escaped string given a page name"
|
||||||
|
[page-name]
|
||||||
|
(string/replace (->page-ref page-name) #"([\[\]])" "\\$1"))
|
||||||
|
|
||||||
(defn get-page-name
|
(defn get-page-name
|
||||||
"Extracts page-name from page-ref string"
|
"Extracts page-name from page-ref string"
|
||||||
[s]
|
[s]
|
||||||
|
|
|
@ -1,9 +1,10 @@
|
||||||
(ns logseq.graph-parser-test
|
(ns logseq.graph-parser-test
|
||||||
(:require [cljs.test :refer [deftest testing is]]
|
(:require [cljs.test :refer [deftest testing is are]]
|
||||||
[clojure.string :as string]
|
[clojure.string :as string]
|
||||||
[logseq.graph-parser :as graph-parser]
|
[logseq.graph-parser :as graph-parser]
|
||||||
[logseq.db :as ldb]
|
[logseq.db :as ldb]
|
||||||
[logseq.graph-parser.block :as gp-block]
|
[logseq.graph-parser.block :as gp-block]
|
||||||
|
[logseq.graph-parser.property :as gp-property]
|
||||||
[datascript.core :as d]))
|
[datascript.core :as d]))
|
||||||
|
|
||||||
(deftest parse-file
|
(deftest parse-file
|
||||||
|
@ -68,3 +69,123 @@
|
||||||
(test-property-order 4))
|
(test-property-order 4))
|
||||||
(testing "Sort order and persistence of 10 properties"
|
(testing "Sort order and persistence of 10 properties"
|
||||||
(test-property-order 10)))
|
(test-property-order 10)))
|
||||||
|
|
||||||
|
(deftest page-properties-persistence
|
||||||
|
(testing "Non-string property values"
|
||||||
|
(let [conn (ldb/start-conn)]
|
||||||
|
(graph-parser/parse-file conn
|
||||||
|
"lythe-of-heaven.md"
|
||||||
|
"rating:: 8\nrecommend:: true\narchive:: false"
|
||||||
|
{})
|
||||||
|
(is (= {:rating 8 :recommend true :archive false}
|
||||||
|
(->> (d/q '[:find (pull ?b [*])
|
||||||
|
:in $
|
||||||
|
:where [?b :block/properties]]
|
||||||
|
@conn)
|
||||||
|
(map (comp :block/properties first))
|
||||||
|
first)))))
|
||||||
|
|
||||||
|
(testing "Other special property cases"
|
||||||
|
(let [conn (ldb/start-conn)]
|
||||||
|
(graph-parser/parse-file conn
|
||||||
|
"foo.md"
|
||||||
|
"- desc:: \"true\""
|
||||||
|
{})
|
||||||
|
(is (= {:desc "\"true\""}
|
||||||
|
(->> (d/q '[:find (pull ?b [*])
|
||||||
|
:in $
|
||||||
|
:where [?b :block/properties]]
|
||||||
|
@conn)
|
||||||
|
(map (comp :block/properties first))
|
||||||
|
first))
|
||||||
|
"Quoted values are unparsed")))
|
||||||
|
|
||||||
|
(testing "Linkable built-in properties"
|
||||||
|
(let [conn (ldb/start-conn)
|
||||||
|
_ (graph-parser/parse-file conn
|
||||||
|
"lol.md"
|
||||||
|
"alias:: 233\ntags:: fun, facts"
|
||||||
|
{})
|
||||||
|
block (->> (d/q '[:find (pull ?b [:block/properties {:block/alias [:block/name]} {:block/tags [:block/name]}])
|
||||||
|
:in $
|
||||||
|
:where [?b :block/name "lol"]]
|
||||||
|
@conn)
|
||||||
|
(map first)
|
||||||
|
first)]
|
||||||
|
|
||||||
|
(is (= {:block/alias [{:block/name "233"}]
|
||||||
|
:block/tags [{:block/name "fun"} {:block/name "facts"}]
|
||||||
|
:block/properties {:alias ["233"] :tags ["fun" "facts"]}}
|
||||||
|
block))
|
||||||
|
|
||||||
|
(is (every? vector? (vals (:block/properties block)))
|
||||||
|
"Linked built-in property values as vectors provides for easier transforms"))))
|
||||||
|
|
||||||
|
(defn- property-relationships-test
|
||||||
|
"Runs tests on page properties and block properties. file-properties is what is
|
||||||
|
visible in a file and db-properties is what is pulled out from the db"
|
||||||
|
[file-properties db-properties user-config]
|
||||||
|
(let [conn (ldb/start-conn)
|
||||||
|
page-content (gp-property/->block-content file-properties)
|
||||||
|
;; Create Block properties from given page ones
|
||||||
|
block-property-transform (fn [m] (update-keys m #(keyword (str "block-" (name %)))))
|
||||||
|
block-content (gp-property/->block-content (block-property-transform file-properties))
|
||||||
|
_ (graph-parser/parse-file conn
|
||||||
|
"property-relationships.md"
|
||||||
|
(str page-content "\n- " block-content)
|
||||||
|
{:extract-options {:user-config user-config}})
|
||||||
|
pages (->> (d/q '[:find (pull ?b [* :block/properties])
|
||||||
|
:in $
|
||||||
|
:where [?b :block/name] [?b :block/properties]]
|
||||||
|
@conn)
|
||||||
|
(map first))
|
||||||
|
_ (assert (= 1 (count pages)))
|
||||||
|
blocks (->> (d/q '[:find (pull ?b [:block/pre-block? :block/properties
|
||||||
|
{:block/refs [:block/original-name]}])
|
||||||
|
:in $
|
||||||
|
:where [?b :block/properties] [(missing? $ ?b :block/name)]]
|
||||||
|
@conn)
|
||||||
|
(map first)
|
||||||
|
(map (fn [m] (update m :block/refs #(map :block/original-name %)))))
|
||||||
|
block-db-properties (block-property-transform db-properties)]
|
||||||
|
|
||||||
|
(is (= db-properties (:block/properties (first pages)))
|
||||||
|
"page has expected properties")
|
||||||
|
|
||||||
|
(is (= [true nil] (map :block/pre-block? blocks))
|
||||||
|
"page has 2 blocks, one of which is a pre-block")
|
||||||
|
|
||||||
|
(is (= [db-properties block-db-properties]
|
||||||
|
(map :block/properties blocks))
|
||||||
|
"pre-block/page and block have expected properties")
|
||||||
|
|
||||||
|
(are [db-props refs]
|
||||||
|
(= (->> (vals db-props)
|
||||||
|
(mapcat identity)
|
||||||
|
(concat (map name (keys db-props)))
|
||||||
|
set)
|
||||||
|
(set refs))
|
||||||
|
; pre-block/page has expected refs
|
||||||
|
db-properties (first (map :block/refs blocks))
|
||||||
|
;; block has expected refs
|
||||||
|
block-db-properties (second (map :block/refs blocks)))))
|
||||||
|
|
||||||
|
(deftest property-relationships
|
||||||
|
(let [properties {:single-link "[[bar]]"
|
||||||
|
:multi-link "[[Logseq]] is the fastest #triples #[[text editor]]"
|
||||||
|
:desc "This is a multiple sentence description. It has one [[link]]"}]
|
||||||
|
(testing "With default config"
|
||||||
|
(property-relationships-test
|
||||||
|
properties
|
||||||
|
{:single-link #{"bar"}
|
||||||
|
:multi-link #{"Logseq" "is the fastest" "triples" "text editor"}
|
||||||
|
:desc #{"This is a multiple sentence description. It has one" "link"}}
|
||||||
|
{}))
|
||||||
|
|
||||||
|
(testing "With :property-values-allow-links-and-text config"
|
||||||
|
(property-relationships-test
|
||||||
|
properties
|
||||||
|
{:single-link #{"bar"}
|
||||||
|
:multi-link #{"Logseq" "triples" "text editor"}
|
||||||
|
:desc #{"link"}}
|
||||||
|
{:property-values-allow-links-and-text? true}))))
|
||||||
|
|
|
@ -6,7 +6,6 @@
|
||||||
[clojure.set :as set]
|
[clojure.set :as set]
|
||||||
[clojure.string :as string]
|
[clojure.string :as string]
|
||||||
[clojure.walk :as walk]
|
[clojure.walk :as walk]
|
||||||
[frontend.state :as state]
|
|
||||||
[frontend.date :as date]
|
[frontend.date :as date]
|
||||||
[frontend.db.model :as model]
|
[frontend.db.model :as model]
|
||||||
[frontend.db.query-react :as query-react]
|
[frontend.db.query-react :as query-react]
|
||||||
|
@ -240,7 +239,7 @@
|
||||||
(let [k (string/replace (name (nth e 1)) "_" "-")
|
(let [k (string/replace (name (nth e 1)) "_" "-")
|
||||||
v (nth e 2)
|
v (nth e 2)
|
||||||
v (if-not (nil? v)
|
v (if-not (nil? v)
|
||||||
(text/parse-property k v (state/get-config))
|
(text/parse-property-value (str v))
|
||||||
v)
|
v)
|
||||||
v (if (coll? v) (first v) v)]
|
v (if (coll? v) (first v) v)]
|
||||||
{:query (list 'property '?b (keyword k) v)
|
{:query (list 'property '?b (keyword k) v)
|
||||||
|
@ -285,7 +284,7 @@
|
||||||
(let [[k v] (rest e)
|
(let [[k v] (rest e)
|
||||||
k (string/replace (name k) "_" "-")]
|
k (string/replace (name k) "_" "-")]
|
||||||
(if (some? v)
|
(if (some? v)
|
||||||
(let [v' (text/parse-property k v (state/get-config))
|
(let [v' (text/parse-property-value (str v))
|
||||||
val (if (coll? v') (first v') v')]
|
val (if (coll? v') (first v') v')]
|
||||||
{:query (list 'page-property '?p (keyword k) val)
|
{:query (list 'page-property '?p (keyword k) val)
|
||||||
:rules [:page-property]})
|
:rules [:page-property]})
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
[clojure.string :as str]
|
[clojure.string :as str]
|
||||||
[frontend.db :as db]
|
[frontend.db :as db]
|
||||||
[frontend.db.query-dsl :as query-dsl]
|
[frontend.db.query-dsl :as query-dsl]
|
||||||
[frontend.test.helper :as test-helper :refer [load-test-files]]))
|
[frontend.test.helper :as test-helper :include-macros true :refer [load-test-files]]))
|
||||||
|
|
||||||
;; TODO: quickcheck
|
;; TODO: quickcheck
|
||||||
;; 1. generate query filters
|
;; 1. generate query filters
|
||||||
|
@ -45,22 +45,8 @@
|
||||||
;; Tests
|
;; Tests
|
||||||
;; =====
|
;; =====
|
||||||
|
|
||||||
(deftest ^:focus block-property-queries
|
(defn- block-property-queries-test
|
||||||
(load-test-files [{:file/path "journals/2022_02_28.md"
|
[]
|
||||||
:file/content "a:: b
|
|
||||||
- b1
|
|
||||||
prop-a:: val-a
|
|
||||||
prop-num:: 2000
|
|
||||||
- b2
|
|
||||||
prop-a:: val-a
|
|
||||||
prop-b:: val-b
|
|
||||||
- b3
|
|
||||||
prop-c:: [[page a]], [[page b]], [[page c]]
|
|
||||||
prop-linked-num:: [[3000]]
|
|
||||||
prop-d:: [[no-space-link]]
|
|
||||||
- b4
|
|
||||||
prop-d:: nada"}])
|
|
||||||
|
|
||||||
(testing "Blocks have given property value"
|
(testing "Blocks have given property value"
|
||||||
(is (= #{"b1" "b2"}
|
(is (= #{"b1" "b2"}
|
||||||
(set (map (comp first str/split-lines :block/content)
|
(set (map (comp first str/split-lines :block/content)
|
||||||
|
@ -112,16 +98,31 @@ prop-d:: nada"}])
|
||||||
(dsl-query "(property prop-d)")))
|
(dsl-query "(property prop-d)")))
|
||||||
"Blocks that have a property"))
|
"Blocks that have a property"))
|
||||||
|
|
||||||
(deftest ^:focus page-property-queries
|
(deftest block-property-queries
|
||||||
(load-test-files [{:file/path "pages/page1.md"
|
(load-test-files [{:file/path "journals/2022_02_28.md"
|
||||||
:file/content "parent:: [[child page 1]], [[child-no-space]]"}
|
:file/content "a:: b
|
||||||
{:file/path "pages/page2.md"
|
- b1
|
||||||
:file/content "foo:: #bar"}
|
prop-a:: val-a
|
||||||
{:file/path "pages/page3.md"
|
prop-num:: 2000
|
||||||
:file/content "parent:: [[child page 1]], [[child page 2]]\nfoo:: bar"}
|
- b2
|
||||||
{:file/path "pages/page4.md"
|
prop-a:: val-a
|
||||||
:file/content "parent:: [[child page 2]]\nfoo:: baz"}])
|
prop-b:: val-b
|
||||||
|
- b3
|
||||||
|
prop-c:: [[page a]], [[page b]], [[page c]]
|
||||||
|
prop-linked-num:: [[3000]]
|
||||||
|
prop-d:: [[no-space-link]]
|
||||||
|
- b4
|
||||||
|
prop-d:: nada"}])
|
||||||
|
|
||||||
|
(testing "block property tests with default config"
|
||||||
|
(test-helper/with-config {}
|
||||||
|
(block-property-queries-test)))
|
||||||
|
(testing "block property tests with property-values-allow-links-and-text? config"
|
||||||
|
(test-helper/with-config {:property-values-allow-links-and-text? true}
|
||||||
|
(block-property-queries-test))))
|
||||||
|
|
||||||
|
(defn- page-property-queries-test
|
||||||
|
[]
|
||||||
(is (= ["page1" "page3" "page4"]
|
(is (= ["page1" "page3" "page4"]
|
||||||
(map :block/name (dsl-query "(page-property parent)")))
|
(map :block/name (dsl-query "(page-property parent)")))
|
||||||
"Pages have given property")
|
"Pages have given property")
|
||||||
|
@ -160,7 +161,33 @@ prop-d:: nada"}])
|
||||||
(map
|
(map
|
||||||
:block/name
|
:block/name
|
||||||
(dsl-query "(and (not (page-property foo bar)) (page-property parent [[child page 2]]))")))
|
(dsl-query "(and (not (page-property foo bar)) (page-property parent [[child page 2]]))")))
|
||||||
"Page property queries nested NOT in first clause"))
|
"Page property queries nested NOT in first clause")
|
||||||
|
|
||||||
|
(testing "boolean values"
|
||||||
|
(is (= ["page1"]
|
||||||
|
(map :block/name (dsl-query "(page-property interesting true)")))
|
||||||
|
"Boolean true")
|
||||||
|
|
||||||
|
(is (= ["page2" "page3"]
|
||||||
|
(map :block/name (dsl-query "(page-property interesting false)")))
|
||||||
|
"Boolean false")))
|
||||||
|
|
||||||
|
(deftest page-property-queries
|
||||||
|
(load-test-files [{:file/path "pages/page1.md"
|
||||||
|
:file/content "parent:: [[child page 1]], [[child-no-space]]\ninteresting:: true"}
|
||||||
|
{:file/path "pages/page2.md"
|
||||||
|
:file/content "foo:: #bar\ninteresting:: false"}
|
||||||
|
{:file/path "pages/page3.md"
|
||||||
|
:file/content "parent:: [[child page 1]], [[child page 2]]\nfoo:: bar\ninteresting:: false"}
|
||||||
|
{:file/path "pages/page4.md"
|
||||||
|
:file/content "parent:: [[child page 2]]\nfoo:: baz"}])
|
||||||
|
|
||||||
|
(testing "page property tests with default config"
|
||||||
|
(test-helper/with-config {}
|
||||||
|
(page-property-queries-test)))
|
||||||
|
(testing "page property tests with property-values-allow-links-and-text? config"
|
||||||
|
(test-helper/with-config {:property-values-allow-links-and-text? true}
|
||||||
|
(page-property-queries-test))))
|
||||||
|
|
||||||
(deftest task-queries
|
(deftest task-queries
|
||||||
(load-test-files [{:file/path "pages/page1.md"
|
(load-test-files [{:file/path "pages/page1.md"
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
(ns frontend.test.helper)
|
||||||
|
|
||||||
|
(defmacro with-config
|
||||||
|
[config & body]
|
||||||
|
`(let [repo# (frontend.state/get-current-repo)]
|
||||||
|
(frontend.state/set-config! repo# ~config)
|
||||||
|
~@body
|
||||||
|
(frontend.state/set-config! repo# nil)))
|
|
@ -229,7 +229,7 @@
|
||||||
;; :property-pages/excludelist
|
;; :property-pages/excludelist
|
||||||
|
|
||||||
;; Allows property values to contain tags, page-refs and free-form text
|
;; Allows property values to contain tags, page-refs and free-form text
|
||||||
;; property-values-allow-links-and-text? true
|
;; :property-values-allow-links-and-text? true
|
||||||
|
|
||||||
;; logbook setup
|
;; logbook setup
|
||||||
;; :logbook/settings
|
;; :logbook/settings
|
||||||
|
|
Loading…
Reference in New Issue