greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: some tests on graph_parser

pull/6529/head
Tienson Qin 2022-08-31 14:26:51 +08:00
parent d64042df81
commit 040fd5510f
8 changed files with 153 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
deps/graph-parser/src/logseq/graph_parser/block.cljs vendored
View File

@ -177,9 +177,18 @@
distinct)))
(defn extract-properties
[properties user-config]
[properties user-config & {:keys [format]
:or {format :markdown}}]
(when (seq properties)
(let [properties (seq properties)
properties (if (= 2 (count (first properties))) ; property value not parsed yet
(map
(fn [[k v]]
(let [mldoc-ast (-> (gp-mldoc/get-references v (gp-mldoc/default-config format))
gp-util/json->clj)]
[k v mldoc-ast]))
properties)
properties)
page-refs (get-page-ref-names-from-properties properties user-config)
*invalid-properties (atom #{})
properties (->> properties
@ -612,7 +621,7 @@
(recur headings (rest blocks) timestamps' properties body))
(gp-property/properties-ast? block)
(let [properties (extract-properties (second block) user-config)]
(let [properties (extract-properties (second block) user-config :format format)]
(recur headings (rest blocks) timestamps properties body))
(heading-block? block)

124
deps/graph-parser/src/logseq/graph_parser/extract.cljc vendored
View File

@ -39,62 +39,69 @@
(or first-block-name file-name)
(or file-name first-block-name)))))))
(defn- build-page-entity
[properties file page-name page ref-tags {:keys [date-formatter db from-page]}]
(defn- extract-page-alias-and-tags
[page-m page page-name properties]
(let [alias (:alias properties)
alias' (if (string? alias) [alias] alias)
alias' (if (coll? alias) alias [(str alias)])
aliases (and alias'
(seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
(string/blank? %)) ;; disable blank alias
alias')))
aliases' (->>
(map
(fn [alias]
(let [page-name (gp-util/page-name-sanity-lc alias)
aliases (distinct
(conj
(remove #{alias} aliases)
page))
aliases (when (seq aliases)
(map
(fn [alias]
{:block/name (gp-util/page-name-sanity-lc alias)})
aliases))]
(if (seq aliases)
{:block/name page-name
:block/alias aliases}
{:block/name page-name})))
aliases)
(remove nil?))
[*valid-properties *invalid-properties]
aliases' (keep
(fn [alias]
(let [page-name (gp-util/page-name-sanity-lc alias)
aliases (distinct
(conj
(remove #{alias} aliases)
page))
aliases (when (seq aliases)
(map
(fn [alias]
{:block/name (gp-util/page-name-sanity-lc alias)})
aliases))]
(if (seq aliases)
{:block/name page-name
:block/original-name alias
:block/alias aliases}
{:block/name page-name
:block/original-name alias})))
aliases)
result (cond-> page-m
(seq aliases')
(assoc :block/alias aliases')
(:tags properties)
(assoc :block/tags (let [tags (:tags properties)
tags (if (coll? tags) tags [(str tags)])
tags (remove string/blank? tags)]
(map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
:block/original-name tag})
tags))))]
(update result :block/properties #(dissoc % :tags :alias))))
(defn- build-page-map
[properties file page page-name {:keys [date-formatter db from-page]}]
(let [[*valid-properties *invalid-properties]
((juxt filter remove)
(fn [[k _v]] (gp-property/valid-property-name? (str k))) properties)
valid-properties (into {} *valid-properties)
invalid-properties (set (map (comp name first) *invalid-properties))]
valid-properties (-> (into {} *valid-properties)
(dissoc :tags :alias))
invalid-properties (set (map (comp name first) *invalid-properties))
page-m (->
(gp-util/remove-nils
(assoc
(gp-block/page-name->map page false db true date-formatter
:from-page from-page)
:block/file {:file/path (gp-util/path-normalize file)}))
(extract-page-alias-and-tags page page-name properties))]
(cond->
(gp-util/remove-nils
(assoc
(gp-block/page-name->map page false db true date-formatter
:from-page from-page)
:block/file {:file/path (gp-util/path-normalize file)}))
page-m
(seq valid-properties)
(assoc :block/properties valid-properties)
(seq valid-properties)
(assoc :block/properties valid-properties)
(seq invalid-properties)
(assoc :block/invalid-properties invalid-properties)
(seq aliases')
(assoc :block/alias aliases')
(:tags properties)
(assoc :block/tags (let [tags (:tags properties)
tags (if (string? tags) [tags] tags)
tags (remove string/blank? tags)]
(swap! ref-tags set/union (set tags))
(map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
:block/original-name tag})
tags))))))
(seq invalid-properties)
(assoc :block/invalid-properties invalid-properties))))
;; TODO: performance improvement
(defn- extract-pages-and-blocks
@ -102,10 +109,14 @@
(try
(let [page (get-page-name file ast page-name-order)
[page page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
(gp-block/with-parent-and-left {:block/name page-name}))
options' (-> options
(assoc :page-name page-name
:original-page-name page)
(dissoc :page-name-order))
blocks (->> (gp-block/extract-blocks ast content false format options')
(gp-block/with-parent-and-left {:block/name page-name})
(vec))
ref-pages (atom #{})
ref-tags (atom #{})
blocks (map (fn [block]
(if (contains? #{"macro"} (:block/type block))
block
@ -122,22 +133,19 @@
:block/refs block-ref-pages
:block/path-refs block-path-ref-pages)))))
blocks)
page-entity (build-page-entity properties file page-name page ref-tags
(assoc options :from-page page))
namespace-pages (let [page (:block/original-name page-entity)]
properties (if (:block/pre-block? (first blocks))
(:block/properties (first blocks))
properties)
page-map (build-page-map properties file page page-name (assoc options' :from-page page))
namespace-pages (let [page (:block/original-name page-map)]
(when (text/namespace-page? page)
(->> (gp-util/split-namespace-pages page)
(map (fn [page]
(-> (gp-block/page-name->map page true db true date-formatter)
(assoc :block/format format)))))))
pages (->> (concat
[page-entity]
[page-map]
@ref-pages
(map
(fn [page]
{:block/original-name page
:block/name (gp-util/page-name-sanity-lc page)})
@ref-tags)
namespace-pages)
;; remove block references
(remove vector?)

4
deps/graph-parser/src/logseq/graph_parser/mldoc.cljc vendored
View File

@ -114,7 +114,7 @@
[k v mldoc-ast]))
properties-ast)]
(if (seq properties)
(cons [["Property_Drawer" properties] nil] other-ast)
(cons [["Properties" properties] nil] other-ast)
original-ast))))
(defn ->edn
@ -145,7 +145,7 @@
[])))
(defn ast-link?
[{:keys [type link]}]
[[type link]]
(let [[ref-type ref-value] (:url link)]
(and (= "Link" type)
(or

3
deps/graph-parser/src/logseq/graph_parser/text.cljs vendored
View File

@ -141,6 +141,9 @@
(string/blank? v)
nil
(and (string? v) (gp-util/wrapped-by-quotes? v))
v
(seq refs)
refs

41
deps/graph-parser/test/logseq/graph_parser/block_test.cljs vendored
View File

@ -6,28 +6,29 @@
(are [x y] (= (:properties (gp-block/extract-properties x {})) y)
;; Built-in properties
[["background-color" "#000000"]] {:background-color "#000000"}
[["alias" "name/with space"]] {:alias #{"name/with space"}}
[["tags" "foo, bar"]] {:tags #{"foo" "bar"}}
[["tags" "'bar'"]] {:tags #{"'bar'"}}
[["alias" "[[name/with space]]"]] {:alias #{"name/with space"}}
[["tags" "[[foo]], [[bar]]"]] {:tags #{"foo" "bar"}}
[["tags" "[[foo]] [[bar]]"]] {:tags #{"foo" "bar"}}
[["tags" "bar"]] {:tags "bar"}
[["file-path" "file:///home/x, y.pdf"]] {:file-path "file:///home/x, y.pdf"}
;; User properties
[["year" "1000"]] {:year 1000}
[["year" "\"1000\""]] {:year "\"1000\""}
[["year" "1000"] ["alias" "name/with space"]] {:year 1000, :alias #{"name/with space"}}
[["year" "1000"] ["tags" "name/with space"]] {:year 1000, :tags #{"name/with space"}}
[["year" "1000"] ["tags" "name/with space, another"]] {:year 1000, :tags #{"name/with space" "another"}}
[["year" "1000"] ["alias" "name/with space, another"]] {:year 1000, :alias #{"name/with space" "another"}}
[["year" "1000"] ["alias" "name/with space, [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
[["year" "1000"] ["alias" "name/with space, [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
[["year" "1000"] ["alias" "[[name/with space]]"]] {:year 1000, :alias #{"name/with space"}}
[["year" "1000"] ["tags" "[[name/with space]]"]] {:year 1000, :tags #{"name/with space"}}
[["year" "1000"] ["tags" "[[name/with space]], [[another]]"]] {:year 1000, :tags #{"name/with space" "another"}}
[["year" "1000"] ["alias" "[[name/with space]], [[another]]"]] {:year 1000, :alias #{"name/with space" "another"}}
[["year" "1000"] ["alias" "[[name/with space]], [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
[["year" "1000"] ["alias" "[[name/with space]], [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
[["foo" "bar"]] {:foo "bar"}
[["foo" "bar, baz"]] {:foo #{"bar" "baz"}}
[["foo" "bar, [[baz]]"]] {:foo #{"bar" "baz"}}
[["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
[["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
[["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
[["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
[["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
[["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
[["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
[["foo" "[[bar]], [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
[["foo" "[[bar]], [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
(testing "page-refs"
(are [x y] (= (vec (:page-refs
@ -35,18 +36,18 @@
[["year" "1000"]] ["year"]
[["year" "\"1000\""]] ["year"]
[["year" "1000"] ["month" "12"]] ["year" "month"]
[["foo" "[[bar]] test"]] ["bar" "test" "foo"]
[["foo" "[[bar]] test [[baz]]"]] ["bar" "test" "baz" "foo"]
[["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "test" "baz" "nested [[baz]]" "foo"]
[["foo" "[[bar]] test"]] ["bar" "foo"]
[["foo" "[[bar]] test [[baz]]"]] ["bar" "baz" "foo"]
[["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "baz" "nested [[baz]]" "foo"]
[["foo" "#bar, #baz"]] ["bar" "baz" "foo"]
[["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "test" "foo"])
[["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "foo"])
(are [x y] (= (vec (:page-refs
(gp-block/extract-properties x {:property-pages/enabled? false}))) y)
[["year" "1000"]] []
[["year" "1000"] ["month" "12"]] []
[["foo" "[[bar]] test"]] ["bar" "test"])
[["foo" "[[bar]] test"]] ["bar"])
(is (= ["year"]
(:page-refs
@ -64,7 +65,7 @@
(is (= ["foo" "bar"]
(:page-refs
(gp-block/extract-properties
;; tags is linkable and background-color is not
[["tags" "foo, bar"] ["background-color" "#008000"]]
;; tags is linkable and background-color is not
[["tags" "[[foo]], [[bar]]"] ["background-color" "#008000"]]
{:property-pages/enabled? true})))
"Only editable linkable built-in properties have page-refs in property values")))

46
deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs vendored
View File

@ -1,5 +1,6 @@
(ns logseq.graph-parser.mldoc-test
(:require [logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.text :as gp-text]
[clojure.string :as string]
[logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
[logseq.graph-parser.cli :as gp-cli]
@ -55,10 +56,10 @@
"Basic src example")
(is (= [["Src"
{:lines [" hello" "\n" " world" "\n"],
:pos_meta {:start_pos 7, :end_pos 25},
:full_content "```\nhello\nworld\n```"}]
{:start_pos 1, :end_pos 29}]
{:lines [" hello" "\n" " world" "\n"],
:pos_meta {:start_pos 7, :end_pos 25},
:full_content "```\nhello\nworld\n```"}]
{:start_pos 1, :end_pos 29}]
(second (gp-mldoc/->edn "
```
hello
@ -67,25 +68,32 @@
" md-config)))
"Src example with leading whitespace"))
(defn- get-properties
[x]
(->> (gp-mldoc/->edn x md-config)
ffirst second
(map (fn [[k v ast]]
[(keyword k) (gp-text/parse-property k v ast {})]))
(into {})))
(deftest md-properties-test
(are [x y] (= [["Properties" y] nil]
(first (gp-mldoc/->edn x md-config)))
(are [x y] (= y (get-properties x))
;; comma separates values
"property:: foo, bar"
{:property #{"foo" "bar"}}
;; comma separates values
"property:: [[foo]], [[bar]]"
{:property #{"foo" "bar"}}
;; alias property
"alias:: foo,, bar"
{:alias ["foo" "bar"]}
;; alias property
"alias:: [[foo]], #bar"
{:alias #{"foo" "bar"}}
;; tags property
"tags:: foo,bar,foo"
{:tags ["foo" "bar"]}
;; tags property
"tags:: #foo,#bar,#foo"
{:tags #{"foo" "bar"}}
;; title property
"title:: comma, is ok"
{:title "comma, is ok"}))
;; title property
"title:: comma, is ok"
{:title "comma, is ok"}))
(deftest name-definition-test
(is (= [["List"
@ -110,7 +118,7 @@
(testing "just title"
(let [content "#+TITLE: some title "
props (parse-properties content)]
(is (= "some title " (:title props)))))
(is (= "some title " (second (first props))))))
(testing "filetags"
(let [content "#+FILETAGS: :tag1:tag2:@tag:

23
deps/graph-parser/test/logseq/graph_parser/text_test.cljs vendored
View File

@ -1,6 +1,8 @@
(ns logseq.graph-parser.text-test
(:require [cljs.test :refer [are deftest testing]]
[logseq.graph-parser.text :as text]))
[logseq.graph-parser.text :as text]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.util :as gp-util]))
(deftest test-get-page-name
[]
@ -59,23 +61,28 @@
"**foobar" "foobar"
"*********************foobar" "foobar")))
(defn- parse-property
[k v]
(let [references (-> (gp-mldoc/get-references v (gp-mldoc/default-config :markdown))
(gp-util/json->clj))]
(text/parse-property k v references {})))
(deftest test-parse-property
(testing "parse-property"
(are [k v y] (= (text/parse-property k v [] {}) y)
(are [k v y] (= (parse-property k v) y)
:tags "foo" "foo"
:tags "foo, bar" #{"foo" "bar"}
:tags "foo,bar" #{"foo" "bar"}
:tags "[[foo]], [[bar]]" #{"foo" "bar"}
:tags "[[foo]],[[bar]]" #{"foo" "bar"}
:tags "[[foo]]" #{"foo"}
:tags "[[foo]] [[bar]]" #{"foo" "bar"}
:tags "[[foo]], [[bar]]" #{"foo" "bar"}
:tags "[[foo]], [[bar]], #baz" #{"foo" "bar" "baz"}
:tags "#baz, [[foo]], [[bar]]" #{"foo" "bar" "baz"}
:tags "[[foo [[bar]]]]" #{"foo [[bar]]"}
:tags "[[foo [[bar]]]], baz" #{"baz" "foo [[bar]]"}))
:tags "[[foo [[bar]]]], [[baz]]" #{"baz" "foo [[bar]]"}))
(testing "parse-property with quoted strings"
(are [k v y] (= (text/parse-property k v [] {}) y)
(are [k v y] (= (parse-property k v) y)
:tags "\"foo, bar\"" "\"foo, bar\""
:tags "\"[[foo]], [[bar]]\"" "\"[[foo]], [[bar]]\""
:tags "baz, \"[[foo]], [[bar]]\"" #{"baz"})))
:tags "\"[[foo]], [[bar]]\"" "\"[[foo]], [[bar]]\"")))
#_(cljs.test/test-ns 'logseq.graph-parser.text-test)

14
src/main/frontend/db/query_dsl.cljs
View File

@ -238,12 +238,14 @@
(defn parse-property-value
"Parses non-string property values or any page-ref like values"
[v]
(let [v (string/trim v)]
(if-some [res (text/parse-non-string-property-value v)]
res
(if (string/starts-with? v "#")
(subs v 1)
(or (page-ref/get-page-name v) v)))))
(let [result (if-some [res (text/parse-non-string-property-value v)]
res
(if (string/starts-with? v "#")
(subs v 1)
(or (page-ref/get-page-name v) v)))]
(if (string? result)
(string/trim result)
result)))
(defn- build-property-two-arg
[e]