enhance: make journals easier to generate with :build/journal

Use it in tests that more appropriately belong in db dep.
Also did minor cleanup around page prep for page-uuids
experiment/tanstack-table
Gabriel Horner 2024-06-04 11:30:19 -04:00
parent 9c87edc802
commit 14222a546f
5 changed files with 227 additions and 180 deletions

View File

@ -8,6 +8,7 @@
(:require [logseq.db.sqlite.util :as sqlite-util]
[logseq.db.frontend.property.build :as db-property-build]
[logseq.common.util :as common-util]
[logseq.common.util.date-time :as date-time-util]
[clojure.string :as string]
[clojure.set :as set]
[datascript.core :as d]
@ -54,7 +55,8 @@
(into {})))
(defn- create-page-uuids
"Creates maps of unique page names, block contents and property names to their uuids"
"Creates maps of unique page names, block contents and property names to their uuids. Used to
provide user references for translate-property-value"
[pages-and-blocks]
(->> pages-and-blocks
(map :page)
@ -126,7 +128,7 @@
new-properties-tx (vec
(mapcat
(fn [[prop-name {:build/keys [schema-classes] :as prop-m}]]
(if-let [closed-values (:build/closed-values prop-m)]
(if-let [closed-values (seq (map #(merge {:uuid (random-uuid)} %) (:build/closed-values prop-m)))]
(let [db-ident (get-ident all-idents prop-name)]
(db-property-build/build-closed-values
db-ident
@ -202,9 +204,15 @@
(def Page-blocks
[:map
{:closed true}
[:page [:map
[:block/original-name :string]
[:build/properties {:optional true} User-properties]]]
[:page [:and
[:map
[:block/original-name {:optional true} :string]
[:build/journal {:optional true} :int]
[:build/properties {:optional true} User-properties]]
[:fn {:error/message ":block/original-name or :build/journal required"
:error/path [:block/original-name]}
(fn [m]
(or (:block/original-name m) (:build/journal m)))]]]
[:blocks {:optional true}
[:vector [:map
[:block/content :string]
@ -221,7 +229,7 @@
{:optional true}
[:vector [:map
[:value [:or :string :double]]
[:uuid :uuid]
[:uuid {:optional true} :uuid]
[:icon {:optional true} :map]]]]
[:build/schema-classes {:optional true} [:vector Class]]]])
@ -289,50 +297,37 @@
(defn- build-pages-and-blocks-tx
[pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties]
:or {page-id-fn :db/id}}]
(let [new-pages-from-refs
(->> pages-and-blocks
(mapcat
(fn [{:keys [blocks]}]
(->> blocks
(mapcat #(extract-content-refs (:block/content %)))
(remove page-uuids))))
(map #(hash-map :page {:block/original-name % :block/uuid (random-uuid)})))
pages-and-blocks' (concat pages-and-blocks new-pages-from-refs)
;; TODO: Make page-uuids' available to all fns once pages only take :block/original-name
page-uuids' (into page-uuids (map #(vector (get-in % [:page :block/original-name])
(get-in % [:page :block/uuid]))
new-pages-from-refs))]
(vec
(mapcat
(fn [{:keys [page blocks]}]
(let [new-page (merge
{:db/id (or (:db/id page) (new-db-id))
:block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
:block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
:block/format :markdown}
(dissoc page :build/properties :db/id :block/name :block/original-name))
pvalue-tx-m (->property-value-tx-m new-page (:build/properties page) properties all-idents)]
(into
(vec
(mapcat
(fn [{:keys [page blocks]}]
(let [new-page (merge
{:db/id (or (:db/id page) (new-db-id))
:block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
:block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
:block/format :markdown}
(dissoc page :build/properties :db/id :block/name :block/original-name))
pvalue-tx-m (->property-value-tx-m new-page (:build/properties page) properties all-idents)]
(into
;; page tx
(cond-> []
(seq pvalue-tx-m)
(into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
true
(conj
(sqlite-util/block-with-timestamps
(merge
new-page
(when (seq (:build/properties page))
(->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
page-uuids'
all-idents))))))
(cond-> []
(seq pvalue-tx-m)
(into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
true
(conj
(sqlite-util/block-with-timestamps
(merge
new-page
(when (seq (:build/properties page))
(->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
page-uuids
all-idents))))))
;; blocks tx
(reduce (fn [acc m]
(into acc
(->block-tx m properties page-uuids' all-idents (page-id-fn new-page))))
[]
blocks))))
pages-and-blocks'))))
(reduce (fn [acc m]
(into acc
(->block-tx m properties page-uuids all-idents (page-id-fn new-page))))
[]
blocks))))
pages-and-blocks)))
(defn- split-blocks-tx
"Splits a vec of maps tx into maps that can immediately be transacted,
@ -355,15 +350,49 @@
{:init-tx init-tx
:block-props-tx block-props-tx}))
(defn- add-new-pages-from-refs
[pages-and-blocks]
(let [existing-pages (->> pages-and-blocks (keep #(get-in % [:page :block/original-name])) set)
new-pages-from-refs
(->> pages-and-blocks
(mapcat
(fn [{:keys [blocks]}]
(->> blocks
(mapcat #(extract-content-refs (:block/content %)))
(remove existing-pages))))
(map #(hash-map :page {:block/original-name %})))]
(when (seq new-pages-from-refs)
(println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/original-name]) new-pages-from-refs))))
(concat pages-and-blocks new-pages-from-refs)))
(defn- pre-build-pages-and-blocks
"Pre builds :pages-and-blocks before any indexes like page-uuids are made"
[pages-and-blocks]
(let [;; add uuids for page-uuids
ensure-uuids (fn [{:keys [page blocks]}]
(cond-> {:page (merge {:block/uuid (random-uuid)} page)}
(seq blocks)
(assoc :blocks (mapv #(merge {:block/uuid (random-uuid)} %) blocks))))
expand-journal (fn [m]
(if-let [date-int (get-in m [:page :build/journal])]
(update m :page
(fn [page]
(let [page-name (date-time-util/int->journal-title date-int "MMM do, yyyy")]
(-> (dissoc page :build/journal)
(merge {:block/journal-day date-int
:block/original-name page-name
:block/type "journal"})))))
m))]
(->> pages-and-blocks
(map expand-journal)
add-new-pages-from-refs
(map ensure-uuids)
vec)))
(defn- build-blocks-tx*
[{:keys [pages-and-blocks properties classes graph-namespace]
:as options}]
(let [;; add uuids before tx for refs in :properties
pages-and-blocks' (mapv (fn [{:keys [page blocks]}]
(cond-> {:page (merge {:block/uuid (random-uuid)} page)}
(seq blocks)
(assoc :blocks (mapv #(merge {:block/uuid (random-uuid)} %) blocks))))
pages-and-blocks)
(let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks)
page-uuids (create-page-uuids pages-and-blocks')
all-idents (create-all-idents properties classes graph-namespace)
properties-tx (build-properties-tx properties page-uuids all-idents)
@ -429,9 +458,11 @@
(build-blocks-tx* options))
(defn create-blocks
"Builds txs with build-blocks-tx and transacts them. Usually used for testing"
"Builds txs with build-blocks-tx and transacts them. Also provides a shorthand
version of options that are useful for testing"
[conn options]
(let [{:keys [init-tx block-props-tx]} (build-blocks-tx options)]
(let [options' (if (vector? options) {:pages-and-blocks options} options)
{:keys [init-tx block-props-tx]} (build-blocks-tx options')]
(d/transact! conn init-tx)
(when (seq block-props-tx)
(d/transact! conn block-props-tx))))

View File

@ -0,0 +1,123 @@
(ns logseq.db.frontend.inputs-test
(:require [cljs.test :refer [deftest is]]
[cljs-time.core :as t]
[datascript.core :as d]
[logseq.db.frontend.rules :as rules]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.frontend.inputs :as db-inputs]
[logseq.db.sqlite.build :as sqlite-build]))
(defn- custom-query [db {:keys [inputs query input-options]}]
(let [q-args (cond-> (mapv #(db-inputs/resolve-input db % input-options) inputs)
(contains? (set query) '%)
(conj (rules/extract-rules rules/db-query-dsl-rules [:between])))]
(->> (apply d/q query db q-args)
(map first))))
(deftest resolve-input-for-page-and-block-inputs
(let [conn (d/create-conn db-schema/schema-for-db-based-graph)
parent-uuid (random-uuid)
_ (sqlite-build/create-blocks
conn
[{:page {:block/original-name "page1"}
:blocks [{:block/content "parent"
:block/uuid parent-uuid}
{:block/content "child 1"
:block/parent {:db/id [:block/uuid parent-uuid]}}
{:block/content "child 2"
:block/parent {:db/id [:block/uuid parent-uuid]}}]}])]
(is (= ["child 2" "child 1" "parent"]
(map :block/content
(custom-query @conn
{:inputs [:current-page]
:query '[:find (pull ?b [*])
:in $ ?current-page
:where [?b :block/page ?bp]
[?bp :block/name ?current-page]]
:input-options {:current-page-fn (constantly "page1")}})))
":current-page input resolves to current page name")
(is (= []
(map :block/content
(custom-query @conn
{:inputs [:current-page]
:query '[:find (pull ?b [*])
:in $ ?current-page
:where [?b :block/page ?bp]
[?bp :block/name ?current-page]]})))
":current-page input doesn't resolve when :current-page-fn not provided")
(is (= ["child 1" "child 2"]
(let [block-uuid (-> (d/q '[:find (pull ?b [:block/uuid])
:where [?b :block/content "parent"]] @conn)
ffirst
:block/uuid)]
(map :block/content
(custom-query @conn
{:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]
:input-options {:current-block-uuid block-uuid}}))))
":current-block input resolves to current block's :db/id")
(is (= []
(map :block/content
(custom-query @conn
{:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]})))
":current-block input doesn't resolve when :current-block-uuid is not provided")
(is (= []
(map :block/content
(custom-query @conn
{:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]
:input-options {:current-block-uuid :magic}})))
":current-block input doesn't resolve when current-block-uuid is invalid")
(is (= ["parent"]
(let [block-uuid (-> (d/q '[:find (pull ?b [:block/uuid])
:where [?b :block/content "child 1"]] @conn)
ffirst
:block/uuid)]
(map :block/content
(custom-query @conn
{:inputs [:parent-block]
:query '[:find (pull ?parent-block [*])
:in $ ?parent-block
:where [?parent-block :block/parent]]
:input-options {:current-block-uuid block-uuid}}))))
":parent-block input resolves to parent of current blocks's :db/id")))
(deftest resolve-input-for-journal-date-inputs
(let [conn (d/create-conn db-schema/schema-for-db-based-graph)
_ (sqlite-build/create-blocks
conn
[{:page {:build/journal 20230101}
:blocks [{:block/content "b1"}]}
{:page {:build/journal 20230107}
:blocks [{:block/content "b2"}]}])]
(is (= ["b2"]
(with-redefs [t/today (constantly (t/date-time 2023 1 7))]
(map :block/content
(custom-query @conn
{:inputs [:3d-before :today]
:query '[:find (pull ?b [*])
:in $ ?start ?end %
:where (between ?b ?start ?end)]}))))
":Xd-before and :today resolve to correct journal range")
(is (= ["b1"]
(with-redefs [t/today (constantly (t/date-time 2022 12 31))]
(map :block/content
(custom-query @conn
{:inputs [:tomorrow :4d-after]
:query '[:find (pull ?b [*])
:in $ ?start ?end %
:where (between ?b ?start ?end)]}))))
":tomorrow and :Xd-after resolve to correct journal range")))

View File

@ -24,16 +24,15 @@
[parent-uuid child-uuid] (repeatedly 2 random-uuid)
_ (sqlite-build/create-blocks
conn
{:pages-and-blocks
[{:page {:block/original-name "bar"}}
{:page {:block/original-name "page1"}
:blocks [{:block/content "parent [[foo]]"
:block/uuid parent-uuid}
{:block/content "child [[baz]]"
:block/uuid child-uuid
:block/parent {:db/id [:block/uuid parent-uuid]}}
{:block/content "grandchild [[bing]]"
:block/parent {:db/id [:block/uuid child-uuid]}}]}]})
[{:page {:block/original-name "bar"}}
{:page {:block/original-name "page1"}
:blocks [{:block/content "parent [[foo]]"
:block/uuid parent-uuid}
{:block/content "child [[baz]]"
:block/uuid child-uuid
:block/parent {:db/id [:block/uuid parent-uuid]}}
{:block/content "grandchild [[bing]]"
:block/parent {:db/id [:block/uuid child-uuid]}}]}])
blocks (get-blocks @conn)
;; Update parent block to replace 'foo' with 'bar' ref
new-tag-id (ffirst (d/q '[:find ?b :where [?b :block/original-name "bar"]] @conn))

View File

@ -18,14 +18,12 @@
(defn- date-journal-title [date]
(date-time-util/int->journal-title (date-time-util/date->int date) "MMM do, yyyy"))
(def date-journal-day date-time-util/date->int)
(defn- subtract-days
[date days]
(new js/Date (- (.getTime date) (* days 24 60 60 1000))))
(defn- build-closed-values-config
[{:keys [dates]}]
[_opts]
{:default-closed
(mapv #(hash-map :value %
:uuid (random-uuid)
@ -43,19 +41,9 @@
(mapv #(hash-map :value %
:uuid (random-uuid))
["Page 1" "Page 2" "Page 3"])
;; If this is enabled again, :uuid translation support would need to be added for :build/closed-values
:date-closed
;; TODO: Consider capitalized journal name when confirming queries work
(mapv #(hash-map :value (date-journal-title %)
:uuid (random-uuid))
dates)})
(defn- create-journal-page
[date journal-name-uuid-map]
(let [journal-name (date-journal-title date)]
{:block/original-name journal-name
:block/uuid (or (journal-name-uuid-map journal-name) (throw (ex-info "No uuid for journal name" {})))
:block/type "journal"
:block/journal-day (date-journal-day date)}))
{}})
(defn- create-init-data
[]
@ -66,7 +54,6 @@
page-values-tx (mapv #(hash-map :page
{:block/uuid (:uuid %) :block/original-name (:value %)})
(:page-closed closed-values-config))
journal-name->uuid (into {} (map (juxt :value :uuid) (:date-closed closed-values-config)))
;; Stores random closed values for use with queries
closed-values (atom {})
random-closed-value #(let [val (-> closed-values-config % rand-nth)]
@ -79,13 +66,13 @@
page-values-tx
;; Journals
[{:page
(create-journal-page today journal-name->uuid)
{:build/journal (date-time-util/date->int today)}
:blocks
[{:block/content "[[Block Properties]]"}
{:block/content "[[Block Property Queries]]"}
{:block/content "[[Page Property Queries]]"}]}
{:page (create-journal-page yesterday journal-name->uuid)}
{:page (create-journal-page two-days-ago journal-name->uuid)}
{:page {:build/journal (date-time-util/date->int yesterday)}}
{:page {:build/journal (date-time-util/date->int two-days-ago)}}
;; Block property blocks and queries
{:page {:block/original-name "Block Properties"}
@ -106,7 +93,7 @@
;; #_{:block/content "page-closed property block" :build/properties {:page-closed (random-closed-value :page-closed)}}
{:block/content "date property block" :build/properties {:date [:page (date-journal-title today)]}}
{:block/content "date-many property block" :build/properties {:date-many #{[:page (date-journal-title today)]
[:page (date-journal-title yesterday)]}}}
[:page (date-journal-title yesterday)]}}}
#_{:block/content "date-closed property block" :build/properties {:date-closed (random-closed-value :date-closed)}}]}
{:page {:block/original-name "Block Property Queries"}
:blocks
@ -143,7 +130,7 @@
;; #_{:page {:block/original-name "page-closed page" :build/properties {:page-closed (random-closed-value :page-closed)}}}
{:page {:block/original-name "date page" :build/properties {:date [:page (date-journal-title today)]}}}
{:page {:block/original-name "date-many page" :build/properties {:date-many #{[:page (date-journal-title today)]
[:page (date-journal-title yesterday)]}}}}
[:page (date-journal-title yesterday)]}}}}
#_{:page {:block/original-name "date-closed page" :build/properties {:date-closed (random-closed-value :date-closed)}}}
{:page {:block/original-name "Page Property Queries"}
:blocks

View File

@ -65,99 +65,6 @@ adds rules that users often use"
{:current-page-fn (constantly current-page)})))
;; TODO: Move most resolve-input tests to deps/db when a load-test-files helper is available for deps
(deftest resolve-input-for-page-and-block-inputs
(load-test-files [{:file/path "pages/page1.md"
:file/content
"- parent
- child 1
- child 2"}])
(is (= ["child 2" "child 1" "parent"]
(with-redefs [state/get-current-page (constantly "page1")]
(map :block/content
(custom-query {:inputs [:current-page]
:query '[:find (pull ?b [*])
:in $ ?current-page
:where [?b :block/page ?bp]
[?bp :block/name ?current-page]]}))))
":current-page input resolves to current page name")
(is (= []
(map :block/content
(custom-query {:inputs [:current-page]
:query '[:find (pull ?b [*])
:in $ ?current-page
:where [?b :block/page ?bp]
[?bp :block/name ?current-page]]}
{:current-page-fn nil})))
":current-page input doesn't resolve when not present")
(is (= ["child 1" "child 2"]
(let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
:where [?b :block/content "parent"]])
ffirst
:block/uuid)]
(map :block/content
(custom-query {:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]}
{:current-block-uuid block-uuid}))))
":current-block input resolves to current block's :db/id")
(is (= []
(map :block/content
(custom-query {:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]})))
":current-block input doesn't resolve when current-block-uuid is not provided")
(is (= []
(map :block/content
(custom-query {:inputs [:current-block]
:query '[:find (pull ?b [*])
:in $ ?current-block
:where [?b :block/parent ?current-block]]}
{:current-block-uuid :magic})))
":current-block input doesn't resolve when current-block-uuid is invalid")
(is (= ["parent"]
(let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
:where [?b :block/content "child 1"]])
ffirst
:block/uuid)]
(map :block/content
(custom-query {:inputs [:parent-block]
:query '[:find (pull ?parent-block [*])
:in $ ?parent-block
:where [?parent-block :block/parent]]}
{:current-block-uuid block-uuid}))))
":parent-block input resolves to parent of current blocks's :db/id"))
(deftest resolve-input-for-journal-date-inputs
(load-test-files [{:file/path "journals/2023_01_01.md"
:file/content "- b1"}
{:file/path "journals/2023_01_07.md"
:file/content "- b2"}])
(is (= ["b2"]
(with-redefs [t/today (constantly (t/date-time 2023 1 7))]
(map :block/content
(custom-query {:inputs [:3d-before :today]
:query '[:find (pull ?b [*])
:in $ ?start ?end
:where (between ?b ?start ?end)]}))))
":Xd-before and :today resolve to correct journal range")
(is (= ["b1"]
(with-redefs [t/today (constantly (t/date-time 2022 12 31))]
(map :block/content
(custom-query {:inputs [:tomorrow :4d-after]
:query '[:find (pull ?b [*])
:in $ ?start ?end
:where (between ?b ?start ?end)]}))))
":tomorrow and :Xd-after resolve to correct journal range"))
;; These tests rely on seeding timestamps with properties. If this ability goes
;; away we could still test page-level timestamps