enhance: make journals easier to generate with :build/journal

Use it in tests that more appropriately belong in db dep.
Also did minor cleanup around page prep for page-uuids

deps/db/src/logseq/db/sqlite/build.cljs

@@ -8,6 +8,7 @@
   (:require [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
             [clojure.string :as string]
             [clojure.set :as set]
             [datascript.core :as d]
@@ -54,7 +55,8 @@
    (into {})))
 
 (defn- create-page-uuids
-  "Creates maps of unique page names, block contents and property names to their uuids"
+  "Creates maps of unique page names, block contents and property names to their uuids. Used to
+   provide user references for translate-property-value"
   [pages-and-blocks]
   (->> pages-and-blocks
        (map :page)
@@ -126,7 +128,7 @@
         new-properties-tx (vec
                            (mapcat
                             (fn [[prop-name {:build/keys [schema-classes] :as prop-m}]]
-                              (if-let [closed-values (:build/closed-values prop-m)]
+                              (if-let [closed-values (seq (map #(merge {:uuid (random-uuid)} %) (:build/closed-values prop-m)))]
                                 (let [db-ident (get-ident all-idents prop-name)]
                                   (db-property-build/build-closed-values
                                    db-ident
@@ -202,9 +204,15 @@
 (def Page-blocks
   [:map
    {:closed true}
-   [:page [:map
+   [:page [:and
-           [:block/original-name :string]
+           [:map
-           [:build/properties {:optional true} User-properties]]]
+            [:block/original-name {:optional true} :string]
+            [:build/journal {:optional true} :int]
+            [:build/properties {:optional true} User-properties]]
+           [:fn {:error/message ":block/original-name or :build/journal required"
+                 :error/path [:block/original-name]}
+            (fn [m]
+              (or (:block/original-name m) (:build/journal m)))]]]
    [:blocks {:optional true}
     [:vector [:map
               [:block/content :string]
@@ -221,7 +229,7 @@
      {:optional true}
      [:vector [:map
                [:value [:or :string :double]]
-               [:uuid :uuid]
+               [:uuid {:optional true} :uuid]
                [:icon {:optional true} :map]]]]
     [:build/schema-classes {:optional true} [:vector Class]]]])
 
@@ -289,50 +297,37 @@
 (defn- build-pages-and-blocks-tx
   [pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties]
                                            :or {page-id-fn :db/id}}]
-  (let [new-pages-from-refs
+  (vec
-        (->> pages-and-blocks
+   (mapcat
-             (mapcat
+    (fn [{:keys [page blocks]}]
-              (fn [{:keys [blocks]}]
+      (let [new-page (merge
-                (->> blocks
+                      {:db/id (or (:db/id page) (new-db-id))
-                     (mapcat #(extract-content-refs (:block/content %)))
+                       :block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
-                     (remove page-uuids))))
+                       :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
-             (map #(hash-map :page {:block/original-name % :block/uuid (random-uuid)})))
+                       :block/format :markdown}
-        pages-and-blocks' (concat pages-and-blocks new-pages-from-refs)
+                      (dissoc page :build/properties :db/id :block/name :block/original-name))
-        ;; TODO: Make page-uuids' available to all fns once pages only take :block/original-name
+            pvalue-tx-m (->property-value-tx-m new-page (:build/properties page) properties all-idents)]
-        page-uuids' (into page-uuids (map #(vector (get-in % [:page :block/original-name])
+        (into
-                                                   (get-in % [:page :block/uuid]))
-                                          new-pages-from-refs))]
-    (vec
-     (mapcat
-      (fn [{:keys [page blocks]}]
-        (let [new-page (merge
-                        {:db/id (or (:db/id page) (new-db-id))
-                         :block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
-                         :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
-                         :block/format :markdown}
-                        (dissoc page :build/properties :db/id :block/name :block/original-name))
-              pvalue-tx-m (->property-value-tx-m new-page (:build/properties page) properties all-idents)]
-          (into
          ;; page tx
-           (cond-> []
+         (cond-> []
-             (seq pvalue-tx-m)
+           (seq pvalue-tx-m)
-             (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
+           (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
-             true
+           true
-             (conj
+           (conj
-              (sqlite-util/block-with-timestamps
+            (sqlite-util/block-with-timestamps
-               (merge
+             (merge
-                new-page
+              new-page
-                (when (seq (:build/properties page))
+              (when (seq (:build/properties page))
-                  (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                                      page-uuids'
+                                    page-uuids
-                                      all-idents))))))
+                                    all-idents))))))
          ;; blocks tx
-           (reduce (fn [acc m]
+         (reduce (fn [acc m]
-                     (into acc
+                   (into acc
-                           (->block-tx m properties page-uuids' all-idents (page-id-fn new-page))))
+                         (->block-tx m properties page-uuids all-idents (page-id-fn new-page))))
-                   []
+                 []
-                   blocks))))
+                 blocks))))
-      pages-and-blocks'))))
+    pages-and-blocks)))
 
 (defn- split-blocks-tx
   "Splits a vec of maps tx into maps that can immediately be transacted,
@@ -355,15 +350,49 @@
     {:init-tx init-tx
      :block-props-tx block-props-tx}))
 
+(defn- add-new-pages-from-refs
+  [pages-and-blocks]
+  (let [existing-pages (->> pages-and-blocks (keep #(get-in % [:page :block/original-name])) set)
+        new-pages-from-refs
+        (->> pages-and-blocks
+             (mapcat
+              (fn [{:keys [blocks]}]
+                (->> blocks
+                     (mapcat #(extract-content-refs (:block/content %)))
+                     (remove existing-pages))))
+             (map #(hash-map :page {:block/original-name %})))]
+    (when (seq new-pages-from-refs)
+      (println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/original-name]) new-pages-from-refs))))
+    (concat pages-and-blocks new-pages-from-refs)))
+
+(defn- pre-build-pages-and-blocks
+  "Pre builds :pages-and-blocks before any indexes like page-uuids are made"
+  [pages-and-blocks]
+  (let [;; add uuids for page-uuids
+        ensure-uuids (fn [{:keys [page blocks]}]
+                       (cond-> {:page (merge {:block/uuid (random-uuid)} page)}
+                         (seq blocks)
+                         (assoc :blocks (mapv #(merge {:block/uuid (random-uuid)} %) blocks))))
+        expand-journal (fn [m]
+                         (if-let [date-int (get-in m [:page :build/journal])]
+                           (update m :page
+                                   (fn [page]
+                                     (let [page-name (date-time-util/int->journal-title date-int "MMM do, yyyy")]
+                                       (-> (dissoc page :build/journal)
+                                           (merge {:block/journal-day date-int
+                                                   :block/original-name page-name
+                                                   :block/type "journal"})))))
+                           m))]
+    (->> pages-and-blocks
+         (map expand-journal)
+         add-new-pages-from-refs
+         (map ensure-uuids)
+         vec)))
+
 (defn- build-blocks-tx*
   [{:keys [pages-and-blocks properties classes graph-namespace]
     :as options}]
-  (let [;; add uuids before tx for refs in :properties
+  (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks)
-        pages-and-blocks' (mapv (fn [{:keys [page blocks]}]
-                                  (cond-> {:page (merge {:block/uuid (random-uuid)} page)}
-                                    (seq blocks)
-                                    (assoc :blocks (mapv #(merge {:block/uuid (random-uuid)} %) blocks))))
-                                pages-and-blocks)
         page-uuids (create-page-uuids pages-and-blocks')
         all-idents (create-all-idents properties classes graph-namespace)
         properties-tx (build-properties-tx properties page-uuids all-idents)
@@ -429,9 +458,11 @@
   (build-blocks-tx* options))
 
 (defn create-blocks
-  "Builds txs with build-blocks-tx and transacts them. Usually used for testing"
+  "Builds txs with build-blocks-tx and transacts them. Also provides a shorthand
+  version of options that are useful for testing"
   [conn options]
-  (let [{:keys [init-tx block-props-tx]} (build-blocks-tx options)]
+  (let [options' (if (vector? options) {:pages-and-blocks options} options)
+        {:keys [init-tx block-props-tx]} (build-blocks-tx options')]
     (d/transact! conn init-tx)
     (when (seq block-props-tx)
       (d/transact! conn block-props-tx))))

deps/db/test/logseq/db/frontend/inputs_test.cljs

@@ -0,0 +1,123 @@
+(ns logseq.db.frontend.inputs-test
+  (:require [cljs.test :refer [deftest is]]
+            [cljs-time.core :as t]
+            [datascript.core :as d]
+            [logseq.db.frontend.rules :as rules]
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.frontend.inputs :as db-inputs]
+            [logseq.db.sqlite.build :as sqlite-build]))
+
+(defn- custom-query [db {:keys [inputs query input-options]}]
+  (let [q-args (cond-> (mapv #(db-inputs/resolve-input db % input-options) inputs)
+                 (contains? (set query) '%)
+                 (conj (rules/extract-rules rules/db-query-dsl-rules [:between])))]
+    (->> (apply d/q query db q-args)
+         (map first))))
+
+(deftest resolve-input-for-page-and-block-inputs
+  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+        parent-uuid (random-uuid)
+        _ (sqlite-build/create-blocks
+           conn
+           [{:page {:block/original-name "page1"}
+             :blocks [{:block/content "parent"
+                       :block/uuid parent-uuid}
+                      {:block/content "child 1"
+                       :block/parent {:db/id [:block/uuid parent-uuid]}}
+                      {:block/content "child 2"
+                       :block/parent {:db/id [:block/uuid parent-uuid]}}]}])]
+    (is (= ["child 2" "child 1" "parent"]
+           (map :block/content
+                (custom-query @conn
+                              {:inputs [:current-page]
+                               :query '[:find (pull ?b [*])
+                                        :in $ ?current-page
+                                        :where [?b :block/page ?bp]
+                                        [?bp :block/name ?current-page]]
+                               :input-options {:current-page-fn (constantly "page1")}})))
+        ":current-page input resolves to current page name")
+
+    (is (= []
+           (map :block/content
+                (custom-query @conn
+                              {:inputs [:current-page]
+                               :query '[:find (pull ?b [*])
+                                        :in $ ?current-page
+                                        :where [?b :block/page ?bp]
+                                        [?bp :block/name ?current-page]]})))
+        ":current-page input doesn't resolve when :current-page-fn not provided")
+
+    (is (= ["child 1" "child 2"]
+           (let [block-uuid (-> (d/q '[:find (pull ?b [:block/uuid])
+                                       :where [?b :block/content "parent"]] @conn)
+                                ffirst
+                                :block/uuid)]
+             (map :block/content
+                  (custom-query @conn
+                                {:inputs [:current-block]
+                                 :query '[:find (pull ?b [*])
+                                          :in $ ?current-block
+                                          :where [?b :block/parent ?current-block]]
+                                 :input-options {:current-block-uuid block-uuid}}))))
+        ":current-block input resolves to current block's :db/id")
+
+    (is (= []
+           (map :block/content
+                (custom-query @conn
+                              {:inputs [:current-block]
+                               :query '[:find (pull ?b [*])
+                                        :in $ ?current-block
+                                        :where [?b :block/parent ?current-block]]})))
+        ":current-block input doesn't resolve when :current-block-uuid is not provided")
+
+    (is (= []
+           (map :block/content
+                (custom-query @conn
+                              {:inputs [:current-block]
+                               :query '[:find (pull ?b [*])
+                                        :in $ ?current-block
+                                        :where [?b :block/parent ?current-block]]
+                               :input-options {:current-block-uuid :magic}})))
+        ":current-block input doesn't resolve when current-block-uuid is invalid")
+
+    (is (= ["parent"]
+           (let [block-uuid (-> (d/q '[:find (pull ?b [:block/uuid])
+                                       :where [?b :block/content "child 1"]] @conn)
+                                ffirst
+                                :block/uuid)]
+             (map :block/content
+                  (custom-query @conn
+                                {:inputs [:parent-block]
+                                 :query '[:find (pull ?parent-block [*])
+                                          :in $ ?parent-block
+                                          :where [?parent-block :block/parent]]
+                                 :input-options {:current-block-uuid block-uuid}}))))
+        ":parent-block input resolves to parent of current blocks's :db/id")))
+
+(deftest resolve-input-for-journal-date-inputs
+  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+        _ (sqlite-build/create-blocks
+           conn
+           [{:page {:build/journal 20230101}
+             :blocks [{:block/content "b1"}]}
+            {:page {:build/journal 20230107}
+             :blocks [{:block/content "b2"}]}])]
+    (is (= ["b2"]
+           (with-redefs [t/today (constantly (t/date-time 2023 1 7))]
+             (map :block/content
+                  (custom-query @conn
+                                {:inputs [:3d-before :today]
+                                 :query '[:find (pull ?b [*])
+                                          :in $ ?start ?end %
+                                          :where (between ?b ?start ?end)]}))))
+        ":Xd-before and :today resolve to correct journal range")
+
+    (is (= ["b1"]
+           (with-redefs [t/today (constantly (t/date-time 2022 12 31))]
+             (map :block/content
+                  (custom-query @conn
+                                {:inputs [:tomorrow :4d-after]
+                                 :query '[:find (pull ?b [*])
+                                          :in $ ?start ?end %
+                                          :where (between ?b ?start ?end)]}))))
+        ":tomorrow and :Xd-after resolve to correct journal range")))

deps/outliner/test/logseq/outliner/pipeline_test.cljs

@@ -24,16 +24,15 @@
           [parent-uuid child-uuid] (repeatedly 2 random-uuid)
           _ (sqlite-build/create-blocks
              conn
-             {:pages-and-blocks
+             [{:page {:block/original-name "bar"}}
-              [{:page {:block/original-name "bar"}}
+              {:page {:block/original-name "page1"}
-               {:page {:block/original-name "page1"}
+               :blocks [{:block/content "parent [[foo]]"
-                :blocks [{:block/content "parent [[foo]]"
+                         :block/uuid parent-uuid}
-                          :block/uuid parent-uuid}
+                        {:block/content "child [[baz]]"
-                         {:block/content "child [[baz]]"
+                         :block/uuid child-uuid
-                          :block/uuid child-uuid
+                         :block/parent {:db/id [:block/uuid parent-uuid]}}
-                          :block/parent {:db/id [:block/uuid parent-uuid]}}
+                        {:block/content "grandchild [[bing]]"
-                         {:block/content "grandchild [[bing]]"
+                         :block/parent {:db/id [:block/uuid child-uuid]}}]}])
-                          :block/parent {:db/id [:block/uuid child-uuid]}}]}]})
           blocks (get-blocks @conn)
           ;; Update parent block to replace 'foo' with 'bar' ref
           new-tag-id (ffirst (d/q '[:find ?b :where [?b :block/original-name "bar"]] @conn))

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -18,14 +18,12 @@
 (defn- date-journal-title [date]
   (date-time-util/int->journal-title (date-time-util/date->int date) "MMM do, yyyy"))
 
-(def date-journal-day date-time-util/date->int)
-
 (defn- subtract-days
   [date days]
   (new js/Date (- (.getTime date) (* days 24 60 60 1000))))
 
 (defn- build-closed-values-config
-  [{:keys [dates]}]
+  [_opts]
   {:default-closed
    (mapv #(hash-map :value %
                     :uuid (random-uuid)
@@ -43,19 +41,9 @@
    (mapv #(hash-map :value %
                     :uuid (random-uuid))
          ["Page 1" "Page 2" "Page 3"])
+   ;; If this is enabled again, :uuid translation support would need to be added for :build/closed-values
    :date-closed
-   ;; TODO: Consider capitalized journal name when confirming queries work
+   {}})
-   (mapv #(hash-map :value (date-journal-title %)
-                    :uuid (random-uuid))
-         dates)})
-
-(defn- create-journal-page
-  [date journal-name-uuid-map]
-  (let [journal-name (date-journal-title date)]
-    {:block/original-name journal-name
-     :block/uuid (or (journal-name-uuid-map journal-name) (throw (ex-info "No uuid for journal name" {})))
-     :block/type "journal"
-     :block/journal-day (date-journal-day date)}))
 
 (defn- create-init-data
   []
@@ -66,7 +54,6 @@
         page-values-tx (mapv #(hash-map :page
                                         {:block/uuid (:uuid %) :block/original-name (:value %)})
                              (:page-closed closed-values-config))
-        journal-name->uuid (into {} (map (juxt :value :uuid) (:date-closed closed-values-config)))
         ;; Stores random closed values for use with queries
         closed-values (atom {})
         random-closed-value #(let [val (-> closed-values-config % rand-nth)]
@@ -79,13 +66,13 @@
       page-values-tx
       ;; Journals
       [{:page
-        (create-journal-page today journal-name->uuid)
+        {:build/journal (date-time-util/date->int today)}
         :blocks
         [{:block/content "[[Block Properties]]"}
          {:block/content "[[Block Property Queries]]"}
          {:block/content "[[Page Property Queries]]"}]}
-       {:page (create-journal-page yesterday journal-name->uuid)}
+       {:page {:build/journal (date-time-util/date->int yesterday)}}
-       {:page (create-journal-page two-days-ago journal-name->uuid)}
+       {:page {:build/journal (date-time-util/date->int two-days-ago)}}
 
        ;; Block property blocks and queries
        {:page {:block/original-name "Block Properties"}
@@ -106,7 +93,7 @@
         ;;  #_{:block/content "page-closed property block" :build/properties {:page-closed (random-closed-value :page-closed)}}
          {:block/content "date property block" :build/properties {:date [:page (date-journal-title today)]}}
          {:block/content "date-many property block" :build/properties {:date-many #{[:page (date-journal-title today)]
-                                                                              [:page (date-journal-title yesterday)]}}}
+                                                                                    [:page (date-journal-title yesterday)]}}}
          #_{:block/content "date-closed property block" :build/properties {:date-closed (random-closed-value :date-closed)}}]}
        {:page {:block/original-name "Block Property Queries"}
         :blocks
@@ -143,7 +130,7 @@
       ;;  #_{:page {:block/original-name "page-closed page" :build/properties {:page-closed (random-closed-value :page-closed)}}}
        {:page {:block/original-name "date page" :build/properties {:date [:page (date-journal-title today)]}}}
        {:page {:block/original-name "date-many page" :build/properties {:date-many #{[:page (date-journal-title today)]
-                                                                      [:page (date-journal-title yesterday)]}}}}
+                                                                                     [:page (date-journal-title yesterday)]}}}}
        #_{:page {:block/original-name "date-closed page" :build/properties {:date-closed (random-closed-value :date-closed)}}}
        {:page {:block/original-name "Page Property Queries"}
         :blocks

src/test/frontend/db/query_react_test.cljs

@@ -65,99 +65,6 @@ adds rules that users often use"
                                     {:current-page-fn (constantly current-page)})))
 
 ;; TODO: Move most resolve-input tests to deps/db when a load-test-files helper is available for deps
-(deftest resolve-input-for-page-and-block-inputs
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content
-                     "- parent
-   - child 1
-   - child 2"}])
-
-  (is (= ["child 2" "child 1" "parent"]
-         (with-redefs [state/get-current-page (constantly "page1")]
-           (map :block/content
-                (custom-query {:inputs [:current-page]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?current-page
-                                        :where [?b :block/page ?bp]
-                                               [?bp :block/name ?current-page]]}))))
-      ":current-page input resolves to current page name")
-
-  (is (= []
-         (map :block/content
-              (custom-query {:inputs [:current-page]
-                             :query '[:find (pull ?b [*])
-                                      :in $ ?current-page
-                                      :where [?b :block/page ?bp]
-                                      [?bp :block/name ?current-page]]}
-                            {:current-page-fn nil})))
-      ":current-page input doesn't resolve when not present")
-
-  (is (= ["child 1" "child 2"]
-         (let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
-                                            :where [?b :block/content "parent"]])
-                              ffirst
-                              :block/uuid)]
-           (map :block/content
-                (custom-query {:inputs [:current-block]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?current-block
-                                        :where [?b :block/parent ?current-block]]}
-                              {:current-block-uuid block-uuid}))))
-      ":current-block input resolves to current block's :db/id")
-
-  (is (= []
-         (map :block/content
-              (custom-query {:inputs [:current-block]
-                             :query '[:find (pull ?b [*])
-                                      :in $ ?current-block
-                                      :where [?b :block/parent ?current-block]]})))
-      ":current-block input doesn't resolve when current-block-uuid is not provided")
-
-  (is (= []
-         (map :block/content
-              (custom-query {:inputs [:current-block]
-                             :query '[:find (pull ?b [*])
-                                      :in $ ?current-block
-                                      :where [?b :block/parent ?current-block]]}
-                            {:current-block-uuid :magic})))
-      ":current-block input doesn't resolve when current-block-uuid is invalid")
-
-  (is (= ["parent"]
-         (let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
-                                            :where [?b :block/content "child 1"]])
-                              ffirst
-                              :block/uuid)]
-           (map :block/content
-                (custom-query {:inputs [:parent-block]
-                               :query '[:find (pull ?parent-block [*])
-                                        :in $ ?parent-block
-                                        :where [?parent-block :block/parent]]}
-                              {:current-block-uuid block-uuid}))))
-      ":parent-block input resolves to parent of current blocks's :db/id"))
-
-(deftest resolve-input-for-journal-date-inputs
-  (load-test-files [{:file/path "journals/2023_01_01.md"
-                     :file/content "- b1"}
-                    {:file/path "journals/2023_01_07.md"
-                     :file/content "- b2"}])
-
-  (is (= ["b2"]
-         (with-redefs [t/today (constantly (t/date-time 2023 1 7))]
-           (map :block/content
-                (custom-query {:inputs [:3d-before :today]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?start ?end
-                                        :where (between ?b ?start ?end)]}))))
-      ":Xd-before and :today resolve to correct journal range")
-
-  (is (= ["b1"]
-         (with-redefs [t/today (constantly (t/date-time 2022 12 31))]
-           (map :block/content
-                (custom-query {:inputs [:tomorrow :4d-after]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?start ?end
-                                        :where (between ?b ?start ?end)]}))))
-      ":tomorrow and :Xd-after resolve to correct journal range"))
 
 ;; These tests rely on seeding timestamps with properties. If this ability goes
 ;; away we could still test page-level timestamps