fix: sort-by filter for db graphs

Also fixed a dsl-query sorting bug where nil property values where
incorrectly being treated as greater than non-nil values. Also
fixed a bug where blocks with an explicit :block/created-at weren't
being set for sqlite-build

deps/db/src/logseq/db/sqlite/build.cljs

@@ -109,8 +109,8 @@
       (seq pvalue-tx-m)
       (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
       true
-      (conj (merge (dissoc m :build/properties)
-                   (sqlite-util/block-with-timestamps new-block)
+      (conj (merge (sqlite-util/block-with-timestamps new-block)
+                   (dissoc m :build/properties)
                    (when (seq properties)
                      (->block-properties (merge properties (db-property-build/build-properties-with-ref-values pvalue-tx-m))
                                          page-uuids all-idents))

src/main/frontend/db/query_dsl.cljs

@@ -10,8 +10,6 @@
             [frontend.db.model :as model]
             [frontend.db.query-react :as query-react]
             [frontend.db.utils :as db-utils]
-            [frontend.db.conn :as conn]
-            [datascript.core :as d]
             [logseq.db.frontend.rules :as rules]
             [frontend.template :as template]
             [logseq.graph-parser.text :as text]
@@ -370,7 +368,11 @@
         order (if (contains? #{:asc :desc} order*)
                 order*
                 :desc)
-        comp (if (= order :desc) >= <=)]
+        comp (if (= order :desc)
+               ;; Handle nil so that is always less than a value e.g. treated as a "" for a string.
+               ;; Otherwise sort bugs occur that prevent non-nil values from being sorted
+               #(if (nil? %2) true (>= %1 %2))
+               #(if (nil? %1) true (<= %1 %2)))]
     (reset! sort-by_
             (fn sort-results [result property-val-fn]
               ;; first because there is one binding result in query-wrapper
@@ -615,18 +617,20 @@ Some bindings in this fn:
   [col]
   ;; Only modify result shapes that we know of
   (if (map? (ffirst col))
-    (let [prop-uuid-names (->> (d/datoms (conn/get-db) :avet :block/type "property")
-                               (map :e)
-                               (db-utils/pull-many '[:block/uuid :block/name])
-                               (map #(vector (:block/uuid %) (keyword (:block/name %))))
-                               (into {}))]
     (map (fn [blocks]
            (mapv (fn [block]
                    (assoc block
                           :block/properties-by-name
-                            (update-keys (:block/properties block) #(prop-uuid-names % %))))
+                          (->> (db-property/properties block)
+                               (map (fn [[k v]]
+                                      [(:block/original-name (db-utils/entity k))
+                                       (or (some->> (:db/id v)
+                                                    db-utils/entity
+                                                    db-property/get-property-value-name)
+                                           v)]))
+                               (into {}))))
                  blocks))
-           col))
+         col)
     col))
 
 (defn get-db-property-value
@@ -636,7 +640,14 @@ Some bindings in this fn:
   (case prop
     :created-at (:block/created-at m)
     :updated-at (:block/updated-at m)
-    (get-in m [:block/properties-by-name prop])))
+    (get-in m [:block/properties-by-name (name prop)])))
+
+(def db-block-attrs
+  "Like ldb/block-attrs but for query dsl an db graphs"
+  ;; '*' needed as we need to pull user properties and don't know their names in advance
+  '[*
+    {:block/page [:db/id :block/name :block/original-name :block/journal-day]}
+    {:block/_parent ...}])
 
 (defn query
   "Runs a dsl query with query as a string. Primary use is from '{{query }}'"
@@ -645,7 +656,9 @@ Some bindings in this fn:
   ([repo query-string query-opts]
    (when (and (string? query-string) (not= "\"\"" query-string))
      (let [{:keys [query rules sort-by blocks? sample]} (parse-query query-string)]
-       (when-let [query' (some-> query (query-wrapper {:blocks? blocks?}))]
+       (when-let [query' (some-> query (query-wrapper {:blocks? blocks?
+                                                       :block-attrs (when (config/db-based-graph? repo)
+                                                                      db-block-attrs)}))]
          (let [random-samples (if @sample
                                 (fn [col]
                                   (take @sample (shuffle col)))
@@ -670,8 +683,10 @@ Some bindings in this fn:
   [repo query-m query-opts]
   (when (seq (:query query-m))
     (let [query-string (template/resolve-dynamic-template! (pr-str (:query query-m)))
-          {:keys [query sort-by blocks? rules]} (parse query-string {:db-graph? (config/db-based-graph? repo)})]
-      (when-let [query' (some-> query (query-wrapper {:blocks? blocks?}))]
+          db-graph? (config/db-based-graph? repo)
+          {:keys [query sort-by blocks? rules]} (parse query-string {:db-graph? db-graph?})]
+      (when-let [query' (some-> query (query-wrapper {:blocks? blocks?
+                                                      :block-attrs (when db-graph? db-block-attrs)}))]
         (query-react/react-query repo
                                  (merge
                                   query-m
@@ -681,7 +696,7 @@ Some bindings in this fn:
                                   query-opts
                                   (when sort-by
                                     {:transform-fn
-                                     (if (config/db-based-graph? repo)
+                                     (if db-graph?
                                        (comp (fn [col] (sort-by col get-db-property-value)) sort-by-prep)
                                        #(sort-by % (fn [m prop] (get-in m [:block/properties prop]))))})))))))
 

src/test/frontend/db/query_dsl_test.cljs

@@ -550,33 +550,34 @@ created-at:: 1608968448115
 - 26-b4
 created-at:: 1608968448116
 "}])
+  (let [task-filter (if js/process.env.DB_GRAPH "(task todo done)" "(task later done)")]
     (testing "sort-by user block property fruit"
-    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit))")
+      (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit))"))
                         (map #(get-property-value % :fruit)))]
         (is (= ["plum" "apple" nil]
                result)
             "sort-by correctly defaults to desc"))
 
-    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit desc))")
+      (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit desc))"))
                         (map #(get-property-value % :fruit)))]
         (is (= ["plum" "apple" nil]
                result)
             "sort-by desc"))
 
-    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit asc))")
+      (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit asc))"))
                         (map #(get-property-value % :fruit)))]
-        (is (= ["apple" "plum" nil]
+        (is (= [nil "apple" "plum"]
                result)
             "sort-by asc")))
 
     (testing "sort-by hidden, built-in block property created-at"
-      (let [result (->> (dsl-query "(and (task now later done) (sort-by created-at desc))")
+      (let [result (->> (dsl-query (str "(and " task-filter " (sort-by created-at desc))"))
                         (map #(get-property-value % :created-at)))]
         (is (= [1608968448115 1608968448114 1608968448113]
                result))
         "sorted-by desc")
 
-      (let [result (->> (dsl-query "(and (todo now later done) (sort-by created-at asc))")
+      (let [result (->> (dsl-query (str "(and " task-filter " (sort-by created-at asc))"))
                         (map #(get-property-value % :created-at)))]
         (is (= [1608968448113 1608968448114 1608968448115]
                result)
@@ -585,9 +586,9 @@ created-at:: 1608968448116
     (testing "user page property rating"
       (is (= [10 8]
              (->> (dsl-query "(and (page-property rating) (sort-by rating))")
-                  (map #(get-property-value % :rating)))))))
+                  (map #(get-property-value % :rating))))))))
 
-(deftest simplify-query
+(deftest ^:done simplify-query
   (are [x y] (= (query-dsl/simplify-query x) y)
     '(and [[foo]])
     '[[foo]]