Fix sort-by not working for blocks

- Also add sort-by support for pages which fixes #5643
- Add sort-by tests
- Fix sample producing weird queries e.g. mixed block and page clauses
- Clean up last, hard to read test setup in query-dsl

src/main/frontend/db/query_dsl.cljs

@@ -38,11 +38,7 @@
 ;; project (block, TBD)
 
 ;; Sort by (field, asc/desc):
-
+;; (sort-by created-at asc)
-;; created_at
-;; last_modified_at
-
-;; (sort-by last_modified_at asc)
 
 ;; (between -7d +7d)
 
@@ -151,13 +147,16 @@
 
       :else
       (->> clauses
-           (map (fn [result]
+           (mapcat (fn [result]
-                  (if (list? result)
+                     (cond
-                    result
+                       ;; rule like (task ?b #{"NOW"})
-                    (let [result (if (vector? (ffirst result))
+                       (list? result)
-                                   (apply concat result)
+                       [result]
-                                   result)]
+                       ;; datalog clause like [[?b :block/uuid]]
-                      (cons 'and (seq result))))))
+                       (vector? result)
+                       result
+                       :else
+                       [(cons 'and (seq result))])))
            (apply list fe)))
 
     :else
@@ -320,33 +319,24 @@
   (when-let [num (second e)]
     (when (integer? num)
       (reset! sample num)
-      {:query [['?p :block/uuid]]})))
+      ;; blank b/c this post-process filter doesn't effect query
+      {})))
 
 (defn- build-sort-by
   [e sort-by_]
-  (let [[k order] (rest e)
+  (let [[k order*] (map keyword (rest e))
-             order (if (and order (contains? #{:asc :desc}
+        order (if (contains? #{:asc :desc} order*)
-                                             (keyword (string/lower-case (name order)))))
+                order*
-                     (keyword (string/lower-case (name order)))
+                :desc)
-                     :desc)
+        comp (if (= order :desc) >= <=)]
-             k (-> (string/lower-case (name k))
+    (reset! sort-by_
-                   (string/replace "_" "-"))
+            (fn sort-results [result]
-             get-value (cond
+              ;; first because there is one binding result in query-wrapper
-                         (= k "created-at")
+              (sort-by #(-> % first (get-in [:block/properties k]))
-                         :block/created-at
+                       comp
-
+                       result)))
-                         (= k "updated-at")
+    ;; blank b/c this post-process filter doesn't effect query
-                         :block/updated-at
+    {}))
-
-                         :else
-                         #(get-in % [:block/properties k]))
-             comp (if (= order :desc) >= <=)]
-         (reset! sort-by_
-                 (fn [result]
-                   (->> result
-                        flatten
-                        (sort-by get-value comp))))
-         nil))
 
 (defn- build-page
   [e]
@@ -392,7 +382,7 @@ Some bindings in this fn:
          page-ref? (page-ref/page-ref? e)]
      (when (or (and page-ref?
                     (not (contains? #{'page-property 'page-tags} (:current-filter env))))
-               (contains? #{'between 'property 'todo 'task 'priority 'sort-by 'page} fe)
+               (contains? #{'between 'property 'todo 'task 'priority 'page} fe)
                (and (not page-ref?) (string? e)))
        (reset! blocks? true))
      (cond

src/test/frontend/db/query_dsl_test.cljs

@@ -235,10 +235,10 @@ prop-d:: nada"}])
 
   (is (= 1
          (count (dsl-query "(and (task todo) (sample 1))")))
-      "Correctly limits results")
+      "Correctly limits block results")
-  (is (= 2
+  (is (= 1
-         (count (dsl-query "(and (task todo) (sample blarg))")))
+         (count (dsl-query "(and (page-property foo) (sample 1))")))
-      "Non-integer arg is ignored"))
+      "Correctly limits page results"))
 
 (deftest priority-queries
   (load-test-files [{:file/path "pages/page1.md"
@@ -447,76 +447,30 @@ tags: other
          (map :block/content
               (dsl-query "(and [[Parent page]] (not [[Child page]]))")))))
 
-(defn- load-test-files-with-timestamps
+(deftest between-queries
-  []
+  (load-test-files [{:file/path "journals/2020_12_26.md"
-  (let [files [{:file/path "journals/2020_12_26.md"
+                     :file/content "- DONE 26-b1
-                :file/content "---
-title: Dec 26th, 2020
----
-- DONE 26-b1
 created-at:: 1608968448113
-last-modified-at:: 1608968448113
 - LATER 26-b2-modified-later
 created-at:: 1608968448114
-last-modified-at:: 1608968448120
 - DONE 26-b3
 created-at:: 1608968448115
-last-modified-at:: 1608968448115
+- 26-b4
-"}
+created-at:: 1608968448116
-               {:file/path "journals/2020_12_27.md"
+"}])
-                :file/content "---
-title: Dec 27th, 2020
----
-- NOW b1
-created-at:: 1609052958714
-last-modified-at:: 1609052958714
-- LATER b2-modified-later
-created-at:: 1609052959376
-last-modified-at:: 1609052974285
-- b3
-created-at:: 1609052959954
-last-modified-at:: 1609052959954
-- b4
-created-at:: 1609052961569
-last-modified-at:: 1609052961569
-- b5
-created-at:: 1609052963089
-last-modified-at:: 1609052963089"}
-               {:file/path "journals/2020_12_28.md"
-                :file/content "---
-title: Dec 28th, 2020
----
-- 28-b1
-created-at:: 1609084800000
-last-modified-at:: 1609084800000
-- 28-b2-modified-later
-created-at:: 1609084800001
-last-modified-at:: 1609084800020
-- 28-b3
-created-at:: 1609084800002
-last-modified-at:: 1609084800002"}]]
-    (load-test-files files)))
-
-(deftest between-queries
-  (load-test-files-with-timestamps)
 
   (are [x y] (= (count (dsl-query x)) y)
        "(and (task now later done) (between [[Dec 26th, 2020]] tomorrow))"
-       5
+       3
 
        ;; between with journal pages
-       "(and (task now later done) (between [[Dec 27th, 2020]] [[Dec 28th, 2020]]))"
+       "(and (task now later done) (between [[Dec 26th, 2020]] [[Dec 27th, 2020]]))"
-       2
+       3
 
        ;; ;; between with created-at
        ;; "(and (task now later done) (between created-at [[Dec 26th, 2020]] tomorrow))"
-       ;; 5
+       ;; 3
-
+       ))
-       ;; ;; between with last-modified-at
-       ;; "(and (task now later done) (between last-modified-at [[Dec 26th, 2020]] tomorrow))"
-       ;; 5
-       )
-  )
 
 (deftest custom-query-test
   (load-test-files [{:file/path "pages/page1.md"
@@ -533,63 +487,63 @@ last-modified-at:: 1609084800002"}]]
          (map :block/content (custom-query {:query (list 'and '(task later) "b")})))
       "Query with rule that can't be derived from the form itself"))
 
-#_(deftest sort-by-queries
+(deftest sort-by-queries
-    (load-test-files-with-timestamps)
+  (load-test-files [{:file/path "journals/2020_02_25.md"
-    ;; (testing "sort-by (created-at defaults to desc)"
+                     :file/content "rating:: 10"}
-    ;;   (db/clear-query-state!)
+                    {:file/path "journals/2020_12_26.md"
-    ;;   (let [result (->> (dsl-query "(and (task now later done)
+                     :file/content "rating:: 8
-    ;;                              (sort-by created-at))")
+- DONE 26-b1
-    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+created-at:: 1608968448113
-    ;;     (is (= result
+fruit:: plum
-    ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
+- LATER 26-b2-modified-later
+created-at:: 1608968448114
+fruit:: apple
+- DONE 26-b3 has no fruit to test sorting of absent property value
+created-at:: 1608968448115
+- 26-b4
+created-at:: 1608968448116
+"}])
 
-    ;; (testing "sort-by (created-at desc)"
+  (testing "sort-by user block property fruit"
-    ;;   (db/clear-query-state!)
+    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit))")
-    ;;   (let [result (->> (dsl-query "(and (todo now later done)
+                      (map #(get-in % [:block/properties :fruit])))]
-    ;;                              (sort-by created-at desc))")
+      (is (= ["plum" "apple" nil]
-    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+             result)
-    ;;     (is (= result
+          "sort-by correctly defaults to desc"))
-    ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
 
-    ;; (testing "sort-by (created-at asc)"
+    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit desc))")
-    ;;   (db/clear-query-state!)
+                      (map #(get-in % [:block/properties :fruit])))]
-    ;;   (let [result (->> (dsl-query "(and (todo now later done)
+      (is (= ["plum" "apple" nil]
-    ;;                              (sort-by created-at asc))")
+             result)
-    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+          "sort-by desc"))
-    ;;     (is (= result
-    ;;            '(1608968448113 1608968448114 1608968448115 1609052958714 1609052959376)))))
 
-    ;; (testing "sort-by (last-modified-at defaults to desc)"
+    (let [result (->> (dsl-query "(and (task now later done) (sort-by fruit asc))")
-    ;;   (db/clear-query-state!)
+                      (map #(get-in % [:block/properties :fruit])))]
-    ;;   (let [result (->> (dsl-query "(and (todo now later done)
+      (is (= ["apple" "plum" nil]
-    ;;                              (sort-by last-modified-at))")
+             result)
-    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+          "sort-by asc")))
-    ;;     (is (= result
-    ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
 
-    ;; (testing "sort-by (last-modified-at desc)"
+  (testing "sort-by hidden, built-in block property created-at"
-    ;;   (db/clear-query-state!)
+    (let [result (->> (dsl-query "(and (task now later done) (sort-by created-at desc))")
-    ;;   (let [result (->> (dsl-query "(and (todo now later done)
+                      (map #(get-in % [:block/properties :created-at])))]
-    ;;                              (sort-by last-modified-at desc))")
+      (is (= [1608968448115 1608968448114 1608968448113]
-    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+             result))
-    ;;     (is (= result
+      "sorted-by desc")
-    ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
 
-    ;; (testing "sort-by (last-modified-at desc)"
+    (let [result (->> (dsl-query "(and (todo now later done) (sort-by created-at asc))")
-    ;;   (db/clear-query-state!)
+                      (map #(get-in % [:block/properties :created-at])))]
-    ;;   (let [result (->> (dsl-query "(and (todo now later done)
+      (is (= [1608968448113 1608968448114 1608968448115]
-    ;;                              (sort-by last-modified-at asc))")
+             result)
-    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+          "sorted-by asc")))
-    ;;     (is (= result
-    ;;            '(1608968448113 1608968448115 1608968448120 1609052958714 1609052974285)))))
-    )
 
-#_(cljs.test/run-tests)
+  (testing "user page property rating"
+    (is (= [10 8]
+           (->> (dsl-query "(and (page-property rating) (sort-by rating))")
+                (map #(get-in % [:block/properties :rating])))))))
 
 (comment
  (require '[clojure.pprint :as pprint])
  (test-helper/start-test-db!)
- (load-test-files-with-timestamps)
 
  (query-dsl/query test-db "(task done)")