add group-errors option to validation script

Reduces error noise and helped fix last two bugs.
Also improved schema by making left, parent and format
required for all normal blocks

scripts/src/logseq/tasks/db_graph/validate_client_db.cljs

@@ -1,6 +1,5 @@
 (ns logseq.tasks.db-graph.validate-client-db
-  "Script that validates the datascript db of a db graph.
-   Currently only validates :block/schema but it will validate much more ..."
+  "Script that validates the datascript db of a db graph"
   (:require [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.sqlite.db :as sqlite-db]
             [datascript.core :as d]
@@ -38,46 +37,47 @@
    [:block/tx-id {:optional true} :int]])
 
 (def page-block
-  (into
-   [[:block/name :string]
-    [:block/original-name :string]
-    [:block/type {:optional true} [:enum "property" "class" "object" "whiteboard"]]
-    [:block/namespace {:optional true} :int]
+  (vec
+   (concat
+    [:map {:closed false}]
+    [[:block/name :string]
+     [:block/original-name :string]
+     [:block/type {:optional true} [:enum "property" "class" "object" "whiteboard"]]
+     [:block/namespace {:optional true} :int]
     ;; TODO: journal?, journal-day and format optional b/c of property
-    [:block/journal? {:optional true} :boolean]
-    [:block/journal-day {:optional true} :int]
-    [:block/format {:optional true} [:enum :markdown]]
+     [:block/journal? {:optional true} :boolean]
+     [:block/journal-day {:optional true} :int]
+    ;; block/format optional b/c of property, objects and built-in pages
+     [:block/format {:optional true} [:enum :markdown]]
     ;; TODO: Should this be here?
-    [:block/path-refs {:optional true} :any]
+     [:block/path-refs {:optional true} :any]
     ;; TODO: collapsed only for linked
-    [:block/collapsed? {:optional true} :boolean]
+     [:block/collapsed? {:optional true} :boolean]
     ;; TODO: Required for property and class types
-    [:block/schema
-     {:optional true}
-     [:map
-      {:closed false}
+     [:block/schema
+      {:optional true}
+      [:map
+       {:closed false}
       ;; TODO: only validate most of these for property blocks
-      [:type {:optional true} :keyword]
-      [:enum-config {:optional true} :map] ;; TODO
-      [:cardinality {:optional true} [:enum :one :many]]
-      [:classes {:optional true} [:set :uuid]]
-      [:description {:optional true} :string]
-      [:hide? {:optional true} :boolean]
+       [:type {:optional true} :keyword]
+       [:enum-config {:optional true} :map] ;; TODO
+       [:cardinality {:optional true} [:enum :one :many]]
+       [:classes {:optional true} [:set :uuid]]
+       [:description {:optional true} :string]
+       [:hide? {:optional true} :boolean]
       ;; TODO: require this for class blocks
-      [:properties {:optional true} [:vector :uuid]]]]]
-   page-or-block-attrs))
+       [:properties {:optional true} [:vector :uuid]]]]]
+    page-or-block-attrs)))
 
 (def block-attrs
   "Common attributes for normal blocks"
   (into
+   ;; refs
    [[:block/page :int]
-    ;; refs
-    ;; TODO: left, parent are only optional b/c of dummy blocks
-    [:block/left {:optional true} :int]
-    [:block/parent {:optional true} :int]
     [:block/path-refs {:optional true} :any] ;;TODO
-    [:block/format {:optional true} [:enum :markdown]]
+    [:block/link {:optional true} :int]
     ;; other
+    [:block/format [:enum :markdown]]
     [:block/marker {:optional true} :string]
     [:block/priority {:optional true} :string]
     [:block/collapsed? {:optional true} :boolean]]
@@ -85,16 +85,31 @@
 
 (def normal-block
   "A normal block is a block with content and a page"
-  (into block-attrs
-        [[:block/content :string]
-         [:block/link {:optional true} :int]]))
+  (vec
+   (concat
+    [:map {:closed false}]
+    block-attrs
+    [[:block/content :string]
+     [:block/left :int]
+     [:block/parent :int]])))
+
+;; TODO: Remove this when bug is fixed with blank content that has no
+;; :block/left or :block/parent
+(def normal-dummy-block
+  (vec
+   (concat
+    [:map {:closed false}]
+    block-attrs
+    [[:block/content [:= ""]]])))
 
 ;; TODO: Figure out where this is coming from
 (def unknown-empty-block
-  [[:block/uuid :uuid]])
+  [:map {:closed true}
+   [:block/uuid :uuid]])
 
 (def file-block
-  [[:block/uuid :uuid]
+  [:map {:closed true}
+   [:block/uuid :uuid]
    [:block/tx-id {:optional true} :int]
    [:file/content :string]
    [:file/path :string]
@@ -104,14 +119,40 @@
 (def client-db-schema
   [:sequential
    [:or
-    (into [:map {:closed false}] page-block)
-    (into [:map {:closed false}] normal-block)
-    (into [:map {:closed true}] file-block)
-    (into [:map {:closed true}] unknown-empty-block)]])
+    page-block
+    normal-block
+    normal-dummy-block
+    file-block
+    unknown-empty-block]])
+
+(defn- build-grouped-errors [db full-maps errors]
+  (->> errors
+       (group-by #(-> % :in first))
+       (map (fn [[idx errors']]
+              {:entity (update (get full-maps idx)
+                               ;; Provide additional page info for debugging
+                               :block/page
+                               (fn [id] (when id
+                                          (select-keys (d/entity db id)
+                                                       [:block/name :block/type :db/id :block/created-at]))))
+               ;; Group by type to reduce verbosity
+               :errors-by-type
+               (->> (group-by :type errors')
+                    (map (fn [[type' type-errors]]
+                           [type'
+                            {:in-value-distinct (->> type-errors
+                                                     (map #(select-keys % [:in :value]))
+                                                     distinct
+                                                     vec)
+                             :schema-distinct (->> (map :schema type-errors)
+                                                   (map m/form)
+                                                   distinct
+                                                   vec)}]))
+                    (into {}))}))))
 
 (defn validate-client-db
   "Validate datascript db as a vec of entity maps"
-  [ent-maps {:keys [closed-maps verbose]}]
+  [db ent-maps {:keys [closed-maps verbose group-errors]}]
   (let [schema (if closed-maps
                  (walk/postwalk (fn [e]
                                   (if (and (vector? e)
@@ -129,11 +170,15 @@
         (println "Found" (count errors) "errors:")
         (if verbose
           (let [full-maps (vec (vals ent-maps))]
-            (pprint/pprint
-             (map #(assoc %
-                          :entity (get full-maps (-> % :in first))
-                          :schema (m/form (:schema %)))
-                  errors)))
+            (if group-errors
+              (let [ent-errors (build-grouped-errors db full-maps errors)]
+                (pprint/pprint ent-errors)
+                (println "Found" (count ent-errors) "entities in errors"))
+              (pprint/pprint
+               (map #(assoc %
+                            :entity (get full-maps (-> % :in first))
+                            :schema (m/form (:schema %)))
+                    errors))))
           (pprint/pprint errors))
         (js/process.exit 1))
       (println "Valid!"))))
@@ -153,7 +198,9 @@
    :verbose {:alias :v
              :desc "Print more info"}
    :closed-maps {:alias :c
-                 :desc "Validate maps marked with closed as :closed"}})
+                 :desc "Validate maps marked with closed as :closed"}
+   :group-errors {:alias :g
+                  :desc "Groups errors by their entity id with --verbose enabled"}})
 
 (defn- validate-graph [graph-dir options]
   (let [[dir db-name] (if (string/includes? graph-dir "/")
@@ -167,7 +214,7 @@
         datoms (d/datoms @conn :eavt)
         ent-maps (datoms->entity-maps datoms)]
     (println "Read graph" (str db-name " with " (count datoms) " datoms!"))
-    (validate-client-db ent-maps options)))
+    (validate-client-db @conn ent-maps options)))
 
 (defn -main [argv]
   (let [{:keys [args opts]} (cli/parse-args argv {:spec spec})